From 50b5495d26c28947fcf20666942048dae7e05ab8 Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Fri, 12 Jun 2026 12:38:31 +1000 Subject: [PATCH] chore: more comment cleanup --- internal/eksgw/client.go | 14 +- internal/fipsboot/fipsboot.go | 12 +- internal/tlsconfig/curves.go | 31 +- internal/tlsconfig/curves_test.go | 14 +- spinifex/admin/admin.go | 94 +--- spinifex/admin/dial.go | 17 +- spinifex/admin/email.go | 8 +- spinifex/admin/images_remove.go | 44 +- spinifex/admin/ipsec.go | 48 +- spinifex/admin/ipsec_test.go | 12 +- spinifex/admin/network_detect.go | 10 +- spinifex/awsec2query/parser.go | 33 +- spinifex/awserrors/awserrors.go | 15 +- spinifex/awserrors/awserrors_test.go | 7 +- spinifex/config/config.go | 53 +- spinifex/daemon/daemon.go | 531 +++++------------- spinifex/daemon/daemon_handlers_eni.go | 16 +- .../daemon/daemon_handlers_iam_profile.go | 32 +- spinifex/daemon/daemon_system_dispatch.go | 31 +- spinifex/daemon/eks_adapter.go | 8 +- spinifex/daemon/eks_deps.go | 40 +- spinifex/daemon/eks_worker_launch.go | 14 +- spinifex/daemon/local_state.go | 40 +- spinifex/daemon/ownership.go | 20 +- spinifex/daemon/peer_health.go | 32 +- spinifex/daemon/reconcile.go | 32 +- spinifex/daemon/resource.go | 111 +--- spinifex/daemon/state.go | 8 +- spinifex/daemon/vm_adapters.go | 58 +- spinifex/filterutil/filterutil.go | 22 +- spinifex/formation/formation.go | 7 +- spinifex/formation/helpers.go | 12 +- spinifex/gpu/bind.go | 5 +- spinifex/gpu/manager.go | 65 +-- spinifex/gpu/mig.go | 8 +- spinifex/gpu/reset.go | 15 +- spinifex/handlers/acm/service.go | 6 +- spinifex/handlers/acm/service_nats.go | 3 +- spinifex/handlers/acm/store.go | 9 +- spinifex/handlers/ec2/eip/service_impl.go | 25 +- .../handlers/ec2/eip/service_impl_test.go | 17 +- spinifex/handlers/ec2/eip/types.go | 6 +- spinifex/handlers/ec2/igw/service_impl.go | 18 +- .../handlers/ec2/igw/service_impl_test.go | 7 +- spinifex/handlers/ec2/image/service_impl.go | 28 +- .../handlers/ec2/image/service_impl_test.go | 25 +- spinifex/handlers/ec2/instance/service.go | 33 +- .../handlers/ec2/instance/service_impl.go | 436 +++++--------- .../ec2/instance/service_impl_test.go | 55 +- .../handlers/ec2/instance/service_nats.go | 11 +- spinifex/handlers/ec2/key/service_impl.go | 6 +- .../ec2/placementgroup/service_impl.go | 4 +- .../ec2/placementgroup/service_impl_test.go | 9 +- .../handlers/ec2/routetable/service_impl.go | 94 +--- .../ec2/routetable/service_impl_test.go | 55 +- spinifex/handlers/ec2/volume/service_impl.go | 38 +- .../handlers/ec2/volume/service_impl_test.go | 5 +- spinifex/handlers/ec2/vpc/eni.go | 29 +- spinifex/handlers/ec2/vpc/external_ipam.go | 12 +- .../handlers/ec2/vpc/external_ipam_test.go | 4 +- spinifex/handlers/ec2/vpc/invariants_test.go | 18 +- spinifex/handlers/ec2/vpc/ipam.go | 8 +- spinifex/handlers/ec2/vpc/security_group.go | 87 +-- .../handlers/ec2/vpc/security_group_test.go | 80 +-- spinifex/handlers/ec2/vpc/service_impl.go | 26 +- .../handlers/ec2/vpc/service_impl_test.go | 20 +- .../handlers/ec2/vpc/service_nats_test.go | 7 +- spinifex/handlers/eks/access_entry.go | 45 +- spinifex/handlers/eks/addon_catalog.go | 23 +- spinifex/handlers/eks/addon_installer.go | 35 +- spinifex/handlers/eks/addon_state.go | 32 +- spinifex/handlers/eks/addons.go | 24 +- spinifex/handlers/eks/awsgw_endpoint.go | 9 +- spinifex/handlers/eks/claim.go | 14 +- spinifex/handlers/eks/cluster_reconciler.go | 93 ++- spinifex/handlers/eks/cluster_state.go | 125 ++--- spinifex/handlers/eks/k3s_ha_control_plane.go | 114 ++-- spinifex/handlers/eks/k3s_server_vm.go | 267 +++------ spinifex/handlers/eks/nats_bootstrap.go | 82 +-- spinifex/handlers/eks/nlb.go | 86 +-- spinifex/handlers/eks/nodegroup.go | 27 +- spinifex/handlers/eks/nodegroup_userdata.go | 22 +- spinifex/handlers/eks/oidc_keypair.go | 38 +- spinifex/handlers/eks/reconciler_registry.go | 29 +- spinifex/handlers/eks/security_groups.go | 73 +-- spinifex/handlers/eks/service.go | 17 +- spinifex/handlers/eks/service_impl.go | 199 ++----- spinifex/handlers/elbv2/agent_types.go | 29 +- spinifex/handlers/elbv2/arn_test.go | 7 +- spinifex/handlers/elbv2/haproxy.go | 118 ++-- spinifex/handlers/elbv2/haproxy_test.go | 8 +- spinifex/handlers/elbv2/health_checker.go | 24 +- spinifex/handlers/elbv2/launcher.go | 7 +- .../elbv2/lifecycle_invariants_test.go | 34 +- spinifex/handlers/elbv2/nginx.go | 26 +- spinifex/handlers/elbv2/service_impl.go | 414 ++++---------- .../handlers/elbv2/service_impl_attributes.go | 7 +- .../elbv2/service_impl_health_test.go | 7 +- .../handlers/elbv2/service_impl_lb_network.go | 65 +-- .../elbv2/service_impl_listener_certs.go | 6 +- .../handlers/elbv2/service_impl_nlb_sg.go | 36 +- spinifex/handlers/elbv2/service_impl_rules.go | 24 +- .../elbv2/service_impl_rules_extra_test.go | 9 +- .../handlers/elbv2/service_impl_rules_test.go | 7 +- spinifex/handlers/elbv2/service_impl_tags.go | 21 +- spinifex/handlers/elbv2/service_impl_test.go | 62 +- .../handlers/elbv2/service_impl_vpc_test.go | 37 +- spinifex/handlers/elbv2/service_nats.go | 9 +- spinifex/handlers/elbv2/store.go | 56 +- .../elbv2/system_instance_launcher_nats.go | 23 +- spinifex/handlers/elbv2/types.go | 49 +- spinifex/handlers/iam/imds_rpc.go | 10 +- spinifex/handlers/iam/instance_profiles.go | 22 +- spinifex/handlers/iam/oidc_providers.go | 50 +- spinifex/handlers/iam/roles.go | 58 +- spinifex/handlers/iam/service_impl.go | 52 +- spinifex/handlers/iam/types.go | 14 +- spinifex/handlers/imds/buckets.go | 19 +- spinifex/handlers/imds/constants.go | 4 +- spinifex/handlers/imds/credentials.go | 30 +- spinifex/handlers/imds/instance_lookup.go | 12 +- spinifex/handlers/imds/metadata.go | 91 +-- spinifex/handlers/imds/nats_clients.go | 26 +- spinifex/handlers/imds/resolver.go | 57 +- spinifex/handlers/imds/server.go | 86 +-- spinifex/handlers/imds/service.go | 39 +- spinifex/handlers/imds/token.go | 18 +- spinifex/handlers/imds/veth_store.go | 11 +- spinifex/handlers/sts/assume_role.go | 98 ++-- .../sts/assume_role_with_web_identity.go | 76 +-- spinifex/handlers/sts/get_caller_identity.go | 6 +- spinifex/handlers/sts/get_session_token.go | 42 +- spinifex/handlers/sts/oidc_jwks.go | 32 +- spinifex/handlers/sts/presigned_url_verify.go | 95 +--- spinifex/handlers/sts/service.go | 62 +- spinifex/handlers/sts/service_impl.go | 16 +- spinifex/handlers/sts/session_credentials.go | 80 +-- .../handlers/sts/web_identity_trust_policy.go | 85 +-- spinifex/handlers/sysinstance/launcher.go | 58 +- spinifex/instancetypes/definitions.go | 35 +- spinifex/instancetypes/eni_limits.go | 26 +- spinifex/instancetypes/generate.go | 27 +- spinifex/lbagent/agent.go | 46 +- spinifex/lbagent/health.go | 11 +- spinifex/lbagent/healthcheck.go | 20 +- spinifex/migrate/001_security_group.go | 12 +- spinifex/migrate/migrate.go | 30 +- spinifex/nbd/nbd.go | 7 - spinifex/network/external/allocator.go | 7 +- spinifex/network/external/dhcp/client.go | 14 +- spinifex/network/external/dhcp/client_test.go | 6 +- spinifex/network/external/dhcp/fake.go | 6 +- spinifex/network/external/dhcp/kv.go | 6 +- .../network/external/dhcp/lrp_allocator.go | 34 +- spinifex/network/external/dhcp/manager.go | 44 +- spinifex/network/external/dhcp/nats_client.go | 12 +- spinifex/network/external/dhcp/nclient4.go | 26 +- .../network/external/dhcp/pool_allocator.go | 16 +- spinifex/network/external/dhcp/promisc.go | 21 +- .../network/external/dhcp/promisc_test.go | 7 +- spinifex/network/external/dhcp/wire.go | 13 +- spinifex/network/external/igw.go | 92 +-- spinifex/network/external/igw_test.go | 22 +- .../external/imds_datapath_contract_test.go | 37 +- spinifex/network/external/imds_topology.go | 33 +- spinifex/network/external/pool.go | 12 +- spinifex/network/external/static_pool.go | 18 +- spinifex/network/external/static_pool_test.go | 5 +- spinifex/network/host/gateway_claim.go | 20 +- spinifex/network/host/host.go | 7 +- spinifex/network/host/imds_names.go | 31 +- spinifex/network/host/imds_netns_unit_test.go | 21 +- spinifex/network/host/imds_veth.go | 84 +-- spinifex/network/host/imds_veth_test.go | 14 +- spinifex/network/host/ipsec.go | 8 +- spinifex/network/host/neigh.go | 31 +- spinifex/network/host/plumber.go | 20 +- spinifex/network/ovn/client.go | 85 +-- spinifex/network/ovn/live.go | 67 +-- spinifex/network/policy/acl.go | 17 +- spinifex/network/policy/nat.go | 73 +-- spinifex/network/policy/nat_test.go | 21 +- spinifex/network/policy/policy.go | 10 +- spinifex/network/policy/route.go | 60 +- spinifex/network/policy/sg.go | 10 +- spinifex/network/policy/sg_test.go | 8 +- spinifex/network/reconcile/apply.go | 62 +- spinifex/network/reconcile/apply_test.go | 11 +- spinifex/network/reconcile/intent.go | 42 +- spinifex/network/reconcile/intent_test.go | 35 +- spinifex/network/reconcile/invariants_test.go | 25 +- spinifex/network/reconcile/liveness_test.go | 31 +- spinifex/network/reconcile/lock.go | 6 +- spinifex/network/reconcile/lock_test.go | 8 +- spinifex/network/reconcile/reconcile.go | 46 +- spinifex/network/subscribers/events.go | 35 +- spinifex/network/subscribers/subscribers.go | 4 +- .../network/subscribers/subscribers_test.go | 24 +- spinifex/network/subscribers/topology.go | 18 +- spinifex/network/topology/dhcp_options.go | 15 +- .../network/topology/dhcp_options_test.go | 6 +- spinifex/network/topology/manager.go | 12 +- spinifex/network/topology/manager_live.go | 17 +- spinifex/network/topology/names.go | 12 +- spinifex/objectstore/objectstore.go | 30 +- spinifex/qmp/qmp.go | 8 +- spinifex/services/awsgw/awsgw.go | 9 +- spinifex/services/awsgw/imds_absent_test.go | 7 +- spinifex/services/nats/nats.go | 3 - spinifex/services/nats/nats_test.go | 12 +- spinifex/services/predastore/predastore.go | 1 - .../predastore/predastore_integration_test.go | 39 +- .../services/predastore/predastore_test.go | 6 +- spinifex/services/spinifexui/proxy.go | 6 +- spinifex/services/viperblockd/viperblockd.go | 31 +- .../viperblockd_integration_test.go | 12 +- .../services/viperblockd/viperblockd_test.go | 5 +- spinifex/systemd/units_invariants_test.go | 8 +- spinifex/tags/tags.go | 7 +- spinifex/testutil/nats.go | 28 +- spinifex/types/ec2.go | 10 +- spinifex/types/eni.go | 12 +- spinifex/types/node.go | 6 +- spinifex/utils/account.go | 14 +- spinifex/utils/account_test.go | 13 +- spinifex/utils/encryption.go | 10 +- spinifex/utils/images.go | 101 +--- spinifex/utils/nats.go | 115 +--- spinifex/utils/nats_test.go | 14 +- spinifex/utils/nbd.go | 27 +- spinifex/utils/pidfile.go | 36 +- spinifex/utils/process.go | 12 +- spinifex/utils/sudo.go | 11 +- spinifex/utils/utils.go | 57 +- spinifex/utils/utils_test.go | 10 +- spinifex/vm/crash_recovery.go | 32 +- spinifex/vm/device_map.go | 36 +- spinifex/vm/eni_hotplug.go | 42 +- spinifex/vm/firmware.go | 21 +- spinifex/vm/instance_cleaner.go | 13 +- spinifex/vm/lifecycle.go | 132 ++--- spinifex/vm/manager.go | 112 ++-- spinifex/vm/migrate.go | 19 +- spinifex/vm/network_plumber.go | 7 +- spinifex/vm/qmp_device.go | 55 +- spinifex/vm/restore.go | 110 +--- spinifex/vm/shutdown.go | 107 +--- spinifex/vm/state.go | 17 +- spinifex/vm/state_store.go | 10 +- spinifex/vm/vm.go | 62 +- spinifex/vm/volumes.go | 54 +- spinifex/vm/watchdog.go | 8 +- spinifex/vpcd/vpcd.go | 246 ++------ tests/e2e/ddil/doc.go | 17 +- tests/e2e/ddil/fault/fault.go | 28 +- tests/e2e/ddil/fault/profiles.go | 15 +- tests/e2e/ddil/fault/snapshot.go | 43 +- tests/e2e/ddil/harness/quarantine.go | 19 +- tests/e2e/ddil/scenarios/a_nats_kill_test.go | 16 +- .../ddil/scenarios/b_daemon_restart_test.go | 28 +- tests/e2e/ddil/scenarios/c_partition_test.go | 17 +- .../e2e/ddil/scenarios/coverage_drift_test.go | 47 +- tests/e2e/ddil/scenarios/main_test.go | 70 +-- tests/e2e/eks/eks_test.go | 45 +- tests/e2e/harness/account.go | 21 +- tests/e2e/harness/artifacts.go | 4 +- tests/e2e/harness/aws.go | 46 +- tests/e2e/harness/cluster.go | 23 +- tests/e2e/harness/daemon_client.go | 46 +- tests/e2e/harness/diag.go | 33 +- tests/e2e/harness/discover.go | 15 +- tests/e2e/harness/ec2helpers.go | 39 +- tests/e2e/harness/eks_mgmt.go | 16 +- tests/e2e/harness/env.go | 13 +- tests/e2e/harness/fixtures.go | 51 +- tests/e2e/harness/imds_diag.go | 26 +- tests/e2e/harness/log.go | 23 +- tests/e2e/harness/multinode_aws_via.go | 10 +- tests/e2e/harness/multinode_cluster.go | 20 +- tests/e2e/harness/multinode_diag.go | 11 +- tests/e2e/harness/multinode_distribution.go | 18 +- tests/e2e/harness/multinode_guest_ssh.go | 10 +- tests/e2e/harness/multinode_node_lifecycle.go | 20 +- tests/e2e/harness/ovn.go | 15 +- tests/e2e/harness/peer_ssh.go | 7 +- tests/e2e/harness/poll.go | 10 +- tests/e2e/harness/rawhttp.go | 14 +- tests/e2e/harness/reset.go | 57 +- tests/e2e/harness/spxcli.go | 21 +- tests/e2e/harness/ssh.go | 14 +- tests/e2e/harness/state.go | 7 +- tests/e2e/harness/sweep.go | 12 +- tests/e2e/harness/tls.go | 7 +- tests/e2e/harness/wait.go | 10 +- tests/e2e/harness/witness.go | 73 +-- tests/e2e/lb/lb_test.go | 134 ++--- tests/e2e/multinode/baseline_test.go | 52 +- tests/e2e/multinode/cluster_health_test.go | 24 +- tests/e2e/multinode/crossnode_ops_test.go | 12 +- tests/e2e/multinode/distribution_test.go | 17 +- tests/e2e/multinode/gateway_test.go | 11 +- tests/e2e/multinode/helpers_test.go | 33 +- tests/e2e/multinode/ipsec_test.go | 28 +- tests/e2e/multinode/main_test.go | 19 +- tests/e2e/multinode/node_failure_test.go | 20 +- tests/e2e/multinode/node_recovery_test.go | 16 +- tests/e2e/multinode/placement_nat_test.go | 115 +--- tests/e2e/multinode/preflight_test.go | 13 +- tests/e2e/multinode/tests_test.go | 85 +-- tests/e2e/multinode/volume_test.go | 12 +- tests/e2e/multinode/vpc_test.go | 28 +- tests/e2e/reboot/reboot_test.go | 187 ++---- tests/e2e/single/account_test.go | 81 +-- .../single/assumed_role_enforcement_test.go | 31 +- tests/e2e/single/baseline_test.go | 82 +-- tests/e2e/single/helpers_test.go | 22 +- tests/e2e/single/iam_helpers_test.go | 17 +- tests/e2e/single/iam_test.go | 19 +- tests/e2e/single/tests_test.go | 85 +-- tests/e2e/single/vpc_test.go | 6 +- 320 files changed, 3282 insertions(+), 8676 deletions(-) diff --git a/internal/eksgw/client.go b/internal/eksgw/client.go index dd7441067..11b6845cb 100644 --- a/internal/eksgw/client.go +++ b/internal/eksgw/client.go @@ -1,8 +1,6 @@ -// Package eksgw is the on-VM SigV4 client EKS control-plane helpers use to -// reach the AWS gateway, instead of dialing core NATS. Both eks-gateway-publish -// (bootstrap + state relay) and eks-token-webhook (TokenReview relay) sign with -// the system (Predastore) credentials for service "eks" and POST over HTTPS; -// the gateway authorizes via IAM and relays to the host-side NATS/STS/KV. +// Package eksgw is the on-VM SigV4 client used by EKS helpers (eks-gateway-publish, +// eks-token-webhook) to POST over HTTPS to the AWS gateway instead of NATS. +// Signs with Predastore credentials for service "eks". package eksgw import ( @@ -73,10 +71,8 @@ func New(baseURL, caPath, accessKey, secretKey, region string) (*Client, error) }, nil } -// Post SigV4-signs and sends a single POST of body to path (e.g. -// "/clusters/alpha/token-review"). It returns the response body on a 2xx, or an -// error (including the gateway status + body) otherwise. No retry — callers that -// need it wrap Post in their own loop. +// Post SigV4-signs and sends body to path, returning the response body on 2xx. +// Errors include the gateway status and body. No retry; callers wrap as needed. func (c *Client) Post(path string, body []byte) ([]byte, error) { req, err := http.NewRequest(http.MethodPost, c.baseURL+path, bytes.NewReader(body)) if err != nil { diff --git a/internal/fipsboot/fipsboot.go b/internal/fipsboot/fipsboot.go index f7614e4f2..c3d711909 100644 --- a/internal/fipsboot/fipsboot.go +++ b/internal/fipsboot/fipsboot.go @@ -1,12 +1,6 @@ -// Package fipsboot enforces FIPS 140-3 mode at process startup. -// -// Blank-import this package from every binary's main: -// -// import _ "github.com/mulgadc/spinifex/internal/fipsboot" -// -// Builds with GOFIPS140=v1.0.0 link the Go Cryptographic Module v1.0.0 -// (NIST CMVP cert #5247). At runtime FIPS mode can still be turned off -// via GODEBUG=fips140=off; init() panics in that case to close the hole. +// Package fipsboot enforces FIPS 140-3 at startup via blank import in every binary. +// Requires GOFIPS140=v1.0.0 at build time; panics if GODEBUG=fips140=off disables +// the runtime FIPS mode. package fipsboot import ( diff --git a/internal/tlsconfig/curves.go b/internal/tlsconfig/curves.go index f6d1a346a..7bc30ad60 100644 --- a/internal/tlsconfig/curves.go +++ b/internal/tlsconfig/curves.go @@ -1,34 +1,13 @@ // Package tlsconfig pins the cluster-wide TLS 1.3 curve preference list. -// -// The list is hardcoded because mixed classification on a single cluster is -// not supported and the certified FIPS build is itself the per-deployment -// choice. A typo in a config string would silently downgrade where a missing -// constant cannot. +// Hardcoded to prevent silent downgrade via misconfiguration; the FIPS build +// is the per-deployment choice. package tlsconfig import "crypto/tls" -// Curves is the fixed TLS 1.3 curve allowlist: -// -// - X25519MLKEM768 — Cat 3 ML-KEM hybrid, wins under Go 1.26 stdlib filter -// - SecP384r1MLKEM1024 — Cat 5 ML-KEM hybrid, fallback for peers that prefer it -// - X25519 — classical TLS 1.3 fallback for SDKs without ML-KEM (aws CLI v2 today) -// - CurveP384 — classical TLS 1.3 fallback (Java/.NET enterprise SDKs) -// -// PQ-capable peers (browsers since 2024, Go aws-sdk v2 with Go 1.24+) always -// negotiate the hybrid first; classical fallback only triggers for peers that -// cannot do ML-KEM. Those connections still get TLS 1.3 + AES-256-GCM but -// remain HNDL-exposed — accepted trade-off until AWS CLI v2 / bundled -// OpenSSL ships ML-KEM (currently lagging upstream OpenSSL 3.5). -// -// Weak/niche entries from the Go stdlib default (SecP256r1MLKEM768, P-256, -// P-521) are intentionally excluded so they cannot be advertised. -// -// Go 1.26 stdlib treats tls.Config.CurvePreferences as an allowlist filter -// applied to the stdlib default order, not as a user-supplied order. -// -// Callers must not mutate this slice. Assign it directly to -// tls.Config.CurvePreferences; do not append to it. +// Curves is the fixed TLS 1.3 curve allowlist: two ML-KEM hybrids (PQ-capable +// peers always negotiate these) followed by two classical fallbacks for SDKs +// without ML-KEM. Weak Go defaults (SecP256r1MLKEM768, P-256, P-521) excluded. var Curves = []tls.CurveID{ tls.X25519MLKEM768, tls.SecP384r1MLKEM1024, diff --git a/internal/tlsconfig/curves_test.go b/internal/tlsconfig/curves_test.go index dc86d3b71..0dd7091b4 100644 --- a/internal/tlsconfig/curves_test.go +++ b/internal/tlsconfig/curves_test.go @@ -46,11 +46,9 @@ func TestCurves_ExcludesWeakPrimitives(t *testing.T) { } } -// TestIntegration_NegotiatesPQHybrid spins up an httptest TLS server -// configured with Curves + TLS 1.3 minimum, dials it, and asserts a PQ -// hybrid is negotiated at TLS 1.3. Per Go 1.26 stdlib filter behavior the -// winner is X25519MLKEM768; the assertion checks for any approved hybrid -// to remain stable across stdlib reorderings of our 4 entries. +// TestIntegration_NegotiatesPQHybrid verifies that a Curves-configured server +// negotiates a PQ hybrid at TLS 1.3; checks for any approved hybrid to stay +// stable across Go stdlib reorderings. func TestIntegration_NegotiatesPQHybrid(t *testing.T) { cert := selfSignedCert(t) srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -95,10 +93,8 @@ func TestIntegration_NegotiatesPQHybrid(t *testing.T) { } } -// TestIntegration_TLS12ClientRejected confirms MinVersion: tls.VersionTLS13 -// is enforced — a TLS 1.2-pinned client gets a handshake error rather than -// silently downgrading. This is the negative test that closes the HNDL gap -// on external surfaces. +// TestIntegration_TLS12ClientRejected confirms MinVersion TLS 1.3 is enforced: +// a TLS 1.2-pinned client must get a handshake error, not a silent downgrade. func TestIntegration_TLS12ClientRejected(t *testing.T) { cert := selfSignedCert(t) srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { diff --git a/spinifex/admin/admin.go b/spinifex/admin/admin.go index ef507de09..389055f11 100644 --- a/spinifex/admin/admin.go +++ b/spinifex/admin/admin.go @@ -44,13 +44,12 @@ type ConfigSettings struct { LogDir string ConfigDir string - // Add more fields as needed Node string Az string Port string BindIP string - // AdvertiseIP is the off-host dial target rendered into the local node's - // [nodes.X].advertise field. Empty → callers fall back to BindIP. + // AdvertiseIP is the off-host dial target rendered into [nodes.X].advertise. + // Empty → callers fall back to BindIP. AdvertiseIP string // Cluster settings @@ -82,10 +81,8 @@ type ConfigSettings struct { PoolPrefixLen int // Subnet prefix length (default 24) PoolDNSServers []string // DNS servers for VM DHCP (auto-detected from host) - // OperatorEmail is the address collected at install time (TUI, SPINIFEX_EMAIL, - // or `spx admin init --email`). Written under [operator] in spinifex.toml so - // reset-dev-env.sh can preserve it across wipes. Empty means no operator - // identity was supplied. + // OperatorEmail is the address collected at install time. Written under [operator] + // in spinifex.toml so it survives wipes. Empty means no identity was supplied. OperatorEmail string // Other nodes in the cluster (for config source of truth) @@ -104,9 +101,8 @@ type ConfigSettings struct { // Sets gpu_passthrough = true under [nodes..daemon]. GPUPassthrough bool - // IPSecEnabled toggles cluster-wide OVN native IPsec on intra-AZ Geneve - // (default true). Surfaced under [network] in spinifex.toml so the value - // is visible/auditable; the daemon reads it via cluster config. + // IPSecEnabled toggles cluster-wide OVN native IPsec on intra-AZ Geneve. + // Written under [network] in spinifex.toml; daemon reads it via cluster config. IPSecEnabled bool } @@ -133,22 +129,19 @@ func GenerateConfigFiles(configs []ConfigFile, configSettings ConfigSettings) er return nil } -// generateConfigFile creates a configuration file from a template +// GenerateConfigFile creates a configuration file from a template. func GenerateConfigFile(configPath string, configTemplate string, configSettings ConfigSettings) error { - // Parse the embedded template tmpl, err := template.New("config").Parse(configTemplate) if err != nil { return fmt.Errorf("failed to parse template: %w", err) } - // Create file with secure permissions f, err := os.OpenFile(configPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { return fmt.Errorf("failed to create config file: %w", err) } defer f.Close() - // Execute template if err := tmpl.Execute(f, configSettings); err != nil { return fmt.Errorf("failed to execute template: %w", err) } @@ -157,7 +150,6 @@ func GenerateConfigFile(configPath string, configTemplate string, configSettings } func GenerateCertificatesIfNeeded(configDir string, force bool, bindIP string) (caCertPath string) { - // Certificate paths caCertPath = filepath.Join(configDir, "ca.pem") caKeyPath := filepath.Join(configDir, "ca.key") serverCertPath := filepath.Join(configDir, "server.pem") @@ -171,7 +163,6 @@ func GenerateCertificatesIfNeeded(configDir string, force bool, bindIP string) ( if needsGeneration { fmt.Println("\n🔐 Generating Certificate Authority and SSL certificates...") - // Step 1: Generate CA certificate if err := GenerateCACert(caCertPath, caKeyPath); err != nil { fmt.Fprintf(os.Stderr, "Error generating CA certificate: %v\n", err) os.Exit(1) @@ -180,7 +171,6 @@ func GenerateCertificatesIfNeeded(configDir string, force bool, bindIP string) ( fmt.Printf(" CA Certificate: %s\n", caCertPath) fmt.Printf(" CA Key: %s\n", caKeyPath) - // Step 2: Generate server certificate signed by CA (with bind IP in SANs) if err := GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, bindIP); err != nil { fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err) os.Exit(1) @@ -211,17 +201,14 @@ func GenerateServerCertOnly(configDir string, bindIP string) error { serverCertPath := filepath.Join(configDir, "server.pem") serverKeyPath := filepath.Join(configDir, "server.key") - // Verify CA files exist if !FileExists(caCertPath) || !FileExists(caKeyPath) { return fmt.Errorf("CA files not found in %s", configDir) } - // Generate server cert signed by CA with this node's bind IP return GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, bindIP) } func CreateServiceDirectories(spxRoot string) { - // Create additional directories dirs := []string{ filepath.Join(spxRoot, "images"), filepath.Join(spxRoot, "amis"), @@ -238,7 +225,6 @@ func CreateServiceDirectories(spxRoot string) { fmt.Println("\n📁 Creating directory structure...") for _, dir := range dirs { - // Check if directory exists if _, err := os.Stat(dir); os.IsNotExist(err) { if err := os.MkdirAll(dir, 0750); err != nil { fmt.Fprintf(os.Stderr, "Warning: Could not create %s: %v\n", dir, err) @@ -248,16 +234,12 @@ func CreateServiceDirectories(spxRoot string) { fmt.Printf("✅ Directory structure created in %s\n", spxRoot) } -// Helper functions - func FileExists(path string) bool { _, err := os.Stat(path) return err == nil } -// ChownRecursive changes ownership of a path and all its contents to the -// specified username. Used after init to hand production directories back -// to the service user when init runs as root via sudo. +// ChownRecursive changes ownership of path and its contents to username. // Best-effort: errors are logged but do not halt the operation. func ChownRecursive(path, username string) { u, err := user.Lookup(username) @@ -306,7 +288,6 @@ func ChownRecursive(path, username string) { // SetServiceOwnership sets per-service ownership on data/config directories // and shared config files to root:spinifex with correct modes. -// Keep in sync with setup.sh create_directories() and plan doc section 2. func SetServiceOwnership() { grp, err := user.LookupGroup("spinifex") if err != nil { @@ -379,12 +360,11 @@ func SetServiceOwnership() { } } -// updateAWSINIFile updates or creates an AWS INI file section with given key-value pairs +// UpdateAWSINIFile updates or creates an AWS INI file section with the given key-value pairs. func UpdateAWSINIFile(path, section string, values map[string]string) error { var cfg *ini.File var err error - // Load existing file or create new one if FileExists(path) { cfg, err = ini.Load(path) if err != nil { @@ -394,25 +374,21 @@ func UpdateAWSINIFile(path, section string, values map[string]string) error { cfg = ini.Empty() } - // Get or create section sec, err := cfg.NewSection(section) if err != nil { - // Section already exists, get it + // Section already exists, get it. sec, err = cfg.GetSection(section) if err != nil { return fmt.Errorf("failed to get section: %w", err) } } - // Set key-value pairs for key, value := range values { sec.Key(key).SetValue(value) } - // Write atomically with 0600. ini.SaveTo uses os.Create (0666 & ~umask, - // typically 0644), which leaves ~/.aws/credentials world-readable. Render - // to a sibling temp file (os.CreateTemp is 0600) and rename into place so - // the secrets are never exposed, even briefly. + // Write atomically: ini.SaveTo uses os.Create (world-readable); render to a + // sibling temp file (0600) and rename to avoid briefly exposing secrets. dir := filepath.Dir(path) tmp, err := os.CreateTemp(dir, ".aws-ini-*.tmp") if err != nil { @@ -433,8 +409,7 @@ func UpdateAWSINIFile(path, section string, values map[string]string) error { return nil } -// generateAWSAccessKey generates an AWS-style access key -// Format: AKIA + 16 random uppercase alphanumeric characters +// GenerateAWSAccessKey generates an AWS-style access key (AKIA + 16 random alphanumeric chars). func GenerateAWSAccessKey() (string, error) { const prefix = "AKIA" const charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" @@ -452,8 +427,7 @@ func GenerateAWSAccessKey() (string, error) { return prefix + string(result), nil } -// generateAWSSecretKey generates an AWS-style secret key -// 40 character base64-encoded string +// GenerateAWSSecretKey generates a 40-character base64-encoded AWS-style secret key. func GenerateAWSSecretKey() (string, error) { bytes := make([]byte, 30) // 30 bytes = 40 chars in base64 if _, err := rand.Read(bytes); err != nil { @@ -488,15 +462,13 @@ func GenerateNATSToken() (string, error) { return "nats_" + base64.URLEncoding.EncodeToString(bytes)[:32], nil } -// GenerateCACert generates a Certificate Authority certificate and key +// GenerateCACert generates a Certificate Authority certificate and key. func GenerateCACert(caCertPath, caKeyPath string) error { - // Generate CA private key caPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096) if err != nil { return fmt.Errorf("failed to generate CA private key: %w", err) } - // Create CA certificate template notBefore := time.Now() notAfter := notBefore.Add(3650 * 24 * time.Hour) // 10 years @@ -519,13 +491,11 @@ func GenerateCACert(caCertPath, caKeyPath string) error { MaxPathLen: 1, } - // Self-sign the CA certificate caDerBytes, err := x509.CreateCertificate(rand.Reader, &caTemplate, &caTemplate, &caPrivateKey.PublicKey, caPrivateKey) if err != nil { return fmt.Errorf("failed to create CA certificate: %w", err) } - // Write CA certificate to file caCertOut, err := os.Create(caCertPath) if err != nil { return fmt.Errorf("failed to create CA cert file: %w", err) @@ -536,7 +506,6 @@ func GenerateCACert(caCertPath, caKeyPath string) error { return fmt.Errorf("failed to write CA cert: %w", err) } - // Write CA private key to file caKeyOut, err := os.OpenFile(caKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { return fmt.Errorf("failed to create CA key file: %w", err) @@ -752,15 +721,13 @@ func GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath string, return nil } -// generateSelfSignedCert generates a self-signed SSL certificate (legacy, kept for compatibility) +// GenerateSelfSignedCert generates a self-signed SSL certificate (legacy, kept for compatibility). func GenerateSelfSignedCert(certPath, keyPath string) error { - // Generate private key privateKey, err := rsa.GenerateKey(rand.Reader, 4096) if err != nil { return fmt.Errorf("failed to generate private key: %w", err) } - // Create certificate template notBefore := time.Now() notAfter := notBefore.Add(3650 * 24 * time.Hour) // 10 years @@ -784,13 +751,11 @@ func GenerateSelfSignedCert(certPath, keyPath string) error { IPAddresses: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}, } - // Create certificate derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey) if err != nil { return fmt.Errorf("failed to create certificate: %w", err) } - // Write certificate to file certOut, err := os.Create(certPath) if err != nil { return fmt.Errorf("failed to create cert file: %w", err) @@ -801,7 +766,6 @@ func GenerateSelfSignedCert(certPath, keyPath string) error { return fmt.Errorf("failed to write cert: %w", err) } - // Write private key to file keyOut, err := os.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { return fmt.Errorf("failed to create key file: %w", err) @@ -821,11 +785,6 @@ func GenerateSelfSignedCert(certPath, keyPath string) error { } // SetupAWSCredentials updates ~/.aws/credentials and ~/.aws/config. -// bindIP is the IP the AWS gateway listens on. If empty or "0.0.0.0", the -// operator's local ~/.aws/config endpoint_url falls back to "localhost" (this -// runs on the same box as the gateway). wanIP is reserved for a future -// --operator-endpoint flag that will let remote operators point their CLI at -// the host's WAN IP; today it is accepted but unused. // When running under sudo, writes to SUDO_USER's home instead of root's. func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP, wanIP string) error { _ = wanIP @@ -834,8 +793,6 @@ func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP, wanIP s return err } - // When running under sudo, write to the invoking user's home directory - // so the operator can use AWS_PROFILE=spinifex without sudo. sudoUser := os.Getenv("SUDO_USER") if os.Getuid() == 0 && sudoUser != "" { if u, err := user.Lookup(sudoUser); err == nil { @@ -851,21 +808,8 @@ func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP, wanIP s credPath := filepath.Join(awsDir, "credentials") configPath := filepath.Join(awsDir, "config") - // Determine profile name - //profileName := "default" - - // Use Spinifex as the default profile profileName := "spinifex" - if FileExists(credPath) { - // Check if default profile already exists - cfg, err := ini.Load(credPath) - if err == nil && cfg.HasSection("default") { - profileName = "spinifex" - } - } - - // Update credentials file if err := UpdateAWSINIFile(credPath, profileName, map[string]string{ "aws_access_key_id": accessKey, "aws_secret_access_key": secretKey, @@ -873,7 +817,6 @@ func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP, wanIP s return err } - // Update config file configSection := profileName if profileName != "default" { configSection = "profile " + profileName @@ -961,9 +904,8 @@ func ParsePredastoreNodeIDFromConfig(tomlContent string, ip string) int { return FindNodeIDByIP(cfg.DB, ip) } -// SetMIGProfile idempotently writes mig_profile = "" for the given -// node into spinifex.toml, preserving all other content and comments. -// An empty profile clears the setting (writes mig_profile = ""). +// SetMIGProfile idempotently writes mig_profile = "" for the given node +// into spinifex.toml. An empty profile clears the setting. func SetMIGProfile(tomlPath, node, profile string) error { raw, err := os.ReadFile(tomlPath) if err != nil { diff --git a/spinifex/admin/dial.go b/spinifex/admin/dial.go index ccaa96d9a..4db248d27 100644 --- a/spinifex/admin/dial.go +++ b/spinifex/admin/dial.go @@ -2,13 +2,8 @@ package admin import "net" -// DialTarget returns a host:port suitable for an in-process client to dial, -// given a configured listen address. Inputs of the form "0.0.0.0:N" or "[::]:N" -// are rewritten to "127.0.0.1:N" or "[::1]:N" — a service listening on the -// wildcard is reachable on loopback, and loopback is the TLS SAN that the -// daemon's own cert always covers. Loopback, specific IPs, and bare hostnames -// pass through unchanged. Malformed input (no host:port) passes through so -// callers still see the original value in error messages. +// DialTarget returns a host:port for in-process dialing. Wildcard listeners +// ("0.0.0.0:N", "[::]:N") are rewritten to loopback; specific IPs pass through. func DialTarget(listenAddr string) string { host, port, err := net.SplitHostPort(listenAddr) if err != nil { @@ -23,12 +18,8 @@ func DialTarget(listenAddr string) string { return listenAddr } -// AdvertiseHost picks the host that off-host clients should dial for a service -// listening on listenAddr. If the listen host is a wildcard (0.0.0.0, ::, or -// empty), the caller-supplied advertiseIP is returned; otherwise the listen -// host itself is returned. Port is intentionally dropped — callers append the -// port appropriate for the protocol (some services listen and advertise on -// different ports). +// AdvertiseHost returns the host off-host clients should dial. Wildcard listeners +// return advertiseIP; specific listen hosts return themselves (port is dropped). func AdvertiseHost(listenAddr, advertiseIP string) string { host, _, err := net.SplitHostPort(listenAddr) if err != nil { diff --git a/spinifex/admin/email.go b/spinifex/admin/email.go index c99a28536..09c9b1586 100644 --- a/spinifex/admin/email.go +++ b/spinifex/admin/email.go @@ -26,12 +26,8 @@ func ValidateEmail(addr string) error { return nil } -// ReadOperatorEmail extracts the [operator].email scalar from a spinifex.toml -// file, or returns "" if the file doesn't exist, can't be read, or has no -// [operator] section. Used by `spx admin init --force` (and reset flows) to -// preserve a previously-captured email across re-inits when --email isn't -// supplied on the new invocation. Minimal text scan — no TOML parser, since -// we only need one scalar from one section. +// ReadOperatorEmail extracts the [operator].email scalar from spinifex.toml, +// returning "" on any error or missing section. Uses a text scan, not a full TOML parser. func ReadOperatorEmail(tomlPath string) string { f, err := os.Open(tomlPath) if err != nil { diff --git a/spinifex/admin/images_remove.go b/spinifex/admin/images_remove.go index ac12b08cb..45642d71a 100644 --- a/spinifex/admin/images_remove.go +++ b/spinifex/admin/images_remove.go @@ -46,10 +46,8 @@ func (d Dependents) Empty() bool { return len(d.Snapshots) == 0 && len(d.Volumes) == 0 && len(d.AMIs) == 0 } -// RemovePreview captures everything the CLI needs to render the confirmation -// prompt and decide whether to proceed: AMI metadata (or zero values when the -// config is missing/corrupt), backing-storage byte counts, and the dependent -// resource set. PreviewRemoveSystemImage performs no deletions. +// RemovePreview captures AMI metadata, byte counts, and dependents for the +// CLI confirmation prompt. PreviewRemoveSystemImage performs no deletions. type RemovePreview struct { ImageID string Name string @@ -74,11 +72,8 @@ func SnapPrefix(imageID string) string { return "snap-" + imageID } -// PreviewRemoveSystemImage walks the object store and gathers everything the -// caller needs to decide whether to remove an admin-imported AMI. It performs -// no mutations. Returned errors indicate unexpected I/O failures only — -// AMI-not-found, corrupt config, and account-owned AMIs are reflected in the -// preview fields, not as errors. +// PreviewRemoveSystemImage gathers AMI metadata and dependents without mutating +// any state. Not-found, corrupt config, and account-owned AMI are preview fields. func PreviewRemoveSystemImage(store objectstore.ObjectStore, bucket, imageID string) (*RemovePreview, error) { if !strings.HasPrefix(imageID, "ami-") { return nil, errors.New(awserrors.ErrorInvalidAMIIDMalformed) @@ -125,16 +120,8 @@ func PreviewRemoveSystemImage(store objectstore.ObjectStore, bucket, imageID str return preview, nil } -// FindAMIDependents walks the bucket to find every resource that transitively -// depends on the given admin-imported AMI: -// -// 1. snap-*/metadata.json whose VolumeID == imageID → derivedSnaps -// (these are CopyImage-created snapshots that point back at the system AMI) -// 2. vol-*/config.json whose VolumeMetadata.SnapshotID ∈ {snap-ami-} ∪ derivedSnaps -// 3. ami-*/config.json whose AMIMetadata.SnapshotID ∈ derivedSnaps -// -// The walk terminates one hop deep. CopyImage of an account AMI uses its own -// VolumeID, not the system AMI's, so further recursion is not required today. +// FindAMIDependents returns snapshots, volumes, and AMIs that depend on the +// given system AMI. Walk terminates one hop deep. func FindAMIDependents(store objectstore.ObjectStore, bucket, imageID string) (Dependents, error) { var deps Dependents @@ -220,16 +207,9 @@ func FindAMIDependents(store objectstore.ObjectStore, bucket, imageID string) (D return deps, nil } -// RemoveSystemImage deletes an admin-imported AMI and its backing storage -// after re-validating the preconditions: -// -// 1. ami-/config.json exists and parses (skipped under --force = salvage) -// 2. AMI is system-owned, not account-owned (--force bypasses) -// 3. No volumes / snapshots / AMIs depend on it (--force bypasses) -// -// Deletion order is intentional: the config.json is deleted FIRST so the AMI -// disappears from DescribeImages immediately, preventing new RunInstances -// from racing the block cleanup. +// RemoveSystemImage deletes an admin-imported AMI after re-validating that it +// is system-owned and has no dependents (bypassed by --force). config.json is +// deleted first so the AMI vanishes from DescribeImages before block cleanup. func RemoveSystemImage(store objectstore.ObjectStore, bucket string, opts RemoveImageOpts) (*RemoveImageResult, error) { if !strings.HasPrefix(opts.ImageID, "ami-") { return nil, errors.New(awserrors.ErrorInvalidAMIIDMalformed) @@ -344,10 +324,8 @@ func readAMIConfig(store objectstore.ObjectStore, bucket, imageID string) (viper return state.VolumeConfig.AMIMetadata, nil } -// errCorruptVolumeConfig lets the dependency walk distinguish a vol-*/config.json -// that can't be parsed (recovery is operator-owned, walk continues) from a -// transient transport error (walk must fail closed — silently dropping a -// dependent could lead to deleting blocks underneath a live volume). +// errCorruptVolumeConfig distinguishes an unparse-able config (walk continues) +// from a transport error (walk fails closed to prevent deleting live blocks). var errCorruptVolumeConfig = errors.New("corrupt volume config") // readVolumeConfig reads vol-/config.json into VolumeConfig. diff --git a/spinifex/admin/ipsec.go b/spinifex/admin/ipsec.go index c08acf63e..040c9114f 100644 --- a/spinifex/admin/ipsec.go +++ b/spinifex/admin/ipsec.go @@ -33,33 +33,16 @@ const ( // strongSwan config consumes them as-is. var oidExtKeyUsageIPSecIKE = asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 3, 17} -// IPSecCertPaths returns the canonical paths for the per-node IPsec peer -// credential pair under configDir. Both paths live in -// {configDir}/ipsec/. The CA trust anchor is the existing cluster CA in -// configDir/ca.pem; ovs-monitor-ipsec is pointed at that path directly. +// IPSecCertPaths returns the canonical cert/key paths for the per-node IPsec peer +// credential pair under {configDir}/ipsec/. func IPSecCertPaths(configDir string) (certPath, keyPath string) { dir := filepath.Join(configDir, ipsecDirName) return filepath.Join(dir, ipsecCertFileName), filepath.Join(dir, ipsecKeyFileName) } -// GenerateIPSecPeerCert issues an IKEv2 peer certificate for this node, -// signed by the existing cluster CA. The cert carries the IPSec-IKE -// extended key usage so strongSwan accepts it as a peer credential. -// -// hostname is the node's primary hostname (used as Common Name + DNS SAN). -// nodeIP is the node's primary cluster IP (added as an IP SAN). Additional -// SANs discovered from the local machine's interfaces are not added here — -// IPSec peer identity should be the explicit cluster identity only, not -// every interface address, to keep peer-identity matching strict. -// -// Cross-cutting invariant: this dnsName SAN must equal the OVS chassis-id -// (Open_vSwitch.external_ids:system-id) on every node, because -// ovs-monitor-ipsec generates strongSwan configs with leftid=@ -// / rightid=@ and charon validates those names against the -// peer cert's dnsName SAN. setup-ovn.sh pins system-id to `hostname -s` -// for exactly this reason — see the comment in setup-ovn.sh near -// `system-id.conf`. Changing the cert identity here without updating -// setup-ovn.sh (or vice versa) breaks IKE with AUTHENTICATION_FAILED. +// GenerateIPSecPeerCert issues an IKEv2 peer certificate for this node signed +// by the existing cluster CA. The cert carries the IPSec-IKE EKU so strongSwan +// accepts it. The dnsName SAN must equal the OVS chassis-id (system-id). func GenerateIPSecPeerCert(configDir, caCertPath, caKeyPath, hostname, nodeIP string) error { if hostname == "" { return fmt.Errorf("ipsec peer cert: hostname required") @@ -127,14 +110,8 @@ func GenerateIPSecPeerCert(configDir, caCertPath, caKeyPath, hostname, nodeIP st return fmt.Errorf("ipsec peer cert: write key: %w", err) } - // Symlink the cluster CA into charon's default cacerts scan dir. - // ovs-monitor-ipsec writes per-conn `cacert=` references, but - // strongSwan 6.0 only loads CAs declared in standalone `ca` sections - // or files placed in /etc/ipsec.d/cacerts at startup. Without this, - // charon authenticates the local side OK but rejects every inbound - // peer cert as `no trusted RSA public key found for ''` and - // no SAs negotiate. The dir is on charon's default AppArmor allowlist - // so no profile change is needed (unlike /etc/spinifex/). + // strongSwan 6.0 only loads CAs placed in /etc/ipsec.d/cacerts at startup; + // per-conn cacert= references are insufficient. Install the cluster CA there. if err := installCAIntoCharonTrustStore(caCertPath); err != nil { return fmt.Errorf("ipsec peer cert: install CA into charon trust store: %w", err) } @@ -166,15 +143,8 @@ func installCAIntoCharonTrustStore(caCertPath string) error { return fmt.Errorf("rename %s -> %s: %w", staging, link, err) } - // strongSwan scans /etc/ipsec.d/cacerts only at charon startup. If - // openvswitch-ipsec.service started before this install (the common - // case — the unit is enabled at provision time), charon is running - // with zero CAs loaded and every IKE_AUTH ends in `no trusted RSA - // public key found for ''`. Trigger a stroke rereadcacerts so - // the symlink we just placed is loaded without bouncing charon. - // Silent failure is expected when charon isn't running yet (admin - // init can run before setup-ovn.sh enables the unit); a later charon - // start will scan cacerts naturally. + // Trigger rereadcacerts so charon picks up the symlink without restart. + // Silent failure is expected when charon isn't running yet. if err := charonRereadCAs(); err != nil { slog.Debug("ipsec: rereadcacerts skipped", "err", err) } diff --git a/spinifex/admin/ipsec_test.go b/spinifex/admin/ipsec_test.go index faefbc3ee..c0684ead2 100644 --- a/spinifex/admin/ipsec_test.go +++ b/spinifex/admin/ipsec_test.go @@ -127,12 +127,8 @@ func TestIPSecCertPaths(t *testing.T) { assert.Equal(t, "/etc/spinifex/ipsec/peer.key", keyPath) } -// TestInstallCAIntoCharonTrustStore_TriggersRereadCAs locks the fix for the -// race where openvswitch-ipsec started before admin init's CA install: every -// IKE_AUTH failed with `no trusted RSA public key found for ''` because -// charon had scanned cacerts before the symlink was placed. The trust-store -// installer must trigger `ipsec rereadcacerts` so an already-running charon -// picks up the CA without a restart. +// TestInstallCAIntoCharonTrustStore_TriggersRereadCAs ensures the installer +// calls `ipsec rereadcacerts` so an already-running charon picks up the CA. func TestInstallCAIntoCharonTrustStore_TriggersRereadCAs(t *testing.T) { origTrustDir := charonCATrustDir charonCATrustDir = t.TempDir() @@ -160,9 +156,7 @@ func TestInstallCAIntoCharonTrustStore_TriggersRereadCAs(t *testing.T) { } // TestInstallCAIntoCharonTrustStore_SwallowsRereadError verifies that a failed -// rereadcacerts (e.g. charon not running, ipsec binary missing on a fresh -// install) does not break admin init. The CA symlink must still be in place so -// a later charon start picks it up via the normal cacerts scan. +// rereadcacerts (charon not running) doesn't abort admin init. func TestInstallCAIntoCharonTrustStore_SwallowsRereadError(t *testing.T) { origTrustDir := charonCATrustDir charonCATrustDir = t.TempDir() diff --git a/spinifex/admin/network_detect.go b/spinifex/admin/network_detect.go index 06de6753f..c1973fecb 100644 --- a/spinifex/admin/network_detect.go +++ b/spinifex/admin/network_detect.go @@ -54,7 +54,6 @@ func DetectNetwork() (*DetectedNetwork, error) { return nil, fmt.Errorf("could not parse default route: %s", defaultRoute) } - // Get all routes to find interfaces and their subnets allRoutes, err := exec.Command("ip", "-4", "route", "show", "scope", "link").Output() if err != nil { return nil, fmt.Errorf("failed to get routes: %w", err) @@ -149,7 +148,6 @@ func SuggestPoolRange(wan *DetectedInterface) (start, end string) { return "", "" } - // Calculate broadcast address ip := ipNet.IP.To4() mask := ipNet.Mask broadcast := make(net.IP, 4) @@ -182,13 +180,9 @@ func SuggestPoolRange(wan *DetectedInterface) (start, end string) { return startIP.String(), endIP.String() } -// isVirtualInterface returns true for interfaces that shouldn't be -// considered as physical NICs. The defaultRouteDev parameter is the -// interface carrying the default route — it's always considered physical -// even if its name matches a virtual prefix (e.g., br-wan). +// isVirtualInterface returns true for interfaces that shouldn't be considered +// physical NICs. defaultRouteDev is always treated as physical. func isVirtualInterface(name string, defaultRouteDev string) bool { - // The default route interface is always considered physical — it's the - // WAN uplink, even if it's a bridge (br-wan, br-external, etc.). if defaultRouteDev != "" && name == defaultRouteDev { return false } diff --git a/spinifex/awsec2query/parser.go b/spinifex/awsec2query/parser.go index d1a3a3141..3d25925c7 100644 --- a/spinifex/awsec2query/parser.go +++ b/spinifex/awsec2query/parser.go @@ -13,27 +13,6 @@ import ( // Callers should map this to AWS's MalformedQueryString error code. var ErrSliceTooLarge = errors.New("list parameter exceeds maximum entries") -/* - -Sample args - -args["Action"] = "RunInstance" -args["ImageId"] = "ami-123456789" -args["MinCount"] = "1" -args["MaxCount"] = "1" -args["InstanceType"] = "t2.micro" -args["KeyName"] = "my-key-pair" -args["SecurityGroup.1"] = "sg-12345678" -args["SecurityGroup.2"] = "sg-87654321" -args["TagSpecification.1.ResourceType"] = "instance" -args["TagSpecification.1.Tag.1.Key"] = "Name" -args["TagSpecification.1.Tag.1.Value"] = "MyInstance" -args["TagSpecification.2.ResourceType"] = "volume" -args["TagSpecification.2.Tag.1.Key"] = "Environment" -args["TagSpecification.2.Tag.1.Value"] = "Production" - -*/ - func QueryParamsToStruct(params map[string]string, out any) error { v := reflect.ValueOf(out) @@ -67,10 +46,8 @@ func setStructFields(v reflect.Value, params map[string]string, prefix string) e // Check for locationName tag (AWS SDK uses this for query parameter names) locationName := fieldType.Tag.Get("locationName") - // Try the field name, locationName, and title-cased locationName. - // AWS query params use title case (e.g. "ResourceId") but some SDK - // structs use camelCase locationName (e.g. "resourceId" in DeleteTagsInput - // vs "ResourceId" in CreateTagsInput). + // Try field name, locationName, and title-cased locationName. + // AWS query params use title case but some SDK structs use camelCase locationName. queryKeys := []string{prefix + fieldName} if locationName != "" && locationName != fieldName { queryKeys = append(queryKeys, prefix+locationName) @@ -153,13 +130,12 @@ func setFieldValue(field reflect.Value, value string) error { elem := field.Type().Elem() // Special case: []byte if elem.Kind() == reflect.Uint8 { - // First try base64 (what the AWS API expects for PublicKeyMaterial). + // Try base64 first (AWS API expectation); fall back to raw bytes. trimmed := strings.TrimSpace(value) if decoded, err := base64.StdEncoding.DecodeString(trimmed); err == nil { field.SetBytes(decoded) return nil } - // Some clients may send raw text. Fall back to raw bytes. field.SetBytes([]byte(value)) return nil } @@ -195,8 +171,7 @@ func setFieldValue(field reflect.Value, value string) error { const maxSliceLen = 1024 func setSliceField(field reflect.Value, params map[string]string, prefix string) error { - // Find all indexed items for this slice. - // Supports both EC2-style (Prefix.N) and IAM/ELBv2-style (Prefix.member.N) formats. + // Collect indexed items; supports EC2-style (Prefix.N) and IAM/ELBv2-style (Prefix.member.N). indices := make(map[int]bool) useMember := false for key := range params { diff --git a/spinifex/awserrors/awserrors.go b/spinifex/awserrors/awserrors.go index 554bac72c..52d2c8a55 100644 --- a/spinifex/awserrors/awserrors.go +++ b/spinifex/awserrors/awserrors.go @@ -221,9 +221,7 @@ var ( ErrorInvalidOptionConflict = "InvalidOption.Conflict" ErrorInvalidPaginationToken = "InvalidPaginationToken" ErrorInvalidParameter = "InvalidParameter" - // ACM (AWS JSON 1.1). Suffix-less; the JSON error envelope appends - // "Exception" on the wire (InvalidArnException), matching aws-sdk-go acm - // ErrCode* values. Not-found reuses the shared ErrorResourceNotFound. + // ACM error: the JSON envelope appends "Exception" on the wire; not-found reuses ErrorResourceNotFound. ErrorACMInvalidArn = "InvalidArn" ErrorInvalidParameterCombination = "InvalidParameterCombination" ErrorInvalidParameterDependency = "InvalidParameterDependency" @@ -372,9 +370,7 @@ var ( ErrorResourceCountExceeded = "ResourceCountExceeded" ErrorResourceCountLimitExceeded = "ResourceCountLimitExceeded" ErrorResourceLimitExceeded = "ResourceLimitExceeded" - // ErrorResourceNotFound ("ResourceNotFound") is the ACM not-found code. EKS - // must use ErrorEKSResourceNotFound ("ResourceNotFoundException", below) — - // the restjson code the aws eks client expects. Do not cross-wire the two. + // ErrorResourceNotFound is the ACM not-found code. EKS must use ErrorEKSResourceNotFound — do not cross-wire. ErrorResourceNotFound = "ResourceNotFound" ErrorEKSResourceInUse = "ResourceInUseException" ErrorEKSResourceNotFound = "ResourceNotFoundException" @@ -495,11 +491,8 @@ func ValidErrorCode(code string) string { return ErrorServerInternal } -// IsErrorCode reports whether err carries the named AWS error code. Service -// handlers return these codes as plain strings (errors.New(code)), commonly -// wrapped with %w by orchestration callers, so the match is a substring test -// against err.Error() rather than equality. Use it to classify a delete error -// as idempotent (already-gone) or retryable (still-in-use) in teardown paths. +// IsErrorCode reports whether err carries the named AWS error code. +// Uses substring matching because handlers return codes as plain strings, sometimes wrapped with %w. func IsErrorCode(err error, code string) bool { return err != nil && strings.Contains(err.Error(), code) } diff --git a/spinifex/awserrors/awserrors_test.go b/spinifex/awserrors/awserrors_test.go index 4f060d879..c848bb2ea 100644 --- a/spinifex/awserrors/awserrors_test.go +++ b/spinifex/awserrors/awserrors_test.go @@ -2,11 +2,8 @@ package awserrors import "testing" -// TestErrorLookup_Structure asserts invariants of the ErrorLookup map without -// duplicating every entry. Catches wholesale-deletion accidents, invalid HTTP -// codes, empty messages, and regressions of SDK-surfaced codes — the failure -// modes that actually matter — without locking in a 1:1 mirror that has to be -// updated in lockstep with every new error code. +// TestErrorLookup_Structure asserts ErrorLookup invariants: minimum size, valid HTTP codes, non-empty messages. +// Avoids a 1:1 mirror that would need updating with every new error code. func TestErrorLookup_Structure(t *testing.T) { if len(ErrorLookup) < 400 { t.Fatalf("ErrorLookup unexpectedly small: %d entries", len(ErrorLookup)) diff --git a/spinifex/config/config.go b/spinifex/config/config.go index 2d00d1022..4fb7ae28d 100644 --- a/spinifex/config/config.go +++ b/spinifex/config/config.go @@ -33,12 +33,8 @@ type ExternalPool struct { DNSServers []string `mapstructure:"dns_servers"` // DNS servers for VM DHCP (auto-detected from host; fallback: 8.8.8.8, 1.1.1.1) Region string `mapstructure:"region"` // Scope to region (optional — empty means any region) AZ string `mapstructure:"az"` // Scope to AZ (optional — empty means any AZ in region) - // GwLrpRangeStart/End reserve a sub-range of the LAN for OVN gateway LRP IPs - // in centralized NAT mode (veth). Each VPC consumes one IP from this - // range so its gateway router port can ARP with a sender IP on the LAN - // subnet — link-local 169.254.0.1/30 makes upstream routers reject ARP per - // RFC 826. Must NOT overlap [RangeStart, RangeEnd] or IPAM and vpcd will - // fight over the same IP. + // GwLrpRangeStart/End reserve a sub-range for OVN gateway LRP IPs in centralized NAT mode. + // Must NOT overlap [RangeStart, RangeEnd] — link-local 169.254/16 is rejected by upstream routers. GwLrpRangeStart string `mapstructure:"gw_lrp_range_start"` GwLrpRangeEnd string `mapstructure:"gw_lrp_range_end"` } @@ -47,15 +43,12 @@ type ExternalPool struct { type NetworkConfig struct { ExternalMode string `mapstructure:"external_mode"` // "pool" or "" (disabled) ExternalPools []ExternalPool `mapstructure:"external_pools"` // One or more IP pools - // IPSecEnabled toggles OVN native IPsec on every node for intra-AZ Geneve - // (AES-256-GCM via strongSwan). Default true. Operators can flip off for - // trusted single-rack lab topologies; production edge stays on. + // IPSecEnabled toggles OVN native IPsec (AES-256-GCM) on every node. Default true; disable only for trusted lab topologies. IPSecEnabled bool `mapstructure:"ipsec_enabled"` } // BootstrapConfig holds the default VPC infrastructure IDs written by admin init. -// vpcd reads this on startup to ensure OVN topology exists for the bootstrap VPC, -// covering the case where admin init ran before services were started. +// vpcd reads this on startup to ensure OVN topology exists for the bootstrap VPC. type BootstrapConfig struct { AccountID string `mapstructure:"account_id"` VpcId string `mapstructure:"vpc_id"` @@ -70,8 +63,7 @@ type Config struct { // Node config Node string `json:"Node" mapstructure:"node"` Host string `json:"Host" mapstructure:"host"` // Unique hostname or IP of this node - // AdvertiseIP is the off-host dial target for this node. Empty → callers - // fall back to Host for backward compat with pre-siv-8 cluster configs. + // AdvertiseIP is the off-host dial target. Empty falls back to Host for backward compat. AdvertiseIP string `json:"AdvertiseIP" mapstructure:"advertise"` Region string `json:"Region" mapstructure:"region"` AZ string `json:"AZ" mapstructure:"az"` @@ -102,12 +94,8 @@ type AWSGWConfig struct { type ViperblockConfig struct { ShardWAL *bool `json:"ShardWAL" mapstructure:"shardwal"` // Enable sharded WAL (default false when nil) - // EncryptionKeyFile is the path to the shared 32-byte AES-256 master key - // for viperblock at-rest encryption (chunks, WAL records, VBState/snapshot - // metadata). When empty, volumes are written in cleartext (legacy mode). - // When set, every VB constructed on this node — daemons and AWS handlers - // alike — must load it via masterkey.LoadShared so existing volumes can - // still be opened. + // EncryptionKeyFile is the path to the shared 32-byte AES-256 master key for viperblock at-rest encryption. + // Empty means cleartext. When set, all VB instances must load it via masterkey.LoadShared. EncryptionKeyFile string `json:"EncryptionKeyFile" mapstructure:"encryption_key_file"` } @@ -139,11 +127,9 @@ type GPUModelOverride struct { Manufacturer string `json:"Manufacturer" mapstructure:"manufacturer"` Name string `json:"Name" mapstructure:"name"` MemoryMiB int64 `json:"MemoryMiB" mapstructure:"memory_mib"` - // XVGAOff forces x-vga=off for this GPU in QEMU passthrough, overriding the - // default (on for consumer GPUs, off for known datacenter cards). + // XVGAOff forces x-vga=off in QEMU passthrough, overriding the per-GPU default. XVGAOff bool `json:"XVGAOff" mapstructure:"xvga_off"` - // MIGProfile overrides the daemon-level MIGProfile for this specific GPU. - // Takes the same format as DaemonConfig.MIGProfile (e.g. "1g.10gb"). + // MIGProfile overrides the daemon-level MIGProfile for this GPU (same format, e.g. "1g.10gb"). MIGProfile string `json:"MIGProfile" mapstructure:"mig_profile"` } @@ -156,9 +142,7 @@ type DaemonConfig struct { MgmtBridge string `json:"MgmtBridge" mapstructure:"mgmt_bridge"` // Linux bridge for system instance control plane (default "br-mgmt") GPUPassthrough bool `json:"GPUPassthrough" mapstructure:"gpu_passthrough"` // Enable VFIO GPU passthrough for g5.* instance types GPUModelOverrides []GPUModelOverride `json:"GPUModelOverrides" mapstructure:"gpu_model_overrides"` // Dev/test GPU mappings not in the production model list - // MIGProfile enables NVIDIA MIG partitioning on all eligible GPUs on this node. - // Set to a profile name such as "1g.10gb" to activate; empty disables MIG. - // Individual GPUs can override this via GPUModelOverrides[].MIGProfile. + // MIGProfile enables NVIDIA MIG on all eligible GPUs (e.g. "1g.10gb"); empty disables. Per-GPU override via GPUModelOverrides[].MIGProfile. MIGProfile string `json:"MIGProfile" mapstructure:"mig_profile"` } @@ -180,8 +164,7 @@ type NATSSub struct { Subject string `json:"Subject" mapstructure:"subject"` } -// NodeBaseDir returns the BaseDir for the current node, or "" if the config -// is nil, the node name is unset, or the node is not found in the Nodes map. +// NodeBaseDir returns the BaseDir for the current node, or "" if config is nil, node is unset, or not found. func (cc *ClusterConfig) NodeBaseDir() string { if cc == nil || cc.Node == "" { slog.Warn("NodeBaseDir: no config or node name set, using global PID path") @@ -201,8 +184,7 @@ func (cc *ClusterConfig) NodeBaseDir() string { // AllServices is the default service list when Services is empty (backward compat). var AllServices = []string{"nats", "predastore", "viperblock", "daemon", "awsgw", "vpcd", "ui"} -// HasService reports whether the node runs the named service. -// An empty Services list means all services (backward compat). +// HasService reports whether the node runs the named service (empty list means all services). func (c Config) HasService(name string) bool { services := c.Services if len(services) == 0 { @@ -225,8 +207,7 @@ func LoadConfig(configPath string) (*ClusterConfig, error) { viper.SetEnvPrefix("SPINIFEX") viper.AutomaticEnv() - // network.ipsec_enabled defaults to true (intra-AZ Geneve encrypted via OVN - // native IPsec). Operators must explicitly set false to disable. + // Default ipsec_enabled to true; operators must explicitly set false to disable. viper.SetDefault("network.ipsec_enabled", true) // Try to load config file if it exists @@ -251,9 +232,7 @@ func LoadConfig(configPath string) (*ClusterConfig, error) { return nil, fmt.Errorf("error unmarshaling config: %w", err) } - // Normalize the local node's bind address: 0.0.0.0 means "listen on all - // interfaces" but is not a valid connect address. Only rewrite for the - // local node — remote nodes use real IPs that must not be changed. + // Rewrite 0.0.0.0 in Predastore.Host to 127.0.0.1 for the local node only (not a valid connect address). if local, ok := config.Nodes[config.Node]; ok { if strings.HasPrefix(local.Predastore.Host, "0.0.0.0") { local.Predastore.Host = strings.Replace(local.Predastore.Host, "0.0.0.0", "127.0.0.1", 1) @@ -268,9 +247,7 @@ func LoadConfig(configPath string) (*ClusterConfig, error) { return &config, nil } -// validateClusterConfig rejects legacy upstream-DHCP config and validates -// external pool ranges. The upstream-DHCP-client model was replaced by static -// WAN-pool allocation; legacy keys must be removed from the operator's TOML. +// validateClusterConfig rejects legacy DHCP config keys and validates external pool ranges. func validateClusterConfig(cc *ClusterConfig) error { if viper.IsSet("network.external_dhcp") { return fmt.Errorf("config: [network] external_dhcp is no longer supported; remove the key (static WAN-pool allocation only)") diff --git a/spinifex/daemon/daemon.go b/spinifex/daemon/daemon.go index 4e83866a1..7f560ae17 100644 --- a/spinifex/daemon/daemon.go +++ b/spinifex/daemon/daemon.go @@ -85,33 +85,26 @@ type EBS struct { // when full, it unsubscribes so NATS routes requests to other nodes. type ResourceManager struct { mu sync.RWMutex - // hostVCPU / hostMemGB are the host figures guests schedule against: - // physical cores (physicalCoreCount, not SMT threads) and /proc/meminfo. - // Schedulable capacity for guest VMs is host - reserved - allocated. + // hostVCPU / hostMemGB: physical cores (not SMT threads) and total RAM. + // Schedulable capacity = host - reserved - allocated. hostVCPU int hostMemGB float64 - // reservedVCPU / reservedMem are held back from guest scheduling for - // the spinifex daemon and co-located services (NATS, predastore, - // viperblock, vpcd, awsgw, ui). See hostReserve / defaultHostReserve. + // reservedVCPU / reservedMem: resources held back for the daemon and + // co-located services. See hostReserve / defaultHostReserve. reservedVCPU int reservedMem float64 allocatedVCPU int allocatedMem float64 - // nbdkitMainMiB / nbdkitAuxMiB are the per-volume nbdkit memory charges - // (main vs aux), added to every instance's guest -m at admission so the - // nbdkit processes backing its volumes are accounted, not absorbed by the - // fixed reserve. See resolveNbdkitCharge / instanceMemChargeMiB. RG-6. + // nbdkitMainMiB / nbdkitAuxMiB: per-volume nbdkit memory charged at + // admission so nbdkit backing a guest's volumes is accounted explicitly. nbdkitMainMiB int nbdkitAuxMiB int instanceTypes map[string]*ec2.InstanceTypeInfo gpuManager *gpu.Manager // nil if GPU passthrough is disabled or no GPUs present - // readMemAvailableGB is the live second admission gate. The static accounting - // above counts only guest -m, so co-located storage VMs, nbdkit (grows with - // guest disk I/O), the daemon stack and page cache are invisible to it. - // MemAvailable sees all of them and already nets reclaimable cache, catching - // real overcommit the accounting would greenlight. nil disables the gate - // (SPINIFEX_ADMISSION_LIVE_MEM=0); a read failure fails open. + // readMemAvailableGB is the live second admission gate (MemAvailable from + // /proc/meminfo). Catches real overcommit the static -m accounting misses. + // nil disables it (SPINIFEX_ADMISSION_LIVE_MEM=0); read failure fails open. readMemAvailableGB func() (float64, bool) // Dynamic instance-type subscription management @@ -156,12 +149,9 @@ type Daemon struct { cancel context.CancelFunc shutdownWg sync.WaitGroup - // systemDispatchWg tracks in-flight host-pinned system.LaunchInstance / - // system.TerminateInstance handlers. Each request runs in its own goroutine - // so a long VM boot never head-of-line blocks the responder's subscription - // (267.4). Not awaited at shutdown — these are request-scoped and the reply - // is bounded by the requester's timeout; the WaitGroup exists so tests can - // deterministically await dispatch completion. + // systemDispatchWg tracks in-flight system.LaunchInstance / TerminateInstance + // handlers. Each runs in its own goroutine so a slow VM boot never blocks + // the NATS subscription. Used by tests to await dispatch completion. systemDispatchWg sync.WaitGroup // vmMgr owns the in-memory map of VMs running on this node. @@ -182,9 +172,8 @@ type Daemon struct { // JetStream manager for KV state storage (nil if JetStream disabled) jsManager *JetStreamManager - // stateStore is the vm.StateStore-shaped view over jsManager. Both the - // vm.Manager and daemon-side handlers route VM-instance state I/O - // through it. Initialized after initJetStream succeeds. + // stateStore is the vm.StateStore view over jsManager, initialized after + // initJetStream succeeds. stateStore vm.StateStore // Delay after QMP device_del before blockdev-del (default 1s, 0 in tests) @@ -193,85 +182,60 @@ type Daemon struct { // NATS connect retry options (nil uses defaults: 5min max, 500ms initial delay) natsRetryOpts []utils.RetryOption - // requireNATSTimeout caps the first connectNATS attempt when the - // SPINIFEX_REQUIRE_NATS=1 strict-startup env var is set (§1d-strict). - // Default 30s; tests override to a shorter value to keep the strict-mode - // abort path fast. + // requireNATSTimeout caps the first connectNATS attempt under + // SPINIFEX_REQUIRE_NATS=1. Default 30s; tests use a shorter value. requireNATSTimeout time.Duration - // exitFunc is invoked when SPINIFEX_REQUIRE_NATS=1 strict-startup is - // requested and the bounded first connect fails. Defaults to os.Exit; - // tests override to observe the abort without killing the test process. + // exitFunc is called when SPINIFEX_REQUIRE_NATS=1 and the first connect + // times out. Defaults to os.Exit; tests override to avoid killing the process. exitFunc func(int) // networkPlumber handles tap device lifecycle for VPC and management networking networkPlumber vm.NetworkPlumber - // Management NIC infrastructure: bridge IP + IP allocator for system instances. - // Populated at startup when br-mgmt is detected; nil/empty otherwise. + // mgmtBridgeIP / mgmtIPAllocator: management NIC infrastructure for system + // instances. Populated at startup when br-mgmt is detected. mgmtBridgeIP string mgmtIPAllocator *MgmtIPAllocator - // mgmtRouteVia is the AWSGW bind IP that system instances must route via the - // management NIC. Set when AWSGW binds to a specific IP (multi-node). + // mgmtRouteVia: AWSGW bind IP system instances route via br-mgmt (multi-node). mgmtRouteVia string - // gpuProbe holds the result of the always-on startup GPU hardware probe. - // Populated regardless of whether gpu_passthrough is enabled in config. + // gpuProbe: startup GPU hardware probe result, always populated. gpuProbe gpuProbeResult - // gpuManager handles VFIO bind/unbind lifecycle for GPU passthrough. - // Nil when GPUPassthrough is disabled in config or no recognised GPUs are found. + // gpuManager: VFIO bind/unbind for GPU passthrough. Nil when disabled or no GPUs found. gpuManager *gpu.Manager - // shuttingDown is set to true during coordinated cluster shutdown (GATE phase) - // or during SIGTERM-based shutdown. When true, the daemon rejects new work, - // crash handlers bail out, and setupShutdown skips redundant VM stops. + // shuttingDown: set during GATE or SIGTERM; daemon rejects new work and + // crash handlers bail out. shuttingDown atomic.Bool - // ready is set to true once NATS connection, JetStream, and all services - // are fully initialized. The health endpoint reports "starting" until ready. + // ready: set once NATS, JetStream, and all services are initialized. + // /health reports "starting" until true. ready atomic.Bool - // natsConnected is true iff the daemon's NATS client TCP connection is - // live. Flipped by onNATSDisconnect / onNATSReconnect and by connectNATS - // success. False at construction (cluster bootstrap hasn't run yet). + // natsConnected: true when the NATS TCP connection is live. natsConnected atomic.Bool - // peersReachable is true iff at least one peer daemon's /health responded - // in the most recent peer-health probe cycle. Managed by - // monitorPeerReachability. Single-node clusters keep this permanently - // true (no peers to lose); multi-node clusters start false and flip true - // after the first successful probe. + // peersReachable: true when at least one peer /health responded in the + // last probe cycle. Pinned true on single-node clusters. peersReachable atomic.Bool - // natsRetryCount counts disconnect→reconnect cycles since process start. - // Bumped from onNATSReconnect; surfaced via NATSRetryCount() for the - // /local/status endpoint added in 1b. + // natsRetryCount: disconnect→reconnect cycles since process start. natsRetryCount atomic.Int64 - // stateRevision is bumped on every successful local-state write. Surfaced - // via /local/status so observers can detect changes without diffing payloads. + // stateRevision: incremented on each successful WriteState. stateRevision atomic.Uint64 - // kvSyncFailures counts best-effort JetStream KV sync failures (timeout or - // put error) since process start. Bumped from RecordKVSyncFailure; surfaced - // via /local/status and the spinifex_daemon_kv_sync_failures_total metric. + // kvSyncFailures: best-effort JetStream KV sync failures since start. kvSyncFailures atomic.Int64 - // lastKVSyncAt holds the unix-nano timestamp of the most recent successful - // best-effort KV sync. Zero means "never synced since process start". + // lastKVSyncAt: unix-nano timestamp of the last successful KV sync. lastKVSyncAt atomic.Int64 - // lastKVSyncError holds the most recent best-effort KV sync error message - // as a string. Cleared back to "" on the next successful sync. + // lastKVSyncError: most recent KV sync error message; "" on success. lastKVSyncError atomic.Value - // reconciling coalesces concurrent reconcileOnHeal invocations. Both - // the NATS reconnect callback and the peer-probe heal edge may fire - // near-simultaneously on the same heal; the second caller observes - // the flag set and returns. + // reconciling: coalesces concurrent reconcileOnHeal calls. reconciling atomic.Bool - // stateWriteMu serialises WriteState. Concurrent callers (each terminate / - // stop goroutine triggers TransitionState → WriteState) share a single - // path + ".tmp" staging file; without serialisation one rename races the - // other and fails ENOENT, which aborts the cleanup chain and leaks ENIs. + // stateWriteMu: serialises WriteState to prevent races on the .tmp staging file. stateWriteMu sync.Mutex mu sync.Mutex @@ -283,18 +247,9 @@ const ( DaemonModeCluster = "cluster" ) -// Mode returns the daemon's current connectivity mode. "cluster" iff both the -// local NATS link is up AND at least one peer responded to the most recent -// /health probe. Any other combination (NATS down, peers unreachable, or -// both) reports "standalone". Safe to call from any goroutine. -// -// The two-signal design covers the DDIL Tier 1 failure modes: -// - NATS-only outage (Scenario A): natsConnected=false ⇒ standalone. -// - Daemon restart with NATS down (Scenario B): natsConnected=false ⇒ -// standalone until both NATS and peers return. -// - Clean partition with local NATS still up (Scenario C): -// peersReachable=false ⇒ standalone even though the client socket is -// healthy. +// Mode returns "cluster" iff both natsConnected and peersReachable are true; +// otherwise "standalone". Two signals are required so a NATS-up partition +// (DDIL Scenario C) still reports standalone when no peer responds. func (d *Daemon) Mode() string { if d.natsConnected.Load() && d.peersReachable.Load() { return DaemonModeCluster @@ -349,9 +304,7 @@ func (d *Daemon) RecordKVSyncSuccess(_ string) { } // RecordKVSyncFailure implements KVSyncObserver. Bumps the failure counter and -// records the error message for /local/status. The bucket arg is reserved for -// future per-bucket labelling; the current call site is the instance-state -// bucket and the value is logged via the manager's slog.Warn line. +// records the error message for /local/status. func (d *Daemon) RecordKVSyncFailure(_ string, err error) { d.kvSyncFailures.Add(1) if err != nil { @@ -376,10 +329,7 @@ func (d *Daemon) onNATSReconnect(_ *nats.Conn) { go d.reconcileOnHeal("nats-reconnect") } -// execCommand wraps exec.Command so tests can substitute a fake builder. -// Mirrors the utils.SudoCommand seam — getSystemMemory shells out to -// sysctl (darwin) or grep /proc/meminfo (linux), neither of which can be -// faked without an indirection that the test can swap. +// execCommand wraps exec.Command so tests can substitute a fake implementation. var execCommand = exec.Command // getSystemMemory returns the total system memory in GB @@ -425,14 +375,9 @@ func getSystemMemory() (float64, error) { } } -// physicalCoreCount returns the number of physical CPU cores. Guest vCPUs are -// scheduled 1:1 against physical cores rather than SMT threads: -// runtime.NumCPU() counts logical threads, so on an SMT or hybrid host (e.g. -// 12 cores / 16 threads) it would let guest vCPUs pack past the physical core -// count and thrash them during the boot spike. Parses /proc/cpuinfo, counting -// distinct (physical id, core id) pairs. Falls back to runtime.NumCPU() on -// non-Linux hosts or when the topology fields are absent (some VMs/containers -// omit them). +// physicalCoreCount returns the number of physical CPU cores by parsing +// distinct (physical id, core id) pairs from /proc/cpuinfo. Falls back to +// runtime.NumCPU() on non-Linux or when topology fields are absent. func physicalCoreCount() int { logical := runtime.NumCPU() if runtime.GOOS != "linux" { @@ -451,9 +396,8 @@ func physicalCoreCount() int { } // parsePhysicalCores counts distinct (physical id, core id) pairs in -// /proc/cpuinfo contents — i.e. physical cores, with SMT siblings collapsed. -// Returns ok=false when the topology fields are absent (some VMs/containers -// omit them) so the caller can fall back to the logical CPU count. +// /proc/cpuinfo, collapsing SMT siblings. Returns ok=false when topology +// fields are absent so the caller can fall back to the logical CPU count. func parsePhysicalCores(data []byte) (int, bool) { cores := make(map[string]struct{}) var phys, core string @@ -488,22 +432,12 @@ func parsePhysicalCores(data []byte) (int, bool) { return len(cores), true } -// NewResourceManager creates a new resource manager with system capabilities. -// Returns an error if system memory cannot be detected, since an incorrect -// default would either under-provision (large servers) or over-commit (small devices). -// Also returns an error if the host is too small to satisfy the daemon's -// reserve — clamping silently would defeat the reserve and look like a -// runtime bug. -// gpuModels is the list of recognised GPU models present on the host; pass nil if -// GPU passthrough is disabled or no GPUs were found. +// NewResourceManager creates a new ResourceManager. Errors if memory detection +// fails or if the host is too small to satisfy the daemon's reserve. func NewResourceManager(gpuModels []instancetypes.GPUModel, migProfiles []instancetypes.MIGProfileSpec, gpuMgr *gpu.Manager) (*ResourceManager, error) { - // Schedule guest vCPUs against physical cores, not SMT threads (see - // physicalCoreCount), so a hyperthreaded/hybrid host can't be packed past - // what it can actually run. SPINIFEX_HOST_VCPU overrides detection for - // hosts that misreport topology. + // Use physical cores (not SMT threads); SPINIFEX_HOST_VCPU overrides. hostVCPU := resolveHostVCPU(os.Getenv, physicalCoreCount()) - // Get system memory (in GB) totalMemGB, err := getSystemMemory() if err != nil { return nil, fmt.Errorf("detect system memory: %w", err) @@ -518,14 +452,11 @@ func NewResourceManager(gpuModels []instancetypes.GPUModel, migProfiles []instan return nil, fmt.Errorf("validate host reserve: %w", err) } - // Determine architecture arch := "x86_64" if runtime.GOARCH == "arm64" { arch = "arm64" } - // Detect CPU generation and generate matching instance types (including GPU - // families when gpuModels is non-nil). instanceTypes := instancetypes.DetectAndGenerate(instancetypes.HostCPU{}, arch, gpuModels) if len(migProfiles) > 0 { maps.Copy(instanceTypes, instancetypes.GenerateMIGTypes(migProfiles, arch)) @@ -537,8 +468,6 @@ func NewResourceManager(gpuModels []instancetypes.GPUModel, migProfiles []instan "schedulableVCPU", hostVCPU-reservedVCPU, "schedulableMemGB", totalMemGB-reservedMem, "instanceTypes", len(instanceTypes)) - // Live-memory admission gate: enabled by default, reads /proc/meminfo - // MemAvailable at each admission. nil when disabled so liveMemGate is a no-op. var memReader func() (float64, bool) if liveMemAdmissionEnabled(os.Getenv) { memReader = readMemAvailableGB @@ -575,11 +504,8 @@ func instanceTypeMemoryMiB(it *ec2.InstanceTypeInfo) int64 { return 0 } -// instanceMemChargeMiB is the full memory admission charges one instance of it: -// its guest -m plus the nbdkit processes backing its volumes (RG-6). Every -// accounting gate (both canAllocateCount sites, liveMemGate, allocate, -// deallocate) divides remaining memory by — or moves the running total by — -// this value, so the charge and release can never drift. +// instanceMemChargeMiB is the full per-instance memory charge: guest -m plus +// nbdkit processes for its volumes. All admission gates use this value. func (rm *ResourceManager) instanceMemChargeMiB(it *ec2.InstanceTypeInfo) int64 { return instanceTypeMemoryMiB(it) + nbdkitChargeMiB(defaultMainVolumes, defaultAuxVolumes, rm.nbdkitMainMiB, rm.nbdkitAuxMiB) @@ -668,12 +594,9 @@ func (rm *ResourceManager) GetAvailableInstanceTypeInfos(showCapacity bool) []*e return infos } -// GetSupportedInstanceTypeInfos returns every instance type this node is -// configured to run, irrespective of current free capacity. It mirrors -// AWS's DescribeInstanceTypes semantics: callers asking "what types do you -// support?" must see a stable answer even when every slot is occupied. -// System types and entries with incomplete CPU/memory metadata are still -// skipped. +// GetSupportedInstanceTypeInfos returns every supported instance type regardless +// of current capacity, mirroring AWS DescribeInstanceTypes semantics. System +// types and entries with incomplete metadata are skipped. func (rm *ResourceManager) GetSupportedInstanceTypeInfos() []*ec2.InstanceTypeInfo { rm.mu.RLock() defer rm.mu.RUnlock() @@ -693,10 +616,8 @@ func (rm *ResourceManager) GetSupportedInstanceTypeInfos() []*ec2.InstanceTypeIn return infos } -// GetResourceStats returns current resource allocation stats for the node status response. -// totalVCPU / totalMemGB are the raw host figures; reservedVCPU / reservedMemGB are -// held back from guest scheduling. Per-type caps reflect host - reserved - allocated, -// matching what the admission path will actually permit. +// GetResourceStats returns host resource figures, reservation, allocation, and +// per-type capacity caps for the node status response. func (rm *ResourceManager) GetResourceStats() (totalVCPU int, totalMemGB float64, reservedVCPU int, reservedMemGB float64, allocVCPU int, allocMemGB float64, caps []types.InstanceTypeCap) { rm.mu.RLock() defer rm.mu.RUnlock() @@ -788,16 +709,9 @@ func NewDaemon(cfg *config.ClusterConfig) (*Daemon, error) { requireNATSTimeout: 30 * time.Second, exitFunc: os.Exit, } - // Initialise peersReachable optimistically. Mode() also requires - // natsConnected, which starts false, so the optimistic init never - // causes Mode() to falsely report cluster — but it does prevent the - // first probe tick from triggering a spurious "heal" edge on - // multi-node clusters at startup (zero-value false → first-successful- - // probe true would otherwise fire reconcileOnHeal at boot, - // concurrently with startCluster's own bootstrap and perturbing the - // timing the DDIL harness relies on). Single-node clusters never - // run a probe at all, so this also keeps the prior single-node - // behaviour where Mode() flips to cluster the moment NATS comes up. + // Initialise peersReachable true so the first probe tick never fires a + // spurious reconcileOnHeal at startup. Mode() still requires natsConnected + // (starts false), so this can't falsely report cluster mode. d.peersReachable.Store(true) return d, nil } @@ -1054,18 +968,9 @@ func (d *Daemon) subscribeAll() error { return nil } -// Start brings the daemon up in two phases (DDIL Tier 1, §1d): -// -// 1. startLocal — bootstraps everything that does not need NATS (cluster -// manager HTTPS, mgmt bridge + IP allocator, OVS plumber, OOM score, -// local instance state via 1a). The daemon is reachable on /local/* and -// /health as soon as this returns. -// 2. startCluster — runs in the background, retries NATS forever, then -// initialises JetStream + cluster-scoped services, restores instances, -// and subscribes to NATS topics. Mode flips to "cluster" once connected. -// -// Process-exit on NATS failure is no longer possible; staying up degraded is -// always better than killing the local VM management plane. +// Start bootstraps the daemon in two phases (DDIL Tier 1): startLocal brings +// up HTTPS + local state without NATS, then startCluster retries NATS +// indefinitely and joins the cluster once connected. func (d *Daemon) Start() error { if err := d.startLocal(); err != nil { return err @@ -1083,20 +988,12 @@ func (d *Daemon) Start() error { return nil } -// startLocal performs the no-NATS bootstrap: HTTPS cluster manager, -// management bridge detection, network plumber, OOM protection, and local -// instance-state recovery. Failures here are fatal — these are local -// configuration errors (TLS misconfig, bad config path) that retry would not -// fix. The daemon is reachable via /local/* and /health once this returns. +// startLocal performs the no-NATS bootstrap. Failures here are fatal (local +// config errors that retry cannot fix). The daemon is reachable via /local/* +// and /health once this returns. // -// Invariant (DDIL §1e-audit): no code in startLocal may touch JetStream KV. -// The 25 cluster-scoped / read-cache / expendable buckets enumerated in -// daemon-local-autonomy.md §1e-audit are initialised exclusively from -// startCluster() — touching any of them here would defeat Tier 1 autonomy by -// blocking on NATS at boot. The only KV bucket the daemon owns at Tier 1 is -// spinifex-instance-state, and even that is read from the local file (see 1a), -// not JetStream, in this phase. assertNoClusterServicesInitialised below -// enforces the invariant at runtime. +// DDIL §1e-audit: no JetStream KV must be touched here. All KV buckets are +// initialised in startCluster. assertNoClusterServicesInitialised enforces this. func (d *Daemon) startLocal() error { // ClusterManager serves /health and /local/* over HTTPS. NATS-independent. if err := d.ClusterManager(); err != nil { @@ -1134,10 +1031,7 @@ func (d *Daemon) startLocal() error { slog.Warn("Failed to set daemon OOM score", "err", err) } - // Recover local instance state from disk (1a). Read-only — KV migrations, - // QMP reconnects, NATS subscriptions and crashed-VM relaunches happen in - // startCluster() once NATS is reachable. Fatal if the file is corrupt: - // silently dropping instance state would orphan running VMs. + // Recover local instance state from disk. Fatal on corruption — orphaned VMs. if err := d.LoadState(); err != nil { return fmt.Errorf("load local instance state: %w", err) } @@ -1153,10 +1047,8 @@ func (d *Daemon) startLocal() error { return fmt.Errorf("startLocal Tier 1 invariant violated: %w", err) } - // Peer-health probe runs in startLocal because Mode() needs a partition - // signal even when NATS never connects (DDIL Scenario C). The probe is - // NATS-independent — it dials each peer's /health over the cluster - // network — so it does not violate the Tier 1 invariant asserted above. + // Peer-health probe is NATS-independent (dials /health directly) and must + // start here so Mode() can detect partitions even if NATS never connects. d.shutdownWg.Go(d.monitorPeerReachability) d.ready.Store(true) @@ -1164,12 +1056,8 @@ func (d *Daemon) startLocal() error { return nil } -// assertNoClusterServicesInitialised guards the DDIL §1e-audit Tier 1 -// invariant: at the end of startLocal, no NATS-dependent or KV-backed handle -// may exist. A non-nil field here means a future edit accidentally hoisted a -// cluster-scoped initialiser into the no-NATS phase, which would re-introduce -// the boot-time NATS dependency that 1d removed. Cheap nil sweep — runs once -// per process, on the bootstrap path only. +// assertNoClusterServicesInitialised enforces the DDIL §1e-audit invariant: +// no NATS-dependent handle may exist at the end of startLocal. func (d *Daemon) assertNoClusterServicesInitialised() error { switch { case d.natsConn != nil: @@ -1210,16 +1098,9 @@ func (d *Daemon) assertNoClusterServicesInitialised() error { return nil } -// startCluster performs the cluster-integration phase asynchronously. It -// retries NATS indefinitely (cap 60s backoff) and only returns once the node -// is fully participating in the cluster or d.ctx is cancelled. Errors here -// are logged, never propagated as a process-exit. -// -// Invariant (DDIL §1e-audit): every JetStream KV bucket — the 18 cluster-scoped -// buckets, 4 read-cache (IAM) buckets, and 2 expendable buckets enumerated in -// daemon-local-autonomy.md §1e-audit — is initialised here, never in -// startLocal. Adding a new cluster-scoped service belongs in this function; -// hoisting one into startLocal trips assertNoClusterServicesInitialised. +// startCluster retries NATS indefinitely and initialises all cluster-scoped +// services. Errors are logged, not fatal. All JetStream KV buckets (DDIL +// §1e-audit) must be initialised here, never in startLocal. func (d *Daemon) startCluster() error { if os.Getenv("SPINIFEX_REQUIRE_NATS") == "1" { // §1d-strict opt-in: bounded first connect, abort on timeout. Restores @@ -1239,18 +1120,14 @@ func (d *Daemon) startCluster() error { return fmt.Errorf("initialize JetStream: %w", err) } - // Tombstone the spinifex-dhcp-leases bucket left over from the upstream-DHCP - // client removed in Phase 2.3. Idempotent — first daemon start on an upgraded - // cluster sweeps the bucket; later restarts are no-ops. + // Remove the obsolete spinifex-dhcp-leases bucket (idempotent). if js, jsErr := d.natsConn.JetStream(); jsErr == nil { if err := utils.DeleteKVBucketIfExists(js, "spinifex-dhcp-leases"); err != nil { slog.Warn("Failed to delete obsolete spinifex-dhcp-leases KV bucket", "err", err) } } - // Enable OVN native IPsec on intra-AZ Geneve when the cluster flag is set. - // Idempotent — ovs-monitor-ipsec materialises strongSwan configs from the - // cert pointers each time ovn-controller programs a tunnel. + // Enable OVN native IPsec when configured (idempotent). if d.clusterConfig != nil && d.clusterConfig.Network.IPSecEnabled { if err := host.EnableOVNIPSec(d.configPath, d.clusterConfig); err != nil { slog.Warn("Failed to enable OVN native IPsec; intra-AZ Geneve will be plaintext", "err", err) @@ -1319,8 +1196,7 @@ func (d *Daemon) startCluster() error { return fmt.Errorf("failed to initialize VPC service: %w", err) } - // Wire the eni-by-vpc-ip reverse index so the ENI controller keeps the - // IMDS source-IP→ENI lookup in sync on Create/DeleteNetworkInterface. + // Wire eni-by-vpc-ip reverse index for IMDS source-IP→ENI lookup. if vpcJS, jsErr := d.natsConn.JetStream(); jsErr != nil { slog.Warn("Failed to get JetStream for eni-by-ip index", "err", jsErr) } else if eniByIPKV, kvErr := handlers_imds.InitENIByIPBucket(vpcJS, 1); kvErr != nil { @@ -1346,8 +1222,7 @@ func (d *Daemon) startCluster() error { return fmt.Errorf("failed to initialize NatGateway service: %w", err) } - // Initialize external IPAM if pool mode is configured (per-VM public IPs). - // NAT mode only uses SNAT via a shared gateway IP — no per-VM allocation needed. + // Initialize external IPAM for pool mode (per-VM public IPs). if d.clusterConfig != nil && d.clusterConfig.Network.ExternalMode == "pool" { js, jsErr := d.natsConn.JetStream() if jsErr != nil { @@ -1414,15 +1289,9 @@ func (d *Daemon) startCluster() error { } } - // Wire deps needed by InstanceService.TerminateStoppedInstance now that - // volumeService / vpcService / externalIPAM are constructed. d.instanceService.SetTerminationDeps(d.volumeService, d.vpcService, d.externalIPAM) - - // Wire deps for InstanceService.PrepareRunInstances (AMI/key/VPC/IPAM). d.instanceService.SetRunInstancesDeps(d.imageService, d.keyService, &daemonENICreator{d: d}, d.externalIPAM) - // Wire GPU claimer for InstanceService.StartStoppedInstance — nil when no - // passthrough hardware is configured. if d.gpuManager != nil { d.instanceService.SetGPUClaimer(&daemonGPUClaimer{d: d}) } @@ -1444,27 +1313,16 @@ func (d *Daemon) startCluster() error { d.elbv2Service.VPCService = d.vpcService } - // Wire LB VM lifecycle: route system VM launches through NATS so they - // fan out across the cluster instead of always running on whichever node - // handled the upstream CreateLoadBalancer call. The daemon's own - // system.LaunchInstance.* subscriptions (registered by ResourceManager) - // will pick up the request locally when this node has capacity, or hand - // off to another node via the spinifex-workers queue group otherwise. + // Route system VM launches through NATS so they fan out across the cluster. d.elbv2Service.InstanceLauncher = handlers_elbv2.NewNATSSystemInstanceLauncher(d.natsConn, 0) - // Wire system credentials + gateway URL for LB agent SigV4 auth. d.wireLBAgentConfig() - // System VMs (LB, NAT GW) use the dedicated sys.micro instance type. d.elbv2Service.SetSystemInstanceTypeFunc(func() string { return "sys.micro" }) - // Invalidate persisted target HealthState before subscriptions go live. - // Stale "healthy" entries from a pre-restart cluster otherwise satisfy - // DescribeTargetHealth waiters before any actual post-restart health - // observation has been recorded. Best-effort: a failure here just leaves - // the old behavior in place rather than blocking daemon startup. + // Invalidate stale "healthy" target state from before restart. Best-effort. if err := d.elbv2Service.ResetTargetHealthOnStartup(context.Background()); err != nil { slog.Warn("ELBv2: target-health reset failed; continuing with stale state", "err", err) @@ -1512,8 +1370,6 @@ func (d *Daemon) startCluster() error { d.ensureDefaultVPCInfrastructure(failedDefaultVPCs) } - // Wire vm.Manager collaborators now that NATS, JetStream, network plumber, - // volume service, and resource manager are all ready. d.vmMgr.SetDeps(d.buildVMManagerDeps()) d.waitForClusterReady() @@ -1522,8 +1378,7 @@ func (d *Daemon) startCluster() error { return fmt.Errorf("restore instances: %w", err) } - // Rebuild mgmt IP allocator from restored VMs so we don't re-allocate IPs - // that are already in use by running system instances. + // Rebuild mgmt IP allocator so already-allocated IPs aren't reused. if d.mgmtIPAllocator != nil { d.mgmtIPAllocator.Rebuild(d.vmMgr.SnapshotMap()) slog.Info("Rebuilt mgmt IP allocator from restored instances", "allocated", d.mgmtIPAllocator.AllocatedCount()) @@ -1533,9 +1388,7 @@ func (d *Daemon) startCluster() error { return fmt.Errorf("failed to subscribe to NATS topics: %w", err) } - // Initialize dynamic per-instance-type subscriptions for capacity-aware routing. - // Each instance type gets its own NATS topic (ec2.RunInstances.{type}) so requests - // are only routed to nodes with available capacity. + // Initialize per-instance-type NATS subscriptions for capacity-aware routing. d.resourceMgr.initSubscriptions(d.natsConn, d.handleEC2RunInstances, d.handleSystemLaunchInstance, d.node) d.startHeartbeat() @@ -1551,12 +1404,8 @@ func (d *Daemon) startCluster() error { return nil } -// connectNATS establishes a connection to the NATS server. Defaults to -// infinite retry with exponential backoff (cap 60s) so the daemon stays up -// in standalone mode through extended NATS outages instead of process-exiting -// (DDIL Tier 1). Tests override d.natsRetryOpts to bound the wait. Callers -// (e.g. §1d-strict) may pass extraOpts that are applied after d.natsRetryOpts -// so they win on conflicting fields like WithMaxWait. +// connectNATS connects to NATS with infinite retry (cap 60s backoff). Tests +// override d.natsRetryOpts; extraOpts override any conflicting fields. func (d *Daemon) connectNATS(extraOpts ...utils.RetryOption) error { opts := append([]utils.RetryOption{ utils.WithMaxWait(0), // infinite retry; cancelled via d.ctx @@ -1578,10 +1427,8 @@ func (d *Daemon) connectNATS(extraOpts ...utils.RetryOption) error { return nil } -// initJetStream initializes JetStream with retry/backoff and upgrades replicas -// for multi-node clusters. On multi-node clusters, JetStream requires NATS -// cluster quorum which may take several minutes if nodes start at different -// times. This retries for up to 5 minutes to allow late-joining nodes. +// initJetStream initialises JetStream KV stores, retrying up to 5 minutes to +// allow late-joining nodes to reach quorum. func (d *Daemon) initJetStream() error { const maxWait = 5 * time.Minute retryDelay := 500 * time.Millisecond @@ -1622,16 +1469,11 @@ func (d *Daemon) initJetStream() error { d.stateStore = newStateStoreAdapter(d.jsManager) - // Replica upgrade is deferred to after all services have created their - // KV buckets and the cluster is ready (see upgradeJetStreamReplicas). - return nil } -// upgradeJetStreamReplicas bumps the replication factor on ALL KV_* streams -// to match the cluster size. This runs after all services have created their -// buckets and after waitForClusterReady so that enough NATS peers are online -// to accept the new replica count. +// upgradeJetStreamReplicas bumps KV_* stream replication to match the cluster +// size. Runs after all buckets are created and the cluster is ready. func (d *Daemon) upgradeJetStreamReplicas() { clusterSize := len(d.clusterConfig.Nodes) if clusterSize <= 1 || d.jsManager == nil { @@ -1642,15 +1484,11 @@ func (d *Daemon) upgradeJetStreamReplicas() { } } -// initRetrySleep is the sleep seam used by initServiceWithRetry. Tests -// override it to drive backoff cadence without the real 35s wall-clock -// budget that 7 doublings (500ms→1s→2s→4s→8s→10s→10s) would impose. +// initRetrySleep is the sleep seam used by initServiceWithRetry; tests override it. var initRetrySleep = time.Sleep -// initServiceWithRetry initializes a service using the provided init function, -// retrying with exponential backoff (500ms→10s) for up to 5 minutes. During -// cluster restarts, JetStream KV may be temporarily unavailable while NATS -// routes re-establish and the cluster forms quorum. +// initServiceWithRetry initialises a service with exponential backoff (500ms→10s) +// for up to 5 minutes, allowing time for JetStream quorum during cluster restarts. func initServiceWithRetry[T any](name string, initFn func() (T, error)) (T, error) { const maxWait = 5 * time.Minute retryDelay := 500 * time.Millisecond @@ -1679,9 +1517,8 @@ func initServiceWithRetry[T any](name string, initFn func() (T, error)) (T, erro } } -// waitForClusterReady waits until dependent infrastructure services are reachable -// before starting VM recovery. This prevents races where VMs try to mount volumes -// before viperblock/predastore are ready. +// waitForClusterReady blocks until viperblock and predastore are reachable, +// preventing races during VM recovery. func (d *Daemon) waitForClusterReady() { slog.Info("Waiting for cluster readiness...") maxWait := 2 * time.Minute @@ -1716,8 +1553,7 @@ func (d *Daemon) waitForClusterReady() { slog.Warn("Cluster readiness timeout, proceeding with recovery anyway", "maxWait", maxWait) } -// checkViperblockReady checks if viperblock is reachable by verifying -// the NATS connection is up (viperblock subscribes to ebs topics on NATS). +// checkViperblockReady reports whether viperblock is reachable via NATS. func (d *Daemon) checkViperblockReady() bool { if d.natsConn == nil { return false @@ -1739,11 +1575,8 @@ func (d *Daemon) checkPredastoreReady() bool { return true } -// LoadState loads the instance state from the local file. Missing file is the -// fresh-install signal (start with empty map). Corrupt or unknown-schema files -// are fatal — caller refuses start rather than silently losing data. There is -// no KV fallback: a fresh node has empty KV anyway, so silent KV-fallback on -// file loss would just mask bugs. +// LoadState loads instance state from disk. Missing file → empty map (fresh +// install). Corrupt or unknown-schema files are fatal. func (d *Daemon) LoadState() error { path := d.localStatePath() state, err := ReadLocalState(path) @@ -1763,11 +1596,8 @@ func (d *Daemon) LoadState() error { return nil } -// restoreInstances delegates to vm.Manager.Restore. vm.Manager only -// persists to the cluster StateStore (JetStream); the daemon's local -// crash-recovery file is owned by daemon.WriteState, so we sync it here -// to preserve the pre-2b invariant that local state == in-memory state -// after restore. Phase 2f folds this back into vm/. +// restoreInstances delegates to vm.Manager.Restore and syncs the local state +// file so it matches in-memory state. func (d *Daemon) restoreInstances() error { d.vmMgr.Restore() if err := d.WriteState(); err != nil { @@ -1786,9 +1616,8 @@ func (d *Daemon) awaitShutdown() { <-done } -// computeConfigHash computes SHA256 hash of the shared cluster config (excluding node-specific fields) +// computeConfigHash computes a SHA256 hash of the shared cluster config. func (d *Daemon) computeConfigHash() (string, error) { - // Only hash the shared cluster data, not the node-specific top-level field sharedData := types.SharedClusterData{ Epoch: d.clusterConfig.Epoch, Version: d.clusterConfig.Version, @@ -1803,9 +1632,8 @@ func (d *Daemon) computeConfigHash() (string, error) { return hex.EncodeToString(hash[:]), nil } -// ClusterManager starts the HTTP cluster management server +// ClusterManager starts the HTTPS cluster management server. func (d *Daemon) ClusterManager() error { - // Get daemon host from config daemonHost := d.config.Daemon.Host if daemonHost == "" { return fmt.Errorf("daemon.host not configured") @@ -1813,7 +1641,6 @@ func (d *Daemon) ClusterManager() error { r := chi.NewRouter() - // Health endpoint - responds to HTTP and NATS r.Get("/health", func(w http.ResponseWriter, r *http.Request) { configHash, err := d.computeConfigHash() if err != nil { @@ -1827,7 +1654,6 @@ func (d *Daemon) ClusterManager() error { for _, svc := range d.config.GetServices() { serviceHealth[svc] = "ok" } - // For remote dependencies, check connectivity if !d.config.HasService("nats") { if d.natsConn != nil && d.natsConn.IsConnected() { serviceHealth["nats"] = "remote_ok" @@ -1836,7 +1662,6 @@ func (d *Daemon) ClusterManager() error { } } - // Check OVN networking readiness ovnHealth := host.HealthStatus() if ovnHealth.BrIntExists { serviceHealth["br-int"] = "ok" @@ -1872,10 +1697,7 @@ func (d *Daemon) ClusterManager() error { d.registerLocalRoutes(r) - // Load TLS certificate. - // Resolve relative cert paths against config directory (cert lives alongside spinifex.toml). - // For binary installs, systemd sets absolute paths via env vars; for dev, the config - // stores relative paths like "config/server.pem" which need resolution. + // Resolve relative cert paths against the config directory. tlsCert := d.config.Daemon.TLSCert tlsKey := d.config.Daemon.TLSKey if tlsCert == "" || tlsKey == "" { @@ -1923,9 +1745,7 @@ func (d *Daemon) ClusterManager() error { return nil } -// kvSyncTimeout bounds the best-effort cluster sync so a degraded NATS does -// not stall every state transition. 1s is well above healthy KV.Put latency -// and well below a user-visible delay. +// kvSyncTimeout caps the best-effort KV sync; 1s is above healthy Put latency. const kvSyncTimeout = time.Second // localStatePath returns the on-disk path to this daemon's instance state file. @@ -1936,14 +1756,8 @@ func (d *Daemon) localStatePath() string { return LocalStatePath(d.config.DataDir) } -// WriteState persists the instance state. Local file is the source of truth; -// JetStream KV is best-effort cluster cache. The local write is fatal on -// failure; KV failures are logged and swallowed so partition-time clients -// never see "KV down" errors. -// -// Both wire forms are marshalled inside vmMgr.View so json.Marshal sees a -// stable VM-field snapshot. Marshaling outside the lock would race against -// concurrent TransitionState writers under the data race detector. +// WriteState persists instance state. Local file is the source of truth; KV is +// best-effort. Both forms are marshalled inside vmMgr.View to avoid data races. func (d *Daemon) WriteState() error { d.stateWriteMu.Lock() defer d.stateWriteMu.Unlock() @@ -2059,16 +1873,10 @@ func (d *Daemon) setupShutdown() { <-sigChan slog.Info("Received shutdown signal, cleaning up...") - // Cancel context to stop heartbeat and other goroutines d.cancel() - // DDIL Tier 1: SIGTERM alone never stops local VMs. - // `systemctl restart spinifex-daemon` must preserve every QEMU child so - // the new daemon can reattach via the persisted local state file - // (Scenario B). VMs are only stopped when coordinated cluster shutdown - // has explicitly drained them — handleShutdownDrain sets shuttingDown - // after calling StopAll, and crash/restart handlers also gate on this - // flag to bail out. + // DDIL Tier 1: SIGTERM alone never stops VMs — the new daemon reattaches + // via the local state file. VMs stop only after coordinated DRAIN. if d.shuttingDown.Load() { slog.Info("Coordinated shutdown in progress, skipping VM stop (already handled by DRAIN phase)") } else { @@ -2076,46 +1884,37 @@ func (d *Daemon) setupShutdown() { d.shuttingDown.Store(true) } - // Stop ELBv2 background goroutines if d.elbv2Service != nil { d.elbv2Service.Close() } - // Stop EKS per-cluster reconciler + bootstrap goroutines. if d.eksService != nil { d.eksService.Shutdown() } - // Final cleanup for _, sub := range d.natsSubscriptions { - // Unsubscribe from each subscription slog.Info("Unsubscribing from NATS", "subject", sub.Subject) if err := sub.Unsubscribe(); err != nil { slog.Error("Error unsubscribing from NATS", "err", err) } } - // Write shutdown marker to cluster state KV if d.jsManager != nil { if err := d.jsManager.WriteShutdownMarker(d.node); err != nil { slog.Error("Failed to write shutdown marker", "err", err) } } - // Write state to JetStream before closing NATS connection err := d.WriteState() if err != nil { slog.Error("Failed to write state", "err", err) } - // Close NATS connection. natsConn is nil when the daemon was started - // with NATS unreachable and never managed an initial connect — that - // is the DDIL Scenario B path (daemon restart while NATS is down). + // natsConn is nil when NATS was unreachable at startup (DDIL Scenario B). if d.natsConn != nil { d.natsConn.Close() } - // Shutdown cluster manager if d.clusterServer != nil { slog.Info("Shutting down cluster manager...") if err := d.clusterServer.Shutdown(context.Background()); err != nil { @@ -2180,14 +1979,8 @@ func (rm *ResourceManager) canAllocateLocked(instanceType *ec2.InstanceTypeInfo, return rm.liveMemGate(n, instanceType) } -// liveMemGate clamps an accounting-derived launch count by the host's current -// MemAvailable, so admission also refuses launches that fit the static -m budget -// but would overcommit real memory (untracked storage VMs, nbdkit, daemon stack, -// cache). Returns n unchanged when the gate is disabled (readMemAvailableGB is -// nil) or the live read fails — failing open keeps a transient /proc/meminfo -// error from wedging all launches; the accounting gate still applies. The bound -// holds at least reservedMem physically available after the launch, read live on -// each call so back-to-back launches see headroom shrink as guests fault in. +// liveMemGate clamps n by current MemAvailable, catching overcommit that the +// static -m budget misses. Returns n unchanged when disabled or on read error. func (rm *ResourceManager) liveMemGate(n int, instanceType *ec2.InstanceTypeInfo) int { if rm.readMemAvailableGB == nil || n <= 0 { return n @@ -2237,9 +2030,7 @@ func (rm *ResourceManager) deallocate(instanceType *ec2.InstanceTypeInfo) { rm.updateInstanceSubscriptions() } -// Allocate, Deallocate, CanAllocate, InstanceTypes are the exported facade -// satisfying handlers_ec2_instance.InstanceTypeAllocator. The unexported -// variants stay for internal daemon callers. +// Allocate, Deallocate, CanAllocate satisfy handlers_ec2_instance.InstanceTypeAllocator. func (rm *ResourceManager) Allocate(it *ec2.InstanceTypeInfo) error { return rm.allocate(it) } func (rm *ResourceManager) Deallocate(it *ec2.InstanceTypeInfo) { rm.deallocate(it) } func (rm *ResourceManager) CanAllocate(it *ec2.InstanceTypeInfo, count int) int { @@ -2252,11 +2043,8 @@ func (rm *ResourceManager) InstanceTypes() map[string]*ec2.InstanceTypeInfo { return rm.instanceTypes } -// initSubscriptions sets up dynamic per-instance-type NATS subscriptions. -// Called once during daemon startup after NATS is connected. -// reloadGPUTypes replaces GPU instance types in-place and updates NATS subscriptions. -// Called on SIGHUP when gpu_passthrough is toggled. Mutates the existing map so that -// all holders of the map reference (e.g. instanceService) see the updated types. +// reloadGPUTypes replaces GPU instance types in the shared map (in-place, so +// all holders see the update) and refreshes NATS subscriptions. Called on SIGHUP. func (rm *ResourceManager) reloadGPUTypes(models []instancetypes.GPUModel, migProfiles []instancetypes.MIGProfileSpec, mgr *gpu.Manager) { arch := "x86_64" if runtime.GOARCH == "arm64" { @@ -2290,24 +2078,9 @@ func (rm *ResourceManager) initSubscriptions(nc *nats.Conn, handler nats.MsgHand rm.updateInstanceSubscriptions() } -// updateInstanceSubscriptions recalculates which instance types can fit on this -// node and subscribes/unsubscribes from the corresponding NATS topics. -// -// Customer instance types use the ec2.RunInstances.* subject root and the -// rm.handler callback. Each customer type gets: -// - ec2.RunInstances.{type} with spinifex-workers queue group -// - ec2.RunInstances.{type}.{nodeId} without queue group (targeted) -// -// System instance types (sys.*) are not exposed via the customer EC2 API and -// instead route under system.LaunchInstance.* with the rm.systemHandler -// callback. Same shape (queue + node-targeted topic) so ELBv2's ALB-VM -// launches load-balance across the cluster instead of piling on the handler -// node that processed the upstream CreateLoadBalancer call. -// -// NATS only routes requests to nodes whose subscription is live, so capacity -// pressure on the queue (anycast) subject naturally re-distributes load when a -// host fills up. The node-targeted (unicast) subject is deliberately exempt -// from that gating — see the per-subject rationale below. +// updateInstanceSubscriptions subscribes/unsubscribes per-type NATS topics +// based on current capacity. Customer types use ec2.RunInstances.*; system +// types use system.LaunchInstance.*. Each gets a queue-group and a node-targeted topic. func (rm *ResourceManager) updateInstanceSubscriptions() { if rm.natsConn == nil { return @@ -2352,18 +2125,9 @@ func (rm *ResourceManager) updateInstanceSubscriptions() { } if rm.nodeID != "" { - // The node-targeted (unicast) launch subject stays subscribed for any - // type this node can ever host, independent of current free capacity. - // Unlike the queue (anycast) subject above — where dropping the - // subscription when full lets NATS reroute the launch to a node with - // room — a node-targeted request names THIS node specifically (a - // committed spread/placement reservation, e.g. an EKS HA control-plane - // leg). If the subscription were torn down under capacity pressure that - // request would fail with 'no responders' instead of reaching the - // daemon, even when the reserved slot is still good. Capacity is - // enforced at launch time by allocate(), which rejects an over-capacity - // launch with an insufficient-capacity error the caller can act on. - // Subscribe once when the type first fits; never drop it on capacity. + // Node-targeted topic stays subscribed even when capacity is full: + // a committed reservation (e.g. spread placement) must not fail with + // "no responders". Capacity is enforced at launch time by allocate(). nodeTopic := fmt.Sprintf("%s.%s.%s", subjectRoot, typeName, rm.nodeID) if _, nodeSubscribed := rm.instanceSubs[nodeTopic]; canFit && !nodeSubscribed { sub, err := rm.natsConn.Subscribe(nodeTopic, handler) @@ -2379,13 +2143,8 @@ func (rm *ResourceManager) updateInstanceSubscriptions() { } // wireLBAgentConfig loads system credentials, resolves the gateway URL, -// reads the CA certificate, and wires them into the ELBv2 service so LB -// VMs get SigV4 credentials and gateway URL injected via cloud-init. +// and wires them into the ELBv2 service for LB VM cloud-init injection. func (d *Daemon) wireLBAgentConfig() { - // Use system credentials from spinifex.toml (predastore section). - // These are the same service-to-service credentials written by admin init - // into both spinifex.toml and system-credentials.json. Reading from the - // config avoids file permission issues with the separate JSON file. if d.config.Predastore.AccessKey != "" && d.config.Predastore.SecretKey != "" { d.systemAccessKey = d.config.Predastore.AccessKey d.systemSecretKey = d.config.Predastore.SecretKey @@ -2396,16 +2155,6 @@ func (d *Daemon) wireLBAgentConfig() { slog.Warn("System credentials missing from spinifex.toml predastore section — LB VMs will not have SigV4 credentials for agent auth") } - // Resolve gateway URL — the address LB VMs use to reach the AWS gateway. - // Host selection is centralized in resolveGatewayHost so the OIDC issuer - // host and EKS NATS URL come from the same source (see M7). The only - // LB-specific extra here is the multi-node mgmt host route: when the host - // resolves to a dedicated AWSGW bind IP reachable only over br-mgmt, the - // lb-agent needs a bootcmd /32 route via br-mgmt. We deliberately do NOT - // add that route when the host is AdvertiseIP — WAN host IPs may share the - // advertiseIP and a /32 would steal the return path for host-initiated ALB - // connections (reply egresses mgmt with the VM's 10.x source, bypassing - // OVN's SNAT, mismatching the open TCP socket dialed against the EIP). awsgwBindIP := "" if d.config.AWSGW.Host != "" { if h, _, splitErr := net.SplitHostPort(d.config.AWSGW.Host); splitErr == nil { @@ -2417,14 +2166,11 @@ func (d *Daemon) wireLBAgentConfig() { gatewayHost := d.resolveGatewayHost() - // Multi-node mgmt-dedicated AWSGW: host resolved to the bind IP over - // br-mgmt (case 1 in resolveGatewayHost). Loopback / no-mgmt / advertiseIP - // paths can't satisfy all three guards, so this matches only that case. + // Set mgmtRouteVia when AWSGW is only reachable via br-mgmt (multi-node). if gatewayHost != "" && gatewayHost == awsgwBindIP && d.mgmtBridgeIP != "" && awsgwBindIP != advertiseIP { d.mgmtRouteVia = awsgwBindIP } - // Extract port from AWSGW host config (e.g. "0.0.0.0:9999" → "9999"). gatewayPort := "9999" if d.config.AWSGW.Host != "" { if _, port, splitErr := net.SplitHostPort(d.config.AWSGW.Host); splitErr == nil && port != "" { @@ -2441,24 +2187,14 @@ func (d *Daemon) wireLBAgentConfig() { "awsgwBindIP", awsgwBindIP, "mgmtBridgeIP", d.mgmtBridgeIP, "advertiseIP", advertiseIP) } - // Pass mgmt route info so buildMicrovmNICs can add a host route for - // internal LBs that reach the AWSGW via the management NIC. if d.mgmtRouteVia != "" { d.elbv2Service.MgmtRouteGateway = d.mgmtBridgeIP d.elbv2Service.MgmtRouteTarget = d.mgmtRouteVia } - // Always expose mgmtBridgeIP and advertiseIP so buildMicrovmNICs can - // synthesize a mgmt-NIC fallback route for internal-scheme LBs on - // single-node setups (where MgmtRoute{Gateway,Target} stay empty because - // internet-facing LBs reach AWSGW via VPC + EIP SNAT). Internal LBs have - // no EIP, so without this fallback the agent has no return path and the - // LB stays in provisioning forever. d.elbv2Service.MgmtBridgeIP = d.mgmtBridgeIP d.elbv2Service.AdvertiseIP = advertiseIP - // Read CA cert so direct-boot microvm guests can verify AWSGW TLS. - // The NATS CA cert signs the AWSGW server cert (same local CA). if d.config.NATS.CACert != "" { if caBytes, err := os.ReadFile(d.config.NATS.CACert); err == nil { d.elbv2Service.CACert = string(caBytes) @@ -2471,14 +2207,8 @@ func (d *Daemon) wireLBAgentConfig() { } } -// buildGPUPool partitions devices into whole-GPU and MIG entries, constructs a -// Manager, and returns GPU models for instance-type generation and MIG profile -// specs for MIG instance-type generation. -// -// MIG-enabled GPUs with existing instances (daemon restart) are recovered via -// AddMIGInstances. Fresh MIG GPUs (no existing instances) are registered via -// AddMIGGPU for dynamic profile-per-request allocation; their slices are created -// on the first Claim and destroyed on the last Release. +// buildGPUPool partitions GPU devices into whole-GPU and MIG entries, constructs +// a Manager, and returns models and MIG profile specs for instance-type generation. func buildGPUPool(devices []gpu.GPUDevice, cfg config.DaemonConfig) (*gpu.Manager, []instancetypes.GPUModel, []instancetypes.MIGProfileSpec) { type migEntry struct { dev gpu.GPUDevice @@ -2552,10 +2282,8 @@ func buildGPUPool(devices []gpu.GPUDevice, cfg config.DaemonConfig) (*gpu.Manage return mgr, models, migProfiles } -// resolveGPUModel maps a discovered GPU to an instance type model. -// Overrides take priority, then the production model list, then a g5 default. -// Any GPU device that reaches the default path is treated as a g5 instance, -// so consumer GPUs used for testing work without explicit config entries. +// resolveGPUModel maps a GPU device to an instance type model. Overrides take +// priority, then the production model list, then a g5 default. func resolveGPUModel(dev gpu.GPUDevice, overrides []config.GPUModelOverride) instancetypes.GPUModel { for i := range overrides { o := &overrides[i] @@ -2573,7 +2301,6 @@ func resolveGPUModel(dev gpu.GPUDevice, overrides []config.GPUModelOverride) ins if m := instancetypes.GPUModelForVendorDevice(dev.VendorID, dev.DeviceID); m != nil { return *m } - // Default: any discovered GPU device maps to g5 using its detected specs. name := dev.Model if name == "" { name = fmt.Sprintf("GPU %s:%s", dev.VendorID, dev.DeviceID) @@ -2613,8 +2340,7 @@ func gpuVendorDisplayName(v gpu.Vendor) string { } } -// gpuProbeResult holds the outcome of the always-on startup hardware probe. -// Populated before any config-gated activation logic runs. +// gpuProbeResult holds the outcome of the startup GPU hardware probe. type gpuProbeResult struct { Capable bool // true when Devices, IOMMUActive, and VFIOPresent are all satisfied IOMMUActive bool @@ -2622,8 +2348,7 @@ type gpuProbeResult struct { Devices []gpu.GPUDevice } -// probeGPU discovers GPU hardware and checks passthrough prerequisites. -// It is read-only and has no side effects. +// probeGPU discovers GPU hardware and checks passthrough prerequisites (read-only). func probeGPU() gpuProbeResult { var r gpuProbeResult diff --git a/spinifex/daemon/daemon_handlers_eni.go b/spinifex/daemon/daemon_handlers_eni.go index ee37eccb7..f94d41204 100644 --- a/spinifex/daemon/daemon_handlers_eni.go +++ b/spinifex/daemon/daemon_handlers_eni.go @@ -15,12 +15,9 @@ import ( "github.com/nats-io/nats.go" ) -// handleAttachNetworkInterface composes the existing KV-side AttachENI -// with the live QMP pipeline vm.Manager.HotPlugENI. The KV update happens -// first so a daemon crash before the QMP pipeline leaves the ENI record -// in attaching state for the Sprint 3d reconciler to converge. On QMP -// failure the KV record is rolled back to available with the error -// reason persisted in LastAttachError. +// handleAttachNetworkInterface updates KV first (crash-safe attaching state), +// then runs the QMP hot-plug pipeline. On QMP failure the KV record rolls +// back to available with the error persisted in LastAttachError. func (d *Daemon) handleAttachNetworkInterface(msg *nats.Msg, command types.EC2InstanceCommand, instance *vm.VM) { slog.Info("Attaching ENI to instance", "instanceId", command.ID) @@ -99,10 +96,9 @@ func (d *Daemon) handleAttachNetworkInterface(msg *nats.Msg, command types.EC2In }) } -// handleDetachNetworkInterface reverses the attach pipeline. The KV -// record is marked detaching first so a daemon crash mid-flight leaves -// state for the reconciler; on QMP success the record returns to -// available. +// handleDetachNetworkInterface marks the KV record as detaching first +// (crash-safe), runs the QMP hot-unplug pipeline, then returns the +// record to available on success. func (d *Daemon) handleDetachNetworkInterface(msg *nats.Msg, command types.EC2InstanceCommand, instance *vm.VM) { slog.Info("Detaching ENI from instance", "instanceId", command.ID) diff --git a/spinifex/daemon/daemon_handlers_iam_profile.go b/spinifex/daemon/daemon_handlers_iam_profile.go index aa3b24124..1078e41ed 100644 --- a/spinifex/daemon/daemon_handlers_iam_profile.go +++ b/spinifex/daemon/daemon_handlers_iam_profile.go @@ -11,10 +11,9 @@ import ( "github.com/nats-io/nats.go" ) -// handleAssociateIamInstanceProfile services the per-instance Associate path -// dispatched from handleEC2Events. The gateway has already resolved the -// profile reference and enforced iam:PassRole; instance ownership has been -// verified by checkInstanceOwnership before this point. +// handleAssociateIamInstanceProfile services the per-instance Associate path. +// Gateway pre-validates the profile reference and iam:PassRole; ownership is +// checked by checkInstanceOwnership before dispatch. func (d *Daemon) handleAssociateIamInstanceProfile(msg *nats.Msg, command types.EC2InstanceCommand, instance *vm.VM) { result, err := d.instanceService.AssociateIamInstanceProfile(instance, command) if err != nil { @@ -24,13 +23,9 @@ func (d *Daemon) handleAssociateIamInstanceProfile(msg *nats.Msg, command types. respondWithJSON(msg, result) } -// handleIamProfileDisassociate services the ec2.IamProfileAssociation.disassociate -// fan-out subject. Every daemon responds: the owner with the populated -// association after mutating vm.VM, non-owners (or daemons whose VM has a -// different account) with JSON null so the gateway's expectedNodes collector -// can exit before the timeout. Errors short-circuit the fan-out — only the -// owner can produce a meaningful error (e.g. persistence failure) since -// non-owners NoOp. +// handleIamProfileDisassociate services the disassociate fan-out. The owning +// daemon responds with the mutated association; non-owners respond with JSON +// null so the gateway's expectedNodes collector can exit early. func (d *Daemon) handleIamProfileDisassociate(msg *nats.Msg) { accountID := utils.AccountIDFromMsg(msg) input := &ec2.DisassociateIamInstanceProfileInput{} @@ -49,10 +44,9 @@ func (d *Daemon) handleIamProfileDisassociate(msg *nats.Msg) { respondWithJSON(msg, result) } -// handleIamProfileReplace services the ec2.IamProfileAssociation.replace -// fan-out subject. Same response contract as handleIamProfileDisassociate: -// every daemon always responds (JSON null on non-owners) so the gateway -// collector exits early when the cluster is healthy. +// handleIamProfileReplace services the replace fan-out. Same contract as +// handleIamProfileDisassociate: every daemon responds (JSON null on +// non-owners) so the gateway collector can exit early. func (d *Daemon) handleIamProfileReplace(msg *nats.Msg) { accountID := utils.AccountIDFromMsg(msg) input := &ec2.ReplaceIamInstanceProfileAssociationInput{} @@ -71,11 +65,9 @@ func (d *Daemon) handleIamProfileReplace(msg *nats.Msg) { respondWithJSON(msg, result) } -// handleIamProfileDescribe services the ec2.IamProfileAssociation.describe -// fan-out subject. Empty IamInstanceProfileAssociations slice is a valid -// response (no matches on this daemon) and counts toward the gateway's -// expectedNodes early-exit collector. The gateway concatenates per-daemon -// slices. +// handleIamProfileDescribe services the describe fan-out. An empty slice is a +// valid response (no matches on this daemon) and counts toward the gateway's +// expectedNodes collector; the gateway concatenates per-daemon slices. func (d *Daemon) handleIamProfileDescribe(msg *nats.Msg) { accountID := utils.AccountIDFromMsg(msg) input := &ec2.DescribeIamInstanceProfileAssociationsInput{} diff --git a/spinifex/daemon/daemon_system_dispatch.go b/spinifex/daemon/daemon_system_dispatch.go index 3db3f036b..2bf57f688 100644 --- a/spinifex/daemon/daemon_system_dispatch.go +++ b/spinifex/daemon/daemon_system_dispatch.go @@ -31,18 +31,9 @@ type systemInstanceTerminateEnvelope struct { Error string `json:"error,omitempty"` } -// handleSystemLaunchInstance is the NATS subscriber for -// system.LaunchInstance.{type}[.{nodeID}] requests. The owning daemon (the -// one whose queue-group subscription wins, or the one whose nodeID-targeted -// subscription is hit) runs LaunchSystemInstance locally and the resulting -// VM stays bound to this node — see handleSystemTerminateInstance for the -// matching teardown path. -// -// The launch runs in its own goroutine so the subscription's delivery goroutine -// returns immediately: a multi-minute VM boot must not head-of-line block the -// host-pinned launch subject, or concurrent launches to the same node fail with -// 'no responders' even though the daemon is up (267.4). The NATS request/reply -// contract is preserved — the goroutine responds with the launch result. +// handleSystemLaunchInstance is the NATS subscriber for system.LaunchInstance. +// The launch runs in its own goroutine so a multi-minute VM boot cannot +// head-of-line block concurrent launches to the same node. func (d *Daemon) handleSystemLaunchInstance(msg *nats.Msg) { d.systemDispatchWg.Go(func() { defer func() { @@ -84,12 +75,9 @@ func (d *Daemon) serveSystemLaunchInstance(msg *nats.Msg) { respondWithSystemLaunchOutput(msg, output) } -// LaunchSystemInstanceOnNode launches a system VM on a specific Spinifex host. -// An empty nodeID or the local node runs the launch in-process (matching -// LaunchSystemInstance); any other node is reached over the node-targeted -// system.LaunchInstance.{type}.{nodeID} subject, where the receiving daemon -// runs the launch locally and binds the per-instance terminate subscription -// (handleSystemLaunchInstance). The VM stays bound to the node that runs it. +// LaunchSystemInstanceOnNode launches a system VM on a specific host. +// An empty nodeID or the local node runs in-process; any other node is +// reached via system.LaunchInstance.{type}.{nodeID} — the VM stays on that node. func (d *Daemon) LaunchSystemInstanceOnNode(nodeID string, input *handlers_elbv2.SystemInstanceInput) (*handlers_elbv2.SystemInstanceOutput, error) { if nodeID == "" || nodeID == d.node { return d.LaunchSystemInstance(input) @@ -156,11 +144,8 @@ func (d *Daemon) subscribeSystemTerminateLocked(instanceID string) error { } // handleSystemTerminateInstance is the NATS subscriber for -// system.TerminateInstance.{instanceID}. Only the daemon that owns the VM -// has a subscription on this subject — other daemons never see the request. -// -// Like the launch path, teardown runs in its own goroutine so a slow terminate -// never head-of-line blocks the responder (267.4). +// system.TerminateInstance.{instanceID}. Only the owning daemon subscribes. +// Teardown runs in its own goroutine to avoid head-of-line blocking. func (d *Daemon) handleSystemTerminateInstance(msg *nats.Msg) { d.systemDispatchWg.Go(func() { defer func() { diff --git a/spinifex/daemon/eks_adapter.go b/spinifex/daemon/eks_adapter.go index c68d3e28f..806154a49 100644 --- a/spinifex/daemon/eks_adapter.go +++ b/spinifex/daemon/eks_adapter.go @@ -9,13 +9,7 @@ import ( handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks" ) -// daemonEKSSubnetResolver adapts the daemon's VPCServiceImpl to the -// handlers_eks subnetVPCResolver-equivalent. handlers/eks declares the -// interface privately; the daemon needs an exported shape only because the -// constructor wires it via EKSServiceDeps.SubnetResolver = ... -// -// The translation is trivial: VPCService.GetSubnet returns a SubnetRecord -// with VpcId, which is the only field the EKS launcher needs. +// daemonEKSSubnetResolver adapts VPCServiceImpl to the EKS SubnetVPCResolver. type daemonEKSSubnetResolver struct { d *Daemon } diff --git a/spinifex/daemon/eks_deps.go b/spinifex/daemon/eks_deps.go index 6a93b8aae..75706a345 100644 --- a/spinifex/daemon/eks_deps.go +++ b/spinifex/daemon/eks_deps.go @@ -11,17 +11,8 @@ import ( "log/slog" ) -// buildEKSServiceDeps assembles the EKSServiceDeps the EKS service needs to -// run CreateCluster/DescribeCluster/ListClusters/DeleteCluster end-to-end. -// All collaborators must already be initialized; callers invoke this from -// Daemon.startCluster after VPC, ELBv2, Instance, and Image services are -// ready. -// -// MasterKey is loaded best-effort from the config dir (the directory holding -// spinifex.toml — same place admin init writes master.key, and the same -// resolution daemon TLS + predastore config use). A missing key disables OIDC -// envelope encryption, which in turn fails CreateCluster at the -// depsReadyForOrchestration gate — the daemon logs and continues. +// buildEKSServiceDeps assembles the EKSServiceDeps. All collaborators must +// already be initialised. MasterKey is loaded best-effort from the config dir. func (d *Daemon) buildEKSServiceDeps() handlers_eks.EKSServiceDeps { masterKey, err := handlers_iam.LoadMasterKey(filepath.Join(filepath.Dir(d.configPath), "master.key")) if err != nil { @@ -30,10 +21,6 @@ func (d *Daemon) buildEKSServiceDeps() handlers_eks.EKSServiceDeps { masterKey = nil } - // CA PEM the K3s server VM uses to verify the AWS gateway's HTTPS cert when - // the eks-gateway-publish helper POSTs its bootstrap envelopes + state - // reports. The same CA signs the AWSGW server cert and the NATS server cert, - // so it is read from the configured NATS CA path. gatewayCA := "" if d.config.NATS.CACert != "" { if caBytes, readErr := os.ReadFile(d.config.NATS.CACert); readErr == nil { @@ -68,22 +55,9 @@ func (d *Daemon) buildEKSServiceDeps() handlers_eks.EKSServiceDeps { } } -// resolveGatewayHost is the single source of truth for the off-host-reachable -// address LB VMs, EKS OIDC issuers, and the EKS NATS URL all dial. startCluster -// and buildEKSServiceDeps both call it so the OIDC issuer host can never -// diverge from the host the lb-agent actually reaches (M7). Precedence: -// -// 1. br-mgmt present + AWSGW on a dedicated non-loopback IP distinct from -// AdvertiseIP (multi-node: AWSGW on a mgmt-only IP, VPC path can't reach -// it) → the AWSGW bind IP. startCluster additionally adds a bootcmd host -// route via br-mgmt for this case only. -// 2. AdvertiseIP set → AdvertiseIP (single-node, or multi-node where AWSGW -// binds the advertised IP; VMs reach it via VPC → external). -// 3. br-mgmt present + AWSGW on 0.0.0.0, no AdvertiseIP → the br-mgmt IP. -// 4. DevNetworking shim → 10.0.2.2. -// 5. AWSGW bound to a specific IP (no br-mgmt, no advertise) → that IP. -// -// Returns "" if no reachable host can be derived. +// resolveGatewayHost returns the single AWSGW host used by LB VMs, EKS OIDC +// issuers, and the EKS NATS URL. Precedence: mgmt-dedicated AWSGW IP → +// AdvertiseIP → br-mgmt IP → DevNetworking → AWSGW bind IP. Returns "". func (d *Daemon) resolveGatewayHost() string { awsgwBindIP := "" if d.config.AWSGW.Host != "" { @@ -109,9 +83,7 @@ func (d *Daemon) resolveGatewayHost() string { return "" } -// resolveGatewayBaseURL is the HTTPS AWSGW endpoint EKS OIDC issuers are built -// from. Returns "" if no reachable host can be derived, in which case -// CreateCluster fails at depsReadyForOrchestration. +// resolveGatewayBaseURL returns the HTTPS AWSGW base URL for EKS OIDC. func (d *Daemon) resolveGatewayBaseURL() string { host := d.resolveGatewayHost() if host == "" { diff --git a/spinifex/daemon/eks_worker_launch.go b/spinifex/daemon/eks_worker_launch.go index 82e2e1893..2ce5839cf 100644 --- a/spinifex/daemon/eks_worker_launch.go +++ b/spinifex/daemon/eks_worker_launch.go @@ -15,12 +15,8 @@ import ( // Compile-time check that Daemon satisfies the EKS worker-launch surface. var _ handlers_eks.WorkerLauncher = (*Daemon)(nil) -// RunWorkerInstance launches a nodegroup worker through the normal customer -// RunInstances path: customer-owned, primary ENI auto-created in input.SubnetId -// with input.SecurityGroupIds, no ManagedBy tag, no mgmt NIC — so the worker is -// visible in DescribeInstances and reclaimed by TerminateInstances like any -// other customer EC2. The EKS service hands us raw cloud-config YAML in -// input.UserData; RunInstances expects it base64-encoded, so wrap it here. +// RunWorkerInstance launches a nodegroup worker via the normal RunInstances path. +// UserData is base64-encoded before forwarding; the worker is customer-visible. func (d *Daemon) RunWorkerInstance(input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error) { if d.instanceService == nil { return nil, errors.New("eks worker: instance service not initialized") @@ -34,10 +30,8 @@ func (d *Daemon) RunWorkerInstance(input *ec2.RunInstancesInput, accountID strin return d.instanceService.RunInstances(input, accountID) } -// TerminateWorkerInstances terminates each nodegroup worker via the same path -// the EC2 NATS terminate handler uses. A worker no longer tracked by the local -// vmMgr is treated as already-gone (idempotent success) so a retried -// DeleteNodegroup / scale-down does not wedge on instances that already drained. +// TerminateWorkerInstances terminates nodegroup workers. Already-gone instances +// are treated as success so retried DeleteNodegroup calls are idempotent. func (d *Daemon) TerminateWorkerInstances(instanceIDs []string, accountID string) error { if d.instanceService == nil { return errors.New("eks worker: instance service not initialized") diff --git a/spinifex/daemon/local_state.go b/spinifex/daemon/local_state.go index 8abba64c1..88f2ad079 100644 --- a/spinifex/daemon/local_state.go +++ b/spinifex/daemon/local_state.go @@ -12,9 +12,7 @@ import ( ) const ( - // LocalStateSchemaVersion is the on-disk schema version for instance-state.json. - // Bump on any breaking change to the LocalState shape; daemon refuses to start - // on an unknown version rather than silently losing data. + // LocalStateSchemaVersion guards on-disk compatibility; bump on breaking changes. LocalStateSchemaVersion = 1 // DefaultLocalStateDir is the default directory under DataDir for local state files. @@ -34,11 +32,8 @@ type LocalState struct { VMS map[string]*vm.VM `json:"vms"` } -// LocalStatePath returns the absolute path to the per-node instance state file -// rooted at dataDir. Empty dataDir falls back to DefaultDataDir -// (/var/lib/spinifex), the platform-shared root. The state directory sits at -// the platform root so it lives next to other shared state (predastore, -// viperblock data dirs) rather than nested under the daemon's own subdir. +// LocalStatePath returns the absolute path to the per-node instance state file. +// Empty dataDir falls back to DefaultDataDir. func LocalStatePath(dataDir string) string { if dataDir == "" { dataDir = DefaultDataDir @@ -46,11 +41,8 @@ func LocalStatePath(dataDir string) string { return filepath.Join(dataDir, DefaultLocalStateDir, LocalStateFileName) } -// MarshalLocalState produces the JSON wire form of vms wrapped with the schema -// version. Callers that already hold a stable view of vms (e.g. inside a -// vm.Manager.View callback) can marshal under the lock and pass the bytes to -// WriteLocalStateBytes — that avoids racing json.Marshal against concurrent -// VM-field mutations. +// MarshalLocalState produces the JSON wire form of vms with schema version. +// Call inside a vm.Manager.View callback to avoid races on VM fields. func MarshalLocalState(vms map[string]*vm.VM) ([]byte, error) { state := LocalState{ SchemaVersion: LocalStateSchemaVersion, @@ -63,14 +55,8 @@ func MarshalLocalState(vms map[string]*vm.VM) ([]byte, error) { return data, nil } -// WriteLocalState atomically writes the instance state to path. vms must be a -// snapshot owned by the caller (e.g. from vm.Manager.SnapshotMap) — this -// function does not lock. Convenience wrapper around MarshalLocalState + -// WriteLocalStateBytes for callers that own an exclusive snapshot. -// -// Atomicity: marshal → write to .tmp → fsync → rename. The rename is -// atomic on POSIX so a concurrent reader sees either the old file or the new -// one, never a half-written file. +// WriteLocalState atomically writes instance state via marshal → tmp → fsync → +// rename. vms must be a caller-owned snapshot; this function does not lock. func WriteLocalState(path string, vms map[string]*vm.VM) error { data, err := MarshalLocalState(vms) if err != nil { @@ -80,8 +66,6 @@ func WriteLocalState(path string, vms map[string]*vm.VM) error { } // WriteLocalStateBytes atomically writes pre-marshalled state JSON to path. -// Used by hot paths that marshal under a short-lived lock and then commit to -// disk lock-free. func WriteLocalStateBytes(path string, data []byte) error { dir := filepath.Dir(path) if err := os.MkdirAll(dir, 0o750); err != nil { @@ -119,14 +103,8 @@ func WriteLocalStateBytes(path string, data []byte) error { return nil } -// ReadLocalState reads the per-node instance state from path. -// -// Returns (nil, nil) if the file does not exist — fresh-install signal, -// caller should start with an empty instance map. -// -// Returns an error on: -// - JSON parse failure (corruption — caller refuses start) -// - Unknown SchemaVersion (caller refuses start; safer than silent migration) +// ReadLocalState reads instance state from path. Returns (nil, nil) if the +// file does not exist (fresh install). Errors on JSON failure or unknown schema. func ReadLocalState(path string) (*LocalState, error) { data, err := os.ReadFile(path) if err != nil { diff --git a/spinifex/daemon/ownership.go b/spinifex/daemon/ownership.go index ea6c38820..5294e3d55 100644 --- a/spinifex/daemon/ownership.go +++ b/spinifex/daemon/ownership.go @@ -8,14 +8,11 @@ import ( "github.com/nats-io/nats.go" ) -// checkInstanceOwnership verifies the caller owns the instance. -// Returns true if access is allowed, false if denied (error already sent). -// Instances with an empty AccountID (legacy/migration data) are only -// visible to root (GlobalAccountID). +// checkInstanceOwnership verifies the caller owns the instance. Returns true if +// allowed; false after sending an error. Empty ownerAccountID is root-only. func checkInstanceOwnership(msg *nats.Msg, instanceID, ownerAccountID string) bool { callerAccountID := utils.AccountIDFromMsg(msg) - // Untenanted instance: only root can access. if ownerAccountID == "" { if callerAccountID != utils.GlobalAccountID { slog.Warn("Untenanted instance access denied (not root)", @@ -26,7 +23,6 @@ func checkInstanceOwnership(msg *nats.Msg, instanceID, ownerAccountID string) bo return true } - // Normal ownership check if callerAccountID != ownerAccountID { slog.Warn("Account does not own instance", "instanceId", instanceID, "callerAccount", callerAccountID, "ownerAccount", ownerAccountID) @@ -36,9 +32,8 @@ func checkInstanceOwnership(msg *nats.Msg, instanceID, ownerAccountID string) bo return true } -// isInstanceVisible checks if the caller can see this instance (for Describe operations). -// Instances with an empty AccountID (legacy/migration data) are only -// visible to root (GlobalAccountID). +// isInstanceVisible reports whether the caller may see this instance. +// Empty ownerAccountID is root-only. func isInstanceVisible(callerAccountID, ownerAccountID string) bool { if ownerAccountID == "" { return callerAccountID == utils.GlobalAccountID @@ -46,11 +41,8 @@ func isInstanceVisible(callerAccountID, ownerAccountID string) bool { return callerAccountID == ownerAccountID } -// volumeVisibleTo reports whether callerAccountID may operate on a volume with -// the given tenantID. Volumes with an empty tenantID (legacy/migration -// data) are root-only — without this, the short-circuit -// `tenantID != "" && tenantID != caller` matches every caller and lets any -// tenant attach a legacy/migration volume by ID alone. +// volumeVisibleTo reports whether callerAccountID may access a volume. +// Empty tenantID is root-only to prevent untenanted volumes from leaking. func volumeVisibleTo(tenantID, callerAccountID string) bool { if tenantID == "" { return callerAccountID == utils.GlobalAccountID diff --git a/spinifex/daemon/peer_health.go b/spinifex/daemon/peer_health.go index 5139e4f01..04c3d0006 100644 --- a/spinifex/daemon/peer_health.go +++ b/spinifex/daemon/peer_health.go @@ -11,10 +11,8 @@ import ( "github.com/mulgadc/spinifex/spinifex/config" ) -// Peer-health probe cadence. Worst-case detection of total partition is -// peerProbeInterval + (peers × peerProbeTimeout); on a 3-node cluster with -// the current settings that lands ~6s after the partition installs — well -// inside the 30s budget DDIL Scenario C asserts on. +// Peer-health probe cadence. Worst-case partition detection on a 3-node +// cluster is ~6s — within the 30s DDIL Scenario C budget. const ( peerProbeInterval = 2 * time.Second peerProbeTimeout = 2 * time.Second @@ -49,14 +47,8 @@ func (d *Daemon) peerNodes() []config.Config { return out } -// monitorPeerReachability polls every peer's /health endpoint over the -// cluster network and updates peersReachable. Runs until d.ctx is cancelled. -// No-op on single-node clusters: peersReachable is pinned true in NewDaemon -// and there is nothing to probe. -// -// Probes are issued serially per tick because (a) typical cluster size is 2–3 -// peers, (b) any peer responding is enough to declare reachability, and -// (c) serial dialling caps in-flight HTTPS connections during steady state. +// monitorPeerReachability polls every peer's /health and updates peersReachable. +// No-op on single-node. Probes are serial: any one responding suffices. func (d *Daemon) monitorPeerReachability() { peers := d.peerNodes() if len(peers) == 0 { @@ -99,23 +91,15 @@ func (d *Daemon) probePeersOnce(client *http.Client, peers []config.Config) { if prev != reachable { slog.Info("peer reachability changed", "reachable", reachable, "peers", len(peers)) if reachable { - // Scenario C heal edge: local NATS client stayed connected - // to its local server throughout the partition so - // onNATSReconnect never fires. The peer-probe flip is the - // only signal that the cluster is back. Goroutine keeps - // the probe ticker non-blocking; reconcileOnHeal coalesces - // with any concurrent NATS-reconnect path. + // DDIL Scenario C: NATS stayed connected, so this flip is + // the only heal signal. Goroutine keeps the ticker non-blocking. go d.reconcileOnHeal("peer-probe-heal") } } } -// peerDaemonPort returns the TCP port to dial on peer p. The toml template -// only renders [nodes..daemon] for the local node — remote-node blocks -// carry just `host = ""` — so a peer's own DaemonConfig is typically -// empty. Clusters are symmetric, so this falls back to the local daemon's -// bound port. Returns "" only when neither the peer nor self has a parseable -// host:port. +// peerDaemonPort returns the TCP port to dial on peer p. Falls back to the +// local daemon's port since remote-node config blocks typically omit it. func (d *Daemon) peerDaemonPort(p config.Config) string { if p.Daemon.Host != "" { if _, port, err := net.SplitHostPort(p.Daemon.Host); err == nil && port != "" { diff --git a/spinifex/daemon/reconcile.go b/spinifex/daemon/reconcile.go index 71d16839b..0e092b71d 100644 --- a/spinifex/daemon/reconcile.go +++ b/spinifex/daemon/reconcile.go @@ -4,19 +4,8 @@ import ( "log/slog" ) -// reconcileOnHeal runs the post-heal resync. Fires from two edges: -// - onNATSReconnect, when the local NATS client reattaches. -// - probePeersOnce, when peer reachability flips false→true (the only -// signal in Scenario C where the NATS client stayed connected to its -// local server throughout the partition). -// -// Both edges may fire near-simultaneously on the same heal. reconciling -// coalesces them: the second caller observes the flag set and returns, -// keeping the work to one round-trip. -// -// The caller is expected to invoke this in a goroutine — both call sites -// run on latency-sensitive callbacks (NATS client thread, peer-probe -// ticker) and the reconcile path performs KV I/O. +// reconcileOnHeal runs the post-heal resync (WriteState to KV). Coalesces +// concurrent calls via reconciling flag. Must be called in a goroutine. func (d *Daemon) reconcileOnHeal(reason string) { if d.jsManager == nil { return @@ -29,25 +18,14 @@ func (d *Daemon) reconcileOnHeal(reason string) { slog.Info("reconcileOnHeal: starting", "reason", reason, "node", d.node) - // Push fresh local running-instance state. This re-establishes our - // row in the cluster-wide instance-state bucket regardless of what - // it held mid-partition (stale, missing, or clobbered by a peer's - // stream snapshot restore). if err := d.WriteState(); err != nil { slog.Warn("reconcileOnHeal: WriteState failed", "reason", reason, "error", err) return } - // NOTE: an earlier revision called d.ensureDefaultVPCInfrastructure() - // here. DDIL Scenario C regressed because the very first peer-probe - // tick on multi-node startup flips peersReachable false→true and - // triggers this path concurrently with startCluster's own - // ensureDefaultVPCInfrastructure call. The Describe→Create→Attach - // sequence is not cluster-wide singleton, so multiple daemons + - // double calls per daemon could race and produce duplicate IGW - // attachments, corrupting default-VPC routing. Heal-time bootstrap - // re-fire needs leader election or a debounce against the - // startup-time path before re-introducing. + // ensureDefaultVPCInfrastructure is intentionally omitted: the peer-probe + // heal edge and startCluster race without singleton protection and can + // produce duplicate IGW attachments. Re-introduce only with leader election. slog.Info("reconcileOnHeal: complete", "reason", reason, "revision", d.Revision()) } diff --git a/spinifex/daemon/resource.go b/spinifex/daemon/resource.go index 919a6812b..140121cc9 100644 --- a/spinifex/daemon/resource.go +++ b/spinifex/daemon/resource.go @@ -17,31 +17,19 @@ import ( // cannot be satisfied. var errInsufficientCapacity = errors.New("insufficient capacity to satisfy MinCount") -// hostReserve is the host CPU and RAM held back from guest scheduling for the -// always-on infrastructure tier only — the daemon and co-located fixed services -// (NATS, predastore, viperblock, vpcd, awsgw, ui). It deliberately does NOT -// cover per-instance consumers such as the nbdkit process backing a guest's -// volumes: those scale with instance count and disk I/O, so folding them into a -// fixed reserve makes it unsizable. Per-instance cost is charged to the instance -// and bounded by the live-memory admission gate (see liveMemGate). Tunable via -// SPINIFEX_RESERVED_VCPU / SPINIFEX_RESERVED_MEM_GB until a `capacity` command -// lifts it into config. +// hostReserve holds CPU and RAM reserved for the daemon and co-located fixed +// services. Per-instance costs (nbdkit) are charged separately. Tunable via +// SPINIFEX_RESERVED_VCPU / SPINIFEX_RESERVED_MEM_GB. type hostReserve struct { vCPU int memGB float64 } -// defaultHostReserve covers the fixed infrastructure tier (predastore + -// viperblock storage VMs under load, plus the daemon/NATS/awsgw/vpcd/ui stack). -// Size it from the measured fixed-tier footprint of the target host via -// SPINIFEX_RESERVED_MEM_GB; the default is a conservative floor, not a ceiling. +// defaultHostReserve is the conservative floor for the fixed infrastructure tier. var defaultHostReserve = hostReserve{vCPU: 2, memGB: 2.0} -// resolveHostReserve returns defaultHostReserve, with vCPU/memGB overridden -// when SPINIFEX_RESERVED_VCPU / SPINIFEX_RESERVED_MEM_GB are set to valid -// non-negative values. Invalid values are logged and ignored — keeps a -// typo'd env from silently widening the reserve. Intended as a stop-gap -// until the planned operator-tunable [capacity] config lands. +// resolveHostReserve returns defaultHostReserve overridden by +// SPINIFEX_RESERVED_VCPU / SPINIFEX_RESERVED_MEM_GB when valid. func resolveHostReserve(getenv func(string) string) hostReserve { r := defaultHostReserve if v := getenv("SPINIFEX_RESERVED_VCPU"); v != "" { @@ -61,13 +49,8 @@ func resolveHostReserve(getenv func(string) string) hostReserve { return r } -// resolveHostVCPU returns the detected core count, overridden by -// SPINIFEX_HOST_VCPU when set to a valid positive integer. The override is an -// escape hatch: it pins the schedulable core count for hosts where -// /proc/cpuinfo topology detection misreports (some VMs/containers expose no -// "core id", or fewer physical cores than the host can actually schedule), and -// lets tests decouple from runner topology. Invalid values are logged and -// ignored so a typo can't silently shrink capacity below the reserve. +// resolveHostVCPU returns detected core count, overridden by SPINIFEX_HOST_VCPU +// when valid. Override is an escape hatch for hosts with misreported topology. func resolveHostVCPU(getenv func(string) string, detected int) int { if v := getenv("SPINIFEX_HOST_VCPU"); v != "" { if n, err := strconv.Atoi(v); err == nil && n > 0 { @@ -78,19 +61,11 @@ func resolveHostVCPU(getenv func(string) string, detected int) int { return detected } -// minHostMemHeadroomGB is the minimum schedulable memory we require above -// the reserve, so a host that just meets the reserve still has a small -// amount left to launch the smallest guest type. +// minHostMemHeadroomGB is the minimum schedulable memory above the reserve. const minHostMemHeadroomGB = 0.5 -// applyHostReserve validates that the host meets the minimum size for the -// given reserve and returns the reserve to apply. Pure function — no locks -// or side effects. Exists as a helper for unit-testability of the -// validation bounds. -// -// vCPU and memory are checked separately so the returned error names the -// failing dimension, letting alerting/log filters distinguish a CPU -// shortfall from a memory shortfall. +// applyHostReserve validates host capacity against the reserve and returns the +// reserve to apply. Checks CPU and memory separately for clear error messages. func applyHostReserve(host hostReserve, totalVCPU int, totalMemGB float64) (vcpu int, mem float64, err error) { if totalVCPU <= host.vCPU { return 0, 0, fmt.Errorf( @@ -107,15 +82,8 @@ func applyHostReserve(host hostReserve, totalVCPU int, totalMemGB float64) (vcpu return host.vCPU, host.memGB, nil } -// canAllocateCount returns how many instances of the given type can fit -// in the remaining capacity, capped at maxCount. Pure aside from a single -// slog.Error when remaining capacity is negative — that condition would -// otherwise be silently clamped to zero, hiding a misconfigured reserve -// or allocation accounting drift. -// -// availGPU is the number of free GPUs in the pool; requiresGPU indicates -// that this instance type needs one. When requiresGPU is true and availGPU -// is zero, the result is always 0. +// canAllocateCount returns how many instances of the given type fit in remaining +// capacity, capped at maxCount. Logs an error if capacity is negative. func canAllocateCount(availVCPU, allocVCPU int, availMem, allocMem float64, vCPUs int64, memMiB int64, maxCount int, availGPU int, requiresGPU bool) int { @@ -150,11 +118,8 @@ func canAllocateCount(availVCPU, allocVCPU int, availMem, allocMem float64, return max(result, 0) } -// liveMemCount clamps an accounting-derived launch count n by live host memory: -// how many guests of memGB fit in (availMemGB − reservedMemGB) right now, never -// more than n. Pure function — the live read and instance-type lookup happen in -// liveMemGate. A negative headroom (host already below the reserve) yields 0, so -// admission refuses rather than admitting into OOM territory. +// liveMemCount clamps n by how many guests of memGB fit in live headroom +// (availMemGB − reservedMemGB). Negative headroom yields 0. func liveMemCount(n int, availMemGB, reservedMemGB, memGB float64) int { if memGB <= 0 { return n @@ -163,10 +128,8 @@ func liveMemCount(n int, availMemGB, reservedMemGB, memGB float64) int { return max(0, min(n, byLive)) } -// liveMemAdmissionEnabled reports whether the live-memory admission gate is on. -// Default on; SPINIFEX_ADMISSION_LIVE_MEM set to a false-y value ("0", "false", -// "off", "no") disables it, falling back to pure accounting — a rollback hatch -// if the live gate proves too conservative on a busy host. +// liveMemAdmissionEnabled reports whether the live-memory gate is enabled. +// Set SPINIFEX_ADMISSION_LIVE_MEM=0/false/off/no to disable. func liveMemAdmissionEnabled(getenv func(string) string) bool { switch strings.ToLower(strings.TrimSpace(getenv("SPINIFEX_ADMISSION_LIVE_MEM"))) { case "0", "false", "off", "no": @@ -176,13 +139,8 @@ func liveMemAdmissionEnabled(getenv func(string) string) bool { } } -// readMemAvailableGB returns the host's current /proc/meminfo MemAvailable in -// GB. ok=false on non-Linux hosts or any read/parse failure, which disables the -// live gate for that call (fail open). MemAvailable is the kernel's own estimate -// of memory available for new workloads without swapping, already netting -// reclaimable page cache — the figure admission should gate on, and one that -// already reflects co-located storage VMs, nbdkit, and the daemon stack that the -// static -m accounting never sees. +// readMemAvailableGB returns /proc/meminfo MemAvailable in GB. Returns ok=false +// on non-Linux or read failure (fail open — accounting gate still applies). func readMemAvailableGB() (float64, bool) { if runtime.GOOS != "linux" { return 0, false @@ -200,9 +158,7 @@ func readMemAvailableGB() (float64, bool) { return float64(kb) / (1024 * 1024), true } -// parseMemAvailableKB extracts the MemAvailable value (in kB) from /proc/meminfo -// contents. Returns ok=false when the field is absent or unparseable. Pure — -// split out for unit-testability without touching the filesystem. +// parseMemAvailableKB extracts MemAvailable (kB) from /proc/meminfo. Pure. func parseMemAvailableKB(data []byte) (int64, bool) { for line := range strings.SplitSeq(string(data), "\n") { key, val, ok := strings.Cut(line, ":") @@ -222,29 +178,20 @@ func parseMemAvailableKB(data []byte) (int64, bool) { return 0, false } -// nbdkit runs one process per attached volume (spinifex/nbd). Measured on a live -// host: a main volume's nbdkit climbs to ~0.75 GB under guest I/O (128 MiB cache -// + plugin working set + in-flight buffers); aux volumes (-cloudinit, -efi; 0 -// cache) stay flat ~32-68 MB. Charged per instance so a busy nbdkit can't -// overcommit memory the static -m budget never saw. Env-tunable; RG-6. +// Default per-volume nbdkit memory charges: main ~0.75 GB under I/O, aux ~96 MiB. +// Env-tunable via SPINIFEX_NBDKIT_{MAIN,AUX}_MIB. const ( defaultNbdkitMainMiB = 768 defaultNbdkitAuxMiB = 96 ) -// defaultMainVolumes / defaultAuxVolumes is the volume layout every instance -// gets: one main root volume plus -cloudinit and -efi aux volumes. Block-device -// mappings add main volumes the static charge can't see at admission; the live -// MemAvailable gate (RG-8) is the backstop for those. RG-6. +// Default volume layout: one main root volume + two aux volumes (cloud-init, efi). const ( defaultMainVolumes = 1 defaultAuxVolumes = 2 ) -// resolveNbdkitCharge returns the per-volume nbdkit memory charge in MiB, with -// main/aux overridable via SPINIFEX_NBDKIT_MAIN_MIB / SPINIFEX_NBDKIT_AUX_MIB so -// an operator can retune from their own nbdkit RSS characterisation without a -// rebuild. Invalid or negative values are logged and ignored. +// resolveNbdkitCharge returns per-volume nbdkit charges, overridable via env. func resolveNbdkitCharge(getenv func(string) string) (mainMiB, auxMiB int) { mainMiB, auxMiB = defaultNbdkitMainMiB, defaultNbdkitAuxMiB if v := getenv("SPINIFEX_NBDKIT_MAIN_MIB"); v != "" { @@ -264,17 +211,13 @@ func resolveNbdkitCharge(getenv func(string) string) (mainMiB, auxMiB int) { return mainMiB, auxMiB } -// nbdkitChargeMiB is the per-instance nbdkit charge — one process per volume over -// the default layout (mainVols main + auxVols aux). Pure; RG-6. +// nbdkitChargeMiB returns the total nbdkit memory charge for mainVols + auxVols. func nbdkitChargeMiB(mainVols, auxVols, mainMiB, auxMiB int) int64 { return int64(mainVols*mainMiB + auxVols*auxMiB) } -// resourceStatsForType computes the InstanceTypeCap for a single instance type -// given the remaining host resources. Pure function — no locks or side effects. -// Callers are responsible for alarming on negative remainVCPU/remainMem -// (see canAllocateCount); resourceStatsForType silently clamps to zero -// because it's invoked in a per-type loop and would otherwise log N times. +// resourceStatsForType computes the InstanceTypeCap for a single type given +// remaining resources. Clamps negative capacity to zero without logging. func resourceStatsForType(remainVCPU int, remainMem float64, it *ec2.InstanceTypeInfo) types.InstanceTypeCap { vCPUs := instanceTypeVCPUs(it) memGB := float64(instanceTypeMemoryMiB(it)) / 1024.0 diff --git a/spinifex/daemon/state.go b/spinifex/daemon/state.go index 1d40cf30b..797299dfa 100644 --- a/spinifex/daemon/state.go +++ b/spinifex/daemon/state.go @@ -8,12 +8,8 @@ import ( ) // TransitionState validates and applies a state transition on the given instance. -// It sets instance.Status, persists via WriteState, and logs. -// On validation failure, the instance status is unchanged and a wrapped -// vm.ErrInvalidTransition is returned so callers can map it to the AWS -// IncorrectInstanceState error code via errors.Is. -// If WriteState fails, the in-memory status retains the new value (the VM has -// physically changed state regardless) and an error is returned. +// Returns vm.ErrInvalidTransition on bad transitions; WriteState failure is also +// returned but in-memory status is kept since the VM has already changed state. func (d *Daemon) TransitionState(instance *vm.VM, target vm.InstanceState) error { var ( current vm.InstanceState diff --git a/spinifex/daemon/vm_adapters.go b/spinifex/daemon/vm_adapters.go index 0203a800d..271c94419 100644 --- a/spinifex/daemon/vm_adapters.go +++ b/spinifex/daemon/vm_adapters.go @@ -18,14 +18,8 @@ import ( "github.com/nats-io/nats.go" ) -// Thin adapters that satisfy the vm.Manager collaborator interfaces. The -// bodies delegate to existing daemon methods or services; the manager -// receives them via vm.Deps so it can drive lifecycle without importing the -// daemon package. - -// stateStoreAdapter satisfies vm.StateStore by delegating to the cluster -// JetStream manager. It also tolerates a nil JetStreamManager during early -// boot so manager construction can happen before initJetStream returns. +// stateStoreAdapter satisfies vm.StateStore by delegating to JetStreamManager. +// Tolerates nil JetStreamManager during early boot. type stateStoreAdapter struct { js *JetStreamManager } @@ -69,11 +63,7 @@ func (a *stateStoreAdapter) ListTerminatedInstances() ([]*vm.VM, error) { } // volumeMounterAdapter satisfies vm.VolumeMounter by routing ebs.mount / -// ebs.unmount NATS requests, mirroring Daemon.MountVolumes and -// Daemon.unmountInstanceVolumes. The mount path mutates -// instance.EBSRequests.Requests[k].NBDURI; unmount also flips boot/data -// volumes back to "available" via VolumeStateUpdater so the AWS API view -// stays consistent. +// ebs.unmount NATS requests. Unmount also updates volume state via VolumeStateUpdater. type volumeMounterAdapter struct { nc *nats.Conn node string @@ -205,11 +195,7 @@ func (a *volumeMounterAdapter) MountOne(req *types.EBSRequest) error { return nil } -// UnmountOne sends ebs.unmount for a single request. Best-effort: errors -// are logged and tolerated. Shared by AttachVolume rollback and the -// DetachVolume ebs.unmount step where the caller cannot meaningfully act -// on a failed unmount. Internal callers that need to surface rollback -// failures use unmountOne directly. +// UnmountOne sends ebs.unmount; errors are logged and swallowed. func (a *volumeMounterAdapter) UnmountOne(req types.EBSRequest) { if err := a.unmountOne(req); err != nil { slog.Error("UnmountOne failed", "volume", req.Name, "err", err) @@ -218,8 +204,7 @@ func (a *volumeMounterAdapter) UnmountOne(req types.EBSRequest) { slog.Info("UnmountOne: volume unmounted successfully", "volume", req.Name) } -// unmountOne sends ebs.unmount and returns any error so callers (e.g. the -// Mount partial-failure rollback) can wrap it into a richer failure. +// unmountOne sends ebs.unmount and returns any error. func (a *volumeMounterAdapter) unmountOne(req types.EBSRequest) error { payload, err := json.Marshal(req) if err != nil { @@ -242,10 +227,8 @@ func (a *volumeMounterAdapter) unmountOne(req types.EBSRequest) error { return nil } -// instanceTypeResolverAdapter satisfies vm.InstanceTypeResolver. It looks up -// the EC2 InstanceTypeInfo on the daemon's ResourceManager and projects the -// QEMU-relevant numbers (vCPU, memory MiB, architecture) into the -// SDK-agnostic vm.InstanceTypeSpec. +// instanceTypeResolverAdapter satisfies vm.InstanceTypeResolver by projecting +// EC2 instance type info into the SDK-agnostic vm.InstanceTypeSpec. type instanceTypeResolverAdapter struct { rm *ResourceManager } @@ -272,9 +255,7 @@ func (a *instanceTypeResolverAdapter) Resolve(name string) (vm.InstanceTypeSpec, }, true } -// resourceControllerAdapter satisfies vm.ResourceController. Both Allocate -// and Deallocate look up the EC2 InstanceTypeInfo by name before delegating -// to the existing capacity-keyed resource manager methods. +// resourceControllerAdapter satisfies vm.ResourceController. type resourceControllerAdapter struct { rm *ResourceManager } @@ -309,13 +290,8 @@ func (a *resourceControllerAdapter) CanAllocate(instanceType string, count int) return a.rm.canAllocate(it, count) } -// onInstanceRecoveringHook returns the daemon's OnInstanceRecovering -// callback. Restore fires it once per instance about to be relaunched so -// concurrent terminate commands can land on this node before launch -// completes — without it, ec2.cmd. would only be subscribed by -// onInstanceUpHook after launch success, leaving a window where the -// instance is reachable by DescribeInstances (in StatePending) but not -// by EC2 commands. +// onInstanceRecoveringHook subscribes ec2.cmd. early during Restore so +// terminate commands can land before the relaunch completes. func (d *Daemon) onInstanceRecoveringHook() func(*vm.VM) { return func(instance *vm.VM) { d.mu.Lock() @@ -334,11 +310,8 @@ func (d *Daemon) onInstanceRecoveringHook() func(*vm.VM) { } } -// consumeCleanShutdownMarker returns the daemon's -// ConsumeCleanShutdownMarker callback. Returns true (and deletes the -// marker) when a clean shutdown was recorded for this node on the -// previous run; returns false otherwise so Restore takes the cautious -// "validate stale PIDs" path. +// consumeCleanShutdownMarker returns the ConsumeCleanShutdownMarker callback. +// Returns true and deletes the marker when a clean shutdown was recorded. func (d *Daemon) consumeCleanShutdownMarker() func() bool { return func() bool { if d.jsManager == nil { @@ -354,11 +327,8 @@ func (d *Daemon) consumeCleanShutdownMarker() func() bool { } } -// onInstanceUpHook returns the daemon's OnInstanceUp callback. Subscribing -// per-instance NATS topics is the only side-effect; the manager fires this -// synchronously after a successful Pending→Running transition. Returns the -// first subscribe error so the manager (specifically reconnectInstance) can -// roll back QMP rather than persist a half-reachable instance to KV. +// onInstanceUpHook subscribes per-instance NATS topics after Pending→Running. +// Returns the first subscribe error so the manager can roll back QMP. func (d *Daemon) onInstanceUpHook() func(*vm.VM) error { return func(instance *vm.VM) error { d.mu.Lock() diff --git a/spinifex/filterutil/filterutil.go b/spinifex/filterutil/filterutil.go index fbad30ede..c914b0677 100644 --- a/spinifex/filterutil/filterutil.go +++ b/spinifex/filterutil/filterutil.go @@ -8,11 +8,8 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" ) -// ParseFilters converts AWS SDK filter types into a map[string][]string for -// easier matching. validNames defines the set of accepted filter names. -// Filter names starting with "tag:" are always accepted. Returns an -// InvalidParameterValue-style error if an unsupported filter name is -// encountered. +// ParseFilters converts AWS SDK filter types to map[string][]string. +// tag: prefixed names are always accepted; other names must be in validNames or an error is returned. func ParseFilters(filters []*ec2.Filter, validNames map[string]bool) (map[string][]string, error) { if len(filters) == 0 { return nil, nil @@ -54,10 +51,8 @@ func MatchesAny(filterValues []string, value string) bool { return false } -// MatchesTags checks whether a resource's tags satisfy all tag:Key filters -// present in the filter map. Each tag:Key filter uses OR logic across its -// values — the resource must have the tag and its value must match at least -// one filter value (with wildcard support). +// MatchesTags checks whether a resource's tags satisfy all tag:Key filters in the map. +// Each tag:Key filter uses OR logic across its values, with wildcard support. func MatchesTags(filters map[string][]string, tags map[string]string) bool { for name, values := range filters { if !strings.HasPrefix(name, "tag:") { @@ -89,9 +84,8 @@ func EC2TagsToMap(tags []*ec2.Tag) map[string]string { return m } -// MatchWildcard matches value against a pattern where * matches zero or more -// characters. Case-sensitive — callers needing case-insensitive matching -// (e.g. IAM policy evaluation) should lower-case both inputs before calling. +// MatchWildcard matches value against a pattern where * matches zero or more characters. +// Case-sensitive; callers needing case-insensitive matching should lower-case both inputs. func MatchWildcard(pattern, value string) bool { if pattern == "*" { return true @@ -103,7 +97,6 @@ func MatchWildcard(pattern, value string) bool { parts := strings.Split(pattern, "*") last := len(parts) - 1 - // Check that value starts with the first part and ends with the last. if !strings.HasPrefix(value, parts[0]) { return false } @@ -111,8 +104,7 @@ func MatchWildcard(pattern, value string) bool { return false } - // Trim prefix and suffix from the search region so middle parts - // cannot consume characters reserved for the anchored ends. + // Trim anchored ends before scanning middle parts. remaining := value[len(parts[0]):] if len(remaining) < len(parts[last]) { return false diff --git a/spinifex/formation/formation.go b/spinifex/formation/formation.go index dc215fe54..e0fdc8f3a 100644 --- a/spinifex/formation/formation.go +++ b/spinifex/formation/formation.go @@ -99,10 +99,9 @@ type SharedCredentials struct { AdminSecretKey string `json:"admin_secret_key,omitempty"` } -// FormationServer is a lightweight HTTP server that coordinates cluster formation. -// Nodes register themselves via POST /formation/join. Once the expected number of -// nodes have joined, the done channel is closed and full cluster data (credentials, -// CA, node list) becomes available via GET /formation/status. +// FormationServer is a lightweight HTTPS server that coordinates cluster formation. +// Nodes register via POST /formation/join; full cluster data is available via +// GET /formation/status once the expected count is reached. type FormationServer struct { mu sync.RWMutex expected int diff --git a/spinifex/formation/helpers.go b/spinifex/formation/helpers.go index 9e0b48507..5279da07a 100644 --- a/spinifex/formation/helpers.go +++ b/spinifex/formation/helpers.go @@ -16,11 +16,9 @@ func hasService(services []string, name string) bool { return slices.Contains(services, name) } -// BuildClusterRoutes returns a sorted list of "IP:4248" NATS cluster routes -// for nodes that run the "nats" service. If ClusterIP is set it is used, -// otherwise BindIP. Nodes are sorted by name for deterministic ordering. +// BuildClusterRoutes returns sorted "IP:4248" NATS cluster routes for nodes +// running the "nats" service. ClusterIP is preferred over BindIP. func BuildClusterRoutes(nodes map[string]NodeInfo) []string { - // Filter to nodes that run NATS natsNodes := make(map[string]NodeInfo) for k, n := range nodes { if hasService(n.Services, "nats") { @@ -39,11 +37,9 @@ func BuildClusterRoutes(nodes map[string]NodeInfo) []string { return routes } -// BuildPredastoreNodes returns a sorted list of PredastoreNodeConfig with -// 1-based IDs for nodes that run the "predastore" service. -// Nodes are sorted by name for deterministic ordering. +// BuildPredastoreNodes returns sorted PredastoreNodeConfig (1-based IDs) for +// nodes running the "predastore" service. func BuildPredastoreNodes(nodes map[string]NodeInfo) []admin.PredastoreNodeConfig { - // Filter to nodes that run Predastore predaNodes := make(map[string]NodeInfo) for k, n := range nodes { if hasService(n.Services, "predastore") { diff --git a/spinifex/gpu/bind.go b/spinifex/gpu/bind.go index 0befb29c2..d69f214bd 100644 --- a/spinifex/gpu/bind.go +++ b/spinifex/gpu/bind.go @@ -94,9 +94,8 @@ func unbindVFIO(sysfsRoot, addr, originalDriver string) error { return nil } -// writeSysfs writes value to a sysfs attribute path. -// Uses O_WRONLY|O_CREATE|O_TRUNC so it works against both real sysfs -// (file exists, kernel handles the write) and test temp dirs (file created). +// writeSysfs writes value to a sysfs attribute path. Works against both real +// sysfs and test temp dirs (file created if absent). func writeSysfs(path, value string) error { return os.WriteFile(path, []byte(value), 0o600) } diff --git a/spinifex/gpu/manager.go b/spinifex/gpu/manager.go index 9e4d4cbfa..0cfbc8cef 100644 --- a/spinifex/gpu/manager.go +++ b/spinifex/gpu/manager.go @@ -50,19 +50,10 @@ func (m *Manager) AddMIGGPU(dev GPUDevice) { m.freeMIGGPUs = append(m.freeMIGGPUs, dev) } -// Claim associates the first available GPU entry with instanceID and returns -// the claimed device. For MIG entries the NVIDIA driver manages isolation via -// mdev; no VFIO bind is performed. For whole-GPU entries the device and all -// IOMMU group members are bound to vfio-pci as before. -// -// profileName is the nvidia-smi MIG profile name (e.g. "1g.10gb") for MIG -// requests, or "" for whole-GPU passthrough. When a matching pre-carved slice -// exists it is claimed immediately. When none exists and a free MIG GPU is -// available, MIG instances are created on-demand (the lock is released during -// the nvidia-smi calls). -// -// The returned *MIGInstance is non-nil only for MIG entries; callers use its -// presence to distinguish attachment modes. +// Claim associates the first available GPU with instanceID. profileName selects +// a MIG profile (e.g. "1g.10gb") or "" for whole-GPU passthrough. A pre-carved +// slice is claimed immediately; otherwise a free MIG GPU is carved on-demand. +// The returned *MIGInstance is non-nil only for MIG entries. func (m *Manager) Claim(instanceID, profileName string) (*GPUDevice, *MIGInstance, error) { m.mu.Lock() @@ -217,14 +208,10 @@ func (m *Manager) Claim(instanceID, profileName string) (*GPUDevice, *MIGInstanc return &entry.Device, nil, nil } -// Release returns the GPU held by instanceID to the available pool. -// For devices that were already bound to vfio-pci before the daemon started -// (e.g. by spx-test-gpu), the device is left vfio-pci bound — QEMU has -// already released the group fd on exit, so the next Claim works immediately. -// For devices that were bound by Claim itself, Release unbinds vfio-pci and -// rebinds to the original driver; if that fails the entry is marked unavailable. -// For MIG slices created dynamically by Claim (tracked in committedProfiles), -// the last release on a GPU destroys all slices and returns the GPU to freeMIGGPUs. +// Release returns the GPU held by instanceID to the pool. Pre-bound devices +// (already vfio-pci before the daemon) are left bound; Claim-bound devices are +// unbound and rebind to their original driver (failure marks them unavailable). +// For dynamically carved MIG GPUs, the last release destroys all slices. func (m *Manager) Release(instanceID string) error { m.mu.Lock() defer m.mu.Unlock() @@ -295,9 +282,8 @@ func (m *Manager) Release(instanceID string) error { return firstErr } -// tryDestroyIdleMIGGPULocked destroys all MIG instances on pciAddr if none are -// currently claimed, then returns the GPU to freeMIGGPUs. Must be called with -// m.mu held; releases and reacquires it around the nvidia-smi call. +// tryDestroyIdleMIGGPULocked destroys MIG instances on pciAddr when none are +// claimed and returns the GPU to freeMIGGPUs. Must be called with m.mu held. func (m *Manager) tryDestroyIdleMIGGPULocked(pciAddr string) { for _, e := range m.pool { if e.Device.PCIAddress == pciAddr && e.MIGInstance != nil && e.InstanceID != "" { @@ -347,9 +333,7 @@ func (m *Manager) tryDestroyIdleMIGGPULocked(pciAddr string) { } // MarkFailed marks a whole-GPU entry as permanently unavailable after a -// VFIO/QEMU launch error. The entry is skipped by Claim until the daemon -// restarts. Call this after releasing a GPU whose QEMU crashed during startup -// so the same broken device is not immediately re-assigned to the next instance. +// VFIO/QEMU launch error. Claim skips it until the daemon restarts. func (m *Manager) MarkFailed(pciAddress string) { m.mu.Lock() defer m.mu.Unlock() @@ -364,9 +348,8 @@ func (m *Manager) MarkFailed(pciAddress string) { } } -// MarkMIGFailed marks a specific MIG slice as permanently unavailable after a -// QEMU launch error. Only the slice identified by mdevPath is affected; other -// slices on the same physical GPU remain available. +// MarkMIGFailed marks a MIG slice unavailable after a QEMU launch error. +// Other slices on the same GPU remain available. func (m *Manager) MarkMIGFailed(mdevPath string) { m.mu.Lock() defer m.mu.Unlock() @@ -381,9 +364,7 @@ func (m *Manager) MarkMIGFailed(mdevPath string) { } } -// AddMIGInstances appends pool entries for each MIG slice on device. Called by -// the daemon after CreateInstances or ListInstances populates the slice set for -// a MIG-enabled GPU. +// AddMIGInstances appends pool entries for each MIG slice on device. func (m *Manager) AddMIGInstances(device GPUDevice, instances []MIGInstance) { m.mu.Lock() defer m.mu.Unlock() @@ -429,8 +410,7 @@ func (m *Manager) TotalCount() int { return len(m.pool) } -// PoolEntry is a read-only snapshot of a single pool slot, safe to inspect -// without holding the manager lock. Returned by Snapshot. +// PoolEntry is a read-only snapshot of a single pool slot, returned by Snapshot. type PoolEntry struct { Device GPUDevice MIGInstance *MIGInstance // nil for whole-GPU entries @@ -438,9 +418,8 @@ type PoolEntry struct { Available bool } -// Snapshot returns a point-in-time copy of every pool slot plus any free -// MIG GPUs (not yet committed to a profile). The caller receives owned copies -// and may inspect them without further synchronisation. +// Snapshot returns a point-in-time copy of every pool slot including free MIG GPUs. +// The caller receives owned copies and may inspect them without synchronisation. func (m *Manager) Snapshot() []PoolEntry { m.mu.Lock() defer m.mu.Unlock() @@ -465,9 +444,8 @@ func (m *Manager) Snapshot() []PoolEntry { return out } -// ReclaimByAddress marks the GPU at addr as claimed by instanceID without -// performing sysfs writes. Used on daemon restart to re-register GPUs that -// are still bound to vfio-pci from a previous run. +// ReclaimByAddress marks the GPU at addr as claimed without sysfs writes. +// Used on daemon restart for GPUs still bound to vfio-pci from a previous run. func (m *Manager) ReclaimByAddress(addr, instanceID string) error { m.mu.Lock() defer m.mu.Unlock() @@ -513,9 +491,8 @@ func (m *Manager) ReclaimByAddress(addr, instanceID string) error { return fmt.Errorf("GPU %s not found in pool", addr) } -// ReclaimByMdev marks the MIG slice at mdevPath as claimed by instanceID -// without performing sysfs writes. Used on daemon restart to re-register slices -// whose VMs are still running. +// ReclaimByMdev marks the MIG slice at mdevPath as claimed without sysfs writes. +// Used on daemon restart for slices whose VMs are still running. func (m *Manager) ReclaimByMdev(mdevPath, instanceID string) error { m.mu.Lock() defer m.mu.Unlock() diff --git a/spinifex/gpu/mig.go b/spinifex/gpu/mig.go index 6c408de88..13f4f1591 100644 --- a/spinifex/gpu/mig.go +++ b/spinifex/gpu/mig.go @@ -166,11 +166,9 @@ func createComputeInstance(pciAddr string, giID int) (int, error) { return parseCreatedCIID(string(out)) } -// enrichMdevPaths walks basePath (/sys/bus/mdev/devices) and matches UUIDs to -// instances by reading each entry's symlink target (which contains the parent -// GPU's PCI address) and the GI ID from the gpu_instance_id sysfs attribute. -// basePath is parameterised so tests can inject a temp directory with fake -// symlinks without touching real sysfs. +// enrichMdevPaths matches mdev UUIDs to instances via symlink target (parent GPU +// PCI address) and gpu_instance_id sysfs attribute. basePath is parameterised +// so tests can inject a temp dir without touching real sysfs. func enrichMdevPaths(basePath, pciAddr string, instances []MIGInstance) error { entries, err := os.ReadDir(basePath) if err != nil { diff --git a/spinifex/gpu/reset.go b/spinifex/gpu/reset.go index 124f15a74..b3ca54659 100644 --- a/spinifex/gpu/reset.go +++ b/spinifex/gpu/reset.go @@ -9,18 +9,9 @@ import ( "strings" ) -// resetPCI issues a PCI Function-Level Reset for addr via sysfs. It is a no-op -// for non-AMD devices: NVIDIA and Intel GPUs handle the unbind/rebind cycle -// without requiring an explicit reset. -// -// AMD Instinct (CDNA) data-centre GPUs expose a reset sysfs node and support -// FLR correctly — writing "1" to it clears the hardware state left behind by -// QEMU, allowing the next Claim to bind a clean device. Consumer AMD GPUs -// (Polaris/Vega/Navi) have a firmware reset bug that FLR does not fully fix; -// those architectures are not the target here. -// -// Returns nil when the reset attribute is absent so callers treat this as -// best-effort. +// resetPCI issues a PCI Function-Level Reset for AMD devices via sysfs. +// No-op for NVIDIA/Intel which handle unbind/rebind without an explicit reset. +// Returns nil when the sysfs reset attribute is absent (best-effort). func resetPCI(sysfsRoot, addr string) error { vendorBytes, _ := os.ReadFile(filepath.Join(sysfsRoot, "bus/pci/devices", addr, "vendor")) if strings.TrimSpace(string(vendorBytes)) != "0x1002" { diff --git a/spinifex/handlers/acm/service.go b/spinifex/handlers/acm/service.go index c4cd937d1..f41a323ce 100644 --- a/spinifex/handlers/acm/service.go +++ b/spinifex/handlers/acm/service.go @@ -2,11 +2,7 @@ package handlers_acm import "github.com/aws/aws-sdk-go/service/acm" -// ACMService is the minimal AWS Certificate Manager-alike control plane: BYO -// certificate import and lifecycle. It covers only what ELBv2 HTTPS listeners -// need to resolve a Certificates[].CertificateArn to PEM material. Request -// flows (RequestCertificate + DNS/email validation), renewal, export, and -// tagging are intentionally out of scope for now. +// ACMService covers BYO certificate import and lifecycle for ELBv2 HTTPS listeners. type ACMService interface { ImportCertificate(input *acm.ImportCertificateInput, accountID string) (*acm.ImportCertificateOutput, error) DescribeCertificate(input *acm.DescribeCertificateInput, accountID string) (*acm.DescribeCertificateOutput, error) diff --git a/spinifex/handlers/acm/service_nats.go b/spinifex/handlers/acm/service_nats.go index da5ad276f..5d04fbe5c 100644 --- a/spinifex/handlers/acm/service_nats.go +++ b/spinifex/handlers/acm/service_nats.go @@ -10,8 +10,7 @@ import ( const defaultTimeout = 30 * time.Second -// NATSACMService is the gateway-side client that forwards ACM operations to -// the daemon over NATS request/response. +// NATSACMService forwards ACM operations to the daemon over NATS request/response. type NATSACMService struct { natsConn *nats.Conn } diff --git a/spinifex/handlers/acm/store.go b/spinifex/handlers/acm/store.go index 7764f045f..bae175a06 100644 --- a/spinifex/handlers/acm/store.go +++ b/spinifex/handlers/acm/store.go @@ -20,10 +20,8 @@ const ( KeyPrefixCert = "cert." ) -// CertRecord is the stored representation of an imported certificate. PEM -// material (leaf, chain, private key) is held alongside the parsed metadata -// surfaced by DescribeCertificate. AccountID scopes ownership so list/describe -// never cross account boundaries. +// CertRecord is the stored representation of an imported certificate. +// AccountID scopes ownership so list/describe never cross account boundaries. type CertRecord struct { CertificateArn string `json:"certificate_arn"` AccountID string `json:"account_id"` @@ -62,8 +60,7 @@ func NewStore(nc *nats.Conn) (*Store, error) { return &Store{kv: kv}, nil } -// certKey derives the KV key from a certificate ARN. The ARN's certificate ID -// (the UUID after "certificate/") is the stable per-record key. +// certKey derives the KV key from a certificate ARN using the UUID after "certificate/". func certKey(certArn string) string { id := certArn if i := strings.LastIndex(certArn, "/"); i >= 0 { diff --git a/spinifex/handlers/ec2/eip/service_impl.go b/spinifex/handlers/ec2/eip/service_impl.go index c52ccd0cb..28a43c376 100644 --- a/spinifex/handlers/ec2/eip/service_impl.go +++ b/spinifex/handlers/ec2/eip/service_impl.go @@ -412,15 +412,13 @@ func addressMatchesFilters(record *EIPRecord, filters map[string][]string) bool return filterutil.MatchesTags(filters, record.Tags) } -// DescribeAddressesAttribute returns per-EIP attributes (e.g. domain-name). -// Spinifex doesn't support reverse-DNS PTR records, so PtrRecord is left nil. -// Unlike DescribeAddresses, this returns an empty list (not an error) when -// requested AllocationIds are not found. This matches real AWS behavior. +// DescribeAddressesAttribute returns per-EIP attributes. PtrRecord is always nil +// (no reverse-DNS support). Returns an empty list, not an error, for missing IDs. func (s *EIPServiceImpl) DescribeAddressesAttribute(input *ec2.DescribeAddressesAttributeInput, accountID string) (*ec2.DescribeAddressesAttributeOutput, error) { var addresses []*ec2.AddressAttribute if len(input.AllocationIds) > 0 { - // Direct lookups — O(n) on requested IDs rather than scanning all EIPs. + // Direct lookups by requested IDs. for _, id := range input.AllocationIds { if id == nil { continue @@ -570,10 +568,9 @@ func (s *EIPServiceImpl) findByAssociationID(accountID, associationID string) (* return nil, "", 0, errors.New(awserrors.ErrorInvalidAssociationIDNotFound) } -// AssociatedPublicIPForInstance returns the public IP of the Elastic IP -// associated with instanceID, if any. The daemon uses it to re-announce a -// post-launch EIP's dnat_and_snat on VM relaunch, where the instance's own -// PublicIP field is unset (only auto-assigned public IPs populate it). +// AssociatedPublicIPForInstance returns the public IP of the EIP associated with +// instanceID, if any. Used by the daemon to re-announce dnat_and_snat on relaunch +// when the instance's own PublicIP field is unset (EIP-assigned vs auto-assigned). func (s *EIPServiceImpl) AssociatedPublicIPForInstance(accountID, instanceID string) (string, bool) { if instanceID == "" { return "", false @@ -602,13 +599,9 @@ func (s *EIPServiceImpl) AssociatedPublicIPForInstance(accountID, instanceID str return "", false } -// publishNATEvent publishes a NAT lifecycle event to NATS for vpcd consumption. -// This is fire-and-forget; errors are logged but do not fail the API response. -// -// PortName must match the OVN logical switch port name ("port-") because -// vpcd sets NAT.LogicalPort to this value in distributed NAT mode (direct -// bridge). A mismatch creates a dnat_and_snat row pointing at a nonexistent -// port, and OVN never programs the DNAT flow. +// publishNATEvent publishes a NAT lifecycle event to NATS for vpcd (fire-and-forget). +// PortName must use topology.Port(eniID) to match the OVN logical switch port name; +// a mismatch causes OVN to never program the DNAT flow. func (s *EIPServiceImpl) publishNATEvent(topic, vpcID, externalIP, logicalIP, eniID, mac string) { utils.PublishEvent(s.natsConn, topic, natEvent{ VpcId: vpcID, diff --git a/spinifex/handlers/ec2/eip/service_impl_test.go b/spinifex/handlers/ec2/eip/service_impl_test.go index 44df84e15..2d844889c 100644 --- a/spinifex/handlers/ec2/eip/service_impl_test.go +++ b/spinifex/handlers/ec2/eip/service_impl_test.go @@ -82,8 +82,8 @@ func TestEIP_Release(t *testing.T) { }, testAccountID) require.NoError(t, err) - // Round-robin: re-allocating does NOT hand back the just-released IP - // (siv-246); the cursor advances, so the new EIP differs. + // Round-robin: re-allocating does NOT hand back the just-released IP; + // the cursor advances, so the new EIP differs. out2, err := svc.AllocateAddress(&ec2.AllocateAddressInput{}, testAccountID) require.NoError(t, err) assert.NotEqual(t, allocatedIP, *out2.PublicIp, "released EIP must not be reused immediately") @@ -558,12 +558,9 @@ func TestEIP_DescribeAddresses_FilterByTag(t *testing.T) { assert.Equal(t, *out.AllocationId, *desc.Addresses[0].AllocationId) } -// TestEIP_PublishNATEvent_PortNameHasPortPrefix is a regression test for a bug -// where EIPServiceImpl.publishNATEvent sent the raw ENI id as PortName. vpcd -// writes PortName into NAT.LogicalPort in distributed NAT mode (direct bridge), -// so a mismatch with the OVN logical switch port name ("port-") produces -// a dnat_and_snat row pointing at a nonexistent port — OVN never programs the -// flow and the EIP black-holes. +// TestEIP_PublishNATEvent_PortNameHasPortPrefix verifies that publishNATEvent +// uses topology.Port(eniID) as PortName. A raw ENI id mismatch creates a +// dnat_and_snat row pointing at a nonexistent OVN port, black-holing the EIP. func TestEIP_PublishNATEvent_PortNameHasPortPrefix(t *testing.T) { svc, _ := setupTestEIP(t) @@ -580,9 +577,7 @@ func TestEIP_PublishNATEvent_PortNameHasPortPrefix(t *testing.T) { var got natEvent require.NoError(t, json.Unmarshal(msg.Data, &got)) assert.Equal(t, "port-"+eniID, got.PortName, - "PortName must match the OVN logical switch port name (port-); "+ - "otherwise vpcd's distributed NAT creates a dnat_and_snat rule "+ - "pointing at a nonexistent logical port and the flow is never programmed") + "PortName must be port- to match the OVN logical switch port name") assert.Equal(t, "10.0.0.5", got.LogicalIP) assert.Equal(t, "198.51.100.10", got.ExternalIP) } diff --git a/spinifex/handlers/ec2/eip/types.go b/spinifex/handlers/ec2/eip/types.go index 13a16f50a..45856eb82 100644 --- a/spinifex/handlers/ec2/eip/types.go +++ b/spinifex/handlers/ec2/eip/types.go @@ -16,10 +16,8 @@ type EIPRecord struct { ENIId string `json:"eni_id,omitempty"` InstanceId string `json:"instance_id,omitempty"` PrivateIp string `json:"private_ip,omitempty"` - // MacAddress is the associated ENI's MAC, captured at associate-time so - // the vpcd reconcile can re-apply the distributed-shape dnat_and_snat - // (external_mac/logical_port) after a host reboot without re-querying the - // ENI. Empty while the EIP is unassociated. + // MacAddress is the ENI's MAC captured at associate-time so vpcd can + // re-apply dnat_and_snat after a host reboot. Empty when unassociated. MacAddress string `json:"mac_address,omitempty"` VpcId string `json:"vpc_id,omitempty"` State string `json:"state"` // "allocated", "associated" diff --git a/spinifex/handlers/ec2/igw/service_impl.go b/spinifex/handlers/ec2/igw/service_impl.go index 5493503e3..d14e3ee32 100644 --- a/spinifex/handlers/ec2/igw/service_impl.go +++ b/spinifex/handlers/ec2/igw/service_impl.go @@ -37,12 +37,9 @@ type IGWRecord struct { CreatedAt time.Time `json:"created_at"` } -// GatePublisher recomputes the per-subnet egress gate/ungate decision for -// every subnet in a VPC. Wired by the daemon after IGW + RouteTable services -// are constructed so AttachInternetGateway / DetachInternetGateway can fan -// out gate decisions immediately instead of waiting up to 5 minutes for the -// reconciler's drift tick to install OVN DROP policies on unrouted subnets. -// Nil-safe: when unset the handler falls back to publish-only behaviour. +// GatePublisher recomputes per-subnet egress gate/ungate decisions for a VPC. +// Wired by the daemon so IGW attach/detach triggers immediate OVN policy updates +// rather than waiting for the reconciler's drift tick. type GatePublisher interface { PublishGateDecisionsForVPC(accountID, vpcID, destCidr string) } @@ -332,13 +329,8 @@ func (s *IGWServiceImpl) AttachInternetGateway(input *ec2.AttachInternetGatewayI } } - // Gate fan-out intentionally skipped on attach. The default-VPC - // bootstrap path attaches the IGW and immediately publishes a - // 0.0.0.0/0 → IGW CreateRoute event; the two race across distinct - // NATS subjects (gate vs ungate) and ungate-then-gate leaves the - // subnet DROPped. Subnets without a default route stay protected by - // the reconciler's drift pass (loadSubnetDropGates). Detach is - // race-free because no follow-up CreateRoute fires. + // Gate fan-out is intentionally skipped on attach to avoid a race with + // the bootstrap CreateRoute path. Detach triggers gate fan-out directly. slog.Info("AttachInternetGateway completed", "internetGatewayId", igwID, "vpcId", vpcID, "accountID", accountID) diff --git a/spinifex/handlers/ec2/igw/service_impl_test.go b/spinifex/handlers/ec2/igw/service_impl_test.go index 6fefe0cef..a16c1ba44 100644 --- a/spinifex/handlers/ec2/igw/service_impl_test.go +++ b/spinifex/handlers/ec2/igw/service_impl_test.go @@ -699,11 +699,8 @@ func TestDeleteInternetGateway_PublishesNoEvent(t *testing.T) { } } -// TestAttachInternetGateway_NoGateFanOut documents that attach intentionally -// skips the gate fan-out to avoid racing the default-VPC bootstrap path, -// which attaches the IGW and immediately adds a 0.0.0.0/0 → IGW route. The -// two events would race across the gate/ungate NATS subjects and leave the -// default subnet DROPped when ungate landed first. +// TestAttachInternetGateway_NoGateFanOut verifies that IGW attach skips gate +// fan-out to avoid racing the bootstrap CreateRoute path. func TestAttachInternetGateway_NoGateFanOut(t *testing.T) { svc, _ := setupTestIGWService(t) pub := &fakeGatePublisher{} diff --git a/spinifex/handlers/ec2/image/service_impl.go b/spinifex/handlers/ec2/image/service_impl.go index 5fd6b4829..4090db445 100644 --- a/spinifex/handlers/ec2/image/service_impl.go +++ b/spinifex/handlers/ec2/image/service_impl.go @@ -744,10 +744,9 @@ func (s *ImageServiceImpl) putSnapshotMetadata(snapshotID, volumeID string, volu return handlers_ec2_snapshot.WriteSnapshotConfig(s.store, s.bucketName, snapshotID, &cfg) } -// CopyImage clones an AMI same-region, metadata-only: the new snapshot shares -// the source's VolumeID and a fresh config.json points at it. -// Source visibility is checked before the O(n) name-uniqueness scan so typos -// and cross-account sources fast-fail without a full AMI listing. +// CopyImage clones an AMI same-region, metadata-only: the new snapshot shares the +// source's VolumeID and a fresh config.json points at it. Source visibility is checked +// before the name-uniqueness scan so cross-account sources fast-fail. func (s *ImageServiceImpl) CopyImage(input *ec2.CopyImageInput, accountID string) (*ec2.CopyImageOutput, error) { if input == nil || input.Name == nil || *input.Name == "" || input.SourceImageId == nil || *input.SourceImageId == "" { @@ -779,12 +778,8 @@ func (s *ImageServiceImpl) CopyImage(input *ec2.CopyImageInput, accountID string } srcSnap, err := handlers_ec2_snapshot.ReadSnapshotConfig(s.store, s.bucketName, srcMeta.SnapshotID) if err != nil { - // Bundled system AMIs (admin-imported via `spx admin images import`) - // store blocks under ami-xxx/ with no standalone snap-xxx/metadata.json - // — the SnapshotID field is a viperblock-internal snap reference. To - // keep copy-image AWS-compatible (copy of a public/shared AMI must - // succeed), synthesize a minimal snap view from the AMI itself: the - // bundled prefix IS the volume, so VolumeID = sourceImageID. + // Bundled system AMIs have no standalone snap-xxx/metadata.json; synthesize + // a minimal snap view using VolumeID = sourceImageID so CopyImage succeeds. if objectstore.IsNoSuchKeyError(err) && srcMeta.ImageOwnerAlias != "" && !utils.IsAccountID(srcMeta.ImageOwnerAlias) { srcSnap = &handlers_ec2_snapshot.SnapshotConfig{ SnapshotID: srcMeta.SnapshotID, @@ -943,10 +938,9 @@ func (s *ImageServiceImpl) clusterEncryptionEnabled() bool { return mkey != nil } -// synthesizeRootBlockDeviceMapping returns /dev/sda1 with size+snapshot from -// AMIMetadata, or nil for non-EBS AMIs. encrypted reflects the cluster-level -// encryption-at-rest posture (master key configured) since AMI metadata does -// not carry a per-image encryption flag of its own. +// synthesizeRootBlockDeviceMapping returns /dev/sda1 with size+snapshot from AMIMetadata, +// or nil for non-EBS AMIs. encrypted reflects the cluster-level encryption posture +// (master key configured); AMI metadata carries no per-image encryption flag. func synthesizeRootBlockDeviceMapping(meta viperblock.AMIMetadata, encrypted bool) []*ec2.BlockDeviceMapping { if meta.RootDeviceType != "ebs" { return nil @@ -1108,10 +1102,8 @@ func (s *ImageServiceImpl) DeregisterImage(input *ec2.DeregisterImageInput, acco return &ec2.DeregisterImageOutput{}, nil } -// ModifyImageAttribute writes a modifiable AMI attribute. Gateway normalises -// into Attribute+Value; only description is writable. Ownership is checked -// before the attribute switch so cross-account callers always see -// UnauthorizedOperation. +// ModifyImageAttribute writes a modifiable AMI attribute; only description is writable. +// Ownership is checked first so cross-account callers always see UnauthorizedOperation. func (s *ImageServiceImpl) ModifyImageAttribute(input *ec2.ModifyImageAttributeInput, accountID string) (*ec2.ModifyImageAttributeOutput, error) { if input == nil || input.ImageId == nil || *input.ImageId == "" || input.Attribute == nil || *input.Attribute == "" { diff --git a/spinifex/handlers/ec2/image/service_impl_test.go b/spinifex/handlers/ec2/image/service_impl_test.go index 2a144bdfa..bf618fb07 100644 --- a/spinifex/handlers/ec2/image/service_impl_test.go +++ b/spinifex/handlers/ec2/image/service_impl_test.go @@ -200,10 +200,8 @@ func TestDescribeImages_AfterCreate(t *testing.T) { assert.Equal(t, testAccountID, *result.Images[0].OwnerId) } -// TestDescribeImages_BootModeProjection asserts that AMIMetadata.BootMode -// round-trips through DescribeImages. Empty BootMode (legacy AMIs registered -// before the field existed) must pass through as an empty string, not get -// synthesized to a default. +// TestDescribeImages_BootModeProjection asserts that AMIMetadata.BootMode round-trips +// through DescribeImages; empty BootMode on legacy AMIs must pass through unchanged. func TestDescribeImages_BootModeProjection(t *testing.T) { svc, store := setupTestImageService(t) @@ -871,10 +869,8 @@ func createTestAMIConfigFull(t *testing.T, store *objectstore.MemoryObjectStore, require.NoError(t, err) } -// TestDescribeImages_FilterBy verifies single-attribute filters. Each subtest -// builds its own AMI fixtures and asserts which IDs the filter selects. -// Multi-filter, wildcard, error-path, and no-filter cases live in their own -// dedicated tests below. +// TestDescribeImages_FilterBy verifies single-attribute filters; each subtest builds +// its own AMI fixtures. Multi-filter, wildcard, and error-path cases have their own tests. func TestDescribeImages_FilterBy(t *testing.T) { tests := []struct { name string @@ -1542,11 +1538,8 @@ func putTestAMIConfigWithSnapshot(t *testing.T, store *objectstore.MemoryObjectS require.NoError(t, err) } -// seedCopyableAMI writes a matching (snapshot, AMI) pair so CopyImage can -// complete end-to-end. Returns the AMI metadata that was persisted (callers -// can customise fields before seeding by tweaking the returned VolumeID / -// tags via preceding calls, but the helper takes the simple path for the -// happy case). +// seedCopyableAMI writes a matching (snapshot, AMI) pair so CopyImage can complete +// end-to-end. func seedCopyableAMI(t *testing.T, store *objectstore.MemoryObjectStore, imageID, name, owner, snapshotID, volumeID string, sizeGiB int64) { t.Helper() cfg := handlers_ec2_snapshot.SnapshotConfig{ @@ -1704,10 +1697,8 @@ func TestCopyImage_SystemAMICopiedIntoCallerAccount(t *testing.T) { assert.Equal(t, "spinifex", srcMeta.ImageOwnerAlias) } -// Bundled system AMIs (admin-imported) have no standalone snap-xxx/metadata.json -// — their SnapshotID refers to a viperblock-internal snap, and blocks live under -// ami-xxx/. CopyImage must still succeed (AWS parity: copy of a public AMI works), -// falling back to synthesizing a snap view where VolumeID = sourceImageID. +// Bundled system AMIs have no standalone snap-xxx/metadata.json; CopyImage must +// still succeed by synthesizing a snap view where VolumeID = sourceImageID. func TestCopyImage_BundledSystemAMINoStandaloneSnap(t *testing.T) { svc, store := setupTestImageService(t) diff --git a/spinifex/handlers/ec2/instance/service.go b/spinifex/handlers/ec2/instance/service.go index 8c7e40d42..400b9dcfd 100644 --- a/spinifex/handlers/ec2/instance/service.go +++ b/spinifex/handlers/ec2/instance/service.go @@ -43,19 +43,9 @@ type TerminateStoppedInstanceOutput struct { InstanceID string `json:"instanceId"` } -// ResourceCapacityProvider exposes per-node instance-type availability used by -// DescribeInstanceTypes. Implemented by daemon.ResourceManager. -// -// GetAvailableInstanceTypeInfos reports types gated by free schedulable -// capacity (one entry per free slot when showCapacity is true). It is used -// when callers ask "what could you launch right now?" — typically with the -// capacity=true filter for aggregated cluster-wide accounting. -// -// GetSupportedInstanceTypeInfos reports every type this node is configured -// to run, regardless of current allocation. This is the AWS-compatible -// answer for plain DescribeInstanceTypes and is what tooling like the -// Terraform AWS provider expects when it performs a metadata lookup after -// RunInstances. +// ResourceCapacityProvider exposes per-node instance-type availability. +// GetAvailableInstanceTypeInfos gates on free capacity; GetSupportedInstanceTypeInfos +// returns all configured types regardless of allocation. type ResourceCapacityProvider interface { GetAvailableInstanceTypeInfos(showCapacity bool) []*ec2.InstanceTypeInfo GetSupportedInstanceTypeInfos() []*ec2.InstanceTypeInfo @@ -129,10 +119,8 @@ type SubnetInfo struct { MapPublicIpOnLaunch bool } -// ENIInfo carries the subset of ENI metadata RunInstances needs to attach a -// pre-created ENI as the primary interface. Translated from -// handlers/ec2/vpc.ENIRecord by the daemon adapter to avoid a cyclic -// instance↔vpc import. +// ENIInfo carries the ENI metadata RunInstances needs for a pre-created primary +// interface. Translated from handlers/ec2/vpc.ENIRecord to avoid a cyclic import. type ENIInfo struct { NetworkInterfaceID string SubnetID string @@ -143,14 +131,9 @@ type ENIInfo struct { SecurityGroupIDs []string } -// ENICreator covers the VPC/ENI operations RunInstances performs while -// auto-attaching a primary interface. Implemented via an adapter over -// handlers/ec2/vpc.VPCServiceImpl on the daemon. DetachENI lives here (not on -// ENIDeleter) because the launch-time NAT-rollback path needs to flip the ENI -// to "available" before ENIDeleter.DeleteNetworkInterface, which rejects -// in-use ENIs. GetENI supports the pre-created-ENI launch path where the -// caller passes NetworkInterfaces[0].NetworkInterfaceId and RunInstances -// attaches the existing ENI instead of creating a new one. +// ENICreator covers the VPC/ENI operations RunInstances uses to auto-attach a +// primary interface. DetachENI is here (not ENIDeleter) so the NAT-rollback +// path can flip the ENI to "available" before deletion. type ENICreator interface { GetDefaultSubnet(accountID string) (*SubnetInfo, error) GetSubnet(accountID, subnetID string) (*SubnetInfo, error) diff --git a/spinifex/handlers/ec2/instance/service_impl.go b/spinifex/handlers/ec2/instance/service_impl.go index 3f31e4814..b2afafe54 100644 --- a/spinifex/handlers/ec2/instance/service_impl.go +++ b/spinifex/handlers/ec2/instance/service_impl.go @@ -89,10 +89,9 @@ instance-id: {{.InstanceID}} local-hostname: {{.Hostname}} ` -// cloudInitNetworkConfigWildcard enables DHCP on all NICs via wildcard match. -// Used when there's no dual-NIC setup (non-VPC or VPC without DEV_NETWORKING). -// The "e*" glob matches both traditional names (eth0, eth1 — Alpine/older -// kernels) and predictable names (ens3, enp0s3 — systemd-based distros). +// cloudInitNetworkConfigWildcard enables DHCP on all NICs via "e*" glob, +// covering both traditional (eth0) and predictable (ens3) names. +// Used when there is no dual-NIC setup (non-VPC or no DEV_NETWORKING). const cloudInitNetworkConfigWildcard = `network: version: 2 ethernets: @@ -103,19 +102,16 @@ const cloudInitNetworkConfigWildcard = `network: dhcp-identifier: mac ` -// cloudInitNetworkConfigDisabled tells cloud-init to do no network rendering -// at all. Used on RHEL-family guests where Spinifex writes NM keyfiles via -// user-data write_files; without this stub cloud-init's RHEL network module -// falls back to autodetect-DHCP and drops a second keyfile next to ours. +// cloudInitNetworkConfigDisabled suppresses cloud-init network rendering. +// Used on RHEL guests where NM keyfiles are written via write_files; +// without this stub cloud-init would drop a competing autoconnect keyfile. const cloudInitNetworkConfigDisabled = `network: config: disabled ` -// selectNetworkConfigForFamily returns the cloud-init network-config payload -// for the given distro family. RHEL-family guests get an explicit -// "config: disabled" stub because their NM keyfiles are written via user-data -// write_files; shipping a netplan config alongside would let cloud-init's RHEL -// network module drop a competing cloud-init-enpXsY.nmconnection. +// selectNetworkConfigForFamily returns the cloud-init network-config for the distro. +// RHEL gets "config: disabled" to prevent cloud-init from dropping a competing +// NM keyfile alongside the ones written via user-data write_files. func selectNetworkConfigForFamily(distroFamily, eniMAC, devMAC, mgmtMAC, mgmtIP string, extraENIMACs []string) string { if distroFamily == "rhel" { return cloudInitNetworkConfigDisabled @@ -128,26 +124,8 @@ func selectNetworkConfigForFamily(distroFamily, eniMAC, devMAC, mgmtMAC, mgmtIP } // generateNetworkConfig produces the cloud-init network-config for the instance. -// -// Per-interface config is generated when eniMAC is present (VPC NIC). This allows -// adding the mgmt NIC with a static IP and optionally the dev NIC with route -// suppression. Without per-interface config, the wildcard fallback does DHCP on -// all NICs — which won't work for the mgmt NIC (no DHCP server on br-mgmt). -// -// extraENIMACs configures additional VPC NICs for multi-subnet system VMs -// (e.g. multi-AZ ALB VMs). Each extra MAC produces a DHCP ethernet block named -// vpc1, vpc2, ... so each interface pulls its address from the subnet it lives in. -// -// The dev NIC still gets an IP via DHCP (needed for hostfwd port forwarding) -// but dhcp4-overrides prevents it from installing routes or DNS. -// -// The primary VPC NIC (vpc0) also carries an on-link route to the IMDS metadata -// IP (169.254.169.254): under the L2 datapath the metadata localport lives on -// the guest's subnet switch and answers ARP directly, so the guest must treat -// the address as on-link rather than steering it via the default gateway. The -// route is reached via the primary ENI, matching AWS; it is not delivered via -// DHCP option 121 (which would force the default route into option 121 per -// RFC 3442) and the default gateway stays option 3. +// Emits per-interface blocks when eniMAC is set; falls back to wildcard DHCP +// when eniMAC is empty. func generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP string, extraENIMACs []string, emitIMDSRoute bool) string { if eniMAC == "" { return cloudInitNetworkConfigWildcard @@ -161,10 +139,8 @@ func generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP string, extraENIMACs dhcp4: true dhcp-identifier: mac `, eniMAC) - // The gateway-less IMDS on-link route renders only under netplan. cloud-init's - // eni renderer (Alpine) crashes on a route with no gateway, taking the whole - // network-config down with it, so eni-rendered guests get this route via - // buildAlpineCloudInit instead (emitIMDSRoute=false). + // Alpine's eni renderer crashes on gateway-less routes; omit here and + // deliver via buildAlpineCloudInit instead (emitIMDSRoute=false). if emitIMDSRoute { cfg += fmt.Sprintf(` routes: - to: %s/32 @@ -203,8 +179,7 @@ func generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP string, extraENIMACs addresses: - "%s/24" `, mgmtMAC, mgmtIP) - // Route for multi-node LB mgmt traffic is delivered to the LB microVM - // via the fw_cfg netcfg blob, not here. + // LB mgmt route is delivered via fw_cfg netcfg, not here. } return cfg @@ -217,46 +192,27 @@ type CloudInitData struct { UserDataCloudConfig string UserDataScript string CACertPEM string - // DistroFamily selects per-distro branches in the cloud-init template. - // "debian" | "rhel" | "alpine". Empty falls through to debian rendering - // (legacy AMIs registered before the field existed). + // DistroFamily selects per-distro rendering: "debian" | "rhel" | "alpine". + // Empty falls through to debian (legacy AMIs without this field). DistroFamily string - // SudoGroup is the OS group cloud-init adds the user to for passwordless - // sudo. "sudo" on debian/ubuntu/alpine, "wheel" on RHEL-family. + // SudoGroup is "sudo" on debian/ubuntu/alpine, "wheel" on RHEL-family. SudoGroup string - // RHELWriteFiles is the pre-indented YAML block of NM keyfile entries - // (one per NIC) appended under the merged write_files: key. Empty on - // non-RHEL guests; the network-config ISO file carries the YAML netplan - // equivalent instead. + // RHELWriteFiles is the pre-indented YAML NM keyfile block (one per NIC). + // Empty on non-RHEL guests; those use the network-config ISO instead. RHELWriteFiles string - // RHELRunCmd is the pre-indented YAML block of runcmd entries (restorecon - // + nmcli reload/up) needed to load and bring up the keyfiles. Empty on - // non-RHEL guests. + // RHELRunCmd is the pre-indented runcmd YAML (restorecon + nmcli reload/up). + // Empty on non-RHEL guests. RHELRunCmd string - // AlpineWriteFiles / AlpineRunCmd carry the IMDS on-link route delivery for - // Alpine guests (a persistent /etc/local.d script + OpenRC `local` enable), - // since the eni renderer cannot emit the gateway-less route in the - // network-config. Empty on non-Alpine guests and on non-VPC instances. + // AlpineWriteFiles / AlpineRunCmd deliver the IMDS on-link route via a + // local.d script (eni renderer can't emit gateway-less routes). + // Empty on non-Alpine or non-VPC instances. AlpineWriteFiles string AlpineRunCmd string } // buildRHELCloudInit produces the cloud-init write_files (NM keyfiles) and -// runcmd (restorecon + nmcli) blocks needed to bring up networking on a -// RHEL-family guest. Returns empty strings when eniMAC is empty (wildcard / -// non-VPC path) so NM's default autoconnect handles the interface. -// -// Mirrors generateNetworkConfig's per-interface structure: vpc0 (DHCP + on-link -// IMDS route) + optional vpc1..N for extra ENIs, dev0 (DHCP with route/DNS -// suppression), mgmt0 (static). Owning the keyfile bytes directly avoids relying on cloud-init's -// v2-to-NM renderer round-tripping dhcp-identifier: mac → dhcp-client-id=mac, -// which OVN's MAC-keyed DHCP requires. -// -// File mode 0600 root:root is load-bearing — NM ignores keyfiles otherwise. -// restorecon defers to host SELinux policy (no-op on non-SELinux systems); -// nmcli reload forces NM to rescan system-connections after write_files -// completes in cloud-init's config stage, and nmcli up brings each interface -// online without waiting on autoconnect heuristics. +// runcmd blocks for RHEL networking. Returns empty strings when eniMAC is empty. +// File mode 0600 is required — NM ignores world-readable keyfiles. func buildRHELCloudInit(eniMAC, devMAC, mgmtMAC, mgmtIP string, extraENIMACs []string) (writeFiles, runCmd string) { if eniMAC == "" { return "", "" @@ -309,15 +265,12 @@ func appendRHELDHCPKeyfile(b *strings.Builder, name, mac string, suppressDefault b.WriteString(" method=auto\n") b.WriteString(" dhcp-client-id=mac\n") if suppressDefaults { - // Equivalent to netplan dhcp4-overrides {use-routes: false, use-dns: false}: - // dev NIC gets an IP for hostfwd but must not install a default route or DNS. + // Dev NIC gets DHCP IP for hostfwd but must not install routes or DNS. b.WriteString(" never-default=true\n") b.WriteString(" ignore-auto-dns=true\n") } if imdsRoute { - // On-link route to the IMDS metadata IP so the guest ARPs 169.254.169.254 - // directly on this NIC; the per-subnet localport answers. No next hop = - // link scope. Matches the netplan path in generateNetworkConfig. + // On-link IMDS route so guest ARPs 169.254.169.254 on this NIC (link scope). fmt.Fprintf(b, " route1=%s/32\n", handlers_imds.MetaDataServerIP) } b.WriteString(" [ipv6]\n") @@ -341,18 +294,9 @@ func appendRHELStaticKeyfile(b *strings.Builder, name, mac, ip string) { b.WriteString(" method=disabled\n") } -// buildAlpineCloudInit produces the cloud-init write_files + runcmd blocks that -// deliver the IMDS on-link route on Alpine guests. Returns empty strings when -// eniMAC is empty (wildcard / non-VPC path). -// -// Alpine renders networking via cloud-init's eni backend, which crashes on a -// gateway-less route (it assumes every route has a next hop), so the route is -// omitted from the netplan network-config (see generateNetworkConfig) and added -// here instead. A persistent /etc/local.d script keeps the on-link route present -// across reboots — runcmd runs once per instance, so the OpenRC `local` service -// re-applies it on every boot. cloud-init's eni renderer renames the primary -// VPC NIC to vpc0 via a 70-persistent-net.rules udev rule, so the device name is -// stable. `ip route replace` is idempotent. +// buildAlpineCloudInit produces the write_files + runcmd blocks that install the +// IMDS on-link route on Alpine guests. Alpine's eni renderer crashes on +// gateway-less routes, so the route is delivered via a persistent local.d script. func buildAlpineCloudInit(eniMAC string) (writeFiles, runCmd string) { if eniMAC == "" { return "", "" @@ -364,10 +308,8 @@ func buildAlpineCloudInit(eniMAC string) (writeFiles, runCmd string) { wf.WriteString(" permissions: '0755'\n") wf.WriteString(" content: |\n") wf.WriteString(" #!/bin/sh\n") - // Resolve the primary VPC NIC via its default route, not by name: the - // persistent-net udev rename to vpc0 doesn't apply on the Alpine AMI, so the - // kernel name is eth0 at first boot. The mgmt NIC (eth1) carries no default - // route, so this always picks the VPC egress NIC — the one IMDS rides. + // Resolve via default route: udev rename to vpc0 hasn't run yet at first boot; + // the mgmt NIC has no default route so this always selects the VPC egress NIC. wf.WriteString(" dev=$(ip route show default | awk '{print $5; exit}')\n") fmt.Fprintf(&wf, " [ -n \"$dev\" ] && ip route replace %s/32 dev \"$dev\" scope link\n", handlers_imds.MetaDataServerIP) @@ -403,13 +345,9 @@ type volumeParams struct { deleteOnTermination bool } -// floorVolumeSizeToAMI ensures requested is at least the AMI's snapshot size. -// Cloud images embed a fixed-size partition table; truncating below it (e.g. -// a 10 GiB Rocky image requested at the 4 GiB default) hangs the guest in -// dracut waiting on a root UUID that lives past the cut-off. AWS rejects -// under-sized requests with InvalidParameterValue — spinifex silently rounds -// up since we don't surface that error code today and the user-visible -// failure (dracut hang) is worse than a slightly bigger volume. +// floorVolumeSizeToAMI ensures the requested size is at least the AMI's snapshot +// size. Under-sizing hangs the guest in dracut; we round up silently rather than +// returning InvalidParameterValue, since a slightly larger volume is less harmful. func floorVolumeSizeToAMI(loader AMIMetaLoader, imageID string, requested int) int { if loader == nil || !strings.HasPrefix(imageID, "ami-") { return requested @@ -592,14 +530,9 @@ func (s *InstanceServiceImpl) RunInstance(input *ec2.RunInstancesInput) (*vm.VM, return instance, ec2Instance, nil } -// PrepareRunInstances validates RunInstances input, allocates per-node capacity, -// creates per-instance VM + ec2.Instance metadata, auto-resolves the subnet, -// creates the primary ENI, and auto-assigns a public IP when the subnet has -// MapPublicIpOnLaunch. It does NOT touch vmMgr, WriteState, or NATS -// subscriptions — those are daemon concerns so the daemon can respond to AWS -// between Prepare and Launch (preserving the original respond-then-launch -// timing). Returns the reservation, the prepared VMs, and the instance-type -// info so the caller can deallocate on failure. +// PrepareRunInstances validates input, allocates capacity, creates VM metadata, +// auto-creates the primary ENI, and auto-assigns a public IP when needed. +// Does NOT touch vmMgr or NATS — callers insert VMs then call LaunchRunInstances. func (s *InstanceServiceImpl) PrepareRunInstances(input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, []*vm.VM, *ec2.InstanceTypeInfo, error) { if accountID == "" { return nil, nil, nil, errors.New(awserrors.ErrorServerInternal) @@ -627,8 +560,7 @@ func (s *InstanceServiceImpl) PrepareRunInstances(input *ec2.RunInstancesInput, slog.Error("PrepareRunInstances: AMI not found", "imageId", *input.ImageId, "err", err) return nil, nil, nil, errors.New(awserrors.ErrorInvalidAMIIDNotFound) } - // Caller must own the AMI or it must carry a non-account-ID owner alias - // (e.g. "self", "spinifex", "") indicating a system/legacy AMI. + // Caller must own the AMI or the owner alias must be non-account-ID (e.g. "self", "spinifex", ""). amiOwner := amiMeta.ImageOwnerAlias if amiOwner != "" && amiOwner != accountID && utils.IsAccountID(amiOwner) { slog.Warn("PrepareRunInstances: AMI not owned by caller", "imageId", *input.ImageId, "amiOwner", amiOwner, "accountID", accountID) @@ -658,9 +590,8 @@ func (s *InstanceServiceImpl) PrepareRunInstances(input *ec2.RunInstancesInput, launchCount := allocatableCount slog.Info("PrepareRunInstances: count determined", "min", minCount, "max", maxCount, "launching", launchCount) - // Allocate is atomic per call (check + commit under a single write lock), - // so this loop can never overcommit. Under contention it may allocate - // fewer than allocatableCount; the < minCount branch below rolls back. + // Each Allocate is atomic; under contention we may get fewer than + // allocatableCount — the < minCount branch below rolls back. var allocatedCount int for i := 0; i < launchCount; i++ { if err := s.resourceMgr.Allocate(instanceType); err != nil { @@ -692,8 +623,7 @@ func (s *InstanceServiceImpl) PrepareRunInstances(input *ec2.RunInstancesInput, } instance.BootMode = amiMeta.BootMode - // Terraform with associate_public_ip_address sends subnet/SG via - // NetworkInterfaces[0]; lift to top-level so the rest works uniformly. + // Terraform may pass subnet/SG via NetworkInterfaces[0]; lift to top-level. if (input.SubnetId == nil || *input.SubnetId == "") && len(input.NetworkInterfaces) > 0 && input.NetworkInterfaces[0] != nil { nic := input.NetworkInterfaces[0] @@ -705,11 +635,8 @@ func (s *InstanceServiceImpl) PrepareRunInstances(input *ec2.RunInstancesInput, } } - // Pre-created ENI path: when the caller passes - // NetworkInterfaces[0].NetworkInterfaceId, attach that existing ENI - // instead of auto-creating one. AWS-parity behaviour; consumed by the - // EKS cluster launcher which needs the ENI to live under the customer - // account before the K3s server VM is created. + // Pre-created ENI: attach the existing ENI instead of auto-creating one + // (AWS parity; used by EKS launcher to pre-create the ENI in customer VPC). preExistingENIID := "" if len(input.NetworkInterfaces) > 0 && input.NetworkInterfaces[0] != nil { preExistingENIID = aws.StringValue(input.NetworkInterfaces[0].NetworkInterfaceId) @@ -760,9 +687,8 @@ func (s *InstanceServiceImpl) PrepareRunInstances(input *ec2.RunInstancesInput, instance.ENIMac = *eni.MacAddress if _, attachErr := s.eniCreator.AttachENI(accountID, instance.ENIId, instance.ID, 0); attachErr != nil { - // Without the attachment, RegisterTargets (TargetType=instance) - // resolves ENIs via attachment.instance-id, finds none, and - // silently drops the target; terminate leaks the auto-EIP. + // Without the attachment RegisterTargets silently drops the target; + // aborting prevents a leak of the auto-assigned EIP. slog.Error("PrepareRunInstances: AttachENI failed — aborting launch", "eniId", instance.ENIId, "instanceId", instance.ID, "err", attachErr) if _, delErr := s.eniDeleter.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{ @@ -813,12 +739,8 @@ func (s *InstanceServiceImpl) PrepareRunInstances(input *ec2.RunInstancesInput, az := s.config.AZ publicIP, poolName, allocErr := s.ipAllocator.AllocateIP(region, az, handlers_ec2_vpc.PurposeENIPublic, "", *eni.NetworkInterfaceId, instance.ID) if allocErr != nil { - // No qemu-hostfwd fallback exists: an instance on a - // MapPublicIpOnLaunch subnet with no public IP is - // unreachable, so fail the launch instead of booting a - // running-but-unreachable instance. The ENI is attached - // with a primary IP — detach before delete (in-use ENIs - // reject deletion). + // Fail rather than boot an unreachable instance; + // detach before delete since in-use ENIs reject deletion. slog.Error("PrepareRunInstances: public IP allocation failed — aborting launch", "instanceId", instance.ID, "eniId", *eni.NetworkInterfaceId, "err", allocErr) if detErr := s.eniCreator.DetachENI(accountID, *eni.NetworkInterfaceId); detErr != nil { @@ -842,7 +764,7 @@ func (s *InstanceServiceImpl) PrepareRunInstances(input *ec2.RunInstancesInput, if natErr := utils.AddNAT(s.natsConn, *eni.VpcId, publicIP, *eni.PrivateIpAddress, portName, *eni.MacAddress); natErr != nil { slog.Error("PrepareRunInstances: vpc.add-nat failed — rolling back public IP to avoid surfacing an unreachable address", "instanceId", instance.ID, "publicIp", publicIP, "pool", poolName, "err", natErr) - // Timeout may have committed the rule after our window; neutralise before releasing the IP. + // Neutralise before releasing in case timeout committed the rule. utils.PublishNATEvent(s.natsConn, "vpc.delete-nat", *eni.VpcId, publicIP, *eni.PrivateIpAddress, portName, *eni.MacAddress) s.rollbackAutoAssignedPublicIP(accountID, instance.ID, *eni.NetworkInterfaceId, publicIP, poolName) lastRunErr = natErr @@ -897,12 +819,9 @@ func (s *InstanceServiceImpl) PrepareRunInstances(input *ec2.RunInstancesInput, return reservation, instances, instanceType, nil } -// attachPreCreatedENI looks up an existing ENI in accountID, verifies it is -// not already in-use, attaches it as device-0 to instance, and populates the -// VM + ec2.Instance with the ENI's network identity (private IP, MAC, subnet, -// VPC, security groups). Returns an error without mutating any state if the -// lookup or attach fails. Public-IP auto-assignment is skipped on the -// pre-created path because the caller is expected to manage that out-of-band. +// attachPreCreatedENI verifies the ENI is available, attaches it as device-0, +// and populates the VM + ec2.Instance. Public-IP auto-assignment is skipped; +// the caller manages that out-of-band. func (s *InstanceServiceImpl) attachPreCreatedENI(accountID, eniID string, instance *vm.VM, ec2Instance *ec2.Instance) error { eniInfo, err := s.eniCreator.GetENI(accountID, eniID) if err != nil { @@ -953,15 +872,12 @@ func (s *InstanceServiceImpl) attachPreCreatedENI(accountID, eniID string, insta return nil } -// LaunchRunInstances takes the VMs created by PrepareRunInstances (already -// inserted into vmMgr by the caller) and runs the heavyweight launch loop: -// volume preparation, GPU claim, vmMgr.Run. Failures mark the VM as failed -// and deallocate resources but do not abort the loop — partial success -// matches AWS behaviour where some instances in a reservation may fail. +// LaunchRunInstances runs the heavyweight launch loop for VMs already inserted +// into vmMgr: volume prep, GPU claim, vmMgr.Run. Partial failures are tolerated. func (s *InstanceServiceImpl) LaunchRunInstances(instances []*vm.VM, input *ec2.RunInstancesInput, instanceType *ec2.InstanceTypeInfo) { var successCount int for _, instance := range instances { - // Skip if a concurrent request terminated the instance during prepare. + // Skip if a concurrent terminate raced with prepare. status := s.vmMgr.Status(instance) if status != vm.StatePending && status != vm.StateProvisioning { slog.Info("LaunchRunInstances: instance state changed during provisioning, skipping launch", @@ -969,8 +885,7 @@ func (s *InstanceServiceImpl) LaunchRunInstances(instances []*vm.VM, input *ec2. continue } - // Pre-compute dev MAC so cloud-init can generate per-interface netplan - // that suppresses the default route on the dev/hostfwd NIC. + // Pre-compute dev MAC for per-interface cloud-init netplan (route suppression). if s.config.Daemon.DevNetworking && instance.ENIId != "" { instance.DevMAC = vm.GenerateDevMAC(instance.ID) } @@ -1028,10 +943,8 @@ func (s *InstanceServiceImpl) LaunchRunInstances(instances []*vm.VM, input *ec2. slog.Info("LaunchRunInstances: completed", "requested", len(instances), "launched", successCount) } -// RunInstances satisfies the InstanceService interface for non-daemon callers -// (tests, mocks). The daemon's NATS handler bypasses this method and calls -// PrepareRunInstances + LaunchRunInstances directly so it can respond to AWS -// between the two phases, preserving the original respond-then-launch timing. +// RunInstances is for non-daemon callers (tests). The daemon calls +// PrepareRunInstances + LaunchRunInstances directly to respond before launching. func (s *InstanceServiceImpl) RunInstances(input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error) { reservation, instances, instanceType, err := s.PrepareRunInstances(input, accountID) if err != nil { @@ -1044,9 +957,7 @@ func (s *InstanceServiceImpl) RunInstances(input *ec2.RunInstancesInput, account return reservation, nil } -// RebootInstance handles an ec2.cmd reboot for an instance already running on -// this node. Returns nil on success; the dispatching daemon handler maps the -// returned error to a NATS error response and responds with `{}` on nil. +// RebootInstance handles an ec2.cmd reboot for a running instance on this node. func (s *InstanceServiceImpl) RebootInstance(instance *vm.VM, command spxtypes.EC2InstanceCommand) error { slog.Info("RebootInstance: rebooting instance", "id", command.ID) @@ -1068,16 +979,12 @@ func (s *InstanceServiceImpl) RebootInstance(instance *vm.VM, command spxtypes.E return nil } -// StartInstance handles an ec2.cmd start for an instance that is stopped on -// this node (local vmMgr state, not the cross-node shared-KV path). Returns -// nil on success; the dispatching daemon handler responds with the running -// status payload. +// StartInstance handles an ec2.cmd start for a locally stopped instance. func (s *InstanceServiceImpl) StartInstance(instance *vm.VM, command spxtypes.EC2InstanceCommand) error { slog.Info("StartInstance: starting instance", "id", command.ID) status := s.vmMgr.Status(instance) - // StateError is startable: a crash/recovery-failed instance can be manually - // recovered. The manager transitions Error->Pending before relaunch. + // StateError is startable so a crashed instance can be manually recovered. if status != vm.StateStopped && status != vm.StateError { slog.Error("StartInstance: instance not in a startable state", "instanceId", command.ID, "status", status) return errors.New(awserrors.ErrorIncorrectInstanceState) @@ -1091,9 +998,8 @@ func (s *InstanceServiceImpl) StartInstance(instance *vm.VM, command spxtypes.EC } } - // Clear stop attribute before launch so WriteState inside the manager - // persists the correct attributes. Without this, a daemon restart after - // a stop→start cycle would see StopInstance=true and skip reconnecting QEMU. + // Clear stop attribute before launch; a stale StopInstance=true would + // cause the daemon to skip QEMU reconnect on restart. s.vmMgr.UpdateState(instance.ID, func(v *vm.VM) { v.Attributes = command.Attributes }) if err := s.vmMgr.Start(instance.ID); err != nil { @@ -1117,10 +1023,8 @@ func (s *InstanceServiceImpl) StartInstance(instance *vm.VM, command spxtypes.EC return nil } -// StopOrTerminateInstance handles an ec2.cmd stop or terminate for an -// instance running on this node. Validates the transition synchronously, -// stamps the command attributes onto the VM, then dispatches the actual -// Stop/Terminate in a background goroutine so the daemon can ack immediately. +// StopOrTerminateInstance handles an ec2.cmd stop or terminate. Validates the +// transition synchronously, then dispatches Stop/Terminate in a goroutine. func (s *InstanceServiceImpl) StopOrTerminateInstance(instance *vm.VM, command spxtypes.EC2InstanceCommand) error { isTerminate := command.Attributes.TerminateInstance action := "Stopping" @@ -1132,11 +1036,8 @@ func (s *InstanceServiceImpl) StopOrTerminateInstance(instance *vm.VM, command s slog.Info("StopOrTerminateInstance: "+action, "id", command.ID) - // Fold the termination-protection check, idempotency check, transition - // validation, and attribute stamp into one lock acquisition so a racing - // ModifyInstanceAttribute can't clear protection between the gates. The - // async lifecycle goroutine below persists on its own state transitions, - // so no synchronous persist is needed here. + // Single lock acquisition covers protection check, idempotency, transition + // validation, and attribute stamp to prevent races with ModifyInstanceAttribute. var ( protected, raced, stateMismatch bool currentState vm.InstanceState @@ -1156,10 +1057,9 @@ func (s *InstanceServiceImpl) StopOrTerminateInstance(instance *vm.VM, command s return } v.Attributes = command.Attributes - // Auto-disassociate IAM instance profile on terminate (matches AWS). - // Stop/Start preserves the binding; only terminate clears it. Done - // under the same lock as the state transition so DescribeInstances - // never observes a terminated instance still advertising a profile. + // Auto-disassociate IAM profile on terminate (AWS parity; stop/start preserves it). + // Done under the same lock so DescribeInstances never sees a terminated + // instance advertising a profile. if isTerminate && v.IamInstanceProfileArn != "" { slog.Info("IAM instance profile auto-disassociated", "instance_id", v.ID, @@ -1181,13 +1081,12 @@ func (s *InstanceServiceImpl) StopOrTerminateInstance(instance *vm.VM, command s return errors.New(awserrors.ErrorOperationNotPermitted) } if raced { - // Idempotent: a concurrent terminate goroutine is already cleaning up. + // Idempotent: a concurrent terminate is already in progress. slog.Info("StopOrTerminateInstance: instance already shutting down, terminate is idempotent", "instanceId", instance.ID) return nil } if stateMismatch { - // Surface IncorrectInstanceState synchronously so the AWS SDK sees - // 400 instead of a stale 200. The async path re-validates. + // Surface IncorrectInstanceState synchronously (AWS SDK expects 400). slog.Warn("StopOrTerminateInstance: instance in incorrect state for "+strings.ToLower(action), "instanceId", instance.ID, "currentState", string(currentState)) return errors.New(awserrors.ErrorIncorrectInstanceState) @@ -1213,14 +1112,9 @@ func (s *InstanceServiceImpl) StopOrTerminateInstance(instance *vm.VM, command s return nil } -// AssociateIamInstanceProfile attaches an instance profile to a running -// instance. The gateway has already resolved the profile reference to a -// canonical ARN and enforced iam:PassRole; this method validates that the -// instance has no existing profile (returns IamInstanceProfileAlreadyAssociated -// otherwise) and atomically writes the ARN + a freshly generated -// iip-assoc-… ID to vm.VM. AssociationId is never reused — every Associate -// produces a new one. InstanceProfile.Id is left nil; the gateway enriches it -// from its IAMService cache before returning to the caller. +// AssociateIamInstanceProfile attaches an instance profile to a running instance. +// Validates no existing profile, then atomically writes the ARN + new association ID. +// InstanceProfile.Id is left nil; the gateway enriches it from IAMService. func (s *InstanceServiceImpl) AssociateIamInstanceProfile(instance *vm.VM, command spxtypes.EC2InstanceCommand) (*ec2.IamInstanceProfileAssociation, error) { if command.IamProfileAssociationData == nil || command.IamProfileAssociationData.InstanceProfileArn == "" { return nil, errors.New(awserrors.ErrorMissingParameter) @@ -1261,13 +1155,8 @@ func (s *InstanceServiceImpl) AssociateIamInstanceProfile(instance *vm.VM, comma }, nil } -// DisassociateIamProfileAssociation finds the VM carrying the requested -// AssociationId in this daemon's vmMgr and clears both profile fields -// atomically. Returns nil when no local VM owns the ID, or when the caller's -// account doesn't own the matching VM, so the gateway fan-out collector can -// advance past this daemon's NoOp response without waiting on the deadline. -// The accountID scope mirrors checkInstanceOwnership semantics applied -// per-instance on ec2.cmd.. +// DisassociateIamProfileAssociation clears the profile association on the local VM. +// Returns nil when no local VM owns the ID so the gateway fan-out can skip this node. func (s *InstanceServiceImpl) DisassociateIamProfileAssociation(input *ec2.DisassociateIamInstanceProfileInput, accountID string) (*ec2.IamInstanceProfileAssociation, error) { if input == nil || input.AssociationId == nil || *input.AssociationId == "" { return nil, errors.New(awserrors.ErrorMissingParameter) @@ -1281,9 +1170,7 @@ func (s *InstanceServiceImpl) DisassociateIamProfileAssociation(input *ec2.Disas var clearedArn string timestamp := time.Now().UTC() _, err := s.vmMgr.UpdateAndPersist(owner, func(v *vm.VM) bool { - // Re-check under the manager lock: another fan-out (or a concurrent - // terminate) could have cleared or replaced the association between - // our snapshot read and this mutation. + // Re-check under lock: a concurrent terminate may have cleared the association. if v.IamInstanceProfileAssociationId != associationID { return false } @@ -1297,8 +1184,7 @@ func (s *InstanceServiceImpl) DisassociateIamProfileAssociation(input *ec2.Disas return nil, errors.New(awserrors.ErrorServerInternal) } if clearedArn == "" { - // Lost a race with terminate / another disassociate. NoOp so the - // gateway collector treats it as "not this daemon's instance". + // Lost a race; NoOp so the gateway collector skips this node. return nil, nil } @@ -1313,12 +1199,9 @@ func (s *InstanceServiceImpl) DisassociateIamProfileAssociation(input *ec2.Disas }, nil } -// ReplaceIamProfileAssociation finds the VM carrying the requested -// AssociationId and swaps it for a new (ARN, AssociationId) pair atomically. -// The new AssociationId is always freshly generated — AWS never reuses IDs -// across replace. Returns nil when no local VM owns the old ID or when the -// caller's account doesn't own the matching VM. The gateway has already -// resolved IamInstanceProfile.Arn to a canonical ARN before publishing. +// ReplaceIamProfileAssociation atomically swaps the profile ARN and AssociationId. +// Returns nil when no local VM owns the old ID. The gateway has already resolved +// the ARN to canonical form. func (s *InstanceServiceImpl) ReplaceIamProfileAssociation(input *ec2.ReplaceIamInstanceProfileAssociationInput, accountID string) (*ec2.IamInstanceProfileAssociation, error) { if input == nil || input.AssociationId == nil || *input.AssociationId == "" || input.IamInstanceProfile == nil || aws.StringValue(input.IamInstanceProfile.Arn) == "" { @@ -1363,12 +1246,9 @@ func (s *InstanceServiceImpl) ReplaceIamProfileAssociation(input *ec2.ReplaceIam }, nil } -// DescribeIamProfileAssociations walks this daemon's vmMgr and returns every -// live association visible to the caller's account matching the supplied -// filters (empty filters match all). Empty result is a valid response: the -// gateway aggregates per-daemon slices, so missing daemons just yield fewer -// rows. Filter names other than "instance-id" / "state" surface as -// InvalidParameterValue, short-circuiting the fan-out at the gateway. +// DescribeIamProfileAssociations returns live associations on this node visible to +// the caller. Supports "instance-id" and "state" filters; unknown filters return +// InvalidParameterValue. func (s *InstanceServiceImpl) DescribeIamProfileAssociations(input *ec2.DescribeIamInstanceProfileAssociationsInput, accountID string) (*ec2.DescribeIamInstanceProfileAssociationsOutput, error) { assocFilter := stringPtrSliceToSet(input.AssociationIds) instFilter := make(map[string]bool) @@ -1409,8 +1289,7 @@ func (s *InstanceServiceImpl) DescribeIamProfileAssociations(input *ec2.Describe if len(instFilter) > 0 && !instFilter[v.ID] { return } - // v1 only emits "associated" — there are no async transitions, so any - // state filter that excludes "associated" yields an empty result. + // Only "associated" state exists; other state filters yield empty results. const liveState = ec2.IamInstanceProfileAssociationStateAssociated if len(stateFilter) > 0 && !stateFilter[liveState] { return @@ -1425,12 +1304,8 @@ func (s *InstanceServiceImpl) DescribeIamProfileAssociations(input *ec2.Describe return out, nil } -// findInstanceByAssociationID is the lock-protected lookup used by the -// fan-out mutators. It snapshots the matching instance ID under the manager -// lock and returns (id, true) only when the caller's account owns the VM, -// so cross-tenant Disassociate / Replace cannot reach into another account's -// instance. The actual mutation re-validates inside UpdateAndPersist to close -// the read/mutate race window. +// findInstanceByAssociationID returns the VM ID for the given AssociationId +// if the caller's account owns it. Mutations re-validate under UpdateAndPersist. func (s *InstanceServiceImpl) findInstanceByAssociationID(associationID, accountID string) (string, bool) { var owner string s.vmMgr.ForEach(func(v *vm.VM) { @@ -1558,10 +1433,8 @@ func (s *InstanceServiceImpl) newViperblock(volumeName string, size int, volumeC return vb, err } -// restoreSlogDefault re-installs the daemon's Info-level slog handler after -// viperblock.New mutates the global slog default via its SetDebug method -// (see viperblock.go SetDebug — it calls slog.SetDefault with LevelError, -// silencing every Info/Warn in the entire process). +// restoreSlogDefault re-installs the Info-level slog handler after viperblock.New +// resets it to LevelError via SetDebug, silencing all Info/Warn globally. func restoreSlogDefault() { slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ Level: slog.LevelInfo, @@ -1583,12 +1456,8 @@ func (s *InstanceServiceImpl) prepareRootVolume(input *ec2.RunInstancesInput, im return errors.New(awserrors.ErrorServerInternal) } - // Load the state from the remote backend. Only os.ErrNotExist means - // "volume doesn't exist, clone from AMI" — every other error (HMAC - // integrity failure, key mismatch, transient backend 5xx, JSON parse - // failure) must abort. Cloning from AMI on a tamper/mismatch would - // overwrite the legitimate volume's config.json with AMI base-map - // state and lose access to the live data. + // Only os.ErrNotExist means "clone from AMI"; any other error must abort. + // Treating a tamper/mismatch as missing would overwrite live volume state. _, err = vb.LoadStateRequest("") if err != nil && !errors.Is(err, os.ErrNotExist) { slog.Error("Failed to load root volume state from backend", @@ -1673,11 +1542,9 @@ func (s *InstanceServiceImpl) cloneAMIToVolume(input *ec2.RunInstancesInput, siz return nil } -// prepareEFIVolume creates the per-VM EFI variable store. The volume is -// sized to match the firmware's VARS template (QEMU pflash refuses any other -// size) and seeded with the template bytes so EFI variables — BootOrder, -// BootNext, secure-boot state — survive across reboots. arch is the AMI -// architecture string ("x86_64" | "arm64"). +// prepareEFIVolume creates the per-VM EFI variable store, sized exactly to the +// firmware VARS template (pflash requires byte-exact size) and seeded from it. +// arch is "x86_64" | "arm64". func (s *InstanceServiceImpl) prepareEFIVolume(imageId string, volumeConfig viperblock.VolumeConfig, instance *vm.VM, arch string) error { codePath, varsTemplate, varsSize, err := vm.FirmwarePaths(arch) if err != nil { @@ -1698,10 +1565,8 @@ func (s *InstanceServiceImpl) prepareEFIVolume(imageId string, volumeConfig vipe efiVolumeName := fmt.Sprintf("%s-efi", imageId) efiVolumeConfig := volumeConfig efiVolumeConfig.VolumeMetadata.VolumeID = efiVolumeName - // Zero SizeGiB so viperblock's LoadState doesn't reconcile the EFI - // volume's persisted byte-exact size up to the parent root volume's - // GiB-rounded size — QEMU pflash rejects any VARS volume larger than - // the firmware's expected variable region. + // Zero SizeGiB prevents viperblock from rounding the EFI volume size + // up to GiB boundaries; pflash rejects any size beyond the VARS region. efiVolumeConfig.VolumeMetadata.SizeGiB = 0 efiVb, err := s.newViperblock(efiVolumeName, int(varsSize), efiVolumeConfig) @@ -1716,11 +1581,8 @@ func (s *InstanceServiceImpl) prepareEFIVolume(imageId string, volumeConfig vipe return errors.New(awserrors.ErrorServerInternal) } - // Distinguish "not found" (seed from template) from any other backend - // failure (transient S3 5xx, JSON unmarshal, etc.). Treating a transient - // failure as "not found" would silently re-seed a volume on every retry, - // clobbering guest-set BootOrder. Backends signal missing-object by - // wrapping os.ErrNotExist (see viperblock.classifyStateLoad). + // Only os.ErrNotExist means "seed from template"; other errors must abort. + // Treating a transient failure as missing would clobber guest-set BootOrder. _, loadErr := efiVb.LoadStateRequest("") if loadErr != nil && !errors.Is(loadErr, os.ErrNotExist) { slog.Error("Failed to load EFI volume state from backend", "name", efiVolumeName, "err", loadErr) @@ -1755,9 +1617,7 @@ func (s *InstanceServiceImpl) prepareEFIVolume(imageId string, volumeConfig vipe } } - // Close is the durability boundary after WriteAt+Flush; if it fails the - // VARS volume may be partially written and QEMU pflash will refuse to - // start. Fail the launch loudly rather than enqueueing a corrupt volume. + // Close is the durability boundary; a partial VARS write causes pflash to refuse launch. if err := efiVb.Close(); err != nil { slog.Error("Failed to close EFI Viperblock store", "err", err) return errors.New(awserrors.ErrorServerInternal) @@ -1777,10 +1637,8 @@ func (s *InstanceServiceImpl) prepareEFIVolume(imageId string, volumeConfig vipe return nil } -// instanceArchitecture pulls the AWS architecture string off an -// ec2.InstanceTypeInfo. Returns "" when the spec is nil or malformed; the -// caller's firmware probe surfaces that as a clear error rather than a -// silent BIOS fallback. +// instanceArchitecture returns the AWS architecture string from the instance type. +// Returns "" on nil/malformed spec; the caller's firmware probe surfaces the error. func instanceArchitecture(it *ec2.InstanceTypeInfo) string { if it == nil || it.ProcessorInfo == nil || len(it.ProcessorInfo.SupportedArchitectures) == 0 || it.ProcessorInfo.SupportedArchitectures[0] == nil { return "" @@ -1788,9 +1646,8 @@ func instanceArchitecture(it *ec2.InstanceTypeInfo) string { return *it.ProcessorInfo.SupportedArchitectures[0] } -// prepareCloudInitVolume creates cloud-init ISO with SSH keys and user data. -// rootVolumeId is the per-instance root volume ID (not the AMI ID), ensuring -// each instance gets its own cloud-init volume with fresh SSH keys and metadata. +// prepareCloudInitVolume creates the cloud-init ISO with SSH keys and user data. +// Uses rootVolumeId (not AMI ID) so each instance gets its own volume. func (s *InstanceServiceImpl) prepareCloudInitVolume(input *ec2.RunInstancesInput, rootVolumeId string, volumeConfig viperblock.VolumeConfig, instance *vm.VM) error { slog.Info("Creating cloud-init volume") @@ -1875,9 +1732,7 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i // Generate instance metadata hostname := generateHostname(instance.ID) - // Retrieve SSH pubkey from S3 — required for instance access. - // Password authentication is not supported; instances without a key - // pair have no remote access method. + // Retrieve SSH pubkey from S3 — password auth is not supported. keyName := "" if input.KeyName != nil { keyName = *input.KeyName @@ -1906,9 +1761,7 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i } } - // Read CA certificate for injection into guest cloud-init. - // Derive the config directory: BaseDir (e.g. ~/spinifex/spinifex/) sits one - // level below the data root; the CA cert is at /config/ca.pem. + // Read CA cert from /config/ca.pem and inject into guest cloud-init. var caCertPEM string dataRoot := filepath.Dir(strings.TrimSuffix(s.config.BaseDir, "/")) caCertPath := filepath.Join(dataRoot, "config", "ca.pem") @@ -1929,9 +1782,7 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i slog.Error("failed to read CA cert for guest cloud-init injection", "path", caCertPath, "error", err) } - // Re-read AMI metadata for DistroFamily rather than persisting it on - // vm.VM — cloud-init renders once at first launch and the ISO is sealed, - // so no consumer needs DistroFamily after this point. + // Re-read AMI metadata for DistroFamily; not persisted on VM since the ISO is sealed once. var distroFamily string if s.amiLoader != nil && input.ImageId != nil && *input.ImageId != "" { amiMeta, err := s.amiLoader.GetAMIConfig(*input.ImageId) @@ -1988,11 +1839,8 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i } } - // Alpine's eni renderer can't emit the gateway-less IMDS route, so deliver it - // out-of-band via a local.d script. Only when the caller didn't supply its own - // #cloud-config: that payload owns the write_files/runcmd keys (e.g. the EKS - // k3s server VM, which bakes the same route into its own block), and a second - // write_files key here would collide — last key wins, silently dropping a block. + // Inject Alpine IMDS route only when no #cloud-config user-data is present; + // a caller-supplied payload already owns write_files/runcmd and a second key collides. if distroFamily == "alpine" && userData.UserDataCloudConfig == "" { userData.AlpineWriteFiles, userData.AlpineRunCmd = buildAlpineCloudInit(instance.ENIMac) } @@ -2032,11 +1880,8 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i return errors.New(awserrors.ErrorServerInternal) } - // Add network-config: per-interface when VPC+dev (suppresses dev default route), - // wildcard DHCP otherwise. Extra ENI MACs produce additional DHCP NICs for - // multi-subnet system VMs (multi-AZ ALBs). RHEL-family guests get a - // "config: disabled" stub here so cloud-init doesn't drop a second NM - // keyfile alongside the ones we wrote via user-data write_files. + // Add network-config: per-interface for VPC+dev, wildcard DHCP otherwise. + // RHEL gets "config: disabled" to prevent a competing NM keyfile. networkConfig := selectNetworkConfigForFamily(distroFamily, instance.ENIMac, instance.DevMAC, instance.MgmtMAC, instance.MgmtIP, extraMACs) err = writer.AddFile(strings.NewReader(networkConfig), "network-config") if err != nil { @@ -2129,9 +1974,8 @@ var DescribeInstancesValidFilters = map[string]bool{ } // DescribeInstanceStatusValidFilters lists the filter names accepted by -// DescribeInstanceStatus. event.*, instance-status.* and system-status.* -// are rejected: Mulga's static-payload health model has no events and a -// single value per status field, so those filters can't usefully narrow. +// DescribeInstanceStatus. event.* and status.* filters are excluded as +// Mulga's health model has a single static value per status field. var DescribeInstanceStatusValidFilters = map[string]bool{ "availability-zone": true, "instance-state-code": true, @@ -2235,21 +2079,15 @@ func (s *InstanceServiceImpl) DescribeInstances(input *ec2.DescribeInstancesInpu reservationMap := make(map[string]*ec2.Reservation) - // Iterate under the manager lock — VM fields (Status, Instance, Reservation, - // PublicIP, PlacementGroupName) are mutated through manager-locked - // Inspect/UpdateState elsewhere, so reading them lock-free would race. + // Iterate under the manager lock to avoid races on Status, Instance, PublicIP, etc. s.vmMgr.View(func(vms map[string]*vm.VM) { for _, instance := range vms { if !IsInstanceVisible(accountID, instance.AccountID) { continue } - // Platform-managed system VMs (LB HAProxy, EKS control plane) carry a - // ManagedBy tag and must stay out of customer EC2 listings, the same - // way they're filtered from the UI Nodes page. LB VMs are owned by the - // system account so IsInstanceVisible already hides them, but the EKS - // control-plane VM is owned by the customer account (its ENI lives in - // the customer VPC), so guard on ManagedBy here. Root/operator callers - // still see system instances. + // Platform-managed VMs (LB, EKS control plane) are hidden from customer + // listings. LBs are system-account-owned; EKS control-plane VMs are + // customer-account-owned so ManagedBy guards them explicitly. if instance.ManagedBy != "" && accountID != utils.GlobalAccountID { continue } @@ -2316,12 +2154,9 @@ func (s *InstanceServiceImpl) DescribeInstances(input *ec2.DescribeInstancesInpu return &ec2.DescribeInstancesOutput{Reservations: reservations}, nil } -// DescribeInstanceTypes returns instance types this node supports. Without -// the "capacity" filter the response lists every supported type once, -// matching AWS semantics — callers (e.g. the Terraform AWS provider) expect -// a stable answer that does not depend on current allocation. With -// `capacity=true` each type expands to one entry per free schedulable slot -// so callers can aggregate cluster-wide remaining capacity. +// DescribeInstanceTypes returns supported instance types. With filter +// capacity=true, expands each type to one entry per free slot for cluster-wide +// capacity aggregation. func (s *InstanceServiceImpl) DescribeInstanceTypes(input *ec2.DescribeInstanceTypesInput, _ string) (*ec2.DescribeInstanceTypesOutput, error) { slog.Info("Processing DescribeInstanceTypes request from this node") @@ -2443,11 +2278,8 @@ func (s *InstanceServiceImpl) describeInstancesFromKV(input *ec2.DescribeInstanc return &ec2.DescribeInstancesOutput{Reservations: reservations}, nil } -// ModifyInstanceAttribute applies a single attribute change. InstanceType and -// UserData require the instance to be stopped. DisableApiTermination works on -// both running and stopped instances. SourceDestCheck is a networking concept -// that does not apply to bare-metal VMs; it is accepted as a no-op on any -// instance state so Terraform and the AWS CLI do not error out. +// ModifyInstanceAttribute applies a single attribute change. SourceDestCheck +// is a no-op on bare-metal VMs. InstanceType/UserData require a stopped instance. func (s *InstanceServiceImpl) ModifyInstanceAttribute(input *ec2.ModifyInstanceAttributeInput, accountID string) (*ec2.ModifyInstanceAttributeOutput, error) { if input.InstanceId == nil || *input.InstanceId == "" { return nil, errors.New(awserrors.ErrorMissingParameter) @@ -2567,10 +2399,8 @@ func (s *InstanceServiceImpl) ModifyInstanceAttribute(input *ec2.ModifyInstanceA return &ec2.ModifyInstanceAttributeOutput{}, nil } -// StoppedInstanceNode returns the LastNode field stored in the shared KV for a -// stopped instance. Returns "" when the store is unavailable, the instance is -// not found, or the entry has no LastNode set. Used by the daemon's ec2.start -// handler to route start requests back to the original node when possible. +// StoppedInstanceNode returns the node that last hosted the stopped instance, +// used to route start requests back to the original node when possible. func (s *InstanceServiceImpl) StoppedInstanceNode(instanceID string) string { if s.stoppedStore == nil { return "" @@ -2686,10 +2516,8 @@ func (s *InstanceServiceImpl) StartStoppedInstance(input *StartStoppedInstanceIn return &StartStoppedInstanceOutput{Status: "running", InstanceID: instance.ID}, nil } -// TerminateStoppedInstance picks up a stopped instance from shared KV, deletes -// its volumes, releases its public IP and ENI, writes it to the terminated -// bucket, then removes it from the stopped bucket. No QEMU shutdown or unmount -// is needed — the instance was already drained during stop. +// TerminateStoppedInstance terminates a stopped instance: deletes its volumes, +// releases its public IP and ENI, and moves it to the terminated KV bucket. func (s *InstanceServiceImpl) TerminateStoppedInstance(input *TerminateStoppedInstanceInput, accountID string) (*TerminateStoppedInstanceOutput, error) { if input.InstanceID == "" { return nil, errors.New(awserrors.ErrorMissingParameter) @@ -2791,14 +2619,8 @@ func (s *InstanceServiceImpl) deleteInstanceVolumes(instance *vm.VM, instanceID _ = instanceID } -// rollbackAutoAssignedPublicIP unwinds an auto-assign that proceeded past -// ENI create + attach + IPAM allocation + ENI update but failed at the -// vpc.add-nat barrier. Order matters: clear the ENI record first so any -// concurrent DescribeNetworkInterfaces stops surfacing the unreachable IP, -// then release the IPAM lease so the pool slot can be reused, then detach -// and delete the ENI itself — the failing instance never enters vmMgr, so -// terminateCleanup will not run and the ENI would otherwise leak. Detach -// must precede DeleteNetworkInterface; the latter rejects in-use ENIs. +// rollbackAutoAssignedPublicIP unwinds a failed auto-assign: clears the ENI +// public IP record, releases the IPAM lease, then detaches and deletes the ENI. func (s *InstanceServiceImpl) rollbackAutoAssignedPublicIP(accountID, instanceID, eniID, publicIP, poolName string) { if s.eniCreator != nil { if err := s.eniCreator.UpdateENIPublicIP(accountID, eniID, "", ""); err != nil { diff --git a/spinifex/handlers/ec2/instance/service_impl_test.go b/spinifex/handlers/ec2/instance/service_impl_test.go index 8bc15687b..f09eb4bfb 100644 --- a/spinifex/handlers/ec2/instance/service_impl_test.go +++ b/spinifex/handlers/ec2/instance/service_impl_test.go @@ -547,10 +547,8 @@ func TestCloudInitMetaTemplateRendering(t *testing.T) { assert.Contains(t, rendered, "local-hostname:") } -// TestCloudInitVolumeNamePerInstance verifies that AMI-based launches produce -// unique root volume IDs, which in turn produce unique cloud-init volume names. -// This prevents the bug where a cached cloud-init ISO (keyed by AMI) would -// serve stale SSH keys or hostnames to subsequent instances. +// TestCloudInitVolumeNamePerInstance verifies that each launch produces a unique +// root volume ID so cloud-init ISOs are not shared across instances. func TestRunInstance_NoImageId(t *testing.T) { instanceTypes := map[string]*ec2.InstanceTypeInfo{ "t3.micro": {InstanceType: aws.String("t3.micro")}, @@ -857,7 +855,7 @@ func TestCloudInitVolumeNamePerInstance(t *testing.T) { } } -// --- 1b-pre Describe* coverage (siv-22 parts) ------------------------------ +// --- Describe* coverage ----------------------------------------------------- type fakeResourceCapacityProvider struct { types []*ec2.InstanceTypeInfo @@ -2238,7 +2236,7 @@ func TestStartStoppedInstance_GPUClaimFailureRollsBack(t *testing.T) { assert.Empty(t, store.deletedStopped, "stopped-KV entry must remain on rollback") } -// --- PrepareRunInstances / ec2.cmd dispatch tests (1b-pre phase 2d) --------- +// --- PrepareRunInstances / ec2.cmd dispatch tests --------------------------- type fakeAMILoader struct { byID map[string]viperblock.AMIMetadata @@ -2727,12 +2725,9 @@ func TestPrepareRunInstances_PublicIPAutoAssigned(t *testing.T) { } } -// TestPrepareRunInstances_NATFailureRollsBackPublicIP regresses the silent -// corruption where vpc.add-nat NACKs (or vpcd is offline) but the launch -// returned an ec2.Instance carrying the unreachable IP, the ENI record was -// updated with that IP, and the IPAM pool entry stayed allocated. With the -// fix in place, NAT failure must drop the instance from the response, clear -// the ENI public IP, release the IPAM lease, and deallocate capacity. +// TestPrepareRunInstances_NATFailureRollsBackPublicIP verifies that a vpc.add-nat +// failure drops the instance, clears the ENI public IP, releases the IPAM +// lease, and deallocates capacity. func TestPrepareRunInstances_NATFailureRollsBackPublicIP(t *testing.T) { eni := &fakeENICreator{ subnet: &SubnetInfo{SubnetID: "subnet-1", VpcID: "vpc-1", MapPublicIpOnLaunch: true}, @@ -2824,12 +2819,9 @@ func TestPrepareRunInstances_NATFailureRollsBackPublicIP(t *testing.T) { } } -// TestPrepareRunInstances_PublicIPAllocFailureAbortsLaunch regresses the -// silent failure where a failed public-IP allocation (pool exhausted) was -// logged at WARN and ignored, so the instance booted running with no public -// IP. There is no qemu-hostfwd fallback, so such an instance is unreachable. -// The fix must fail the launch: detach + delete the auto-created ENI, -// deallocate capacity, drop the instance, and surface InsufficientAddressCapacity. +// TestPrepareRunInstances_PublicIPAllocFailureAbortsLaunch verifies that a +// failed public-IP allocation aborts the launch: detaches and deletes the ENI, +// deallocates capacity, and returns InsufficientAddressCapacity. func TestPrepareRunInstances_PublicIPAllocFailureAbortsLaunch(t *testing.T) { eni := &fakeENICreator{ subnet: &SubnetInfo{SubnetID: "subnet-1", VpcID: "vpc-1", MapPublicIpOnLaunch: true}, @@ -2896,16 +2888,9 @@ func TestPrepareRunInstances_ENICreateFailureDeallocates(t *testing.T) { require.Len(t, prov.deallocated, 1, "ENI failure must trigger deallocate") } -// TestPrepareRunInstances_ENIAttachFailureRollsBack regresses the silent -// corruption where AttachENI failure was logged-and-ignored, leaving the -// auto-created ENI unattached in vpcService. Later RegisterTargets with -// TargetType=instance enumerates ENIs by attachment.instance-id, finds -// none, and drops the target — the customer's ALB has 0 healthy targets -// with no log line connecting the two events. The dangling ENI also leaks -// its auto-assigned EIP on terminate because DeleteNetworkInterface -// cannot resolve the instance link. With the fix, AttachENI failure must -// delete the auto-created ENI, deallocate capacity, and drop the instance -// from the reservation. +// TestPrepareRunInstances_ENIAttachFailureRollsBack verifies that an AttachENI +// failure deletes the auto-created ENI, deallocates capacity, and drops the +// instance from the reservation. func TestPrepareRunInstances_ENIAttachFailureRollsBack(t *testing.T) { eni := &fakeENICreator{ attachErr: errors.New("vpc attach refused"), @@ -3038,10 +3023,8 @@ func TestStartInstance_AllocateFails(t *testing.T) { assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, err.Error()) } -// TestStartInstance_ErrorStateStartable verifies a crash/recovery-failed -// instance (StateError) is accepted past the startability guard. Reaching the -// resource Allocate step (here forced to fail) proves the guard did not reject -// it with IncorrectInstanceState — manual recovery of errored instances works. +// TestStartInstance_ErrorStateStartable verifies a StateError instance passes +// the startability guard and reaches resource allocation. func TestStartInstance_ErrorStateStartable(t *testing.T) { id := "i-err" mgr := mgrWith(map[string]*vm.VM{ @@ -3068,12 +3051,8 @@ func TestRebootInstance_NotFound(t *testing.T) { assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error()) } -// TestStartInstance_NotFound verifies that when vmMgr.Start cannot find the -// instance (e.g. it was terminated between the daemon's pre-dispatch lookup -// and the manager call), the service returns InvalidInstanceID.NotFound -// rather than collapsing to a generic Server.InternalError. The AWS SDK -// retries 500s, so the prior behaviour produced duplicate Start attempts -// before the client surfaced the failure. +// TestStartInstance_NotFound verifies that a missing instance returns +// InvalidInstanceID.NotFound rather than a generic internal error. func TestStartInstance_NotFound(t *testing.T) { id := "i-missing" mgr := mgrWith(nil) diff --git a/spinifex/handlers/ec2/instance/service_nats.go b/spinifex/handlers/ec2/instance/service_nats.go index 67048141d..919146f10 100644 --- a/spinifex/handlers/ec2/instance/service_nats.go +++ b/spinifex/handlers/ec2/instance/service_nats.go @@ -10,14 +10,9 @@ import ( "github.com/nats-io/nats.go" ) -// NATSInstanceService handles instance operations via NATS messaging. -// -// Note: DescribeInstances and DescribeInstanceTypes subscriptions are -// fan-out (no queue group); a single NATSRequest only captures one -// responder. Cluster-wide aggregation is the gateway's job (see -// spinifex/gateway/ec2/instance). The methods below are kept for symmetry -// with the InstanceService interface and intentionally fail rather than -// silently returning a one-node view. +// NATSInstanceService handles instance operations via NATS. Fan-out methods +// (DescribeInstances, DescribeInstanceTypes) intentionally error — use the +// gateway's scatter-gather helper for cluster-wide results. type NATSInstanceService struct { natsConn *nats.Conn } diff --git a/spinifex/handlers/ec2/key/service_impl.go b/spinifex/handlers/ec2/key/service_impl.go index afac82e1f..76d0af2ab 100644 --- a/spinifex/handlers/ec2/key/service_impl.go +++ b/spinifex/handlers/ec2/key/service_impl.go @@ -367,10 +367,8 @@ func (s *KeyServiceImpl) ValidateKeyPairExists(accountID, keyName string) error return err } -// GetPublicKeyMaterial returns the stored OpenSSH public key line, trimmed to a -// single clean line. NoSuchKey maps to ErrorInvalidKeyPairNotFound; every other -// fault is returned as-is so the caller can distinguish absence (404) from a -// backend error (500). +// GetPublicKeyMaterial returns the stored OpenSSH public key line, trimmed to a single +// line. NoSuchKey maps to ErrorInvalidKeyPairNotFound; other errors are returned as-is. func (s *KeyServiceImpl) GetPublicKeyMaterial(accountID, keyName string) (string, error) { keyPath := fmt.Sprintf("keys/%s/%s", accountID, keyName) result, err := s.store.GetObject(&s3.GetObjectInput{ diff --git a/spinifex/handlers/ec2/placementgroup/service_impl.go b/spinifex/handlers/ec2/placementgroup/service_impl.go index 51342ddf6..ad25badae 100644 --- a/spinifex/handlers/ec2/placementgroup/service_impl.go +++ b/spinifex/handlers/ec2/placementgroup/service_impl.go @@ -317,9 +317,7 @@ func (s *PlacementGroupServiceImpl) UpdatePlacementGroupRecord(accountID, groupN const maxCASRetries = 5 // ReserveSpreadNodes atomically reserves node slots for a spread placement group launch. -// It reads the group record, filters eligible nodes (excluding already-occupied ones), -// selects nodes, writes placeholder entries, and returns the selected nodes. -// Uses CAS with retries following the IPAM pattern. +// Filters occupied nodes, selects up to MaxCount, writes placeholders via CAS with retries. func (s *PlacementGroupServiceImpl) ReserveSpreadNodes(input *ReserveSpreadNodesInput, accountID string) (*ReserveSpreadNodesOutput, error) { if input.GroupName == "" { return nil, errors.New(awserrors.ErrorMissingParameter) diff --git a/spinifex/handlers/ec2/placementgroup/service_impl_test.go b/spinifex/handlers/ec2/placementgroup/service_impl_test.go index 6fc1d3f53..3ff3dc6f4 100644 --- a/spinifex/handlers/ec2/placementgroup/service_impl_test.go +++ b/spinifex/handlers/ec2/placementgroup/service_impl_test.go @@ -562,12 +562,9 @@ func TestRemoveInstance_Success(t *testing.T) { assert.Equal(t, []string{"i-bbb"}, record.NodeInstances["node-b"]) } -// TestRemoveInstance_NodeNotFound / _GroupNotFound pin the idempotency -// contract in service_impl.go:453-497 — RemoveInstance must succeed when the -// node or group is missing, since callers (instance Terminate) may race with -// group deletion or have already moved the instance off the node. Production -// marks both paths with //nolint:nilerr // intentional; without these tests a -// future cleanup could remove the annotations and silently flip the contract. +// TestRemoveInstance_NodeNotFound / _GroupNotFound pin the idempotency contract: +// RemoveInstance must succeed when the node or group is missing (callers may race +// with group deletion or have already moved the instance off the node). func TestRemoveInstance_NodeNotFound(t *testing.T) { svc := setupTestService(t) diff --git a/spinifex/handlers/ec2/routetable/service_impl.go b/spinifex/handlers/ec2/routetable/service_impl.go index ccc4a6535..3c9003b6b 100644 --- a/spinifex/handlers/ec2/routetable/service_impl.go +++ b/spinifex/handlers/ec2/routetable/service_impl.go @@ -128,15 +128,9 @@ func (s *RouteTableServiceImpl) getVPCCidr(accountID, vpcID string) (string, err return vpcRecord.CidrBlock, nil } -// mainRouteTable returns the VPC's main route table or (nil, nil) if none -// exists. Used to enumerate effective routes for subnets that have no -// explicit RT association (AWS implicit-main semantics). -// -// Deterministic across duplicate IsMain=true records: see preferMain. A -// duplicate is a bootstrap-race symptom — the canonical fix lives in -// VPCServiceImpl.createMainRouteTable, but the picker stays defensive so a -// stale dup (e.g. cross-AZ restore, manual KV edit) can't break gate -// recomputation by selecting the orphan non-deterministically. +// mainRouteTable returns the VPC's main route table, or (nil, nil) if none exists. +// Applies AWS implicit-main semantics; deterministic across duplicate IsMain=true +// records via preferMain. func (s *RouteTableServiceImpl) mainRouteTable(accountID, vpcID string) (*RouteTableRecord, error) { rts, err := s.allRouteTablesForVPC(accountID, vpcID) if err != nil { @@ -205,12 +199,9 @@ func (s *RouteTableServiceImpl) subnetsImplicitlyOnMainRT(accountID, vpcID strin return implicit, nil } -// effectiveRouteTable returns the route table whose routes apply to subnetID -// per AWS semantics: explicit non-main association if one exists, otherwise -// the VPC's main RT. Returns (nil, nil) when neither is present. -// -// Main-RT selection is deterministic across IsMain=true duplicates via -// preferMain — see mainRouteTable. +// effectiveRouteTable returns the route table whose routes apply to subnetID per AWS +// semantics: explicit non-main association if present, otherwise the VPC's main RT. +// Returns (nil, nil) when neither is present. Main-RT ties broken via preferMain. func (s *RouteTableServiceImpl) effectiveRouteTable(accountID, vpcID, subnetID string) (*RouteTableRecord, error) { rts, err := s.allRouteTablesForVPC(accountID, vpcID) if err != nil { @@ -230,12 +221,9 @@ func (s *RouteTableServiceImpl) effectiveRouteTable(accountID, vpcID, subnetID s return main, nil } -// preferMain reports whether candidate should replace current as the chosen -// main RT. Ordered tiebreakers when two IsMain=true records survive for one -// VPC: (1) more routes — user-added 0.0.0.0/0 etc. flag the "real" RT vs an -// orphan that only carries the implicit local route; (2) oldest CreatedAt — -// the original wins over a late dup; (3) smaller RouteTableId — total order -// so the same input always produces the same answer across nodes. +// preferMain reports whether candidate should replace current as the chosen main RT. +// Tiebreakers: (1) more routes, (2) oldest CreatedAt, (3) smaller RouteTableId — +// giving a deterministic total order across duplicate IsMain=true records. func preferMain(current, candidate *RouteTableRecord) bool { if current == nil { return true @@ -249,10 +237,8 @@ func preferMain(current, candidate *RouteTableRecord) bool { return candidate.RouteTableId < current.RouteTableId } -// subnetHasDefaultEgress reports whether subnetID's effective RT carries a -// destCidr route pointing at any internet-bound target (IGW or NAT GW). If -// false, the subnet must be gated with a DROP policy because the VPC LR's -// router-wide default static route would otherwise let packets egress. +// subnetHasDefaultEgress reports whether subnetID's effective RT carries a destCidr +// route pointing at an IGW or NAT GW; false means the subnet must be gated with a DROP policy. func (s *RouteTableServiceImpl) subnetHasDefaultEgress(accountID, vpcID, subnetID, destCidr string) (bool, error) { rt, err := s.effectiveRouteTable(accountID, vpcID, subnetID) if err != nil { @@ -275,12 +261,9 @@ func (s *RouteTableServiceImpl) subnetHasDefaultEgress(accountID, vpcID, subnetI return false, nil } -// publishSubnetEgressGateDecision recomputes the gate/ungate decision for one -// subnet+destCidr pair and emits either vpc.gate-subnet-egress (when the -// effective RT lacks an internet-bound route) or vpc.ungate-subnet-egress -// (when one is present). Called after every per-subnet IGW/NATGW event so -// the drop policy state always matches the AWS RT view. Side-effect only; -// errors are logged. +// publishSubnetEgressGateDecision emits vpc.gate-subnet-egress or vpc.ungate-subnet-egress +// for one subnet+destCidr based on whether the effective RT has an internet-bound route. +// Side-effect only; errors are logged. func (s *RouteTableServiceImpl) publishSubnetEgressGateDecision(accountID, vpcID, subnetID, destCidr string) { if s.natsConn == nil || subnetID == "" || vpcID == "" { return @@ -320,11 +303,9 @@ func (s *RouteTableServiceImpl) publishSubnetEgressGateDecision(accountID, vpcID "topic", topic, "vpcId", vpcID, "subnetId", subnetID, "destinationCidr", destCidr) } -// PublishGateDecisionsForVPC recomputes gate/ungate for every subnet in -// vpcID against destCidr by enumerating all subnets in the VPC and resolving -// each one's effective RT. Called from the IGW handler on Attach / Detach so -// per-subnet OVN DROP policies are reinstated immediately instead of waiting -// for the reconciler's drift tick. destCidr defaults to 0.0.0.0/0. +// PublishGateDecisionsForVPC recomputes gate/ungate for every subnet in vpcID against +// destCidr, resolving each subnet's effective RT. Called on IGW Attach/Detach. +// destCidr defaults to 0.0.0.0/0. func (s *RouteTableServiceImpl) PublishGateDecisionsForVPC(accountID, vpcID, destCidr string) { if s.natsConn == nil || vpcID == "" { return @@ -909,10 +890,8 @@ func (s *RouteTableServiceImpl) AssociateRouteTable(input *ec2.AssociateRouteTab return nil, err } - // Subnet leaves implicit main-RT membership: tear down per-subnet rules - // that came from the main RT's routes before the explicit RT's rules - // take effect. Skip if the explicit RT IS the main RT (can't happen via - // public API but DescribeRouteTables permits the query). + // Subnet leaves implicit main-RT membership: tear down main RT's per-subnet rules + // before the explicit RT's rules take effect. if !record.IsMain { if mainRT, err := s.mainRouteTable(accountID, record.VpcId); err != nil { slog.Warn("AssociateRouteTable: main RT lookup failed", "vpcId", record.VpcId, "err", err) @@ -995,10 +974,7 @@ func (s *RouteTableServiceImpl) DisassociateRouteTable(input *ec2.DisassociateRo s.publishNatGatewayEventsForAssociation(accountID, "vpc.delete-nat-gateway", &record, departingSubnetID) s.publishIGWRouteEventsForAssociation(accountID, "vpc.delete-igw-route", &record, departingSubnetID) - // Subnet falls back to implicit main-RT membership: re-install - // per-subnet rules for the main RT's routes (no-op if departing - // table WAS the main, which can't happen since DisassociateMain - // is rejected above). + // Subnet falls back to implicit main-RT membership: re-install main RT's per-subnet rules. if mainRT, err := s.mainRouteTable(accountID, record.VpcId); err != nil { slog.Warn("DisassociateRouteTable: main RT lookup failed", "vpcId", record.VpcId, "err", err) } else if mainRT != nil && mainRT.RouteTableId != record.RouteTableId { @@ -1247,10 +1223,8 @@ func rtbMatchesFilters(record *RouteTableRecord, filters map[string][]string) bo return filterutil.MatchesTags(filters, record.Tags) } -// publishNatGatewayEvents publishes vpc.add-nat-gateway events for each subnet -// associated with this route table, so vpcd creates the SNAT rules. Called by -// CreateRoute when a NAT GW route is added to a table that may already have -// subnet associations. +// publishNatGatewayEvents publishes vpc.add-nat-gateway for each associated subnet +// so vpcd creates the SNAT rules. func (s *RouteTableServiceImpl) publishNatGatewayEvents(accountID string, record *RouteTableRecord, vpcID, natgwID, publicIp, destCidr string) { if s.natsConn == nil { return @@ -1324,10 +1298,8 @@ func (s *RouteTableServiceImpl) publishNatGatewayDeleteEvents(accountID string, } } -// publishNatGatewayEventsForAssociation emits one NAT GW SNAT event per NAT GW -// route on the table, scoped to a single subnet. Called when a subnet joins or -// leaves a route table that already has NAT GW routes so OVN SNAT state tracks -// association lifecycle (terraform creates the route first, then associates). +// publishNatGatewayEventsForAssociation emits one NAT GW SNAT event per NAT GW route +// on the table, scoped to a single subnet joining or leaving the table. func (s *RouteTableServiceImpl) publishNatGatewayEventsForAssociation(accountID, topic string, record *RouteTableRecord, subnetID string) { if s.natsConn == nil || subnetID == "" { return @@ -1398,10 +1370,8 @@ func (s *RouteTableServiceImpl) publishNatGatewayEventForSubnet(accountID, topic slog.Info("NAT GW event published", "topic", topic, "subnetCidr", subnet.CidrBlock, "publicIp", publicIp, "subnetId", subnetID, "destinationCidr", destCidr) } -// publishIGWRouteEvents emits one vpc.{add,delete}-igw-route event per subnet -// associated with the route table. Called by CreateRoute/DeleteRoute when an -// IGW route is added or removed against a table that may already carry -// subnet associations. +// publishIGWRouteEvents emits one vpc.{add,delete}-igw-route event per associated +// subnet when an IGW route is added or removed from a table. func (s *RouteTableServiceImpl) publishIGWRouteEvents(accountID, topic string, record *RouteTableRecord, vpcID, igwID, destCidr string) { if s.natsConn == nil { return @@ -1430,10 +1400,8 @@ func (s *RouteTableServiceImpl) publishIGWRouteEvents(accountID, topic string, r } } -// publishIGWRouteEventsForAssociation emits one IGW route event per IGW route -// on the table, scoped to a single subnet. Called when a subnet joins or leaves -// a route table that already has IGW routes so OVN policy state tracks -// association lifecycle (terraform creates the route first, then associates). +// publishIGWRouteEventsForAssociation emits one IGW route event per IGW route on the +// table, scoped to a single subnet joining or leaving the table. func (s *RouteTableServiceImpl) publishIGWRouteEventsForAssociation(accountID, topic string, record *RouteTableRecord, subnetID string) { if s.natsConn == nil || subnetID == "" { return @@ -1446,10 +1414,8 @@ func (s *RouteTableServiceImpl) publishIGWRouteEventsForAssociation(accountID, t } } -// publishIGWRouteEventForSubnet publishes a single vpc.{add,delete}-igw-route -// event. Side-effect only — logs and swallows errors so a publish failure -// doesn't fail the caller's API response. The subscriber (network/subscribers) -// resolves the gateway nexthop and installs the OVN Logical_Router_Policy. +// publishIGWRouteEventForSubnet publishes a single vpc.{add,delete}-igw-route event. +// Side-effect only — logs and swallows errors so a publish failure doesn't fail the caller. func (s *RouteTableServiceImpl) publishIGWRouteEventForSubnet(topic, subnetID, vpcID, igwID, destCidr string) { if s.natsConn == nil { return diff --git a/spinifex/handlers/ec2/routetable/service_impl_test.go b/spinifex/handlers/ec2/routetable/service_impl_test.go index 792829666..534232ff3 100644 --- a/spinifex/handlers/ec2/routetable/service_impl_test.go +++ b/spinifex/handlers/ec2/routetable/service_impl_test.go @@ -491,10 +491,8 @@ func receiveNatGWEvent(t *testing.T, sub *nats.Subscription, wantCidr string) (n return evt.NatGatewayId, evt.PublicIp } -// TestAssociateRouteTable_PublishesNatGatewayEvent covers the terraform flow: -// CreateRouteTable → CreateRoute(NAT GW) → AssociateRouteTable(subnet). CreateRoute -// runs against an empty-association table so no event fires there; Associate must -// emit the event so vpcd installs the SNAT rule for the joining subnet. +// TestAssociateRouteTable_PublishesNatGatewayEvent covers the terraform flow where +// AssociateRouteTable must emit a NAT GW event so vpcd installs the SNAT rule. func TestAssociateRouteTable_PublishesNatGatewayEvent(t *testing.T) { svc := setupTestService(t) sub, err := svc.natsConn.SubscribeSync("vpc.add-nat-gateway") @@ -577,10 +575,8 @@ func TestDisassociateRouteTable_PublishesNatGatewayDeleteEvent(t *testing.T) { assert.Equal(t, "nat-test1", natgwID) } -// TestDeleteRoute_NATGW_PublishesDeleteForAssociatedSubnets covers DeleteRoute -// on a NATGW route: every subnet associated with the table receives a -// vpc.delete-nat-gateway event so the subscriber tears down both the SNAT rule -// and the per-subnet egress policy (mirror of the IGW DeleteRoute path). +// TestDeleteRoute_NATGW_PublishesDeleteForAssociatedSubnets verifies that every +// associated subnet receives a vpc.delete-nat-gateway event when a NATGW route is removed. func TestDeleteRoute_NATGW_PublishesDeleteForAssociatedSubnets(t *testing.T) { svc := setupTestService(t) addSub, err := svc.natsConn.SubscribeSync("vpc.add-nat-gateway") @@ -707,10 +703,8 @@ func TestCreateRoute_IGW_PublishesAddIGWRouteForAssociatedSubnets(t *testing.T) assert.Equal(t, "igw-test1", igwID) } -// TestAssociateRouteTable_PublishesAddIGWRouteForExistingRoute covers the -// terraform ordering: CreateRoute(IGW) → AssociateRouteTable. CreateRoute runs -// against an empty-association table so no event fires; Associate must emit -// the event so the subscriber installs the policy for the joining subnet. +// TestAssociateRouteTable_PublishesAddIGWRouteForExistingRoute covers the terraform +// ordering: AssociateRouteTable must emit an IGW route event for the joining subnet. func TestAssociateRouteTable_PublishesAddIGWRouteForExistingRoute(t *testing.T) { svc := setupTestService(t) sub, err := svc.natsConn.SubscribeSync("vpc.add-igw-route") @@ -825,10 +819,8 @@ func drainIGWSubnets(t *testing.T, sub *nats.Subscription, count int) map[string return got } -// TestCreateRoute_IGW_OnMainRT_FansOutToImplicitSubnets covers the primary -// publish-gap fixed in this change: CreateRoute(IGW) on a VPC's main RT must -// emit one vpc.add-igw-route event per subnet that has no explicit non-main -// RT association (AWS implicit-main semantics). +// TestCreateRoute_IGW_OnMainRT_FansOutToImplicitSubnets verifies CreateRoute(IGW) on +// the main RT emits one vpc.add-igw-route per implicit-main subnet. func TestCreateRoute_IGW_OnMainRT_FansOutToImplicitSubnets(t *testing.T) { svc := setupTestService(t) sub, err := svc.natsConn.SubscribeSync("vpc.add-igw-route") @@ -882,10 +874,8 @@ func TestCreateRoute_IGW_OnMainRT_SkipsExplicitlyAssociatedSubnets(t *testing.T) assert.False(t, got["subnet-priv1"], "explicitly-associated subnet must NOT receive main-RT event") } -// TestAssociateRouteTable_RemovesMainRTRoutesForJoiningSubnet covers the -// symmetric fix: when a subnet leaves implicit main-RT membership for an -// explicit non-main RT, the main RT's per-subnet rules must be torn down so -// state matches AWS effective-route semantics. +// TestAssociateRouteTable_RemovesMainRTRoutesForJoiningSubnet verifies that the main +// RT's per-subnet rules are torn down when a subnet joins an explicit non-main RT. func TestAssociateRouteTable_RemovesMainRTRoutesForJoiningSubnet(t *testing.T) { svc := setupTestService(t) delSub, err := svc.natsConn.SubscribeSync("vpc.delete-igw-route") @@ -913,13 +903,8 @@ func TestAssociateRouteTable_RemovesMainRTRoutesForJoiningSubnet(t *testing.T) { assert.Len(t, got, 1) } -// TestDisassociateRouteTable_RestoresMainRTRoutesForDepartingSubnet covers -// the symmetric add-back: when a subnet leaves an explicit RT and falls back -// to implicit main-RT membership, the main RT's per-subnet rules must be -// re-installed. -// receiveGateEvent reads one message from sub and returns the (subnet, -// destCidr) it carried. Used for both vpc.gate-subnet-egress and -// vpc.ungate-subnet-egress since the payload shape is identical. +// receiveGateEvent reads one gate/ungate message from sub and returns the +// (subnet, destCidr) it carried. func receiveGateEvent(t *testing.T, sub *nats.Subscription) (subnetID, destCidr string) { t.Helper() msg, err := sub.NextMsg(time.Second) @@ -951,10 +936,8 @@ func drainGateSubnets(t *testing.T, sub *nats.Subscription, limit int) map[strin return got } -// TestCreateRoute_IGW_PublishesUngateForAssociatedSubnets: when a subnet is -// already associated to a route table and an IGW default route is added, -// the gate decision flips to "egress allowed" so the subscriber must remove -// any previously-installed drop policy. +// TestCreateRoute_IGW_PublishesUngateForAssociatedSubnets: adding an IGW default route +// to a table must flip associated subnets to "egress allowed" (ungate). func TestCreateRoute_IGW_PublishesUngateForAssociatedSubnets(t *testing.T) { svc := setupTestService(t) ungate, err := svc.natsConn.SubscribeSync("vpc.ungate-subnet-egress") @@ -1269,13 +1252,9 @@ func TestPublishGateDecisionsForVPC_UngatesSubnetWithIGWRoute(t *testing.T) { assert.Len(t, gateGot, 1) } -// TestEffectiveRouteTable_DeterministicWithDuplicateMain seeds two -// IsMain=true RTs for the same VPC (the bootstrap-race symptom) and asserts -// effectiveRouteTable picks the one with more routes — the "real" RT carries -// any user-added 0.0.0.0/0 entry, while the orphan only has the implicit -// local route. Non-deterministic selection produced the run 26699045536 -// guest-SSH timeout: CreateRoute's post-write gate recompute happened to -// resolve to the orphan and published vpc.gate-subnet-egress. +// TestEffectiveRouteTable_DeterministicWithDuplicateMain seeds two IsMain=true RTs +// for the same VPC and asserts effectiveRouteTable deterministically picks the one +// with more routes (the "real" RT) over the orphan with only the implicit local route. func TestEffectiveRouteTable_DeterministicWithDuplicateMain(t *testing.T) { svc := setupTestService(t) now := time.Now() diff --git a/spinifex/handlers/ec2/volume/service_impl.go b/spinifex/handlers/ec2/volume/service_impl.go index 1160eebe2..40b76ea4b 100644 --- a/spinifex/handlers/ec2/volume/service_impl.go +++ b/spinifex/handlers/ec2/volume/service_impl.go @@ -785,10 +785,9 @@ type volumeModificationResult struct { err error } -// fetchVolumeModificationsByIDs reads each requested volume's config in -// parallel, returning a result slice positionally aligned with volumeIDs. -// Cross-tenant volumes surface as InvalidVolume.NotFound, mirroring -// DescribeVolumes' silent tenant scoping. +// fetchVolumeModificationsByIDs reads each requested volume's config in parallel, +// returning results positionally aligned with volumeIDs. Cross-tenant volumes surface +// as InvalidVolume.NotFound. func (s *VolumeServiceImpl) fetchVolumeModificationsByIDs(volumeIDs []*string, accountID string) []volumeModificationResult { results := make([]volumeModificationResult, len(volumeIDs)) var wg sync.WaitGroup @@ -980,10 +979,8 @@ func (s *VolumeServiceImpl) GetVolumeConfig(volumeID string) (*viperblock.Volume return cfg, err } -// getVolumeConfigAndEncryption reads config.json and surfaces both the -// VolumeConfig and the authoritative VBState.EncryptionEnabled flag. Wrapper- -// only blobs (pre-VBState volumes or freshly-seeded test fixtures) report -// encryptionEnabled=false — they predate encryption-at-rest by construction. +// getVolumeConfigAndEncryption reads config.json and returns the VolumeConfig plus the +// VBState.EncryptionEnabled flag. Pre-VBState blobs report encryptionEnabled=false. func (s *VolumeServiceImpl) getVolumeConfigAndEncryption(volumeID string) (*viperblock.VolumeConfig, bool, error) { configKey := volumeID + "/config.json" @@ -1043,16 +1040,9 @@ func (s *VolumeServiceImpl) putVolumeConfig(volumeID string, cfg *viperblock.Vol return nil } -// mergeVolumeConfig reads existing config.json from S3 and merges the new -// VolumeConfig into it, preserving full VBState when present. If no existing -// VBState is found, it returns a plain volumeConfigWrapper. -// -// Refuses to merge an encrypted-at-rest VBState: the persisted blob carries a -// trailing 16-byte AES-GCM tag bound to the JSON bytes, and re-marshaling -// here without the master key would strip the tag and brick the volume. -// Spinifex does not currently hold the viperblock master key, so the safe -// behavior is to fail loud rather than silently corrupt the on-disk state. -// Track full encryption-aware merge under a separate bead. +// mergeVolumeConfig reads existing config.json from S3 and merges the new VolumeConfig +// into it, preserving full VBState when present. Refuses to merge encrypted VBState — +// re-marshaling without the master key would corrupt the on-disk AES-GCM tag. func (s *VolumeServiceImpl) mergeVolumeConfig(configKey string, cfg *viperblock.VolumeConfig) ([]byte, error) { getResult, err := s.store.GetObject(&s3.GetObjectInput{ Bucket: aws.String(s.bucketName), @@ -1072,10 +1062,8 @@ func (s *VolumeServiceImpl) mergeVolumeConfig(configKey string, cfg *viperblock. return nil, fmt.Errorf("failed to read existing config: %w", err) } - // Decoder (not Unmarshal) so a trailing 16-byte AES-GCM tag does not - // produce a parse error and route the body to the wrapper-only path — - // that would silently overwrite an encrypted config.json with a tag-less - // blob and brick the volume. + // Use Decoder (not Unmarshal): a trailing AES-GCM tag must not cause a parse error + // that would silently route to the wrapper-only path and brick the volume. var state viperblock.VBState if decodeErr := json.NewDecoder(bytes.NewReader(body)).Decode(&state); decodeErr != nil || state.BlockSize == 0 { // Not a full VBState (new volume or wrapper-only) -- write wrapper @@ -1180,10 +1168,8 @@ func (s *VolumeServiceImpl) ModifyVolume(input *ec2.ModifyVolumeInput, accountID } targetIOPS := int64(volMeta.IOPS) - // Persist the modification record alongside the volume metadata so - // DescribeVolumesModifications can read it back. spinifex applies - // modifications synchronously, so the persisted state is always - // completed/100/EndTime==StartTime. + // Persist the modification record so DescribeVolumesModifications can read it back. + // Modifications are synchronous, so state is always completed/100. now := time.Now() cfg.Modification = &viperblock.VolumeModification{ VolumeID: volumeID, diff --git a/spinifex/handlers/ec2/volume/service_impl_test.go b/spinifex/handlers/ec2/volume/service_impl_test.go index fb6827309..84eec0310 100644 --- a/spinifex/handlers/ec2/volume/service_impl_test.go +++ b/spinifex/handlers/ec2/volume/service_impl_test.go @@ -963,10 +963,7 @@ func TestCreateVolume_StampsAccountID(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - // CreateVolume will fail at viperblock layer (no real S3), but the - // VolumeConfig should have TenantID set if we can inspect it. - // We test via the validation path — CreateVolume with invalid size - // should NOT fail because of accountID. + // Test via the validation path — CreateVolume should not fail because of accountID. _, err := svc.CreateVolume(&ec2.CreateVolumeInput{ Size: aws.Int64(1), AvailabilityZone: aws.String("ap-southeast-2a"), diff --git a/spinifex/handlers/ec2/vpc/eni.go b/spinifex/handlers/ec2/vpc/eni.go index e28749bcf..f445d61b1 100644 --- a/spinifex/handlers/ec2/vpc/eni.go +++ b/spinifex/handlers/ec2/vpc/eni.go @@ -575,10 +575,8 @@ func (s *VPCServiceImpl) UpdateENI(accountID, eniId string, fn func(*ENIRecord)) return nil } -// FindENIByAttachment scans the per-account ENI bucket for a record whose -// AttachmentId matches attachmentId. Returns NotFound semantics when no -// match exists. Used by DetachNetworkInterface, whose AWS input identifies -// the attachment by id rather than by ENI id. +// FindENIByAttachment scans the ENI bucket for the record with the given +// AttachmentId. Used by DetachNetworkInterface which identifies by attachment ID. func (s *VPCServiceImpl) FindENIByAttachment(accountID, attachmentId string) (ENIRecord, error) { keys, err := s.eniKV.Keys() if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { @@ -721,10 +719,8 @@ func (s *VPCServiceImpl) publishPortEvent(topic, eniId, subnetId, vpcId, private }) } -// requestPortEvent sends a port lifecycle event via request-reply with the -// shared SG-event timeout. Used for vpc.create-port so vpcd OVSDB failures -// (which leave the LSP outside any SG port group) surface to the caller -// instead of being swallowed. +// requestPortEvent sends a port lifecycle event via request-reply so vpcd +// OVSDB failures surface to the caller rather than being swallowed. func (s *VPCServiceImpl) requestPortEvent(topic, eniId, subnetId, vpcId, privateIP, macAddr string, sgIds []string) error { return utils.RequestEvent(s.natsConn, topic, portEventPayload{ NetworkInterfaceId: eniId, @@ -737,9 +733,7 @@ func (s *VPCServiceImpl) requestPortEvent(topic, eniId, subnetId, vpcId, private } // requestUpdatePortSGsEvent sends a vpc.update-port-sgs event to vpcd via -// request-reply so OVN port-group reconciliation errors surface to the caller -// instead of being swallowed. The payload is declarative — vpcd reads its -// libovsdb cache to compute the diff. +// request-reply. Errors surface to the caller; vpcd computes the OVN diff. func (s *VPCServiceImpl) requestUpdatePortSGsEvent(eniId, privateIP string, sgIds []string) error { return utils.RequestEvent(s.natsConn, "vpc.update-port-sgs", struct { NetworkInterfaceId string `json:"network_interface_id"` @@ -768,16 +762,9 @@ func (s *VPCServiceImpl) publishNATEvent(topic, vpcId, externalIP, logicalIP, po }) } -// validateSGAttachment is the boundary validator shared by RunInstances (via -// CreateNetworkInterface), CreateNetworkInterface itself, and -// ModifyNetworkInterfaceAttribute. It runs *before* any KV write so failures -// don't leave half-state. -// -// AWS contract reproduced here: -// - InvalidGroup.NotFound when an SG ID is unknown to the account. -// - InvalidParameterValue when an SG's VPC differs from the resolved VPC. -// - SecurityGroupsPerInterfaceLimitExceeded when >5 SGs (AWS default). -// - MissingParameter when the list is empty (every ENI must have ≥1 SG). +// validateSGAttachment validates SG IDs before any KV write. Returns +// InvalidGroup.NotFound, InvalidParameterValue, SecurityGroupsPerInterfaceLimitExceeded, +// or MissingParameter per AWS contract. const sgPerInterfaceLimit = 5 func (s *VPCServiceImpl) validateSGAttachment(accountID string, sgIds []string, vpcId string) error { diff --git a/spinifex/handlers/ec2/vpc/external_ipam.go b/spinifex/handlers/ec2/vpc/external_ipam.go index 570a6357b..a7de45d9e 100644 --- a/spinifex/handlers/ec2/vpc/external_ipam.go +++ b/spinifex/handlers/ec2/vpc/external_ipam.go @@ -25,10 +25,8 @@ const ( KVBucketExternalIPAMVersion = external.KVBucketStaticPoolVersion ) -// ExternalPoolConfig is the admin-defined pool from spinifex.toml. -// Duplicated against network/external.ExternalPoolConfig — the duplicate -// stays until ExternalIPAM moves into network/external (deferred L5 -// cleanup tracked separately). +// ExternalPoolConfig is the admin-defined IP pool from spinifex.toml. +// Duplicated from network/external.ExternalPoolConfig pending consolidation. type ExternalPoolConfig struct { Name string Source string // "static" (default) or "dhcp" @@ -47,10 +45,8 @@ type ExternalPoolConfig struct { GwLrpRangeEnd string } -// ExternalIPAM is the AWS-facing entry point for external IP allocation. -// State + CAS logic lives in external.StaticPoolAllocator for static -// pools and dhcp.DHCPPoolAllocator (via vpcd RPC) for dhcp-sourced -// pools; ExternalIPAM dispatches by pool name. +// ExternalIPAM is the AWS-facing entry point for external IP allocation, +// dispatching to StaticPoolAllocator or dhcp.DHCPPoolAllocator per pool name. type ExternalIPAM struct { kv nats.KeyValue pools []ExternalPoolConfig diff --git a/spinifex/handlers/ec2/vpc/external_ipam_test.go b/spinifex/handlers/ec2/vpc/external_ipam_test.go index 0408f995c..e7750a123 100644 --- a/spinifex/handlers/ec2/vpc/external_ipam_test.go +++ b/spinifex/handlers/ec2/vpc/external_ipam_test.go @@ -100,8 +100,8 @@ func TestExternalIPAM_Release(t *testing.T) { err = ipam.ReleaseIP("wan", "192.168.1.151") require.NoError(t, err) - // Round-robin: a released IP is NOT reused immediately (siv-246). Allocation - // resumes past the cursor, so the next IP is .153, not the freed .151. + // Round-robin: a released IP is NOT reused immediately. Allocation resumes + // past the cursor, so the next IP is .153, not the freed .151. ip3, _, err := ipam.AllocateIP("", "", PurposeENIPublic, "", "eni-3", "i-3") require.NoError(t, err) assert.Equal(t, "192.168.1.153", ip3) diff --git a/spinifex/handlers/ec2/vpc/invariants_test.go b/spinifex/handlers/ec2/vpc/invariants_test.go index b33a70d7f..b1b5fddb6 100644 --- a/spinifex/handlers/ec2/vpc/invariants_test.go +++ b/spinifex/handlers/ec2/vpc/invariants_test.go @@ -12,15 +12,9 @@ import ( // IMDS-datapath invariant. -// TestI4_ENIIndexBucketShape asserts the spinifex-network-eni-by-vpc-ip reverse -// index keeps its contract: keys parse as "vpcID/ip" with a valid IP, values -// carry a non-empty eni_id, and no mutable/denormalised fields creep back into -// the value. The index is identity-only (eni_id + the immutable account_id) by -// design — every mutable field (IPs, MAC, instance ID, profile ARN) is read -// live off the ENIRecord + instance record at request time. A denormalised -// field here resurrects the staleness class the design deliberately removed: -// an IMDS handler trusting a stale profile ARN would mint credentials for the -// wrong role. +// TestI4_ENIIndexBucketShape asserts the eni-by-vpc-ip reverse index keeps its +// contract: keys parse as "vpcID/ip", values carry only immutable identity +// fields (eni_id + account_id), and no mutable/denormalised fields are present. func TestI4_ENIIndexBucketShape(t *testing.T) { _, nc := setupTestVPCServiceWithNC(t) kv := openTestENIByIPBucket(t, nc) @@ -30,10 +24,8 @@ func TestI4_ENIIndexBucketShape(t *testing.T) { require.NoError(t, idx.Put("vpc-aaaaaaaa", "10.0.1.5", "eni-aaa", "111122223333")) require.NoError(t, idx.Put("vpc-bbbbbbbb", "10.0.2.9", "eni-bbb", "444455556666")) - // allowedFields is the closed set the value JSON may contain. Adding a field - // to eniByIPValue without updating this set fails the test by design — a - // reviewer must justify any new field as immutable identity, not cached - // mutable state. + // allowedFields is the closed set the value JSON may contain; any new field + // must be immutable identity, not cached mutable state. allowedFields := map[string]struct{}{"eni_id": {}, "account_id": {}} keys, err := kv.Keys() diff --git a/spinifex/handlers/ec2/vpc/ipam.go b/spinifex/handlers/ec2/vpc/ipam.go index b79d27f70..3b20378f7 100644 --- a/spinifex/handlers/ec2/vpc/ipam.go +++ b/spinifex/handlers/ec2/vpc/ipam.go @@ -55,12 +55,8 @@ func NewIPAMWithKV(kv nats.KeyValue) *IPAM { return &IPAM{kv: kv} } -// AllocateIP allocates an IP address from the given subnet. -// It reserves the first 4 and last IP per AWS convention: -// .0=network, .1=gateway, .2=DNS, .3=reserved, .255=broadcast (for /24). -// Uses CAS for conflict-free allocation across nodes. purpose is one of the -// Purpose* constants in purpose.go; ownerID is the ENI / instance / EIP -// resource ID that owns this allocation. +// AllocateIP allocates an IP from the subnet, reserving the first 4 and last +// addresses per AWS convention. Uses CAS for conflict-free multi-node allocation. func (m *IPAM) AllocateIP(subnetId, cidrBlock, purpose, ownerID string) (string, error) { for attempt := range 5 { record, revision, err := m.getRecord(subnetId) diff --git a/spinifex/handlers/ec2/vpc/security_group.go b/spinifex/handlers/ec2/vpc/security_group.go index 05354b628..4cd6b5777 100644 --- a/spinifex/handlers/ec2/vpc/security_group.go +++ b/spinifex/handlers/ec2/vpc/security_group.go @@ -27,12 +27,9 @@ var sgIDRegex = regexp.MustCompile(`^sg-[0-9a-f]{17}$`) // re-implementing the format check. var SGRuleIDRegex = regexp.MustCompile(`^sgr-[0-9a-f]{17}$`) -// validateSGRule rejects values that could break out of an OVN ACL match-expression token. -// CidrIp must be IPv4 and round-trip to canonical form (so "10.0.0.5/8" with host bits set is -// rejected, as is anything containing operators/whitespace that net.ParseCIDR would not accept). -// IPv6 is rejected because the ACL builder in vpcd/acl.go is IPv4-only — accepting an IPv6 CIDR -// would persist a rule that OVN can never program. -// SourceSG must match the spinifex SG-ID format. At least one source must be specified. +// validateSGRule rejects CidrIp values that are non-canonical or IPv6 (OVN ACL +// builder is IPv4-only), and SourceSG values not matching the sg-ID format. +// At least one source must be specified. func validateSGRule(r SGRule) error { if r.CidrIp == "" && r.SourceSG == "" { return errors.New("rule must specify CidrIp or SourceSG") @@ -71,11 +68,8 @@ const ( ) // SGRule represents a single ingress or egress rule in a security group. -// -// RuleId is the persistent AWS-style identifier returned by -// DescribeSecurityGroupRules. It is assigned at insert by Authorize* and by -// the v1→v2 migration for pre-existing rules; sgRuleKey deliberately excludes -// it so duplicate-content rules are still rejected regardless of ID. +// RuleId is assigned on Authorize and excluded from sgRuleKey so +// duplicate-content rules are rejected regardless of ID. type SGRule struct { RuleId string `json:"rule_id"` // sgr-<17 hex>; assigned on Authorize, backfilled by migration IpProtocol string `json:"ip_protocol"` // "tcp", "udp", "icmp", "-1" (all) @@ -262,11 +256,9 @@ func (s *VPCServiceImpl) DeleteSecurityGroup(input *ec2.DeleteSecurityGroupInput return &ec2.DeleteSecurityGroupOutput{}, nil } -// validateSGRuleReferences ensures every SourceSG named in `rules` resolves to -// an SG in the caller's account that lives in the same VPC as the rule-owning -// SG. Cross-VPC and missing references both return InvalidGroup.NotFound, -// matching AWS (peering not supported; AWS uses the same code for "doesn't -// exist" and "exists in another VPC"). +// validateSGRuleReferences returns InvalidGroup.NotFound if any SourceSG in +// rules is missing or belongs to a different VPC (cross-VPC refs not +// supported; AWS uses the same error for both cases). func (s *VPCServiceImpl) validateSGRuleReferences(accountID, ownerVpcId string, rules []SGRule) error { for _, r := range rules { if r.SourceSG == "" { @@ -492,10 +484,8 @@ var describeSecurityGroupRulesValidFilters = map[string]bool{ } // DescribeSecurityGroupRules returns a flat list of SecurityGroupRule objects -// across every security group in the caller's account, optionally narrowed by -// SecurityGroupRuleIds or filters (group-id, security-group-rule-id, tag:*, -// tag-key). MaxResults and NextToken are accepted but ignored — matches the -// existing Describe* precedent in spinifex. +// for the caller's account, optionally narrowed by SecurityGroupRuleIds or +// filters. MaxResults and NextToken are accepted but ignored. func (s *VPCServiceImpl) DescribeSecurityGroupRules(input *ec2.DescribeSecurityGroupRulesInput, accountID string) (*ec2.DescribeSecurityGroupRulesOutput, error) { requested := make(map[string]bool) if input != nil { @@ -579,9 +569,8 @@ func (s *VPCServiceImpl) DescribeSecurityGroupRules(input *ec2.DescribeSecurityG } // sgRuleMatchesFilters applies the DescribeSecurityGroupRules filter set to a -// single rule within its parent record. Rule-level tags are not yet persisted, -// so tag:* and tag-key filters always fail to match (consistent with returning -// the empty set for tag queries until per-rule tagging lands). +// single rule. Rule-level tags are not yet persisted, so tag:* and tag-key +// filters always return false. func sgRuleMatchesFilters(record *SecurityGroupRecord, rule SGRule, filters map[string][]string) bool { for name, values := range filters { if strings.HasPrefix(name, "tag:") { @@ -601,11 +590,8 @@ func sgRuleMatchesFilters(record *SecurityGroupRecord, rule SGRule, filters map[ // rule.Tags is not yet populated; tag-key excludes every rule. return false default: - // Unreachable today because ParseFilters rejects anything not in - // describeSecurityGroupRulesValidFilters or starting with "tag:". - // If a future change adds a name to that map without adding a case - // here, this Error gives a Sentry breadcrumb instead of silently - // returning an empty result set for a documented filter. + // Unreachable: ParseFilters rejects names not in the valid set. + // Logs an error if a future map entry lacks a matching case. slog.Error("sgRuleMatchesFilters: filter accepted by ParseFilters but no case", "filter", name) return false } @@ -614,10 +600,8 @@ func sgRuleMatchesFilters(record *SecurityGroupRecord, rule SGRule, filters map[ } // sgRuleToSecurityGroupRule flattens a stored SGRule into the AWS API shape. -// accountID supplies GroupOwnerId and (for SG-source rules) the -// ReferencedGroupInfo.UserId. VpcId is derived from the parent record — -// spinifex enforces same-VPC SG references at security_group.go:validateSGRuleReferences, -// so the referenced SG's VpcId always equals the parent's. +// accountID supplies GroupOwnerId and ReferencedGroupInfo.UserId; VpcId is +// derived from the parent record (same-VPC references are enforced on write). func sgRuleToSecurityGroupRule(record *SecurityGroupRecord, rule SGRule, isEgress bool, accountID string) *ec2.SecurityGroupRule { out := &ec2.SecurityGroupRule{ SecurityGroupRuleId: aws.String(rule.RuleId), @@ -913,11 +897,8 @@ func (s *VPCServiceImpl) sgRecordToEC2(record *SecurityGroupRecord, accountID st } // sgParseMode selects how ipPermissionsToSGRules handles IPv6 sources. -// Spinifex stores IPv4-only rules (the OVN ACL builder in vpcd/acl.go cannot -// program IPv6), so Ipv6Ranges are unrepresentable. Authorize must reject -// them; Revoke must tolerate them silently because the Terraform AWS provider -// auto-revokes the default ::/0 IPv6 egress on every aws_security_group it -// creates and would otherwise abort every apply that touches an SG. +// Authorize rejects IPv6; Revoke silently skips it (Terraform auto-revokes +// ::/0 IPv6 egress and must not abort on no-op). type sgParseMode int const ( @@ -925,15 +906,9 @@ const ( sgParseRevoke ) -// ipPermissionsToSGRules converts AWS IpPermission slice to SGRule slice, validating -// every tenant-supplied CidrIp/SourceSG. This is the only path that constructs SGRule -// from external input — validating here makes it impossible for a future handler to -// bypass the check. -// -// Permissions whose only source is Ipv6Ranges contribute no rules. In -// sgParseAuthorize they cause an error; in sgParseRevoke they are skipped. -// Mixed v4+v6 permissions keep their v4/SG entries and silently drop the v6 -// part on revoke (no stored rule can match v6 anyway). +// ipPermissionsToSGRules converts AWS IpPermission slice to SGRule slice, +// validating every tenant-supplied CidrIp/SourceSG. IPv6-only permissions +// error in Authorize mode and are silently skipped in Revoke mode. func ipPermissionsToSGRules(perms []*ec2.IpPermission, mode sgParseMode) ([]SGRule, error) { var rules []SGRule for _, perm := range perms { @@ -1049,11 +1024,9 @@ func sgRulesToIpPermissions(rules []SGRule) []*ec2.IpPermission { return result } -// assertRulesPresent returns InvalidPermission.NotFound on the first revoke -// rule that does not match an existing rule. AWS rejects revoke of a -// non-existent rule rather than treating it as a no-op; matching that -// behaviour lets Terraform / SDK consumers distinguish "already gone" from -// "operator typo". +// assertRulesPresent returns InvalidPermission.NotFound if any rule in +// toRevoke is absent from existing, matching AWS's non-idempotent revoke +// behavior. func assertRulesPresent(existing, toRevoke []SGRule) error { if len(toRevoke) == 0 { return nil @@ -1091,23 +1064,17 @@ func sgRuleKey(r SGRule) string { return fmt.Sprintf("%s:%d:%d:%s:%s", r.IpProtocol, r.FromPort, r.ToPort, r.CidrIp, r.SourceSG) } -// vpcdSGEventTimeout bounds the synchronous handler↔vpcd round-trip for SG -// events (vpc.create-sg, vpc.delete-sg, vpc.update-sg, vpc.update-port-sgs). -// On timeout the caller surfaces the error to the API client; the periodic -// reconciler is the safety net that converges any drift afterwards. +// vpcdSGEventTimeout bounds the synchronous vpcd round-trip for SG events. const vpcdSGEventTimeout = 5 * time.Second // requestSGEvent sends a security group lifecycle event to vpcd via -// request-reply and waits for vpcd's success/error response. Used by the -// public SG API surface so vpcd-side ACL/port-group failures are visible to -// the caller instead of swallowed. +// request-reply and surfaces vpcd-side failures to the API caller. func (s *VPCServiceImpl) requestSGEvent(topic string, evt SGEvent) error { return utils.RequestEvent(s.natsConn, topic, evt, vpcdSGEventTimeout) } // createDefaultSecurityGroupInternal provisions the per-VPC default SG with -// AWS-equivalent rules: allow all inbound from self, allow all outbound to -// 0.0.0.0/0. Bypasses the public-API "default" reserved-name guard. Used by +// AWS-equivalent rules, bypassing the public-API reserved-name guard. Used by // CreateVpc and EnsureDefaultVPC. func (s *VPCServiceImpl) createDefaultSecurityGroupInternal(accountID, vpcId string) (string, error) { groupId := utils.GenerateResourceID("sg") diff --git a/spinifex/handlers/ec2/vpc/security_group_test.go b/spinifex/handlers/ec2/vpc/security_group_test.go index 4a5100ce9..688e07151 100644 --- a/spinifex/handlers/ec2/vpc/security_group_test.go +++ b/spinifex/handlers/ec2/vpc/security_group_test.go @@ -294,10 +294,9 @@ func TestRevokeSecurityGroupIngress_NotFound(t *testing.T) { assert.Error(t, err) } -// TestRevokeSecurityGroupIngress_RuleNotFound: revoking a rule that doesn't -// exist in the SG must return InvalidPermission.NotFound, matching AWS. -// Terraform / SDK consumers branch on this code to distinguish "already -// revoked, idempotent" from "operator typo". +// TestRevokeSecurityGroupIngress_RuleNotFound asserts that revoking a missing +// rule returns InvalidPermission.NotFound, matching AWS non-idempotent +// revoke behavior. func TestRevokeSecurityGroupIngress_RuleNotFound(t *testing.T) { svc := setupTestVPCService(t) vpcID := createTestVPC(t, svc, "10.0.0.0/16") @@ -347,11 +346,9 @@ func TestRevokeSecurityGroupEgress_NotFound(t *testing.T) { assert.Error(t, err) } -// TestRevokeSecurityGroupEgress_IPv6OnlyIsNoOp guards against regression of -// the Terraform AWS provider's auto-revoke of the default ::/0 IPv6 egress -// rule. Spinifex stores no IPv6 rules, so the call must succeed as a no-op -// instead of returning InvalidParameterValue (which surfaces as bare -// "UnknownError" to terraform and aborts every aws_security_group apply). +// TestRevokeSecurityGroupEgress_IPv6OnlyIsNoOp asserts that revoking a +// IPv6-only permission succeeds as a no-op; Terraform auto-revokes ::/0 IPv6 +// egress and must not abort when spinifex stores no IPv6 rules. func TestRevokeSecurityGroupEgress_IPv6OnlyIsNoOp(t *testing.T) { svc := setupTestVPCService(t) vpcID := createTestVPC(t, svc, "10.0.0.0/16") @@ -390,9 +387,8 @@ func TestRevokeSecurityGroupIngress_IPv6OnlyIsNoOp(t *testing.T) { require.True(t, aws.BoolValue(out.Return)) } -// TestAuthorizeSecurityGroupEgress_IPv6Rejected guards the inverse: authorize -// with IPv6 sources must still fail. Spinifex cannot enforce IPv6 — accepting -// the call would persist a rule OVN can never program. +// TestAuthorizeSecurityGroupEgress_IPv6Rejected asserts that authorizing +// IPv6 sources is rejected; spinifex cannot program IPv6 OVN ACLs. func TestAuthorizeSecurityGroupEgress_IPv6Rejected(t *testing.T) { svc := setupTestVPCService(t) vpcID := createTestVPC(t, svc, "10.0.0.0/16") @@ -484,9 +480,8 @@ func TestIpPermissionsToSGRules_SourceSG(t *testing.T) { assert.Equal(t, srcSG, rules[0].SourceSG) } -// TestIpPermissionsToSGRules_IPv6OnlyAuthorize rejects an authorize call whose -// only source is Ipv6Ranges. Spinifex's OVN ACL builder is IPv4-only — an IPv6 -// rule would persist as state OVN cannot program. +// TestIpPermissionsToSGRules_IPv6OnlyAuthorize rejects authorize calls whose +// only source is Ipv6Ranges (OVN ACL builder is IPv4-only). func TestIpPermissionsToSGRules_IPv6OnlyAuthorize(t *testing.T) { allProto := "-1" perms := []*ec2.IpPermission{ @@ -502,9 +497,8 @@ func TestIpPermissionsToSGRules_IPv6OnlyAuthorize(t *testing.T) { } // TestIpPermissionsToSGRules_IPv6OnlyRevoke tolerates IPv6-only permissions on -// revoke (returns zero rules, no error). The Terraform AWS provider sends -// exactly this shape when it auto-revokes the default ::/0 IPv6 egress rule -// after every aws_security_group create. +// revoke (returns zero rules, no error), matching Terraform's auto-revoke of +// the default ::/0 IPv6 egress rule. func TestIpPermissionsToSGRules_IPv6OnlyRevoke(t *testing.T) { allProto := "-1" perms := []*ec2.IpPermission{ @@ -520,9 +514,7 @@ func TestIpPermissionsToSGRules_IPv6OnlyRevoke(t *testing.T) { } // TestIpPermissionsToSGRules_MixedV4V6Revoke keeps the v4 rule and drops the -// v6 part on a mixed-source permission. Mixed perms are uncommon from real -// tooling but worth covering — IPv6 must never silently corrupt the IPv4 -// parse output. +// v6 part on a mixed-source permission on revoke. func TestIpPermissionsToSGRules_MixedV4V6Revoke(t *testing.T) { proto := "tcp" perms := []*ec2.IpPermission{ @@ -957,11 +949,8 @@ func TestDescribeSecurityGroups_FilterNoResults(t *testing.T) { // --- Phase 3: default SG lifecycle, dependency checks, quotas --- -// findDefaultSGInVPC returns the default-SG ID a CreateVpc call provisions. -// Walks the SG KV bucket and filters by IsDefault=true so a regression in -// the discriminator field (rather than the GroupName="default" convention) -// is caught — production code (FindDefaultSGForVPC, DeleteVpc cascade, -// DeleteSecurityGroup default guard) keys off IsDefault, not GroupName. +// findDefaultSGInVPC returns the default-SG ID for a VPC by filtering on +// IsDefault=true (production code keys off IsDefault, not GroupName). func findDefaultSGInVPC(t *testing.T, svc *VPCServiceImpl, vpcID string) string { t.Helper() keys, err := svc.sgKV.Keys() @@ -1290,10 +1279,8 @@ func TestAuthorizeSecurityGroupEgress_RuleLimit(t *testing.T) { func TestCreateSecurityGroup_VpcdError_RecordPersists(t *testing.T) { svc, _ := setupTestVPCServiceWithFailingVpcd(t, "forced-create-sg-error") - // Bootstrap a VPC with a working stub by swapping in a success responder - // for vpc.create-sg only during VPC creation. We do that by short-circuit: - // CreateVpc here will fail because it also publishes vpc.create-sg. - // Instead seed a VPC record directly so we can isolate the SG path. + // CreateVpc also publishes vpc.create-sg and would fail; seed the VPC + // record directly to isolate the SG path. seedAvailableVPC(t, svc, "vpc-fail000000000") _, err := svc.CreateSecurityGroup(&ec2.CreateSecurityGroupInput{ @@ -1397,11 +1384,8 @@ func TestRevokeSecurityGroupIngress_VpcdError_Propagated(t *testing.T) { assert.Contains(t, err.Error(), "forced-revoke-error") } -// TestAuthorizeSecurityGroupEgress_VpcdError_Propagated and the matching -// Revoke variant lock Phase 7's sync contract on the egress path. Without -// these, an OVN port-group failure during egress rule provisioning would be -// silently swallowed — the KV record would gain the rule but OVN would never -// enforce it. +// TestAuthorizeSecurityGroupEgress_VpcdError_Propagated asserts that vpcd +// failures on the egress path are surfaced to the caller. func TestAuthorizeSecurityGroupEgress_VpcdError_Propagated(t *testing.T) { svc, nc := setupTestVPCServiceWithNC(t) vpcID := createTestVPC(t, svc, "10.0.0.0/16") @@ -1444,9 +1428,8 @@ func TestRevokeSecurityGroupEgress_VpcdError_Propagated(t *testing.T) { assert.Contains(t, err.Error(), "forced-egress-revoke-error") } -// TestFindDefaultSGForVPC_NoMatch: an account with SGs in vpcA must return "" -// when asked for the default SG of vpcB. The "" return — not an error — -// signals "no default exists; create one" to RunInstances' fallback path. +// TestFindDefaultSGForVPC_NoMatch asserts that a lookup for vpcB's default SG +// returns "" (not an error) when only vpcA has SGs. func TestFindDefaultSGForVPC_NoMatch(t *testing.T) { svc := setupTestVPCService(t) vpcA := createTestVPC(t, svc, "10.0.0.0/16") @@ -1457,10 +1440,8 @@ func TestFindDefaultSGForVPC_NoMatch(t *testing.T) { assert.Equal(t, "", got, "no default SG → empty string, not error") } -// TestFindDefaultSGForVPC_SkipsMalformedRecord: a corrupt SG entry under the -// account prefix must be skipped, not crash the scan. Without this guard a -// single bad write to the SG bucket would brick the default-SG lookup -// cluster-wide. +// TestFindDefaultSGForVPC_SkipsMalformedRecord asserts that a corrupt SG entry +// is skipped rather than aborting the scan. func TestFindDefaultSGForVPC_SkipsMalformedRecord(t *testing.T) { svc := setupTestVPCService(t) vpcID := createTestVPC(t, svc, "10.0.0.0/16") @@ -1517,9 +1498,7 @@ func TestDeleteSecurityGroup_FailsClosedOnCorruptENI(t *testing.T) { // --- Helpers --- -// seedAvailableVPC writes a VPC record directly to KV in "available" state, -// bypassing CreateVpc (which itself depends on a working vpcd stub). Used to -// isolate non-CreateVpc paths in failing-vpcd tests. +// seedAvailableVPC writes a VPC record directly to KV, bypassing CreateVpc. func seedAvailableVPC(t *testing.T, svc *VPCServiceImpl, vpcID string) { t.Helper() rec := VPCRecord{VpcId: vpcID, CidrBlock: "10.0.0.0/16", State: "available"} @@ -1529,9 +1508,7 @@ func seedAvailableVPC(t *testing.T, svc *VPCServiceImpl, vpcID string) { require.NoError(t, err) } -// failingDeleteResponder layers a failing handler for vpc.delete-sg on top of -// the default success stubs (NATS picks the responder that subscribes to the -// queue group; here neither uses a queue, so we drain the success sub first). +// failingDeleteResponder overrides the vpc.delete-sg stub to return a failure. func failingDeleteResponder(t *testing.T, nc *nats.Conn, msg string) { t.Helper() testutil.OverrideVpcdStubResponse(nc, "vpc.delete-sg", []byte(`{"success":false,"error":"`+msg+`"}`)) @@ -1762,11 +1739,8 @@ func TestAuthorizeSecurityGroupIngress_AssignsRuleIDs(t *testing.T) { } } -// TestAuthorizeSecurityGroupIngress_RejectsContentDuplicate verifies the -// key-based dedup still catches duplicate-content rules now that RuleId is in -// the struct. The old slices.Contains check would have missed this because -// freshly-generated IDs make two otherwise-identical rules unequal under Go -// struct equality. +// TestAuthorizeSecurityGroupIngress_RejectsContentDuplicate verifies key-based +// dedup catches duplicate-content rules regardless of generated RuleId values. func TestAuthorizeSecurityGroupIngress_RejectsContentDuplicate(t *testing.T) { svc := setupTestVPCService(t) vpcID := createTestVPC(t, svc, "10.0.0.0/16") diff --git a/spinifex/handlers/ec2/vpc/service_impl.go b/spinifex/handlers/ec2/vpc/service_impl.go index 898ad64f0..2fde4e021 100644 --- a/spinifex/handlers/ec2/vpc/service_impl.go +++ b/spinifex/handlers/ec2/vpc/service_impl.go @@ -35,11 +35,8 @@ const ( KVBucketVNICounterVersion = 1 ) -// VPCRecord represents a stored VPC. AZ stamps the local availability zone -// at create time; the network/reconcile package filters its intent set by -// matching `vpc.AZ == localAZ` (with the legacy rule that empty matches -// every AZ for pre-Phase-2.2 records). New records always populate AZ from -// the daemon's config.AZ. +// VPCRecord represents a stored VPC. AZ is stamped at create time; empty AZ +// is a legacy value that matches every AZ in the reconciler. type VPCRecord struct { VpcId string `json:"vpc_id"` CidrBlock string `json:"cidr_block"` @@ -994,10 +991,8 @@ type BootstrapIDs struct { SubnetId string } -// EnsureDefaultVPC creates a default VPC and subnet if none exists for the given account. -// This matches AWS behavior where a default VPC is present on account creation. -// Safe to call multiple times — no-ops if a default VPC already exists. -// Returns the default VPC info (whether newly created or pre-existing). +// EnsureDefaultVPC creates a default VPC and subnet if none exists for the +// account. Safe to call multiple times — no-ops if already present. func (s *VPCServiceImpl) EnsureDefaultVPC(accountID string, bootstrap ...BootstrapIDs) (*DefaultVPCInfo, error) { if s.vpcKV == nil { return nil, nil // No persistence, skip @@ -1136,16 +1131,9 @@ func (s *VPCServiceImpl) EnsureDefaultVPC(accountID string, bootstrap ...Bootstr }, nil } -// createMainRouteTable writes a main route table record directly to the route table KV bucket. -// This avoids a circular import (routetable package imports vpc package for validation). -// -// Idempotency: when concurrent daemons race EnsureDefaultVPC for the same -// bootstrap-pinned VPC, both would otherwise mint a fresh random rtb-ID -// and Put it, leaving the KV with two IsMain=true records for one VPC. -// effectiveRouteTable then resolves the subnet's RT non-deterministically, -// and CreateRoute's post-write gate recompute can publish -// vpc.gate-subnet-egress against the orphan (route-less) main RT, installing -// a Priority-1100 DROP that kills egress on the default subnet. +// createMainRouteTable writes a main route table record directly to the route +// table KV bucket. Avoids a circular import; idempotent via findMainRouteTableID +// so concurrent EnsureDefaultVPC calls do not mint duplicate main RTs. func (s *VPCServiceImpl) createMainRouteTable(accountID, vpcID, vpcCidr string) error { if existing, err := s.findMainRouteTableID(accountID, vpcID); err != nil { return fmt.Errorf("check existing main route table: %w", err) diff --git a/spinifex/handlers/ec2/vpc/service_impl_test.go b/spinifex/handlers/ec2/vpc/service_impl_test.go index a556887ab..f0e081048 100644 --- a/spinifex/handlers/ec2/vpc/service_impl_test.go +++ b/spinifex/handlers/ec2/vpc/service_impl_test.go @@ -33,10 +33,8 @@ func setupTestVPCService(t *testing.T) *VPCServiceImpl { } // setupTestVPCServiceWithFailingVpcd creates a VPC service whose vpcd stub -// always returns success=false. Used by Phase 7 propagation tests to assert -// vpcd-side errors surface to the API caller. The default-VPC bootstrap is -// not needed here — these tests CreateVpc themselves once they have working -// stubs swapped in. +// always returns success=false. Used to assert vpcd-side errors surface to the +// API caller. func setupTestVPCServiceWithFailingVpcd(t *testing.T, errMsg string) (*VPCServiceImpl, *nats.Conn) { t.Helper() _, nc, _ := testutil.StartTestJetStream(t) @@ -597,11 +595,8 @@ func TestEnsureDefaultVPC_SkipsWhenDefaultExists(t *testing.T) { } // TestCreateMainRouteTable_Idempotent asserts a second createMainRouteTable -// call for the same VPC is a no-op. Without this guard, concurrent multi-node -// EnsureDefaultVPC calls (admin account, bootstrap-pinned VpcId) each Put a -// random rtb-ID with IsMain=true, and the resulting duplicate causes -// CreateRoute's post-write gate recompute to resolve effective RT to the -// orphan (route-less) record and publish vpc.gate-subnet-egress. +// call for the same VPC is a no-op. Concurrent calls otherwise create duplicate +// IsMain=true records, corrupting route-table resolution. func TestCreateMainRouteTable_Idempotent(t *testing.T) { svc := setupTestVPCService(t) vpcID := createTestVPC(t, svc, "10.99.0.0/16") // CreateVpc auto-calls createMainRouteTable @@ -649,11 +644,8 @@ func countMainRouteTablesForVPC(t *testing.T, svc *VPCServiceImpl, vpcID string) } // TestEnsureDefaultVPC_NoVpcdResponder simulates the daemon-startup race where -// EnsureDefaultVPC runs before vpcd has subscribed to vpc.create-sg. Without -// the fix, the synchronous SG round-trip errors out and EnsureDefaultVPC -// returns early, leaving the VPC without a default subnet or main route table. -// After the fix the SG step is best-effort, so subnet + RTB still land in KV -// and vpcd's reconcile-sgs loop converges the OVN port group asynchronously. +// EnsureDefaultVPC runs before vpcd has subscribed. The SG step is best-effort, +// so subnet and RTB must still land in KV when vpcd is absent. func TestEnsureDefaultVPC_NoVpcdResponder(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) svc, err := NewVPCServiceImplWithNATS(nil, nc) diff --git a/spinifex/handlers/ec2/vpc/service_nats_test.go b/spinifex/handlers/ec2/vpc/service_nats_test.go index f8df6c8f1..f8960c3dd 100644 --- a/spinifex/handlers/ec2/vpc/service_nats_test.go +++ b/spinifex/handlers/ec2/vpc/service_nats_test.go @@ -207,11 +207,8 @@ func TestNATSVPCService_Timeout(t *testing.T) { _, nc := setupTestVPCServiceWithNC(t) client := &NATSVPCService{natsConn: nc} - // Use a very short timeout to make the test fast - // We can't easily override the 30s timeout in the production code, - // so instead test with a topic that has no subscriber. - // The standard NATSVPCService uses 30s which is too long for tests. - // Just verify the constructor works. + // Production code uses a 30s timeout; just verify the constructor works + // without exercising the full round-trip. _ = client _ = time.Second // reference time package } diff --git a/spinifex/handlers/eks/access_entry.go b/spinifex/handlers/eks/access_entry.go index 736014afc..894de52fe 100644 --- a/spinifex/handlers/eks/access_entry.go +++ b/spinifex/handlers/eks/access_entry.go @@ -12,10 +12,9 @@ import ( "github.com/nats-io/nats.go" ) -// Access-entry types (Q9). v1 supports STANDARD principals only; the node -// types (EC2_LINUX/EC2_WINDOWS/FARGATE_LINUX) are accepted by AWS but the -// nodegroup join path (Sprint 6d) handles node identity, so they are rejected -// here until that lands. +// Access-entry types. v1 supports STANDARD principals only; node types +// (EC2_LINUX/EC2_WINDOWS/FARGATE_LINUX) are rejected until the nodegroup +// join path wires them. const ( AccessEntryTypeStandard = "STANDARD" @@ -23,9 +22,8 @@ const ( accessScopeNamespace = "namespace" ) -// Supported AWS-managed access policies (Q9). The map value is the K8s -// ClusterRole the policy maps onto; the token webhook + RBAC bindings consume -// it. Any policy ARN outside this set is rejected with InvalidParameterValue. +// supportedAccessPolicies maps each supported AWS-managed policy ARN to the +// K8s ClusterRole it grants. Policy ARNs outside this set are rejected. var supportedAccessPolicies = map[string]string{ "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy": "cluster-admin", "arn:aws:eks::aws:cluster-access-policy/AmazonEKSAdminPolicy": "admin", @@ -33,15 +31,13 @@ var supportedAccessPolicies = map[string]string{ "arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy": "view", } -// ErrAccessEntryNotFound is returned by GetAccessEntryRecord / the CAS helper -// when no entry exists for the principal. Callers translate to the AWS shape -// (ResourceNotFoundException) at the service boundary. +// ErrAccessEntryNotFound is returned when no entry exists for the principal. +// Callers translate it to ResourceNotFoundException at the service boundary. var ErrAccessEntryNotFound = errors.New("eks: access entry not found") -// AccessEntryARN composes the access-entry ARN. AWS uses a per-entry UUID; this -// build keys the discriminator off the principal-ARN hash instead so the ARN is -// deterministic (no RNG) and one-per-principal — clients address access entries -// by principalArn, not by this ARN, so the divergence is informational only. +// AccessEntryARN composes the access-entry ARN, keying the discriminator off +// the principal-ARN hash for determinism. Clients address entries by +// principalArn, not this ARN, so the divergence from AWS's UUID is informational. func AccessEntryARN(region, accountID, cluster, principalARN string) string { return fmt.Sprintf("arn:aws:eks:%s:%s:access-entry/%s/%s", region, accountID, cluster, PrincipalARNHash(principalARN)) @@ -85,8 +81,7 @@ func GetAccessEntryRecord(kv nats.KeyValue, cluster, principalARN string) (*Acce return &rec, nil } -// ListAccessEntryRecords returns every access entry under a cluster, sorted by -// principal ARN for stable output. +// ListAccessEntryRecords returns all access entries under a cluster, sorted by principal ARN. func ListAccessEntryRecords(kv nats.KeyValue, cluster string) ([]*AccessEntryRecord, error) { if cluster == "" { return nil, errors.New("eks: ListAccessEntryRecords empty cluster") @@ -121,8 +116,7 @@ func ListAccessEntryRecords(kv nats.KeyValue, cluster string) ([]*AccessEntryRec return out, nil } -// DeleteAccessEntryRecord removes one entry. Returns ErrAccessEntryNotFound if -// it did not exist so DeleteAccessEntry can surface ResourceNotFoundException. +// DeleteAccessEntryRecord removes one entry; returns ErrAccessEntryNotFound if absent. func DeleteAccessEntryRecord(kv nats.KeyValue, cluster, principalARN string) error { key := AccessEntryKey(cluster, principalARN) if _, err := kv.Get(key); err != nil { @@ -137,8 +131,8 @@ func DeleteAccessEntryRecord(kv nats.KeyValue, cluster, principalARN string) err return nil } -// casUpdateAccessEntry does a revision-checked read-modify-write. mutate returns -// true if it changed a field. Returns ErrAccessEntryNotFound if absent. +// casUpdateAccessEntry does a revision-checked read-modify-write. +// mutate returns true when a field changed. Returns ErrAccessEntryNotFound if absent. func casUpdateAccessEntry(kv nats.KeyValue, cluster, principalARN string, mutate func(*AccessEntryRecord) bool) (*AccessEntryRecord, error) { key := AccessEntryKey(cluster, principalARN) for range maxClusterStateCASRetries { @@ -180,9 +174,8 @@ func sortAccessEntries(recs []*AccessEntryRecord) { } } -// validateAccessScope checks an AccessScope per AWS rules: type ∈ -// {cluster, namespace}; namespace scope requires at least one namespace and -// cluster scope must not carry any. +// validateAccessScope validates an AccessScope: type must be "cluster" or +// "namespace"; namespace scope requires ≥1 namespace, cluster scope must have none. func validateAccessScope(scope *eks.AccessScope) (AccessScope, error) { if scope == nil || aws.StringValue(scope.Type) == "" { return AccessScope{}, errors.New("eks: accessScope.type is required") @@ -237,14 +230,12 @@ func associatedPolicyToAWS(p AssociatedAccessPolicy) *eks.AssociatedAccessPolicy } } -// defaultKubernetesUsername mirrors AWS: when the caller omits username, EKS -// derives it from the principal ARN. +// defaultKubernetesUsername mirrors AWS behaviour: username defaults to principalARN when omitted. func defaultKubernetesUsername(principalARN string) string { return principalARN } -// newAccessEntryRecord builds a record, defaulting the username to the -// principal ARN when unset (AWS behaviour). now stamps Created/Modified. +// newAccessEntryRecord builds a record; defaults username to principalARN when unset. func newAccessEntryRecord(region, accountID, cluster, principalARN, username string, groups []string, entryType string, tags map[string]string, now time.Time) *AccessEntryRecord { if username == "" { username = defaultKubernetesUsername(principalARN) diff --git a/spinifex/handlers/eks/addon_catalog.go b/spinifex/handlers/eks/addon_catalog.go index e9c30d989..50aa2eb5a 100644 --- a/spinifex/handlers/eks/addon_catalog.go +++ b/spinifex/handlers/eks/addon_catalog.go @@ -2,26 +2,20 @@ package handlers_eks import "slices" -// AddonSpec describes one Spinifex-supported managed add-on. The catalog is a -// static in-binary registry (air-gapped; there is no internet add-on catalog), -// so DescribeAddonVersions and CreateAddon validate against these entries. +// AddonSpec describes one Spinifex-supported managed add-on in the static +// in-binary catalog. DescribeAddonVersions and CreateAddon validate against it. type AddonSpec struct { // Name is the AWS add-on name (e.g. "aws-load-balancer-controller"). Name string - // Versions lists the supported versions newest-first; [0] is the default. - Versions []string - // DefaultVersion is the version CreateAddon applies when none is requested. - // Always equal to Versions[0]. + // Versions lists supported versions newest-first; [0] is the default. + Versions []string DefaultVersion string - // RequiresIRSA reports whether the add-on needs an IAM-role-for-service- - // account binding to operate (surfaced as requiresIamPermissions). + // RequiresIRSA indicates the add-on needs an IAM role for service-account binding. RequiresIRSA bool - // Description is a short human-readable summary. - Description string + Description string } -// addonCatalog is the bundled set of supported add-ons. Keep newest version -// first in each Versions slice; newAddonSpec enforces DefaultVersion = [0]. +// addonCatalog is the bundled add-on registry. Keep newest version first per slice. var addonCatalog = buildAddonCatalog( newAddonSpec("aws-load-balancer-controller", true, "Provisions ELBv2 load balancers for Kubernetes Service/Ingress resources.", @@ -31,8 +25,7 @@ var addonCatalog = buildAddonCatalog( "1.11.1"), ) -// newAddonSpec builds a spec, pinning DefaultVersion to the first (newest) -// version. Panics on an empty version list — the catalog is build-time data. +// newAddonSpec builds a spec with DefaultVersion = versions[0]. Panics on empty versions. func newAddonSpec(name string, requiresIRSA bool, description string, versions ...string) AddonSpec { if len(versions) == 0 { panic("eks: addon spec " + name + " has no versions") diff --git a/spinifex/handlers/eks/addon_installer.go b/spinifex/handlers/eks/addon_installer.go index e181bf6ac..b75ff78c3 100644 --- a/spinifex/handlers/eks/addon_installer.go +++ b/spinifex/handlers/eks/addon_installer.go @@ -8,27 +8,19 @@ import ( "github.com/nats-io/nats.go" ) -// AddonInstaller delivers (and removes) a managed add-on's Kubernetes manifests -// to a cluster. It is deliberately transport-agnostic: the add-on state machine -// depends only on this narrow contract so the VM-side delivery transport (which -// drops manifests into the K3s server auto-deploy dir) can drop in without -// touching the op layer. -// -// The reachability constraint (cluster_reconciler.go:73-79) means delivery must -// be VM-side, not a daemon apiserver apply — a private cluster's apiserver is -// not host-reachable. +// AddonInstaller delivers and removes a managed add-on's Kubernetes manifests. +// Transport-agnostic: delivery must be VM-side because a private cluster's +// apiserver is not host-reachable. type AddonInstaller interface { - // Install renders and delivers the add-on described by rec. It does not flip - // the record to ACTIVE — that transition is gated on the cluster's state - // report confirming the manifest applied and pods are ready. + // Install stages the add-on manifest. Does not transition to ACTIVE — that + // is gated on the cluster's state report confirming pods are ready. Install(accountID, cluster string, rec *AddonRecord) error - // Uninstall removes the add-on's delivered manifests from the cluster. + // Uninstall removes the add-on's manifests from the cluster. Uninstall(accountID, cluster, addon string) error } -// stagedManifest is the artifact the VM-side delivery transport consumes. It -// names the bundled add-on + version and carries the operator-supplied config -// so the VM can render the final Kubernetes objects from the baked manifests. +// stagedManifest is consumed by the VM-side delivery transport: add-on name, +// version, and operator-supplied config for rendering the Kubernetes objects. type stagedManifest struct { AddonName string `json:"addonName"` AddonVersion string `json:"addonVersion"` @@ -36,19 +28,16 @@ type stagedManifest struct { ConfigurationValues string `json:"configurationValues,omitempty"` } -// stagingInstaller renders the bundled manifest descriptor for an add-on and -// stages it in the per-account KV at AddonManifestKey. The VM-side delivery -// slice (separate bead) pulls staged manifests and applies them via the K3s -// auto-deploy dir; until then an installed add-on sits CREATING — honest, not a -// false ACTIVE. +// stagingInstaller stages the manifest descriptor in KV at AddonManifestKey. +// The VM-side delivery slice applies it via the K3s auto-deploy dir; until then +// the add-on sits CREATING. type stagingInstaller struct { nc *nats.Conn } var _ AddonInstaller = (*stagingInstaller)(nil) -// newStagingInstaller builds the default installer bound to the daemon's NATS -// connection. +// newStagingInstaller returns a stagingInstaller bound to the daemon NATS connection. func newStagingInstaller(nc *nats.Conn) *stagingInstaller { return &stagingInstaller{nc: nc} } diff --git a/spinifex/handlers/eks/addon_state.go b/spinifex/handlers/eks/addon_state.go index e9467dc14..cefe57c52 100644 --- a/spinifex/handlers/eks/addon_state.go +++ b/spinifex/handlers/eks/addon_state.go @@ -10,8 +10,7 @@ import ( "github.com/nats-io/nats.go" ) -// AddonStatus mirrors the AWS EKS addon.status enum verbatim so clients match -// against the values without translation. +// AddonStatus mirrors the AWS EKS addon.status enum verbatim. type AddonStatus string const ( @@ -23,9 +22,8 @@ const ( AddonStatusCreateFailed AddonStatus = "CREATE_FAILED" ) -// AddonRecord is the persisted-state envelope for a managed add-on bound to a -// cluster. It tracks the AWS-visible lifecycle status plus the IRSA role and -// opaque configuration the installer needs. +// AddonRecord is the persisted state for a managed add-on: lifecycle status, +// IRSA role, and opaque configuration the installer needs. type AddonRecord struct { AddonName string `json:"addonName"` AddonVersion string `json:"addonVersion"` @@ -39,13 +37,11 @@ type AddonRecord struct { ModifiedAt time.Time `json:"modifiedAt"` } -// ErrAddonNotFound is returned by GetAddonRecord / the CAS helper when no -// record exists for the add-on. Callers translate to the AWS shape -// (ResourceNotFoundException) at the service boundary. +// ErrAddonNotFound is returned when no record exists for the add-on. +// Callers translate it to ResourceNotFoundException at the service boundary. var ErrAddonNotFound = errors.New("eks: addon not found") -// AddonARN composes the deterministic ARN for a managed add-on (one per -// add-on name per cluster). +// AddonARN composes the deterministic ARN for a managed add-on. func AddonARN(region, accountID, cluster, addon string) string { return fmt.Sprintf("arn:aws:eks:%s:%s:addon/%s/%s", region, accountID, cluster, addon) } @@ -88,10 +84,8 @@ func GetAddonRecord(kv nats.KeyValue, cluster, addon string) (*AddonRecord, erro return &rec, nil } -// ListAddonRecords returns every add-on record under a cluster, sorted by add-on -// name for stable output. The staged-manifest sub-keys -// (clusters/{c}/addons/{addon}/manifest) are skipped — only the record keys -// (one path segment under the prefix) are returned. +// ListAddonRecords returns all add-on records under a cluster, sorted by name. +// Staged-manifest sub-keys (one extra path segment) are skipped. func ListAddonRecords(kv nats.KeyValue, cluster string) ([]*AddonRecord, error) { if cluster == "" { return nil, errors.New("eks: ListAddonRecords empty cluster") @@ -109,8 +103,7 @@ func ListAddonRecords(kv nats.KeyValue, cluster string) ([]*AddonRecord, error) if !strings.HasPrefix(k, prefix) { continue } - // Skip sub-keys (e.g. the staged manifest); a record key is exactly one - // segment under the prefix. + // Skip sub-keys (e.g. staged manifest); record keys are one segment under the prefix. if strings.Contains(strings.TrimPrefix(k, prefix), "/") { continue } @@ -131,8 +124,7 @@ func ListAddonRecords(kv nats.KeyValue, cluster string) ([]*AddonRecord, error) return out, nil } -// DeleteAddonRecord removes one record (and its staged manifest, if any). -// Returns ErrAddonNotFound if the record did not exist. +// DeleteAddonRecord removes one record and its staged manifest. Returns ErrAddonNotFound if absent. func DeleteAddonRecord(kv nats.KeyValue, cluster, addon string) error { key := AddonKey(cluster, addon) if _, err := kv.Get(key); err != nil { @@ -152,8 +144,8 @@ func DeleteAddonRecord(kv nats.KeyValue, cluster, addon string) error { return nil } -// casUpdateAddon does a revision-checked read-modify-write. mutate returns true -// if it changed a field. Returns ErrAddonNotFound if absent. +// casUpdateAddon does a revision-checked read-modify-write. +// mutate returns true when a field changed. Returns ErrAddonNotFound if absent. func casUpdateAddon(kv nats.KeyValue, cluster, addon string, mutate func(*AddonRecord) bool) (*AddonRecord, error) { key := AddonKey(cluster, addon) for range maxClusterStateCASRetries { diff --git a/spinifex/handlers/eks/addons.go b/spinifex/handlers/eks/addons.go index 8fcb1e0db..c887adc02 100644 --- a/spinifex/handlers/eks/addons.go +++ b/spinifex/handlers/eks/addons.go @@ -32,9 +32,7 @@ func (s *EKSServiceImpl) ListAddons(input *eks.ListAddonsInput, accountID string return &eks.ListAddonsOutput{Addons: aws.StringSlice(names)}, nil } -// DescribeAddonVersions returns the bundled add-on catalog, optionally filtered -// to a single add-on by name. It is cluster-independent (the catalog is static) -// so it does not touch the KV store. +// DescribeAddonVersions returns the static add-on catalog, optionally filtered by name. func (s *EKSServiceImpl) DescribeAddonVersions(input *eks.DescribeAddonVersionsInput, _ string) (*eks.DescribeAddonVersionsOutput, error) { filter := "" if input != nil { @@ -51,9 +49,8 @@ func (s *EKSServiceImpl) DescribeAddonVersions(input *eks.DescribeAddonVersionsI return &eks.DescribeAddonVersionsOutput{Addons: out}, nil } -// CreateAddon validates the add-on against the bundled catalog, persists a -// CREATING record, and stages it for delivery. The record reaches ACTIVE only -// once the cluster's state report confirms delivery (separate slice). +// CreateAddon validates, persists a CREATING record, and stages it for delivery. +// Transitions to ACTIVE once the cluster state report confirms delivery. func (s *EKSServiceImpl) CreateAddon(input *eks.CreateAddonInput, accountID string) (*eks.CreateAddonOutput, error) { if input == nil { return nil, errors.New(awserrors.ErrorInvalidParameterValue) @@ -125,8 +122,7 @@ func (s *EKSServiceImpl) DescribeAddon(input *eks.DescribeAddonInput, accountID return &eks.DescribeAddonOutput{Addon: addonRecordToAWS(cluster, rec)}, nil } -// UpdateAddon CASes the version/config/role onto the record, marks it UPDATING, -// and re-stages it for delivery. +// UpdateAddon CASes new version/config/role onto the record, marks it UPDATING, and re-stages it. func (s *EKSServiceImpl) UpdateAddon(input *eks.UpdateAddonInput, accountID string) (*eks.UpdateAddonOutput, error) { if input == nil { return nil, errors.New(awserrors.ErrorInvalidParameterValue) @@ -177,8 +173,7 @@ func (s *EKSServiceImpl) UpdateAddon(input *eks.UpdateAddonInput, accountID stri }}, nil } -// DeleteAddon marks the record DELETING, removes the staged manifest, and -// deletes the record. +// DeleteAddon marks the record DELETING, removes the staged manifest, then deletes the record. func (s *EKSServiceImpl) DeleteAddon(input *eks.DeleteAddonInput, accountID string) (*eks.DeleteAddonOutput, error) { if input == nil { return nil, errors.New(awserrors.ErrorInvalidParameterValue) @@ -239,8 +234,7 @@ func addonRecordToAWS(cluster string, rec *AddonRecord) *eks.Addon { return out } -// addonSpecToAWS converts a catalog spec to the SDK AddonInfo shape, expanding -// each supported version into an AddonVersionInfo. +// addonSpecToAWS converts a catalog spec to the SDK AddonInfo shape. func addonSpecToAWS(spec AddonSpec) *eks.AddonInfo { versions := make([]*eks.AddonVersionInfo, 0, len(spec.Versions)) for _, v := range spec.Versions { @@ -255,8 +249,7 @@ func addonSpecToAWS(spec AddonSpec) *eks.AddonInfo { } } -// addonInstaller returns the wired installer, defaulting to the staging -// installer bound to the daemon NATS connection when none was injected. +// addonInstaller returns the injected installer or the default stagingInstaller. func (s *EKSServiceImpl) addonInstaller() AddonInstaller { if s.deps.AddonInstaller != nil { return s.deps.AddonInstaller @@ -264,8 +257,7 @@ func (s *EKSServiceImpl) addonInstaller() AddonInstaller { return newStagingInstaller(s.deps.NATSConn) } -// markAddonFailed best-effort flips a record to CREATE_FAILED with the reason -// after an installer error so a failed install is not left looking healthy. +// markAddonFailed best-effort flips a record to CREATE_FAILED with the error reason. func (s *EKSServiceImpl) markAddonFailed(acctKV nats.KeyValue, cluster, addon string, cause error) { now := time.Now().UTC() if _, err := casUpdateAddon(acctKV, cluster, addon, func(r *AddonRecord) bool { diff --git a/spinifex/handlers/eks/awsgw_endpoint.go b/spinifex/handlers/eks/awsgw_endpoint.go index 4031512f3..bd4b0e916 100644 --- a/spinifex/handlers/eks/awsgw_endpoint.go +++ b/spinifex/handlers/eks/awsgw_endpoint.go @@ -7,16 +7,11 @@ import ( "strings" ) -// ClusterOIDCIssuer returns the public OIDC issuer URL for one cluster. This -// is the URL kube-apiserver advertises as service-account-issuer and the URL -// that AWS STS / external IRSA validators fetch /.well-known/openid-configuration -// against. Format per plan §10.6: +// ClusterOIDCIssuer returns the public OIDC issuer URL: // // {gatewayBaseURL}/oidc/eks/{region}/{accountID}/{clusterName} // -// gatewayBaseURL is the awsgw HTTPS endpoint (e.g. "https://10.0.0.1:9999"), -// without a trailing /oidc. Validates that region/accountID/clusterName are -// non-empty and URL-safe. +// gatewayBaseURL is the awsgw HTTPS endpoint without a trailing slash. func ClusterOIDCIssuer(gatewayBaseURL, region, accountID, clusterName string) (string, error) { if gatewayBaseURL == "" { return "", errors.New("eks: ClusterOIDCIssuer empty gatewayBaseURL") diff --git a/spinifex/handlers/eks/claim.go b/spinifex/handlers/eks/claim.go index 51ecb8a22..e9468d5ca 100644 --- a/spinifex/handlers/eks/claim.go +++ b/spinifex/handlers/eks/claim.go @@ -8,15 +8,11 @@ import ( ) // claimKey atomically creates key with payload. owned is true when this caller -// created it (won the claim). On an existing key it returns owned=false with the -// current value + revision so the caller can decide reclaim-vs-conflict; a key -// that vanished between the Create and the Get returns owned=false, existing=nil -// so the caller can retry the claim. -// -// This is the shared idempotency primitive for create handlers: the first -// caller to Create a resource's identity key wins and does the launching work; -// a duplicate (SDK retry, gateway re-publish that spawned a second handler) -// loses and never launches, so concurrent creates cannot double-launch + leak. +// wins the CAS-create. On an existing key it returns owned=false with the +// current value + revision; a key that vanished between Create and Get returns +// owned=false, existing=nil so the caller can retry. +// This is the shared idempotency primitive: the first caller wins and launches; +// duplicates (SDK retries, re-published handlers) lose and never launch. func claimKey(kv nats.KeyValue, key string, payload []byte) (owned bool, existing []byte, rev uint64, err error) { if _, cerr := kv.Create(key, payload); cerr == nil { return true, nil, 0, nil diff --git a/spinifex/handlers/eks/cluster_reconciler.go b/spinifex/handlers/eks/cluster_reconciler.go index 3f0d8edd2..f49fbc3c9 100644 --- a/spinifex/handlers/eks/cluster_reconciler.go +++ b/spinifex/handlers/eks/cluster_reconciler.go @@ -18,44 +18,36 @@ const ( defaultReconcileInterval = 30 * time.Second defaultHealthzTimeout = 5 * time.Second // defaultCreateTimeout bounds how long a cluster may sit in CREATING before - // the reconciler gives up and marks it FAILED. Measured from meta.CreatedAt - // (absolute), so a VM that never boots reaches a terminal state even across - // daemon restarts. Plan Risk #1. + // being marked FAILED. Measured from meta.CreatedAt so a VM that never boots + // reaches a terminal state even across daemon restarts. defaultCreateTimeout = 15 * time.Minute - // defaultStateStaleAfter bounds how old the control plane's last NATS state - // report may be before health is treated as unknown/failed. The CP publishes - // every ~30s; 3x that tolerates a missed tick without flapping. + // defaultStateStaleAfter is the maximum age of a CP state report before + // health is treated as unknown. CP publishes every ~30s; 3× tolerates a missed tick. defaultStateStaleAfter = 90 * time.Second ) -// natsSubscriber is the minimum surface the reconciler needs to consume the -// control-plane state report; *nats.Conn satisfies it, tests stub it. +// natsSubscriber is the minimal subscribe surface the reconciler needs; *nats.Conn satisfies it. type natsSubscriber interface { Subscribe(subject string, cb nats.MsgHandler) (*nats.Subscription, error) } -// ErrReconcilerLeaseLost is returned by Run when the leader lease was lost -// mid-loop. Callers can re-Acquire and re-Run if they want to keep driving -// reconcile from the same node. +// ErrReconcilerLeaseLost is returned by Run when the leader lease is lost mid-loop. var ErrReconcilerLeaseLost = errors.New("eks: reconciler lease lost") -// ErrReconcilerClusterDeleting is returned by Run when meta.Status is -// observed as DELETING; the DeleteCluster service-impl path is responsible -// for teardown, so the reconciler exits without further work. +// ErrReconcilerClusterDeleting is returned by Run when meta.Status is DELETING; +// DeleteCluster owns teardown so the reconciler exits. var ErrReconcilerClusterDeleting = errors.New("eks: cluster deleting") // ErrReconcilerClusterFailed is returned by Run when meta.Status is FAILED. var ErrReconcilerClusterFailed = errors.New("eks: cluster failed") -// HTTPDoer is the minimum http.Client surface the reconciler needs for -// /healthz probes; tests stub it without a TLS server. +// HTTPDoer is the minimal http.Client surface needed for /healthz probes. type HTTPDoer interface { Do(req *http.Request) (*http.Response, error) } -// ClusterReconciler drives one cluster's CREATING → ACTIVE → DELETING -// lifecycle under a leader lease. One instance per (accountID, clusterName); -// the leader lease arbitrates which spinifex node actually drives reconcile. +// ClusterReconciler drives one cluster's CREATING → ACTIVE → DELETING lifecycle +// under a leader lease. One instance per (accountID, clusterName). type ClusterReconciler struct { leaderKV nats.KeyValue acctKV nats.KeyValue @@ -70,17 +62,15 @@ type ClusterReconciler struct { createTimeout time.Duration httpClient HTTPDoer - // State-report health source. When stateSub is non-nil the reconciler gates - // on the CP's NATS self-report (latest) rather than the HTTP /healthz probe — - // the apiserver is bound to the VPC node-ip and unreachable from the host. + // When stateSub is non-nil, health is gated on the CP's NATS self-report + // rather than the HTTP /healthz probe (apiserver is VPC-only, host-unreachable). stateSub natsSubscriber stateSubject string stateStaleAfter time.Duration latest atomic.Pointer[ServerStateReport] } -// ReconcilerOption tunes a ClusterReconciler. Tests use the With* helpers to -// shrink tickers and inject a stub HTTP client. +// ReconcilerOption tunes a ClusterReconciler. type ReconcilerOption func(*ClusterReconciler) // WithReconcileInterval overrides the default reconcile loop period. @@ -88,8 +78,7 @@ func WithReconcileInterval(d time.Duration) ReconcilerOption { return func(r *ClusterReconciler) { r.interval = d } } -// WithLeaseRefresh overrides the default leader-lease refresh period. Should -// be less than KVBucketEKSLeaderTTL. +// WithLeaseRefresh overrides the leader-lease refresh period (should be < KVBucketEKSLeaderTTL). func WithLeaseRefresh(d time.Duration) ReconcilerOption { return func(r *ClusterReconciler) { r.leaseRefresh = d } } @@ -99,8 +88,7 @@ func WithHealthzTimeout(d time.Duration) ReconcilerOption { return func(r *ClusterReconciler) { r.healthTimeout = d } } -// WithCreateTimeout overrides how long a cluster may remain CREATING before the -// reconciler marks it FAILED. +// WithCreateTimeout overrides the CREATING→FAILED timeout. func WithCreateTimeout(d time.Duration) ReconcilerOption { return func(r *ClusterReconciler) { r.createTimeout = d } } @@ -110,9 +98,8 @@ func WithHTTPClient(c HTTPDoer) ReconcilerOption { return func(r *ClusterReconciler) { r.httpClient = c } } -// WithStateSource configures the reconciler to gate health on the control -// plane's NATS state report (subject) consumed via sub, instead of the HTTP -// /healthz probe. The subscription is opened in Run and closed when it returns. +// WithStateSource gates health on the CP's NATS state report instead of an +// HTTP /healthz probe. The subscription is opened in Run and closed on return. func WithStateSource(sub natsSubscriber, subject string) ReconcilerOption { return func(r *ClusterReconciler) { r.stateSub = sub @@ -120,15 +107,13 @@ func WithStateSource(sub natsSubscriber, subject string) ReconcilerOption { } } -// WithStateStaleAfter overrides how old the last state report may be before -// health is treated as failing. +// WithStateStaleAfter overrides the maximum age of a state report before health is treated as failing. func WithStateStaleAfter(d time.Duration) ReconcilerOption { return func(r *ClusterReconciler) { r.stateStaleAfter = d } } -// NewClusterReconciler constructs a ClusterReconciler. healthURL is the -// fully-qualified probe target (e.g. https://nlb-dns:443/healthz) or empty -// to skip /healthz probing (CREATING transitions on bootstrap state alone). +// NewClusterReconciler constructs a ClusterReconciler. healthURL is the probe +// target or empty to skip /healthz probing (bootstrap state alone drives transitions). func NewClusterReconciler(leaderKV, acctKV nats.KeyValue, accountID, clusterName, holderID, healthURL string, opts ...ReconcilerOption) (*ClusterReconciler, error) { if leaderKV == nil { return nil, errors.New("eks: NewClusterReconciler nil leaderKV") @@ -177,9 +162,8 @@ func reconcilerLeaderKey(accountID, clusterName string) string { } // AcquireLease attempts a CAS Create on the leader bucket. Returns -// (release, true) if this caller is now the leader; (nil, false) otherwise. -// The release closure performs a best-effort kv.Delete; the bucket's 60s -// TTL reaps stale leases when release does not run (crashed leader). +// (release, true) on success; (nil, false) otherwise. The bucket TTL reaps +// stale leases when release does not run. func (r *ClusterReconciler) AcquireLease() (func(), bool) { key := reconcilerLeaderKey(r.accountID, r.clusterName) if _, err := r.leaderKV.Create(key, []byte(r.holderID)); err != nil { @@ -196,8 +180,7 @@ func (r *ClusterReconciler) AcquireLease() (func(), bool) { }, true } -// RefreshLease re-asserts ownership via a CAS update on the leader key. -// Returns false if the key was taken by another holder or the CAS conflicts. +// RefreshLease re-asserts ownership via a CAS update. Returns false if another holder won. func (r *ClusterReconciler) RefreshLease() bool { key := reconcilerLeaderKey(r.accountID, r.clusterName) entry, err := r.leaderKV.Get(key) @@ -218,9 +201,8 @@ func (r *ClusterReconciler) RefreshLease() bool { return true } -// Run drives reconcile until ctx is cancelled, the lease is lost, or the -// cluster reaches a terminal state. Caller is expected to AcquireLease first; -// Run does not re-acquire. +// Run drives reconcile until ctx is cancelled, the lease is lost, or the cluster +// reaches a terminal state. Caller must AcquireLease first. func (r *ClusterReconciler) Run(ctx context.Context) error { if r.stateSub != nil && r.stateSubject != "" { sub, err := r.stateSub.Subscribe(r.stateSubject, func(m *nats.Msg) { @@ -327,10 +309,8 @@ func (r *ClusterReconciler) reconcileOnce(ctx context.Context) error { return nil } -// observe returns the current health issue ("" when healthy) and the reported -// node count. With a NATS state source it reads the control plane's latest -// self-report (rejecting a stale or unhealthy one); otherwise it falls back to -// the HTTP /healthz probe (node count is then 0, the probe carries no count). +// observe returns the health issue ("" = healthy) and node count from the CP's +// latest NATS self-report, or from the HTTP /healthz probe when no NATS source is set. func (r *ClusterReconciler) observe(ctx context.Context) (issue string, nodeCount int) { if r.stateSub != nil && r.stateSubject != "" { report := r.latest.Load() @@ -352,11 +332,9 @@ func (r *ClusterReconciler) observe(ctx context.Context) (issue string, nodeCoun return "", 0 } -// failIfCreateTimedOut transitions a still-CREATING cluster to FAILED once it -// has exceeded createTimeout since meta.CreatedAt, and returns -// ErrReconcilerClusterFailed so Run exits the loop. While still within the -// window it returns nil so the next tick retries. A zero or absent CreatedAt -// disables the deadline (no false-positive fail on a malformed record). +// failIfCreateTimedOut marks the cluster FAILED once createTimeout has elapsed +// since meta.CreatedAt. Returns nil while still within the window so the next +// tick retries. A zero CreatedAt disables the deadline. func (r *ClusterReconciler) failIfCreateTimedOut(meta *ClusterMeta, reason string) error { if r.createTimeout <= 0 || meta.CreatedAt.IsZero() { return nil @@ -376,12 +354,9 @@ func (r *ClusterReconciler) failIfCreateTimedOut(meta *ClusterMeta, reason strin return ErrReconcilerClusterFailed } -// bootstrapReady returns true once the four NATS bootstrap KV artifacts are -// present: node token, admin kubeconfig, the OIDC JWKS verified-marker, and the -// CA on meta. The JWKS gate is the verified-marker (written only after the VM's -// published JWKS passes the controller cross-check), NOT the pre-seeded -// OIDCJWKSKey — so a cluster never transitions to ACTIVE on an unverified or -// mismatched signing key. +// bootstrapReady returns true once the four bootstrap KV artifacts are present: +// node token, admin kubeconfig, OIDC JWKS verified-marker, and CA on meta. +// Uses the verified-marker (not OIDCJWKSKey) so ACTIVE always implies a verified key. func (r *ClusterReconciler) bootstrapReady(meta *ClusterMeta) (bool, string) { if meta.CertificateAuthorityB64 == "" { return false, "CA absent on meta" diff --git a/spinifex/handlers/eks/cluster_state.go b/spinifex/handlers/eks/cluster_state.go index aa37114c9..8322cd8c1 100644 --- a/spinifex/handlers/eks/cluster_state.go +++ b/spinifex/handlers/eks/cluster_state.go @@ -10,9 +10,7 @@ import ( "github.com/nats-io/nats.go" ) -// ClusterStatus is the persisted lifecycle state for a cluster. Values match -// the AWS EKS Cluster.status string set so DescribeCluster can pass them -// through verbatim. +// ClusterStatus is the persisted cluster lifecycle state; values match the AWS EKS status enum. type ClusterStatus string const ( @@ -22,26 +20,19 @@ const ( ClusterStatusFailed ClusterStatus = "FAILED" ) -// ClusterVpcConfig is the persisted subset of eks.VpcConfigResponse that the -// reconciler + DescribeCluster need at-rest. +// ClusterVpcConfig is the persisted subset of eks.VpcConfigResponse. type ClusterVpcConfig struct { SubnetIds []string `json:"subnetIds"` SecurityGroupIds []string `json:"securityGroupIds,omitempty"` VpcId string `json:"vpcId,omitempty"` - // EndpointPublicAccess / EndpointPrivateAccess control apiserver endpoint - // exposure (AWS resourcesVpcConfig parity). Public ⇒ internet-facing cluster - // NLB (external-pool front-end IP, LAN-reachable); private ⇒ internal NLB - // (VPC-only). PublicAccessCidrs records the allowed source ranges for the - // public endpoint (stored for parity + Describe; ingress enforcement is a - // follow-up). + // EndpointPublicAccess/EndpointPrivateAccess mirror AWS resourcesVpcConfig. + // PublicAccessCidrs stores allowed source ranges for the public endpoint. EndpointPublicAccess bool `json:"endpointPublicAccess"` EndpointPrivateAccess bool `json:"endpointPrivateAccess"` PublicAccessCidrs []string `json:"publicAccessCidrs,omitempty"` } -// ClusterMeta is the persisted control-plane record for one cluster. The -// blob lives at ClusterMetaKey(name) inside the per-account KV bucket and is -// the source of truth for DescribeCluster. +// ClusterMeta is the persisted control-plane record at ClusterMetaKey(name); source of truth for DescribeCluster. type ClusterMeta struct { Name string `json:"name"` Arn string `json:"arn"` @@ -50,9 +41,8 @@ type ClusterMeta struct { Version string `json:"version"` RoleArn string `json:"roleArn"` Endpoint string `json:"endpoint,omitempty"` - // EndpointIP is the host/LAN-reachable front-end IP the cluster NLB exposes - // (public-access ⇒ external-pool IP; private ⇒ VPC IP). The apiserver serving - // cert SANs this IP, so kubectl reaches Endpoint with TLS verification on. + // EndpointIP is the NLB front-end IP (external-pool for public, VPC IP for private). + // The apiserver serving cert SANs this IP for TLS verification. EndpointIP string `json:"endpointIp,omitempty"` OIDCIssuer string `json:"oidcIssuer,omitempty"` CertificateAuthorityB64 string `json:"certificateAuthorityB64,omitempty"` @@ -60,52 +50,35 @@ type ClusterMeta struct { ControlPlaneInstanceID string `json:"controlPlaneInstanceId,omitempty"` ControlPlaneENIID string `json:"controlPlaneEniId,omitempty"` ControlPlaneENIIP string `json:"controlPlaneEniIp,omitempty"` - // ControlPlaneMgmtIP is the control-plane VM's br-mgmt NIC address. Until - // authoritative DNS (Eclipso/Route53) lands, the host-side reconciler cannot - // resolve or route to the VPC-internal NLB DNS endpoint, so it probes - // /healthz on this host-reachable address instead. + // ControlPlaneMgmtIP is the CP VM's br-mgmt NIC address, used as the /healthz + // probe target until authoritative in-VPC DNS is available. ControlPlaneMgmtIP string `json:"controlPlaneMgmtIp,omitempty"` - // ControlPlaneNodes lists the placed control-plane server VMs. HA spread - // holds one entry per distinct host; the single-CP path holds one. [0] is the - // primary the NLB target + egress SNAT wire to until per-node NLB - // registration (231.7.3). The scalar ControlPlane* fields above mirror [0] - // for readers that predate this field (reconciler, teardown) and for clusters - // persisted before HA spread existed (empty ControlPlaneNodes). + // ControlPlaneNodes lists placed CP server VMs; [0] is the primary for NLB + // and egress. Scalar ControlPlane* fields mirror [0] for back-compat. ControlPlaneNodes []ControlPlaneNode `json:"controlPlaneNodes,omitempty"` - // ControlPlaneSpreadGroup is the spread placement-group name reserving the CP - // hosts; "" for the single-CP fallback. DeleteCluster releases + deletes it. + // ControlPlaneSpreadGroup is the spread placement-group name; "" for single-CP. ControlPlaneSpreadGroup string `json:"controlPlaneSpreadGroup,omitempty"` - // EgressEIPAllocationID / EgressEIPPublicIP track the hidden pool address - // SNAT'd to the control-plane VM for egress-only internet (image pulls). - // Released + the snat removed on DeleteCluster. + // EgressEIPAllocationID / EgressEIPPublicIP track the hidden-pool SNAT address + // for CP VM egress (image pulls). Released on DeleteCluster. EgressEIPAllocationID string `json:"egressEipAllocationId,omitempty"` EgressEIPPublicIP string `json:"egressEipPublicIp,omitempty"` NLBArn string `json:"nlbArn,omitempty"` NLBTargetGroupArn string `json:"nlbTargetGroupArn,omitempty"` CreatedAt time.Time `json:"createdAt"` - // HealthIssue is the last /healthz probe failure reason for an ACTIVE - // cluster; empty means healthy. DescribeCluster surfaces a non-empty value - // as a ClusterHealth issue so a dead control plane is visible behind the - // still-ACTIVE status. LastHealthProbe stamps when the health state changed. + // HealthIssue is the last health failure reason ("" = healthy). + // DescribeCluster surfaces it as a ClusterHealth issue. HealthIssue string `json:"healthIssue,omitempty"` - // No omitempty/omitzero: stdlib encoding/json ignores omitzero (json/v2 - // only) and never treats a time.Time struct as empty, so this field always - // serializes. The tag states that honestly rather than implying it elides. + // No omitempty: stdlib encoding/json never treats a time.Time as empty. LastHealthProbe time.Time `json:"lastHealthProbe"` - // NodeCount is the node total from the control plane's last NATS state - // report (server + joined agents). Surfaces nodegroup join/scale progress - // without the host needing apiserver reachability. + // NodeCount is the node total from the CP's last NATS state report. NodeCount int `json:"nodeCount,omitempty"` - // BuiltinIngress records whether the cluster opted into K3s' bundled - // traefik + servicelb (dev / interim in-VPC app exposure), derived from the - // managed-ingress tag at CreateCluster. Default false = AWS parity (built-ins - // disabled; ingress via the AWS Load Balancer Controller). + // BuiltinIngress is true when the cluster opted into K3s' bundled traefik + servicelb. + // Default false = AWS parity (ingress via the AWS Load Balancer Controller). BuiltinIngress bool `json:"builtinIngress,omitempty"` } -// ControlPlaneNode identifies one placed control-plane server VM and the host -// it landed on. NodeID is the Spinifex host — distinct per entry under HA -// spread, empty for a single control plane launched on the local node. +// ControlPlaneNode identifies one placed CP server VM. NodeID is the Spinifex +// host; empty for a single-CP launched on the local node. type ControlPlaneNode struct { NodeID string `json:"nodeId,omitempty"` InstanceID string `json:"instanceId"` @@ -114,15 +87,13 @@ type ControlPlaneNode struct { MgmtIP string `json:"mgmtIp,omitempty"` } -// ErrClusterNotFound is returned by GetClusterMeta / SetClusterStatus / -// DeleteClusterPrefix when the cluster meta key is absent. Callers translate -// to the AWS shape (ResourceNotFoundException) at the service boundary. +// ErrClusterNotFound is returned when the cluster meta key is absent. +// Callers translate it to ResourceNotFoundException at the service boundary. var ErrClusterNotFound = errors.New("eks: cluster not found") const maxClusterStateCASRetries = 5 -// PutClusterMeta writes the meta record unconditionally. Used at -// CreateCluster time to lay down the initial CREATING record. +// PutClusterMeta writes the meta record unconditionally. func PutClusterMeta(kv nats.KeyValue, meta *ClusterMeta) error { if meta == nil { return errors.New("eks: PutClusterMeta nil meta") @@ -160,11 +131,9 @@ func GetClusterMeta(kv nats.KeyValue, name string) (*ClusterMeta, error) { return &meta, nil } -// casUpdateMeta does a revision-checked read-modify-write of the cluster meta -// record. mutate receives the decoded meta and returns true if it changed a -// field (false short-circuits to a no-op success). Retries on KV CAS conflict -// (concurrent reconciler write) up to maxClusterStateCASRetries. Returns -// ErrClusterNotFound if the meta record was deleted underneath us. +// casUpdateMeta does a revision-checked read-modify-write of the cluster meta. +// mutate returns true when a field changed. Retries on CAS conflict up to +// maxClusterStateCASRetries. Returns ErrClusterNotFound if deleted concurrently. func casUpdateMeta(kv nats.KeyValue, name string, mutate func(*ClusterMeta) bool) error { if name == "" { return errors.New("eks: casUpdateMeta empty name") @@ -200,8 +169,7 @@ func casUpdateMeta(kv nats.KeyValue, name string, mutate func(*ClusterMeta) bool return fmt.Errorf("eks: casUpdateMeta %s exhausted CAS retries", name) } -// SetClusterStatus does a revision-checked update of the meta.Status field. -// Returns ErrClusterNotFound if the meta record was deleted underneath us. +// SetClusterStatus does a CAS update of meta.Status. Returns ErrClusterNotFound if deleted concurrently. func SetClusterStatus(kv nats.KeyValue, name string, status ClusterStatus) error { if name == "" { return errors.New("eks: SetClusterStatus empty name") @@ -215,11 +183,8 @@ func SetClusterStatus(kv nats.KeyValue, name string, status ClusterStatus) error }) } -// MarkClusterFailed transitions a cluster to FAILED with a human-readable -// reason, but only from CREATING — a cluster already DELETING/ACTIVE/FAILED is -// left untouched so a late bootstrap error cannot clobber a concurrent delete -// or an already-healthy cluster. Returns ErrClusterNotFound if the meta record -// was deleted underneath us. +// MarkClusterFailed transitions the cluster to FAILED with a reason, but only +// from CREATING — so a late error cannot clobber a concurrent delete or an active cluster. func MarkClusterFailed(kv nats.KeyValue, name, reason string) error { if name == "" { return errors.New("eks: MarkClusterFailed empty name") @@ -234,10 +199,8 @@ func MarkClusterFailed(kv nats.KeyValue, name, reason string) error { }) } -// SetClusterCertificateAuthority does a revision-checked update of the -// meta.CertificateAuthorityB64 field. The NATS bootstrap subscriber calls -// this once the K3s server VM publishes its CA on the bootstrap bus. Returns -// ErrClusterNotFound if the meta record was deleted underneath us. +// SetClusterCertificateAuthority does a CAS update of meta.CertificateAuthorityB64. +// Called by the bootstrap subscriber when the K3s VM publishes its CA. func SetClusterCertificateAuthority(kv nats.KeyValue, name, caB64 string) error { if name == "" { return errors.New("eks: SetClusterCertificateAuthority empty name") @@ -254,12 +217,8 @@ func SetClusterCertificateAuthority(kv nats.KeyValue, name, caB64 string) error }) } -// SetClusterHealth records the latest /healthz probe outcome for a cluster: -// issue is the failure reason (empty clears it / marks healthy). Writes only -// when the health state changes, stamping LastHealthProbe at the transition so -// a persistently-unhealthy cluster carries the time it first went bad rather -// than churning the KV every probe interval. Returns ErrClusterNotFound if the -// meta record was deleted underneath us. +// SetClusterHealth records the latest health outcome; issue="" means healthy. +// Only writes on a state change, stamping LastHealthProbe at the transition. func SetClusterHealth(kv nats.KeyValue, name, issue string) error { if name == "" { return errors.New("eks: SetClusterHealth empty name") @@ -274,12 +233,8 @@ func SetClusterHealth(kv nats.KeyValue, name, issue string) error { }) } -// SetClusterHealthState records the latest health outcome together with the -// node count from the control-plane state report. issue is the failure reason -// (empty clears it / marks healthy). LastHealthProbe is stamped whenever health -// or node count changes; a steady-state healthy cluster with an unchanging node -// count produces no KV write. Returns ErrClusterNotFound if the meta record was -// deleted underneath us. +// SetClusterHealthState records health + node count. LastHealthProbe is stamped +// on any change; no KV write when both are unchanged. func SetClusterHealthState(kv nats.KeyValue, name, issue string, nodeCount int) error { if name == "" { return errors.New("eks: SetClusterHealthState empty name") @@ -295,10 +250,8 @@ func SetClusterHealthState(kv nats.KeyValue, name, issue string, nodeCount int) }) } -// DeleteClusterPrefix removes every KV key under clusters/{name}/. Called -// from DeleteCluster after the OIDC private key has been zeroized. -// Returns the first delete error encountered but continues sweeping so -// best-effort cleanup proceeds. +// DeleteClusterPrefix removes every KV key under clusters/{name}/. +// Returns the first error but continues sweeping. func DeleteClusterPrefix(kv nats.KeyValue, name string) error { if name == "" { return errors.New("eks: DeleteClusterPrefix empty name") diff --git a/spinifex/handlers/eks/k3s_ha_control_plane.go b/spinifex/handlers/eks/k3s_ha_control_plane.go index e838cde10..37bfbc541 100644 --- a/spinifex/handlers/eks/k3s_ha_control_plane.go +++ b/spinifex/handlers/eks/k3s_ha_control_plane.go @@ -18,15 +18,11 @@ import ( "github.com/nats-io/nats.go" ) -// haControlPlaneCount is the control-plane spread width: an odd quorum of three -// so an embedded-etcd control plane tolerates a single host loss. Fewer than -// this many schedulable hosts falls back to a single control-plane VM (today's -// behaviour) instead of failing the create. +// haControlPlaneCount is the HA spread width (odd quorum tolerating one host loss). +// Fewer schedulable hosts fall back to a single CP VM. const haControlPlaneCount = 3 -// controlPlanePlacer is the spread-placement surface the HA control-plane -// orchestrator needs. handlers/ec2/placementgroup.PlacementGroupService (wired -// by the daemon as NewNATSPlacementGroupService) satisfies it. +// controlPlanePlacer is the spread-placement surface for HA CP orchestration. type controlPlanePlacer interface { CreatePlacementGroup(*ec2.CreatePlacementGroupInput, string) (*ec2.CreatePlacementGroupOutput, error) DeletePlacementGroup(*ec2.DeletePlacementGroupInput, string) (*ec2.DeletePlacementGroupOutput, error) @@ -38,25 +34,17 @@ type controlPlanePlacer interface { var _ controlPlanePlacer = (handlers_ec2_placementgroup.PlacementGroupService)(nil) -// HostScheduler answers the capacity + placement fan-out questions the HA -// control-plane orchestrator asks. The daemon wires a NATS implementation; unit -// tests fake it so the orchestration logic needs no live cluster. +// HostScheduler answers capacity + placement fan-out questions for HA CP placement. type HostScheduler interface { - // SchedulableHosts returns the distinct node IDs that can fit at least one - // VM of instanceType. + // SchedulableHosts returns node IDs that can fit at least one VM of instanceType. SchedulableHosts(instanceType string) []string - // InstanceHosts maps each instance ID to the node currently hosting it. - // Instances absent from the result are not (yet) visible to the fan-out. + // InstanceHosts maps each instance ID to its hosting node; absent entries are not yet visible. InstanceHosts(instanceIDs []string) map[string]string } -// placeControlPlane places the cluster's control-plane VM(s). With at least -// haControlPlaneCount schedulable hosts it spreads that many k3s server VMs -// across distinct hosts, all-or-nothing; otherwise it launches a single -// control-plane VM on the local node (today's behaviour). Returns the placed -// nodes ([0] is the primary the NLB + egress wire to until 231.7.3) and the -// spread placement-group name ("" for the single-CP fallback) the caller -// persists for DeleteCluster teardown. +// placeControlPlane places the cluster's CP VMs. With ≥ haControlPlaneCount +// schedulable hosts it spreads servers all-or-nothing; otherwise falls back to +// a single CP. Returns placed nodes ([0] = primary) and spread group name ("" for single-CP). func (s *EKSServiceImpl) placeControlPlane(accountID, clusterName string, tmpl K3sServerInput) ([]ControlPlaneNode, string, error) { instanceType := tmpl.InstanceType if instanceType == "" { @@ -83,10 +71,7 @@ func (s *EKSServiceImpl) placeControlPlane(accountID, clusterName string, tmpl K MaxCount: haControlPlaneCount, }, pgAccount) if err != nil { - // Could not reserve the full quorum of distinct hosts (capacity raced - // away between the status fan-out and the CAS reservation). Drop the - // group and fall back to a single control plane rather than failing the - // whole create. + // Capacity raced away between fan-out and reservation; fall back to single CP. slog.Warn("placeControlPlane: spread reservation failed, falling back to single control plane", "cluster", clusterName, "group", groupName, "err", err) s.deleteSpreadGroup(groupName, pgAccount) @@ -106,8 +91,7 @@ func (s *EKSServiceImpl) placeControlPlane(accountID, clusterName string, tmpl K launched = append(launched, controlPlaneNode(r.node, r.out)) } - // All-or-nothing: any launch failure rolls back every VM that did come up, - // releases the reservation, drops the group, and fails the create. + // All-or-nothing: any launch failure rolls back all VMs, releases the reservation. if len(launchErrs) > 0 { slog.Error("placeControlPlane: partial spread launch, rolling back", "cluster", clusterName, "launched", len(launched), "failed", len(launchErrs)) @@ -115,10 +99,8 @@ func (s *EKSServiceImpl) placeControlPlane(accountID, clusterName string, tmpl K return nil, "", fmt.Errorf("eks: HA control-plane launch failed: %w", errors.Join(launchErrs...)) } - // Confirm each VM actually landed on its reserved host (acceptance: verified - // via the node-VM fan-out). The node-targeted launch subject already pins - // placement, so an instance not yet visible to the fan-out is tolerated; - // only a definitive wrong-host placement triggers rollback. + // Verify each VM landed on its reserved host. Not-yet-visible instances are + // tolerated; only a definitive wrong-host placement triggers rollback. if err := s.verifyControlPlaneSpread(launched); err != nil { slog.Error("placeControlPlane: placement verification failed, rolling back", "cluster", clusterName, "err", err) @@ -144,8 +126,7 @@ func (s *EKSServiceImpl) placeControlPlane(accountID, clusterName string, tmpl K return launched, groupName, nil } -// launchSingleControlPlane launches one control-plane VM on the local node, the -// pre-HA behaviour. NodeID is empty: the local launch is node-blind. +// launchSingleControlPlane launches one CP VM on the local node (pre-HA fallback). func (s *EKSServiceImpl) launchSingleControlPlane(tmpl K3sServerInput) ([]ControlPlaneNode, string, error) { in := tmpl in.TargetNodeID = "" @@ -162,13 +143,9 @@ type cpLaunchResult struct { err error } -// launchControlPlaneSpread launches the reserved nodes as one etcd quorum: -// nodes[0] is the first server (cluster-init), and its ENI IP is the join -// endpoint the remaining servers register against — so it must come up first. -// nodes[1..] then launch in parallel as join servers (each remote launch blocks -// on a full-AMI boot round trip, so sequential joins would be N×). Results stay -// index-aligned with nodes. If the first server fails, the joins are skipped -// (no endpoint to join) so the caller's all-or-nothing rollback fires. +// launchControlPlaneSpread launches the reserved nodes as one etcd quorum. +// nodes[0] is the cluster-init server whose ENI IP the remaining servers join; +// nodes[1..] launch in parallel. Results are index-aligned with nodes. func (s *EKSServiceImpl) launchControlPlaneSpread(tmpl K3sServerInput, nodes []string) []cpLaunchResult { results := make([]cpLaunchResult, len(nodes)) @@ -200,13 +177,11 @@ func (s *EKSServiceImpl) launchControlPlaneSpread(tmpl K3sServerInput, nodes []s return results } -// errFirstServerFailed marks join-server results skipped when the cluster-init -// server never came up — the joins have no endpoint to register against. +// errFirstServerFailed marks join results skipped when the cluster-init server failed to launch. var errFirstServerFailed = errors.New("eks: first control-plane server failed; join servers skipped") -// verifyControlPlaneSpread confirms every launched VM sits on its reserved host -// and that no two share a host. A VM not yet visible to the fan-out is tolerated -// (listing lag — the node-targeted launch subject already guarantees the host). +// verifyControlPlaneSpread confirms every CP VM sits on its reserved host with +// no duplicates. VMs not yet visible to the fan-out are tolerated. func (s *EKSServiceImpl) verifyControlPlaneSpread(nodes []ControlPlaneNode) error { ids := make([]string, 0, len(nodes)) for _, n := range nodes { @@ -233,10 +208,8 @@ func (s *EKSServiceImpl) verifyControlPlaneSpread(nodes []ControlPlaneNode) erro return nil } -// rollbackControlPlaneSpread tears down a partial HA placement: terminate every -// VM that launched, release the reservation (clears the empty placeholders — -// the record was never finalized on any rollback path), and drop the group. -// Best-effort: a stranded internal placement group is harmless and re-creatable. +// rollbackControlPlaneSpread terminates launched VMs, releases the reservation, +// and drops the group. Best-effort: a leaked internal group is harmless. func (s *EKSServiceImpl) rollbackControlPlaneSpread(accountID, groupName, pgAccount string, launched []ControlPlaneNode, reserved []string) { for _, n := range launched { if err := TerminateK3sServerVM(s.deps.VPCK3s, s.deps.Instance, accountID, n.InstanceID, n.ENIID); err != nil { @@ -252,9 +225,7 @@ func (s *EKSServiceImpl) rollbackControlPlaneSpread(accountID, groupName, pgAcco s.deleteSpreadGroup(groupName, pgAccount) } -// ensureSpreadGroup creates the per-cluster spread placement group. A duplicate -// (a prior attempt left it behind) is reused — ReserveSpreadNodes filters -// already-occupied hosts, so reserving against a stale group is safe. +// ensureSpreadGroup creates the spread placement group, reusing it if it already exists. func (s *EKSServiceImpl) ensureSpreadGroup(groupName, pgAccount string) error { _, err := s.deps.PlacementGroup.CreatePlacementGroup(&ec2.CreatePlacementGroupInput{ GroupName: aws.String(groupName), @@ -278,9 +249,8 @@ func (s *EKSServiceImpl) deleteSpreadGroup(groupName, pgAccount string) { } } -// teardownSpreadGroup removes the cluster's CP instances from the spread group -// and deletes it. Best-effort: a leaked internal group strands nothing billable. -// Runs only for HA clusters (ControlPlaneSpreadGroup set). +// teardownSpreadGroup removes CP instances from the spread group and deletes it. +// No-op for single-CP clusters (ControlPlaneSpreadGroup == ""). func (s *EKSServiceImpl) teardownSpreadGroup(meta *ClusterMeta) { if meta.ControlPlaneSpreadGroup == "" { return @@ -302,9 +272,8 @@ func (s *EKSServiceImpl) teardownSpreadGroup(meta *ClusterMeta) { s.deleteSpreadGroup(meta.ControlPlaneSpreadGroup, pgAccount) } -// controlPlaneTeardownNodes returns the control-plane VMs DeleteCluster must -// tear down. It prefers the multi-node list; for clusters persisted before that -// field existed it synthesizes a single entry from the scalar fields. +// controlPlaneTeardownNodes returns CP VMs for DeleteCluster. Prefers +// ControlPlaneNodes; falls back to scalar fields for older persisted clusters. func controlPlaneTeardownNodes(meta *ClusterMeta) []ControlPlaneNode { if len(meta.ControlPlaneNodes) > 0 { return meta.ControlPlaneNodes @@ -320,10 +289,8 @@ func controlPlaneTeardownNodes(meta *ClusterMeta) []ControlPlaneNode { }} } -// haSpreadGroupName namespaces the spread group by account + cluster. The group -// lives under the system account (admin.SystemAccountID) so it never surfaces in -// a customer's DescribePlacementGroups; the account therefore cannot -// disambiguate same-named clusters across tenants, so it is folded into the name. +// haSpreadGroupName namespaces the spread group by account + cluster so same-named +// clusters across tenants don't collide. Lives under the system account. func haSpreadGroupName(accountID, clusterName string) string { return "eks-cp-" + accountID + "-" + clusterName } @@ -342,27 +309,22 @@ func controlPlaneNode(nodeID string, out *K3sServerOutput) ControlPlaneNode { var _ HostScheduler = (*natsHostScheduler)(nil) -// hostFanoutTimeout bounds a node status / node VMs fan-out. Sized for a LAN -// round trip to every daemon; CreateCluster is already multi-second so a fixed -// short collection window trades a little latency for reliably seeing every host -// (a missed host would wrongly drop the cluster to single-CP). +// hostFanoutTimeout bounds a node status/VMs fan-out. Short fixed window +// trades a little latency for reliably seeing all hosts before placement. const hostFanoutTimeout = time.Second type natsHostScheduler struct { nc *nats.Conn } -// NewNATSHostScheduler builds the NATS fan-out HostScheduler the daemon wires -// into EKSServiceDeps. +// NewNATSHostScheduler returns a NATS fan-out HostScheduler for EKSServiceDeps. func NewNATSHostScheduler(nc *nats.Conn) HostScheduler { return &natsHostScheduler{nc: nc} } func (h *natsHostScheduler) SchedulableHosts(instanceType string) []string { - // The control-plane VM is a system type (sys.*), which node.status omits from - // its per-type capacity list (system types are hidden from customers). A - // system VM still consumes host vCPU/memory like any guest, so size the fit - // against each node's raw schedulable headroom and the type's footprint. + // sys.* types are hidden from customers in node.status but still consume + // vCPU/memory, so fit is checked against raw schedulable headroom. vcpu, memGB, ok := instancetypes.SpecForSystemType(instanceType) if !ok { slog.Warn("SchedulableHosts: unknown system instance type, no schedulable hosts", @@ -385,9 +347,8 @@ func (h *natsHostScheduler) SchedulableHosts(instanceType string) []string { return hosts } -// nodeFitsSystemInstance reports whether a node's schedulable headroom -// (Total - Reserved - Alloc, the same arithmetic node.status documents for guest -// scheduling) admits at least one VM of the given vCPU/memory footprint. +// nodeFitsSystemInstance reports whether a node's headroom (Total - Reserved - Alloc) +// fits at least one VM of the given vCPU/memory footprint. func nodeFitsSystemInstance(st types.NodeStatusResponse, vcpu int, memGB float64) bool { remainVCPU := st.TotalVCPU - st.ReservedVCPU - st.AllocVCPU remainMem := st.TotalMemGB - st.ReservedMemGB - st.AllocMemGB @@ -414,8 +375,7 @@ func (h *natsHostScheduler) InstanceHosts(instanceIDs []string) map[string]strin return out } -// fanout publishes an empty request on subject and invokes handle for every -// reply that arrives within hostFanoutTimeout. +// fanout publishes on subject and calls handle for each reply within hostFanoutTimeout. func (h *natsHostScheduler) fanout(subject string, handle func([]byte)) { if h.nc == nil { return diff --git a/spinifex/handlers/eks/k3s_server_vm.go b/spinifex/handlers/eks/k3s_server_vm.go index c23dab4fc..cb347a6bd 100644 --- a/spinifex/handlers/eks/k3s_server_vm.go +++ b/spinifex/handlers/eks/k3s_server_vm.go @@ -17,108 +17,73 @@ import ( "github.com/mulgadc/spinifex/spinifex/tags" ) -// imdsServerIP is the link-local IMDS address (mirror of -// handlers/imds.MetaDataServerIP, duplicated to avoid an import cycle: -// imds → sts → eks). +// imdsServerIP is the link-local IMDS address (duplicated here to avoid an imds→sts→eks import cycle). const imdsServerIP = "169.254.169.254" -// ErrEKSServerAMINotFound is returned by the launcher when no AMI carrying the -// EKS managed-by tag exists in the account. CreateCluster maps it to a precise -// client error: it signals an operator/config gap (the eks-server image was -// never built/imported), not an unrecoverable internal fault. +// ErrEKSServerAMINotFound is returned when no AMI with the EKS managed-by tag +// exists. Signals an operator/config gap (image not built/imported). var ErrEKSServerAMINotFound = errors.New("eks: eks-server AMI not found") -// k3sVPCProvisioner is the subset of handlers_ec2_vpc.VPCService that the K3s -// server VM launcher needs. Narrow so tests can fake it without implementing -// the full VPC surface. +// k3sVPCProvisioner is the narrow VPC surface the K3s server VM launcher needs. type k3sVPCProvisioner interface { CreateNetworkInterface(input *ec2.CreateNetworkInterfaceInput, accountID string) (*ec2.CreateNetworkInterfaceOutput, error) DeleteNetworkInterface(input *ec2.DeleteNetworkInterfaceInput, accountID string) (*ec2.DeleteNetworkInterfaceOutput, error) } -// k3sInstanceLauncher is the system-instance launch surface the K3s control-plane -// VM needs. The VM boots from the eks-server AMI (BootAMI) and must get a -// management-bridge NIC so it can reach the daemon's NATS endpoint off its -// tenant VPC subnet — which only the system-instance path provides. -// *daemon.Daemon satisfies this interface. +// k3sInstanceLauncher is the system-instance launch surface for the K3s CP VM. +// The VM boots from the eks-server AMI and needs a mgmt-bridge NIC via the +// system-instance path to reach the daemon's NATS endpoint. type k3sInstanceLauncher interface { LaunchSystemInstance(input *sysinstance.SystemInstanceInput) (*sysinstance.SystemInstanceOutput, error) - // LaunchSystemInstanceOnNode places the VM on a specific Spinifex host for - // HA control-plane spread. An empty nodeID (or the local node) launches - // in-process, matching LaunchSystemInstance. + // LaunchSystemInstanceOnNode pins the VM to a specific host for HA spread. + // An empty nodeID launches in-process like LaunchSystemInstance. LaunchSystemInstanceOnNode(nodeID string, input *sysinstance.SystemInstanceInput) (*sysinstance.SystemInstanceOutput, error) TerminateSystemInstance(instanceID string) error } -// k3sAMIResolver is the subset of handlers_ec2_image.ImageService the launcher -// uses to resolve the eks-server AMI name to an AMI ID at launch time. +// k3sAMIResolver is the narrow AMI surface for resolving the eks-server AMI ID. type k3sAMIResolver interface { DescribeImages(input *ec2.DescribeImagesInput, accountID string) (*ec2.DescribeImagesOutput, error) } const ( - // defaultK3sServerInstanceType is the internal system type the k3s - // control-plane VM boots as (2 vCPU / 4 GB / 40 GB root). A sys.* type - // keeps the CP VM out of customer DescribeInstanceTypes and lets the daemon - // register the node-targeted system.LaunchInstance.{type}.{nodeID} subject - // the HA spread path uses. Callers can override via K3sServerInput.InstanceType. + // defaultK3sServerInstanceType is the sys.* type for the CP VM; keeps it out of + // customer DescribeInstanceTypes and enables node-targeted HA launch subjects. defaultK3sServerInstanceType = "sys.medium" - // k3sOIDCSigningKeyPath is the on-VM path where cloud-init drops the - // per-cluster OIDC private key PEM. K3s reads it via the - // service-account-signing-key-file kube-apiserver arg (signs SA tokens). + // k3sOIDCSigningKeyPath is the on-VM OIDC private key PEM path. k3sOIDCSigningKeyPath = "/etc/rancher/k3s/oidc-signing-key.pem" - // k3sOIDCPublicKeyPath is where cloud-init drops the matching public key - // PEM. kube-apiserver's service-account-key-file requires a PUBLIC key - // (it rejects a private-key PEM with "data does not contain any valid RSA - // or ECDSA public keys" and crash-loops), so it must be a separate file. + // k3sOIDCPublicKeyPath is the matching public key PEM; kube-apiserver's + // --service-account-key-file requires a PUBLIC key, so it must be separate. k3sOIDCPublicKeyPath = "/etc/rancher/k3s/oidc-signing-key.pub.pem" - // k3sFirstBootEnvPath is the env file k3s-first-boot.sh sources at boot - // (see scripts/images/eks-server/k3s-first-boot.sh ENVFILE). + // k3sFirstBootEnvPath is the env file k3s-first-boot.sh sources at boot. k3sFirstBootEnvPath = "/etc/spinifex-eks/first-boot.env" - // agentEnvPath is the env file the k3s-agent OpenRC service sources for its - // K3S_URL/K3S_TOKEN/K3S_NODE_NAME/K3S_NODE_LABEL. Path matches the AGENT_ENVFILE - // in scripts/images/eks-node/eks-node-role.sh and k3s-agent.initd. + // agentEnvPath is the env file the k3s-agent OpenRC service sources for + // K3S_URL/K3S_TOKEN/K3S_NODE_NAME/K3S_NODE_LABEL. agentEnvPath = "/etc/spinifex-eks/agent.env" - // k3sGatewayCAPath is the on-VM destination for the AWS gateway TLS CA cert - // PEM. The K3s VM uses it to verify the gateway's HTTPS cert when the - // eks-gateway-publish helper POSTs bootstrap envelopes + state reports. - // Path matches k3s-first-boot.sh EKS_GATEWAY_CA. + // k3sGatewayCAPath is the on-VM gateway TLS CA cert PEM path. k3sGatewayCAPath = "/etc/spinifex-eks/gateway-ca.pem" - // k3sTokenWebhookKubeconfigPath is the apiserver token-webhook config the - // eks-token-webhook service (ordered `before k3s`) writes before the - // apiserver starts. Wired via the authentication-token-webhook-config-file - // apiserver arg so bearer tokens minted by `aws eks get-token` resolve - // through the webhook. Must match the webhook's EKS_WEBHOOK_KUBECONFIG default. + // k3sTokenWebhookKubeconfigPath is the apiserver token-webhook kubeconfig + // written before k3s starts, wired via --authentication-token-webhook-config-file. k3sTokenWebhookKubeconfigPath = "/etc/spinifex-eks/token-webhook.kubeconfig" //nolint:gosec // file path, not a credential - // k3sConfigPath is the K3s server config file cloud-init writes; K3s - // reads it at startup (overrides the AMI-baked config.yaml.skel). + // k3sConfigPath is the K3s server config file written by cloud-init. k3sConfigPath = "/etc/rancher/k3s/config.yaml" - // k3sResolvConfPath is the on-VM resolver file. The Alpine eks-server AMI - // runs dhcpcd, whose resolv.conf hook fails to persist the DHCP-supplied - // nameservers ("can't create /etc/resolv.conf: nonexistent directory"), so - // the VM boots with no resolver and containerd cannot resolve registry-1. - // docker.io to pull the system-pod images. cloud-init writes a static - // resolver here; the dhcpcd hook never clobbers it (it errors before it - // would). Reachable via the cluster's egress SNAT. + // k3sResolvConfPath is the on-VM resolver path. The Alpine AMI's dhcpcd hook + // cannot create /etc/resolv.conf, so cloud-init writes a static resolver here. k3sResolvConfPath = "/etc/resolv.conf" - // k3sResolvConf is the static resolver content. Public anycast resolvers - // reached over the control-plane VM's egress path. + // k3sResolvConf is the static resolver; reached via the cluster's egress SNAT. k3sResolvConf = "nameserver 1.1.1.1\nnameserver 8.8.8.8" ) -// K3sServerInput is the launcher's input shape. AccountID is the customer -// account; the ENI + VM both live there in v1 (SystemAccount-owned VM is -// deferred behind cross-account-ENI work). Region is carried for future -// region-aware AMI lookups but not consumed today. +// K3sServerInput is the K3s server VM launcher's input shape. type K3sServerInput struct { AccountID string ClusterName string @@ -126,49 +91,34 @@ type K3sServerInput struct { SubnetID string ControlPlaneSGID string NLBDNS string - // EndpointIP is the reachable cluster NLB front-end IP. When set it is added - // to the apiserver serving-cert SANs (tls-san) so kubectl reaching the - // cluster on this IP validates TLS. Empty for an internal endpoint with no - // front-end IP read back. + // EndpointIP is the NLB front-end IP added to the apiserver cert SANs for TLS. + // Empty for an internal endpoint with no front-end IP. EndpointIP string OIDCIssuer string OIDCPrivateKeyPEM string OIDCPublicKeyPEM string - // Gateway broker config: the control-plane VM publishes its bootstrap - // envelopes + state reports via SigV4-signed HTTPS POST to the AWS gateway - // (the ELBv2 lb-agent model), not by dialing core NATS. GatewayURL is the - // mgmt-reachable AWSGW endpoint; AccessKey/SecretKey are the system - // (Predastore) SigV4 creds; GatewayCACert is the PEM that signs the gateway - // server cert. + // Gateway broker config: CP VM publishes via SigV4-signed HTTPS POST to AWSGW. + // GatewayURL is the mgmt-reachable endpoint; AccessKey/SecretKey are system + // SigV4 creds; GatewayCACert signs the gateway TLS cert. GatewayURL string AccessKey string SecretKey string GatewayCACert string InstanceType string - // TargetNodeID pins the control-plane VM to a specific Spinifex host for HA - // spread across distinct failure domains. Empty launches on the local - // daemon node (the single-CP default). + // TargetNodeID pins the VM to a specific host for HA spread; empty = local node. TargetNodeID string - // JoinToken is the shared k3s cluster token (server secret). Set on every - // control-plane server so servers 2..N authenticate into the first server's - // embedded-etcd quorum and workers continue to register. Empty preserves - // k3s' per-server auto-generated token (pre-HA single-CP behaviour). + // JoinToken is the shared k3s cluster token so HA servers join the etcd quorum. + // Empty = k3s auto-generated token (single-CP). JoinToken string - // ServerURL, when set, boots this VM as a JOIN server: it registers with the - // existing control plane at this https://:6443 endpoint and - // joins the etcd quorum WITHOUT cluster-init. Empty = the first server, which - // cluster-inits the datastore. A non-empty ServerURL requires JoinToken. + // ServerURL boots this VM as a JOIN server joining the quorum at the given endpoint. + // Empty = first server (cluster-init). Non-empty requires JoinToken. ServerURL string - // BuiltinIngress keeps K3s' bundled traefik + servicelb enabled (interim - // in-VPC app exposure). When false the config disables them for AWS parity, - // where Service type=LoadBalancer / Ingress are satisfied by the AWS Load - // Balancer Controller instead. The CreateCluster default is ON until that - // controller ships (see managedIngressTagKey). + // BuiltinIngress keeps K3s' bundled traefik + servicelb enabled. + // When false, both are disabled for AWS parity (ingress via the LB Controller). BuiltinIngress bool } -// K3sServerOutput carries the identifiers the caller needs to persist into -// ClusterMeta and to register the ENI IP with the cluster NLB target group. +// K3sServerOutput carries identifiers to persist in ClusterMeta and register with the NLB. type K3sServerOutput struct { InstanceID string ENIID string @@ -176,15 +126,9 @@ type K3sServerOutput struct { MgmtIP string } -// LaunchK3sServerVM provisions the K3s control-plane VM for an EKS cluster. -// Sequence: resolve the eks-server AMI, pre-create the customer-account ENI -// in the supplied subnet with the control-plane SG, render cloud-init user -// data (env vars, OIDC PEM, K3s config), then call RunInstances with -// NetworkInterfaces[0].NetworkInterfaceId pointing at the pre-created ENI. -// Returns instance/ENI identity so the caller can register the ENI IP with -// the cluster NLB target group and persist the IDs in ClusterMeta. On -// RunInstances failure the pre-created ENI is deleted best-effort so the -// caller does not leak a customer-account resource. +// LaunchK3sServerVM provisions the K3s CP VM: resolves the AMI, pre-creates the +// ENI, renders cloud-init user-data, then launches via RunInstances. On failure +// the ENI is deleted best-effort to avoid leaking a customer-account resource. func LaunchK3sServerVM( vpcSvc k3sVPCProvisioner, instSvc k3sInstanceLauncher, @@ -267,10 +211,8 @@ func LaunchK3sServerVM( }, nil } -// TerminateK3sServerVM tears down the K3s server VM for a cluster. The -// instance termination cascades ENI detach inside the customer instance path, -// then this helper deletes the ENI explicitly so DeleteCluster does not leak -// it. Missing instance/ENI is a no-op so retries stay safe. +// TerminateK3sServerVM terminates the K3s server VM and deletes the ENI. +// Missing instance/ENI is a no-op for idempotent retries. func TerminateK3sServerVM( vpcSvc k3sVPCProvisioner, instSvc k3sInstanceLauncher, @@ -281,9 +223,7 @@ func TerminateK3sServerVM( } var firstErr error if instanceID != "" { - // A retried DeleteCluster reaches here after the VM already drained, so - // "instance not found" is the steady-state success case, not a failure — - // treat it as idempotent so teardown can proceed to the ENI/SG/KV sweep. + // "instance not found" on a retry is idempotent success; proceed to the ENI/SG/KV sweep. if err := instSvc.TerminateSystemInstance(instanceID); err != nil { if errors.Is(err, sysinstance.ErrSystemInstanceNotFound) { slog.Debug("TerminateK3sServerVM: instance already gone", "instanceId", instanceID) @@ -300,15 +240,10 @@ func TerminateK3sServerVM( switch { case awserrors.IsErrorCode(err, awserrors.ErrorInvalidNetworkInterfaceIDNotFound), awserrors.IsErrorCode(err, awserrors.ErrorInvalidNetworkInterfaceNotFound): - // The instance-terminate cascade (or a prior retry) already - // deleted the ENI. Idempotent success — must NOT block the SG + - // KV sweep, or the cluster wedges in DELETING permanently. + // ENI already gone (instance-terminate cascade or prior retry); idempotent success. slog.Debug("TerminateK3sServerVM: ENI already gone", "eniId", eniID) default: - // InvalidNetworkInterface.InUse (the VM is still terminating - // async and holds the ENI) and any other error are retryable: - // surface so the cluster stays DELETING and the reconciler - // retries once the instance releases the ENI. + // InUse = VM still terminating async; surface error so the reconciler retries. slog.Warn("TerminateK3sServerVM: ENI delete failed", "eniId", eniID, "err", err) if firstErr == nil { firstErr = fmt.Errorf("delete ENI %s: %w", eniID, err) @@ -319,12 +254,8 @@ func TerminateK3sServerVM( return firstErr } -// lookupEKSServerAMI resolves the EKS control-plane AMI by the -// spinifex:managed-by=eks tag the build pipeline stamps on it, rather than a -// brittle exact name. This survives the planned server+agent → single EKS AMI -// unify untouched (the unified AMI keeps the tag; role is chosen per-instance -// at launch). If more than one AMI carries the tag (e.g. server + agent both -// imported pre-unify), the newest by CreationDate wins. +// lookupEKSServerAMI resolves the EKS CP AMI by the spinifex:managed-by=eks tag +// rather than a brittle exact name. If multiple AMIs match, the newest wins. func lookupEKSServerAMI(amiSvc k3sAMIResolver, accountID string) (string, error) { out, err := amiSvc.DescribeImages(&ec2.DescribeImagesInput{ Filters: []*ec2.Filter{ @@ -401,11 +332,8 @@ func validateK3sServerInput(in K3sServerInput) error { return nil } -// GenerateK3sClusterToken mints the shared k3s cluster token (256 bits of -// crypto-random, hex-encoded) seeded into every control-plane server so servers -// 2..N join the first server's etcd quorum and workers register. Generated per -// CreateCluster; not persisted (the node-token derived from it is published on -// the bootstrap bus for workers). +// GenerateK3sClusterToken mints the shared k3s cluster token (256 bits, hex-encoded). +// Servers 2..N and workers use it to join; the derived node-token is published on the bootstrap bus. func GenerateK3sClusterToken() (string, error) { var b [32]byte if _, err := rand.Read(b[:]); err != nil { @@ -414,9 +342,7 @@ func GenerateK3sClusterToken() (string, error) { return hex.EncodeToString(b[:]), nil } -// k3sServerJoinURL builds the registration endpoint a join server dials: the -// first server's ENI private IP on the k3s supervisor port. Token-based join is -// trust-on-first-use, so the IP need not be in the apiserver cert SANs. +// k3sServerJoinURL builds the join endpoint (first server's ENI IP on port 6443). func k3sServerJoinURL(ip string) string { return "https://" + net.JoinHostPort(ip, "6443") } @@ -442,11 +368,7 @@ type userDataFile struct { func buildK3sUserData(in K3sServerInput) string { nlbEndpoint := "https://" + net.JoinHostPort(in.NLBDNS, strconv.FormatInt(clusterNLBListenPort, 10)) - // Role the eks-node-role first-boot selector reads to enable the - // control-plane services. The first server is "server" (cluster-init + - // bootstrap publisher); servers 2..N are "server-join" (register into the - // existing quorum, no bootstrap re-publish). Nodegroup workers seed "agent" - // through their own path. + // "server" = first CP (cluster-init + bootstrap publisher); "server-join" = HA join server. role := "server" if in.ServerURL != "" { role = "server-join" @@ -463,42 +385,15 @@ func buildK3sUserData(in K3sServerInput) string { "EKS_CLUSTER_NAME=" + in.ClusterName, "EKS_NLB_ENDPOINT=" + nlbEndpoint, "EKS_OIDC_ISSUER=" + in.OIDCIssuer, - // Gate deferred built-in ingress: k3s.initd stages a traefik .skip marker - // before boot and the state-reporter removes it once the apiserver is - // stable. Set only for clusters that want built-in ingress. + // Deferred ingress: k3s.initd stages a traefik .skip marker; state-reporter removes it once stable. "EKS_DEFER_TRAEFIK=" + boolFlag(in.BuiltinIngress), }, "\n") - // cluster-init (first server) selects the embedded etcd datastore (required - // for multi-server HA); servers 2..N instead carry `server: ` + the - // shared token and join the quorum without cluster-init. - // etcd-expose-metrics surfaces etcd's own wal_fsync_duration_seconds and - // backend_commit_duration_seconds on 127.0.0.1:2381/metrics so control-plane - // commit latency is measurable directly, not inferred. - // - // anonymous-auth=true: k3s hardens it off by default, which makes the - // apiserver answer 401 to an unauthenticated /healthz. The cluster reconciler - // probes https:///healthz anonymously to gate ACTIVE, so it must be - // reachable; the default RBAC binds only the health/version non-resource - // paths to system:unauthenticated, so this exposes nothing else. - // In real EKS neither traefik nor servicelb exists; Service type=LoadBalancer - // and Ingress are reconciled by the AWS Load Balancer Controller. When a - // cluster does not opt into built-in ingress (BuiltinIngress=false) the K3s - // built-ins are disabled outright for parity. - // - // When a cluster DOES want built-in ingress (dev / interim, until the AWS LB - // controller add-on lands) traefik stays ENABLED in config so k3s writes its - // manifest — but its helm-install Job (klipper-helm image pull + chart - // install) hammers the embedded etcd during the fragile bootstrap window, - // which can tip etcd fsync latency past the apiserver / leaderelection - // deadlines and crash the control plane. So traefik is deferred, NOT disabled: - // k3s.initd stages a deploy-controller .skip marker before the apiserver - // starts (EKS_DEFER_TRAEFIK gates it), and the state-reporter removes that - // marker once the apiserver is sustainedly healthy — traefik then installs - // with etcd headroom. Disabling traefik (`disable: traefik`) is wrong for this: - // k3s never writes traefik.yaml, leaving no manifest to un-skip. servicelb - // (klipper) is lazy — it acts only when a LoadBalancer Service exists — so it - // carries no bootstrap cost and stays enabled whenever built-in ingress is on. + // First server uses cluster-init (embedded etcd); join servers set `server: ` + token. + // etcd-expose-metrics: surfaces etcd fsync/commit latency on 127.0.0.1:2381/metrics. + // anonymous-auth=true: reconciler probes /healthz unauthenticated to gate ACTIVE; RBAC limits exposure. + // When BuiltinIngress=false, traefik+servicelb are disabled for AWS LB Controller parity. + // When true, traefik is deferred (not disabled) to avoid hammering etcd during bootstrap. var configLines []string if in.ServerURL == "" { configLines = append(configLines, "cluster-init: true") @@ -510,13 +405,7 @@ func buildK3sUserData(in K3sServerInput) string { } configLines = append(configLines, "etcd-expose-metrics: true", - // Keep user workloads off the control plane (EKS parity — the CP is never - // a worker). Critical here because the server node is schedulable by - // default: a workload landing on the CP pulls its images onto the same - // NBD/viperblock disk the embedded etcd fsyncs to, starving etcd until the - // apiserver crashes and the VM OOMs. k3s' packaged addons tolerate - // CriticalAddonsOnly, so they still run (here, draining to the nodegroup); - // NoExecute also evicts anything already scheduled, not just future pods. + // Prevent user workloads on the CP (EKS parity). k3s packaged addons tolerate CriticalAddonsOnly. "node-taint:", " - CriticalAddonsOnly=true:NoExecute", ) @@ -528,9 +417,7 @@ func buildK3sUserData(in K3sServerInput) string { ) } configLines = append(configLines, "tls-san:", " - "+in.NLBDNS) - // The reachable front-end IP must be a cert SAN so kubectl reaching the - // cluster on https://:443 validates TLS. k3s accepts IP literals - // in tls-san and classifies them as IP SANs. + // EndpointIP must be a cert SAN for TLS validation via https://:443. if in.EndpointIP != "" { configLines = append(configLines, " - "+in.EndpointIP) } @@ -543,30 +430,21 @@ func buildK3sUserData(in K3sServerInput) string { " - anonymous-auth=true", " - authentication-token-webhook-config-file="+k3sTokenWebhookKubeconfigPath, " - authentication-token-webhook-cache-ttl=5m", - // Pin v1 so the apiserver decodes the webhook's authentication.k8s.io/v1 - // TokenReview response; the default v1beta1 rejects the GVK mismatch (401). + // v1: default v1beta1 rejects authentication.k8s.io/v1 TokenReview response (401). " - authentication-token-webhook-version=v1", ) k3sConfig := strings.Join(configLines, "\n") files := []userDataFile{ - // first-boot.env carries the system SigV4 secret key, so keep it - // root-only (0600). + // 0600: contains system SigV4 secret key. {Path: k3sFirstBootEnvPath, Perms: "0600", Body: envBody}, {Path: k3sOIDCSigningKeyPath, Perms: "0600", Body: strings.TrimRight(in.OIDCPrivateKeyPEM, "\n")}, {Path: k3sOIDCPublicKeyPath, Perms: "0644", Body: strings.TrimRight(in.OIDCPublicKeyPEM, "\n")}, {Path: k3sConfigPath, Perms: "0644", Body: k3sConfig}, {Path: k3sGatewayCAPath, Perms: "0644", Body: strings.TrimRight(in.GatewayCACert, "\n")}, - // IMDS on-link route. Alpine's cloud-init eni renderer crashes on a - // gateway-less route in network-config, so it's delivered out-of-band: - // a persistent local.d script (re-applied every boot by the OpenRC - // `local` service, enabled via runcmd below) ARPs 169.254.169.254 - // directly on the VPC subnet. This block owns the only write_files key, - // so it must carry the route itself — appending a second write_files via - // the generic cloud-init template would collide (last key wins, silently - // dropping the k3s config). The device is resolved via the default route - // (the VPC egress NIC), not a name: the persistent-net rename to vpc0 - // doesn't apply on the Alpine AMI, so the kernel name is eth0 at boot. + // IMDS on-link route via persistent local.d script: Alpine cloud-init crashes on a + // gateway-less network-config route, so it's written here instead. Uses default-route + // device (eth0 on Alpine, not vpc0) to ARP 169.254.169.254 directly on the VPC subnet. { Path: "/etc/local.d/imds-onlink-route.start", Perms: "0755", @@ -579,14 +457,8 @@ func buildK3sUserData(in K3sServerInput) string { var buf strings.Builder buf.WriteString("#cloud-config\n") - // Resolver via bootcmd, NOT write_files: on the Alpine AMI /etc/resolv.conf - // is a dangling symlink (its target dir does not exist — which is why the - // dhcpcd hook cannot persist DHCP DNS), and pointing write_files at it makes - // cloud-init follow the dead link, fail, and abort the WHOLE write_files - // block (dropping first-boot.env + the k3s config). bootcmd runs before - // write_files in the init stage and as a shell, so it can drop the symlink - // and write a real file. Containerd needs this to resolve registry-1.docker. - // io for system-pod image pulls; reachable over the cluster egress SNAT. + // bootcmd (not write_files): /etc/resolv.conf is a dangling symlink on Alpine; write_files + // follows it, fails, and aborts the entire block. bootcmd drops the symlink first. buf.WriteString("bootcmd:\n") buf.WriteString(" - rm -f " + k3sResolvConfPath + "\n") fmt.Fprintf(&buf, " - printf '%s\\n' > %s\n", @@ -604,8 +476,7 @@ func buildK3sUserData(in K3sServerInput) string { } } - // Enable + start the OpenRC `local` service so the IMDS route script runs on - // first boot and is re-applied on every subsequent boot. + // Enable OpenRC `local` so the IMDS route script runs on every boot. buf.WriteString("runcmd:\n") buf.WriteString(" - [ rc-update, add, local, default ]\n") buf.WriteString(" - [ rc-service, local, start ]\n") diff --git a/spinifex/handlers/eks/nats_bootstrap.go b/spinifex/handlers/eks/nats_bootstrap.go index fd7a64dbe..b756684e9 100644 --- a/spinifex/handlers/eks/nats_bootstrap.go +++ b/spinifex/handlers/eks/nats_bootstrap.go @@ -14,19 +14,13 @@ import ( "github.com/nats-io/nats.go" ) -// errBootstrapPermanent marks a bootstrap persist failure that no amount of -// retrying will fix — a malformed envelope, a non-base64 CA, or a JWKS whose -// kid does not match the controller-generated keypair. Transient failures -// (KV write blips, JWKS-not-yet-propagated) are NOT wrapped with this, so -// persistWithRetry keeps retrying them. A permanent error fails the cluster. +// errBootstrapPermanent marks failures that retrying won't fix (malformed envelope, +// non-base64 CA, JWKS kid mismatch). Transient errors (KV blips) are NOT wrapped. var errBootstrapPermanent = errors.New("bootstrap permanent error") const ( - // bootstrapPersistAttempts bounds inline retries of a single subject's - // persist op on transient errors. A core-NATS publication is delivered - // once, so a transient KV blip must be ridden out in-handler rather than - // by tearing down the subscription and waiting for a re-publish that - // never comes. + // bootstrapPersistAttempts bounds inline retries on transient errors. + // Core-NATS delivers each publication once, so KV blips must be retried in-handler. bootstrapPersistAttempts = 3 bootstrapPersistBackoff = 200 * time.Millisecond ) @@ -62,29 +56,21 @@ func BootstrapSubject(accountID, clusterName, kind string) string { return fmt.Sprintf("eks.bus.%s.%s.%s", accountID, clusterName, kind) } -// BootstrapEnvelope is the JSON wire shape every k3s-first-boot.sh -// publication uses. Each subject populates the field matching its kind; the -// rest stay empty. Versioned implicitly by field shape — adding new optional -// fields is backwards-compatible. +// BootstrapEnvelope is the JSON wire shape for k3s-first-boot.sh publications. +// Each subject populates its matching field; unused fields stay empty. type BootstrapEnvelope struct { // Token is the K3s bootstrap node-token for the cluster (k3s-bootstrap-token). Token string `json:"token,omitempty"` - // AdminKubeconfig is the cluster-admin kubeconfig YAML - // (k3s-admin-kubeconfig). + // AdminKubeconfig is the cluster-admin kubeconfig YAML. AdminKubeconfig string `json:"adminKubeconfig,omitempty"` - // JWKS is the K3s-published RFC 7517 JWKS document. The bootstrap - // subscriber cross-checks it against the controller-generated keypair - // (k3s-oidc-jwks). + // JWKS is the K3s-published JWKS document; cross-checked against the controller-generated keypair. JWKS string `json:"jwks,omitempty"` - // CertificateAuthority is the base64-encoded K3s server CA PEM - // (k3s-ca). Written onto meta.CertificateAuthorityB64. + // CertificateAuthority is the base64-encoded K3s server CA PEM; written to meta.CertificateAuthorityB64. CertificateAuthority string `json:"certificateAuthority,omitempty"` } -// NATSBootstrap collects the four single-shot bootstrap publications from a -// K3s server VM, validates the OIDC JWKS against what the controller wrote -// pre-VM, persists each artifact into the per-cluster KV bucket, and writes -// the CA onto meta.CertificateAuthorityB64. The reconciler observes +// NATSBootstrap collects the four single-shot bootstrap publications from the K3s VM, +// validates the OIDC JWKS, and persists each artifact to KV. Reconciler observes // completion via the four KV keys + meta.CertificateAuthorityB64. type NATSBootstrap struct { nc *nats.Conn @@ -121,20 +107,10 @@ func NewNATSBootstrap(nc *nats.Conn, kv nats.KeyValue, masterKey []byte, account }, nil } -// BootstrapPendingKinds reports which bootstrap subjects still lack their -// persisted artifact for a cluster, so a daemon-restart resume re-subscribes -// only to what is missing. The K3s VM publishes each subject once over core -// NATS and never republishes, so re-waiting on an already-arrived subject -// would deadlock. -// -// JWKS is tracked by its verified-marker (OIDCJWKSVerifiedKey), written by -// persistJWKS only after the cross-check passes — NOT by OIDCJWKSKey, which the -// controller pre-seeds at create time. Until the marker exists the cross-check -// has not positively passed, so resume re-subscribes to JWKS like any other -// pending subject. (If the VM published JWKS during the restart window the -// re-subscribe waits for a republish that never comes and the cluster fails on -// the create timeout — the same one-shot limitation every bootstrap subject -// shares, now applied uniformly so ACTIVE always implies a verified key.) +// BootstrapPendingKinds reports which bootstrap artifacts are still missing so +// a daemon-restart resumes only on pending subjects (K3s publishes each once). +// JWKS is tracked via OIDCJWKSVerifiedKey (set after cross-check), not OIDCJWKSKey +// (pre-seeded by the controller), so ACTIVE always implies a verified keypair. func BootstrapPendingKinds(kv nats.KeyValue, clusterName string, meta *ClusterMeta) []string { var pending []string if _, err := kv.Get(NodeTokenKey(clusterName)); err != nil { @@ -192,18 +168,9 @@ func (b *NATSBootstrap) Run(ctx context.Context) error { return b.RunForKinds(ctx, nil) } -// RunForKinds subscribes only to the requested bootstrap subjects (nil = all -// four) and returns when: -// - every requested subject has arrived and persisted successfully (nil), -// - ctx is cancelled (ctx.Err()), -// - one of the persistence handlers returns an error (first error). -// -// The subset form drives daemon-restart recovery: a cluster that already has -// some artifacts in KV re-subscribes only to the missing ones, since the K3s -// VM published each subject once over core NATS and will not republish. -// -// Each subject is consumed at most once; subsequent publications on the same -// subject are dropped. Subscriptions are always cleaned up before return. +// RunForKinds subscribes to the requested bootstrap subjects (nil = all four) and +// returns when all arrive, ctx is cancelled, or a handler errors. Drives daemon-restart +// recovery by re-subscribing only to missing artifacts. Each subject is consumed at most once. func (b *NATSBootstrap) RunForKinds(ctx context.Context, kinds []string) error { subs, err := b.subjectHandlers(kinds) if err != nil { @@ -319,10 +286,7 @@ func (b *NATSBootstrap) persistJWKS(data []byte) error { if err := assertJWKSMatch(existing.Value(), []byte(env.JWKS)); err != nil { return err } - // Record the positive cross-check as a distinct marker. The reconciler - // gates ACTIVE on this key — never on OIDCJWKSKey, which the controller - // pre-seeds at create time — so a cluster cannot reach ACTIVE on an - // unverified or mismatched signing key. + // Reconciler gates ACTIVE on this verified-marker, not OIDCJWKSKey (pre-seeded by controller). if _, err := b.kv.Put(OIDCJWKSVerifiedKey(b.clusterName), []byte("verified")); err != nil { return fmt.Errorf("kv put %s: %w", OIDCJWKSVerifiedKey(b.clusterName), err) } @@ -351,12 +315,8 @@ func unmarshalBootstrapEnvelope(data []byte) (BootstrapEnvelope, error) { return env, nil } -// assertJWKSMatch verifies that every key in the incoming JWKS matches a -// controller-generated key by both key id (kid) and key type (kty). A kid -// mismatch means the VM generated its own signing key instead of consuming our -// PEM; a kty mismatch (e.g. an RSA key carrying our EC key's kid) means the -// published key cannot be the one the controller distributed even though the -// kid collides. Either rejects the publication. +// assertJWKSMatch verifies every incoming JWKS key matches the controller-generated +// keypair by kid and kty; a mismatch rejects the publication. func assertJWKSMatch(existing, incoming []byte) error { var ex, in oidcJWKS if err := json.Unmarshal(existing, &ex); err != nil { diff --git a/spinifex/handlers/eks/nlb.go b/spinifex/handlers/eks/nlb.go index e1fc0c5c1..efa735ee5 100644 --- a/spinifex/handlers/eks/nlb.go +++ b/spinifex/handlers/eks/nlb.go @@ -10,9 +10,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/tags" ) -// nlbProvisioner is the subset of handlers_elbv2.ELBv2Service that the -// cluster NLB helpers need. Narrow so tests can fake with hand-rolled -// recorders without implementing the full ELBv2 surface. +// nlbProvisioner is the narrow ELBv2 surface needed by cluster NLB helpers. type nlbProvisioner interface { CreateLoadBalancer(input *elbv2.CreateLoadBalancerInput, accountID string) (*elbv2.CreateLoadBalancerOutput, error) DescribeLoadBalancers(input *elbv2.DescribeLoadBalancersInput, accountID string) (*elbv2.DescribeLoadBalancersOutput, error) @@ -31,38 +29,28 @@ type nlbProvisioner interface { SetLoadBalancerIngressCIDRs(lbArn string, cidrs []string, accountID string) error } -// AWS load-balancer + target-group names are bounded to 32 alphanumeric or -// hyphen characters. EnsureClusterNLB rejects clusters whose derived name -// would exceed this before issuing the create call so callers see a clear -// error instead of a generic ELBv2 InvalidParameterValue. +// maxELBv2NameLen is the AWS limit on LB/TG names (alphanumeric + hyphens). const maxELBv2NameLen = 32 -// k3sAPIServerPort is the in-VM K3s api-server bind port. The NLB target -// group sends traffic to this port on the K3s server VM's ENI. +// k3sAPIServerPort is the K3s apiserver port; NLB target group forwards here. const k3sAPIServerPort int64 = 6443 // clusterNLBListenPort is the customer-facing port the cluster NLB exposes. // kubectl + the AWS SDK both expect TLS on 443. const clusterNLBListenPort int64 = 443 -// ClusterNLB is the in-process tuple produced by EnsureClusterNLB. Each ARN -// is persisted into ClusterMeta so the reconciler + DeleteCluster can act on -// the resources without re-resolving by name. +// ClusterNLB is the NLB resource tuple produced by EnsureClusterNLB; ARNs are persisted to ClusterMeta. type ClusterNLB struct { LoadBalancerArn string TargetGroupArn string ListenerArn string DNSName string - // FrontendIP is the NLB's reachable front-end address: the external-pool - // public IP for an internet-facing cluster NLB, or the VPC private IP for an - // internal one. Used as the kubeconfig endpoint host + apiserver cert SAN. - // Empty if the backing LB record has no address yet. + // FrontendIP is the public IP (internet-facing) or VPC IP (internal) of the NLB front-end. + // Used as the kubeconfig endpoint host and apiserver cert SAN; empty if not yet provisioned. FrontendIP string } -// ClusterNLBName returns the deterministic NLB name for a cluster. Stable so -// DeleteCluster can locate the NLB from clusterName alone after a daemon -// restart that lost the in-memory ARN. +// ClusterNLBName returns the deterministic NLB name for a cluster. func ClusterNLBName(clusterName string) string { return "eks-" + clusterName } @@ -73,24 +61,9 @@ func ClusterTargetGroupName(clusterName string) string { return "eks-" + clusterName + "-cp" } -// EnsureClusterNLB provisions (or returns) the cluster's NLB + target group + -// :443→:6443 TCP listener. Idempotent on clusterName: existing resources are -// reused so DeleteCluster failure-retry and reconciler crash-recovery both -// converge without duplicate creates. -// -// The K3s server VM ENI IP is NOT registered here — Stage 4 -// (k3s_server_vm.go) calls RegisterClusterTarget once the VM has its ENI IP. -// -// internetFacing selects the NLB scheme: true ⇒ internet-facing (external-pool -// front-end IP, LAN-reachable, for a public cluster endpoint); false ⇒ internal -// (VPC-only). After ensuring the LB, the reachable FrontendIP is read back from -// the backing LB record so the caller can use it as the endpoint host + cert SAN. -// -// publicAccessCidrs narrows who may reach an internet-facing front-end below the -// scheme default (0.0.0.0/0). It maps the cluster's publicAccessCidrs onto the -// NLB managed-SG ingress; ignored for an internal NLB (its ingress already -// tracks the VPC CIDR) and for the wide-open default, which the LB carries out -// of the box. +// EnsureClusterNLB provisions (or reuses) the cluster NLB, target group, and +// :443→:6443 TCP listener. Idempotent; ENI registration is done separately by +// RegisterClusterTarget. publicAccessCidrs narrows ingress for internet-facing LBs. func EnsureClusterNLB(nlbp nlbProvisioner, accountID, clusterName string, subnetIDs []string, internetFacing bool, publicAccessCidrs []string) (*ClusterNLB, error) { if clusterName == "" { return nil, errors.New("eks: EnsureClusterNLB empty cluster name") @@ -118,17 +91,12 @@ func EnsureClusterNLB(nlbp nlbProvisioner, accountID, clusterName string, subnet if err := ensureClusterListener(nlbp, accountID, lbName, out); err != nil { return nil, err } - // Narrow the internet-facing front-end to the cluster's publicAccessCidrs - // when they restrict below the wide-open scheme default the LB already - // carries. The override is idempotent, so re-running on retry converges. if internetFacing && narrowsPublicAccess(publicAccessCidrs) { if err := nlbp.SetLoadBalancerIngressCIDRs(out.LoadBalancerArn, publicAccessCidrs, accountID); err != nil { return nil, fmt.Errorf("eks: set NLB ingress CIDRs for %s: %w", lbName, err) } } - // Read the front-end IP back from the LB record (DescribeLoadBalancers → - // populated LoadBalancerAddresses). Best-effort: an empty IP is not fatal - // here — the caller falls back to the DNS-name endpoint. + // Best-effort front-end IP read-back; caller falls back to DNS name if empty. if lb, err := lookupLBByName(nlbp, accountID, lbName); err == nil && lb != nil { out.FrontendIP = frontendIPFromLB(lb, internetFacing) } else if err != nil { @@ -137,10 +105,7 @@ func EnsureClusterNLB(nlbp nlbProvisioner, accountID, clusterName string, subnet return out, nil } -// narrowsPublicAccess reports whether publicAccessCidrs restrict the front-end -// below the wide-open default an internet-facing NLB already carries. Empty (no -// override) or the lone 0.0.0.0/0 default are no-ops and skip the extra ELBv2 -// round-trip. +// narrowsPublicAccess reports whether publicAccessCidrs restrict below the 0.0.0.0/0 default. func narrowsPublicAccess(cidrs []string) bool { if len(cidrs) == 0 { return false @@ -151,9 +116,8 @@ func narrowsPublicAccess(cidrs []string) bool { return true } -// frontendIPFromLB extracts the reachable front-end address from a described -// load balancer: the public IpAddress for an internet-facing LB, else the VPC -// PrivateIPv4Address. Returns the first non-empty match across AZs. +// frontendIPFromLB returns the public IpAddress (internet-facing) or PrivateIPv4Address (internal) +// from the described LB; first non-empty match across AZs. func frontendIPFromLB(lb *elbv2.LoadBalancer, internetFacing bool) string { for _, az := range lb.AvailabilityZones { for _, addr := range az.LoadBalancerAddresses { @@ -174,8 +138,7 @@ func frontendIPFromLB(lb *elbv2.LoadBalancer, internetFacing bool) string { return "" } -// RegisterClusterTarget attaches one K3s server ENI IP to the cluster TG. Thin -// wrapper over RegisterClusterTargets for callers with a single IP. +// RegisterClusterTarget attaches one ENI IP to the cluster TG. func RegisterClusterTarget(nlbp nlbProvisioner, accountID, tgArn, eniIP string) error { if eniIP == "" { return errors.New("eks: RegisterClusterTarget empty ENI IP") @@ -183,12 +146,8 @@ func RegisterClusterTarget(nlbp nlbProvisioner, accountID, tgArn, eniIP string) return RegisterClusterTargets(nlbp, accountID, tgArn, []string{eniIP}) } -// RegisterClusterTargets attaches every control-plane server ENI IP to the -// cluster TG so the NLB forwards :443 traffic to :6443 on each CP VM. With all -// N HA servers registered, the TG health check (:6443) drains a member when its -// node dies and kubectl/apiserver traffic fails over to the survivors. Empty -// IPs are skipped; the ELBv2 RegisterTargets impl dedupes on (id, port), so -// re-invocation (e.g. by the reconciler) is idempotent. +// RegisterClusterTargets attaches all CP ENI IPs to the cluster TG. Empty IPs are +// skipped; RegisterTargets deduplicates on (id, port) so re-invocation is idempotent. func RegisterClusterTargets(nlbp nlbProvisioner, accountID, tgArn string, eniIPs []string) error { if tgArn == "" { return errors.New("eks: RegisterClusterTargets empty TG arn") @@ -215,9 +174,7 @@ func RegisterClusterTargets(nlbp nlbProvisioner, accountID, tgArn string, eniIPs return nil } -// DeregisterClusterTarget removes the K3s server ENI IP from the cluster TG. -// Called by DeleteCluster before VM termination so health checks stop hitting -// a dying target. +// DeregisterClusterTarget removes an ENI IP from the cluster TG before VM termination. func DeregisterClusterTarget(nlbp nlbProvisioner, accountID, tgArn, eniIP string) error { if tgArn == "" { return errors.New("eks: DeregisterClusterTarget empty TG arn") @@ -237,11 +194,8 @@ func DeregisterClusterTarget(nlbp nlbProvisioner, accountID, tgArn, eniIP string return nil } -// DeleteClusterNLB tears down the cluster NLB + TG. DeleteLoadBalancer -// cascades listener cleanup inside the ELBv2 impl, so listeners do not need -// an explicit delete pass. Missing resources are no-ops to keep DeleteCluster -// retries safe. Both sub-sweeps always run; LB delete error takes precedence -// over TG delete error when both fail. +// DeleteClusterNLB tears down the cluster NLB and TG. DeleteLoadBalancer cascades +// listener cleanup. Missing resources are no-ops; LB error takes precedence if both fail. func DeleteClusterNLB(nlbp nlbProvisioner, accountID, clusterName string) error { if clusterName == "" { return errors.New("eks: DeleteClusterNLB empty cluster name") diff --git a/spinifex/handlers/eks/nodegroup.go b/spinifex/handlers/eks/nodegroup.go index 508d5cbf7..89201b3cc 100644 --- a/spinifex/handlers/eks/nodegroup.go +++ b/spinifex/handlers/eks/nodegroup.go @@ -226,12 +226,7 @@ func (s *EKSServiceImpl) createNodegroup(acctKV nats.KeyValue, input *eks.Create ModifiedAt: now, } - // Atomically claim the nodegroup record before any SG provisioning or worker - // launch. This is the idempotency barrier: a duplicate CreateNodegroup (SDK - // retry / gateway re-publish that spawned a second handler) loses the claim - // here and returns without doing any provisioning work, so concurrent creates - // cannot double-launch workers. The L169 read above stays only as a cheap - // fast-path reject. + // Atomically claim the record; concurrent/duplicate CreateNodegroup requests lose here. owned, err := ClaimNodegroupRecord(acctKV, rec) if err != nil { return nil, err @@ -240,13 +235,8 @@ func (s *EKSServiceImpl) createNodegroup(acctKV nats.KeyValue, input *eks.Create return nil, errors.New(awserrors.ErrorEKSResourceInUse) } - // Claim won — the nodegroup is now CREATING. SG provisioning, token decrypt, - // AMI lookup and worker launch are a multi-minute job that must not run in the - // gateway request path (it head-of-line blocks the shared responder and the - // gateway times out and re-publishes — 267.4). Snapshot the CREATING reply - // before the launch, which mutates rec (InstanceIDs/Status), then run the - // provisioning on a background goroutine. Failures surface via the record's - // CREATE_FAILED status, which DeleteNodegroup reclaims. + // Snapshot the CREATING reply before background launch (launch mutates rec). + // Failures surface as CREATE_FAILED, which DeleteNodegroup reclaims. out := &eks.CreateNodegroupOutput{Nodegroup: nodegroupRecordToAWS(rec)} s.launchWG.Go(func() { @@ -325,9 +315,7 @@ func (s *EKSServiceImpl) launchNodegroupInfra(lc nodegroupLaunchCtx) { return } - // v1 marks ACTIVE on successful RunInstances (instance-level), not on - // k8s node-Ready — node-Ready gating needs a kubectl probe from the - // reconciler and lands in a follow-up. + // v1: ACTIVE on RunInstances success; node-Ready gating is a future improvement. rec.Status = eks.NodegroupStatusActive rec.ModifiedAt = time.Now().UTC() if err := PutNodegroupRecord(acctKV, rec); err != nil { @@ -469,9 +457,7 @@ func (s *EKSServiceImpl) updateNodegroupConfig(acctKV nats.KeyValue, input *eks. } if err := s.reconcileWorkerCount(acctKV, accountID, rec, meta); err != nil { - // reconcileWorkerCount may have launched some workers before failing; their - // IDs are already on rec. Persist so DeleteNodegroup can reclaim them rather - // than leaking the partial scale-up. + // Persist partial IDs so DeleteNodegroup can reclaim a failed scale-up. rec.ModifiedAt = time.Now().UTC() if perr := PutNodegroupRecord(acctKV, rec); perr != nil { slog.Error("updateNodegroupConfig: persist after reconcile failure", @@ -533,8 +519,7 @@ func (s *EKSServiceImpl) reconcileWorkerCount(acctKV nats.KeyValue, accountID st } func (s *EKSServiceImpl) updateNodegroupVersion(input *eks.UpdateNodegroupVersionInput) (*eks.UpdateNodegroupVersionOutput, error) { - // Worker AMI version upgrades (drain + replace) land in a follow-up; v1 - // nodegroups are fixed at their create-time version. + // Worker AMI version upgrades (drain + replace) are not implemented in v1. if input != nil { slog.Info("UpdateNodegroupVersion not implemented in v1", "cluster", aws.StringValue(input.ClusterName), "nodegroup", aws.StringValue(input.NodegroupName)) diff --git a/spinifex/handlers/eks/nodegroup_userdata.go b/spinifex/handlers/eks/nodegroup_userdata.go index 40329d2bd..4925ee72d 100644 --- a/spinifex/handlers/eks/nodegroup_userdata.go +++ b/spinifex/handlers/eks/nodegroup_userdata.go @@ -17,21 +17,10 @@ type agentUserDataInput struct { NodeName string } -// buildAgentUserData renders the cloud-config YAML a nodegroup worker consumes -// at first boot. It mirrors buildK3sUserData: a bootcmd resolver fix, a single -// write_files block (first-boot.env + agent.env + the IMDS on-link route -// script), and a runcmd that enables the OpenRC `local` service. Output is -// unencoded YAML; the caller base64-wraps it for the RunInstances UserData -// field. The eks-node-role first-boot selector reads SPINIFEX_K3S_ROLE=agent -// (and the presence of agent.env) to enable the k3s-agent service, which sources -// agent.env for K3S_URL/K3S_TOKEN. +// buildAgentUserData renders the cloud-config YAML for a nodegroup worker VM. +// Mirrors buildK3sUserData structure; SPINIFEX_K3S_ROLE=agent enables k3s-agent. func buildAgentUserData(in agentUserDataInput) string { - // K3S_URL targets the control-plane ENI's in-VPC private IP on :6443 - // because the cluster NLB DNS endpoint is not resolvable in-VPC yet. - // Workers share the VPC with the server, so this is directly reachable and - // TLS-pinned by the join token's CA hash (SAN/hostname irrelevant for the - // k3s agent join). Switch to the NLB endpoint on :443 once authoritative - // in-VPC DNS is available. + // K3S_URL uses the CP ENI IP directly (NLB DNS not yet resolvable in-VPC). k3sURL := "https://" + net.JoinHostPort(in.ControlPlaneENIIP, "6443") nodeLabel := "eks.amazonaws.com/nodegroup=" + in.NodegroupName @@ -61,10 +50,7 @@ func buildAgentUserData(in agentUserDataInput) string { var buf strings.Builder buf.WriteString("#cloud-config\n") - // Resolver fix via bootcmd (not write_files) for the same reason as the - // server path: /etc/resolv.conf is a dangling symlink on the Alpine AMI, so - // write_files would follow the dead link and abort the whole block. - // Containerd needs a working resolver to pull the worker's system images. + // bootcmd (not write_files): /etc/resolv.conf is a dangling symlink on Alpine; see buildK3sUserData. buf.WriteString("bootcmd:\n") buf.WriteString(" - rm -f " + k3sResolvConfPath + "\n") fmt.Fprintf(&buf, " - printf '%s\\n' > %s\n", diff --git a/spinifex/handlers/eks/oidc_keypair.go b/spinifex/handlers/eks/oidc_keypair.go index fc81999ea..88b8e13a3 100644 --- a/spinifex/handlers/eks/oidc_keypair.go +++ b/spinifex/handlers/eks/oidc_keypair.go @@ -17,11 +17,8 @@ import ( "github.com/nats-io/nats.go" ) -// PublicKeyPEMFromPrivate parses a PKCS#8 private-key PEM and returns the PKIX -// public-key PEM. kube-apiserver's --service-account-key-file requires a PUBLIC -// key (ParsePublicKeysPEM rejects private-key blocks), while the matching -// --service-account-signing-key-file takes the private key; the K3s server VM -// needs both, seeded to separate files. +// PublicKeyPEMFromPrivate parses a PKCS#8 private-key PEM and returns the PKIX public-key PEM. +// kube-apiserver --service-account-key-file requires a public key; signing uses the private key. func PublicKeyPEMFromPrivate(privPEM string) (string, error) { block, _ := pem.Decode([]byte(privPEM)) if block == nil { @@ -42,9 +39,7 @@ func PublicKeyPEMFromPrivate(privPEM string) (string, error) { return string(pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: pubDER})), nil } -// oidcJWK and oidcJWKS are the RFC 7517 wire shapes the per-cluster JWKS -// document uses. handlers_sts decodes the same JSON via its own JWK / JWKS -// types — wire compatibility (not Go type identity) is the contract. +// oidcJWK and oidcJWKS are the RFC 7517 wire shapes for the per-cluster JWKS document. type oidcJWK struct { Kty string `json:"kty"` Kid string `json:"kid"` @@ -59,16 +54,11 @@ type oidcJWKS struct { Keys []oidcJWK `json:"keys"` } -// P-256 coordinate width per SEC1 — JWK requires fixed-length base64url, not -// the variable-length big.Int byte representation. +// p256CoordLen is the P-256 coordinate width; JWK requires fixed-length base64url. const p256CoordLen = 32 -// GenerateClusterOIDCKeypair creates a fresh ECDSA-P256 keypair for the -// cluster, PEM-marshals the private key as PKCS8, envelope-encrypts the PEM -// with masterKey, writes the ciphertext to OIDCSigningKeyKey and the JWKS -// document to OIDCJWKSKey. Returns the plaintext private-key PEM and the JWKS -// bytes so the caller (CreateCluster) can pass them straight to user-data -// without a follow-up KV read + decrypt. +// GenerateClusterOIDCKeypair creates an ECDSA-P256 keypair, encrypts the private key PEM +// into KV, and writes the JWKS document. Returns plaintext PEM + JWKS for immediate use. func GenerateClusterOIDCKeypair(kv nats.KeyValue, clusterName string, masterKey []byte) (privPEM string, jwksBytes []byte, err error) { if clusterName == "" { return "", nil, errors.New("eks: GenerateClusterOIDCKeypair empty cluster name") @@ -106,9 +96,8 @@ func GenerateClusterOIDCKeypair(kv nats.KeyValue, clusterName string, masterKey return string(pemBlock), jwksBytes, nil } -// LoadClusterOIDCPrivateKey reads the encrypted PEM from KV, decrypts with -// masterKey, parses it to *ecdsa.PrivateKey. Returns ErrClusterNotFound when -// the key is absent so DeleteCluster idempotency works. +// LoadClusterOIDCPrivateKey decrypts and parses the cluster OIDC private key from KV. +// Returns ErrClusterNotFound if absent (supports idempotent DeleteCluster). func LoadClusterOIDCPrivateKey(kv nats.KeyValue, clusterName string, masterKey []byte) (*ecdsa.PrivateKey, error) { if clusterName == "" { return nil, errors.New("eks: LoadClusterOIDCPrivateKey empty cluster name") @@ -145,14 +134,9 @@ func LoadClusterOIDCPrivateKey(kv nats.KeyValue, clusterName string, masterKey [ return priv, nil } -// ZeroizeClusterOIDCKey overwrites the encrypted private-key blob with zeros -// of the same length, then purges the key. DeleteCluster calls this first so -// a crash after this point leaves no recoverable key material. Purge (not -// Delete) drops the key's entire revision history — a Delete leaves a tombstone -// over prior revisions, so the original encrypted PEM would survive in history -// until age/limit rolls it off. The account bucket is History=1 regardless -// (see store.go), so this is defense-in-depth against a widened history. -// Absent key is a no-op — supports idempotent DeleteCluster retries. +// ZeroizeClusterOIDCKey overwrites the encrypted key blob with zeros then purges it. +// Purge (not Delete) drops all revision history, leaving no recoverable key material. +// Absent key is a no-op for idempotent DeleteCluster retries. func ZeroizeClusterOIDCKey(kv nats.KeyValue, clusterName string) error { if clusterName == "" { return errors.New("eks: ZeroizeClusterOIDCKey empty cluster name") diff --git a/spinifex/handlers/eks/reconciler_registry.go b/spinifex/handlers/eks/reconciler_registry.go index 5fc72d5a0..1b1bf0336 100644 --- a/spinifex/handlers/eks/reconciler_registry.go +++ b/spinifex/handlers/eks/reconciler_registry.go @@ -9,15 +9,11 @@ import ( "github.com/nats-io/nats.go" ) -// ErrLeaseHeld signals that a reconciler was not spawned because another node -// holds the leader lease. Spawn treats it as benign: it drops the registry -// entry (rather than recording a phantom holder) so a later Spawn re-attempts -// once the holder's lease TTL lapses. +// ErrLeaseHeld signals the leader lease is held by another node. +// Spawn treats it as benign and retries on the next tick. var ErrLeaseHeld = errors.New("eks: reconciler lease held by another node") -// ReconcilerRegistry tracks active per-cluster reconciler goroutines so the -// daemon can spawn, stop, and graceful-shutdown them in unison. One entry per -// (accountID, clusterName) key. +// ReconcilerRegistry tracks active per-cluster reconciler goroutines; one entry per (accountID, clusterName). type ReconcilerRegistry struct { mu sync.Mutex holders map[string]*reconcilerHandle @@ -34,15 +30,10 @@ func NewReconcilerRegistry() *ReconcilerRegistry { return &ReconcilerRegistry{holders: map[string]*reconcilerHandle{}} } -// SpawnReconcilerFn is the per-cluster goroutine starter the registry uses. -// Tests inject a fake; production wires a real ClusterReconciler under the -// hood (acquires lease, then loops via Run). +// SpawnReconcilerFn starts a per-cluster reconciler goroutine; injectable for tests. type SpawnReconcilerFn func(ctx context.Context, accountID, clusterName string) (release func(), err error) -// Spawn launches one reconciler goroutine for (accountID, clusterName). If -// one is already running for that pair, returns nil without spawning a -// duplicate. SpawnFn is responsible for acquiring the leader lease + invoking -// Run; the registry only owns the ctx and the release. +// Spawn launches one reconciler goroutine for (accountID, clusterName), or no-ops if already running. func (r *ReconcilerRegistry) Spawn(parent context.Context, accountID, clusterName string, spawnFn SpawnReconcilerFn) error { if accountID == "" || clusterName == "" { return errors.New("eks: ReconcilerRegistry.Spawn empty ids") @@ -71,8 +62,6 @@ func (r *ReconcilerRegistry) Spawn(parent context.Context, accountID, clusterNam cancel() close(done) if errors.Is(err, ErrLeaseHeld) { - // Lease held elsewhere — not a failure. The entry is already - // dropped above, so a later Spawn retries after the holder's TTL. return nil } return err @@ -146,12 +135,8 @@ func registryKey(accountID, clusterName string) string { return accountID + "/" + clusterName } -// RunClusterReconciler is the production SpawnReconcilerFn: it constructs a -// ClusterReconciler, acquires the leader lease, then drives Run in a -// goroutine. Returns (release, nil) on lease acquired, or (nil, error) on -// constructor failure. If another node holds the lease, returns -// (nil, ErrLeaseHeld) so the registry drops the entry and re-attempts on a -// later tick instead of recording a phantom holder. +// RunClusterReconciler is the production SpawnReconcilerFn: constructs a ClusterReconciler, +// acquires the lease, and drives Run in a goroutine. Returns ErrLeaseHeld if another node wins. func RunClusterReconciler( ctx context.Context, leaderKV, acctKV nats.KeyValue, diff --git a/spinifex/handlers/eks/security_groups.go b/spinifex/handlers/eks/security_groups.go index 79b3567ef..c1fc4dd81 100644 --- a/spinifex/handlers/eks/security_groups.go +++ b/spinifex/handlers/eks/security_groups.go @@ -11,9 +11,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/tags" ) -// sgProvisioner is the subset of handlers_ec2_vpc.VPCService that the cluster -// SG helpers need. Narrow so tests can fake with hand-rolled recorders without -// implementing the full VPCService surface. +// sgProvisioner is the narrow VPC surface needed by cluster SG helpers. type sgProvisioner interface { CreateSecurityGroup(input *ec2.CreateSecurityGroupInput, accountID string) (*ec2.CreateSecurityGroupOutput, error) DescribeSecurityGroups(input *ec2.DescribeSecurityGroupsInput, accountID string) (*ec2.DescribeSecurityGroupsOutput, error) @@ -21,16 +19,13 @@ type sgProvisioner interface { AuthorizeSecurityGroupIngress(input *ec2.AuthorizeSecurityGroupIngressInput, accountID string) (*ec2.AuthorizeSecurityGroupIngressOutput, error) } -// clusterEKSClusterTagKey is stamped on every cluster-scoped resource so the -// DeleteCluster sweep + the UI hide-from-customer filter can group them. +// clusterEKSClusterTagKey groups all cluster-scoped resources for DeleteCluster sweeps. const clusterEKSClusterTagKey = "spinifex:eks-cluster" -// clusterEKSRoleTagKey distinguishes the control-plane SG from the nodegroup -// SG when both live in the same VPC. +// clusterEKSRoleTagKey distinguishes the control-plane SG from the nodegroup SG. const clusterEKSRoleTagKey = "spinifex:eks-role" -// clusterEKSNodegroupTagKey is stamped on worker instances so the nodegroup -// they belong to is recoverable from EC2 tags alone. +// clusterEKSNodegroupTagKey is stamped on worker instances to identify their nodegroup. const clusterEKSNodegroupTagKey = "spinifex:eks-nodegroup" const ( @@ -38,9 +33,7 @@ const ( clusterEKSRoleNodegroup = "nodegroup" ) -// ClusterControlPlaneSGName returns the deterministic SG name used for the -// cluster's control-plane SG. Stable so Stage-6 DeleteCluster can locate it -// from clusterName alone. +// ClusterControlPlaneSGName returns the deterministic control-plane SG name for a cluster. func ClusterControlPlaneSGName(clusterName string) string { return fmt.Sprintf("eks-cluster-%s-control-plane-sg", clusterName) } @@ -51,12 +44,8 @@ func ClusterNodegroupSGName(clusterName string) string { return fmt.Sprintf("eks-cluster-%s-nodegroup-sg", clusterName) } -// EnsureClusterSGs creates (or returns existing) control-plane + nodegroup -// security groups in the customer VPC. Both are tagged -// spinifex:managed-by=eks + spinifex:eks-cluster=. Idempotent: an SG -// whose group-name + vpc-id match the expected pair is reused without a -// second create call. The two creates are independent — a failure on the -// nodegroup SG does not unwind the control-plane SG (DeleteCluster cleans up). +// EnsureClusterSGs creates or reuses the control-plane and nodegroup SGs in the customer VPC. +// Idempotent; the two creates are independent (DeleteCluster handles partial teardown). func EnsureClusterSGs(sgp sgProvisioner, accountID, clusterName, vpcID string) (controlPlaneSGID, nodegroupSGID string, err error) { if clusterName == "" { return "", "", errors.New("eks: EnsureClusterSGs empty cluster name") @@ -84,10 +73,7 @@ func EnsureClusterSGs(sgp sgProvisioner, accountID, clusterName, vpcID string) ( return controlPlaneSGID, nodegroupSGID, nil } -// DeleteClusterSGs removes both cluster SGs in the customer VPC. Missing SGs -// are no-ops so DeleteCluster can retry safely. Delete errors are logged and -// the sweep continues; the first error is returned so the caller surfaces it -// without halting the broader teardown. +// DeleteClusterSGs removes both cluster SGs. Missing SGs are no-ops; first error is returned. func DeleteClusterSGs(sgp sgProvisioner, accountID, clusterName, vpcID string) error { if clusterName == "" { return errors.New("eks: DeleteClusterSGs empty cluster name") @@ -148,18 +134,8 @@ func ensureClusterSG(sgp sgProvisioner, accountID, vpcID, clusterName, name, des return *out.GroupId, nil } -// EnsureNodegroupSGRules authorizes the ingress rules a nodegroup's workers -// need to join the K3s control plane and form the flannel overlay. All rules -// are group-to-group (UserIdGroupPairs) so they stay correct as instances come -// and go. Idempotent: the InvalidPermission.Duplicate error a re-run triggers -// for an already-authorized rule is treated as success. -// -// Rules: -// - agent → server apiserver/supervisor: 6443/tcp (ngSG → cpSG) -// - flannel VXLAN: 8472/udp both directions between cpSG and ngSG, and -// ngSG ↔ ngSG -// - kubelet: 10250/tcp (cpSG → ngSG) -// - intra-nodegroup all traffic: ngSG self-reference +// EnsureNodegroupSGRules authorizes ingress for flannel VXLAN (8472/udp), apiserver +// (6443/tcp), and kubelet (10250/tcp) between CP and nodegroup SGs. Idempotent. func EnsureNodegroupSGRules(sgp sgProvisioner, accountID, clusterName, cpSGID, ngSGID string) error { if cpSGID == "" || ngSGID == "" { return errors.New("eks: EnsureNodegroupSGRules empty SG id") @@ -208,13 +184,8 @@ func EnsureNodegroupSGRules(sgp sgProvisioner, accountID, clusterName, cpSGID, n return nil } -// EnsureControlPlaneIngress authorizes the ingress the cluster NLB needs to -// reach the K3s apiserver. The NLB's backing LB VM forwards the front-end -// listener to the apiserver on k3sAPIServerPort from inside the VPC, so the -// control-plane SG must admit that port from the VPC CIDR. Public exposure is -// gated separately at the NLB front-end by publicAccessCidrs; this rule only -// governs the in-VPC LB→apiserver hop. Idempotent: the -// InvalidPermission.Duplicate error a re-run triggers is treated as success. +// EnsureControlPlaneIngress admits the NLB→apiserver hop from the VPC CIDR on k3sAPIServerPort. +// Public access is gated separately at the NLB front-end. Idempotent. func EnsureControlPlaneIngress(sgp sgProvisioner, accountID, cpSGID, vpcCIDR string) error { if cpSGID == "" { return errors.New("eks: EnsureControlPlaneIngress empty control-plane SG id") @@ -242,24 +213,8 @@ func EnsureControlPlaneIngress(sgp sgProvisioner, accountID, cpSGID, vpcCIDR str return nil } -// EnsureControlPlaneHAIngress authorizes the server-to-server ingress an HA -// control plane needs so a join server's embedded etcd can peer with the rest -// of the quorum. The control-plane SG is self-referencing here: every rule's -// source and target are the same group, so the rules stay correct as CP VMs are -// launched, lost, and rebuilt. A single cluster-init server never exercises -// these ports, which is why they were absent until multi-server HA: the join -// server registers over the supervisor port (already admitted) but its etcd -// member can never replicate the raft log without the peer port, so the quorum -// silently never forms and the node never reports Ready. -// -// Rules (cpSG → cpSG): -// - etcd client+peer: 2379-2380/tcp — embedded-etcd quorum replication -// - kubelet: 10250/tcp — server-to-server kubelet (metrics, logs, exec) -// -// The supervisor/apiserver port (6443) is already admitted from the VPC CIDR by -// EnsureControlPlaneIngress, which covers the CP peers, so it is not repeated. -// Idempotent: the InvalidPermission.Duplicate error a re-run triggers for an -// already-authorized rule is treated as success. +// EnsureControlPlaneHAIngress authorizes self-referencing CP SG rules for HA etcd peering: +// etcd client+peer (2379-2380/tcp) and kubelet (10250/tcp) within cpSG. Idempotent. func EnsureControlPlaneHAIngress(sgp sgProvisioner, accountID, cpSGID string) error { if cpSGID == "" { return errors.New("eks: EnsureControlPlaneHAIngress empty control-plane SG id") diff --git a/spinifex/handlers/eks/service.go b/spinifex/handlers/eks/service.go index aaf17ede8..5938bbaff 100644 --- a/spinifex/handlers/eks/service.go +++ b/spinifex/handlers/eks/service.go @@ -1,19 +1,11 @@ -// Package handlers_eks provides the EKS (Amazon Elastic Kubernetes Service) -// control-plane API surface for Spinifex. The interface declared here is the -// stable contract every gateway and daemon caller binds to; the NATS wrapper -// and concrete service implementation both satisfy it. +// Package handlers_eks implements the EKS control-plane API surface for Spinifex. package handlers_eks import "github.com/aws/aws-sdk-go/service/eks" -// EKSService is the AWS EKS control-plane contract exposed to spinifex -// gateway and daemon callers. Every method maps 1:1 to an AWS EKS API action -// (PascalCase). Signatures use the aws-sdk-go input/output types verbatim so -// downstream callers can pass straight through from generated SDK code. +// EKSService is the EKS control-plane contract; one method per AWS EKS API action. type EKSService interface { - // Cluster. CreateCluster also takes the caller's resolved IAM principal ARN - // (from the X-Principal-ARN header) so it can mint the bootstrap - // cluster-creator-admin AccessEntry; "" skips that step. + // Cluster. callerPrincipalARN is the IAM principal for the bootstrap AccessEntry; "" skips it. CreateCluster(input *eks.CreateClusterInput, accountID, callerPrincipalARN string) (*eks.CreateClusterOutput, error) DescribeCluster(input *eks.DescribeClusterInput, accountID string) (*eks.DescribeClusterOutput, error) ListClusters(input *eks.ListClustersInput, accountID string) (*eks.ListClustersOutput, error) @@ -48,8 +40,7 @@ type EKSService interface { DescribeAddon(input *eks.DescribeAddonInput, accountID string) (*eks.DescribeAddonOutput, error) UpdateAddon(input *eks.UpdateAddonInput, accountID string) (*eks.UpdateAddonOutput, error) - // OIDC identity-provider configs (control-plane API only; the anonymous - // OIDC issuer routes live with the awsgw IRSA wiring). + // OIDC identity-provider configs. AssociateIdentityProviderConfig(input *eks.AssociateIdentityProviderConfigInput, accountID string) (*eks.AssociateIdentityProviderConfigOutput, error) DescribeIdentityProviderConfig(input *eks.DescribeIdentityProviderConfigInput, accountID string) (*eks.DescribeIdentityProviderConfigOutput, error) ListIdentityProviderConfigs(input *eks.ListIdentityProviderConfigsInput, accountID string) (*eks.ListIdentityProviderConfigsOutput, error) diff --git a/spinifex/handlers/eks/service_impl.go b/spinifex/handlers/eks/service_impl.go index 4747400ef..b96e1c4d5 100644 --- a/spinifex/handlers/eks/service_impl.go +++ b/spinifex/handlers/eks/service_impl.go @@ -23,19 +23,13 @@ import ( "github.com/nats-io/nats.go" ) -// SubnetVPCResolver resolves a subnet ID to its parent VPC ID and the VPC's -// CIDR block. Narrow so EKSServiceImpl can stay free of the wider -// handlers/ec2/vpc surface; the daemon adapts VPCServiceImpl onto this -// contract. +// SubnetVPCResolver resolves a subnet to its VPC ID and CIDR. type SubnetVPCResolver interface { GetSubnetVPC(accountID, subnetID string) (vpcID string, err error) GetVPCCIDR(accountID, vpcID string) (cidr string, err error) } -// EKSServiceDeps wires every external collaborator EKSServiceImpl needs. -// The narrow per-area interfaces (sgProvisioner, nlbProvisioner, etc.) are -// already defined alongside their consumers; the daemon adapts each -// service's concrete type onto the interface. +// EKSServiceDeps wires the external collaborators EKSServiceImpl needs. type EKSServiceDeps struct { Config *config.Config NATSConn *nats.Conn @@ -44,13 +38,8 @@ type EKSServiceDeps struct { Region string HolderID string - // Gateway broker config handed to the K3s server VM so it can publish its - // bootstrap envelopes + state reports via SigV4-signed HTTPS POST to the AWS - // gateway (the ELBv2 lb-agent model) instead of dialing core NATS. - // SystemGatewayURL is the mgmt-reachable AWSGW endpoint (distinct from - // GatewayBaseURL, which is the OIDC issuer host); SystemAccessKey/SecretKey - // are the system (Predastore) SigV4 creds; GatewayCACert is the PEM that - // signs the gateway server cert. + // Gateway broker config for the K3s server VM (SigV4 HTTPS to AWSGW). + // SystemGatewayURL is the mgmt-reachable endpoint; GatewayCACert signs its TLS cert. SystemGatewayURL string SystemAccessKey string SystemSecretKey string @@ -65,42 +54,28 @@ type EKSServiceDeps struct { EIP eipProvisioner Worker WorkerLauncher - // PlacementGroup reserves distinct hosts for HA control-plane spread; - // Scheduler answers the capacity + placement fan-out the spread path needs. - // The daemon wires the NATS implementations. + // PlacementGroup and Scheduler support HA control-plane spread (NATS-backed). PlacementGroup controlPlanePlacer Scheduler HostScheduler - // AddonInstaller delivers managed-addon manifests to clusters. Optional: - // when nil the service defaults to the KV staging installer - // (newStagingInstaller) bound to NATSConn. + // AddonInstaller delivers managed-addon manifests; nil defaults to the KV staging installer. AddonInstaller AddonInstaller } -// WorkerLauncher is the narrow EC2 surface CreateNodegroup needs to launch and -// terminate nodegroup worker instances. Workers go through the normal -// customer-owned RunInstances path (no ManagedBy tag, no mgmt NIC) so they are -// visible in DescribeInstances and reclaimed by TerminateInstances like any -// other customer EC2. The daemon adapts its concrete instance service onto this. -// Exported so the daemon can assert *Daemon satisfies it with a compile-time -// check. +// WorkerLauncher is the narrow EC2 surface for launching nodegroup worker instances. +// Workers use the customer-owned RunInstances path (no ManagedBy tag, no mgmt NIC). type WorkerLauncher interface { RunWorkerInstance(input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error) TerminateWorkerInstances(instanceIDs []string, accountID string) error } -// eipProvisioner is the narrow EIP surface CreateCluster needs to give the -// hidden K3s control-plane VM an egress-only public IP (the distributed-NAT -// dev/edge topology has no shared masquerade address). The daemon adapts the -// concrete EIP service onto this. +// eipProvisioner is the narrow EIP surface for allocating a CP VM egress IP. type eipProvisioner interface { AllocateAddress(input *ec2.AllocateAddressInput, accountID string) (*ec2.AllocateAddressOutput, error) ReleaseAddress(input *ec2.ReleaseAddressInput, accountID string) (*ec2.ReleaseAddressOutput, error) } -// EKSServiceImpl is the daemon-side EKSService. CreateCluster / -// DescribeCluster / ListClusters / DeleteCluster have real bodies; every -// other action is still NotImplemented pending follow-up sprints. +// EKSServiceImpl is the daemon-side EKSService implementation. type EKSServiceImpl struct { deps EKSServiceDeps leaderKV nats.KeyValue @@ -110,27 +85,18 @@ type EKSServiceImpl struct { bgCtx context.Context bgCancel context.CancelFunc - // launchWG tracks in-flight asynchronous create launches (control-plane and - // nodegroup VM provisioning kicked off after the fast CREATING response). - // Not part of request handling — exists so tests can deterministically await - // launch completion (WaitLaunches). + // launchWG tracks in-flight async create launches for test determinism (WaitLaunches). launchWG sync.WaitGroup } var _ EKSService = (*EKSServiceImpl)(nil) -// WaitLaunches blocks until all asynchronous create launches started by this -// service have finished. Intended for tests; production create handlers return -// as soon as the CREATING record is claimed. +// WaitLaunches blocks until all async create launches finish. Intended for tests. func (s *EKSServiceImpl) WaitLaunches() { s.launchWG.Wait() } const defaultK8sVersion = "1.32" -// NewEKSServiceImpl wires the Store, leader bucket, reconciler registry, and -// per-cluster background-goroutine context. Validates that MasterKey, -// GatewayBaseURL, Region, and HolderID are set when called from the live -// daemon path; the legacy NewEKSServiceImplWithNATS shim leaves them empty -// and bodies that need them return ServerInternal. +// NewEKSServiceImpl initialises EKSServiceImpl, wiring the leader KV and reconciler registry. func NewEKSServiceImpl(deps EKSServiceDeps) (*EKSServiceImpl, error) { if deps.NATSConn == nil { return nil, errors.New("eks: NewEKSServiceImpl nil NATSConn") @@ -153,10 +119,7 @@ func NewEKSServiceImpl(deps EKSServiceDeps) (*EKSServiceImpl, error) { }, nil } -// NewEKSServiceImplWithNATS is a back-compat shim. Bodies that need -// orchestration deps return ServerInternal when invoked through this shim; -// the daemon-handler routing test calls it to verify NATS subject wiring -// without standing up the full dependency graph. +// NewEKSServiceImplWithNATS is a back-compat shim for tests that only need NATS wiring. func NewEKSServiceImplWithNATS(cfg *config.Config, nc *nats.Conn) (*EKSServiceImpl, error) { return NewEKSServiceImpl(EKSServiceDeps{Config: cfg, NATSConn: nc}) } @@ -173,10 +136,7 @@ func (s *EKSServiceImpl) Shutdown() { s.registry.StopAll() } -// SpawnRegisteredReconcilers enumerates every cluster in every per-account -// bucket and registers a reconciler goroutine for each cluster in -// CREATING or ACTIVE state. Called by the daemon on boot so a node restart -// resumes lifecycle reconcile without waiting for the next CreateCluster. +// SpawnRegisteredReconcilers resumes reconciler goroutines for all CREATING/ACTIVE clusters on daemon boot. func (s *EKSServiceImpl) SpawnRegisteredReconcilers() error { if !s.depsReadyForOrchestration() { slog.Debug("SpawnRegisteredReconcilers: deps not ready, skipping") @@ -210,10 +170,7 @@ func (s *EKSServiceImpl) SpawnRegisteredReconcilers() error { if meta.Status != ClusterStatusCreating && meta.Status != ClusterStatusActive { continue } - // A restart mid-CREATING lost the bootstrap subscriptions; the K3s - // VM publishes its one-shot core-NATS messages once and to nobody. - // Re-subscribe to whichever artifacts are still missing so the - // cluster can still reach ACTIVE instead of hanging in CREATING. + // Re-subscribe to missing artifacts; K3s publishes each one-shot message once. if meta.Status == ClusterStatusCreating { if pending := BootstrapPendingKinds(acctKV, cluster, meta); len(pending) > 0 { slog.Info("SpawnRegisteredReconcilers: resuming bootstrap", @@ -231,9 +188,7 @@ func (s *EKSServiceImpl) depsReadyForOrchestration() bool { return len(s.missingOrchestrationDeps()) == 0 } -// requireOrchestrationDeps returns a client-facing ServiceUnavailable (with the -// specific unmet deps logged at ERROR) when the daemon is missing orchestration -// wiring, so the failure is diagnosable instead of a bare ServerInternal. +// requireOrchestrationDeps returns ServiceUnavailable with missing deps logged when orchestration is not wired. func (s *EKSServiceImpl) requireOrchestrationDeps(op string) error { missing := s.missingOrchestrationDeps() if len(missing) == 0 { @@ -243,9 +198,7 @@ func (s *EKSServiceImpl) requireOrchestrationDeps(op string) error { return errors.New(awserrors.ErrorServiceUnavailable) } -// missingOrchestrationDeps names the deps required for cluster orchestration -// that are unset, so a "deps not ready" rejection can log the precise cause -// instead of an opaque ServerInternal whose only hint is a one-time boot WARN. +// missingOrchestrationDeps returns names of unset orchestration deps. func (s *EKSServiceImpl) missingOrchestrationDeps() []string { var missing []string if s.deps.VPCSG == nil { @@ -295,22 +248,13 @@ func (s *EKSServiceImpl) missingOrchestrationDeps() []string { // --- Cluster --- -// logCreateErr records the precise cause of a CreateCluster step failure at -// ERROR. The request surface collapses these wrapped errors to an opaque -// ServerInternal ("An internal error has occurred... contact AWS re:Post"), so -// without this the wrapped error never reaches the operator. Returns the error -// wrapped with the stage for the caller to surface. +// logCreateErr logs a CreateCluster step failure at ERROR and returns it wrapped with the stage name. func logCreateErr(name, accountID, stage string, err error) error { slog.Error("CreateCluster: "+stage+" failed", "cluster", name, "accountID", accountID, "err", err) return fmt.Errorf("%s: %w", stage, err) } -// managedIngressTagKey is the CreateCluster tag that controls K3s' bundled -// traefik + servicelb (interim in-VPC app exposure). They are ON by default so -// workloads are reachable before the AWS Load Balancer Controller add-on ships; -// value "false" (case-insensitive) opts out for AWS parity. TODO: flip the -// default back off (parity) once the aws-lb-controller add-on lands — see -// docs/development/feature/eks-dataplane-ingress.md. +// managedIngressTagKey controls K3s built-in traefik+servicelb; "false" opts out for AWS parity. const managedIngressTagKey = "spinifex.io/managed-ingress" // builtinIngressEnabled reports whether a cluster keeps the K3s built-in ingress @@ -340,9 +284,7 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID, vpcID, err := s.deps.VPCSubnet.GetSubnetVPC(accountID, subnetIDs[0]) if err != nil { - // Subnet is a caller-supplied parameter — a resolve failure is a client - // fault (bad/foreign subnet), not an internal one. Surface it as such - // instead of collapsing to a retryable ServerInternal the SDK retries. + // Subnet resolve failure is a client fault (bad/foreign subnet). slog.Error("CreateCluster: resolve subnet VPC failed", "cluster", name, "accountID", accountID, "subnet", subnetIDs[0], "err", err) return nil, errors.New(awserrors.ErrorInvalidParameterValue) @@ -370,26 +312,13 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID, BuiltinIngress: builtinIngressEnabled(input.Tags), CreatedAt: time.Now().UTC(), } - // Atomically claim the cluster name before any launching/fallible work. A - // duplicate CreateCluster (the SDK's own retry, or a gateway re-publish that - // spawns a second concurrent handler) loses the claim and is rejected — or, - // for a prior FAILED attempt, reclaims it — so exactly one handler ever - // launches control-plane VMs for a given name. + // Claim the cluster name before any launching; duplicate/retry handlers lose the claim. if err := s.claimClusterName(accountID, acctKV, meta); err != nil { return nil, err } - // Respond with the cluster in CREATING immediately and run the launch - // (SGs, NLB, OIDC, control-plane VM boot, egress, target registration, - // bootstrap, reconciler) off the request goroutine. A multi-minute control- - // plane boot must not occupy the spinifex-workers subscription or the gateway - // times out and re-publishes the request, spawning a duplicate concurrent - // handler (267.4 — the same retry the 267.3 claim guards against). AWS itself - // returns the cluster in CREATING here; the client polls DescribeCluster to - // ACTIVE/FAILED. - // Snapshot the CREATING response before spawning the launch: the goroutine - // owns meta from here on (it mutates and re-persists it), so the request - // goroutine must not touch meta again. + // Respond CREATING immediately; run the slow launch on a background goroutine. + // The goroutine owns meta from here on — do not touch it after this point. out := &eks.CreateClusterOutput{Cluster: clusterMetaToAWS(meta)} s.launchWG.Go(func() { defer func() { @@ -430,11 +359,7 @@ type clusterLaunchCtx struct { acctKV nats.KeyValue } -// failClusterLaunch records an asynchronous control-plane launch failure: it -// marks the cluster FAILED (with reason, so DescribeCluster surfaces the cause -// and the 267.3 reclaim path can recreate cleanly) and logs the stage. The -// launch runs after the CREATING response was already sent, so a failure can -// only surface as status, never as the CreateCluster return value. +// failClusterLaunch logs a launch failure and marks the cluster FAILED so DescribeCluster surfaces the cause. func (s *EKSServiceImpl) failClusterLaunch(kv nats.KeyValue, name, accountID, stage string, err error) { _ = logCreateErr(name, accountID, stage, err) if mErr := MarkClusterFailed(kv, name, stage+": "+err.Error()); mErr != nil && !errors.Is(mErr, ErrClusterNotFound) { @@ -442,11 +367,8 @@ func (s *EKSServiceImpl) failClusterLaunch(kv nats.KeyValue, name, accountID, st } } -// launchClusterInfra is CreateCluster's slow phase, run on a background -// goroutine after the CREATING response is sent: security groups, the cluster -// NLB, OIDC keypair, the control-plane VM(s), egress wiring, NLB target -// registration, the creator-admin AccessEntry, and the bootstrap/reconciler -// goroutines. Every failure path marks the cluster FAILED via failClusterLaunch. +// launchClusterInfra is CreateCluster's slow phase (SGs, NLB, OIDC, CP VMs, egress, bootstrap). +// Runs on a background goroutine; every failure marks the cluster FAILED. func (s *EKSServiceImpl) launchClusterInfra(lc clusterLaunchCtx) { accountID := lc.accountID callerPrincipalARN := lc.callerPrincipalARN @@ -467,9 +389,6 @@ func (s *EKSServiceImpl) launchClusterInfra(lc clusterLaunchCtx) { } meta.ResourcesVpcConfig.SecurityGroupIds = []string{cpSG, ngSG} - // The NLB's backing LB VM forwards the published endpoint to the apiserver - // from inside the VPC, so the control-plane SG must admit that hop or the - // NLB target stays unhealthy and the endpoint never serves. vpcCIDR, err := s.deps.VPCSubnet.GetVPCCIDR(accountID, vpcID) if err != nil { s.failClusterLaunch(acctKV, name, accountID, "resolve vpc cidr", err) @@ -479,24 +398,17 @@ func (s *EKSServiceImpl) launchClusterInfra(lc clusterLaunchCtx) { s.failClusterLaunch(acctKV, name, accountID, "ensure control-plane ingress", err) return } - // HA control planes run servers 2..N as join servers whose embedded etcd must - // peer with the quorum; without these self-referencing CP-SG rules a join - // registers but its etcd never replicates, so the node never reports Ready. if err := EnsureControlPlaneHAIngress(s.deps.VPCSG, accountID, cpSG); err != nil { s.failClusterLaunch(acctKV, name, accountID, "ensure control-plane HA ingress", err) return } - // Public access ⇒ internet-facing NLB (external-pool front-end IP, reachable - // on the LAN/edge network); private-only ⇒ internal NLB (VPC-only). nlb, err := EnsureClusterNLB(s.deps.NLB, accountID, name, subnetIDs, publicAccess, publicCidrs) if err != nil { s.failClusterLaunch(acctKV, name, accountID, "ensure cluster NLB", err) return } - // Prefer the reachable front-end IP as the kubeconfig endpoint host so the - // apiserver serving cert (which SANs this IP) validates with TLS verification - // on. Fall back to the synthetic NLB DNS name when no IP was read back. + // Prefer the NLB front-end IP for TLS SAN validation; fall back to DNS name. endpointHost := nlb.FrontendIP if endpointHost == "" { endpointHost = nlb.DNSName @@ -506,11 +418,7 @@ func (s *EKSServiceImpl) launchClusterInfra(lc clusterLaunchCtx) { meta.NLBArn = nlb.LoadBalancerArn meta.NLBTargetGroupArn = nlb.TargetGroupArn - // Persist the NLB ARNs now, before any further fallible step. Without this - // a failure between here and the final PutClusterMeta would leave the NLB + - // target group + listener + backing LB VM created but unrecorded, so - // DeleteCluster (which keys teardown off the persisted ARNs) could not - // reclaim them. + // Persist NLB ARNs early so DeleteCluster can reclaim them on any later failure. if err := PutClusterMeta(acctKV, meta); err != nil { s.failClusterLaunch(acctKV, name, accountID, "persist NLB arns", err) return @@ -534,9 +442,6 @@ func (s *EKSServiceImpl) launchClusterInfra(lc clusterLaunchCtx) { return } - // Shared cluster token seeded into every control-plane server so the HA - // spread's servers 2..N join the first server's etcd quorum (and workers - // register). Single-CP clusters carry it too — harmless, one code path. joinToken, err := GenerateK3sClusterToken() if err != nil { s.failClusterLaunch(acctKV, name, accountID, "generate k3s cluster token", err) @@ -569,9 +474,7 @@ func (s *EKSServiceImpl) launchClusterInfra(lc clusterLaunchCtx) { s.failClusterLaunch(acctKV, name, accountID, "launch K3s VM", err) return } - // Mirror the primary ([0]) into the scalar fields the reconciler, egress - // wiring, and teardown read today. All CP nodes are NLB target-registered - // (failover); egress is still primary-only (separate hidden-pool /32 wire). + // Mirror primary CP ([0]) into scalar fields; all nodes are NLB-registered for failover. primary := cpNodes[0] meta.ControlPlaneNodes = cpNodes meta.ControlPlaneSpreadGroup = spreadGroup @@ -580,19 +483,12 @@ func (s *EKSServiceImpl) launchClusterInfra(lc clusterLaunchCtx) { meta.ControlPlaneENIIP = primary.ENIIP meta.ControlPlaneMgmtIP = primary.MgmtIP - // Persist the CP VM + ENI + spread-group refs now, before any further fallible - // step. The VMs are live the moment placeControlPlane returns; without this a - // failure between here and the next PutClusterMeta (e.g. egress allocation) - // leaves them launched but unrecorded, so neither DeleteCluster nor the - // FAILED-cluster reclaim could reach them and the sys.medium VMs leak. + // Persist CP IDs immediately; VMs are live now and must not be orphaned on later failure. if err := PutClusterMeta(acctKV, meta); err != nil { s.failClusterLaunch(acctKV, name, accountID, "persist control-plane ids", err) return } - // Egress: the control-plane VM must pull container images. Allocate a - // hidden pool address and wire an egress-only SNAT for its /32 (no DNAT — - // the VM stays unreachable inbound; the NLB is its only front door). egressIP, egressAllocID, err := s.allocateClusterEgressIP(accountID, name) if err != nil { s.failClusterLaunch(acctKV, name, accountID, "allocate cluster egress IP", err) @@ -607,8 +503,7 @@ func (s *EKSServiceImpl) launchClusterInfra(lc clusterLaunchCtx) { ExternalIp: egressIP, }) - // Record the egress allocation before registering the target so a failed - // register (or anything after) leaves it recoverable by DeleteCluster. + // Persist egress allocation so DeleteCluster can reclaim it on later failure. if err := PutClusterMeta(acctKV, meta); err != nil { s.failClusterLaunch(acctKV, name, accountID, "persist egress allocation", err) return @@ -628,10 +523,7 @@ func (s *EKSServiceImpl) launchClusterInfra(lc clusterLaunchCtx) { return } - // bootstrapClusterCreatorAdminPermissions (default true): grant the caller - // system:masters via an AccessEntry so the cluster creator can immediately - // authenticate through the token webhook (Q9). Keyed by the caller's exact - // principal ARN — the webhook looks the entry up by the same ARN STS returns. + // Seed creator system:masters AccessEntry so the token webhook can authenticate the creator immediately. if bootstrapCreatorAdmin(input) && callerPrincipalARN != "" { rec := newAccessEntryRecord(region, accountID, name, callerPrincipalARN, "", []string{"system:masters"}, AccessEntryTypeStandard, nil, time.Now().UTC()) @@ -789,17 +681,8 @@ func (s *EKSServiceImpl) DeleteCluster(input *eks.DeleteClusterInput, accountID return &eks.DeleteClusterOutput{Cluster: clusterMetaToAWS(meta)}, nil } -// claimClusterName atomically takes ownership of the cluster's meta key before -// CreateCluster does any launching work, closing the duplicate-handler race (an -// SDK retry or a gateway re-publish spawning a second concurrent CreateCluster). -// The fresh CREATING meta is laid down with kv.Create: the winner owns it. A -// loser finds the key already present and either rejects (a live cluster) or, -// for a prior FAILED attempt, reclaims it — flipping FAILED→CREATING via CAS so -// exactly one concurrent reclaimer wins, then purging the old attempt's infra -// (keeping the meta key as its lock) before laying down the fresh record. The -// old resource refs stay in the record until teardown confirms the infra is -// gone, so a crash mid-purge leaves them recoverable rather than orphaned. -// Returns an AWS-shaped error ready to hand back to the caller. +// claimClusterName atomically claims the cluster meta key before any launch work. +// A prior FAILED attempt is reclaimed via CAS (FAILED→CREATING); live clusters reject with ResourceInUse. func (s *EKSServiceImpl) claimClusterName(accountID string, acctKV nats.KeyValue, meta *ClusterMeta) error { name := meta.Name data, err := json.Marshal(meta) @@ -813,10 +696,7 @@ func (s *EKSServiceImpl) claimClusterName(accountID string, acctKV nats.KeyValue return logCreateErr(name, accountID, "claim cluster name", cerr) } - // Key already present. Reclaim only a prior FAILED attempt; flip its status - // to CREATING via CAS so exactly one concurrent reclaimer wins. The mutate - // changes only Status, so the old resource refs survive in the record for - // teardown below (and for a retry if we crash before clearing them). + // Key already present: reclaim a FAILED attempt via CAS; old refs survive for teardown. var oldRefs *ClusterMeta reclaimed := false if err := casUpdateMeta(acctKV, name, func(m *ClusterMeta) bool { @@ -1034,9 +914,7 @@ func (s *EKSServiceImpl) DeleteNodegroup(input *eks.DeleteNodegroupInput, accoun // --- AccessEntry + AccessPolicy --- -// acctKVForCluster opens the per-account bucket and verifies the cluster exists, -// translating absence to the AWS ResourceNotFoundException shape. The -// AccessEntry handlers all need both. +// acctKVForCluster opens the per-account bucket and verifies the cluster exists. func (s *EKSServiceImpl) acctKVForCluster(accountID, cluster string) (nats.KeyValue, error) { if cluster == "" { return nil, errors.New(awserrors.ErrorInvalidParameterValue) @@ -1072,8 +950,7 @@ func (s *EKSServiceImpl) CreateAccessEntry(input *eks.CreateAccessEntryInput, ac entryType = AccessEntryTypeStandard } if entryType != AccessEntryTypeStandard { - // Node-type entries (EC2_LINUX/etc.) need the nodegroup join path; reject - // until Sprint 6d wires them. + // Non-standard types (EC2_LINUX etc.) are not yet implemented. return nil, errors.New(awserrors.ErrorInvalidParameterValue) } acctKV, err := s.acctKVForCluster(accountID, cluster) diff --git a/spinifex/handlers/elbv2/agent_types.go b/spinifex/handlers/elbv2/agent_types.go index bc4fd6fbb..7b0f7e9f2 100644 --- a/spinifex/handlers/elbv2/agent_types.go +++ b/spinifex/handlers/elbv2/agent_types.go @@ -5,16 +5,14 @@ import ( "github.com/mulgadc/spinifex/spinifex/lbagent" ) -// LBAgentHeartbeatInput is sent by the LB agent on each heartbeat tick. -// The agent includes its health report (HAProxy backend server statuses) so -// the daemon can update target health without polling. +// LBAgentHeartbeatInput is sent by the LB agent on each heartbeat tick, +// including its health report so the daemon can update target health. type LBAgentHeartbeatInput struct { LBID *string `locationName:"LBID" type:"string"` Servers []*LBAgentServerStatus `locationName:"Servers" type:"list"` } -// LBAgentServerStatus represents a single backend server's health status -// as reported by the LB agent. Maps to lbagent.ServerStatus for processing. +// LBAgentServerStatus represents a single backend server's health status as reported by the LB agent. type LBAgentServerStatus struct { Backend *string `locationName:"Backend" type:"string"` Server *string `locationName:"Server" type:"string"` @@ -32,11 +30,9 @@ type GetLBConfigInput struct { LBID *string `locationName:"LBID" type:"string"` } -// GetLBConfigOutput returns the pre-computed data-plane config and its hash, -// plus any TLS certificate files the config references. Engine selects the -// data plane the agent runs: "haproxy" (ALB) or "nginx" (NLB). HealthTargets -// is populated for NLBs only: nginx `stream` has no active upstream probing, -// so the agent probes these and reports the results via heartbeat. +// GetLBConfigOutput returns the pre-computed data-plane config, its hash, and +// any TLS certificate files. Engine is "haproxy" (ALB) or "nginx" (NLB). +// HealthTargets is NLB-only: the agent actively probes these targets. type GetLBConfigOutput struct { ConfigText *string `type:"string"` ConfigHash *string `type:"string"` @@ -45,12 +41,8 @@ type GetLBConfigOutput struct { HealthTargets []*HealthTarget `locationName:"HealthTargets" type:"list"` } -// HealthTarget is one backend the nginx agent must actively health-check. -// ServerName is daemon-computed (sanitizeName("srv", target.Id)) so it matches -// the name the health checker keys on — the agent echoes it back verbatim and -// needs no ELBv2 naming logic. Address is the ip:port to probe; Protocol is -// the target group's health-check protocol (TCP/HTTP/HTTPS); Path is the -// HTTP(S) health-check path. +// HealthTarget is one backend the nginx agent actively health-checks. +// ServerName matches sanitizeName("srv", target.Id); the agent echoes it back verbatim. type HealthTarget struct { ServerName *string `locationName:"ServerName" type:"string"` Address *string `locationName:"Address" type:"string"` @@ -58,9 +50,8 @@ type HealthTarget struct { Path *string `locationName:"Path" type:"string"` } -// CertFile is one TLS certificate PEM delivered to the LB agent. Path is the -// absolute destination (under lbagent.CertDir) the agent writes 0600 before -// reload; PEM is the combined cert+chain+key material. +// CertFile is one TLS certificate PEM delivered to the LB agent. +// Path is the absolute destination the agent writes 0600 before reload. type CertFile struct { Path *string `locationName:"Path" type:"string"` PEM *string `locationName:"PEM" type:"string"` diff --git a/spinifex/handlers/elbv2/arn_test.go b/spinifex/handlers/elbv2/arn_test.go index ae8c6d489..617dea7bd 100644 --- a/spinifex/handlers/elbv2/arn_test.go +++ b/spinifex/handlers/elbv2/arn_test.go @@ -242,10 +242,9 @@ func TestBuildMicrovmNICs_ExtraENIs(t *testing.T) { assert.False(t, nics[2].IsDefault) } -// TestBuildMicrovmNICs_MgmtRoute covers the same single-node/multi-node matrix -// as TestLBVMUserData_InternalSingleNodeFallback: internet-facing single-node -// must NOT emit a /32 via mgmt to AdvertiseIP because that steals the host's -// WAN return path for reply traffic, breaking same-chassis ingress. +// TestBuildMicrovmNICs_MgmtRoute verifies the single-node/multi-node mgmt-route matrix: +// internet-facing single-node must not emit a /32 via mgmt to AdvertiseIP, as that +// would steal the host's WAN return path and break same-chassis ingress. func TestBuildMicrovmNICs_MgmtRoute(t *testing.T) { mk := func() *ELBv2ServiceImpl { return &ELBv2ServiceImpl{ diff --git a/spinifex/handlers/elbv2/haproxy.go b/spinifex/handlers/elbv2/haproxy.go index f56321028..d78914526 100644 --- a/spinifex/handlers/elbv2/haproxy.go +++ b/spinifex/handlers/elbv2/haproxy.go @@ -12,12 +12,9 @@ import ( "github.com/mulgadc/spinifex/spinifex/lbagent" ) -// Every interpolated string field in the templates below MUST be regex-validated -// at its input boundary. A newline or HAProxy keyword in any unvalidated field -// would inject arbitrary directives into the rendered config. The current -// surface is HealthCheckPath and HealthCheckMatcher (validated in -// CreateTargetGroup); all other interpolated identifiers go through -// sanitizeName or are integers. +// All interpolated fields are regex-validated at the input boundary. +// HealthCheckPath and HealthCheckMatcher are validated in CreateTargetGroup; +// all other interpolated identifiers go through sanitizeName or are integers. const haproxyConfigTemplate = `# Auto-generated by spinifex ELBv2 — do not edit global log stdout format short daemon @@ -82,25 +79,22 @@ type HAProxyFrontend struct { Rules []HAProxyRule } -// HAProxyRule represents one routing rule emitted as ACL declarations + a -// use_backend directive. Multiple ACLs sharing a Name are OR'd by HAProxy; -// distinct ConditionNames in the use_backend "if" clause are AND'd. +// HAProxyRule represents one routing rule as ACL declarations + a use_backend +// directive. Same-name ACLs are OR'd; distinct ConditionNames are AND'd. type HAProxyRule struct { Backend string ACLs []HAProxyACL ConditionNames []string } -// HAProxyACL is one acl line. Name may repeat across the rule's ACL slice to -// express OR semantics over multiple match expressions. +// HAProxyACL is one acl line. Repeated names express OR over multiple match expressions. type HAProxyACL struct { Name string Expr string } -// HAProxyBackend represents a target group backend, or a synthetic backend that -// emits a fixed response or a redirect (when FixedResponse / Redirect is set, -// the server fields are unused). +// HAProxyBackend represents a target group backend, or a synthetic redirect / +// fixed-response backend (server fields unused when those are set). type HAProxyBackend struct { Name string HealthCheckPath string @@ -110,8 +104,7 @@ type HAProxyBackend struct { Redirect *HAProxyRedirect } -// HAProxyFixedResponse renders an `http-request return` directive for a -// fixed-response listener default action. All fields are validated before use. +// HAProxyFixedResponse renders an `http-request return` directive. All fields are validated before use. type HAProxyFixedResponse struct { StatusCode string ContentType string @@ -119,8 +112,7 @@ type HAProxyFixedResponse struct { } // HAProxyRedirect renders an `http-request redirect` directive. Exactly one of -// Scheme (the scheme-only fast path, e.g. HTTP→HTTPS) or Location (a full -// rebuilt URL with HAProxy log-format fetches) is set. +// Scheme (scheme-only, e.g. HTTP→HTTPS) or Location (full rebuilt URL) is set. type HAProxyRedirect struct { Scheme string Location string @@ -137,22 +129,16 @@ type HAProxyServer struct { Rise int64 } -// GenerateHAProxyConfig builds an HAProxy config string from the LB, its -// listeners, target groups, and per-listener rules. For NLBs it generates a -// TCP-mode config and ignores rules (NLB has no L7 routing). TLS termination -// is off (no certificates resolved) — use GenerateHAProxyConfigWithCerts for -// HTTPS listeners. +// GenerateHAProxyConfig builds an HAProxy config from the LB, listeners, and +// target groups. No TLS — use GenerateHAProxyConfigWithCerts for HTTPS listeners. func GenerateHAProxyConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord, tgByArn map[string]*TargetGroupRecord, rulesByListener map[string][]*RuleRecord, bindAddr string) (string, error) { config, _, err := GenerateHAProxyConfigWithCerts(lb, listeners, tgByArn, rulesByListener, bindAddr, nil) return config, err } // GenerateHAProxyConfigWithCerts is GenerateHAProxyConfig plus TLS termination. -// certPEMByArn maps each listener certificate ARN to its combined PEM; secure -// frontends render `ssl crt ` and the returned certFiles map carries each -// path → PEM for delivery to the LB agent. A secure listener whose default cert -// ARN is absent from certPEMByArn renders without `ssl crt` (HAProxy would fail -// to bind, surfacing the missing material). +// certPEMByArn maps certificate ARNs to combined PEMs; secure frontends emit +// `ssl crt ` and the returned certFiles carries path→PEM for the LB agent. func GenerateHAProxyConfigWithCerts(lb *LoadBalancerRecord, listeners []*ListenerRecord, tgByArn map[string]*TargetGroupRecord, rulesByListener map[string][]*RuleRecord, bindAddr string, certPEMByArn map[string]string) (string, map[string]string, error) { // NLBs (L4) render an nginx `stream` config — HAProxy load-balances no UDP. if lb.Type == LoadBalancerTypeNetwork { @@ -177,8 +163,7 @@ func generateALBConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord, tgBy } // buildHAProxyConfig constructs the HAProxyConfig struct for an ALB (L7). -// rulesByListener provides per-listener rules sorted by priority. NLBs use the -// nginx data plane (generateNLBStreamConfig), not this builder. +// NLBs use the nginx data plane (generateNLBStreamConfig), not this builder. func buildHAProxyConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord, tgByArn map[string]*TargetGroupRecord, rulesByListener map[string][]*RuleRecord, bindAddr string, certPEMByArn map[string]string) (HAProxyConfig, error) { socketDir := filepath.Join(os.TempDir(), "spinifex-haproxy") // LoadBalancerID already carries the "lb-" prefix; must match the agent's socket path. @@ -262,9 +247,7 @@ func buildHAProxyConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord, tgB if isForwardDefault { defaultBackendName = sanitizeName("bk", da.TargetGroupArn) } else { - // redirect / fixed-response (or any non-forward default, e.g. the - // shared-ingress 404): synthesize a backend so HAProxy has a valid - // default_backend instead of a dangling reference. + // Non-forward default: synthesize a backend for HAProxy's default_backend. defaultBackendName = sanitizeName("bkdefault", l.ListenerArn) } @@ -276,10 +259,7 @@ func buildHAProxyConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord, tgB DefaultBackend: defaultBackendName, } - // Secure listeners terminate TLS: resolve the default cert to a stable - // per-listener PEM path and stage it for delivery. Without resolved - // material the frontend renders plain (HAProxy then fails to bind, - // surfacing the gap rather than silently serving cleartext). + // Resolve the default cert and stage it for delivery to the agent. if protocolRequiresCert(l.Protocol) { if pem, path := resolveFrontendCert(lb, l, certPEMByArn); pem != "" { frontend.CertPath = path @@ -320,16 +300,13 @@ func buildHAProxyConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord, tgB return cfg, nil } -// frontendCertPath returns the stable absolute path of a listener's combined -// PEM file under the agent's cert dir. Daemon-rendered and agent-written paths -// must match exactly. +// frontendCertPath returns the stable absolute PEM path under the agent's cert dir. func frontendCertPath(lb *LoadBalancerRecord, l *ListenerRecord) string { return filepath.Join(lbagent.CertDir, fmt.Sprintf("%s-%s.pem", lb.LoadBalancerID, l.ListenerID)) } -// resolveFrontendCert picks a listener's default certificate, resolves its PEM -// from certPEMByArn, and returns (pem, path). Returns ("", "") when the listener -// has no certificate or its default ARN is unresolved. +// resolveFrontendCert returns (pem, path) for the listener's default certificate, +// or ("", "") when absent or unresolved. func resolveFrontendCert(lb *LoadBalancerRecord, l *ListenerRecord, certPEMByArn map[string]string) (pem, path string) { if len(l.Certificates) == 0 || certPEMByArn == nil { return "", "" @@ -348,9 +325,7 @@ func resolveFrontendCert(lb *LoadBalancerRecord, l *ListenerRecord, certPEMByArn return pem, frontendCertPath(lb, l) } -// registerRuleBackend resolves the HAProxy backend a rule's single action -// routes to, registering a synthetic backend for redirect / fixed-response -// rule actions, and returns its name. +// registerRuleBackend registers the backend for a rule's single action and returns its name. func registerRuleBackend( r *RuleRecord, l *ListenerRecord, @@ -386,10 +361,8 @@ func registerRuleBackend( } } -// buildHAProxyRule converts a RuleRecord's conditions into the rendered -// ACL+use_backend shape, routing to backendName (already resolved/registered -// by registerRuleBackend). Returns an error if any condition value fails the -// defence-in-depth re-validation (HAProxy meta-character rejection). +// buildHAProxyRule converts a RuleRecord's conditions into ACL+use_backend shape. +// Returns an error if any condition value fails render-safe validation. func buildHAProxyRule(r *RuleRecord, backendName string) (HAProxyRule, error) { rule := HAProxyRule{Backend: backendName} @@ -408,9 +381,8 @@ func buildHAProxyRule(r *RuleRecord, backendName string) (HAProxyRule, error) { return rule, nil } -// buildHAProxyACLExprs returns one or more HAProxy ACL expressions for a -// single rule condition. Multiple expressions represent OR semantics over the -// condition's values (HAProxy ORs same-name ACL declarations). +// buildHAProxyACLExprs returns HAProxy ACL expressions for a rule condition. +// Multiple expressions represent OR semantics over the condition's values. func buildHAProxyACLExprs(c RuleCondition) ([]string, error) { switch c.Field { case RuleFieldHostHeader: @@ -481,10 +453,7 @@ func hostOrPathExprs(prefix string, values []string) ([]string, error) { } // wildcardMatch maps a host/path pattern to a HAProxy match flag + literal. -// Supports leading *, trailing *, or no wildcard. Embedded `?` or interior `*` -// fall through to substring match (`sub`) — AWS treats `?` as single-char -// wildcard; HAProxy lacks an exact equivalent so we widen to substring (false -// positives over false negatives is the right failure mode for a router). +// `?` and interior `*` widen to substring — false positives over false negatives. func wildcardMatch(v string) (flag, literal string) { hasLead := strings.HasPrefix(v, "*") hasTrail := strings.HasSuffix(v, "*") @@ -502,8 +471,7 @@ func wildcardMatch(v string) (flag, literal string) { } } -// contentTypeRegex matches a conservative MIME type (type/subtype) for the -// fixed-response content-type header rendered into the HAProxy config. +// contentTypeRegex matches a conservative type/subtype MIME string. var contentTypeRegex = regexp.MustCompile(`^[a-zA-Z0-9!#$&^_.+-]+/[a-zA-Z0-9!#$&^_.+-]+$`) // validHTTPStatusCode accepts a 3-digit 2xx–5xx HTTP status. @@ -524,9 +492,7 @@ func validContentType(s string) bool { return contentTypeRegex.MatchString(s) } -// validFixedResponseBody rejects bytes that would terminate the quoted HAProxy -// string or inject a directive. Spaces are allowed (the body renders inside -// double quotes). +// validFixedResponseBody rejects bytes that would terminate or inject into the HAProxy directive. func validFixedResponseBody(s string) bool { if s == "" { return false @@ -539,11 +505,9 @@ func validFixedResponseBody(s string) bool { return true } -// buildRedirect translates a stored RedirectAction into an HAProxy redirect -// directive in the context of listener l (which resolves the `#{protocol}` and -// `#{port}` placeholders). The common scheme-only redirect (e.g. HTTP→HTTPS -// with every other field left at its AWS default) renders as `redirect scheme` -// so HAProxy preserves host/path/query; anything else rebuilds a `location`. +// buildRedirect translates a RedirectAction into an HAProxy redirect directive. +// Scheme-only redirects (e.g. HTTP→HTTPS, all other fields default) render as +// `redirect scheme`; everything else rebuilds a `location`. func buildRedirect(rd *RedirectAction, l *ListenerRecord) (*HAProxyRedirect, error) { if rd == nil { return nil, fmt.Errorf("redirect action missing config") @@ -605,8 +569,7 @@ func orDefault(s, def string) string { return s } -// redirectScheme resolves the redirect scheme literal, mapping the `#{protocol}` -// placeholder (or an empty protocol) to the listener's own protocol. +// redirectScheme resolves the redirect scheme, mapping `#{protocol}` to the listener's protocol. func redirectScheme(proto string, l *ListenerRecord) string { switch proto { case ProtocolHTTP, ProtocolHTTPS: @@ -616,8 +579,7 @@ func redirectScheme(proto string, l *ListenerRecord) string { } } -// redirectPlaceholders substitutes the AWS redirect placeholders with HAProxy -// log-format fetches. `#{protocol}` and `#{port}` resolve from the listener. +// redirectPlaceholders substitutes AWS redirect placeholders with HAProxy log-format fetches. func redirectPlaceholders(s string, l *ListenerRecord) string { r := strings.NewReplacer( "#{protocol}", strings.ToLower(l.Protocol), @@ -629,10 +591,8 @@ func redirectPlaceholders(s string, l *ListenerRecord) string { return r.Replace(s) } -// validRedirectField rejects bytes that would break the HAProxy redirect -// directive. The known `#{...}` placeholders are stripped first, so any -// remaining `#`, `%`, bracket, quote, backslash, whitespace, or control byte -// (which would indicate an unknown placeholder or an injection attempt) fails. +// validRedirectField rejects bytes that would break the HAProxy redirect directive. +// Known `#{...}` placeholders are stripped first; any remaining forbidden byte fails. func validRedirectField(s string) bool { known := []string{"#{protocol}", "#{host}", "#{port}", "#{path}", "#{query}"} tmp := s @@ -651,10 +611,8 @@ func validRedirectField(s string) bool { return true } -// validateRenderSafe rejects bytes that could break ACL parsing or inject new -// directives. Mirrors the write-time validators in service_impl_rules.go; -// duplicated here so a future migration that bypasses the API cannot leak a -// malicious string into the generated config. +// validateRenderSafe rejects bytes that could break ACL parsing or inject directives. +// Duplicated from write-time validators so API-bypass cannot leak malicious strings. func validateRenderSafe(v string) error { if v == "" { return fmt.Errorf("empty rule token") @@ -678,7 +636,7 @@ func sanitizeACLToken(id string) string { return sb.String() } -// sanitizeName creates a safe identifier from an ARN or ID for use in HAProxy config. +// sanitizeName creates a safe HAProxy identifier from an ARN or ID. func sanitizeName(prefix, id string) string { // Use last segment of ARN or ID, replace non-alphanumeric with underscore parts := strings.Split(id, "/") diff --git a/spinifex/handlers/elbv2/haproxy_test.go b/spinifex/handlers/elbv2/haproxy_test.go index bfce81e86..084b294f0 100644 --- a/spinifex/handlers/elbv2/haproxy_test.go +++ b/spinifex/handlers/elbv2/haproxy_test.go @@ -161,11 +161,9 @@ func TestGenerateHAProxyConfig_SharedTargetGroup(t *testing.T) { assert.Equal(t, 1, strings.Count(config, "balance roundrobin")) } -// TestGenerateHAProxyConfig_FixedResponseDefault mirrors the shared-ingress -// shape: a listener whose default action is fixed-response (no target group), -// with host-header rules forwarding to per-app target groups. The generator -// must synthesize a default backend instead of emitting a dangling -// `default_backend bk_`, which makes HAProxy fail to start. +// TestGenerateHAProxyConfig_FixedResponseDefault covers a fixed-response default action +// with host-header rules forwarding to target groups. The generator must synthesize a +// default backend; a dangling `default_backend bk_` makes HAProxy fail to start. func TestGenerateHAProxyConfig_FixedResponseDefault(t *testing.T) { lb := &LoadBalancerRecord{LoadBalancerID: "lb-ingress"} listenerArn := "arn:aws:elasticloadbalancing:ap-southeast-2:1:listener/app/wd-ingress/lb-ingress/lst-616760baf6a3031c3" diff --git a/spinifex/handlers/elbv2/health_checker.go b/spinifex/handlers/elbv2/health_checker.go index 14aeec0b8..949dac8e2 100644 --- a/spinifex/handlers/elbv2/health_checker.go +++ b/spinifex/handlers/elbv2/health_checker.go @@ -9,13 +9,8 @@ import ( "github.com/mulgadc/spinifex/spinifex/lbagent" ) -// healthChecker processes target health reports from LB agents. Reports arrive -// via LBAgentHeartbeat (the agent pushes health data on each heartbeat). This -// checker maps HAProxy backend server statuses back to registered targets and -// updates HealthState in the store. -// -// This mirrors the AWS model where the LB itself health-checks the targets, -// rather than the control plane probing them directly. +// healthChecker processes target health reports from LB agents pushed via +// LBAgentHeartbeat, mapping backend statuses back to targets in the store. type healthChecker struct { store *Store @@ -37,7 +32,6 @@ func newHealthChecker(store *Store) *healthChecker { } // handleHealthReport unmarshals a JSON-encoded health report and processes it. -// This is the []byte entry point used by tests and any future wire-format callers. func (hc *healthChecker) handleHealthReport(data []byte) { var report lbagent.HealthReport if err := json.Unmarshal(data, &report); err != nil { @@ -48,8 +42,7 @@ func (hc *healthChecker) handleHealthReport(data []byte) { hc.handleHealthReportDirect(report) } -// handleHealthReportDirect processes a health report from an lb-agent without -// an intermediate JSON round-trip. Called directly from LBAgentHeartbeat. +// handleHealthReportDirect processes a health report without a JSON round-trip. func (hc *healthChecker) handleHealthReportDirect(report lbagent.HealthReport) { if len(report.Servers) == 0 { return @@ -61,8 +54,7 @@ func (hc *healthChecker) handleHealthReportDirect(report lbagent.HealthReport) { serverUp[srv.Server] = srv.Status == "UP" } - // Look up only the target groups attached to this LB's listeners. - // Falls back to scanning all target groups if LBID is missing. + // Look up target groups for this LB; fall back to all TGs if LBID is missing. var tgs []*TargetGroupRecord var err error if report.LBID != "" { @@ -75,8 +67,7 @@ func (hc *healthChecker) handleHealthReportDirect(report lbagent.HealthReport) { return } - // Collect changed TGs under the lock, then persist outside to avoid - // holding the mutex across KV store network I/O. + // Collect changed TGs under the lock; persist outside to avoid holding it across KV I/O. hc.mu.Lock() var changedTGs []*TargetGroupRecord for _, tg := range tgs { @@ -142,9 +133,8 @@ func (hc *healthChecker) handleHealthReportDirect(report lbagent.HealthReport) { } // evaluateHealth applies threshold logic to determine a target's new state. -// From "initial", a single healthy probe transitions to healthy (fast start, -// matching AWS behavior for newly registered targets). From "healthy" or -// "unhealthy", full threshold counts are required. +// From "initial", one healthy probe transitions to healthy (AWS fast-start); +// from "healthy"/"unhealthy", full threshold counts are required. func evaluateHealth(current string, ctr *targetCounter, cfg HealthCheckConfig) (string, string) { healthyThreshold := cfg.HealthyThreshold if healthyThreshold == 0 { diff --git a/spinifex/handlers/elbv2/launcher.go b/spinifex/handlers/elbv2/launcher.go index 26d65c184..14bb393d3 100644 --- a/spinifex/handlers/elbv2/launcher.go +++ b/spinifex/handlers/elbv2/launcher.go @@ -2,11 +2,8 @@ package handlers_elbv2 import "github.com/mulgadc/spinifex/spinifex/handlers/sysinstance" -// The system-instance launch contract now lives in the boot-agnostic -// handlers/sysinstance package so both ELBv2 and EKS can launch system VMs -// without an eks→elbv2 dependency. These aliases keep existing ELBv2 source -// (which builds direct-boot microVM inputs) compiling unchanged. ELBv2 always -// launches with BootMode=BootDirect (the zero value). +// These aliases re-export sysinstance types so ELBv2 and EKS share launch +// logic without an eks→elbv2 dependency. ELBv2 always uses BootMode=BootDirect. type ( SystemInstanceLauncher = sysinstance.SystemInstanceLauncher SystemInstanceInput = sysinstance.SystemInstanceInput diff --git a/spinifex/handlers/elbv2/lifecycle_invariants_test.go b/spinifex/handlers/elbv2/lifecycle_invariants_test.go index b2ac8b6e6..3d642e102 100644 --- a/spinifex/handlers/elbv2/lifecycle_invariants_test.go +++ b/spinifex/handlers/elbv2/lifecycle_invariants_test.go @@ -11,13 +11,9 @@ import ( "github.com/stretchr/testify/require" ) -// TestRLC1_ELBv2DeleteIdempotentOnAbsent enforces Common Resource Lifecycle -// Contract rule #1 (idempotent delete) across the ELBv2 surface: deleting an -// absent resource is success, not a NotFound error. This mirrors AWS ELBv2 -// delete semantics and is required for tofu destroy to converge on retry. -// -// Every ELBv2 delete endpoint MUST have a case here. When a new delete -// endpoint lands, add it; a missing case is an idempotency gap. +// TestRLC1_ELBv2DeleteIdempotentOnAbsent enforces RLC rule #1 (idempotent delete): +// deleting an absent ELBv2 resource is success, not NotFound. Every delete endpoint +// must have a case here — a missing case is an idempotency gap. func TestRLC1_ELBv2DeleteIdempotentOnAbsent(t *testing.T) { const arnBase = "arn:aws:elasticloadbalancing:us-east-1:123456789012:" @@ -57,10 +53,9 @@ func TestRLC1_ELBv2DeleteIdempotentOnAbsent(t *testing.T) { } } -// TestRLC2_ELBv2NoOrphanAfterDeleteLB enforces ADR-0002 §5 no-orphan -// completeness: after DeleteLoadBalancer, no listener or rule owned by that LB -// may survive in the store. Uses two listeners (each with a rule) so a -// single-listener cascade can't pass by accident. +// TestRLC2_ELBv2NoOrphanAfterDeleteLB enforces ADR-0002 §5 no-orphan completeness: +// after DeleteLoadBalancer no listener or rule owned by the LB may remain. Two +// listeners are used so a single-listener cascade can't pass by accident. func TestRLC2_ELBv2NoOrphanAfterDeleteLB(t *testing.T) { svc := setupTestService(t) @@ -102,13 +97,9 @@ func TestRLC2_ELBv2NoOrphanAfterDeleteLB(t *testing.T) { require.Emptyf(t, rules, "ADR-0002 §5 no-orphan completeness: no rule owned by a deleted LB may remain") } -// TestRLC3_ELBv2DeleteLBDoesNotBypassListenerCascade enforces ADR-0002 §5 no -// store-bypass cascade: DeleteLoadBalancer must tear listeners (and their -// rules) down through the shared deleteListenerCascade helper, never by -// calling store.DeleteListener directly — a direct store delete orphans the -// listener's rules. The store field is a concrete type with no injection seam, -// so the contract is enforced structurally against the source rather than with -// a runtime spy. +// TestRLC3_ELBv2DeleteLBDoesNotBypassListenerCascade enforces ADR-0002 §5 no-store-bypass +// cascade: DeleteLoadBalancer must use deleteListenerCascade, not store.DeleteListener +// directly (a direct call orphans rules). Enforced structurally against source. func TestRLC3_ELBv2DeleteLBDoesNotBypassListenerCascade(t *testing.T) { src, err := os.ReadFile("service_impl.go") require.NoError(t, err) @@ -148,10 +139,9 @@ func stripComments(src string) string { return b.String() } -// TestRLC4_ELBv2TGDeletableAfterLBTeardown enforces ADR-0002 §5 TG deletability -// after LB teardown: create LB+listener+rule→TG, delete the LB, then the target -// group must delete successfully rather than staying pinned as ResourceInUse by -// an orphaned rule or stale listener default action. +// TestRLC4_ELBv2TGDeletableAfterLBTeardown enforces ADR-0002 §5 TG deletability: +// after LB+listener+rule teardown, the target group must not remain pinned as +// ResourceInUse by an orphaned rule or stale listener default action. func TestRLC4_ELBv2TGDeletableAfterLBTeardown(t *testing.T) { svc := setupTestService(t) diff --git a/spinifex/handlers/elbv2/nginx.go b/spinifex/handlers/elbv2/nginx.go index 46daab107..d1d6e115f 100644 --- a/spinifex/handlers/elbv2/nginx.go +++ b/spinifex/handlers/elbv2/nginx.go @@ -11,12 +11,9 @@ import ( "github.com/mulgadc/spinifex/spinifex/lbagent" ) -// NLB (L4) data plane. HAProxy load-balances no UDP in any release, so NLB -// listeners (TCP/UDP/TLS/TCP_UDP) render an nginx `stream` config instead; -// ALBs stay on HAProxy. See docs/development/feature/nlb-nginx-l4-dataplane.md. -// -// Health checks are passive only (max_fails / fail_timeout) — OSS nginx stream -// has no active upstream probing. +// NLB (L4) data plane. NLB listeners render an nginx `stream` config because HAProxy +// has no UDP support; ALBs stay on HAProxy. Health checks are passive only +// (max_fails/fail_timeout) — OSS nginx stream has no active upstream probing. const nginxStreamConfigTemplate = `# Auto-generated by spinifex ELBv2 (NLB) — do not edit load_module /usr/lib/nginx/modules/ngx_stream_module.so; @@ -100,11 +97,9 @@ func engineForType(lbType string) string { return EngineHAProxy } -// generateNLBStreamConfig builds an nginx `stream` config for an NLB. Secure -// (TLS) listeners terminate TLS at nginx and forward plaintext to the target; -// the returned certFiles map carries each cert path → combined PEM for delivery -// to the LB agent. bindAddr is unused (nginx listens on all addresses, matching -// the HAProxy `bind *` behaviour). +// generateNLBStreamConfig builds an nginx `stream` config for an NLB; TLS listeners +// terminate at nginx. certFiles maps cert path → combined PEM for delivery to the +// agent. bindAddr is unused — nginx listens on all addresses like HAProxy's `bind *`. func generateNLBStreamConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord, tgByArn map[string]*TargetGroupRecord, _ string, certPEMByArn map[string]string) (string, map[string]string, error) { cfg := nginxConfig{PIDPath: lbagent.NginxPIDPath} var certFiles map[string]string @@ -187,12 +182,9 @@ func nginxCertPath(lb *LoadBalancerRecord, l *ListenerRecord) string { return filepath.Join(lbagent.NginxCertDir, fmt.Sprintf("%s-%s.pem", lb.LoadBalancerID, l.ListenerID)) } -// buildNLBHealthTargets computes the active health-check targets for an NLB's -// backends. nginx `stream` has no active upstream probing, so the agent probes -// these and reports the results via heartbeat. ServerName matches the health -// checker's key (sanitizeName("srv", target.Id)); the probe port resolves -// "traffic-port" to the target's port. Sorted by server name for deterministic -// delivery. The tgByArn map is the same set used to render the upstreams. +// buildNLBHealthTargets returns backends the agent actively probes (nginx stream +// has no built-in active probing). ServerName matches sanitizeName("srv", target.Id); +// sorted by name for deterministic delivery. func buildNLBHealthTargets(tgByArn map[string]*TargetGroupRecord) []HealthTargetSpec { var out []HealthTargetSpec for _, tg := range tgByArn { diff --git a/spinifex/handlers/elbv2/service_impl.go b/spinifex/handlers/elbv2/service_impl.go index 3eee3bfb1..b497a1c31 100644 --- a/spinifex/handlers/elbv2/service_impl.go +++ b/spinifex/handlers/elbv2/service_impl.go @@ -36,15 +36,11 @@ const ( // elbv2LBTag is the tag key storing the parent LB ARN on managed ENIs. elbv2LBTag = tags.LBARNKey - // heartbeatPersistInterval controls how often a no-op heartbeat (no state - // change) writes the full LoadBalancerRecord back to KV. State transitions - // always persist immediately. + // heartbeatPersistInterval is how often a no-op heartbeat writes to KV. heartbeatPersistInterval = 60 * time.Second - // Health check fields are interpolated verbatim into the HAProxy template - // (see haproxy.go); restrict them to characters that cannot terminate a - // directive or introduce a new one. Length caps bound abuse and match the - // surface AWS exposes for these fields. + // Health check fields are interpolated into the HAProxy template; restrict + // to characters that cannot terminate or inject directives. maxHealthCheckPathLen = 1024 maxHealthCheckMatcherLen = 64 ) @@ -68,21 +64,9 @@ func validateHealthCheckMatcher(m string) error { return nil } -// resolveMgmtRoute returns the (gateway, target) pair for the host route the -// lb-agent uses to reach AWSGW from inside the LB microVM, or empty strings -// to skip the route. Consumed by buildMicrovmNICs to populate NIC[1]'s -// RouteDst/RouteVia for the netcfg fw_cfg blob. -// -// Multi-node (AWSGW on dedicated mgmt IP): MgmtRoute{Gateway,Target} are set -// by the daemon for both schemes — return them. -// -// Single-node (AWSGW on advertiseIP, MgmtRoute fields empty): internet-facing -// LBs reach AWSGW via VPC → external (their EIP gives OVN a SNAT pair for the -// reply), so they need no mgmt route — and adding a /32 to advertiseIP would -// steal the host's WAN return path. Internal LBs have no EIP and no SNAT pair, -// so the VPC egress reply targets the VM's private IP from the host's WAN with -// no route back → packet drops, lb-agent never heartbeats, LB sticks in -// provisioning. Force the mgmt route for internal scheme via the br-mgmt IP. +// resolveMgmtRoute returns the (gateway, target) for the lb-agent's host route to AWSGW. +// Multi-node: returns MgmtRoute{Gateway,Target}. Single-node internet-facing: no route +// (EIP provides SNAT). Single-node internal: route forced via br-mgmt IP. func (s *ELBv2ServiceImpl) resolveMgmtRoute(scheme string) (gateway, target string) { if s.MgmtRouteGateway != "" && s.MgmtRouteTarget != "" { return s.MgmtRouteGateway, s.MgmtRouteTarget @@ -132,10 +116,9 @@ type ELBv2ServiceImpl struct { recoveryLBIndexMu sync.Mutex recoveryLBIndex map[string]*LoadBalancerRecord - // launchWG tracks in-flight asynchronous LB-VM launches spawned by - // CreateLoadBalancer. Not awaited at shutdown (a multi-minute boot must not - // block Close); exposed via WaitLaunches purely so tests can join the - // background launch deterministically. + // launchWG tracks in-flight LB-VM launches from CreateLoadBalancer; not awaited + // at shutdown to avoid blocking Close on a multi-minute boot. Exposed via + // WaitLaunches so tests can join the background launch deterministically. launchWG sync.WaitGroup } @@ -150,10 +133,7 @@ func NewELBv2ServiceImplWithNATS(cfg *config.Config, nc *nats.Conn) (*ELBv2Servi return nil, fmt.Errorf("failed to create ELBv2 store: %w", err) } - // ACM store shares the same JetStream KV — read cert PEM directly, no - // cross-service NATS hop. Non-fatal: a failure here only disables HTTPS - // termination (resolveCertPEM returns an error), it must not block the - // daemon from serving ELBv2 control-plane requests. + // ACM store shares JetStream KV. Non-fatal: failure only disables HTTPS termination. acmStore, acmErr := handlers_acm.NewStore(nc) if acmErr != nil { slog.Warn("ELBv2: ACM store unavailable, HTTPS listeners cannot resolve certs", "err", acmErr) @@ -192,29 +172,9 @@ func (s *ELBv2ServiceImpl) Close() { } } -// ResetTargetHealthOnStartup invalidates persisted target HealthState across -// all target groups by resetting every non-draining target to "initial". Run -// once during daemon startup before the service begins serving requests. -// -// Rationale: target HealthState is persisted in JetStream KV, but the -// healthChecker's per-target counters live only in process memory. After a -// daemon restart (e.g. host reboot), the counters are gone but the stored -// "healthy" claim remains — DescribeTargetHealth returns "healthy" -// immediately even though the daemon has zero active health observations -// and the LB system VM's lb-agent hasn't posted a fresh report yet. Tests -// that gate on WaitForTargetsHealthy advance prematurely, then ALB traffic -// lands on a backend that isn't actually up. -// -// Resetting to "initial" is the correct AWS-semantics representation of -// "we don't know yet, give us a moment" — the next lb-agent heartbeat -// drives the state machine forward through evaluateHealth. -// -// Multi-node caveat: in a multi-daemon cluster, each daemon that restarts -// resets the SHARED KV state. A sibling daemon's in-memory counters would -// then be out-of-sync with the persisted "initial" state until the next -// heartbeat re-converges. Acceptable trade-off — heartbeats arrive within -// one health-check interval (default 5-30s) and round-robin transiently -// excluding a backend is safer than serving traffic to a dead one. +// ResetTargetHealthOnStartup resets every non-draining target to "initial" so +// stale "healthy" state doesn't mislead DescribeTargetHealth after a restart; +// the next lb-agent heartbeat re-evaluates and converges within one interval. func (s *ELBv2ServiceImpl) ResetTargetHealthOnStartup(ctx context.Context) error { if s == nil || s.store == nil { return nil @@ -232,8 +192,7 @@ func (s *ELBv2ServiceImpl) ResetTargetHealthOnStartup(ctx context.Context) error changed := false for i := range tg.Targets { t := &tg.Targets[i] - // Skip targets actively being drained — that state is driven by - // DeregisterTargets and must not be clobbered by a daemon restart. + // Draining state is driven by DeregisterTargets; don't clobber it. if t.HealthState == TargetHealthDraining { continue } @@ -261,8 +220,7 @@ func (s *ELBv2ServiceImpl) ResetTargetHealthOnStartup(ctx context.Context) error return nil } -// SetSystemInstanceTypeFunc sets a function that resolves the smallest available -// instance type. Called at request time so it adapts to node capacity. +// SetSystemInstanceTypeFunc sets a function that resolves the smallest available instance type. func (s *ELBv2ServiceImpl) SetSystemInstanceTypeFunc(fn func() string) { s.systemInstanceTypeMu.Lock() defer s.systemInstanceTypeMu.Unlock() @@ -271,9 +229,7 @@ func (s *ELBv2ServiceImpl) SetSystemInstanceTypeFunc(fn func() string) { s.systemInstanceTypeResolved = false } -// getSystemInstanceType returns the instance type for system VMs. -// Only caches non-empty results so the resolver retries when no capacity -// is available yet. +// getSystemInstanceType returns the instance type for system VMs, caching only non-empty results. func (s *ELBv2ServiceImpl) getSystemInstanceType() string { s.systemInstanceTypeMu.Lock() defer s.systemInstanceTypeMu.Unlock() @@ -296,9 +252,8 @@ func (s *ELBv2ServiceImpl) buildLBAgentEnv(lbID string) string { lbID, s.GatewayURL, s.SystemAccessKey, s.SystemSecretKey, s.region) } -// subnetCIDRForIP computes "ip/prefixlen" given a host IP and the subnet's -// CIDR block (e.g. "10.0.1.0/24" → "10.0.1.5/24"). Returns empty string on -// parse failure so the caller can log and continue without panicking. +// subnetCIDRForIP returns "ip/prefixlen" from a host IP and subnet CIDR block. +// Returns empty string on parse failure. func subnetCIDRForIP(hostIP, subnetCIDR string) string { _, ipNet, err := net.ParseCIDR(subnetCIDR) if err != nil { @@ -325,17 +280,9 @@ func subnetGatewayIP(subnetCIDR string) string { return gwCopy.String() } -// buildMicrovmNICs constructs the NIC slice for a direct-boot microvm launch. -// -// NIC[0] is the primary VPC ENI (IsDefault=true): MAC/IP from the ENI record, -// CIDR from ip+subnet prefix, gateway = subnet network address +1. -// -// NIC[1] is the management NIC (IsDefault=false): MAC and CIDR are left empty -// here — the daemon injects them after allocating the per-instance mgmt IP. -// RouteDst/RouteVia encode the host route the lb-agent needs to reach AWSGW -// via the management interface (same logic as resolveMgmtRoute). -// -// NIC[2+] are extra VPC ENIs for multi-subnet ALBs. +// buildMicrovmNICs constructs the NIC slice for a direct-boot microvm. +// NIC[0] is the primary VPC ENI; NIC[1] is the mgmt NIC (MAC/CIDR filled +// post-allocation); NIC[2+] are extra ENIs for multi-subnet ALBs. func (s *ELBv2ServiceImpl) buildMicrovmNICs(primaryIP, primaryMAC, primarySubnetID, _, scheme string, extraENIs []ExtraENIInput, accountID string) []NICConfig { // Resolve primary subnet CIDR for NIC[0] prefix and gateway. primaryCIDR := "" @@ -359,11 +306,8 @@ func (s *ELBv2ServiceImpl) buildMicrovmNICs(primaryIP, primaryMAC, primarySubnet }, } - // NIC[1]: management NIC — daemon fills MAC/CIDR after IP allocation. - // RouteDst = AWSGW IP (destination), RouteVia = br-mgmt IP (next-hop). - // Honor resolveMgmtRoute's deliberate empty return for single-node - // internet-facing: adding a /32 to AdvertiseIP via mgmt steals the host's - // WAN return path for reply traffic, breaking same-chassis ingress. + // NIC[1]: mgmt NIC — daemon fills MAC/CIDR. Empty return from resolveMgmtRoute + // intentionally skips the route for single-node internet-facing LBs. mgmtRouteGW, mgmtRouteTarget := s.resolveMgmtRoute(scheme) nics = append(nics, NICConfig{ IsDefault: false, @@ -394,10 +338,8 @@ func (s *ELBv2ServiceImpl) buildMicrovmNICs(primaryIP, primaryMAC, primarySubnet return nics } -// describeENIs resolves the supplied ENI IDs in one VPC describe call and -// returns them keyed by NetworkInterfaceId. Returns an empty map when the -// VPC service is unwired or the describe errors — callers must tolerate -// missing entries. +// describeENIs resolves ENI IDs in one VPC call, keyed by NetworkInterfaceId. +// Returns an empty map when the VPC service is unavailable or the call errors. func (s *ELBv2ServiceImpl) describeENIs(eniIDs []string, accountID string) map[string]*ec2.NetworkInterface { eniDetails := make(map[string]*ec2.NetworkInterface, len(eniIDs)) if s.VPCService == nil || len(eniIDs) == 0 { @@ -422,11 +364,8 @@ func (s *ELBv2ServiceImpl) describeENIs(eniIDs []string, accountID string) map[s return eniDetails } -// buildExtraENIInputs threads every non-primary ENI in eniIDs into an -// ExtraENIInput, using eniDetails as the MAC/IP/SubnetID source. eniIDs[0] -// is the primary ENI and is intentionally excluded. ENIs missing from -// eniDetails are skipped (matching CreateLoadBalancer's original behaviour -// when DescribeNetworkInterfaces drops an entry). +// buildExtraENIInputs builds ExtraENIInput for all non-primary ENIs (eniIDs[0] excluded). +// ENIs missing from eniDetails are skipped. func buildExtraENIInputs(eniIDs []string, eniDetails map[string]*ec2.NetworkInterface) []ExtraENIInput { if len(eniIDs) <= 1 { return nil @@ -447,9 +386,8 @@ func buildExtraENIInputs(eniIDs []string, eniDetails map[string]*ec2.NetworkInte return extras } -// lbVMLaunch is the outcome of booting the system VM that backs a load -// balancer. failed is true when the VM could not be launched (missing system -// credentials or a launcher error) so the caller can mark the LB failed. +// lbVMLaunch is the outcome of booting the LB system VM. failed is true when +// the VM could not be launched. type lbVMLaunch struct { instanceID string vpcIP string @@ -458,12 +396,9 @@ type lbVMLaunch struct { failed bool } -// launchLBVM boots the system VM backing a load balancer with the given ENI -// set. The first ENI is the primary NIC; any additional ENIs are passed as -// ExtraENIs so the daemon sets up a tap + QEMU NIC per subnet, giving the LB VM -// data-plane presence in every subnet it spans. When no launcher is configured -// the call is a no-op (zero value, failed=false) so launcher-less deployments -// keep the LB Active. Shared by CreateLoadBalancer and the SetSubnets relaunch. +// launchLBVM boots the system VM for a load balancer. The first ENI is the +// primary NIC; extras give multi-subnet data-plane presence. No-op when no +// launcher is configured. Shared by CreateLoadBalancer and SetSubnets. func (s *ELBv2ServiceImpl) launchLBVM(lbID, scheme string, eniIDs, subnets []string, accountID string) lbVMLaunch { var res lbVMLaunch if s.InstanceLauncher == nil || len(eniIDs) == 0 || len(subnets) == 0 { @@ -522,10 +457,8 @@ func (s *ELBv2ServiceImpl) launchLBVM(lbID, scheme string, eniIDs, subnets []str return res } -// loadRecoveryLBIndex returns the instanceID->LB map, populated lazily on -// first successful call. Errors are not cached: a transient JetStream -// failure at startup must not condemn every subsequent recovery candidate -// in the same batch — the next caller retries. +// loadRecoveryLBIndex returns the instanceID→LB map, populated lazily. +// Errors are not cached so transient JetStream failures don't condemn recovery. func (s *ELBv2ServiceImpl) loadRecoveryLBIndex() (map[string]*LoadBalancerRecord, error) { s.recoveryLBIndexMu.Lock() defer s.recoveryLBIndexMu.Unlock() @@ -546,10 +479,8 @@ func (s *ELBv2ServiceImpl) loadRecoveryLBIndex() (map[string]*LoadBalancerRecord return s.recoveryLBIndex, nil } -// RebuildSystemInstanceInput reconstructs the launch input for an existing -// system instance during host-reboot recovery. The instance->LB map is -// memoised in s.recoveryLBIndex so N concurrent recovery candidates share a -// single ListLoadBalancers call instead of issuing N×L JetStream round-trips. +// RebuildSystemInstanceInput reconstructs the launch input for host-reboot recovery. +// The instance→LB map is memoised so concurrent recovery candidates share one ListLoadBalancers call. func (s *ELBv2ServiceImpl) RebuildSystemInstanceInput(ctx RecoveryContext) (*SystemInstanceInput, error) { index, err := s.loadRecoveryLBIndex() if err != nil { @@ -564,10 +495,7 @@ func (s *ELBv2ServiceImpl) RebuildSystemInstanceInput(ctx RecoveryContext) (*Sys } eniDetails := s.describeENIs(lb.ENIs, lb.AccountID) - // Multi-ENI rebuild needs an authoritative MAC/IP/subnet per extra, which - // only the VPC store supplies. Single-ENI ALBs can fall back to ctx.ENIMac - // + lb.VPCIP for the primary, so we only enforce the strict count match - // when extras are present. + // For multi-ENI LBs the VPC store must supply all MAC/IP/subnet details. if len(lb.ENIs) > 1 && len(eniDetails) < len(lb.ENIs) { return nil, fmt.Errorf("describe lb %s ENIs: got %d/%d", lb.LoadBalancerID, len(eniDetails), len(lb.ENIs)) } @@ -583,10 +511,7 @@ func (s *ELBv2ServiceImpl) RebuildSystemInstanceInput(ctx RecoveryContext) (*Sys extraENIs := buildExtraENIInputs(lb.ENIs, eniDetails) nics := s.buildMicrovmNICs(lb.VPCIP, primaryMAC, lb.Subnets[0], lb.ENIs[0], lb.Scheme, extraENIs, lb.AccountID) - // Re-inject the daemon-allocated mgmt NIC values so writeFwCfgBlobs - // emits the same netcfg the original launch produced — buildMicrovmNICs - // leaves NIC[1] MAC/CIDR blank because they are only known after the - // daemon's per-instance mgmt IP allocation. + // Re-inject mgmt NIC MAC/CIDR — buildMicrovmNICs leaves them blank. if len(nics) > 1 && ctx.MgmtMAC != "" { nics[1].MAC = ctx.MgmtMAC if ctx.MgmtIP != "" { @@ -630,10 +555,8 @@ func (s *ELBv2ServiceImpl) resolveENIBindAddr(lb *LoadBalancerRecord) string { return "" } -// updateStoredConfig generates the HAProxy config for an ALB, hashes it, and -// stores both ConfigText and ConfigHash on the LB record. The agent pulls this -// on its next heartbeat when it detects a hash change. Safe to call when -// instanceLauncher is nil or the LB has no VM. +// updateStoredConfig generates the data-plane config, hashes it, and stores it +// on the LB record. The agent fetches it on the next heartbeat when the hash changes. func (s *ELBv2ServiceImpl) updateStoredConfig(lb *LoadBalancerRecord) error { if lb.InstanceID == "" { return nil @@ -701,8 +624,7 @@ func (s *ELBv2ServiceImpl) updateStoredConfig(lb *LoadBalancerRecord) error { lb.ConfigText = configContent lb.ConfigHash = hash lb.CertFiles = certFiles - // NLBs run nginx, which has no active upstream probing — the agent - // probes these targets and reports health. ALBs report via HAProxy stats. + // NLBs: the agent actively probes targets (nginx has no upstream probing). if lb.Type == LoadBalancerTypeNetwork { lb.HealthTargets = buildNLBHealthTargets(tgByArn) } else { @@ -719,9 +641,8 @@ func (s *ELBv2ServiceImpl) updateStoredConfig(lb *LoadBalancerRecord) error { return nil } -// resolveListenerCerts resolves every distinct certificate ARN referenced by -// the listeners to its combined PEM via the ACM store. Returns nil when no -// listener carries a certificate (the common HTTP-only case). +// resolveListenerCerts resolves each distinct certificate ARN to its combined PEM. +// Returns nil for HTTP-only listeners. func (s *ELBv2ServiceImpl) resolveListenerCerts(listeners []*ListenerRecord, accountID string) (map[string]string, error) { var out map[string]string for _, l := range listeners { @@ -742,10 +663,8 @@ func (s *ELBv2ServiceImpl) resolveListenerCerts(listeners []*ListenerRecord, acc return out, nil } -// resolveCertPEM loads a certificate by ARN from the ACM store and returns its -// combined PEM (leaf + chain + key) in the order HAProxy `ssl crt` expects. -// Ownership is enforced: a cert belonging to another account is treated as -// absent. +// resolveCertPEM loads a certificate from ACM and returns its combined PEM +// (leaf + chain + key). Cross-account certs are treated as absent. func (s *ELBv2ServiceImpl) resolveCertPEM(arn, accountID string) (string, error) { if s.acmStore == nil { return "", errors.New(awserrors.ErrorELBv2CertificateNotFound) @@ -770,13 +689,9 @@ func (s *ELBv2ServiceImpl) resolveCertPEM(arn, accountID string) (string, error) return b.String(), nil } -// validateListenerCerts confirms every certificate ARN resolves in the ACM -// store and is owned by the account, rejecting an unknown ARN at the API -// boundary (CreateListener/ModifyListener) with ErrorELBv2CertificateNotFound. -// Without this, an unresolvable ARN is accepted into the listener and only -// fails later at config-render time, which aborts updateStoredConfig wholesale -// and silently freezes the LB's data-plane convergence. Skipped when the ACM -// store is unavailable (cannot validate; matches the nil-safe resolve path). +// validateListenerCerts confirms every certificate ARN resolves in the ACM store +// and is owned by the account. Rejects at the API boundary to avoid silently +// freezing data-plane convergence at config-render time. func (s *ELBv2ServiceImpl) validateListenerCerts(certs []ListenerCertificate, accountID string) error { if s.acmStore == nil { return nil @@ -789,9 +704,8 @@ func (s *ELBv2ServiceImpl) validateListenerCerts(certs []ListenerCertificate, ac return nil } -// configCertHash hashes the rendered config plus every delivered cert file -// (path + content, in sorted path order) so a cert rotation changes the hash -// and triggers an agent reload even when the config text is unchanged. +// configCertHash hashes the config and all cert files so a cert rotation triggers +// an agent reload even when the config text is unchanged. func configCertHash(configContent string, certFiles map[string]string) string { h := sha256.New() h.Write([]byte(configContent)) @@ -827,7 +741,7 @@ func (s *ELBv2ServiceImpl) updateStoredConfigForTargetGroup(tgArn string) error } } - // Rules can independently forward to a TG even if the listener default does not. + // Rules may forward to a TG even if the listener default action does not. allRules, rulesErr := s.store.ListRules() if rulesErr != nil { slog.Error("updateStoredConfigForTargetGroup: failed to list rules", "err", rulesErr) @@ -860,10 +774,8 @@ func (s *ELBv2ServiceImpl) updateStoredConfigForTargetGroup(tgArn string) error return nil } -// LBAgentHeartbeat processes a heartbeat from an LB agent. On first heartbeat, -// transitions LB from provisioning → active. Always updates LastHeartbeat and -// processes the health report. Returns the current config hash so the agent -// knows whether to fetch new config. +// LBAgentHeartbeat processes a heartbeat from an LB agent. On first heartbeat +// transitions LB provisioning→active, processes health, and returns the config hash. func (s *ELBv2ServiceImpl) LBAgentHeartbeat(input *LBAgentHeartbeatInput, accountID string) (*LBAgentHeartbeatOutput, error) { if input.LBID == nil || *input.LBID == "" { return nil, errors.New(awserrors.ErrorMissingParameter) @@ -877,10 +789,7 @@ func (s *ELBv2ServiceImpl) LBAgentHeartbeat(input *LBAgentHeartbeatInput, accoun return nil, errors.New(awserrors.ErrorServerInternal) } if lb == nil || (lb.AccountID != accountID && accountID != utils.GlobalAccountID) { - // The heartbeat reached AWSGW (so the mgmt return path works) but - // cannot transition any LB. Without this the reject is silent and a - // stuck-in-provisioning LB looks identical to one whose heartbeat - // never arrived. + // Log to distinguish a stuck-in-provisioning LB from one whose heartbeat never arrived. slog.Warn("LBAgentHeartbeat: LB not found or account mismatch", "lbId", lbID, "accountId", accountID, "found", lb != nil) return nil, errors.New(awserrors.ErrorELBv2LoadBalancerNotFound) @@ -900,21 +809,14 @@ func (s *ELBv2ServiceImpl) LBAgentHeartbeat(input *LBAgentHeartbeatInput, accoun switch { case stateChanged: - // Activation is the LB's readiness signal: InstanceID is set and the - // agent is alive to consume config. The reactive updateStoredConfig - // calls during the create burst no-op while the VM is still launching - // (InstanceID empty), so build the data-plane config now from the - // listeners/targets created during provisioning. updateStoredConfig - // persists the record (state, config, hash, health targets) in one - // write, so the ConfigHash returned below is fresh. + // Build data-plane config on activation: reactive calls during create + // no-op while InstanceID is empty, so we build it now from existing listeners/targets. if err := s.updateStoredConfig(lb); err != nil { slog.Error("LBAgentHeartbeat: failed to build config on activation", "lbId", lbID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } case heartbeatStale: - // Steady state: refresh the heartbeat timestamp only. Avoids writing - // the full record (including ConfigText) every 5 seconds when nothing - // changed. + // Refresh the heartbeat timestamp only; avoid writing the full record on every tick. if err := s.store.PutLoadBalancer(lb); err != nil { slog.Error("LBAgentHeartbeat: failed to persist LB", "lbId", lbID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -975,8 +877,7 @@ func healthTargetsToSDK(specs []HealthTargetSpec) []*HealthTarget { return out } -// certFilesToSDK converts the stored path→PEM map into the sorted CertFile -// slice delivered to the agent. Sorted for deterministic output. +// certFilesToSDK converts the stored path→PEM map into a sorted CertFile slice. func certFilesToSDK(certFiles map[string]string) []*CertFile { if len(certFiles) == 0 { return nil @@ -1031,11 +932,9 @@ func (s *ELBv2ServiceImpl) resolveTargetIP(instanceID, accountID string) string return "" } -// resolveRegisteredTargetIP returns the private IP to route to for a target -// being registered, based on the target group's target type. For ip targets the -// supplied ID is the IP itself; for instance targets it is resolved via the -// instance's primary ENI. Returns an error when the ID's shape does not match -// the target type. +// resolveRegisteredTargetIP returns the private IP for a target being registered. +// For ip-type targets the ID is the IP; for instance targets it is resolved via +// the primary ENI. Returns an error if the ID doesn't match the target type. func (s *ELBv2ServiceImpl) resolveRegisteredTargetIP(targetType, id, accountID string) (string, error) { switch targetType { case TargetTypeIP: @@ -1071,12 +970,8 @@ const ( elbv2ResourceListenerRule = "listener-rule" ) -// elbv2ResourceTypeFromArn extracts the resource type from an ELBv2 ARN. -// ELBv2 ARNs have the form -// "arn:aws:elasticloadbalancing:{region}:{account}:{type}/...". Returns one -// of "loadbalancer", "targetgroup", "listener", "listener-rule" — anything -// else yields ErrorInvalidParameterValue so callers can surface a proper API -// error. +// elbv2ResourceTypeFromArn extracts the resource type from an ELBv2 ARN, +// returning one of "loadbalancer", "targetgroup", "listener", "listener-rule". func elbv2ResourceTypeFromArn(arn string) (string, error) { parts := strings.SplitN(arn, ":", 6) if len(parts) < 6 || parts[0] != "arn" || parts[2] != "elasticloadbalancing" { @@ -1096,9 +991,7 @@ func elbv2ResourceTypeFromArn(arn string) (string, error) { } } -// isCompatibleProtocol checks whether a listener protocol is compatible with a -// target group protocol. ALB: HTTP/HTTPS listeners can forward to HTTP or HTTPS -// TGs. NLB: TCP→TCP, UDP→UDP, TLS→TCP or TLS, TCP_UDP→TCP_UDP. +// isCompatibleProtocol reports whether a listener protocol is compatible with a target group protocol. func isCompatibleProtocol(listenerProto, tgProto string) bool { switch listenerProto { case ProtocolHTTP, ProtocolHTTPS: @@ -1166,11 +1059,8 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp } dnsName := fmt.Sprintf("%s%s-%s.%s.elb.spinifex.local", dnsPrefix, name, lbID, s.region) - // Atomically claim the LB name before any ENI/VM work. A duplicate - // CreateLoadBalancer (SDK retry / gateway re-publish that spawned a second - // handler) loses the claim and never launches an LB VM; a claim left by a - // crashed prior create (no live record) is reclaimed. Every failure path - // below releases the claim so a failed create does not block the name. + // Atomically claim the name before ENI/VM work. SDK retries lose the claim; + // orphaned claims from crashed creates are reclaimed. Every failure releases it. claimOK, claimDup, claimErr := s.store.ClaimLBName(name, accountID, lbID) if claimErr != nil { slog.Error("CreateLoadBalancer: name claim failed", "name", name, "err", claimErr) @@ -1205,10 +1095,9 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp vpcID := "" var nlbManagedSGID string if s.VPCService != nil && len(subnets) > 0 { - // NLBs reject customer SGs (rejected above), so the ENIs would fall back - // to the VPC default SG whose intra-SG-only ingress drops inbound - // listener traffic. Mint a dedicated managed SG up front and attach it to - // every LB ENI; CreateListener opens the listener ports on it. + // NLBs reject customer SGs, so mint a dedicated managed SG for all LB ENIs; + // CreateListener opens listener ports on it. Without this, ENIs fall back + // to the VPC default SG and inbound listener traffic is dropped. eniGroups := securityGroups if lbType == LoadBalancerTypeNetwork { sgID, sgErr := s.createNLBManagedSG(lbID, lbArn, subnets[0], accountID) @@ -1234,18 +1123,15 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp }, }, } - // SG enforcement gates inbound traffic by the ENI's port-group - // membership, which is set at ENI creation. For NLBs this is the - // managed SG minted above; for ALBs it is the customer SGs. Empty - // Groups intentionally falls back to the VPC default SG inside - // eni.CreateNetworkInterface. + // SG enforcement gates inbound traffic via ENI port-group membership set at + // creation. NLBs use the managed SG; ALBs use customer SGs. Empty Groups + // falls back to the VPC default SG inside CreateNetworkInterface. if len(eniGroups) > 0 { eniIn.Groups = aws.StringSlice(eniGroups) } eniOut, eniErr := s.VPCService.CreateNetworkInterface(eniIn, accountID) if eniErr != nil { - // All-or-nothing: tear down the ENIs/SG already created so a - // failed create leaks nothing and frees the name claim. + // Rollback so a partial ENI creation doesn't leak resources. s.rollbackLBInfra(eniIDs, nlbManagedSGID, name, accountID) slog.Error("CreateLoadBalancer: failed to create ENI", "subnet", subnetID, "err", eniErr) return nil, errors.New(awserrors.ErrorELBv2SubnetNotFound) @@ -1268,24 +1154,14 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp } } - // Determine the data-plane state synchronously. When a launcher is configured - // the LB is backed by a system VM whose boot is a multi-minute job: running it - // inline head-of-line blocks the shared gateway responder and the gateway - // times out and re-publishes the create (267.4). So the VM boot runs on a - // background goroutine and the create returns provisioning immediately, which - // matches AWS (CreateLoadBalancer returns the LB provisioning; the AZ/VpcId in - // the response come from the ENIs created synchronously above). Launcher-less - // deployments (no InstanceLauncher / no ENIs) have no data plane to boot and - // are active on the spot. + // VM boot is async to avoid blocking the gateway responder. The create + // returns provisioning immediately; launcher-less deployments are active on the spot. state := StateActive willLaunch := s.InstanceLauncher != nil && len(eniIDs) > 0 && len(subnets) > 0 if willLaunch { state = StateProvisioning - // Internal LBs only leave provisioning once the lb-agent heartbeats, - // which needs the mgmt return route. If it can't be resolved the LB - // hangs in provisioning forever — record the failed state up front so the - // broken return path is visible immediately. The VM still boots (matching - // the synchronous path) so the data plane exists for diagnosis. + // Internal LBs without a mgmt return route would hang in provisioning; + // mark failed immediately so the broken path is visible. if scheme == SchemeInternal { if gw, tgt := s.resolveMgmtRoute(scheme); gw == "" || tgt == "" { slog.Error("CreateLoadBalancer: internal LB has no mgmt return route; marking failed (lb-agent cannot heartbeat AWSGW)", @@ -1297,11 +1173,8 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp } } - // Attributes are intentionally left nil — DescribeLoadBalancerAttributes - // derives per-type defaults from lb.Type via DefaultLoadBalancerAttributes. - // Callers that want a non-default value must call ModifyLoadBalancerAttributes. - // InstanceID/VPCIP/HostPorts and the per-AZ PublicIP are filled in by the - // asynchronous launch below once the VM boots. + // Attributes left nil — defaults derived from lb.Type on read. + // InstanceID/VPCIP/HostPorts are filled by the async launch. record := &LoadBalancerRecord{ LoadBalancerArn: lbArn, LoadBalancerID: lbID, @@ -1325,15 +1198,13 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp if err := s.store.PutLoadBalancer(record); err != nil { slog.Error("CreateLoadBalancer: failed to persist record", "lbId", lbID, "err", err) - // The ENIs and managed SG exist but no record was persisted to own them — - // roll them back so they are not orphaned. + // Rollback so unowned ENIs/SG don't leak. s.rollbackLBInfra(eniIDs, nlbManagedSGID, name, accountID) return nil, errors.New(awserrors.ErrorServerInternal) } if willLaunch { - // Snapshot the launch inputs; the goroutine must not read the record the - // caller is about to return. + // Snapshot inputs so the goroutine doesn't alias the caller's record. lc := lbLaunchCtx{lbID: lbID, lbArn: lbArn, scheme: scheme, eniIDs: eniIDs, subnets: subnets, accountID: accountID} s.launchWG.Go(func() { defer func() { @@ -1346,8 +1217,6 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp }) } - // Agent heartbeat will transition provisioning → active on first contact. - slog.Info("CreateLoadBalancer accepted", "name", name, "lbArn", lbArn, "enis", len(eniIDs), "state", state, "accountID", accountID) return &elbv2.CreateLoadBalancerOutput{ @@ -1365,11 +1234,8 @@ type lbLaunchCtx struct { accountID string } -// launchLBVMAsync boots the LB-VM on a background goroutine and folds the result -// back into the persisted record: on success the instance/VPC-IP/host-ports (and -// the per-AZ public IP) are recorded and the LB stays provisioning until the -// lb-agent heartbeats; on failure the record is marked failed so the API -// reflects the broken data plane. Runs after CreateLoadBalancer has returned. +// launchLBVMAsync boots the LB-VM and folds the result into the persisted record. +// On failure marks the LB failed; on success stays provisioning until lb-agent heartbeats. func (s *ELBv2ServiceImpl) launchLBVMAsync(lc lbLaunchCtx) { launch := s.launchLBVM(lc.lbID, lc.scheme, lc.eniIDs, lc.subnets, lc.accountID) @@ -1398,12 +1264,8 @@ func (s *ELBv2ServiceImpl) launchLBVMAsync(lc lbLaunchCtx) { } } -// rollbackLBInfra tears down the infrastructure a partial CreateLoadBalancer -// created before it could persist an owning record: every ENI minted so far, the -// NLB managed front-end SG (if any), and the name claim. Without this an -// ENI/SG-create-then-persist-fail path orphans those resources with no record -// for DeleteLoadBalancer or any reclaim to reach. Each step is best-effort and -// idempotent — a NotFound is success. +// rollbackLBInfra tears down ENIs, the NLB managed SG, and the name claim created +// by a partial CreateLoadBalancer. Each step is best-effort and idempotent. func (s *ELBv2ServiceImpl) rollbackLBInfra(eniIDs []string, nlbManagedSGID, name, accountID string) { if s.VPCService != nil { for _, eniID := range eniIDs { @@ -1418,8 +1280,7 @@ func (s *ELBv2ServiceImpl) rollbackLBInfra(eniIDs []string, nlbManagedSGID, name s.releaseLBNameClaim(name, accountID) } -// markLBFailed reloads the LB record and flips it to the failed state, used by -// the async launch panic recovery. +// markLBFailed reloads the LB record and flips it to the failed state. func (s *ELBv2ServiceImpl) markLBFailed(lbArn string) { record, err := s.store.GetLoadBalancerByArn(lbArn) if err != nil || record == nil { @@ -1442,17 +1303,11 @@ func (s *ELBv2ServiceImpl) DeleteLoadBalancer(input *elbv2.DeleteLoadBalancerInp return nil, errors.New(awserrors.ErrorServerInternal) } if lb == nil || lb.AccountID != accountID { - // Idempotent: AWS ELBv2 delete returns success on an absent (or - // not-owned) load balancer, so tofu destroy retries converge. + // Idempotent: return success for absent/unowned LBs. return &elbv2.DeleteLoadBalancerOutput{}, nil } - // Delete all listeners (and their rules) for this LB before tearing down - // backing artifacts. Cascade through the shared helper, not - // store.DeleteListener directly, so rules don't survive and pin their - // target groups as ResourceInUse after the LB is gone. A list or cascade - // failure aborts the delete with the record intact, so tofu retries - // converge instead of leaving orphaned rules behind. + // Cascade-delete all listeners and their rules so no orphan pins a TG as ResourceInUse. listeners, err := s.store.ListListenersByLB(lb.LoadBalancerArn) if err != nil { slog.Error("DeleteLoadBalancer: failed to list listeners for cascade", "lbArn", lb.LoadBalancerArn, "err", err) @@ -1465,16 +1320,10 @@ func (s *ELBv2ServiceImpl) DeleteLoadBalancer(input *elbv2.DeleteLoadBalancerInp } } - // Reap the LB VM's floating-IP dnat_and_snat deterministically. The VM/ENI - // teardown paths reap it only conditionally — DeleteNetworkInterface defers - // EIP-owned addresses to the EIP lifecycle, and that lifecycle's reap runs - // in the async terminate goroutine below — so without this, stale rules - // accumulate on the VPC router across repeated create/delete cycles. + // Reap floating-IP NAT now; conditional VM/ENI teardown paths can miss it. s.reapFloatingIPNAT(lb) - // Terminate ALB VM in background (VM termination also cleans up tap device). - // Must be async because VM shutdown can take seconds (QMP powerdown + QEMU exit) - // and we don't want to block the NATS request handler. + // Terminate ALB VM in background; shutdown can take seconds (QMP powerdown). if lb.InstanceID != "" && s.InstanceLauncher != nil { instanceID := lb.InstanceID go func() { @@ -1514,20 +1363,16 @@ func (s *ELBv2ServiceImpl) DeleteLoadBalancer(input *elbv2.DeleteLoadBalancerInp return &elbv2.DeleteLoadBalancerOutput{}, nil } -// releaseLBNameClaim drops the per-account LB name claim. Failure is logged but -// not fatal: a leaked claim is reclaimed by the crash-orphan path on the next -// same-name create (its owner lbID resolves to no live record). +// releaseLBNameClaim drops the per-account LB name claim. Failures are logged but +// not fatal; leaked claims are reclaimed by the crash-orphan path on next create. func (s *ELBv2ServiceImpl) releaseLBNameClaim(name, accountID string) { if err := s.store.ReleaseLBName(name, accountID); err != nil { slog.Warn("failed to release LB name claim", "name", name, "accountID", accountID, "err", err) } } -// reapFloatingIPNAT removes the OVN dnat_and_snat rule mapping an -// internet-facing LB's floating (public) IP to its VM's VPC IP. DeleteEIP -// matches on external_ip+logical_ip and is idempotent, so a redundant publish -// from the EIP/ENI teardown paths is harmless. Internal LBs carry no floating -// IP and are a no-op. +// reapFloatingIPNAT removes the OVN dnat_and_snat rule for an internet-facing +// LB's floating IP. Idempotent; internal LBs are a no-op. func (s *ELBv2ServiceImpl) reapFloatingIPNAT(lb *LoadBalancerRecord) { publicIP := "" for _, az := range lb.AvailZones { @@ -1785,15 +1630,11 @@ func (s *ELBv2ServiceImpl) DeleteTargetGroup(input *elbv2.DeleteTargetGroupInput return nil, errors.New(awserrors.ErrorServerInternal) } if tg == nil || tg.AccountID != accountID { - // Idempotent: AWS ELBv2 delete returns success on an absent (or - // not-owned) target group, so tofu destroy retries converge. + // Idempotent: return success for absent/unowned TGs. return &elbv2.DeleteTargetGroupOutput{}, nil } - // Only a *live* listener or rule pins the target group. A listener (or - // rule) whose owning load balancer no longer exists is treated as already - // torn down, not a live reference, so an orphan left by a partial teardown - // can never pin this target group as ResourceInUse permanently. + // Only live listeners/rules (whose LB still exists) pin the TG as ResourceInUse. lbs, err := s.store.ListLoadBalancers() if err != nil { slog.Error("DeleteTargetGroup: failed to list load balancers for in-use check", "err", err) @@ -2074,10 +1915,8 @@ func (s *ELBv2ServiceImpl) DescribeTargetHealth(input *elbv2.DescribeTargetHealt // --- Listener operations --- -// listenerActionFromSDK converts an SDK listener action into the stored form, -// preserving fixed-response config so HAProxy generation and tofu read-back see -// the full action (a forward-only conversion drops the fixed-response default -// and produces a dangling backend + perpetual plan diff). +// listenerActionFromSDK converts an SDK listener action to stored form, +// preserving fixed-response/redirect config to avoid dangling backends. func listenerActionFromSDK(a *elbv2.Action) ListenerAction { action := ListenerAction{} if a.Type != nil { @@ -2124,11 +1963,8 @@ func listenerActionFromSDK(a *elbv2.Action) ListenerAction { return action } -// validateListenerAction enforces the per-type action contract for listener -// default actions and rule actions. Forward actions are validated against the -// target group elsewhere; this covers redirect (status code + render-safe -// fields) so invalid input fails at the API instead of being silently -// defaulted by the renderer. +// validateListenerAction enforces the per-type action contract. Redirect fields +// are validated here so invalid input fails at the API, not at render time. func validateListenerAction(a ListenerAction) error { if a.Type == ActionTypeRedirect { if a.Redirect == nil { @@ -2139,9 +1975,7 @@ func validateListenerAction(a ListenerAction) error { return nil } -// validateRedirectAction rejects an unsupported status code or any field that -// would break the HAProxy redirect directive once the known AWS placeholders -// are stripped. +// validateRedirectAction rejects an unsupported status code or render-unsafe fields. func validateRedirectAction(rd *RedirectAction) error { if rd == nil { return errors.New(awserrors.ErrorMissingParameter) @@ -2159,11 +1993,8 @@ func validateRedirectAction(rd *RedirectAction) error { return nil } -// buildListenerCertificates validates and normalises the certificate set and -// SSL policy for a listener of the given protocol. Secure protocols (HTTPS, -// TLS) require at least one certificate and receive a default SslPolicy when -// unset; non-secure protocols must carry neither certificates nor an SslPolicy. -// Exactly one certificate is marked default. +// buildListenerCertificates validates certificates and SSL policy for a listener. +// Secure protocols require exactly one default cert and receive a default SslPolicy when unset. func buildListenerCertificates(protocol string, in []*elbv2.Certificate, sslPolicy *string) ([]ListenerCertificate, string, error) { policy := "" if sslPolicy != nil { @@ -2361,11 +2192,9 @@ func (s *ELBv2ServiceImpl) CreateListener(input *elbv2.CreateListenerInput, acco }, nil } -// rollbackListener removes a just-persisted listener whose post-persist -// data-plane wiring failed, so a failed CreateListener leaves no orphan record -// and no dangling SG authorization. authorizedCIDRs is non-empty only when the -// NLB front-end port was already opened (config-update failure path), in which -// case the port is revoked first. Best-effort: cleanup errors are logged. +// rollbackListener removes a listener persisted before post-persist wiring failed. +// authorizedCIDRs is non-empty when the NLB port was already opened — it is +// revoked first. Cleanup errors are logged but not returned. func (s *ELBv2ServiceImpl) rollbackListener(record *ListenerRecord, lb *LoadBalancerRecord, protocol string, port int64, authorizedCIDRs []string, accountID string) { if len(authorizedCIDRs) > 0 { if err := s.revokeNLBListenerPort(lb, protocol, port, authorizedCIDRs, accountID); err != nil { @@ -2407,8 +2236,7 @@ func (s *ELBv2ServiceImpl) DeleteListener(input *elbv2.DeleteListenerInput, acco return nil, errors.New(awserrors.ErrorServerInternal) } if listener == nil || listener.AccountID != accountID { - // Idempotent: AWS ELBv2 delete returns success on an absent (or - // not-owned) listener, so tofu destroy retries converge. + // Idempotent: return success for absent/unowned listeners. return &elbv2.DeleteListenerOutput{}, nil } @@ -2548,9 +2376,7 @@ func (s *ELBv2ServiceImpl) ModifyListener(input *elbv2.ModifyListenerInput, acco } } - // Certificates / SSL policy. The effective protocol after any change drives - // the requirement: switching to a non-secure protocol clears cert material; - // staying on or switching to a secure protocol validates and defaults it. + // Cert/policy: switching to non-secure clears material; staying/switching to secure validates it. if !protocolRequiresCert(updated.Protocol) { if len(input.Certificates) > 0 || (input.SslPolicy != nil && *input.SslPolicy != "") { return nil, errors.New(awserrors.ErrorInvalidParameterValue) @@ -2639,12 +2465,8 @@ func (s *ELBv2ServiceImpl) DescribeListeners(input *elbv2.DescribeListenersInput // --- Tag operations --- -// DescribeTags returns tags for one or more ELBv2 resources (load balancers, -// target groups, listeners, listener rules). Tag data is read from the existing -// record stores — Spinifex doesn't have a separate tag KV. Untagged resources -// return an empty Tags slice (matches AWS behaviour). Cross-account or unknown -// ARNs return the per-resource not-found error so existence isn't leaked across -// accounts. +// DescribeTags returns tags for ELBv2 resources read from the record stores. +// Cross-account or unknown ARNs return a not-found error. func (s *ELBv2ServiceImpl) DescribeTags(input *elbv2.DescribeTagsInput, accountID string) (*elbv2.DescribeTagsOutput, error) { if input == nil || len(input.ResourceArns) == 0 { return nil, errors.New(awserrors.ErrorMissingParameter) @@ -2734,8 +2556,7 @@ func (s *ELBv2ServiceImpl) DescribeTags(input *elbv2.DescribeTagsInput, accountI }, nil } -// tagsFromSDK converts an SDK Tag slice into a tag map, skipping entries with a -// nil key or value. Returns a non-nil (possibly empty) map for storage. +// tagsFromSDK converts an SDK Tag slice into a tag map, skipping nil key/value entries. func tagsFromSDK(tags []*elbv2.Tag) map[string]string { m := make(map[string]string, len(tags)) for _, tag := range tags { @@ -2746,9 +2567,7 @@ func tagsFromSDK(tags []*elbv2.Tag) map[string]string { return m } -// tagsMapToSDK converts a tag map into the SDK Tag slice with deterministic -// (sorted-by-key) ordering. Returns nil for empty/nil input so the response -// matches AWS for untagged resources. +// tagsMapToSDK converts a tag map into a sorted SDK Tag slice; returns nil for empty input. func tagsMapToSDK(tags map[string]string) []*elbv2.Tag { if len(tags) == 0 { return nil @@ -2784,9 +2603,7 @@ func (s *ELBv2ServiceImpl) lbRecordToSDK(r *LoadBalancerRecord) *elbv2.LoadBalan lb.SecurityGroups = append(lb.SecurityGroups, aws.String(sg)) } - // Build a subnet → private IP map by looking up each ENI. This lets each - // AvailabilityZone entry surface the private IP of the ENI that lives in - // its subnet, not just the public IP recorded at create time. + // Build subnet → private IP map so each AZ entry surfaces the ENI's private IP. privateIPBySubnet := map[string]string{} if s.VPCService != nil && len(r.ENIs) > 0 { eniPtrs := make([]*string, 0, len(r.ENIs)) @@ -2873,9 +2690,8 @@ func (s *ELBv2ServiceImpl) listenerRecordToSDK(r *ListenerRecord) *elbv2.Listene return listener } -// listenerActionToSDK converts a stored action into the AWS SDK shape, -// preserving forward / fixed-response / redirect detail so DescribeListeners -// and DescribeRules round-trip cleanly. Shared by listeners and rules. +// listenerActionToSDK converts a stored action to the AWS SDK shape. +// Shared by listeners and rules; preserves all action types for round-trip fidelity. func listenerActionToSDK(a ListenerAction) *elbv2.Action { action := &elbv2.Action{Type: aws.String(a.Type)} if a.TargetGroupArn != "" { diff --git a/spinifex/handlers/elbv2/service_impl_attributes.go b/spinifex/handlers/elbv2/service_impl_attributes.go index 362c469e5..07eb0d192 100644 --- a/spinifex/handlers/elbv2/service_impl_attributes.go +++ b/spinifex/handlers/elbv2/service_impl_attributes.go @@ -50,10 +50,9 @@ func (r attrResource[R]) load() (*R, error) { return rec, nil } -// modifyResourceAttributes validates and merges submitted attributes into the -// record, persisting only when something actually changed. rawCount is the -// number of attributes the caller submitted before nil filtering, used to -// distinguish "nothing sent" from "everything sent was invalid". +// modifyResourceAttributes validates and merges submitted attributes, persisting only +// on change. rawCount is pre-nil-filter count, used to distinguish "nothing sent" +// from "everything sent was invalid". func modifyResourceAttributes[R any](res attrResource[R], pairs []attrPair, rawCount int) ([]attrPair, error) { if res.arn == "" { return nil, errors.New(awserrors.ErrorMissingParameter) diff --git a/spinifex/handlers/elbv2/service_impl_health_test.go b/spinifex/handlers/elbv2/service_impl_health_test.go index 19553fe75..9a797edee 100644 --- a/spinifex/handlers/elbv2/service_impl_health_test.go +++ b/spinifex/handlers/elbv2/service_impl_health_test.go @@ -161,10 +161,9 @@ func TestRegisterTargets_MultipleInstances(t *testing.T) { assert.Equal(t, "10.0.1.51", tg.Targets[1].PrivateIP) } -// TestResetTargetHealthOnStartup_TransitionsAllNonDrainingToInitial verifies -// that the startup reset rewrites every non-draining target's HealthState -// back to "initial" so a daemon restart can't surface stale "healthy" claims -// before the lb-agent has posted a fresh report. +// TestResetTargetHealthOnStartup_TransitionsAllNonDrainingToInitial verifies that +// startup resets non-draining targets to "initial", preventing stale "healthy" +// claims before the lb-agent has posted a fresh report after a daemon restart. func TestResetTargetHealthOnStartup_TransitionsAllNonDrainingToInitial(t *testing.T) { store := setupTestNATS(t) svc := &ELBv2ServiceImpl{store: store} diff --git a/spinifex/handlers/elbv2/service_impl_lb_network.go b/spinifex/handlers/elbv2/service_impl_lb_network.go index 215d8e40d..d266d13db 100644 --- a/spinifex/handlers/elbv2/service_impl_lb_network.go +++ b/spinifex/handlers/elbv2/service_impl_lb_network.go @@ -11,9 +11,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/awserrors" ) -// SetIpAddressType sets the load balancer's IP address type. Spinifex ALBs are -// IPv4-only, so the only accepted value is "ipv4"; dualstack variants are -// rejected with InvalidConfigurationRequest. The call is idempotent. +// SetIpAddressType sets the LB IP address type; only "ipv4" is accepted. func (s *ELBv2ServiceImpl) SetIpAddressType(input *elbv2.SetIpAddressTypeInput, accountID string) (*elbv2.SetIpAddressTypeOutput, error) { if input == nil || input.LoadBalancerArn == nil || *input.LoadBalancerArn == "" { return nil, errors.New(awserrors.ErrorMissingParameter) @@ -47,11 +45,8 @@ func (s *ELBv2ServiceImpl) SetIpAddressType(input *elbv2.SetIpAddressTypeInput, }, nil } -// SetSecurityGroups replaces the security groups associated with an -// (application) load balancer. The new groups are re-attached to every ENI the -// ALB spans via ModifyNetworkInterfaceAttribute, which validates them against -// the ENI's VPC and pushes the change to the live data-plane port before the -// record is persisted. +// SetSecurityGroups replaces the security groups on an ALB, re-attaching them +// to every ENI via ModifyNetworkInterfaceAttribute before persisting. func (s *ELBv2ServiceImpl) SetSecurityGroups(input *elbv2.SetSecurityGroupsInput, accountID string) (*elbv2.SetSecurityGroupsOutput, error) { if input == nil || input.LoadBalancerArn == nil || *input.LoadBalancerArn == "" { return nil, errors.New(awserrors.ErrorMissingParameter) @@ -82,10 +77,7 @@ func (s *ELBv2ServiceImpl) SetSecurityGroups(input *elbv2.SetSecurityGroupsInput sgs = append(sgs, *sg) } - // Re-attach the new groups to each ALB ENI. This validates the groups - // against the ENI's VPC and fires the live port-SG update; a failure here - // (e.g. unknown SG) aborts before the record is persisted. All ENIs share - // the LB's VPC and groups, so a successful first apply implies the rest. + // Re-attach to each ENI; failure aborts before the record is persisted. if s.VPCService != nil { for _, eniID := range lb.ENIs { if _, err := s.VPCService.ModifyNetworkInterfaceAttribute(&ec2.ModifyNetworkInterfaceAttributeInput{ @@ -109,13 +101,8 @@ func (s *ELBv2ServiceImpl) SetSecurityGroups(input *elbv2.SetSecurityGroupsInput }, nil } -// SetSubnets enables the subnets (and their backing ENIs) for a load balancer. -// It is a full add+remove: a subnet in the request but not on the LB gets a new -// managed ENI; a subnet on the LB but absent from the request has its ENI -// removed. The system VM binds its data-plane taps at boot and the launcher -// exposes no live ENI hotplug, so the new ENI set is applied by relaunching the -// LB VM — a brief data-plane interruption. The record is mutated only after the -// new ENIs exist and the VM has been relaunched. +// SetSubnets does a full add+remove of the LB's subnets and their ENIs. +// Because ENI hotplug is not supported, the LB VM is relaunched with the new ENI set. func (s *ELBv2ServiceImpl) SetSubnets(input *elbv2.SetSubnetsInput, accountID string) (*elbv2.SetSubnetsOutput, error) { if input == nil || input.LoadBalancerArn == nil || *input.LoadBalancerArn == "" { return nil, errors.New(awserrors.ErrorMissingParameter) @@ -157,8 +144,7 @@ func (s *ELBv2ServiceImpl) SetSubnets(input *elbv2.SetSubnetsInput, accountID st return s.setSubnetsOutput(lb), nil // idempotent — no change } - // Without a VPC service we cannot manage ENIs; record the requested subnet - // set so the API stays consistent for launcher-less / test deployments. + // No VPC service: just record the subnet set (launcher-less / test deployments). if s.VPCService == nil { lb.Subnets = desired lb.AvailZones = rebuildAvailZones(desired, lb.AvailZones, nil) @@ -169,8 +155,7 @@ func (s *ELBv2ServiceImpl) SetSubnets(input *elbv2.SetSubnetsInput, accountID st return s.setSubnetsOutput(lb), nil } - // Create ENIs for the added subnets first; roll them back on any failure so - // a partial apply never leaks ENIs or mutates the live LB. + // Create ENIs for added subnets; roll back on failure to avoid leaks. newENIBySubnet := make(map[string]string, len(toAdd)) newAZBySubnet := make(map[string]string, len(toAdd)) rollbackNewENIs := func() { @@ -203,8 +188,7 @@ func (s *ELBv2ServiceImpl) SetSubnets(input *elbv2.SetSubnetsInput, accountID st } } - // Terminate the LB VM before reshaping its ENI set, then relaunch it on the - // new set. + // Terminate the LB VM before reshaping its ENI set, then relaunch on the new set. if lb.InstanceID != "" && s.InstanceLauncher != nil { if err := s.InstanceLauncher.TerminateSystemInstance(lb.InstanceID); err != nil { rollbackNewENIs() @@ -213,10 +197,7 @@ func (s *ELBv2ServiceImpl) SetSubnets(input *elbv2.SetSubnetsInput, accountID st } } - // TerminateSystemInstance tears down the VM but leaves its ENIs marked - // in-use, so detach the existing ENIs explicitly: retained ones must be free - // to re-attach to the relaunched VM, and removed ones must be detached before - // they can be deleted. + // Detach all ENIs explicitly: TerminateSystemInstance doesn't clear in-use status. for _, eniID := range current { if detachErr := s.VPCService.DetachENI(accountID, eniID); detachErr != nil { slog.Warn("SetSubnets: failed to detach ENI before relaunch", "eni", eniID, "err", detachErr) @@ -256,8 +237,7 @@ func (s *ELBv2ServiceImpl) SetSubnets(input *elbv2.SetSubnetsInput, accountID st return s.setSubnetsOutput(lb), nil } -// desiredSubnetSet flattens the request's Subnets and SubnetMappings into a -// de-duplicated, order-preserving subnet-ID list. +// desiredSubnetSet flattens Subnets and SubnetMappings into a deduplicated, ordered list. func desiredSubnetSet(input *elbv2.SetSubnetsInput) []string { seen := make(map[string]bool) var out []string @@ -280,9 +260,7 @@ func desiredSubnetSet(input *elbv2.SetSubnetsInput) []string { return out } -// subnetENIMap pairs each of the LB's current subnets with its backing ENI. -// CreateLoadBalancer and SetSubnets write Subnets and ENIs in lockstep, so the -// parallel arrays are authoritative for which ENI lives in which subnet. +// subnetENIMap pairs each current subnet with its ENI using the parallel Subnets/ENIs arrays. func subnetENIMap(lb *LoadBalancerRecord) map[string]string { m := make(map[string]string, len(lb.Subnets)) for i, sn := range lb.Subnets { @@ -293,9 +271,7 @@ func subnetENIMap(lb *LoadBalancerRecord) map[string]string { return m } -// createLBENI creates a single managed ENI for the LB in the given subnet, -// tagged and security-grouped like the create-time ENIs. Returns the ENI ID and -// its availability zone. +// createLBENI creates a managed ENI in the given subnet, returning the ENI ID and AZ. func (s *ELBv2ServiceImpl) createLBENI(subnetID string, lb *LoadBalancerRecord, accountID string) (eniID, az string, err error) { eniIn := &ec2.CreateNetworkInterfaceInput{ SubnetId: aws.String(subnetID), @@ -321,10 +297,8 @@ func (s *ELBv2ServiceImpl) createLBENI(subnetID string, lb *LoadBalancerRecord, return aws.StringValue(eni.NetworkInterfaceId), aws.StringValue(eni.AvailabilityZone), nil } -// rebuildAvailZones produces the AvailZoneInfo list for the new subnet set in -// order, preserving the existing zone name for retained subnets and using -// newAZBySubnet for added ones. PublicIP is cleared — it is re-derived from the -// relaunched VM by the caller. +// rebuildAvailZones builds the AZ list for the new subnet set, preserving existing +// zone names and using newAZBySubnet for additions. PublicIP is cleared. func rebuildAvailZones(subnets []string, existing []AvailZoneInfo, newAZBySubnet map[string]string) []AvailZoneInfo { bySubnet := make(map[string]AvailZoneInfo, len(existing)) for _, az := range existing { @@ -341,9 +315,8 @@ func rebuildAvailZones(subnets []string, existing []AvailZoneInfo, newAZBySubnet return out } -// lbStateAfterLaunch mirrors CreateLoadBalancer's post-launch state gate: a VM -// that came up enters provisioning until the lb-agent heartbeats; an internal -// LB with no mgmt return route is failed loud; a failed launch is failed. +// lbStateAfterLaunch returns the post-launch state: provisioning if the VM came up, +// failed if the launch failed or if an internal LB has no mgmt return route. func (s *ELBv2ServiceImpl) lbStateAfterLaunch(launch lbVMLaunch, scheme string) string { if launch.instanceID == "" { if launch.failed { @@ -361,9 +334,7 @@ func (s *ELBv2ServiceImpl) lbStateAfterLaunch(launch lbVMLaunch, scheme string) return StateProvisioning } -// setSubnetsOutput builds the SetSubnets response from the persisted record, -// reusing the SDK availability-zone projection (which surfaces per-AZ private -// IPs) so the response matches DescribeLoadBalancers. +// setSubnetsOutput builds the SetSubnets response from the persisted record. func (s *ELBv2ServiceImpl) setSubnetsOutput(lb *LoadBalancerRecord) *elbv2.SetSubnetsOutput { sdk := s.lbRecordToSDK(lb) return &elbv2.SetSubnetsOutput{ diff --git a/spinifex/handlers/elbv2/service_impl_listener_certs.go b/spinifex/handlers/elbv2/service_impl_listener_certs.go index 2ba76fe6b..e4bb233dd 100644 --- a/spinifex/handlers/elbv2/service_impl_listener_certs.go +++ b/spinifex/handlers/elbv2/service_impl_listener_certs.go @@ -21,10 +21,8 @@ func listenerCertsToSDK(certs []ListenerCertificate) []*elbv2.Certificate { return out } -// AddListenerCertificates attaches additional (SNI) certificates to a secure -// listener. The default certificate is set at create time and is unaffected; -// added certificates are non-default. Re-adding an existing certificate is a -// no-op. +// AddListenerCertificates attaches additional (SNI) certificates to a secure listener. +// Added certificates are non-default; re-adding an existing certificate is a no-op. func (s *ELBv2ServiceImpl) AddListenerCertificates(input *elbv2.AddListenerCertificatesInput, accountID string) (*elbv2.AddListenerCertificatesOutput, error) { if input == nil || input.ListenerArn == nil || *input.ListenerArn == "" { return nil, errors.New(awserrors.ErrorMissingParameter) diff --git a/spinifex/handlers/elbv2/service_impl_nlb_sg.go b/spinifex/handlers/elbv2/service_impl_nlb_sg.go index a1a2f92b5..f3aa743a7 100644 --- a/spinifex/handlers/elbv2/service_impl_nlb_sg.go +++ b/spinifex/handlers/elbv2/service_impl_nlb_sg.go @@ -17,10 +17,9 @@ func nlbManagedSGName(lbID string) string { return "spinifex-nlb-" + lbID } -// lbENIGroups returns the security groups to attach to an LB's data-plane ENIs. -// NLBs use the internally-minted managed SG (customer SGs are rejected at -// create time); ALBs use the customer-supplied SGs. Shared by the create-time -// ENI loop and the SetSubnets relaunch path so both attach the same SG. +// lbENIGroups returns the SGs to attach to an LB's data-plane ENIs: the managed SG +// for NLBs (customer SGs are rejected), or customer SGs for ALBs. Shared by the +// create-time ENI loop and SetSubnets relaunch so both paths attach the same SG. func lbENIGroups(lb *LoadBalancerRecord) []string { if lb.Type == LoadBalancerTypeNetwork { if lb.NLBManagedSGID != "" { @@ -31,10 +30,9 @@ func lbENIGroups(lb *LoadBalancerRecord) []string { return lb.SecurityGroups } -// createNLBManagedSG mints the internal front-end SG for an NLB in the LB's VPC -// and returns its ID. The VPC is resolved from the LB's first subnet. The SG is -// tagged like the LB's other managed resources so the ownership sweep can find -// it. +// createNLBManagedSG mints the managed front-end SG for an NLB; the VPC is resolved +// from the first subnet. The SG is tagged like other managed resources so the +// ownership sweep can find it. func (s *ELBv2ServiceImpl) createNLBManagedSG(lbID, lbArn, subnetID, accountID string) (string, error) { subnet, err := s.VPCService.GetSubnet(accountID, subnetID) if err != nil { @@ -65,10 +63,9 @@ func (s *ELBv2ServiceImpl) createNLBManagedSG(lbID, lbArn, subnetID, accountID s return *out.GroupId, nil } -// deleteNLBManagedSG best-effort deletes an NLB's managed SG. Callable on the -// create-rollback and delete paths; a nil VPC service or empty ID is a no-op, -// and a delete failure is logged but not surfaced (the SG can only be deleted -// once its ENIs are gone, which the delete path guarantees). +// deleteNLBManagedSG best-effort deletes an NLB's managed SG; nil VPC service or +// empty ID is a no-op. Failures are logged but not returned — the SG can only +// be deleted after its ENIs are gone, which the delete path ensures. func (s *ELBv2ServiceImpl) deleteNLBManagedSG(sgID, accountID string) { if s.VPCService == nil || sgID == "" { return @@ -132,10 +129,9 @@ func listenerIPProtocols(protocol string) []string { } } -// authorizeNLBListenerPort opens a listener port on the NLB's managed SG for -// each (protocol, CIDR) pair. Each rule is authorized in its own call so an -// already-present rule (Duplicate, swallowed) never blocks a sibling rule — -// keeping re-create / reconcile idempotent. No-op when there is no managed SG. +// authorizeNLBListenerPort opens a listener port on the NLB's managed SG for each +// (protocol, CIDR) pair; each rule is authorized separately so a Duplicate error +// never blocks a sibling rule. No-op when there is no managed SG. func (s *ELBv2ServiceImpl) authorizeNLBListenerPort(lb *LoadBalancerRecord, protocol string, port int64, cidrs []string, accountID string) error { if s.VPCService == nil || lb.NLBManagedSGID == "" { return nil @@ -188,11 +184,9 @@ func (s *ELBv2ServiceImpl) revokeNLBListenerPort(lb *LoadBalancerRecord, protoco return nil } -// SetLoadBalancerIngressCIDRs overrides the client CIDRs the NLB's listener -// ports are opened to (e.g. an EKS public endpoint's publicAccessCidrs) and -// re-authorizes every existing listener accordingly. Passing nil/empty reverts -// to the scheme-based default. Idempotent: the old rules are revoked and the -// new ones authorized, both swallowing the benign duplicate / not-found errors. +// SetLoadBalancerIngressCIDRs overrides the client CIDRs for an NLB's listener ports, +// revoking old rules and authorizing new ones. Passing nil/empty reverts to the +// scheme-based default (0.0.0.0/0 internet-facing, VPC CIDR internal). func (s *ELBv2ServiceImpl) SetLoadBalancerIngressCIDRs(lbArn string, cidrs []string, accountID string) error { if lbArn == "" { return errors.New(awserrors.ErrorMissingParameter) diff --git a/spinifex/handlers/elbv2/service_impl_rules.go b/spinifex/handlers/elbv2/service_impl_rules.go index c1818a868..533488412 100644 --- a/spinifex/handlers/elbv2/service_impl_rules.go +++ b/spinifex/handlers/elbv2/service_impl_rules.go @@ -16,10 +16,8 @@ import ( "github.com/mulgadc/spinifex/spinifex/utils" ) -// Regex guards for rule condition values. Any byte outside these classes is -// rejected at write time so the HAProxy renderer can interpolate them without -// re-validating. The renderer still rejects newline/CR/comment/quote -// defensively (see haproxy.go) in case the store is bypassed. +// Regex guards for rule condition values — rejected at write time so the renderer +// can interpolate them safely. The renderer still validates defensively. var ( ruleHostPathRegex = regexp.MustCompile(`^[A-Za-z0-9*?._/-]+$`) ruleHeaderValueRegex = regexp.MustCompile(`^[\x20-\x7E]+$`) // printable ASCII, no control bytes @@ -262,8 +260,7 @@ func (s *ELBv2ServiceImpl) SetRulePriorities(input *elbv2.SetRulePrioritiesInput return nil, errors.New(awserrors.ErrorMissingParameter) } - // Resolve all rules first so duplicate-priority checks can run before any - // write hits the store. + // Resolve all rules first so duplicate-priority checks run before any write. type pendingUpdate struct { rule *RuleRecord newPriority int @@ -304,8 +301,7 @@ func (s *ELBv2ServiceImpl) SetRulePriorities(input *elbv2.SetRulePrioritiesInput seen[p.newPriority] = struct{}{} } - // Detect collisions with rules on the same listener that are NOT being - // renumbered in this call. + // Detect collisions with rules not being renumbered in this call. all, err := s.store.ListRulesByListener(listenerArn) if err != nil { slog.Error("SetRulePriorities: failed to list rules", "listenerArn", listenerArn, "err", err) @@ -493,9 +489,8 @@ func validateCondition(c *elbv2.RuleCondition) (RuleCondition, error) { return out, nil } -// extractStringValues pulls the canonical Values slice for host-header / -// path-pattern conditions, accepting both the flat input.Values and the -// nested {Host,Path}HeaderConfig.Values shapes the AWS SDK emits. +// extractStringValues returns the canonical Values for host-header / path-pattern +// conditions, accepting both flat and nested SDK shapes. func extractStringValues(c *elbv2.RuleCondition) ([]string, error) { var vals []string switch *c.Field { @@ -591,11 +586,8 @@ func (s *ELBv2ServiceImpl) validateAndConvertRuleActions(in []*elbv2.Action, lis return out, nil } -// buildRuleArn rewrites a listener ARN into a listener-rule ARN by replacing -// the leading "listener/" resource segment with "listener-rule/" and appending -// the rule short ID. AWS format: -// -// arn:aws:elasticloadbalancing:{region}:{account}:listener-rule/{app}/{lbName}/{lbID}/{listenerID}/{ruleID} +// buildRuleArn rewrites a listener ARN to a listener-rule ARN by replacing "listener/" +// with "listener-rule/" and appending the rule short ID. func buildRuleArn(listenerArn, ruleID string) (string, error) { const old = ":listener/" before, after, ok := strings.Cut(listenerArn, old) diff --git a/spinifex/handlers/elbv2/service_impl_rules_extra_test.go b/spinifex/handlers/elbv2/service_impl_rules_extra_test.go index 1f8c03f21..bb10da76a 100644 --- a/spinifex/handlers/elbv2/service_impl_rules_extra_test.go +++ b/spinifex/handlers/elbv2/service_impl_rules_extra_test.go @@ -398,12 +398,9 @@ func TestRegisterRuleBackend_RejectsUnknownAction(t *testing.T) { require.Error(t, err) } -// TestTargetGroupsForLB_IncludesRuleTGs guards against regressing the -// rule-TG visibility bug: without rule-action enumeration, the health -// checker cannot resolve rule-only TGs back to a TG record and never -// transitions them from initial -> healthy. Reproduces the e2e failure -// where tgB (referenced only by a CreateRule action) timed out at 0/1 -// healthy because TargetGroupsForLB walked only DefaultActions. +// TestTargetGroupsForLB_IncludesRuleTGs guards against the rule-TG visibility bug: +// without rule-action enumeration, the health checker never resolves rule-only TGs +// and they time out at 0/1 healthy (TargetGroupsForLB walked only DefaultActions). func TestTargetGroupsForLB_IncludesRuleTGs(t *testing.T) { env := newRuleTestEnv(t, "tgs-rule") _, err := env.svc.CreateRule(&elbv2.CreateRuleInput{ diff --git a/spinifex/handlers/elbv2/service_impl_rules_test.go b/spinifex/handlers/elbv2/service_impl_rules_test.go index 4bc334d7a..0a8a59220 100644 --- a/spinifex/handlers/elbv2/service_impl_rules_test.go +++ b/spinifex/handlers/elbv2/service_impl_rules_test.go @@ -70,10 +70,9 @@ func TestCreateRule_PathPattern(t *testing.T) { assert.Contains(t, *out.Rules[0].RuleArn, ":listener-rule/") } -// TestDeleteLoadBalancer_CascadesRuleDeletion is a regression guard: -// DeleteLoadBalancer must cascade through listener rule deletion. Previously it -// called store.DeleteListener directly, bypassing the rule cascade, so a rule's -// target group stayed pinned as ResourceInUse and was permanently undeletable. +// TestDeleteLoadBalancer_CascadesRuleDeletion guards against the regression where +// DeleteLoadBalancer called store.DeleteListener directly, bypassing rule cascade, +// leaving a rule's target group pinned as ResourceInUse permanently. func TestDeleteLoadBalancer_CascadesRuleDeletion(t *testing.T) { env := newRuleTestEnv(t, "del-cascade") diff --git a/spinifex/handlers/elbv2/service_impl_tags.go b/spinifex/handlers/elbv2/service_impl_tags.go index 5238397d2..c447083eb 100644 --- a/spinifex/handlers/elbv2/service_impl_tags.go +++ b/spinifex/handlers/elbv2/service_impl_tags.go @@ -17,10 +17,9 @@ type taggable struct { save func(map[string]string) error } -// resolveTaggable loads the ELBv2 resource named by arn and returns a handle for -// reading and persisting its tags. found is false when the resource doesn't -// exist; notFoundError carries the per-type API error for that case. Store -// failures are logged and returned as ErrorServerInternal. +// resolveTaggable loads the ELBv2 resource by ARN and returns a tag read/write handle. +// found is false when the resource doesn't exist; notFoundError is the per-type error. +// Store failures are logged and returned as ErrorServerInternal. func (s *ELBv2ServiceImpl) resolveTaggable(arn string) (h taggable, notFoundError string, found bool, err error) { resourceType, terr := elbv2ResourceTypeFromArn(arn) if terr != nil { @@ -89,10 +88,9 @@ func (s *ELBv2ServiceImpl) resolveTaggable(arn string) (h taggable, notFoundErro return h, notFoundError, found, nil } -// AddTags adds or overwrites tags on one or more ELBv2 resources. Tags are -// validated up-front so a malformed key can't leave a partial apply spanning -// the ResourceArns. Cross-account or unknown ARNs yield the per-resource -// not-found error. +// AddTags adds or overwrites tags on ELBv2 resources. Tags are validated up-front +// so a malformed key can't leave a partial apply; cross-account or unknown ARNs +// yield the per-resource not-found error. func (s *ELBv2ServiceImpl) AddTags(input *elbv2.AddTagsInput, accountID string) (*elbv2.AddTagsOutput, error) { if input == nil || len(input.ResourceArns) == 0 || len(input.Tags) == 0 { return nil, errors.New(awserrors.ErrorMissingParameter) @@ -138,10 +136,9 @@ func (s *ELBv2ServiceImpl) AddTags(input *elbv2.AddTagsInput, accountID string) return &elbv2.AddTagsOutput{}, nil } -// RemoveTags removes the named tag keys from one or more ELBv2 resources. -// Removal is idempotent — keys absent from a resource are silently ignored, -// matching AWS. Cross-account or unknown ARNs yield the per-resource not-found -// error. +// RemoveTags removes tag keys from ELBv2 resources; absent keys are silently ignored +// (idempotent, matching AWS). Cross-account or unknown ARNs yield the per-resource +// not-found error. func (s *ELBv2ServiceImpl) RemoveTags(input *elbv2.RemoveTagsInput, accountID string) (*elbv2.RemoveTagsOutput, error) { if input == nil || len(input.ResourceArns) == 0 || len(input.TagKeys) == 0 { return nil, errors.New(awserrors.ErrorMissingParameter) diff --git a/spinifex/handlers/elbv2/service_impl_test.go b/spinifex/handlers/elbv2/service_impl_test.go index c53dc1e0a..a5efa3b32 100644 --- a/spinifex/handlers/elbv2/service_impl_test.go +++ b/spinifex/handlers/elbv2/service_impl_test.go @@ -1547,11 +1547,9 @@ func TestDescribeListeners_FilterByLBArn(t *testing.T) { assert.Len(t, desc.Listeners, 2) } -// TestDescribeListeners_AccountIsolation pins the independent account filter -// in DescribeListeners (service_impl.go:1470). LB-level isolation is tested -// by TestDescribeLoadBalancers_AccountIsolation, but listener filtering is -// enforced separately on each ListenerRecord — a regression that drops the -// listener-side check would not be caught by the LB-level test. +// TestDescribeListeners_AccountIsolation pins the per-ListenerRecord account filter. +// LB-level isolation is tested separately; a regression dropping the listener-side +// check would not be caught by the LB-level test. func TestDescribeListeners_AccountIsolation(t *testing.T) { svc := setupTestService(t) @@ -1836,12 +1834,9 @@ func TestCreateListener_PushConfig_NoNATS(t *testing.T) { require.NoError(t, err) // No panic, no error — updateStoredConfig skipped gracefully } -// TestDeleteLoadBalancer_NoTerminateWhenEmptyInstanceID pins the nil-safe -// branch in DeleteLoadBalancer: an LB created without a systemAMI never -// launched an ALB VM (InstanceID == ""), so DeleteLoadBalancer must skip -// TerminateSystemInstance rather than call it with an empty string. The -// positive case (terminate IS called when InstanceID is set) is covered by -// TestDeleteLoadBalancer_TerminatesVM_WithPublicIP in service_impl_vpc_test.go. +// TestDeleteLoadBalancer_NoTerminateWhenEmptyInstanceID pins the nil-safe branch: +// an LB without a systemAMI has InstanceID=="" and must skip TerminateSystemInstance. +// The positive case is covered by TestDeleteLoadBalancer_TerminatesVM_WithPublicIP. func TestDeleteLoadBalancer_NoTerminateWhenEmptyInstanceID(t *testing.T) { svc := setupTestService(t) @@ -2008,12 +2003,9 @@ func TestLBAgentHeartbeat_ProcessesHealthReport(t *testing.T) { assert.Equal(t, TargetHealthHealthy, stored.Targets[0].HealthState) } -// TestLBAgentHeartbeat_BuildsConfigOnActivation covers the create-burst race: -// the reactive updateStoredConfig calls during CreateListener/RegisterTargets -// no-op while the LB VM is still launching (InstanceID empty). The agent's -// first heartbeat (provisioning→active) is the readiness signal that must build -// the data-plane config from the listeners/targets created during provisioning, -// so the agent receives a non-empty ConfigHash + backend list on its first poll. +// TestLBAgentHeartbeat_BuildsConfigOnActivation covers the create-burst race where +// updateStoredConfig no-ops while InstanceID is empty; the first heartbeat +// (provisioning→active) must build the full config so the agent gets a ConfigHash. func TestLBAgentHeartbeat_BuildsConfigOnActivation(t *testing.T) { svc := setupTestService(t) @@ -2590,11 +2582,9 @@ func TestDescribeLoadBalancerAttributes_NLBDefaults(t *testing.T) { assert.Equal(t, "false", attrMap["load_balancing.cross_zone.enabled"]) } -// TestDefaultLoadBalancerAttributes_ALBCoversTerraformKeys guards against -// regressions where terraform's default ModifyLoadBalancerAttributes call -// hits ValidationError because the ALB default attribute set is missing a -// key that the AWS provider sends. Every key here is one the aws-sdk-go -// elbv2 API documents for ALBs. +// TestDefaultLoadBalancerAttributes_ALBCoversTerraformKeys guards against missing ALB +// attribute keys that Terraform's default ModifyLoadBalancerAttributes call sends, +// which would surface as ValidationError. func TestDefaultLoadBalancerAttributes_ALBCoversTerraformKeys(t *testing.T) { attrs := DefaultLoadBalancerAttributes(LoadBalancerTypeApplication) @@ -2719,11 +2709,8 @@ func TestModifyLoadBalancerAttributes_EmptyStringClears(t *testing.T) { } // --- Attribute mirror-pair tests (table-driven over TG/LB) --- -// -// ModifyTargetGroupAttributes/ModifyLoadBalancerAttributes (and their Describe -// counterparts) are mirror pairs differing only by record type, store methods, -// default set and not-found error. The behavioural tests below run once per -// kind via t.Run instead of being hand-copied per resource. +// Modify*/Describe* are mirror pairs differing only by record type, store methods, +// default set, and not-found error; tests run once per kind via t.Run. // rawAttr models one submitted SDK attribute, including the invalid shapes a // handler must skip: a nil slice element, a nil Key, or a nil Value. @@ -3027,10 +3014,8 @@ func TestAttributeSkipsInvalidEntries(t *testing.T) { } } -// TestAttributeAllInvalidReturnsError guards against the silent-success case -// where every submitted attribute trips the nil guard and the handler returns -// 200 OK with an empty body — the caller would think the write landed when -// nothing was applied. +// TestAttributeAllInvalidReturnsError guards against silent success when every +// attribute trips the nil guard, returning 200 OK with nothing applied. func TestAttributeAllInvalidReturnsError(t *testing.T) { for _, k := range attrKinds() { t.Run(k.name, func(t *testing.T) { @@ -3047,11 +3032,9 @@ func TestAttributeAllInvalidReturnsError(t *testing.T) { } } -// TestAttributeSequentialMerge verifies that successive Modify calls accumulate -// keys instead of replacing the entire attribute map. A refactor that did -// `rec.Attributes = newMap` would pass every other test (single-call round-trip -// covers the happy path) but silently wipe previous attributes on every -// subsequent Modify — the most likely real-world bug. +// TestAttributeSequentialMerge verifies successive Modify calls accumulate keys. +// `rec.Attributes = newMap` would pass single-call tests but silently wipe +// previous attributes on every subsequent call. func TestAttributeSequentialMerge(t *testing.T) { for _, k := range attrKinds() { t.Run(k.name, func(t *testing.T) { @@ -3112,10 +3095,9 @@ func TestAttributeNoopSkipsPersist(t *testing.T) { } } -// TestAttributeRejectsUnknownKey guards against silently persisting typo'd or -// cross-product attribute keys. AWS rejects unknown keys with ValidationError; -// we match that so Terraform surfaces the typo at plan time instead of letting -// it drift into KV forever. +// TestAttributeRejectsUnknownKey guards against silently persisting typo'd keys. +// AWS rejects unknowns with ValidationError; matching that surfaces typos at plan +// time rather than letting them drift into KV. func TestAttributeRejectsUnknownKey(t *testing.T) { for _, k := range attrKinds() { t.Run(k.name, func(t *testing.T) { diff --git a/spinifex/handlers/elbv2/service_impl_vpc_test.go b/spinifex/handlers/elbv2/service_impl_vpc_test.go index 4c45ea2bc..1db8937af 100644 --- a/spinifex/handlers/elbv2/service_impl_vpc_test.go +++ b/spinifex/handlers/elbv2/service_impl_vpc_test.go @@ -723,11 +723,9 @@ func TestENI_RequesterManagedFlag(t *testing.T) { } } -// TestCreateLoadBalancer_AttachesSpecifiedSecurityGroupsToENI verifies that -// when the caller supplies SecurityGroups on the LB input, the underlying ALB -// ENI is created with those SGs (not the VPC default SG). Without this the -// ENI's OVN port-group never gains the user's allow rules and inbound traffic -// is dropped by the SG enforcement default-deny ACL. +// TestCreateLoadBalancer_AttachesSpecifiedSecurityGroupsToENI verifies caller-supplied +// SecurityGroups are applied to the ALB ENI; without this the OVN port-group misses +// the user's allow rules and the default-deny ACL drops all inbound traffic. func TestCreateLoadBalancer_AttachesSpecifiedSecurityGroupsToENI(t *testing.T) { svc, vpcSvc := setupTestServiceWithVPC(t) @@ -804,11 +802,9 @@ func TestCreateLoadBalancer_NoSecurityGroupsFallsBackToDefault(t *testing.T) { assert.Equal(t, defaultSGID, *albENI.Groups[0].GroupId) } -// TestRebuildSystemInstanceInput_HappyPath verifies that a recovering ALB -// VM gets the same SystemInstanceInput shape the launch path produced — -// otherwise writeFwCfgBlobs would emit a different netcfg/cacert blob than -// the persisted QEMU command line references, and the relaunch would fail -// at fw_cfg load time the same way the tmpfs wipe does today. +// TestRebuildSystemInstanceInput_HappyPath verifies a recovering ALB VM gets the +// same SystemInstanceInput as the original launch; a mismatch causes writeFwCfgBlobs +// to emit a different netcfg/cacert blob than the persisted QEMU command line. func TestRebuildSystemInstanceInput_HappyPath(t *testing.T) { svc, vpcSvc := setupTestServiceWithVPC(t) @@ -854,10 +850,8 @@ func TestRebuildSystemInstanceInput_HappyPath(t *testing.T) { assert.Equal(t, originalInput.SubnetID, rebuilt.SubnetID) assert.Equal(t, originalInput.ENIID, rebuilt.ENIID) assert.Equal(t, originalInput.ENIMac, rebuilt.ENIMac) - // ENIIP comes from the persisted lb.VPCIP (which the launcher's PrivateIP - // return value sets); in production the launcher echoes back the ENI's - // private IP, so this matches the original launch input. The test mock - // returns a synthetic value, so assert against the mock's PrivateIP. + // ENIIP comes from lb.VPCIP (set by the launcher's PrivateIP return value); + // the test mock returns a synthetic value, so assert against the mock's IP. assert.Equal(t, "10.0.1.42", rebuilt.ENIIP) assert.Equal(t, originalInput.Scheme, rebuilt.Scheme) assert.Equal(t, originalInput.AccountID, rebuilt.AccountID) @@ -870,10 +864,9 @@ func TestRebuildSystemInstanceInput_HappyPath(t *testing.T) { assert.Equal(t, "172.31.0.7/24", rebuilt.NICs[1].CIDR, "mgmt NIC CIDR must come from RecoveryContext") } -// TestRebuildSystemInstanceInput_NoLBRecord verifies the error path used by -// refreshSystemInstanceState to flag the instance recovery_failed when the -// LB record was wiped from KV but the VM record survived — recovery must -// not invent a fake input and silently boot the VM into an unowned state. +// TestRebuildSystemInstanceInput_NoLBRecord verifies the recovery_failed path when +// the LB record is gone from KV but the VM record survives; recovery must not +// silently boot the VM into an unowned state. func TestRebuildSystemInstanceInput_NoLBRecord(t *testing.T) { svc := setupTestService(t) @@ -885,11 +878,9 @@ func TestRebuildSystemInstanceInput_NoLBRecord(t *testing.T) { assert.Contains(t, err.Error(), "no LB record references instance i-ghost") } -// TestRebuildSystemInstanceInput_MultiENI verifies that extras survive the -// rebuild for multi-subnet ALBs — the create path passes one ExtraENIInput -// per non-primary subnet, and recovery must produce the same shape so the -// daemon wires the same set of taps and NICs the persisted QEMU args -// reference. +// TestRebuildSystemInstanceInput_MultiENI verifies extras survive the rebuild for +// multi-subnet ALBs; recovery must produce the same ExtraENIInput shape so the +// daemon wires the same taps and NICs the persisted QEMU args reference. func TestRebuildSystemInstanceInput_MultiENI(t *testing.T) { svc, vpcSvc := setupTestServiceWithVPC(t) diff --git a/spinifex/handlers/elbv2/service_nats.go b/spinifex/handlers/elbv2/service_nats.go index 903cfdf30..0ae0af0f1 100644 --- a/spinifex/handlers/elbv2/service_nats.go +++ b/spinifex/handlers/elbv2/service_nats.go @@ -10,12 +10,9 @@ import ( const ( defaultTimeout = 30 * time.Second - // longRunningTimeout covers operations that synchronously launch or tear - // down ALB frontend VMs (DHCP lease + QEMU boot / QMP shutdown + QEMU exit). - // A healthy launch is ~45s on a warm box; shutdown can take tens of seconds - // if QMP is slow. Until CreateLoadBalancer is made fully async, we keep a - // wide margin so Terraform's AWS provider doesn't retry mid-flight and then - // hit DuplicateLoadBalancerName on the second attempt. + // longRunningTimeout covers synchronous ALB VM launch/teardown (~45 s on a warm + // box). Wide margin prevents Terraform retrying mid-flight and hitting + // DuplicateLoadBalancerName before CreateLoadBalancer is made async. longRunningTimeout = 5 * time.Minute ) diff --git a/spinifex/handlers/elbv2/store.go b/spinifex/handlers/elbv2/store.go index 0b76867e2..7edec5006 100644 --- a/spinifex/handlers/elbv2/store.go +++ b/spinifex/handlers/elbv2/store.go @@ -22,14 +22,12 @@ const ( KeyPrefixTG = "tg." KeyPrefixListener = "listener." KeyPrefixRule = "rule." - // KeyPrefixLBName is the per-account LB-name claim used for - // CreateLoadBalancer idempotency: it makes the LB name (not the generated - // lbID) an atomically claimable identity so concurrent same-name creates - // cannot both launch an LB VM. + // KeyPrefixLBName is the per-account LB-name claim key that makes the LB + // name an atomically claimable identity, preventing concurrent same-name + // creates from both launching a VM. KeyPrefixLBName = "lbname." - // maxLBNameClaimRetries bounds the crash-orphan CAS-reclaim loop in - // ClaimLBName (Create raced a delete, or a CAS-reclaim lost the revision). + // maxLBNameClaimRetries bounds the crash-orphan CAS-reclaim loop in ClaimLBName. maxLBNameClaimRetries = 5 ) @@ -65,12 +63,9 @@ func LBNameKey(name, accountID string) string { return KeyPrefixLBName + accountID + "." + name } -// ClaimLBName atomically claims the LB name for accountID with lbID as owner. -// ok=true ⇒ this caller owns the name and may proceed. dup=true ⇒ a live LB -// already holds the name (a genuine duplicate). A name claim whose owner lbID -// resolves to no live record is a crashed prior create — it is reclaimed for -// this caller (ok=true). This is the idempotency barrier for CreateLoadBalancer: -// a duplicate concurrent create loses the claim and never launches a VM. +// ClaimLBName atomically claims the LB name; ok=true means this caller owns it, +// dup=true means a live LB already holds it. An orphaned claim (owner resolves to +// no record) is reclaimed via CAS. Idempotency barrier for CreateLoadBalancer. func (s *Store) ClaimLBName(name, accountID, lbID string) (ok bool, dup bool, err error) { key := LBNameKey(name, accountID) for range maxLBNameClaimRetries { @@ -158,10 +153,9 @@ func (s *Store) ListLoadBalancers() ([]*LoadBalancerRecord, error) { return listByPrefix[LoadBalancerRecord](s.kv, KeyPrefixLB) } -// GetLoadBalancerByArn finds a load balancer by its ARN via a direct KV lookup -// on the short ID embedded in the ARN's final path segment. Returns (nil, nil) -// if the ARN can't be parsed or no record matches. Terraform hits -// Describe*Attributes on every plan/refresh so this must be O(1), not O(n). +// GetLoadBalancerByArn finds a load balancer by the short ID in the ARN's final +// path segment. Returns (nil, nil) if unparseable or not found. O(1) — Terraform +// hits Describe*Attributes on every plan/refresh. func (s *Store) GetLoadBalancerByArn(arn string) (*LoadBalancerRecord, error) { // ELBv2 LB ARN: arn:aws:elasticloadbalancing:{region}:{account}:loadbalancer/{app,net}/{name}/{lbID} idx := strings.LastIndex(arn, "/") @@ -176,10 +170,7 @@ func (s *Store) GetLoadBalancerByArn(arn string) (*LoadBalancerRecord, error) { if lb == nil { return nil, nil } - // Defence-in-depth: ensure the record actually belongs to this ARN. - // Short IDs are random hex, so collisions are effectively impossible, but - // a mismatch here would indicate KV corruption and must not be silently - // served as a successful lookup. + // Defence-in-depth: ARN mismatch indicates KV corruption; never serve it silently. if lb.LoadBalancerArn != arn { slog.Error("load balancer KV record ARN mismatch", "requested_arn", arn, "stored_arn", lb.LoadBalancerArn, "lb_id", lbID) @@ -244,10 +235,8 @@ func (s *Store) ListTargetGroups() ([]*TargetGroupRecord, error) { return listByPrefix[TargetGroupRecord](s.kv, KeyPrefixTG) } -// GetTargetGroupByArn finds a target group by its ARN via a direct KV lookup -// on the short ID embedded in the ARN's final path segment. See -// GetLoadBalancerByArn for the motivation — Terraform's per-plan -// DescribeTargetGroupAttributes storm must be O(1). +// GetTargetGroupByArn finds a target group by the short ID in the ARN's final +// path segment. O(1) — see GetLoadBalancerByArn for motivation. func (s *Store) GetTargetGroupByArn(arn string) (*TargetGroupRecord, error) { // ELBv2 TG ARN: arn:aws:elasticloadbalancing:{region}:{account}:targetgroup/{name}/{tgID} idx := strings.LastIndex(arn, "/") @@ -286,10 +275,8 @@ func (s *Store) TargetGroupsForLB(lbID string) ([]*TargetGroupRecord, error) { return nil, fmt.Errorf("list listeners for %s: %w", lbID, err) } - // Collect unique TG IDs from listener default actions and rule actions. - // Rule TGs must be included so the health checker can update their state - // when HAProxy reports server status — HAProxy probes every backend it - // renders, including rule backends. + // Collect unique TG IDs from default actions and rule actions; rule TGs must + // be included so the health checker can update state for all probed backends. seen := make(map[string]struct{}) collect := func(tgArn string) { if tgArn == "" { @@ -399,11 +386,8 @@ func (s *Store) ListListenersByLB(lbArn string) ([]*ListenerRecord, error) { return result, nil } -// GetListenerByArn finds a listener by its ARN via a direct KV lookup on the -// short ID embedded in the ARN's final path segment. See GetLoadBalancerByArn -// for the motivation — Terraform calls DescribeTags on every aws_lb_listener -// on every plan, and DescribeTags can carry many ARNs per call, so this must -// be O(1) per ARN, not an O(n) listener-KV scan. +// GetListenerByArn finds a listener by the short ID in the ARN's final path segment. +// O(1) per ARN — Terraform calls DescribeTags for every listener on every plan. func (s *Store) GetListenerByArn(arn string) (*ListenerRecord, error) { // ELBv2 listener ARN: arn:aws:elasticloadbalancing:{region}:{account}:listener/{app,net}/{name}/{lbID}/{listenerID} idx := strings.LastIndex(arn, "/") @@ -481,10 +465,8 @@ func (s *Store) ListRulesByListener(listenerArn string) ([]*RuleRecord, error) { return result, nil } -// GetRuleByArn finds a rule by its ARN. Falls back to linear scan because the -// listener-rule ARN structure embeds the rule short ID after several -// listener-specific segments; parsing it is brittle and rules-per-account is -// bounded. +// GetRuleByArn finds a rule by its ARN via linear scan; the rule short ID is +// embedded after several listener-specific segments, making direct parsing brittle. func (s *Store) GetRuleByArn(arn string) (*RuleRecord, error) { rules, err := s.ListRules() if err != nil { diff --git a/spinifex/handlers/elbv2/system_instance_launcher_nats.go b/spinifex/handlers/elbv2/system_instance_launcher_nats.go index 12fcdcd1a..b1df236d2 100644 --- a/spinifex/handlers/elbv2/system_instance_launcher_nats.go +++ b/spinifex/handlers/elbv2/system_instance_launcher_nats.go @@ -8,19 +8,9 @@ import ( "github.com/nats-io/nats.go" ) -// natsSystemInstanceLauncher implements SystemInstanceLauncher by -// dispatching LaunchSystemInstance / TerminateSystemInstance over NATS so -// ALB-VMs fan out across the cluster instead of piling on the daemon node -// that handled the upstream CreateLoadBalancer call. -// -// Subjects: -// - system.LaunchInstance.{type} (queue group spinifex-workers) -// - system.TerminateInstance.{id} (single subscriber: the owning daemon) -// -// The daemons subscribe with capacity-aware logic in -// ResourceManager.updateInstanceSubscriptions; only nodes with free -// vCPU/memory for the requested type carry the launch subscription, so -// over-full nodes naturally drop out of the queue group. +// natsSystemInstanceLauncher implements SystemInstanceLauncher over NATS request/reply, +// fanning ALB-VM launches across the cluster via the spinifex-workers queue group +// (system.LaunchInstance.{type}); terminate goes directly to the owning daemon. type natsSystemInstanceLauncher struct { nc *nats.Conn timeout time.Duration @@ -28,10 +18,9 @@ type natsSystemInstanceLauncher struct { const defaultSystemInstanceTimeout = 60 * time.Second -// NewNATSSystemInstanceLauncher returns a SystemInstanceLauncher that -// publishes requests over NATS request/reply. timeout caps each round trip; -// pass 0 to use the default (60s, sized for direct-boot microVM start + -// EIP allocation latency on a busy cluster). +// NewNATSSystemInstanceLauncher returns a NATS-based SystemInstanceLauncher. +// Pass timeout=0 to use the 60 s default (sized for direct-boot microVM +// start + EIP allocation on a busy cluster). func NewNATSSystemInstanceLauncher(nc *nats.Conn, timeout time.Duration) SystemInstanceLauncher { if timeout <= 0 { timeout = defaultSystemInstanceTimeout diff --git a/spinifex/handlers/elbv2/types.go b/spinifex/handlers/elbv2/types.go index e11682691..d02512325 100644 --- a/spinifex/handlers/elbv2/types.go +++ b/spinifex/handlers/elbv2/types.go @@ -109,10 +109,9 @@ type LoadBalancerRecord struct { State string `json:"state"` // "provisioning", "active", "failed" VpcId string `json:"vpc_id"` SecurityGroups []string `json:"security_groups"` - // NLBManagedSGID is the internal security group the ELBv2 service mints for - // a network LB (NLBs reject customer SGs, so this is created and owned here). - // It is attached to every LB ENI and is the SG listener-port ingress is - // authorized on. Not surfaced by DescribeLoadBalancers. + // NLBManagedSGID is the managed SG minted for NLBs (customer SGs are rejected). + // Attached to every LB ENI; listener-port ingress is authorized on it. + // Not surfaced by DescribeLoadBalancers. NLBManagedSGID string `json:"nlb_managed_sg_id,omitempty"` // NLBIngressCIDRs overrides the scheme-based default client CIDRs that // listener ports are opened to on NLBManagedSGID. Empty ⇒ default @@ -137,10 +136,9 @@ type LoadBalancerRecord struct { CreatedAt time.Time `json:"created_at"` } -// HealthTargetSpec is a backend the nginx agent actively health-checks. It is -// computed when the NLB config is stored and delivered to the agent via -// GetLBConfig. ServerName matches the health checker's key -// (sanitizeName("srv", target.Id)). +// HealthTargetSpec is a backend the nginx agent actively health-checks, computed +// at NLB config store time and delivered via GetLBConfig. ServerName matches +// the health checker's key (sanitizeName("srv", target.Id)). type HealthTargetSpec struct { ServerName string `json:"server_name"` Address string `json:"address"` // ip:port to probe @@ -234,10 +232,9 @@ type ListenerRecord struct { AccountID string `json:"account_id"` CreatedAt time.Time `json:"created_at"` - // Certificates holds the listener's TLS certificates for a secure - // protocol (ALB HTTPS / NLB TLS). The first IsDefault entry is the - // default cert; the rest are additional SNI certs. Empty for non-secure - // listeners. SslPolicy is the negotiated security policy name. + // Certificates holds TLS certs for HTTPS/TLS listeners; the first IsDefault + // entry is the default cert, the rest are SNI certs. Empty for non-secure + // listeners. SslPolicy is the negotiated policy name. Certificates []ListenerCertificate `json:"certificates,omitempty"` SslPolicy string `json:"ssl_policy,omitempty"` @@ -309,17 +306,9 @@ type RuleQueryStringKV struct { Value string `json:"value"` } -// DefaultLoadBalancerAttributes returns the default attribute set for a load -// balancer of the given type. ALBs default load_balancing.cross_zone.enabled to -// "true"; NLBs (and any unknown type) default it to "false". This is the single -// source of truth — CreateLoadBalancer no longer seeds attributes, and -// DescribeLoadBalancerAttributes derives the per-type defaults from lb.Type on -// read. -// -// The key set must be broad enough to satisfy terraform AWS provider's -// default ModifyLoadBalancerAttributes call after aws_lb creation: the -// provider sends every attribute it knows about, and any key missing here -// gets rejected with ValidationError, surfacing as "UnknownError" in tofu. +// DefaultLoadBalancerAttributes returns the default attribute map for a load balancer. +// ALBs cross-zone default is "true"; NLBs/unknown default to "false". Key set covers +// every attribute Terraform's AWS provider sends on post-create Modify calls. func DefaultLoadBalancerAttributes(lbType string) map[string]string { switch lbType { case LoadBalancerTypeApplication: @@ -336,22 +325,16 @@ func DefaultTargetGroupAttributes() map[string]string { return maps.Clone(targetGroupAttributeDefaults) } -// Default attribute sets are package-level so Describe calls clone a prebuilt -// map instead of rebuilding it on every request. Callers receive a clone, never -// the shared map, so mutation cannot leak back into the defaults. -// -// access_logs.s3.bucket / .prefix (and the connection_logs equivalents) default -// to empty strings: that matches real AWS, where logging is disabled by default -// and the bucket/prefix are unset until the operator opts in. Terraform sends -// these empty strings on every apply, so they must be present as known keys. +// Default attribute sets are package-level vars; callers receive a clone so +// mutation never leaks back. Empty-string log keys match real AWS defaults +// and must be present as known keys for Terraform compatibility. var ( - // Common to all load balancer types; also the fallback for unknown types. + // lbBaseAttributeDefaults is common to all LB types and the fallback for unknown types. lbBaseAttributeDefaults = map[string]string{ "deletion_protection.enabled": "false", "load_balancing.cross_zone.enabled": "false", } - // ALB defaults match real AWS values as of the aws-sdk-go 1.55 elbv2 API. albAttributeDefaults = map[string]string{ "deletion_protection.enabled": "false", "load_balancing.cross_zone.enabled": "true", diff --git a/spinifex/handlers/iam/imds_rpc.go b/spinifex/handlers/iam/imds_rpc.go index 9cb1bc48e..208644f76 100644 --- a/spinifex/handlers/iam/imds_rpc.go +++ b/spinifex/handlers/iam/imds_rpc.go @@ -10,9 +10,8 @@ import ( // awsgw in the cluster, matching the cluster-wide worker convention. const imdsResponderQueue = "spinifex-workers" -// SubjectResolveInstanceProfile and SubjectGetRole are the internal, ACL-gated -// subjects the IMDS handler (in vpcd) calls to resolve an instance profile and its -// role from awsgw's in-process IAM. Only these two read RPCs are surfaced. +// SubjectResolveInstanceProfile and SubjectGetRole are the ACL-gated internal RPC subjects +// vpcd uses to resolve an instance profile and role from awsgw's IAM service. const ( SubjectResolveInstanceProfile = "imds.iam.resolve_instance_profile" SubjectGetRole = "imds.iam.get_role" @@ -30,9 +29,8 @@ type GetRoleRequest struct { Input *iam.GetRoleInput `json:"input"` } -// SubscribeIMDSResponders wires the profile/role responders onto nc, -// queue-grouped so any awsgw can answer. The returned subscriptions are for -// caller-side cleanup. +// SubscribeIMDSResponders wires profile/role responders onto nc, queue-grouped. +// Returns subscriptions for caller-side cleanup. func (s *IAMServiceImpl) SubscribeIMDSResponders(nc *nats.Conn) ([]*nats.Subscription, error) { profileSub, err := nc.QueueSubscribe(SubjectResolveInstanceProfile, imdsResponderQueue, func(msg *nats.Msg) { utils.ServeNATSRequest(msg, func(req *ResolveInstanceProfileRequest) (*InstanceProfile, error) { diff --git a/spinifex/handlers/iam/instance_profiles.go b/spinifex/handlers/iam/instance_profiles.go index 12a445d7f..6d46edd32 100644 --- a/spinifex/handlers/iam/instance_profiles.go +++ b/spinifex/handlers/iam/instance_profiles.go @@ -245,14 +245,8 @@ func (s *IAMServiceImpl) ListInstanceProfilesForRole(accountID string, input *ia }, nil } -// ResolveInstanceProfile dereferences an instance-profile reference (the -// {Name, Arn} shape from RunInstancesInput.IamInstanceProfile and related -// EC2 association ops) to the canonical InstanceProfile record. Accepts -// either the friendly name or the full ARN. -// -// When given an ARN, the embedded account ID must match the caller — this -// is a defence-in-depth check; the gateway also rejects cross-account -// references with AccessDenied before reaching this helper. +// ResolveInstanceProfile resolves an instance-profile name or ARN to its record. +// When given an ARN, the embedded account ID must match accountID. func (s *IAMServiceImpl) ResolveInstanceProfile(accountID, nameOrARN string) (*InstanceProfile, error) { if nameOrARN == "" { return nil, errors.New(awserrors.ErrorIAMInvalidInput) @@ -272,10 +266,7 @@ func (s *IAMServiceImpl) ResolveInstanceProfile(accountID, nameOrARN string) (*I return s.getInstanceProfile(accountID, profileName) } -// parseInstanceProfileARN extracts the account ID and profile name from an -// IAM instance-profile ARN of the form -// `arn:aws:iam:::instance-profile//` (path optional). -// Returns InvalidIamInstanceProfileArn.Malformed for any deviation. +// parseInstanceProfileARN extracts accountID and profile name from an IAM instance-profile ARN. func parseInstanceProfileARN(arn string) (accountID, name string, err error) { parts := strings.SplitN(arn, ":", 6) if len(parts) != 6 || parts[0] != "arn" || parts[1] != "aws" || parts[2] != "iam" || parts[3] != "" { @@ -315,11 +306,8 @@ func (s *IAMServiceImpl) getInstanceProfile(accountID, profileName string) (*Ins return &profile, nil } -// profileToSDK converts the internal InstanceProfile record into the AWS SDK -// shape. The SDK requires a Roles array (0 or 1 element in our model); when -// a role is attached the role's full record is dereferenced and embedded. -// Propagates errors from the role lookup — DeleteRole's no-orphan-ref guard -// makes this an invariant violation that must not be silently degraded. +// profileToSDK converts the internal InstanceProfile to the AWS SDK shape. +// Dereferences the attached role when present; propagates lookup errors. func (s *IAMServiceImpl) profileToSDK(accountID string, p *InstanceProfile) (*iam.InstanceProfile, error) { out := &iam.InstanceProfile{ InstanceProfileName: aws.String(p.InstanceProfileName), diff --git a/spinifex/handlers/iam/oidc_providers.go b/spinifex/handlers/iam/oidc_providers.go index c1e8109fa..d65dd403b 100644 --- a/spinifex/handlers/iam/oidc_providers.go +++ b/spinifex/handlers/iam/oidc_providers.go @@ -18,22 +18,12 @@ import ( "github.com/nats-io/nats.go" ) -// Per-account IAM JetStream KV bucket holds resources that are inherently -// account-scoped and not appropriate for the global IAM buckets — currently -// the OIDC identity-provider registry consumed by STS AssumeRoleWithWebIdentity -// and by IAM's CreateOpenIDConnectProvider CRUD. -// -// Lazy creation: callers that read MUST tolerate ErrBucketNotFound (no provider -// has been registered for the account yet); callers that write MUST use -// GetOrCreateIAMAccountBucket so the bucket appears on first use. +// Per-account IAM bucket holds account-scoped resources (currently the OIDC provider registry). +// Readers must tolerate ErrBucketNotFound; writers must use GetOrCreateIAMAccountBucket. const ( KVBucketIAMAccountPrefix = "iam-account-" KVBucketIAMAccountVersion = 1 - // oidcProvidersKeyPrefix matches the eks-v1.md Q4 layout: - // "iam-account-{accountID}/oidc-providers/{issuerHash}". Keep the literal - // here (not strings.Join) so a stray refactor that introduces a different - // separator fails compile rather than silently writing to a sibling path. oidcProvidersKeyPrefix = "oidc-providers/" ) @@ -43,40 +33,25 @@ func IAMAccountBucketName(accountID string) string { return KVBucketIAMAccountPrefix + accountID } -// OIDCProviderKey returns the KV key for an OIDC provider entry. The key is -// derived from the SHA-256 hex digest of the issuer URL — the Mulga -// convention (AWS uses the SHA-1 thumbprint of the IdP root cert; we hash the -// issuer URL because we are also the IdP and have no separate cert chain to -// pin against). +// OIDCProviderKey returns the KV key for an OIDC provider, derived from the SHA-256 of the issuer URL. func OIDCProviderKey(issuer string) string { sum := sha256.Sum256([]byte(issuer)) return oidcProvidersKeyPrefix + hex.EncodeToString(sum[:]) } -// OIDCProviderARN composes the AWS-format ARN used as the Federated principal -// value in role trust policies that authorise the named issuer: -// -// arn:aws:iam::{accountID}:oidc-provider/{issuerHostPath} -// -// issuerHostPath is the issuer URL with the scheme stripped (no leading -// "https://") — matches the AWS convention so customers can paste an existing -// IRSA trust policy unchanged. +// OIDCProviderARN returns the arn:aws:iam::{accountID}:oidc-provider/{issuerHostPath} ARN. +// issuerHostPath is the issuer URL with the https:// scheme stripped. func OIDCProviderARN(accountID, issuerHostPath string) string { return fmt.Sprintf("arn:aws:iam::%s:oidc-provider/%s", accountID, issuerHostPath) } -// GetOrCreateIAMAccountBucket opens the per-account IAM bucket, creating it on -// first use. Writers (CreateOpenIDConnectProvider) MUST use this; readers that -// must tolerate a not-yet-created bucket should call js.KeyValue directly and -// treat nats.ErrBucketNotFound as "empty". +// GetOrCreateIAMAccountBucket opens the per-account IAM bucket, creating it on first use. func GetOrCreateIAMAccountBucket(js nats.JetStreamContext, accountID string, replicas int) (nats.KeyValue, error) { return getOrCreateBucket(js, IAMAccountBucketName(accountID), 1, max(replicas, 1)) } // OIDCProviderRecord is the stored shape of a registered OIDC identity provider. -// Url carries the full issuer URL including scheme (the exact string the -// cluster apiserver places in the token `iss` claim); the KV key is its hash, -// and the provider ARN is derived by stripping the scheme. +// Url is the full issuer URL; the KV key is its SHA-256 hash. type OIDCProviderRecord struct { Url string `json:"url"` ClientIDList []string `json:"clientIDList,omitempty"` @@ -85,10 +60,7 @@ type OIDCProviderRecord struct { Tags []Tag `json:"tags,omitempty"` } -// issuerFromOIDCProviderARN reverses OIDCProviderARN: it extracts the -// scheme-less host/path suffix and prepends https:// to reconstruct the issuer -// URL (and therefore its KV key). v1 issuers are always https, so this -// round-trips Create's input.Url. +// issuerFromOIDCProviderARN reverses OIDCProviderARN, reconstructing the full https issuer URL. func issuerFromOIDCProviderARN(arn string) (string, error) { _, hostPath, ok := strings.Cut(arn, ":oidc-provider/") if !ok { @@ -100,8 +72,7 @@ func issuerFromOIDCProviderARN(arn string) (string, error) { return "https://" + hostPath, nil } -// validateOIDCProviderURL enforces the v1 issuer shape: a parseable https URL -// with a host. We are the IdP, so no thumbprint/cert-chain validation applies. +// validateOIDCProviderURL enforces the issuer shape: a parseable https URL with a non-empty host. func validateOIDCProviderURL(raw string) error { if raw == "" { return errors.New("url is required") @@ -290,8 +261,7 @@ func stripIssuerScheme(issuer string) string { return strings.TrimPrefix(issuer, "https://") } -// tagsToSDK converts stored Tags into the SDK shape used by OIDC-provider -// responses. +// tagsToSDK converts stored Tags into the SDK shape. func tagsToSDK(tags []Tag) []*iam.Tag { if len(tags) == 0 { return nil diff --git a/spinifex/handlers/iam/roles.go b/spinifex/handlers/iam/roles.go index 8fb2caebd..b0256c56a 100644 --- a/spinifex/handlers/iam/roles.go +++ b/spinifex/handlers/iam/roles.go @@ -331,16 +331,9 @@ func (s *IAMServiceImpl) ListAttachedRolePolicies(accountID string, input *iam.L }, nil } -// GetRolePolicies resolves the managed policy documents attached to a role, -// the symmetric twin of GetUserPolicies. The gateway uses it to evaluate an -// assumed-role session's permissions. Inline role policies do not exist yet, -// so this resolves attached managed policies only. -// -// Fails closed: a missing role (NoSuchEntity) or any unresolvable attached -// policy returns an error so the caller denies access rather than evaluating -// against a partial policy set. A role with no attached policies returns no -// documents, which the evaluator treats as an implicit deny — AWS parity, as -// assumability does not imply permissions. +// GetRolePolicies resolves the managed policy documents attached to a role. +// Used by the gateway for policy evaluation. Fails closed: any unresolvable +// policy returns an error so the caller denies access rather than using a partial set. func (s *IAMServiceImpl) GetRolePolicies(accountID, roleName string) ([]PolicyDocument, error) { role, err := s.getRole(accountID, roleName) if err != nil { @@ -351,12 +344,9 @@ func (s *IAMServiceImpl) GetRolePolicies(accountID, roleName string) ([]PolicyDo for _, arn := range role.AttachedPolicies { policy, err := s.getPolicyByARN(accountID, arn) if err != nil { - // Fail closed: if we can't resolve a policy, we can't make a safe - // access decision. Return an error so the caller denies access. - return nil, fmt.Errorf("resolve policy %s: %w", arn, err) + return nil, fmt.Errorf("resolve policy %s: %w", arn, err) // fail closed } - // policy already validated at creation, so we skip validating again var doc PolicyDocument if err := json.Unmarshal([]byte(policy.PolicyDocument), &doc); err != nil { return nil, fmt.Errorf("parse policy %s: %w", policy.PolicyName, err) @@ -450,30 +440,12 @@ func roleToSDK(r *Role) *iam.Role { return out } -// STSActionAssumeRoleWithWebIdentity is the only Action that may carry a -// Condition block in v1 (IRSA `{iss}:sub` / `{iss}:aud` checks). Lives in -// this package so the validator and the STS-side evaluator agree on the -// exact spelling without a cross-package constant chase. +// STSActionAssumeRoleWithWebIdentity is the only action that may carry a Condition block. const STSActionAssumeRoleWithWebIdentity = "sts:AssumeRoleWithWebIdentity" -// ValidateTrustPolicyDocument parses and validates an AssumeRolePolicyDocument -// JSON string. Trust-policy evaluation (Principal semantics, Action being -// sts:AssumeRole, etc.) is deferred to STS — this layer only checks the -// document shape. -// -// Categories of fields rejected at write time rather than silently -// accepted-then-ignored at evaluation time: NotPrincipal and NotAction — -// each would otherwise let an author write a policy whose runtime behaviour -// silently diverges from its stated intent (NotPrincipal-Allow grants the -// universe; NotAction is unimplemented). -// -// Condition blocks are accepted only for the narrow IRSA case: the statement -// Action must be exactly `["sts:AssumeRoleWithWebIdentity"]` (no wildcards, -// no co-listed actions) and the only operator key allowed is StringEquals. -// Any wider acceptance would let an author write an ExternalId-protected -// sts:AssumeRole policy that evalTrustPolicy silently ignores at runtime — -// the same "loud-fail-here-or-silent-allow-there" trade-off the original -// blanket rejection guarded. +// ValidateTrustPolicyDocument parses and validates an AssumeRolePolicyDocument JSON string. +// Rejects NotPrincipal and NotAction at write time. Condition blocks are accepted only for +// sts:AssumeRoleWithWebIdentity with StringEquals — any wider shape would be silently ignored at runtime. func ValidateTrustPolicyDocument(docJSON string) (*TrustPolicyDocument, error) { if len(docJSON) > maxTrustPolicyDocumentSize { return nil, fmt.Errorf("trust policy exceeds maximum size of %d bytes", maxTrustPolicyDocumentSize) @@ -523,11 +495,8 @@ func ValidateTrustPolicyDocument(docJSON string) (*TrustPolicyDocument, error) { return &doc, nil } -// validateWebIdentityCondition gates the narrow IRSA Condition allowance: -// Action must be exactly `["sts:AssumeRoleWithWebIdentity"]` (no other entry, -// no wildcard) and the Condition operator must be exclusively StringEquals. -// Any wider shape would re-open the "silently-accepted Condition on -// sts:AssumeRole" hole the original blanket rejection closed. +// validateWebIdentityCondition gates Condition blocks to the IRSA case: +// Action must be exactly sts:AssumeRoleWithWebIdentity and operator must be StringEquals only. func validateWebIdentityCondition(i int, stmt TrustStatement) error { if len(stmt.Action) != 1 || stmt.Action[0] != STSActionAssumeRoleWithWebIdentity { return fmt.Errorf("statement %d: trust policy Condition blocks are only supported on statements whose Action is exactly [%q]; v1 does not evaluate conditions for other actions", i, STSActionAssumeRoleWithWebIdentity) @@ -544,11 +513,8 @@ func validateWebIdentityCondition(i int, stmt TrustStatement) error { return nil } -// isRawJSONNonEmpty reports whether a json.RawMessage carries a meaningful -// value. Whitespace, the JSON null literal, and the empty object/array -// literals all count as empty — they would otherwise let `Condition: {}` -// trip the Condition rejection and let `Principal: {}` slip past the -// Principal-is-required check. +// isRawJSONNonEmpty reports whether a json.RawMessage carries a meaningful value. +// Whitespace, null, and empty object/array all count as empty. func isRawJSONNonEmpty(raw json.RawMessage) bool { if len(raw) == 0 { return false diff --git a/spinifex/handlers/iam/service_impl.go b/spinifex/handlers/iam/service_impl.go index 39b9fe950..605991327 100644 --- a/spinifex/handlers/iam/service_impl.go +++ b/spinifex/handlers/iam/service_impl.go @@ -40,11 +40,8 @@ const ( maxAccessKeysPerUser = 2 - // LongLivedAccessKeyIDPrefix is the AWS-defined prefix for long-lived - // IAM access keys. The access-keys bucket rejects writes whose key does - // not start with it — a record under any other prefix in this bucket - // would be resolved by the SigV4 AKIA path without the X-Amz-Security-Token - // check that the ASIA path enforces, a silent privilege upgrade. + // LongLivedAccessKeyIDPrefix is the AWS-defined prefix for long-lived IAM access keys. + // The access-keys bucket rejects writes with any other prefix to prevent silent privilege escalation. LongLivedAccessKeyIDPrefix = "AKIA" ) @@ -87,18 +84,14 @@ type IAMServiceImpl struct { instanceProfilesBucket nats.KeyValue masterKey []byte decrypter *Decrypter - // replicas is the JetStream replication factor for lazily-created - // per-account buckets (the OIDC-provider registry). Matches the factor - // the eagerly-created buckets above were built with. + // replicas is the JetStream replication factor for lazily-created per-account buckets. replicas int } var _ IAMService = (*IAMServiceImpl)(nil) // NewIAMServiceImpl creates a new IAM service backed by NATS JetStream KV. -// clusterSize sets the JetStream replication factor for KV buckets. For -// multi-node clusters this must match the number of NATS servers so that -// buckets survive node failures. Pass 1 for single-node or test setups. +// clusterSize sets the replication factor; pass 1 for single-node or test setups. func NewIAMServiceImpl(natsConn *nats.Conn, masterKey []byte, clusterSize int) (*IAMServiceImpl, error) { if len(masterKey) != 32 { return nil, fmt.Errorf("master key must be 32 bytes, got %d", len(masterKey)) @@ -441,7 +434,6 @@ func (s *IAMServiceImpl) CreateAccessKey(accountID string, input *iam.CreateAcce return nil, fmt.Errorf("store access key: %w", err) } - // Update user's access key list user.AccessKeys = append(user.AccessKeys, accessKeyID) userData, err := json.Marshal(user) if err != nil { @@ -521,7 +513,6 @@ func (s *IAMServiceImpl) DeleteAccessKey(accountID string, input *iam.DeleteAcce return nil, err } - // Find and remove the access key reference from the user found := false remaining := make([]string, 0, len(user.AccessKeys)) for _, keyID := range user.AccessKeys { @@ -536,8 +527,7 @@ func (s *IAMServiceImpl) DeleteAccessKey(accountID string, input *iam.DeleteAcce return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) } - // Update user record first, then delete the key. This avoids orphaning the - // reference if a crash occurs between the two operations. + // Update user record first to avoid orphaning the reference on crash. user.AccessKeys = remaining userData, err := json.Marshal(user) if err != nil { @@ -623,13 +613,8 @@ func (s *IAMServiceImpl) DecryptSecret(ciphertext string) (string, error) { return s.decrypter.Decrypt(ciphertext) } -// SeedBootstrap consumes bootstrap data to create the system root IAM user and -// (optionally) the default admin account in NATS KV. Uses conditional create -// (put-if-not-exists) for multi-node race safety — the first node wins; others -// skip silently. Also creates the system account record (000000000000). -// If data.Admin is non-nil, it also seeds the admin account, user, access key, -// AdministratorAccess policy, and sets the account counter so subsequent -// CreateAccount calls start at 000000000002. +// SeedBootstrap seeds the system root user and optional admin account into NATS KV. +// Uses conditional create for multi-node safety; first node wins, others skip silently. func (s *IAMServiceImpl) SeedBootstrap(data *BootstrapData) error { // --- Seed system account (000000000000) --- systemAccount := Account{ @@ -844,8 +829,7 @@ func (s *IAMServiceImpl) IsEmpty() (bool, error) { // Account Operations // --------------------------------------------------------------------------- -// CreateAccount creates a new account with a sequentially assigned 12-digit ID. -// Uses a CAS loop on the counter bucket for safe concurrent ID assignment. +// CreateAccount creates a new account with a sequentially assigned 12-digit ID using a CAS loop. func (s *IAMServiceImpl) CreateAccount(name string) (*Account, error) { if name == "" { return nil, errors.New(awserrors.ErrorIAMInvalidInput) @@ -853,10 +837,9 @@ func (s *IAMServiceImpl) CreateAccount(name string) (*Account, error) { var accountID string for { - // Read current counter value entry, err := s.accountCounterBucket.Get("next_id") if errors.Is(err, nats.ErrKeyNotFound) { - // First account ever — start at 1 (000000000000 is the global account) + // First account ever; 000000000000 is the global account. accountID = fmt.Sprintf("%012d", 1) if _, err := s.accountCounterBucket.Create("next_id", []byte("2")); err != nil { if errors.Is(err, nats.ErrKeyExists) { @@ -876,7 +859,6 @@ func (s *IAMServiceImpl) CreateAccount(name string) (*Account, error) { } accountID = fmt.Sprintf("%012d", nextID) - // CAS update: increment counter, fail if revision changed newVal := []byte(strconv.FormatInt(nextID+1, 10)) if _, err := s.accountCounterBucket.Update("next_id", newVal, entry.Revision()); err != nil { if errors.Is(err, nats.ErrKeyExists) { @@ -1095,9 +1077,7 @@ func (s *IAMServiceImpl) ListPolicyVersions(accountID string, input *iam.ListPol return nil, err } - // CreatePolicyVersion is unrouted, so every policy has exactly one immutable - // default version (v1). Real IAM omits the document from list entries — - // callers fetch it via GetPolicyVersion — so Document is left nil here. + // Every policy has exactly one immutable version (v1); Document is omitted per AWS convention. createdAt := parseCreatedAt(policy.CreatedAt) return &iam.ListPolicyVersionsOutput{ Versions: []*iam.PolicyVersion{{ @@ -1121,7 +1101,6 @@ func (s *IAMServiceImpl) ListPolicies(accountID string, input *iam.ListPoliciesI return nil, fmt.Errorf("list policy keys: %w", err) } - // Build attachment counts once for the whole account instead of per-policy. attachCounts, err := s.buildAttachmentCounts(accountID) if err != nil { return nil, fmt.Errorf("build attachment counts: %w", err) @@ -1204,7 +1183,6 @@ func (s *IAMServiceImpl) AttachUserPolicy(accountID string, input *iam.AttachUse policyARN := *input.PolicyArn userKVKey := accountID + "." + userName - // Verify policy exists if _, err := s.getPolicyByARN(accountID, policyARN); err != nil { return nil, err } @@ -1214,8 +1192,7 @@ func (s *IAMServiceImpl) AttachUserPolicy(accountID string, input *iam.AttachUse return nil, err } - // Idempotent — if already attached, succeed silently - if slices.Contains(user.AttachedPolicies, policyARN) { + if slices.Contains(user.AttachedPolicies, policyARN) { // idempotent return &iam.AttachUserPolicyOutput{}, nil } @@ -1308,12 +1285,10 @@ func (s *IAMServiceImpl) GetUserPolicies(accountID, userName string) ([]PolicyDo for _, arn := range user.AttachedPolicies { policy, err := s.getPolicyByARN(accountID, arn) if err != nil { - // Fail closed: if we can't resolve a policy, we can't make a safe - // access decision. Return an error so the caller denies access. + // Fail closed: unresolvable policy means we cannot make a safe access decision. return nil, fmt.Errorf("resolve policy %s: %w", arn, err) } - // policy already validated at creation, so we skip validating again var doc PolicyDocument if err := json.Unmarshal([]byte(policy.PolicyDocument), &doc); err != nil { return nil, fmt.Errorf("parse policy %s: %w", policy.PolicyName, err) @@ -1329,12 +1304,10 @@ func (s *IAMServiceImpl) GetUserPolicies(accountID, userName string) ([]PolicyDo // --------------------------------------------------------------------------- func (s *IAMServiceImpl) getPolicyByARN(accountID, policyARN string) (*Policy, error) { - // Extract policy name from ARN: arn:aws:iam::000000000000:policy/path/PolicyName parts := strings.SplitN(policyARN, ":policy", 2) if len(parts) != 2 || parts[1] == "" { return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) } - // The name is the last segment after the final / segments := strings.Split(parts[1], "/") policyName := segments[len(segments)-1] if policyName == "" { @@ -1355,7 +1328,6 @@ func (s *IAMServiceImpl) getPolicyByARN(accountID, policyARN string) (*Policy, e return nil, fmt.Errorf("unmarshal policy: %w", err) } - // Verify the full ARN matches (path may differ) if policy.ARN != policyARN { return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) } diff --git a/spinifex/handlers/iam/types.go b/spinifex/handlers/iam/types.go index 96ac99177..dda671d00 100644 --- a/spinifex/handlers/iam/types.go +++ b/spinifex/handlers/iam/types.go @@ -72,8 +72,7 @@ type Statement struct { } // Role is an assumable IAM identity stored in JetStream KV. -// The trust policy (AssumeRolePolicyDocument) is stored opaque in v1; -// it is parsed only for shape validation and is not evaluated until STS lands. +// AssumeRolePolicyDocument is stored opaque; parsed for shape validation only. type Role struct { RoleName string `json:"role_name"` RoleID string `json:"role_id"` @@ -110,13 +109,8 @@ type TrustPolicyDocument struct { } // TrustStatement is a single statement within a trust policy document. -// -// NotPrincipal and NotAction are accepted by the JSON unmarshal so the -// validator can see — and reject — them. Without these fields a policy -// containing `{"Effect":"Allow","NotPrincipal":{...}}` would unmarshal -// with NotPrincipal silently dropped, leaving the validator unable to -// distinguish it from a (still-empty) Principal and effectively turning -// every NotPrincipal-Allow into a universe-allow at runtime. +// NotPrincipal and NotAction are present so the validator can detect and reject them; +// without them a policy with NotPrincipal would silently unmarshal as an empty Principal. type TrustStatement struct { Sid string `json:"Sid,omitempty"` Effect string `json:"Effect"` @@ -132,14 +126,12 @@ type StringOrArr []string // UnmarshalJSON implements custom unmarshaling for string-or-array fields. func (s *StringOrArr) UnmarshalJSON(data []byte) error { - // Try string first var single string if err := json.Unmarshal(data, &single); err == nil { *s = []string{single} return nil } - // Try array var arr []string if err := json.Unmarshal(data, &arr); err != nil { return err diff --git a/spinifex/handlers/imds/buckets.go b/spinifex/handlers/imds/buckets.go index 6e2b68fad..d04762fa1 100644 --- a/spinifex/handlers/imds/buckets.go +++ b/spinifex/handlers/imds/buckets.go @@ -8,22 +8,16 @@ import ( ) const ( - // KVBucketIMDSSubnetVeth is the canonical record that a subnet has an IMDS - // localport installed. Keyed by subnetID, the value carries the localport - // spec the BindManager replays to materialise host-side state on every chassis. + // KVBucketIMDSSubnetVeth records per-subnet IMDS localport state replayed on each chassis. KVBucketIMDSSubnetVeth = "spinifex-network-imds-subnet-veth" KVBucketIMDSSubnetVethVersion = 1 - // KVBucketENIByVPCIP is a pure reverse index (vpcID/ip → eniID) so the IMDS - // handler can resolve a request's datapath-attested source IP to an ENI in - // one KV read instead of an O(N) scan of the ENI bucket. + // KVBucketENIByVPCIP is a reverse index (vpcID/ip → eniID) for O(1) source-IP resolution. KVBucketENIByVPCIP = "spinifex-network-eni-by-vpc-ip" KVBucketENIByVPCIPVersion = 1 ) -// InitBuckets opens (or creates) both IMDS KV buckets and runs any pending -// migrations. History is fixed at 1: both are write-once-per-key, so retained -// revisions only waste storage and lengthen tombstone lifetime. +// InitBuckets opens or creates both IMDS KV buckets and runs pending migrations. func InitBuckets(js nats.JetStreamContext, replicas int) (subnetVeth, eniByIP nats.KeyValue, err error) { subnetVeth, err = initIMDSBucket(js, KVBucketIMDSSubnetVeth, KVBucketIMDSSubnetVethVersion, replicas) if err != nil { @@ -38,15 +32,12 @@ func InitBuckets(js nats.JetStreamContext, replicas int) (subnetVeth, eniByIP na return subnetVeth, eniByIP, nil } -// InitENIByIPBucket opens (or creates) just the eni-by-vpc-ip reverse-index -// bucket, for the ENI controller (in the daemon), which may start before the -// IMDS service and so needs an init path independent of InitBuckets. Idempotent. +// InitENIByIPBucket opens or creates the eni-by-vpc-ip bucket independently of InitBuckets. func InitENIByIPBucket(js nats.JetStreamContext, replicas int) (nats.KeyValue, error) { return initIMDSBucket(js, KVBucketENIByVPCIP, KVBucketENIByVPCIPVersion, replicas) } -// initIMDSBucket opens (or creates) a single IMDS KV bucket and runs any -// pending migrations. +// initIMDSBucket opens or creates a single IMDS KV bucket and runs pending migrations. func initIMDSBucket(js nats.JetStreamContext, bucket string, version, replicas int) (nats.KeyValue, error) { if replicas < 1 { replicas = 1 diff --git a/spinifex/handlers/imds/constants.go b/spinifex/handlers/imds/constants.go index d5e0e7ea9..e7abc46bc 100644 --- a/spinifex/handlers/imds/constants.go +++ b/spinifex/handlers/imds/constants.go @@ -1,6 +1,4 @@ package handlers_imds -// MetaDataServerIP is the link-local address the EC2 Instance Metadata Service is -// served on, identical to AWS. Exported so the network topology layer can claim it -// on the subnet-switch localport that answers IMDS over L2 for every guest. +// MetaDataServerIP is the standard EC2 link-local metadata address. const MetaDataServerIP = "169.254.169.254" diff --git a/spinifex/handlers/imds/credentials.go b/spinifex/handlers/imds/credentials.go index 1a57e01bf..eb3e5d011 100644 --- a/spinifex/handlers/imds/credentials.go +++ b/spinifex/handlers/imds/credentials.go @@ -11,18 +11,14 @@ import ( ) const ( - // credDurationSeconds is the lifetime of instance-role credentials. 1 hour - // is the AWS default and matches the storage-backed STS v1 design. + // credDurationSeconds is the lifetime of instance-role credentials (AWS default: 1 hour). credDurationSeconds = 3600 - // credRefreshWindow is how far before expiry a cached credential is treated - // as stale and re-minted. AWS SDKs refresh at the same 5-minute mark. + // credRefreshWindow is how early a cached credential is re-minted before expiry. credRefreshWindow = 5 * time.Minute ) -// instanceCredential is the JSON body served at -// /latest/meta-data/iam/security-credentials/, byte-for-byte matching the -// AWS shape so unmodified SDK credential providers parse it. +// instanceCredential is the JSON body served at /latest/meta-data/iam/security-credentials/. type instanceCredential struct { Code string `json:"Code"` LastUpdated string `json:"LastUpdated"` @@ -34,23 +30,18 @@ type instanceCredential struct { AccountId string `json:"AccountId"` } -// stsAssumer is the narrow slice of STSService the IMDS credential path needs: -// the in-process, EC2-service-principal AssumeRole entry point. Narrowed to an -// interface so the cred cache is unit-testable with a fake. +// stsAssumer is the narrow STSService slice needed by the IMDS credential path. type stsAssumer interface { AssumeRoleForInstance(accountID, roleARN, instanceID string, durationSeconds int64) (*sts.AssumeRoleOutput, error) } -// cachedCred holds a rendered credential body and the instant at which it should -// be re-minted (expiry minus the refresh window). +// cachedCred holds a rendered credential body and the time it should be re-minted. type cachedCred struct { body []byte refreshAt time.Time } -// credCache memoises minted instance-role credentials per (ENI, role) until the -// refresh window. It keeps STS off the hot path: the typical request returns a -// cached body, and only a near-expiry request triggers a fresh assume. +// credCache memoises instance-role credentials per (ENI, role) until the refresh window. type credCache struct { sts stsAssumer mu sync.Mutex @@ -61,9 +52,7 @@ func newCredCache(assumer stsAssumer) *credCache { return &credCache{sts: assumer, entries: make(map[string]cachedCred)} } -// get returns the credential JSON body for an instance's role, minting fresh -// credentials via STS when the cache is cold or within the refresh window. The -// cache key is (eniID, roleName) so two ENIs sharing a role don't collide. +// get returns the credential JSON body, minting via STS when the cache is cold or near expiry. func (c *credCache) get(eni *eniFacts, roleName, roleARN string, now time.Time) ([]byte, error) { key := eni.eniID + "\x00" + roleName @@ -105,10 +94,7 @@ func (c *credCache) get(eni *eniFacts, roleName, roleARN string, now time.Time) return body, nil } -// sweep drops fully-expired entries (past refreshAt + the refresh window, i.e. -// past the credential's actual expiry), bounding the cache against ENI churn: -// a terminated instance's entry is never overwritten and would otherwise leak. -// Evicting a live entry would only force a re-mint, so this is always safe. +// sweep removes fully-expired entries to bound the cache against ENI churn. func (c *credCache) sweep(now time.Time) { c.mu.Lock() defer c.mu.Unlock() diff --git a/spinifex/handlers/imds/instance_lookup.go b/spinifex/handlers/imds/instance_lookup.go index c00364160..fbeb2db51 100644 --- a/spinifex/handlers/imds/instance_lookup.go +++ b/spinifex/handlers/imds/instance_lookup.go @@ -11,9 +11,7 @@ import ( "github.com/nats-io/nats.go" ) -// natsInstanceLookup resolves the instance-only metadata fields via the same -// account-scoped NATS fan-out the gateway's DescribeInstances path uses, since -// the instance record lives in each daemon's in-memory manager, not a central KV. +// natsInstanceLookup resolves instance-only metadata fields via DescribeInstances fan-out. type natsInstanceLookup struct { nc *nats.Conn expectedNodes int @@ -31,9 +29,7 @@ func (l *natsInstanceLookup) describe(accountID, instanceID string) (*instanceFa inst := firstInstance(out) if inst == nil { - // The ENI references an instance the daemons no longer report (mid - // terminate, or not yet visible). Treated as a miss → 404. - return nil, nil + return nil, nil // terminating or not yet visible; treat as miss } facts := &instanceFacts{ @@ -49,9 +45,7 @@ func (l *natsInstanceLookup) describe(accountID, instanceID string) (*instanceFa return facts, nil } -// userData fetches the base64 user-data blob via DescribeInstanceAttribute and -// decodes it. A miss or decode failure yields nil — /latest/user-data then 404s, -// matching AWS for instances launched without user-data. +// userData fetches and decodes the instance's base64 user-data, returning nil on miss or error. func (l *natsInstanceLookup) userData(accountID, instanceID string) []byte { attr, err := gateway_ec2_instance.DescribeInstanceAttribute(&ec2.DescribeInstanceAttributeInput{ InstanceId: aws.String(instanceID), diff --git a/spinifex/handlers/imds/metadata.go b/spinifex/handlers/imds/metadata.go index 00872e3b7..e7937649f 100644 --- a/spinifex/handlers/imds/metadata.go +++ b/spinifex/handlers/imds/metadata.go @@ -16,8 +16,6 @@ import ( "github.com/mulgadc/spinifex/spinifex/utils" ) -// HTTP paths. IMDSv2 token issuance is a PUT; everything else is a token-gated -// GET under /latest/. const ( pathToken = "/latest/api/token" //nolint:gosec // URL path, not a credential prefixMetaData = "/latest/meta-data/" @@ -34,9 +32,7 @@ const ( hdrForwardedFor = "X-Forwarded-For" ) -// rejectForwarded enforces AWS IMDS's SSRF defence: any request carrying an -// X-Forwarded-For header is refused with 403 before token or identity checks, -// blocking the classic "trick a server-side app into relaying metadata" attack. +// rejectForwarded enforces the IMDS SSRF defence: requests with X-Forwarded-For are 403'd. func rejectForwarded(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get(hdrForwardedFor) != "" { @@ -47,9 +43,7 @@ func rejectForwarded(next http.Handler) http.Handler { }) } -// handleToken serves PUT /latest/api/token. It binds a fresh token to the -// requesting ENI (resolved from the datapath-attested source IP) and returns it -// as a text/plain body, matching AWS IMDSv2. +// handleToken serves PUT /latest/api/token, issuing a fresh ENI-bound IMDSv2 token. func (s *IMDSServiceImpl) handleToken(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPut { w.Header().Set("Allow", http.MethodPut) @@ -81,8 +75,7 @@ func (s *IMDSServiceImpl) handleToken(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(token)) } -// handleMetadata serves every token-gated read path. It resolves the caller's -// ENI, enforces the IMDSv2 token, then dispatches on the request path. +// handleMetadata serves token-gated GET requests, resolving the ENI and dispatching. func (s *IMDSServiceImpl) handleMetadata(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { w.Header().Set("Allow", http.MethodGet) @@ -96,9 +89,7 @@ func (s *IMDSServiceImpl) handleMetadata(w http.ResponseWriter, r *http.Request) return } - // IMDSv2: a valid, ENI-bound token is mandatory on every read. A missing, - // unknown, expired, or wrong-ENI token is an indistinguishable 401 so a - // guest cannot probe which tokens exist. + // IMDSv2: valid ENI-bound token required; any failure is 401 to prevent probing. if !s.tokens.validate(r.Header.Get(hdrToken), eni.eniID, s.now()) { w.WriteHeader(http.StatusUnauthorized) return @@ -107,11 +98,8 @@ func (s *IMDSServiceImpl) handleMetadata(w http.ResponseWriter, r *http.Request) s.dispatch(w, r, eni) } -// resolveCaller maps the request's (vpcID-from-context, source-IP) to the owning -// ENI, or nil when no mapping exists (logged for operator triage). The listener -// identifies the subnet; its statically-resolved VPC is what keys the -// eni-by-vpc-ip index. A backend error is also surfaced as nil → the caller -// 404s, never 500s, matching AWS's opaque "eventually available" boot posture. +// resolveCaller maps (vpcID-from-context, source-IP) to the owning ENI. +// Returns nil on miss or backend error, producing a 404 instead of 500. func (s *IMDSServiceImpl) resolveCaller(r *http.Request) *eniFacts { vpcID, _ := r.Context().Value(ctxKeyVPCID).(string) subnetID, _ := r.Context().Value(ctxKeySubnetID).(string) @@ -137,16 +125,11 @@ func (s *IMDSServiceImpl) resolveCaller(r *http.Request) *eniFacts { func (s *IMDSServiceImpl) dispatch(w http.ResponseWriter, r *http.Request, eni *eniFacts) { path := r.URL.Path - // Credential fetch for a specific role: /iam/security-credentials/. if strings.HasPrefix(path, prefixSecurityCreds) && len(path) > len(prefixSecurityCreds) { s.serveRoleCredentials(w, eni, strings.TrimPrefix(path, prefixSecurityCreds)) return } - // SSH public keys: /public-keys/, /public-keys/0/, /public-keys/0/openssh-key. - // The two directory listings resolve from the instance's launch key name; only - // the material leaf fetches it. (The bare /public-keys, sans trailing slash, is - // handled in the switch below.) if sub, ok := strings.CutPrefix(path, prefixPublicKeys); ok { s.servePublicKeys(w, eni, sub) return @@ -188,8 +171,6 @@ func (s *IMDSServiceImpl) dispatch(w http.ResponseWriter, r *http.Request, eni * case pathUserData: s.serveUserData(w, eni) default: - // /latest/dynamic/instance-identity/document and - // /latest/meta-data/network/interfaces/... are out of scope in v1. w.WriteHeader(http.StatusNotFound) } } @@ -204,8 +185,7 @@ func (s *IMDSServiceImpl) serveInstanceField(w http.ResponseWriter, eni *eniFact writeText(w, field(inst)) } -// serveUserData writes the instance's user-data, or 404 when there is none — -// matching AWS for instances launched without user-data. +// serveUserData writes the instance's user-data, or 404 if absent. func (s *IMDSServiceImpl) serveUserData(w http.ResponseWriter, eni *eniFacts) { inst := s.instanceFor(w, eni) if inst == nil { @@ -219,8 +199,7 @@ func (s *IMDSServiceImpl) serveUserData(w http.ResponseWriter, eni *eniFacts) { _, _ = w.Write(inst.userData) } -// serveIAMInfo writes {InstanceProfileArn, InstanceProfileId}, 404 when the -// instance has no instance profile, or 500 when the profile can't be resolved. +// serveIAMInfo writes the instance profile ARN and ID, or 404 if none is attached. func (s *IMDSServiceImpl) serveIAMInfo(w http.ResponseWriter, eni *eniFacts) { profile, err := s.profileFor(eni) if err != nil { @@ -237,9 +216,8 @@ func (s *IMDSServiceImpl) serveIAMInfo(w http.ResponseWriter, eni *eniFacts) { }) } -// serveSecurityCredentialsList writes the role name(s) under the profile, one -// per line. No profile/role is an empty 200 (matching AWS); a backend failure is -// a 500, never an empty 200 that would silently strip the instance's creds. +// serveSecurityCredentialsList writes the role name(s) under the profile. +// No profile/role returns an empty 200; a backend failure returns 500. func (s *IMDSServiceImpl) serveSecurityCredentialsList(w http.ResponseWriter, eni *eniFacts) { profile, err := s.profileFor(eni) if err != nil { @@ -253,9 +231,8 @@ func (s *IMDSServiceImpl) serveSecurityCredentialsList(w http.ResponseWriter, en writeText(w, profile.RoleName) } -// serveRoleCredentials mints (or serves cached) credentials for the named role. -// AWS only accepts the actual role name as the path parameter, not the profile -// name, so a mismatch is 404; a backend failure resolving the profile is 500. +// serveRoleCredentials mints or returns cached credentials for the named role. +// A name mismatch is 404; a backend failure resolving the profile is 500. func (s *IMDSServiceImpl) serveRoleCredentials(w http.ResponseWriter, eni *eniFacts, roleParam string) { profile, err := s.profileFor(eni) if err != nil { @@ -276,9 +253,7 @@ func (s *IMDSServiceImpl) serveRoleCredentials(w http.ResponseWriter, eni *eniFa body, err := s.creds.get(eni, profile.RoleName, *role.Role.Arn, s.now()) if err != nil { - // AWS surfaces a trust-policy failure as a JSON body with Code:"Failed"; - // the SDK reports it as "EC2RoleProvider failed". Mirror that rather than - // a bare HTTP error so the SDK error message is AWS-accurate. + // Mirror the AWS Code:"Failed" JSON body so SDKs report a recognisable error. slog.Warn("IMDS: AssumeRoleForInstance failed", "account_id", eni.accountID, "role", profile.RoleName, "instance_id", eni.instanceID, "err", err) writeJSON(w, map[string]string{ "Code": "Failed", @@ -292,20 +267,8 @@ func (s *IMDSServiceImpl) serveRoleCredentials(w http.ResponseWriter, eni *eniFa _, _ = w.Write(body) } -// servePublicKeys serves the /public-keys subtree, where sub is the path after -// "/public-keys/" ("" for the bare directory). Spinifex binds one key pair per -// instance, so the index is always 0: -// -// "" → "0=" (key list) -// "0", "0/" → "openssh-key" (formats list for index 0) -// "0/openssh-key" → the SSH public key material -// -// The two listings resolve from the instance's launch key name with no material -// RPC; only the leaf fetches it. A no-key instance, or any index other than 0, -// is 404 — matching AWS. On the material path, a deleted key is 404 (definitive -// absence) but any other backend fault is 500: a 404 tells cloud-init the -// instance has no key and it boots keyless without retry, so a transient fault -// must never collapse to 404 (mirrors serveIAMInfo/serveRoleCredentials). +// servePublicKeys serves the /public-keys subtree (index always 0). +// A deleted key is 404; any other backend fault is 500 to avoid cloud-init booting keyless. func (s *IMDSServiceImpl) servePublicKeys(w http.ResponseWriter, eni *eniFacts, sub string) { inst := s.instanceFor(w, eni) if inst == nil { @@ -354,9 +317,8 @@ func (s *IMDSServiceImpl) instanceFor(w http.ResponseWriter, eni *eniFacts) *ins return inst } -// profileFor resolves the instance's IAM instance profile. Returns (profile, nil) -// when resolvable, (nil, nil) when genuinely absent (404/empty), (nil, err) on -// backend failure (500) — collapsing err into absent would silently drop creds. +// profileFor resolves the IAM instance profile for an ENI. +// Returns (nil, nil) when absent, (nil, err) on backend failure — never collapses errors to absent. func (s *IMDSServiceImpl) profileFor(eni *eniFacts) (*resolvedProfile, error) { inst, err := s.resolver.resolveInstance(eni) if err != nil { @@ -369,10 +331,7 @@ func (s *IMDSServiceImpl) profileFor(eni *eniFacts) (*resolvedProfile, error) { profile, err := s.iam.ResolveInstanceProfile(eni.accountID, inst.iamInstanceProfileArn) if err != nil { if err.Error() == awserrors.ErrorIAMNoSuchEntity { - // The profile was deleted out from under the instance. AWS treats this - // as "no instance role", not a backend fault — NoSuchEntity is raised - // only for a genuinely absent record, never a transient. - return nil, nil + return nil, nil // profile deleted; treat as no role } slog.Error("IMDS: resolve instance profile failed", "account_id", eni.accountID, "arn", inst.iamInstanceProfileArn, "err", err) return nil, err @@ -383,9 +342,7 @@ func (s *IMDSServiceImpl) profileFor(eni *eniFacts) (*resolvedProfile, error) { return &resolvedProfile{ARN: profile.ARN, InstanceProfileID: profile.InstanceProfileID, RoleName: profile.RoleName}, nil } -// resolvedProfile is the slice of an IAM instance profile the metadata surface -// serves, decoupled from the handlers_iam type so this package needn't import -// it for the field access. +// resolvedProfile is the slice of an IAM instance profile served by the metadata surface. type resolvedProfile struct { ARN string InstanceProfileID string @@ -404,8 +361,7 @@ func parseTokenTTL(raw string) (time.Duration, bool) { return time.Duration(n) * time.Second, true } -// regionFromAZ derives the region from an availability-zone name by stripping -// the trailing AZ letter ("ap-southeast-2a" → "ap-southeast-2"). +// regionFromAZ strips the trailing AZ letter from an AZ name to get the region. func regionFromAZ(az string) string { if az == "" { return "" @@ -417,8 +373,7 @@ func regionFromAZ(az string) string { return az } -// synthHostname builds the AWS-style internal hostname -// "ip-10-0-1-5..compute.internal" from a private IP and region. +// synthHostname builds "ip-10-0-1-5..compute.internal" from a private IP and region. func synthHostname(ip, region string) string { if ip == "" || region == "" { return "" @@ -441,9 +396,7 @@ func writeJSON(w http.ResponseWriter, v any) { _, _ = w.Write(data) } -// ctxKey is the unexported context-key type for values the bind manager threads -// into each request: the subnet the listener serves and that subnet's -// statically-resolved owning VPC (which keys the eni-by-vpc-ip index). +// ctxKey is the unexported context-key type used to thread subnet and VPC into each request. type ctxKey int const ( diff --git a/spinifex/handlers/imds/nats_clients.go b/spinifex/handlers/imds/nats_clients.go index 0788f8924..a0b7c6b9f 100644 --- a/spinifex/handlers/imds/nats_clients.go +++ b/spinifex/handlers/imds/nats_clients.go @@ -18,18 +18,14 @@ const ( // imdsRPCTimeout bounds each internal control-plane round-trip to awsgw. imdsRPCTimeout = 10 * time.Second - // profileCacheTTL memoises profile/role lookups. Both mappings are effectively - // static, so a short TTL keeps the iam/* GETs off a NATS round-trip per request. + // profileCacheTTL memoises profile/role lookups to avoid per-request NATS round-trips. profileCacheTTL = 5 * time.Minute - // pubKeyCacheTTL memoises SSH public-key material, immutable for a key's - // lifetime; separate from profileCacheTTL so the two are tunable independently. + // pubKeyCacheTTL memoises SSH public-key material (immutable for a key's lifetime). pubKeyCacheTTL = 5 * time.Minute ) -// NATSSTSAssumer is the NATS-backed stsAssumer. It mints instance-role credentials -// via awsgw's SubjectAssumeRoleForInstance responder, keeping no local cache — -// credCache already memoises minted credentials, so STS stays off the hot path. +// NATSSTSAssumer is the NATS-backed stsAssumer for minting instance-role credentials. type NATSSTSAssumer struct { nc *nats.Conn } @@ -50,9 +46,7 @@ func (a *NATSSTSAssumer) AssumeRoleForInstance(accountID, roleARN, instanceID st }, imdsRPCTimeout, accountID) } -// NATSProfileLookup is the NATS-backed profileLookup. It resolves instance -// profiles and roles via awsgw's IAM responders, fronted by a short-TTL cache -// so repeat iam/* GETs don't take a round-trip each. +// NATSProfileLookup is the NATS-backed profileLookup with a short-TTL cache. type NATSProfileLookup struct { nc *nats.Conn profiles *ttlCache[*handlers_iam.InstanceProfile] @@ -102,10 +96,7 @@ func (p *NATSProfileLookup) GetRole(accountID string, input *iam.GetRoleInput) ( return out, nil } -// NATSPublicKeyLookup is the NATS-backed publicKeyLookup. It fetches an -// instance's launch SSH public key from the daemon-side key service via the -// imds.ec2.get_public_key responder, fronted by a success-only TTL cache so -// repeat material fetches don't take a round-trip each. +// NATSPublicKeyLookup is the NATS-backed publicKeyLookup with a success-only TTL cache. type NATSPublicKeyLookup struct { nc *nats.Conn cache *ttlCache[string] @@ -131,16 +122,13 @@ func (p *NATSPublicKeyLookup) GetPublicKey(accountID, keyName string) (string, e KeyName: keyName, }, imdsRPCTimeout, accountID) if err != nil { - // Errors are never cached — a transient miss must not pin a key as absent. - return "", err + return "", err // errors not cached: transient miss must not pin key as absent } p.cache.put(key, out.OpenSSHKey) return out.OpenSSHKey, nil } -// ttlCache is a minimal concurrency-safe map with per-entry expiry, used to -// keep effectively-static IAM lookups off the NATS round-trip. now is a field -// so tests can drive expiry deterministically. +// ttlCache is a concurrency-safe map with per-entry expiry; now is injectable for tests. type ttlCache[V any] struct { mu sync.Mutex ttl time.Duration diff --git a/spinifex/handlers/imds/resolver.go b/spinifex/handlers/imds/resolver.go index 93dc16d16..23919bbdd 100644 --- a/spinifex/handlers/imds/resolver.go +++ b/spinifex/handlers/imds/resolver.go @@ -9,20 +9,11 @@ import ( "github.com/nats-io/nats.go" ) -// kvBucketENIs is the ENI source-of-truth bucket (handlers_ec2_vpc.KVBucketENIs), -// duplicated as a literal rather than imported because importing handlers/ec2/vpc -// back would close an import cycle (the veth_store record type does likewise). +// kvBucketENIs and kvBucketSecurityGroups are duplicated as literals to avoid an import cycle. const kvBucketENIs = "spinifex-vpc-enis" - -// kvBucketSecurityGroups is the security-group source-of-truth bucket -// (handlers_ec2_vpc.KVBucketSecurityGroups). Duplicated as a literal for the -// same import-cycle reason as kvBucketENIs. const kvBucketSecurityGroups = "spinifex-vpc-security-groups" -// eniFacts is the subset of an ENIRecord the IMDS metadata surface serves -// directly, plus the account ID recovered from the ENI bucket key. Everything -// here is read live off the ENI record, so a change (e.g. a re-associated EIP) -// is reflected on the next request. +// eniFacts is the ENI fields served by the IMDS metadata surface, read live from the ENI record. type eniFacts struct { eniID string accountID string @@ -36,9 +27,7 @@ type eniFacts struct { securityGroupIDs []string } -// instanceFacts carries the four metadata fields not present on the ENI record. -// They live in the daemon's in-memory instance manager, so they are fetched via -// the account-scoped DescribeInstances fan-out behind instanceLookup. +// instanceFacts carries metadata fields not present on the ENI record, fetched via instanceLookup. type instanceFacts struct { instanceType string imageID string @@ -47,32 +36,23 @@ type instanceFacts struct { userData []byte } -// instanceLookup resolves the instance-only metadata fields by instance ID. It -// is an interface so the metadata handlers stay unit-testable without a live -// NATS cluster; the production implementation (natsInstanceLookup) fans out -// DescribeInstances + DescribeInstanceAttribute. +// instanceLookup resolves instance-only metadata fields by instance ID. type instanceLookup interface { describe(accountID, instanceID string) (*instanceFacts, error) } -// eniIndexValue mirrors the stored shape of a spinifex-network-eni-by-vpc-ip -// row (handlers_ec2_vpc.eniByIPValue). Duplicated for the same cycle reason as -// kvBucketENIs. Both fields are the ENI's immutable identity; everything mutable -// is read live off the ENI record. +// eniIndexValue mirrors the eni-by-vpc-ip row shape, duplicated to avoid an import cycle. type eniIndexValue struct { ENIId string `json:"eni_id"` AccountID string `json:"account_id"` } -// sgNameRecord is the slice of handlers_ec2_vpc.SecurityGroupRecord the metadata -// surface needs: the human-readable group name AWS serves at -// /latest/meta-data/security-groups (not the sg-* ID). +// sgNameRecord holds the human-readable group name for the /security-groups path. type sgNameRecord struct { GroupName string `json:"group_name"` } -// eniRecord is the subset of handlers_ec2_vpc.ENIRecord the resolver reads. The -// full record is owned by the ENI controller; only these fields feed IMDS. +// eniRecord is the ENI record subset the resolver reads from the ENI bucket. type eniRecord struct { NetworkInterfaceId string `json:"network_interface_id"` SubnetId string `json:"subnet_id"` @@ -85,15 +65,8 @@ type eniRecord struct { SecurityGroupIds []string `json:"security_group_ids,omitempty"` } -// metadataResolver maps a datapath-attested (vpcID, srcIP) to the ENI + instance -// facts the metadata surface serves. The chain is: -// -// (vpcID, ip) → {eniID, accountID} via the eni-by-vpc-ip reverse index -// (account, eniID) → ENIRecord via a direct KV Get -// (account, instanceID) → facts via the account-scoped DescribeInstances fan-out -// -// The index carries the ENI's immutable identity so the account resolves in one -// Get; every mutable field is read live off the record, so there's no staleness. +// metadataResolver maps a datapath-attested (vpcID, srcIP) to ENI + instance facts. +// Resolution chain: (vpcID,ip)→eniID via reverse index → ENIRecord → instanceFacts. type metadataResolver struct { index nats.KeyValue // spinifex-network-eni-by-vpc-ip eniKV nats.KeyValue // spinifex-vpc-enis @@ -103,10 +76,7 @@ type metadataResolver struct { var _ eniResolver = (*metadataResolver)(nil) -// resolveENI returns the ENI facts for a request's (vpcID, srcIP), or (nil, nil) -// when no mapping exists — the caller maps a miss to 404, matching AWS's -// "eventually available during boot" posture. A non-nil error is reserved for -// genuine backend failures. +// resolveENI returns ENI facts for (vpcID, srcIP), or (nil, nil) on miss. func (r *metadataResolver) resolveENI(vpcID, srcIP string) (*eniFacts, error) { entry, err := r.index.Get(vpcID + "/" + srcIP) if err != nil { @@ -151,8 +121,7 @@ func (r *metadataResolver) resolveENI(vpcID, srcIP string) (*eniFacts, error) { }, nil } -// resolveInstance fetches the instance-only metadata fields for an ENI's -// attached instance. Returns (nil, nil) when the ENI has no attached instance. +// resolveInstance fetches instance-only fields for an ENI's attached instance, or (nil, nil) if unattached. func (r *metadataResolver) resolveInstance(eni *eniFacts) (*instanceFacts, error) { if eni.instanceID == "" { return nil, nil @@ -160,9 +129,7 @@ func (r *metadataResolver) resolveInstance(eni *eniFacts) (*instanceFacts, error return r.lookup.describe(eni.accountID, eni.instanceID) } -// resolveSGNames maps security-group IDs to the group names AWS serves at -// /latest/meta-data/security-groups. Best-effort and order-preserving: a missing -// record or nil sgKV falls back to the raw ID rather than failing the request. +// resolveSGNames maps SG IDs to group names for /security-groups. Best-effort; falls back to IDs. func (r *metadataResolver) resolveSGNames(accountID string, sgIDs []string) []string { names := make([]string, len(sgIDs)) for i, id := range sgIDs { diff --git a/spinifex/handlers/imds/server.go b/spinifex/handlers/imds/server.go index edc2bf9e5..c2d698a03 100644 --- a/spinifex/handlers/imds/server.go +++ b/spinifex/handlers/imds/server.go @@ -19,18 +19,13 @@ import ( "golang.org/x/sys/unix" ) -// ensureVethFunc / removeVethFunc are the per-subnet host veth lifecycle hooks, -// injected (not imported) to avoid an import cycle via network/host. vpcd passes -// host.EnsureIMDSVeth / host.RemoveIMDSVeth; tests pass root-free fakes. ensure -// takes the subnet CIDR (added on-link in the netns so the reply resolves the -// guest by ARP) and returns the per-subnet netns the listener must be created in -// plus the host-end veth name (now living inside that netns). +// ensureVethFunc is the per-subnet host veth lifecycle hook injected to avoid an import cycle. +// Returns the per-subnet netns name and the host-end veth name. type ensureVethFunc func(ctx context.Context, subnetID string, cidr netip.Prefix) (netnsName, hostEnd string, err error) type removeVethFunc func(ctx context.Context, subnetID string) error -// listenFunc binds a TCP listener on 169.254.169.254:80 inside the subnet's netns, -// scoped to its host veth. Swappable in tests so bind-manager logic runs without -// CAP_NET_ADMIN or a real netns. An empty netnsName binds in the current netns. +// listenFunc binds a TCP listener on 169.254.169.254:80 inside the subnet's netns. +// An empty netnsName binds in the current netns; swapped in tests to avoid CAP_NET_ADMIN. type listenFunc func(ctx context.Context, netnsName, hostEnd string) (net.Listener, error) // activeBinding is the per-subnet realised state: a listener bound to the subnet's @@ -40,9 +35,8 @@ type activeBinding struct { server *http.Server } -// bindManager realises the per-chassis IMDS listener stack. It runs on every -// chassis and eagerly materialises a veth + OVS port + listener for every subnet in -// the subnet-veth bucket — lifecycle is "watch the bucket, ensure local state". +// bindManager realises the per-chassis IMDS listener stack, materialising a listener +// per subnet by watching the subnet-veth bucket. type bindManager struct { kv nats.KeyValue // spinifex-network-imds-subnet-veth store *VethStore // reads the subnet CIDR the host on-link route needs @@ -67,8 +61,8 @@ func newBindManager(kv nats.KeyValue, handler http.Handler, ensure ensureVethFun } } -// sync reads the full subnet-veth bucket and binds a listener for every entry. Run -// once at startup; idempotent, so a re-sync after a transient error is safe. +// sync reads the subnet-veth bucket and binds a listener for every entry. +// Idempotent; safe to re-call after a transient error. func (b *bindManager) sync(ctx context.Context) error { keys, err := b.kv.Keys() if err != nil { @@ -79,7 +73,7 @@ func (b *bindManager) sync(ctx context.Context) error { } for _, subnetID := range keys { if subnetID == utils.VersionKey { - continue // schema-version marker written by migrate.RunKV, not a subnet + continue // schema-version marker, not a subnet } vpcID, cidr, err := b.subnetBinding(subnetID) if err != nil { @@ -93,12 +87,8 @@ func (b *bindManager) sync(ctx context.Context) error { return nil } -// subnetBinding reads the persisted veth record for subnetID and returns the two -// facts bind needs: the owning VPC (the subnet→VPC static lookup — the listener -// identifies the subnet, the eni-by-vpc-ip index is keyed vpcID/ip) and the -// subnet CIDR (added on-link in the netns so the reply resolves the guest by -// ARP). A record missing either field is malformed; bind cannot proceed without -// them, and the reconciler re-publishes a complete record to converge. +// subnetBinding returns the owning VPC ID and subnet CIDR for the given subnetID. +// Returns an error if the veth record is missing or incomplete. func (b *bindManager) subnetBinding(subnetID string) (vpcID string, cidr netip.Prefix, err error) { rec, err := b.store.Get(subnetID) if err != nil { @@ -117,9 +107,7 @@ func (b *bindManager) subnetBinding(subnetID string) (vpcID string, cidr netip.P return rec.VPCID, cidr, nil } -// watch reacts to subnet-veth bucket changes for the life of ctx: a Put binds the -// subnet (if not already bound), a Delete unbinds it and tears down the veth. Both -// transitions are idempotent. +// watch reacts to subnet-veth bucket changes: Put binds, Delete/Purge unbinds. Idempotent. func (b *bindManager) watch(ctx context.Context) { watcher, err := b.kv.WatchAll(nats.Context(ctx)) if err != nil { @@ -137,12 +125,10 @@ func (b *bindManager) watch(ctx context.Context) { return } if entry == nil { - // nil marks the end of the initial replay; sync() already - // covered those, so there is nothing extra to do. - continue + continue // end of initial replay; sync() already handled those } if entry.Key() == utils.VersionKey { - continue // schema-version marker written by migrate.RunKV, not a subnet + continue // schema-version marker, not a subnet } switch entry.Operation() { case nats.KeyValuePut: @@ -161,10 +147,8 @@ func (b *bindManager) watch(ctx context.Context) { } } -// bind ensures the subnet's host veth (with the subnet CIDR on-link for the reply -// path) then opens a device-scoped listener for subnetID. vpcID is the subnet's -// statically-resolved owning VPC, threaded into every request so the handler can -// key the eni-by-vpc-ip index. A no-op if already bound. +// bind ensures the subnet's host veth, then opens a device-scoped listener. +// vpcID is threaded into every request for ENI resolution. No-op if already bound. func (b *bindManager) bind(ctx context.Context, subnetID, vpcID string, cidr netip.Prefix) error { b.mu.Lock() if _, ok := b.active[subnetID]; ok { @@ -186,9 +170,6 @@ func (b *bindManager) bind(ctx context.Context, subnetID, vpcID string, cidr net server := &http.Server{ Handler: b.handler, ReadHeaderTimeout: 10 * time.Second, - // Tag every request from this listener with the subnet it serves and that - // subnet's owning VPC. The handler pairs the VPC with the datapath-attested - // source IP to resolve the ENI; the subnet ID rides along for triage. BaseContext: func(net.Listener) context.Context { c := context.WithValue(ctx, ctxKeyVPCID, vpcID) return context.WithValue(c, ctxKeySubnetID, subnetID) @@ -196,8 +177,7 @@ func (b *bindManager) bind(ctx context.Context, subnetID, vpcID string, cidr net } b.mu.Lock() - // Re-check under lock in case a concurrent bind won the race. - if _, ok := b.active[subnetID]; ok { + if _, ok := b.active[subnetID]; ok { // re-check under lock b.mu.Unlock() _ = listener.Close() return nil @@ -233,10 +213,7 @@ func (b *bindManager) unbind(ctx context.Context, subnetID string) { slog.Info("IMDS: listener unbound", "subnet_id", subnetID) } -// shutdown closes every active listener. The host veths are intentionally left -// in place: they are cheap, idempotent to re-ensure on the next start, and -// tearing them down on a clean vpcd restart would needlessly disrupt any other -// chassis-local consumer. +// shutdown closes every active listener. Host veths are left in place for cheap re-ensure on restart. func (b *bindManager) shutdown() { b.mu.Lock() defer b.mu.Unlock() @@ -248,16 +225,9 @@ func (b *bindManager) shutdown() { } } -// bindLocalListener opens a TCP listener on 169.254.169.254:80 inside the subnet's -// netns, where the host-end veth carries 169.254.169.254/30 with the subnet CIDR -// on-link — so the host-served reply resolves the guest by ARP over the localport -// on the guest's own L2 switch (SO_BINDTODEVICE alone in the root netns could not -// route the SYN-ACK). The netns also isolates subnets whose CIDRs overlap, since -// each is its own routing domain. The socket must be *created* in the netns, so we -// enter it on a locked thread for the socket call and restore the root netns before -// serving. -// SO_BINDTODEVICE (belt-and-braces — there is only one veth in the netns) and -// SO_REUSEADDR (fast rebind on vpcd restart) are still set on the fd. +// bindLocalListener opens 169.254.169.254:80 inside the subnet's netns. +// The socket is created inside the netns (locked thread) so the fd belongs to it; +// SO_BINDTODEVICE and SO_REUSEADDR are also set. func bindLocalListener(ctx context.Context, netnsName, hostEnd string) (net.Listener, error) { lc := net.ListenConfig{ Control: func(_, _ string, c syscall.RawConn) error { @@ -279,8 +249,7 @@ func bindLocalListener(ctx context.Context, netnsName, hostEnd string) (net.List } if netnsName == "" { - // No netns (tests, or a degraded host): bind in the current netns. - return bind() + return bind() // no netns: bind in current netns (tests or degraded host) } var ln net.Listener @@ -295,12 +264,8 @@ func bindLocalListener(ctx context.Context, netnsName, hostEnd string) (net.List return ln, nil } -// inNetns runs fn on a dedicated OS thread switched into the named network -// namespace, then restores the root netns. fn's socket-creating syscalls happen -// in the target netns, so the listener fd belongs to it; once created, the fd is -// served normally from any thread. If the root netns cannot be restored the -// thread is poisoned, so the goroutine returns while still locked and the Go -// runtime retires the thread rather than returning it to the pool. +// inNetns runs fn on a locked OS thread switched into the named netns, then restores root netns. +// If root netns cannot be restored the goroutine exits locked so the runtime retires the thread. func inNetns(netnsName string, fn func() error) error { const netnsDir = "/var/run/netns/" errCh := make(chan error, 1) @@ -333,8 +298,7 @@ func inNetns(netnsName string, fn func() error) error { fnErr := fn() if err := unix.Setns(int(orig.Fd()), unix.CLONE_NEWNET); err != nil { - // Cannot return to the root netns — leave the thread locked so the - // runtime destroys it when this goroutine exits. + // Thread poisoned — leave it locked so the runtime retires it. errCh <- errors.Join(fnErr, fmt.Errorf("restore root netns: %w", err)) return } diff --git a/spinifex/handlers/imds/service.go b/spinifex/handlers/imds/service.go index 178f562e0..7d7f84d7b 100644 --- a/spinifex/handlers/imds/service.go +++ b/spinifex/handlers/imds/service.go @@ -16,25 +16,18 @@ import ( // tokenSweepInterval is how often abandoned/expired IMDSv2 tokens are swept. const tokenSweepInterval = time.Minute -// profileLookup is the slice of IAMService the IMDS credential + iam/* paths -// need: dereference an instance-profile ARN to its record, and resolve a role -// name to its canonical ARN. Narrowed to an interface for unit testing. +// profileLookup is the IAMService slice needed for credential + iam/* paths. type profileLookup interface { ResolveInstanceProfile(accountID, nameOrARN string) (*handlers_iam.InstanceProfile, error) GetRole(accountID string, input *iam.GetRoleInput) (*iam.GetRoleOutput, error) } -// publicKeyLookup fetches an instance's launch SSH public key material by -// (accountID, keyName), serving the public-keys/0/openssh-key path. Narrowed to -// an interface for unit testing; the production implementation -// (NATSPublicKeyLookup) fronts the daemon-side key service over a NATS RPC. +// publicKeyLookup fetches the SSH public key material for the public-keys path. type publicKeyLookup interface { GetPublicKey(accountID, keyName string) (string, error) } -// IMDSService is the host-served EC2 Instance Metadata Service. It runs in -// vpcd on every chassis, serving 169.254.169.254 to local guest VMs over -// per-subnet veths. Run owns the listener lifecycle and blocks until ctx is done. +// IMDSService is the host-served EC2 Instance Metadata Service (169.254.169.254). type IMDSService interface { Run(ctx context.Context) error } @@ -42,16 +35,13 @@ type IMDSService interface { var _ IMDSService = (*IMDSServiceImpl)(nil) // eniResolver maps a datapath-attested (vpcID, srcIP) to ENI + instance facts. -// An interface so the HTTP handlers are unit-testable without a live KV; -// metadataResolver is the production implementation. type eniResolver interface { resolveENI(vpcID, srcIP string) (*eniFacts, error) resolveInstance(eni *eniFacts) (*instanceFacts, error) resolveSGNames(accountID string, sgIDs []string) []string } -// IMDSServiceImpl is the in-process IMDS implementation. It is not part of the -// HTTPS gateway dispatch surface — it owns its own per-VPC listener stack. +// IMDSServiceImpl is the in-process IMDS implementation with its own per-subnet listener stack. type IMDSServiceImpl struct { resolver eniResolver tokens *tokenStore @@ -62,9 +52,7 @@ type IMDSServiceImpl struct { now func() time.Time } -// NewIMDSServiceImpl wires the IMDS service. natsConn backs the KV reads and the -// account-scoped instance fan-out (sized by expectedNodes); ensureVeth/removeVeth -// are injected rather than imported to avoid an import cycle via network/host. +// NewIMDSServiceImpl wires the IMDS service. ensureVeth/removeVeth are injected to avoid an import cycle. func NewIMDSServiceImpl(natsConn *nats.Conn, sts stsAssumer, iamSvc profileLookup, pubKeys publicKeyLookup, expectedNodes int, ensureVeth ensureVethFunc, removeVeth removeVethFunc) (*IMDSServiceImpl, error) { if natsConn == nil { return nil, errors.New("nil NATS connection") @@ -100,9 +88,7 @@ func NewIMDSServiceImpl(natsConn *nats.Conn, sts stsAssumer, iamSvc profileLooku return nil, fmt.Errorf("open %s bucket: %w", KVBucketIMDSSubnetVeth, err) } - // The SG bucket is owned by the VPC service and only feeds the non-critical - // security-groups path. Open it best-effort: if it isn't up yet, IMDS still - // starts and resolveSGNames degrades to serving raw IDs. + // Open SG bucket best-effort; IMDS starts fine without it, degrading to raw IDs. sgKV, err := js.KeyValue(kvBucketSecurityGroups) if err != nil { slog.Warn("IMDS: security-group bucket unavailable, /security-groups will serve IDs", "bucket", kvBucketSecurityGroups, "err", err) @@ -130,8 +116,7 @@ func NewIMDSServiceImpl(natsConn *nats.Conn, sts stsAssumer, iamSvc profileLooku return svc, nil } -// httpHandler builds the shared mux used by every per-VPC listener. The token -// PUT is the only path reachable without a token; everything else is gated. +// httpHandler builds the shared mux for every per-subnet listener. func (s *IMDSServiceImpl) httpHandler() http.Handler { mux := http.NewServeMux() mux.HandleFunc(pathToken, s.handleToken) @@ -139,10 +124,8 @@ func (s *IMDSServiceImpl) httpHandler() http.Handler { return rejectForwarded(mux) } -// Run starts the bind manager (sync + watch) and the token-sweep ticker, then -// blocks until ctx is cancelled. The initial sync failing is logged but not -// fatal — other vpcd services must not have their readiness gated on IMDS -// plumbing, and SDKs retry while it converges. +// Run starts the bind manager (sync + watch) and token-sweep ticker, blocks until ctx is done. +// Initial sync failure is non-fatal; vpcd readiness must not be gated on IMDS. func (s *IMDSServiceImpl) Run(ctx context.Context) error { if err := s.bind.sync(ctx); err != nil { slog.Warn("IMDS: initial bind sync failed, continuing", "err", err) @@ -155,9 +138,7 @@ func (s *IMDSServiceImpl) Run(ctx context.Context) error { return ctx.Err() } -// sweepExpired evicts expired IMDSv2 tokens and stale credential-cache entries -// on a ticker for the life of ctx. Both stores are in-memory and accrete on VM -// churn, so a single sweep keeps them bounded without a per-store goroutine. +// sweepExpired evicts expired tokens and stale credential-cache entries on a ticker. func (s *IMDSServiceImpl) sweepExpired(ctx context.Context) { ticker := time.NewTicker(tokenSweepInterval) defer ticker.Stop() diff --git a/spinifex/handlers/imds/token.go b/spinifex/handlers/imds/token.go index 5dd30280f..ce3916d37 100644 --- a/spinifex/handlers/imds/token.go +++ b/spinifex/handlers/imds/token.go @@ -17,17 +17,15 @@ const ( tokenBytes = 32 ) -// tokenEntry binds an issued IMDSv2 token to the ENI that requested it (not the -// IP — so the session survives a DHCP-renew IP change but dies on ENI detach) -// and its expiry. +// tokenEntry binds an IMDSv2 token to its issuing ENI and expiry. +// Bound to the ENI (not the IP) so it survives DHCP-renew but dies on ENI detach. type tokenEntry struct { eniID string expiresAt time.Time } -// tokenStore holds outstanding IMDSv2 tokens in memory, per process. Tokens are -// a CSRF defence, not a security artefact, so they are never persisted; a -// vpcd restart drops them and SDKs reissue transparently. +// tokenStore holds outstanding IMDSv2 tokens in memory. Tokens are a CSRF defence, +// not persisted; a vpcd restart drops them and SDKs reissue transparently. type tokenStore struct { mu sync.Mutex tokens map[string]tokenEntry @@ -51,9 +49,8 @@ func (s *tokenStore) issue(eniID string, ttl time.Duration, now time.Time) (stri return token, nil } -// validate reports whether token is currently valid AND was issued to -// expectedENI. A wrong-ENI token is rejected exactly like an unknown one so the -// caller cannot probe which tokens exist. Expired tokens are evicted lazily. +// validate reports whether token is valid and was issued to expectedENI. +// Wrong-ENI tokens are rejected identically to unknown ones to prevent probing. func (s *tokenStore) validate(token, expectedENI string, now time.Time) bool { if token == "" { return false @@ -72,8 +69,7 @@ func (s *tokenStore) validate(token, expectedENI string, now time.Time) bool { return entry.eniID == expectedENI } -// sweep deletes every token that expired at or before now. Called on a 1-minute -// ticker so abandoned tokens (an SDK that issues then never reads) don't leak. +// sweep deletes expired tokens to prevent abandoned tokens from leaking. func (s *tokenStore) sweep(now time.Time) { s.mu.Lock() defer s.mu.Unlock() diff --git a/spinifex/handlers/imds/veth_store.go b/spinifex/handlers/imds/veth_store.go index 4e574457c..1098b14cc 100644 --- a/spinifex/handlers/imds/veth_store.go +++ b/spinifex/handlers/imds/veth_store.go @@ -8,11 +8,8 @@ import ( "github.com/nats-io/nats.go" ) -// SubnetVethRecord is the persisted per-subnet IMDS plumbing record -// (KVBucketIMDSSubnetVeth, keyed by subnet ID): the signal that a subnet's IMDS -// localport is installed, replayed by each chassis's BindManager. It carries the -// LSP MAC and the subnet CIDR the host can't re-derive, plus the owning VPC for -// the subnet→VPC reverse lookup. +// SubnetVethRecord is the per-subnet IMDS plumbing record stored in KVBucketIMDSSubnetVeth. +// Replayed by each chassis's BindManager; carries MAC and CIDR fields the host cannot re-derive. type SubnetVethRecord struct { SubnetID string `json:"subnet_id"` ShortSubnetID string `json:"short_subnet_id"` @@ -22,9 +19,7 @@ type SubnetVethRecord struct { CreatedAt string `json:"created_at"` } -// VethStore is the persistence surface for SubnetVethRecord rows in -// KVBucketIMDSSubnetVeth. Backed by a NATS JetStream KV bucket opened by -// InitBuckets in production; tests inject the handle via NewVethStore. +// VethStore is the persistence surface for SubnetVethRecord rows in KVBucketIMDSSubnetVeth. type VethStore struct { kv nats.KeyValue } diff --git a/spinifex/handlers/sts/assume_role.go b/spinifex/handlers/sts/assume_role.go index 1fe54a704..1f22f9ff4 100644 --- a/spinifex/handlers/sts/assume_role.go +++ b/spinifex/handlers/sts/assume_role.go @@ -30,19 +30,16 @@ const ( stsActionWildcard = "sts:*" globalWildcard = "*" - // principalTypeAssumedRole is the SessionCredential.PrincipalType for a role - // session — the default kind, also assumed when a stored value is empty. + // principalTypeAssumedRole is the SessionCredential.PrincipalType for a role session; + // empty stored values are also treated as this type. principalTypeAssumedRole = "assumed-role" - // principalTypeUser is the SessionCredential.PrincipalType for a session - // minted by GetSessionToken, which resolves back to the calling IAM user. - // Mirrors the gateway's principalTypeUser so the ASIA resolve path can map - // the stored value straight onto the user principal context. + // principalTypeUser is the SessionCredential.PrincipalType for sessions minted by + // GetSessionToken, which resolve back to the calling IAM user. principalTypeUser = "user" - // ec2ServicePrincipal is the synthetic caller used by AssumeRoleForInstance — the - // only service principal that matches a trust policy in v1. The HTTPS AssumeRole - // path supplies an empty principalSource, so service principals never match there. + // ec2ServicePrincipal is the synthetic caller used by AssumeRoleForInstance. + // The HTTPS AssumeRole path supplies an empty principalSource, so service principals never match there. ec2ServicePrincipal = "ec2.amazonaws.com" minDurationSeconds int64 = 900 @@ -58,10 +55,8 @@ const ( mintMaxAttempts = 3 ) -// roleSessionNameRegex enforces AWS's documented session-name character set -// expressed as a literal ASCII range: `\w` would silently widen to Unicode -// under a future build tag and is avoided. `:` and `/` are excluded so the -// synthesised assumed-role ARN and AssumedRoleId stay unambiguously parseable. +// roleSessionNameRegex enforces AWS's documented session-name character set as a literal +// ASCII range. `:` and `/` are excluded so the assumed-role ARN stays unambiguously parseable. var roleSessionNameRegex = regexp.MustCompile(`^[A-Za-z0-9_+=,.@-]{2,64}$`) // AssumeRole mints temporary credentials after evaluating the target role's @@ -114,8 +109,7 @@ func (s *STSServiceImpl) AssumeRole(callerAccountID, callerARN, callerIdentity s } // AssumeRoleForInstance is the in-process IMDS entry point, not reachable over HTTPS. -// The caller is synthesised as the EC2 service principal (principalSource = -// ec2.amazonaws.com); the session name is the instanceID, yielding an AWS-style ARN. +// Caller is synthesised as ec2.amazonaws.com; the session name is the instanceID. func (s *STSServiceImpl) AssumeRoleForInstance(accountID, roleARN, instanceID string, durationSeconds int64) (*sts.AssumeRoleOutput, error) { if accountID == "" || roleARN == "" || instanceID == "" { return nil, errors.New(awserrors.ErrorMissingParameter) @@ -140,9 +134,9 @@ func (s *STSServiceImpl) AssumeRoleForInstance(accountID, roleARN, instanceID st return out, nil } -// assumeRoleForCaller is the shared core of AssumeRole (HTTPS) and AssumeRoleForInstance -// (IMDS): resolves the role, validates the duration (0 → default), evaluates the trust -// policy, and mints credentials. principalSource is "" for HTTPS, ec2.amazonaws.com for IMDS. +// assumeRoleForCaller is the shared core of AssumeRole and AssumeRoleForInstance: +// resolves the role, clamps the duration, evaluates the trust policy, and mints credentials. +// principalSource is "" for HTTPS, ec2.amazonaws.com for IMDS. func (s *STSServiceImpl) assumeRoleForCaller(callerAccountID, callerARN, principalSource, roleARN, sessionName, sourceIdentity string, requestedDuration int64) (*sts.AssumeRoleOutput, error) { roleAccountID, roleName, err := auth.ParseRoleARN(roleARN) if err != nil { @@ -151,10 +145,7 @@ func (s *STSServiceImpl) assumeRoleForCaller(callerAccountID, callerARN, princip roleOut, err := s.iamSvc.GetRole(roleAccountID, &iam.GetRoleInput{RoleName: aws.String(roleName)}) if err != nil { - // Same-account miss leaks no information the caller couldn't get via - // ListRoles; cross-account miss is masked to AccessDenied to prevent - // enumeration. Comparing against the bare error code matches how - // IAMService surfaces NoSuchEntity (errors.New(awserrors.ErrorIAMNoSuchEntity)). + // Cross-account miss is masked to AccessDenied to prevent role enumeration. if err.Error() == awserrors.ErrorIAMNoSuchEntity && callerAccountID != roleAccountID { return nil, errors.New(awserrors.ErrorAccessDenied) } @@ -202,16 +193,13 @@ func (s *STSServiceImpl) assumeRoleForCaller(callerAccountID, callerARN, princip }, nil } -// evalTrustPolicy implements AWS's explicit-deny-wins semantics: a first pass -// scans every Deny statement (returning AccessDenied on any match), a second -// pass scans every Allow statement (returning nil on the first match). A -// single-pass "return on first Allow" loop would silently skip a later Deny -// and is a real authorisation bug. +// evalTrustPolicy implements AWS's explicit-deny-wins semantics: first pass scans all +// Deny statements, second pass scans all Allows. A single-pass loop would skip a later +// Deny and silently grant access. func evalTrustPolicy(docJSON, callerARN, principalSource string) error { doc, err := handlers_iam.ValidateTrustPolicyDocument(docJSON) if err != nil { - // Stored docs were validated at CreateRole / UpdateAssumeRolePolicy - // time, so reaching here implies on-disk corruption — fail closed. + // Docs are validated at write time; reaching here implies on-disk corruption. return fmt.Errorf("stored trust policy invalid: %w", err) } @@ -259,9 +247,8 @@ func matchTrustAction(actions []string) bool { return false } -// matchTrustPrincipal evaluates each top-level Principal key independently (AWS -// treats the map as an OR). AWS matches callerARN, Service matches principalSource -// (set only by the IMDS path); unsupported keys (Federated) skip, not fail. +// matchTrustPrincipal evaluates each Principal key as an OR. AWS matches callerARN; +// Service matches principalSource (IMDS path only); unsupported keys (Federated) skip. func matchTrustPrincipal(raw json.RawMessage, callerARN, principalSource string) (bool, error) { if len(raw) == 0 { return false, nil @@ -298,10 +285,7 @@ func matchTrustPrincipal(raw json.RawMessage, callerARN, principalSource string) } // matchServicePrincipal matches a Service principal against the synthesised caller. -// v1 trusts only the EC2 service principal: any other principalSource (including -// the empty one used by the HTTPS path) is rejected up front, so unsupported -// service principals are denied even if a future caller synthesised one. -// Expanding the whitelist (ecs-tasks, lambda) is a one-line change to this guard. +// Only the EC2 service principal is trusted; any other principalSource is denied. func matchServicePrincipal(raw json.RawMessage, principalSource string) (bool, error) { if principalSource != ec2ServicePrincipal { return false, nil @@ -334,11 +318,8 @@ func matchAWSPrincipal(raw json.RawMessage, callerARN string) (bool, error) { return false, nil } -// matchAWSPrincipalEntry resolves one principal clause against the caller's -// ARN, handling four forms: the wildcard, a bare 12-digit account ID (treated -// as :root shorthand), exact ARN match, :root in the same account, and the -// chained-assume auto-expansion where a role ARN matches any session of that -// role. +// matchAWSPrincipalEntry resolves one principal clause: wildcard, bare account ID +// (treated as :root), exact ARN, :root shorthand, and role-ARN session expansion. func matchAWSPrincipalEntry(clause, callerARN string) bool { if clause == globalWildcard { return true @@ -359,17 +340,14 @@ func matchAWSPrincipalEntry(clause, callerARN string) bool { return false } - // :root → any principal in the same account. Scoped to the IAM service - // segment; arn:aws:s3::A:root (or any other service) must not match — - // validators upstream do not check service-segment shape, so a malformed - // ARN pasted into a trust policy must fail closed here. + // :root matches any principal in the same account; scoped to the IAM service + // segment so arn:aws:s3::A:root (or any other service) does not match. if clauseARN.service == "iam" && clauseARN.resource == "root" && clauseARN.account == callerParts.account { return true } - // Role-ARN auto-expansion: `arn:aws:iam::A:role/.../X` matches - // `arn:aws:sts::A:assumed-role/X/` for any session. + // Role-ARN auto-expansion: arn:aws:iam::A:role/X matches any assumed-role session of X. if clauseARN.service == "iam" && strings.HasPrefix(clauseARN.resource, "role/") && callerParts.service == "sts" && strings.HasPrefix(callerParts.resource, "assumed-role/") && clauseARN.account == callerParts.account { @@ -409,10 +387,8 @@ func lastPathSegment(s string) string { } // sessionEnvelope is the resolved identity a session credential is minted for. -// It decouples mintSession from AssumeRole's *iam.Role so GetSessionToken can -// mint a user-bound session: the role paths fill the assumed-role fields, the -// user path sets PrincipalType "user" + SessionName = the IAM user name and -// leaves the assumed-role fields empty. +// Role paths fill the assumed-role fields; GetSessionToken sets PrincipalType "user" +// and leaves the assumed-role fields empty. type sessionEnvelope struct { PrincipalType string AccountID string @@ -426,10 +402,8 @@ type sessionEnvelope struct { AssumedRoleID string } -// assumedRoleEnvelope derives the session envelope for a role session, shared by -// AssumeRole, AssumeRoleForInstance, and AssumeRoleWithWebIdentity. It owns the -// assumed-role ARN / AssumedRoleId construction so every role path stays -// consistent. +// assumedRoleEnvelope derives the session envelope for a role session, shared by all +// AssumeRole variants. It centralises assumed-role ARN and AssumedRoleId construction. func assumedRoleEnvelope(role *iam.Role, roleAccountID, sessionName, sourceIdentity string) sessionEnvelope { roleID := aws.StringValue(role.RoleId) roleName := aws.StringValue(role.RoleName) @@ -445,10 +419,8 @@ func assumedRoleEnvelope(role *iam.Role, roleAccountID, sessionName, sourceIdent } } -// mintSession generates a fresh ASIA AKID, secret, and session token, encrypts -// the secret with the IAM master key, HMACs the session token, and persists the -// record described by env. Retries on AKID collision; the birthday bound for -// 64-bit AKID entropy makes that practically unreachable. +// mintSession generates a fresh ASIA AKID, encrypts the secret, HMACs the token, +// and persists the credential. Retries up to mintMaxAttempts on AKID collision. func (s *STSServiceImpl) mintSession(env sessionEnvelope, duration int64) (*SessionCredential, string, string, error) { plainSecret, err := generateRandomBase64(sessionSecretBytes) if err != nil { @@ -515,11 +487,9 @@ func generateRandomBase64(n int) (string, error) { return base64.StdEncoding.EncodeToString(b), nil } -// computeTokenHMAC HMACs the wire-form (base64) session token with the IAM -// master key. The SigV4 verifier recomputes this over the -// X-Amz-Security-Token header and constant-time-compares against the stored -// value — a bucket read by any process that lacks the master key cannot -// recover a usable token. +// computeTokenHMAC HMACs the wire-form session token with the master key. +// The SigV4 verifier recomputes and constant-time-compares; a bucket read without +// the master key cannot recover a valid token. func computeTokenHMAC(key []byte, token string) string { mac := hmac.New(sha256.New, key) mac.Write([]byte(token)) diff --git a/spinifex/handlers/sts/assume_role_with_web_identity.go b/spinifex/handlers/sts/assume_role_with_web_identity.go index cce646ac8..c6a711df9 100644 --- a/spinifex/handlers/sts/assume_role_with_web_identity.go +++ b/spinifex/handlers/sts/assume_role_with_web_identity.go @@ -21,30 +21,15 @@ import ( "github.com/nats-io/nats.go" ) -// webIdentityClaims is the RegisteredClaims subset the IRSA flow consumes. -// The wider set of K8s ServiceAccount-token claims (kubernetes.io/*) is -// ignored: STS only needs iss/sub/aud/exp to authorise the role assumption. +// webIdentityClaims is the RegisteredClaims subset the IRSA flow consumes (iss/sub/aud/exp). type webIdentityClaims struct { jwt.RegisteredClaims } -// AssumeRoleWithWebIdentity exchanges an OIDC ID token (typically a projected -// Kubernetes ServiceAccount token signed by an EKS cluster's per-cluster -// ECDSA-P256 signing key) for short-lived AWS credentials bound to the target -// IAM role. -// -// Verification order (each step fails closed on any error): -// -// 1. Validate input shape. -// 2. Parse JWT with ES256 keyfunc that resolves issuer → registered OIDC -// provider → cluster JWKS → JWK by kid → *ecdsa.PublicKey. -// 3. Validate `aud` contains `sts.amazonaws.com` (IRSA convention). -// 4. Resolve and look up the target role. -// 5. Evaluate the role's trust policy under web-identity semantics. -// 6. Mint a session credential bound to the role. -// -// Anonymous action — caller identity is the JWT, not SigV4. The gateway -// dispatcher does not gate this action with checkPolicy. +// AssumeRoleWithWebIdentity exchanges an OIDC ID token (typically a K8s ServiceAccount +// token) for short-lived credentials bound to the target IAM role. Validates input, +// verifies the JWT (ES256, registered OIDC provider, aud must contain sts.amazonaws.com), +// evaluates the trust policy, and mints a session. Anonymous — identity is the JWT, not SigV4. func (s *STSServiceImpl) AssumeRoleWithWebIdentity(input *sts.AssumeRoleWithWebIdentityInput) (*sts.AssumeRoleWithWebIdentityOutput, error) { if input == nil { return nil, errors.New(awserrors.ErrorMissingParameter) @@ -71,10 +56,8 @@ func (s *STSServiceImpl) AssumeRoleWithWebIdentity(input *sts.AssumeRoleWithWebI parser := jwt.NewParser( jwt.WithValidMethods([]string{"ES256"}), jwt.WithExpirationRequired(), - // Audience is validated explicitly below — the IRSA contract requires - // sts.amazonaws.com membership, and the jwt/v5 default audience check - // is a string-equality match on a configurable expected value that - // does not capture set-membership semantics on multi-aud tokens. + // Audience validated explicitly below; jwt/v5 default check is + // string-equality, which misses set-membership semantics on multi-aud tokens. ) _, err = parser.ParseWithClaims(rawToken, claims, s.webIdentityKeyFunc(roleAccountID)) if err != nil { @@ -94,9 +77,7 @@ func (s *STSServiceImpl) AssumeRoleWithWebIdentity(input *sts.AssumeRoleWithWebI roleOut, err := s.iamSvc.GetRole(roleAccountID, &iam.GetRoleInput{RoleName: aws.String(roleName)}) if err != nil { - // Cross-account miss → AccessDenied (per the AssumeRole pattern — - // prevents cross-account role enumeration). Same-account NoSuchEntity - // surfaces as-is. + // All misses → AccessDenied to prevent cross-account role enumeration. if err.Error() == awserrors.ErrorIAMNoSuchEntity { return nil, errors.New(awserrors.ErrorAccessDenied) } @@ -169,18 +150,9 @@ func (s *STSServiceImpl) AssumeRoleWithWebIdentity(input *sts.AssumeRoleWithWebI }, nil } -// webIdentityKeyFunc returns a jwt.Keyfunc bound to the role's account. The -// `iss` claim drives JWKS discovery, but the OIDC-provider registry is read -// from the role-account's IAM bucket (per Q4 — cross-account IRSA against an -// issuer registered in a different account is not supported v1). -// -// Resolution path: iss → ParseEKSIssuerURL → (issuerAccountID, clusterName) -// → registered-provider check in iam-account-{roleAccountID} → FetchClusterJWKS -// from eks-account-{issuerAccountID} → JWK by kid → *ecdsa.PublicKey. -// -// Failures surface as plain errors to the parser; the caller maps them all -// to ErrorInvalidIdentityToken so the token is rejected with a single -// auditable error code regardless of which step failed. +// webIdentityKeyFunc returns a jwt.Keyfunc that resolves iss → OIDC provider registry +// (role account) → JWKS (issuer account) → JWK by kid → *ecdsa.PublicKey. +// All failures map to ErrorInvalidIdentityToken at the call site. func (s *STSServiceImpl) webIdentityKeyFunc(roleAccountID string) jwt.Keyfunc { return func(t *jwt.Token) (any, error) { kid, _ := t.Header["kid"].(string) @@ -217,12 +189,9 @@ func (s *STSServiceImpl) webIdentityKeyFunc(roleAccountID string) jwt.Keyfunc { } } -// verifyOIDCProviderRegistered looks up the issuer in the role-account's IAM -// OIDC-provider registry. Bucket-not-found is treated identically to -// key-not-found: the account has no providers registered yet, so no token -// from any issuer can succeed. The error message intentionally does not -// distinguish the two — leaking "your bucket exists" leaks account -// activation status to anonymous callers. +// verifyOIDCProviderRegistered checks the issuer in the role-account's OIDC-provider +// registry. Bucket-not-found and key-not-found return the same error to avoid leaking +// account activation status to anonymous callers. func (s *STSServiceImpl) verifyOIDCProviderRegistered(roleAccountID, issuer string) error { bucketName := handlers_iam.IAMAccountBucketName(roleAccountID) kv, err := s.js.KeyValue(bucketName) @@ -242,10 +211,8 @@ func (s *STSServiceImpl) verifyOIDCProviderRegistered(roleAccountID, issuer stri return nil } -// jwkToECDSAPublicKey decodes a JWK (RFC 7517) EC entry into an -// *ecdsa.PublicKey. Only ES256 / P-256 is supported v1 — matches the per- -// cluster signing key generated at CreateCluster (eks-v1.md Q8). Any other -// shape fails closed. +// jwkToECDSAPublicKey decodes a JWK EC entry (RFC 7517) into an *ecdsa.PublicKey. +// Only ES256 / P-256 is supported; any other shape fails closed. func jwkToECDSAPublicKey(jwk *JWK) (*ecdsa.PublicKey, error) { if jwk == nil { return nil, errors.New("nil JWK") @@ -276,18 +243,13 @@ func jwkToECDSAPublicKey(jwk *JWK) (*ecdsa.PublicKey, error) { return pub, nil } -// claimAudienceContains is a constant-time membership test on the JWT `aud` -// claim. jwt/v5 already canonicalises string-or-array into ClaimStrings; -// this just iterates. +// claimAudienceContains tests membership of want in the JWT aud claim. func claimAudienceContains(aud jwt.ClaimStrings, want string) bool { return slices.Contains(aud, want) } -// stripIssuerScheme drops the `https://` prefix from an issuer URL to produce -// the form AWS uses as the suffix of `arn:aws:iam::{accountID}:oidc-provider/...`. -// Returns the input unchanged if the prefix is absent (defensive — upstream -// ParseEKSIssuerURL already enforces https, but stripping a non-match would -// produce an empty path). +// stripIssuerScheme drops the https:// prefix to produce the scheme-less form +// AWS uses in oidc-provider ARN suffixes. func stripIssuerScheme(issuer string) string { return strings.TrimPrefix(issuer, "https://") } diff --git a/spinifex/handlers/sts/get_caller_identity.go b/spinifex/handlers/sts/get_caller_identity.go index 3e11ce0d9..64fc148bc 100644 --- a/spinifex/handlers/sts/get_caller_identity.go +++ b/spinifex/handlers/sts/get_caller_identity.go @@ -5,10 +5,8 @@ import ( "github.com/aws/aws-sdk-go/service/sts" ) -// GetCallerIdentity returns the resolved caller identity. AWS allows every -// authenticated principal to call this — the gateway layer extracts identity -// from the SigV4 context and passes the three plain strings in. The input -// struct is empty in the AWS SDK and exists only for SDK uniformity. +// GetCallerIdentity returns the resolved caller identity. The gateway passes the +// three identity strings extracted from the SigV4 context. func (s *STSServiceImpl) GetCallerIdentity(callerAccountID, callerARN, callerUserID string, _ *sts.GetCallerIdentityInput) (*sts.GetCallerIdentityOutput, error) { return &sts.GetCallerIdentityOutput{ Account: aws.String(callerAccountID), diff --git a/spinifex/handlers/sts/get_session_token.go b/spinifex/handlers/sts/get_session_token.go index fa762387d..a1fbc2422 100644 --- a/spinifex/handlers/sts/get_session_token.go +++ b/spinifex/handlers/sts/get_session_token.go @@ -11,30 +11,18 @@ import ( ) const ( - // GetSessionToken duration bounds differ from AssumeRole: AWS permits a - // session up to 36h (vs. the 12h role-session ceiling) and defaults to 12h. - // The floor is the STS-wide minimum (minDurationSeconds, 900s). + // GetSessionToken allows up to 36h (vs. the 12h role-session ceiling), defaulting to 12h. getSessionTokenMaxDuration int64 = 129600 // 36h getSessionTokenDefaultDuration int64 = 43200 // 12h ) -// GetSessionToken exchanges the calling IAM user's long-lived credentials for -// short-lived ASIA session credentials bound to the SAME user identity. Unlike -// AssumeRole, the resulting session resolves back to arn:aws:iam::A:user/N and -// IAM policy is evaluated against that user — see the user branch of the SigV4 -// ASIA resolve path. The caller identity is resolved by the gateway and passed -// in as plain strings to keep the handler unit-testable. -// -// AWS forbids calling GetSessionToken with temporary credentials, so only a -// long-lived user principal (AKIA → principalType "user") is accepted. Assumed -// -role callers are rejected by principal type; session callers — including a -// GetSessionToken session, which resolves back to principalType "user" — are -// rejected by their ASIA access-key prefix. Root is a long-lived user and, per -// AWS, may call GetSessionToken. +// GetSessionToken exchanges a long-lived IAM user credential for short-lived ASIA +// credentials that still resolve to the same user identity. Only long-lived users +// (AKIA prefix, principalType "user") may call this; assumed-role and session callers +// are rejected. Caller identity is resolved by the gateway and passed as plain strings. func (s *STSServiceImpl) GetSessionToken(callerAccountID, callerUserName, callerPrincipalType, callerAccessKeyID string, input *sts.GetSessionTokenInput) (*sts.GetSessionTokenOutput, error) { if input == nil { - // `aws sts get-session-token` with no arguments sends an empty body; - // an absent input is the common case, not an error. + // An absent body is the common case from `aws sts get-session-token` with no args. input = &sts.GetSessionTokenInput{} } @@ -44,24 +32,21 @@ func (s *STSServiceImpl) GetSessionToken(callerAccountID, callerUserName, caller return nil, errors.New(awserrors.ErrorAccessDenied) } if strings.HasPrefix(callerAccessKeyID, SessionAccessKeyIDPrefix) { - // A GetSessionToken session resolves back to principalType "user", so the - // check above cannot catch it — the ASIA prefix is the only signal that - // the caller holds temporary credentials. Rejecting it stops a session - // from minting a fresh session and rolling its lifetime forward forever. + // A GetSessionToken session resolves to principalType "user", so the ASIA prefix + // is the only signal the caller holds temporary credentials. Rejecting prevents + // a session from minting a fresh session and rolling its lifetime forward. slog.Warn("GetSessionToken denied: caller is using temporary (session) credentials", "account_id", callerAccountID, "akid", callerAccessKeyID) return nil, errors.New(awserrors.ErrorAccessDenied) } if callerAccountID == "" || callerUserName == "" { - // A user principal with no account or name is a mis-wired caller, not a - // client error — fail loud rather than mint a session with an empty name. + // Mis-wired caller, not a client error — fail loud rather than mint a nameless session. slog.Error("GetSessionToken: user principal missing account or name", "account_id", callerAccountID, "user_name", callerUserName) return nil, errors.New(awserrors.ErrorInternalError) } - // MFA (SerialNumber/TokenCode) is out of scope: reject rather than silently - // ignore so a caller never believes an MFA condition was enforced. + // MFA is out of scope: reject rather than silently ignore, so callers don't believe MFA was enforced. if aws.StringValue(input.SerialNumber) != "" || aws.StringValue(input.TokenCode) != "" { return nil, errors.New(awserrors.ErrorInvalidParameterValue) } @@ -93,9 +78,8 @@ func (s *STSServiceImpl) GetSessionToken(callerAccountID, callerUserName, caller }, nil } -// clampGetSessionTokenDuration coerces a requested duration into the permitted -// [900, 129600] window. A nil DurationSeconds is handled by the caller (default -// 12h); this only sees an explicit value, which AWS clamps rather than rejects. +// clampGetSessionTokenDuration coerces a duration into [900, 129600]. +// A nil DurationSeconds is handled by the caller; AWS clamps explicit values rather than rejecting them. func clampGetSessionTokenDuration(requested int64) int64 { if requested < minDurationSeconds { return minDurationSeconds diff --git a/spinifex/handlers/sts/oidc_jwks.go b/spinifex/handlers/sts/oidc_jwks.go index 175b745cd..8eb837f78 100644 --- a/spinifex/handlers/sts/oidc_jwks.go +++ b/spinifex/handlers/sts/oidc_jwks.go @@ -12,10 +12,8 @@ import ( "github.com/nats-io/nats.go" ) -// JWK is one entry of an RFC 7517 JSON Web Key Set. EKS publishes EC P-256 -// keys (kty=EC, crv=P-256, alg=ES256); the optional RSA fields are kept so a -// future cluster published with a different alg still round-trips without -// breaking the decoder. +// JWK is one entry of an RFC 7517 JSON Web Key Set. EKS publishes EC P-256 keys; +// RSA fields are kept for future compatibility. type JWK struct { Kty string `json:"kty"` Kid string `json:"kid"` @@ -60,22 +58,12 @@ const ( eksIssuerSegCluster = 4 ) -// ParseEKSIssuerURL extracts the (accountID, clusterName) pair encoded in a -// spinifex EKS OIDC issuer URL of the form: +// ParseEKSIssuerURL extracts (accountID, clusterName) from a spinifex EKS OIDC issuer URL: // // https://{gateway-host}/oidc/eks/{region}/{accountID}/{clusterName} // -// This matches the URL ClusterOIDCIssuer (handlers/eks) emits and that awsgw -// serves /.well-known/openid-configuration + /keys under. The host is the -// awsgw gateway (an IP:port on bare-metal/on-prem, not an `oidc.eks.*` vhost), -// so host trust is established by the IAM OIDC-provider registration — the full -// issuer string is the registry key in verifyOIDCProviderRegistered — and by -// the JWKS signature check, not by a host name pattern. -// -// Returns an error for any URL that does not match this exact 5-segment path -// structure: defensive parsing pins accountID and clusterName at fixed -// positions so a maliciously crafted `iss` claim cannot steer the JWKS lookup -// at a different cluster bucket. +// Strict 5-segment path parsing pins accountID and clusterName at fixed positions so a +// crafted `iss` claim cannot steer the JWKS lookup to a different cluster bucket. func ParseEKSIssuerURL(issuer string) (accountID, clusterName string, err error) { u, err := url.Parse(issuer) if err != nil { @@ -98,14 +86,8 @@ func ParseEKSIssuerURL(issuer string) (accountID, clusterName string, err error) return segments[eksIssuerSegAccountID], segments[eksIssuerSegCluster], nil } -// FetchClusterJWKS reads the JWKS document for an EKS cluster from the -// per-account EKS KV bucket. The lookup is keyed by `{accountID, clusterName}` -// extracted from the IRSA token's `iss` claim and resolved via the EKS store -// helpers (single source of truth for bucket + key naming). -// -// Returns (nil, nil) when the cluster has no JWKS published yet — callers -// translate that miss to InvalidIdentityToken so unverified tokens always -// fail closed without leaking cluster-existence information. +// FetchClusterJWKS reads the JWKS for an EKS cluster from the per-account EKS KV bucket. +// Returns (nil, nil) when the cluster has no JWKS published; callers map that to InvalidIdentityToken. func FetchClusterJWKS(js nats.JetStreamContext, accountID, clusterName string) (*JWKS, error) { if js == nil { return nil, errors.New("nil JetStream context") diff --git a/spinifex/handlers/sts/presigned_url_verify.go b/spinifex/handlers/sts/presigned_url_verify.go index c605c58f4..27224debc 100644 --- a/spinifex/handlers/sts/presigned_url_verify.go +++ b/spinifex/handlers/sts/presigned_url_verify.go @@ -29,20 +29,16 @@ const ( presignedSignedHeader = "x-k8s-aws-id" presignedHostHeader = "host" - // presignedMaxExpiresSeconds is the AWS-imposed ceiling on - // X-Amz-Expires for SigV4-presigned URLs. The aws eks get-token output - // defaults to 900s; rejecting anything beyond the SigV4 ceiling fails - // a forged URL that lies about its TTL. + // presignedMaxExpiresSeconds is the AWS-imposed ceiling on X-Amz-Expires. + // Rejecting beyond this ceiling catches forged URLs that lie about their TTL. presignedMaxExpiresSeconds = 7 * 24 * 60 * 60 - // presignedClockSkew matches the SigV4 wire-protocol skew - // allowance (used during X-Amz-Date sanity check). + // presignedClockSkew is the SigV4 wire-protocol clock-skew allowance. presignedClockSkew = 5 * time.Minute ) -// PresignedCallerIdentity is the result of validating a SigV4-presigned -// `sts:GetCallerIdentity` URL — the token shape produced by `aws eks -// get-token` and consumed by the EKS token webhook (`eks-token-webhook`). +// PresignedCallerIdentity is the result of validating a SigV4-presigned GetCallerIdentity +// URL — the token shape produced by `aws eks get-token`. type PresignedCallerIdentity struct { AccountID string ARN string @@ -51,31 +47,12 @@ type PresignedCallerIdentity struct { PrincipalType string } -// presignedTimeNow is the time-source seam — tests override it to pin the -// clock for replay / expiry assertions. Production calls go through here so -// the wire-protocol skew check is not flaky. +// presignedTimeNow is the time-source seam; tests override it to pin the clock. var presignedTimeNow = func() time.Time { return time.Now().UTC() } -// VerifyPresignedGetCallerIdentity validates a SigV4-presigned URL for the -// `sts:GetCallerIdentity` action and resolves the calling principal. The EKS -// token webhook calls this over NATS as part of TokenReview processing. -// -// Verification (each step fails closed): -// -// 1. Parse and validate envelope query parameters (algorithm, credential, -// date, expires, signed-headers, signature). -// 2. Enforce that x-k8s-aws-id is in X-Amz-SignedHeaders — skipping this is -// a cross-cluster auth bypass (eks-v1.md Q10 mandatory check). -// 3. Validate timestamp and expiry against the wire clock. -// 4. Resolve the access key to a secret (session via ASIA path, otherwise -// long-lived IAM). -// 5. Reconstruct the canonical request with X-K8s-Aws-Id pinned to -// expectedClusterName, recompute the SigV4 signature, constant-time -// compare against the wire signature. A mismatch here is the -// cross-cluster-replay defence: a token signed under cluster-A's name -// produces a different signature when reconstructed under cluster-B's -// name. -// 6. Build a PresignedCallerIdentity from the resolved principal. +// VerifyPresignedGetCallerIdentity validates a SigV4-presigned GetCallerIdentity URL +// and resolves the calling principal. Called by the EKS token webhook over NATS. +// Reconstructs the canonical request with expectedClusterName to prevent cross-cluster replay. func (s *STSServiceImpl) VerifyPresignedGetCallerIdentity(presignedURL, expectedClusterName string) (*PresignedCallerIdentity, error) { if presignedURL == "" || expectedClusterName == "" { return nil, errors.New(awserrors.ErrorMissingParameter) @@ -114,19 +91,14 @@ func (s *STSServiceImpl) VerifyPresignedGetCallerIdentity(presignedURL, expected now := presignedTimeNow() if now.Sub(env.signedTime).Abs() > presignedClockSkew+time.Duration(env.expiresSeconds)*time.Second { - // Wide allowance — exact expiry is checked below; this guards a - // nonsense X-Amz-Date that's months out (signature would still verify - // against the wire if we let it through, but the token would be - // trivially "expired" and we surface a clearer error code). + // Guards a nonsense X-Amz-Date (months out); exact expiry is checked below. return nil, errors.New(awserrors.ErrorExpiredToken) } if now.After(env.signedTime.Add(time.Duration(env.expiresSeconds) * time.Second)) { return nil, errors.New(awserrors.ErrorExpiredToken) } if env.signedTime.UTC().Format(presignedDateFmt) != env.credential.date { - // Credential-scope date must match the X-Amz-Date date portion; - // AWS-CLI enforces this; mismatch is the classic "signed yesterday - // presented today" bug. Fail closed. + // Credential-scope date must match the X-Amz-Date date portion. return nil, errors.New(awserrors.ErrorInvalidIdentityToken) } @@ -143,10 +115,7 @@ func (s *STSServiceImpl) VerifyPresignedGetCallerIdentity(presignedURL, expected canonicalQuery, canonicalHeaders, signedHeadersList, - // botocore / aws-sdk presign `sts:GetCallerIdentity` (a non-S3, empty-body - // request) with the SHA256 of the empty string as the payload hash. - // "UNSIGNED-PAYLOAD" is an S3-only convention and would mismatch every - // real `aws eks get-token`. + // Non-S3 presigned requests use SHA256("") as the payload hash; "UNSIGNED-PAYLOAD" is S3-only. emptyStringSHA256, }, "\n") @@ -171,9 +140,7 @@ func (s *STSServiceImpl) VerifyPresignedGetCallerIdentity(presignedURL, expected return principal, nil } -// presignedEnvelope is the parsed-and-validated set of envelope facts from -// a SigV4-presigned URL's query string. The fields are pure parse outputs; -// signature verification consumes them. +// presignedEnvelope holds the parsed envelope facts from a SigV4-presigned URL query string. type presignedEnvelope struct { credential presignedCredential amzDate string // raw X-Amz-Date value (used in StringToSign) @@ -263,11 +230,8 @@ func parsePresignedEnvelope(q url.Values) (presignedEnvelope, error) { }, nil } -// canonicalQueryStringExcludingSignature returns the SigV4 canonical query -// string with X-Amz-Signature dropped. The remaining parameters are -// percent-encoded per RFC 3986 and sorted lexicographically by key, then by -// value for duplicate keys. The aws-sdk-go url.QueryEscape produces the same -// output for these envelope params (alphanumerics + the safe set). +// canonicalQueryStringExcludingSignature returns the SigV4 canonical query string +// with X-Amz-Signature dropped, per-RFC-3986 encoded, sorted by key then value. func canonicalQueryStringExcludingSignature(q url.Values) string { type kv struct { k string @@ -296,10 +260,8 @@ func canonicalQueryStringExcludingSignature(q url.Values) string { return strings.Join(parts, "&") } -// awsQueryEscape encodes per SigV4: every byte outside the unreserved set -// (A-Z, a-z, 0-9, '-', '_', '.', '~') is percent-encoded as uppercase hex. -// net/url.QueryEscape encodes spaces as '+' and lower-cases hex digits, both -// of which would diverge from AWS canonicalization. +// awsQueryEscape encodes per SigV4 (unreserved set, uppercase hex). +// net/url.QueryEscape encodes spaces as '+' and lower-cases hex, both diverging from AWS. func awsQueryEscape(s string) string { var b strings.Builder b.Grow(len(s)) @@ -322,10 +284,7 @@ func hexByte(b byte) string { return string([]byte{hexDigits[b>>4], hexDigits[b&0x0f]}) } -// canonicalURI returns the canonical URI per SigV4. For sts.GetCallerIdentity -// presigned URLs the path is "/" (action lives in the query string); paths -// beyond root are accepted and pass through unmodified per the canonical -// rules — empty path normalises to "/". +// canonicalURI returns the SigV4 canonical URI; empty path normalises to "/". func canonicalURI(u *url.URL) string { if u.Path == "" { return "/" @@ -333,12 +292,8 @@ func canonicalURI(u *url.URL) string { return u.Path } -// buildCanonicalHeaders constructs the canonical-headers + signed-headers -// pair for the presigned-URL request as it would have been canonicalised by -// the original signer. The wire URL does not carry the header values — they -// are reconstructed from known inputs: host comes from the URL itself, the -// X-K8s-Aws-Id value is the expected cluster name (constant-time-bound by -// the eventual signature compare). +// buildCanonicalHeaders reconstructs the canonical-headers/signed-headers pair for +// the presigned URL, using the URL host and expectedClusterName for X-K8s-Aws-Id. func buildCanonicalHeaders(host, xK8sAwsID string) (canonicalHeaders, signedHeadersList string) { headers := map[string]string{ presignedHostHeader: host, @@ -364,8 +319,7 @@ func hexSHA256(s string) string { return hex.EncodeToString(sum[:]) } -// emptyStringSHA256 is the SigV4 payload hash for an empty body — the value -// botocore/aws-sdk use when presigning non-S3 requests like GetCallerIdentity. +// emptyStringSHA256 is the SigV4 payload hash for an empty body. var emptyStringSHA256 = hexSHA256("") func hmacSHA256(key []byte, data string) []byte { @@ -381,11 +335,8 @@ func deriveSigningKey(secret, date, region, service string) []byte { return hmacSHA256(kService, presignedTerminator) } -// resolvePrincipalForVerify resolves an access key to its plaintext secret -// plus a PresignedCallerIdentity skeleton (X-K8s-Aws-Id field filled in by -// the caller after signature verification succeeds). Branches on AKID -// prefix — the same invariant the gateway SigV4 path relies on — so a -// misfiled record cannot be resolved by the wrong path. +// resolvePrincipalForVerify resolves an access key to its plaintext secret and a +// PresignedCallerIdentity skeleton. Branches on AKID prefix (session vs long-lived). func (s *STSServiceImpl) resolvePrincipalForVerify(accessKeyID string) (*PresignedCallerIdentity, string, error) { switch { case strings.HasPrefix(accessKeyID, SessionAccessKeyIDPrefix): diff --git a/spinifex/handlers/sts/service.go b/spinifex/handlers/sts/service.go index acc65f8d3..31b438e6b 100644 --- a/spinifex/handlers/sts/service.go +++ b/spinifex/handlers/sts/service.go @@ -5,63 +5,39 @@ import ( ) // STSService defines the interface for AWS STS operations exposed by spinifex. -// -// The implementation lives only in awsgw: STS shares the IAM at-rest envelope -// key and resolves roles via the in-process IAMService, so a daemon-side -// instance would need access to the master key — the very thing the awsgw / -// daemon trust boundary keeps separated. +// Implementation lives only in awsgw because STS shares the IAM master key, +// which must not cross the awsgw/daemon trust boundary. type STSService interface { - // AssumeRole mints short-lived temporary credentials bound to the target - // role after evaluating the role's trust policy against the caller. The - // caller identity is resolved by the gateway and passed in as plain - // strings to keep the handler unit-testable. + // AssumeRole mints temporary credentials after evaluating the role's trust policy. + // Caller identity is resolved by the gateway and passed as plain strings. AssumeRole(callerAccountID, callerARN, callerIdentity string, input *sts.AssumeRoleInput) (*sts.AssumeRoleOutput, error) - // AssumeRoleForInstance mints role-bound temporary credentials for an EC2 instance. - // It is the in-process IMDS entry point, NOT reachable over HTTPS: the caller is the - // synthesised EC2 service principal (trust must allow Service ec2.amazonaws.com). + // AssumeRoleForInstance mints instance-role credentials via the IMDS in-process path. + // Not reachable over HTTPS; trust policy must allow Service ec2.amazonaws.com. AssumeRoleForInstance(accountID, roleARN, instanceID string, durationSeconds int64) (*sts.AssumeRoleOutput, error) - // AssumeRoleWithWebIdentity exchanges an OIDC ID token (typically a - // projected K8s ServiceAccount token signed by an EKS cluster's - // per-cluster signing key) for short-lived AWS credentials bound to the - // target IAM role. Called anonymously — the caller is identified by the - // iss/sub/aud claims of the supplied JWT, not by SigV4. + // AssumeRoleWithWebIdentity exchanges an OIDC ID token for short-lived credentials + // bound to the target role. Anonymous — caller identity comes from JWT claims, not SigV4. AssumeRoleWithWebIdentity(input *sts.AssumeRoleWithWebIdentityInput) (*sts.AssumeRoleWithWebIdentityOutput, error) - // GetCallerIdentity returns the authenticated principal's account / ARN / - // UserId. AWS allows every authenticated principal to call this; the - // gateway does not gate it with checkPolicy. + // GetCallerIdentity returns the authenticated principal's account, ARN, and UserId. + // Allowed for every authenticated caller; not gated by checkPolicy. GetCallerIdentity(callerAccountID, callerARN, callerUserID string, input *sts.GetCallerIdentityInput) (*sts.GetCallerIdentityOutput, error) - // GetSessionToken exchanges the calling IAM user's long-lived credentials for - // short-lived session credentials bound to the SAME user identity — resolving - // back to arn:aws:iam::A:user/N, unlike AssumeRole. callerPrincipalType and - // callerAccessKeyID let the handler enforce that only a long-lived user - // (never a session) may call it. The gateway resolves these from the SigV4 - // context. + // GetSessionToken exchanges a long-lived user credential for short-lived ASIA credentials + // that still resolve to the same user identity. Only long-lived users (not sessions) may call it. GetSessionToken(callerAccountID, callerUserName, callerPrincipalType, callerAccessKeyID string, input *sts.GetSessionTokenInput) (*sts.GetSessionTokenOutput, error) - // VerifyPresignedGetCallerIdentity validates a SigV4-presigned URL for - // the sts:GetCallerIdentity action — the token shape produced by `aws - // eks get-token` — and resolves the calling principal. The EKS token - // webhook calls this over NATS as part of TokenReview processing. - // expectedClusterName is constant-time-compared against the signed - // X-K8s-Aws-Id header value to prevent cross-cluster replay. + // VerifyPresignedGetCallerIdentity validates a SigV4-presigned GetCallerIdentity URL + // (the `aws eks get-token` shape) and resolves the calling principal. + // expectedClusterName is constant-time-compared against X-K8s-Aws-Id to prevent cross-cluster replay. VerifyPresignedGetCallerIdentity(presignedURL, expectedClusterName string) (*PresignedCallerIdentity, error) - // LookupSessionCredential resolves an access-key ID to its stored - // SessionCredential record. Used by the SigV4 middleware to verify - // requests that carry an X-Amz-Security-Token. Returns (nil, nil) when - // the AKID is not a session credential — neither an ASIA-prefixed AKID - // nor a hit in the session-credentials bucket. The caller decides how to - // translate that miss (the SigV4 verifier maps it to InvalidClientTokenId - // on the ASIA path). + // LookupSessionCredential resolves an AKID to its stored SessionCredential. + // Returns (nil, nil) when the AKID is not a session credential or has no stored record. LookupSessionCredential(accessKeyID string) (*SessionCredential, error) - // VerifySessionToken constant-time-compares the wire-form session token - // (the X-Amz-Security-Token header) against the HMAC stored on cred. - // Returns true on match. Encapsulating the HMAC in STSService keeps the - // master key inside the service and off the gateway surface. + // VerifySessionToken constant-time-compares the wire token against the stored HMAC. + // Keeping the HMAC inside STSService prevents the master key from crossing the gateway surface. VerifySessionToken(cred *SessionCredential, wireToken string) bool } diff --git a/spinifex/handlers/sts/service_impl.go b/spinifex/handlers/sts/service_impl.go index a91fd318b..8386a3d4d 100644 --- a/spinifex/handlers/sts/service_impl.go +++ b/spinifex/handlers/sts/service_impl.go @@ -11,11 +11,9 @@ import ( const masterKeySize = 32 // AES-256, must match handlers_iam. -// STSServiceImpl implements STS operations using NATS JetStream KV. It -// resolves roles + IAM crypto through the in-process IAMService and persists -// session credentials in its own bucket. The master key is supplied by the -// awsgw startup path and shared by reference with IAMServiceImpl; STS does -// not own rotation. +// STSServiceImpl implements STS operations backed by NATS JetStream KV. +// Roles and IAM crypto are resolved through the in-process IAMService; the master key +// is shared with IAMServiceImpl and not rotated by STS. type STSServiceImpl struct { natsConn *nats.Conn js nats.JetStreamContext @@ -26,11 +24,9 @@ type STSServiceImpl struct { var _ STSService = (*STSServiceImpl)(nil) -// NewSTSServiceImpl constructs an STSServiceImpl. masterKey must be the same -// 32-byte secret IAM uses for at-rest envelope encryption; both services -// share it (see § Crypto reuse in the STS v1 plan). clusterSize sets the -// JetStream replication factor for the session bucket; pass 1 for -// single-node or test setups. +// NewSTSServiceImpl constructs an STSServiceImpl. masterKey must be the 32-byte +// key shared with IAMServiceImpl. clusterSize sets the JetStream replication factor; +// pass 1 for single-node or test setups. func NewSTSServiceImpl(natsConn *nats.Conn, iamSvc handlers_iam.IAMService, masterKey []byte, clusterSize int) (*STSServiceImpl, error) { if natsConn == nil { return nil, errors.New("nil NATS connection") diff --git a/spinifex/handlers/sts/session_credentials.go b/spinifex/handlers/sts/session_credentials.go index 8023cba09..f75d36697 100644 --- a/spinifex/handlers/sts/session_credentials.go +++ b/spinifex/handlers/sts/session_credentials.go @@ -21,42 +21,28 @@ const ( KVBucketSessionCredentials = "spinifex-iam-session-credentials" KVBucketSessionCredentialsVersion = 1 - // SessionAccessKeyIDPrefix is the AWS-defined prefix for STS-issued - // temporary credentials. Long-lived IAM access keys use AKIA*; the two - // namespaces live in disjoint KV buckets so a SigV4 prefix-first - // dispatch cannot be confused by a misfiled record. + // SessionAccessKeyIDPrefix is the AWS-defined prefix for STS temporary credentials. + // Long-lived keys use AKIA; the two namespaces live in separate KV buckets. SessionAccessKeyIDPrefix = "ASIA" - // janitorInterval is how often the sweep runs. Tight enough that - // expired-then-resurrected AKIDs would have to wait < this for cleanup; - // loose enough that the iterate-all-keys cost is amortised. + // janitorInterval is how often the credential sweep runs. janitorInterval = 5 * time.Minute - // janitorGracePeriod keeps just-expired records around so a client whose - // clock is slightly ahead still gets ExpiredToken (a diagnosable error) - // rather than InvalidClientTokenId (which masks the real cause). + // janitorGracePeriod keeps just-expired records so a client with a slightly fast + // clock gets ExpiredToken (diagnosable) rather than InvalidClientTokenId. janitorGracePeriod = 1 * time.Hour ) -// SessionCredential is the on-disk record for an STS-issued temporary -// credential. The raw SessionToken is returned to the client once and never -// persisted; only an HMAC of it is stored, so a bucket read by a process that -// lacks the master key cannot recover the token. +// SessionCredential is the on-disk record for an STS-issued temporary credential. +// Only the HMAC of the session token is stored; the raw token is returned once and never persisted. type SessionCredential struct { AccessKeyID string `json:"access_key_id"` SecretEncrypted string `json:"secret_encrypted"` SessionTokenHMAC string `json:"session_token_hmac"` AccountID string `json:"account_id"` - // PrincipalType records what kind of identity this session represents: - // "assumed-role" (minted by AssumeRole) or "user" (minted by - // GetSessionToken, resolving back to the calling IAM user). It governs how - // the SigV4 ASIA path rebuilds the caller identity and evaluates policy. - // - // Backward compat: an empty/absent value is treated as "assumed-role". - // Records are write-once and expire within hours (default 12h, max 36h), - // so no bucket-version bump or data migration is needed — the empty-string - // default covers any in-flight records minted before this field existed. + // PrincipalType is "assumed-role" or "user" (GetSessionToken). Empty is treated as "assumed-role" + // for backward compatibility with in-flight records minted before this field existed. PrincipalType string `json:"principal_type,omitempty"` AssumedRoleARN string `json:"assumed_role_arn"` @@ -69,13 +55,8 @@ type SessionCredential struct { CreatedAt time.Time `json:"created_at"` } -// initSessionCredentialsBucket opens (or creates) the session-credentials KV -// bucket and runs any pending migrations. -// -// History is fixed at 1: session credentials are write-once at mint and -// delete-once at expiry, never updated, so retained revisions waste storage -// and lengthen tombstone lifetime with no recovery benefit. This is -// deliberately lower than the IAM buckets (5-10). +// initSessionCredentialsBucket opens (or creates) the session-credentials KV bucket. +// History is fixed at 1: credentials are write-once at mint and delete-once at expiry. func initSessionCredentialsBucket(js nats.JetStreamContext, replicas int) (nats.KeyValue, error) { if replicas < 1 { replicas = 1 @@ -102,13 +83,8 @@ func initSessionCredentialsBucket(js nats.JetStreamContext, replicas int) (nats. return kv, nil } -// putSessionCredential persists a SessionCredential via a CAS-safe create. -// All writes to the session-credentials bucket MUST go through this helper: -// the ASIA-prefix invariant is enforced here at the writer, and a regression -// is caught by the bucket-prefix invariants test. -// -// Returns nats.ErrKeyExists on AKID collision so callers can retry with a -// freshly generated AKID. +// putSessionCredential persists a SessionCredential via CAS create, enforcing the ASIA-prefix +// invariant. Returns nats.ErrKeyExists on collision so callers can retry. func putSessionCredential(bucket nats.KeyValue, cred *SessionCredential) error { if cred == nil { return errors.New("nil session credential") @@ -127,10 +103,8 @@ func putSessionCredential(bucket nats.KeyValue, cred *SessionCredential) error { return nil } -// VerifySessionToken recomputes the HMAC of the wire-form session token under -// the IAM master key and constant-time-compares it against the stored value on -// cred. Returns true on match. A bucket-read by any process that lacks the -// master key cannot recover a usable token: the wire token is never persisted. +// VerifySessionToken recomputes the HMAC of the wire token and constant-time-compares +// it against the stored value. Returns true on match. func (s *STSServiceImpl) VerifySessionToken(cred *SessionCredential, wireToken string) bool { if cred == nil || wireToken == "" { return false @@ -143,17 +117,13 @@ func (s *STSServiceImpl) VerifySessionToken(cred *SessionCredential, wireToken s } got, err := base64.StdEncoding.DecodeString(computeTokenHMAC(s.masterKey, wireToken)) if err != nil { - // computeTokenHMAC always emits valid base64; defence in depth. - return false + return false // computeTokenHMAC always emits valid base64; defence in depth } return subtle.ConstantTimeCompare(got, expected) == 1 } -// LookupSessionCredential resolves an access-key ID to its stored -// SessionCredential. Returns (nil, nil) when the AKID does not start with -// "ASIA" or when no record exists for it — the SigV4 verifier translates -// that miss into InvalidClientTokenId on the ASIA path. Any other failure -// (unmarshal, transport) is returned as an error. +// LookupSessionCredential resolves an AKID to its stored SessionCredential. +// Returns (nil, nil) when the AKID lacks the ASIA prefix or has no record. func (s *STSServiceImpl) LookupSessionCredential(accessKeyID string) (*SessionCredential, error) { if !strings.HasPrefix(accessKeyID, SessionAccessKeyIDPrefix) { return nil, nil @@ -172,12 +142,8 @@ func (s *STSServiceImpl) LookupSessionCredential(accessKeyID string) (*SessionCr return &cred, nil } -// RunJanitor periodically removes session credentials whose ExpiresAt is -// further in the past than janitorGracePeriod. The sweep is idempotent — -// delete-of-already-deleted is a JetStream no-op — so multiple awsgw -// instances may run independently without coordination. -// -// Blocks until ctx is cancelled. Intended to be invoked as `go svc.RunJanitor(ctx)`. +// RunJanitor periodically removes expired session credentials. Idempotent; multiple +// awsgw instances may run it concurrently. Blocks until ctx is cancelled. func (s *STSServiceImpl) RunJanitor(ctx context.Context) { ticker := time.NewTicker(janitorInterval) defer ticker.Stop() @@ -197,10 +163,8 @@ func (s *STSServiceImpl) RunJanitor(ctx context.Context) { } } -// sweepExpired iterates the session-credentials bucket and deletes every -// record whose ExpiresAt is older than now - janitorGracePeriod. Per-key -// errors are logged and skipped — one corrupt record must not stall the -// sweep. Returns the number of records deleted (exposed for tests). +// sweepExpired deletes all records whose ExpiresAt is past the grace period. +// Per-key errors are logged and skipped; returns the delete count. func (s *STSServiceImpl) sweepExpired(now time.Time) int { cutoff := now.Add(-janitorGracePeriod) diff --git a/spinifex/handlers/sts/web_identity_trust_policy.go b/spinifex/handlers/sts/web_identity_trust_policy.go index bdda8a2f0..60602650d 100644 --- a/spinifex/handlers/sts/web_identity_trust_policy.go +++ b/spinifex/handlers/sts/web_identity_trust_policy.go @@ -12,53 +12,31 @@ import ( ) const ( - // stsActionAssumeRoleWithWebIdentity is the only Action whose statements - // may carry a Condition block (per the validator in handlers_iam). The - // constant is duplicated here so the action-match path stays in this - // package and a future rename surfaces as a compile-time mismatch. + // stsActionAssumeRoleWithWebIdentity is the only action that may carry a Condition block. stsActionAssumeRoleWithWebIdentity = "sts:AssumeRoleWithWebIdentity" - // IRSA convention: pods bake a single audience value into the projected - // ServiceAccount token. The handler accepts any token whose `aud` claim - // contains this exact string (claim is a JSON string or array — handled - // upstream via jwt.ClaimStrings). + // irsaExpectedAudience is the audience value IRSA pods bake into ServiceAccount tokens. irsaExpectedAudience = "sts.amazonaws.com" ) -// webIdentityContext is the resolved-and-verified set of facts about an -// AssumeRoleWithWebIdentity caller that the trust-policy evaluator needs to -// match against. The handler builds it after JWT signature + claim validation; -// the evaluator does not re-verify, it only matches. +// webIdentityContext holds the JWT-verified facts the trust-policy evaluator matches against. type webIdentityContext struct { - // federatedPrincipalARN is `arn:aws:iam::{roleAccountID}:oidc-provider/{issuerHostPath}` - // (the value the role trust policy's Principal.Federated entry must equal). + // federatedPrincipalARN is the oidc-provider ARN Principal.Federated must equal. federatedPrincipalARN string - // issuer is the literal `iss` claim value. Used to build the condition-key - // prefix `{iss}:sub` / `{iss}:aud` per the IRSA convention. + // issuer is the literal `iss` claim; used to build condition-key prefixes like `{iss}:sub`. issuer string - // subject is the literal `sub` claim value. + // subject is the literal `sub` claim. subject string - // audience is the list of audience values from the JWT `aud` claim. - // Membership semantics — a single Condition StringEquals on `{iss}:aud` - // matches if any wired audience equals the condition value. + // audience holds the JWT `aud` values; a StringEquals condition matches if any entry equals the expected value. audience []string } -// evalTrustPolicyForWebIdentity is the sibling of evalTrustPolicy for the -// AssumeRoleWithWebIdentity action. AWS semantics: explicit-deny wins across -// the document, then any matching Allow grants. A statement matches when: -// -// - Action contains sts:AssumeRoleWithWebIdentity (wildcards accepted — -// sts:* and "*" — same as evalTrustPolicy for symmetry). -// - Principal.Federated equals webIdentityContext.federatedPrincipalARN. -// - Every Condition operator entry holds (currently only StringEquals). -// -// Returns nil on grant; awserrors.ErrorAccessDenied on no-match or explicit -// deny; a wrapped error on stored-doc corruption (matches evalTrustPolicy's -// fail-closed-loudly contract). +// evalTrustPolicyForWebIdentity evaluates an AssumeRoleWithWebIdentity trust policy. +// Explicit-deny wins; a statement matches when Action, Principal.Federated, and all Conditions hold. +// Returns nil on grant or AccessDenied/error on deny/corruption. func evalTrustPolicyForWebIdentity(docJSON string, ctx webIdentityContext) error { doc, err := handlers_iam.ValidateTrustPolicyDocument(docJSON) if err != nil { @@ -120,23 +98,16 @@ func matchWebIdentityAction(actions []string) bool { return false } -// matchFederatedPrincipal accepts Principal.Federated as either a string or -// an array of strings, mirroring the AWS Principal.AWS shape. Returns false -// on Principal: "*" — Federated does not honour the bare wildcard the way -// Principal: "*" does; a web-identity policy that means "any OIDC issuer" -// must say so explicitly via a Principal.Federated entry per issuer ARN. -// Unrelated top-level keys (Service, AWS) skip at the entry level — same -// open-set semantics as matchTrustPrincipal in assume_role.go. +// matchFederatedPrincipal accepts Principal.Federated as a string or array. +// Returns false for Principal: "*" — Federated does not honour the bare wildcard. +// Unrelated top-level keys (Service, AWS) are skipped. func matchFederatedPrincipal(raw json.RawMessage, expectedARN string) (bool, error) { if len(raw) == 0 { return false, nil } var asString string if err := json.Unmarshal(raw, &asString); err == nil { - // Principal: "*" → does NOT match Federated. The web-identity action - // has no analogue to the AWS-account-wide root principal and the bare - // wildcard would silently grant any OIDC issuer that could fish a - // trust policy. Fail closed. + // Principal: "*" does not match Federated — fail closed to prevent any-issuer grants. return false, nil } var m map[string]json.RawMessage @@ -163,19 +134,8 @@ func matchFederatedPrincipal(raw json.RawMessage, expectedARN string) (bool, err return false, nil } -// conditionsHold evaluates a Condition block against the web-identity context. -// The validator restricts the operator to StringEquals — anything else is a -// stored-doc inconsistency and the safe default is "do not match". -// -// IRSA-shaped keys: -// -// {iss}:sub → exact equality with the JWT `sub` claim -// {iss}:aud → set-membership against the JWT `aud` claim list -// (`aud` is a ClaimStrings — one or many) -// -// Each StringEquals entry value can be a single string or a list — both -// shapes match if any entry compares equal. Missing-condition = grant -// (no Condition → unrestricted Allow on the Federated principal). +// conditionsHold evaluates a Condition block. Only StringEquals is supported; +// keys are {iss}:sub (equality) and {iss}:aud (set-membership). No Condition = grant. func conditionsHold(raw json.RawMessage, ctx webIdentityContext) bool { if !isCondRawNonEmpty(raw) { return true @@ -186,9 +146,7 @@ func conditionsHold(raw json.RawMessage, ctx webIdentityContext) bool { } for op, kv := range ops { if op != "StringEquals" { - // Validator rejects other operators at write time; reaching here - // means a stored doc was tampered with bypassing validation. Fail - // closed rather than silently grant. + // Validator rejects other operators at write time; a non-StringEquals here means tampering. return false } issPrefix := strings.TrimSuffix(ctx.issuer, "/") @@ -207,11 +165,7 @@ func conditionsHold(raw json.RawMessage, ctx webIdentityContext) bool { return false } default: - // Unknown condition key — fail closed. v1 only models the two - // IRSA-shaped keys above; an unknown key in a stored doc most - // likely means the policy author wanted a stricter check the - // evaluator cannot honour, so refusing the grant is safer - // than ignoring the key. + // Unknown condition key — fail closed rather than silently ignore a stricter check. return false } } @@ -219,8 +173,7 @@ func conditionsHold(raw json.RawMessage, ctx webIdentityContext) bool { return true } -// isCondRawNonEmpty mirrors the validator's empty-detection so an empty `{}` -// Condition (validator-accepted) does not enter the operator loop. +// isCondRawNonEmpty reports whether a Condition RawMessage has meaningful content. func isCondRawNonEmpty(raw json.RawMessage) bool { if len(raw) == 0 { return false diff --git a/spinifex/handlers/sysinstance/launcher.go b/spinifex/handlers/sysinstance/launcher.go index 7e6cb1f52..855ca7368 100644 --- a/spinifex/handlers/sysinstance/launcher.go +++ b/spinifex/handlers/sysinstance/launcher.go @@ -1,29 +1,12 @@ -// Package sysinstance holds the boot-agnostic contract for launching -// system-managed VMs. System instances are hidden from customer -// DescribeInstances calls — BootDirect VMs (LB) via system-account ownership, -// BootAMI VMs (EKS control plane, owned by the customer account so their ENI -// can live in the customer VPC) via their ManagedBy tag. Two boot styles are -// supported: -// -// - BootDirect: a direct-boot microVM (bundled vmlinuz+initramfs, per-VM -// config delivered via QEMU fw_cfg blobs; no AMI, volume, or cloud-init). -// Used by the ELBv2 LB agent. -// - BootAMI: a full AMI boot (root volume cloned from an AMI, cloud-init -// user-data + network-config seed, q35 machine). Used by the EKS K3s -// control-plane VM. -// -// The types live here (not in handlers/elbv2) so both ELBv2 and EKS can launch -// system instances without an eks→elbv2 dependency. handlers/elbv2 re-exports -// them as type aliases for source compatibility. +// Package sysinstance holds the boot-agnostic contract for system-managed VMs. +// Hidden from DescribeInstances; supports two boot styles: BootDirect (fw_cfg microVM) +// and BootAMI (AMI + cloud-init). Types live here to avoid an eks→elbv2 import cycle. package sysinstance import "errors" -// ErrSystemInstanceNotFound is returned by TerminateSystemInstance when the -// target instance is unknown to the daemon — already terminated, or never -// present on this node. Teardown callers treat it as idempotent success so a -// retried delete (e.g. EKS DeleteCluster after the VM already drained) does not -// wedge forever on a VM that is legitimately gone. +// ErrSystemInstanceNotFound is returned by TerminateSystemInstance when the instance is unknown. +// Callers treat it as idempotent success so a retried delete does not block on a gone VM. var ErrSystemInstanceNotFound = errors.New("sysinstance: instance not found") // BootMode selects how a system instance boots. @@ -36,44 +19,29 @@ const ( BootAMI BootMode = "ami" ) -// SystemInstanceLauncher is the subset of daemon functionality needed to launch -// and terminate system-managed VMs. Defined here to avoid a circular import -// between the launching handlers (elbv2, eks) and daemon. +// SystemInstanceLauncher launches and terminates system-managed VMs. +// Defined here to avoid a circular import between elbv2/eks and daemon. type SystemInstanceLauncher interface { - // LaunchSystemInstance creates and starts a system-managed VM. The VM is - // assigned to the system account and not visible to customers. Returns the - // instance ID and private IP once the VM is running. + // LaunchSystemInstance creates and starts a system-managed VM, returning its ID and private IP. LaunchSystemInstance(input *SystemInstanceInput) (*SystemInstanceOutput, error) // TerminateSystemInstance stops and cleans up a system-managed VM. TerminateSystemInstance(instanceID string) error } -// SystemInstanceInput describes a system VM to launch. The BootMode field -// selects which subset of fields is consumed: -// -// - BootDirect uses NICs, LBAgentEnv, CACert (fw_cfg blobs); ImageID/UserData -// are ignored. -// - BootAMI uses ImageID and UserData (cloud-init); NICs/LBAgentEnv/CACert are -// ignored. The mgmt NIC is allocated by the daemon and rendered into the -// cloud-init network-config from the VM record. -// -// JSON tags are explicit because this struct crosses the -// system.LaunchInstance.* NATS subject root when a launch is handed off to the -// cluster — see daemon/daemon_system_dispatch.go. +// SystemInstanceInput describes a system VM to launch. +// BootDirect uses NICs/LBAgentEnv/CACert; BootAMI uses ImageID/UserData. +// JSON tags are explicit as this struct crosses NATS subjects. type SystemInstanceInput struct { - // BootMode selects the boot style. Empty defaults to BootDirect for - // backward compatibility with the original ELBv2-only contract. + // BootMode selects the boot style; empty defaults to BootDirect. BootMode BootMode `json:"boot_mode,omitempty"` - // ManagedBy is the spinifex:managed-by tag value stamped on the instance so - // the UI/customer listings hide it (e.g. "elbv2", "eks"). + // ManagedBy is the spinifex:managed-by tag value (e.g. "elbv2", "eks"). ManagedBy string `json:"managed_by,omitempty"` InstanceType string `json:"instance_type"` // e.g. "sys.micro" SubnetID string `json:"subnet_id"` // VPC subnet for networking - // ENI fields — the VM always uses a pre-created ENI (the primary ENI). ENIID string `json:"eni_id"` ENIMac string `json:"eni_mac"` ENIIP string `json:"eni_ip"` diff --git a/spinifex/instancetypes/definitions.go b/spinifex/instancetypes/definitions.go index 4657d3c91..2f3567877 100644 --- a/spinifex/instancetypes/definitions.go +++ b/spinifex/instancetypes/definitions.go @@ -68,14 +68,8 @@ type cpuGeneration struct { } // vendorSiblingFamily maps an x86_64 family to its cross-vendor counterpart -// (Intel ↔ AMD) within the same generation tier. Resource shape (vCPU / -// memory ratio per size) is identical between siblings, so a host that can -// run t3.medium can run t3a.medium on the same KVM substrate. Used by -// generateForGeneration so any x86_64 host accepts requests for either -// vendor family — load balances across mixed Intel + AMD clusters. -// -// ARM (Graviton t4g/c?g/m?g/r?g) and legacy Intel-only (t2, c4, m4, r4) -// families have no cross-vendor sibling and are omitted. +// (Intel ↔ AMD) within the same generation tier. Siblings share the same vCPU/memory +// ratios, so a mixed Intel+AMD cluster can serve either vendor family. var vendorSiblingFamily = map[string]string{ "t3": "t3a", "t3a": "t3", "c5": "c5a", "c5a": "c5", @@ -130,8 +124,6 @@ type instanceFamilyDef struct { currentGen bool } -// Size tables for each instance category - var burstableSizes = []instanceSize{ {"nano", 2, 0.5}, {"micro", 2, 1}, @@ -299,26 +291,9 @@ var systemSizes = []instanceSize{ {"medium", 2, 4}, // 2 vCPU, 4 GB — EKS k3s control-plane VMs } -// instanceFamilyDefs defines all supported instance families with their vendor and sizes. -// -// We support the core families across burstable, general purpose, compute optimized, -// and memory optimized categories. The following AWS family categories are intentionally -// excluded because they require specialized hardware not available on standard bare-metal hosts: -// -// - Local disk variants (d/n suffixes): c5d, c5ad, c5n, m5d, m5ad, m5n, m5dn, m5zn, r5d, r5ad, -// r5n, r5dn, r5b, c6gd, c6gn, c6id, c6in, m6gd, m6id, m6idn, m6in, r6gd, r6id, r6idn, r6in, -// c7gd, c7gn, c7i-flex, m7gd, m7i-flex, r7gd, r7iz, c8gd, c8gn, c8i-flex, m8gd, m8i-flex, -// r8gd, r8gn, r8gb, r8i-flex — require NVMe instance storage or enhanced networking -// - Multi-GPU only: g4dn 12xlarge, p3 8/16xlarge, p3dn 24xlarge, p4d/p4de 24xlarge, p5/p5e 48xlarge -// - AWS-proprietary accelerators: inf1, inf2, trn1, trn2, dl1, dl2q — Inferentia/Trainium chips, AWS-only hardware -// - Unsupported GPU: g2, g6f, gr6f, p2, p6 — too old (g2/p2), future/unannounced (p6), or insufficient hardware availability (g6f/gr6f) -// - Storage optimized: d2, d3, d3en, h1, i2-i8g, i7ie, i8ge, im4gn, is4gen — require dense HDD/NVMe -// - FPGA: f1, f2 — require FPGA hardware -// - High memory: u-*, u7i-*, x1, x1e, x2gd, x2idn, x2iedn, x2iezn, x8g — require TB-scale memory -// - High frequency: z1d — specialized high clock-speed instances -// - (unsupported) Dedicated host: mac*, hpc* — require macOS/Apple hardware or HPC interconnects -// - (unsupported) Video: vt1 — requires video transcoding hardware -// - Legacy (pre-gen4): a1, c1, c3, cc1, cc2, cg1, cr1, hi1, hs1, m1, m2, m3, r3, t1 +// instanceFamilyDefs lists all supported instance families. Excluded families require +// hardware not available on bare-metal: local NVMe, Inferentia/Trainium, FPGAs, TB-scale +// memory, multi-GPU-only sizes, or macOS/HPC interconnects. var instanceFamilyDefs = []instanceFamilyDef{ // Burstable {name: "t2", sizes: burstableSizes, currentGen: false}, diff --git a/spinifex/instancetypes/eni_limits.go b/spinifex/instancetypes/eni_limits.go index cfe0aa34e..e1cdd0a40 100644 --- a/spinifex/instancetypes/eni_limits.go +++ b/spinifex/instancetypes/eni_limits.go @@ -2,14 +2,10 @@ package instancetypes import "strings" -// defaultMaxENIs is the fallback ENI cap for instance types not enumerated -// in maxENIsByType / maxENIsByFamily. Matches AWS's "small general purpose" -// envelope; deliberately conservative so undersized hosts cannot exhaust -// host taps from a single VM. +// defaultMaxENIs is the conservative fallback ENI cap for unrecognized types. const defaultMaxENIs = 4 -// maxENIsByType overrides maxENIsByFamily for size-specific caps where AWS -// docs publish a different number from the family default. +// maxENIsByType overrides maxENIsByFamily for size-specific AWS ENI caps. var maxENIsByType = map[string]int{ "m5.large": 3, "m5.xlarge": 3, @@ -17,10 +13,7 @@ var maxENIsByType = map[string]int{ "m5.4xlarge": 8, } -// maxENIsByFamily is consulted when no type-specific entry exists. Sized to -// match AWS's published per-family ENI cap for representative members of -// each family; coarse on purpose for v1 — refine when a customer hits the -// cap. Keys are the dot-stripped family prefix (e.g. "t3", "m5"). +// maxENIsByFamily is the per-family fallback when no type-specific entry exists. var maxENIsByFamily = map[string]int{ "t2": 3, "t3": 3, @@ -39,11 +32,8 @@ var maxENIsByFamily = map[string]int{ "m8g": 8, } -// MaxENIsForType returns the total ENI cap for the given instance type -// (primary + secondaries). Lookup precedence: explicit type → family → -// defaultMaxENIs. The value is the hardware-imposed AWS limit, not a -// per-host quota; the host's hot-plug slot pool is sized as -// MaxENIsForType - 1 (the primary ENI consumes no hot-plug slot). +// MaxENIsForType returns the total ENI cap (primary + secondaries) for an instance type. +// Precedence: explicit type → family → defaultMaxENIs. func MaxENIsForType(instanceType string) int { if n, ok := maxENIsByType[instanceType]; ok { return n @@ -57,10 +47,8 @@ func MaxENIsForType(instanceType string) int { return defaultMaxENIs } -// HotPlugENISlotsForType returns the number of PCIe root ports that must be -// pre-allocated at VM start for runtime ENI hot-plug. Equals -// MaxENIsForType - 1 since the primary ENI is wired at boot via the fixed -// netdev path and does not consume a hot-plug slot. Always ≥ 0. +// HotPlugENISlotsForType returns MaxENIsForType - 1 (the primary ENI occupies +// no hot-plug slot). Always ≥ 0. func HotPlugENISlotsForType(instanceType string) int { n := MaxENIsForType(instanceType) - 1 if n < 0 { diff --git a/spinifex/instancetypes/generate.go b/spinifex/instancetypes/generate.go index c49ecc563..00b39d156 100644 --- a/spinifex/instancetypes/generate.go +++ b/spinifex/instancetypes/generate.go @@ -17,12 +17,8 @@ func IsSystemType(name string) bool { return strings.HasPrefix(name, "sys.") } -// SpecForSystemType returns the vCPU and memory (GB) footprint of a system -// instance type (sys.*). System VMs consume host capacity like any guest, so -// the EKS HA control-plane scheduler sizes spread placement against these -// figures rather than the customer-facing node.status type list (which omits -// sys.* types). ok is false for an unknown name or a non-system type. The -// footprint is arch-independent, so the host GOARCH is used for the lookup. +// SpecForSystemType returns the vCPU and memory (GB) footprint of a sys.* type. +// ok is false for unknown or non-system types. func SpecForSystemType(name string) (vcpu int, memGB float64, ok bool) { if !IsSystemType(name) { return 0, 0, false @@ -36,18 +32,10 @@ func SpecForSystemType(name string) (vcpu int, memGB float64, ok bool) { true } -// generateForGeneration creates the instance type map for the given CPU generation. -// It generates all instance families matching the generation's family list across -// burstable, general purpose, compute optimized, and memory optimized categories. -// -// On x86_64 hosts the cross-vendor sibling family for each member of -// gen.families is also emitted (e.g. Intel Skylake's "t3" pulls in "t3a" so -// AMD-targeted RunInstances requests can land on this host). ARM and -// legacy Intel-only families have no sibling and are unaffected. +// generateForGeneration creates instance types for the given CPU generation. +// Cross-vendor siblings are included on x86_64 so a mixed Intel+AMD cluster +// can serve either vendor family. func generateForGeneration(gen cpuGeneration, arch string) map[string]*ec2.InstanceTypeInfo { - // Build a set of allowed families for fast lookup. Include cross-vendor - // siblings so a mixed Intel+AMD cluster routes RunInstances of either - // vendor family to any x86_64 host with capacity. allowed := make(map[string]bool, len(gen.families)*2) for _, f := range gen.families { allowed[f] = true @@ -124,11 +112,8 @@ func generateSystemTypes(arch string) map[string]*ec2.InstanceTypeInfo { return types } -// generateGPUTypes generates instance types for the discovered GPU models. -// For each unique GPU family, it emits all sizes for that family with GpuInfo populated. -// Duplicates (multiple GPUs of the same model) produce only one set of types. // GenerateGPUTypes returns InstanceTypeInfo entries for each GPU model with GpuInfo populated. -// It is exported for use by the daemon's hot-reload path. +// Exported for use by the daemon's hot-reload path. func GenerateGPUTypes(models []GPUModel, arch string) map[string]*ec2.InstanceTypeInfo { return generateGPUTypes(models, arch) } diff --git a/spinifex/lbagent/agent.go b/spinifex/lbagent/agent.go index 3e7ad797c..b31605e8c 100644 --- a/spinifex/lbagent/agent.go +++ b/spinifex/lbagent/agent.go @@ -31,16 +31,11 @@ const ( DefaultConfigPath = "/etc/haproxy/haproxy.cfg" DefaultPIDPath = "/run/haproxy.pid" - // CertDir is the fixed directory holding TLS certificate PEM files for - // HTTPS listeners. The daemon renders `ssl crt` paths under this dir; the - // agent writes the delivered PEM files here (0600) before reload and - // rejects any delivered path that escapes it. + // CertDir holds TLS PEM files for HTTPS listeners (0600). Delivered paths + // are constrained to this dir to prevent traversal writes. CertDir = "/etc/haproxy/certs" - // Nginx paths for the NLB (L4) data plane. NLBs render an nginx `stream` - // config because HAProxy cannot load-balance UDP; ALBs keep using the - // HAProxy paths above. NginxCertDir holds TLS PEM files for TLS listeners - // (referenced by both ssl_certificate and ssl_certificate_key). + // Nginx paths for the NLB (L4) data plane. ALBs use the HAProxy paths above. NginxConfigPath = "/etc/nginx/nginx.conf" NginxPIDPath = "/run/nginx.pid" NginxCertDir = "/etc/nginx/certs" @@ -49,10 +44,7 @@ const ( pollInterval = 5 * time.Second ) -// Data-plane engines, duplicated from handlers/elbv2 to avoid an import cycle -// (handlers/elbv2 imports this package for path constants). The agent runs -// HAProxy for ALBs and nginx `stream` for NLBs, selected by the GetLBConfig -// response. +// Data-plane engine names (duplicated from handlers/elbv2 to avoid an import cycle). const ( EngineHAProxy = "haproxy" EngineNginx = "nginx" @@ -339,11 +331,8 @@ func (a *Agent) signedPost(params url.Values) ([]byte, error) { return respBody, nil } -// writeCertFiles writes each delivered TLS certificate PEM to its path 0600, -// creating certDir as needed. Every path is constrained to certDir — a -// delivered path that escapes it (traversal, absolute elsewhere) is rejected -// rather than written, so a malformed/hostile response can't drop files across -// the filesystem. +// writeCertFiles writes each delivered TLS PEM to its path (0600) under certDir. +// Paths that escape certDir are rejected to prevent traversal writes. func (a *Agent) writeCertFiles(certDir string, certs []certFile) error { if len(certs) == 0 { return nil @@ -398,10 +387,9 @@ func reloadHAProxy(configPath, pidPath string) error { var cmd *exec.Cmd if oldPID > 0 { - // Graceful reload: new worker starts, old workers finish in-flight requests + // Graceful reload: new worker replaces old in-flight requests. cmd = exec.Command("haproxy", "-f", configPath, "-p", pidPath, "-D", "-sf", strconv.Itoa(oldPID)) } else { - // Fresh start cmd = exec.Command("haproxy", "-f", configPath, "-p", pidPath, "-D") } @@ -413,14 +401,9 @@ func reloadHAProxy(configPath, pidPath string) error { } // reloadNginx validates then starts or reloads nginx for the NLB data plane. -// `nginx -t` fails closed before any reload, mirroring HAProxy's -// validate-before-bind guarantee (a bad config never replaces a good one). +// `nginx -t` fails closed before any reload (a bad config never replaces a good one). func reloadNginx(configPath, pidPath string) error { - // On a stripped Alpine microvm initramfs the nginx runtime prefix dirs may - // be absent, so `nginx -t`/start fails closed on a missing pid dir or temp - // path — and the agent never enters nginx mode, never probes, and reports - // empty health (NLB targets stuck at 0/N). Pre-create them, mirroring - // reloadHAProxy's socket-dir pre-create. + // Pre-create runtime dirs; stripped Alpine microvms may not have them. for _, dir := range []string{filepath.Dir(pidPath), "/var/lib/nginx", "/var/lib/nginx/tmp", "/var/log/nginx"} { _ = os.MkdirAll(dir, 0o750) } @@ -432,20 +415,13 @@ func reloadNginx(configPath, pidPath string) error { var cmd *exec.Cmd if readPID(pidPath) > 0 { - // Master running: signal a graceful reload. cmd = exec.Command("nginx", "-s", "reload", "-c", configPath) } else { - // Fresh start (nginx daemonizes; pid path comes from the config). cmd = exec.Command("nginx", "-c", configPath) } - // nginx daemonizes on start, and the rendered config uses `error_log stderr`, - // so the forked master and workers inherit and keep this command's stderr - // open. CombinedOutput pipes stderr and blocks on Wait until EOF — which the - // long-lived daemon never sends — hanging the agent here forever before it - // ever reports health, leaving NLB targets stuck at 0/N. Capture to a temp - // file instead: Wait then blocks only on the short-lived foreground process, - // which exits once it has daemonized. + // nginx daemonizes and inherits stderr, so CombinedOutput blocks forever. + // Capture to a temp file; Wait returns once the foreground launcher exits. logFile, err := os.CreateTemp("", "nginx-reload-*.log") if err != nil { return fmt.Errorf("create nginx log: %w", err) diff --git a/spinifex/lbagent/health.go b/spinifex/lbagent/health.go index 74a73b59c..7510a7954 100644 --- a/spinifex/lbagent/health.go +++ b/spinifex/lbagent/health.go @@ -22,15 +22,8 @@ type HealthReport struct { Servers []ServerStatus `json:"servers"` } -// queryHAProxyStats connects to the HAProxy stats socket, runs "show stat", -// and parses the CSV output to extract backend server health status. -// -// HAProxy CSV format: pxname,svname,... fields. We need: -// - Column 0: pxname (backend name) -// - Column 1: svname (server name) -// - Column 17: status (UP, DOWN, etc.) -// -// Rows where svname is "FRONTEND" or "BACKEND" are aggregates — skip them. +// queryHAProxyStats reads backend server health from the HAProxy stats socket. +// Parses CSV "show stat" output; cols 0/1/17 = pxname/svname/status. func queryHAProxyStats(socketPath string) ([]ServerStatus, error) { conn, err := net.DialTimeout("unix", socketPath, 2*time.Second) if err != nil { diff --git a/spinifex/lbagent/healthcheck.go b/spinifex/lbagent/healthcheck.go index 94ad0371e..14424755e 100644 --- a/spinifex/lbagent/healthcheck.go +++ b/spinifex/lbagent/healthcheck.go @@ -10,11 +10,8 @@ import ( "time" ) -// HealthTarget is one backend the agent must actively health-check, delivered -// by GetLBConfig for the nginx (NLB) data plane. OSS nginx `stream` has no -// active upstream probing, so the agent probes each target every tick and -// reports the result. ServerName is the daemon-computed name the health checker -// keys on — echoed back verbatim so the agent needs no ELBv2 naming logic. +// HealthTarget is a backend the agent actively health-checks for the nginx NLB +// data plane (OSS nginx stream has no built-in upstream probing). type HealthTarget struct { ServerName string `xml:"ServerName"` Address string `xml:"Address"` // ip:port to probe @@ -25,11 +22,8 @@ type HealthTarget struct { // probeTimeout bounds each individual target probe. const probeTimeout = 3 * time.Second -// probeHealthTargets actively probes every delivered health target and returns -// one ServerStatus per target (Status "UP"/"DOWN"). The Server field carries -// the daemon-supplied ServerName so the daemon health checker matches it -// directly. An empty target list yields a nil report (the daemon early-returns -// on empty reports, leaving health untouched). +// probeHealthTargets probes each health target and returns one ServerStatus per +// target. An empty list returns nil. func probeHealthTargets(targets []HealthTarget) []ServerStatus { if len(targets) == 0 { return nil @@ -45,10 +39,8 @@ func probeHealthTargets(targets []HealthTarget) []ServerStatus { return out } -// probeOne performs a single health check against one target. TCP succeeds on a -// completed dial; HTTP/HTTPS succeeds on a response status < 400 (HTTPS skips -// certificate verification, matching how AWS NLB HTTPS health checks treat the -// target's self-signed cert). +// probeOne performs a single health check. TCP: completed dial. HTTP/HTTPS: +// status < 400 (HTTPS skips cert verification, matching AWS NLB behavior). func probeOne(t HealthTarget) bool { switch strings.ToUpper(t.Protocol) { case "HTTP", "HTTPS": diff --git a/spinifex/migrate/001_security_group.go b/spinifex/migrate/001_security_group.go index 8c8f41090..61ca3679d 100644 --- a/spinifex/migrate/001_security_group.go +++ b/spinifex/migrate/001_security_group.go @@ -68,11 +68,8 @@ func init() { }) } -// backfillSGRuleIDs assigns a fresh sgr- ID to any rule on the record at key -// whose RuleId is empty. Records with all IDs already populated are skipped -// (no write, no revision bump) so the migration is idempotent. CAS conflicts -// are retried up to sgMigrationMaxRetries; on exhaustion the most recent -// underlying error is wrapped into the returned failure. +// backfillSGRuleIDs assigns sgr- IDs to any rule missing one. Skips records +// already fully populated. CAS conflicts retry up to sgMigrationMaxRetries. func backfillSGRuleIDs(ctx KVContext, key string) error { var lastErr error for attempt := range sgMigrationMaxRetries { @@ -112,10 +109,7 @@ func backfillSGRuleIDs(ctx KVContext, key string) error { return fmt.Errorf("marshal %s: %w", key, err) } if _, err := ctx.KV.Update(key, data, entry.Revision()); err != nil { - // nats.ErrKeyExists is the sentinel returned by KV.Update on a - // revision mismatch: it wraps a JetStream APIError with - // ErrorCode JSStreamWrongLastSequence (10071), which APIError.Is - // matches on. + // nats.ErrKeyExists = revision mismatch (JSStreamWrongLastSequence). if errors.Is(err, nats.ErrKeyExists) { ctx.Logger.Warn("SG migration CAS conflict, retrying", "key", key, "attempt", attempt+1) lastErr = err diff --git a/spinifex/migrate/migrate.go b/spinifex/migrate/migrate.go index 41f695352..cafc84821 100644 --- a/spinifex/migrate/migrate.go +++ b/spinifex/migrate/migrate.go @@ -107,17 +107,8 @@ func (r *Registry) RunKVWithJetStream(bucket string, kv nats.KeyValue, js nats.J return r.runKV(bucket, kv, js, targetVersion) } -// RunKV executes all pending migrations for a KV bucket up to targetVersion. -// It reads the current version via utils.ReadVersion, runs each migration in -// order, and stamps the version via utils.WriteVersion after each step. -// -// When no migrations are registered for a bucket, RunKV stamps targetVersion -// directly — equivalent to a bare utils.WriteVersion. This is the common case -// for buckets that have not yet needed a schema change. -// -// If targetVersion is ahead of the registered migration chain, RunKV returns -// an error — migration authors must register migrations before bumping version -// constants. +// RunKV applies pending KV migrations up to targetVersion. Stamps directly when +// no migrations are registered (fresh bucket). Errors if the chain is incomplete. func (r *Registry) RunKV(bucket string, kv nats.KeyValue, targetVersion int) error { return r.runKV(bucket, kv, nil, targetVersion) } @@ -134,28 +125,17 @@ func (r *Registry) runKV(bucket string, kv nats.KeyValue, js nats.JetStreamConte all := r.kvMigrations[bucket] - // Fresh bucket with no registered migrations — stamp directly. This is - // the common first-init path for buckets that have not yet needed a - // schema change. Equivalent to the pre-framework utils.WriteVersion call. - // Note: we do NOT stamp directly for a non-fresh bucket with no migrations - // (e.g. someone bumped a BucketVersion constant from 1 to 2 without adding - // a migration) — that case falls through and errors with "no migrations - // registered" to force the author to add the migration. + // Fresh bucket, no migrations: stamp directly (common first-init path). if current == 0 && len(all) == 0 { return utils.WriteVersion(kv, targetVersion) } - // Fresh bucket with migrations registered: there is no v0 schema by - // convention (BucketVersion constants start at 1). Treat the bucket as - // being at the bottom of the registered chain so the chain validation - // below accepts the first migration. Without this, every first-ever - // KV migration would trip the "chain gap" error on fresh deployments. + // Fresh bucket with migrations: no v0 schema by convention; start at chain bottom. if current == 0 { current = all[0].FromVersion // sorted ascending by FromVersion } - // Migrations are registered (or an existing bucket has a missing chain) — - // require a complete chain from current to target. + // Require a complete chain from current to target. var pending []KVMigration for _, m := range all { if m.FromVersion >= current && m.ToVersion <= targetVersion { diff --git a/spinifex/nbd/nbd.go b/spinifex/nbd/nbd.go index 56382021f..707bc0865 100644 --- a/spinifex/nbd/nbd.go +++ b/spinifex/nbd/nbd.go @@ -71,13 +71,6 @@ func (cfg *NBDKitConfig) buildArgs() ([]string, error) { } func (cfg *NBDKitConfig) Execute() (*exec.Cmd, error) { - // TODO: Consider setting threads to X% of the host availability (preference for VMs) - /* - --threads=THREADS - Set the number of threads to be used per connection, which in turn controls the number of outstanding requests that can - be processed at once. Only matters for plugins with thread_model=parallel (where it defaults to 16). To force - serialized behavior (useful if the client is not prepared for out-of-order responses), set this to 1. - */ args, err := cfg.buildArgs() if err != nil { return nil, err diff --git a/spinifex/network/external/allocator.go b/spinifex/network/external/allocator.go index 5c8ccc03b..7834cd07f 100644 --- a/spinifex/network/external/allocator.go +++ b/spinifex/network/external/allocator.go @@ -5,10 +5,9 @@ import ( "net/netip" ) -// AllocateRequest carries the per-allocation identity used by both static -// and DHCP-backed allocators. PoolName picks the pool; the AWS identity -// (AllocationID / ENIID / InstanceID) is recorded on the IPAM entry for -// static pools and used as the DHCP client-ID for DHCP pools. +// AllocateRequest carries the allocation identity for static and DHCP allocators. +// PoolName selects the pool; AWS identity is recorded on the IPAM entry (static) +// or used as the DHCP client-ID. type AllocateRequest struct { PoolName string Purpose string diff --git a/spinifex/network/external/dhcp/client.go b/spinifex/network/external/dhcp/client.go index c69a47334..46f3b7b9a 100644 --- a/spinifex/network/external/dhcp/client.go +++ b/spinifex/network/external/dhcp/client.go @@ -1,14 +1,6 @@ -// Package dhcp provides an in-process DHCPv4 client used by spinifex-vpcd -// to obtain, renew and release real server-bound leases on behalf of the -// external IPAM pool. -// -// The production client (NewNClient4) opens an AF_PACKET socket per DORA -// on the target bridge via github.com/insomniacslk/dhcp/dhcpv4/nclient4. -// Tests inject Fake instead. -// -// Retransmission lives one layer up (DHCPManager in vpcd, Q3) — this -// package performs single-attempt DORA/RENEW/RELEASE and persists -// server-bound leases to a per-AZ KV bucket. +// Package dhcp provides an in-process DHCPv4 client (spinifex-vpcd) for +// single-attempt DORA/RENEW/RELEASE with KV-persisted leases. Retransmission +// is owned by DHCPManager one layer up. package dhcp import ( diff --git a/spinifex/network/external/dhcp/client_test.go b/spinifex/network/external/dhcp/client_test.go index 5487d1df6..8f248d28f 100644 --- a/spinifex/network/external/dhcp/client_test.go +++ b/spinifex/network/external/dhcp/client_test.go @@ -151,10 +151,8 @@ func TestNClient4ValidatesInputs(t *testing.T) { } } -// Regression: nightly cell 1 failed with "dhcp acquire: hw_addr is required" -// because DHCPGatewayLRPAllocator passes only ClientID, never HWAddr. The -// real client must derive chaddr from the client-id rather than rejecting -// the request. +// Regression: NClient4Client must derive chaddr from ClientID when HWAddr is +// absent — DHCPGatewayLRPAllocator never passes HWAddr. func TestDeriveMAC_StableAndLocallyAdministered(t *testing.T) { got1, err := dhcp.DeriveMAC("dhcp-gw-lrp-vpc-1") if err != nil { diff --git a/spinifex/network/external/dhcp/fake.go b/spinifex/network/external/dhcp/fake.go index 5f2c7187a..14e1512cd 100644 --- a/spinifex/network/external/dhcp/fake.go +++ b/spinifex/network/external/dhcp/fake.go @@ -8,10 +8,8 @@ import ( "time" ) -// Fake is a configurable in-memory Client for unit tests. It mimics the -// DORA / RENEW / RELEASE lifecycle well enough to exercise DHCPManager -// (Q3) and DHCPPoolAllocator (Q4) without opening real sockets. Hooks -// let tests inject errors or custom responses at each step. +// Fake is a configurable in-memory Client for unit tests. Hooks let tests +// inject errors or custom responses without opening real sockets. type Fake struct { mu sync.Mutex diff --git a/spinifex/network/external/dhcp/kv.go b/spinifex/network/external/dhcp/kv.go index 6d1438438..8f545cb8b 100644 --- a/spinifex/network/external/dhcp/kv.go +++ b/spinifex/network/external/dhcp/kv.go @@ -113,10 +113,8 @@ func (s *Store) Delete(clientID string) error { return nil } -// LookupByIP scans the bucket for an entry whose lease IP matches ip -// (and pool name, when non-empty). Returns (nil, nats.ErrKeyNotFound) -// when no match. O(N) over leases — release is the only caller and -// runs at allocation-lifecycle frequency, not on the hot path. +// LookupByIP scans the bucket for an entry whose lease IP matches ip (and +// pool name when non-empty). O(N) is fine — only called on release. func (s *Store) LookupByIP(poolName, ip string) (*Entry, error) { if ip == "" { return nil, errors.New("dhcp store lookup: ip required") diff --git a/spinifex/network/external/dhcp/lrp_allocator.go b/spinifex/network/external/dhcp/lrp_allocator.go index ca04d50f9..122c60222 100644 --- a/spinifex/network/external/dhcp/lrp_allocator.go +++ b/spinifex/network/external/dhcp/lrp_allocator.go @@ -10,8 +10,7 @@ import ( ) // Purpose tags written to the per-AZ lease bucket so operators can -// distinguish lease classes (mirrors the schema in -// docs/development/improvements/external-pool-dhcp-source.md). +// distinguish lease classes. const ( PurposeGatewayLRP = "gw-lrp" PurposeEIP = "eip" @@ -19,19 +18,13 @@ const ( PurposeNATGWExternal = "natgw-external" ) -// GatewayLRPClientID returns the DHCP client-id used by -// DHCPGatewayLRPAllocator. Operators see this string in dnsmasq.leases; -// the "dhcp-" prefix keeps it disjoint from OVN object names (ADR-0006 -// S4 reserves "gw-" for L2-owned OVN identifiers). +// GatewayLRPClientID returns the DHCP client-id for vpcID. The "dhcp-" prefix +// keeps it disjoint from OVN object names ("gw-" is L2-owned). func GatewayLRPClientID(vpcID string) string { return "dhcp-gw-lrp-" + vpcID } -// DHCPGatewayLRPAllocator obtains the per-VPC gateway LRP IP via DORA -// against the upstream DHCP server. Idempotent on vpcID — the manager -// short-circuits to the persisted lease on repeated calls. -// -// Used by vpcd's IGWManager when pool.Source == external.SourceDHCP -// and NAT mode is centralized. Distributed NAT keeps using -// external.LinkLocalAllocator (LRP IP never goes on wire). +// DHCPGatewayLRPAllocator obtains the per-VPC gateway LRP IP via DORA. +// Used when pool.Source == SourceDHCP with centralized NAT; distributed NAT +// uses LinkLocalAllocator instead (LRP IP never goes on wire). type DHCPGatewayLRPAllocator struct { mgr *Manager } @@ -43,12 +36,9 @@ func NewDHCPGatewayLRPAllocator(mgr *Manager) *DHCPGatewayLRPAllocator { return &DHCPGatewayLRPAllocator{mgr: mgr} } -// Allocate DORAs once per vpcID. Returns ok=false when pool is nil or -// not DHCP-sourced so IGWManager can fall back to link-local. -// Nexthop is the first router from the DHCP ACK; empty Routers is a hard -// error — silent link-local fallback would yield 169.254.0.2 on the WAN -// subnet, which is unreachable and was the root cause of the -// extdhcp-restore-dhcp-source CI failures. +// Allocate DORAs once per vpcID; returns ok=false for non-DHCP pools. +// Empty Routers in the ACK is a hard error — fallback to link-local would +// yield 169.254.0.2 on the WAN subnet, which is unreachable. func (a *DHCPGatewayLRPAllocator) Allocate(ctx context.Context, vpcID string, pool *external.ExternalPoolConfig) (string, int, string, bool, error) { if a == nil || a.mgr == nil { return "", 0, "", false, errors.New("dhcp gw-lrp allocator: nil manager") @@ -91,10 +81,8 @@ func (a *DHCPGatewayLRPAllocator) Allocate(ctx context.Context, vpcID string, po return entry.Lease.IP.String(), prefix, nexthop, true, nil } -// firstRouter returns the first non-nil router in routers, or an error -// when none is present. The OVN default route needs an on-link nexthop; -// without one the VPC would have to fall back to link-local, which is -// unreachable on the WAN subnet. +// firstRouter returns the first non-nil router, or an error if none present. +// OVN's default route requires an on-link nexthop. func firstRouter(routers []net.IP) (string, error) { for _, r := range routers { if r != nil { diff --git a/spinifex/network/external/dhcp/manager.go b/spinifex/network/external/dhcp/manager.go index 875171024..2e39a921c 100644 --- a/spinifex/network/external/dhcp/manager.go +++ b/spinifex/network/external/dhcp/manager.go @@ -15,9 +15,8 @@ import ( "golang.org/x/sync/singleflight" ) -// renewJitter is the ± window applied to every renewal/rebind/expiry -// timer so cluster-wide synchronised wake-ups don't pile DORA traffic -// onto the upstream DHCP server at the same instant. +// renewJitter is the ± window applied to renewal timers to spread cluster-wide +// wake-ups and avoid piling DORA traffic on the upstream server. const renewJitter = time.Second // postExpiryBackoff is the cool-off applied after a re-DORA fails post @@ -69,10 +68,9 @@ type Manager struct { wg sync.WaitGroup } -// leaseLoop is the handle stored in Manager.loops. The pointer-identity -// lets the run() defer disambiguate its own loop from a successor loop -// that may have been registered after a cancel (e.g. expired lease -// re-acquired with same client-id). +// leaseLoop is the handle stored in Manager.loops. Pointer-identity lets +// run() distinguish its own loop from a successor (e.g. re-acquired lease +// with the same client-id). type leaseLoop struct { cancel context.CancelFunc } @@ -109,10 +107,8 @@ func NewManager(cfg ManagerConfig) (*Manager, error) { }, nil } -// Start scans the KV bucket and, for every non-expired lease, kicks -// off a renewal goroutine that first re-issues a RENEW to confirm the -// upstream server still honours the binding (RFC 2131 §4.3.2 -// INIT-REBOOT equivalent — nclient4 unicasts to ServerID from raw_ack). +// Start scans the KV bucket and spawns a renewal goroutine per live lease, +// re-issuing a RENEW to confirm the upstream binding (RFC 2131 INIT-REBOOT). // Expired entries are deleted. Repeated calls return an error. func (m *Manager) Start(ctx context.Context) error { m.mu.Lock() @@ -166,11 +162,9 @@ func (m *Manager) Stop() { m.wg.Wait() } -// Subscribe registers the two daemon-facing handlers via a per-AZ -// queue group so that exactly one vpcd answers each request even when -// multiple vpcds in the AZ subscribe. Without the queue group every -// vpcd would DORA in parallel with the same derived chaddr, causing -// upstream-server NAKs / lease leaks (real-multi CI cells 8, 10). +// Subscribe registers NATS handlers under a per-AZ queue group so exactly +// one vpcd answers each request. Without a queue group all vpcds would DORA +// in parallel with the same chaddr, causing NAKs and lease leaks. func (m *Manager) Subscribe(nc *nats.Conn) ([]*nats.Subscription, error) { if nc == nil { return nil, errors.New("dhcp manager: nats conn required") @@ -317,11 +311,9 @@ func (m *Manager) doAcquire(ctx context.Context, e *Entry) error { return nil } -// acquireWithBackoff drives client.Acquire under a per-attempt deadline -// schedule, capped by AcquireBudget wallclock. Each attempt gets a fresh -// child ctx with timeout schedule[i] ± acquireAttemptJitter so concurrent -// acquires don't synchronise. Returns the last error if every attempt -// fails or the budget is exhausted; ctx.Err() short-circuits immediately. +// acquireWithBackoff drives client.Acquire with per-attempt timeouts from +// the schedule, capped by AcquireBudget. Jitter prevents synchronised wakes. +// ctx.Err() short-circuits; returns last error when all attempts fail. func (m *Manager) acquireWithBackoff(ctx context.Context, req AcquireRequest) (*Lease, error) { if len(m.acquireSchedule) == 0 { return m.client.Acquire(ctx, req) @@ -487,13 +479,9 @@ func (m *Manager) acquireLocked(ctx context.Context, req acquireWireRequest) (*E return &entry, nil } -// DrainAll DHCPRELEASEs every lease currently held in this vpcd's per-AZ -// store and deletes the KV records. Best-effort: a per-lease release -// failure is logged and skipped so one stuck lease can't block the rest. -// Returns the number of leases successfully released. Used by the teardown -// path so an env reset returns its leases to the upstream pool rather than -// stranding them until TTL (Stop() deliberately preserves leases for -// adopt-on-reboot, which is wrong on a destructive teardown). +// DrainAll DHCPRELEASEs every lease in this vpcd's per-AZ store and deletes +// the KV records. Best-effort (per-lease failures logged, not fatal). Used on +// destructive teardown; Stop() preserves leases for adopt-on-reboot. func (m *Manager) DrainAll(ctx context.Context) (int, error) { entries, err := m.store.List() if err != nil { diff --git a/spinifex/network/external/dhcp/nats_client.go b/spinifex/network/external/dhcp/nats_client.go index 78dcf1224..1ec215f21 100644 --- a/spinifex/network/external/dhcp/nats_client.go +++ b/spinifex/network/external/dhcp/nats_client.go @@ -11,10 +11,8 @@ import ( "github.com/nats-io/nats.go" ) -// NATSClient is the daemon-side wrapper around vpcd's Manager. It -// marshals acquire/release requests over NATS request/reply and -// unmarshals the Lease returned by the bridge-owning vpcd. Used by -// ExternalIPAM (Q4) for DHCP-sourced pools. +// NATSClient is the daemon-side DHCP wrapper: marshals acquire/release RPC +// calls over NATS to the bridge-owning vpcd. type NATSClient struct { nc *nats.Conn timeout time.Duration @@ -30,10 +28,8 @@ func NewNATSClient(nc *nats.Conn, timeout time.Duration) *NATSClient { return &NATSClient{nc: nc, timeout: timeout} } -// AcquireParams is the wire-level acquire payload. Bridge/HWAddr are -// passed through to the vpcd-side Client.Acquire; Purpose/PoolName/ -// VPCID let the manager persist the lease against the right pool -// record. +// AcquireParams is the wire-level acquire payload; fields map to Client.Acquire +// and the KV record's purpose/pool/vpc metadata. type AcquireParams struct { Bridge string ClientID string diff --git a/spinifex/network/external/dhcp/nclient4.go b/spinifex/network/external/dhcp/nclient4.go index 28e0ea658..8706ec526 100644 --- a/spinifex/network/external/dhcp/nclient4.go +++ b/spinifex/network/external/dhcp/nclient4.go @@ -12,15 +12,9 @@ import ( "github.com/mulgadc/spinifex/spinifex/utils" ) -// NClient4Client is the production DHCP client backed by -// github.com/insomniacslk/dhcp/dhcpv4/nclient4. Each Acquire/Renew/Release -// opens an AF_PACKET socket on the target bridge for the duration of the -// handshake and closes it when done — no long-lived per-lease process. -// -// Single-shot per call: Manager.acquireWithBackoff drives the outer DORA -// schedule and per-attempt deadlines, so the leaf does not retry. This -// keeps the budget arithmetic in one place instead of being multiplied -// by nclient4's internal retry count. +// NClient4Client is the production DHCP client (AF_PACKET per call, no +// long-lived socket). Single-shot: Manager.acquireWithBackoff owns retries +// so budget arithmetic stays in one place. type NClient4Client struct { timeout time.Duration } @@ -75,10 +69,8 @@ func (c *NClient4Client) Acquire(ctx context.Context, req AcquireRequest) (*Leas } defer func() { _ = client.Close() }() - // Without the broadcast flag, the server sends a unicast OFFER to the - // generated chaddr MAC. The physical NIC drops it in hardware (not its - // MAC), so the AF_PACKET socket on the bridge never sees the frame. - // Setting the broadcast flag forces ff:ff:ff:ff:ff:ff. + // Broadcast flag forces ff:ff:ff:ff:ff:ff destination — without it the + // server unicasts to the derived chaddr MAC which the NIC drops in hw. mods := append(identityModifiers(req.ClientID, req.Hostname, req.VendorClass), dhcpv4.WithBroadcast(true)) lease, err := client.Request(ctx, mods...) if err != nil { @@ -168,11 +160,9 @@ func (c *NClient4Client) Release(_ context.Context, lease *Lease) error { return nil } -// DeriveMAC returns the deterministic locally-administered unicast MAC -// (02:xx:xx:xx:xx:xx) used as chaddr when an allocator has no NIC of -// its own — DHCPGatewayLRPAllocator and DHCPPoolAllocator on first -// acquire. Wraps utils.HashMAC so the same 0x02-prefixed scheme used -// for OVN router/port MACs is also visible in upstream dnsmasq leases. +// DeriveMAC returns the deterministic 02:xx:xx:xx:xx:xx chaddr for clientID, +// using the same HashMAC scheme as OVN router/port MACs so leases are legible +// in upstream dnsmasq logs. func DeriveMAC(clientID string) (net.HardwareAddr, error) { hw, err := net.ParseMAC(utils.HashMAC(clientID)) if err != nil { diff --git a/spinifex/network/external/dhcp/pool_allocator.go b/spinifex/network/external/dhcp/pool_allocator.go index 398e55976..3c914445d 100644 --- a/spinifex/network/external/dhcp/pool_allocator.go +++ b/spinifex/network/external/dhcp/pool_allocator.go @@ -11,13 +11,9 @@ import ( "github.com/mulgadc/spinifex/spinifex/network/external" ) -// DHCPPoolAllocator implements external.Allocator for a single -// DHCP-sourced pool. Runs daemon-side and routes acquire/release through -// vpcd over NATS (the bridge-owning vpcd answers). -// -// Each call DORAs once per AWS identity; the client-id is derived from -// req.AllocationID / ENIID / InstanceID, in that order. Vpcd's Manager is -// idempotent on client-id, so retries return the same IP. +// DHCPPoolAllocator implements external.Allocator for a DHCP-sourced pool, +// routing acquire/release via NATS to the bridge-owning vpcd. Client-id is +// derived from AllocationID / ENIID / InstanceID; Manager is idempotent. type DHCPPoolAllocator struct { client *NATSClient pool external.ExternalPoolConfig @@ -85,10 +81,8 @@ func (a *DHCPPoolAllocator) Allocate(ctx context.Context, req external.AllocateR return addr.Unmap(), nil } -// Release looks up the lease holding ip and issues vpc.dhcp.release. -// Static-pool semantics — error when poolName doesn't match this -// allocator's pool — so ExternalIPAM's dispatch table doesn't accidentally -// release a different pool's lease. +// Release issues vpc.dhcp.release for ip. Errors if poolName doesn't match +// this allocator's pool to prevent cross-pool release. func (a *DHCPPoolAllocator) Release(ctx context.Context, poolName string, ip netip.Addr) error { if a == nil || a.client == nil { return errors.New("dhcp pool allocator: nil client") diff --git a/spinifex/network/external/dhcp/promisc.go b/spinifex/network/external/dhcp/promisc.go index 555d6773f..e0c314129 100644 --- a/spinifex/network/external/dhcp/promisc.go +++ b/spinifex/network/external/dhcp/promisc.go @@ -10,24 +10,9 @@ import ( "golang.org/x/sys/unix" ) -// enableBridgePromisc puts iface into IFF_PROMISC and returns a release -// callback the caller defers. Reference-counted across concurrent DORAs -// on the same bridge so the first caller flips PROMISC on and the last -// caller flips it off — concurrent leases don't fight over flag state. -// -// Rationale: nclient4 sends DHCPDISCOVER with the derived 02:xx:xx:xx:xx:xx -// chaddr (see DeriveMAC) which doesn't match the bridge's own MAC. Many -// upstream DHCP servers reply unicast to chaddr regardless of the -// broadcast flag, so the OFFER egresses the Linux bridge with a destination -// MAC the bridge has never learned. Without IFF_PROMISC the kernel -// AF_PACKET socket bound to the bridge drops those frames in software and -// the DORA times out with "no matching response packet received". With -// PROMISC the bridge surfaces every received frame to the socket and -// nclient4 sees the OFFER. -// -// On failure (typically EPERM when vpcd lost CAP_NET_ADMIN) the function -// returns the error so the caller can decide whether to proceed — the -// bridge may already be in PROMISC via operator config. +// enableBridgePromisc puts iface into IFF_PROMISC and returns a release callback. +// Reference-counted so concurrent DORAs share one flag; needed because servers +// unicast OFFERs to the derived chaddr, which the bridge drops without PROMISC. var enableBridgePromisc = func(iface string) (func() error, error) { if iface == "" { return nil, errors.New("promisc: iface required") diff --git a/spinifex/network/external/dhcp/promisc_test.go b/spinifex/network/external/dhcp/promisc_test.go index f2fb1e7db..362ab64ab 100644 --- a/spinifex/network/external/dhcp/promisc_test.go +++ b/spinifex/network/external/dhcp/promisc_test.go @@ -9,11 +9,8 @@ import ( ) // TestPromiscRefCount verifies the bridge stays in IFF_PROMISC across -// overlapping DORAs: first caller flips on, intermediate callers piggy-back -// without touching the flag, last caller flips off. Catches the regression -// where nightly cell 1 timed out on "no matching response packet received" -// because chaddr-derived 02:xx unicast OFFERs never reach AF_PACKET on a -// non-promisc Linux bridge. +// overlapping DORAs (first caller flips on, last flips off). Regression +// guard: unicast OFFERs to a derived chaddr are dropped on non-promisc bridges. func TestPromiscRefCount(t *testing.T) { defer resetPromiscState() var ( diff --git a/spinifex/network/external/dhcp/wire.go b/spinifex/network/external/dhcp/wire.go index 47c485c36..029d7d923 100644 --- a/spinifex/network/external/dhcp/wire.go +++ b/spinifex/network/external/dhcp/wire.go @@ -7,10 +7,8 @@ import ( "time" ) -// NATS subjects for daemon ↔ vpcd DHCP RPCs. Plain (non-AZ-prefixed) -// because the vpcd that owns br-wan is exactly one process per AZ; an -// AZ-prefixed subject would just push the same constraint into the -// subject string. +// NATS subjects for daemon ↔ vpcd DHCP RPCs. Non-AZ-prefixed: the +// bridge-owning vpcd is exactly one process per AZ. const ( TopicAcquire = "vpc.dhcp.acquire" TopicRelease = "vpc.dhcp.release" @@ -41,10 +39,9 @@ type acquireWireReply struct { Lease *wireLease `json:"lease,omitempty"` } -// releaseWireRequest carries either a direct ClientID, or a (PoolName, IP) -// pair the manager uses to reverse-lookup the ClientID from the per-AZ -// lease store. ExternalIPAM's Allocator.Release signature only has the -// IP, so the daemon side uses the IP path; in-process callers use ClientID. +// releaseWireRequest carries either a ClientID or (PoolName, IP) for reverse- +// lookup. Daemon-side callers use IP (that's all Allocator.Release provides); +// in-process callers use ClientID. type releaseWireRequest struct { ClientID string `json:"client_id,omitempty"` PoolName string `json:"pool_name,omitempty"` diff --git a/spinifex/network/external/igw.go b/spinifex/network/external/igw.go index 62075025b..2fb34dd63 100644 --- a/spinifex/network/external/igw.go +++ b/spinifex/network/external/igw.go @@ -22,37 +22,24 @@ type FlowsBarrier func() error type IGWManager interface { AttachIGW(ctx context.Context, spec IGWSpec) error DetachIGW(ctx context.Context, vpcID string) error - // EnsureSubnetEgress installs an OVN Logical_Router_Policy on the VPC - // router so traffic sourced from subnetID and destined for prefix - // reroutes via the IGW's gateway port. Called from the route-table - // subscriber when CreateRoute / AssociateRouteTable wires a subnet to a - // route table carrying a 0.0.0.0/0 -> igw-X route. Idempotent. + // EnsureSubnetEgress installs a Logical_Router_Policy rerouting traffic + // from subnetID matching prefix via the IGW's gateway port. Idempotent. EnsureSubnetEgress(ctx context.Context, vpcID, subnetID string, prefix netip.Prefix) error // RemoveSubnetEgress is the inverse. Idempotent. RemoveSubnetEgress(ctx context.Context, vpcID, subnetID string, prefix netip.Prefix) error - // EnsureNATGatewaySubnetEgress installs the per-subnet egress policy that - // gives private-subnet packets a default route to leave the LR after - // NATGW SNAT. Reuses the IGW's gateway port + wan nexthop (NATGW SNAT - // happens on the same VPC router); priority SubnetEgressPriorityNATGW - // (lower than IGW so an IGW route on the same subnet would win). - // Requires AttachIGW to have run first (NATGW depends on IGW per AWS). + // EnsureNATGatewaySubnetEgress installs the NATGW egress policy at + // SubnetEgressPriorityNATGW (lower than IGW). Requires AttachIGW first. EnsureNATGatewaySubnetEgress(ctx context.Context, vpcID, subnetID string, prefix netip.Prefix) error // RemoveNATGatewaySubnetEgress is the inverse. Idempotent. RemoveNATGatewaySubnetEgress(ctx context.Context, vpcID, subnetID string, prefix netip.Prefix) error - // EnsureSubnetEgressDrop installs a DROP policy at - // SubnetEgressPriorityDrop for subnets whose effective route table lacks - // a 0.0.0.0/0 entry. The drop fires after lr_in_ip_routing has matched - // the VPC LR's router-wide default static route, killing the packet - // before egress. Idempotent. + // EnsureSubnetEgressDrop installs a DROP policy for subnets lacking a + // 0.0.0.0/0 route table entry. Fires after lr_in_ip_routing. Idempotent. EnsureSubnetEgressDrop(ctx context.Context, vpcID, subnetID string, prefix netip.Prefix) error // RemoveSubnetEgressDrop is the inverse. Idempotent. RemoveSubnetEgressDrop(ctx context.Context, vpcID, subnetID string, prefix netip.Prefix) error - // EnsureSystemInstanceEgress wires egress-only internet access for a - // single system instance (e.g. an EKS K3s server VM) sourced from - // instanceIP in subnetID, SNAT'd to externalIP. Installs a /32 reroute - // policy at SystemInstanceEgressPriority (above the drop gate so it works - // even on a drop-gated subnet) plus a plain snat (no DNAT, no inbound). - // Requires AttachIGW to have run first. Idempotent. + // EnsureSystemInstanceEgress wires SNAT-only egress for a single instance + // (instanceIP → externalIP). Installs a /32 reroute above the drop gate. + // Requires AttachIGW first. Idempotent. EnsureSystemInstanceEgress(ctx context.Context, vpcID, subnetID, instanceIP, externalIP string) error // RemoveSystemInstanceEgress is the inverse. Idempotent. RemoveSystemInstanceEgress(ctx context.Context, vpcID, subnetID, instanceIP, externalIP string) error @@ -284,10 +271,8 @@ func (m *igwManager) DetachIGW(ctx context.Context, vpcID string) error { return nil } -// EnsureSubnetEgress installs a per-subnet egress policy on the VPC router -// rerouting (inport == "rtr-" && ip4.dst == prefix) via the IGW -// nexthop out the gateway LRP. Idempotent (drift-replace via the underlying -// policy.RouteManager). +// EnsureSubnetEgress installs a per-subnet egress reroute policy on the VPC router. +// Idempotent. func (m *igwManager) EnsureSubnetEgress(ctx context.Context, vpcID, subnetID string, prefix netip.Prefix) error { return m.ensureSubnetEgressAtPriority(ctx, vpcID, subnetID, prefix, policy.SubnetEgressPriorityIGW, "EnsureSubnetEgress") } @@ -301,16 +286,14 @@ func (m *igwManager) RemoveSubnetEgress(ctx context.Context, vpcID, subnetID str return m.routes.DeleteSubnetEgress(ctx, vpcID, subnetID, prefix, m.rerouteExcludeCIDRs(ctx, vpcID)) } -// EnsureNATGatewaySubnetEgress is the NATGW priority sibling of -// EnsureSubnetEgress. Same nexthop and output port; lower priority so an -// IGW route on the same subnet (if ever both present) wins. +// EnsureNATGatewaySubnetEgress installs the NATGW-priority egress policy +// (lower than IGW so IGW wins if both apply). func (m *igwManager) EnsureNATGatewaySubnetEgress(ctx context.Context, vpcID, subnetID string, prefix netip.Prefix) error { return m.ensureSubnetEgressAtPriority(ctx, vpcID, subnetID, prefix, policy.SubnetEgressPriorityNATGW, "EnsureNATGatewaySubnetEgress") } -// RemoveNATGatewaySubnetEgress deletes the per-subnet egress policy at any -// priority for (subnetID, prefix). policy.RouteManager.DeleteSubnetEgress -// already deletes both IGW and NATGW priorities defensively. +// RemoveNATGatewaySubnetEgress deletes the egress policy for (subnetID, prefix). +// Idempotent. func (m *igwManager) RemoveNATGatewaySubnetEgress(ctx context.Context, vpcID, subnetID string, prefix netip.Prefix) error { if vpcID == "" || subnetID == "" { return errors.New("RemoveNATGatewaySubnetEgress: vpcID and subnetID required") @@ -318,12 +301,8 @@ func (m *igwManager) RemoveNATGatewaySubnetEgress(ctx context.Context, vpcID, su return m.routes.DeleteSubnetEgress(ctx, vpcID, subnetID, prefix, m.rerouteExcludeCIDRs(ctx, vpcID)) } -// EnsureSubnetEgressDrop installs a drop policy at SubnetEgressPriorityDrop -// for a subnet whose effective route table lacks a 0.0.0.0/0 entry. The drop -// fires in lr_in_policy AFTER the VPC LR's router-wide default static route -// has already matched in lr_in_ip_routing, killing the packet before egress. -// Excludes the VPC CIDR (intra-VPC traffic untouched), 169.254.0.0/16 (DHCP -// / link-local / metadata), and 224.0.0.0/4 (multicast). +// EnsureSubnetEgressDrop installs a drop policy for subnets lacking a 0.0.0.0/0 +// route. Fires in lr_in_policy after routing; excludes VPC CIDR, link-local, multicast. func (m *igwManager) EnsureSubnetEgressDrop(ctx context.Context, vpcID, subnetID string, prefix netip.Prefix) error { if vpcID == "" || subnetID == "" { return errors.New("EnsureSubnetEgressDrop: vpcID and subnetID required") @@ -365,12 +344,8 @@ func (m *igwManager) ensureSubnetEgressAtPriority(ctx context.Context, vpcID, su }) } -// EnsureSystemInstanceEgress installs an egress-only path for one system -// instance: a /32 reroute policy out the gateway port plus a plain snat -// (instanceIP -> externalIP). The reroute sits above the subnet drop gate so -// it works even where the instance's subnet is otherwise private; where the -// subnet already has a 1000 reroute it is harmless (same nexthop). The snat is -// not dnat_and_snat, so the instance is never reachable inbound. +// EnsureSystemInstanceEgress installs a /32 reroute above the drop gate plus a plain +// snat (instanceIP → externalIP). No DNAT: the instance is not reachable inbound. func (m *igwManager) EnsureSystemInstanceEgress(ctx context.Context, vpcID, subnetID, instanceIP, externalIP string) error { if vpcID == "" || subnetID == "" { return errors.New("EnsureSystemInstanceEgress: vpcID and subnetID required") @@ -427,19 +402,16 @@ func (m *igwManager) RemoveSystemInstanceEgress(ctx context.Context, vpcID, subn return firstErr } -// linkLocalCIDR / multicastCIDR are appended to drop-policy excludes so -// link-local (DHCP and friends — traffic that should never egress the gateway) -// and multicast destinations are not killed by the per-subnet gate. +// linkLocalCIDR / multicastCIDR are appended to drop-policy excludes so link-local +// and multicast are not killed by the per-subnet drop gate. var ( linkLocalCIDR = netip.MustParsePrefix("169.254.0.0/16") multicastCIDR = netip.MustParsePrefix("224.0.0.0/4") defaultRoutePrefix = netip.MustParsePrefix("0.0.0.0/0") ) -// dropExcludeCIDRs returns the exclusion list for a per-subnet drop policy: -// VPC CIDR (if discoverable) plus link-local and multicast. The drop policy is -// a default-deny within the subnet's egress so it must spare any class the -// platform legitimately uses outside the VPC. +// dropExcludeCIDRs returns the exclusion list for a drop policy: VPC CIDR, +// link-local, and multicast. func (m *igwManager) dropExcludeCIDRs(ctx context.Context, vpcID string) []netip.Prefix { excludes := []netip.Prefix{linkLocalCIDR, multicastCIDR} if vpc := m.vpcExcludeCIDRs(ctx, vpcID); len(vpc) > 0 { @@ -448,22 +420,15 @@ func (m *igwManager) dropExcludeCIDRs(ctx context.Context, vpcID string) []netip return excludes } -// rerouteExcludeCIDRs returns the exclusion list for a per-subnet egress -// reroute policy: VPC CIDR (if discoverable) plus link-local. The reroute -// matches 0.0.0.0/0 — the widest possible scope — so it would otherwise divert -// link-local (169.254.0.0/16) out the gateway, where it has no business going. -// Multicast is not excluded here (unlike the drop policy): a reroute only -// diverts traffic that already has somewhere to go, so leaking multicast to the -// gateway is harmless, whereas the drop policy would kill it outright. +// rerouteExcludeCIDRs returns the exclusion list for an egress reroute policy: +// VPC CIDR plus link-local (0.0.0.0/0 would otherwise divert link-local out the gateway). func (m *igwManager) rerouteExcludeCIDRs(ctx context.Context, vpcID string) []netip.Prefix { excludes := m.vpcExcludeCIDRs(ctx, vpcID) return append(excludes, linkLocalCIDR) } -// vpcExcludeCIDRs fetches the VPC's primary CIDR from the LR ExternalIDs -// label so per-subnet egress reroute policies skip in-VPC destinations. -// Returns nil on lookup failure or unset label — caller installs a policy -// without exclusions, preserving prior behaviour. +// vpcExcludeCIDRs returns the VPC's primary CIDR from the LR ExternalIDs. +// Returns nil on lookup failure; caller installs policy without exclusions. func (m *igwManager) vpcExcludeCIDRs(ctx context.Context, vpcID string) []netip.Prefix { router, err := m.ovn.GetLogicalRouter(ctx, topology.VPCRouter(vpcID)) if err != nil || router == nil { @@ -480,8 +445,7 @@ func (m *igwManager) vpcExcludeCIDRs(ctx context.Context, vpcID string) []netip. return []netip.Prefix{prefix} } -// resolveGatewayNetwork picks LRP Networks/nexthop/IP. Distributed: link-local. -// Centralised: WAN-subnet IP (LRP is on-wire). +// resolveGatewayNetwork resolves the LRP network/nexthop/IP for the VPC gateway port. func (m *igwManager) resolveGatewayNetwork(ctx context.Context, vpcID string) (network, nexthop, gwIP string, err error) { network = linkLocalGatewayNetwork nexthop = linkLocalGatewayNexthop diff --git a/spinifex/network/external/igw_test.go b/spinifex/network/external/igw_test.go index 1e46e85d9..601944741 100644 --- a/spinifex/network/external/igw_test.go +++ b/spinifex/network/external/igw_test.go @@ -86,11 +86,9 @@ func TestAttachIGW_Distributed_LinkLocalLRP(t *testing.T) { _, hasGwIP := lrp.ExternalIDs[gatewayIPExtIDKey] assert.False(t, hasGwIP, "link-local LRP must not record a gateway IP") - // OVN pipeline runs lr_in_ip_routing (table 15) before lr_in_policy - // (table 17) — without a router-wide default static route every external - // destination drops in routing before EnsureSubnetEgress's reroute policy - // gets a chance. AttachIGW installs `0.0.0.0/0 -> pool.Gateway` on the - // VPC router so policy lookup can refine per-subnet egress on top. + // lr_in_ip_routing runs before lr_in_policy: without a default static route + // every external destination drops before the reroute policy fires. + // AttachIGW must install 0.0.0.0/0 → pool.Gateway on the VPC router. route, err := m.FindStaticRoute(ctx, topology.VPCRouter("vpc-1"), "0.0.0.0/0") require.NoError(t, err) require.NotNil(t, route, "AttachIGW must install the default static route so routing forwards to gw-port") @@ -272,10 +270,9 @@ func TestAttachIGW_AllocatorFailureUnwindsExtSwitch(t *testing.T) { assert.Error(t, err, "external switch must be unwound on allocator failure") } -// TestEnsureNATGatewaySubnetEgress installs an LR policy at the NATGW -// priority (lower than IGW) reusing the IGW's gateway port + nexthop. -// Without this policy, NATGW SNAT rewrites the source IP but the packet -// has no route to leave the LR — private subnets cannot reach the internet. +// TestEnsureNATGatewaySubnetEgress installs an LR policy at NATGW priority +// (lower than IGW) reusing the gateway port. Without it, SNAT rewrites the +// source but the packet has no route out of the LR. func TestEnsureNATGatewaySubnetEgress(t *testing.T) { ctx := context.Background() m := mock.New() @@ -362,10 +359,9 @@ func TestEnsureNATGatewaySubnetEgress_RequiresAttachedIGW(t *testing.T) { require.Error(t, err, "missing gateway nexthop must surface as an error") } -// TestRemoveNATGatewaySubnetEgress_PropagatesDeleteError verifies the manager -// surfaces an OVN delete error (e.g. router missing). policy.RouteManager. -// DeleteSubnetEgress targets the VPC router by name; with no router seeded the -// mock returns "router not found", which must bubble up. +// TestRemoveNATGatewaySubnetEgress_PropagatesDeleteError verifies that an OVN +// delete error (e.g. router missing) bubbles up; DeleteSubnetEgress targets the +// VPC router by name and the mock returns "router not found". func TestRemoveNATGatewaySubnetEgress_PropagatesDeleteError(t *testing.T) { ctx := context.Background() m := mock.New() diff --git a/spinifex/network/external/imds_datapath_contract_test.go b/spinifex/network/external/imds_datapath_contract_test.go index d126cfdb1..218b8e0b0 100644 --- a/spinifex/network/external/imds_datapath_contract_test.go +++ b/spinifex/network/external/imds_datapath_contract_test.go @@ -16,21 +16,9 @@ import ( "github.com/mulgadc/spinifex/spinifex/network/topology" ) -// IMDS datapath contract — desk-speed oracle. -// -// Every IMDS datapath bug in the v1 grind (the egress-reroute hijack of the -// /32, the multi-hop reply break) lived inside the VPC logical-router pipeline. -// The L2 datapath takes IMDS off the router entirely: the localport that claims -// 169.254.169.254 sits directly on the guest's subnet switch, so the request is -// answered over a single L2 hop and never enters lr_in_*. This test asserts that -// structural property — the localport is on the subnet LS and NO IMDS object -// (LRP, /32 route, dedicated switch) exists on the VPC router — for two VPCs that -// share the *identical* CIDR (the overlapping-CIDR case per-subnet identity is -// built for). -// -// The LR-pipeline simulator below is retained for TestIMDSDatapathContract_OracleHasTeeth, -// which proves a broad lr_in_policy reroute WOULD have hijacked IMDS had it ever -// been routed — documenting exactly why moving IMDS to L2 removes the bug class. +// IMDS datapath contract: 169.254.169.254 is served by a localport on the +// subnet switch (one L2 hop, never enters lr_in_*). No IMDS LRP, /32 route, +// or dedicated switch on the VPC router — the LR pipeline cannot shadow it. const imdsContractCIDR = "10.211.0.0/16" @@ -72,10 +60,8 @@ func TestIMDSDatapathContract_AnsweredAtL2_OverlappingCIDRs(t *testing.T) { } // TestIMDSDatapathContract_OracleHasTeeth proves the LR-pipeline simulator -// actually detects the regression it documents: a 0.0.0.0/0 egress reroute that -// does NOT exclude link-local (the pre-#19 bug) must be reported as diverting -// IMDS. Under L2 this can no longer reach IMDS — but the simulator keeps the -// historical hazard legible and the oracle non-vacuous. +// catches the pre-#19 bug: a 0.0.0.0/0 reroute without link-local exclusion +// must be detected as diverting IMDS, keeping the oracle non-vacuous. func TestIMDSDatapathContract_OracleHasTeeth(t *testing.T) { ctx := context.Background() m := mock.New() @@ -101,10 +87,8 @@ func TestIMDSDatapathContract_OracleHasTeeth(t *testing.T) { "simulator must detect a link-local-unaware reroute as hijacking IMDS; if this fails the oracle is blind") } -// assertIMDSAnsweredAtL2 is the contract: 169.254.169.254 is claimed by a -// localport on the guest's subnet switch (so it is answered over one L2 hop), -// and NO IMDS object exists on the VPC router (so the LR pipeline cannot shadow -// it — the entire v1 bug class is structurally impossible). +// assertIMDSAnsweredAtL2 enforces the L2 contract: IMDS localport must exist on +// the subnet switch, and no IMDS LRP/route may exist on the VPC router. func assertIMDSAnsweredAtL2(t *testing.T, m *mock.Client, vpcID, subnetID string) { t.Helper() spec := IMDSSpecForSubnet(subnetID) @@ -144,11 +128,8 @@ func resolveEgress(t *testing.T, m *mock.Client, vpcID, subnetID, dstIP string) return resolveEgressFrom(t, m, vpcID, topology.SubnetRouterPort(subnetID), dstIP) } -// resolveEgressFrom models OVN's two ingress stages in order: -// -// lr_in_policy — Logical_Router_Policy, highest Priority first, first -// applicable wins (reroute => diverted, drop => dropped). -// lr_in_ip_routing — longest-prefix-match over static routes. +// resolveEgressFrom models OVN lr_in_policy (highest-priority first) then +// lr_in_ip_routing (longest-prefix-match). func resolveEgressFrom(t *testing.T, m *mock.Client, vpcID, inport, dstIP string) egressResult { t.Helper() dst, err := netip.ParseAddr(dstIP) diff --git a/spinifex/network/external/imds_topology.go b/spinifex/network/external/imds_topology.go index 7d65af0a1..798a16081 100644 --- a/spinifex/network/external/imds_topology.go +++ b/spinifex/network/external/imds_topology.go @@ -16,10 +16,8 @@ import ( "github.com/mulgadc/spinifex/spinifex/utils" ) -// IMDSSubnetSpec is the resolved set of OVN + host names for a subnet's IMDS -// localport. Every field is a deterministic function of the subnet ID, so -// callers can rebuild it from the subnet ID alone (BindManager does this per -// bucket entry). +// IMDSSubnetSpec is the resolved OVN + host names for a subnet's IMDS localport. +// All fields are deterministic functions of subnetID. type IMDSSubnetSpec struct { SubnetID string ShortSubnetID string // last 8 chars of SubnetID @@ -30,10 +28,8 @@ type IMDSSubnetSpec struct { HostEndName string // imds-h-{shortSubnetID} } -// IMDSTopologyManager installs and removes the per-subnet localport that claims -// 169.254.169.254 directly on the guest's subnet switch, so metadata is reached -// over a single L2 hop on the guest's own broadcast domain — no router in the -// path. Idempotent. +// IMDSTopologyManager installs the per-subnet localport for 169.254.169.254 +// on the guest's subnet switch (one L2 hop, no router). Idempotent. type IMDSTopologyManager interface { EnsureForSubnet(ctx context.Context, subnetID, vpcID string, cidr netip.Prefix) (IMDSSubnetSpec, error) RemoveForSubnet(ctx context.Context, subnetID string) error @@ -73,12 +69,8 @@ func IMDSSpecForSubnet(subnetID string) IMDSSubnetSpec { } // EnsureForSubnet installs the IMDS localport on subnet-{subnetID} and publishes -// the subnet-veth record. Idempotent: a present record short-circuits, and the -// localport is created only when absent so a lost record still converges. The -// subnet switch must already exist (the subnet lifecycle owns it). cidr is -// persisted in the record so the host reply path resolves the guest on-link; -// vpcID is persisted so the IMDS handler can resolve the subnet→VPC mapping the -// eni-by-vpc-ip index needs (the index stays keyed vpcID/ip). +// the subnet-veth record. Idempotent: present record short-circuits; lost record +// reconverges. Subnet switch must already exist; cidr and vpcID are persisted. func (m *imdsTopologyManager) EnsureForSubnet(ctx context.Context, subnetID, vpcID string, cidr netip.Prefix) (IMDSSubnetSpec, error) { if subnetID == "" { return IMDSSubnetSpec{}, errors.New("EnsureForSubnet: subnetID required") @@ -103,11 +95,8 @@ func (m *imdsTopologyManager) EnsureForSubnet(ctx context.Context, subnetID, vpc extIDs := map[string]string{"spinifex:subnet_id": subnetID, "spinifex:role": "imds"} - // Host-owned localport on the guest's subnet switch: no port_security (the - // host sources 169.254.169.254 frames, and port_security would make - // ovn-controller drop reply frames from the host veth's MAC) and no SG - // membership (so the guest's SG ACLs never match it). localport binds on - // every chassis. + // localport on the guest's switch: no port_security (host sources IMDS frames; + // port_security would drop replies) and no SG membership. Binds on every chassis. if _, err := m.ovn.GetLogicalSwitchPort(ctx, spec.LSPName); err != nil { if cErr := m.ovn.CreateLogicalSwitchPort(ctx, spec.LSName, &nbdb.LogicalSwitchPort{ Name: spec.LSPName, @@ -135,10 +124,8 @@ func (m *imdsTopologyManager) EnsureForSubnet(ctx context.Context, subnetID, vpc return spec, nil } -// RemoveForSubnet deletes the IMDS localport and the subnet-veth record. The -// subnet switch is left in place (the subnet lifecycle owns it). Idempotent and -// drift-tolerant: a missing localport is logged and skipped so the record delete -// always runs. Must run before the subnet switch goes away. +// RemoveForSubnet deletes the IMDS localport and subnet-veth record. Idempotent: +// missing localport is logged and skipped so the record delete always runs. func (m *imdsTopologyManager) RemoveForSubnet(ctx context.Context, subnetID string) error { if subnetID == "" { return errors.New("RemoveForSubnet: subnetID required") diff --git a/spinifex/network/external/pool.go b/spinifex/network/external/pool.go index 220419fc7..2c8ce2c6e 100644 --- a/spinifex/network/external/pool.go +++ b/spinifex/network/external/pool.go @@ -10,10 +10,8 @@ import ( "github.com/mulgadc/spinifex/spinifex/network/topology" ) -// linkLocalGatewayNetwork is the link-local /30 the IGW gateway LRP carries -// in distributed-NAT mode. LRP IP never goes on the wire — per-VM dnat_and_snat -// with external_mac/logical_port handles upstream ARP per chassis. -// Centralised NAT requires a WAN-subnet IP from gw_lrp_range instead. +// linkLocalGatewayNetwork is the link-local /30 for the IGW gateway LRP in +// distributed-NAT mode. Centralised NAT uses a WAN-subnet IP from gw_lrp_range. const linkLocalGatewayNetwork = "169.254.0.1/30" const linkLocalGatewayNexthop = "169.254.0.2" @@ -44,10 +42,8 @@ func FindPool(pools []ExternalPoolConfig, region, az string) *ExternalPoolConfig return nil } -// GatewayIPAllocator resolves the gateway LRP IP for a VPC. -// ok=false means caller falls back to link-local; error means abort. -// nexthop is the upstream gateway the VPC's default route should point at; -// empty means caller falls back to pool.Gateway / link-local. +// GatewayIPAllocator resolves the gateway LRP IP for a VPC. ok=false falls +// back to link-local; error aborts. nexthop empty falls back to pool.Gateway. type GatewayIPAllocator interface { Allocate(ctx context.Context, vpcID string, pool *ExternalPoolConfig) (ip string, prefixLen int, nexthop string, ok bool, err error) Release(ctx context.Context, vpcID string) error diff --git a/spinifex/network/external/static_pool.go b/spinifex/network/external/static_pool.go index bc00d13c8..3ff85a87b 100644 --- a/spinifex/network/external/static_pool.go +++ b/spinifex/network/external/static_pool.go @@ -52,11 +52,8 @@ type PoolRecord struct { GwLrpRangeStart string `json:"gw_lrp_range_start,omitempty"` GwLrpRangeEnd string `json:"gw_lrp_range_end,omitempty"` Allocated map[string]ExternalIPAllocation `json:"allocated"` - // Cursor is the last address handed out. Allocation resumes just past it - // and wraps at RangeEnd, so a released IP is not reused until the cursor - // cycles the whole range — by which time the host ARP cache for the prior - // owner has decayed and its OVN gateway state is gone. Empty on a fresh - // pool (older records unmarshal with no cursor) → first cycle is lowest-free. + // Cursor is the last address handed out; allocation resumes just past it, + // wrapping at RangeEnd so a released IP is not immediately reused. Cursor string `json:"cursor,omitempty"` } @@ -280,13 +277,8 @@ func (a *StaticPoolAllocator) getRecord(poolName string) (*PoolRecord, uint64, e return &record, entry.Revision(), nil } -// nextAvailableIP picks the next unallocated address in the pool's range, -// skipping addresses inside [GwLrpRangeStart, GwLrpRangeEnd]. Allocation -// resumes one past record.Cursor and wraps at RangeEnd so a just-released IP -// is not handed straight back — it only becomes a candidate again after the -// cursor cycles the whole range. An empty cursor (fresh pool, or an older KV -// record) starts at RangeStart, preserving lowest-free order for the first -// cycle. Carved from the pre-Q1 handlers/ec2/vpc.nextAvailableExternalIP. +// nextAvailableIP picks the next unallocated address, skipping [GwLrpRangeStart, +// GwLrpRangeEnd]. Resumes one past Cursor, wrapping at RangeEnd. func nextAvailableIP(record *PoolRecord) (string, error) { startIP := net.ParseIP(record.RangeStart).To4() endIP := net.ParseIP(record.RangeEnd).To4() @@ -313,7 +305,7 @@ func nextAvailableIP(record *PoolRecord) (string, error) { } total := endInt - startInt + 1 - // Resume just past the cursor; empty/out-of-range cursor → start of range. + // Empty or out-of-range cursor starts at RangeStart. offset := uint32(0) if c := net.ParseIP(record.Cursor).To4(); c != nil { ci := ipv4ToUint32(c) diff --git a/spinifex/network/external/static_pool_test.go b/spinifex/network/external/static_pool_test.go index fd85992b8..33cbb7eb4 100644 --- a/spinifex/network/external/static_pool_test.go +++ b/spinifex/network/external/static_pool_test.go @@ -63,9 +63,8 @@ func TestStaticPool_GatewayReserved(t *testing.T) { } // TestStaticPool_NoInstantReuseAfterRelease pins the round-robin reuse policy: -// a released IP must not be handed straight back (the recycle that left a -// freshly-reassigned EIP unreachable from the same host — siv-246). It is only -// reusable after the cursor cycles the range. +// a released IP must not be handed back immediately; only after the cursor +// cycles the range. func TestStaticPool_NoInstantReuseAfterRelease(t *testing.T) { a := newStaticAllocator(t, []ExternalPoolConfig{wanPool()}) ctx := context.Background() diff --git a/spinifex/network/host/gateway_claim.go b/spinifex/network/host/gateway_claim.go index 5ba4cbf1d..ea0c32a54 100644 --- a/spinifex/network/host/gateway_claim.go +++ b/spinifex/network/host/gateway_claim.go @@ -8,11 +8,8 @@ import ( "github.com/mulgadc/spinifex/spinifex/utils" ) -// GatewayClaimProber checks and repairs the Southbound chassis claim for OVN -// gateway router ports by shelling out to ovn-sbctl / ovn-appctl. It backs the -// reconcile.GatewayClaimVerifier interface; the compile-time check lives at the -// wiring site (vpcd), since this host-layer package cannot import the reconcile -// cross-cutter without re-tangling the layer tree. +// GatewayClaimProber checks the SB chassis claim for OVN gateway router ports. +// Backs reconcile.GatewayClaimVerifier; compile-time check lives in vpcd. type GatewayClaimProber struct { sbAddr string } @@ -24,21 +21,16 @@ func NewGatewayClaimProber(sbAddr string) *GatewayClaimProber { } // GatewayPortClaimed reports whether the SB Port_Binding for crPortName has a -// non-empty chassis. crPortName is the chassisredirect (cr-) port of a -// distributed gateway port: the bare LRP binding stays distributed/chassis-less, -// so the chassisredirect port is the one that carries the gateway-chassis claim. -// An unclaimed binding (chassis == []) means ovn-controller has not installed -// flows for the gateway redirect, so the floating IPs behind it are unreachable. -// ctx is part of the verifier contract; ovn-sbctl is a short-lived local query -// and is not cancelled mid-flight. +// non-empty chassis. Uses the chassisredirect (cr-) port — the bare LRP stays +// chassis-less. Unclaimed binding means flows are not installed and EIPs unreachable. func (p *GatewayClaimProber) GatewayPortClaimed(_ context.Context, crPortName string) (bool, error) { args := []string{"--no-leader-only"} if p.sbAddr != "" { args = append(args, "--db="+p.sbAddr) } args = append(args, "--bare", "--columns=chassis", "find", "Port_Binding", "logical_port="+crPortName) - // Output() not CombinedOutput(): sudo PAM audit noise on stderr would - // otherwise be read as a non-empty chassis value. + // Output() not CombinedOutput(): sudo PAM noise on stderr would be + // misread as a non-empty chassis value. out, err := utils.SudoCommand("ovn-sbctl", args...).Output() if err != nil { return false, fmt.Errorf("ovn-sbctl find Port_Binding %s: %w", crPortName, err) diff --git a/spinifex/network/host/host.go b/spinifex/network/host/host.go index def2527db..620cde9d7 100644 --- a/spinifex/network/host/host.go +++ b/spinifex/network/host/host.go @@ -1,7 +1,6 @@ -// Package host is L0 of the spinifex network stack: kernel/OVS host wiring. -// Owns OVS bridges (br-int, br-ext), the uplink port attaching br-ext to WAN, -// and the external CIDR. Bridge mode (physical vs veth) is resolved here and -// invisible above L0. See ADR-0006 (S2, S3). +// Package host is L0 of the spinifex network stack: OVS bridges (br-int, +// br-ext), uplink port, and external CIDR. Bridge mode resolved here +// and invisible above L0 (ADR-0006 S2, S3). package host import ( diff --git a/spinifex/network/host/imds_names.go b/spinifex/network/host/imds_names.go index fea55b4c5..45824c51d 100644 --- a/spinifex/network/host/imds_names.go +++ b/spinifex/network/host/imds_names.go @@ -1,13 +1,10 @@ package host -// IMDS per-subnet veth naming. The names live here (not in network/topology) so -// the IMDS BindManager can derive them without importing topology, which would -// form an import cycle (topology imports handlers/imds for MetaDataServerIP). +// IMDS per-subnet veth naming lives here (not in network/topology) to avoid an +// import cycle: topology imports handlers/imds for MetaDataServerIP. -// IMDSShortSubnetID is the last 8 chars of the subnet ID. Linux IFNAMSIZ caps -// interface names at 15 chars, so the IMDS veth names key off this short form. -// Deterministic and stable: every chassis serving IMDS for subnet X derives the -// same names. +// IMDSShortSubnetID returns the last 8 chars of subnetID. Linux IFNAMSIZ caps +// interface names at 15 chars, so veth names key off this short form. func IMDSShortSubnetID(subnetID string) string { if len(subnetID) <= 8 { return subnetID @@ -15,22 +12,14 @@ func IMDSShortSubnetID(subnetID string) string { return subnetID[len(subnetID)-8:] } -// IMDSOVSPortName is the br-int side of the per-subnet IMDS veth pair, bound to the -// imds-port LSP via external_ids:iface-id. The 7-char prefix keeps "imds-o-" + the -// 8-char short subnet ID at exactly IFNAMSIZ-1 (15); longer fails `ip link add`. +// IMDSOVSPortName is the br-int side of the per-subnet IMDS veth pair, bound via +// external_ids:iface-id. "imds-o-" + 8-char short ID = 15 chars (IFNAMSIZ-1). func IMDSOVSPortName(subnetID string) string { return "imds-o-" + IMDSShortSubnetID(subnetID) } -// IMDSHostVethName is the host-end side of the per-subnet IMDS veth pair — the -// device the IMDS listener binds to via SO_BINDTODEVICE. Same 7-char prefix -// budget as IMDSOVSPortName: "imds-h-" + 8-char short subnet ID = 15 chars. It -// lives inside the subnet's netns (see IMDSNetnsName), not the root netns. +// IMDSHostVethName is the host-end of the IMDS veth pair (SO_BINDTODEVICE). +// "imds-h-" + 8-char short ID = 15 chars; lives in the subnet netns. func IMDSHostVethName(subnetID string) string { return "imds-h-" + IMDSShortSubnetID(subnetID) } -// IMDSNetnsName is the per-subnet network namespace the IMDS host-end veth and its -// listener live in. The netns gives the reply path a real L3 next-hop — the host -// end carries 169.254.169.254/30 with the subnet CIDR on-link, so the reply to the -// guest resolves by ARP over the localport — and structurally isolates subnets -// with overlapping CIDRs, since each netns is its own routing domain. Netns names -// are filesystem-scoped, not IFNAMSIZ-bound, so "imds-" + 8-char short subnet ID -// (13 chars) is comfortably within budget. +// IMDSNetnsName is the per-subnet network namespace for the IMDS host-end veth. +// Isolates overlapping CIDRs into separate routing domains; not IFNAMSIZ-bound. func IMDSNetnsName(subnetID string) string { return "imds-" + IMDSShortSubnetID(subnetID) } diff --git a/spinifex/network/host/imds_netns_unit_test.go b/spinifex/network/host/imds_netns_unit_test.go index 47fd99cb3..5a250d1f7 100644 --- a/spinifex/network/host/imds_netns_unit_test.go +++ b/spinifex/network/host/imds_netns_unit_test.go @@ -8,9 +8,8 @@ import ( "testing" ) -// repoRoot resolves the spinifex submodule root from this test file's location -// (spinifex/network/host/ → three levels up) so the systemd-unit invariants can -// read build/ artifacts regardless of the test's working directory. +// repoRoot resolves the spinifex module root from this file's location +// so systemd-unit invariant tests can read build/ artifacts. func repoRoot(t *testing.T) string { t.Helper() _, file, _, ok := runtime.Caller(0) @@ -33,24 +32,16 @@ func readUnit(t *testing.T, root, rel string) string { return string(b) } -// TestIMDS_NetnsHostMountNSInvariant pins the host-mount-namespace contract for -// per-VPC IMDS netns (see spinifex-vpcd.service + bead js-133): vpcd creates the -// netns via `ip netns add` and binds the listener via in-process setns, so the -// /run/netns/ bind-mount must live in the HOST mount namespace — host-visible -// for diagnostics and durable across a vpcd restart. Any filesystem-sandbox -// directive forks a PRIVATE mount ns that traps the bind-mount inside vpcd, so the -// host sees only a stale handle that fails setns(2) EINVAL (CI runs 26920490454, -// 26922587471). The earlier MountFlags=shared + spinifex-netns.service shared-mount -// workaround was proven insufficient and removed; vpcd now simply shares the host -// mount ns. Guard that shape: re-adding any mount-ns directive reintroduces the bug. +// TestIMDS_NetnsHostMountNSInvariant pins the host-mount-namespace contract: vpcd +// creates netns via `ip netns add` into the HOST mount namespace. A sandbox +// directive forks a private mount ns, trapping the bind-mount (setns EINVAL). func TestIMDS_NetnsHostMountNSInvariant(t *testing.T) { root := repoRoot(t) vpcd := readUnit(t, root, "build/systemd/spinifex-vpcd.service") target := readUnit(t, root, "build/systemd/spinifex.target") - // No filesystem-sandbox directive may be set on vpcd: each forks a private - // mount namespace and re-traps the netns bind-mount. + // No filesystem-sandbox directive: each forks a private mount ns and traps the bind-mount. for _, banned := range []string{ "ProtectSystem", "ProtectHome", "PrivateTmp", "ProtectKernelTunables", "ProtectKernelModules", "ProtectKernelLogs", "ProtectControlGroups", diff --git a/spinifex/network/host/imds_veth.go b/spinifex/network/host/imds_veth.go index 94a8679b5..8f2ab39c6 100644 --- a/spinifex/network/host/imds_veth.go +++ b/spinifex/network/host/imds_veth.go @@ -10,33 +10,22 @@ import ( "github.com/mulgadc/spinifex/spinifex/utils" ) -// imdsNetnsAddr is the host/listener side of the IMDS address, assigned to the -// veth host end inside the per-subnet netns. .254 is MetaDataServerIP; the /30 is -// kept (no longer load-bearing for a next-hop) and the reply path resolves the -// guest via the subnet CIDR added on-link out the same veth. +// imdsNetnsAddr is the IMDS address assigned to the veth host end inside the netns. +// Reply path resolves the guest via the subnet CIDR added on-link. const imdsNetnsAddr = "169.254.169.254/30" -// imdsPortLSPName is the OVN logical-switch-port the OVS-side veth binds to via -// external_ids:iface-id. It mirrors topology.IMDSPort but is duplicated here to -// avoid an import cycle (host ← handlers/imds ← topology). +// imdsPortLSPName returns the LSP name the OVS-side veth binds to. +// Duplicated from topology.IMDSPort to avoid a host←handlers/imds←topology cycle. func imdsPortLSPName(subnetID string) string { return "imds-port-" + subnetID } -// imdsLSPMAC is the logical MAC OVN advertises for 169.254.169.254 (the localport -// address). The netns veth host end MUST carry this exact MAC: OVN resolves .254 -// to it and delivers IMDS request frames with dl_dst set to it, so a kernel-random -// veth MAC makes the netns drop every frame not-for-me and the listener never sees -// a SYN. Mirrors external.IMDSSpecForSubnet's LSPMAC, duplicated here (like -// imdsPortLSPName) to avoid the import cycle. +// imdsLSPMAC returns the logical MAC OVN uses for 169.254.169.254. The host-end veth +// must carry this MAC exactly — OVN delivers frames with dl_dst set to it. +// Duplicated from external.IMDSSpecForSubnet to avoid the import cycle. func imdsLSPMAC(subnetID string) string { return utils.HashMAC("imds-" + subnetID) } -// EnsureIMDSVeth idempotently creates the per-subnet IMDS veth pair, moves the host -// end into a dedicated netns with 169.254.169.254/30 + the subnet CIDR on-link, and -// attaches the OVS end to br-int so ovn-controller binds the localport here. The -// netns gives the host-served reply a real L3 next-hop (SO_BINDTODEVICE alone cannot -// route the SYN-ACK): the on-link subnet route makes the guest directly reachable, so -// the reply resolves by ARP over the localport on the guest's own L2 switch. The netns -// also isolates overlapping subnet CIDRs into separate routing domains. Returns the -// netns name and host-end name the listener enters and binds. +// EnsureIMDSVeth idempotently creates the per-subnet IMDS veth pair: moves the host +// end into a dedicated netns with 169.254.169.254/30 + subnet CIDR on-link, and +// attaches the OVS end to br-int. Returns netns and host-end names. func EnsureIMDSVeth(ctx context.Context, subnetID string, cidr netip.Prefix) (netnsName, hostEndName string, err error) { if !cidr.IsValid() { return "", "", fmt.Errorf("EnsureIMDSVeth: subnet %s requires a valid CIDR", subnetID) @@ -45,13 +34,9 @@ func EnsureIMDSVeth(ctx context.Context, subnetID string, cidr netip.Prefix) (ne hostEnd := IMDSHostVethName(subnetID) netns := IMDSNetnsName(subnetID) - // Idempotency probe: the full plumbing (veth pair, netns, host-end address + - // route) exists from a prior boot only when the OVS end is a port on br-int - // AND the netns is enterable. A live OVS port over a stale netns (name - // present but setns(2) fails EINVAL — e.g. a crash between umount and unlink, - // or a tmpfs remount across reboot) leaves the listener permanently - // unbindable, so that case must NOT short-circuit: tear the inconsistent - // plumbing down and rebuild it below. + // Idempotency: plumbing is complete only when both the OVS port is on br-int + // AND the netns is enterable. A live OVS port over an unenterable netns + // (stale /run/netns handle, setns EINVAL) must not short-circuit — tear and rebuild. if imdsOVSPortOnBrInt(ovsEnd) { if netnsEnterable(netns) { slog.Debug("IMDS veth already present", "subnet", subnetID, "ovs_end", ovsEnd, "host_end", hostEnd, "netns", netns) @@ -95,15 +80,9 @@ func EnsureIMDSVeth(ctx context.Context, subnetID string, cidr netip.Prefix) (ne return netns, hostEnd, nil } -// configureIMDSNetns moves the host end into the netns, sets its MAC to the -// localport's logical MAC, brings it (and lo) up, and assigns the IMDS address + -// the subnet CIDR on-link so the host-served reply resolves the guest by ARP over -// the localport (a single L2 hop on the guest's own switch — no router next-hop). -// The OVS end stays in the root netns for ovn-controller to bind. The MAC set is -// the request-path contract: OVN delivers frames with dl_dst = the localport MAC, -// so the veth host end must own that MAC or the netns silently drops them. The -// addr/route adds tolerate "File exists" so a re-run after a partial failure -// converges rather than wedging. +// configureIMDSNetns moves the host end into the netns, sets its MAC (OVN delivers +// frames with dl_dst = localport MAC, so the veth must own it), brings both lo and +// hostEnd up, and adds the IMDS addr + subnet CIDR on-link. Tolerates "File exists". func configureIMDSNetns(netns, hostEnd, ovsEnd, hostEndMAC string, cidr netip.Prefix) error { if out, err := utils.SudoCommand("ip", "link", "set", ovsEnd, "up").CombinedOutput(); err != nil { return fmt.Errorf("bring up IMDS OVS end %s: %s: %w", ovsEnd, strings.TrimSpace(string(out)), err) @@ -125,11 +104,8 @@ func configureIMDSNetns(netns, hostEnd, ovsEnd, hostEndMAC string, cidr netip.Pr return ipNetnsTolerate(netns, "File exists", "route", "add", cidr.String(), "dev", hostEnd) } -// ensureNetns creates the netns, treating "already exists" as success — but -// only when the pre-existing handle is actually enterable. A stale -// /run/netns/ bind-mount (name resolves yet setns(2) fails EINVAL) returns -// "File exists" from `ip netns add` while being unusable, which would leave the -// IMDS listener permanently unbindable; that handle is torn down and recreated. +// ensureNetns creates the netns, treating "already exists" as success only if the +// handle is enterable. A stale bind-mount (setns EINVAL) is torn down and recreated. func ensureNetns(netns string) error { out, err := utils.SudoCommand("ip", "netns", "add", netns).CombinedOutput() if err == nil { @@ -153,23 +129,20 @@ func ensureNetns(netns string) error { return nil } -// imdsOVSPortOnBrInt reports whether the IMDS OVS-end veth is currently a port -// on br-int — the cheap signal that prior plumbing exists. +// imdsOVSPortOnBrInt reports whether the OVS-end veth is on br-int. func imdsOVSPortOnBrInt(ovsEnd string) bool { out, err := utils.SudoCommand("ovs-vsctl", "port-to-br", ovsEnd).CombinedOutput() return err == nil && strings.TrimSpace(string(out)) == "br-int" } -// netnsEnterable reports whether netns can be entered via setns(2). A stale -// bind-mount resolves by name but fails EINVAL; `ip -n link show lo` -// performs the setns and surfaces it. A truly-absent netns also reports false, -// which is the correct signal for both callers (recreate / rebuild). +// netnsEnterable reports whether netns can be entered via setns(2). +// A stale bind-mount or absent netns both return false. func netnsEnterable(netns string) bool { return utils.SudoCommand("ip", "-n", netns, "link", "show", "lo").Run() == nil } // ipNetnsTolerate runs `ip -n `, treating output containing -// tolerate as success (for idempotent re-runs of addr/route adds). +// tolerate as success (idempotent addr/route adds). func ipNetnsTolerate(netns, tolerate string, args ...string) error { full := append([]string{"-n", netns}, args...) if out, err := utils.SudoCommand("ip", full...).CombinedOutput(); err != nil { @@ -181,16 +154,14 @@ func ipNetnsTolerate(netns, tolerate string, args ...string) error { return nil } -// RemoveIMDSVeth detaches the OVS port, deletes the netns (which destroys the -// host end and thus the veth pair), and clears any root-netns remnant. Idempotent: -// safe to call for a subnet that never had a veth on this chassis. +// RemoveIMDSVeth detaches the OVS port, deletes the netns, and clears any leftover +// veth. Idempotent. func RemoveIMDSVeth(ctx context.Context, subnetID string) error { return removeIMDSPlumbing(IMDSOVSPortName(subnetID), IMDSHostVethName(subnetID), IMDSNetnsName(subnetID)) } -// removeIMDSPlumbing tears down the OVS port, the netns, and any leftover veth. -// Deleting the netns destroys the host end (and its peer); the trailing link del -// covers the partial state where the host end was never moved into the netns. +// removeIMDSPlumbing removes the OVS port, netns, and any leftover veth. +// The trailing link del covers the case where the host end was never moved to the netns. func removeIMDSPlumbing(ovsEnd, hostEnd, netns string) error { if out, err := utils.SudoCommand("ovs-vsctl", "--if-exists", "del-port", ovsEnd).CombinedOutput(); err != nil { slog.Warn("Failed to remove IMDS veth from OVS", "ovs_end", ovsEnd, "err", err, "out", strings.TrimSpace(string(out))) @@ -198,8 +169,7 @@ func removeIMDSPlumbing(ovsEnd, hostEnd, netns string) error { if out, err := utils.SudoCommand("ip", "netns", "del", netns).CombinedOutput(); err != nil { msg := strings.TrimSpace(string(out)) - // "No such file or directory" — already gone. Anything else is logged - // but not fatal: the link del below still runs. + // "No such file or directory" — already gone; other errors logged but not fatal. if !strings.Contains(msg, "No such file") { slog.Warn("Failed to delete IMDS netns", "netns", netns, "err", err, "out", msg) } diff --git a/spinifex/network/host/imds_veth_test.go b/spinifex/network/host/imds_veth_test.go index 55f846b07..d41527da6 100644 --- a/spinifex/network/host/imds_veth_test.go +++ b/spinifex/network/host/imds_veth_test.go @@ -109,10 +109,9 @@ func TestEnsureIMDSVeth_Idempotent(t *testing.T) { } } -// TestEnsureIMDSVeth_StaleNetnsBehindLivePort_Rebuilds guards Fix #2: a live -// OVS port over an unenterable netns (stale /run/netns handle, setns EINVAL) -// must NOT short-circuit. The inconsistent plumbing is torn down and rebuilt so -// the IMDS listener becomes bindable again, rather than wedging forever. +// TestEnsureIMDSVeth_StaleNetnsBehindLivePort_Rebuilds guards against a live OVS +// port with an unenterable netns (setns EINVAL): plumbing must be torn down +// and rebuilt so the IMDS listener is bindable rather than wedging forever. func TestEnsureIMDSVeth_StaleNetnsBehindLivePort_Rebuilds(t *testing.T) { calls := recordSudo(t, func(name string, args []string) *exec.Cmd { if name == "ovs-vsctl" && len(args) >= 1 && args[0] == "port-to-br" { @@ -333,10 +332,9 @@ func TestRemoveIMDSVeth_MissingDeviceIsNotError(t *testing.T) { } } -// TestIMDSVethNamesWithinIFNAMSIZ guards the load-bearing constraint that both -// veth-end names fit Linux's IFNAMSIZ-1 (15 chars). A longer name makes -// `ip link add` fail with "name too long" for every subnet, silently taking IMDS -// offline cluster-wide — and the SudoCommand mock would never catch it. +// TestIMDSVethNamesWithinIFNAMSIZ guards that both veth-end names fit IFNAMSIZ-1 +// (15 chars). A longer name causes `ip link add` to fail cluster-wide, silently +// taking IMDS offline in a way the SudoCommand mock would never catch. func TestIMDSVethNamesWithinIFNAMSIZ(t *testing.T) { const ifnamsizMax = 15 for _, subnetID := range []string{ diff --git a/spinifex/network/host/ipsec.go b/spinifex/network/host/ipsec.go index 1fbf24e78..86997c93b 100644 --- a/spinifex/network/host/ipsec.go +++ b/spinifex/network/host/ipsec.go @@ -35,11 +35,9 @@ func EnableIPSecEncapsulation() error { return nil } -// SetNBGlobalIPSec writes NB_Global.ipsec on the local OVN NB DB. Triggers -// ovn-controller on every chassis to add options:remote_name to Geneve tunnel -// ports, which is what ovs-monitor-ipsec keys off to materialise strongSwan -// connections. Only the management node has a local NB socket; callers gate -// the call on socket presence. +// SetNBGlobalIPSec writes NB_Global.ipsec on the local OVN NB DB, triggering +// ovn-controller to add options:remote_name to Geneve tunnels for strongSwan. +// Only the management node has a local NB socket; callers gate on presence. func SetNBGlobalIPSec(enable bool) error { val := "false" if enable { diff --git a/spinifex/network/host/neigh.go b/spinifex/network/host/neigh.go index 9057b0270..2223442c6 100644 --- a/spinifex/network/host/neigh.go +++ b/spinifex/network/host/neigh.go @@ -6,18 +6,9 @@ import ( "strings" ) -// FlushNeigh removes the kernel neighbour (ARP) entry for ip on dev so the next -// packet to ip re-resolves L2. Wraps `ip neigh flush to dev `, which -// is idempotent — flushing zero entries still succeeds. -// -// ARP refresh for EIP recycle: ovn-controller emits its automatic GARP only on -// LSP binding-chassis migration, so a same-chassis external_ip rebind leaves the -// host neighbour cache pointed at the prior owner's MAC until the kernel ARP -// timeout (60-300s). Flushing the entry on EIP attach and detach forces a fresh -// resolution against the current owner. -// -// L0 method (ADR-0006 S2) — only network/host/ may shell out to host tools. -// Best-effort: callers must treat errors as warnings, not failures. +// FlushNeigh removes the kernel ARP entry for ip on dev, forcing re-resolution. +// Same-chassis EIP rebind skips OVN's automatic GARP, so flushing is required. +// Best-effort: callers should treat errors as warnings. func FlushNeigh(ctx context.Context, runner Runner, dev, ip string) error { if dev == "" || ip == "" { return fmt.Errorf("host.FlushNeigh: dev and ip required") @@ -32,19 +23,9 @@ func FlushNeigh(ctx context.Context, runner Runner, dev, ip string) error { return nil } -// ReplaceNeigh installs (or overwrites) the kernel neighbour entry mapping ip to -// mac on dev in NUD_REACHABLE state. Wraps `ip neigh replace lladdr -// dev nud reachable`. -// -// This is the deterministic counterpart to FlushNeigh for EIP attach: when an -// external IP is recycled onto a new owner, OVN advertises the new MAC only via -// a GARP it emits on LSP binding-chassis migration — a same-chassis rebind emits -// none, so no node answers the host's re-ARP. Flushing then waits for an ARP -// reply that never comes; programming the known external_mac directly skips the -// round-trip, and inbound traffic refreshes the entry from there on. -// -// L0 method (ADR-0006 S2) — only network/host/ may shell out to host tools. -// Best-effort: callers must treat errors as warnings, not failures. +// ReplaceNeigh installs the kernel ARP entry for ip→mac on dev (NUD_REACHABLE). +// OVN emits no GARP on same-chassis EIP rebind, so programming the MAC directly +// avoids an ARP round-trip. Best-effort: callers treat errors as warnings. func ReplaceNeigh(ctx context.Context, runner Runner, dev, ip, mac string) error { if dev == "" || ip == "" || mac == "" { return fmt.Errorf("host.ReplaceNeigh: dev, ip and mac required") diff --git a/spinifex/network/host/plumber.go b/spinifex/network/host/plumber.go index 51e365696..c92d51be6 100644 --- a/spinifex/network/host/plumber.go +++ b/spinifex/network/host/plumber.go @@ -22,10 +22,8 @@ var _ vm.NetworkPlumber = (*OVSPlumber)(nil) func NewOVSPlumber() *OVSPlumber { return &OVSPlumber{} } // SetupTap creates the kernel tap, brings it up, and attaches it to spec.Bridge. -// The pre-create del-port is unconditional because OVS conf.db survives reboot -// while kernel taps don't, so a /sys/class/net check would miss orphan ports. -// Rejected alternative: `--may-exist add-port` — it would silently keep stale -// external_ids from a prior launch with a different ENI/MAC. +// Pre-create del-port is unconditional: OVS conf.db survives reboot but kernel +// taps don't, and --may-exist would silently keep stale external_ids. func (p *OVSPlumber) SetupTap(spec vm.TapSpec) error { if err := utils.SudoCommand("ovs-vsctl", "--if-exists", "del-port", spec.Bridge, spec.Name).Run(); err != nil { slog.Warn("Pre-create del-port failed (continuing)", "tap", spec.Name, "bridge", spec.Bridge, "err", err) @@ -36,17 +34,9 @@ func (p *OVSPlumber) SetupTap(spec vm.TapSpec) error { } } - // When the daemon runs non-root (production: User=spinifex-daemon), child - // QEMU processes inherit that uid without CAP_NET_ADMIN. The kernel's tun - // attach check then requires the tap to be owned by the calling euid - // (drivers/net/tun.c: tun_chr_open + TUNSETIFF perm check). Without the - // `user` flag the tap defaults to root:root and qemu's TUNSETIFF fails - // with EPERM ("could not configure /dev/net/tun: Operation not permitted"). - // Pass numeric uid/gid directly — user.Current() can fail silently under - // hardened systemd units / static builds, dropping the flags and leaving - // the tap root-owned (LB microvm in cell-17/tofu-examples). Numeric values - // always work and skip the NSS lookup. Root daemons skip this — they have - // CAP_NET_ADMIN ambient. + // Non-root daemon: QEMU inherits the uid without CAP_NET_ADMIN, so the tap + // must be owned by the calling euid (kernel TUNSETIFF check). Use numeric + // uid/gid to avoid NSS lookup failures in hardened systemd units. addArgs := []string{"tuntap", "add", "dev", spec.Name, "mode", "tap"} if uid := os.Geteuid(); uid != 0 { addArgs = append(addArgs, "user", strconv.Itoa(uid), "group", strconv.Itoa(os.Getegid())) diff --git a/spinifex/network/ovn/client.go b/spinifex/network/ovn/client.go index afc9ae887..3775772c6 100644 --- a/spinifex/network/ovn/client.go +++ b/spinifex/network/ovn/client.go @@ -37,13 +37,8 @@ type Client interface { // Logical Switch (subnet) CreateLogicalSwitch(ctx context.Context, ls *nbdb.LogicalSwitch) error - // EnsureLogicalSwitch atomically creates a logical switch if no row with - // the same Name already exists, or returns the existing row. Wraps the - // insert in an OVSDB wait-op that asserts "zero rows match Name == ls.Name" - // at commit time; concurrent writers across nodes are serialised by - // ovsdb-server rather than by the local libovsdb cache (which can be - // stale relative to another node's just-committed insert). See - // EnsureLogicalRouter for the rationale. + // EnsureLogicalSwitch atomically creates a logical switch or returns the existing + // row. Uses an OVSDB wait-op (see EnsureLogicalRouter for rationale). EnsureLogicalSwitch(ctx context.Context, ls *nbdb.LogicalSwitch) (*nbdb.LogicalSwitch, error) DeleteLogicalSwitch(ctx context.Context, name string) error GetLogicalSwitch(ctx context.Context, name string) (*nbdb.LogicalSwitch, error) @@ -51,14 +46,9 @@ type Client interface { // Logical Switch Port (VM/ENI) CreateLogicalSwitchPort(ctx context.Context, switchName string, lsp *nbdb.LogicalSwitchPort) error - // CreateLogicalSwitchPortInGroups creates an LSP, adds it to its switch, - // and joins it to the named port groups — all in a single OVSDB - // transaction. Required for SG enforcement: a non-atomic create-then-join - // leaves a window where the LSP exists outside any port group (OVN - // default = unrestricted). portGroupNames may be empty (e.g. router/ - // localnet ports). The per-port-group `_ip4`/`_ip6` Address_Set rows in - // SB are auto-derived by ovn-northd from each port group's port - // addresses; SG-to-SG match expressions resolve against those. + // CreateLogicalSwitchPortInGroups creates an LSP and joins it to port groups in + // one transaction — prevents the window where the LSP exists outside any group + // (OVN default = unrestricted). portGroupNames may be empty. CreateLogicalSwitchPortInGroups(ctx context.Context, switchName string, lsp *nbdb.LogicalSwitchPort, portGroupNames []string) error DeleteLogicalSwitchPort(ctx context.Context, switchName string, portName string) error GetLogicalSwitchPort(ctx context.Context, name string) (*nbdb.LogicalSwitchPort, error) @@ -66,28 +56,15 @@ type Client interface { // Logical Router (VPC router) CreateLogicalRouter(ctx context.Context, lr *nbdb.LogicalRouter) error - // EnsureLogicalRouter atomically creates a logical router if no row with - // the same Name already exists, or returns the existing row. The insert - // is bundled with an OVSDB wait-op that asserts "zero rows match - // Name == lr.Name" at commit time, so two writers (different goroutines - // or different nodes) that both observe absence cannot both succeed — - // ovsdb-server rejects the second transaction. The returned pointer is - // the canonical row: newly created with the supplied fields, or the - // pre-existing row when a competing writer raced us. - // - // Required because OVN NB has no unique constraint on Name. Without the - // wait-op, the cross-handler / cross-node race between vpc.create and - // vpc.create-subnet's defensive EnsureVPC call produces duplicate routers - // per VPC. + // EnsureLogicalRouter atomically creates a logical router or returns the existing + // row. OVSDB wait-op serialises concurrent writers — NB has no unique-Name + // constraint, so without it concurrent vpc.create calls produce duplicates. EnsureLogicalRouter(ctx context.Context, lr *nbdb.LogicalRouter) (*nbdb.LogicalRouter, error) DeleteLogicalRouter(ctx context.Context, name string) error GetLogicalRouter(ctx context.Context, name string) (*nbdb.LogicalRouter, error) ListLogicalRouters(ctx context.Context) ([]nbdb.LogicalRouter, error) - // UpdateLogicalRouterExternalIDs rewrites the ExternalIDs map of an - // existing logical router. Used by topology.EnsureVPC to backfill the - // `spinifex:cidr` / `spinifex:vni` keys when the row was first created by - // a defensive EnsureVPC call that did not carry the parent VPC's full - // metadata. The whole map is replaced; callers must pass the merged set. + // UpdateLogicalRouterExternalIDs replaces the ExternalIDs map on an existing + // router. Callers must pass the full merged set. UpdateLogicalRouterExternalIDs(ctx context.Context, name string, externalIDs map[string]string) error // Logical Router Port @@ -126,46 +103,28 @@ type Client interface { // Port Groups (security group enforcement) CreatePortGroup(ctx context.Context, name string, ports []string) error - // EnsurePortGroup is the wait-op-protected create-or-get analogue of - // CreatePortGroup. Returns the canonical row. See EnsureLogicalRouter for - // the wait-op pattern and rationale. + // EnsurePortGroup atomically creates a port group or returns the existing row. + // See EnsureLogicalRouter for the wait-op rationale. EnsurePortGroup(ctx context.Context, name string, ports []string) (*nbdb.PortGroup, error) DeletePortGroup(ctx context.Context, name string) error - // UpdatePortGroupMemberships applies all port-group joins and leaves for a - // single LSP in one atomic OVSDB transaction. Required by reconcilePortSGs - // so a 5-SG → different-5-SG modify never exposes an intermediate state - // with fewer port groups (which would let the OVN default = unrestricted - // apply for the gap). The per-port-group `_ip4`/`_ip6` Address_Set rows - // in SB are auto-derived by ovn-northd from each port group's port - // addresses; no explicit address-set update is required here. + // UpdatePortGroupMemberships applies all port-group joins and leaves for an LSP + // in one transaction, preventing an intermediate state with fewer groups + // (OVN default = unrestricted). UpdatePortGroupMemberships(ctx context.Context, lspName string, addPGs, removePGs []string) error - // ListPortGroupsForPort returns the names of every port group whose Ports - // set contains the given LSP. Used by reconcilePortSGs to discover current - // membership before computing the add/remove diff against desired. + // ListPortGroupsForPort returns the names of every port group containing the LSP. ListPortGroupsForPort(ctx context.Context, lspName string) ([]string, error) - // GetPortGroup returns the port group with the given name, or an error if - // it doesn't exist. Used by the reconciler to detect SGs whose port group - // has gone missing in OVN NB. + // GetPortGroup returns the port group with the given name, or an error if absent. GetPortGroup(ctx context.Context, name string) (*nbdb.PortGroup, error) - // ListPortGroups returns every port group in OVN NB. Used by the - // reconciler's orphan-PG scan to detect spinifex-managed port groups - // (`sg_*`) that no longer have a matching SG record in KV. + // ListPortGroups returns every port group in OVN NB. ListPortGroups(ctx context.Context) ([]nbdb.PortGroup, error) - // ACLs (attached to port groups). AddACLs creates all rows and links - // them to the port group in one OVSDB transaction — important when a - // single SG can carry up to 60 ingress + 60 egress rules. + // ACLs (attached to port groups). AddACLs creates all rows and links them + // to the port group in one transaction. AddACLs(ctx context.Context, portGroupName string, specs []ACLSpec) error ClearACLs(ctx context.Context, portGroupName string) error - // ReplaceACLs atomically swaps the port group's ACL set: detach + delete - // the existing rows and attach freshly-created rows in ONE OVSDB - // transaction. SG mutations must use this instead of ClearACLs followed - // by AddACLs — the latter leaves the port group with zero ACLs between - // transactions, and a no-ACL port group on a tenant LSP path defaults - // to drop, producing observable mid-flight egress drops on connectionless - // traffic (ICMP) and on TCP SYNs that don't match an existing conntrack - // entry. + // ReplaceACLs atomically swaps the port group's ACL set in one transaction. + // Use instead of ClearACLs+AddACLs to avoid a zero-ACL window (defaults to drop). ReplaceACLs(ctx context.Context, portGroupName string, specs []ACLSpec) error // Gateway Chassis (HA scheduling for gateway router ports) diff --git a/spinifex/network/ovn/live.go b/spinifex/network/ovn/live.go index 79b974b89..5bdc531e4 100644 --- a/spinifex/network/ovn/live.go +++ b/spinifex/network/ovn/live.go @@ -24,11 +24,9 @@ const ensureRefetchInterval = 20 * time.Millisecond // refetch path on conflict. const ensureWaitTimeoutMS = 0 -// ensureNamedRowOps returns ops that atomically "insert createObj unless a row -// with this Name exists". OVN NB lacks a unique-Name constraint; the paired -// wait+create lets ovsdb-server serialise concurrent writers, and the -// `until=!=` inverse formulation is required because RFC 7047 §5.2.6 mandates -// a non-empty `rows` member (libovsdb drops `[]` via `omitempty`). +// ensureNamedRowOps returns ops that insert createObj unless a row with the same +// Name exists. The wait-op serialises concurrent writers; `until=!=` is required +// because RFC 7047 §5.2.6 mandates a non-empty `rows` member. func (c *LiveClient) ensureNamedRowOps(table, name string, createObj model.Model) ([]ovsdb.Operation, error) { createOps, err := c.client.Create(createObj) if err != nil { @@ -68,10 +66,8 @@ func (c *LiveClient) transactOps(ctx context.Context, ops []ovsdb.Operation) err return err } -// namedUUID builds a valid OVSDB named-uuid by replacing every non -// [_a-zA-Z0-9] byte with `_`. Required for ACL UUIDs that embed Match -// expressions like "outport == @pg-sg-XYZ && ip4"; without this, OVSDB -// rejects them with "Type mismatch for member 'uuid-name'". +// namedUUID builds a valid OVSDB named-uuid by replacing non-[_a-zA-Z0-9] +// bytes with `_`. Required for ACL UUIDs embedding match expressions. func namedUUID(prefix, name string) string { s := prefix + name result := make([]byte, len(s)) @@ -235,7 +231,6 @@ func (c *LiveClient) ListLogicalSwitches(ctx context.Context) ([]nbdb.LogicalSwi } func (c *LiveClient) CreateLogicalSwitchPort(ctx context.Context, switchName string, lsp *nbdb.LogicalSwitchPort) error { - // Named UUID lets the same transaction reference the port. if lsp.UUID == "" { lsp.UUID = namedUUID("lsp_", lsp.Name) } @@ -268,9 +263,7 @@ func (c *LiveClient) CreateLogicalSwitchPort(ctx context.Context, switchName str } // CreateLogicalSwitchPortInGroups bundles LSP create, switch-ports mutate, and -// per-port-group ports mutates into one transaction so the LSP's named UUID -// resolves across all ops. The per-PG `_ip4`/`_ip6` Address_Set rows are -// auto-derived by ovn-northd; vpcd must not write them (see provisionSG). +// per-PG ports mutates into one transaction so the LSP's named UUID resolves across all ops. func (c *LiveClient) CreateLogicalSwitchPortInGroups(ctx context.Context, switchName string, lsp *nbdb.LogicalSwitchPort, portGroupNames []string) error { if lsp.UUID == "" { lsp.UUID = namedUUID("lsp_", lsp.Name) @@ -495,7 +488,6 @@ func (c *LiveClient) ListLogicalRouters(ctx context.Context) ([]nbdb.LogicalRout } func (c *LiveClient) CreateLogicalRouterPort(ctx context.Context, routerName string, lrp *nbdb.LogicalRouterPort) error { - // Named UUID lets the same transaction reference the port. if lrp.UUID == "" { lrp.UUID = namedUUID("lrp_", lrp.Name) } @@ -594,8 +586,7 @@ func (c *LiveClient) UpdateLogicalRouterPort(ctx context.Context, lrp *nbdb.Logi } // SetGatewayChassis binds a chassis to an LRP for HA gateway scheduling. -// Read-then-decide so reconcile/IGW-attach can re-run safely: absent → -// create + LRP-mutate; same priority → no-op; differing priority → mutate. +// Idempotent: absent → create+mutate; same priority → no-op; different priority → update. func (c *LiveClient) SetGatewayChassis(ctx context.Context, lrpName string, chassisName string, priority int) error { gcName := lrpName + "-" + chassisName existing, err := c.GetGatewayChassisByName(ctx, gcName) @@ -672,8 +663,7 @@ func (c *LiveClient) GetGatewayChassisByName(ctx context.Context, name string) ( return &rows[0], nil } -// ListGatewayChassis returns every Gateway_Chassis row; reconcile uses this -// to find rows whose chassis_name no longer exists in SBDB. +// ListGatewayChassis returns every Gateway_Chassis row. func (c *LiveClient) ListGatewayChassis(ctx context.Context) ([]nbdb.GatewayChassis, error) { var rows []nbdb.GatewayChassis if err := c.client.List(ctx, &rows); err != nil { @@ -822,9 +812,7 @@ func (c *LiveClient) AddNAT(ctx context.Context, routerName string, nat *nbdb.NA } // DeleteNAT removes a NAT rule matching (natType, logicalIP) from routerName. -// Lookup is scoped to routerName.NAT — rules owned by other routers with the -// same logical IP are left alone. Required because AWS subnet CIDRs repeat -// across VPCs, so (natType, logicalIP) is not globally unique. +// Scoped to routerName.NAT — AWS CIDRs repeat across VPCs so the pair is not globally unique. func (c *LiveClient) DeleteNAT(ctx context.Context, routerName string, natType, logicalIP string) error { lr, err := c.GetLogicalRouter(ctx, routerName) if err != nil { @@ -872,11 +860,8 @@ func (c *LiveClient) DeleteNAT(ctx context.Context, routerName string, natType, return nil } -// DeleteNATByExternalIP removes a NAT rule matching the given external IP from -// routerName only. Lookup is scoped to routerName.NAT — rules on other routers -// with the same external IP are left alone (use DeleteAllNATsByExternalIP for -// cross-router cleanup). Returns ErrNATNotFound if no matching rule is found -// on this router (callers can ignore this). +// DeleteNATByExternalIP removes a NAT rule matching (natType, externalIP) from +// routerName only. Returns ErrNATNotFound if absent on this router. func (c *LiveClient) DeleteNATByExternalIP(ctx context.Context, routerName string, natType, externalIP string) error { lr, err := c.GetLogicalRouter(ctx, routerName) if err != nil { @@ -926,10 +911,8 @@ func (c *LiveClient) DeleteNATByExternalIP(ctx context.Context, routerName strin return nil } -// DeleteAllNATsByExternalIP removes NAT rules matching externalIP from every -// router that references them, handling cross-VPC stale rules left when -// vpc.delete-nat hasn't been processed before the IP is reused. Returns the -// number deleted. +// DeleteAllNATsByExternalIP removes NAT rules matching externalIP across all routers. +// Handles cross-VPC stale rules. Returns the count deleted. func (c *LiveClient) DeleteAllNATsByExternalIP(ctx context.Context, natType, externalIP string) (int, error) { var nats []nbdb.NAT if err := c.client.WhereCache(func(n *nbdb.NAT) bool { @@ -980,8 +963,7 @@ func (c *LiveClient) DeleteAllNATsByExternalIP(ctx context.Context, natType, ext return len(nats), nil } -// FindNATByExternalIP returns the first NAT rule matching (natType, -// externalIP) or nil if none exists. +// FindNATByExternalIP returns the first NAT rule matching (natType, externalIP), or nil. func (c *LiveClient) FindNATByExternalIP(ctx context.Context, natType, externalIP string) (*nbdb.NAT, error) { var nats []nbdb.NAT if err := c.client.WhereCache(func(n *nbdb.NAT) bool { @@ -1027,9 +1009,8 @@ func (c *LiveClient) AddStaticRoute(ctx context.Context, routerName string, rout return nil } -// FindStaticRoute returns the first owned static route on routerName whose -// IPPrefix matches, or (nil, nil). Reconcile callers must dedupe via this -// helper because AddStaticRoute leaves a duplicate row on every retry. +// FindStaticRoute returns the first owned static route matching ipPrefix on routerName, +// or (nil, nil). Prevents duplicate rows from AddStaticRoute retries. func (c *LiveClient) FindStaticRoute(ctx context.Context, routerName, ipPrefix string) (*nbdb.LogicalRouterStaticRoute, error) { lr, err := c.GetLogicalRouter(ctx, routerName) if err != nil { @@ -1351,7 +1332,7 @@ func (c *LiveClient) getPortGroup(ctx context.Context, name string) (*nbdb.PortG return &pgs[0], nil } -// GetPortGroup is the exported wrapper around getPortGroup. +// GetPortGroup returns the named port group. func (c *LiveClient) GetPortGroup(ctx context.Context, name string) (*nbdb.PortGroup, error) { return c.getPortGroup(ctx, name) } @@ -1364,8 +1345,7 @@ func (c *LiveClient) ListPortGroups(ctx context.Context) ([]nbdb.PortGroup, erro return pgs, nil } -// ListPortGroupsForPort returns the names of port groups whose Ports set -// contains the LSP's UUID. Empty slice (not error) when no memberships. +// ListPortGroupsForPort returns port group names containing the LSP. Empty (not error) when none. func (c *LiveClient) ListPortGroupsForPort(ctx context.Context, lspName string) ([]string, error) { lsp, err := c.GetLogicalSwitchPort(ctx, lspName) if err != nil { @@ -1398,9 +1378,7 @@ func (c *LiveClient) AddACLs(ctx context.Context, portGroupName string, specs [] var ops []ovsdb.Operation uuids := make([]string, 0, len(specs)) for i, spec := range specs { - // Index `i` disambiguates ACLs that collapse to the same - // (direction,match) — without it OVSDB rejects the second insert - // with "duplicate uuid-name". + // Index i disambiguates ACLs with the same (direction, match). acl := &nbdb.ACL{ UUID: namedUUID("acl_", fmt.Sprintf("%s_%d_%s", portGroupName, i, spec.Direction)), Direction: spec.Direction, @@ -1478,11 +1456,8 @@ func (c *LiveClient) ClearACLs(ctx context.Context, portGroupName string) error return nil } -// ReplaceACLs atomically swaps the port group's ACL set in a single OVSDB -// transaction: detach + delete every existing ACL row, create the new rows, -// attach them to the port group. Equivalent to ClearACLs followed by AddACLs -// without the mid-flight window where the port group has zero ACLs (which -// would otherwise default-drop tenant traffic on the egress test path). +// ReplaceACLs atomically swaps the port group's ACL set in one transaction. +// Avoids the zero-ACL window that ClearACLs+AddACLs would expose (default-drop). func (c *LiveClient) ReplaceACLs(ctx context.Context, portGroupName string, specs []ACLSpec) error { pg, err := c.getPortGroup(ctx, portGroupName) if err != nil { diff --git a/spinifex/network/policy/acl.go b/spinifex/network/policy/acl.go index 16d43e2c2..f9366fa8e 100644 --- a/spinifex/network/policy/acl.go +++ b/spinifex/network/policy/acl.go @@ -42,10 +42,8 @@ type Rule struct { SourceSG string } -// InfrastructureACLs returns the platform ACLs every PG carries: the -// 900/800 logged default-denies (CMMC SC.L1-3.13.1) and the 1050 DHCPv4 -// client/server allows (OVN's DHCP responder runs after the PG ACL stage; -// without these a narrow-egress SG would drop DHCPDISCOVER). +// InfrastructureACLs returns the platform ACLs every PG carries: logged +// 900/800 default-denies (CMMC SC.L1-3.13.1) and 1050 DHCPv4 allows. func InfrastructureACLs(portGroupName string) []ovn.ACLSpec { return []ovn.ACLSpec{ denyIngressACL(portGroupName), @@ -55,8 +53,7 @@ func InfrastructureACLs(portGroupName string) []ovn.ACLSpec { } } -// RuleACLSpecs builds priority-1000 allow ACLs. Action is "allow-related" -// so reply traffic on established flows passes without a reverse rule. +// RuleACLSpecs builds priority-1000 allow ACLs with "allow-related" action. func RuleACLSpecs(portGroupName string, ingress, egress []Rule) []ovn.ACLSpec { specs := make([]ovn.ACLSpec, 0, len(ingress)+len(egress)) for _, rule := range ingress { @@ -78,8 +75,7 @@ func RuleACLSpecs(portGroupName string, ingress, egress []Rule) []ovn.ACLSpec { return specs } -// BuildIngressACLMatch builds an OVN to-lport match. Tenant CIDR/SourceSG -// MUST be validated upstream; values are interpolated verbatim. +// BuildIngressACLMatch builds an OVN to-lport match expression. func BuildIngressACLMatch(portGroupName string, rule Rule) string { parts := []string{fmt.Sprintf("outport == @%s", portGroupName), "ip4"} parts = appendProtocolMatch(parts, rule) @@ -105,9 +101,8 @@ func BuildEgressACLMatch(portGroupName string, rule Rule) string { return strings.Join(parts, " && ") } -// addressSetName returns the ovn-northd-derived SB Address_Set name for a -// PG's IPv4 addresses. Spinifex must NOT create NB Address_Set rows under -// these names — duplicate SB sync wedges northd in a commit-failed loop. +// addressSetName returns the ovn-northd-derived SB Address_Set name for a PG's +// IPv4 addresses. Do NOT create NB Address_Set rows with this name — it wedges northd. func addressSetName(portGroupName string) string { return portGroupName + "_ip4" } diff --git a/spinifex/network/policy/nat.go b/spinifex/network/policy/nat.go index 799885363..75c4737c7 100644 --- a/spinifex/network/policy/nat.go +++ b/spinifex/network/policy/nat.go @@ -54,10 +54,8 @@ type NATManager interface { // DeleteSNAT removes the IGW default-outbound snat for vpcCIDR. DeleteSNAT(ctx context.Context, vpcID, vpcCIDR string) error - // AddSystemInstanceSNAT installs an egress-only snat rewriting a single - // system instance's /32 (logicalIP) to externalIP. Unlike AddEIP it is a - // plain snat, not dnat_and_snat, so there is no inbound path to the - // instance. Idempotent; tagged role=system-instance-egress. + // AddSystemInstanceSNAT installs an egress-only snat for a /32 logicalIP → + // externalIP. Plain snat (not dnat_and_snat), so no inbound path. Idempotent. AddSystemInstanceSNAT(ctx context.Context, vpcID, logicalIP, externalIP string) error // DeleteSystemInstanceSNAT removes the egress-only snat by logicalIP; @@ -65,27 +63,18 @@ type NATManager interface { DeleteSystemInstanceSNAT(ctx context.Context, vpcID, logicalIP string) error } -// FlowsBarrier blocks until ovn-northd has compiled NB → SB and every -// chassis installed flows. Production wires a closure over -// `ovn-nbctl --wait=hv sync`; tests leave it nil. +// FlowsBarrier blocks until every chassis has installed flows. Production +// wires `ovn-nbctl --wait=hv sync`; tests leave it nil. type FlowsBarrier func() error -// NeighFlusher invalidates the host neighbour (ARP) entry for externalIP so the -// next dial re-resolves L2. Called on both EIP attach and detach: it keeps a -// recycled external IP reachable, preventing the host ARP cache from pointing at -// the prior owner's MAC until the kernel ARP timeout expires (60-300s). -// -// Best-effort: implementations return errors but callers warn and proceed. +// NeighFlusher invalidates the host ARP entry for externalIP so a recycled +// external IP isn't shadowed by the prior owner's MAC (kernel ARP timeout 60-300s). +// Best-effort: callers warn and proceed on error. type NeighFlusher func(externalIP string) error -// NeighPrimer installs the host neighbour entry for a distributed EIP directly, -// mapping ExternalIP to its external_mac. Preferred over NeighFlusher on attach -// when the MAC is known (distributed mode): a flush merely re-arms ARP, but no -// node answers the host's re-ARP for a same-chassis recycled IP, so the entry -// would just decay back to FAILED. Programming the known MAC makes the recycled -// IP reachable without an ARP round-trip. -// -// Best-effort: implementations return errors but callers warn and proceed. +// NeighPrimer programs the host ARP entry for a distributed EIP directly. +// Preferred over NeighFlusher when the MAC is known: a flush triggers re-ARP that +// no node answers for a same-chassis recycled IP. Best-effort; callers warn and proceed. type NeighPrimer func(eip EIPSpec) error type Option func(*natManager) @@ -100,10 +89,8 @@ func WithFlowsBarrier(b FlowsBarrier) Option { } } -// WithNeighFlusher injects the host neighbour-flush hook fired on EIP detach (and -// on attach when the external_mac is unknown). Without it, recycled external IPs -// stay shadowed by the prior owner's MAC in the host ARP cache until the kernel -// ARP timeout expires. +// WithNeighFlusher injects the ARP-flush hook fired on EIP detach and on attach +// when external_mac is unknown. func WithNeighFlusher(f NeighFlusher) Option { return func(m *natManager) { if f != nil { @@ -112,10 +99,8 @@ func WithNeighFlusher(f NeighFlusher) Option { } } -// WithNeighPrimer injects the host neighbour-prime hook fired on EIP attach in -// distributed mode (external_mac known). Preferred over the flusher there: it -// programs the recycled IP's MAC directly instead of waiting on an ARP reply -// that no node sends for a same-chassis recycled IP. +// WithNeighPrimer injects the ARP-prime hook fired on EIP attach in distributed +// mode; preferred over the flusher when external_mac is known. func WithNeighPrimer(p NeighPrimer) Option { return func(m *natManager) { if p != nil { @@ -184,16 +169,13 @@ func (m *natManager) AddEIP(ctx context.Context, eip EIPSpec) error { existing.LogicalPort != nil && *existing.LogicalPort == eip.PortName)) { slog.Info("policy: AddEIP idempotent skip — rule current, re-priming reachability", "router", router, "external_ip", eip.ExternalIP, "logical_ip", eip.LogicalIP) - // Skip the delete-then-add row churn, but still re-announce on the wire: - // a stop->start or daemon reboot-recovery re-attaches the same EIP, and - // without a fresh prime the host neigh / OVS datapath stays dark until - // the kernel ARP entry times out (60-300s). + // Skip row churn but still re-prime: stop->start re-attaches the same EIP + // and the host neigh stays dark until ARP times out without a fresh prime. m.primeReachability(eip, distributed) return nil } - // Search every router for stale rules — vpc.delete-nat is - // fire-and-forget and may not have run before IP reuse. + // Search every router for stale rules — vpc.delete-nat is fire-and-forget. if removed, err := m.ovn.DeleteAllNATsByExternalIP(ctx, "dnat_and_snat", eip.ExternalIP); err != nil { slog.Warn("policy: stale NAT cleanup failed before AddEIP", "external_ip", eip.ExternalIP, "err", err) } else if removed > 0 { @@ -210,12 +192,8 @@ func (m *natManager) AddEIP(ctx context.Context, eip EIPSpec) error { return nil } -// primeReachability re-announces an EIP on the wire so the host neigh table and -// OVS datapath learn its external_mac. Distributed mode programs the neighbour -// entry directly instead of flushing and waiting on an ARP reply no node sends -// for a same-chassis recycled IP; centralised mode (no known MAC) falls back to -// a flush. Best-effort — every step is logged, none fatal — so it is safe to run -// on the idempotent-skip path as well as after a fresh AddNAT. +// primeReachability re-announces an EIP: distributed mode programs the ARP entry +// directly; centralised mode (no MAC) flushes and re-arms ARP. Best-effort. func (m *natManager) primeReachability(eip EIPSpec, distributed bool) { if distributed { if err := m.neighPrime(eip); err != nil { @@ -233,9 +211,7 @@ func (m *natManager) DeleteEIP(ctx context.Context, vpcID, externalIP, logicalIP return fmt.Errorf("delete dnat_and_snat %s on %s: %w", logicalIP, router, err) } } - // Flush the host ARP entry for the released IP so the next owner is not - // shadowed by this rule's stale MAC. Best-effort, and safe on the - // not-found path (the entry may still be cached from a prior binding). + // Flush host ARP for the released IP so the next owner isn't shadowed. Best-effort. if externalIP != "" { if err := m.neigh(externalIP); err != nil { slog.Warn("policy: DeleteEIP neighbour flush failed", "external_ip", externalIP, "logical_ip", logicalIP, "err", err) @@ -259,8 +235,7 @@ func (m *natManager) AddNATGateway(ctx context.Context, gw NATGWSpec) error { if err := m.ovn.AddNAT(ctx, router, snatRule); err != nil { return fmt.Errorf("add NAT GW snat %s -> %s on %s: %w", gw.SubnetCIDR, gw.PublicIP, router, err) } - // Block until SB + chassis have the SNAT flow; otherwise first packets - // from the private subnet drop. + // Block until SB + chassis have the SNAT flow. if err := m.barrier(); err != nil { slog.Warn("policy: AddNATGateway flows barrier failed", "public_ip", gw.PublicIP, "subnet_cidr", gw.SubnetCIDR, "err", err) @@ -319,8 +294,7 @@ func (m *natManager) AddSystemInstanceSNAT(ctx context.Context, vpcID, logicalIP "spinifex:role": "system-instance-egress", }, } - // Skip when the existing row already matches; avoids the duplicate-append - // on a re-published add (the snat is keyed by its unique pool external IP). + // Skip when the existing row already matches (idempotent re-publish guard). if existing, err := m.ovn.FindNATByExternalIP(ctx, "snat", externalIP); err != nil { slog.Warn("policy: AddSystemInstanceSNAT idempotency lookup failed", "external_ip", externalIP, "err", err) } else if existing != nil && existing.LogicalIP == logicalIP { @@ -332,8 +306,7 @@ func (m *natManager) AddSystemInstanceSNAT(ctx context.Context, vpcID, logicalIP if err := m.ovn.AddNAT(ctx, router, snatRule); err != nil { return fmt.Errorf("add system-instance snat %s -> %s on %s: %w", logicalIP, externalIP, router, err) } - // Block until SB + chassis have the SNAT flow; otherwise first egress - // packets from the instance drop. + // Block until SB + chassis have the SNAT flow. if err := m.barrier(); err != nil { slog.Warn("policy: AddSystemInstanceSNAT flows barrier failed", "logical_ip", logicalIP, "external_ip", externalIP, "err", err) diff --git a/spinifex/network/policy/nat_test.go b/spinifex/network/policy/nat_test.go index 3f5bdd191..7e4922c04 100644 --- a/spinifex/network/policy/nat_test.go +++ b/spinifex/network/policy/nat_test.go @@ -104,9 +104,8 @@ func TestNATManager_AddEIP_IdempotentSkip_RePrimesReachability(t *testing.T) { PortName: "port-eni-abc", MAC: "aa:bb:cc:dd:ee:ff", } require.NoError(t, nm.AddEIP(ctx, spec)) - // Re-attach the same EIP (stop->start / reboot-recovery): the DNAT row is - // unchanged so the row write is skipped, but reachability must be re-primed - // or the host neigh entry goes dark until the kernel ARP times out. + // Re-attach same EIP (stop->start): row write is skipped but reachability + // must be re-primed or the host neigh goes dark until ARP times out. require.NoError(t, nm.AddEIP(ctx, spec)) assert.Equal(t, 1, barrierCalls, "FlowsBarrier must not fire on idempotent skip") @@ -329,13 +328,9 @@ func TestNATManager_AddNATGateway_AndDelete(t *testing.T) { require.NoError(t, nm.DeleteNATGateway(ctx, "vpc-1", "10.0.1.0/24")) } -// TestNATManager_DeleteNATGateway_CrossRouterIsolation guards against a NAT -// row owned by router B being mutated/deleted when DeleteNATGateway is called -// on router A and both routers carry the same subnet CIDR. AWS allows subnet -// CIDRs to repeat per-VPC, so (snat, logicalIP) is not globally unique. Bug -// observed in CI Phase 8d: shared 172.31.x CIDRs across VPCs caused -// DeleteEIP/DeleteNAT to hit the wrong NAT row, leaving orphaned rules that -// corrupted ARP/conntrack for the surviving NATGW EIP. +// TestNATManager_DeleteNATGateway_CrossRouterIsolation guards against deleting +// a NAT row on router B when DeleteNATGateway targets router A. AWS allows +// overlapping subnet CIDRs across VPCs, so (snat, logicalIP) is not globally unique. func TestNATManager_DeleteNATGateway_CrossRouterIsolation(t *testing.T) { ctx := context.Background() m := mock.New() @@ -366,10 +361,8 @@ func TestNATManager_DeleteNATGateway_CrossRouterIsolation(t *testing.T) { assert.Equal(t, "nat-b", survived.ExternalIDs["spinifex:nat_gateway_id"]) } -// TestNATManager_DeleteEIP_CrossRouterIsolation is the dnat_and_snat analogue -// of the cross-router NATGW isolation test. Two VPCs each with a private VM -// at 10.0.0.5 (legal — VPC CIDRs may overlap). DeleteEIP on vpc-a must not -// touch vpc-b's NAT row. +// TestNATManager_DeleteEIP_CrossRouterIsolation is the dnat_and_snat analogue: +// two VPCs with overlapping private IPs — DeleteEIP on vpc-a must not touch vpc-b. func TestNATManager_DeleteEIP_CrossRouterIsolation(t *testing.T) { ctx := context.Background() m := mock.New() diff --git a/spinifex/network/policy/policy.go b/spinifex/network/policy/policy.go index 7a7434847..5fca12355 100644 --- a/spinifex/network/policy/policy.go +++ b/spinifex/network/policy/policy.go @@ -14,14 +14,12 @@ type NATMode int const ( NATModeUnknown NATMode = iota - // NATModeDistributed sets ExternalMAC + LogicalPort on every - // dnat_and_snat so OVN processes DNAT on the VM's own chassis. Required - // by UplinkModePhysical. + // NATModeDistributed sets ExternalMAC+LogicalPort so OVN processes DNAT on + // the VM's own chassis. Required by UplinkModePhysical. NATModeDistributed - // NATModeCentralized leaves ExternalMAC/LogicalPort unset so the gateway - // chassis owns SNAT/DNAT. Required by UplinkModeVeth (bridge breaks - // distributed-NAT hairpin routing). + // NATModeCentralized leaves ExternalMAC/LogicalPort unset; gateway chassis + // owns SNAT/DNAT. Required by UplinkModeVeth. NATModeCentralized ) diff --git a/spinifex/network/policy/route.go b/spinifex/network/policy/route.go index fcf5a0fac..04e58eb90 100644 --- a/spinifex/network/policy/route.go +++ b/spinifex/network/policy/route.go @@ -13,43 +13,29 @@ import ( const ( defaultRoutePrefix = "0.0.0.0/0" - // SubnetEgressPriorityIGW is the priority assigned to per-subnet default - // egress policies routing via an IGW. NATGW egress sits at a different - // priority so the route precedence is deterministic when both apply. - // SubnetEgressPriorityDrop sits above both reroute priorities; a subnet - // whose effective route table lacks 0.0.0.0/0 gets a drop policy at this - // priority to override the VPC LR's router-wide default static route - // (which is required so lr_in_ip_routing doesn't drop at priority 0 - // before lr_in_policy fires). + // SubnetEgressPriorityDrop sits above both reroute priorities; subnets + // without a 0.0.0.0/0 route get a drop policy to override the VPC LR's + // router-wide default static route. SubnetEgressPriorityDrop = 1100 SubnetEgressPriorityIGW = 1000 SubnetEgressPriorityNATGW = 900 - // SystemInstanceEgressPriority sits above the drop gate (1100) so a single - // system instance (e.g. an EKS K3s server VM) egresses even when its subnet - // is otherwise drop-gated. The policy is scoped to the instance's /32 - // source, so peer instances in the same subnet are unaffected — they still - // hit the 1100 drop (or the subnet's own 1000 reroute where one exists). + // SystemInstanceEgressPriority sits above the 1100 drop gate so a system + // instance egresses even on an otherwise drop-gated subnet. Scoped to /32. SystemInstanceEgressPriority = 1200 ) -// RouteSpec is a static route on a VPC's LogicalRouter. OutputPort is -// required when Nexthop isn't directly-connected (e.g. IGW default route on -// 169.254.0.1/30) — ovn-northd silently drops the route from SB otherwise. +// RouteSpec is a static route on a VPC's LogicalRouter. OutputPort is required +// for non-directly-connected nexthops — ovn-northd silently drops the SB route otherwise. type RouteSpec struct { Prefix netip.Prefix Nexthop string OutputPort string } -// SubnetEgressSpec describes a per-subnet egress override installed as an -// OVN Logical_Router_Policy. The match is built from SubnetID (inport == -// "rtr-") AND ip4.dst == Prefix; the action is "reroute" via -// Nexthop, egressing through OutputPort. ExcludeCIDRs are appended as -// `&& ip4.dst != ` clauses so in-VPC traffic skips the reroute and -// follows the directly-connected route instead — without this, NATGW egress -// policies on a `0.0.0.0/0` prefix would intercept return traffic from -// peer subnets (bastion -> private SSH) and forward it to the public IP. +// SubnetEgressSpec describes a per-subnet egress reroute policy. ExcludeCIDRs +// are appended as `ip4.dst != ` clauses so in-VPC return traffic skips +// the reroute — without them, NATGW 0.0.0.0/0 policies intercept peer-subnet traffic. type SubnetEgressSpec struct { SubnetID string Prefix netip.Prefix @@ -59,11 +45,9 @@ type SubnetEgressSpec struct { ExcludeCIDRs []netip.Prefix } -// SystemInstanceEgressSpec describes a per-instance egress override installed -// as an OVN Logical_Router_Policy at SystemInstanceEgressPriority. The match -// adds an `ip4.src == /32` clause to the subnet-egress shape so the -// reroute is confined to a single system instance; peer instances in the same -// subnet are untouched. Action is "reroute" via Nexthop out OutputPort. +// SystemInstanceEgressSpec describes a per-instance reroute at +// SystemInstanceEgressPriority. The /32 src match confines the reroute to one +// system instance; peers in the same subnet are untouched. type SystemInstanceEgressSpec struct { SubnetID string SrcIP netip.Addr @@ -73,9 +57,7 @@ type SystemInstanceEgressSpec struct { ExcludeCIDRs []netip.Prefix } -// RouteManager owns static routes on VPC LogicalRouters. Adds are -// idempotent (no-op on match, delete-then-add on drift) so the reconciler -// can replay safely. +// RouteManager owns static routes on VPC LogicalRouters. Adds are idempotent. type RouteManager interface { AddDefaultRoute(ctx context.Context, vpcID, nexthop, outputPort string) error DeleteDefaultRoute(ctx context.Context, vpcID string) error @@ -89,14 +71,9 @@ type RouteManager interface { DeleteSystemInstanceEgress(ctx context.Context, vpcID, subnetID string, srcIP netip.Addr, prefix netip.Prefix, excludeCIDRs []netip.Prefix) error } -// SubnetEgressDropSpec describes a per-subnet egress DROP override installed -// as an OVN Logical_Router_Policy at SubnetEgressPriorityDrop. Match shape -// mirrors SubnetEgressSpec (inport == rtr- && ip4.dst == prefix with -// optional ip4.dst != clauses) but the action is "drop" with no -// nexthop/output port. Used to gate subnets whose effective route table -// lacks a 0.0.0.0/0 entry; the policy fires after lr_in_ip_routing has -// already matched the VPC LR's router-wide default static route, killing -// the packet before egress. +// SubnetEgressDropSpec describes a per-subnet DROP policy at SubnetEgressPriorityDrop. +// Used to gate subnets whose route table lacks a 0.0.0.0/0 entry, overriding the +// VPC LR's router-wide default static route before egress. type SubnetEgressDropSpec struct { SubnetID string Prefix netip.Prefix @@ -165,8 +142,7 @@ func (m *routeManager) DeleteStaticRoute(ctx context.Context, vpcID string, pref return m.deleteByPrefixString(ctx, vpcID, prefix.String()) } -// deleteByPrefixString returns nil when the route is already absent (every -// Delete* entrypoint needs this for fire-and-forget event flows). +// deleteByPrefixString returns nil when the route is already absent (idempotent). func (m *routeManager) deleteByPrefixString(ctx context.Context, vpcID, prefix string) error { router := topology.VPCRouter(vpcID) existing, err := m.ovn.FindStaticRoute(ctx, router, prefix) diff --git a/spinifex/network/policy/sg.go b/spinifex/network/policy/sg.go index 9f3b0fb05..f8d460af7 100644 --- a/spinifex/network/policy/sg.go +++ b/spinifex/network/policy/sg.go @@ -24,8 +24,7 @@ type SecurityGroupManager interface { // EnsureSG sets the full ACL set; idempotent. EnsureSG(ctx context.Context, sg SGSpec) error - // UpdateSG replaces the SG's ACL set. Identical to EnsureSG; kept - // distinct to mirror §8.1 of the parent plan. + // UpdateSG replaces the SG's ACL set. Semantically identical to EnsureSG. UpdateSG(ctx context.Context, sg SGSpec) error // DeleteSG clears every ACL on the SG's port group; the PG itself @@ -60,11 +59,8 @@ func (m *sgManager) DeleteSG(ctx context.Context, groupID string) error { return nil } -// applyACLs atomically replaces the PG's ACL set with infra + tenant ACLs in -// a single OVSDB transaction. The previous ClearACLs+AddACLs split left the -// port group with zero ACLs between transactions, defaulting tenant LSP -// traffic to drop on connectionless flows (ICMP) and on TCP SYNs that -// don't match an existing conntrack entry. +// applyACLs atomically replaces infra + tenant ACLs in one OVSDB transaction. +// ClearACLs+AddACLs split left zero-ACL gaps that defaulted traffic to drop. func (m *sgManager) applyACLs(ctx context.Context, sg SGSpec) error { pg := topology.SecurityGroupPortGroup(sg.GroupID) specs := append(InfrastructureACLs(pg), RuleACLSpecs(pg, sg.IngressRules, sg.EgressRules)...) diff --git a/spinifex/network/policy/sg_test.go b/spinifex/network/policy/sg_test.go index 58fbc0913..c7e966e9a 100644 --- a/spinifex/network/policy/sg_test.go +++ b/spinifex/network/policy/sg_test.go @@ -95,12 +95,8 @@ func TestSGManager_EnsureSG_MissingPortGroup_ReturnsError(t *testing.T) { assert.Contains(t, err.Error(), "port group") } -// TestSGManager_UpdateSG_AtomicReplace asserts EnsureSG / UpdateSG route -// through ovn.Client.ReplaceACLs (single OVSDB transaction) and never call -// ClearACLs. ClearACLs+AddACLs leaves the port group with zero ACLs between -// transactions, defaulting tenant LSP traffic to drop on connectionless -// flows (ICMP) and on TCP SYNs that miss conntrack — the symptom behind -// TestSecurityGroupEgress flakes. +// TestSGManager_UpdateSG_AtomicReplace asserts EnsureSG/UpdateSG use +// ReplaceACLs (single OVSDB transaction) and never call ClearACLs. func TestSGManager_UpdateSG_AtomicReplace(t *testing.T) { ctx := context.Background() m, pg := newSGMockWithPG(t, "sg-web") diff --git a/spinifex/network/reconcile/apply.go b/spinifex/network/reconcile/apply.go index 6432b3409..1f3df8e82 100644 --- a/spinifex/network/reconcile/apply.go +++ b/spinifex/network/reconcile/apply.go @@ -11,9 +11,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/utils" ) -// Bounds for the post-rebind Southbound chassis-claim wait. A host reboot's SB -// claim normally converges within a few seconds once ovn-controller reconnects; -// the recompute nudge covers the stuck case. Package vars so tests can shorten. +// Bounds for the post-rebind SB chassis-claim wait. Package vars so tests can shorten. var ( gatewayClaimTimeout = 30 * time.Second gatewayClaimInterval = 2 * time.Second @@ -42,8 +40,8 @@ func (r *reconciler) applyVPCs(ctx context.Context, intent IntentState, actual A } } -// applySubnets ensures every intent Subnet has a LogicalSwitch, parent-LRP, -// router-side LSP, and DHCPOptions row. Each step is independently idempotent. +// applySubnets ensures every intent subnet has a LogicalSwitch, LRP, router LSP, +// and DHCPOptions row. Each step is idempotent. func (r *reconciler) applySubnets(ctx context.Context, intent IntentState, actual ActualState) { for subnetID, spec := range intent.Subnets { switchName := topology.SubnetSwitch(subnetID) @@ -110,8 +108,7 @@ func (r *reconciler) applySubnets(ctx context.Context, intent IntentState, actua } } - // The IMDS localport rides on the subnet switch (idempotent via the - // subnet-veth bucket gate); the switch must exist first. + // IMDS localport rides on the subnet switch; idempotent. if r.imds != nil { if _, err := r.imds.EnsureForSubnet(ctx, subnetID, spec.VPCID, spec.CIDR); err != nil { slog.Error("reconcile/apply: IMDS EnsureForSubnet failed", "subnet_id", subnetID, "err", err) @@ -137,11 +134,9 @@ func (r *reconciler) applySubnets(ctx context.Context, intent IntentState, actua } } -// applySGs ensures every intent SG has a port group, then (when -// pruneOrphans) deletes sg_* PGs without a matching intent SG. -// pruneOrphans=false is startup mode: a leader can load intent before peer -// subscribers have finished creating the PGs, so pruning at boot would -// delete in-flight resources. Drift always prunes. +// applySGs ensures every intent SG has a port group; when pruneOrphans is true, +// deletes sg_* PGs with no matching intent SG. Startup passes false to avoid +// deleting in-flight resources before peer subscribers have converged. func (r *reconciler) applySGs(ctx context.Context, intent IntentState, actual ActualState, pruneOrphans bool) { for groupID, spec := range intent.SGs { if err := r.topology.EnsureSGPortGroup(ctx, groupID); err != nil { @@ -178,9 +173,8 @@ func (r *reconciler) applySGs(ctx context.Context, intent IntentState, actual Ac } } -// applyPorts ensures each intent ENI has an LSP with PG memberships matching -// its SGIDs. Existing ports get a diff-based UpdatePortGroupMemberships so an -// SG swap never exposes an "unrestricted" gap (OVN default). +// applyPorts ensures each intent ENI has an LSP with PG memberships matching its +// SGIDs. Existing ports use diff-based UpdatePortGroupMemberships to avoid gaps. func (r *reconciler) applyPorts(ctx context.Context, intent IntentState, actual ActualState) { for portID, spec := range intent.Ports { portName := topology.Port(portID) @@ -232,9 +226,8 @@ func (r *reconciler) applyPorts(ctx context.Context, intent IntentState, actual } } -// applyIGWs ensures every intent IGW's OVN topology (external switch, -// localnet LSP, gateway LRP, default route, Gateway_Chassis bindings) and -// rebinds chassis on existing IGWs. AttachIGW is idempotent. +// applyIGWs ensures every intent IGW has OVN topology and rebinds chassis on +// existing IGWs. AttachIGW is idempotent. func (r *reconciler) applyIGWs(ctx context.Context, intent IntentState, actual ActualState) { for vpcID, spec := range intent.IGWs { if _, ok := actual.ExternalSwch[vpcID]; !ok { @@ -248,8 +241,7 @@ func (r *reconciler) applyIGWs(ctx context.Context, intent IntentState, actual A } } -// rebindGatewayChassis re-asserts (chassis,priority) tuples on an existing -// gateway LRP. SetGatewayChassis is idempotent. +// rebindGatewayChassis re-asserts chassis priority tuples on the gateway LRP. func (r *reconciler) rebindGatewayChassis(ctx context.Context, vpcID string) { if len(r.chassis) == 0 { return @@ -267,17 +259,9 @@ func (r *reconciler) rebindGatewayChassis(ctx context.Context, vpcID string) { r.ensureGatewayClaimed(ctx, topology.GatewayChassisRedirectPort(vpcID)) } -// ensureGatewayClaimed drives the gateway port's Southbound Port_Binding to a -// claimed chassis after SetGatewayChassis. crPortName is the chassisredirect -// (cr-) port, not the bare LRP: for a distributed gateway port the LRP binding -// stays distributed/chassis-less, and only the chassisredirect port carries the -// gateway-chassis claim. SetGatewayChassis only asserts the Northbound intent; -// after a host reboot ovn-controller can leave the SB binding unclaimed, -// installing no flows for the gateway redirect so every floating IP on the VPC -// is unreachable. This polls the SB claim and, once, nudges ovn-controller to -// recompute, then logs and returns on timeout rather than blocking reconcile -// indefinitely. No-op when no verifier is wired (the steady-state path is a -// single claimed check per gateway port per cycle). +// ensureGatewayClaimed polls the SB chassisredirect binding after SetGatewayChassis. +// An unclaimed binding after reboot makes floating IPs unreachable; nudges recompute +// once, then gives up. No-op when no verifier is wired. func (r *reconciler) ensureGatewayClaimed(ctx context.Context, crPortName string) { if r.gwClaim == nil { return @@ -316,8 +300,7 @@ func (r *reconciler) ensureGatewayClaimed(ctx context.Context, crPortName string } } -// applyEIPs runs every intent EIP through NATManager.AddEIP (idempotent; -// stale dnat_and_snat rules are cleaned first). +// applyEIPs runs every intent EIP through NATManager.AddEIP; idempotent. func (r *reconciler) applyEIPs(ctx context.Context, intent IntentState, _ ActualState) { for _, spec := range intent.EIPs { if err := r.nat.AddEIP(ctx, spec); err != nil { @@ -327,7 +310,6 @@ func (r *reconciler) applyEIPs(ctx context.Context, intent IntentState, _ Actual } // applyNATGWs runs every intent NAT gateway through NATManager.AddNATGateway. -// Duplicate (snat,SubnetCIDR) tuples are rejected by the underlying client. func (r *reconciler) applyNATGWs(ctx context.Context, intent IntentState, _ ActualState) { for _, spec := range intent.NATGWs { if err := r.nat.AddNATGateway(ctx, spec); err != nil { @@ -337,10 +319,8 @@ func (r *reconciler) applyNATGWs(ctx context.Context, intent IntentState, _ Actu } } -// applyIGWRoutes installs per-subnet egress reroute policies for every IGW -// route in intent. Closes the bootstrap race where the publisher fires -// vpc.add-igw-route before subscribers attach: events vanish, KV retains the -// route, this pass re-derives and installs. +// applyIGWRoutes installs per-subnet egress reroute policies from intent. +// Closes the bootstrap race: events fire before subscribers attach; KV retains the route. func (r *reconciler) applyIGWRoutes(ctx context.Context, intent IntentState, _ ActualState) { for _, spec := range intent.IGWRoutes { if err := r.igw.EnsureSubnetEgress(ctx, spec.VPCID, spec.SubnetID, spec.DestCIDR); err != nil { @@ -360,10 +340,8 @@ func (r *reconciler) applyNATGWRoutes(ctx context.Context, intent IntentState, _ } } -// applyDropGates installs a DROP policy at SubnetEgressPriorityDrop for every -// subnet whose VPC has an attached IGW but whose effective route table lacks -// a 0.0.0.0/0 IGW/NATGW entry. Without this, the VPC LR's router-wide default -// static route would let the subnet egress. +// applyDropGates installs DROP policies for subnets with an attached IGW but +// no 0.0.0.0/0 route, preventing unintended egress via the VPC default route. func (r *reconciler) applyDropGates(ctx context.Context, intent IntentState, _ ActualState) { for _, spec := range intent.DropGates { if err := r.igw.EnsureSubnetEgressDrop(ctx, spec.VPCID, spec.SubnetID, spec.DestCIDR); err != nil { diff --git a/spinifex/network/reconcile/apply_test.go b/spinifex/network/reconcile/apply_test.go index c6a2c6bbe..c00b7b045 100644 --- a/spinifex/network/reconcile/apply_test.go +++ b/spinifex/network/reconcile/apply_test.go @@ -159,8 +159,8 @@ func TestReconcile_OrphanPortGroupRemoved(t *testing.T) { } } -// ReconcileApplyOnly must not prune managed sg_* PGs when intent is empty -// (startup race vs. peer subscribers); full Reconcile must still prune. +// ReconcileApplyOnly must not prune sg_* PGs on empty intent (startup race); +// full Reconcile must still prune. func TestReconcile_ApplyOnlyKeepsOrphanPortGroup(t *testing.T) { rec, m := newTestReconciler(t) ctx := context.Background() @@ -256,10 +256,9 @@ func TestReconcile_ChassisRebindOnExistingIGW(t *testing.T) { } } -// TestReconcile_GatewayClaimChecksChassisRedirectPort pins the claim verifier to -// the chassisredirect (cr-) Port_Binding. The distributed gateway LRP binding is -// always chassis-less; checking it instead made ensureGatewayClaimed report -// "unclaimed" forever and fire a recompute every cycle, churning the EIP datapath. +// TestReconcile_GatewayClaimChecksChassisRedirectPort pins the verifier to the +// cr- Port_Binding. The LRP binding is chassis-less; checking it caused infinite +// recomputes and churned the EIP datapath. func TestReconcile_GatewayClaimChecksChassisRedirectPort(t *testing.T) { withFastClaimBounds(t) m := mock.New() diff --git a/spinifex/network/reconcile/intent.go b/spinifex/network/reconcile/intent.go index d97ef29fe..45c0c1298 100644 --- a/spinifex/network/reconcile/intent.go +++ b/spinifex/network/reconcile/intent.go @@ -37,9 +37,8 @@ type IntentState struct { DropGates map[string]SubnetEgressIntent // per (subnet, dest=0.0.0.0/0) drop policy } -// SubnetEgressIntent is a per-subnet default-route policy installed on the -// VPC LR. Mirrors the publishIGWRouteEvent fan-out: one entry per (subnet, -// destCIDR) that the source route table directs at an IGW or NAT gateway. +// SubnetEgressIntent is a per-subnet default-route policy entry for the VPC LR, +// keyed by (subnet, destCIDR) pointing at an IGW or NAT gateway. type SubnetEgressIntent struct { VPCID string SubnetID string @@ -51,9 +50,8 @@ func subnetEgressKey(subnetID string, prefix netip.Prefix) string { return subnetID + "|" + prefix.String() } -// LoadIntentFromKV assembles IntentState scoped to localAZ. Missing buckets -// (first boot) are treated as empty. AZ filter: `vpc.AZ == "" || vpc.AZ == -// localAZ`. Children inherit the filter transitively via their parent VPC. +// LoadIntentFromKV assembles IntentState for localAZ. Missing buckets are empty. +// AZ filter: `vpc.AZ == "" || vpc.AZ == localAZ`; children inherit it transitively. func LoadIntentFromKV(ctx context.Context, js nats.JetStreamContext, localAZ string) (IntentState, error) { intent := IntentState{ VPCs: make(map[string]topology.VPCSpec), @@ -101,11 +99,8 @@ func LoadIntentFromKV(ctx context.Context, js nats.JetStreamContext, localAZ str return intent, nil } -// loadSubnetEgressRoutes fans IGW/NATGW routes out over the subnets that -// inherit them — explicit non-main associations plus, for the main RT, -// subnets in the same VPC with no explicit non-main association. Mirrors -// publishIGWRouteEvents/publishNatGatewayEvents so the reconcile pass -// reaches the same per-subnet set as the runtime event path. +// loadSubnetEgressRoutes fans IGW/NATGW routes over associated subnets (explicit +// non-main + implicit main-RT subnets). Mirrors the runtime event-publisher fan-out. func loadSubnetEgressRoutes( localVPCs map[string]struct{}, subnets map[string]topology.SubnetSpec, @@ -184,11 +179,8 @@ func loadSubnetEgressRoutes( } } -// loadSubnetDropGates emits one drop intent per subnet whose VPC has an -// attached IGW but which has no 0.0.0.0/0 reroute path (neither IGWRoutes -// nor NATGWRoutes carry an entry for it). VPCs without an attached IGW -// have no router-wide default static route, so lr_in_ip_routing already -// kills the packet — no drop policy needed. +// loadSubnetDropGates emits a drop intent for each subnet whose VPC has an IGW +// but lacks a 0.0.0.0/0 reroute. VPCs without an IGW need no drop policy. func loadSubnetDropGates( localVPCs map[string]struct{}, subnets map[string]topology.SubnetSpec, @@ -491,18 +483,14 @@ func loadEIPs(js nats.JetStreamContext, localVPCs map[string]struct{}, out map[s if rec.ENIId != "" { spec.PortName = topology.Port(rec.ENIId) } - // MAC drives the distributed dnat_and_snat shape in policy.AddEIP; an - // empty MAC re-applies the rule centralised (external_mac/logical_port - // unset), which is the host-reboot connectivity regression this avoids. + // MAC drives distributed dnat_and_snat; empty falls back to centralised. spec.MAC = rec.MacAddress out[rec.PrivateIp] = spec } return nil } -// loadRouteTables snapshots every route table whose VPC is local. Used by -// loadNATGWs to fan out NATGW SNAT specs over associated subnets — mirrors the -// event-driven publisher in handlers/ec2/routetable. +// loadRouteTables snapshots every local-VPC route table. func loadRouteTables(js nats.JetStreamContext, localVPCs map[string]struct{}) ([]handlers_ec2_routetable.RouteTableRecord, error) { kv, err := js.KeyValue(handlers_ec2_routetable.KVBucketRouteTables) if err != nil { @@ -539,17 +527,13 @@ func loadRouteTables(js nats.JetStreamContext, localVPCs map[string]struct{}) ([ return out, nil } -// natgwSpecKey is the intent-map key for NATGWSpec. The reconciler needs one -// spec per (natgwID, subnetCIDR) because a NATGW may serve multiple subnets -// (route-table associations) and OVN stores one snat row per subnet CIDR. +// natgwSpecKey is the intent-map key for NATGWSpec: one entry per (natgwID, subnetCIDR). func natgwSpecKey(natgwID, subnetCIDR string) string { return natgwID + "|" + subnetCIDR } -// loadNATGWs emits one NATGWSpec per (NATGW, associated subnet) pair. Mirrors -// publishNatGatewayEventsForAssociation in handlers/ec2/routetable: a NATGW's -// SNAT rule rewrites traffic from the *private* subnets routed through it, -// not from the NATGW's own public subnet. +// loadNATGWs emits one NATGWSpec per (NATGW, associated private subnet) pair. +// SNAT rewrites traffic from the private subnets routed through the NATGW, not its home subnet. func loadNATGWs( js nats.JetStreamContext, localVPCs map[string]struct{}, diff --git a/spinifex/network/reconcile/intent_test.go b/spinifex/network/reconcile/intent_test.go index e208c9290..be3b72b62 100644 --- a/spinifex/network/reconcile/intent_test.go +++ b/spinifex/network/reconcile/intent_test.go @@ -162,12 +162,9 @@ func TestLoadIntentFromKV_IGWAttachedFilter(t *testing.T) { } } -// TestLoadIntentFromKV_NATGWUsesAssociatedSubnet enforces that loadNATGWs -// emits one NATGWSpec per *associated* private subnet (via route-table -// associations) — NOT the NATGW's own public home subnet. Bug observed in CI -// Phase 8d: reconciler installed snat for the NATGW's public-subnet CIDR -// (172.31.0.0/20) instead of the routed private-subnet CIDR (172.31.16.0/20), -// corrupting conntrack reverse-NAT and producing 100% packet loss. +// TestLoadIntentFromKV_NATGWUsesAssociatedSubnet enforces that loadNATGWs emits +// specs for associated *private* subnets, not the NATGW's own public home subnet. +// Wrong CIDR corrupts conntrack reverse-NAT and causes 100% packet loss. func TestLoadIntentFromKV_NATGWUsesAssociatedSubnet(t *testing.T) { _, _, js := testutil.StartTestJetStream(t) @@ -217,10 +214,8 @@ func TestLoadIntentFromKV_NATGWUsesAssociatedSubnet(t *testing.T) { } } -// TestLoadIntentFromKV_NATGWNoAssociationSkips guards against installing a -// SNAT for a NATGW that no route table points to. Without an associated -// private subnet there is no traffic to translate, and emitting the home -// subnet's CIDR would corrupt the SNAT table. +// TestLoadIntentFromKV_NATGWNoAssociationSkips guards against SNAT install for +// a NATGW with no route-table association; emitting the home-subnet CIDR would corrupt the SNAT table. func TestLoadIntentFromKV_NATGWNoAssociationSkips(t *testing.T) { _, _, js := testutil.StartTestJetStream(t) @@ -260,12 +255,9 @@ func TestLoadIntentFromKV_NoBucketsTolerated(t *testing.T) { } } -// TestLoadIntentFromKV_IGWRoutesFanOutMainRT covers the bootstrap race: -// publisher fires vpc.add-igw-route before subscribers attach, the event -// is dropped, and KV is the only source of truth left. The reconcile pass -// must rediscover the per-subnet egress intent from the main RT and fan it -// out to every subnet (including implicit-main subnets with no explicit -// non-main association). +// TestLoadIntentFromKV_IGWRoutesFanOutMainRT covers the bootstrap race: events +// dropped before subscribers attach; reconcile must re-derive per-subnet egress +// intent from the main RT, including implicit-main subnets. func TestLoadIntentFromKV_IGWRoutesFanOutMainRT(t *testing.T) { _, _, js := testutil.StartTestJetStream(t) @@ -316,10 +308,8 @@ func TestLoadIntentFromKV_IGWRoutesFanOutMainRT(t *testing.T) { } } -// TestLoadIntentFromKV_DropGatesForUnroutedSubnetWithIGW: a subnet whose -// effective RT has no 0.0.0.0/0 entry, in a VPC with an attached IGW, must -// surface a DropGates intent so the reconciler installs the per-subnet -// drop policy that gates the VPC LR's router-wide default static route. +// TestLoadIntentFromKV_DropGatesForUnroutedSubnetWithIGW: a subnet with no +// 0.0.0.0/0 route in an IGW-attached VPC must produce a DropGates intent. func TestLoadIntentFromKV_DropGatesForUnroutedSubnetWithIGW(t *testing.T) { _, _, js := testutil.StartTestJetStream(t) @@ -374,9 +364,8 @@ func TestLoadIntentFromKV_DropGatesForUnroutedSubnetWithIGW(t *testing.T) { } } -// TestLoadIntentFromKV_NoDropGateWithoutIGW: VPC without an attached IGW -// has no router-wide default static route, so unrouted subnets are already -// dropped by lr_in_ip_routing priority-0 — no drop policy needed. +// TestLoadIntentFromKV_NoDropGateWithoutIGW: VPC with no IGW has no router-wide +// default static route; lr_in_ip_routing already drops, so no drop policy needed. func TestLoadIntentFromKV_NoDropGateWithoutIGW(t *testing.T) { _, _, js := testutil.StartTestJetStream(t) diff --git a/spinifex/network/reconcile/invariants_test.go b/spinifex/network/reconcile/invariants_test.go index 8afa80b77..5af062ce2 100644 --- a/spinifex/network/reconcile/invariants_test.go +++ b/spinifex/network/reconcile/invariants_test.go @@ -10,20 +10,13 @@ import ( "github.com/mulgadc/spinifex/spinifex/network/topology" ) -// IMDS-datapath invariants. A guest LSP is created by two -// independent paths — the live topology manager (EnsurePort) and the reconciler -// (applyPorts) — and the per-subnet DHCPOptions row likewise. Both paths are -// exercised here so neither can drift away from the contract the IMDS handler -// trusts. These live in the reconcile package because it is the only network -// package that can import both topology (the live path) and itself (the -// reconciler path) without a cycle. +// IMDS-datapath invariants. Guest LSPs and DHCPOptions rows are created by two +// independent paths (live topology manager and reconciler); both are exercised +// here so neither can drift from the contract the IMDS handler trusts. -// TestI1_GuestLSPMustHavePortSecurity asserts that every guest-attached LSP, -// created by either path, carries port_security equal to its addresses -// (" "). This is the load-bearing security boundary: ovn-controller -// drops any frame whose eth.src/ip4.src doesn't match port_security at the -// ingress LSP, so a compromised guest cannot forge a peer's source IP. Without -// it the IMDS (VPC-ID, source-IP) → ENI mapping is forgeable. +// TestI1_GuestLSPMustHavePortSecurity asserts every guest LSP carries +// port_security == " ". Without it a compromised guest can forge a +// peer's source IP, breaking the IMDS (VPC-ID, source-IP) → ENI mapping. func TestI1_GuestLSPMustHavePortSecurity(t *testing.T) { ctx := context.Background() @@ -63,8 +56,7 @@ func TestI1_GuestLSPMustHavePortSecurity(t *testing.T) { }) } -// assertPortSecurity fails with the ADR clause when the LSP lacks port_security -// equal to its addresses. +// assertPortSecurity fails when the LSP's port_security doesn't match its addresses. func assertPortSecurity(t *testing.T, m *mock.Client, portName, want string) { t.Helper() lsp, err := m.GetLogicalSwitchPort(context.Background(), portName) @@ -76,8 +68,7 @@ func assertPortSecurity(t *testing.T, m *mock.Client, portName, want string) { "port_security allow source-IP spoofing, breaking the IMDS "+ "(VPC-ID, source-IP) → ENI mapping", portName, lsp.PortSecurity, want) } - // port_security must mirror addresses exactly, or the enforced identity and - // the advertised identity diverge. + // port_security must mirror addresses exactly. if len(lsp.Addresses) != 1 || lsp.Addresses[0] != lsp.PortSecurity[0] { t.Errorf("guest LSP %s Addresses %v != PortSecurity %v: enforced and advertised "+ "identity must match", portName, lsp.Addresses, lsp.PortSecurity) diff --git a/spinifex/network/reconcile/liveness_test.go b/spinifex/network/reconcile/liveness_test.go index c6abc1585..088df9f39 100644 --- a/spinifex/network/reconcile/liveness_test.go +++ b/spinifex/network/reconcile/liveness_test.go @@ -13,14 +13,9 @@ import ( "github.com/mulgadc/spinifex/spinifex/network/topology" ) -// TestL1_ReconcilerConverges enforces ADR-0006 liveness clause L1. -// -// "While both the NATS KV and OVN NB DB are reachable, the reconciler -// eventually drives the delta to zero. Idempotent EnsureX methods make -// repeated runs safe." -// -// Two reconcile passes over IntentStates of N={1,3,8} VPCs: state on pass 2 -// must equal state on pass 1. +// TestL1_ReconcilerConverges enforces ADR-0006 L1: while KV and OVN NB are +// reachable, repeated Reconcile calls converge. Two passes over N={1,3,8} VPCs; +// state on pass 2 must equal state on pass 1. func TestL1_ReconcilerConverges(t *testing.T) { for _, n := range []int{1, 3, 8} { t.Run(fmt.Sprintf("N=%d", n), func(t *testing.T) { @@ -55,14 +50,8 @@ func TestL1_ReconcilerConverges(t *testing.T) { } } -// TestL2_DriftIntervalBounded enforces ADR-0006 liveness clause L2. -// -// "While the leader is active, the periodic drift-detection loop -// eventually re-runs and re-applies any divergence introduced since -// the last reconciler run." -// -// Structural-only: bounds DriftInterval to (0, 30m]. Full behavioural -// coverage blocked on a clock-injectable DriftLoop signature. +// TestL2_DriftIntervalBounded enforces ADR-0006 L2 structurally: bounds +// DriftInterval to (0, 30m]. Full coverage blocked on a clock-injectable DriftLoop. func TestL2_DriftIntervalBounded(t *testing.T) { if DriftInterval <= 0 { t.Fatalf("ADR-0006 L2: DriftInterval must be > 0; got %v", DriftInterval) @@ -75,19 +64,15 @@ func TestL2_DriftIntervalBounded(t *testing.T) { // Blocked on a clock-injectable DriftLoop signature. } -// TestL3_FederationReEnablesOnLinkRecovery is a placeholder for ADR-0006 L3. -// -// "Once an inter-AZ link reaches ≥ Degraded class (ADR-0003), L4 -// eventually calls BringUpLink and establishes the OVN-IC tunnel to -// the peer AZ." +// TestL3_FederationReEnablesOnLinkRecovery is a placeholder for ADR-0006 L3 +// (OVN-IC tunnel re-establishment on inter-AZ link recovery). func TestL3_FederationReEnablesOnLinkRecovery(t *testing.T) { t.Skip(`ADR-0006 L3 deferred: network/federation/ has not yet been built. ` + `Once it lands, replace this stub with a test that drives a fake ` + `LinkObserver from Down → Degraded and asserts BringUpLink fires.`) } -// scaledIntent produces an IntentState with n VPCs, each holding one -// subnet, one ENI port, and one SG. +// scaledIntent produces an IntentState with n VPCs, each with one subnet, port, and SG. func scaledIntent(t *testing.T, n int) IntentState { t.Helper() intent := IntentState{ diff --git a/spinifex/network/reconcile/lock.go b/spinifex/network/reconcile/lock.go index 37b4b5c19..1e06c02d5 100644 --- a/spinifex/network/reconcile/lock.go +++ b/spinifex/network/reconcile/lock.go @@ -33,10 +33,8 @@ func AcquireLeader(nc *nats.Conn, holder string) (func(), bool) { ) deadline := time.Now().Add(leaderRetryFor) for { - // Get-or-create: CreateKeyValue alone returns "stream name already in - // use" once the bucket exists, so every reconcile after the first would - // hit the deadline and never elect. Attach to the existing bucket first, - // create only when it is genuinely absent. + // Get-or-create: CreateKeyValue returns "stream name already in use" if + // the bucket exists; attach first, create only when genuinely absent. kv, err = js.KeyValue(reconcileLeaderBucket) if errors.Is(err, nats.ErrBucketNotFound) { kv, err = js.CreateKeyValue(&nats.KeyValueConfig{ diff --git a/spinifex/network/reconcile/lock_test.go b/spinifex/network/reconcile/lock_test.go index 2c2a48d44..52715b63e 100644 --- a/spinifex/network/reconcile/lock_test.go +++ b/spinifex/network/reconcile/lock_test.go @@ -8,11 +8,9 @@ import ( "github.com/stretchr/testify/require" ) -// TestAcquireLeader_GetOrCreateAcrossCalls pins the get-or-create fix: the -// first acquire creates the bucket and wins, a second acquire while the lock is -// held must attach to the now-existing bucket (not hang on "stream name already -// in use") and lose cleanly, and once the winner releases a later acquire wins -// again. +// TestAcquireLeader_GetOrCreateAcrossCalls pins the get-or-create fix: first +// acquire creates the bucket and wins; second must attach (not hang on "stream +// name already in use") and lose; after release a new acquire wins. func TestAcquireLeader_GetOrCreateAcrossCalls(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) diff --git a/spinifex/network/reconcile/reconcile.go b/spinifex/network/reconcile/reconcile.go index 1807e7417..c191d9b42 100644 --- a/spinifex/network/reconcile/reconcile.go +++ b/spinifex/network/reconcile/reconcile.go @@ -1,8 +1,6 @@ -// Package reconcile is the network stack's intent-actual reconciliation -// layer. It loads desired state from the JetStream KV snapshot (scoped to -// the local AZ) and applies the diff against OVN NB in a single -// topologically-sorted pass (VPC → Subnet → SG → Port → IGW → EIP → -// NATGW). Deletes run reverse order. Drift fires every 5 minutes. +// Package reconcile is the network stack's intent-actual reconciliation layer. +// It loads desired state from JetStream KV and applies the diff against OVN NB +// in topological order (VPC→Subnet→SG→Port→IGW→EIP→NATGW). Drift every 5m. package reconcile import ( @@ -27,20 +25,12 @@ type Reconciler interface { ReconcileApplyOnly(ctx context.Context, intent IntentState) error } -// GatewayClaimVerifier confirms that ovn-controller has claimed the Southbound -// chassisredirect Port_Binding for a gateway router port, and nudges a recompute -// when it has not. After a host reboot the Northbound gateway_chassis intent can -// be correct while the SB binding is left unclaimed — ovn-controller then -// installs no logical flows for the gateway redirect and the VPC's floating IPs -// go dark. The reconciler runs this after SetGatewayChassis to drive the binding -// to claimed. Implementations shell out to ovn-sbctl/ovn-appctl; the -// compile-time check lives at the wiring site (vpcd) since the host -// implementation cannot import this cross-cutter package. +// GatewayClaimVerifier confirms ovn-controller has claimed the SB chassisredirect +// Port_Binding and nudges recompute if not. After a reboot the SB binding can be +// unclaimed while NB intent is correct, making VPC floating IPs unreachable. type GatewayClaimVerifier interface { - // GatewayPortClaimed reports whether the SB Port_Binding for crPortName has a - // non-empty chassis. crPortName is the chassisredirect (cr-) port: the - // distributed gateway LRP binding stays chassis-less, so only the - // chassisredirect port reflects the gateway-chassis claim. + // GatewayPortClaimed reports whether the SB Port_Binding for crPortName (the + // chassisredirect port) has a non-empty chassis. GatewayPortClaimed(ctx context.Context, crPortName string) (bool, error) // NudgeRecompute asks the local ovn-controller to re-evaluate logical flows. NudgeRecompute(ctx context.Context) error @@ -55,22 +45,17 @@ type Config struct { Routes policy.RouteManager IGW external.IGWManager Topology topology.Manager - // IMDS installs per-VPC IMDS OVN topology during the VPC apply pass. - // Optional: nil skips IMDS plumbing (focused reconcile tests leave it - // unset; production always wires it). + // IMDS installs IMDS OVN topology per-VPC. Optional: nil skips IMDS (tests). IMDS external.IMDSTopologyManager LocalAZ string // NodeHostname is the holder identity for leader-election CAS. NodeHostname string // Chassis is the SBDB-discovered chassis list for gateway LRP rebinding. Chassis []string - // GatewayClaim verifies/repairs the Southbound chassis claim for gateway - // ports after rebinding. Optional: nil skips the post-reboot claim check - // (focused reconcile tests leave it unset; production wires the host prober). + // GatewayClaim verifies/repairs the SB chassis claim after rebinding. Optional. GatewayClaim GatewayClaimVerifier - // DNSServer is the OVN dhcp_options dns_server value ("{a, b}") emitted on - // subnet DHCPOptions rows. Empty falls back to the topology default, so the - // reconciler and live topology paths emit the same value (no drift). + // DNSServer is the OVN dhcp_options dns_server value ("{a, b}"). Empty falls + // back to the topology default to keep both code paths in sync. DNSServer string } @@ -130,11 +115,8 @@ func New(cfg Config) (Reconciler, error) { }, nil } -// Reconcile diffs intent vs. actual OVN state and applies create/delete -// per-resource: VPC → Subnet → SG → Port → IGW → EIP → NATGW (reverse for -// deletes). SG precedes Port so ENI LSPs join their PGs atomically via -// CreateLogicalSwitchPortInGroups. Per-stage errors are logged and the next -// stage runs; only an actual-state scan failure is returned. +// Reconcile diffs intent vs. actual OVN state and applies in topological order. +// Per-stage errors are logged; only a scan failure is returned. func (r *reconciler) Reconcile(ctx context.Context, intent IntentState) error { return r.reconcile(ctx, intent, true) } diff --git a/spinifex/network/subscribers/events.go b/spinifex/network/subscribers/events.go index 6e12c8198..993e0ce18 100644 --- a/spinifex/network/subscribers/events.go +++ b/spinifex/network/subscribers/events.go @@ -40,9 +40,8 @@ type SubnetEvent struct { CidrBlock string `json:"cidr_block"` } -// PortEvent: vpc.create-port / vpc.delete-port. SecurityGroupIds set on -// create so OVN PG membership is wired atomically with the LSP; empty on -// delete (handleDeletePort reads current memberships from the cache). +// PortEvent: vpc.create-port / vpc.delete-port. SecurityGroupIds set on create +// for atomic PG wiring; empty on delete. type PortEvent struct { NetworkInterfaceId string `json:"network_interface_id"` SubnetId string `json:"subnet_id"` @@ -71,9 +70,7 @@ type NATEvent struct { // NATGatewayEvent is published on vpc.add-nat-gateway / vpc.delete-nat-gateway. // SubnetId + DestinationCidr identify the per-subnet egress policy installed -// alongside the SNAT rule so the LR has a default route for the private -// subnet (priority SubnetEgressPriorityNATGW). Without that policy, SNAT -// rewrites the source IP but the packet has no route to leave the LR. +// alongside the SNAT rule so the LR can route packets out. type NATGatewayEvent struct { VpcId string `json:"vpc_id"` NatGatewayId string `json:"nat_gateway_id"` @@ -83,10 +80,8 @@ type NATGatewayEvent struct { DestinationCidr string `json:"destination_cidr"` // route destination, typically 0.0.0.0/0 } -// IGWRouteEvent is published on vpc.add-igw-route / vpc.delete-igw-route -// when a route table associated with SubnetId carries (or loses) a route -// to InternetGatewayId for DestinationCidr. The subscriber installs (or -// removes) an OVN Logical_Router_Policy scoped to the subnet's inport. +// IGWRouteEvent is published on vpc.add-igw-route / vpc.delete-igw-route when a +// route table gains or loses a route to InternetGatewayId for DestinationCidr. type IGWRouteEvent struct { VpcId string `json:"vpc_id"` SubnetId string `json:"subnet_id"` @@ -94,12 +89,8 @@ type IGWRouteEvent struct { InternetGatewayId string `json:"internet_gateway_id"` } -// SubnetEgressGateEvent is published on vpc.gate-subnet-egress when the -// effective route table of SubnetId loses (or never had) a DestinationCidr -// entry pointing at any egress target (IGW / NATGW). The subscriber installs -// an OVN Logical_Router_Policy DROP at SubnetEgressPriorityDrop so the -// subnet cannot reach external destinations via the VPC LR's router-wide -// default static route. +// SubnetEgressGateEvent is published on vpc.gate-subnet-egress when SubnetId +// loses its 0.0.0.0/0 egress target; subscriber installs a DROP policy. type SubnetEgressGateEvent struct { VpcId string `json:"vpc_id"` SubnetId string `json:"subnet_id"` @@ -107,20 +98,16 @@ type SubnetEgressGateEvent struct { } // SubnetEgressUngateEvent is published on vpc.ungate-subnet-egress when a -// previously-gated subnet acquires an egress target in its effective route -// table. The subscriber removes the DROP policy installed by the gate event. +// gated subnet acquires an egress target; subscriber removes the DROP policy. type SubnetEgressUngateEvent struct { VpcId string `json:"vpc_id"` SubnetId string `json:"subnet_id"` DestinationCidr string `json:"destination_cidr"` } -// SystemEgressEvent is published on vpc.add-system-egress / -// vpc.delete-system-egress to give a single hidden system instance (e.g. an -// EKS K3s server VM) egress-only internet access. The subscriber installs (or -// removes) a /32 reroute policy at SystemInstanceEgressPriority plus a plain -// snat (InstanceIp -> ExternalIp) — no DNAT, so the instance stays unreachable -// inbound. +// SystemEgressEvent is published on vpc.add/delete-system-egress for egress-only +// internet access to a single system instance. Subscriber installs a /32 reroute +// + plain snat (no DNAT, so the instance stays unreachable inbound). type SystemEgressEvent struct { VpcId string `json:"vpc_id"` SubnetId string `json:"subnet_id"` diff --git a/spinifex/network/subscribers/subscribers.go b/spinifex/network/subscribers/subscribers.go index ccd785b4e..25c55ee97 100644 --- a/spinifex/network/subscribers/subscribers.go +++ b/spinifex/network/subscribers/subscribers.go @@ -33,9 +33,7 @@ type Config struct { EIP external.EIPManager NATGW external.NATGWManager IGW external.IGWManager - // IMDS installs per-VPC IMDS OVN topology on vpc.create / removes it on - // vpc.delete. Optional: nil disables IMDS plumbing (focused topology tests - // leave it unset; production always wires it). + // IMDS installs/removes IMDS OVN topology on vpc.create/delete. Optional. IMDS external.IMDSTopologyManager } diff --git a/spinifex/network/subscribers/subscribers_test.go b/spinifex/network/subscribers/subscribers_test.go index e528cb285..979d764b6 100644 --- a/spinifex/network/subscribers/subscribers_test.go +++ b/spinifex/network/subscribers/subscribers_test.go @@ -115,12 +115,9 @@ func TestSubscribe_RegistersAllTopics(t *testing.T) { } } -// TestHandleAddNATGateway_InstallsPerSubnetEgress drives the regression: the -// subscriber must install a Logical_Router_Policy at NATGW priority alongside -// the SNAT rule, otherwise private-subnet packets have no route off the LR -// and NAT GW egress fails (Phase 8d). Verifies the policy is installed -// against the IGW's gateway port, with the inport scoped to the private -// subnet, after a vpc.add-nat-gateway event. +// TestHandleAddNATGateway_InstallsPerSubnetEgress drives the regression: subscriber +// must install a Logical_Router_Policy at NATGW priority alongside the SNAT rule, +// scoped to the private subnet inport. func TestHandleAddNATGateway_InstallsPerSubnetEgress(t *testing.T) { ctx := context.Background() _, nc := testutil.StartTestNATS(t) @@ -322,9 +319,8 @@ func TestHandleDeleteNATGateway_InvalidCIDRStillDetaches(t *testing.T) { }, time.Second, 20*time.Millisecond, "SNAT must be removed even when CIDR is invalid") } -// TestHandleAddNATGateway_NoSubnetIDSkipsPolicy: legacy events without a -// subnet (or AWS gateway types that don't bind to a specific subnet) must -// install the SNAT rule and short-circuit before any policy work. +// TestHandleAddNATGateway_NoSubnetIDSkipsPolicy: events without a SubnetId +// must install the SNAT but skip policy work. func TestHandleAddNATGateway_NoSubnetIDSkipsPolicy(t *testing.T) { ctx := context.Background() _, nc := testutil.StartTestNATS(t) @@ -436,11 +432,8 @@ func TestHandleAddIGWRoute_InstallsPolicy(t *testing.T) { assert.Equal(t, policy.SubnetEgressPriorityIGW, policies[0].Priority) } -// TestHandleDeleteNATGateway_MissingRouterStillReturns covers the failure -// path where the underlying router has been torn down already: the handler -// must log + return rather than panic. Exercises both the -// RemoveNATGatewaySubnetEgress error branch and the DetachNATGateway error -// branch, and the empty-DestinationCidr default of 0.0.0.0/0. +// TestHandleDeleteNATGateway_MissingRouterStillReturns: missing router must not +// panic; exercises both error branches and the empty-DestinationCidr default. func TestHandleDeleteNATGateway_MissingRouterStillReturns(t *testing.T) { _, nc := testutil.StartTestNATS(t) sub, _ := newTestSubscriber(t) @@ -543,8 +536,7 @@ func TestHandleAddIGWRoute_InvalidCIDR(t *testing.T) { assert.Empty(t, policies) } -// Every request/reply topic must return a structured error envelope on bad -// JSON. Fire-and-forget topics (NAT gateway) just must not panic. +// Every request/reply topic must return a structured error on bad JSON. func TestBadJSON_AllRequestReplies(t *testing.T) { _, nc := testutil.StartTestNATS(t) sub, _ := newTestSubscriber(t) diff --git a/spinifex/network/subscribers/topology.go b/spinifex/network/subscribers/topology.go index 18d44639e..de7e084df 100644 --- a/spinifex/network/subscribers/topology.go +++ b/spinifex/network/subscribers/topology.go @@ -71,8 +71,7 @@ func (s *Subscriber) handleSubnetCreate(msg *nats.Msg) { } ctx := context.Background() // vpc.create and vpc.create-subnet have no ordering guarantee; pre-ensure - // the VPC router so a tight bootstrap doesn't fail with "router not - // found". EnsureVPC is idempotent. + // the VPC router for tight bootstraps. EnsureVPC is idempotent. if err := s.topology.EnsureVPC(ctx, topology.VPCSpec{VPCID: evt.VpcId}); err != nil { slog.Error("subscribers: EnsureVPC (subnet pre-create) failed", "vpc_id", evt.VpcId, "err", err) respond(msg, err) @@ -87,9 +86,7 @@ func (s *Subscriber) handleSubnetCreate(msg *nats.Msg) { respond(msg, err) return } - // The IMDS localport lives on every subnet switch (guests reach metadata over - // one L2 hop). Install is best-effort — the reconciler re-ensures it - // idempotently, so a transient OVN failure here must not fail CreateSubnet. + // IMDS localport lives on every subnet switch. Best-effort; reconciler re-ensures it. if s.imds != nil { if _, err := s.imds.EnsureForSubnet(ctx, evt.SubnetId, evt.VpcId, cidr); err != nil { slog.Warn("subscribers: IMDS EnsureForSubnet failed; reconciler will converge", "subnet_id", evt.SubnetId, "err", err) @@ -112,8 +109,7 @@ func (s *Subscriber) handleSubnetDelete(msg *nats.Msg) { } } ctx := context.Background() - // Remove the IMDS localport before the subnet switch goes away — the - // localport lives on subnet-{subnetID}. + // Remove the IMDS localport before the subnet switch goes away. if s.imds != nil { if err := s.imds.RemoveForSubnet(ctx, evt.SubnetId); err != nil { slog.Error("subscribers: IMDS RemoveForSubnet failed", "subnet_id", evt.SubnetId, "err", err) @@ -302,10 +298,7 @@ func (s *Subscriber) handleAddNATGateway(msg *nats.Msg) { "vpc_id", evt.VpcId, "natgw_id", evt.NatGatewayId, "public_ip", evt.PublicIp, "subnet_cidr", evt.SubnetCidr) - // SNAT rewrites src IP but the LR still needs a route to egress. Install - // a per-subnet LR policy at NATGW priority so private-subnet packets are - // rerouted out the IGW gateway port (NATGW SNAT happens on the same LR - // before egress). + // SNAT rewrites src IP but the LR still needs a per-subnet reroute policy. if evt.SubnetId == "" { return } @@ -372,8 +365,7 @@ func (s *Subscriber) handleDeleteNATGateway(msg *nats.Msg) { return } ctx := context.Background() - // Remove the per-subnet policy first so no packets are routed to the LR - // uplink after the SNAT rule is gone (which would leak un-NAT'd private IPs). + // Remove per-subnet policy before SNAT to avoid leaking un-NAT'd private IPs. if evt.SubnetId != "" { destCidr := evt.DestinationCidr if destCidr == "" { diff --git a/spinifex/network/topology/dhcp_options.go b/spinifex/network/topology/dhcp_options.go index b1c323539..4bd946e3b 100644 --- a/spinifex/network/topology/dhcp_options.go +++ b/spinifex/network/topology/dhcp_options.go @@ -1,17 +1,8 @@ package topology -// BuildSubnetDHCPOptions returns the OVN DHCPOptions Options map emitted for a -// subnet's DHCPOptions row. Both the live topology manager and the reconciler -// call it so the two paths cannot drift — they previously hard-coded divergent -// dns_server values (live used the configured server, the reconciler "8.8.8.8"). -// -// IMDS reachability (169.254.169.254) is not steered via DHCP option 121: the -// default gateway stays option 3 (router), and the localport on the subnet -// switch answers ARP for the metadata IP directly (one L2 hop, no router). The -// guest's on-link route to 169.254.169.254 is delivered out-of-band by the -// cloud-init network-config the instance handler renders (NoCloud seed) — see -// generateNetworkConfig in handlers/ec2/instance — so it reaches DHCP and static -// guests alike without folding the default route into option 121 (RFC 3442). +// BuildSubnetDHCPOptions returns the OVN DHCPOptions map for a subnet. Shared +// by the live manager and reconciler to prevent dns_server drift. IMDS is not +// steered via option 121; the subnet-switch localport answers L2 ARP directly. func BuildSubnetDHCPOptions(gwIP, routerMAC, dnsServer string) map[string]string { return map[string]string{ "server_id": gwIP, diff --git a/spinifex/network/topology/dhcp_options_test.go b/spinifex/network/topology/dhcp_options_test.go index 2c72051f9..7cc0c8ecb 100644 --- a/spinifex/network/topology/dhcp_options_test.go +++ b/spinifex/network/topology/dhcp_options_test.go @@ -2,10 +2,8 @@ package topology import "testing" -// IMDS reachability is served by a localport on the subnet switch (answered over -// L2), not via DHCP option 121, so BuildSubnetDHCPOptions must no longer emit a -// classless_static_route key. The default gateway is still carried by option 3 -// (router), and dns_server/server_id remain present. +// IMDS is served by a subnet-switch localport over L2, not option 121, so +// BuildSubnetDHCPOptions must not emit classless_static_route. func TestBuildSubnetDHCPOptions_NoClasslessStaticRoute(t *testing.T) { opts := BuildSubnetDHCPOptions("10.0.1.1", "02:00:00:00:00:01", "{8.8.8.8, 1.1.1.1}") diff --git a/spinifex/network/topology/manager.go b/spinifex/network/topology/manager.go index 7c487a1bb..5b05d2b44 100644 --- a/spinifex/network/topology/manager.go +++ b/spinifex/network/topology/manager.go @@ -1,7 +1,6 @@ -// Package topology is L2 of the spinifex network stack: the logical network -// model. It translates VPC/subnet/ENI/SG API objects into OVN logical network -// objects via L1 (network/ovn). Higher layers (policy, external, federation) -// build on this; lower layers must not call here. +// Package topology is L2 of the spinifex network stack. It translates +// VPC/subnet/ENI/SG objects into OVN logical network objects via L1. +// Higher layers (policy, external, federation) build on this. package topology import ( @@ -50,9 +49,8 @@ type SubnetSpec struct { CIDR netip.Prefix } -// PortSpec describes an ENI / VM port at L2. PrivateIP+MAC bind to LSP -// Addresses/PortSecurity at create. SGIDs is initial membership; -// changes go through SetPortSecurityGroups. +// PortSpec describes an ENI port at L2. PrivateIP+MAC bind to LSP Addresses/ +// PortSecurity at create; SG membership changes go through SetPortSecurityGroups. type PortSpec struct { PortID string SubnetID string diff --git a/spinifex/network/topology/manager_live.go b/spinifex/network/topology/manager_live.go index fe6f576c3..b298803fd 100644 --- a/spinifex/network/topology/manager_live.go +++ b/spinifex/network/topology/manager_live.go @@ -43,8 +43,7 @@ var _ Manager = (*liveManager)(nil) func defaultDNSServer() string { return "{8.8.8.8, 1.1.1.1}" } -// EnsureVPC idempotently creates the OVN logical router for the VPC via the -// wait-op-protected EnsureLogicalRouter primitive (cross-node race-safe). +// EnsureVPC idempotently creates the OVN logical router for the VPC. // Backfills ExternalIDs if the surviving row was created with empty metadata. func (m *liveManager) EnsureVPC(ctx context.Context, spec VPCSpec) error { if m.ovn == nil { @@ -81,8 +80,8 @@ func (m *liveManager) EnsureVPC(ctx context.Context, spec VPCSpec) error { return nil } -// backfillRouterMetadata fills empty ExternalIDs on the existing row from spec -// so the race-loser's richer metadata isn't lost (EnsureLogicalRouter is single-shot). +// backfillRouterMetadata fills empty ExternalIDs on the existing row so metadata +// from the race-loser isn't lost. func (m *liveManager) backfillRouterMetadata(ctx context.Context, existing *nbdb.LogicalRouter, spec map[string]string) error { if existing.ExternalIDs == nil { existing.ExternalIDs = map[string]string{} @@ -270,8 +269,7 @@ func (m *liveManager) DeleteSubnet(ctx context.Context, spec SubnetSpec) error { } // EnsurePort creates the ENI's LSP and joins initial SG port groups atomically. -// If the LSP exists from a prior crashed attempt, converges SG memberships in -// place — the gap would leave a port with zero ACLs (OVN default = unrestricted). +// If the LSP already exists, converges SG memberships to avoid zero-ACL gaps. func (m *liveManager) EnsurePort(ctx context.Context, spec PortSpec) error { if m.ovn == nil { return fmt.Errorf("OVN client not connected") @@ -363,8 +361,8 @@ func (m *liveManager) EnsureSGPortGroup(ctx context.Context, groupID string) err return nil } -// DeleteSGPortGroup clears ACLs then removes the port-group row. Idempotent. -// ClearACLs must precede DeletePortGroup — libovsdb rejects dangling refs. +// DeleteSGPortGroup clears ACLs then removes the port-group row; idempotent. +// ClearACLs must run first — libovsdb rejects dangling refs. func (m *liveManager) DeleteSGPortGroup(ctx context.Context, groupID string) error { if groupID == "" { return fmt.Errorf("DeleteSGPortGroup: empty groupID") @@ -409,8 +407,7 @@ func (m *liveManager) SetPortSecurityGroups(ctx context.Context, portID string, return nil } -// reconcilePortSGs applies the SG diff atomically — an N→M modify must never -// expose an intermediate gap (OVN default = unrestricted). Returns true if changed. +// reconcilePortSGs applies the SG diff atomically to avoid zero-ACL gaps. Returns true if changed. func (m *liveManager) reconcilePortSGs(ctx context.Context, lspName string, desiredSGs []string) (bool, error) { desired := make(map[string]struct{}, len(desiredSGs)) for _, sgID := range desiredSGs { diff --git a/spinifex/network/topology/names.go b/spinifex/network/topology/names.go index 2ed43dea2..73a22e980 100644 --- a/spinifex/network/topology/names.go +++ b/spinifex/network/topology/names.go @@ -28,10 +28,8 @@ func GatewayRouterPort(vpcID string) string { return "gw-" + vpcID } // peered with GatewayRouterPort. func GatewaySwitchPort(vpcID string) string { return "gw-port-" + vpcID } -// GatewayChassisRedirectPort is the OVN chassisredirect Port_Binding for the -// VPC's distributed gateway port. OVN derives it as "cr-"+LRP when the LRP has a -// gateway_chassis; it is the only binding that carries the gateway-chassis claim -// (the LRP binding itself stays distributed/chassis-less). +// GatewayChassisRedirectPort is the OVN chassisredirect Port_Binding (cr-). +// Only this binding carries the gateway-chassis claim; the LRP binding stays chassis-less. func GatewayChassisRedirectPort(vpcID string) string { return "cr-" + GatewayRouterPort(vpcID) } // ExternalSwitch is the OVN logical switch name for the per-VPC external @@ -42,10 +40,8 @@ func ExternalSwitch(vpcID string) string { return "ext-" + vpcID } // external switch onto the host uplink. func ExternalLocalnetPort(vpcID string) string { return "ext-port-" + vpcID } -// IMDSPort is the host-owned localport LSP that claims 169.254.169.254 directly -// on the subnet switch (SubnetSwitch). ovn-controller binds it on every chassis -// with a matching iface-id OVS port, so each chassis self-serves IMDS for its -// local VMs over a single L2 hop on the guest's own broadcast domain. +// IMDSPort is the host-owned localport LSP that claims 169.254.169.254 on the +// subnet switch. Bound on every chassis; serves IMDS over a single L2 hop. func IMDSPort(subnetID string) string { return "imds-port-" + subnetID } // SecurityGroupPortGroup is the OVN port group name for an SG. diff --git a/spinifex/objectstore/objectstore.go b/spinifex/objectstore/objectstore.go index 105eddafb..561fbc00b 100644 --- a/spinifex/objectstore/objectstore.go +++ b/spinifex/objectstore/objectstore.go @@ -1,6 +1,5 @@ -// Package objectstore provides a common abstraction for S3-like object storage operations. -// This package enables handlers to work with either real S3 backends (Predastore) or -// in-memory stores for testing, without changing the handler implementation. +// Package objectstore provides a common S3-like storage abstraction used by handlers +// to work with real backends (Predastore) or in-memory stores for testing. package objectstore import ( @@ -30,30 +29,21 @@ func (e *NoSuchKeyError) Code() string { return "NoSuchKey" } -// IsNoSuchKeyError checks if an error is a NoSuchKey error +// IsNoSuchKeyError reports whether err is a NoSuchKey error. func IsNoSuchKeyError(err error) bool { var noSuchKey *NoSuchKeyError return errors.As(err, &noSuchKey) } // ObjectStore defines the interface for S3-like object storage operations. -// This abstraction allows for mocking in tests without requiring actual S3 connectivity. type ObjectStore interface { - // GetObject retrieves an object from storage GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) - - // PutObject stores an object in storage PutObject(input *s3.PutObjectInput) (*s3.PutObjectOutput, error) - - // DeleteObject removes an object from storage DeleteObject(input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) - - // ListObjectsV2 lists objects in a bucket using the V2 API ListObjectsV2(input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) } -// NewS3ObjectStoreFromConfig creates an S3ObjectStore from Predastore connection parameters, -// eliminating the duplicated TLS+HTTP/2+session boilerplate in each service. +// NewS3ObjectStoreFromConfig creates an S3ObjectStore from Predastore connection parameters. func NewS3ObjectStoreFromConfig(host, region, accessKey, secretKey string) *S3ObjectStore { sess := session.Must(session.NewSession(&aws.Config{ Endpoint: aws.String(host), @@ -101,7 +91,7 @@ func (s *S3ObjectStore) ListObjectsV2(input *s3.ListObjectsV2Input) (*s3.ListObj return s.client.ListObjectsV2(input) } -// MemoryObjectStore implements ObjectStore using in-memory storage for testing +// MemoryObjectStore implements ObjectStore using in-memory storage for testing. type MemoryObjectStore struct { objects map[string][]byte // key: bucket/key -> value: object data mu sync.RWMutex @@ -181,25 +171,16 @@ func (m *MemoryObjectStore) ListObjectsV2(input *s3.ListObjectsV2Input) (*s3.Lis prefixes := make(map[string]bool) for key, data := range m.objects { - // Check if key belongs to this bucket if !strings.HasPrefix(key, bucket+"/") { continue } - - // Extract the key part (remove bucket/) objectKey := key[len(bucket)+1:] - - // Check prefix filter if prefix != "" && !strings.HasPrefix(objectKey, prefix) { continue } - - // Handle delimiter (common prefixes) if delimiter != "" { - // Find the position after the prefix where delimiter appears afterPrefix := objectKey[len(prefix):] if idx := strings.Index(afterPrefix, delimiter); idx >= 0 { - // This object is in a "subdirectory", add to common prefixes commonPrefix := objectKey[:len(prefix)+idx+len(delimiter)] prefixes[commonPrefix] = true continue @@ -212,7 +193,6 @@ func (m *MemoryObjectStore) ListObjectsV2(input *s3.ListObjectsV2Input) (*s3.Lis }) } - // Convert prefixes map to CommonPrefixes list var commonPrefixes []*s3.CommonPrefix for p := range prefixes { commonPrefixes = append(commonPrefixes, &s3.CommonPrefix{ diff --git a/spinifex/qmp/qmp.go b/spinifex/qmp/qmp.go index 89e9f0d70..2db876dc3 100644 --- a/spinifex/qmp/qmp.go +++ b/spinifex/qmp/qmp.go @@ -28,9 +28,6 @@ type Status struct { Running bool `json:"running"` } -// (QEMU) query-block -// {"return": [{"io-status": "ok", "device": "os", "locked": false, "removable": false, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 4294967296, "filename": "nbd://127.0.0.1:44801", "format": "raw"}, "iops_wr": 0, "ro": false, "node-name": "#block192", "backing_file_depth": 0, "drv": "raw", "iops": 0, "bps_wr": 0, "write_threshold": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "cache": {"no-flush": false, "direct": false, "writeback": true}, "file": "nbd://127.0.0.1:44801"}, "qdev": "/machine/peripheral-anon/device[0]/virtio-backend", "type": "unknown"}, {"io-status": "ok", "device": "cloudinit", "locked": false, "removable": false, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 1048576, "filename": "nbd://127.0.0.1:42911", "format": "raw"}, "iops_wr": 0, "ro": true, "node-name": "#block312", "backing_file_depth": 0, "drv": "raw", "iops": 0, "bps_wr": 0, "write_threshold": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "cache": {"no-flush": false, "direct": false, "writeback": true}, "file": "nbd://127.0.0.1:42911"}, "qdev": "/machine/peripheral-anon/device[3]/virtio-backend", "type": "unknown"}, {"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "qdev": "/machine/unattached/device[24]", "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "qdev": "/machine/unattached/device[18]", "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "type": "unknown"}]} - type QMPQueryBlockResponse struct { Return []BlockDevice `json:"return"` Error *QMPError `json:"error,omitempty"` @@ -119,7 +116,7 @@ func NewQMPClient(path string) (*QMPClient, error) { return nil, err } - // Set deadline for greeting — a hung QEMU would block forever without this + // Deadline prevents a hung QEMU from blocking forever. if err := conn.SetReadDeadline(time.Now().Add(30 * time.Second)); err != nil { _ = conn.Close() return nil, fmt.Errorf("set read deadline: %w", err) @@ -144,8 +141,5 @@ func NewQMPClient(path string) (*QMPClient, error) { return nil, fmt.Errorf("clear read deadline: %w", err) } - // enable capabilities - //client.Send(QMPCommand{Execute: "qmp_capabilities"}) - return client, nil } diff --git a/spinifex/services/awsgw/awsgw.go b/spinifex/services/awsgw/awsgw.go index 29daa81a1..1ddc3f0a6 100644 --- a/spinifex/services/awsgw/awsgw.go +++ b/spinifex/services/awsgw/awsgw.go @@ -104,8 +104,6 @@ func loadThrottleConfig(path string) (ratelimit.Config, error) { func launchService(config *config.ClusterConfig) error { nodeConfig := config.Nodes[config.Node] - // Connect to NATS for service communication. On concurrent startup the - // local NATS server may not be listening yet, so retry with backoff. natsConn, err := utils.ConnectNATSWithRetry(admin.DialTarget(nodeConfig.NATS.Host), nodeConfig.NATS.ACL.Token, nodeConfig.NATS.CACert) if err != nil { return err @@ -198,7 +196,6 @@ func launchService(config *config.ClusterConfig) error { slog.Warn("Failed to load throttle config, throttling disabled", "err", err) } - // Create gateway with NATS connection gw := gateway.GatewayConfig{ Debug: nodeConfig.AWSGW.Debug, DisableLogging: false, @@ -277,10 +274,8 @@ func initIAMService(natsConn *nats.Conn, masterKey []byte, clusterSize int) (*ha } } -// findBootstrapFile returns the path to bootstrap.json, checking the data -// directory first (production), then the awsgw subdir (dev mode), then the -// legacy config dir. Returns the first path that exists, or the primary -// location if none exist (so the caller gets a clean "not found" error). +// findBootstrapFile returns the first existing bootstrap.json candidate path, +// or the primary path if none exist. func findBootstrapFile(baseDir string) string { candidates := []string{ filepath.Join(baseDir, "bootstrap.json"), diff --git a/spinifex/services/awsgw/imds_absent_test.go b/spinifex/services/awsgw/imds_absent_test.go index 6640e703e..cb6c1da14 100644 --- a/spinifex/services/awsgw/imds_absent_test.go +++ b/spinifex/services/awsgw/imds_absent_test.go @@ -6,11 +6,8 @@ import ( "testing" ) -// TestIMDSRuntimeAbsentFromAWSGW guards against re-introducing the privileged -// IMDS listener stack into awsgw, whose systemd unit is the most hardened in -// the fleet (NoNewPrivileges=yes, empty CapabilityBoundingSet). The veth + -// privileged-socket bind cannot succeed there; the runtime lives in vpcd, and -// awsgw only answers the IMDS control-plane RPCs (STS + IAM). +// TestIMDSRuntimeAbsentFromAWSGW guards against importing the privileged IMDS +// listener stack (NewIMDSServiceImpl) into awsgw; that runtime lives in vpcd. func TestIMDSRuntimeAbsentFromAWSGW(t *testing.T) { src, err := os.ReadFile("awsgw.go") if err != nil { diff --git a/spinifex/services/nats/nats.go b/spinifex/services/nats/nats.go index 0d48a0cf9..683621c22 100644 --- a/spinifex/services/nats/nats.go +++ b/spinifex/services/nats/nats.go @@ -104,12 +104,9 @@ func launchService(config *Config) (err error) { return err } - // Configure the logger based on the flags. ns.ConfigureLogger() - // Start things up. Block here until done. if err := server.Run(ns); err != nil { - // Will exit() here server.PrintAndDie(err.Error()) } diff --git a/spinifex/services/nats/nats_test.go b/spinifex/services/nats/nats_test.go index d1fbe44ee..bcd6db009 100644 --- a/spinifex/services/nats/nats_test.go +++ b/spinifex/services/nats/nats_test.go @@ -26,10 +26,8 @@ func templatePath(t *testing.T) string { return filepath.Join(filepath.Dir(thisFile), "..", "..", "..", "cmd", "spinifex", "cmd", "templates", "nats.conf") } -// TestRenderedConfig_EnforcesAuth renders the production nats.conf template with -// a known token, starts an embedded NATS server from the resulting config, and -// verifies that unauthenticated and wrong-token connections are rejected while -// correct-token connections succeed. +// TestRenderedConfig_EnforcesAuth renders the production nats.conf template, +// starts an embedded NATS server, and verifies token authentication. func TestRenderedConfig_EnforcesAuth(t *testing.T) { token := "nats_test-secret-token-1234" tmpDir := t.TempDir() @@ -107,10 +105,8 @@ func TestRenderedConfig_EnforcesAuth(t *testing.T) { } // TestRenderedConfig_HasMigrationVersionMarker ensures the nats.conf template -// stamps the current migration version on the first line. Without this marker, -// the migration framework treats fresh installs as version 0 and tries to run -// the 0→1 migration on next upgrade. The marker must be the literal first line -// — NATSConfVersionReader only checks line 0. +// stamps the migration version on the first line (NATSConfVersionReader only +// checks line 0; missing marker causes fresh installs to run the 0→1 migration). func TestRenderedConfig_HasMigrationVersionMarker(t *testing.T) { raw, err := os.ReadFile(templatePath(t)) require.NoError(t, err) diff --git a/spinifex/services/predastore/predastore.go b/spinifex/services/predastore/predastore.go index 7ba811212..b3f624b08 100644 --- a/spinifex/services/predastore/predastore.go +++ b/spinifex/services/predastore/predastore.go @@ -82,7 +82,6 @@ func (svc *Service) Start() (int, error) { svc.server = server - // Start server asynchronously if err := server.ListenAndServeAsync(); err != nil { slog.Error("Failed to start predastore server", "error", err) return 0, err diff --git a/spinifex/services/predastore/predastore_integration_test.go b/spinifex/services/predastore/predastore_integration_test.go index 83b7da3ec..c64a224f1 100644 --- a/spinifex/services/predastore/predastore_integration_test.go +++ b/spinifex/services/predastore/predastore_integration_test.go @@ -1,9 +1,7 @@ package predastore -// Integration tests for predastore service -// -// These tests start a real predastore daemon and test S3 operations against it. -// Tests use AWS SDK Go v1 to verify bucket operations, authentication, and file operations. +// Integration tests for predastore: start a real daemon and verify S3 bucket, +// auth, and file operations using AWS SDK Go v1. import ( "bytes" @@ -57,11 +55,8 @@ var ( sharedConfig *Config ) -// generateTestCertificate generates a self-signed TLS certificate for testing -// and returns a CertPool that trusts it. Predastore's QUIC client (commit -// dce8618) verifies server certs against the OS trust store after -// InsecureSkipVerify was dropped, so the test must inject the ephemeral CA -// via quicclient.SetDefaultRootCAs before any dial. +// generateTestCertificate generates a self-signed TLS certificate for testing. +// The CA must be injected via quicclient.SetDefaultRootCAs before any dial. func generateTestCertificate(certPath, keyPath string) (*x509.CertPool, error) { priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { @@ -161,21 +156,11 @@ func startPredastoreServer(t *testing.T) *Config { t.Fatalf("Failed to generate certificate: %v", err) } - // Predastore's in-process QUIC client (s3d → shard nodes) verifies the - // server cert against the OS trust store. The ephemeral test cert is - // not in that store, so inject it as a package-level default. Persists - // for the lifetime of the test process — startPredastoreServer is a - // once-per-process singleton. + // Inject the ephemeral cert for the QUIC client (s3d → shard nodes). quicclient.SetDefaultRootCAs(caPool) - // Predastore's in-process s3db REST client (port 6660) has no public - // hook to override its RootCAs — it uses the OS trust store via the - // default tls.Config (RootCAs nil ⇒ x509.SystemCertPool, which Go reads - // from SSL_CERT_FILE on Unix). Point it at the ephemeral cert so the - // in-process bucket/object metadata calls trust their own server. - // x509.SystemCertPool is sync.Once-cached, so the pool is loaded on - // the first dial below while SSL_CERT_FILE is still set, and stays - // valid for the rest of the process even after t.Setenv reverts. + // SSL_CERT_FILE injects the cert for the s3db REST client's OS trust store + // (sync.Once-cached, must be set before the first dial). t.Setenv("SSL_CERT_FILE", certPath) // Predastore mandates a 32-byte master key at mode 0600 (rejected otherwise @@ -189,10 +174,7 @@ func startPredastoreServer(t *testing.T) *Config { t.Fatalf("Failed to write test encryption key: %v", err) } - // Create config file. Five storage nodes are declared so predastore's dev-mode - // path launches all QUIC servers locally as goroutines (server.go:629). Buckets - // are created via the S3 API after startup — config buckets are no longer - // surfaced by ListBuckets (predastore commit 0711e7d, account-scoped auth). + // Five nodes trigger dev-mode: all QUIC shards start as local goroutines. configPath := filepath.Join(testDir, "predastore_test.toml") configContent := `version = "1.0" region = "us-east-1" @@ -256,10 +238,7 @@ account_id = "123456789012" TlsCert: certPath, TlsKey: keyPath, EncryptionKeyFile: encryptionKeyPath, - // Predastore rejects NodeID 0; -1 triggers dev mode, which launches - // every configured QUIC node in-process. The test config defines - // five nodes, so this is the only sane choice for a single-process - // integration test. + // NodeID -1 triggers dev mode: all QUIC nodes run in-process. NodeID: -1, } sharedConfig = cfg diff --git a/spinifex/services/predastore/predastore_test.go b/spinifex/services/predastore/predastore_test.go index 4f2ed5e1d..c8ea9bb62 100644 --- a/spinifex/services/predastore/predastore_test.go +++ b/spinifex/services/predastore/predastore_test.go @@ -237,10 +237,8 @@ func TestServiceStartWithoutConfig(t *testing.T) { t.Skip("Skipping test that requires actual predastore config file - covered by integration tests") } -// Start() must fail fast (before touching the pid file or trying to bind a -// port) when no encryption key file is configured. Predastore itself -// enforces this at the s3.Server layer, but a clear pre-check error at the -// spinifex service boundary saves operators an opaque downstream failure. +// TestServiceStartRejectsMissingEncryptionKey ensures Start fails fast when +// EncryptionKeyFile is unset, before touching the pid file or binding. func TestServiceStartRejectsMissingEncryptionKey(t *testing.T) { cfg := &Config{ ConfigPath: "/tmp/test-config.toml", diff --git a/spinifex/services/spinifexui/proxy.go b/spinifex/services/spinifexui/proxy.go index 9eff300c6..f4d031eb6 100644 --- a/spinifex/services/spinifexui/proxy.go +++ b/spinifex/services/spinifexui/proxy.go @@ -38,10 +38,8 @@ func newProxyTransport(caCertPath string) (*http.Transport, error) { }, nil } -// newReverseProxy creates a reverse proxy that forwards requests to the given -// backend host:port after stripping the pathPrefix from the request path. -// The proxy sets req.Host to the backend address so SigV4 signature verification -// succeeds (the gateway uses r.Host for the canonical host header). +// newReverseProxy forwards requests to backendHost after stripping pathPrefix. +// Sets req.Host to the backend so SigV4 canonical-host verification passes. func newReverseProxy(backendHost, pathPrefix string, transport *http.Transport) http.Handler { target := &url.URL{ Scheme: "https", diff --git a/spinifex/services/viperblockd/viperblockd.go b/spinifex/services/viperblockd/viperblockd.go index 310abeac9..7ccf33233 100644 --- a/spinifex/services/viperblockd/viperblockd.go +++ b/spinifex/services/viperblockd/viperblockd.go @@ -24,20 +24,15 @@ import ( "github.com/nats-io/nats.go" ) -// loadStateRetryAttempts and loadStateRetryBaseDelay tune the mount-time -// retry loop for transient backend errors (predastore restarting, NATS -// disconnect, etc.). 5 attempts at 200ms * 1.5^n caps total wait at ~3.7s, -// well under the daemon's 30s ebs.mount NATS timeout. See mulga-siv-25. +// loadStateRetryAttempts / loadStateRetryBaseDelay tune the mount-time retry +// loop (5 attempts at 200ms * 1.5^n ≈ 3.7s; well under the 30s NATS timeout). const ( loadStateRetryAttempts = 5 loadStateRetryBaseDelay = 200 * time.Millisecond ) -// retryLoadState invokes loadFn up to attempts times, retrying with -// exponential backoff (delay * 3/2 each step) only when the returned error -// is viperblock.ErrStateBackendUnavailable. Success, ErrStateNotFound, and -// any unclassified error return immediately so callers see them. Extracted -// for unit testability — see loadStateWithRetry for the production caller. +// retryLoadState invokes loadFn with exponential backoff (delay * 3/2 each step) +// on ErrStateBackendUnavailable only; other errors return immediately. func retryLoadState(volume string, attempts int, baseDelay time.Duration, sleep func(time.Duration), loadFn func() error) error { delay := baseDelay var err error @@ -109,10 +104,8 @@ type Config struct { // ShardWAL enables sharded WAL for mounted volumes (default false) ShardWAL bool - // EncryptionKeyFile is the path to the shared 32-byte AES-256 master key - // used to seal viperblock WAL records, chunk blocks, and metadata. When - // empty the daemon mounts volumes in cleartext mode (legacy). Loaded once - // at launchService start; the parsed key is cached in masterKey. + // EncryptionKeyFile is the path to the shared AES-256 master key for at-rest + // encryption. Empty → cleartext mode (legacy). EncryptionKeyFile string masterKey *masterkey.Key @@ -226,7 +219,6 @@ func (svc *Service) Reload() (err error) { } func launchService(cfg *Config) (err error) { - // Connect to NATS nc, err := utils.ConnectNATSWithRetry(admin.DialTarget(cfg.NatsHost), cfg.NatsToken, cfg.NatsCACert) if err != nil { slog.Error("Failed to connect to NATS", "err", err) @@ -326,7 +318,6 @@ func launchService(cfg *Config) (err error) { if _, err := unmountSubscribe(unmountTopic, func(msg *nats.Msg) { slog.Info("Received message", "data", string(msg.Data)) - // Parse the message var ebsRequest types.EBSRequest if err := json.Unmarshal(msg.Data, &ebsRequest); err != nil { slog.Error("Failed to unmarshal ebs.unmount message", "err", err) @@ -453,7 +444,6 @@ func launchService(cfg *Config) (err error) { if _, err := mountSubscribe(mountTopic, func(msg *nats.Msg) { slog.Info("Received message:", "data", string(msg.Data)) - // Parse the message var ebsRequest types.EBSRequest if err := json.Unmarshal(msg.Data, &ebsRequest); err != nil { slog.Error("Failed to unmarshal ebs.mount message", "err", err) @@ -531,10 +521,7 @@ func launchService(cfg *Config) (err error) { return } - // Next, connect to the volume and confirm the state - // First, fetch the state from the remote backend. Retry on transient - // backend errors (predastore restart races, NATS hiccups) so daemon - // recovery doesn't tip a healthy volume into the cleanup path. + // Retry on transient backend errors so daemon recovery doesn't tip a healthy volume into cleanup. err = loadStateWithRetry(vb, ebsRequest.Name) if err != nil { @@ -543,9 +530,6 @@ func launchService(cfg *Config) (err error) { return } - // Next, mount the volume using nbdkit - - // Determine transport type (default to socket) useTCP := cfg.NBDTransport == types.NBDTransportTCP var nbdURI string @@ -649,7 +633,6 @@ func launchService(cfg *Config) (err error) { slog.Error("NBDKit exited", "code", exitCode) }() - // Wait for startup result pid := <-processChan if pid == 0 { diff --git a/spinifex/services/viperblockd/viperblockd_integration_test.go b/spinifex/services/viperblockd/viperblockd_integration_test.go index 222a41401..362421927 100644 --- a/spinifex/services/viperblockd/viperblockd_integration_test.go +++ b/spinifex/services/viperblockd/viperblockd_integration_test.go @@ -1,16 +1,8 @@ package viperblockd -// Integration tests for viperblockd service using embedded NATS server -// -// These tests verify the service's behavior with real NATS messaging while mocking -// external dependencies (S3, nbdkit). Each test starts a goroutine running launchService() -// which blocks waiting for SIGTERM. The goroutines are cleaned up when the test process exits. -// -// IMPORTANT: Run these tests with a timeout flag to prevent hanging: +// Integration tests for viperblockd using an embedded NATS server. Run with: // go test ./spinifex/services/viperblockd/... -timeout 40s -// -// To skip integration tests and run only unit tests: -// go test -short ./spinifex/services/viperblockd/... +// go test -short ./spinifex/services/viperblockd/... # unit tests only import ( "encoding/json" diff --git a/spinifex/services/viperblockd/viperblockd_test.go b/spinifex/services/viperblockd/viperblockd_test.go index 093a084d2..d48847942 100644 --- a/spinifex/services/viperblockd/viperblockd_test.go +++ b/spinifex/services/viperblockd/viperblockd_test.go @@ -444,9 +444,8 @@ func TestMountedVolumePortRange(t *testing.T) { } } -// TestRetryLoadState pins the retry policy used by ebs.mount: transient -// backend errors retry with backoff, ErrStateNotFound fails fast, and the -// retry budget is honoured. Required by mulga-siv-25 medium-term fix. +// TestRetryLoadState pins the retry policy: transient errors retry with backoff, +// ErrStateNotFound fails fast, and the retry budget is honoured. func TestRetryLoadState(t *testing.T) { t.Run("success_first_attempt_no_sleep", func(t *testing.T) { calls := 0 diff --git a/spinifex/systemd/units_invariants_test.go b/spinifex/systemd/units_invariants_test.go index cfa55f676..f4d07d1bf 100644 --- a/spinifex/systemd/units_invariants_test.go +++ b/spinifex/systemd/units_invariants_test.go @@ -62,11 +62,9 @@ func hasDirective(unit, line string) bool { return false } -// TestRG9_TierConfinement asserts the RG-9 per-tier least-privilege contract: -// the locked-down storage/control tier drops all capabilities; the daemon and -// vpcd are the two privileged exceptions, each confined to exactly what its role -// needs and no broader. Adding a capability/device to any tier — or dropping the -// baseline from the locked-down tier — without updating this test fails CI. +// TestRG9_TierConfinement asserts the RG-9 least-privilege contract. +// Storage/control tier drops all caps; daemon and vpcd are the two privileged exceptions. +// Adding capabilities or weakening the locked-down baseline without updating this test fails CI. func TestRG9_TierConfinement(t *testing.T) { dir := unitsDir(t) diff --git a/spinifex/tags/tags.go b/spinifex/tags/tags.go index d705d798b..d42f294e4 100644 --- a/spinifex/tags/tags.go +++ b/spinifex/tags/tags.go @@ -1,8 +1,5 @@ -// Package tags defines tag keys and values used to mark -// Spinifex system-owned resources (ENIs, EC2 instances, AMIs). -// -// The UI filters resources carrying these tags out of customer-facing -// listings. Operators can append ?system=1 to the URL to surface them. +// Package tags defines tag keys for Spinifex system-owned resources. +// The UI filters these out of customer-facing listings; operators append ?system=1 to surface them. package tags const ( diff --git a/spinifex/testutil/nats.go b/spinifex/testutil/nats.go index 1955c0881..8196542f5 100644 --- a/spinifex/testutil/nats.go +++ b/spinifex/testutil/nats.go @@ -60,10 +60,8 @@ func StartTestJetStream(t *testing.T) (*server.Server, *nats.Conn, nats.JetStrea return ns, nc, js } -// vpcdStubTopics lists every synchronous vpcd request topic the unit-test -// stub auto-replies to. CreateSecurityGroup / Authorize / Revoke / Modify / -// CreateNetworkInterface all block on the 5s vpcd round-trip, so the stub -// has to cover each one or the tests time out. +// vpcdStubTopics lists every synchronous vpcd topic the stub auto-replies to. +// All block on a 5 s round-trip, so missing any one causes test timeouts. var vpcdStubTopics = []string{ "vpc.create-sg", "vpc.delete-sg", @@ -72,37 +70,27 @@ var vpcdStubTopics = []string{ "vpc.create-port", } -// vpcdStubRegistry holds the per-conn response map shared between the -// subscribers and the OverrideVpcdStubResponse helper. Stored per-conn so -// concurrent tests with separate NATS conns don't trip on each other. +// vpcdStubRegistry holds the per-conn response map. Per-conn so concurrent tests with separate conns don't interfere. var ( vpcdStubMu sync.RWMutex vpcdStubRegistry = map[*nats.Conn]map[string][]byte{} ) -// StubVpcdSGResponder stands in for vpcd in unit tests by auto-replying -// success to the synchronous vpcd request topics. Use -// OverrideVpcdStubResponse to swap a topic's reply mid-test (negative-path -// tests) — that updates the same single subscriber rather than layering a -// second one, which would race the success replier. +// StubVpcdSGResponder auto-replies success to all synchronous vpcd topics. +// Use OverrideVpcdStubResponse to swap a reply mid-test without adding a racing second subscriber. func StubVpcdSGResponder(t *testing.T, nc *nats.Conn) { t.Helper() registerVpcdStub(t, nc, []byte(`{"success":true}`)) } -// StubVpcdSGFailingResponder is the negative-path counterpart to -// StubVpcdSGResponder: every stubbed topic replies success=false so tests -// can assert the handler propagates vpcd errors instead of swallowing them. +// StubVpcdSGFailingResponder is the negative-path counterpart: all topics reply success=false. func StubVpcdSGFailingResponder(t *testing.T, nc *nats.Conn, errMsg string) { t.Helper() registerVpcdStub(t, nc, []byte(`{"success":false,"error":"`+errMsg+`"}`)) } -// OverrideVpcdStubResponse changes the stub's reply for one topic on the -// given conn. The change is in-place on the existing subscriber, so there -// is exactly one responder per topic — no race with a layered second -// subscriber. Use to make a single-topic negative-path assertion against -// an otherwise-success stub. +// OverrideVpcdStubResponse changes the stub's reply for one topic on the given conn. +// Updates the existing subscriber in-place — no second responder, no race. func OverrideVpcdStubResponse(nc *nats.Conn, topic string, payload []byte) { vpcdStubMu.Lock() defer vpcdStubMu.Unlock() diff --git a/spinifex/types/ec2.go b/spinifex/types/ec2.go index 8c914d938..00b1f5d66 100644 --- a/spinifex/types/ec2.go +++ b/spinifex/types/ec2.go @@ -1,9 +1,7 @@ package types // EC2InstanceCommand is the NATS wire format for EC2 instance commands -// (stop, terminate, start, attach-volume, detach-volume, attach-eni, -// detach-eni). It replaces direct use of qmp.Command on the gateway→daemon -// boundary. +// (stop, terminate, start, attach/detach-volume, attach/detach-eni). type EC2InstanceCommand struct { ID string `json:"id"` Attributes EC2CommandAttributes `json:"attributes"` @@ -54,10 +52,8 @@ type DetachENIData struct { } // IamProfileAssociationData carries parameters for an associate-iam-instance-profile -// command. The gateway has already resolved the profile reference to a canonical -// ARN and enforced iam:PassRole — the daemon only needs the ARN to persist on -// vm.VM. The new AssociationId is generated daemon-side under the vmMgr lock, -// so it does not appear in the request payload. +// command. The gateway resolves the ARN and enforces iam:PassRole; the daemon +// only needs the ARN to persist on vm.VM. type IamProfileAssociationData struct { InstanceProfileArn string `json:"instance_profile_arn"` } diff --git a/spinifex/types/eni.go b/spinifex/types/eni.go index 27e3aff9f..78ab97b30 100644 --- a/spinifex/types/eni.go +++ b/spinifex/types/eni.go @@ -2,15 +2,9 @@ package types import "sync" -// ENIRequests tracks the PCIe hot-plug slot allocator for a single VM's -// hot-plugged ENIs. AvailableSlots is the free-list of slot indices (1-based, -// matching the pcie-root-port id=hotplug-eni{N} pre-allocated at VM start). -// AttachedByENIID maps eniID → assigned slot so the detach path can free the -// slot without re-scanning, and so the post-restart reconciler can rebuild -// the in-memory state from KV truth. -// -// Boot-time ENIs (primary + ExtraENIs) use fixed slot addresses configured -// at VM start and do NOT consume the hot-plug slot pool. +// ENIRequests tracks PCIe hot-plug slot allocation for a VM's hot-plugged ENIs. +// AvailableSlots is the free-list; AttachedByENIID maps eniID → slot for detach +// and post-restart reconciliation. Boot-time ENIs use fixed slots outside this pool. type ENIRequests struct { Mu sync.Mutex `json:"-"` AvailableSlots []int `json:"available_slots"` diff --git a/spinifex/types/node.go b/spinifex/types/node.go index 4adac276c..a108d0c5d 100644 --- a/spinifex/types/node.go +++ b/spinifex/types/node.go @@ -37,11 +37,7 @@ type VMGPUInfo struct { } // NodeStatusResponse is returned by the spinifex.node.status NATS topic (fan-out). -// -// ReservedVCPU / ReservedMemGB are held back from guest scheduling for the -// spinifex daemon and co-located services. Schedulable capacity is -// TotalVCPU - ReservedVCPU - AllocVCPU (and the equivalent for memory); -// per-type counts in InstanceTypes already account for the reserve. +// Schedulable capacity = TotalVCPU - ReservedVCPU - AllocVCPU (same for memory). type NodeStatusResponse struct { Node string `json:"node"` Status string `json:"status"` diff --git a/spinifex/utils/account.go b/spinifex/utils/account.go index 6f8e940b7..e904e54d3 100644 --- a/spinifex/utils/account.go +++ b/spinifex/utils/account.go @@ -11,8 +11,7 @@ import ( // VersionKey is the well-known KV key used to store a bucket's schema version. const VersionKey = "_version" -// WriteVersion writes the schema version to a bucket. -// Only writes if the key doesn't exist or the stored version is older. +// WriteVersion writes the schema version to a bucket, only if missing or older. // Returns an error if the stored value is corrupt (non-integer). func WriteVersion(kv nats.KeyValue, version int) error { entry, err := kv.Get(VersionKey) @@ -32,9 +31,8 @@ func WriteVersion(kv nats.KeyValue, version int) error { return err } -// ReadVersion reads the schema version from a bucket. -// Returns 0 if the key is not set. Returns an error on network failures -// or corrupt values so callers can distinguish "not set" from "unreachable". +// ReadVersion reads the schema version from a bucket; returns 0 if not set. +// Errors distinguish network failures and corrupt values from "not set". func ReadVersion(kv nats.KeyValue) (int, error) { entry, err := kv.Get(VersionKey) if err != nil { @@ -50,7 +48,7 @@ func ReadVersion(kv nats.KeyValue) (int, error) { return v, nil } -// GlobalAccountID is the root/system account. Pre-Phase4 resources are owned by this account. +// GlobalAccountID is the root/system account ID. const GlobalAccountID = "000000000000" // AccountKey returns a KV key scoped to an account: "{accountID}.{resourceID}". @@ -86,9 +84,7 @@ func GetOrCreateKVBucket(js nats.JetStreamContext, bucketName string, history in return kv, nil } -// DeleteKVBucketIfExists deletes a NATS KV bucket by name. Returns nil if the -// bucket does not exist (idempotent). Used by daemon startup to tombstone -// obsolete buckets after a subsystem is removed in a schema migration. +// DeleteKVBucketIfExists deletes a NATS KV bucket by name, returning nil if it does not exist. func DeleteKVBucketIfExists(js nats.JetStreamContext, bucketName string) error { err := js.DeleteKeyValue(bucketName) if err == nil { diff --git a/spinifex/utils/account_test.go b/spinifex/utils/account_test.go index 5c848d098..5eb396031 100644 --- a/spinifex/utils/account_test.go +++ b/spinifex/utils/account_test.go @@ -67,17 +67,8 @@ func startJSNATSServer(t *testing.T) (*nats.Conn, nats.JetStreamContext) { return nc, js } -// TestVersionStateMachine walks WriteVersion/ReadVersion through every state -// transition the migration helper guarantees: -// - unset → ReadVersion returns 0 with no error -// - first write persists the value -// - same version is a no-op (idempotent) -// - higher version replaces the stored value (upgrade) -// - lower version is a no-op (no downgrade) -// -// One bucket is reused across steps so each Write is exercised against the -// state the prior Write left behind — that's the only way to catch a -// regression that, say, made WriteVersion unconditionally overwrite. +// TestVersionStateMachine covers unset→0, first write, idempotent same write, upgrade, and no-downgrade. +// One bucket is reused so each step runs against the prior state — the only way to catch unconditional-overwrite regressions. func TestVersionStateMachine(t *testing.T) { _, js := startJSNATSServer(t) kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "test-version-fsm"}) diff --git a/spinifex/utils/encryption.go b/spinifex/utils/encryption.go index c824c4f1a..5c53076e9 100644 --- a/spinifex/utils/encryption.go +++ b/spinifex/utils/encryption.go @@ -11,18 +11,14 @@ import ( "github.com/mulgadc/predastore/pkg/masterkey" ) -// viperblockKeyCache memoises masterkey.LoadShared by path so the AWS gateway -// handlers (volume / instance / image) do not re-stat and re-parse the same -// 32-byte file on every VB.New. Process-wide; the underlying *Key holds an -// AEAD which is safe for concurrent use per the crypto/cipher contract. +// viperblockKeyCache memoises masterkey.LoadShared by path. The *Key holds an AEAD safe for concurrent use. var ( viperblockKeyCacheMu sync.Mutex viperblockKeyCache = map[string]*masterkey.Key{} ) -// LoadViperblockMasterKey returns the cached *masterkey.Key for the given -// path, loading it via masterkey.LoadShared on first use. An empty path -// returns (nil, nil) — encryption is treated as disabled. +// LoadViperblockMasterKey returns the cached *masterkey.Key for path, loading on first use. +// An empty path returns (nil, nil), meaning encryption is disabled. func LoadViperblockMasterKey(path string) (*masterkey.Key, error) { if path == "" { return nil, nil diff --git a/spinifex/utils/images.go b/spinifex/utils/images.go index e103f7db5..0a1dcfb3a 100644 --- a/spinifex/utils/images.go +++ b/spinifex/utils/images.go @@ -26,8 +26,6 @@ import ( "time" ) -// Helper functions for OS images - var ErrQCOWDetected = errors.New("qcow format detected") const sumsFileMaxSize = 1 * 1024 * 1024 // 1 MB, matches formation.go JoinRequest cap. @@ -39,26 +37,15 @@ var ( ErrChecksumFetchFailed = errors.New("checksum fetch failed") ) -// checksumExtraRootCAs is a test-only hook so unit tests can stand up an -// httptest.NewTLSServer (which uses a self-signed cert) without relaxing the -// HTTPS enforcement. Production builds leave this nil and the verification -// client uses the system trust store. +// checksumExtraRootCAs is a test-only hook for httptest TLS servers; nil in production. var checksumExtraRootCAs *x509.CertPool -// checksumFetchTimeout is a var (not const) so tests can shrink it to exercise -// the context-deadline path without waiting 30s. +// checksumFetchTimeout is a var so tests can shrink it to exercise the deadline path. var checksumFetchTimeout = 30 * time.Second -// VerifyImageChecksum fetches the sums file at checksumURL, locates the entry -// for imagePath's basename, hashes imagePath with the algorithm named by -// checksumType ("sha256" or "sha512"), and compares digests. -// -// Fails closed: every error path returns without accepting the image. A -// non-HTTPS scheme, non-2xx status, transport error, response over 1 MB, or -// cross-scheme redirect all wrap ErrChecksumFetchFailed. Unknown algorithm -// wraps ErrUnsupportedChecksumType. Missing filename entry wraps -// ErrChecksumNotFound. Digest mismatch wraps ErrChecksumMismatch and the -// wrapped error's %v includes expected and actual hex. +// VerifyImageChecksum fetches the sums file at checksumURL, finds the entry for imagePath's basename, +// hashes the file with checksumType ("sha256" or "sha512"), and compares digests. +// Fails closed: non-HTTPS, non-2xx, oversized response, or digest mismatch all return a wrapped error. func VerifyImageChecksum(imagePath, checksumURL, checksumType string) error { hasher, err := newHasher(checksumType) if err != nil { @@ -71,8 +58,7 @@ func VerifyImageChecksum(imagePath, checksumURL, checksumType string) error { return err } - // Distinct error for a catalog/sums-file algorithm mismatch — without this - // it'd surface as "tampering" via ConstantTimeCompare's length-0 return. + // Catch algorithm mismatch before ConstantTimeCompare; a length mismatch would look like tampering. if len(expected) != hasher.Size()*2 { return fmt.Errorf("%w: digest length %d from sums file does not match %s output length %d", ErrChecksumFetchFailed, len(expected), checksumType, hasher.Size()*2) @@ -108,10 +94,8 @@ func newHasher(checksumType string) (hash.Hash, error) { } } -// fetchExpectedDigest downloads the sums file and returns the hex digest for -// the given filename. Enforces HTTPS on initial URL and every redirect hop, -// caps redirects at 10, and truncates response at sumsFileMaxSize+1 so the -// size check is unambiguous. +// fetchExpectedDigest downloads the sums file and returns the hex digest for filename. +// Enforces HTTPS on every redirect hop, caps at 10 redirects, and limits response to sumsFileMaxSize. func fetchExpectedDigest(checksumURL, filename string) (string, error) { parsed, err := url.Parse(checksumURL) if err != nil { @@ -182,21 +166,9 @@ func fetchExpectedDigest(checksumURL, filename string) (string, error) { return digest, nil } -// parseSumsFile scans a sums file and returns the hex digest matching filename. -// Filename match is case-sensitive: upstream Debian/Ubuntu/Alpine sums are -// consistently lowercase and a divergence is a real signal. -// -// Accepts four on-the-wire shapes: " " (coreutils text mode), -// " *" (coreutils binary mode), " () = " (BSD -// style, used by Rocky/RHEL/Alma/Fedora/CentOS Stream CHECKSUM files), and a -// bare single-token "" line (single-file .sha512 from Alpine, which does -// not carry a filename). -// -// GPG cleartext-signed armor is tolerated implicitly: the BEGIN/END markers -// and "Hash:" header don't match any shape's structural guards, and signature -// body lines are individually single tokens but always appear in pairs -// (base64 + the =CRC trailer at minimum), so bareCount stays ≥ 2 and the -// bare-digest fallback never fires on a signed file. +// parseSumsFile scans a sums file and returns the hex digest matching filename (case-sensitive). +// Accepts GNU coreutils text/binary, BSD-style, and bare single-token (Alpine .sha512) formats. +// GPG cleartext-signed armor is tolerated: its body lines always appear in pairs, keeping bareCount ≥ 2. func parseSumsFile(body []byte, filename string) (string, error) { var bareDigest string bareCount := 0 @@ -218,10 +190,7 @@ func parseSumsFile(body []byte, filename string) (string, error) { return fields[0], nil } case 4: - // BSD-style: " () = ". The "=" guard also rejects - // armor lines like "-----BEGIN PGP SIGNED MESSAGE-----" that happen - // to tokenise into 4 fields. Filenames have no spaces in upstream - // usage (Rocky/Alma/Fedora pin dated builds). + // BSD-style: " () = ". The "=" guard rejects PGP armor lines that happen to tokenise to 4 fields. if fields[2] != "=" || !strings.HasPrefix(fields[1], "(") || !strings.HasSuffix(fields[1], ")") { continue } @@ -230,17 +199,14 @@ func parseSumsFile(body []byte, filename string) (string, error) { return fields[3], nil } default: - // Unrecognised shape. Skip rather than reject outright — signed sums - // files occasionally have trailing commentary we want to tolerate. + // Skip unrecognised shapes; signed sums files may have trailing commentary. } } if err := scanner.Err(); err != nil { return "", fmt.Errorf("%w: scan sums file: %v", ErrChecksumFetchFailed, err) } - // Alpine single-file .sha512 fallback: exactly one bare-digest line in the - // whole file. Any more and we refuse, because we can't tell which line is - // the right one without a filename. + // Alpine single-file fallback: accept only if exactly one bare-digest line in the file. if bareCount == 1 { return bareDigest, nil } @@ -248,10 +214,8 @@ func parseSumsFile(body []byte, filename string) (string, error) { return "", fmt.Errorf("%w: %s", ErrChecksumNotFound, filename) } -// hashImageFile streams the image through hasher and returns the digest as -// lowercase hex. Zero-byte files are rejected outright so a truncated -// download surfaces as an explicit error rather than an opaque "checksum -// mismatch". +// hashImageFile streams imagePath through hasher and returns lowercase hex. +// Rejects zero-byte files so a truncated download surfaces as an explicit error. func hashImageFile(imagePath string, hasher hash.Hash) (string, error) { f, err := os.Open(imagePath) if err != nil { @@ -285,15 +249,11 @@ type Images struct { Checksum string `json:"checksum"` ChecksumType string `json:"checksum_type"` BootMode string `json:"boot_mode"` - // Tags are copied onto the imported AMI's AMIMetadata.Tags so the UI - // can filter/classify the image. Used to mark system-owned AMIs - // (e.g. the LB/HAProxy image) via spinifex:managed-by. + // Tags are copied onto the imported AMI's metadata for UI filtering (e.g. spinifex:managed-by). Tags map[string]string `json:"tags,omitempty"` } -// distroFamilies maps a distro name to its cloud-init family. Family selects -// the per-distro branches in the cloud-init template (sudoers group, -// NetworkManager keyfile vs netplan). Keep keys lowercase; callers normalise. +// distroFamilies maps a distro name to its cloud-init family. Keys are lowercase. var distroFamilies = map[string]string{ "debian": "debian", "ubuntu": "debian", @@ -305,11 +265,8 @@ var distroFamilies = map[string]string{ "alpine": "alpine", } -// DistroFamily returns the cloud-init family for distro. Unknown or empty -// distro maps to "debian" (today's default rendering) and logs a warning so -// operators using --file imports for custom appliances aren't broken by a -// missing --distro flag; explicit RHEL-family rendering requires -// --distro rocky|rhel|alma|fedora|centos. +// DistroFamily returns the cloud-init family for distro. +// Unknown or empty distro defaults to "debian" with a warning; RHEL family requires an explicit distro flag. func DistroFamily(distro string) string { d := strings.ToLower(strings.TrimSpace(distro)) if family, ok := distroFamilies[d]; ok { @@ -463,12 +420,7 @@ var AvailableImages = map[string]Images{ Tags: map[string]string{"gpu-vendor": "amd"}, }, - // EKS node system AMI. The control-plane + node-group launchers resolve the - // boot AMI by the spinifex:managed-by=eks tag (not by name), so importing - // this catalog entry — which copies the tag onto the AMI — is enough for - // CreateCluster/CreateNodegroup to find it. Built from - // scripts/images/eks-node/manifest.conf (Alpine bios base); published to R2 - // with scripts/publish-system-image.sh. + // EKS node system AMI. Resolved by spinifex:managed-by=eks tag — importing this entry is sufficient for CreateCluster/CreateNodegroup. "spinifex-eks-node": { Name: "spinifex-eks-node", Description: "Mulga EKS node image — Alpine 3.21.7 + K3s v1.32.5 + eks-token-webhook (server|agent role selected at first boot)", @@ -485,34 +437,23 @@ var AvailableImages = map[string]Images{ }, } -// AMI / image extraction utils func ExtractDiskImageFromFile(imagepath string, tmpdir string) (diskimage string, err error) { var args []string var execCmd string - // Confirm file exists _, err = os.Stat(imagepath) - if err != nil { return diskimage, err } - // Extract the filepath imagefile := filepath.Base(imagepath) - // Already in raw/image formt, confirm the file contains a valid disk image/MBR if strings.HasSuffix(imagefile, ".raw") || strings.HasSuffix(imagefile, ".img") || strings.HasSuffix(imagefile, ".qcow2") || strings.HasSuffix(imagefile, ".qcow") { path, err := filepath.Abs(imagepath) - if err != nil { return path, err } - - // Validate the specified filename is indeed a disk image / MBR err = validateDiskImagePath(path) - - // Check error response - if errors.Is(err, ErrQCOWDetected) { extractpath := fmt.Sprintf("%s/%s", tmpdir, imagefile) extractpath = strings.TrimSuffix(extractpath, ".qcow2") + ".raw" diff --git a/spinifex/utils/nats.go b/spinifex/utils/nats.go index 7990b87de..458d487d2 100644 --- a/spinifex/utils/nats.go +++ b/spinifex/utils/nats.go @@ -16,34 +16,21 @@ import ( "github.com/nats-io/nats.go" ) -// Sentinel errors for TLS certificate configuration failures in ConnectNATS. -// Callers can use errors.Is to detect permanent TLS errors without relying -// on error message text. +// Sentinel errors for TLS configuration failures in ConnectNATS. var ( ErrCACertRead = errors.New("failed to read CA cert") ErrCACertParse = errors.New("failed to parse CA cert") ) -// ErrClusterUnavailable is returned by NATS request helpers when the underlying -// connection is not currently connected. Callers (gateway, scatter-gather -// fan-out) use this to fail fast instead of waiting for per-call timeouts. +// ErrClusterUnavailable is returned when the NATS connection is not currently connected. var ErrClusterUnavailable = errors.New("cluster unavailable: NATS disconnected") -// natsRetryEscalateAttempt is the attempt count past which ConnectNATSWithRetry -// promotes the per-attempt log line from slog.Warn to slog.Error (rate-limited -// to once a minute). With the 60s backoff cap, ~30 attempts corresponds to -// ~30 minutes of continuous disconnection — long enough to suspect a config -// error rather than a routine NATS restart. +// natsRetryEscalateAttempt is the threshold at which retry logs escalate from Warn to Error (rate-limited to 1/min). +// ~30 attempts at the 60 s backoff cap ≈ ~30 min disconnected, suggesting config error not transient restart. const natsRetryEscalateAttempt = 30 -// ConnectNATS establishes a connection to a NATS server with standard reconnect -// handling and logging. If token is non-empty, token authentication is used. -// If caCertPath is non-empty, TLS is enabled using the given CA certificate. -// -// Optional callback hooks (WithDisconnectHandler / WithReconnectHandler) wrap -// the default log lines so callers can react to connectivity changes (e.g. the -// daemon flips its cluster/standalone mode field) without losing the existing -// disconnect/reconnect log output. +// ConnectNATS connects to a NATS server with reconnect handling. Supports token auth and TLS via caCertPath. +// WithDisconnectHandler/WithReconnectHandler wrap the default log lines for callers that need to react to state changes. func ConnectNATS(host, token, caCertPath string, opts ...RetryOption) (*nats.Conn, error) { cfg := retryConfig{} for _, o := range opts { @@ -97,10 +84,6 @@ func ConnectNATS(host, token, caCertPath string, opts ...RetryOption) (*nats.Con } // retryConfig holds parameters for ConnectNATS / ConnectNATSWithRetry. -// -// retry-related fields tune the outer reconnect-with-backoff loop; -// callback fields wrap nats.go's connection-state handlers so callers can -// react to disconnects/reconnects without losing the default log lines. type retryConfig struct { maxWait time.Duration retryDelay time.Duration @@ -119,7 +102,7 @@ func WithMaxWait(d time.Duration) RetryOption { return func(c *retryConfig) { c.maxWait = d } } -// WithRetryDelay sets the initial delay between retries (doubles each attempt, capped at 10s). +// WithRetryDelay sets the initial retry delay (exponentially doubled, capped at 10 s). func WithRetryDelay(d time.Duration) RetryOption { return func(c *retryConfig) { c.retryDelay = d } } @@ -130,22 +113,18 @@ func WithMaxRetryDelay(d time.Duration) RetryOption { return func(c *retryConfig) { c.maxRetryDelay = d } } -// WithDisconnectHandler registers an optional callback invoked after the -// default disconnect log line. Callback runs on a NATS client goroutine; keep -// it non-blocking (atomic stores, channel sends, goroutine spawns). +// WithDisconnectHandler registers a callback invoked after the default disconnect log line. +// Runs on a NATS goroutine; keep it non-blocking. func WithDisconnectHandler(fn func(*nats.Conn, error)) RetryOption { return func(c *retryConfig) { c.onDisconnect = fn } } -// WithReconnectHandler registers an optional callback invoked after the -// default reconnect log line. Same goroutine constraints as WithDisconnectHandler. +// WithReconnectHandler registers a callback invoked after the default reconnect log line. Same goroutine constraints as WithDisconnectHandler. func WithReconnectHandler(fn func(*nats.Conn)) RetryOption { return func(c *retryConfig) { c.onReconnect = fn } } -// WithAttemptErrHandler registers an optional callback invoked after each -// failed connect attempt during ConnectNATSWithRetry's outer loop. Used by the -// daemon to surface initial-connect retries as a counter on /local/status. +// WithAttemptErrHandler registers a callback invoked after each failed attempt in ConnectNATSWithRetry. func WithAttemptErrHandler(fn func(err error, attempt int)) RetryOption { return func(c *retryConfig) { c.onAttemptErr = fn } } @@ -156,11 +135,8 @@ func WithContext(ctx context.Context) RetryOption { return func(c *retryConfig) { c.ctx = ctx } } -// ConnectNATSWithRetry calls ConnectNATS in a retry loop with exponential -// backoff. It retries for up to 5 minutes (default) before giving up; pass -// WithMaxWait(0) to retry indefinitely (cancel via WithContext). TLS -// configuration errors (ErrCACertRead, ErrCACertParse) are permanent and -// cause an immediate return without retrying. +// ConnectNATSWithRetry calls ConnectNATS with exponential backoff, retrying up to 5 min by default. +// Pass WithMaxWait(0) to retry indefinitely (cancel via WithContext). TLS errors return immediately. func ConnectNATSWithRetry(host, token, caCertPath string, opts ...RetryOption) (*nats.Conn, error) { cfg := retryConfig{ maxWait: 5 * time.Minute, @@ -201,10 +177,7 @@ func ConnectNATSWithRetry(host, token, caCertPath string, opts ...RetryOption) ( return nil, fmt.Errorf("NATS connect failed after %s: %w", elapsed.Round(time.Second), err) } - // Past the retry-escalation threshold (~30 attempts ≈ ~30 min disconnected - // at the 60s backoff cap) the warn-on-every-retry pattern can hide a - // stuck NATS config in routine warning noise. Escalate to slog.Error, - // rate-limited to once a minute, so the operator's error log surfaces it. + // Past the escalation threshold, promote from Warn to Error (rate-limited to 1/min). if attempt > natsRetryEscalateAttempt { if lastEscalatedLog.IsZero() || time.Since(lastEscalatedLog) >= time.Minute { slog.Error("NATS still disconnected", "error", err, "disconnected_for", elapsed.Round(time.Second), "attempt", attempt) @@ -231,9 +204,7 @@ func ConnectNATSWithRetry(host, token, caCertPath string, opts ...RetryOption) ( // AWS account ID from the gateway to daemon handlers. const AccountIDHeader = "X-Account-ID" -// PrincipalARNHeader carries the caller's resolved IAM principal ARN from the -// gateway to daemon handlers that attribute an action to its caller (e.g. the -// EKS bootstrap-creator-admin AccessEntry). +// PrincipalARNHeader carries the caller's resolved IAM principal ARN from gateway to daemon handlers. const PrincipalARNHeader = "X-Principal-ARN" // NATSHeader is an extra request header passed to NATSRequest beyond the @@ -250,10 +221,7 @@ func PrincipalARNFromMsg(msg *nats.Msg) string { } // NATSRequest performs a NATS request-response with JSON marshaling. -// It marshals the input, sends to the given subject with the X-Account-ID -// header (plus any extra headers), validates the response for error payloads, -// and unmarshals the successful response into Out. Handlers can ignore the -// account ID if the operation is unscoped (e.g. DescribeInstanceTypes). +// Sends with X-Account-ID (plus any extra headers) and unmarshals the successful response into Out. func NATSRequest[Out any](conn *nats.Conn, subject string, input any, timeout time.Duration, accountID string, headers ...NATSHeader) (*Out, error) { if conn == nil || !conn.IsConnected() { return nil, ErrClusterUnavailable @@ -294,9 +262,7 @@ func NATSRequest[Out any](conn *nats.Conn, subject string, input any, timeout ti return &output, nil } -// ServeNATSRequest is the responder-side counterpart to NATSRequest: it unmarshals -// the request payload into *I, invokes fn, and replies with the JSON result or an -// awserrors envelope. The payload carries its own context; no X-Account-ID header. +// ServeNATSRequest unmarshals the request into *I, invokes fn, and replies with JSON or an awserrors envelope. func ServeNATSRequest[I any, O any](msg *nats.Msg, fn func(*I) (*O, error)) { input := new(I) if errResp := UnmarshalJsonPayload(input, msg.Data); errResp != nil { @@ -323,22 +289,15 @@ func respondNATS(msg *nats.Msg, data []byte) { } const ( - // maxScatterGatherResponseSize is the maximum allowed size for a single - // scatter-gather response payload (10 MB). Responses exceeding this are - // skipped to prevent OOM from buggy nodes. + // maxScatterGatherResponseSize caps a single scatter-gather response at 10 MB to prevent OOM. maxScatterGatherResponseSize = 10 * 1024 * 1024 - - // maxScatterGatherUnboundedResponses is the hard cap on responses collected - // when expectedNodes is 0 (unbounded fan-out). + // maxScatterGatherUnboundedResponses caps responses when expectedNodes is 0. maxScatterGatherUnboundedResponses = 256 ) -// NATSScatterGather publishes a fan-out request and collects responses from -// multiple nodes. It returns the first successful (non-error) response. Error -// payloads from individual nodes are skipped. If all responses are errors, the -// last error is returned. If no responses arrive before the deadline, a timeout -// error is returned. When expectedNodes > 0, collection exits early once that -// many responses have been received. +// NATSScatterGather fans out a request and returns the first successful response. +// Skips per-node error payloads; returns the last error if all fail or a timeout if none arrive. +// Exits early when expectedNodes > 0 responses have been received. func NATSScatterGather[Out any](conn *nats.Conn, subject string, input any, timeout time.Duration, expectedNodes int, accountID string) (*Out, error) { if conn == nil || !conn.IsConnected() { return nil, ErrClusterUnavailable @@ -421,8 +380,7 @@ func NATSScatterGather[Out any](conn *nats.Conn, subject string, input any, time return nil, fmt.Errorf("scatter-gather timeout: no responses received for %s", subject) } -// PublishEvent marshals event as JSON and publishes it to the given NATS topic. -// Errors are logged but not returned (fire-and-forget). A nil connection is a no-op. +// PublishEvent marshals event as JSON and publishes to topic (fire-and-forget; nil conn is a no-op). func PublishEvent(nc *nats.Conn, topic string, event any) { if nc == nil { return @@ -446,15 +404,8 @@ type natEvent struct { MAC string `json:"mac"` } -// AddNAT requests vpcd commit the OVN dnat_and_snat rule for (externalIP → -// logicalIP) via NATS request-reply and waits up to 10 s for the ack. A -// non-nil return means the rule was either NOT committed (NACK / no -// responders) OR may have been committed after the timeout fired (vpcd's -// waitForFlowsHV barrier is unbounded). Callers MUST roll back any state -// that assumes the public IP is routable AND publish a best-effort -// vpc.delete-nat for the same (vpcID, externalIP, logicalIP, portName) to -// neutralise a half-committed rule — there is no reconciler that repairs -// a black-holed external IP after the fact. +// AddNAT requests vpcd commit the OVN dnat_and_snat rule via NATS request-reply (10 s timeout). +// A non-nil return means the rule may not be committed; callers must roll back and publish vpc.delete-nat. func AddNAT(nc *nats.Conn, vpcID, externalIP, logicalIP, portName, mac string) error { return RequestEvent(nc, "vpc.add-nat", natEvent{ VpcId: vpcID, ExternalIP: externalIP, LogicalIP: logicalIP, @@ -462,12 +413,8 @@ func AddNAT(nc *nats.Conn, vpcID, externalIP, logicalIP, portName, mac string) e }, 10*time.Second) } -// PublishNATEvent sends a NAT lifecycle event (vpc.add-nat or vpc.delete-nat) -// to NATS. vpc.add-nat uses request-reply to commit the OVN NAT rule before -// returning (prevents ARP propagation races); vpc.delete-nat is fire-and-forget. -// -// Callers that MUST react to an add-nat failure (e.g. roll back a launch) -// should use AddNAT instead — this helper only logs the failure. +// PublishNATEvent sends a NAT lifecycle event. vpc.add-nat uses request-reply (prevents ARP races); +// vpc.delete-nat is fire-and-forget. Use AddNAT directly when failure must trigger a rollback. func PublishNATEvent(nc *nats.Conn, topic, vpcID, externalIP, logicalIP, portName, mac string) { evt := natEvent{ VpcId: vpcID, ExternalIP: externalIP, LogicalIP: logicalIP, @@ -484,10 +431,7 @@ func PublishNATEvent(nc *nats.Conn, topic, vpcID, externalIP, logicalIP, portNam PublishEvent(nc, topic, evt) } -// RequestEvent marshals event as JSON and sends a NATS request, waiting for a -// response. This ensures the subscriber has processed the event before the -// caller continues. Returns an error if the request times out or the responder -// reports an error. +// RequestEvent marshals event as JSON and sends a synchronous NATS request, blocking until the subscriber acks. func RequestEvent(nc *nats.Conn, topic string, event any, timeout time.Duration) error { if nc == nil { return fmt.Errorf("%s: nats connection not initialized", topic) @@ -514,8 +458,7 @@ func RequestEvent(nc *nats.Conn, topic string, event any, timeout time.Duration) return nil } -// AccountIDFromMsg extracts the caller's account ID from a NATS message header. -// Returns the account ID, or empty string if the header is not set. +// AccountIDFromMsg extracts the caller's account ID from a NATS message header, or "" if absent. func AccountIDFromMsg(msg *nats.Msg) string { if msg == nil || msg.Header == nil { return "" diff --git a/spinifex/utils/nats_test.go b/spinifex/utils/nats_test.go index 5aff70481..6b36ace0a 100644 --- a/spinifex/utils/nats_test.go +++ b/spinifex/utils/nats_test.go @@ -772,11 +772,8 @@ func startTestNATSOnPort(t *testing.T, port int) *server.Server { return ns } -// TestConnectNATSWithRetry_LogEscalatesPastThreshold drives ConnectNATSWithRetry -// against a closed port with a tight retry loop until the attempt count exceeds -// natsRetryEscalateAttempt, then asserts that at least one escalated slog.Error -// line was emitted with disconnected_for context (rate-limited to once per -// minute) while earlier attempts stayed at slog.Warn. +// TestConnectNATSWithRetry_LogEscalatesPastThreshold verifies that log lines escalate from Warn to Error +// once attempt count exceeds natsRetryEscalateAttempt, while earlier attempts stay at Warn. func TestConnectNATSWithRetry_LogEscalatesPastThreshold(t *testing.T) { var buf bytes.Buffer prev := slog.Default() @@ -847,11 +844,8 @@ func TestAddNAT_Success(t *testing.T) { assert.Equal(t, "02:00:00:00:00:01", got.MAC) } -// TestAddNAT_NACK is the core regression for the silent-corruption bug: when -// vpcd reports failure, callers MUST see a non-nil error so they can roll -// back the IPAM allocation and the ENI public IP record. Before the fix the -// helper logged a warning and returned nothing, leaving an unreachable -// public IP surfaced to the AWS client. +// TestAddNAT_NACK is the regression for the silent-corruption bug: a vpcd failure must return a non-nil error +// so callers can roll back IPAM and ENI public IP state (previously the helper only logged a warning). func TestAddNAT_NACK(t *testing.T) { ns := startTestNATSServer(t) nc, err := nats.Connect(ns.ClientURL()) diff --git a/spinifex/utils/nbd.go b/spinifex/utils/nbd.go index 075a61577..859ecb112 100644 --- a/spinifex/utils/nbd.go +++ b/spinifex/utils/nbd.go @@ -26,9 +26,7 @@ func GenerateSocketFile(name string) (string, error) { return filepath.Join(pidPath, fmt.Sprintf("%s.sock", name)), nil } -// GenerateUniqueSocketFile generates a unique socket file path with unix nano timestamp. -// Format: nbd-{volname}-{unixnano}.sock -// This ensures each mount operation gets a unique socket path. +// GenerateUniqueSocketFile generates a unique socket path with format nbd-{volname}-{unixnano}.sock. func GenerateUniqueSocketFile(volname string) (string, error) { if volname == "" { return "", errors.New("volume name is required") @@ -44,9 +42,7 @@ func GenerateUniqueSocketFile(volname string) (string, error) { return filepath.Join(dir, filename), nil } -// NBDSocketDir returns the directory for NBD Unix sockets. -// Under systemd installs, this is /run/spinifex/nbd/. -// In dev mode (no /run/spinifex/nbd/), falls back to pidPath(). +// NBDSocketDir returns the NBD socket directory (/run/spinifex/nbd under systemd; pidPath() as fallback). func NBDSocketDir() string { const systemdNBDDir = "/run/spinifex/nbd" if dirExists(systemdNBDDir) { @@ -55,27 +51,23 @@ func NBDSocketDir() string { return pidPath() } -// IsSocketURI returns true if the NBD URI is a Unix socket path. -// Socket URIs end with ".sock" or contain "unix:". +// IsSocketURI reports whether the NBD URI refers to a Unix socket (ends with ".sock" or contains "unix:"). func IsSocketURI(nbdURI string) bool { return strings.HasSuffix(nbdURI, ".sock") || strings.Contains(nbdURI, "unix:") } -// FormatNBDSocketURI formats a socket path as an NBD URI for QEMU. -// Returns format: nbd:unix:/path/to/socket.sock +// FormatNBDSocketURI formats a socket path as an NBD URI (nbd:unix:/path/to/socket.sock). func FormatNBDSocketURI(socketPath string) string { return fmt.Sprintf("nbd:unix:%s", socketPath) } -// FormatNBDTCPURI formats a host:port as an NBD URI for QEMU. -// Returns format: nbd://host:port +// FormatNBDTCPURI formats a host:port as an NBD TCP URI (nbd://host:port). func FormatNBDTCPURI(host string, port int) string { return "nbd://" + net.JoinHostPort(host, strconv.Itoa(port)) } -// WaitForNBDReady polls until the NBD endpoint at uri is reachable or the -// timeout expires. Unix sockets use existence-on-disk to avoid consuming -// the listener's accept queue before QEMU dials; TCP uses a dial-and-close. +// WaitForNBDReady polls until the NBD endpoint is reachable or timeout expires. +// Unix: checks existence on disk (avoids consuming the accept queue); TCP: dial-and-close. func WaitForNBDReady(uri string, timeout time.Duration) error { serverType, path, host, port, err := ParseNBDURI(uri) if err != nil { @@ -106,10 +98,7 @@ func waitForTCPListener(addr string, timeout time.Duration) error { } } -// ParseNBDURI parses an NBD URI into its components for QMP blockdev-add. -// Supported formats: -// - "nbd:unix:/path/to/socket.sock" → serverType="unix", path="/path/to/socket.sock" -// - "nbd://host:port" → serverType="inet", host="host", port= +// ParseNBDURI parses an NBD URI into components: nbd:unix:/path → ("unix", path); nbd://host:port → ("inet", host, port). func ParseNBDURI(nbdURI string) (serverType, path, host string, port int, err error) { if after, ok := strings.CutPrefix(nbdURI, "nbd:unix:"); ok { path = after diff --git a/spinifex/utils/pidfile.go b/spinifex/utils/pidfile.go index e4c253ac0..29dfd4a6b 100644 --- a/spinifex/utils/pidfile.go +++ b/spinifex/utils/pidfile.go @@ -20,7 +20,6 @@ func ReadPidFile(name string) (int, error) { return 0, err } - // Strip whitespace and /r or /n pidFile = bytes.TrimSpace(pidFile) return strconv.Atoi(string(pidFile)) @@ -41,7 +40,6 @@ func GeneratePidFile(name string) (string, error) { } func WritePidFile(name string, pid int) error { - // Write PID to file, check XDG, otherwise user home directory ~/spinifex/ pidFilename, err := GeneratePidFile(name) if err != nil { @@ -63,10 +61,8 @@ func WritePidFile(name string, pid int) error { return nil } -// WritePidFileTo writes a PID file to a specific directory. If dir is empty, -// falls back to the default pidPath(). Used by services that know their own -// data directory (e.g. predastore's BasePath) to avoid PID file collisions -// when multiple nodes run on the same host. +// WritePidFileTo writes a PID file to dir (or pidPath() if empty). +// Per-service data directories prevent PID file collisions on multi-node hosts. func WritePidFileTo(dir string, name string, pid int) error { if dir == "" { return WritePidFile(name, pid) @@ -126,10 +122,7 @@ func ServiceStatus(dir, name string) (string, error) { return fmt.Sprintf("running (pid: %d)", pid), nil } -// StopProcessAt stops a process using a PID file in a specific directory. -// If dir is empty, falls back to the default pidPath(). The PID file is -// always removed, even if the process is already dead, to prevent stale -// PID files from accumulating across restarts. +// StopProcessAt stops a process using its PID file. Always removes the PID file, even if the process is already dead. func StopProcessAt(dir string, name string) error { pid, err := ReadPidFileFrom(dir, name) if err != nil { @@ -138,8 +131,6 @@ func StopProcessAt(dir string, name string) error { killErr := KillProcess(pid) - // Always remove the PID file to avoid stale entries. If the process is - // already dead, the PID file is stale and must be cleaned up. if removeErr := RemovePidFileAt(dir, name); removeErr != nil && killErr == nil { return removeErr } @@ -173,10 +164,8 @@ func pidPath() string { return os.TempDir() } -// WaitForProcessExit polls until the given PID is no longer alive or the -// timeout expires. Unlike WaitForPidFileRemoval, this checks the process -// itself via kill(pid, 0), so it works after SIGKILL where the target -// cannot clean up its own PID file. +// WaitForProcessExit polls until the PID is no longer alive or timeout expires. +// Uses kill(pid,0) — works after SIGKILL where the process can't clean up its PID file. func WaitForProcessExit(pid int, timeout time.Duration) error { timer := time.NewTimer(timeout) defer timer.Stop() @@ -200,12 +189,7 @@ func WaitForProcessExit(pid int, timeout time.Duration) error { } // WaitForPidFile polls until QEMU writes its pidfile or the timeout expires. -// Direct-boot microvms need ~50ms in the happy path, but under post-reboot -// recovery load (multiple VMs launching, nbdkit binding sockets, fwcfg blobs -// being written) the kernel may not schedule QEMU's pidfile write within a -// single settle window. A short blocking poll keeps the fast-start latency -// intact while avoiding premature launch-failure cascades that tear down the -// tap before QEMU finishes opening it. +// A short poll avoids premature failure cascades under recovery load while keeping fast-start latency. func WaitForPidFile(instanceID string, timeout time.Duration) (int, error) { deadline := time.Now().Add(timeout) for { @@ -220,11 +204,8 @@ func WaitForPidFile(instanceID string, timeout time.Duration) (int, error) { } } -// WaitForUnixSocket polls until a Unix socket exists at path or the timeout -// expires. QEMU binds its QMP socket after writing the pidfile, so callers -// that gate on WaitForPidFile can still race the socket bind under recovery -// load. Existence is checked via os.Stat rather than a dial probe so we don't -// consume the listener's accept queue before the real client connects. +// WaitForUnixSocket polls until a Unix socket exists at path or timeout expires. +// Uses os.Stat rather than a dial probe to avoid consuming the accept queue before the real client. func WaitForUnixSocket(path string, timeout time.Duration) error { deadline := time.Now().Add(timeout) for { @@ -252,7 +233,6 @@ func WaitForPidFileRemoval(instanceID string, timeout time.Duration) error { case <-ticker.C: _, err := ReadPidFile(instanceID) if err != nil { - // PID file no longer exists return nil } } diff --git a/spinifex/utils/process.go b/spinifex/utils/process.go index 1b2c901b3..c8cb15513 100644 --- a/spinifex/utils/process.go +++ b/spinifex/utils/process.go @@ -31,7 +31,6 @@ func StopProcess(serviceName string) error { return err } - // Remove PID file err = RemovePidFile(serviceName) if err != nil { return err @@ -47,14 +46,11 @@ func KillProcess(pid int) error { return err } - // Send SIGTERM first (graceful) - err = process.Signal(syscall.SIGTERM) + err = process.Signal(syscall.SIGTERM) // graceful shutdown if err != nil { return err } - // Check process terminated - checks := 0 for { time.Sleep(1 * time.Second) @@ -67,15 +63,13 @@ func KillProcess(pid int) error { err = process.Signal(syscall.Signal(0)) if err != nil { - // Signal(0) error means the process has terminated — this is the expected outcome. - break + break // Signal(0) error means process has terminated } checks++ - // If process is still running after 120 seconds, force kill if checks > 120 { - err = process.Kill() // SIGKILL + err = process.Kill() // SIGKILL after 120 s if err != nil { return err diff --git a/spinifex/utils/sudo.go b/spinifex/utils/sudo.go index f7f0e4929..542f72349 100644 --- a/spinifex/utils/sudo.go +++ b/spinifex/utils/sudo.go @@ -5,9 +5,7 @@ import ( "os/exec" ) -// sudoCommand is the runtime implementation. Kept private so tests in other -// packages can't reassign it directly — reassign linter rule. Use -// SetSudoCommandForTest from tests. +// sudoCommand is the private runtime implementation; use SetSudoCommandForTest in tests. var sudoCommand = func(name string, args ...string) *exec.Cmd { if os.Getuid() == 0 { return exec.Command(name, args...) @@ -22,11 +20,8 @@ func SudoCommand(name string, args ...string) *exec.Cmd { return sudoCommand(name, args...) } -// SetSudoCommandForTest swaps the underlying command builder for the duration -// of a test, returning a restore func that callers pass to t.Cleanup. The -// indirection prevents cross-package reassignment of an exported var, which -// the reassign linter flags. Running the live binary against the dev host's -// OVS would mutate external_ids on the running cluster, so tests must stub. +// SetSudoCommandForTest swaps the command builder for a test, returning a restore func for t.Cleanup. +// Tests must stub this — running against real OVS would mutate the live cluster. func SetSudoCommandForTest(stub func(name string, args ...string) *exec.Cmd) (restore func()) { orig := sudoCommand sudoCommand = stub diff --git a/spinifex/utils/utils.go b/spinifex/utils/utils.go index dbc93dacc..0120d06fc 100644 --- a/spinifex/utils/utils.go +++ b/spinifex/utils/utils.go @@ -40,7 +40,6 @@ func GenerateResourceID(prefix string) string { return prefix + "-" + hex.EncodeToString(b)[:17] } -// Convert interface to XML func MarshalToXML(payload any) ([]byte, error) { var buf bytes.Buffer enc := xml.NewEncoder(&buf) @@ -58,7 +57,7 @@ func MarshalToXML(payload any) ([]byte, error) { return buf.Bytes(), nil } -// wrapWithLocation decorates payload with the requested locationName tag. +// GenerateXMLPayload wraps payload with the requested locationName tag. func GenerateXMLPayload(locationName string, payload any) any { t := reflect.StructOf([]reflect.StructField{ { @@ -73,10 +72,8 @@ func GenerateXMLPayload(locationName string, payload any) any { return v.Interface() } -// GenerateIAMXMLPayload wraps IAM output with proper *Result element structure. -// IAM responses have format: content... +// GenerateIAMXMLPayload wraps IAM output in the ... structure. func GenerateIAMXMLPayload(action string, payload any) any { - // First wrap with ActionResult (e.g., CreateUserResult) resultName := action + "Result" resultWrapper := reflect.StructOf([]reflect.StructField{ { @@ -88,7 +85,6 @@ func GenerateIAMXMLPayload(action string, payload any) any { resultV := reflect.New(resultWrapper).Elem() resultV.Field(0).Set(reflect.ValueOf(payload)) - // Then wrap with ActionResponse (e.g., CreateUserResponse) responseName := action + "Response" responseWrapper := reflect.StructOf([]reflect.StructField{ { @@ -103,12 +99,10 @@ func GenerateIAMXMLPayload(action string, payload any) any { return responseV.Interface() } -// Generate JSON Error Payload +// GenerateErrorPayload serializes an ec2.ResponseError with the given code as JSON. func GenerateErrorPayload(code string) (jsonResponse []byte) { var responseError ec2.ResponseError responseError.Code = aws.String(code) - - // Return as JSON, to simulate the NATS response jsonResponse, err := json.Marshal(responseError) if err != nil { slog.Error("GenerateErrorPayload could not marshal JSON payload", "err", err) @@ -118,7 +112,7 @@ func GenerateErrorPayload(code string) (jsonResponse []byte) { return jsonResponse } -// Validate the payload is an ec2.ResponseError +// ValidateErrorPayload decodes payload as an ec2.ResponseError and returns an error when a non-nil Code is detected. func ValidateErrorPayload(payload []byte) (responseError ec2.ResponseError, err error) { decoder := json.NewDecoder(bytes.NewReader(payload)) decoder.DisallowUnknownFields() @@ -126,22 +120,15 @@ func ValidateErrorPayload(payload []byte) (responseError ec2.ResponseError, err err = decoder.Decode(&responseError) if err == nil && responseError.Code != nil { - // Successfully decoded as ResponseError AND has a non-nil Code field - // This is a real error response return responseError, errors.New("ResponseError detected") } - - // Either failed to decode (not an error structure) or Code is nil (empty valid response) return responseError, nil } -// Unmarshal payload - +// UnmarshalJsonPayload decodes jsonData into input (already a pointer) using strict field checking. func UnmarshalJsonPayload(input any, jsonData []byte) []byte { decoder := json.NewDecoder(bytes.NewReader(jsonData)) decoder.DisallowUnknownFields() - - // input is already a pointer, don't take address again err := decoder.Decode(input) if err != nil { return GenerateErrorPayload(awserrors.ErrorValidationError) @@ -153,8 +140,6 @@ func UnmarshalJsonPayload(input any, jsonData []byte) []byte { func MarshalJsonPayload(input any, jsonData []byte) []byte { decoder := json.NewDecoder(bytes.NewReader(jsonData)) decoder.DisallowUnknownFields() - - // input is already a pointer, don't take address again err := decoder.Decode(input) if err != nil { return GenerateErrorPayload(awserrors.ErrorValidationError) @@ -163,21 +148,13 @@ func MarshalJsonPayload(input any, jsonData []byte) []byte { return nil } -// ValidateKeyPairName validates that a key pair name only contains allowed characters: -// - Uppercase A-Z -// - Lowercase a-z -// - Digits 0-9 -// - Hyphen (-) -// - Underscore (_) -// - Period (.) -// This prevents path traversal attacks and invalid characters like /etc/passwd, ../../../, etc. -// Returns ErrorInvalidKeyPairFormat if validation fails +// ValidateKeyPairName validates that a key pair name contains only [A-Za-z0-9._-]. +// Rejects empty names and returns ErrorInvalidKeyPairFormat on any invalid character. func ValidateKeyPairName(name string) error { if name == "" { return errors.New("key name cannot be empty") } - // Check each character is in the allowed set for _, char := range name { valid := (char >= 'A' && char <= 'Z') || (char >= 'a' && char <= 'z') || @@ -206,10 +183,7 @@ func CreateS3Client(cfg *config.Config) *s3.S3 { return s3.New(sess) } -// Download helper - func DownloadFileWithProgress(url string, name string, filename string, timeout time.Duration) (err error) { - // Context with optional timeout and Ctrl+C cancel ctx, cancel := context.WithCancel(context.Background()) if timeout > 0 { ctx, cancel = context.WithTimeout(ctx, timeout) @@ -244,7 +218,6 @@ func DownloadFileWithProgress(url string, name string, filename string, timeout cl := resp.ContentLength - // Known size: progress bar with total if cl > 0 { bar, _ := pterm.DefaultProgressbar. WithTitle(fmt.Sprintf("Downloading %s", name)). @@ -252,7 +225,6 @@ func DownloadFileWithProgress(url string, name string, filename string, timeout Start() reader := io.TeeReader(resp.Body, progressWriter(func(n int) { - // Update progress bar with the number of bytes read in this chunk bar.Add(n) })) @@ -263,7 +235,6 @@ func DownloadFileWithProgress(url string, name string, filename string, timeout } return err } else { - // Unknown size: spinner that shows bytes downloaded spin, _ := pterm.DefaultSpinner. WithText("Downloading (size unknown)..."). Start() @@ -306,18 +277,8 @@ func HumanBytes(b uint64) string { return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPEZY"[exp]) } -// HashMAC returns a deterministic locally-administered unicast MAC derived -// from id. Output is always "02:xx:xx:xx:xx:xx" — first octet is pinned to -// 0x02 (bit0=0 unicast, bit1=1 LAA, all other bits zero) so generated MACs -// are immediately recognisable as ours and cannot collide with registered -// vendor OUI space. The remaining 40 bits are SHA-256-derived. Birthday- -// paradox 1% collision at ~150k ids, 50% at ~1.2M. -// -// id must be globally unique across all MACs the deployment produces. No -// per-deployment salt is mixed in. Callers that share a base id between -// resource classes (e.g. an instance's dev vs mgmt NIC) must compose a -// class tag into id ("dev:"+instanceID, "mgmt:"+instanceID). Hardcoded or -// low-entropy ids will collide. +// HashMAC returns a deterministic locally-administered unicast MAC for id (SHA-256; first octet 0x02). +// id must be globally unique; callers sharing a base id across resource classes must compose a class tag (e.g. "dev:"+id). func HashMAC(id string) string { sum := sha256.Sum256([]byte(id)) b := make([]byte, 6) diff --git a/spinifex/utils/utils_test.go b/spinifex/utils/utils_test.go index e004ee9b0..851368024 100644 --- a/spinifex/utils/utils_test.go +++ b/spinifex/utils/utils_test.go @@ -1191,10 +1191,7 @@ func TestWaitForProcessExit_Timeout(t *testing.T) { } func TestHashMAC_FirstOctetPinnedTo02(t *testing.T) { - // First octet must be literal 0x02 across all inputs. Anything else - // (e.g. 0x4a, 0xfe — technically valid LAA) visually encroaches on - // vendor OUI space and is rejected by ops/CI. Second-octet entropy is - // covered by the distribution test. + // First octet must be 0x02 (not just any LAA value) to stay out of vendor OUI space. for i := range 1000 { hw, err := net.ParseMAC(HashMAC(fmt.Sprintf("id-%d", i))) require.NoError(t, err) @@ -1598,10 +1595,7 @@ func TestParseSumsFile(t *testing.T) { }) t.Run("gpg-armored body with no sums returns not found", func(t *testing.T) { - // Signature-block lines are individually single tokens. The - // bare-digest fallback only fires when bareCount == 1; PGP signatures - // always have ≥ 2 (the base64 block + the =CRC trailer), so the - // fallback stays dormant and we correctly report NotFound. + // PGP signatures always have ≥ 2 bare-token lines (base64 + =CRC), so bareCount stays ≥ 2 and NotFound is returned. body := []byte(`-----BEGIN PGP SIGNED MESSAGE----- Hash: SHA256 diff --git a/spinifex/vm/crash_recovery.go b/spinifex/vm/crash_recovery.go index 80409e565..eb1b34323 100644 --- a/spinifex/vm/crash_recovery.go +++ b/spinifex/vm/crash_recovery.go @@ -21,9 +21,8 @@ const ( restartBackoffMax = 2 * time.Minute ) -// restartAfterFunc is the scheduling seam used by MaybeRestart. Tests -// override it to capture the scheduled callback without firing a real -// timer (the backoff is multi-second). +// restartAfterFunc is a var so tests can capture the restart callback without +// firing a real timer (backoff is multi-second). var restartAfterFunc = time.AfterFunc // RestartBackoff computes the exponential backoff delay for the given @@ -39,9 +38,8 @@ func RestartBackoff(restartCount int) time.Duration { return delay } -// ClassifyCrashReason extracts a human-readable crash reason from the error -// returned by cmd.Wait(). Uses exec.ExitError + syscall.WaitStatus to -// distinguish OOM kills (SIGKILL), segfaults (SIGSEGV), etc. +// ClassifyCrashReason extracts a human-readable crash reason from cmd.Wait()'s +// error, distinguishing OOM kills, segfaults, aborts, and other signals. func ClassifyCrashReason(waitErr error) string { if waitErr == nil { return "clean-exit" @@ -77,13 +75,9 @@ func ClassifyCrashReason(waitErr error) string { return "unknown" } -// HandleCrash is the QEMU exit goroutine's reaction when cmd.Wait() returns -// during runtime (after startup was confirmed). It detects the crash reason, -// transitions the instance to error state, releases resources, unmounts -// volumes, and triggers MaybeRestart. -// -// Wired as Deps.CrashHandler so the launch goroutine in lifecycle.go can -// invoke it without importing the manager into a separate goroutine spawner. +// HandleCrash reacts to an unexpected QEMU exit after startup was confirmed: +// classifies the reason, transitions to error, releases resources, unmounts +// volumes, and calls MaybeRestart. Wired as Deps.CrashHandler. func (m *Manager) HandleCrash(instance *VM, waitErr error) { if status := m.Status(instance); status != StateRunning { slog.Debug("QEMU exited but instance not in running state, skipping crash handler", @@ -142,9 +136,8 @@ func (m *Manager) HandleCrash(instance *VM, waitErr error) { m.MaybeRestart(instance) } -// MaybeRestart checks restart policy and schedules a restart via -// time.AfterFunc when allowed. Bails out on coordinated shutdown, unknown -// instance type, exhausted restart window, or insufficient host capacity. +// MaybeRestart checks restart policy and schedules a restart via time.AfterFunc. +// No-op on shutdown, unknown instance type, exceeded restart window, or no capacity. func (m *Manager) MaybeRestart(instance *VM) { if m.deps.ShutdownSignal != nil && m.deps.ShutdownSignal() { slog.Info("Skipping restart during shutdown", "instance", instance.ID) @@ -210,10 +203,9 @@ func (m *Manager) MaybeRestart(instance *VM) { }) } -// RestartCrashedInstance re-verifies the instance is still in error state -// and relaunches it via Manager.Run. Reallocates resources before relaunch -// (HandleCrash deallocates them) and rolls back the transition + allocation -// if the relaunch fails. +// RestartCrashedInstance re-verifies the instance is in error state, reallocates +// resources (HandleCrash deallocated them), and relaunches via Manager.Run. +// Rolls back the transition and allocation if relaunch fails. func (m *Manager) RestartCrashedInstance(instance *VM) { var skipReason string var restartCount int diff --git a/spinifex/vm/device_map.go b/spinifex/vm/device_map.go index 7e7fc3036..49081443e 100644 --- a/spinifex/vm/device_map.go +++ b/spinifex/vm/device_map.go @@ -18,23 +18,12 @@ import ( var pciAddrRegexp = regexp.MustCompile(`device\[(\d+)\]`) // hotplugPortRegexp extracts the EBS hot-plug port number from a QDev path. -// Example: "/machine/peripheral/vdisk-vol-xxx/hotplug-ebs3/virtio-backend" or -// -// "/machine/peripheral/vdisk-vol-xxx/virtio-backend" (bus assigned by QEMU) -// -// The chassis number from the QEMU command line determines PCI slot order. -// ENI hot-plug ports (`hotplug-eni{N}`) are explicitly excluded — virtio-net -// devices are not surfaced through query-block, but the explicit suffix -// guards against future query callers conflating the two pools. +// Example: "/machine/peripheral/vdisk-vol-xxx/hotplug-ebs3/virtio-backend" → 3 +// ENI ports (hotplug-eni{N}) are excluded to avoid conflating the two pools. var hotplugPortRegexp = regexp.MustCompile(`hotplug-ebs(\d+)`) -// queryGuestDeviceMap uses QMP query-block to build a map from QEMU device ID -// (e.g. "os", "cloudinit", "vdisk-vol-xxx") to the guest device path -// (e.g. "/dev/vda", "/dev/vdb", "/dev/vdc"). -// -// The mapping is derived from PCI address order: virtio-blk-pci devices are -// enumerated by the guest kernel in PCI bus order, which corresponds to the -// device index in the QDev path. +// queryGuestDeviceMap builds a map from QEMU device ID to guest device path via +// QMP query-block. Ordering follows PCI bus order (QDev device index). func queryGuestDeviceMap(qmpClient *qmp.QMPClient, instanceID string) (map[string]string, error) { resp, err := sendQMPCommand(qmpClient, qmp.QMPCommand{Execute: "query-block"}, instanceID) if err != nil { @@ -49,14 +38,11 @@ func queryGuestDeviceMap(qmpClient *qmp.QMPClient, instanceID string) (map[strin return buildDeviceMap(devices), nil } -// deviceMapRetrySleep is the sleep seam used by queryGuestDeviceMapWait. -// Tests override it to drive the retry cadence without burning real -// wall-clock time on each iteration. +// deviceMapRetrySleep is a var so tests can drive retry cadence without real sleeps. var deviceMapRetrySleep = time.Sleep // queryGuestDeviceMapWait retries queryGuestDeviceMap until expectedDevice -// appears in the result. This handles the race where query-block is called -// immediately after device_add, before QEMU has registered the new device. +// appears, handling the race between device_add and QEMU registration. func queryGuestDeviceMapWait(qmpClient *qmp.QMPClient, instanceID, expectedDevice string) (map[string]string, error) { const maxAttempts = 5 const retryDelay = 200 * time.Millisecond @@ -80,14 +66,8 @@ func queryGuestDeviceMapWait(qmpClient *qmp.QMPClient, instanceID, expectedDevic return queryGuestDeviceMap(qmpClient, instanceID) } -// buildDeviceMap takes a list of BlockDevices from QMP and returns a map -// from QEMU device ID to guest /dev/vdX path, sorted by PCI address. -// -// Two QDev path formats exist: -// - Boot-time: /machine/peripheral-anon/device[N]/virtio-backend -// device field contains the ID (e.g. "os", "cloudinit") -// - Hot-plugged: /machine/peripheral//virtio-backend -// device field is empty; ID is extracted from the QDev path +// buildDeviceMap maps QEMU device ID → /dev/vdX sorted by PCI address. +// Boot-time devices use device[N] paths; hot-plugged use peripheral/ paths. func buildDeviceMap(devices []qmp.BlockDevice) map[string]string { type deviceEntry struct { name string diff --git a/spinifex/vm/eni_hotplug.go b/spinifex/vm/eni_hotplug.go index 0b15c626a..4e846d229 100644 --- a/spinifex/vm/eni_hotplug.go +++ b/spinifex/vm/eni_hotplug.go @@ -32,28 +32,14 @@ var newDeviceController = func(v *VM) DeviceController { return NewQMPDeviceController(v.QMPClient, v.ID) } -// HotPlugENIResult carries the slot index assigned to a successful -// hot-plug. Returned to the daemon handler so the AttachmentId KV record -// can be annotated with HotPlugSlot before the NATS success event fires. +// HotPlugENIResult carries the slot index assigned to a successful hot-plug. type HotPlugENIResult struct { Slot int } -// HotPlugENI runs the 4-step attach pipeline against the live VM: -// -// 1. tap device + OVS port (NetworkPlumber.SetupTap — atomic tap+br-int) -// 2. QMP netdev_add -// 3. QMP device_add -// 4. poll query-pci until materialized -// -// On step-N failure, steps 1..N-1 are rolled back in reverse order and -// the slot is returned to ENIRequests.AvailableSlots. The instance must -// be running and have a live QMPClient. -// -// Caller (daemon handler) owns the KV side-effects: marking -// AttachmentStatus=attached and publishing the -// vpc.eni-hotplug.attached.{instanceID} event happens after this returns -// nil. +// HotPlugENI runs the 4-step attach pipeline: (1) tap+OVS, (2) netdev_add, +// (3) device_add, (4) poll query-pci. On failure, prior steps are rolled back +// and the slot is returned. Instance must be running with a live QMPClient. func (m *Manager) HotPlugENI(instance *VM, eniID, mac string) (HotPlugENIResult, error) { if instance == nil { return HotPlugENIResult{}, ErrInstanceNotFound @@ -140,10 +126,8 @@ func (m *Manager) HotPlugENI(instance *VM, eniID, mac string) (HotPlugENIResult, return HotPlugENIResult{Slot: slot}, nil } -// HotUnplugENI runs the reverse 4-step detach pipeline. force=true -// shortens the post-device_del poll budget so an unresponsive guest does -// not block the call — at the cost of leaving the guest with a dangling -// virtio-net device until reboot if the kernel never releases it. +// HotUnplugENI runs the reverse 4-step detach pipeline. force=true shortens +// the poll budget at the cost of potentially leaving a dangling device. func (m *Manager) HotUnplugENI(instance *VM, eniID string, force bool) error { if instance == nil { return ErrInstanceNotFound @@ -211,9 +195,8 @@ func (m *Manager) releaseSlotLocked(instance *VM, eniID string, slot int) { instance.ENIRequests.AvailableSlots = append(instance.ENIRequests.AvailableSlots, slot) } -// waitForPCIDevice polls QueryPCI until presence (wantPresent=true) or -// absence (wantPresent=false) of deviceID is observed. Attach and detach -// use distinct poll budgets via the package-level eniPipeline settings. +// waitForPCIDevice polls QueryPCI until deviceID is present or absent. +// Attach and detach use distinct budgets from the eniPipeline settings. func waitForPCIDevice(dc DeviceController, deviceID string, wantPresent bool) error { interval := eniPipeline.AttachPollInterval maxAttempts := eniPipeline.AttachPollMax @@ -247,13 +230,8 @@ func waitForPCIDevice(dc DeviceController, deviceID string, wantPresent bool) er // override it to drive poll cadence without burning wall-clock time. var eniPipelineSleep = time.Sleep -// SetHotPlugTestSeams swaps the device-controller factory and the -// query-pci poll sleep for the duration of a test, returning a restore -// func that callers pass to t.Cleanup. Mirrors utils.SetSudoCommandForTest: -// the indirection lets tests in other packages drive the hot-plug -// pipeline against a StubDeviceController without reassigning -// unexported vars (which the reassign linter forbids). Pass nil for -// either argument to leave the corresponding seam untouched. +// SetHotPlugTestSeams swaps the device-controller factory and poll sleep for a +// test, returning a restore func for t.Cleanup. Pass nil to leave a seam untouched. func SetHotPlugTestSeams(factory func(*VM) DeviceController, sleep func(time.Duration)) (restore func()) { origFactory := newDeviceController origSleep := eniPipelineSleep diff --git a/spinifex/vm/firmware.go b/spinifex/vm/firmware.go index ab7452c3f..af939a9be 100644 --- a/spinifex/vm/firmware.go +++ b/spinifex/vm/firmware.go @@ -14,12 +14,9 @@ type FirmwareCandidate struct { VarsTemplate string } -// FirmwarePathCandidates lists where UEFI firmware lives on supported hosts, -// keyed by Architecture string ("x86_64" | "arm64"). First match wins so -// distro-native packages take precedence over edk2 fallbacks. -// -// Exported so tests across packages can swap entries via t.Cleanup. The RHEL -// host plan extends this map by appending candidates. +// FirmwarePathCandidates lists UEFI firmware locations keyed by architecture. +// First match wins so distro packages take precedence over edk2 fallbacks. +// Exported so tests can swap entries via t.Cleanup. var FirmwarePathCandidates = map[string][]FirmwareCandidate{ "x86_64": { {Code: "/usr/share/OVMF/OVMF_CODE_4M.fd", VarsTemplate: "/usr/share/OVMF/OVMF_VARS_4M.fd"}, @@ -31,15 +28,9 @@ var FirmwarePathCandidates = map[string][]FirmwareCandidate{ }, } -// FirmwarePaths returns the first (code, varsTemplate, varsSize) tuple from -// FirmwarePathCandidates[arch] where both files exist on disk. varsSize is -// the on-disk size of the VARS template — QEMU pflash requires the per-VM -// VARS volume to match the template byte-for-byte, so the caller sizes the -// viperblock volume from this return. -// -// Returns an error mentioning the arch on miss; the launch path treats this -// as a hard failure rather than falling back to SeaBIOS (a UEFI guest booted -// under SeaBIOS panics on missing ESP). +// FirmwarePaths returns the first (code, varsTemplate, varsSize) from +// FirmwarePathCandidates[arch] where both files exist. varsSize is used to +// size the per-VM VARS volume. Returns an error on miss — no SeaBIOS fallback. func FirmwarePaths(arch string) (code, varsTemplate string, varsSize int64, err error) { candidates, ok := FirmwarePathCandidates[arch] if !ok { diff --git a/spinifex/vm/instance_cleaner.go b/spinifex/vm/instance_cleaner.go index fd98b7885..5c59ae9b7 100644 --- a/spinifex/vm/instance_cleaner.go +++ b/spinifex/vm/instance_cleaner.go @@ -1,15 +1,8 @@ package vm -// InstanceCleaner releases the per-instance external resources that the -// manager itself does not own: storage volumes, the management TAP/IP, the -// public IP allocation, the primary ENI, and placement-group membership. -// The manager invokes these from Stop/Terminate after QEMU has shut down -// and volumes have been unmounted; methods are best-effort and log their -// own errors. -// -// The live implementation lives in the daemon package and delegates to the -// existing volume / VPC / EIP / placement-group services. Tests inject a -// recording fake. +// InstanceCleaner releases per-instance external resources (volumes, mgmt TAP/IP, +// public IP, ENI, placement group) that the manager does not own. Invoked from +// Stop/Terminate after QEMU shuts down; methods are best-effort and self-logging. type InstanceCleaner interface { // DeleteVolumes deletes EFI / cloud-init internal volumes via ebs.delete // and user volumes flagged DeleteOnTermination via the volume service. diff --git a/spinifex/vm/lifecycle.go b/spinifex/vm/lifecycle.go index 65c22dfd2..8ae176d30 100644 --- a/spinifex/vm/lifecycle.go +++ b/spinifex/vm/lifecycle.go @@ -21,18 +21,15 @@ import ( "github.com/mulgadc/viperblock/viperblock" ) -// RG-4 guest OOM tiers. Under host memory pressure the kernel reaps a user -// guest QEMU first; system instances (ELBv2 LB-VMs, EKS control-plane nodes) -// back many user VMs, so they rank above user guests but below the infra tier -// (units carry OOMScoreAdjust=-500). The host system.slice is reserved below all. +// RG-4 guest OOM tiers: user guests are reaped first; system instances (ELBv2, +// EKS) rank above user guests but below infra (OOMScoreAdjust=-500). const ( oomScoreUserGuest = 500 // ManagedBy == "" — customer instance, reaped first oomScoreSystemInstance = 0 // ManagedBy != "" — elbv2 / eks, protected above user guests ) -// guestOOMScore returns the oom_score_adj for a guest QEMU per the RG-4 ladder, -// keyed on whether the instance is platform-managed (ManagedBy set). Pure — -// split out for unit-testability of the tiering. +// guestOOMScore returns the oom_score_adj per the RG-4 ladder. +// Pure function; split out for unit-testability. func guestOOMScore(managedBy string) int { if managedBy == "" { return oomScoreUserGuest @@ -40,28 +37,22 @@ func guestOOMScore(managedBy string) int { return oomScoreSystemInstance } -// Run launches a VM through the manager's lifecycle pipeline: validate state, -// mount volumes, exec QEMU, attach QMP, transition to Running, fire -// OnInstanceUp. The instance need not be in the manager's map yet — Run -// inserts it before transitioning. Used by RunInstances, the start-stopped -// handler, restore, and crash recovery. +// Run launches a VM: validate state, mount volumes, exec QEMU, attach QMP, +// transition to Running, fire OnInstanceUp. Inserts the instance before +// transitioning. Used by RunInstances, start-stopped handler, restore, and crash recovery. func (m *Manager) Run(instance *VM) error { return m.launch(instance) } -// Start re-launches a stopped instance held in the manager's map by id. Used -// by the EC2 StartInstances handler when the instance is already local. -// Returns ErrInstanceNotFound when id is unknown so callers can map the -// failure to InvalidInstanceID.NotFound rather than a generic 500. +// Start re-launches a stopped instance by id. Returns ErrInstanceNotFound when +// id is unknown so callers can map the failure to InvalidInstanceID.NotFound. func (m *Manager) Start(id string) error { instance, ok := m.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrInstanceNotFound, id) } - // A crash/recovery-failed instance sits in StateError, which launchStillValid - // rejects. Move it to pending first (the same Error->Pending step the - // auto-restart path takes) so launch proceeds; resource re-allocation is the - // caller's responsibility, mirroring the stopped-start flow. + // StateError is rejected by launchStillValid; move to Pending first so + // launch proceeds (resource re-allocation is the caller's responsibility). if m.Status(instance) == StateError && m.deps.TransitionState != nil { if err := m.deps.TransitionState(instance, StatePending); err != nil { return err @@ -70,10 +61,8 @@ func (m *Manager) Start(id string) error { return m.launch(instance) } -// Reboot issues a QMP system_reset to a running instance. The VM stays -// in StateRunning across the reset; QEMU re-runs firmware and the guest -// kernel reboots in place. Returns ErrInstanceNotFound when id is unknown -// and ErrInvalidTransition when the instance is not Running. +// Reboot issues a QMP system_reset; the VM stays in StateRunning while QEMU +// re-runs firmware. Returns ErrInstanceNotFound or ErrInvalidTransition as appropriate. func (m *Manager) Reboot(id string) error { instance, ok := m.Get(id) if !ok { @@ -89,11 +78,8 @@ func (m *Manager) Reboot(id string) error { return nil } -// launchStillValid returns true while the launch pipeline may continue -// setting up resources for instance. Returns false if a concurrent terminate -// has flipped status out of pending/stopped/provisioning — at that point the -// terminate goroutine owns cleanup and launch must bail without further side -// effects. +// launchStillValid returns true when status is still pending/stopped/provisioning. +// Returns false if a concurrent terminate took ownership; launch must bail. func (m *Manager) launchStillValid(instance *VM) bool { status := m.Status(instance) if status == StatePending || status == StateStopped || status == StateProvisioning { @@ -127,10 +113,8 @@ func (m *Manager) launch(instance *VM) error { return err } - // Re-check status — Mount can take 30+s on cold AMIs (NBD clone), and a - // terminate can race in during that window. Skip the remaining setup so - // the concurrent terminate goroutine doesn't fight SetupTapDevice and leak - // resources. + // Re-check status — Mount can take 30+s on cold AMIs, and a terminate may + // race in during that window; bail to avoid resource contention. if !m.launchStillValid(instance) { return nil } @@ -160,10 +144,8 @@ func (m *Manager) launch(instance *VM) error { m.Insert(instance) - // Final race check — QEMU is up, but if a terminate fired during start / - // QMP attach, the concurrent goroutine has already transitioned status to - // shutting-down. Attempting StateRunning here would log a spurious error; - // let the terminate cleanup own teardown. + // Final race check — a concurrent terminate may have already transitioned to + // shutting-down; let that goroutine own cleanup. if !m.launchStillValid(instance) { return nil } @@ -485,10 +467,8 @@ func (m *Manager) startQEMU(instance *VM) error { } } - // QEMU writes its pidfile after parsing args and mmap'ing the kernel/initrd/ - // fwcfg blobs. Direct-boot is usually <50ms, but under post-reboot recovery - // load it can exceed the settle wait — block briefly so we don't tear down - // the tap before QEMU finishes attaching to it. + // Wait for the pidfile so we don't tear down the tap before QEMU finishes + // attaching to it (can lag under post-reboot recovery load). if _, err := utils.WaitForPidFile(instance.ID, 3*time.Second); err != nil { slog.Error("Failed to read PID file", "err", err) return err @@ -497,11 +477,8 @@ func (m *Manager) startQEMU(instance *VM) error { return nil } -// findFreePort is bound to a var (not called directly) so tests can swap in -// a deterministic stub — appendDevHostfwdNIC's failure paths (split error, -// Atoi error, per-entry failures continuing) are otherwise unreachable -// because viperblock.FindFreePort always succeeds with a well-formed -// host:port from the OS. +// findFreePort is a var so tests can swap in a stub to reach appendDevHostfwdNIC's +// failure paths (viperblock.FindFreePort always succeeds in production). var findFreePort = viperblock.FindFreePort // appendDevHostfwdNIC adds a user-mode NIC with SSH hostfwd for dev access. @@ -553,10 +530,8 @@ func (m *Manager) appendDevHostfwdNIC(instance *VM) { "bindIP", bindIP, "port", sshDebugPort, "mac", devMac, "instanceId", instance.ID) } -// AttachQMP connects a QMP client to a QEMU process that already exists -// (e.g. a daemon restart finding a still-running QEMU) and starts the -// heartbeat goroutine. AttachQMP is the matching seam for reconnect -// callers; the launch path calls the same helper inline. +// AttachQMP connects a QMP client to an already-running QEMU process and +// starts the heartbeat goroutine. Used by reconnect callers on daemon restart. func (m *Manager) AttachQMP(instance *VM) error { client, err := newQMPClientWithHandshake(instance) if err != nil { @@ -567,15 +542,11 @@ func (m *Manager) AttachQMP(instance *VM) error { return nil } -// newQMPClientWithHandshake dials the instance's QMP socket and runs the -// qmp_capabilities handshake. The caller owns starting the heartbeat -// goroutine on the returned client. +// newQMPClientWithHandshake dials the QMP socket and runs the qmp_capabilities +// handshake. The caller is responsible for starting the heartbeat goroutine. func newQMPClientWithHandshake(v *VM) (*qmp.QMPClient, error) { - // QEMU binds the QMP listening socket after writing its pidfile and parsing - // args; under post-reboot recovery load the bind can lag behind startQEMU's - // pidfile gate, leaving an "ENOENT on dial" race that tears down the VM - // before it finishes coming up. Block briefly so the dial only runs once - // the socket inode exists. + // QMP socket bind lags the pidfile under recovery load; wait for the + // socket inode to exist before dialling to avoid an ENOENT race. if err := utils.WaitForUnixSocket(v.Config.QMPSocket, 3*time.Second); err != nil { return nil, fmt.Errorf("connect QMP socket %s: %w", v.Config.QMPSocket, err) } @@ -591,9 +562,8 @@ func newQMPClientWithHandshake(v *VM) (*qmp.QMPClient, error) { return client, nil } -// qmpHeartbeat sends a query-status QMP command every 30s to confirm the -// QEMU process is still healthy. Exits when the instance reaches a terminal -// or transitional state. Closes the QMP connection on exit. +// qmpHeartbeat sends query-status every 30s. Exits and closes the QMP +// connection when the instance reaches a terminal or transitional state. func (m *Manager) qmpHeartbeat(instance *VM) { for { time.Sleep(30 * time.Second) @@ -621,8 +591,7 @@ func (m *Manager) qmpHeartbeat(instance *VM) { } } -// sendQMPCommand encodes cmd onto client and decodes the matching response, -// skipping informational events. +// sendQMPCommand encodes cmd and decodes the response, skipping event messages. func sendQMPCommand(q *qmp.QMPClient, cmd qmp.QMPCommand, instanceID string) (*qmp.QMPResponse, error) { if q == nil || q.Encoder == nil || q.Decoder == nil { return nil, fmt.Errorf("QMP client is not initialized") @@ -669,25 +638,13 @@ func sendQMPCommand(q *qmp.QMPClient, cmd qmp.QMPCommand, instanceID string) (*q } } -// EBSHotPlugSlotCount is the fixed number of PCIe root ports pre-allocated -// for EBS hot-plug. Matches the /dev/sd[f-p] AWS device-letter range that -// AttachVolume targets. Cannot grow without QEMU restart (Linux PCIe -// hot-plug requires pre-allocated ports), so the count is deliberately -// equal to the AWS device-letter ceiling. +// EBSHotPlugSlotCount is the fixed number of PCIe root ports pre-allocated for +// EBS hot-plug, matching the /dev/sd[f-p] range. Cannot grow without QEMU restart. const EBSHotPlugSlotCount = 11 -// buildBaseVMConfig creates a vm.Config with base QEMU settings and PCIe -// hotplug root ports. bootMode is the AMI's boot mode string ("bios" | "uefi" -// | "uefi-preferred"); "uefi" and "uefi-preferred" flip cfg.UseUEFI. Any other -// value (including "") defaults to BIOS. -// -// PCIe root ports are pre-allocated in two pools: -// -// - hotplug-ebs{1..EBSHotPlugSlotCount} on chassis 1..EBSHotPlugSlotCount -// for AttachVolume targets (/dev/sd[f-p]). -// - hotplug-eni{1..N} on chassis EBSHotPlugSlotCount+1.. where N is -// instancetypes.HotPlugENISlotsForType(instanceType) — the AWS-published -// ENI cap for the type minus the primary ENI, which is wired at boot. +// buildBaseVMConfig creates a Config with base QEMU settings and two pre-allocated +// PCIe root-port pools: hotplug-ebs{1..N} for EBS (/dev/sd[f-p]) and +// hotplug-eni{1..M} for ENI hot-plug. bootMode "uefi"/"uefi-preferred" sets UseUEFI. func buildBaseVMConfig(instanceID, instanceType, pidFile, consoleLogPath, serialSocket, architecture, bootMode string, vCPUs, memoryMiB int) Config { cfg := Config{ Name: instanceID, @@ -722,11 +679,9 @@ func buildBaseVMConfig(instanceID, instanceType, pidFile, consoleLogPath, serial return cfg } -// initENIRequests resets the per-VM ENI hot-plug slot allocator before the -// VM boots. The free-list mirrors the hotplug-eni{1..N} root ports emitted -// by buildBaseVMConfig for the instance type. AttachedByENIID is preserved -// across restart so the reconciler (Sprint 3d) can match KV truth against -// the in-memory map; on a cold boot it starts empty. +// initENIRequests resets the per-VM ENI slot free-list to mirror the +// hotplug-eni{1..N} root ports in buildBaseVMConfig. AttachedByENIID is +// preserved across restart; on a cold boot it starts empty. func (m *Manager) initENIRequests(instance *VM) { eniSlots := instancetypes.HotPlugENISlotsForType(instance.InstanceType) instance.ENIRequests.Mu.Lock() @@ -740,10 +695,9 @@ func (m *Manager) initENIRequests(instance *VM) { } } -// buildDrives converts EBS volume requests into QEMU drive, iothread, and -// device configurations. Returns an error if any volume is missing its -// NBDURI. EFI volumes are emitted as pflash unit=1 (per-VM EFI variable -// store); the readonly pflash CODE blob (unit=0) is added by Config.Execute. +// buildDrives converts EBS requests into QEMU drive/iothread/device configs. +// Returns an error if any volume is missing its NBDURI. EFI volumes emit +// pflash unit=1; the readonly CODE blob (unit=0) is added by Config.Execute. func buildDrives(requests []types.EBSRequest, cpuCount int, machineType string) ([]Drive, []IOThread, []Device, error) { var drives []Drive var iothreads []IOThread diff --git a/spinifex/vm/manager.go b/spinifex/vm/manager.go index 32894fe90..5a2cb26db 100644 --- a/spinifex/vm/manager.go +++ b/spinifex/vm/manager.go @@ -6,36 +6,23 @@ import ( "time" ) -// ManagerHooks are callbacks the manager fires synchronously on every -// running/terminated transition. The daemon uses these to drive per-instance -// NATS subscription/unsubscription. Hook fields may be nil; nil hooks are no-ops. -// -// OnInstanceUp returns an error so callers can distinguish a soft launch -// (subscribe failures only logged) from a reconnect (failures must roll back -// QMP and abort the reconnect). Callers that don't care about the error may -// ignore it. +// ManagerHooks are callbacks the manager fires synchronously on running/terminated +// transitions. Hook fields may be nil (no-ops). OnInstanceUp returns an error so +// reconnect callers can roll back QMP on subscribe failure; launch callers may ignore it. type ManagerHooks struct { OnInstanceUp func(*VM) error OnInstanceDown func(id string) - // OnInstanceRecovering fires from Restore once per instance that is - // about to be relaunched. The daemon subscribes only the command - // topic (ec2.cmd.) here so concurrent terminate commands can - // reach this node while the relaunch is in flight; the subsequent - // OnInstanceUp on launch success is idempotent and reinstalls both - // the command and console subscriptions. + // OnInstanceRecovering fires from Restore before each relaunch so the daemon + // can early-subscribe ec2.cmd. and handle concurrent terminates in flight. OnInstanceRecovering func(*VM) - // BeforeInstanceRelaunch fires from Restore once per recovery - // candidate, after OnInstanceRecovering and immediately before - // m.Run. Daemons use this to refresh ephemeral on-host state - // (tmpfs-backed config blobs, dynamic firmware files) that did not - // survive a host reboot. A non-nil error aborts the relaunch and - // marks the instance recovery_failed. + // BeforeInstanceRelaunch fires from Restore immediately before m.Run. Daemons + // use this to refresh ephemeral on-host state that did not survive a reboot. + // A non-nil error aborts the relaunch and marks the instance recovery_failed. BeforeInstanceRelaunch func(*VM) error } -// Deps bundles every collaborator the manager uses to drive lifecycle. -// Fields may be nil where the manager does not yet require them; call sites -// guard against nil so partial wiring during construction is safe. +// Deps bundles collaborators the manager uses to drive lifecycle. +// Fields may be nil; call sites guard against nil so partial wiring is safe. type Deps struct { NodeID string @@ -72,11 +59,8 @@ type Deps struct { // Zero is acceptable in tests; production uses 1s. DetachDelay time.Duration - // ConsumeCleanShutdownMarker reports whether the previous daemon run - // recorded a clean shutdown marker for this node, deleting it as a - // side effect. Restore uses the result to decide whether to wait - // briefly for stale QEMU PIDs to die before classifying state. Nil - // is treated as "no marker" (cautious recovery). + // ConsumeCleanShutdownMarker reports whether the previous run wrote a clean + // shutdown marker, consuming it. Nil is treated as "no marker" (cautious recovery). ConsumeCleanShutdownMarker func() bool } @@ -89,9 +73,8 @@ type Manager struct { goroutineWg sync.WaitGroup } -// NewManager returns a Manager with no collaborators wired. Production code -// uses NewManagerWithDeps; tests that only exercise the in-memory map keep -// this convenience. +// NewManager returns a Manager with no collaborators wired. Tests that only +// exercise the in-memory map use this; production code uses NewManagerWithDeps. func NewManager() *Manager { return &Manager{vms: make(map[string]*VM)} } @@ -101,10 +84,8 @@ func NewManagerWithDeps(deps Deps) *Manager { return &Manager{vms: make(map[string]*VM), deps: deps} } -// SetDeps replaces the manager's dependencies. The daemon constructs the -// manager early (before NATS / JetStream / network plumber are available) -// and calls SetDeps once those collaborators exist. Must not be called -// concurrently with lifecycle methods. +// SetDeps replaces the manager's dependencies. Called once collaborators are +// available after early construction. Must not be called concurrently. func (m *Manager) SetDeps(deps Deps) { m.deps = deps } // NodeID returns the node identifier the manager was constructed with. @@ -145,8 +126,7 @@ func (m *Manager) Delete(id string) { } // DeleteIf deletes the entry for id only if the stored pointer matches want. -// Returns true if the delete happened. Used by stop/terminate handlers to guard -// against the slot being reclaimed by a concurrent start handler. +// Returns true if the delete happened; false if the slot was reclaimed concurrently. func (m *Manager) DeleteIf(id string, want *VM) bool { m.mu.Lock() defer m.mu.Unlock() @@ -165,9 +145,8 @@ func (m *Manager) Count() int { return len(m.vms) } -// ForEach calls fn for each VM under the manager lock. fn must not call back -// into Manager methods that take the lock — that will deadlock. For lock-free -// iteration, use Snapshot. +// ForEach calls fn for each VM under the manager lock. fn must not re-enter +// locked Manager methods (deadlock). For lock-free iteration use Snapshot. func (m *Manager) ForEach(fn func(*VM)) { m.mu.Lock() defer m.mu.Unlock() @@ -176,9 +155,8 @@ func (m *Manager) ForEach(fn func(*VM)) { } } -// Snapshot returns the current set of VMs as a slice. Callers must not mutate -// the slice's *VM entries' map-related fields; the underlying VM pointers are -// shared with the manager. +// Snapshot returns the current VMs as a slice. The returned pointers are shared +// with the manager; callers must not mutate map-related fields. func (m *Manager) Snapshot() []*VM { m.mu.Lock() defer m.mu.Unlock() @@ -189,8 +167,7 @@ func (m *Manager) Snapshot() []*VM { return out } -// SnapshotMap returns a copy of the id→VM map. Used by the state persistence -// adapter so serialization can happen without holding the manager lock. +// SnapshotMap returns a copy of the id→VM map for serialization outside the lock. func (m *Manager) SnapshotMap() map[string]*VM { m.mu.Lock() defer m.mu.Unlock() @@ -212,9 +189,8 @@ func (m *Manager) Filter(pred func(*VM) bool) []*VM { return out } -// UpdateState looks up id and, if found, runs fn(v) under lock. Used for -// atomic check-then-mutate or read-under-lock on a single VM. Returns true -// if the VM was found and fn was invoked. +// UpdateState looks up id and, if found, runs fn(v) under lock. +// Returns true if the VM was found and fn was invoked. func (m *Manager) UpdateState(id string, fn func(*VM)) bool { m.mu.Lock() defer m.mu.Unlock() @@ -226,14 +202,10 @@ func (m *Manager) UpdateState(id string, fn func(*VM)) bool { return true } -// UpdateAndPersist looks up id and runs fn(v) under the manager lock. fn -// returns true to request a state-store persist after the lock is released -// (matching the writeRunningState pattern used by AttachVolume / Stop / -// Terminate); false signals a no-op or early-return mutator and skips the -// persist entirely. Returns (false, nil) when the VM is not in the running -// map; fn is not invoked. On persist failure the in-memory mutation has -// already taken effect — the caller's API-level error masks the drift but -// memory will be ahead of disk until the next successful persist. +// UpdateAndPersist looks up id and runs fn(v) under lock. fn returns true to +// trigger a state-store persist after the lock is released; false skips it. +// Returns (false, nil) when the VM is absent. On persist failure the in-memory +// mutation already took effect — memory stays ahead of disk until next persist. func (m *Manager) UpdateAndPersist(id string, fn func(*VM) bool) (bool, error) { var changed bool found := m.UpdateState(id, func(v *VM) { @@ -245,22 +217,17 @@ func (m *Manager) UpdateAndPersist(id string, fn func(*VM) bool) (bool, error) { return true, m.writeRunningState() } -// Status returns v.Status under the manager lock. Replaces the dominant -// "Inspect to read Status" pattern with a typed accessor. Read-only — no -// membership check, so callers may pass a pointer to an instance no longer -// in the map (e.g. mid-cleanup) and still get a consistent read. +// Status returns v.Status under the manager lock. No membership check — +// callers may pass a pointer to an instance no longer in the map. func (m *Manager) Status(v *VM) InstanceState { m.mu.Lock() defer m.mu.Unlock() return v.Status } -// Inspect runs fn(v) under the manager lock for an already-resolved VM -// pointer. Used by call sites that hold a *VM (e.g. from a NATS handler -// dispatch) and need to read or mutate its fields with the same memory- -// ordering guarantee as map-keyed access. Prefer UpdateState (membership- -// checked) for mutation and Status for the read-Status pattern; Inspect -// remains for closures that mutate fields on a possibly-orphaned VM. +// Inspect runs fn(v) under the manager lock for an already-resolved VM pointer. +// Prefer UpdateState for membership-checked mutation; Inspect is for closures +// that may run on a possibly-orphaned VM (e.g. MarkFailed). func (m *Manager) Inspect(v *VM, fn func(*VM)) { m.mu.Lock() defer m.mu.Unlock() @@ -268,18 +235,14 @@ func (m *Manager) Inspect(v *VM, fn func(*VM)) { } // View runs fn with the live id→VM map under the manager lock. fn must not -// mutate the map nor retain references after returning. Used by serialization -// paths that need every VM's fields to be stable for the duration of the -// encode (e.g. JSON marshal during state persistence). +// mutate the map or retain references after returning. func (m *Manager) View(fn func(map[string]*VM)) { m.mu.Lock() defer m.mu.Unlock() fn(m.vms) } -// Replace bulk-resets the manager to the given set of VMs. The supplied map -// is copied; callers may mutate it after the call returns. Used by Restore -// after loading persisted state. +// Replace bulk-resets the manager to the given VMs (copied). Used by Restore. func (m *Manager) Replace(vms map[string]*VM) { m.mu.Lock() defer m.mu.Unlock() @@ -287,9 +250,8 @@ func (m *Manager) Replace(vms map[string]*VM) { maps.Copy(m.vms, vms) } -// WaitForBackgroundWork blocks until all background goroutines started by -// MarkFailed and MarkRecoveryFailed have completed. Intended for tests that -// need the temp directory to be quiescent before cleanup runs. +// WaitForBackgroundWork blocks until all goroutines started by MarkFailed and +// MarkRecoveryFailed complete. Used by tests to drain before cleanup. func (m *Manager) WaitForBackgroundWork() { m.goroutineWg.Wait() } diff --git a/spinifex/vm/migrate.go b/spinifex/vm/migrate.go index d0340674c..2b1112808 100644 --- a/spinifex/vm/migrate.go +++ b/spinifex/vm/migrate.go @@ -2,17 +2,10 @@ package vm import "log/slog" -// MigrateStoppedToSharedKV writes instance to the cluster-shared "stopped" -// KV bucket and removes it from the local running map. Returns true only -// when both the shared write succeeded AND the local entry was removed -// under this caller's ownership of the supplied *VM pointer. -// -// Returns false on KV write failure or when a concurrent handler reclaimed -// the slot (DeleteIf no longer matches the supplied pointer). Callers -// should fire OnInstanceDown / persist running-state only on true: a slot -// reclaim means the id now resolves to a different live instance, and -// firing the down hook would tear down the new instance's per-id -// resources (e.g. NATS subscriptions). +// MigrateStoppedToSharedKV writes instance to the "stopped" KV bucket and +// removes it from the local map. Returns true only when both the write and +// DeleteIf succeeded — a concurrent slot reclaim reports false to prevent +// firing OnInstanceDown against a different live instance at the same id. func (m *Manager) MigrateStoppedToSharedKV(instance *VM) bool { if m.deps.StateStore == nil { return false @@ -31,9 +24,7 @@ func (m *Manager) MigrateTerminatedToKV(instance *VM) bool { } // migrateInstanceToKV is the shared body of MigrateStoppedToSharedKV and -// MigrateTerminatedToKV. Returns true only when both the shared write -// succeeded and DeleteIf removed the local entry — see the public -// methods' doc comments for why slot-reclaim is reported as failure. +// MigrateTerminatedToKV. func (m *Manager) migrateInstanceToKV(instance *VM, writeFn func(string, *VM) error, label string) bool { instance.LastNode = m.deps.NodeID if err := writeFn(instance.ID, instance); err != nil { diff --git a/spinifex/vm/network_plumber.go b/spinifex/vm/network_plumber.go index 455c89ae7..ef45cc58b 100644 --- a/spinifex/vm/network_plumber.go +++ b/spinifex/vm/network_plumber.go @@ -1,10 +1,7 @@ package vm -// NetworkPlumber handles tap device and OVS bridge operations. Defined here -// so the manager can hold the collaborator without importing network/host; -// host.OVSPlumber satisfies it structurally. Both methods must be idempotent — -// callers rely on safe reinvocation across host reboots and terminate-during- -// pending races. +// NetworkPlumber handles tap device and OVS bridge operations. Defined in vm +// so the manager avoids importing network/host. Both methods must be idempotent. type NetworkPlumber interface { SetupTap(spec TapSpec) error CleanupTap(name string) error diff --git a/spinifex/vm/qmp_device.go b/spinifex/vm/qmp_device.go index ca31c8d94..4cc4bfa86 100644 --- a/spinifex/vm/qmp_device.go +++ b/spinifex/vm/qmp_device.go @@ -9,16 +9,9 @@ import ( "github.com/mulgadc/spinifex/spinifex/qmp" ) -// DeviceController abstracts the QEMU Machine Protocol surface the ENI -// hot-plug pipeline uses: device_add / device_del / netdev_add / netdev_del / -// query-pci. Sprint 3a lands the interface and the in-memory stub so the -// downstream pipeline (Sprint 3b) can be unit-tested without a live QEMU. -// -// The production backend (QMPDeviceController) wraps the existing -// sendQMPCommand helper so wire-level serialization continues to live on the -// underlying *qmp.QMPClient mutex. The stub backend (StubDeviceController) -// keeps an in-memory map of attached devices/netdevs for assertions and lets -// tests pre-program failure responses per execute name. +// DeviceController abstracts the QMP surface the ENI hot-plug pipeline uses: +// device_add / device_del / netdev_add / netdev_del / query-pci. +// QMPDeviceController is the production backend; StubDeviceController is for tests. type DeviceController interface { DeviceAdd(args map[string]any) error DeviceDel(deviceID string) error @@ -27,19 +20,15 @@ type DeviceController interface { QueryPCI() ([]PCIDevice, error) } -// PCIDevice is the trimmed projection of QEMU's query-pci response that the -// hot-plug pipeline consumes. Full response parsing lands in Sprint 3b; the -// stub backend populates QDevID only, which is the field the pipeline checks -// to confirm device materialization. +// PCIDevice is the trimmed projection of QEMU's query-pci response used by +// the hot-plug pipeline to confirm device materialization (QDevID field). type PCIDevice struct { Bus int `json:"bus"` Slot int `json:"slot"` QDevID string `json:"qdev_id,omitempty"` } -// QMPDeviceController is the production DeviceController bound to a live -// QEMU instance. Construct one per VM via NewQMPDeviceController and reuse -// it across the hot-plug / hot-unplug pipeline for the same VM. +// QMPDeviceController is the production DeviceController for a live QEMU instance. type QMPDeviceController struct { client *qmp.QMPClient instanceID string @@ -77,13 +66,9 @@ func (c *QMPDeviceController) NetdevDel(netdevID string) error { return err } -// QueryPCI issues query-pci and flattens the nested QEMU response into a -// list of PCIDevice entries. The response is a per-bus list whose devices -// may themselves carry a "pci_bridge" subfield containing a nested -// "devices" list (every pcie-root-port surfaces this way), so the parse -// recurses through bridges. Only devices with a non-empty qdev_id are -// returned — the hot-plug pipeline only needs to detect materialization -// of a known id. +// QueryPCI issues query-pci and flattens the response into a PCIDevice list, +// recursing through pci_bridge entries. Only devices with a non-empty qdev_id +// are returned. func (c *QMPDeviceController) QueryPCI() ([]PCIDevice, error) { resp, err := sendQMPCommand(c.client, qmp.QMPCommand{Execute: "query-pci"}, c.instanceID) if err != nil { @@ -100,10 +85,8 @@ func (c *QMPDeviceController) QueryPCI() ([]PCIDevice, error) { return out, nil } -// pciBusInfo and pciDeviceInfo mirror only the fields the hot-plug -// pipeline reads from query-pci. Full QEMU output is much richer -// (class_info, regions, irq, …) but we ignore everything else to keep the -// parser stable against QMP version drift. +// pciBusInfo and pciDeviceInfo mirror only the fields the hot-plug pipeline +// reads from query-pci; everything else is ignored for stability against QMP drift. type pciBusInfo struct { Bus int `json:"bus"` Devices []pciDeviceInfo `json:"devices"` @@ -130,10 +113,9 @@ func appendPCIDevices(dst []PCIDevice, devs []pciDeviceInfo) []PCIDevice { return dst } -// StubDeviceController is an in-memory DeviceController for unit tests. It -// records every call in order, enforces duplicate-id and unknown-id errors -// the way real QEMU does, and lets tests inject a one-shot failure per -// execute name via FailNext. +// StubDeviceController is an in-memory DeviceController for tests. It records +// calls, enforces duplicate/unknown-id errors like real QEMU, and supports +// one-shot failure injection via SetFailNext. type StubDeviceController struct { mu sync.Mutex devices map[string]map[string]any @@ -142,9 +124,7 @@ type StubDeviceController struct { failNext map[string]error } -// StubCall records one DeviceController invocation. Args is a shallow copy -// of the args map passed in (or {"id": …} for the *Del methods) so tests -// can assert on payload without worrying about caller mutation. +// StubCall records one DeviceController invocation with a shallow copy of Args. type StubCall struct { Execute string Args map[string]any @@ -160,9 +140,8 @@ func NewStubDeviceController() *StubDeviceController { var _ DeviceController = (*StubDeviceController)(nil) -// SetFailNext primes the stub to return err on the next call matching -// execute (one of "device_add", "device_del", "netdev_add", "netdev_del", -// "query-pci"). Cleared after the call fires. +// SetFailNext primes the stub to return err on the next call matching execute. +// Cleared after the call fires. func (s *StubDeviceController) SetFailNext(execute string, err error) { s.mu.Lock() defer s.mu.Unlock() diff --git a/spinifex/vm/restore.go b/spinifex/vm/restore.go index aef3790dd..c9cd9ada0 100644 --- a/spinifex/vm/restore.go +++ b/spinifex/vm/restore.go @@ -15,25 +15,13 @@ import ( "github.com/mulgadc/spinifex/spinifex/utils" ) -// maxConcurrentRecovery limits how many VMs are relaunched in parallel during -// recovery. Cold-AMI clones plus QEMU exec start are I/O-heavy; bounding the -// fan-out keeps the host responsive while still parallelising the slow path. +// maxConcurrentRecovery bounds the recovery fan-out; cold-AMI clones are I/O-heavy. const maxConcurrentRecovery = 2 -// Restore loads persisted VM state and re-launches instances that are -// neither terminated nor flagged as user-stopped. Steps: -// -// 1. Consume the clean-shutdown marker (deps callback). Without one, sleep -// briefly so any stale QEMU PIDs from a crashed previous run finish dying -// before isInstanceProcessRunning is consulted. -// 2. Load the per-node running snapshot via StateStore. -// 3. Classify each instance — terminated/stopped go to their shared KV -// buckets; running with live QEMU + valid sockets reconnects via QMP; -// transitional states finalise; everything else queues for relaunch. -// 4. Fan out Manager.Run across the relaunch queue with a semaphore. -// -// All errors are logged; Restore never fails fatally — it always leaves the -// daemon in a usable state. +// Restore loads persisted VM state and re-launches instances that are neither +// terminated nor user-stopped. Terminated/stopped instances migrate to shared KV; +// running instances with live QEMU reconnect via QMP; others relaunch via Run. +// All errors are logged; Restore never fails fatally. func (m *Manager) Restore() { cleanShutdown := false if m.deps.ConsumeCleanShutdownMarker != nil { @@ -62,9 +50,8 @@ func (m *Manager) Restore() { } } -// loadRunningState replaces the manager's running set with the snapshot -// persisted for this node. Returns an error only if the StateStore -// surfaced one — a missing key produces an empty map and no error. +// loadRunningState replaces the manager's running set from the persisted snapshot. +// A missing key returns an empty map and no error. func (m *Manager) loadRunningState() error { if m.deps.StateStore == nil { return fmt.Errorf("StateStore not wired") @@ -77,12 +64,9 @@ func (m *Manager) loadRunningState() error { return nil } -// classifyRestoredInstances walks every loaded VM and routes it: stopped -// and terminated instances go to their shared KV buckets and are dropped -// from the local map; running instances with live QEMU + reachable NBD -// sockets reconnect via QMP; transitional states (Stopping, ShuttingDown) -// finalise to their stable counterparts; the remainder is returned for -// relaunch via Manager.Run. +// classifyRestoredInstances routes each VM: stopped/terminated migrate to KV; +// running with live QEMU+NBD reconnects; transitional states finalize; +// the remainder is returned for relaunch via Manager.Run. func (m *Manager) classifyRestoredInstances() []*VM { var toLaunch []*VM @@ -193,9 +177,8 @@ func (m *Manager) classifyRestoredInstances() []*VM { return toLaunch } -// markUnschedulable flips an instance to Stopped with an -// InsufficientInstanceCapacity StateReason so DescribeInstances surfaces a -// useful error after a node loses the ability to host the type. +// markUnschedulable flips an instance to Stopped with InsufficientInstanceCapacity +// so DescribeInstances reports a useful error when the node can no longer host the type. func markUnschedulable(instance *VM, reason string) { instance.Status = StateStopped if instance.Instance != nil { @@ -205,15 +188,9 @@ func markUnschedulable(instance *VM, reason string) { } } -// killOrphanedQEMU SIGKILLs a QEMU process whose NBD storage no longer -// works (viperblock restarted with fresh sockets). Returns true when the -// process is gone (or was never running) and the caller should proceed -// with classification; false when the kill failed and the instance -// should be skipped this cycle. -// -// SIGKILL directly: orphaned QEMU with dead storage has no state worth a -// graceful shutdown, and KillProcess's 120s SIGTERM timeout would block -// daemon startup. +// killOrphanedQEMU SIGKILLs a QEMU whose NBD storage is no longer reachable. +// Returns true when the process is gone and classification can proceed. +// SIGKILL is used directly to avoid the 120s SIGTERM timeout blocking startup. func killOrphanedQEMU(instance *VM) bool { pid, pidErr := utils.ReadPidFile(instance.ID) if pidErr != nil || pid <= 0 { @@ -235,11 +212,9 @@ func killOrphanedQEMU(instance *VM) bool { return true } -// finalizeTransitionalRestore resolves an instance whose QEMU is gone but -// whose persisted state was Stopping or ShuttingDown — flipping it to its -// stable counterpart and migrating to the appropriate shared KV bucket. -// Returns true on a clean migration; false signals the caller to retry on -// the next restart. +// finalizeTransitionalRestore advances a Stopping/ShuttingDown instance (whose +// QEMU is gone) to its stable state and migrates it to the appropriate KV bucket. +// Returns true on success; false signals the caller to retry on next restart. func (m *Manager) finalizeTransitionalRestore(instance *VM) bool { prevStatus := instance.Status if instance.Status == StateStopping { @@ -265,12 +240,8 @@ func (m *Manager) finalizeTransitionalRestore(instance *VM) bool { return true } -// relaunchAll kicks the recovery launch loop. Each instance is announced -// via OnInstanceRecovering before launching so the daemon can -// early-subscribe ec2.cmd.; that lets a concurrent terminate reach -// this node while the relaunch is still pending. The OnInstanceUp hook -// fired after a successful launch reinstalls both per-instance subs -// idempotently. +// relaunchAll fires OnInstanceRecovering for each instance (for early +// ec2.cmd. subscription) then fans out Manager.Run under a semaphore. func (m *Manager) relaunchAll(toLaunch []*VM) { if m.deps.Hooks.OnInstanceRecovering != nil { for _, instance := range toLaunch { @@ -322,25 +293,14 @@ func (m *Manager) relaunchAll(toLaunch []*VM) { wg.Wait() } -// attachQMPForReconnect is a test seam over (*Manager).AttachQMP so unit -// tests can drive reconnectInstance's success/failure branches without -// spawning the qmpHeartbeat goroutine (which sleeps 30s with no -// cancellation and otherwise leaks under goleak). +// attachQMPForReconnect is a test seam over (*Manager).AttachQMP so tests can +// drive reconnectInstance without spawning the 30s heartbeat goroutine (goleak). var attachQMPForReconnect = (*Manager).AttachQMP -// reconnectInstance re-establishes the QMP connection for an instance whose -// QEMU survived the daemon restart, fires OnInstanceUp so the daemon -// reinstalls per-instance NATS subscriptions, and persists the running -// state. Bypasses the state-machine validation because reconnect is not a -// modelled transition. -// -// Subscribe failure during the hook is treated as a hard error: the QMP -// connection is closed (so the heartbeat goroutine exits on the next tick -// when status leaves StateRunning) and the error is propagated. The caller -// in classifyRestoredInstances logs and moves on; the instance remains in -// the loaded map and will be retried on the next daemon restart. Status is -// only flipped to StateRunning after subscribes are confirmed live so a -// failed reconnect does not advertise a half-working instance to peers. +// reconnectInstance re-establishes QMP for a surviving QEMU, fires OnInstanceUp +// to reinstall NATS subscriptions, and persists running state. Subscribe failure +// closes QMP and propagates the error; status is only set to Running after +// subscriptions are confirmed live to avoid advertising a broken instance. func (m *Manager) reconnectInstance(instance *VM) error { if err := attachQMPForReconnect(m, instance); err != nil { return fmt.Errorf("failed to reconnect QMP: %w", err) @@ -366,9 +326,8 @@ func (m *Manager) reconnectInstance(instance *VM) error { return nil } -// isInstanceProcessRunning checks whether the QEMU process recorded in -// the instance's PID file is still alive. Returns false on any failure -// path (missing PID file, dead PID, signal-0 error). +// isInstanceProcessRunning reports whether the QEMU process in the PID file +// is still alive. Returns false on any failure (missing file, dead PID). func isInstanceProcessRunning(instance *VM) bool { pid, err := utils.ReadPidFile(instance.ID) if err != nil || pid <= 0 { @@ -381,16 +340,9 @@ func isInstanceProcessRunning(instance *VM) bool { return process.Signal(syscall.Signal(0)) == nil } -// AreVolumeSocketsValid reports whether the NBD Unix sockets backing the -// instance's volumes are reachable. A dial probe (not just os.Stat) is -// required because viperblock may restart with sockets at the same paths -// — the file exists but no process is listening on the old fd that QEMU -// holds. TCP and unparseable URIs are treated as valid (we can't probe -// remote viperblockd from the recovery path). -// -// Exported so the daemon-side test suite can assert socket-validity -// behaviour through the existing test helper without reaching into -// manager internals. +// AreVolumeSocketsValid dials each NBD Unix socket to confirm viperblock is +// still listening. A stat-only check is insufficient since viperblock may restart +// leaving stale socket files. TCP and unparseable URIs are treated as valid. func AreVolumeSocketsValid(instance *VM) bool { instance.EBSRequests.Mu.Lock() defer instance.EBSRequests.Mu.Unlock() diff --git a/spinifex/vm/shutdown.go b/spinifex/vm/shutdown.go index f47c09e1f..417a2393c 100644 --- a/spinifex/vm/shutdown.go +++ b/spinifex/vm/shutdown.go @@ -14,20 +14,13 @@ import ( "github.com/mulgadc/spinifex/spinifex/utils" ) -// pidFileRemovalTimeout is how long Stop/Terminate wait for QEMU's PID file -// to disappear after a graceful system_powerdown before resorting to -// SIGKILL. 20s is enough for an ACPI shutdown — guests that haven't -// responded by then won't. +// pidFileRemovalTimeout is how long Stop/Terminate wait for the PID file to +// disappear after system_powerdown before resorting to SIGKILL. const pidFileRemovalTimeout = 20 * time.Second -// Stop transitions a running instance to stopped: graceful QMP shutdown, -// volume unmount, tap teardown, resource deallocation. Persists to the -// cluster-shared "stopped" KV bucket and removes the instance from the -// local running map. Fires OnInstanceDown. -// -// Returns ErrInstanceNotFound if id is unknown, ErrInvalidTransition if -// the instance is not in a state that admits Stopping. Persistence errors -// after the in-memory transition succeeded are logged but not surfaced. +// Stop transitions a running instance to stopped: graceful QMP shutdown, volume +// unmount, tap teardown, resource deallocation. Migrates to the "stopped" KV +// bucket and fires OnInstanceDown. Returns ErrInstanceNotFound or ErrInvalidTransition. func (m *Manager) Stop(id string) error { instance, ok := m.Get(id) if !ok { @@ -53,16 +46,10 @@ func (m *Manager) Stop(id string) error { return nil } -// stopOne runs the per-instance stop sequence shared by Stop and StopAll: -// transition to Stopping → stopCleanup → transition to Stopped → migrate -// to the cluster-shared "stopped" KV bucket → fire OnInstanceDown. -// Returns (true, nil) when the migration removed the instance from the -// local map under this caller's ownership; the caller must then persist -// the running state. Returns (false, nil) when migration was skipped -// (KV write failure or slot reclaim) — see MigrateStoppedToSharedKV for -// why OnInstanceDown must not fire in that case. Returns (false, err) -// when the precheck rejects the transition; non-Running VMs surface as -// ErrInvalidTransition so fan-out callers can skip them. +// stopOne runs the stop sequence shared by Stop and StopAll: +// Stopping → stopCleanup → Stopped → migrate to "stopped" KV → OnInstanceDown. +// Returns (true, nil) when migration removed the instance (caller must persist). +// Returns (false, nil) on KV failure or slot reclaim; (false, err) on precheck failure. func (m *Manager) stopOne(instance *VM) (bool, error) { if err := m.transitionWithPrecheck(instance, StateStopping); err != nil { return false, err @@ -92,19 +79,9 @@ func (m *Manager) stopOne(instance *VM) (bool, error) { return true, nil } -// StopAll fans stopOne out across every VM the manager currently holds. -// Used by the coordinated shutdown DRAIN phase. Each instance transitions -// through Stopping → Stopped, is migrated to the cluster-shared "stopped" -// KV bucket, and fires OnInstanceDown — matching Stop's contract so that -// a daemon restart sees the instances as stopped rather than promoting -// them through the failed-recovery path. The fan-out runs one goroutine -// per VM so total wall-time is bounded by the slowest, and per-VM errors -// (e.g. non-Running precheck failures) are logged but never abort the -// fan-out. writeRunningState is called once at the end rather than per -// VM to avoid O(N²) marshalling of the running map. -// -// Volume + tap teardown only — AWS resources (ENI, public IP, placement -// group) are not released because the instance is not being terminated. +// StopAll fans stopOne across every VM for the coordinated shutdown DRAIN phase. +// Runs one goroutine per VM; per-VM errors are logged but do not abort the fan-out. +// AWS resources (ENI, public IP, placement group) are not released on stop. func (m *Manager) StopAll() error { snapshot := m.Snapshot() if len(snapshot) == 0 { @@ -133,14 +110,9 @@ func (m *Manager) StopAll() error { return nil } -// Terminate transitions an instance to terminated: graceful shutdown, -// volume + ENI + IP cleanup, placement group removal. Persists to the -// cluster-shared "terminated" KV bucket and removes the instance from -// the local running map. Fires OnInstanceDown. -// -// Idempotent on already-shutting-down (the failed-launch goroutine is -// already cleaning up). Returns ErrInstanceNotFound if id is unknown, -// ErrInvalidTransition if the current state does not permit termination. +// Terminate transitions an instance to terminated: graceful shutdown, volume + +// ENI + IP cleanup, placement group removal. Idempotent on already-shutting-down. +// Returns ErrInstanceNotFound or ErrInvalidTransition as appropriate. func (m *Manager) Terminate(id string) error { instance, ok := m.Get(id) if !ok { @@ -161,15 +133,9 @@ func (m *Manager) Terminate(id string) error { return m.finalizeTerminated(instance) } -// MarkFailed sets a failure reason, transitions to shutting-down -// synchronously, then runs the cleanup chain in a goroutine. Used when a -// launch errors mid-way: callers (NATS RunInstances handler, recovery -// worker, pending watchdog, system-instance launcher) get back control -// immediately and do not block on volume unmount, ENI delete, or KV -// writes. -// -// Tolerates instances already in a cleanup state (no-op) and instances -// that may or may not be present in the running-VM map. +// MarkFailed sets a failure reason, transitions to shutting-down synchronously, +// then runs the cleanup chain in a goroutine so callers return immediately. +// Tolerates instances already in a cleanup state (no-op). func (m *Manager) MarkFailed(instance *VM, reason string) { skip := false var observed InstanceState @@ -210,19 +176,10 @@ func (m *Manager) MarkFailed(instance *VM, reason string) { }) } -// MarkRecoveryFailed handles the case where daemon-restart recovery cannot -// bring a previously-running instance back online (Run error during -// relaunch, or QMP reconnect failure on a surviving QEMU). It transitions -// the instance to StateError with a Server.RecoveryFailed StateReason, -// then runs the non-destructive cleanup chain (graceful unmount + tap -// teardown + GPU release + resource deallocation) in a goroutine. Unlike -// MarkFailed, it does NOT delete volumes, release public IPs, detach -// ENIs, or remove placement-group membership — those resources are -// preserved so the operator can either retry recovery or issue an -// explicit ec2.TerminateInstances (which will then honour -// DeleteOnTermination as normal). The instance stays in the local map -// and OnInstanceDown is NOT fired so the ec2.cmd. subscription -// remains live for operator action. +// MarkRecoveryFailed transitions an instance to StateError after a failed +// daemon-restart recovery. Runs non-destructive cleanup (unmount, tap teardown, +// GPU/resource release) in a goroutine. Unlike MarkFailed, volumes, ENIs, and +// IPs are preserved for operator retry or explicit TerminateInstances. func (m *Manager) MarkRecoveryFailed(instance *VM, reason string) { skip := false var observed InstanceState @@ -264,10 +221,8 @@ func (m *Manager) MarkRecoveryFailed(instance *VM, reason string) { }) } -// finalizeTerminated transitions instance to terminated, writes the -// terminated KV entry, removes the instance from the local map, fires -// OnInstanceDown, and persists the running set. Shared by Terminate and -// MarkFailed. +// finalizeTerminated transitions to terminated, writes the KV entry, removes +// from the local map, fires OnInstanceDown, and persists the running set. func (m *Manager) finalizeTerminated(instance *VM) error { // Inspect (not UpdateState): MarkFailed may invoke this for an // instance that was never inserted into the local map. @@ -415,12 +370,9 @@ func (m *Manager) deallocateResources(instance *VM) { m.deps.Resources.Deallocate(instance.InstanceType) } -// transitionWithPrecheck validates the transition first, then calls the -// daemon-supplied TransitionState. Pre-validation lets us surface -// ErrInvalidTransition cleanly (so callers can map it to the AWS -// IncorrectInstanceState error code) and treat any post-precheck error -// as a persistence failure on a transition whose memory mutation -// already succeeded. +// transitionWithPrecheck validates the transition then calls TransitionState. +// Surfaces ErrInvalidTransition cleanly; post-precheck errors are persistence +// failures on a transition whose in-memory mutation already succeeded. func (m *Manager) transitionWithPrecheck(instance *VM, target InstanceState) error { current := m.Status(instance) if !IsValidTransition(current, target) { @@ -446,9 +398,8 @@ func (m *Manager) transitionWithPrecheck(instance *VM, target InstanceState) err return nil } -// writeRunningState persists the current running-VM map via the StateStore. -// The View callback holds the manager lock across the marshal+put so VM -// fields can't change mid-encode; splitting marshal from put is deferred. +// writeRunningState persists the running-VM map. View holds the lock across +// marshal+put to prevent field changes mid-encode. func (m *Manager) writeRunningState() error { if m.deps.StateStore == nil { return nil diff --git a/spinifex/vm/state.go b/spinifex/vm/state.go index 50edf8585..bf83a2616 100644 --- a/spinifex/vm/state.go +++ b/spinifex/vm/state.go @@ -23,8 +23,7 @@ type EC2StateInfo struct { } // EC2StateCodes maps each InstanceState to its EC2 API code and name. -// Note: StateError and StateProvisioning are Spinifex-specific states with no direct -// AWS EC2 equivalent. Their Code/Name values are best-effort mappings. +// StateError and StateProvisioning have no AWS equivalent; their mappings are best-effort. var EC2StateCodes = map[InstanceState]EC2StateInfo{ StateProvisioning: {Code: 0, Name: "pending"}, StatePending: {Code: 0, Name: "pending"}, @@ -36,13 +35,9 @@ var EC2StateCodes = map[InstanceState]EC2StateInfo{ StateError: {Code: 0, Name: "error"}, } -// EC2APIState returns the AWS-API-facing state code and name for status, -// projecting Spinifex-internal states with no AWS equivalent onto the closest -// valid lifecycle state. AWS InstanceStateName is a closed enum, so leaking the -// internal "error" label breaks SDK/UI clients. A StateError instance (crash or -// recovery-failed: QEMU dead, resources released, volumes unmounted) is, to the -// outside world, stopped — it can be started or terminated. ok is false only for -// a status with no mapping, letting callers apply their own fallback. +// EC2APIState returns the AWS-API state for status. StateError maps to "stopped" +// because AWS InstanceStateName is a closed enum and StateError instances can be +// started or terminated like stopped ones. ok is false for unmapped statuses. func EC2APIState(status InstanceState) (EC2StateInfo, bool) { if status == StateError { return EC2StateCodes[StateStopped], true @@ -51,8 +46,8 @@ func EC2APIState(status InstanceState) (EC2StateInfo, bool) { return info, ok } -// ValidTransitions defines the allowed state transitions for an instance. -// StateTerminated is intentionally absent — it is a terminal state with no valid transitions. +// ValidTransitions defines allowed state transitions. StateTerminated is absent +// as it is a terminal state. var ValidTransitions = map[InstanceState][]InstanceState{ StateProvisioning: {StateRunning, StateError, StateShuttingDown}, StatePending: {StateRunning, StateError, StateShuttingDown}, diff --git a/spinifex/vm/state_store.go b/spinifex/vm/state_store.go index be9c88084..d91814cef 100644 --- a/spinifex/vm/state_store.go +++ b/spinifex/vm/state_store.go @@ -1,12 +1,8 @@ package vm -// StateStore persists VM state to the per-node "running" bucket and to the -// cluster-shared "stopped" / "terminated" buckets. -// -// The interface lives in the vm package; the live implementation is a thin -// adapter in the daemon package wrapping the JetStream-backed bucket. Both -// vm.Manager and daemon-side handlers route VM-instance state I/O through -// this interface so the daemon can be exercised against an in-memory fake. +// StateStore persists VM state to the per-node "running" bucket and the +// cluster-shared "stopped"/"terminated" buckets. The live implementation is a +// JetStream adapter in the daemon package; tests inject an in-memory fake. type StateStore interface { // SaveRunningState writes the snapshot of VMs currently running on the // given node. The supplied map is owned by the caller and must not be diff --git a/spinifex/vm/vm.go b/spinifex/vm/vm.go index 6218c8240..cec36d2e4 100644 --- a/spinifex/vm/vm.go +++ b/spinifex/vm/vm.go @@ -49,13 +49,10 @@ type VM struct { // User attributes (user initiated stop/delete) Attributes types.EC2CommandAttributes `json:"attributes"` - // EC2 API metadata - stored for AWS API compatibility - // RunInstancesInput contains the original request parameters (ImageId, KeyName, UserData, etc.) + // EC2 API metadata stored for AWS API compatibility. RunInstancesInput *ec2.RunInstancesInput `json:"run_instances_input,omitempty"` - // Reservation contains the reservation metadata (ReservationId, OwnerId, etc.) - Reservation *ec2.Reservation `json:"reservation,omitempty"` - // Instance contains the current instance state and metadata - Instance *ec2.Instance `json:"instance,omitempty"` + Reservation *ec2.Reservation `json:"reservation,omitempty"` + Instance *ec2.Instance `json:"instance,omitempty"` // LastNode records which daemon node last ran this instance. // Set when ownership is released on stop for shared KV storage. @@ -67,30 +64,24 @@ type VM struct { // Health tracks crash detection and auto-restart state Health InstanceHealthState `json:"health"` - // AccountID is the AWS account that owns this instance. - // Empty AccountID identifies legacy/migration resources that are only - // visible to the root account. + // AccountID is the AWS account that owns this instance. Empty for legacy + // resources visible only to the root account. AccountID string `json:"account_id,omitempty"` // VPC networking: ENI attached to this instance (set by RunInstances when VPC mode is active) ENIId string `json:"eni_id,omitempty"` ENIMac string `json:"eni_mac,omitempty"` - // ExtraENIs lists additional VPC NICs beyond the primary ENIId/ENIMac. - // Used by multi-AZ system VMs (ALBs with subnets in multiple subnets) — - // each entry gets its own tap device on br-int and its own QEMU NIC. - // Empty for customer EC2 instances and single-subnet ALBs. + // ExtraENIs lists additional VPC NICs beyond the primary ENI. + // Each gets its own tap on br-int. Empty for single-ENI instances. ExtraENIs []ExtraENI `json:"extra_enis,omitempty"` // Public IP auto-assigned from external IPAM pool (released on termination) PublicIP string `json:"public_ip,omitempty"` PublicIPPool string `json:"public_ip_pool,omitempty"` - // PublicIPAllocID / PublicIPAssocID are set when the public IP was - // allocated through the EIP service (handlers/ec2/eip) rather than - // directly via externalIPAM. Presence signals that teardown must go - // back through the EIP service (DisassociateAddress + ReleaseAddress) - // so the EIP KV record is removed. + // PublicIPAllocID / PublicIPAssocID are set when the IP was allocated via the + // EIP service; teardown must go through DisassociateAddress + ReleaseAddress. PublicIPAllocID string `json:"public_ip_alloc_id,omitempty"` PublicIPAssocID string `json:"public_ip_assoc_id,omitempty"` @@ -123,33 +114,25 @@ type VM struct { // attached. Auto-cleared on TerminateInstances; preserved across stop/start. IamInstanceProfileArn string `json:"iam_instance_profile_arn,omitempty"` - // IamInstanceProfileAssociationId is the synthesized AWS-style association - // ID (iip-assoc-...) used by DescribeIamInstanceProfileAssociations and - // Disassociate/Replace. Empty when no profile is attached. Regenerated on - // every Associate/Replace — never reused. + // IamInstanceProfileAssociationId is the association ID used by + // DescribeIamInstanceProfileAssociations. Regenerated on every Associate/Replace. IamInstanceProfileAssociationId string `json:"iam_instance_profile_association_id,omitempty"` - // DirectBoot signals that this VM was configured with a pre-built - // vm.Config by the launcher (microvm direct-boot path). When true, - // startQEMU skips buildBaseVMConfig and uses the existing Config - // instead. False for all PC-machine VMs. + // DirectBoot signals that Config was pre-built by the launcher (microvm path). + // When true, startQEMU skips buildBaseVMConfig. DirectBoot bool `json:"direct_boot,omitempty"` - // GPUAttachments describes how GPUs are attached to this instance. Empty for - // non-GPU instances. Each entry carries either a PCI address (whole-GPU VFIO - // passthrough) or an mdev path (MIG slice), never both. + // GPUAttachments describes GPU attachments. Each entry has either a PCI address + // (whole-GPU VFIO passthrough) or an mdev path (MIG slice), never both. GPUAttachments []gpu.GPUAttachment `json:"gpu_attachments,omitempty"` - // BootMode captures the AMI's boot mode ("bios" | "uefi" | "uefi-preferred") - // at launch time so the firmware choice survives daemon restart and host - // migration without re-querying AMIMetadata. Empty for legacy VMs launched - // before this field existed; treated as "bios" by the launch path. + // BootMode captures the AMI's boot mode at launch so firmware choice survives + // restarts. Empty for legacy VMs; treated as "bios" by the launch path. BootMode string `json:"boot_mode,omitempty"` } -// ResetNodeLocalState zeroes out fields that are specific to the daemon node -// that last ran this instance. Must be called after deserializing a VM from -// shared KV before launching it on a new node. +// ResetNodeLocalState zeroes node-specific fields after deserializing a VM +// from shared KV before launching it on a new node. func (v *VM) ResetNodeLocalState() { v.MetadataServerAddress = "" v.QMPClient = &qmp.QMPClient{} @@ -217,11 +200,8 @@ type Config struct { InstanceType string `json:"instance_type"` Architecture string `json:"architecture"` - // UseUEFI requests UEFI firmware (OVMF on x86_64, AAVMF on arm64) instead of - // SeaBIOS / no-firmware boot. Execute() probes firmwarePathCandidates and - // returns an error if no matching code+vars pair is installed — no silent - // SeaBIOS fallback, since a UEFI-only guest booted under SeaBIOS panics on - // missing ESP. + // UseUEFI requests UEFI firmware. Execute() probes FirmwarePathCandidates and + // returns an error if no pair is found — no silent SeaBIOS fallback. UseUEFI bool `json:"use_uefi,omitempty"` KernelImage string `json:"kernel_image,omitempty"` // path to vmlinuz; emits -kernel when set diff --git a/spinifex/vm/volumes.go b/spinifex/vm/volumes.go index 5110fc8a5..c2b4ae43f 100644 --- a/spinifex/vm/volumes.go +++ b/spinifex/vm/volumes.go @@ -19,29 +19,17 @@ import ( // gives a 20s budget. const blockdevDelMaxAttempts = 20 -// AttachVolumeResult carries the device names produced by AttachVolume: -// the AWS-API name (/dev/sd[f-p], possibly auto-allocated when the -// caller passed an empty device) and the in-guest path discovered from -// QMP query-block (/dev/vd*). Daemon handlers echo DeviceName in the -// AttachVolume response and into volume metadata so AWS-spec round-trips -// (Volume.Attachments[].Device, the attachment.device filter on -// DescribeVolumes) match. GuestDevice is retained for diagnostics and -// for any internal callers that need to map back to the guest's virtio -// enumeration. +// AttachVolumeResult carries the AWS-API device name (/dev/sd[f-p]) and the +// in-guest path (/dev/vd*) discovered via QMP. DeviceName is echoed in the +// API response and volume metadata; GuestDevice is used for diagnostics. type AttachVolumeResult struct { DeviceName string GuestDevice string } -// AttachVolume hot-plugs a volume into a running instance via the -// three-phase QMP pipeline (mount → blockdev-add → device_add). On -// failure mid-pipeline, partial state is rolled back. The instance must -// be in StateRunning. If device is empty, the next free /dev/sd[f-p] -// slot is allocated. -// -// Daemon handlers retain volume-side validation (existence, ownership, -// availability, AZ) and emit the AWS API response; the manager owns -// every QMP and persistence side-effect. +// AttachVolume hot-plugs a volume via the QMP pipeline (mount → blockdev-add → +// device_add). Partial state is rolled back on failure. If device is empty, the +// next free /dev/sd[f-p] slot is allocated. Instance must be in StateRunning. func (m *Manager) AttachVolume(id, volumeID, device string) (AttachVolumeResult, error) { instance, ok := m.Get(id) if !ok { @@ -242,15 +230,9 @@ func (m *Manager) AttachVolume(id, volumeID, device string) (AttachVolumeResult, return AttachVolumeResult{DeviceName: device, GuestDevice: guestDevice}, nil } -// DetachVolume hot-unplugs a volume from a running instance via the -// three-phase QMP pipeline (device_del → blockdev-del → object-del → -// ebs.unmount). Returns the recorded AWS-API device name on success so -// the daemon handler can echo it in the response. -// -// device may be empty (no cross-check) or must equal the recorded -// DeviceName. force=true tolerates a failing device_del; blockdev-del -// is always required because tearing down the NBD server underneath a -// live block node would crash the VM. +// DetachVolume hot-unplugs a volume via the QMP pipeline (device_del → +// blockdev-del → object-del → ebs.unmount). Returns the recorded AWS-API device +// name. force=true tolerates device_del failure; blockdev-del is always required. func (m *Manager) DetachVolume(id, volumeID, device string, force bool) (string, error) { instance, ok := m.Get(id) if !ok { @@ -375,11 +357,8 @@ func (m *Manager) DetachVolume(id, volumeID, device string, force bool) (string, return ebsReq.DeviceName, nil } -// tryBlockdevDel issues blockdev-del with bounded retry on "is in use" -// errors. After device_del, NBD client teardown and any in-flight guest -// I/O can briefly hold the block node; QEMU surfaces this as a -// GenericError carrying "Node is in use". Polling at DetachDelay -// gives the I/O drain enough time without a hardcoded long sleep. +// tryBlockdevDel issues blockdev-del with bounded retry on "is in use" errors. +// QEMU surfaces this as GenericError while NBD teardown and in-flight I/O drain. func (m *Manager) tryBlockdevDel(instance *VM, nodeName string) error { var lastErr error for attempt := 1; attempt <= blockdevDelMaxAttempts; attempt++ { @@ -410,9 +389,8 @@ func (m *Manager) tryBlockdevDel(instance *VM, nodeName string) error { return lastErr } -// isQMPDeviceNotFound returns true when err is a QMP DeviceNotFound class -// error from device_del. Used to make device_del idempotent across detach -// retries. +// isQMPDeviceNotFound reports whether err is a QMP DeviceNotFound error, +// making device_del idempotent across retries. func isQMPDeviceNotFound(err error) bool { if err == nil { return false @@ -420,10 +398,8 @@ func isQMPDeviceNotFound(err error) bool { return strings.Contains(err.Error(), "DeviceNotFound") } -// isQMPNodeInUse returns true when err is a QMP GenericError reporting a -// block node still in use. blockdev-del fails this way while NBD client -// teardown and queued I/O drain after device_del. QEMU phrases this as -// both "X is in use" and "X is still in use" — match "in use". +// isQMPNodeInUse reports whether err is a QMP GenericError for a node still in +// use. Matches both "is in use" and "is still in use" phrasing. func isQMPNodeInUse(err error) bool { if err == nil { return false diff --git a/spinifex/vm/watchdog.go b/spinifex/vm/watchdog.go index f54274f17..892cd0651 100644 --- a/spinifex/vm/watchdog.go +++ b/spinifex/vm/watchdog.go @@ -16,11 +16,9 @@ const ( PendingWatchdogTimeout = 5 * time.Minute ) -// StartPendingWatchdog spawns a background goroutine that periodically -// scans for instances stuck in Pending/Provisioning beyond -// PendingWatchdogTimeout and marks them failed via Manager.MarkFailed. The -// goroutine exits when ctx is cancelled. Safe to call once per Manager -// (additional calls would duplicate the goroutine). +// StartPendingWatchdog spawns a goroutine that marks instances stuck in +// Pending/Provisioning beyond PendingWatchdogTimeout as failed. Exits when +// ctx is cancelled. Call once per Manager to avoid duplicating the goroutine. func (m *Manager) StartPendingWatchdog(ctx context.Context) { ticker := time.NewTicker(PendingWatchdogInterval) go func() { diff --git a/spinifex/vpcd/vpcd.go b/spinifex/vpcd/vpcd.go index 726366e1d..9d1c57067 100644 --- a/spinifex/vpcd/vpcd.go +++ b/spinifex/vpcd/vpcd.go @@ -29,32 +29,18 @@ import ( "github.com/nats-io/nats.go" ) -// Bridge mode constants for external connectivity. Bridge mode selects how -// the WAN NIC reaches the OVS integration bridge: -// - Direct: WAN NIC is added directly to br-external as an OVS port. -// Enables distributed NAT. Only safe when the WAN NIC is NOT the SSH/ -// management NIC. -// - Veth: a veth pair links a Linux bridge (br-wan) to the OVS bridge -// (br-ext). Requires centralized NAT — the Linux bridge intermediary -// breaks distributed NAT hairpin routing. +// Bridge mode selects how the WAN NIC reaches the OVS bridge. +// Direct: WAN NIC added to OVS (distributed NAT, not safe on mgmt NIC). +// Veth: veth pair links a Linux bridge to OVS (requires centralized NAT). const ( BridgeModeDirect = "direct" BridgeModeVeth = "veth" - // OvnExternalBridge is the OVS bridge that ovn-bridge-mappings targets - // for the "external" localnet. Owned by setup-ovn.sh and independent of - // the WAN bridge (which is Linux-side in veth mode). + // OvnExternalBridge is the OVS bridge targeted by ovn-bridge-mappings for the "external" localnet. OvnExternalBridge = "br-ext" ) -// waitForFlowsHV shells out to `ovn-nbctl --wait=hv sync`, bumping -// NB_Global.nb_cfg and blocking until every connected chassis has acknowledged -// the new sequence — i.e. ovn-northd has compiled NB→SB and ovn-controller -// has installed the resulting flows. Used as the NAT manager's flows barrier -// so newly-launched VMs aren't unreachable while their gateway chassis is -// still catching up. Bounded by ovn-nbctl --timeout=30; on overrun we log -// a Warn and return nil so the caller continues. -// -// Declared as a var so tests can stub it. +// waitForFlowsHV runs `ovn-nbctl --wait=hv sync`, blocking until all chassis acknowledge the new NB sequence. +// Bounded at 30 s; overruns log a Warn and return nil. Declared as a var so tests can stub it. var waitForFlowsHV = func() error { start := time.Now() cmd := sudoCommand("ovn-nbctl", @@ -75,9 +61,7 @@ var waitForFlowsHV = func() error { return nil } -// sudoCommand wraps exec.Command with sudo when running as non-root. -// OVS/OVN commands require elevated privileges; when running as root -// (Docker, production) no wrapper is needed. +// sudoCommand wraps exec.Command with sudo when not root; OVS/OVN commands require elevated privileges. func sudoCommand(name string, args ...string) *exec.Cmd { if os.Getuid() == 0 { return exec.Command(name, args...) @@ -87,13 +71,10 @@ func sudoCommand(name string, args ...string) *exec.Cmd { var serviceName = "vpcd" -// host.GatewayClaimProber backs the reconciler's SB chassis-claim verifier; the -// check lives here because host (L0) cannot import reconcile (cross-cutter). +// Compile-time check: host.GatewayClaimProber implements reconcile.GatewayClaimVerifier. var _ reconcile.GatewayClaimVerifier = (*host.GatewayClaimProber)(nil) -// BootstrapVPC holds the default VPC infrastructure IDs from spinifex.toml. -// vpcd uses this to ensure OVN topology exists on first boot (before NATS KV -// has any data) and on subsequent boots where OVN state may have been lost. +// BootstrapVPC holds the default VPC IDs from spinifex.toml used to seed OVN topology on first boot. type BootstrapVPC struct { AccountID string VpcId string @@ -216,24 +197,16 @@ var checkBrInt = func() error { return nil } -// checkOVNController verifies that ovn-controller is running on this host. -// OVN 22.03+ moved the control socket from /var/run/openvswitch/ to /var/run/ovn/, -// so the short-form "ovs-appctl -t ovn-controller" fails on newer packages. -// We try the legacy path first, then the new path, then fall back to systemctl. +// checkOVNController verifies ovn-controller is running. Tries legacy socket path, then OVN 22.03+ path, then systemctl. var checkOVNController = func() error { - // Legacy path: works when socket is in /var/run/openvswitch/ if sudoCommand("ovs-appctl", "-t", "ovn-controller", "version").Run() == nil { return nil } - - // OVN 22.03+: socket moved to /var/run/ovn/ if matches, _ := filepath.Glob("/var/run/ovn/ovn-controller.*.ctl"); len(matches) > 0 { if sudoCommand("ovs-appctl", "-t", matches[0], "version").Run() == nil { return nil } } - - // Fallback: check if the service is active via systemctl if sudoCommand("systemctl", "is-active", "--quiet", "ovn-controller").Run() == nil { return nil } @@ -241,16 +214,9 @@ var checkOVNController = func() error { return fmt.Errorf("ovn-controller is not running: run ./scripts/setup-ovn.sh --management") } -// localSystemID returns the OVS external-ids:system-id, which is the chassis -// name that the local ovn-controller registers in the Southbound DB. -// -// Uses Output() (stdout only) for the same reason as portToBr: vpcd.service's -// AmbientCapabilities trips sudo's PAM into emitting audit warnings on stderr, -// which CombinedOutput would merge into stdout and poison the system-id. -// A corrupted localID causes discoverChassis to skip the live chassis as -// "stale", leaving gateway_chassis pointing at a fallback name that no real -// chassis owns → cr-gw* chassisredirect ports stay unbound → no proxy-ARP -// for EIPs. +// localSystemID returns the OVS external-ids:system-id (the chassis name in the Southbound DB). +// Uses Output() not CombinedOutput(): AmbientCapabilities causes sudo's PAM to emit stderr noise +// that would corrupt the system-id and cause discoverChassis to misidentify the local chassis. var localSystemID = func() (string, error) { out, err := sudoCommand("ovs-vsctl", "get", "open_vswitch", ".", "external-ids:system-id").Output() if err != nil { @@ -260,15 +226,11 @@ var localSystemID = func() (string, error) { } return "", fmt.Errorf("ovs-vsctl get system-id: %s: %w", stderr, err) } - // ovs-vsctl wraps the value in quotes return strings.Trim(strings.TrimSpace(string(out)), "\""), nil } -// discoverChassis queries the OVN Southbound DB for registered chassis names. -// It cross-references with the local OVS system-id to filter out stale chassis -// entries on this host. When the system-id is changed (e.g. setup-ovn.sh re-run), -// the old Chassis row persists in the SBDB — using it for gateway scheduling -// causes the gateway port to bind to a chassis that no ovn-controller owns. +// discoverChassis queries the OVN Southbound DB for chassis names, filtering out stale local entries. +// Stale rows (from a system-id change) must be excluded to prevent gateway ports binding to phantom chassis. var discoverChassis = func(sbAddr string) ([]string, error) { localID, err := localSystemID() if err != nil { @@ -280,11 +242,9 @@ var discoverChassis = func(sbAddr string) ([]string, error) { if sbAddr != "" { args = append(args, "--db="+sbAddr) } - // OVN 25.03+ removed the "list-chassis" convenience command. - // Use "--columns=name,hostname list Chassis" which works on all versions. + // OVN 25.03+ removed "list-chassis"; use "--columns=name,hostname list Chassis" instead. + // Output() not CombinedOutput(): sudo PAM stderr would corrupt name/hostname parsing. args = append(args, "--bare", "--columns=name,hostname", "list", "Chassis") - // Output() not CombinedOutput(): sudo PAM audit noise on stderr would - // otherwise be parsed as chassis name/hostname pairs. out, err := sudoCommand("ovn-sbctl", args...).Output() if err != nil { var stderr string @@ -297,16 +257,14 @@ var discoverChassis = func(sbAddr string) ([]string, error) { return parseChassisList(string(out), localID, localHostname), nil } -// parseChassisList parses ovn-sbctl --bare --columns=name,hostname output and -// filters out stale chassis on the local host. The output format is pairs of -// name/hostname lines separated by blank lines. +// parseChassisList parses ovn-sbctl --bare --columns=name,hostname output (name/hostname pairs separated by blank lines) +// and filters out stale chassis on the local host. func parseChassisList(raw, localID, localHostname string) []string { var names []string var pair []string for line := range strings.SplitSeq(strings.TrimSpace(raw), "\n") { line = strings.TrimSpace(line) if line == "" { - // Blank line = row separator; process the accumulated pair if len(pair) == 2 { names = appendIfActive(names, pair[0], pair[1], localID, localHostname) } @@ -315,8 +273,7 @@ func parseChassisList(raw, localID, localHostname string) []string { } pair = append(pair, line) } - // Handle last row (no trailing blank line) - if len(pair) == 2 { + if len(pair) == 2 { // last row may have no trailing blank line names = appendIfActive(names, pair[0], pair[1], localID, localHostname) } return names @@ -341,12 +298,8 @@ func preflightOVN() error { return nil } -// externalCIDRFromBridge returns the first IPv4 CIDR assigned to the named -// bridge interface. Used to discover the host's uplink network at startup -// (the OS assigns this via netplan static config or systemd-networkd DHCP -// before Spinifex starts). -// -// Injected as a var so tests can stub it without requiring a real interface. +// externalCIDRFromBridge returns the first IPv4 CIDR on the named bridge. +// Injected as a var so tests can stub it without a real interface. var externalCIDRFromBridge = func(bridge string) (netip.Prefix, error) { iface, err := net.InterfaceByName(bridge) if err != nil { @@ -372,14 +325,8 @@ var externalCIDRFromBridge = func(bridge string) (netip.Prefix, error) { return netip.Prefix{}, fmt.Errorf("no IPv4 address on %q", bridge) } -// resolveExternalCIDR blocks until the WAN bridge has an IPv4 address or the -// timeout elapses. This guards the boot race where vpcd starts before -// systemd-networkd finishes DHCP on br-wan, or before netplan applies a -// static config. Returns the resolved CIDR for downstream consumers. -// -// Forward-compatible with the ADR-0006 L0 contract: this becomes the backing -// implementation of HostWiring.ExternalCIDR() once the network/host package -// lands. Until then the value is only used to fail-start on missing uplink. +// resolveExternalCIDR blocks until the WAN bridge has an IPv4 address or timeout elapses. +// Guards the boot race where vpcd starts before systemd-networkd or netplan assigns the uplink address. func resolveExternalCIDR(ctx context.Context, bridge string, timeout time.Duration) (netip.Prefix, error) { const retryDelay = 500 * time.Millisecond deadline := time.Now().Add(timeout) @@ -407,13 +354,8 @@ func resolveExternalCIDR(ctx context.Context, bridge string, timeout time.Durati } } -// ensureExternalCIDRReady blocks at startup until the WAN bridge has an IPv4 -// address, or returns an error if resolution fails within the timeout. The OS -// (netplan static or systemd-networkd DHCP) assigns the uplink address before -// Spinifex starts in steady-state; a missing address at this point means a -// boot race or misconfiguration. Bounded retry so systemd's -// Restart=on-failure does not flap through transient gaps. No-op when -// externalMode is empty (overlay-only deployments). +// ensureExternalCIDRReady blocks until the WAN bridge has an IPv4 address within the timeout. +// No-op when externalMode is empty (overlay-only). Missing address indicates boot race or misconfiguration. func ensureExternalCIDRReady(ctx context.Context, externalMode, bridge string) error { if externalMode == "" { return nil @@ -433,14 +375,12 @@ func launchService(cfg *Config) error { "nats_host", cfg.NatsHost, ) - // OVN preflight: verify br-int and ovn-controller before proceeding if err := preflightOVN(); err != nil { slog.Error("OVN preflight check failed — vpcd cannot start without OVN", "err", err) return err } slog.Info("OVN preflight passed (br-int exists, ovn-controller running)") - // Connect to NATS nc, err := utils.ConnectNATSWithRetry(admin.DialTarget(cfg.NatsHost), cfg.NatsToken, cfg.NatsCACert) if err != nil { slog.Error("Failed to connect to NATS", "err", err) @@ -448,7 +388,6 @@ func launchService(cfg *Config) error { } defer nc.Close() - // Connect to OVN NB DB (required — vpcd is useless without it) if cfg.OVNNBAddr == "" { return fmt.Errorf("OVN NB DB address not configured (ovn_nb_addr is empty)") } @@ -469,8 +408,6 @@ func launchService(cfg *Config) error { return err } - // Resolve external CIDR from the WAN bridge before any reconcile. Skipped - // when external networking is disabled (overlay-only deployments). if err := ensureExternalCIDRReady(ctx, cfg.ExternalMode, wanBridge); err != nil { return err } @@ -478,11 +415,7 @@ func launchService(cfg *Config) error { if cfg.ExternalMode != "" { slog.Info("External network enabled", "mode", cfg.ExternalMode, "pools", len(cfg.ExternalPools)) } - // Discover chassis from OVN SBDB. The OVS-managed system-id (persisted at - // /etc/openvswitch/system-id.conf and re-applied on every boot) is the - // canonical chassis identity; SBDB is the only source of truth. Fail-start - // rather than guess — a missing chassis means ovn-controller hasn't - // registered yet (boot race) and systemd's Restart=on-failure will retry. + // Fail-start if no chassis found — missing chassis means ovn-controller hasn't registered yet (boot race). chassisNames, err := discoverChassis(cfg.OVNSBAddr) if err != nil { return fmt.Errorf("vpcd: discover OVN chassis: %w", err) @@ -498,10 +431,6 @@ func launchService(cfg *Config) error { } natMode := policy.NATModeFromUplinkMode(uplinkMode) - // Construct the network manager stack. The reconciler is the single - // intent-driven pass that runs once at startup (leader-gated) and then - // on a 5-minute drift ticker; together they replace the five separate - // passes the old vpcd ran on boot. var topoOpts []topology.Option if dns := pickDNSServer(cfg.ExternalPools); dns != "" { topoOpts = append(topoOpts, topology.WithDNSServer(func() string { return dns })) @@ -530,12 +459,8 @@ func launchService(cfg *Config) error { return fmt.Errorf("get JetStream context: %w", err) } - // IMDS per-subnet localport installer. Create the KV bucket at a single - // replica; the daemon's deferred upgradeJetStreamReplicas bumps all KV_* - // streams to the cluster size once the cluster is ready. Using the chassis - // count here requested more replicas than a single-node JetStream could - // satisfy (chassis count can exceed NATS node count), failing the create - // with "bucket not found". + // Create IMDS KV at replica=1; daemon upgrades replicas later. Using chassis count here + // could exceed the NATS node count and fail bucket creation. imdsVethKV, _, err := handlers_imds.InitBuckets(js, 1) if err != nil { return fmt.Errorf("init imds buckets: %w", err) @@ -545,9 +470,7 @@ func launchService(cfg *Config) error { return fmt.Errorf("construct IMDS topology manager: %w", err) } - // IMDS host-served datapath. vpcd holds the network capabilities the listener - // stack needs (the awsgw sandbox can't grant them); STS/IAM stay in awsgw over - // NATS RPCs. Run blocks, so it gets its own goroutine. + // vpcd holds the network capabilities needed for IMDS; STS/IAM stay in awsgw over NATS. imdsCtx, cancelIMDS := context.WithCancel(ctx) defer cancelIMDS() imdsSvc, err := handlers_imds.NewIMDSServiceImpl( @@ -603,12 +526,8 @@ func launchService(cfg *Config) error { return fmt.Errorf("construct NATGW manager: %w", err) } - // Elect a single vpcd to run startup reconcile. Without this, N vpcds in - // a multi-node cluster all hit Get-then-Create on Logical_Router with no - // atomicity, producing duplicate rows that ovn-nbctl rejects with - // "Multiple logical routers named '...'. Use a UUID." - // Runtime VPC events still fan out via the vpcd-workers queue group, so - // non-leaders remain functional after Subscribe below. + // Elect one vpcd for startup reconcile; without this, concurrent Get-then-Create on Logical_Router + // produces duplicate rows that ovn-nbctl rejects. Runtime events still fan out via queue groups. holder, _ := os.Hostname() releaseLeader, isLeader := reconcile.AcquireLeader(nc, holder) @@ -652,20 +571,9 @@ func launchService(cfg *Config) error { return fmt.Errorf("construct reconciler: %w", err) } - // Startup reconcile: leader-gated read of NATS KV intent state, applied - // once against OVN NB DB. The drift loop below handles the recovery case - // (KV not yet populated when this fires — daemon's EnsureDefaultVPC may - // race with vpcd's startup). Per-event NATS subscribers route through - // the same network/* managers, so the startup pass and the runtime fast - // path share one implementation. - // - // ReconcileApplyOnly (not Reconcile) — orphan pruning is unsafe at - // startup because intent may be stale: daemon EnsureDefaultVPC on a - // peer node can be mid-flight (KV write committed but not yet visible - // here) while a peer subscriber concurrently creates port groups from - // its vpc.create-sg event. A startup prune then sweeps those port - // groups as orphans, breaking every subsequent RunInstances until the - // 5-minute drift tick recreates them. Drift uses full Reconcile. + // Startup reconcile (leader-gated, apply-only). Orphan pruning is skipped because intent may be stale: + // a peer's vpc.create-sg could be mid-flight and a prune would sweep those port groups as orphans. + // Drift loop uses full Reconcile. if isLeader { intent, intentErr := reconcile.LoadIntentFromKV(ctx, js, cfg.AZ) if intentErr != nil { @@ -676,9 +584,7 @@ func launchService(cfg *Config) error { releaseLeader() } - // Periodic drift detection. Leader-gated on the shared reconcile bucket - // so only one vpcd in the cluster scans at a time; cancelled when the - // parent ctx is. + // Periodic drift detection, leader-gated so only one vpcd scans at a time. loopCtx, loopCancel := context.WithCancel(ctx) defer loopCancel() loopDone := make(chan struct{}) @@ -690,7 +596,6 @@ func launchService(cfg *Config) error { slog.Info("vpcd service started, waiting for VPC lifecycle events", "subscriptions", len(subs)) - // Wait for shutdown signal sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) <-sigChan @@ -701,10 +606,8 @@ func launchService(cfg *Config) error { return nil } -// pickDNSServer returns the OVN dhcp_options dns_server string ("{a, b}") -// from the first unscoped (no region, no AZ) pool with DNS servers set. -// Empty string falls back to topology.NewLiveManager's default -// ("{8.8.8.8, 1.1.1.1}"). +// pickDNSServer returns the OVN dhcp_options dns_server from the first unscoped pool with DNS servers. +// Empty falls back to topology.NewLiveManager's default. func pickDNSServer(pools []ExternalPoolConfig) string { for _, p := range pools { if p.Region == "" && p.AZ == "" && len(p.DNSServers) > 0 { @@ -714,9 +617,8 @@ func pickDNSServer(pools []ExternalPoolConfig) string { return "" } -// startDHCPManagerIfNeeded constructs and starts the per-AZ DHCP Manager -// when any configured pool has Source="dhcp", and subscribes its NATS -// handlers. Returns (nil, nil, nil) when no pool needs DHCP. +// startDHCPManagerIfNeeded starts the per-AZ DHCP Manager when any pool has Source="dhcp". +// Returns (nil, nil, nil) when not needed. func startDHCPManagerIfNeeded(ctx context.Context, nc *nats.Conn, js nats.JetStreamContext, cfg *Config) (*dhcp.Manager, []*nats.Subscription, error) { if cfg == nil || cfg.ExternalMode == "" { return nil, nil, nil @@ -755,11 +657,7 @@ func startDHCPManagerIfNeeded(ctx context.Context, nc *nats.Conn, js nats.JetStr return mgr, subs, nil } -// pickGatewayAllocator chooses the GatewayIPAllocator wired into -// IGWManager. DHCP-sourced pools with a started Manager get the -// DHCPGatewayLRPAllocator; everything else keeps StaticRangeAllocator -// (centralized + static) or LinkLocalAllocator selection inside the -// resolveGatewayNetwork code path. +// pickGatewayAllocator returns a DHCPGatewayLRPAllocator for DHCP-sourced pools; otherwise StaticRangeAllocator. func pickGatewayAllocator(pool *external.ExternalPoolConfig, ovnClient ovn.Client, mgr *dhcp.Manager) external.GatewayIPAllocator { if pool.IsDHCP() && mgr != nil { return dhcp.NewDHCPGatewayLRPAllocator(mgr) @@ -767,10 +665,7 @@ func pickGatewayAllocator(pool *external.ExternalPoolConfig, ovnClient ovn.Clien return external.NewStaticRangeAllocator(ovnClient) } -// externalPoolConfigToShared translates the vpcd-local ExternalPoolConfig -// into the network/external package's shared shape. The two will merge -// once topology.go is split (see external.go's type doc); until then -// this is the seam. +// externalPoolConfigToShared translates vpcd's ExternalPoolConfig into the network/external shared type. func externalPoolConfigToShared(p ExternalPoolConfig) external.ExternalPoolConfig { return external.ExternalPoolConfig{ Name: p.Name, @@ -789,11 +684,7 @@ func externalPoolConfigToShared(p ExternalPoolConfig) external.ExternalPoolConfi } } -// resolveBridgeConfig picks the bridge mode and WAN bridge name to use, -// auto-detecting mode when unset. Empty mode stays empty — verifyBridgeMode -// rejects it with a list of supported values (D12). The WAN bridge name is -// always "br-wan" (the consumer-router convention) since vpcd no longer -// accepts a per-node override. +// resolveBridgeConfig picks bridge mode (auto-detecting when unset) and always uses "br-wan" as the WAN bridge. func resolveBridgeConfig(cfgBridgeMode, externalIface string) (string, string) { bridgeMode := cfgBridgeMode if bridgeMode == "" && externalIface != "" { @@ -802,11 +693,8 @@ func resolveBridgeConfig(cfgBridgeMode, externalIface string) (string, string) { return bridgeMode, "br-wan" } -// neighFlusher builds the post-AddEIP / post-DeleteEIP host ARP-flush hook. It -// flushes the kernel neighbour entry for the external IP on the Linux WAN -// bridge so a recycled IP re-resolves L2 against the current owner, keeping -// EIPs reachable without waiting on the kernel ARP timeout. No-op when -// wanBridge is unset. +// neighFlusher builds the ARP-flush hook for AddEIP/DeleteEIP so recycled IPs re-resolve L2 immediately. +// No-op when wanBridge is unset. func neighFlusher(wanBridge string) policy.NeighFlusher { return func(externalIP string) error { if wanBridge == "" { @@ -818,11 +706,8 @@ func neighFlusher(wanBridge string) policy.NeighFlusher { } } -// neighPrimer builds the post-AddEIP host ARP-prime hook for distributed EIPs. -// It programs the kernel neighbour entry for the external IP to the EIP's -// external_mac on the Linux WAN bridge, so a recycled IP is reachable -// immediately without an ARP reply that no node sends for a same-chassis -// recycled IP. No-op when wanBridge or the MAC is unset. +// neighPrimer builds the ARP-prime hook for distributed EIPs so recycled IPs are reachable immediately +// without waiting for an ARP reply. No-op when wanBridge or MAC is unset. func neighPrimer(wanBridge string) policy.NeighPrimer { return func(eip policy.EIPSpec) error { if wanBridge == "" || eip.MAC == "" { @@ -839,14 +724,8 @@ var ifaceExists = func(name string) bool { return exec.Command("ip", "link", "show", name).Run() == nil } -// detectBridgeMode checks how the WAN bridge is wired: -// - veth: veth-wan-ovs interface exists (Linux bridge linked to OVS via veth pair) -// - direct: physical NIC is added directly to the OVS bridge -// -// Each decision point logs at Info so `journalctl -u spinifex-vpcd | grep -// bridge` surfaces the full trail. The fall-through case logs at Warn — the -// silent Debug fall-through is what let the veth-persistence bug hide for -// weeks (mulga-998.b Fix 2). +// detectBridgeMode infers bridge mode: veth when veth-wan-ovs exists, direct otherwise. +// Each branch logs at Info/Warn so `journalctl | grep bridge` shows the full detection trail. func detectBridgeMode(externalIface string) string { if ifaceExists("veth-wan-ovs") { slog.Info("vpcd: detected veth pair linking Linux bridge to OVS", "mode", BridgeModeVeth) @@ -858,13 +737,8 @@ func detectBridgeMode(externalIface string) string { return BridgeModeDirect } -// portToBr returns the OVS bridge that owns `port`. Returns "" when the port -// is not in OVSDB. Used by the post-detect sanity checks. -// -// Uses Output() (stdout only) because vpcd.service runs with AmbientCapabilities -// set, which causes sudo's PAM to emit "sudo: unable to send audit message" -// warnings on stderr. CombinedOutput would merge those into stdout and poison -// the bridge-name compare. +// portToBr returns the OVS bridge owning port, or "" if not in OVSDB. +// Uses Output() not CombinedOutput(): AmbientCapabilities causes sudo PAM stderr that would corrupt the bridge name. var portToBr = func(port string) (string, error) { out, err := sudoCommand("ovs-vsctl", "port-to-br", port).Output() if err != nil { @@ -877,8 +751,7 @@ var portToBr = func(port string) (string, error) { return strings.TrimSpace(string(out)), nil } -// readLinkMaster returns the master of a kernel link by reading -// /sys/class/net//master. Returns "" if the link has no master. +// readLinkMaster returns the master of a kernel link via /sys/class/net//master, or "" if none. var readLinkMaster = func(iface string) (string, error) { target, err := os.Readlink(filepath.Join("/sys/class/net", iface, "master")) if err != nil { @@ -887,18 +760,9 @@ var readLinkMaster = func(iface string) (string, error) { return filepath.Base(target), nil } -// verifyBridgeMode is the post-detect sanity check. It refuses to start vpcd -// when the chosen bridge mode does not match the host plumbing (D4+D18): -// -// - direct: ExternalInterface must be an OVS port on the WAN bridge. That -// is the whole contract of direct mode. -// - veth: (a) veth-wan-ovs must be an OVS port on OvnExternalBridge — the -// OVN side, owned by setup-ovn.sh's ovn-bridge-mappings. (b) veth-wan-br -// must be enslaved to the WAN bridge — the Linux side. -// - empty / unknown: fail with the list of supported values. -// -// Fail-start, not soft-degrade — the distributed-NAT-on-veth-host footgun is -// exactly what this plan set out to kill. +// verifyBridgeMode validates that the chosen mode matches host plumbing. Fail-start on mismatch: +// direct requires ExternalInterface on the WAN OVS bridge; veth requires veth-wan-ovs on OvnExternalBridge +// and veth-wan-br enslaved to the WAN Linux bridge. func verifyBridgeMode(mode, externalIface, wanBridge string) error { switch mode { case BridgeModeDirect: diff --git a/tests/e2e/ddil/doc.go b/tests/e2e/ddil/doc.go index 11f56121a..63f1b7751 100644 --- a/tests/e2e/ddil/doc.go +++ b/tests/e2e/ddil/doc.go @@ -1,15 +1,4 @@ -// Package ddil is the root of the DDIL (Denied, Disrupted, Intermittent, -// Limited) E2E test harness. -// -// See docs/development/improvements/ddil-e2e-test-harness.md in the mulga -// monorepo for the full design. -// -// Sub-packages: -// - harness: typed primitives (cluster, SSH, fault injection, witness VMs, -// snapshot/state assertions, cleanup/retry, daemon client). -// - scenarios: test files (build tag e2e) that exercise the hardening -// epics via the harness primitives. -// -// All non-scaffolding files in this tree are tagged //go:build e2e so that -// default go build ./... skips them. +// Package ddil is the root of the DDIL (Denied, Disrupted, Intermittent, Limited) +// E2E test harness. Sub-packages: harness (typed primitives) and scenarios (test files). +// All non-scaffolding files carry //go:build e2e so default builds skip them. package ddil diff --git a/tests/e2e/ddil/fault/fault.go b/tests/e2e/ddil/fault/fault.go index 54b1c6ad4..188a2443d 100644 --- a/tests/e2e/ddil/fault/fault.go +++ b/tests/e2e/ddil/fault/fault.go @@ -15,17 +15,9 @@ import ( "github.com/mulgadc/spinifex/tests/e2e/harness" ) -// PartitionNode isolates target from peers by installing per-peer iptables -// DROP rules for both directions. The orchestrator's SSH source IP must not -// be one of peers (it normally isn't: the peers are cluster members, and the -// orchestrator is the CI runner). After applying, a sanity-check SSH echo -// confirms the control plane is still reachable; a lost control-plane SSH -// is a loud error rather than a silent hang. -// -// The rule set is additive (iptables -I INPUT/OUTPUT) so PartitionNode -// composes with any prior non-DDIL rules. HealNode's flush is still the -// correct teardown because the harness owns the target's firewall during a -// scenario. +// PartitionNode isolates target from peers with per-peer iptables DROP rules (INPUT+OUTPUT). +// A post-partition SSH echo confirms the orchestrator's control-plane path was not severed; +// rules are additive so PartitionNode composes with existing non-DDIL firewall state. func PartitionNode(ctx context.Context, ssh harness.SSH, target harness.Node, peers []harness.Node) error { if len(peers) == 0 { return fmt.Errorf("ddil fault: PartitionNode %s: no peers supplied", target.Name) @@ -43,9 +35,7 @@ func PartitionNode(ctx context.Context, ssh harness.SSH, target harness.Node, pe return fmt.Errorf("ddil fault: partition %s: %w", target.Name, err) } - // Sanity-check: if our SSH path ran through a peer IP, the rules we just - // installed would have severed it. Fail loudly so a future infra change - // (e.g. orchestrator moved onto the cluster network) surfaces immediately. + // Confirm orchestrator SSH still works; if not, the orchestrator shares a peer IP. if _, err := ssh.Run(ctx, target, "echo ok"); err != nil { return fmt.Errorf("ddil fault: partition %s severed orchestrator SSH "+ "(orchestrator may share peer IPs): %w", target.Name, err) @@ -62,9 +52,8 @@ func HealNode(ctx context.Context, ssh harness.SSH, target harness.Node) error { return nil } -// KillNATS stops spinifex-nats on node without touching spinifex-daemon. -// This is the scenario shape the current happy-path suite cannot express — -// `systemctl stop spinifex.target` brings both down together. +// KillNATS stops spinifex-nats without touching spinifex-daemon (unlike systemctl stop +// spinifex.target which brings both down). func KillNATS(ctx context.Context, ssh harness.SSH, node harness.Node) error { if _, err := ssh.Run(ctx, node, "sudo systemctl stop spinifex-nats"); err != nil { return fmt.Errorf("ddil fault: kill nats on %s: %w", node.Name, err) @@ -80,9 +69,8 @@ func StartNATS(ctx context.Context, ssh harness.SSH, node harness.Node) error { return nil } -// RestartDaemonOnly restarts spinifex-daemon without touching spinifex-nats. -// Used by Scenario B to exercise the daemon-without-NATS startup path -// (daemon-local-autonomy §1d). +// RestartDaemonOnly restarts spinifex-daemon without touching spinifex-nats, +// exercising the daemon-without-NATS startup path. func RestartDaemonOnly(ctx context.Context, ssh harness.SSH, node harness.Node) error { if _, err := ssh.Run(ctx, node, "sudo systemctl restart spinifex-daemon"); err != nil { return fmt.Errorf("ddil fault: restart daemon on %s: %w", node.Name, err) diff --git a/tests/e2e/ddil/fault/profiles.go b/tests/e2e/ddil/fault/profiles.go index 65eaf8a4f..ed2054336 100644 --- a/tests/e2e/ddil/fault/profiles.go +++ b/tests/e2e/ddil/fault/profiles.go @@ -4,10 +4,7 @@ package fault import "time" -// LinkProfile describes a tc netem configuration approximating a real-world -// tactical link. Values are unvalidated approximations derived from public -// specifications; see docs/development/improvements/ddil-e2e-test-harness.md -// §2 and spinifex/tests/e2e/TEST_COVERAGE.md for validation tracking. +// LinkProfile describes a tc netem configuration approximating a real-world tactical link. type LinkProfile struct { Name string Delay time.Duration @@ -17,18 +14,14 @@ type LinkProfile struct { Flapping *FlapSpec } -// FlapSpec toggles link up/down for profiles that simulate jammed or -// intermittent RF. The flap loop itself is not driven by ApplyNetem — it is -// orchestrated by a higher-level helper (added with the scenarios that use -// it) so scenarios keep full control over when the flap stops. +// FlapSpec toggles link up/down for profiles simulating jammed or intermittent RF. +// The flap loop is driven by a higher-level helper, not by ApplyNetem. type FlapSpec struct { Up time.Duration Down time.Duration } -// Authoritative DDIL link profile values. See the §2 table in the design doc -// for the rationale behind each row. Replace with measured values when -// real-hardware access is available. +// Predefined DDIL link profiles. Replace with measured hardware values when available. var ( LAN = LinkProfile{ Name: "LAN", diff --git a/tests/e2e/ddil/fault/snapshot.go b/tests/e2e/ddil/fault/snapshot.go index d22a7ff40..f5657c005 100644 --- a/tests/e2e/ddil/fault/snapshot.go +++ b/tests/e2e/ddil/fault/snapshot.go @@ -10,23 +10,13 @@ import ( "github.com/mulgadc/spinifex/tests/e2e/harness" ) -// InstanceSnapshot captures a point-in-time view of a single daemon's locally -// known instances. Scenarios take a snapshot before injecting a fault and a -// second snapshot after the cluster heals, then call AssertPreserved to prove -// the daemon did not lose or regress any instance it had been tracking. -// -// Reuses the LocalInstance type from harness/daemon_client.go. Once -// daemon-local-autonomy §1a lands, that placeholder will be replaced by the -// daemon's authoritative struct and this alias will pick up the real schema -// without scenario changes. +// InstanceSnapshot captures a point-in-time view of a daemon's locally known instances. +// Take one before and after a fault/heal cycle, then call AssertPreserved to prove no +// instances were lost or regressed to a terminal state. type InstanceSnapshot []harness.LocalInstance -// TakeSnapshot reads /local/instances on the given node and returns the -// response as an InstanceSnapshot. -// -// The DaemonClient is passed explicitly (rather than through a package-level -// singleton) so each scenario owns its connection reuse and so dry-run -// scenarios can stub with a fake client once the endpoint exists. +// TakeSnapshot reads /local/instances on node and returns the result as an InstanceSnapshot. +// DaemonClient is passed explicitly so each scenario controls its own connection reuse. func TakeSnapshot(ctx context.Context, d *harness.DaemonClient, node harness.Node) (InstanceSnapshot, error) { xs, err := d.Instances(ctx, node) if err != nil { @@ -35,16 +25,8 @@ func TakeSnapshot(ctx context.Context, d *harness.DaemonClient, node harness.Nod return InstanceSnapshot(xs), nil } -// AssertPreserved fails t if any instance present in the pre-snapshot is -// absent from post, or if any instance regressed from a live state (pending, -// running) to a terminal state (shutting-down, stopped, terminated) without -// explicit scenario intent. -// -// EC2-style state ordering is used rather than the daemon's internal enum -// because /local/instances reports the same `state` string the gateway -// exposes over `ec2.DescribeInstances`. When daemon-local-autonomy §1a -// replaces LocalInstance with the daemon type, the stateOrdinal map below -// should be pointed at the authoritative constants. +// AssertPreserved fails t if any instance present in pre is absent from post or has +// regressed from a live state (pending/running) to a terminal state without explicit scenario intent. func (pre InstanceSnapshot) AssertPreserved(t *testing.T, post InstanceSnapshot) { t.Helper() @@ -78,9 +60,7 @@ func (pre InstanceSnapshot) AssertPreserved(t *testing.T, post InstanceSnapshot) t.FailNow() } -// stateOrdinal maps EC2 instance states to a monotonic lifecycle position. -// Unknown states return -1 so we can distinguish them from "pending" (0) and -// refuse to flag a regression we can't reason about. +// stateOrdinal maps EC2 instance states to lifecycle position; -1 for unknown. func stateOrdinal(s string) int { switch s { case "pending": @@ -100,11 +80,8 @@ func stateOrdinal(s string) int { } } -// stateRegressed reports whether a live instance (pending/running) moved to -// a terminal state (stopping/stopped/shutting-down/terminated). Moves within -// the live band or within the terminal band are not flagged, and unknown -// states are treated as non-regressing so the assertion does not fail on -// schema drift. +// stateRegressed reports whether an instance moved from a live state (pending/running) +// to a terminal state. Unknown states are treated as non-regressing to tolerate schema drift. func stateRegressed(pre, post string) bool { preOrd := stateOrdinal(pre) postOrd := stateOrdinal(post) diff --git a/tests/e2e/ddil/harness/quarantine.go b/tests/e2e/ddil/harness/quarantine.go index a9dcefc14..03f6f490d 100644 --- a/tests/e2e/ddil/harness/quarantine.go +++ b/tests/e2e/ddil/harness/quarantine.go @@ -15,18 +15,9 @@ import ( "github.com/mulgadc/spinifex/tests/e2e/harness" ) -// Run wraps a scenario with reset-and-retry plus quarantine handling. It is -// the primary entry point used by the files in scenarios/: -// -// harness.Run(t, cluster, ssh, "A", func(t *testing.T) { ... }) -// -// Quarantine: scenarios whose letter appears in DDIL_QUARANTINED are skipped -// instead of executed. Per the design doc, a quarantined scenario must have -// a linked bead tracking its fix; the live status is tracked in -// tests/e2e/TEST_COVERAGE.md. -// -// Retry: delegates to harness.RetryWithReset so the shared package owns the -// reset-and-rerun mechanics; only quarantine policy stays DDIL-specific. +// Run wraps a scenario with reset-and-retry and quarantine handling. +// Scenarios listed in DDIL_QUARANTINED are skipped; others are delegated to +// harness.RetryWithReset which owns the reset-and-rerun mechanics. func Run(t *testing.T, c *harness.Cluster, ssh harness.SSH, letter string, fn func(*testing.T)) { t.Helper() @@ -38,9 +29,7 @@ func Run(t *testing.T, c *harness.Cluster, ssh harness.SSH, letter string, fn fu harness.RetryWithReset(t, c, ssh, "scenario "+letter, fn) } -// IsQuarantined reports whether letter appears in DDIL_QUARANTINED. -// Comparison is case-insensitive; whitespace and empty entries are tolerated -// so the env var can be edited by humans without worrying about formatting. +// IsQuarantined reports whether letter appears in DDIL_QUARANTINED (case-insensitive, comma-separated). func IsQuarantined(letter string) bool { raw := os.Getenv("DDIL_QUARANTINED") if raw == "" { diff --git a/tests/e2e/ddil/scenarios/a_nats_kill_test.go b/tests/e2e/ddil/scenarios/a_nats_kill_test.go index f49916651..5b4f74074 100644 --- a/tests/e2e/ddil/scenarios/a_nats_kill_test.go +++ b/tests/e2e/ddil/scenarios/a_nats_kill_test.go @@ -7,16 +7,13 @@ import ( "testing" "time" - "github.com/mulgadc/spinifex/tests/e2e/harness" - ddilh "github.com/mulgadc/spinifex/tests/e2e/ddil/harness" "github.com/mulgadc/spinifex/tests/e2e/ddil/fault" + ddilh "github.com/mulgadc/spinifex/tests/e2e/ddil/harness" + "github.com/mulgadc/spinifex/tests/e2e/harness" ) -// TestScenarioA_NATSKill — stop spinifex-nats on a single node without -// touching spinifex-daemon, verify the daemon stays up in standalone -// mode, serves its local API, and the surviving 2-node quorum keeps -// serving cluster-wide requests. See -// docs/development/improvements/ddil-e2e-test-harness.md §3 Scenario A. +// TestScenarioA_NATSKill stops spinifex-nats on one node without touching the daemon, +// verifies the daemon enters standalone mode, and confirms the 2-node quorum keeps serving. func TestScenarioA_NATSKill(t *testing.T) { deps := requireLiveCluster(t) c, ssh, dc, w := deps.cluster, deps.ssh, deps.dc, deps.witness @@ -60,10 +57,7 @@ func TestScenarioA_NATSKill(t *testing.T) { t.Fatalf("%s /local/instances regressed: pre=%d post=%d", node3.Name, len(pre), len(local3)) } - // Surviving 2-node quorum: each majority daemon's HTTPS listener is - // reachable and reports healthy. /health is NATS-independent so - // this proves the daemons are up; the witness counters below prove - // the data plane (gateway → daemon → QEMU) is still progressing. + // /health is NATS-independent; witness counters prove the data plane is still progressing. for _, n := range []harness.Node{node1, node2} { if _, err := dc.Health(ctx, n); err != nil { t.Fatalf("majority %s /health: %v", n.Name, err) diff --git a/tests/e2e/ddil/scenarios/b_daemon_restart_test.go b/tests/e2e/ddil/scenarios/b_daemon_restart_test.go index ebaaf237d..ee5a43b12 100644 --- a/tests/e2e/ddil/scenarios/b_daemon_restart_test.go +++ b/tests/e2e/ddil/scenarios/b_daemon_restart_test.go @@ -7,23 +7,14 @@ import ( "testing" "time" - "github.com/mulgadc/spinifex/tests/e2e/harness" - ddilh "github.com/mulgadc/spinifex/tests/e2e/ddil/harness" "github.com/mulgadc/spinifex/tests/e2e/ddil/fault" + ddilh "github.com/mulgadc/spinifex/tests/e2e/ddil/harness" + "github.com/mulgadc/spinifex/tests/e2e/harness" ) -// TestScenarioB_DaemonRestartWithoutNATS — kill spinifex-nats, restart -// spinifex-daemon, verify the daemon starts within 30s (not the old -// 5-minute NATS-wait abort) and recovers its instances from the local -// state file. See -// docs/development/improvements/ddil-e2e-test-harness.md §3 Scenario B. -// -// Plan §3 Scenario B step 8 originally asked for "same PID, same command -// line" across the restart. Operationally only recovery matters — PID is -// just a label and would only stay stable if the daemon unit set -// KillMode=process (today it uses systemd's default control-group). -// Asserting recovery + a live post-restart QEMU process gives the same -// regression coverage without coupling the test to the unit file. +// TestScenarioB_DaemonRestartWithoutNATS kills spinifex-nats, restarts spinifex-daemon, and +// verifies the daemon enters standalone mode within 30s and recovers its instances from the +// local state file. PID/command-line identity is not asserted; recovery + live QEMU is sufficient. func TestScenarioB_DaemonRestartWithoutNATS(t *testing.T) { deps := requireLiveCluster(t) c, ssh, dc, w := deps.cluster, deps.ssh, deps.dc, deps.witness @@ -54,9 +45,7 @@ func TestScenarioB_DaemonRestartWithoutNATS(t *testing.T) { _ = fault.StartNATS(cctx, ssh, node3) }) - // 1d's promise is "daemon comes up without NATS". The 30s budget is - // the plan's threshold — if it slips, the new startup path has - // regressed back to the 5-minute NATS-wait abort. + // 30s budget: if the daemon misses it, the startup path has regressed to the 5-minute NATS-wait abort. restartStart := time.Now() if err := fault.RestartDaemonOnly(ctx, ssh, node3); err != nil { t.Fatalf("restart daemon on %s: %v", node3.Name, err) @@ -72,10 +61,7 @@ func TestScenarioB_DaemonRestartWithoutNATS(t *testing.T) { } pre.AssertPreserved(t, post) - // Each recovered instance must have a live QEMU process. /local/instances - // reports PID from utils.ReadPidFile (local_api.go:112) — a non-zero - // value proves the daemon found and re-attached to (or relaunched) - // the QEMU pidfile. Zero means no QEMU on this host. + // Non-zero PID proves the daemon re-attached to the QEMU pidfile after restart. for _, inst := range post { if inst.PID == 0 { t.Errorf("recovered instance %s has no live QEMU PID", inst.InstanceID) diff --git a/tests/e2e/ddil/scenarios/c_partition_test.go b/tests/e2e/ddil/scenarios/c_partition_test.go index 43e2df0e2..4b7826d43 100644 --- a/tests/e2e/ddil/scenarios/c_partition_test.go +++ b/tests/e2e/ddil/scenarios/c_partition_test.go @@ -4,19 +4,10 @@ package scenarios import "testing" -// TestScenarioC_CleanPartition — iptables-DROP node3 away from node1 and -// node2, verify the majority keeps serving API, the isolated node -// reports standalone mode, and heal converges state without duplicate -// or orphaned VMs. See -// docs/development/improvements/ddil-e2e-test-harness.md §3 Scenario C. -// -// Quarantined: witness VM block I/O is backed by distributed predastore, -// so the partition that severs node3 ↔ peer traffic also severs the -// witness disk writes (BLOCK_IO_ERROR → daemon terminates the instance -// → AssertProgressed reads a dead sshd → handshake EOF). Needs -// predastore-ddil-hardening §2 (write-quorum with deferred parity -// repair) so writes succeed at data-shard quorum and the witness disk -// survives the split. +// TestScenarioC_CleanPartition partitions node3 from node1+node2 via iptables DROP, verifies +// the majority keeps serving and the isolated node enters standalone, then asserts heal +// converges without duplicate or orphaned VMs. Quarantined until predastore write-quorum lands +// (partition also severs witness disk I/O, causing BLOCK_IO_ERROR → instance termination). func TestScenarioC_CleanPartition(t *testing.T) { scenarioSkip(t, "C", "predastore-ddil-hardening §2") } diff --git a/tests/e2e/ddil/scenarios/coverage_drift_test.go b/tests/e2e/ddil/scenarios/coverage_drift_test.go index 0a1608151..09eb243fe 100644 --- a/tests/e2e/ddil/scenarios/coverage_drift_test.go +++ b/tests/e2e/ddil/scenarios/coverage_drift_test.go @@ -15,30 +15,16 @@ import ( "testing" ) -// scenarioFuncRe matches the TestScenario_ functions that make -// up the DDIL suite. The capture group is the single-letter identifier -// (A..F) used everywhere in the plan, bead, and TEST_COVERAGE.md. +// scenarioFuncRe matches TestScenario_ functions; capture group is the scenario letter. var scenarioFuncRe = regexp.MustCompile(`^TestScenario([A-Z])_[A-Za-z0-9]+$`) -// coverageDocRe matches the scenario-letter column in the DDIL table in -// TEST_COVERAGE.md (e.g. `| A | ...`). Anchoring on a leading pipe + -// whitespace keeps the pattern from matching prose mentions of "A" or -// "B" elsewhere in the doc. +// coverageDocRe matches the scenario-letter column in the DDIL table (e.g. `| A |`), +// anchored on a leading pipe to avoid matching prose mentions. var coverageDocRe = regexp.MustCompile(`(?m)^\|\s*([A-F])\s*\|`) -// TestCoverageDrift enforces TEST_COVERAGE.md and the scenarios package -// stay in lock-step. It fails if either: -// - A TestScenario_... function exists in the scenarios package but -// TEST_COVERAGE.md's DDIL table has no row for letter . -// - TEST_COVERAGE.md's DDIL table has a row for letter but no -// TestScenario_... function exists. -// -// Drift surfaces the most common mistake in rolling out Phase 2/3 -// hardening: flipping a scenario's Skip to a real assertion without -// updating the coverage table (or vice versa). -// -// Runs unconditionally — including under DDIL_DRY_RUN=1 — because it -// only reads local source and has no cluster dependency. +// TestCoverageDrift enforces TEST_COVERAGE.md and the scenarios package stay in sync: +// fails if a TestScenario_... function exists without a matching DDIL table row or +// vice versa. Runs unconditionally (no cluster dependency). func TestCoverageDrift(t *testing.T) { scenarioDir := callerDir(t) @@ -69,10 +55,7 @@ func TestCoverageDrift(t *testing.T) { } } -// callerDir returns the directory holding this test file. Used to locate -// sibling *_test.go files and to compute the relative path to -// TEST_COVERAGE.md without hard-coding a fragile relative path at the -// test-binary CWD. +// callerDir returns the directory of this test file, used to locate siblings and TEST_COVERAGE.md. func callerDir(t *testing.T) string { t.Helper() _, self, _, ok := runtime.Caller(0) @@ -82,10 +65,8 @@ func callerDir(t *testing.T) string { return filepath.Dir(self) } -// lettersFromSource AST-parses every *_test.go file in dir (except this -// file and main_test.go, which hold no scenarios) and returns the -// deduplicated sorted set of scenario letters declared by top-level -// TestScenario_... functions. +// lettersFromSource AST-parses scenario *_test.go files and returns the deduplicated +// sorted set of letters declared by top-level TestScenario_... functions. func lettersFromSource(dir string) ([]string, error) { entries, err := os.ReadDir(dir) if err != nil { @@ -121,10 +102,8 @@ func lettersFromSource(dir string) ([]string, error) { return sortedKeys(seen), nil } -// lettersFromDoc reads TEST_COVERAGE.md and returns the deduplicated -// sorted set of scenario letters its DDIL table declares. Rows are -// matched by coverageDocRe so prose mentions elsewhere in the file do -// not trigger false positives. +// lettersFromDoc reads TEST_COVERAGE.md and returns deduplicated sorted scenario letters +// from its DDIL table (matched by coverageDocRe, ignoring prose mentions). func lettersFromDoc(path string) ([]string, error) { data, err := os.ReadFile(path) if err != nil { @@ -146,9 +125,7 @@ func sortedKeys(m map[string]struct{}) []string { return out } -// setDiff returns elements in a that are not in b. Both inputs must be -// sorted (they always are, coming from sortedKeys); the linear merge -// keeps the drift check O(n+m) rather than O(nm). +// setDiff returns elements in a that are not in b (both inputs must be sorted). func setDiff(a, b []string) []string { var out []string i, j := 0, 0 diff --git a/tests/e2e/ddil/scenarios/main_test.go b/tests/e2e/ddil/scenarios/main_test.go index b1a397929..76a835fa8 100644 --- a/tests/e2e/ddil/scenarios/main_test.go +++ b/tests/e2e/ddil/scenarios/main_test.go @@ -1,11 +1,8 @@ //go:build e2e -// Package scenarios holds the DDIL E2E scenario tests. Each scenario -// (A..F) exercises one failure mode documented in -// docs/development/improvements/ddil-e2e-test-harness.md. Scenarios start -// as t.Skip stubs and are flipped to real assertions by the hardening -// epics that make them runnable (daemon-local-autonomy, -// predastore-ddil-hardening). +// Package scenarios holds the DDIL E2E scenario tests (A..F), each exercising one +// failure mode. Scenarios start as t.Skip stubs and are flipped to real assertions +// by the hardening epics that make them runnable. package scenarios import ( @@ -21,16 +18,11 @@ import ( "github.com/mulgadc/spinifex/tests/e2e/harness" ) -// scenarioLetters is the canonical ordered list of scenarios the suite -// knows about. It is used for DDIL_DRY_RUN=1 logging and by -// TestCoverageDrift as the source-of-truth set when cross-checking -// TEST_COVERAGE.md. Keep in sync with the TestScenario_* functions in -// this package — TestCoverageDrift fails loudly if it drifts. +// scenarioLetters is the canonical scenario list used for DDIL_DRY_RUN logging and +// TestCoverageDrift cross-checking. Keep in sync with TestScenario_* functions. var scenarioLetters = []string{"A", "B", "C", "D", "E", "F"} -// Package-level state shared between TestMain's signal handler and -// TestHarnessSmoke. Populated by setupCluster when DDIL_NODES is set and -// DDIL_DRY_RUN is unset; both remain nil otherwise. +// Package-level cluster state. Populated by setupCluster when DDIL_NODES is set; nil in dry-run. var ( clusterOnce sync.Once cluster *harness.Cluster @@ -42,9 +34,7 @@ var ( // TestHarnessSmoke skip cluster-touching work in dry-run mode. func dryRun() bool { return os.Getenv("DDIL_DRY_RUN") == "1" } -// setupCluster builds the Cluster/SSH pair once per test binary. Callers -// that need cluster access gate on both (cluster, err): cluster is nil in -// dry-run or when DDIL_NODES is unset. +// setupCluster builds the Cluster/SSH pair once. Returns nil cluster in dry-run or when DDIL_NODES is unset. func setupCluster() (*harness.Cluster, harness.SSH, error) { clusterOnce.Do(func() { if dryRun() { @@ -69,12 +59,9 @@ func setupCluster() (*harness.Cluster, harness.SSH, error) { return cluster, sshXport, clusterErr } -// TestMain is the suite-level entry point. It installs a signal handler -// that best-effort resets the cluster on SIGINT/SIGTERM so an aborted CI -// run does not leave iptables DROP rules or tc netem qdiscs in place, -// then delegates to go test's default runner. DDIL_DRY_RUN=1 logs the -// planned scenarios and skips cluster initialisation, leaving -// TestCoverageDrift as the only meaningful assertion. +// TestMain installs a SIGINT/SIGTERM handler that resets the cluster before exit +// (clears iptables DROP rules and tc netem qdiscs), then runs the suite. +// DDIL_DRY_RUN=1 skips cluster init and leaves only TestCoverageDrift active. func TestMain(m *testing.M) { if dryRun() { log.Printf("ddil: DDIL_DRY_RUN=1 — planned scenarios: %v (TestCoverageDrift will still execute)", scenarioLetters) @@ -95,11 +82,8 @@ func TestMain(m *testing.M) { os.Exit(code) } -// installSignalHandler traps SIGINT/SIGTERM and, when a cluster is -// available, calls ResetAllNodes before exiting non-zero. Without a -// cluster (dry-run or missing env) it still exits non-zero so the parent -// shell sees the signal. Returns a stop function the caller must defer to -// unregister the handler. +// installSignalHandler traps SIGINT/SIGTERM; calls ResetAllNodes when a cluster is +// available, then exits non-zero. Returns a stop func the caller must defer. func installSignalHandler(c *harness.Cluster, s harness.SSH) func() { ch := make(chan os.Signal, 1) signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM) @@ -128,12 +112,8 @@ func installSignalHandler(c *harness.Cluster, s harness.SSH) func() { } } -// TestHarnessSmoke exercises the reset + clean-state round trip without -// running a scenario. It satisfies the Phase 1 acceptance criterion that -// "full run against real cluster completes with AssertCleanState + -// ResetAllNodes round-tripping cleanly" and gives hardening PR authors a -// fast command for validating their tofu-cluster before touching -// scenarios (go test -run TestHarnessSmoke). +// TestHarnessSmoke exercises the ResetAllNodes + AssertCleanState round trip without +// running a scenario. Fast validation command for a live cluster. func TestHarnessSmoke(t *testing.T) { if dryRun() { t.Skipf("DDIL_DRY_RUN=1") @@ -155,18 +135,13 @@ func TestHarnessSmoke(t *testing.T) { harness.AssertCleanState(ctx, t, c, s) } -// scenarioSkip is the shared skip helper used by each TestScenario_* -// stub. Centralising the message format keeps Phase 2/3 hardening PRs to -// a single place when they flip a scenario: delete the scenarioSkip call, -// add the real body. +// scenarioSkip is the shared skip helper for scenario stubs. func scenarioSkip(t *testing.T, letter, dep string) { t.Helper() t.Skipf("scenario %s requires %s", letter, dep) } -// scenarioDeps bundles the live cluster handles every flipped scenario -// needs. Returned by requireLiveCluster; nil on dry-run / missing env so -// the caller's t.Skip path does not have to dereference it. +// scenarioDeps bundles live cluster handles needed by flipped scenarios. type scenarioDeps struct { cluster *harness.Cluster ssh harness.SSH @@ -174,10 +149,8 @@ type scenarioDeps struct { witness *harness.Witness } -// requireLiveCluster wires up the cluster, SSH, daemon client, and witness -// factory a flipped scenario needs. Skips (rather than fails) on dry-run, -// missing DDIL_NODES, or missing AWS_REGION so the suite can run on a -// laptop without provisioning the tofu cluster — Phase 1's smoke path. +// requireLiveCluster wires up cluster, SSH, daemon client, and witness factory. +// Skips on dry-run, missing DDIL_NODES, or insufficient cluster size. func requireLiveCluster(t *testing.T) scenarioDeps { t.Helper() if dryRun() { @@ -205,11 +178,8 @@ func requireLiveCluster(t *testing.T) scenarioDeps { } } -// launchWitnesses launches one counter VM on each requested host, registers -// a t.Cleanup that terminates them, and returns the slice in input order. -// A launch failure fails the test rather than skipping — the harness -// retries placement up to maxPlacementAttempts internally, so a hard error -// here means the cluster cannot serve a witness at all. +// launchWitnesses launches one counter VM per host, registers t.Cleanup termination, +// and returns VMs in input order. Fatals on launch failure (placement retries are internal). func launchWitnesses(ctx context.Context, t *testing.T, w *harness.Witness, hosts ...harness.Node) []*harness.WitnessVM { t.Helper() out := make([]*harness.WitnessVM, 0, len(hosts)) diff --git a/tests/e2e/eks/eks_test.go b/tests/e2e/eks/eks_test.go index bb29df336..4f7cf41c0 100644 --- a/tests/e2e/eks/eks_test.go +++ b/tests/e2e/eks/eks_test.go @@ -30,16 +30,9 @@ const ( getTokenTpl = "k8s-aws-v1." ) -// TestEKS drives the EKS control-plane lifecycle against the local awsgw -// endpoint: a real VPC/subnet, CreateCluster → ACTIVE (which boots the K3s -// server VM + NLB), kubeconfig artifact assembly, AccessEntry CRUD, get-token, -// and DeleteCluster → gone. Phase 1 is API-level only (no kubectl); the -// kubectl/IRSA data-plane subtests land in Phase 2 once apiserver reachability -// from the runner is wired. -// -// One cluster fixture is shared across the subtests (create once, delete in -// Cleanup) — control-plane boot is the slowest step, so re-creating per subtest -// would blow the suite timeout on dev nodes. +// TestEKS drives the EKS control-plane lifecycle: VPC/subnet creation, CreateCluster → ACTIVE +// (K3s VM + NLB boot), kubeconfig artifact assembly, AccessEntry CRUD, get-token, and +// DeleteCluster. One fixture is shared across subtests to avoid repeated control-plane boot cost. func TestEKS(t *testing.T) { env := harness.LoadEnv(t) artifacts := harness.ArtifactDir(t, env) @@ -122,23 +115,16 @@ func TestEKS(t *testing.T) { t.Run("KubectlGetNodes", func(t *testing.T) { requireClusterReady(t, fx) - // Reach the apiserver through the published cluster endpoint (the - // internet-facing NLB front-end IP:443) with TLS verification ON — the - // serving cert SANs this IP, so no insecure-skip-tls-verify and no - // mgmt-IP shim. A 401 here means the get-token webhook chain regressed; - // a TLS error means the endpoint/cert-SAN wiring regressed. + // Reach the apiserver at the published endpoint with TLS verification ON. + // A 401 means the get-token webhook chain regressed; a TLS error means + // the cert-SAN wiring regressed. require.NotEmpty(t, aws.StringValue(fx.Cluster.Endpoint), "cluster must publish a reachable endpoint") kcPath := writeKubeconfig(t, artifacts, fx.Cluster) kc := harness.NewKubectl(t, kcPath, getTokenEnv(t, env)) - // Auth + reachability: poll until the apiserver answers and reports a - // Ready node. A 401 here means the get-token webhook chain regressed. - // Generous envelope: the cluster flips ACTIVE as soon as the apiserver - // first answers, but k3s can crash once during bootstrap (embedded-etcd - // fsync latency under I/O contention blows the apiserver/leaderelection - // deadlines) and is respawned by supervise-daemon. The control plane - // then stabilises once etcd latency settles — a few minutes on a busy - // node — so wait well past the first ACTIVE before failing. + // Poll until a Ready node appears. k3s may crash once during bootstrap + // (etcd fsync latency under I/O contention) and be respawned; the generous + // envelope allows the control plane to stabilise after the first ACTIVE. harness.EventuallyErr(t, func() error { out, err := kc.Run(30*time.Second, "get", "nodes", "-o", `jsonpath={range .items[*]}{.metadata.name}{"="}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}`) @@ -253,10 +239,9 @@ func deleteSubnet(t *testing.T, c *harness.AWSClient, fx *clusterFixture) { } } -// deleteClusterBestEffort tears the cluster down if a subtest left it standing -// (the DeleteCluster subtest already removed it on the happy path). VPC/subnet -// cleanup must wait for the cluster's NLB + VM to release them, so this runs -// before the VPC/subnet Cleanups (Cleanups run LIFO; this is registered last). +// deleteClusterBestEffort tears the cluster down if the DeleteCluster subtest did not. +// Registered last so it runs before VPC/subnet Cleanups (LIFO), ensuring the NLB + VM +// release the subnet before the VPC is removed. func deleteClusterBestEffort(t *testing.T, c *harness.AWSClient, fx *clusterFixture) { if fx.Deleted { return @@ -274,10 +259,8 @@ func deleteClusterBestEffort(t *testing.T, c *harness.AWSClient, fx *clusterFixt // --- kubeconfig artifact -------------------------------------------------- -// writeKubeconfig assembles the kubeconfig `aws eks update-kubeconfig` would -// produce from DescribeCluster output and writes it to the artifact dir. The -// suite asserts the structure (server, CA, exec block) rather than shelling out -// to update-kubeconfig so the assertion is hermetic. +// writeKubeconfig builds a kubeconfig from DescribeCluster output and writes it to the artifact +// dir. Avoids shelling to `aws eks update-kubeconfig` so the structure assertion is hermetic. func writeKubeconfig(t *testing.T, artifacts string, cl *eks.Cluster) string { t.Helper() name := aws.StringValue(cl.Name) diff --git a/tests/e2e/harness/account.go b/tests/e2e/harness/account.go index 0c11487fd..9ed44bba3 100644 --- a/tests/e2e/harness/account.go +++ b/tests/e2e/harness/account.go @@ -9,10 +9,8 @@ import ( ) // Profile bundles a tenant account's credentials with a pre-built AWSClient -// scoped to that account. The Client field bypasses AWS_PROFILE shared-config -// lookup — credentials are injected statically via NewAWSClientWithCreds so a -// single test binary can drive many tenants concurrently without mutating -// ~/.aws/credentials. +// scoped to that account. Credentials are injected statically so a single test +// binary can drive many tenants concurrently without mutating ~/.aws/credentials. type Profile struct { Name string AccountID string @@ -21,10 +19,8 @@ type Profile struct { } // AccountCarousel manages multiple Profile entries created via -// `spx admin account create`. Used by Phase 8 account-scoping where the test -// repeatedly switches between Alpha/Beta/Gamma to assert isolation. -// -// Concurrent reads via Get/Names are safe; Add must not race with itself. +// `spx admin account create`. Concurrent reads via Get/Names are safe; +// Add must not race with itself. type AccountCarousel struct { mu sync.RWMutex profiles map[string]*Profile @@ -35,10 +31,8 @@ func NewAccountCarousel() *AccountCarousel { return &AccountCarousel{profiles: make(map[string]*Profile)} } -// Add registers a new profile under the given name (e.g. "team-alpha") and -// returns it. Duplicate names overwrite — caller is expected to use unique -// labels per tenant. info.AccessKeyID and info.SecretAccessKey must be set; -// NewAWSClientWithCreds t.Fatals otherwise. +// Add registers a new profile under name, returning it. Duplicate names +// overwrite. info.AccessKeyID and info.SecretAccessKey must be non-empty. func (a *AccountCarousel) Add(t *testing.T, env *Env, name string, info AccountInfo) *Profile { t.Helper() client := NewAWSClientWithCreds(t, env, info.AccessKeyID, info.SecretAccessKey) @@ -61,8 +55,7 @@ func (a *AccountCarousel) Get(name string) *Profile { return a.profiles[name] } -// Names returns the registered profile names in sorted order — stable for -// snapshot-style logging without depending on map iteration order. +// Names returns registered profile names in sorted order. func (a *AccountCarousel) Names() []string { a.mu.RLock() defer a.mu.RUnlock() diff --git a/tests/e2e/harness/artifacts.go b/tests/e2e/harness/artifacts.go index 4d595d7d5..0ef2e0beb 100644 --- a/tests/e2e/harness/artifacts.go +++ b/tests/e2e/harness/artifacts.go @@ -12,9 +12,7 @@ import ( ) // ArtifactDir returns the per-test artifact directory and registers a Cleanup -// that prunes the directory if the test passed (to avoid filling disk on CI). -// Call once per test. Safe to call from subtests — uses t.Name() which is -// unique per subtest. +// that prunes it on test pass. Safe to call from subtests. func ArtifactDir(t *testing.T, env *Env) string { t.Helper() safeName := strings.ReplaceAll(t.Name(), "/", "_") diff --git a/tests/e2e/harness/aws.go b/tests/e2e/harness/aws.go index b465ad116..25e3c29e4 100644 --- a/tests/e2e/harness/aws.go +++ b/tests/e2e/harness/aws.go @@ -21,35 +21,31 @@ import ( "github.com/aws/aws-sdk-go/service/sts" ) -// AWSClient bundles the SDK service handles a scenario typically needs against -// the local Spinifex AWS gateway. Region is fixed to ap-southeast-2 to match -// the bash scripts; override via SPINIFEX_AWS_REGION. +// AWSClient bundles the SDK service handles for the local Spinifex AWS gateway. +// Region defaults to ap-southeast-2; override via SPINIFEX_AWS_REGION. type AWSClient struct { // EC2 is wrapped so RunInstances auto-serialises against cluster capacity - // (see capacityRetryEC2). EC2Conf is the underlying SDK client, kept for - // the few call sites that read its Config (region, creds, transport) or - // mutate it (IgnoreCertErrors); both share one *ec2.EC2. + // (see capacityRetryEC2). EC2Conf is the underlying SDK client shared by + // both; kept for call sites that read or mutate its Config. EC2 ec2iface.EC2API EC2Conf *ec2.EC2 ELBv2 *elbv2.ELBV2 IAM *iam.IAM STS *sts.STS - EKS *eks.EKS + EKS *eks.EKS } -// NewAWSClient builds clients pointed at https://:/ using the -// spinifex CA for TLS verification. Credentials come from the AWS_PROFILE env -// var (matching the bash scripts) — default `spinifex`. Override via -// SPINIFEX_AWS_ACCESS_KEY_ID / SPINIFEX_AWS_SECRET_ACCESS_KEY for static creds. +// NewAWSClient builds clients pointed at the spinifex gateway using the +// spinifex CA. Credentials come from AWS_PROFILE (default "spinifex") or +// SPINIFEX_AWS_ACCESS_KEY_ID / SPINIFEX_AWS_SECRET_ACCESS_KEY. func NewAWSClient(t *testing.T, env *Env) *AWSClient { t.Helper() id, secret := os.Getenv("SPINIFEX_AWS_ACCESS_KEY_ID"), os.Getenv("SPINIFEX_AWS_SECRET_ACCESS_KEY") return newAWSClient(t, env, id, secret, "") } -// NewAWSClientWithCreds builds an AWSClient with explicit static credentials -// — used by AccountCarousel to scope a client to a tenant account created -// via `spx admin account create`. Bypasses AWS_PROFILE shared-config lookup. +// NewAWSClientWithCreds builds an AWSClient with explicit static credentials, +// bypassing AWS_PROFILE lookup. func NewAWSClientWithCreds(t *testing.T, env *Env, accessKey, secretKey string) *AWSClient { t.Helper() if accessKey == "" || secretKey == "" { @@ -58,10 +54,8 @@ func NewAWSClientWithCreds(t *testing.T, env *Env, accessKey, secretKey string) return newAWSClient(t, env, accessKey, secretKey, "") } -// NewAWSClientWithSessionCreds builds an AWSClient with static temporary -// credentials issued by sts:AssumeRole. The SDK signs every request with the -// supplied session token in X-Amz-Security-Token, driving the gateway's ASIA -// auth path. +// NewAWSClientWithSessionCreds builds an AWSClient with STS temporary +// credentials, signing requests with the session token (ASIA auth path). func NewAWSClientWithSessionCreds(t *testing.T, env *Env, accessKey, secretKey, sessionToken string) *AWSClient { t.Helper() if accessKey == "" || secretKey == "" || sessionToken == "" { @@ -83,10 +77,8 @@ func newAWSClient(t *testing.T, env *Env, accessKey, secretKey, sessionToken str } region := getenv("SPINIFEX_AWS_REGION", "ap-southeast-2") - // Runner-resident scenarios (e.g. reboot suite running outside the VM) - // don't have the spinifex CA cert on disk. SPINIFEX_AWS_INSECURE=1 skips - // the CA load entirely and uses InsecureSkipVerify — trust validation is - // already covered by the cert suite, so this is safe for non-cert tests. + // SPINIFEX_AWS_INSECURE=1 skips CA load for runner-resident scenarios that + // lack the spinifex cert on disk. Trust is covered by the cert suite. tlsCfg := &tls.Config{} if os.Getenv("SPINIFEX_AWS_INSECURE") == "1" { tlsCfg.InsecureSkipVerify = true //nolint:gosec // explicit opt-in for runner-resident E2E @@ -115,9 +107,7 @@ func newAWSClient(t *testing.T, env *Env, accessKey, secretKey, sessionToken str opts := session.Options{Config: *cfg} if accessKey != "" && secretKey != "" { - // Static creds bypass shared-config lookup — required for the - // per-tenant carousel where each Profile holds its own access key, - // and for STS-issued session credentials (sessionToken non-empty). + // Static creds: used for per-tenant carousel profiles and STS session creds. cfg.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, sessionToken) opts.Config = *cfg } else { @@ -137,12 +127,12 @@ func newAWSClient(t *testing.T, env *Env, accessKey, secretKey, sessionToken str ELBv2: elbv2.New(sess), IAM: iam.New(sess), STS: sts.New(sess), - EKS: eks.New(sess), + EKS: eks.New(sess), } } -// IgnoreCertErrors disables TLS verification on the bundled clients. Use only -// for fault-injection scenarios — the cert scenario already covers trust path. +// IgnoreCertErrors disables TLS verification on all bundled clients. +// Use only for fault-injection scenarios. func (c *AWSClient) IgnoreCertErrors() { tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} hc := &http.Client{Transport: tr} diff --git a/tests/e2e/harness/cluster.go b/tests/e2e/harness/cluster.go index 43e1d9f86..dc854b81e 100644 --- a/tests/e2e/harness/cluster.go +++ b/tests/e2e/harness/cluster.go @@ -10,10 +10,7 @@ import ( ) // Node identifies a single cluster member addressable over SSH and HTTPS. -// -// Index is the 1-based position the node occupies in the SPINIFEX_NODES list; -// scenarios reference nodes by index (node1, node2, node3) so a fixed -// ordering is required. +// Index is 1-based; scenarios reference nodes by index so ordering is fixed. type Node struct { Index int // 1-based position in the cluster Name string // human-readable tag, e.g. "node1" @@ -21,10 +18,6 @@ type Node struct { } // Cluster is an ordered set of Nodes plus shared SSH credentials. -// -// Scenarios receive a *Cluster from main_test.go and use its Nodes slice to -// address individual peers. All helpers that mutate node state take a Node -// (not an index) to keep call sites self-describing. type Cluster struct { Nodes []Node SSHUser string @@ -45,11 +38,7 @@ func (c *Cluster) Peers(target Node) []Node { } // ClusterFromEnv builds a Cluster from SPINIFEX_NODES / SPINIFEX_SSH_USER / -// SPINIFEX_SSH_KEY. Nodes are named node1..nodeN in list order. -// -// Returns an error if any required variable is missing or if SPINIFEX_NODES -// contains no non-empty entries. main_test.go is expected to call this once -// and pass the result into every scenario. +// SPINIFEX_SSH_KEY. Returns an error if any required variable is missing. func ClusterFromEnv() (*Cluster, error) { nodesRaw := envWithLegacy("SPINIFEX_NODES", "DDIL_NODES") user := envWithLegacy("SPINIFEX_SSH_USER", "DDIL_SSH_USER") @@ -92,9 +81,7 @@ func ClusterFromEnv() (*Cluster, error) { }, nil } -// NodeFromEnv returns the Nth node (1-based) from the cluster defined by -// SPINIFEX_NODES. Convenience wrapper for single-node helpers; most scenarios -// should call ClusterFromEnv once and index c.Nodes directly. +// NodeFromEnv returns the Nth node (1-based) from the cluster in SPINIFEX_NODES. func NodeFromEnv(index int) (Node, error) { c, err := ClusterFromEnv() if err != nil { @@ -107,9 +94,7 @@ func NodeFromEnv(index int) (Node, error) { } // envWithLegacy returns os.Getenv(canonical), falling back to legacy on miss. -// DDIL_* names are accepted for back-compat; rename to SPINIFEX_* once all -// CI workflows are migrated. A one-line stderr note flags the deprecation -// so leftover callers surface in CI logs. +// A stderr note flags use of the deprecated DDIL_* names. func envWithLegacy(canonical, legacy string) string { if v := os.Getenv(canonical); v != "" { return v diff --git a/tests/e2e/harness/daemon_client.go b/tests/e2e/harness/daemon_client.go index 8c146bd2c..a79e108e5 100644 --- a/tests/e2e/harness/daemon_client.go +++ b/tests/e2e/harness/daemon_client.go @@ -17,12 +17,8 @@ import ( ) // WaitForMode polls the daemon's /local/status until it reports the expected -// mode or timeout expires. Poll interval is 1s — fast enough that a -// 30-second timeout sees ~30 attempts, slow enough not to flood a recovering -// daemon. -// -// Depends on daemon-local-autonomy §1b. Until that endpoint ships, this -// function will time out; callers gate on t.Skip. +// mode or timeout expires (1s poll interval). +// Requires the /local/status endpoint; callers gate on t.Skip until it ships. func WaitForMode(ctx context.Context, dc *DaemonClient, node Node, want DaemonMode, timeout time.Duration) error { deadline := time.Now().Add(timeout) const interval = 1 * time.Second @@ -55,17 +51,11 @@ func WaitForMode(ctx context.Context, dc *DaemonClient, node Node, want DaemonMo } } -// daemonPort is the cluster-manager HTTPS port the daemon binds for /health -// and /local/* (see spinifex/daemon/daemon.go ClusterManager, configured via -// [nodes..daemon].host). 4432 is the deployed default; 8443 is predastore. +// daemonPort is the HTTPS port for /health and /local/* (deployed default; 8443 is predastore). const daemonPort = 4432 // DaemonMode mirrors the daemon's operating mode reported by /local/status. -// -// The typed enum avoids stringly-typed assertions in scenarios. The concrete -// endpoint and JSON schema land with daemon-local-autonomy §1b; this file -// predates that PR and its types will be swapped for the daemon's own when -// it merges. +// The typed enum avoids stringly-typed assertions. type DaemonMode string const ( @@ -75,11 +65,7 @@ const ( ) // LocalStatus is the response shape from GET /local/status. -// -// Placeholder fields only — the authoritative struct ships with -// daemon-local-autonomy §1b. Scenarios that depend on these fields remain -// t.Skip until that epic replaces this definition with an import from the -// daemon package. +// Placeholder — scenarios that depend on these fields gate on t.Skip. type LocalStatus struct { Node string `json:"node"` Mode DaemonMode `json:"mode"` @@ -87,24 +73,20 @@ type LocalStatus struct { NATS string `json:"nats"` // "connected" | "disconnected" } -// LocalInstance is one row of GET /local/instances. Placeholder — see -// LocalStatus. +// LocalInstance is one row of GET /local/instances. Placeholder. type LocalInstance struct { InstanceID string `json:"instance_id"` State string `json:"state"` PID int `json:"pid,omitempty"` } -// DaemonClient is a thin HTTPS client targeting a single daemon's local -// endpoints. It uses one *http.Client per node so connection reuse works -// across calls within a scenario. +// DaemonClient is a thin HTTPS client for a single daemon's local endpoints. type DaemonClient struct { http *http.Client } // NewDaemonClient returns a DaemonClient with TLS verification disabled -// (the tofu-cluster uses self-signed per-node certs). Timeouts are set to -// 5s so a wedged daemon fails fast enough for scenario retry logic. +// (self-signed per-node certs) and a 5s timeout. func NewDaemonClient() *DaemonClient { return &DaemonClient{ http: &http.Client{ @@ -143,9 +125,7 @@ func (d *DaemonClient) getJSON(ctx context.Context, node Node, path string, out } // Status returns the daemon's self-reported local status. -// -// Depends on daemon-local-autonomy §1b. Until that lands, the daemon will -// return 404 and callers should t.Skip. +// Returns 404 until the /local/status endpoint ships; callers should t.Skip. func (d *DaemonClient) Status(ctx context.Context, node Node) (LocalStatus, error) { var s LocalStatus if err := d.getJSON(ctx, node, "/local/status", &s); err != nil { @@ -155,9 +135,7 @@ func (d *DaemonClient) Status(ctx context.Context, node Node) (LocalStatus, erro } // Instances returns the daemon's locally-known instance list. -// -// Depends on daemon-local-autonomy §1a. Until that lands, the daemon will -// return 404 and callers should t.Skip. +// Returns 404 until the /local/instances endpoint ships; callers should t.Skip. func (d *DaemonClient) Instances(ctx context.Context, node Node) ([]LocalInstance, error) { var xs []LocalInstance if err := d.getJSON(ctx, node, "/local/instances", &xs); err != nil { @@ -166,9 +144,7 @@ func (d *DaemonClient) Instances(ctx context.Context, node Node) ([]LocalInstanc return xs, nil } -// Health hits the daemon's existing /health endpoint. Useful during -// scaffolding validation before the /local/* endpoints exist — confirms -// the harness can reach the daemon at all. +// Health hits the daemon's /health endpoint to confirm basic reachability. func (d *DaemonClient) Health(ctx context.Context, node Node) (types.NodeHealthResponse, error) { var h types.NodeHealthResponse if err := d.getJSON(ctx, node, "/health", &h); err != nil { diff --git a/tests/e2e/harness/diag.go b/tests/e2e/harness/diag.go index 1ac307dcf..4e990da9a 100644 --- a/tests/e2e/harness/diag.go +++ b/tests/e2e/harness/diag.go @@ -38,21 +38,10 @@ type VPCDiagnosticsOpts struct { ArtifactDir string } -// DumpVPCFlowDiagnostics emits a triage bundle for VPC/IGW datapath failures -// (e.g. Phase8b SSH-to-public-IP timeout). Dumps the guest's console tail -// (boot + cloud-init state) plus a slim OVN state snapshot to identify -// whether the gateway router has converged. Non-fatal — runs purely for log -// signal so the test's primary Fatal still wins. -// -// Tracked product bugs surfaced by this signal: -// - mulga-siv-105 (handleIGWAttach lacks flows-ready barrier) -// - mulga-siv-106 (handleVPCCreate doesn't store spinifex:cidr) -// - mulga-siv-107 (dnat_and_snat double-delete race) -// - mulga-siv-111 (EIP-recycle stale OVS state; this is the Phase 1 -// diagnostics surface that closes the bead's first acceptance gate) -// -// Skips silently if OVN tooling / passwordless sudo aren't available, so the -// helper is safe to call from developer laptops too. +// DumpVPCFlowDiagnostics emits a triage bundle for VPC/IGW datapath failures. +// Dumps the guest console tail and a slim OVN state snapshot to identify +// whether the gateway router has converged. Non-fatal — log signal only. +// Skips silently if OVN tooling / passwordless sudo aren't available. func DumpVPCFlowDiagnostics(t *testing.T, c *AWSClient, instanceID, label string, opts VPCDiagnosticsOpts) { t.Helper() fmt.Printf("\n%s%s── DIAGNOSTICS: %s (instance=%s) ──%s\n", @@ -162,16 +151,10 @@ func dumpOVNState(t *testing.T, instanceID string) { } } -// dumpDatapathState captures OVS / kernel-side state keyed on the -// external IP being probed. The signal disambiguates between three -// hypotheses on the residual SSH-timeout-despite-barrier failure -// (mulga-siv-111): (1) stale conntrack, (2) stale upstream ARP, (3) -// missing OF flow install. Each capture also writes to a separate -// file under opts.ArtifactDir when set, so the Stage 2 analyzer -// bundler picks them up. -// -// Skipped silently when the IP-specific inputs aren't supplied or the -// shell tools aren't available. +// dumpDatapathState captures OVS / kernel-side state keyed on the external IP: +// conntrack, upstream ARP, and OF flow install. Each capture is also written +// to opts.ArtifactDir when set. Skipped silently when inputs or shell tools +// are unavailable. func dumpDatapathState(t *testing.T, opts VPCDiagnosticsOpts) { t.Helper() if opts.ExternalIP == "" { diff --git a/tests/e2e/harness/discover.go b/tests/e2e/harness/discover.go index 299d5b64f..02c7b5bde 100644 --- a/tests/e2e/harness/discover.go +++ b/tests/e2e/harness/discover.go @@ -1,12 +1,7 @@ //go:build e2e -// Cluster-discovery helpers that memoize their answer on the Fixture. -// -// These wrap the read-only EC2 calls the single-node e2e suite uses to learn -// the per-cluster catalog (AZ, nano instance type, Ubuntu AMI) so any top-level -// Test* can resolve them without depending on an earlier phase having stashed -// the result on a shared Fixture field. Memo keys are stable per fixture, so -// the first call pays the API cost and later callers hit the cache. +// Cluster-discovery helpers that memoize EC2 catalog lookups on the Fixture. +// First call pays the API cost; later callers in the same process hit the cache. package harness import ( @@ -78,10 +73,8 @@ func DiscoverNanoInstanceType(t *testing.T, fx *Fixture) (instanceType, arch str } // DiscoverUbuntuAMI returns the AMI ID for the architecture-appropriate Ubuntu -// gold image bootstrap-install.sh staged via `spx admin images import --file`. -// Tries the v6+ ubuntu-26.04 name first, falls back to the v3 ubuntu-24.04 -// candidate. Routes the discovered ID through EnsureAMI so the state-available -// poll + memoization match every other AMI consumer in the fixture. +// gold image. Tries ubuntu-26.04 first, falls back to ubuntu-24.04. +// Routes through EnsureAMI so the state-available poll and memoization apply. func DiscoverUbuntuAMI(t *testing.T, fx *Fixture, arch string) string { t.Helper() if arch == "" { diff --git a/tests/e2e/harness/ec2helpers.go b/tests/e2e/harness/ec2helpers.go index d881098f4..14ca22259 100644 --- a/tests/e2e/harness/ec2helpers.go +++ b/tests/e2e/harness/ec2helpers.go @@ -14,11 +14,9 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" ) -// SSHTarget is the minimal addressing tuple needed by LsblkRootGiB. The -// existing harness.SSH interface requires a Cluster.Node, but the single-host -// e2e suite addresses individual instances by (host, port, key) discovered at -// runtime — those don't fit a Cluster. Defined here rather than in ssh.go to -// keep the production SSH transport untouched. +// SSHTarget is the minimal addressing tuple used by LsblkRootGiB. Unlike +// harness.SSH / Cluster.Node, instances here are addressed by (host, port, key) +// discovered at runtime, which doesn't fit the Cluster model. type SSHTarget struct { User string Host string @@ -27,14 +25,9 @@ type SSHTarget struct { } // InstancePublicSSHHost returns (host, 22) for an SSH connection to inst's -// public IP. -// -// There is deliberately NO qemu-hostfwd fallback. hostfwd (127.0.0.1 -> -// guest:22) is a dev_networking shortcut that bypasses the OVN datapath -// entirely — no SG ACL, no IGW, no SNAT/DNAT — so a test that reached a guest -// through it would validate nothing real and mask exactly the networking -// regressions these tests exist to catch. An instance with no public IP is a -// hard failure: the routing it depends on is broken or unconfigured. +// public IP. No qemu-hostfwd fallback: hostfwd bypasses the OVN datapath (no +// SG ACL, no IGW, no SNAT/DNAT), masking the networking regressions these +// tests exist to catch. No public IP is a hard failure. func InstancePublicSSHHost(t *testing.T, inst *ec2.Instance) (string, int) { t.Helper() if inst == nil { @@ -49,11 +42,8 @@ func InstancePublicSSHHost(t *testing.T, inst *ec2.Instance) (string, int) { return pub, 22 } -// LsblkRootGiB SSHes into the VM and returns the root disk size in GiB, -// cross-checking the value the API reports against the guest's view. -// -// Equivalent of run-e2e.sh's lsblk pipeline (findmnt → lsblk PKNAME → lsblk -// -b -d). Returns the GiB rounded down (same math as bash: bytes / 1<<30). +// LsblkRootGiB SSHes into the VM and returns the root disk size in GiB +// (bytes / 1<<30, rounded down) by running findmnt + lsblk in the guest. func LsblkRootGiB(t *testing.T, tgt SSHTarget) int { t.Helper() cmd := `SRC=$(findmnt -n -o SOURCE /); PKN=$(lsblk -n -o PKNAME "$SRC" 2>/dev/null | head -1); DEV=${PKN:-$(basename "$SRC")}; lsblk -b -d -n -o SIZE "/dev/$DEV"` @@ -117,10 +107,8 @@ func DisableSerialConsoleAccess(t *testing.T, c *AWSClient) bool { return aws.BoolValue(out.SerialConsoleAccessEnabled) } -// DiscoverDefaultVPC returns (vpcID, defaultSGID, defaultSubnetID) for the -// account's default VPC. t.Fatal if the default VPC is missing — the suite -// can't run without one, and silently falling back masks the real failure -// (daemon didn't create the default VPC on account bootstrap). +// DiscoverDefaultVPC returns (vpcID, defaultSGID, defaultSubnetID). Fatals if +// the default VPC is absent — its absence indicates a bootstrap failure. func DiscoverDefaultVPC(t *testing.T, c *AWSClient) (vpcID, sgID, subnetID string) { t.Helper() vpcs, err := c.EC2.DescribeVpcs(&ec2.DescribeVpcsInput{ @@ -215,10 +203,9 @@ func AuthorizeICMPIngress(t *testing.T, c *AWSClient, sgID string) { t.Fatalf("authorize-security-group-ingress icmp on %s: %v", sgID, err) } -// EnsureDefaultSGOpen authorizes tcp/22 + ICMP on the default VPC's default -// SG. AWS default SGs only admit same-SG members; e2e probes come from the -// test runner's external IP, so without this every SSH/ping hits the OVN -// port-group ACL drop. Mirrors the Phase-5 block in run-e2e.sh. +// EnsureDefaultSGOpen authorizes tcp/22 + ICMP on the default SG. Default SGs +// only admit same-SG members; without this, runner-originated SSH/ping is dropped +// by the OVN port-group ACL. func EnsureDefaultSGOpen(t *testing.T, c *AWSClient) { t.Helper() _, sgID, _ := DiscoverDefaultVPC(t, c) diff --git a/tests/e2e/harness/eks_mgmt.go b/tests/e2e/harness/eks_mgmt.go index fc031da87..31d5207fb 100644 --- a/tests/e2e/harness/eks_mgmt.go +++ b/tests/e2e/harness/eks_mgmt.go @@ -13,14 +13,9 @@ import ( "github.com/mulgadc/spinifex/spinifex/utils" ) -// ControlPlaneMgmtIP resolves the EKS control-plane VM's host-bridge (br-mgmt) -// address — the only address reachable from the host/runner. The cluster -// endpoint in DescribeCluster is the internal NLB DNS, which is NXDOMAIN here -// and whose :443->:6443 passthrough is not wired; the VPC NIC (172.31.x) is -// likewise unreachable. The mgmt IP is persisted unencrypted on the cluster's -// KV meta record (ClusterMeta.ControlPlaneMgmtIP), so read it directly over -// NATS. This is the Phase-2 reachability shim — see the eks-6g-e2e plan; the -// real fix (NLB/Route53 passthrough) is tracked separately. +// ControlPlaneMgmtIP resolves the EKS control-plane VM's br-mgmt address — +// the only address reachable from the runner. The NLB endpoint is not wired +// for external access; the mgmt IP is read directly from the NATS KV record. func ControlPlaneMgmtIP(t *testing.T, env *Env, accountID, clusterName string) string { t.Helper() host, token, ca := natsConn(t, env) @@ -49,9 +44,8 @@ func ControlPlaneMgmtIP(t *testing.T, env *Env, accountID, clusterName string) s return meta.ControlPlaneMgmtIP } -// natsConn resolves the NATS host/token/CA for the harness's own JetStream -// reads. SPINIFEX_NATS_* env overrides win (CI / runner-resident runs); else it -// reads the local node's stanza from spinifex.toml. +// natsConn resolves NATS host/token/CA. SPINIFEX_NATS_* env vars win; otherwise +// reads from spinifex.toml. func natsConn(t *testing.T, env *Env) (host, token, ca string) { t.Helper() host = os.Getenv("SPINIFEX_NATS_URL") diff --git a/tests/e2e/harness/env.go b/tests/e2e/harness/env.go index 86dc4595f..aa8141c91 100644 --- a/tests/e2e/harness/env.go +++ b/tests/e2e/harness/env.go @@ -1,10 +1,7 @@ //go:build e2e // Package harness provides shared E2E test primitives for the Spinifex scenario -// suites (cert, lb, multinode, baremetal, reboot, tofu). Each scenario package -// imports this harness; bash drivers under spinifex/tests/e2e/ are being -// retired in favour of these Go scenarios — see -// docs/development/improvements/e2e-go-harness.md. +// suites (cert, lb, multinode, baremetal, reboot, tofu). package harness import ( @@ -66,9 +63,8 @@ func LoadEnv(t *testing.T) *Env { nodeIPs := splitCSV(os.Getenv("SPINIFEX_NODE_IPS")) if len(nodeIPs) == 0 { - // Single-node parity with run-cert-e2e.sh: loopback + every - // non-loopback global IPv4 so SAN checks see all addresses the - // cert is expected to cover. + // Include loopback + every non-loopback global IPv4 so SAN checks see + // all addresses the cert covers. if mode == ModeSingle { nodeIPs = discoverSingleNodeIPs() } else { @@ -77,8 +73,7 @@ func LoadEnv(t *testing.T) *Env { } serviceIPs := splitCSV(os.Getenv("SPINIFEX_SERVICE_IPS")) if len(serviceIPs) == 0 { - // awsgw.host in spinifex.toml is the only IP the gateway actually - // listens on; cert TLS checks must hit that IP, not the loopback. + // Use the awsgw bind IP so TLS checks hit the actual listener, not loopback. if bind := awsgwBindIP(configDir); bind != "" && bind != "0.0.0.0" { serviceIPs = []string{bind} } else { diff --git a/tests/e2e/harness/fixtures.go b/tests/e2e/harness/fixtures.go index ff05b3336..256ce5bce 100644 --- a/tests/e2e/harness/fixtures.go +++ b/tests/e2e/harness/fixtures.go @@ -2,21 +2,10 @@ // Memoized, cleanup-aware resource fixtures for the e2e harness. // -// Background: the bash phase scripts (single-node, multinode) and their first -// Go ports bundle two distinct concerns — provisioning (CreateImage, CreateVpc, -// RunInstances) and assertion (lifecycle checks). Downstream phases inherit -// state via a single shared Fixture struct, so a vpcd-only diff still re-runs -// the AMI + instance chain. -// -// Ensure* helpers separate the two. Each helper is idempotent: first call -// creates the resource and registers teardown; subsequent calls (within the -// same Fixture) return the cached ID. Concurrent callers see one create via -// singleflight. Cleanup is registered on the Fixture's parent test so memoized -// resources survive child-subtest lifetimes. -// -// Bead 2 (e2e-fixtures-harness) of -// docs/development/improvements/e2e-targeted-suite-selection.md. Wired into -// real tests by Bead 3 (e2e-single-fixture-migration). +// Ensure* helpers are idempotent: the first call creates the resource and +// registers teardown; subsequent calls return the cached ID. Concurrent callers +// share one create via singleflight. Cleanup routes to the Fixture's parent test +// so memoized resources outlive child-subtest lifetimes. package harness import ( @@ -83,8 +72,7 @@ type Fixture struct { processCleanups []func() } -// Compile-time interface checks — the real SDK clients must satisfy the -// subset the Fixture stores (CLAUDE.md project standard). +// Compile-time interface checks. var ( _ ec2iface.EC2API = (*ec2.EC2)(nil) _ elbv2iface.ELBV2API = (*elbv2.ELBV2)(nil) @@ -197,15 +185,9 @@ func (f *Fixture) RegisterCleanup(fn func()) { f.registerCleanup(fn) } -// ensureOnce is the shared backbone of every Ensure*. It: -// 1. Returns the cached resource ID if one exists for key. -// 2. Otherwise enters singleflight, calls create exactly once across -// concurrent callers, registers cleanup against fx.parent, and caches -// the result. -// -// create returns (id, cleanupFn). cleanupFn runs once at parent teardown. -// If create fails, no cleanup is registered and the cache is untouched, so -// a retry triggers a fresh create. +// ensureOnce returns the cached resource ID for key, or calls create exactly +// once (via singleflight) and caches the result. On failure no cleanup is +// registered and the cache is untouched, so a retry triggers a fresh create. func (f *Fixture) ensureOnce(t *testing.T, key string, create func() (string, func() error, error)) (string, error) { t.Helper() @@ -260,11 +242,8 @@ func (f *Fixture) resourceName(prefix string) string { return fmt.Sprintf("%s-%s", prefix, f.scratch) } -// pollUntil is the local readiness poller used by Ensure*. Re-implements -// the (timeout, interval, cond) loop on top of the interface-typed EC2 -// client so unit-test fakes can drive it deterministically. Caller-side -// reuse of harness.WaitFor* (which takes a concrete *AWSClient) is -// deliberately avoided here. +// pollUntil polls cond at interval until it returns true or timeout elapses. +// Uses the interface-typed EC2 client so unit-test fakes can drive it. func pollUntil(t *testing.T, timeout, interval time.Duration, cond func() (bool, error)) error { t.Helper() deadline := time.Now().Add(timeout) @@ -334,9 +313,8 @@ func EnsureKeyPair(t *testing.T, fx *Fixture, artifactsDir string) (string, stri // EnsureAMI // ---------------------------------------------------------------------------- -// AMISource describes how to materialise an AMI. Exactly one of Existing or -// CreateFrom must be set: Existing pins a pre-baked image; CreateFrom drives -// CreateImage against a snapshotable source instance. +// AMISource describes how to materialise an AMI. Set exactly one of Existing +// (pre-baked image ID) or CreateFrom (drives CreateImage from a source instance). type AMISource struct { // Existing is a pre-baked AMI ID to verify + return as-is. Existing string @@ -349,9 +327,8 @@ type AMICreateSpec struct { Name string Description string Architecture string - // NoReboot mirrors CreateImage's NoReboot flag. Set true when the - // source instance must keep running across the AMI bake (e.g. Phase 5e - // hands the instance off to Phase 6/7 lifecycle assertions). + // NoReboot mirrors CreateImage's NoReboot flag. Set true when the source + // instance must stay running across the bake. NoReboot bool } diff --git a/tests/e2e/harness/imds_diag.go b/tests/e2e/harness/imds_diag.go index 4f960fac7..242f09937 100644 --- a/tests/e2e/harness/imds_diag.go +++ b/tests/e2e/harness/imds_diag.go @@ -11,29 +11,9 @@ import ( ) // DumpIMDSDatapathDiagnostics emits a triage bundle for an IMDS reachability -// failure (guest curl to 169.254.169.254 times out). It captures both halves of -// the per-subnet L2 datapath so a triager can tell request-path from reply-path: -// -// - OVN realisation: is imds-port- bound/up on this chassis, is the -// localport LSP defined as expected. There is no VPC-LR static route to check -// under L2 — the localport sits directly on the subnet switch, so the request -// never enters a router. -// - Host realisation (inside the per-subnet netns imds-): the -// imds-h- veth's address/route/neighbour state and the listener socket -// — the load-bearing question is whether the netns has a real L2 path back to -// guestIP. The veth carries 169.254.169.254/30 plus the subnet CIDR on-link, so -// a healthy dump shows the IPv4 address, an on-link reply route out the veth -// (no .253 gateway hop), and a resolved neighbour entry for guestIP; a broken -// dump shows an addressless veth that accepted the SYN but never returned the -// SYN-ACK. -// - In-flight evidence: conntrack (per-netns) for 169.254.169.254 (a stuck -// SYN_RECV is the smoking gun that the host received the SYN and replied but -// the ACK never came back) and the br-int flows touching the IMDS / guest -// addresses. -// -// Non-fatal — runs purely for log/artifact signal so the test's own Fatal still -// wins. Skips silently when OVN tooling / passwordless sudo aren't available, so -// it's safe to call from developer laptops. +// failure. Captures OVN port-binding state, per-subnet netns veth/route/neigh, +// conntrack, and br-int flows for both request and reply paths. +// Non-fatal — best-effort diagnostics; skips if OVN tooling or sudo unavailable. func DumpIMDSDatapathDiagnostics(t *testing.T, subnetID, guestIP, artifactDir string) { t.Helper() fmt.Printf("\n%s%s── DIAGNOSTICS: IMDS datapath (subnet=%s guest=%s) ──%s\n", diff --git a/tests/e2e/harness/log.go b/tests/e2e/harness/log.go index d3bc31ea5..f62b092b1 100644 --- a/tests/e2e/harness/log.go +++ b/tests/e2e/harness/log.go @@ -9,10 +9,7 @@ import ( "testing" ) -// ANSI color codes — rendered live by the Actions UI; raw escapes pass -// through to artifact log files harmlessly. Disabled when stdout is not -// a TTY and GITHUB_ACTIONS isn't set, so piped local `go test -v` stays -// clean. +// ANSI color codes. Disabled when stdout is not a TTY and GITHUB_ACTIONS is unset. var ( colorBold = "\x1b[1m" colorDim = "\x1b[2m" @@ -28,25 +25,15 @@ func init() { } } -// Phase writes a colored section banner directly to stdout, bypassing -// t.Logf so the line lands flush-left without the testing framework's -// `::` prefix or forced indent. Banners are pure visual -// structure — the diagnostic resource IDs themselves stay on t.Logf -// (see Detail) so they remain attributed to the owning test in JUnit XML. -// -// Use sparingly: one line per logical phase, not per attribute. +// Phase writes a colored section banner to stdout, bypassing t.Logf to avoid +// the framework's indentation. Use one line per logical phase, not per attribute. func Phase(t *testing.T, format string, args ...any) { t.Helper() fmt.Fprintf(os.Stdout, "\n%s%s━━ %s ━━%s\n", colorBold, colorCyan, fmt.Sprintf(format, args...), colorReset) } -// Step emits a single-line sub-phase marker directly to os.Stdout so it -// streams live in CI. Bypasses t.Logf because the testing framework -// buffers per-test and releases only on PASS/FAIL — long subtests -// (lb's Internal_NLB ~60s, Phase8Acct_AccountScoping ~115s) would -// otherwise go silent for the whole subtest. go-junit-report parses -// stdout between `=== RUN` and `--- PASS/FAIL` markers, so per-test -// attribution is preserved without the buffered duplicate. +// Step emits a sub-phase marker directly to stdout so it streams live in CI. +// Bypasses t.Logf, which buffers until PASS/FAIL and goes silent for long subtests. func Step(t *testing.T, format string, args ...any) { t.Helper() fmt.Fprintf(os.Stdout, "%s· %s%s\n", colorDim, fmt.Sprintf(format, args...), colorReset) diff --git a/tests/e2e/harness/multinode_aws_via.go b/tests/e2e/harness/multinode_aws_via.go index 76aa64158..795adcc65 100644 --- a/tests/e2e/harness/multinode_aws_via.go +++ b/tests/e2e/harness/multinode_aws_via.go @@ -4,14 +4,8 @@ package harness import "testing" -// AWSClientForGateway returns an AWSClient pointed at the named node's -// gateway endpoint instead of env.ServiceIPs[0]. Used by multinode phases -// 6 and 7 (cross-node gateway access + cross-node operations) where the -// test issues describe/stop/start against every node in turn. -// -// Bash equivalent: aws_via in run-multinode-e2e.sh — sets AWS_ENDPOINT_URL -// per call. Here we clone the *Env so the swap is scoped to one call site -// and concurrent callers can each address a different node safely. +// AWSClientForGateway returns an AWSClient scoped to a single node's gateway +// endpoint. Clones the Env so concurrent callers can safely address different nodes. func AWSClientForGateway(t *testing.T, env *Env, node Node) *AWSClient { t.Helper() if env == nil { diff --git a/tests/e2e/harness/multinode_cluster.go b/tests/e2e/harness/multinode_cluster.go index 39c95e4d2..2388d5b45 100644 --- a/tests/e2e/harness/multinode_cluster.go +++ b/tests/e2e/harness/multinode_cluster.go @@ -20,28 +20,16 @@ const ( awsgwHealthPort = 9999 ) -// natsRoutezResponse models the subset of /routez we care about. Bash polled -// `jq '.routes | unique_by(.remote_id) | length'` — we match by parsing the -// route slice and counting distinct remote_id values. +// natsRoutezResponse models the NATS /routez subset used to count distinct peers. type natsRoutezResponse struct { Routes []struct { RemoteID string `json:"remote_id"` } `json:"routes"` } -// WaitNATSPeers polls every node's NATS /routez endpoint until each reports -// at least want distinct peers. Default timeout 60s / interval 2s — covers -// the post-join settle window observed in run-multinode-e2e.sh phase 2. -// -// NATS monitor binds 127.0.0.1:8222 only (cmd/spinifex/cmd/templates/nats.conf), -// so the routez query has to land on each node's loopback. Bash uses -// `peer_ssh "$ip" curl http://127.0.0.1:8222/routez`; we mirror that via -// PeerSSH + curl rather than dialling node.Addr:8222 directly (which fails -// with connection refused). -// -// Use want=2 on a healthy 3-node cluster (each peer sees the other two via -// one route — NATS dedupes by remote_id so a 3-node mesh shows 2 unique -// peers per node). Drops to want=1 after a single-node failure. +// WaitNATSPeers polls every node's NATS /routez until each reports at least want +// distinct peers (timeout 60s, interval 2s). NATS monitor binds 127.0.0.1:8222 +// only, so queries run via PeerSSH + curl rather than dialling node.Addr directly. func (c *Cluster) WaitNATSPeers(t *testing.T, want int, opts ...PollOpt) { t.Helper() cfg := applyOpts(pollCfg{timeout: 60 * time.Second, interval: 2 * time.Second}, opts...) diff --git a/tests/e2e/harness/multinode_diag.go b/tests/e2e/harness/multinode_diag.go index 7da1db670..67d1e779a 100644 --- a/tests/e2e/harness/multinode_diag.go +++ b/tests/e2e/harness/multinode_diag.go @@ -33,13 +33,10 @@ func DumpAllNodeLogs(t *testing.T, c *Cluster, artifactsDir string) { } } -// SpxGetNodesAcrossCluster runs `spx get nodes` best-effort and returns the -// raw output stripped of trailing whitespace. Bash phase 9 (run-multinode-e2e -// .sh:1018) appends `|| echo ""` to swallow a non-zero spx exit — the CLI ↔ -// NATS dial races the cluster join right after node restart and can return -// "no servers available for connection" even though the data path is healthy -// (gateway+daemon already verified). Mirror that lenience so phase 9 doesn't -// fail on a known-flaky CLI dial. +// SpxGetNodesAcrossCluster runs `spx get nodes` best-effort and returns trimmed +// output. CLI ↔ NATS dial can race the cluster join after a node restart and +// return "no servers available" even when the data path is healthy; errors are +// swallowed intentionally. func SpxGetNodesAcrossCluster(t *testing.T) string { t.Helper() return strings.TrimSpace(SpxRunBestEffort(t, "get", "nodes", "--timeout", "5s")) diff --git a/tests/e2e/harness/multinode_distribution.go b/tests/e2e/harness/multinode_distribution.go index d0932a9a0..45d602488 100644 --- a/tests/e2e/harness/multinode_distribution.go +++ b/tests/e2e/harness/multinode_distribution.go @@ -11,16 +11,8 @@ import ( "time" ) -// InstanceHostingNode returns the cluster node currently running the named -// instance's qemu process. Mirrors the bash find_instance_node helper: -// SSH each node, `ps auxw | grep | grep qemu-system`. Returns -// nil if no node owns the instance (caller decides whether that's fatal). -// -// Skips fix-the-process for the local node (index 0) because the bash uses a -// direct `ps` instead of SSH for self — but Go runs everywhere as part of -// the same test binary, so we accept the small redundancy and SSH back to -// self too. Avoids a special case that the bash had only because of script -// invocation context. +// InstanceHostingNode returns the cluster node running the named instance's +// qemu process (ps auxw | grep qemu-system). Returns nil if no node owns it. func InstanceHostingNode(t *testing.T, c *Cluster, instanceID string) *Node { t.Helper() ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) @@ -64,15 +56,11 @@ func AssertSpreadAcrossNodes(t *testing.T, c *Cluster, instanceIDs []string, min } // runPSGrep runs the ps+grep pipeline on a node and returns combined output. -// Bash uses `ps auxw | grep | grep qemu-system | grep -v grep`; we -// preserve the same pipeline so future schedulers that change the qemu -// binary name (e.g. qemu-system-aarch64) surface a single-place change. func runPSGrep(ctx context.Context, ssh *PeerSSH, n Node, instanceID string) (string, error) { cmd := fmt.Sprintf("ps auxw | grep %q | grep qemu-system | grep -v grep", instanceID) out, err := ssh.Run(ctx, n.Addr, cmd) if err != nil { - // `grep` returns exit 1 when no match; treat that as "no match", - // not a transport error. + // exit 1 from grep means no match, not a transport error. if exitErr, ok := asExitErr(err); ok && exitErr.ExitCode() == 1 { return "", nil } diff --git a/tests/e2e/harness/multinode_guest_ssh.go b/tests/e2e/harness/multinode_guest_ssh.go index 5a19af634..bd915c740 100644 --- a/tests/e2e/harness/multinode_guest_ssh.go +++ b/tests/e2e/harness/multinode_guest_ssh.go @@ -14,13 +14,9 @@ import ( "github.com/aws/aws-sdk-go/aws" ) -// GuestSSHEndpoint resolves (host, 22) for SSH-ing into a guest VM via its -// public IP. -// -// There is deliberately NO qemu-hostfwd fallback (see InstancePublicSSHHost): -// hostfwd bypasses the OVN datapath, so reaching a guest through it would mask -// the networking regressions these tests exist to catch. A running instance -// with no public IP is a hard failure. +// GuestSSHEndpoint resolves (host, 22) for SSH-ing into a guest VM. No +// qemu-hostfwd fallback: hostfwd bypasses the OVN datapath (see +// InstancePublicSSHHost). A running instance with no public IP is a hard failure. func GuestSSHEndpoint(t *testing.T, c *AWSClient, cluster *Cluster, instanceID string, opts ...PollOpt) (host string, port int) { t.Helper() _ = cluster // retained for signature stability; node hostfwd lookup removed diff --git a/tests/e2e/harness/multinode_node_lifecycle.go b/tests/e2e/harness/multinode_node_lifecycle.go index 3376bce42..dd23edff7 100644 --- a/tests/e2e/harness/multinode_node_lifecycle.go +++ b/tests/e2e/harness/multinode_node_lifecycle.go @@ -9,16 +9,12 @@ import ( "time" ) -// StopNode stops every spinifex unit on a remote node via -// `sudo systemctl stop spinifex.target`. Used by phase 8 (node failure). -// Bash issues the same command and tolerates a non-zero exit because the -// cluster shutdown sequence racily kills the SSH connection — we mirror -// that lenience by logging instead of fatal-ing. +// StopNode stops every spinifex unit on a remote node via systemctl. Non-fatal: +// the cluster shutdown sequence can racily kill the SSH connection. func StopNode(t *testing.T, node Node) { t.Helper() ssh := NewPeerSSH() - // spinifex.target has 6+ units (predastore, NATS, awsgw, vpcd, daemon, ui). - // stop takes 30-60s in practice; budget 3min to cover slow shutdowns. + // 3min covers slow 6+ unit shutdowns (predastore, NATS, awsgw, vpcd, daemon, ui). ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) defer cancel() if _, err := ssh.Run(ctx, node.Addr, "sudo systemctl stop spinifex.target"); err != nil { @@ -26,16 +22,12 @@ func StopNode(t *testing.T, node Node) { } } -// StartNode brings the spinifex.target back up on a remote node. Used by -// phase 9 (recovery) and as a t.Cleanup safety net in phase 8 so a -// cancelled run doesn't leave the cluster degraded for the next suite. +// StartNode brings spinifex.target back up on a remote node. Also used as a +// t.Cleanup safety net so a cancelled run doesn't leave the cluster degraded. func StartNode(t *testing.T, node Node) { t.Helper() ssh := NewPeerSSH() - // systemctl start on spinifex.target blocks until every dependent unit - // is Active=active — predastore + NATS + awsgw + vpcd + daemon + ui can - // take 60-90s combined. 30s was too tight and produced "signal: killed" - // SSH terminations under context cancel; bump to 5min. + // 5min: systemctl blocks until all units are Active; 6 units can take 60-90s. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() if _, err := ssh.Run(ctx, node.Addr, "sudo systemctl start spinifex.target"); err != nil { diff --git a/tests/e2e/harness/ovn.go b/tests/e2e/harness/ovn.go index 105363cc5..106130422 100644 --- a/tests/e2e/harness/ovn.go +++ b/tests/e2e/harness/ovn.go @@ -39,11 +39,7 @@ func OvnSbctl(t *testing.T, args ...string) string { } // OvnTrace runs `sudo -n ovn-trace ` and returns the detailed pipeline -// trace. Usage is `ovn-trace [options] DATAPATH MICROFLOW`; pass any `--ct=...` -// options first, then the datapath, then the microflow. The detailed trace names -// each logical-flow table it traverses (`ls_in_*`, `lr_in_*`, …) and prints the -// final `/* output to "" */`, so callers can assert both the delivery port -// and the absence of router stages. +// trace. Pass `--ct=...` options first, then DATAPATH, then MICROFLOW. func OvnTrace(t *testing.T, args ...string) string { t.Helper() return runOvn(t, "ovn-trace", args...) @@ -62,13 +58,8 @@ func runOvn(t *testing.T, tool string, args ...string) string { return strings.TrimSpace(stdout.String()) } -// WaitForPortGroupMember polls until the OVN port_group pg contains the -// logical-switch-port lsp. Default 10s/1s — northd propagation is normally -// sub-second but tests run on busy nodes. -// -// Lookup uses ovn-nbctl --bare --columns=ports find port_group name= -// (same shape as run-e2e.sh:3311) and checks for the lsp's UUID in the -// returned set. +// WaitForPortGroupMember polls until OVN port_group pg contains lsp (10s/1s). +// Uses ovn-nbctl to look up the LSP UUID and confirm it appears in the port_group. func WaitForPortGroupMember(t *testing.T, pg, lsp string, opts ...PollOpt) { t.Helper() cfg := applyOpts(pollCfg{timeout: 10 * time.Second, interval: 1 * time.Second}, opts...) diff --git a/tests/e2e/harness/peer_ssh.go b/tests/e2e/harness/peer_ssh.go index 182be0f6f..4702bfd19 100644 --- a/tests/e2e/harness/peer_ssh.go +++ b/tests/e2e/harness/peer_ssh.go @@ -11,11 +11,8 @@ import ( "time" ) -// PeerSSH is a minimal SSH wrapper that shells out to /usr/bin/ssh with the -// flags the bash E2E scripts use. It exists so LB/multinode scenarios can -// drive remote nodes without depending on the DDIL harness. Step 1.5 of -// docs/development/improvements/e2e-go-harness.md replaces this with the -// typed x/crypto/ssh client hoisted from ddil/harness/ssh.go. +// PeerSSH is a minimal SSH wrapper for driving remote cluster nodes in +// LB/multinode scenarios. type PeerSSH struct { User string KeyPath string diff --git a/tests/e2e/harness/poll.go b/tests/e2e/harness/poll.go index 42bee7146..dbae1aee9 100644 --- a/tests/e2e/harness/poll.go +++ b/tests/e2e/harness/poll.go @@ -11,10 +11,8 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" ) -// PollOpt tunes the default timeout / interval for a Wait* helper. Defaults -// are tuned per-resource based on the bash `sleep N; check` loops in -// run-e2e.sh — instance/volume = 5min, snapshot/image = 10min, all polled -// every 2s. +// PollOpt tunes the default timeout / interval for a Wait* helper. +// Resource defaults: instance/volume = 5min, snapshot/image = 10min, 2s interval. type PollOpt func(*pollCfg) type pollCfg struct { @@ -36,9 +34,7 @@ func applyOpts(def pollCfg, opts ...PollOpt) pollCfg { } // WaitForInstanceState polls DescribeInstances until State.Name == target. -// Returns the latest *ec2.Instance on success; t.Fatal on timeout. Separate -// from lb.go's WaitForInstanceRunning so callers can wait for stopped / -// terminated / pending without one helper per state. +// Returns the latest instance on success; t.Fatal on timeout. func WaitForInstanceState(t *testing.T, c *AWSClient, id, target string, opts ...PollOpt) *ec2.Instance { t.Helper() cfg := applyOpts(pollCfg{timeout: 5 * time.Minute, interval: 2 * time.Second}, opts...) diff --git a/tests/e2e/harness/rawhttp.go b/tests/e2e/harness/rawhttp.go index 70faed0e2..87a93cded 100644 --- a/tests/e2e/harness/rawhttp.go +++ b/tests/e2e/harness/rawhttp.go @@ -36,9 +36,7 @@ func PostAWSAction(t *testing.T, env *Env, c *AWSClient, action string, params m form := url.Values{} form.Set("Action", action) if _, ok := params["Version"]; !ok { - // EC2 query API version — same value the SDK injects on every call. - // Hard-coded here because we are bypassing the SDK entirely. - form.Set("Version", "2016-11-15") + form.Set("Version", "2016-11-15") // EC2 query API version; bypassing SDK. } for k, v := range params { form.Set(k, v) @@ -56,14 +54,11 @@ func PostAWSAction(t *testing.T, env *Env, c *AWSClient, action string, params m t.Fatalf("PostAWSAction: no credentials on client") } signer := v4.NewSigner(creds) - // SignedReader keeps the body intact for replay; v4.Sign reads-then-resets. if _, err := signer.Sign(req, bytes.NewReader(bodyBytes), "ec2", region, time.Now()); err != nil { t.Fatalf("PostAWSAction: sign: %v", err) } - // Reuse the EC2 client's HTTP transport so TLS trust matches the rest of - // the suite. Fall back to a CA-loaded transport if the embedded client - // is using the SDK default. + // Reuse the EC2 client's transport so TLS trust is consistent. httpClient := c.EC2Conf.Config.HTTPClient if httpClient == nil { httpClient = &http.Client{ @@ -89,9 +84,8 @@ func PostAWSAction(t *testing.T, env *Env, c *AWSClient, action string, params m return status, body, awsCode } -// EC2 returns either … or the SDK-style -// … envelope. Try both shapes; the -// gateway has shipped one then the other across releases. +// EC2 error envelopes come in two shapes; the gateway has returned both. +// ec2ErrorResponse covers , sdkErrorResponse covers . type ec2ErrorResponse struct { XMLName xml.Name `xml:"Response"` Errors struct { diff --git a/tests/e2e/harness/reset.go b/tests/e2e/harness/reset.go index 34351d232..9f2866588 100644 --- a/tests/e2e/harness/reset.go +++ b/tests/e2e/harness/reset.go @@ -12,28 +12,17 @@ import ( "time" ) -// routezResponse is the subset of the NATS /routez monitoring payload the -// harness consumes. Only remote_name is required because the harness counts -// unique peer names (not raw route count — NATS opens multiple TCP routes -// per peer and num_routes would overcount). The shell suite's -// verify_nats_cluster helper uses the same predicate. +// routezResponse is the subset of NATS /routez used to count unique peer names. +// Raw route count is not used; NATS opens multiple TCP routes per peer. type routezResponse struct { Routes []struct { RemoteName string `json:"remote_name"` } `json:"routes"` } -// ResetAllNodes returns the cluster to a known clean state: every node has -// all iptables rules flushed, every tc netem qdisc removed, and both -// spinifex-nats and spinifex-daemon running. After the per-node reset, it -// waits for cluster health (every daemon /health returns 200 and each node's -// NATS monitor reports len(Nodes)-1 unique route peers) before returning. -// -// Idempotent — safe against a clean cluster. Three callers: -// 1. harness.Run, between a failed scenario and its retry. -// 2. AssertCleanState, when a pre-scenario baseline is dirty. -// 3. The suite-level signal handler in TestMain, so SIGINT/SIGTERM do not -// leave the cluster partitioned or netem-shaped. +// ResetAllNodes returns the cluster to a known clean state: iptables flushed, +// tc netem removed, and spinifex-nats + spinifex-daemon running on every node. +// Idempotent — safe to call on a clean cluster. func ResetAllNodes(ctx context.Context, c *Cluster, ssh SSH) error { for _, n := range c.Nodes { if err := resetNode(ctx, ssh, n); err != nil { @@ -44,17 +33,12 @@ func ResetAllNodes(ctx context.Context, c *Cluster, ssh SSH) error { } func resetNode(ctx context.Context, ssh SSH, n Node) error { - // Flush every DDIL-owned firewall rule. iptables -F on the default - // (filter) table covers INPUT/OUTPUT DROPs installed by PartitionNode; - // iptables -X removes any transient user chains a scenario happens to - // leave behind. This matches HealNode's teardown. + // -F flushes INPUT/OUTPUT DROP rules from PartitionNode; -X removes stale user chains. if _, err := ssh.Run(ctx, n, "sudo iptables -F && sudo iptables -X"); err != nil { return fmt.Errorf("e2e harness: reset iptables on %s: %w", n.Name, err) } - // Remove netem qdiscs only on interfaces that actually carry one. - // Scenarios may pick different NICs (eth0, ens18, br-mgmt) and a fixed - // interface list here would silently miss whichever one was used. + // Remove netem qdiscs only from interfaces that carry one; NIC names vary. ifaces, err := netemIfaces(ctx, ssh, n) if err != nil { return err @@ -73,12 +57,8 @@ func resetNode(ctx context.Context, ssh SSH, n Node) error { return nil } -// netemIfaces lists every interface on node whose root qdisc is a netem -// qdisc. Used by resetNode to decide what to tear down and by -// AssertCleanState to detect a dirty baseline. +// netemIfaces lists every interface on node that has a netem root qdisc. func netemIfaces(ctx context.Context, ssh SSH, n Node) ([]string, error) { - // `tc qdisc show` prints one row per qdisc; `qdisc netem 8003: dev eth0 - // root ...` → awk $5 yields the interface name. out, err := ssh.Run(ctx, n, "tc qdisc show | awk '/qdisc netem/ {print $5}' | sort -u") if err != nil { return nil, fmt.Errorf("e2e harness: list netem qdiscs on %s: %w", n.Name, err) @@ -130,9 +110,8 @@ func checkClusterHealthy(ctx context.Context, c *Cluster, ssh SSH, dc *DaemonCli return nil } -// natsPeerCount SSHes into node, curls the node-local NATS monitor endpoint -// (bound to 127.0.0.1:8222 by the spinifex NATS config), and returns the -// number of unique peer names it reports. +// natsPeerCount returns the number of unique peer names from the node-local +// NATS /routez endpoint (127.0.0.1:8222). func natsPeerCount(ctx context.Context, ssh SSH, n Node) (int, error) { out, err := ssh.Run(ctx, n, "curl -fsS http://127.0.0.1:8222/routez") if err != nil { @@ -152,11 +131,9 @@ func natsPeerCount(ctx context.Context, ssh SSH, n Node) (int, error) { return len(seen), nil } -// AssertCleanState verifies no node carries a DDIL-owned firewall rule or -// netem qdisc before a scenario begins. A dirty baseline is not itself a -// test failure — it usually means a previous scenario's t.Cleanup did not -// run (panic, SIGKILL, CI runner killed) — so the helper logs details and -// calls ResetAllNodes to recover. Only a failure of that reset is fatal. +// AssertCleanState verifies no node has leftover DROP rules or netem qdiscs. +// A dirty baseline is logged and reset via ResetAllNodes; only a reset failure +// is fatal. func AssertCleanState(ctx context.Context, t *testing.T, c *Cluster, ssh SSH) { t.Helper() @@ -187,12 +164,8 @@ func AssertCleanState(ctx context.Context, t *testing.T, c *Cluster, ssh SSH) { } } -// countDropRules reports how many DROP rules are installed in INPUT/OUTPUT -// on node. `iptables -S` without a chain prints every chain in -// iptables-save format (`-A INPUT ... -j DROP`); filtering for the two -// chains we care about keeps the count scoped to PartitionNode's targets. -// `2>/dev/null` swallows sudo/iptables stderr so a noisy environment does -// not poison the grep input, and `|| true` covers grep's no-match exit 1. +// countDropRules counts DROP rules in INPUT/OUTPUT installed by PartitionNode. +// `|| true` handles grep's no-match exit 1; `2>/dev/null` suppresses sudo noise. func countDropRules(ctx context.Context, ssh SSH, n Node) (int, error) { const cmd = `sudo iptables -S 2>/dev/null | grep -E '^-A (INPUT|OUTPUT) ' | grep -c -- '-j DROP' || true` out, err := ssh.Run(ctx, n, cmd) diff --git a/tests/e2e/harness/spxcli.go b/tests/e2e/harness/spxcli.go index e72959709..cedd6c0ac 100644 --- a/tests/e2e/harness/spxcli.go +++ b/tests/e2e/harness/spxcli.go @@ -30,16 +30,9 @@ func SpxBin() string { } // spxChildEnv returns a sanitized environment for `spx` child processes. -// spx CLI calls viper.AutomaticEnv() with prefix "SPINIFEX" inside -// config.LoadConfig (spinifex/spinifex/config/config.go:212), so any -// SPINIFEX_* env var the harness sets to wire up its own ClusterFromEnv / -// Env loaders silently overrides the matching cluster.toml field — most -// notably SPINIFEX_NODES (csv of IPs) clobbers the `nodes` map and leaves -// NATS.Host empty, surfacing as `nats: no servers available for -// connection` even though the cluster is healthy (mulga-siv-90 run -// 26195522455). Strip every SPINIFEX_* except the small set the spx root -// command actually binds via BindEnv (kept for parity with operator -// muscle-memory). +// Harness SPINIFEX_* vars would override cluster.toml fields via viper +// AutomaticEnv, so we strip them except for the small set spx explicitly +// binds via BindEnv. func spxChildEnv() []string { keep := map[string]struct{}{ "SPINIFEX_CONFIG_PATH": {}, @@ -102,11 +95,9 @@ func SpxGetVMs(t *testing.T) string { return SpxRun(t, false, "get", "vms") } -// SpxRunBestEffort runs `spx ` and returns combined output ignoring -// the exit code. Matches the bash `2>/dev/null` + no exit-check pattern used -// by `spx get vms` in run-multinode-e2e.sh phases 2-3 — the CLI's NATS dial -// can race the cluster join shortly after bootstrap without indicating a -// data-path fault, so a transient non-zero exit shouldn't fail the suite. +// SpxRunBestEffort runs `spx ` and returns combined output, ignoring +// the exit code. CLI NATS dial can race cluster join and return an error even +// when the data path is healthy. func SpxRunBestEffort(t *testing.T, args ...string) string { t.Helper() var buf bytes.Buffer diff --git a/tests/e2e/harness/ssh.go b/tests/e2e/harness/ssh.go index 6971f4dfb..1c9c39aa5 100644 --- a/tests/e2e/harness/ssh.go +++ b/tests/e2e/harness/ssh.go @@ -37,13 +37,9 @@ type sshClient struct { var _ SSH = (*sshClient)(nil) -// NewSSH returns an SSH transport configured from a Cluster. -// -// The private key at cluster.SSHKeyPath must be in a format understood by -// ssh.ParsePrivateKey (OpenSSH or PEM). Host keys are not verified — the -// harness runs against short-lived test infrastructure where TOFU would add -// noise and no security benefit. Do not reuse this transport for production -// access. +// NewSSH returns an SSH transport configured from a Cluster. Host keys are not +// verified: the harness targets short-lived infra where TOFU adds noise and no +// security benefit. func NewSSH(cluster *Cluster) (SSH, error) { keyBytes, err := os.ReadFile(cluster.SSHKeyPath) if err != nil { @@ -95,9 +91,7 @@ func (s *sshClient) Run(ctx context.Context, node Node, cmd string) ([]byte, err session.Stdout = &out session.Stderr = &out - // Propagate context cancellation by signalling the session. ssh.Session - // has no Kill; Signal(KILL) is the portable way to interrupt a running - // command. + // Signal(KILL) is the portable way to interrupt a running command. done := make(chan struct{}) defer close(done) go func() { diff --git a/tests/e2e/harness/state.go b/tests/e2e/harness/state.go index 91dd95a39..b7b08db76 100644 --- a/tests/e2e/harness/state.go +++ b/tests/e2e/harness/state.go @@ -8,11 +8,8 @@ import ( "testing" ) -// LocalStateRevision reads the daemon's local state file revision via -// /local/status. The revision is a monotonic counter bumped on every write -// to the local state file (see daemon-local-autonomy §1a); AssertMonotonic -// uses it to prove that a fault did not cause the daemon to rewind or reset -// its persisted state. +// LocalStateRevision reads the daemon's local state revision via /local/status. +// AssertMonotonic uses it to verify a fault didn't rewind persisted state. func LocalStateRevision(ctx context.Context, d *DaemonClient, node Node) (uint64, error) { s, err := d.Status(ctx, node) if err != nil { diff --git a/tests/e2e/harness/sweep.go b/tests/e2e/harness/sweep.go index 750ae74cb..b3e9b5b8a 100644 --- a/tests/e2e/harness/sweep.go +++ b/tests/e2e/harness/sweep.go @@ -1,14 +1,10 @@ //go:build e2e -// Run-scoped resource tagging + teardown sweep (mulga-siv-277). +// Run-scoped resource tagging and teardown sweep. // -// Every resource the Ensure* fixtures create is tagged e2e:run= when -// SPINIFEX_E2E_RUN_ID is set (CI: github.run_id; local: a temp id). A thin -// teardown.test binary then sweeps everything carrying that tag at the end of a -// run — reclaiming resources leaked by a crashed suite, an interrupted run, or -// a persistent (non-tofu-destroyed) cluster. The run-id scoping keeps the sweep -// precise: with no run id, tagging and sweeping both no-op, so the sweep can -// never delete an untagged or foreign resource. +// Ensure* fixtures tag resources e2e:run= when SPINIFEX_E2E_RUN_ID is +// set. A teardown binary sweeps everything with that tag at run end. Without a +// run id, tagging and sweeping both no-op. package harness import ( diff --git a/tests/e2e/harness/tls.go b/tests/e2e/harness/tls.go index 495a7d2b9..c84832367 100644 --- a/tests/e2e/harness/tls.go +++ b/tests/e2e/harness/tls.go @@ -53,11 +53,8 @@ func HTTPSGet(url string, caPool *x509.CertPool, timeout time.Duration) (int, [] return resp.StatusCode, body, nil } -// FetchTLSPosture opens a TLS connection to host:port and returns the -// negotiated version and curve. Trust is bypassed because the goal is to -// observe the handshake, not validate the cert chain. Used by e2e tests to -// assert that deployed services actually negotiate TLS 1.3 + PQ hybrid -// (see internal/tlsconfig.Curves) under the production FIPS build. +// FetchTLSPosture opens a TLS connection and returns the negotiated version and +// curve. Trust is bypassed to observe the handshake, not validate the cert chain. func FetchTLSPosture(host string, port int, timeout time.Duration) (version uint16, curve tls.CurveID, err error) { dialer := &net.Dialer{Timeout: timeout} conn, err := tls.DialWithDialer(dialer, "tcp", fmt.Sprintf("%s:%d", host, port), &tls.Config{ diff --git a/tests/e2e/harness/wait.go b/tests/e2e/harness/wait.go index 9189dafc9..d3b3c0813 100644 --- a/tests/e2e/harness/wait.go +++ b/tests/e2e/harness/wait.go @@ -46,14 +46,8 @@ func EventuallyErr(t *testing.T, cond func() error, timeout, interval time.Durat } } -// RetryWithReset runs fn as an "attempt-1" subtest; on failure it calls -// ResetAllNodes and retries fn as "attempt-2". Both attempts surface in the -// test output so CI logs preserve the transient-vs-deterministic signal — -// the retry is a diagnostic aid, not a way to mask flakes. -// -// Used by DDIL's quarantine wrapper (ddil/harness.Run) and any other scenario -// that wants the same retry-after-reset shape without taking on DDIL's -// quarantine semantics. +// RetryWithReset runs fn as "attempt-1"; on failure it calls ResetAllNodes and +// retries as "attempt-2". The retry is a diagnostic aid, not a flake mask. func RetryWithReset(t *testing.T, c *Cluster, ssh SSH, label string, fn func(*testing.T)) { t.Helper() if t.Run("attempt-1", fn) { diff --git a/tests/e2e/harness/witness.go b/tests/e2e/harness/witness.go index 9a4474a8b..71264ce4c 100644 --- a/tests/e2e/harness/witness.go +++ b/tests/e2e/harness/witness.go @@ -50,28 +50,10 @@ type Witness struct { keyName string } -// NewWitness constructs a Witness from environment-supplied settings: -// -// - AWS_REGION (required) — passed to the AWS SDK client. -// - DDIL_WITNESS_AMI (optional) — specific AMI ID; if unset, an -// ami-ubuntu-* image is discovered at launch time. -// - DDIL_WITNESS_INSTANCE_TYPE (optional) — explicit instance type; if -// unset, the smallest registered type (by memory, then vCPUs) is -// auto-discovered at launch time. Auto-discovery beats hard-coding -// `t2.micro` because tofu clusters seed `*.nano` variants under -// names that change between releases. -// - DDIL_WITNESS_KEY_NAME (default spinifex-key) — EC2 key pair name -// the daemon injects via cloud-init. -// - DDIL_GUEST_SSH_USER (default ec2-user) — user for guest SSH; matches -// the default account on the cluster's ami-ubuntu-* images (shell E2E -// uses the same login at tests/e2e/run-multinode-e2e.sh:650). -// - DDIL_GUEST_SSH_KEY (required) — private key for guest SSH; must -// pair with the public material registered under DDIL_WITNESS_KEY_NAME -// so authorized_keys on the cloud-init guest accepts it. -// -// Credentials for the SDK come from the default chain (AWS_ACCESS_KEY_ID/ -// AWS_SECRET_ACCESS_KEY or shared profile), matching the tofu-cluster -// convention used by the shell E2E suites. +// NewWitness constructs a Witness from environment variables. Required: +// AWS_REGION, DDIL_GUEST_SSH_KEY. Optional: DDIL_WITNESS_AMI, +// DDIL_WITNESS_INSTANCE_TYPE (defaults to smallest registered type), +// DDIL_WITNESS_KEY_NAME (default spinifex-key), DDIL_GUEST_SSH_USER (default ec2-user). func NewWitness(cluster *Cluster, transport SSH) (*Witness, error) { region := os.Getenv("AWS_REGION") if region == "" { @@ -324,12 +306,8 @@ func (w *Witness) resolveAMI(ctx context.Context) (string, error) { } // resolveInstanceType returns the witness instance type, caching the result. -// When DDIL_WITNESS_INSTANCE_TYPE is set it is honoured verbatim; otherwise -// DescribeInstanceTypes is consulted and the smallest registered type — by -// memory first, vCPUs as tiebreaker — is selected. The sort key is -// quantitative on purpose: naming conventions (`t2.nano`, `c6i.large`, …) -// shift between cluster releases, but smallest-by-resource always picks the -// cheapest valid launch target. +// Falls back to the smallest registered type (by memory, then vCPUs) so the +// choice doesn't depend on naming conventions that shift between releases. func (w *Witness) resolveInstanceType(ctx context.Context) (string, error) { if w.instanceType != "" { return w.instanceType, nil @@ -399,25 +377,18 @@ func (w *Witness) waitForRunning(ctx context.Context, id string, timeout time.Du } } -// findHost walks every cluster node looking for the QEMU process that owns -// instanceID. The daemon embeds the instance ID in `-pidfile`, `-qmp`, and -// `-name guest=` on the QEMU command line, so grepping for the ID identifies -// exactly one process per cluster regardless of net mode (user-hostfwd vs -// tap-mode bridge). +// findHost walks cluster nodes looking for the QEMU process that owns instanceID +// (the instance ID appears in -pidfile, -qmp, and -name on the QEMU command line). func (w *Witness) findHost(ctx context.Context, instanceID string) (Node, error) { cmd := fmt.Sprintf("ps auxw | grep %s | grep qemu-system | grep -v grep", ShellQuote(instanceID)) - // Give the daemon a short window to actually spawn QEMU after the - // EC2 state flip to running, since /aws/ec2 reports "running" before - // the daemon has finished forking the process on some nodes. + // EC2 "running" state is reported before QEMU is forked; give it a window. deadline := time.Now().Add(30 * time.Second) const interval = 1 * time.Second for { for _, n := range w.cluster.Nodes { out, err := w.ssh.Run(ctx, n, cmd) if err != nil { - // `grep | grep` exits 1 when no match — our SSH wrapper treats - // that as error; try the next node rather than bailing. - continue + continue // grep exits 1 on no match; try next node } if strings.Contains(string(out), instanceID) { return n, nil @@ -477,11 +448,8 @@ func (w *Witness) terminate(ctx context.Context, id string) error { return nil } -// readCounter tunnels SSH through host to publicIP:22 and reads -// /var/lib/counter. The cluster's external pool (MapPublicIpOnLaunch) sits -// on an OVN-managed subnet that isn't routable from the off-cluster runner; -// the host node, however, reaches it via OVN's gateway router. Using the -// VM's hosting node as the jump keeps the relay inside the cluster. +// readCounter tunnels SSH through host to publicIP:22 and reads /var/lib/counter. +// The runner can't reach the EIP directly; the hosting node jumps via OVN's gateway. func readCounter(ctx context.Context, w *Witness, host Node, publicIP string) (int, error) { hostClient, err := dialHost(ctx, host, w.cluster.SSHUser, w.hostSigner) if err != nil { @@ -530,16 +498,9 @@ func readCounter(ctx context.Context, w *Witness, host Node, publicIP string) (i return n, nil } -// awaitBaseline polls readCounter through the host-tunnelled SSH session -// until the guest answers with a parseable counter value or timeout -// expires. Three layers must come up before the baseline can be read: -// sshd binding :22, the OVN dnat_and_snat rule landing for the EIP, and -// cloud-init's runcmd enabling witness-counter.service (which seeds -// /var/lib/counter on first ExecStartPre). All three are async to -// RunInstances, so we retry on the transport errors plus the -// "No such file or directory" the counter service emits before its -// ExecStartPre runs. Auth and parse errors fail fast — they won't fix -// themselves with more polling. +// awaitBaseline polls readCounter until the guest returns a parseable counter +// value. Retries on transient transport errors (sshd not yet up, EIP rule not +// yet landed, counter file not yet created). Auth and parse errors fail fast. func awaitBaseline(ctx context.Context, w *Witness, host Node, publicIP string, timeout time.Duration) (int, error) { deadline := time.Now().Add(timeout) const interval = 2 * time.Second @@ -551,10 +512,6 @@ func awaitBaseline(ctx context.Context, w *Witness, host Node, publicIP string, } lastErr = err msg := strings.ToLower(err.Error()) - // "connect failed" covers x/crypto/ssh's - // `ssh: rejected: connect failed (Connection refused)` - // emitted when the SSH jump host's DialContext fails — the - // canonical signal that the guest hasn't bound :22 yet. transient := strings.Contains(msg, "connect failed") || strings.Contains(msg, "connection refused") || strings.Contains(msg, "connection reset") || diff --git a/tests/e2e/lb/lb_test.go b/tests/e2e/lb/lb_test.go index 6c467270a..23c2ab988 100644 --- a/tests/e2e/lb/lb_test.go +++ b/tests/e2e/lb/lb_test.go @@ -47,21 +47,15 @@ const ( kindNLB lbKind = "NLB" ) -// TestLoadBalancer ports run-lb-e2e.sh. Each of the 4 LB variants (ALB/NLB × -// internet-facing/internal) runs as its own sequential subtest with its own -// LB, listener, target group, and (for internal) client VM. Sequential -// scheduling keeps peak instance count low (2 app + 1 LB + 1 client = 4) so -// the suite passes on capacity-constrained dev nodes — see mulga-siv-77 for -// the underlying placement bug. +// TestLoadBalancer runs 4 LB variants (ALB/NLB × internet-facing/internal) as sequential +// subtests, each with its own LB + listener + TG. Sequential scheduling keeps peak instance +// count low (≤4) so the suite passes on capacity-constrained dev nodes. func TestLoadBalancer(t *testing.T) { env := harness.LoadEnv(t) skipIfDevNetworking(t, env) - // Resolve peer availability BEFORE the shared fixture build (VPC + 2 - // app VMs + probe client = minutes). Single-node mode has no peer, so - // the user sees the "internet-facing will skip" message immediately - // instead of attributing the fixture-setup wait to the first subtest - // name that gotestsum prints once subtests start. + // Resolve peer availability before building the shared fixture so the "skip" message + // appears immediately rather than after minutes of VPC/VM setup. peer := pickPeer(env) var ssh *harness.PeerSSH if peer != "" { @@ -94,10 +88,8 @@ func TestLoadBalancer(t *testing.T) { } runLBSuite(t, client, fixture, kindNLB, "internet-facing", ssh, peer) }) - // Every internal subtest depends on the same mgmt-return datapath and shared - // fixture. If the first (Internal_ALB) never leaves provisioning, the rest - // time out identically — gate them on it so the suite fails fast instead of - // burning ~5 minutes per subtest on a foregone outcome. + // Gate remaining internal subtests on Internal_ALB: if the LB never reaches active, + // the rest will time out identically — fail fast instead of burning ~5min each. albOK := t.Run("Internal_ALB", func(t *testing.T) { runLBSuite(t, client, fixture, kindALB, "internal", nil, "") }) @@ -108,11 +100,8 @@ func TestLoadBalancer(t *testing.T) { } t.Run("Internal_NLB", func(t *testing.T) { skipIfInternalBroken(t) - // DescribeLoadBalancers reports the ALB gone before the sys.micro VM's - // vCPU/memory allocation is actually reclaimed. Without this settle, - // NLB's createLB races the deallocate on capacity-tight dev hosts and - // trips the sys.micro reserve, ending up in terminal state=failed. - // Cheaper than cold-booting a fresh probe client per suite. + // Allow time for the ALB sys.micro VM's resources to be reclaimed before + // NLB races the deallocate on capacity-tight hosts. time.Sleep(15 * time.Second) runLBSuite(t, client, fixture, kindNLB, "internal", nil, "") }) @@ -345,10 +334,8 @@ func deleteSubnet(t *testing.T, c *harness.AWSClient, f *sharedFixture) { } } -// addPublicRoute installs 0.0.0.0/0 → IGW on the VPC's main route table. -// Without this, the daemon classifies the subnet as private and installs an -// OVN LR policy DROP on the subnet's egress (priority 1100), breaking the -// return path for inbound DNATed probes to the public IP. +// addPublicRoute installs 0.0.0.0/0 → IGW on the VPC's main route table so the daemon +// classifies the subnet as public and does not install an OVN LR egress DROP. func addPublicRoute(t *testing.T, c *harness.AWSClient, f *sharedFixture) { t.Helper() out, err := c.EC2.DescribeRouteTables(&ec2.DescribeRouteTablesInput{ @@ -395,10 +382,8 @@ func configureDefaultSG(t *testing.T, c *harness.AWSClient, f *sharedFixture) { f.SecurityGroup = aws.StringValue(out.SecurityGroups[0].GroupId) t.Logf("default SG: %s", f.SecurityGroup) - // Use structured IpPermissions form — spinifex's vpcd ignores the - // top-level IpProtocol/FromPort/ToPort/CidrIp shortcut and silently - // returns newRules=0, so the OVN ACL is never installed and ingress - // drops at the port group. Filed as a separate bead on the daemon side. + // Use structured IpPermissions form — vpcd ignores the top-level shorthand fields + // so the OVN ACL would never be installed otherwise. for _, port := range []int64{httpPort, tcpPort, triggerPort} { _, err := c.EC2.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{ GroupId: aws.String(f.SecurityGroup), @@ -478,11 +463,9 @@ func terminateInstances(t *testing.T, c *harness.AWSClient, ids []string) { // --- LB suite: one LB (ALB or NLB) × one scheme (internet-facing/internal) -// runLBSuite creates one TG + LB + listener, asserts scheme/DNS/ENI -// invariants, runs the traffic test (local + remote for internet-facing, -// client-VM for internal), then tears it all down before returning. Each -// suite is fully self-contained so the 4 subtests run sequentially without -// piling up LB system instances on capacity-constrained dev nodes. +// runLBSuite creates one TG + LB + listener, asserts scheme/DNS/ENI invariants, runs +// traffic tests, then tears down before returning. Self-contained so subtests don't +// accumulate LB system instances on capacity-constrained dev nodes. func runLBSuite(t *testing.T, c *harness.AWSClient, f *sharedFixture, kind lbKind, scheme string, ssh *harness.PeerSSH, peer string) { t.Helper() suffix := "int" @@ -544,12 +527,9 @@ func runLBSuite(t *testing.T, c *harness.AWSClient, f *sharedFixture, kind lbKin runInternalTrafficViaClient(t, c, f, kind, priv) } -// runUDPNLBSuite exercises an internal NLB with a UDP listener end-to-end. This -// is the regression gate for nginx NLB target health: nginx `stream` has no -// active upstream probing, so the lb-agent actively probes the targets (here -// over TCP/9000) and reports health — without it WaitForTargetsHealthy would -// never pass for any NLB. Once healthy, a UDP datagram round-trip through the -// VIP proves the L4 data plane (which HAProxy cannot serve for UDP). +// runUDPNLBSuite exercises an internal NLB with a UDP listener. The lb-agent probes +// targets over TCP/9000 (nginx stream has no active health-check); once healthy, a +// UDP datagram round-trip through the VIP proves the L4 data plane. func runUDPNLBSuite(t *testing.T, c *harness.AWSClient, f *sharedFixture) { t.Helper() const label = "NLB internal UDP" @@ -639,11 +619,8 @@ func createTargetGroup(t *testing.T, c *harness.AWSClient, f *sharedFixture, nam return arn } -// createUDPTargetGroup creates a UDP target group. AWS (and spinifex) require a -// UDP target group's health check to run over TCP/HTTP/HTTPS — UDP itself can't -// be probed — so the check targets the app's TCP echo port (9000) while the -// data plane forwards UDP on udpPort. This exercises the nginx agent's active -// TCP prober, which is what advances NLB targets to healthy. +// createUDPTargetGroup creates a UDP TG with a TCP health check (UDP can't be probed), +// exercising the nginx agent's active TCP prober that advances NLB targets to healthy. func createUDPTargetGroup(t *testing.T, c *harness.AWSClient, f *sharedFixture, name string) string { t.Helper() out, err := c.ELBv2.CreateTargetGroup(&elbv2.CreateTargetGroupInput{ @@ -779,10 +756,8 @@ func deleteLB(t *testing.T, c *harness.AWSClient, lb lbInfo) { } } -// waitForLBGone polls DescribeLoadBalancers until the daemon reports -// LoadBalancerNotFound (or the LB no longer appears in the result), which -// in spinifex corresponds to the underlying system VM having finished -// terminating and released its capacity. +// waitForLBGone polls until DescribeLoadBalancers returns LoadBalancerNotFound, +// indicating the underlying system VM has terminated and released its capacity. func waitForLBGone(t *testing.T, c *harness.AWSClient, arn string, timeout time.Duration) { t.Helper() deadline := time.Now().Add(timeout) @@ -871,15 +846,8 @@ func describeListenerPort(t *testing.T, c *harness.AWSClient, listenerArn string return aws.Int64Value(out.Listeners[0].Port) } -// runModifyListenerSuite verifies that an in-place listener edit reroutes -// traffic without dropping the listener — the regression mulga-944 fixes -// (frontend previously delete+created on every edit). -// -// - Start: listener HTTP:80 → tgA, probes round-robin across 2 app instances. -// - ModifyListener port=8090 → same tgA. Probe at 8090 must succeed -// immediately (port: 80 must not be active any more, port 8090 must serve). -// - ModifyListener defaultActions → tgB (same backends). Probe still -// round-robins; proves DefaultActions swap is honoured by HAProxy. +// runModifyListenerSuite verifies in-place listener edits: port change and DefaultActions swap. +// Regression gate: the listener must stay alive across edits (not delete+recreate). func runModifyListenerSuite(t *testing.T, c *harness.AWSClient, f *sharedFixture) { t.Helper() const label = "ALB internal ModifyListener" @@ -927,19 +895,9 @@ func runModifyListenerSuite(t *testing.T, c *harness.AWSClient, f *sharedFixture probeAtPort(t, f, priv, altPort, label+" after TG-swap (still port 8090, now tgB)") } -// runListenerRulesSuite exercises ALB listener rules end-to-end: -// -// - Two TGs with disjoint backends (app#0 -> tgA, app#1 -> tgB) so the probe -// responder identifies which TG actually served the request. -// - CreateRule path-pattern /alpha* -> tgB. Probe at /alpha/index.html must -// hit app#1 only; probe at / must still hit app#0 (default). -// - ModifyRule path-pattern /alpha* -> /beta*. Probe at /alpha/ now falls to -// the default rule (app#0). -// - DescribeRules lists the rule plus the synthetic default. -// - DeleteRule removes the rule entirely; probes route to default. -// -// Uses host-header on a parallel branch to verify a second condition field -// reaches HAProxy as an ACL. +// runListenerRulesSuite exercises ALB listener rules: CreateRule, ModifyRule, DescribeRules, +// and DeleteRule. Two TGs with disjoint backends so probes confirm which TG served. +// Also tests host-header condition to verify a second ACL field reaches HAProxy. func runListenerRulesSuite(t *testing.T, c *harness.AWSClient, f *sharedFixture) { t.Helper() const label = "ALB internal ListenerRules" @@ -1097,10 +1055,9 @@ func singleResponder(r harness.TrafficResult) string { return "" } -// probeAtPath drives the shared client to issue probesPerRun probes at -// lbIP:port with the given path and optional Host header, asserting that at -// least half succeed. Returns the parsed distribution so callers can assert -// which backend(s) served. +// probeAtPath drives the shared client to issue probesPerRun probes at lbIP:port with the +// given path and optional Host header. Asserts at least half succeed and returns the +// distribution so callers can assert which backend(s) served. func probeAtPath(t *testing.T, f *sharedFixture, lbIP string, port int64, path, host, label string) harness.TrafficResult { t.Helper() r, err := probeOnceAt(f, lbIP, port, path, host, probesPerRun) @@ -1130,10 +1087,9 @@ func waitForPathServing(t *testing.T, f *sharedFixture, lbIP string, port int64, }, timeout, 2*time.Second) } -// waitForPathRoutedAway polls a path until responses stop coming from -// awayFrom. Use after CreateRule/ModifyRule/DeleteRule to wait out the -// lb-agent's reconcile window. Requires at least one successful probe so -// we don't pass on a transient outage. +// waitForPathRoutedAway polls a path until responses stop coming from awayFrom. +// Use after CreateRule/ModifyRule/DeleteRule to wait out lb-agent reconcile. +// Requires at least one successful probe to avoid passing on a transient outage. func waitForPathRoutedAway(t *testing.T, f *sharedFixture, lbIP string, port int64, path, awayFrom, label string, timeout time.Duration) { t.Helper() harness.EventuallyErr(t, func() error { @@ -1204,10 +1160,9 @@ func probeOnceAt(f *sharedFixture, lbIP string, port int64, path, host string, c return harness.VerifyResultsLines(results, "http"), nil } -// waitForListenerServing polls the shared client with a tiny round of probes -// until at least one returns a successful instance_id payload. Used after -// ModifyListener calls so the assertive probeAtPort doesn't race the -// lb-agent's poll-driven HAProxy reload (5s tick + reload latency). +// waitForListenerServing polls the shared client until at least one probe returns +// a successful instance_id payload. Used after ModifyListener to avoid racing +// the lb-agent's poll-driven HAProxy reload (5s tick + reload latency). func waitForListenerServing(t *testing.T, f *sharedFixture, lbIP string, port int64, label string, timeout time.Duration) { t.Helper() harness.EventuallyErr(t, func() error { @@ -1317,11 +1272,9 @@ func lbENI(t *testing.T, c *harness.AWSClient, prefix string, lb lbInfo) *ec2.Ne return eni } -// captureLBConsoleOnFailure registers an on-failure dump of the NLB VM's serial -// console to the subtest artifact dir. The lb-agent's nginx engine activation -// (and any `reload nginx:` error) lands on the guest console, not in the host -// daemon journal — so an NLB "0/N healthy" timeout is otherwise undiagnosable -// from CI artifacts. Call after the LB is active (the VM/ENI exists by then). +// captureLBConsoleOnFailure registers an on-failure dump of the NLB VM's serial console. +// nginx activation errors land on the guest console, not the host journal — so an +// "0/N healthy" timeout is otherwise undiagnosable from CI artifacts. func captureLBConsoleOnFailure(t *testing.T, c *harness.AWSClient, eniDescPrefix string, lb lbInfo) { t.Helper() eni := lbENI(t, c, eniDescPrefix, lb) @@ -1447,10 +1400,9 @@ func runNLBDeregisterDraining(t *testing.T, c *harness.AWSClient, tgArn, targetI // --- Shared probe client + per-suite trigger ---------------------------- -// launchSharedProbeClient launches one client VM whose user-data exposes -// http.server on :80 (results dir) and a JSON trigger endpoint on :9090. -// Both internal suites hit the same client — avoids the ~60s second cold -// boot of a per-suite client. +// launchSharedProbeClient launches one client VM whose user-data exposes http.server on :80 +// and a JSON trigger endpoint on :9090. Both internal suites share the same client +// to avoid the ~60s cold-boot cost of a per-suite client. func launchSharedProbeClient(t *testing.T, c *harness.AWSClient, f *sharedFixture) { t.Helper() out, err := c.EC2.RunInstances(&ec2.RunInstancesInput{ diff --git a/tests/e2e/multinode/baseline_test.go b/tests/e2e/multinode/baseline_test.go index 3acaecefc..2495638fb 100644 --- a/tests/e2e/multinode/baseline_test.go +++ b/tests/e2e/multinode/baseline_test.go @@ -19,31 +19,17 @@ import ( "github.com/stretchr/testify/require" ) -// Fresh-install reachability baselines for the multi-node cluster. These run -// right after preflight and before any test triggers needInstanceTrio (which -// authorizes SSH on the default SG), so the default SG / subnet / route table -// are observed in their pristine, out-of-box state. +// Fresh-install reachability baselines. Run before needInstanceTrio so the +// default SG/subnet/route-table are in their pristine state. // -// Two facts they pin: -// -// - runMultinodeDefaultSGReachabilityBaseline: an instance in the default -// subnet behind a dedicated default-deny SG is NOT reachable from the -// runner until tcp/22 is authorized — the SG, not routing, is the gate. -// -// - runMultinodeSameSGCrossHostComms: two instances on DIFFERENT nodes that -// share the default SG can reach each other over the VPC datapath with no -// ingress rule added — the default SG's self-reference rule is enforced as -// an OVN address set spanning chassis. The probe is ICMP, which only the -// default SG's self-ingress permits, so the signal can't be confounded by -// the tcp/22 rule other tests add to the default SG later. -// -// Neither mutates a default resource: dedicated SGs are authorized, the -// default SG is only read / used for membership. +// - runMultinodeDefaultSGReachabilityBaseline: proves the SG (not routing) +// gates external tcp/22 — blocked before authorize, reachable after. +// - runMultinodeSameSGCrossHostComms: proves the default SG self-reference +// rule spans chassis — ICMP between two instances on different nodes +// succeeds with no added ingress rule. -// baselineLaunch launches one instance with the given SGs into subnetID and -// registers a terminate-and-wait cleanup. Returns the instance ID once -// "running". Self-cleaning so the cluster VM inventory is unperturbed for -// sibling tests. +// baselineLaunch launches one instance with the given SGs into subnetID, +// registers a terminate-and-wait cleanup, and returns the instance ID once "running". func baselineLaunch(t *testing.T, fix *Fixture, amiID, instType, keyName, subnetID string, sgIDs []string) string { t.Helper() sgs := make([]*string, 0, len(sgIDs)) @@ -98,8 +84,7 @@ func instancePrivateIP(t *testing.T, fix *Fixture, id string) string { return ip } -// sshCapture runs cmd over SSH and returns combined output + error without -// fataling, so callers can assert on the exit status (ping success/failure). +// sshCapture runs cmd over SSH and returns combined output + error without fataling. func sshCapture(pem, user, host string, port int, cmd string) (string, error) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -160,11 +145,9 @@ func runMultinodeDefaultSGReachabilityBaseline(t *testing.T, fix *Fixture) { assert.Containsf(t, out, "ec2-user", "ssh id after authorize\n%s", out) } -// runMultinodeSameSGCrossHostComms launches two instances on different nodes, -// both in the default SG plus a dedicated runner-SSH SG, and asserts that one -// can ICMP-ping the other over the VPC. ICMP is permitted only by the default -// SG's self-reference rule, so success proves that rule is enforced across -// chassis with no default-SG mutation. +// runMultinodeSameSGCrossHostComms launches two instances on different nodes sharing +// the default SG and asserts ICMP-ping succeeds across hosts. ICMP is permitted only +// by the default SG self-reference rule, proving it is enforced across chassis. func runMultinodeSameSGCrossHostComms(t *testing.T, fix *Fixture) { harness.Phase(t, "Multinode — Baseline: same default-SG instances communicate across hosts") @@ -173,14 +156,13 @@ func runMultinodeSameSGCrossHostComms(t *testing.T, fix *Fixture) { amiID := needAMI(t, fix, arch) keyName, keyPath := needKeyPair(t, fix) - // Dedicated SG so the runner can SSH into a probe instance without touching - // the default SG. Opens tcp/22 only — ICMP between guests still depends on - // the default SG's self-ingress, keeping the cross-host signal clean. + // Dedicated SG for runner SSH — opens tcp/22 only; ICMP between guests + // still depends on the default SG self-ingress, keeping the signal clean. runnerSG := harness.EnsureSG(t, fix.Harness, vpcID, "baseline-runnersg") harness.AuthorizeSSHIngress(t, fix.AWS, runnerSG) - // Launch instances until two land on distinct nodes (best-effort scheduler - // spread). Bounded so a degenerate single-node placement can't loop. + // Launch instances until two land on distinct nodes. Bounded so a degenerate + // single-node placement can't loop forever. type placed struct { id string node string diff --git a/tests/e2e/multinode/cluster_health_test.go b/tests/e2e/multinode/cluster_health_test.go index d38bc2ee9..50d6465d4 100644 --- a/tests/e2e/multinode/cluster_health_test.go +++ b/tests/e2e/multinode/cluster_health_test.go @@ -9,14 +9,8 @@ import ( "github.com/mulgadc/spinifex/tests/e2e/harness" ) -// runClusterHealth is the Go port of cluster health checks -// (run-multinode-e2e.sh:426-510). Verifies the four mesh services -// (NATS, Predastore, gateway, daemon) are healthy on every node and -// the `spx get` CLI agrees the cluster is formed. -// -// Bash version uses `pass_test` / `fail_test` to accumulate per-check -// pass counts; the Go port lets each failing sub-check fail the whole -// test (t.Fatalf) — a single dead node should fail fast. +// runClusterHealth verifies the four mesh services (NATS, Predastore, gateway, +// daemon) are healthy on every node and that `spx get` agrees the cluster is formed. func runClusterHealth(t *testing.T, fix *Fixture) { harness.Phase(t, "Multinode — Cluster Health") @@ -32,13 +26,8 @@ func runClusterHealth(t *testing.T, fix *Fixture) { harness.Step(t, "Daemon ready per gateway (DescribeInstanceTypes)") fix.Cluster.WaitDaemonReady(t, fix.Env) - // Bash phase 2 (run-multinode-e2e.sh:494) wraps `spx get nodes` with - // `2>/dev/null` and routes a <3-Ready result through `fail_test` which - // merely increments a counter — the bash run keeps going. The Go port - // must mirror that lenience: the spx CLI ↔ NATS dial races the cluster - // join on cold-bootstrapped CI runners (mulga-siv-90 run 26163994815) - // without affecting the data path (gateway+daemon checks above already - // confirmed end-to-end NATS reachability). Downgrade to WARN. + // spx get nodes is best-effort: the CLI ↔ NATS dial can race cluster join + // on cold-bootstrapped runners without affecting the data path. Downgrade to WARN. harness.Step(t, "spx get nodes shows 3 Ready (best-effort)") nodesOut := harness.SpxRunBestEffort(t, "get", "nodes", "--timeout", "5s") harness.Detail(t, "spx_get_nodes", strings.TrimSpace(nodesOut)) @@ -47,9 +36,8 @@ func runClusterHealth(t *testing.T, fix *Fixture) { ready, len(fix.Cluster.Nodes), nodesOut) } - // Best-effort: bash phase 2 runs `spx get vms --timeout 5s 2>/dev/null` - // and never checks the exit code — the CLI ↔ NATS dial can transiently - // fail right after cluster join without affecting the data path. + // Best-effort: the CLI ↔ NATS dial can transiently fail right after cluster + // join without affecting the data path. harness.Step(t, "spx get vms returns (any result, even empty)") vmsOut := harness.SpxRunBestEffort(t, "get", "vms") harness.Detail(t, "spx_get_vms_bytes", len(vmsOut)) diff --git a/tests/e2e/multinode/crossnode_ops_test.go b/tests/e2e/multinode/crossnode_ops_test.go index fb96b6b7c..acfeb6e65 100644 --- a/tests/e2e/multinode/crossnode_ops_test.go +++ b/tests/e2e/multinode/crossnode_ops_test.go @@ -12,15 +12,9 @@ import ( "github.com/stretchr/testify/require" ) -// runCrossNodeOps is the Go port of cross-node stop/start -// (run-multinode-e2e.sh:854-895). Picks the first trio instance, finds the -// node currently hosting it, then drives stop+start through gateways on -// OTHER nodes — proving the cross-node control path actually reaches the -// hosting daemon, not just the local one. -// -// State guarantee: instance ends running. Other Test*s may share the trio -// (guest SSH, etc.) so leaving it stopped would cascade-fail downstream -// tests. Bash has the same guarantee implicitly via the start poll at end. +// runCrossNodeOps stops and starts the first trio instance through gateways on +// OTHER nodes than the one hosting it, proving cross-node control path routing. +// Instance is guaranteed to end running so sibling tests sharing the trio are unaffected. func runCrossNodeOps(t *testing.T, fix *Fixture) { harness.Phase(t, "Multinode — Cross-Node Stop/Start") diff --git a/tests/e2e/multinode/distribution_test.go b/tests/e2e/multinode/distribution_test.go index 8b55498a1..5b8210c0c 100644 --- a/tests/e2e/multinode/distribution_test.go +++ b/tests/e2e/multinode/distribution_test.go @@ -9,13 +9,9 @@ import ( "github.com/mulgadc/spinifex/tests/e2e/harness" ) -// runInstanceDistribution is the Go port of instance lifecycle + distribution -// (run-multinode-e2e.sh:513-623). Launches a trio of instances and checks at -// least two cluster nodes host them. Distribution is non-deterministic -// (scheduler may stack on one node under load) so the bash version -// warns-not-fails when all three land on the same host. The Go port mirrors: -// minNodes=1 fatal, log "spread=N" so flaky CI surfaces in metrics without -// blocking the run. +// runInstanceDistribution launches the trio and checks at least two cluster nodes host them. +// Distribution is non-deterministic so all-on-one-node is a WARN not a failure, mirroring +// the bash driver behaviour. func runInstanceDistribution(t *testing.T, fix *Fixture) { harness.Phase(t, "Multinode — Instance Distribution") @@ -29,11 +25,8 @@ func runInstanceDistribution(t *testing.T, fix *Fixture) { t.Logf("WARN: all %d instances on one node (%v) — non-fatal, scheduler quirk", len(ids), counts) } - // `spx get vms` is best-effort. Bash phase 3 invokes it with `2>/dev/null` - // and downgrades a missing instance to WARN — the underlying CLI ↔ NATS - // connection can race the cluster join (mulga-siv-90 CI run 26161792146) - // without affecting correctness of the data path. Mirror that: accept - // non-zero exit, log when an ID is missing rather than fail. + // spx get vms is best-effort: CLI ↔ NATS can race cluster join without + // affecting the data path. Accept non-zero exit; log missing IDs as WARN. harness.Step(t, "spx get vms includes every launched instance (best-effort)") vms := harness.SpxRunBestEffort(t, "get", "vms") for _, id := range ids { diff --git a/tests/e2e/multinode/gateway_test.go b/tests/e2e/multinode/gateway_test.go index ec6c96eb2..e88de385f 100644 --- a/tests/e2e/multinode/gateway_test.go +++ b/tests/e2e/multinode/gateway_test.go @@ -10,14 +10,9 @@ import ( "github.com/stretchr/testify/require" ) -// runCrossNodeGateway is the Go port of cross-node gateway access -// (run-multinode-e2e.sh:828-851). Drives DescribeInstances through every -// node's gateway and asserts each returns the same instance count as node1 -// (the test runner's local gateway). -// -// Catches the regression where the daemon answers only locally-hosted -// instances instead of fanning out via NATS — i.e. a stale awsgw routing -// table or a queue-group misconfiguration. +// runCrossNodeGateway drives DescribeInstances through every node's gateway and asserts +// each returns the same instance count as node1. Catches the regression where the daemon +// answers only locally-hosted instances instead of fanning out via NATS. func runCrossNodeGateway(t *testing.T, fix *Fixture) { harness.Phase(t, "Multinode — Cross-Node Gateway Access") diff --git a/tests/e2e/multinode/helpers_test.go b/tests/e2e/multinode/helpers_test.go index 055f59172..7e57d6ab4 100644 --- a/tests/e2e/multinode/helpers_test.go +++ b/tests/e2e/multinode/helpers_test.go @@ -15,9 +15,7 @@ import ( "github.com/stretchr/testify/require" ) -// needAZ is a package-local shorthand for the discovered default AZ. -// Memoized on the harness fixture so every Test* gets the same answer -// regardless of execution order. +// needAZ returns the discovered default AZ, memoized on the harness fixture. func needAZ(t *testing.T, fix *Fixture) string { t.Helper() return harness.DiscoverDefaultAZ(t, fix.Harness) @@ -37,26 +35,22 @@ func needAMI(t *testing.T, fix *Fixture, arch string) string { return harness.DiscoverUbuntuAMI(t, fix.Harness, arch) } -// needKeyPair ensures a test-scoped EC2 key pair and returns its name plus -// the on-disk PEM path. Memoized on the harness fixture, so every Test* -// across the package shares the same key (PEM file written once). +// needKeyPair ensures a test-scoped EC2 key pair and returns its name and PEM path. +// Memoized on the harness fixture so all tests share one key. func needKeyPair(t *testing.T, fix *Fixture) (name, pemPath string) { t.Helper() return harness.EnsureKeyPair(t, fix.Harness, fix.Artifacts) } -// Package-scoped trio. runInstanceDistribution launches; runGuestSSH / -// runCrossNodeGateway / runCrossNodeOps / runVolumeLifecycle reuse the same -// IDs. Mirrors the iam_helpers_test sync.Once pattern from tests/e2e/single. +// Package-scoped trio. runInstanceDistribution launches; downstream tests reuse the same IDs. var ( trioOnce sync.Once trioIDs []string trioErr error ) -// needInstanceTrio returns the package singleton trio of nano instances on -// the default VPC, launching them once on first call and registering -// terminate-on-process-exit. Mirrors bash phase 3 which launches 3 stagger. +// needInstanceTrio returns the package singleton trio of nano instances on the default +// VPC, launching them once and registering process-exit termination cleanup. func needInstanceTrio(t *testing.T, fix *Fixture) []string { t.Helper() trioOnce.Do(func() { @@ -67,13 +61,9 @@ func needInstanceTrio(t *testing.T, fix *Fixture) []string { require.NotEmpty(t, def.SGID, "default SG required") harness.AuthorizeSSHIngress(t, fix.AWS, def.SGID) - // Bash phase 3 launches with a 2s stagger between calls. Go port - // also retries each launch on InsufficientInstanceCapacity — on a - // cold-bootstrapped CI cluster (mulga-siv-90 run 26163994815) the - // per-node resourceMgr can briefly report 0 capacity right after - // daemon start before its first inventory scan completes, even - // though DescribeInstanceTypes already answers. AWS itself surfaces - // the same error transiently; treat as retryable. + // Each launch is retried on InsufficientInstanceCapacity: the resourceMgr + // can briefly report 0 capacity right after daemon start before its first + // inventory scan completes. Treat as transiently retryable. input := &ec2.RunInstancesInput{ ImageId: aws.String(amiID), InstanceType: aws.String(instType), @@ -132,10 +122,7 @@ func needInstanceTrio(t *testing.T, fix *Fixture) []string { return trioIDs } -// readyNodeCount counts the number of "Ready" lines in `spx get nodes` -// output. Bash phase 2 used `grep -c "Ready"`; we match the same -// (substring, case-sensitive) so cluster-status string drift surfaces in -// both tracks identically. +// readyNodeCount counts "Ready" lines in spx get nodes output (substring, case-sensitive). func readyNodeCount(out string) int { n := 0 for _, line := range strings.Split(out, "\n") { diff --git a/tests/e2e/multinode/ipsec_test.go b/tests/e2e/multinode/ipsec_test.go index 8300de0c9..8e7d3418c 100644 --- a/tests/e2e/multinode/ipsec_test.go +++ b/tests/e2e/multinode/ipsec_test.go @@ -12,27 +12,9 @@ import ( "github.com/mulgadc/spinifex/tests/e2e/harness" ) -// runIPSec verifies OVN native IPsec is live on every node. Three checks: -// -// 1. OVS DB on each node carries the cert/key/CA pointers plus -// other_config:ipsec_encapsulation=true. This is what -// daemon.enableOVNIPSec writes; if any node lost it (rolling restart, -// manual ovs-vsctl unset, missing peer cert) the SAs below would -// silently drop to plaintext. -// -// 2. `ip xfrm state` on each node shows at least one AEAD SA negotiated -// in AES-GCM mode. ovs-monitor-ipsec drives strongSwan to bring SAs up -// against every peer ovn-controller has a Geneve tunnel to, so on a -// 3-node mesh we expect 2N SAs per node (in+out per peer) — but the -// test only asserts >=1 to stay tolerant of ovs-monitor-ipsec startup -// ordering quirks. -// -// 3. Best-effort tcpdump for ESP (proto 50) traffic on the underlay. -// Logged only — geneve_sys_6081 may be idle during the capture window -// if no overlay VMs are running yet. -// -// Skips cleanly if the cluster was bootstrapped with --ipsec=false (no -// other_config:ipsec_encapsulation key on the first node). +// runIPSec verifies OVN native IPsec is live on every node: OVS DB carries cert/key/CA +// pointers, xfrm shows AES-GCM SAs, and ESP traffic is observed (best-effort tcpdump). +// Skips if the cluster was bootstrapped with --ipsec=false. func runIPSec(t *testing.T, fix *Fixture) { harness.Phase(t, "Multinode — OVN Native IPsec") @@ -83,9 +65,7 @@ func runIPSec(t *testing.T, fix *Fixture) { if !strings.Contains(s, "aead") { return fmt.Errorf("%s xfrm has no AEAD SAs:\n%s", n.Name, strings.TrimSpace(s)) } - // Kernel renders the GCM AEAD as either `rfc4106(gcm(aes))` - // (RFC 4106 ESP AES-GCM, what strongSwan negotiates by default) - // or the bare `gcm(aes)` template name. Accept either. + // Kernel renders GCM AEAD as rfc4106(gcm(aes)) or gcm(aes); accept either. if !strings.Contains(s, "gcm(aes)") { return fmt.Errorf("%s xfrm SAs not AES-GCM:\n%s", n.Name, strings.TrimSpace(s)) } diff --git a/tests/e2e/multinode/main_test.go b/tests/e2e/multinode/main_test.go index afab15a32..28c4c0d66 100644 --- a/tests/e2e/multinode/main_test.go +++ b/tests/e2e/multinode/main_test.go @@ -16,9 +16,8 @@ import ( "github.com/mulgadc/spinifex/tests/e2e/harness" ) -// Package-scoped singleton fixture. Mirrors single/main_test.go. Built -// lazily by the first call to requireMultiNodeFixture so a test run with -// SPINIFEX_E2E unset stays cheap (no AWS dial, no temp dir). +// Package-scoped singleton fixture, built lazily by requireMultiNodeFixture. +// A run with SPINIFEX_E2E unset stays cheap (no AWS dial, no temp dir). var ( pkgFixOnce sync.Once pkgFix *Fixture @@ -26,8 +25,6 @@ var ( ) // TestMain owns process-level lifecycle for the singleton fixture. -// harness.Fixture.Close drains every cleanup at process exit, regardless -// of which Test* created it. func TestMain(m *testing.M) { code := m.Run() if pkgFix != nil { @@ -41,10 +38,8 @@ func TestMain(m *testing.M) { os.Exit(code) } -// requireMultiNodeFixture returns the package-scoped Fixture singleton, -// building it on first call. Skips the calling test if SPINIFEX_E2E is -// unset or the cluster mode is not multinode. Fails the test if init -// itself errors (AWS client construction, cluster build). +// requireMultiNodeFixture returns the package-scoped Fixture singleton, building +// it on first call. Skips if SPINIFEX_E2E is unset or mode is not multinode. func requireMultiNodeFixture(t *testing.T) *Fixture { t.Helper() pkgFixOnce.Do(func() { @@ -89,10 +84,8 @@ func requireMultiNodeFixture(t *testing.T) *Fixture { return pkgFix } -// Fixture carries the per-process state shared across every Test* in this -// package. Only environment-level slots — every per-phase resource ID is -// memoized on Harness (harness.Fixture) and surfaced via the package-local -// need* helpers / harness.Ensure* + harness.Discover*. +// Fixture carries the per-process state shared across every Test* in this package. +// Per-phase resource IDs are memoized on Harness via need* / harness.Ensure* helpers. type Fixture struct { Env *harness.Env AWS *harness.AWSClient diff --git a/tests/e2e/multinode/node_failure_test.go b/tests/e2e/multinode/node_failure_test.go index fbe378126..d9fccd99a 100644 --- a/tests/e2e/multinode/node_failure_test.go +++ b/tests/e2e/multinode/node_failure_test.go @@ -11,14 +11,9 @@ import ( "github.com/stretchr/testify/require" ) -// runNodeFailure is the Go port of node-failure injection -// (run-multinode-e2e.sh:898-957). Stops spinifex.target on node2, asserts the -// cluster degrades cleanly: surviving nodes still serve DescribeInstances, -// NATS reports 1 peer instead of 2, and the trio remains addressable. -// -// Registers a t.Cleanup that restarts node2 unconditionally so a cancelled -// recovery test doesn't leave the cluster broken for downstream tests. -// runNodeRecovery itself runs the recovery assertions explicitly. +// runNodeFailure stops spinifex.target on node2 and asserts clean degradation: +// surviving nodes still serve DescribeInstances and NATS reports 1 peer. +// Unconditionally restarts node2 in t.Cleanup so downstream tests are unaffected. func runNodeFailure(t *testing.T, fix *Fixture) { harness.Phase(t, "Multinode — Node Failure") @@ -33,15 +28,10 @@ func runNodeFailure(t *testing.T, fix *Fixture) { harness.Step(t, "stop spinifex.target on %s (%s)", node2.Name, node2.Addr) harness.StopNode(t, node2) t.Cleanup(func() { - // Best-effort: ensure node2 comes back even if assertions below fail - // or the run is cancelled. runNodeRecovery calls StartNode again — - // systemctl start is idempotent. - harness.StartNode(t, node2) + harness.StartNode(t, node2) // idempotent; runNodeRecovery may also call this }) - // NATS detection of a peer drop is delayed by the heartbeat — bash - // uses `sleep 10`. WaitNATSPeers polls so we converge faster on a - // healthy run while still tolerating a slow detection. + // NATS peer-drop detection is delayed by heartbeat; poll so we converge faster than a fixed sleep. harness.Step(t, "wait NATS to report 1 peer (degraded)") fix.Cluster.WaitNATSPeers(t, 1, harness.WithTimeout(30*time.Second), harness.WithPoll(2*time.Second)) diff --git a/tests/e2e/multinode/node_recovery_test.go b/tests/e2e/multinode/node_recovery_test.go index 63f7f5df4..e1d9d43dd 100644 --- a/tests/e2e/multinode/node_recovery_test.go +++ b/tests/e2e/multinode/node_recovery_test.go @@ -10,14 +10,9 @@ import ( "github.com/stretchr/testify/require" ) -// runNodeRecovery is the Go port of node-recovery validation -// (run-multinode-e2e.sh:961-1037). Starts spinifex.target on node2, waits for -// NATS to reform to 2 peers, gateway to answer, spx get nodes to show 3 -// Ready, and node2's gateway to answer DescribeInstanceTypes. -// -// Idempotent: if runNodeFailure didn't actually take node2 down (e.g. -// `go test -run TestMultinodeNodeRecovery` in isolation), the StartNode call -// no-ops and the assertions still hold. +// runNodeRecovery starts spinifex.target on node2 and asserts full cluster reformation: +// NATS 2 peers, gateway answering, and DescribeInstanceTypes succeeding via node2. +// Idempotent if node2 was never stopped. func runNodeRecovery(t *testing.T, fix *Fixture) { harness.Phase(t, "Multinode — Node Recovery") @@ -33,10 +28,7 @@ func runNodeRecovery(t *testing.T, fix *Fixture) { harness.Step(t, "wait %s gateway to answer HTTPS", node2.Name) harness.WaitNodeServiceReady(t, node2, harness.WithTimeout(60*time.Second), harness.WithPoll(2*time.Second)) - // Bash (run-multinode-e2e.sh:1018) appends `|| echo ""` to swallow a - // non-zero spx exit and never checks Ready count strictly — recovery is - // gated on the gateway DescribeInstanceTypes call below, which actually - // proves end-to-end NATS routing through node2. Downgrade spx to WARN. + // spx get nodes is best-effort; real recovery gate is DescribeInstanceTypes via node2. harness.Step(t, "spx get nodes shows %d Ready after recovery (best-effort)", len(fix.Cluster.Nodes)) out := harness.SpxGetNodesAcrossCluster(t) harness.Detail(t, "spx_get_nodes", out) diff --git a/tests/e2e/multinode/placement_nat_test.go b/tests/e2e/multinode/placement_nat_test.go index 43eb83a1e..262e2c9af 100644 --- a/tests/e2e/multinode/placement_nat_test.go +++ b/tests/e2e/multinode/placement_nat_test.go @@ -16,29 +16,12 @@ import ( "github.com/stretchr/testify/require" ) -// Spread placement + NAT GW (the largest section of run-multinode-e2e.sh, -// originally phase 11) runs as one top-level Test with 6 t.Run sub-tests -// sharing the VPC + bastion + private trio + NAT GW graph. Setup happens -// once in the outer scope; sub-tests carry the assertions. The 8 bash -// sub-steps map as: -// -// bash step 1 (VPC + subnets + IGW + SGs + route) -> sub-test "VPCSetup" -// bash step 2 (bastion launch + SSH + scp key) -> sub-test "BastionSSH" -// bash steps 3+4 (spread PG + 3 priv VMs + assert) -> sub-test "SpreadPlacement" -// bash step 5 (ping pre-NAT, expect fail) -> sub-test "PreNATIsolation" -// bash steps 6+7 (NAT GW + ping post-NAT) -> sub-test "NATGatewayInternet" -// bash step 8 (explicit teardown ordering) -> sub-test "NATCleanupOrdering" -// -// Earlier mulga-siv-127 iterations split this into 6 separate top-level Tests -// each rebuilding the full graph — that ballooned phase-11 runtime to ~60min -// (6 × ~10min setups). The shared-setup layout keeps sub-test granularity in -// JUnit while restoring the ~15min monolithic budget. Outer Test stays -// sequential (no t.Parallel) — owns VPC CIDR 10.100.0.0/16 + EIP pool. +// Spread placement + NAT GW runs as one top-level Test with 6 sub-tests sharing +// one VPC + bastion + private trio + NAT GW graph. Shared setup avoids rebuilding +// the full graph per sub-test. Outer Test is sequential (owns CIDR 10.100.0.0/16 + EIP pool). -// runSpread is the Go port of the spread-placement + NAT GW section from -// run-multinode-e2e.sh:1255-1594. See file-level doc for the sub-test map. -// Wrapper TestMultinodeSpread lives in tests_test.go so source-order -// controls execution position relative to the rest of the suite. +// runSpread drives the spread-placement + NAT GW suite. +// Wrapper TestMultinodeSpread in tests_test.go controls execution order. func runSpread(t *testing.T, fix *Fixture) { harness.Phase(t, "Multinode — Spread Placement + NAT Gateway") c := fix.AWS @@ -46,9 +29,7 @@ func runSpread(t *testing.T, fix *Fixture) { v := spreadSetupVPC(t, fix) t.Run("VPCSetup", func(t *testing.T) { - // Setup helper required the CreateRoute call so the test would - // already fatal on error; the DescribeRouteTables read here is the - // explicit oracle matching this sub-test's name. + // DescribeRouteTables is the explicit oracle for this sub-test; setup already fataled on create errors. out, err := c.EC2.DescribeRouteTables(&ec2.DescribeRouteTablesInput{ RouteTableIds: []*string{aws.String(v.MainRTBID)}, }) @@ -68,9 +49,7 @@ func runSpread(t *testing.T, fix *Fixture) { b := spreadSetupBastion(t, fix, v) t.Run("BastionSSH", func(t *testing.T) { - // spreadSetupBastion already waited for SSH + scp'd the key. The - // sub-test is the JUnit marker recording that those steps passed - // for this run. + // JUnit marker: SSH + scp completed in spreadSetupBastion. t.Logf("bastion %s SSH+scp ok at %s", b.ID, b.PublicIP) }) @@ -168,9 +147,8 @@ type spreadVPC struct { MainRTBID string } -// spreadSetupVPC creates the phase-11 VPC graph (VPC + subnets + IGW + main -// RT route + SGs) and registers LIFO cleanup against t. Fatals on any AWS -// error. Mirrors run-multinode-e2e.sh phase 11 step 1. +// spreadSetupVPC creates the VPC graph (VPC + subnets + IGW + main RT route + SGs) +// and registers LIFO cleanup against t. func spreadSetupVPC(t *testing.T, fix *Fixture) spreadVPC { t.Helper() c := fix.AWS @@ -332,10 +310,8 @@ type spreadBastion struct { KeyPath string } -// spreadSetupBastion launches the bastion in the public subnet, waits for SSH, -// and copies the test PEM to /tmp/key.pem so the bastion can re-key into -// private VMs. Registers terminate + diagnostic-on-failure cleanup. Mirrors -// run-multinode-e2e.sh phase 11 step 2. +// spreadSetupBastion launches the bastion in the public subnet, waits for SSH, and copies +// the test PEM to /tmp/key.pem so the bastion can re-key into private VMs. func spreadSetupBastion(t *testing.T, fix *Fixture, v spreadVPC) spreadBastion { t.Helper() c := fix.AWS @@ -372,10 +348,7 @@ func spreadSetupBastion(t *testing.T, fix *Fixture, v spreadVPC) spreadBastion { "bastion %s has no PublicIpAddress (pool mode required for phase 11)", bastionID) harness.Detail(t, "bastion", bastionID, "bastion_pub_ip", bastionPubIP) - // veth bridge mode (run 26198221557 journal) routes EIP ingress through - // the priority-20 gateway-chassis; with a 3-node cluster the gw chassis - // is the same node as the bastion only ~33% of the time, so ARP + - // geneve traversal need observing to localise SYN drops on failure. + // Dump datapath diagnostics on failure to help localise ARP/geneve SYN drops. t.Cleanup(func() { if t.Failed() { dumpPlacementNATDiag(t, fix, bastionPubIP, bastionID) @@ -405,8 +378,7 @@ type spreadPrivate struct { } // spreadSetupPrivateTrio creates the spread placement group + 3 private VMs, -// waits for SSH-via-bastion, and resolves each instance's hosting node and -// private IP. Mirrors run-multinode-e2e.sh phase 11 step 3. +// waits for SSH-via-bastion, and resolves each instance's hosting node and private IP. func spreadSetupPrivateTrio(t *testing.T, fix *Fixture, b spreadBastion) spreadPrivate { t.Helper() c := fix.AWS @@ -502,10 +474,8 @@ func spreadSetupPrivateTrio(t *testing.T, fix *Fixture, b spreadBastion) spreadP } } -// spreadNAT carries the NAT GW + private RT IDs. The *bool fields are the -// "already torn down" latches used by TestMultinodePhase11Cleanup to flip -// the deferred cleanup closures into no-ops as each explicit teardown step -// succeeds. +// spreadNAT carries the NAT GW + private RT IDs. The *bool fields are already-torn-down +// latches so deferred cleanup skips explicit teardown steps that already ran. type spreadNAT struct { Private spreadPrivate NatGWID string @@ -520,10 +490,8 @@ type spreadNAT struct { RouteDeleted *bool } -// spreadSetupNATGW allocates an EIP, creates the NAT GW in the public subnet, -// builds the private RT + association + 0.0.0.0/0 -> NAT GW route, and waits -// for the NAT GW to reach `available`. Mirrors run-multinode-e2e.sh phase 11 -// step 6. +// spreadSetupNATGW allocates an EIP, creates the NAT GW in the public subnet, builds the +// private RT + route, and waits for the NAT GW to reach `available`. func spreadSetupNATGW(t *testing.T, fix *Fixture, p spreadPrivate) spreadNAT { t.Helper() c := fix.AWS @@ -628,9 +596,7 @@ func spreadSetupNATGW(t *testing.T, fix *Fixture, p spreadPrivate) spreadNAT { // --- helpers -------------------------------------------------------------- -// uniqueCount returns the number of distinct non-empty / non-"unknown" -// strings in xs. Matches the bash `sort -u | wc -l` line that gates the -// spread-placement pass. +// uniqueCount returns the number of distinct non-empty, non-"unknown" strings in xs. func uniqueCount(xs []string) int { seen := map[string]bool{} for _, x := range xs { @@ -642,18 +608,9 @@ func uniqueCount(xs []string) int { return len(seen) } -// waitForBastionSSH polls SSH `true` until the handshake completes. Bash -// equivalent: phase 11 Step 2's 30-attempt 5s loop = 150s budget. Go port -// uses a wider 5min budget because a brand-new VPC's OVN flow install on -// the external switch lags the instance's "running" state on the tofu -// cluster (mulga-siv-90 runs 26195522455, 26197248656, 26198221557 — SSH -// timed out at exactly 150s even after the dnat_and_snat race fix -// mulga-siv-124 landed). 5min covers both cloud-init + OVN warm-up. -// -// Logs a single ICMP probe per iteration before the SSH dial so a future -// failure log distinguishes "datapath down" (ping fails) from "sshd not -// up yet" (ping ok, ssh refused/timeout) — the smoking-gun split per -// the RCA agent's fix candidate 3. +// waitForBastionSSH polls SSH until the handshake completes. Uses a 5min budget to cover +// both cloud-init and OVN flow install on the external switch lagging instance "running". +// Logs an ICMP probe each iteration to distinguish "datapath down" from "sshd not up yet". func waitForBastionSSH(t *testing.T, host, keyPath string, timeout time.Duration) { t.Helper() harness.Step(t, "wait bastion SSH %s", host) @@ -706,7 +663,7 @@ func scpKeyToBastion(t *testing.T, keyPath, host string) { if out, err := exec.CommandContext(ctx, "scp", scpArgs...).CombinedOutput(); err != nil { t.Fatalf("scp %s -> %s:/tmp/key.pem: %v\n%s", keyPath, host, err, string(out)) } - // chmod 600 — sshd refuses keys with loose perms. + // chmod 600: sshd refuses keys with loose permissions. ctx2, cancel2 := context.WithTimeout(context.Background(), 30*time.Second) defer cancel2() sshArgs := []string{ @@ -750,9 +707,7 @@ func sshViaBastion(keyPath, bastionHost, privIP, cmd string) (string, error) { return string(out), err } -// waitForSSHViaBastion polls hostname-through-bastion until cloud-init on -// the private VM has finished bringing sshd up. label is a triage tag for -// the log line; bash uses a similar attempt loop with 30 tries x 5s. +// waitForSSHViaBastion polls hostname-through-bastion until sshd is up on the private VM. func waitForSSHViaBastion(t *testing.T, keyPath, bastionHost, privIP, label string) { t.Helper() harness.EventuallyErr(t, func() error { @@ -773,8 +728,7 @@ func pingViaBastion(keyPath, bastionHost, privIP string) (string, error) { return sshViaBastion(keyPath, bastionHost, privIP, "ping -c 1 -W 3 8.8.8.8") } -// --- NAT GW state pollers (multinode-local; phase 8d single-node has its -// own copies — promote to harness/poll.go once a third consumer lands). +// --- NAT GW state pollers (promote to harness/poll.go when a third consumer appears) func waitForNATGatewayStateFatal(t *testing.T, c *harness.AWSClient, id, target string) { t.Helper() @@ -824,21 +778,9 @@ func waitForNATGatewayStateBest(c *harness.AWSClient, id, target string, timeout } } -// dumpPlacementNATDiag fans out a fixed shortlist of datapath probes to -// every cluster node and writes results to /diag--.log. -// Triggered only on test failure (via t.Cleanup wrapper) so a green run -// leaves no extra files. Best-effort: per-node SSH errors are logged but -// never re-fail the test. -// -// Probe set targets the centralized-NAT EIP datapath: -// - arp -an : did ARP for the EIP resolve on this node? -// - ip route get : kernel routing decision for outbound to EIP -// - ovs-vsctl show : bridges + ports (br-int, br-ex) -// - ovs-vsctl get ... ovn-bridge-mappings : physnet binding -// - ovn-sbctl list chassis : encap IPs + gateway hostnames -// - ovn-nbctl list nat : dnat_and_snat rule state -// - ovn-nbctl list gateway_chassis : priority election state -// - ovn-nbctl find logical_router_port : DGP gateway chassis attachment +// dumpPlacementNATDiag fans out datapath probes (ARP, xfrm, OVS/OVN state) to every +// cluster node on failure and writes results under Artifacts. Best-effort: SSH errors +// are logged but never re-fail the test. func dumpPlacementNATDiag(t *testing.T, fix *Fixture, bastionPubIP, bastionID string) { t.Helper() if fix.Artifacts == "" { @@ -906,8 +848,7 @@ func dumpPlacementNATDiag(t *testing.T, fix *Fixture, bastionPubIP, bastionID st } } -// waitForInstanceStateBest is the cleanup-time analogue of -// harness.WaitForInstanceState — no t.Fatal, just polls + returns. +// waitForInstanceStateBest is the cleanup-safe analogue of harness.WaitForInstanceState: polls without t.Fatal. func waitForInstanceStateBest(c *harness.AWSClient, id, target string, timeout time.Duration) error { deadline := time.Now().Add(timeout) for { diff --git a/tests/e2e/multinode/preflight_test.go b/tests/e2e/multinode/preflight_test.go index fa934ff7d..875dcab37 100644 --- a/tests/e2e/multinode/preflight_test.go +++ b/tests/e2e/multinode/preflight_test.go @@ -11,13 +11,8 @@ import ( "github.com/mulgadc/spinifex/tests/e2e/harness" ) -// runPreflight is the Go port of pre-flight validation -// (run-multinode-e2e.sh:394-423). Two checks: -// - /dev/kvm exists and is writable on the local node (we run on node1). -// - SSH to every peer node succeeds with `hostname`. -// -// Failing this exposes a misconfigured VM image or broken peer networking -// before any AWS API churn. +// runPreflight checks /dev/kvm is writable on node1 and SSH `hostname` succeeds on every peer. +// Catches misconfigured images or broken peer networking before any AWS API churn. func runPreflight(t *testing.T, fix *Fixture) { harness.Phase(t, "Multinode — Pre-flight") @@ -32,9 +27,7 @@ func runPreflight(t *testing.T, fix *Fixture) { ssh := harness.NewPeerSSH() ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - // Bash runs this loop from node1 and skips itself; mirror that — the - // runner host doesn't necessarily accept SSH back into its own loopback - // and a local hostname call would test nothing about peer reachability. + // Skip node1 (the runner): SSH back to its own loopback tests nothing about peer reachability. for _, n := range fix.Cluster.Nodes[1:] { if _, err := ssh.Run(ctx, n.Addr, "hostname"); err != nil { t.Fatalf("peer_ssh %s (%s): %v", n.Name, n.Addr, err) diff --git a/tests/e2e/multinode/tests_test.go b/tests/e2e/multinode/tests_test.go index a971bfc6a..db5508577 100644 --- a/tests/e2e/multinode/tests_test.go +++ b/tests/e2e/multinode/tests_test.go @@ -4,53 +4,18 @@ package multinode import "testing" -// Top-level Test* wrappers. Each delegates to a runX function in the matching -// _test.go file. Names follow the single-node convention (TestX) so -// isolated runs via `go test -run TestMultinodeClusterHealth` are stable. -// -// Spread placement + NAT GW (formerly bash phase 11) lives in -// placement_nat_test.go as a single TestMultinodeSpread with 6 t.Run -// sub-tests sharing one VPC + bastion + private trio + NAT GW setup chain. -// Sub-test layout keeps JUnit granularity without paying 6× setup cost. -// -// Parallelism: -// -// Bucket #1 (parallel): the read-only / independent Tests below call -// t.Parallel(). They share the package-singleton trio (sync.Once gated) and -// the harness Fixture but never mutate it — DescribeInstances / SSH probe / -// VPC creation in independent CIDR space. -// -// Sequential (no t.Parallel — would race the parallel bucket): -// - TestMultinodePreflight : runs first; initialises pkg fixture. -// - TestMultinodeVolumeLifecycle : touches predastore state. -// - TestMultinodeCrossNodeGateway : asserts equality between baseline -// DescribeInstances and per-gateway DescribeInstances; concurrent VPC -// test launches/terminates instances mid-assert, breaking equality. -// Bead spec listed it in bucket #1 but the snapshot assumption fails -// under parallel state churn; keep sequential. -// - TestMultinodeCrossNodeOps : stops/starts trio[0]. Only waits -// for state "running", not for sshd; GuestSSH must not probe the trio -// until the guest has settled. Bead spec listed it in bucket #1 but the -// trio mutation makes that unsafe; keep sequential. -// - TestMultinodeNodeFailure/Recovery : StopNode/StartNode mutate cluster. -// - TestMultinodeSpread : owns EIP pool + VPC CIDR -// 10.100.0.0/16; sub-tests share the setup chain sequentially. -// - TestMultinodeGuestSSH : iterates every trio member over -// SSH. Declared last so it runs after CrossNodeOps + NodeFailure/Recovery -// + Spread have restabilised the cluster — the restarted trio[0] is then -// long-booted and there is no concurrent VM churn to delay sshd. +// Top-level Test* wrappers, each delegating to a runX function. Parallel tests +// share the read-only singleton trio; sequential tests mutate cluster state +// (stop/start nodes, predastore, EIP pool) and are ordered so the cluster is +// fully restabilised before GuestSSH probes every trio member. -// TestMultinodePreflight runs sequentially because it initialises the -// package fixture singleton. +// TestMultinodePreflight runs sequentially first to initialise the package fixture singleton. func TestMultinodePreflight(t *testing.T) { runPreflight(t, requireMultiNodeFixture(t)) } -// Fresh-install reachability baselines run right after preflight and before -// any test calls needInstanceTrio (which authorizes SSH on the default SG), -// so the default SG / subnet / route table are exercised pristine. Both own -// their mutable resources (dedicated SGs, self-cleaning instances) and never -// mutate a default resource. +// Baseline tests run before needInstanceTrio so the default SG/subnet/route table +// are in pristine state. Both own dedicated SGs and self-cleaning instances. func TestMultinodeDefaultSGReachabilityBaseline(t *testing.T) { runMultinodeDefaultSGReachabilityBaseline(t, requireMultiNodeFixture(t)) } @@ -69,22 +34,19 @@ func TestMultinodeInstanceDistribution(t *testing.T) { runInstanceDistribution(t, requireMultiNodeFixture(t)) } -// TestMultinodeVolumeLifecycle is sequential — touches predastore state -// shared with other suites. +// TestMultinodeVolumeLifecycle is sequential: touches predastore state. func TestMultinodeVolumeLifecycle(t *testing.T) { runVolumeLifecycle(t, requireMultiNodeFixture(t)) } -// TestMultinodeCrossNodeGateway is sequential — asserts a stable -// instance-count snapshot across gateways, which the parallel VPC test -// would break by launching/terminating its own instances mid-assert. +// TestMultinodeCrossNodeGateway is sequential: asserts a stable per-gateway instance count +// that concurrent launches/terminates would break. func TestMultinodeCrossNodeGateway(t *testing.T) { runCrossNodeGateway(t, requireMultiNodeFixture(t)) } -// TestMultinodeCrossNodeOps is sequential — stops/starts trio[0]. It waits -// only for state "running", so GuestSSH (declared last) probes the trio after -// the cluster restabilises rather than racing this restart. +// TestMultinodeCrossNodeOps is sequential: stops/starts trio[0]; GuestSSH is declared +// after this so it probes the trio only after the cluster has restabilised. func TestMultinodeCrossNodeOps(t *testing.T) { runCrossNodeOps(t, requireMultiNodeFixture(t)) } @@ -97,36 +59,25 @@ func TestMultinodeNodeRecovery(t *testing.T) { runNodeRecovery(t, requireMultiNodeFixture(t)) } -// TestMultinodeSpread runs after NodeRecovery so the cluster is fully -// healthy + degrade-tested before this Test launches its 4-VM + NAT GW + -// custom-VPC graph. Sequential — owns 10.100.0.0/16 + EIP pool; sub-tests -// share the setup chain (see placement_nat_test.go). +// TestMultinodeSpread is sequential after NodeRecovery; owns 10.100.0.0/16 + EIP pool. func TestMultinodeSpread(t *testing.T) { runSpread(t, requireMultiNodeFixture(t)) } -// TestMultinodeGuestSSH is sequential and declared last in the sequential -// bucket so it runs after the cluster has fully restabilised. It iterates -// every trio member, and CrossNodeOps stop/starts trio[0] — which only waits -// for state "running", not for sshd to answer. Running parallel, GuestSSH -// resumed immediately after CrossNodeOps and probed a guest still booting, -// yielding kex_exchange_identification resets and 2m timeouts. Ordering it -// after NodeFailure/Recovery + Spread gives every trio member ample settle -// time and removes concurrent VM churn from the parallel bucket. +// TestMultinodeGuestSSH is sequential and declared last so it runs after the cluster has +// fully restabilised. CrossNodeOps stops/starts trio[0] and only waits for "running" — not +// sshd — so GuestSSH must not probe until the guest has settled. func TestMultinodeGuestSSH(t *testing.T) { runGuestSSH(t, requireMultiNodeFixture(t)) } -// TestMultinodeVPCNetworking owns its own 10.200.0.0/16 VPC (no EIP use) -// so it's safe alongside bucket #1. +// TestMultinodeVPCNetworking owns 10.200.0.0/16 (no EIP use); safe to run in parallel. func TestMultinodeVPCNetworking(t *testing.T) { t.Parallel() runVPCNetworking(t, requireMultiNodeFixture(t)) } -// TestMultinodeIPSec is read-only over SSH — verifies the OVN native IPsec -// wiring (OVS DB cert pointers, xfrm SAs, ESP traffic) without launching -// any AWS resources. Safe to run alongside the parallel bucket. +// TestMultinodeIPSec is read-only over SSH; launches no AWS resources. func TestMultinodeIPSec(t *testing.T) { t.Parallel() runIPSec(t, requireMultiNodeFixture(t)) diff --git a/tests/e2e/multinode/volume_test.go b/tests/e2e/multinode/volume_test.go index d166302ca..9fe976eef 100644 --- a/tests/e2e/multinode/volume_test.go +++ b/tests/e2e/multinode/volume_test.go @@ -12,15 +12,9 @@ import ( "github.com/stretchr/testify/require" ) -// runVolumeLifecycle is the Go port of volume lifecycle -// (run-multinode-e2e.sh:731-825). Creates a 10GiB volume, attaches to the -// first trio instance at /dev/sdf, detaches, deletes. Each step polls a -// matching state target (in-use/attached → available → 404). -// -// Volume is created + torn down inline (not via EnsureVolume) because the -// test exercises the full CRUD cycle including DeleteVolume — wiring it -// through Ensure* would have the fixture cleanup race the in-test delete -// and surface as InvalidVolume.NotFound. +// runVolumeLifecycle creates a 10GiB volume, attaches it to trio[0] at /dev/sdf, +// detaches, and deletes. Managed inline (not via EnsureVolume) to exercise the full +// CRUD cycle — Ensure* cleanup would race the in-test delete. func runVolumeLifecycle(t *testing.T, fix *Fixture) { harness.Phase(t, "Multinode — Volume Lifecycle") diff --git a/tests/e2e/multinode/vpc_test.go b/tests/e2e/multinode/vpc_test.go index 516ebece0..e72255265 100644 --- a/tests/e2e/multinode/vpc_test.go +++ b/tests/e2e/multinode/vpc_test.go @@ -17,15 +17,9 @@ import ( // IPs must come from 10.200.1.0/24. Compiled once. var vpcSubnetCIDR = regexp.MustCompile(`^10\.200\.1\.[0-9]+$`) -// runVPCNetworking is the Go port of VPC networking validation -// (run-multinode-e2e.sh:1042-1251). Stands up a fresh non-default VPC + -// subnet, launches 3 instances into the subnet, asserts each gets a unique -// PrivateIpAddress within 10.200.1.0/24, then stops + restarts all three and -// re-asserts the same IPs persist. -// -// Independent of the package-level trio (which lives on the default VPC) — -// this test needs full control over subnet allocation, so instances are -// launched, terminated, and cleaned up locally via t.Cleanup. +// runVPCNetworking stands up a fresh VPC 10.200.0.0/16 + subnet, launches 3 instances, +// asserts unique PrivateIpAddresses within 10.200.1.0/24, then stops + restarts and +// re-asserts IP persistence. Independent of the package-level trio. func runVPCNetworking(t *testing.T, fix *Fixture) { harness.Phase(t, "Multinode — VPC Networking") @@ -63,9 +57,7 @@ func runVPCNetworking(t *testing.T, fix *Fixture) { }) harness.Detail(t, "subnet", subnetID, "cidr", "10.200.1.0/24") - // OVN topology programming is async — bash sleeps 2s after create-subnet - // before launching. Mirror that pause to avoid flakes during the first - // RunInstances. + // OVN topology programming is async; 2s pause avoids first-RunInstances flakes. time.Sleep(2 * time.Second) // --- Launch 3 instances ------------------------------------------------ @@ -86,8 +78,7 @@ func runVPCNetworking(t *testing.T, fix *Fixture) { require.NotEmptyf(t, id, "run-instances #%d: empty InstanceId", i+1) instIDs = append(instIDs, id) harness.Detail(t, "instance", id) - // Per-instance cleanup — terminate so the subnet/VPC can drop in - // later cleanup phases. Best-effort; we're already unwinding. + // Terminate so the subnet/VPC can be deleted in later cleanup phases. idCopy := id t.Cleanup(func() { _, _ = c.EC2.TerminateInstances(&ec2.TerminateInstancesInput{ @@ -159,10 +150,8 @@ func runVPCNetworking(t *testing.T, fix *Fixture) { harness.Detail(t, "ips_restarted", restartedIPs) } -// describeAndCollectIPs runs DescribeInstances for each id, asserts the -// Subnet/VPC binding matches expectations, and returns PrivateIpAddress -// values in the same order as instIDs. Fatal on any missing field — bash -// also fatals on these. +// describeAndCollectIPs describes each instance, asserts Subnet/VPC binding, and returns +// PrivateIpAddress values in instIDs order. func describeAndCollectIPs(t *testing.T, c *harness.AWSClient, instIDs []string, wantSubnet, wantVPC string) []string { t.Helper() ips := make([]string, 0, len(instIDs)) @@ -184,8 +173,7 @@ func describeAndCollectIPs(t *testing.T, c *harness.AWSClient, instIDs []string, return ips } -// needArch returns the discovered arch for the nano instance type. -// Convenience wrapper because needAMI takes arch, not (t, fix). +// needArch returns the architecture for the discovered nano instance type. func needArch(t *testing.T, fix *Fixture) string { t.Helper() _, arch := needInstanceTypeArch(t, fix) diff --git a/tests/e2e/reboot/reboot_test.go b/tests/e2e/reboot/reboot_test.go index a3558dcb1..9064ac0f9 100644 --- a/tests/e2e/reboot/reboot_test.go +++ b/tests/e2e/reboot/reboot_test.go @@ -1,17 +1,8 @@ //go:build e2e -// Package reboot ports run-reboot-e2e.sh — the single-node host-reboot -// resilience driver (cell #18). -// -// Unlike the cert/lb/single suites which ship their compiled .test binary to -// the spinifex VM and run in-VM, this suite MUST run on the GitHub Actions -// runner (or any host outside the cluster). `sudo systemctl reboot` kills any -// in-VM process mid-step, so the driver has to live somewhere the reboot -// can't touch. -// -// All AWS API traffic crosses the WAN to https://WAN_IP:9999 via the SDK; -// SSH-level ops (reboot, ovn-nbctl, journalctl, file mutations) hop via -// harness.PeerSSH. +// Package reboot is the single-node host-reboot resilience suite. It MUST run outside +// the cluster (on the CI runner) because systemctl reboot kills any in-VM process. +// AWS API traffic crosses the WAN via the SDK; SSH ops use harness.PeerSSH. package reboot import ( @@ -50,10 +41,8 @@ const ( rebootDoctype = "reboot-e2e" ) -// appUserData provisions a systemd-managed HTTP responder on the guest VM. -// Installed as a unit rather than nohup-from-cloud-init so the responder -// survives the guest's own restart after the host reboot — cloud-init's -// per-instance semaphore only fires once. +// appUserData provisions a systemd-managed HTTP responder. Using a unit (not nohup) ensures +// the responder survives the guest's own restart after host reboot. const appUserData = `#!/bin/bash set -e INSTANCE_ID=$(hostname) @@ -131,12 +120,8 @@ type timeouts struct { lbRecover time.Duration } -// TestRebootResilience is the Go port of run-reboot-e2e.sh. -// -// Phases run sequentially in one top-level test rather than parallel subtests -// — the reboot itself is a global event and later phases assume earlier -// phases succeeded. Each phase emits a Phase banner so failures localise -// cleanly in JUnit XML. +// TestRebootResilience runs the 8-phase reboot resilience sequence. +// Phases are sequential: the reboot is a global event and each phase depends on the prior. func TestRebootResilience(t *testing.T) { env := harness.LoadEnv(t) requireRunnerMode(t, env) @@ -150,18 +135,9 @@ func TestRebootResilience(t *testing.T) { rebootWait: durationEnv("REBOOT_WAIT_SECS", 300*time.Second), daemonReady: durationEnv("DAEMON_READY_SECS", 180*time.Second), instanceRunning: durationEnv("INSTANCE_RUNNING_SECS", 120*time.Second), - // 5min, not bash's 180s. Pre-reboot the test creates the LB - // AFTER the apps are already running, so by the time HAProxy - // starts probing the app responders are already serving — ~75s - // to healthy. Post-reboot the daemon's recovery scan relaunches - // app VMs and the ALB sys.micro VM simultaneously, so HC starts - // probing within seconds of app boot beginning. Both targets - // honestly report unhealthy until the app's responder unit - // finishes starting (cloud-init's per-instance semaphore means - // no shortcut path on relaunch). The bash driver masked this by - // reading stale "healthy" from the daemon's pre-reboot TG cache - // (now fixed via ResetTargetHealthOnStartup — mulga-siv-119); - // the honest wait surfaces the real recovery window. + // 5min budget: post-reboot the daemon relaunches app VMs and the ALB sys.micro + // simultaneously; HC probing starts immediately but targets honestly report + // unhealthy until the responder unit finishes starting (no cloud-init shortcut). lbRecover: durationEnv("LB_RECOVER_SECS", 300*time.Second), }, } @@ -216,10 +192,7 @@ func TestRebootResilience(t *testing.T) { // --- Setup helpers ------------------------------------------------------- -// requireRunnerMode bails early if the env doesn't look runner-resident. -// WAN_IP empty would still work but signals a config mistake — better to fail -// fast than walk through 30s of AWS-endpoint timeouts before the operator -// notices. +// requireRunnerMode bails early if the env isn't runner-resident (SPINIFEX_WAN_IP or endpoint missing). func requireRunnerMode(t *testing.T, env *harness.Env) { t.Helper() if env.WANHost == "" { @@ -233,12 +206,8 @@ func requireRunnerMode(t *testing.T, env *harness.Env) { } } -// resolveTrust attempts to SCP the spinifex CA off the VM to a tmp file so -// the AWS SDK uses real TLS verification. On any failure (no SSH key, scp -// non-zero exit, file empty) it logs the reason and flips -// SPINIFEX_AWS_INSECURE=1 so the SDK skips verification. CA trust isn't what -// the reboot suite is testing — the cert suite already validates trust — so -// the fallback is intentional rather than a quiet degradation. +// resolveTrust SCPs the spinifex CA to a tmp file for real TLS verification. +// Falls back to SPINIFEX_AWS_INSECURE=1 on any error; CA trust is the cert suite's concern. func resolveTrust(t *testing.T, fix *fixture) { t.Helper() if os.Getenv("SPINIFEX_AWS_INSECURE") == "1" { @@ -294,8 +263,7 @@ func phase0Prereqs(t *testing.T, fix *fixture) { fix.amiID = discoverAMI(t, fix.aws) harness.Detail(t, "ami", fix.amiID) - // Key pair — capture KeyMaterial so post-reboot diagnostics can SSH into - // guest VMs via the host's QEMU hostfwd port when health probes fail. + // Capture KeyMaterial so post-reboot diagnostics can SSH into guest VMs. _, _ = fix.aws.EC2.DeleteKeyPair(&ec2.DeleteKeyPairInput{KeyName: aws.String(keyName)}) kpOut, err := fix.aws.EC2.CreateKeyPair(&ec2.CreateKeyPairInput{KeyName: aws.String(keyName)}) require.NoErrorf(t, err, "create-key-pair %s", keyName) @@ -344,8 +312,7 @@ func phase1Network(t *testing.T, fix *fixture) { require.NoError(t, err, "create-security-group") fix.sgID = aws.StringValue(sgOut.GroupId) - // Structured IpPermissions form — vpcd ignores the top-level shortcut - // (mulga-siv-79) so port 80 ingress must use IpPermissions to land in OVN. + // Use IpPermissions form — vpcd ignores the top-level shorthand field. // tcp/22 is opened so on-failure diagnostics can SSH into app guests. _, err = fix.aws.EC2.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{ GroupId: aws.String(fix.sgID), @@ -493,11 +460,8 @@ func phase5Snapshot(t *testing.T, fix *fixture) { harness.Detail(t, "pre_switches", fix.preSwitches, "pre_ports", fix.prePorts) harness.DumpFile(t, fix.artifacts, "ovn-pre.txt", out) - // Capture the OVN chassis identity + claim state PRE-reboot so the - // post-reboot dump can be diffed against it. The post-reboot claim failure - // (run 27254808154) shows the SB Chassis _uuid churns across reboot while - // SB persists, orphaning every Port_Binding claim. The pre value is the - // baseline that proves (or refutes) the churn. + // Capture OVN chassis identity pre-reboot so the post-reboot dump can be diffed; + // SB Chassis _uuid churn across reboot orphans every Port_Binding claim. cstate, err := fix.ssh.Run(ctx, fix.env.WANHost, ovnChassisStateBundle) if err != nil { t.Logf("ovn chassis pre-snapshot: %v (non-fatal)", err) @@ -512,14 +476,12 @@ func phase6Reboot(t *testing.T, fix *fixture) { fix.mu.Unlock() harness.Step(t, "issuing reboot via ssh (connection will drop)") - // Best-effort: the ssh process is expected to die mid-command when the - // host starts shutting down. Don't fail on it. + // SSH dies mid-command when the host shuts down; ignore the error. ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() _, _ = fix.ssh.Run(ctx, fix.env.WANHost, "sudo systemctl reboot") - // Let the host start shutting down before we begin polling so we don't - // accept the pre-reboot SSH window as "back". + // Wait for shutdown to begin so the pre-reboot SSH window isn't mistaken for "back up". time.Sleep(5 * time.Second) harness.Step(t, "polling TCP/22 (timeout %s)", fix.timeouts.rebootWait) @@ -606,14 +568,8 @@ func phase8Asserts(t *testing.T, fix *fixture) { // 8.4 ALB active harness.WaitForLBActive(t, fix.aws, fix.albArn, "ALB post-reboot", fix.timeouts.lbRecover) - // 8.5 targets healthy — on timeout, capture diagnostics BEFORE failing. - // The lb-agent's "context deadline exceeded while awaiting headers" symptom - // cannot, on its own, tell a network-level drop from an app/gateway stall. - // The heartbeat capture watches both the real heartbeat path (gateway port - // on any iface) and the north-south floating-IP path on br-wan; the OVN - // dataplane capture then shows whether br-int is actually forwarding - // external->tap traffic (flows installed) or only the control-plane/NAT - // config + neigh recovered while forwarding stays dead. + // 8.5 targets healthy — capture diagnostics before failing so a timeout can be + // pinned to network-level drop vs. app/gateway stall. if !pollTargetsHealthy(fix.aws, fix.tgArn, 2, fix.timeouts.lbRecover) { harness.Step(t, "ALB post-reboot: targets NOT healthy — capturing diagnostics") dctx, dcancel := context.WithTimeout(context.Background(), 120*time.Second) @@ -626,16 +582,8 @@ func phase8Asserts(t *testing.T, fix *fixture) { } harness.Step(t, "ALB post-reboot: 2 targets healthy") - // 8.6 ALB actually serves traffic from BOTH backends. - // WaitForTargetsHealthy reads cached state from spinifex's TG store — - // post-reboot the cache may still show pre-reboot "healthy" until the - // lb-agent inside the ALB system VM sends a fresh report. A naive - // "ANY response" probe lets the burst fire when only one backend is - // up (the second is still bringing HAProxy/responder online), and the - // burst then lands 20/0 instead of 10/10. Probe until both backends' - // instance_ids each appear at least once — confirms the recovery path - // has actually reached both before we assert round-robin. Refs the - // daemon-side TG cache invalidation bug tracked separately. + // 8.6 Probe until both backends' instance_ids are seen before asserting round-robin: + // healthy-cache state may show stale "healthy" until both responders are actually up. harness.Step(t, "waiting for ALB to serve HTTP from both backends") probeCtx, probeCancel := context.WithTimeout(context.Background(), 180*time.Second) defer probeCancel() @@ -656,10 +604,7 @@ func phase8Asserts(t *testing.T, fix *fixture) { runHTTPBurst(t, fix, "ALB post-reboot") - // 8.6.5 ALWAYS-RUN guest probe — captures in-guest responder state even - // when the test passes, so we can confirm whether the responder is genuinely - // running on the relaunched VMs vs. some upstream cache satisfying HC. - // Refs investigation into "tcp/22 SG ingress fixes post-reboot HTTP". + // 8.6.5 Always capture in-guest responder state (pass or fail) to confirm genuine liveness. dumpAppDiagnostics(t, fix) // 8.7 ovn-nbctl drift check @@ -895,11 +840,8 @@ func dumpDiagnostics(t *testing.T, fix *fixture, since time.Time) { } } -// dumpAppDiagnostics probes each app instance's IGW-NAT'd public IP from the -// host VM. App VMs use pure OVN tap interfaces (no QEMU hostfwd), so SSH from -// host is impossible — public-IP probes are the only no-cost path. Localises -// post-reboot failures to one of: VM not running, OVN DNAT missing, responder -// down, or responder up but reachable only from sys.micro (path asymmetry). +// dumpAppDiagnostics probes each app instance's public IP from the host. +// App VMs have no QEMU hostfwd, so public-IP probes are the only available path. func dumpAppDiagnostics(t *testing.T, fix *fixture) { t.Helper() if len(fix.appInstanceIDs) == 0 { @@ -924,9 +866,7 @@ func dumpAppDiagnostics(t *testing.T, fix *fixture) { } } -// dumpHostNetState captures post-reboot OVN/OVS/route state on the spinifex -// host so failures show whether the gateway router, ext bridge, and host -// routing table all came back as expected. Single round-trip. +// dumpHostNetState captures OVN/OVS/route state on the host in a single round-trip. func dumpHostNetState(ctx context.Context, t *testing.T, fix *fixture) { t.Helper() bundle := `set +e @@ -953,8 +893,7 @@ sudo ovs-vsctl list-ports br-int 2>&1 } // pollTargetsHealthy is the non-fatal twin of harness.WaitForTargetsHealthy: -// returns true once `expected` targets report healthy, false on timeout — so the -// caller can capture failure-time diagnostics before aborting. +// returns true on healthy, false on timeout, so the caller can capture diagnostics before aborting. func pollTargetsHealthy(c *harness.AWSClient, tgArn string, expected int, timeout time.Duration) bool { deadline := time.Now().Add(timeout) for { @@ -979,16 +918,9 @@ func pollTargetsHealthy(c *harness.AWSClient, tgArn string, expected int, timeou } } -// dumpHeartbeatDiagnostics captures, at post-reboot failure time, whether the -// ALB system-VM lb-agent's heartbeat to the gateway (:AWSGWPort) is failing at -// the NETWORK layer or the APP/GATEWAY layer. The lb-agent posts to the host -// gateway endpoint on whatever host interface holds the advertise IP — NOT the -// ALB floating IP, and NOT necessarily br-wan — so the primary tcpdump watches -// the gateway port on `any`. A second tcpdump on br-wan for the ALB floating IP -// covers the north-south path. It then pulls the lb-agent console lines and the -// awsgw journal to see whether the gateway even received the heartbeat. Gateway -// packets seen + gateway answering => app/gateway-layer; silence on the gateway -// port => network-level drop (corroborate with diag-ovn-flows.txt). +// dumpHeartbeatDiagnostics captures whether the lb-agent's heartbeat to the gateway is +// failing at the network layer or the app/gateway layer. Tcpdumps both the gateway port +// (any iface) and the br-wan north-south path; also pulls lb-agent console and awsgw journal. func dumpHeartbeatDiagnostics(ctx context.Context, t *testing.T, fix *fixture) { t.Helper() if fix.albPublicIP == "" { @@ -1017,7 +949,7 @@ sudo journalctl -u spinifex-awsgw --no-pager --since "3 min ago" 2>/dev/null | g return } body := string(out) - // "IP a.b.c.d. >" appears for any L4 packet on the gateway port. + // Detect any L4 packet on the gateway port. sawGatewayL4 := strings.Contains(body, fmt.Sprintf(".%d:", fix.env.AWSGWPort)) || strings.Contains(body, fmt.Sprintf(".%d >", fix.env.AWSGWPort)) || strings.Contains(body, fmt.Sprintf("> %s.%d", fix.env.WANHost, fix.env.AWSGWPort)) @@ -1029,14 +961,8 @@ sudo journalctl -u spinifex-awsgw --no-pager --since "3 min ago" 2>/dev/null | g } } -// dumpOVNDataplane captures the OVN/OVS dataplane state at post-reboot failure -// time so a network-level heartbeat drop can be pinned to the restore step: -// control-plane/NAT config + neigh recovered, but br-int flows never reprogrammed -// for the restored taps. It dumps ovs-vsctl topology, br-int OpenFlow flows, -// ovn-controller chassis + port bindings, the dnat_and_snat NAT rows, and the -// host neigh table, then probes reachability of the ALB floating IP from the -// host. Flows present + reachable => forwarding OK (look elsewhere); config -// present but flows/forwarding absent => br-int flow-restore gap. +// dumpOVNDataplane captures OVN/OVS dataplane state at failure time: topology, br-int flows, +// Port_Bindings, NAT rows, neigh table, and ALB floating-IP reachability from the host. func dumpOVNDataplane(ctx context.Context, t *testing.T, fix *fixture) { t.Helper() bundle := fmt.Sprintf(`set +e @@ -1076,12 +1002,8 @@ sudo ovs-appctl ofproto/trace br-int "in_port=LOCAL,ip,nw_dst=%[1]s" 2>&1 | tail } } -// ovnChassisStateBundle captures the OVN chassis identity + SB chassis records. -// Run both pre-reboot (phase5Snapshot) and post-reboot (dumpOVNClaimState) so -// the two can be diffed: if the SB Chassis _uuid (or the OVS system-id) differs -// across the reboot, the chassis identity churned and every persisted -// Port_Binding claim is orphaned — the root-cause signal for the post-reboot -// "claims nothing" failure. +// ovnChassisStateBundle captures OVN chassis identity + SB records for pre/post diff. +// _uuid churn across reboot orphans every Port_Binding claim. const ovnChassisStateBundle = `set +e echo "=== OVS system-id (chassis identity; must be stable across reboot) ===" sudo cat /etc/openvswitch/system-id.conf 2>&1 @@ -1095,15 +1017,9 @@ sudo ovn-nbctl get-connection 2>&1 sudo ovn-sbctl get-connection 2>&1 ` -// dumpOVNClaimState captures, at post-reboot failure time, the evidence needed -// to explain WHY ovn-controller on the live chassis is not claiming local -// Port_Bindings (run 27254808154: ENIs stuck on a dead chassis, cr-gw chassis -// empty, NAT external_mac empty, all while NB intent is correct and the taps -// carry the right iface-id). The ovn-controller vlog lives under /var/log/ovn -// (NOT journald), so it is invisible to the systemd-journal artifacts; this -// pulls it directly, plus the controller's SB connection state and the -// ovn-central-vs-ovn-controller boot ordering — disambiguating chassis-identity -// churn from an SB/NB startup-ordering race. +// dumpOVNClaimState captures evidence for why ovn-controller is not claiming Port_Bindings. +// Pulls /var/log/ovn directly (not journald), SB connection state, and boot ordering +// to distinguish chassis-identity churn from an SB/NB startup-ordering race. func dumpOVNClaimState(ctx context.Context, t *testing.T, fix *fixture) { t.Helper() bundle := ovnChassisStateBundle + `echo "=== ovn-controller SB connection-status ===" @@ -1190,9 +1106,7 @@ func diagnoseAppInstance(ctx context.Context, t *testing.T, fix *fixture, id str harness.Detail(t, id, "qemu running") } - // Serial console — captures cloud-init, kernel oops, systemd boot, anything - // the guest emitted to ttyS0. Tail to cap size; the full log lives on the - // host until cleanup. + // Serial console (ttyS0) — cloud-init, kernel oops, systemd boot events. consoleOut, _ := fix.ssh.Run(ctx, fix.env.WANHost, fmt.Sprintf("sudo tail -c 262144 /run/spinifex/console-%s.log 2>/dev/null || true", id)) harness.DumpFile(t, fix.artifacts, fmt.Sprintf("diag-%s-console.log", id), consoleOut) @@ -1236,16 +1150,12 @@ rm -f /tmp/diag-body-%[2]s harness.Detail(t, id, "probe inconclusive — see artifact") } - // Try SSH-in via the IGW path. tcp/22 SG ingress is wired in phase1. - // On success this dumps the responder unit + cloud-init state, which is - // the only path to root-cause a "tcp open but responder silent" failure. + // SSH-in via IGW; dumps responder unit + cloud-init state inside the guest. sshIntoApp(ctx, t, fix, id, pub) } -// sshIntoApp uploads the key pair private key to the host, then SSHes from -// the host to the app guest at its IGW public IP and dumps responder / -// cloud-init / network state. Best-effort: silently no-ops if key wasn't -// captured or SSH fails. +// sshIntoApp uploads the key to the host then SSHes into the app guest to dump +// responder, cloud-init, and network state. Best-effort: no-ops if key unavailable or SSH fails. func sshIntoApp(ctx context.Context, t *testing.T, fix *fixture, id, pub string) { t.Helper() if fix.privateKeyPath == "" { @@ -1309,8 +1219,7 @@ ls -l /var/lib/cloud/instances/ 2>&1 echo "=== /var/lib/cloud/instances/*/sem ===" ls -l /var/lib/cloud/instances/*/sem/ 2>&1 ` - // Outer ssh ships one cmd; inside, write the key, then ssh again to guest - // with a heredoc. Single round trip from runner. + // Single outer SSH: write the key then hop into the guest. One round trip. outerCmd := fmt.Sprintf(`set +e umask 077 && printf '%%s' '%s' | base64 -d > %s && chmod 600 %s ssh -i %s -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ @@ -1332,9 +1241,8 @@ rm -f %s // --- Cleanup ------------------------------------------------------------- -// cleanup runs t.Cleanup logic — LIFO ordering of the bash cleanup() trap. -// SG deletion runs LAST because it can't be deleted while instance/ALB ENIs -// still reference it. +// cleanup tears down all test resources; SG is deleted last as it can't be removed +// while instance or ALB ENIs still reference it. func cleanup(t *testing.T, fix *fixture) { t.Helper() harness.Phase(t, "Cleanup") @@ -1401,8 +1309,7 @@ func durationEnv(key string, def time.Duration) time.Duration { if v == "" { return def } - // Accept either a bare integer (seconds, per the bash defaults) or any - // time.ParseDuration string (e.g. "5m"). Bash sets REBOOT_WAIT_SECS=300. + // Accept a bare integer (treated as seconds) or a time.ParseDuration string. if d, err := time.ParseDuration(v); err == nil { return d } diff --git a/tests/e2e/single/account_test.go b/tests/e2e/single/account_test.go index 6bc97cf1e..be05c6e31 100644 --- a/tests/e2e/single/account_test.go +++ b/tests/e2e/single/account_test.go @@ -14,33 +14,10 @@ import ( "github.com/stretchr/testify/require" ) -// runAccountScoping ports run-e2e.sh Phase 8 EC2 Account Scoping -// (lines 1915–2624). Validates that EC2 resources (instances, volumes, key -// pairs, snapshots, VPCs, IGWs, EIGWs) are isolated between tenant accounts -// created via `spx admin account create`. -// -// Three admin accounts are created (Team Alpha, Team Beta, Team Gamma). All -// resources are launched through their scoped *AWSClient (statically-credentialed, -// bypasses AWS_PROFILE). A single t.Cleanup at the outer function tears down -// every resource across all three accounts regardless of which sub-step -// failed — every delete is best-effort / not-found tolerant. There is no -// `spx admin account delete` so the account rows themselves persist beyond -// the test; the carousel entry is local-only state and goes out of scope -// with the function. -// -// Sub-steps: -// -// Step1_AccountSetup — create Alpha + Beta, smoke-test auth -// Step2_InstanceScoping — instance describe + cross-account ops blocked -// Step3_VolumeScoping — volume describe + attach/detach/modify -// Step4_KeyPairScoping — key pair namespace + cross-account delete no-op -// Step5_SnapshotScoping — snapshot describe + OwnerId + delete -// Step6_VPCSubnetScoping — VPC/subnet describe + cross-account ops -// Step7_IGWEIGWScoping — IGW/EIGW describe + attach/detach -// Step8_AccountSettings — EBS-encryption-by-default per-tenant -// Step9_GlobalResources — regions/AZs/instance types identical -// Step10_EdgeCases — empty Gamma + non-existent resource ids -// Step11_Cleanup — explicit best-effort teardown (also runs via t.Cleanup) +// runAccountScoping validates that EC2 resources (instances, volumes, key +// pairs, snapshots, VPCs, IGWs, EIGWs) are isolated between tenant accounts. +// Three accounts are created (Alpha, Beta, Gamma). All resources use scoped +// clients with static credentials. t.Cleanup tears down everything best-effort. func runAccountScoping(t *testing.T, fix *Fixture) { harness.Phase(t, "Single — EC2 Account Scoping") @@ -50,8 +27,7 @@ func runAccountScoping(t *testing.T, fix *Fixture) { carousel := harness.NewAccountCarousel() - // State threaded across Steps. The teardown below references whatever - // is populated at failure time; zero values are skipped. + // State threaded across Steps; zero values are skipped by teardown. var ( alphaInst, betaInst string alphaVol, betaVol string @@ -66,11 +42,7 @@ func runAccountScoping(t *testing.T, fix *Fixture) { betaEncryptionLeftEnabled bool ) - // t.Cleanup runs LIFO regardless of which Step failed. Every call below - // swallows not-found errors so a partial Step 11 run + this cleanup is - // idempotent. Step 11's explicit cleanup is duplicated here intentionally: - // the bash script's `2>/dev/null || true` pattern translates to "ignore - // any error" in Go, which is exactly what these closures do. + // t.Cleanup runs LIFO. All deletes below swallow not-found errors (idempotent). t.Cleanup(func() { harness.Step(t, "Phase 8 acct cleanup (deferred)") alpha := carousel.Get("spx-team-alpha") @@ -224,8 +196,7 @@ func runAccountScoping(t *testing.T, fix *Fixture) { require.NoError(t, err, "beta describe-instances (auth check)") }) - // Bail out if Step 1 didn't seat both profiles — every subsequent step - // derefs them. Step 11 still runs via t.Cleanup with whatever state exists. + // Every subsequent step derefs these; bail if Step 1 failed to register them. alpha := carousel.Get("spx-team-alpha") beta := carousel.Get("spx-team-beta") if alpha == nil || beta == nil { @@ -493,7 +464,6 @@ func runAccountScoping(t *testing.T, fix *Fixture) { require.NotEmpty(t, betaShared, "beta shared key id") assert.NotEqual(t, alphaShared, betaShared, "same KeyPairId across accounts") - // Cross-account delete is idempotent (no error) but must not affect the other account. harness.Step(t, "beta deletes alpha-key (idempotent, no cross-account effect)") _, _ = beta.Client.EC2.DeleteKeyPair(&ec2.DeleteKeyPairInput{KeyName: aws.String(alphaKey)}) assert.Equal(t, alphaKeyID, describeKeyPairID(t, alpha.Client, alphaKey), @@ -555,8 +525,7 @@ func runAccountScoping(t *testing.T, fix *Fixture) { assert.Equal(t, alpha.AccountID, aws.StringValue(alphaSnaps.Snapshots[0].OwnerId), "alpha snapshot OwnerId mismatch") - // Cross-account delete — bash expects UnauthorizedOperation (NOT the - // usual NotFound). This is a Snapshot-specific gateway behavior. + // DeleteSnapshot returns UnauthorizedOperation (not NotFound) across accounts. harness.Step(t, "cross-account delete-snapshot blocked (UnauthorizedOperation)") harness.ExpectError(t, "UnauthorizedOperation", func() error { _, err := beta.Client.EC2.DeleteSnapshot(&ec2.DeleteSnapshotInput{ @@ -763,9 +732,7 @@ func runAccountScoping(t *testing.T, fix *Fixture) { } assert.NotContains(t, alphaEIGWIDs, betaEIGW, "alpha saw beta's EIGW") - // Cross-account EIGW delete — bash uses `expect_error "" ... || true` - // (any error or none is acceptable). The hard assertion is that - // alpha's EIGW survives. + // Any error or none is acceptable; the contract is that alpha's EIGW survives. _, _ = beta.Client.EC2.DeleteEgressOnlyInternetGateway(&ec2.DeleteEgressOnlyInternetGatewayInput{ EgressOnlyInternetGatewayId: aws.String(alphaEIGW), }) @@ -780,8 +747,7 @@ func runAccountScoping(t *testing.T, fix *Fixture) { } assert.True(t, stillThere, "alpha's EIGW was deleted by beta") - // Cross-account EIGW creation in other's VPC — bash also accepts any - // error or none; the existence of alpha's EIGW is the contract. + // Best-effort cross-account EIGW creation attempt in alpha's VPC. _, _ = beta.Client.EC2.CreateEgressOnlyInternetGateway(&ec2.CreateEgressOnlyInternetGatewayInput{ VpcId: aws.String(alphaVPC), }) @@ -861,9 +827,6 @@ func runAccountScoping(t *testing.T, fix *Fixture) { } assert.Empty(t, gInstIDs, "gamma has instances") - // Volume check is skipped — bash comments "root-account volumes - // (empty TenantID) are visible to all accounts by design". - gKeys := describeKeyPairNames(t, gamma.Client) assert.Empty(t, gKeys, "gamma has key pairs") @@ -912,10 +875,7 @@ func runAccountScoping(t *testing.T, fix *Fixture) { // --------------------------------------------------------------------- // Step 11: EC2 Account Scoping Cleanup (bash 2546–2624) // --------------------------------------------------------------------- - // Bash performs an explicit ordered teardown here. We do the same — but - // the outer t.Cleanup also runs (idempotently) so a Step 11 failure - // won't leak. Keeping Step 11 in-line preserves the bash structure and - // the explicit assertions a triager expects when reading the log. + // Explicit teardown — outer t.Cleanup also runs idempotently on failure. t.Run("Step11_Cleanup", func(t *testing.T) { harness.Step(t, "terminate alpha instance") if alphaInst != "" { @@ -1034,17 +994,12 @@ func runAccountScoping(t *testing.T, fix *Fixture) { betaVPC = "" } - // Clear the instance IDs so the outer t.Cleanup doesn't redundantly - // terminate them (a second terminate on a terminated id is a no-op - // at the gateway, but skipping the round-trip keeps logs clean). alphaInst = "" betaInst = "" }) } -// waitTerminated polls until id reaches "terminated" or vanishes (NotFound -// is treated as success — some gateway versions garbage-collect terminated -// rows). Bounded by a short timeout so a stuck termination surfaces fast. +// waitTerminated polls until id reaches "terminated" or NotFound (60s cap). func waitTerminated(t *testing.T, c *harness.AWSClient, id string) { t.Helper() deadline := time.Now().Add(60 * time.Second) @@ -1056,7 +1011,6 @@ func waitTerminated(t *testing.T, c *harness.AWSClient, id string) { if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "InvalidInstanceID.NotFound" { return } - // Other errors — keep polling until the deadline. } else if len(out.Reservations) > 0 && len(out.Reservations[0].Instances) > 0 { state := aws.StringValue(out.Reservations[0].Instances[0].State.Name) if state == "terminated" { @@ -1134,9 +1088,7 @@ func vpcIDs(vpcs []*ec2.Vpc) []string { return ids } -// describeRegionNames returns sorted region names. Bash compares by direct -// equality after a sort-by-position — same here, since the gateway returns -// the regions in a stable order. +// describeRegionNames returns region names in gateway-returned order. func describeRegionNames(t *testing.T, c *harness.AWSClient) []string { t.Helper() out, err := c.EC2.DescribeRegions(&ec2.DescribeRegionsInput{}) @@ -1160,8 +1112,7 @@ func describeAZNames(t *testing.T, c *harness.AWSClient) []string { return names } -// describeInstanceTypeNames returns sorted instance-type names — bash sorts -// before comparing so the assertion is order-insensitive. +// describeInstanceTypeNames returns sorted instance-type names. func describeInstanceTypeNames(t *testing.T, c *harness.AWSClient) []string { t.Helper() out, err := c.EC2.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{}) @@ -1170,13 +1121,11 @@ func describeInstanceTypeNames(t *testing.T, c *harness.AWSClient) []string { for _, it := range out.InstanceTypes { names = append(names, aws.StringValue(it.InstanceType)) } - // Sort to mirror bash's `| tr '\t' '\n' | sort` step. sortStrings(names) return names } -// sortStrings sorts in place — kept local to avoid an extra import in the -// already-busy file. Equivalent to sort.Strings. +// sortStrings sorts xs in place (insertion sort). func sortStrings(xs []string) { for i := 1; i < len(xs); i++ { for j := i; j > 0 && xs[j-1] > xs[j]; j-- { diff --git a/tests/e2e/single/assumed_role_enforcement_test.go b/tests/e2e/single/assumed_role_enforcement_test.go index c2a2961b3..eab9d0648 100644 --- a/tests/e2e/single/assumed_role_enforcement_test.go +++ b/tests/e2e/single/assumed_role_enforcement_test.go @@ -13,9 +13,6 @@ import ( "github.com/stretchr/testify/require" ) -// Identifiers for the assumed-role control-plane enforcement suite. The are- -// prefix keeps them clear of the STS suite (sts-e2e-) and the IAM-roles suite -// (app-role / other-role). const ( areRoleName = "are-e2e-role" arePolicyName = "are-e2e-ec2-describe-regions" @@ -26,29 +23,16 @@ const ( arePolicyDoc = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"ec2:DescribeRegions","Resource":"*"}]}` ) -// runAssumedRoleControlPlaneEnforcement proves the gateway evaluates an -// assumed-role session's managed policies against the control plane — the -// symmetric twin of predastore's assumed-role S3 enforcement. It exercises the -// full SigV4-session → resolve-role-policies → evaluate path through the real -// gateway: -// -// - a zero-policy role is denied (assumability does not imply permissions); -// - after attaching a policy granting ec2:DescribeRegions, the SAME live -// session is permitted — the gateway resolves the role's current managed -// policies at request time, so already-minted credentials pick up the grant. -// -// Owns its own role + policy, so it is safe alongside the other IAM/STS suites, -// but registered sequential so its grant doesn't race a parallel evaluation. +// runAssumedRoleControlPlaneEnforcement proves the gateway evaluates assumed-role +// sessions against managed policies: a zero-policy role is denied, and the same +// live session is allowed once a policy is attached (policies resolved per request). func runAssumedRoleControlPlaneEnforcement(t *testing.T, fix *Fixture) { harness.Phase(t, "Single — Assumed-role control-plane enforcement") adminAccount := iamEnsureAdminAccountID(t, fix) roleARN := iamRoleARN(adminAccount, areRoleName) policyARN := iamPolicyARN(adminAccount, arePolicyName) - // Defensive pre-sweep + teardown — a previous failed run may have left - // fragments. iamDeleteRoleAndProfilesBestEffort detaches the policy before - // deleting the role; the explicit DeletePolicy then removes the managed - // policy itself. + // Pre-sweep in case a previous run left fragments. sweep := func() { iamDeleteRoleAndProfilesBestEffort(fix, areRoleName, nil, policyARN) _, _ = fix.AWS.IAM.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: aws.String(policyARN)}) @@ -56,8 +40,6 @@ func runAssumedRoleControlPlaneEnforcement(t *testing.T, fix *Fixture) { sweep() fix.Harness.RegisterCleanup(sweep) - // Role assumable by anyone in-account (wildcard trust), with NO permission - // policy yet. harness.Step(t, "create-role %q (no permission policy)", areRoleName) _, err := fix.AWS.IAM.CreateRole(&iam.CreateRoleInput{ RoleName: aws.String(areRoleName), @@ -80,8 +62,6 @@ func runAssumedRoleControlPlaneEnforcement(t *testing.T, fix *Fixture) { aws.StringValue(creds.SecretAccessKey), aws.StringValue(creds.SessionToken)) - // Zero-policy role → implicit deny. The session authenticates (SigV4 ASIA - // path) but the resolved role grants nothing. harness.Step(t, "describe-regions with zero-policy role (expect AccessDenied)") harness.ExpectError(t, "AccessDenied", func() error { _, e := sessionCli.EC2.DescribeRegions(&ec2.DescribeRegionsInput{}) @@ -101,9 +81,6 @@ func runAssumedRoleControlPlaneEnforcement(t *testing.T, fix *Fixture) { }) require.NoError(t, err, "attach-role-policy") - // The SAME live session is now permitted: the gateway resolves the role's - // current managed policies per request, so an already-minted session picks - // up the new grant. harness.Step(t, "describe-regions after grant (expect success)") _, err = sessionCli.EC2.DescribeRegions(&ec2.DescribeRegionsInput{}) require.NoError(t, err, "describe-regions must be allowed once the role grants it") diff --git a/tests/e2e/single/baseline_test.go b/tests/e2e/single/baseline_test.go index 22dc8b9d0..57baef75c 100644 --- a/tests/e2e/single/baseline_test.go +++ b/tests/e2e/single/baseline_test.go @@ -19,30 +19,12 @@ import ( "github.com/stretchr/testify/require" ) -// Fresh-install reachability baselines. +// Fresh-install reachability baselines. Tests #1 and #2 pin two independent +// barriers a user hits on a fresh cluster: // -// The rest of the suite calls harness.AuthorizeSSHIngress before every SSH -// probe, so it never exercises the out-of-box default configuration. These -// two tests pin the two independent barriers a real user hits when they -// launch an instance and expect to reach it: -// -// 1. SG barrier (runDefaultSGReachabilityBaseline): an instance in the -// fully-routed default subnet is NOT reachable from outside until an -// ingress rule is added — AWS default-SG semantics admit only same-SG -// members. Routes are fine; the SG is the gate. -// -// 2. Route/egress barrier (runNewVPCEgressBaseline): a brand-new VPC where -// the 0.0.0.0/0 -> IGW route is added to the MAIN route table BEFORE the -// public subnet is created. The subnet implicitly joins the main RT, which -// already carries the IGW route, but CreateSubnet does not recompute the -// per-subnet OVN egress policy — so the subnet's egress LRP is never -// installed and the instance has no datapath to the IGW. The default VPC -// masks this: its main RT is prebaked with the IGW route at bootstrap and -// the vpcd drift loop reconciles any implicit-main subnets, so a subnet -// created in it is reachable regardless of the create-time gap. -// -// Both own all their resources (dedicated SG + instance; #2 owns a whole VPC) -// so they don't perturb the singleton VM or the shared default SG. +// 1. SG barrier: default-SG only admits same-SG members; routes are fine. +// 2. Route/egress gap: 0.0.0.0/0 added to the main RT before the subnet +// exists; CreateSubnet doesn't recompute the per-subnet OVN egress LRP. // tcpReachable reports whether a TCP connect to host:port succeeds within // timeout. An OVN ACL drop yields a dial timeout (no RST); a reject yields @@ -56,12 +38,9 @@ func tcpReachable(host string, port int, timeout time.Duration) bool { return true } -// launchBaselineInstance launches one instance into subnetID with the given -// SGs and registers a terminate-and-wait cleanup so the VM is gone before the -// next test runs. Unlike harness.EnsureInstance (which defers teardown to -// fixture cleanup at suite end), these baselines own their VMs for the test's -// duration only — they must not perturb cluster-VM-count assertions in -// sibling tests. +// launchBaselineInstance launches one instance and registers a terminate-and-wait +// cleanup so the VM is gone before the next test. Not memoized — each baseline +// owns its VM for its own duration only. func launchBaselineInstance(t *testing.T, fix *Fixture, ami, instType, keyName, subnetID string, sgIDs []string) string { t.Helper() sgs := make([]*string, 0, len(sgIDs)) @@ -124,10 +103,8 @@ func sshCapture(tgt harness.SSHTarget, cmd string) (string, error) { return string(out), err } -// instancePublicIP returns the instance's routable public IP. A missing public -// IP is a hard failure: these baselines must exercise the real OVN datapath, -// and the qemu-hostfwd shortcut (which would otherwise stand in) is disabled -// suite-wide because it bypasses SG ACLs / IGW / SNAT and masks regressions. +// instancePublicIP returns the instance's routable public IP. Missing is fatal: +// baselines must exercise the real OVN datapath (hostfwd is disabled suite-wide). func instancePublicIP(t *testing.T, fix *Fixture, instanceID string) string { t.Helper() out, err := fix.AWS.EC2.DescribeInstances(&ec2.DescribeInstancesInput{ @@ -156,8 +133,7 @@ func runDefaultSGReachabilityBaseline(t *testing.T, fix *Fixture) { keyName, keyPath := needKeyPair(t, fix) ami := needAMI(t, fix) - // Dedicated SG with NO ingress rules. CreateSecurityGroup seeds allow-all - // egress, so the SSH return path is unaffected — ingress is the only gate. + // No ingress rules; egress is allow-all by default so only inbound is gated. sgID := harness.EnsureSG(t, fix.Harness, vpcID, "baseline-denysg") harness.Detail(t, "vpc", vpcID, "subnet", subnetID, "sg", sgID) @@ -166,8 +142,7 @@ func runDefaultSGReachabilityBaseline(t *testing.T, fix *Fixture) { pubIP := instancePublicIP(t, fix, instanceID) harness.Detail(t, "instance", instanceID, "public_ip", pubIP) - // Phase A — default-deny SG: tcp/22 must stay blocked. Poll for a window - // so a slow datapath converging to "open" would still be caught. + // Poll for 30s so a slow-converging datapath would still be caught. harness.Step(t, "asserting tcp/22 stays blocked under default-deny SG") deadline := time.Now().Add(30 * time.Second) for time.Now().Before(deadline) { @@ -176,8 +151,6 @@ func runDefaultSGReachabilityBaseline(t *testing.T, fix *Fixture) { time.Sleep(3 * time.Second) } - // Phase B — authorize tcp/22, then a full handshake must succeed. Proves - // the subnet routes to the IGW and only the SG was gating reachability. harness.Step(t, "authorizing tcp/22 ingress, expecting reachability") harness.AuthorizeSSHIngress(t, fix.AWS, sgID) require.Truef(t, trySSHReady(pubIP, 22, keyPath, 3*time.Minute), @@ -206,22 +179,10 @@ func mainRouteTableID(t *testing.T, c *harness.AWSClient, vpcID string) string { return id } -// runNewVPCEgressBaseline stands up a brand-new VPC and reproduces the IGW -// egress gap that the default-VPC path masks. The bug is order-sensitive: the -// 0.0.0.0/0 -> IGW route is added to the MAIN route table BEFORE the public -// subnet exists, so when the subnet is created it implicitly joins a main RT -// that already carries the route. CreateSubnet does not recompute per-subnet -// egress, so the subnet's OVN egress LRP is never installed and the instance -// has no datapath to the IGW. With an open SG, routing is the only remaining -// barrier, so an unreachable instance pins the gap. -// -// runVPCSubnetE2E does NOT catch this: it uses a custom RT, associates the -// subnet, then CreateRoute — so the route mutation enumerates the already- -// associated subnet and installs the LRP. The default-VPC path doesn't catch -// it either (route prebaked at bootstrap + drift-loop reconcile). -// -// PoolMode-gated like runVPCSubnetE2E: a fresh VPC needs OVN IPAM to hand the -// instance a routable public IP, which dev_networking single-node lacks. +// runNewVPCEgressBaseline reproduces the IGW egress gap when a 0.0.0.0/0 route +// is added to the main RT before the subnet is created. The subnet implicitly +// joins the RT but CreateSubnet doesn't install the per-subnet OVN egress LRP, +// leaving the instance with no datapath to the IGW. Pool-mode gated. func runNewVPCEgressBaseline(t *testing.T, fix *Fixture) { if !fix.PoolMode { t.Skip("fresh-VPC egress baseline requires pool-mode networking") @@ -329,12 +290,9 @@ func runNewVPCEgressBaseline(t *testing.T, fix *Fixture) { assert.Containsf(t, idOut, "ec2-user", "ssh id in fresh-VPC subnet\n%s", idOut) } -// runSameSGComms launches two instances in the default SG (plus a dedicated -// runner-SSH SG so the runner can shell into one) and asserts that one can -// ICMP-ping the other over the VPC. ICMP between them is permitted only by the -// default SG's self-reference rule, so success proves that rule is enforced on -// the OVN datapath — the east-west counterpart to the external-reach baseline. -// No default resource is mutated (only the dedicated runner SG is authorized). +// runSameSGComms verifies that two instances in the default SG can ICMP-ping +// each other (permitted by the default SG's self-reference rule) while a +// separate runner SG handles tcp/22 from outside. No default resource mutated. func runSameSGComms(t *testing.T, fix *Fixture) { harness.Phase(t, "Single — Baseline: same default-SG instances communicate") @@ -343,8 +301,6 @@ func runSameSGComms(t *testing.T, fix *Fixture) { keyName, keyPath := needKeyPair(t, fix) ami := needAMI(t, fix) - // Dedicated SG opens tcp/22 only, so ICMP between guests still depends on - // the default SG's self-ingress — keeps the signal unambiguous. runnerSG := harness.EnsureSG(t, fix.Harness, vpcID, "baseline-runnersg") harness.AuthorizeSSHIngress(t, fix.AWS, runnerSG) harness.Detail(t, "vpc", vpcID, "default_sg", defSGID, "runner_sg", runnerSG) diff --git a/tests/e2e/single/helpers_test.go b/tests/e2e/single/helpers_test.go index 73d0b3d7b..752ae20c0 100644 --- a/tests/e2e/single/helpers_test.go +++ b/tests/e2e/single/helpers_test.go @@ -11,13 +11,8 @@ import ( "github.com/stretchr/testify/require" ) -// Package-local prereq bootstrappers. Each wraps the matching -// harness.Discover* / harness.Ensure* call so a top-level Test* in this -// package can self-bootstrap whatever the phase under test depends on, -// without threading state through a shared Fixture struct. -// -// All wrapped helpers are memoized by the underlying harness.Fixture, so -// repeated need* calls from different tests hit the cache. +// Package-local bootstrappers wrapping harness.Discover*/Ensure* so each Test* +// can self-bootstrap its prerequisites. Results are memoized by harness.Fixture. // needAZ returns the discovered default availability zone. func needAZ(t *testing.T, fix *Fixture) string { @@ -39,16 +34,12 @@ func needAMI(t *testing.T, fix *Fixture) string { } // needKeyPair returns the memoized primary EC2 key pair name + PEM path. -// Routes through harness.EnsureKeyPair so the PEM lands in fix.TmpDir and -// is cleaned up at fixture teardown. func needKeyPair(t *testing.T, fix *Fixture) (name, pemPath string) { t.Helper() return harness.EnsureKeyPair(t, fix.Harness, fix.TmpDir) } -// customAMIName is the well-known name the bash driver bakes when exercising -// the CreateImage flow. Lifted out as a constant so both phase5e (creator) -// and phase8 8o (duplicate-name negative) reference the same literal. +// customAMIName is the well-known AMI name used by CreateImage tests. const customAMIName = "e2e-custom-ami" // needCustomAMI returns the memoized CreateImage-built custom AMI ID, @@ -76,9 +67,7 @@ func ensureCustomAMI(t *testing.T, fix *Fixture, sourceInstanceID, name, desc st } // needInstance bootstraps the suite's primary running instance and returns -// the running *ec2.Instance plus its root volume ID. Idempotent via -// harness.EnsureInstance — the first caller pays the launch cost, every -// other caller hits the cache. +// the *ec2.Instance plus its root volume ID. Idempotent via harness.EnsureInstance. func needInstance(t *testing.T, fix *Fixture) (inst *ec2.Instance, rootVolumeID string) { t.Helper() instType, _ := needInstanceTypeArch(t, fix) @@ -105,8 +94,7 @@ func needInstance(t *testing.T, fix *Fixture) (inst *ec2.Instance, rootVolumeID require.NotEmpty(t, descOut.Reservations[0].Instances, "no instances for %s", instanceID) inst = descOut.Reservations[0].Instances[0] - // Root volume: the BDM whose DeviceName matches RootDeviceName. Falls back - // to the first BDM if RootDeviceName is empty (some drivers don't echo it). + // Match BDM by RootDeviceName; fall back to first BDM if empty. rootDev := aws.StringValue(inst.RootDeviceName) for _, bdm := range inst.BlockDeviceMappings { if rootDev != "" && aws.StringValue(bdm.DeviceName) != rootDev { diff --git a/tests/e2e/single/iam_helpers_test.go b/tests/e2e/single/iam_helpers_test.go index a16a8a523..77d5070a2 100644 --- a/tests/e2e/single/iam_helpers_test.go +++ b/tests/e2e/single/iam_helpers_test.go @@ -10,15 +10,9 @@ import ( "github.com/aws/aws-sdk-go/service/iam" ) -// IAM state shared across the IAM phase tests. Each phase test is its own -// top-level Test*, so cross-phase state (users, primary access keys, and the -// derived admin account ID) is memoized at package scope behind sync.Once. -// LIFO cleanup is registered against fix.Harness so it runs at fixture -// teardown rather than at end of the first caller's subtest. -// -// All helpers tolerate "already exists" outcomes so a caller that races a -// phase test which is itself exercising create-then-delete (e.g. IAM1) still -// converges on a populated state. +// IAM state shared across the IAM phase tests, memoized at package scope via +// sync.Once. Cleanup is registered on fix.Harness to run at fixture teardown. +// All helpers tolerate "already exists" outcomes so concurrent phases converge. var ( iamAliceOnce sync.Once @@ -155,8 +149,7 @@ func iamEnsureAdminAccountID(t *testing.T, fix *Fixture) string { return iamAdminAccountID } -// iamCreateUserIdempotent creates a user, swallowing EntityAlreadyExists so -// the helper can run after a phase test that already created the user. +// iamCreateUserIdempotent creates a user, treating EntityAlreadyExists as success. func iamCreateUserIdempotent(fix *Fixture, name, path string) error { input := &iam.CreateUserInput{UserName: aws.String(name)} if path != "" { @@ -166,8 +159,6 @@ func iamCreateUserIdempotent(fix *Fixture, name, path string) error { if err == nil { return nil } - // EntityAlreadyExists: user came from a parallel phase test. Treat as - // success — we just need the named user to exist. if iamIsEntityAlreadyExists(err) { return nil } diff --git a/tests/e2e/single/iam_test.go b/tests/e2e/single/iam_test.go index 5e6ac393e..31ceda871 100644 --- a/tests/e2e/single/iam_test.go +++ b/tests/e2e/single/iam_test.go @@ -323,9 +323,8 @@ func runIAMAccessKeyLifecycle(t *testing.T, fix *Fixture) { func runIAMUserAuthentication(t *testing.T, fix *Fixture) { harness.Phase(t, "Single — IAM User Authentication") // Daemon does not yet honour active-key signatures created via the IAM - // API — every scoped DescribeInstances returns 403. Skip-gate until - // the handler lands; mulga-siv-100 tracks the daemon-side work. - t.Skip("daemon IAM signature/principal lookup not implemented — mulga-siv-100") + // API — every scoped DescribeInstances returns 403. + t.Skip("daemon IAM signature/principal lookup not implemented") aliceKeyID, aliceSecret := iamEnsureAliceKey(t, fix) harness.Step(t, "scoped client (alice) describe-instances — active key OK") @@ -559,10 +558,9 @@ func runIAMPolicyCRUD(t *testing.T, fix *Fixture) { // bypass / prefix wildcard / FullAdmin), DetachUserPolicy. func runIAMPolicyAttachmentEnforcement(t *testing.T, fix *Fixture) { harness.Phase(t, "Single — IAM Policy Attachment IAM Phase 5 — Policy Attachment & Enforcement Enforcement") - // Enforcement depends on scoped-credential signing (Phase 3) which the - // daemon doesn't honour yet — mulga-siv-100. Skip-gate the whole phase - // until the upstream gap closes. - t.Skip("daemon IAM scoped-credential enforcement not implemented — mulga-siv-100") + // Enforcement depends on scoped-credential signing which the daemon + // doesn't honour yet. + t.Skip("daemon IAM scoped-credential enforcement not implemented") adminAccount := iamEnsureAdminAccountID(t, fix) aliceKeyID, aliceSecret := iamEnsureAliceKey(t, fix) bobKeyID, bobSecret := iamEnsureBobKey(t, fix) @@ -733,10 +731,9 @@ func runIAMPolicyAttachmentEnforcement(t *testing.T, fix *Fixture) { // delete the policy is gone (NoSuchEntity). func runIAMPolicyLifecycle(t *testing.T, fix *Fixture) { harness.Phase(t, "Single — IAM Policy Lifecycle (Detach IAM Phase 6 — Policy Lifecycle (Detach & Delete) Delete)") - // DetachUserPolicy returns 404 NoSuchEntity even for attachments - // confirmed by Phase 5 — daemon's attachment ledger isn't persisted - // across the API boundary. Skip-gate; mulga-siv-100 tracks the fix. - t.Skip("daemon DetachUserPolicy not implemented — mulga-siv-100") + // DetachUserPolicy returns 404 NoSuchEntity even for confirmed attachments — + // the daemon's attachment ledger isn't persisted across the API boundary. + t.Skip("daemon DetachUserPolicy not implemented") adminAccount := iamEnsureAdminAccountID(t, fix) aliceKeyID, aliceSecret := iamEnsureAliceKey(t, fix) diff --git a/tests/e2e/single/tests_test.go b/tests/e2e/single/tests_test.go index e3a49a2cd..032369957 100644 --- a/tests/e2e/single/tests_test.go +++ b/tests/e2e/single/tests_test.go @@ -5,38 +5,13 @@ package single import "testing" // Top-level Test* entry points. Each delegates to a runX function in the -// matching _test.go file. Names are number-free (TestX) so isolated -// runs via `go test -run TestKeyPairs` are stable. +// matching _test.go file. Every run* self-bootstraps prerequisites so +// `go test -run TestX` works in isolation. // -// Every run* function self-bootstraps prerequisites via harness.Discover* / -// harness.Ensure* (or the package-local need* / iamEnsure* helpers), so a -// targeted `go test -run TestX` invocation works in isolation without -// relying on a sibling Test* having stashed state. -// -// Parallelism (mulga-siv-128): -// -// Bucket #1 (parallel): read-only or own-everything Tests below call -// t.Parallel(). They share the package fixture but never mutate the -// singleton EC2 instance, default VPC, or default SG. -// -// Sequential (no t.Parallel — would race the singleton VM or shared state): -// - TestClusterStatsCLI : asserts baseline cluster state -// (0/ CPU, "No VMs found"); must run before any launches. -// - TestInstanceLaunch : first sequential — boots singleton. -// - TestInstanceClusterStats / Metadata -// / SSHProbe / ConsoleOutput : read singleton. -// - TestVolumeLifecycle / VolumeStatus : attach to singleton. -// - TestSnapshotLifecycle / -// SnapshotBackedLaunch / CreateImage : snapshot singleton. -// - TestSecurityGroupEgress : mutates default SG. -// - TestStopStart / AttachToStoppedError -// / ModifyInstanceAttribute / -// RebootInstance / RunInstancesMultiCount : mutate singleton state. -// - TestNegativeErrorPaths : sub-tests already parallel inside. -// - TestNATGateway / TestSGToSGDatapath : share EIP pool. -// - All TestIAM* : share IAM policy state via the -// package-scoped IAM fixture; sub-tests inside each already parallel. -// - TestFinalClusterStats : final sanity gate. +// Parallel bucket (t.Parallel): read-only or own-everything tests that never +// mutate the singleton instance, default VPC, or default SG. +// Sequential: tests that boot, mutate, or snapshot the singleton VM; IAM tests +// that share package-scoped IAM state; TestFinalClusterStats (last gate). // --- Bucket #1: parallel-safe --- @@ -87,17 +62,13 @@ func TestRouteTableValidation(t *testing.T) { // --- Sequential: singleton VM lifecycle --- -// TestClusterStatsCLI runs before TestInstanceLaunch because it asserts -// baseline cluster state (0/ CPU used, "No VMs found") that doesn't hold -// once the singleton instance is up. +// TestClusterStatsCLI asserts 0-VM baseline state before any instance launches. func TestClusterStatsCLI(t *testing.T) { runClusterStatsCLI(t, requireSingleNodeFixture(t)) } -// Fresh-install reachability baselines run right after the 0-VM cluster-stats -// gate and before the singleton launch. They own all mutable resources -// (dedicated SG / subnet / self-cleaning instance) and never mutate the -// default SG, subnet, or route table. +// TestDefaultSGReachabilityBaseline, TestNewVPCEgressBaseline, and TestSameSGComms +// own all mutable resources and run before the singleton launch. func TestDefaultSGReachabilityBaseline(t *testing.T) { runDefaultSGReachabilityBaseline(t, requireSingleNodeFixture(t)) } @@ -218,51 +189,39 @@ func TestIAMCleanup(t *testing.T) { runIAMCleanup(t, requireSingleNodeFixture(t)) } -// TestIAMRolesAndProfiles + TestIAMInstanceProfileAssociation form the -// companion-plan suite (docs/development/archive/feature/iam-roles-v1.md and -// docs/development/feature/iam-roles-v1-ec2.md). Sequential: the second test -// recreates the same role/profile names the first tore down, so concurrent -// runs would race the EntityAlreadyExists / NoSuchEntity boundaries. +// TestIAMRolesAndProfiles + TestIAMInstanceProfileAssociation are sequential: +// the second recreates role/profile names the first tore down. func TestIAMRolesAndProfiles(t *testing.T) { runIAMRolesAndProfiles(t, requireSingleNodeFixture(t)) } -// TestIAMInstanceProfileAssociation also briefly mutates the singleton VM -// (Associate/Disassociate), so it can't share the parallel bucket with -// instance-mutation tests. +// TestIAMInstanceProfileAssociation mutates the singleton VM, so it cannot +// share the parallel bucket. func TestIAMInstanceProfileAssociation(t *testing.T) { runIAMInstanceProfileAssociation(t, requireSingleNodeFixture(t)) } -// TestSTSAssumeRoleAndGetCallerIdentity exercises the STS v1 surface. Owns -// its own role, so safe alongside the IAM Roles tests above, but sequential -// so trust-policy mutations don't race a parallel AssumeRole. +// TestSTSAssumeRoleAndGetCallerIdentity is sequential to avoid racing trust-policy +// mutations against a parallel AssumeRole. func TestSTSAssumeRoleAndGetCallerIdentity(t *testing.T) { runSTS(t, requireSingleNodeFixture(t)) } -// TestAssumedRoleControlPlaneEnforcement verifies the gateway evaluates an -// assumed-role session's managed policies against the control plane: a -// zero-policy role is denied, and the same live session is permitted once a -// granting policy is attached. Owns its own role + policy, but sequential so -// the mid-test grant doesn't race a parallel evaluation. +// TestAssumedRoleControlPlaneEnforcement verifies a zero-policy assumed-role +// is denied and permitted once a policy is attached. Sequential to avoid +// racing the mid-test grant. func TestAssumedRoleControlPlaneEnforcement(t *testing.T) { runAssumedRoleControlPlaneEnforcement(t, requireSingleNodeFixture(t)) } -// TestIMDS exercises the host-served IMDSv2 surface end-to-end: token -// issuance, the v2-only stance, the metadata surface, the -// instance-role credential path + wire round-trip, the per-subnet L2 localport -// datapath (ovn-trace round-trip), and cross-VPC source-IP isolation. Owns its -// own role/profile + a second VPC, but sequential: it launches profile-bound VMs -// and a fresh VPC, so it must not race the singleton-mutation tests above. +// TestIMDS exercises IMDSv2 end-to-end: token issuance, metadata surface, +// instance-role credentials, OVN datapath, and cross-VPC isolation. Sequential +// because it launches profile-bound VMs and creates a fresh VPC. func TestIMDS(t *testing.T) { runIMDS(t, requireSingleNodeFixture(t)) } -// TestFinalClusterStats runs as the last sequential test (parallel bucket -// runs afterwards, but those Tests are read-only / own-everything so a -// later cluster-stats snapshot would still be valid). +// TestFinalClusterStats runs as the last sequential test. func TestFinalClusterStats(t *testing.T) { runFinalClusterStats(t, requireSingleNodeFixture(t)) } diff --git a/tests/e2e/single/vpc_test.go b/tests/e2e/single/vpc_test.go index 8a9752d68..f6317e28c 100644 --- a/tests/e2e/single/vpc_test.go +++ b/tests/e2e/single/vpc_test.go @@ -197,7 +197,7 @@ func runVPCSubnetE2E(t *testing.T, fix *Fixture) { // with this rule by admin init, but a newly-created VPC has not. // Without this step Phase 8b SSH times out via an ACL drop between OVN // tables 13 (DNAT) and 17 (NAT commit) even when every datapath barrier - // is satisfied (tracking mulga-siv-111). + // is satisfied. harness.Step(t, "describe-security-groups (default) vpc=%s", vpcID) sgOut, err := c.EC2.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{ Filters: []*ec2.Filter{ @@ -247,7 +247,7 @@ func runVPCSubnetE2E(t *testing.T, fix *Fixture) { // --- SSH + external connectivity --------------------------------------- // Use the non-fatal probe so we can dump VPC/IGW datapath diagnostics // before Fatal. Same 3min budget — fresh-VPC unreachability beyond that - // is a real product bug (mulga-siv-105 + mulga-siv-111), not test flake. + // is a real product bug, not test flake. if !trySSHReady(pubIP, 22, keyPath, 3*time.Minute) { harness.DumpVPCFlowDiagnostics(t, c, instID, fmt.Sprintf("Phase 8b SSH timeout — vpc=%s igw=%s pub=%s", vpcID, igwID, pubIP), @@ -256,7 +256,7 @@ func runVPCSubnetE2E(t *testing.T, fix *Fixture) { LogicalIP: privIP, ArtifactDir: fix.ArtifactDir(t), }) - t.Fatalf("SSH handshake %s:22 never completed within 3min (see diagnostics above; tracking mulga-siv-111)", pubIP) + t.Fatalf("SSH handshake %s:22 never completed within 3min (see diagnostics above)", pubIP) } tgt := harness.SSHTarget{User: "ec2-user", Host: pubIP, Port: 22, KeyPath: keyPath}