gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-15 17:11:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: IMDS v1 (#266)

Browse Source
This commit is contained in:
Julian Sommer
2026-06-04 13:32:44 +10:00
committed by GitHub
parent f14fb64562
commit bcb49cd771
74 changed files with 6127 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build/systemd/spinifex-vpcd.service
+25 -15
View File
@@ -16,25 +16,34 @@ OOMScoreAdjust=-500
Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/
# Filesystem access — NoNewPrivileges omitted (vpcd uses sudo for ovs-vsctl, ovn-nbctl, systemctl)
ProtectSystem=full
# vpcd MUST share the HOST mount namespace. IMDS binds one listener per VPC inside
# a per-VPC netns entered in-process via setns(CLONE_NEWNET); the /run/netns/<vpc>
# bind-mount that `ip netns add` creates must be host-visible (for diagnostics) and
# must survive a vpcd restart (so the listener can rebind). ANY filesystem-sandbox
# directive — ProtectSystem / ProtectHome / PrivateTmp / ProtectKernel* /
# ProtectControlGroups / ProtectProc / Read{Only,Write}Paths — forks a PRIVATE
# mount namespace that traps that bind-mount inside vpcd, leaving the host a stale
# handle that fails setns(2) with EINVAL (CI runs 26920490454, 26922587471). Those
# directives are therefore omitted by design, and so is the MountFlags=shared /
# spinifex-netns.service propagation workaround they required.
#
# Boundary that REMAINS: DAC (User=spinifex-vpcd) plus the seccomp / capability /
# namespace / address-family restrictions below. The durable fix that restores
# filesystem sandboxing AND drops CAP_SYS_ADMIN is the process-per-netns model
# (`ip netns exec` helper instead of in-process setns) — see
# docs/development/improvements/imds-l2-datapath.md Phase 5.
PrivateDevices=no
ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/ca.pem
ReadWritePaths=/var/lib/spinifex/vpcd /var/log/spinifex
# Hardening
ProtectHome=yes
PrivateTmp=yes
# Hardening that does NOT fork a mount namespace (kept)
RestrictSUIDSGID=no
LockPersonality=yes
RestrictRealtime=yes
RestrictNamespaces=yes
ProtectClock=yes
ProtectKernelTunables=yes
ProtectKernelModules=yes
ProtectKernelLogs=yes
ProtectControlGroups=yes
ProtectProc=invisible
# IMDS enters the per-VPC netns via setns(CLONE_NEWNET); the `ip -n <netns>` /
# `ip netns exec` plumbing also unshares a mount namespace (CLONE_NEWNS, to give
# the netns its own /sys), so `mnt` must be allowlisted alongside `net` or the
# first in-netns command fails with EPERM.
RestrictNamespaces=net mnt
SystemCallArchitectures=native
MemoryDenyWriteExecute=no
# AF_PACKET: required by the in-process DHCP client (github.com/insomniacslk/dhcp
@@ -44,9 +53,10 @@ RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK AF_PACKET
# CAP_NET_RAW: open AF_PACKET sockets.
# CAP_NET_BIND_SERVICE: covers unicast RENEW to UDP :68.
# CAP_NET_ADMIN: already used for ovs-vsctl / ip link wiring on the bridge.
# CAP_SYS_ADMIN: setns(CLONE_NEWNET) into per-VPC IMDS netns to bind the listener.
# Note, disabled for now, sudo issue with ovs
AmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID
CapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID
AmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID CAP_SYS_ADMIN
CapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID CAP_SYS_ADMIN
NoNewPrivileges=no
ProtectHostname=yes
KeyringMode=private
docs/COMMANDS.md
+109 -158
View File
@@ -8,162 +8,102 @@ Platform management commands not exposed via the AWS gateway API. These are CLI-
Service lifecycle commands for starting, stopping, and checking status of all Spinifex cluster services. Each service subcommand supports `start`, `stop`, and `status` operations.
| Command | Flags | Prerequisites | Basic Logic | Status |
|---------|-------|---------------|-------------|--------|
| `spx service predastore start` | `--port` (default: 8443), `--host` (default: 0.0.0.0), `--base-path`, `--config-path`, `--debug`, `--tls-cert`, `--tls-key`, `--backend` (distributed/filesystem, default: distributed), `--node-id` (default: 0), `--pprof`, `--pprof-output` | TLS cert/key, base-path, config-path required | Creates predastore service instance with S3-compatible storage backend → starts service | **DONE** |
| `spx service predastore stop` | — | Predastore must be running | Stops the predastore service | **DONE** |
| `spx service predastore status` | — | None | Reports predastore service status | **DONE** |
| `spx service viperblock start` | `--s3-host` (default: 0.0.0.0:8443), `--s3-bucket` (default: predastore), `--s3-region` (default: ap-southeast-2), `--plugin-path` (auto-detected via `nbdkit --dump-config plugindir`; typically `/usr/lib/x86_64-linux-gnu/nbdkit/plugins/nbdkit-viperblock-plugin.so` on amd64, overridable via `SPINIFEX_VIPERBLOCK_PLUGIN_PATH` in `/etc/spinifex/systemd.env`) | `--config` required, plugin-path must exist on disk | Loads cluster config → connects to NATS and Predastore → starts viperblock block storage service with NBD plugin | **DONE** |
| `spx service viperblock stop` | — | Viperblock must be running | Stops the viperblock service | **DONE** |
| `spx service viperblock status` | — | None | Reports viperblock service status | **DONE** |
| `spx service nats start` | `--port` (default: 4222), `--host` (default: 0.0.0.0), `--debug`, `--data-dir`, `--jetstream` | None | Starts embedded NATS server with optional JetStream | **DONE** |
| `spx service nats stop` | — | NATS must be running | Stops the NATS service | **DONE** |
| `spx service nats status` | — | None | Reports NATS service status | **DONE** |
| `spx service spinifex start` | `--wal-dir` | `--config` required | Loads cluster config → starts spinifex daemon (VM orchestration, NATS subscriptions, health endpoint) | **DONE** |
| `spx service spinifex stop` | — | Spinifex daemon must be running | Stops the spinifex daemon service | **DONE** |
| `spx service spinifex status` | — | None | Reports spinifex daemon service status | **DONE** |
| `spx service awsgw start` | `--host` (default: 0.0.0.0:9999), `--tls-cert`, `--tls-key`, `--debug` | `--config` required | Loads cluster config → starts AWS-compatible gateway with SigV4 auth, IAM policy enforcement, TLS | **DONE** |
| `spx service awsgw stop` | — | AWS gateway must be running | Stops the AWS gateway service | **DONE** |
| `spx service awsgw status` | — | None | Reports AWS gateway service status | **DONE** |
| `spx service spinifex-ui start` | `--port` (default: 3000), `--host` (default: 0.0.0.0), `--tls-cert`, `--tls-key` | None | Starts embedded web UI server serving the React frontend. Aliases: `ui`, `spinifexui` | **DONE** |
| `spx service spinifex-ui stop` | — | spinifex-ui must be running | Stops the spinifex-ui service | **DONE** |
| `spx service spinifex-ui status` | — | None | Reports spinifex-ui service status | **DONE** |
| `spx service vpcd start` | — | `--config` required, OVN/OVS installed | Loads cluster config → starts VPC daemon (subscribes to `vpc.*` NATS events, translates to OVN logical switches/ports/routers) | **DONE** |
| `spx service vpcd stop` | — | vpcd must be running | Stops the vpcd service | **DONE** |
| `spx service vpcd status` | — | None | Reports vpcd service status | **DONE** |
| Command | Flags | Description |
|---------|-------|-------------|
| `spx service predastore start` | `--port` (default: 8443), `--host` (default: 0.0.0.0), `--base-path`, `--config-path`, `--debug`, `--tls-cert`, `--tls-key`, `--backend` (distributed/filesystem, default: distributed), `--node-id` (default: 0), `--pprof`, `--pprof-output` | Creates predastore service instance with S3-compatible storage backend → starts service |
| `spx service predastore stop` | — | Stops the predastore service |
| `spx service predastore status` | — | Reports predastore service status |
| `spx service viperblock start` | `--s3-host` (default: 0.0.0.0:8443), `--s3-bucket` (default: predastore), `--s3-region` (default: ap-southeast-2), `--plugin-path` (auto-detected via `nbdkit --dump-config plugindir`; typically `/usr/lib/x86_64-linux-gnu/nbdkit/plugins/nbdkit-viperblock-plugin.so` on amd64, overridable via `SPINIFEX_VIPERBLOCK_PLUGIN_PATH` in `/etc/spinifex/systemd.env`) | Loads cluster config → connects to NATS and Predastore → starts viperblock block storage service with NBD plugin |
| `spx service viperblock stop` | — | Stops the viperblock service |
| `spx service viperblock status` | — | Reports viperblock service status |
| `spx service nats start` | `--port` (default: 4222), `--host` (default: 0.0.0.0), `--debug`, `--data-dir`, `--jetstream` | Starts embedded NATS server with optional JetStream |
| `spx service nats stop` | — | Stops the NATS service |
| `spx service nats status` | — | Reports NATS service status |
| `spx service spinifex start` | `--wal-dir` | Loads cluster config → starts spinifex daemon (VM orchestration, NATS subscriptions, health endpoint) |
| `spx service spinifex stop` | — | Stops the spinifex daemon service |
| `spx service spinifex status` | — | Reports spinifex daemon service status |
| `spx service awsgw start` | `--host` (default: 0.0.0.0:9999), `--tls-cert`, `--tls-key`, `--debug` | Loads cluster config → starts AWS-compatible gateway with SigV4 auth, IAM policy enforcement, TLS |
| `spx service awsgw stop` | — | Stops the AWS gateway service |
| `spx service awsgw status` | — | Reports AWS gateway service status |
| `spx service spinifex-ui start` | `--port` (default: 3000), `--host` (default: 0.0.0.0), `--tls-cert`, `--tls-key` | Starts embedded web UI server serving the React frontend. Aliases: `ui`, `spinifexui` |
| `spx service spinifex-ui stop` | — | Stops the spinifex-ui service |
| `spx service spinifex-ui status` | — | Reports spinifex-ui service status |
| `spx service vpcd start` | — | Loads cluster config → starts VPC daemon (subscribes to `vpc.*` NATS events, translates to OVN logical switches/ports/routers) |
| `spx service vpcd stop` | — | Stops the vpcd service |
| `spx service vpcd status` | — | Reports vpcd service status |
### Cluster Inspection
Operational commands for inspecting cluster state. These fan out NATS requests to all nodes and aggregate responses.
| Command | Flags | Prerequisites | Basic Logic | Output Columns | Status |
|---------|-------|---------------|-------------|----------------|--------|
| `spx get nodes` | `--timeout` (default: 3s) | Cluster must be running (NATS) | Loads config → publishes to `spinifex.node.status` fan-out topic → collects responses within timeout → merges config-known nodes with NATS responders → nodes that don't respond shown as `NotReady` | NAME, STATUS, IP, REGION, AZ, UPTIME, VMs, SERVICES | **DONE** |
| `spx get vms` | `--timeout` (default: 3s) | Cluster must be running (NATS) | Publishes to `spinifex.node.vms` fan-out topic → collects VM info from all nodes → sorts by node then instance ID → prints table. Alias: `spx get instances` | INSTANCE, STATUS, TYPE, VCPU, MEM, NODE, IP, AGE | **DONE** |
| Command | Flags | Description |
|---------|-------|-------------|
| `spx get nodes` | `--timeout` (default: 3s) | Loads config → publishes to `spinifex.node.status` fan-out topic → collects responses within timeout → merges config-known nodes with NATS responders → nodes that don't respond shown as `NotReady` |
| `spx get vms` | `--timeout` (default: 3s) | Publishes to `spinifex.node.vms` fan-out topic → collects VM info from all nodes → sorts by node then instance ID → prints table. Alias: `spx get instances` |
### Resource Monitoring
| Command | Flags | Prerequisites | Basic Logic | Output | Status |
|---------|-------|---------------|-------------|--------|--------|
| `spx top nodes` | `--timeout` (default: 3s) | Cluster must be running (NATS) | Publishes to `spinifex.node.status` fan-out topic → collects CPU/memory usage per node → aggregates instance type capacity across all nodes → prints two tables: per-node resource usage and cluster-wide instance type availability | Table 1: NAME, CPU (used/total), MEM (used/total), GPU (used/total), VMs — GPU column shows `allocated/total` when passthrough is enabled, `0/count*` when the node is GPU-capable but passthrough is disabled, or `-` when no GPU is present. Table 2: INSTANCE TYPE, AVAILABLE, VCPU, MEMORY | **DONE** |
| Command | Flags | Description |
|---------|-------|-------------|
| `spx top nodes` | `--timeout` (default: 3s) | Publishes to `spinifex.node.status` fan-out topic → collects CPU/memory usage per node → aggregates instance type capacity across all nodes → prints two tables: per-node resource usage and cluster-wide instance type availability |
### Cluster Initialization
| Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
|---------|-------|---------------|-------------|------------|--------|
| `spx admin init` | `--nodes`, `--node`, `--bind`, `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--services`, `--formation-timeout`, `--token-ttl`, `--force` | None (first-time setup) | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → generates join token (written to `join-token` file, displayed in join command) → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/` | 1. Init creates all config files<br>2. Root credentials printed once<br>3. master.key is 32 bytes, mode 0600<br>4. bootstrap.json consumed on first start<br>5. `--force` re-initializes existing config<br>6. AWS CLI profile `spx` auto-configured<br>7. Multi-node init generates join token and writes to `<config-dir>/join-token` | **DONE** |
| `spx admin join` | `--host` (required), `--node` (required), `--token` (required), `--bind`, `--port`, `--region`, `--az`, `--cluster-bind`, `--cluster-routes`, `--data-dir`, `--services` | Leader node must be running | Connects to leader node with join token (Authorization: Bearer header) → retrieves cluster configuration → configures local node to join cluster and participate in distributed operations | 1. Join existing cluster<br>2. Missing host (error)<br>3. Missing node name (error)<br>4. Missing token (error)<br>5. Invalid token (401)<br>6. Expired token (401) | **DONE** |
| Command | Flags | Description |
|---------|-------|-------------|
| `spx admin init` | `--nodes`, `--node`, `--bind`, `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--services`, `--formation-timeout`, `--token-ttl`, `--force` | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → generates join token (written to `join-token` file, displayed in join command) → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/` |
| `spx admin join` | `--host` (required), `--node` (required), `--token` (required), `--bind`, `--port`, `--region`, `--az`, `--cluster-bind`, `--cluster-routes`, `--data-dir`, `--services` | Connects to leader node with join token (Authorization: Bearer header) → retrieves cluster configuration → configures local node to join cluster and participate in distributed operations |
### Version
| Command | Flags | Prerequisites | Basic Logic | Status |
|---------|-------|---------------|-------------|--------|
| `spx version` | — | None | Prints Spinifex version, commit hash, OS, and architecture (populated via build-time ldflags) | **DONE** |
| Command | Flags | Description |
|---------|-------|-------------|
| `spx version` | — | Prints Spinifex version, commit hash, OS, and architecture (populated via build-time ldflags) |
### Cluster Operations
| Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
|---------|-------|---------------|-------------|------------|--------|
| `spx admin cluster shutdown` | `--force` (shutdown even if nodes don't respond), `--timeout` (max wait per phase, default 120s), `--dry-run` (print phase plan without executing) | Cluster must be running | Performs coordinated, phased shutdown of entire cluster. Phases execute in order: GATE (stop API/UI) → DRAIN (stop VMs) → STORAGE (stop viperblock) → PERSIST (stop predastore) → INFRA (stop NATS/daemon). Each phase waits for all nodes to ACK before proceeding. Uses JetStream state tracking. | 1. Shutdown running cluster<br>2. All nodes stop cleanly<br>3. Force shutdown with unresponsive nodes<br>4. Dry-run prints plan | **DONE** |
| Command | Flags | Description |
|---------|-------|-------------|
| `spx admin cluster shutdown` | `--force` (shutdown even if nodes don't respond), `--timeout` (max wait per phase, default 120s), `--dry-run` (print phase plan without executing) | Performs coordinated, phased shutdown of entire cluster. Phases execute in order: GATE (stop API/UI) → DRAIN (stop VMs) → STORAGE (stop viperblock) → PERSIST (stop predastore) → INFRA (stop NATS/daemon). Each phase waits for all nodes to ACK before proceeding. Uses JetStream state tracking. |
### Certificate Management
| Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
|---------|-------|---------------|-------------|------------|--------|
| `spx admin cert renew` | `--extra-ip` (additional IPs for SANs), `--extra-dns` (additional DNS names for SANs) | Existing CA from `spx admin init` | Reads existing CA → regenerates server certificate with all current network interface IPs and machine hostname in SANs → writes new cert. Use after adding a new network interface or changing IP addresses. | 1. Renew with auto-detected IPs<br>2. Renew with extra IPs<br>3. Renew with extra DNS names | **DONE** |
| Command | Flags | Description |
|---------|-------|-------------|
| `spx admin cert renew` | `--extra-ip` (additional IPs for SANs), `--extra-dns` (additional DNS names for SANs) | Reads existing CA → regenerates server certificate with all current network interface IPs and machine hostname in SANs → writes new cert. Use after adding a new network interface or changing IP addresses. |
### Upgrade Management
| Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
|---------|-------|---------------|-------------|------------|--------|
| `spx admin upgrade` | `--yes` (apply without prompting), `--config-dir` (persistent, default: `~/spinifex/config`), `--spinifex-dir` (persistent, default: `~/spinifex`) | Existing installation (`spinifex.toml` must exist in config dir) | Reads current config versions from registry → computes pending config migrations (from→to per target) → prints version summary and pending list → prompts for confirmation unless `--yes` → runs `migrate.DefaultRegistry.RunAllConfig()` to apply migrations to config files. Intended for upgrades between Spinifex versions. Operators can skip migrations during install by setting `INSTALL_SPINIFEX_SKIP_MIGRATE=1`, then run `spx admin upgrade` manually to review and apply. After completion, services must be restarted with `sudo systemctl restart spinifex.target`. Invoked non-interactively by `scripts/setup.sh` with `--yes`. | 1. No installation present (error, suggests `spx admin init`)<br>2. No pending migrations (prints "No pending config migrations")<br>3. Pending migrations with interactive confirmation (y/yes applies, anything else aborts)<br>4. Pending migrations with `--yes` (applies without prompt)<br>5. Migration failure surfaces error and exits non-zero | **DONE** |
| Command | Flags | Description |
|---------|-------|-------------|
| `spx admin upgrade` | `--yes` (apply without prompting), `--config-dir` (persistent, default: `~/spinifex/config`), `--spinifex-dir` (persistent, default: `~/spinifex`) | Reads current config versions from registry → computes pending config migrations (from→to per target) → prints version summary and pending list → prompts for confirmation unless `--yes` → runs `migrate.DefaultRegistry.RunAllConfig()` to apply migrations to config files. Intended for upgrades between Spinifex versions. Operators can skip migrations during install by setting `INSTALL_SPINIFEX_SKIP_MIGRATE=1`, then run `spx admin upgrade` manually to review and apply. After completion, services must be restarted with `sudo systemctl restart spinifex.target`. Invoked non-interactively by `scripts/setup.sh` with `--yes`. |
### Account Management
| Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
|---------|-------|---------------|-------------|------------|--------|
| `spx admin account create` | `--name` | Cluster must be running (NATS), master key must exist | Connects to NATS → CAS loop on `spinifex-account-counter:next_id` for sequential 12-digit ID → creates Account record in `spinifex-accounts` KV → creates `admin` user in new account → creates access key for admin → creates AdministratorAccess policy (Action:*, Resource:*) → attaches policy → prints credentials | 1. Create first account (ID 000000000001)<br>2. Sequential IDs (000000000002, etc.)<br>3. Admin user has full access<br>4. Credentials printed once | **DONE** |
| `spx admin account list` | — | Cluster must be running (NATS), master key must exist | Connects to NATS → IAMService.ListAccounts() → prints table with Account ID, Name, Status, Created | 1. List all accounts<br>2. Empty list (only global account) | **DONE** |
| Command | Flags | Description |
|---------|-------|-------------|
| `spx admin account create` | `--name` | Connects to NATS → CAS loop on `spinifex-account-counter:next_id` for sequential 12-digit ID → creates Account record in `spinifex-accounts` KV → creates `admin` user in new account → creates access key for admin → creates AdministratorAccess policy (Action:*, Resource:*) → attaches policy → prints credentials |
| `spx admin account list` | — | Connects to NATS → IAMService.ListAccounts() → prints table with Account ID, Name, Status, Created |
### Image Management
| Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
|---------|-------|---------------|-------------|------------|--------|
| `spx admin images import` | `--name`, `--ami-name`, `--file`, `--distro`, `--version`, `--arch`, `--platform`, `--boot-mode` (bios/uefi/uefi-preferred), `--tag`, `--force`, `--skip-verify` | Cluster must be running; either `--name` (catalog download) or `--file` (operator-supplied media). `--file` mode additionally requires `--distro`, `--version`, `--arch`, `--boot-mode`. | Catalog imports (`--name`) download the image, fetch the catalog `Checksum` URL, verify the SHA-256/SHA-512 digest, and inherit `BootMode` from the catalog entry. `--boot-mode` overrides the catalog value when set. Mismatch fails closed; the cached file is left on disk and `--force` re-downloads. `--file` imports skip checksum verification (operator-supplied media is outside Spinifex's trust boundary, the skip is logged at INFO for audit) and require an explicit `--boot-mode` because there is no catalog metadata to inherit from. `--ami-name` overrides the registered AMI name (the `DescribeImages` name), which otherwise defaults to `ami-{distro}-{version}-{arch}`; use it for locally-built appliance images (e.g. `--ami-name spinifex-eks-node`) not in the catalog. `--skip-verify` bypasses verification for catalog imports and emits a WARN slog + stderr notice; use only for debugging or when upstream mirrors are confirmed-broken. | 1. Import valid catalog image (verifies checksum)<br>2. Tampered cache hit fails with `ErrChecksumMismatch`<br>3. `--force` recovers after a mismatch<br>4. `--file` import skips verification<br>5. `--skip-verify` bypasses checksum with WARN<br>6. `--file` without `--boot-mode` rejected<br>7. `--boot-mode=invalid` rejected | **DONE** |
| `spx admin images list` | — | None | Lists available OS images that can be imported or downloaded | 1. List available images | **DONE** |
| `spx admin images remove` | `--image-id` (required), `--force`, `--yes` | Cluster must be running; predastore reachable. Targets AMIs imported via `spx admin images import` (non-account `ImageOwnerAlias`, e.g. `"system"`). | Loads `ami-<id>/config.json`, walks transitive dependents — copied snapshots whose `VolumeID == imageID`, volumes whose `SnapshotID` references the internal `snap-ami-<id>` or any derived snap, and account AMIs created via `CopyImage` whose `SnapshotID` is a derived snap — then prompts (skipped with `--yes`) before deleting `ami-<id>/config.json` (the DescribeImages barrier) followed by the rest of `ami-<id>/` and `snap-ami-<id>/`. Account-owned AMIs are refused with a hint pointing at `aws ec2 deregister-image` + `aws ec2 delete-snapshot`. `--force` bypasses the dependency, ownership and config-corrupt checks for salvage of orphaned blocks. | 1. Happy path: system AMI, no deps → deleted<br>2. Account-owned → refused with AWS-flow hint<br>3. Missing/corrupt config → `InvalidAMIID.NotFound` (salvageable with `--force`)<br>4. Dependent volume (direct or via copied snap) → refused<br>5. Dependent account AMI → refused<br>6. `--force` overrides dependents<br>7. Idempotent re-run after partial delete | **DONE** |
#### Image integrity verification (CMMC SI.L1-3.14.2)
Catalog imports (`spx admin images import --name <name>`) verify the image
against the catalog-declared SHA-256/SHA-512 digest before extraction. The sums
file is fetched from the catalog `Checksum` URL over HTTPS only (cross-scheme
redirects refused), and verification runs on both fresh downloads and cache
hits so a poisoned cache is caught on the next import.
On mismatch the import exits non-zero, the cached file is left on disk for
inspection, and the printed guidance is `spx admin images import --name <name>
--force` to re-download.
`--file` imports skip verification by design: operator-supplied media is
outside Spinifex's trust boundary and the operator is responsible for
integrity (e.g. `sha256sum` against a trusted upstream digest before import).
The skip is recorded as an INFO `slog` event with `reason=local-file-import`
so a CMMC assessor can audit the decision from journald.
`--skip-verify` bypasses the checksum step for catalog imports. The command
still downloads via the catalog URL but does not compare the image digest
against the sums file. Intended for narrow cases such as debugging upstream
mirror issues or running against a transiently-broken `latest/` path; the
skip is logged at WARN with `reason=skip-verify-flag` and printed to stderr
so operators and assessors see it. Prefer `--file` with an out-of-band
verified image over `--skip-verify` whenever possible.
**Limitation:** verification confirms the image matches the digest the mirror
served. A mirror compromise that swaps both image and sums file is not
detected; closing that gap requires GPG signature verification of the sums
file, deferred to a later phase.
#### `spx admin images remove` caveats
Admin-imported AMIs (`ImageOwnerAlias = "system"`) live
under the `ami-<id>/` S3 prefix and use a viperblock-internal snap checkpoint
at `snap-ami-<id>/` — there is no `snap-<id>/metadata.json`. The AWS handlers
(`DeregisterImage`, `DeleteSnapshot`) reject system owners with
`UnauthorizedOperation`, which is the right behaviour for tenant API callers
but leaves no AWS-flow path to reclaim space. `spx admin images remove` is
the admin-trust-boundary counterpart that performs the dependency walk and
hard-deletes the blocks directly against predastore.
`CopyImage` of a system AMI is metadata-only: it writes a fresh
`snap-<acct>/metadata.json` whose `VolumeID` points at `ami-<sys>` and a new
`ami-<acct>/config.json` referencing that snap. Volumes launched from the
copied AMI read transitively from `ami-<sys>/chunks/...`. The remove command
walks this transitive set and refuses if anything references the target.
**TOCTOU window:** between the safety scan and the `config.json` delete a
concurrent `RunInstances` against the AMI could create a new dependent
volume. The window is sub-second on a healthy cluster. The admin running
this command is expected to know the fleet's operational state; if the race
fires the result is a `vol-<id>` with deleted backing blocks, recovered by
terminating the orphaned instance.
**`--force` bypasses every safety check** (dependents, ownership, missing /
corrupt `config.json`). Use only for salvage of orphaned blocks. Running it
against a live system AMI corrupts every dependent volume on the next disk
read.
| Command | Flags | Description |
|---------|-------|-------------|
| `spx admin images import` | `--name`, `--file`, `--distro`, `--version`, `--arch`, `--platform`, `--boot-mode` (bios/uefi/uefi-preferred), `--tag`, `--force`, `--skip-verify` | Catalog imports (`--name`) download the image, fetch the catalog `Checksum` URL, verify the SHA-256/SHA-512 digest, and inherit `BootMode` from the catalog entry. `--boot-mode` overrides the catalog value when set. Mismatch fails closed; the cached file is left on disk and `--force` re-downloads. `--file` imports skip checksum verification (operator-supplied media is outside Spinifex's trust boundary, the skip is logged at INFO for audit) and require an explicit `--boot-mode` because there is no catalog metadata to inherit from. `--skip-verify` bypasses verification for catalog imports and emits a WARN slog + stderr notice; use only for debugging or when upstream mirrors are confirmed-broken. |
| `spx admin images list` | — | Lists available OS images that can be imported or downloaded |
| `spx admin images remove` | `--image-id` (required), `--force`, `--yes` | Loads `ami-<id>/config.json`, walks transitive dependents — copied snapshots whose `VolumeID == imageID`, volumes whose `SnapshotID` references the internal `snap-ami-<id>` or any derived snap, and account AMIs created via `CopyImage` whose `SnapshotID` is a derived snap — then prompts (skipped with `--yes`) before deleting `ami-<id>/config.json` (the DescribeImages barrier) followed by the rest of `ami-<id>/` and `snap-ami-<id>/`. Account-owned AMIs are refused with a hint pointing at `aws ec2 deregister-image` + `aws ec2 delete-snapshot`. `--force` bypasses the dependency, ownership and config-corrupt checks for salvage of orphaned blocks. |
### GPU Management
| Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
|---------|-------|---------------|-------------|------------|--------|
| `spx admin gpu status` | `--node` (default: local node) | Cluster must be running (NATS) | Queries `spinifex.node.status` fan-out → finds the target node response → prints Node, GPU hardware (model list or "none detected"), IOMMU state, vfio-pci state, passthrough enabled/disabled, and GPU pool allocation (`allocated/total`). Also lists GPU-capable instance types when passthrough is active. | 1. Local node with passthrough enabled (shows pool and instance types)<br>2. Local node capable but passthrough disabled (shows "run enable")<br>3. Node without GPU hardware (shows "none detected")<br>4. `--node` targets a remote node<br>5. Node not found or not responding (error) | **DONE** |
| `spx admin gpu enable` | — | Cluster must be running (NATS); node must be GPU-capable (IOMMU + vfio-pci configured) | Checks current passthrough state via NATS → errors if already enabled or prerequisites not met (directs to `setup`) → writes `gpu_passthrough = true` to `spinifex.toml` via `admin.SetGPUPassthrough` → sends SIGHUP to `spinifex-daemon` via `systemctl kill -s HUP` → polls node status for up to 30 s until daemon confirms new state → prints final `gpu status` output. Must be run directly on the target host. | 1. Enable on capable node<br>2. Already enabled (no-op)<br>3. Prerequisites not met (error, suggests `setup`)<br>4. Daemon does not confirm within 30 s (timeout error) | **DONE** |
| `spx admin gpu disable` | — | Cluster must be running (NATS); no GPU instances may be running | Checks current passthrough state via NATS → errors if already disabled or if `AllocGPUs > 0` (must terminate GPU instances first) → writes `gpu_passthrough = false` → sends SIGHUP to `spinifex-daemon` → polls for up to 30 s → prints final `gpu status` output. Must be run directly on the target host. | 1. Disable with no GPU instances running<br>2. Already disabled (no-op)<br>3. GPU instances running (error with count)<br>4. Daemon does not confirm within 30 s (timeout error) | **DONE** |
| `spx admin gpu setup` | — | Must be run as root; NVIDIA or AMD GPU must be present | Idempotent host configuration for GPU passthrough. Steps (skipped if already applied): detect GPUs via `gpu.Discover` → collect PCI IDs for all IOMMU-group siblings → check/enable IOMMU in GRUB (`intel_iommu=on iommu=pt` or `amd_iommu=on iommu=pt`) → write vfio udev rule (`/etc/udev/rules.d/99-spinifex-vfio.rules`) → blacklist nouveau (`/etc/modprobe.d/blacklist-nouveau.conf`) → blacklist amdgpu if AMD GPU present (`/etc/modprobe.d/blacklist-amdgpu.conf`) → write vfio-pci early binding config (`/etc/modprobe.d/vfio-pci.conf`) → add vfio modules to initramfs. If any change requires a reboot: runs `update-initramfs -u` and exits with reboot instructions. After reboot: verifies `vfio_pci` module is loaded → verifies each GPU is bound to `vfio-pci` (binds explicitly via `driver_override` if unbound) → calls `gpu enable` to activate passthrough. | 1. First run on unconfigured host (GRUB updated, reboot required)<br>2. Re-run after reboot (verifies bindings, enables passthrough)<br>3. Already fully configured (all steps skipped, enables passthrough)<br>4. AMD GPU present (blacklists amdgpu in addition to nouveau)<br>5. Not run as root (error)<br>6. No GPU hardware detected (error) | **DONE** |
| `spx admin gpu mig status` | — (must be run on target host) | NVIDIA GPU with MIG support present | Runs `gpu.Discover()` locally → for each GPU prints: PCI address, model, MIG capability, MIG mode (enabled/disabled/N/A); for GPUs with MIG enabled lists active MIG slices with GI ID, profile name, and mdev path. | 1. Node with MIG enabled (shows slices)<br>2. MIG capable but not enabled (shows disabled)<br>3. No MIG-capable GPU (shows N/A for MIG fields) | **DONE** |
| `spx admin gpu mig enable` | `--profile <name>` (required, e.g. `1g.10gb`), `--gpu <pci-addr>` (optional, default: all MIG-capable GPUs) | Must be run as root; MIG-capable GPU present; no GPU instances running | Checks no GPU instances running (NATS) → discovers MIG-capable GPUs via `gpu.Discover()` (filtered by `--gpu` if set) → enables MIG mode on each target (`gpu.EnableMIGMode`) → lists available profiles (`gpu.ListProfiles`) and validates requested profile name → destroys any existing instances (`gpu.DestroyAllInstances`) → creates new instances filling GPU capacity (`gpu.CreateInstances`) → writes `mig_profile` to `spinifex.toml` via `admin.SetMIGProfile` → sends SIGHUP to `spinifex-daemon`. Must be run directly on the target host. | 1. Enable MIG with valid profile<br>2. Invalid profile name (error with available profile list)<br>3. No MIG-capable GPU (error)<br>4. GPU instances running (error)<br>5. `--gpu` targets specific GPU<br>6. Not run as root (error) | **DONE** |
| `spx admin gpu mig disable` | `--gpu <pci-addr>` (optional, default: all MIG-capable GPUs) | Must be run as root; no GPU instances running | Checks no GPU instances running (NATS) → discovers MIG-capable GPUs via `gpu.Discover()` (filtered by `--gpu` if set) → destroys all GPU instances (`gpu.DestroyAllInstances`) → disables MIG mode (`gpu.DisableMIGMode`) → clears `mig_profile` in `spinifex.toml` via `admin.SetMIGProfile` → sends SIGHUP to `spinifex-daemon`. Must be run directly on the target host. | 1. Disable with no running GPU instances<br>2. GPU instances running (error)<br>3. MIG not enabled on GPU (skips that GPU)<br>4. `--gpu` targets specific GPU<br>5. Not run as root (error) | **DONE** |
| Command | Flags | Description |
|---------|-------|-------------|
| `spx admin gpu status` | `--node` (default: local node) | Queries `spinifex.node.status` fan-out → finds the target node response → prints Node, GPU hardware (model list or "none detected"), IOMMU state, vfio-pci state, passthrough enabled/disabled, and GPU pool allocation (`allocated/total`). Also lists GPU-capable instance types when passthrough is active. |
| `spx admin gpu enable` | — | Checks current passthrough state via NATS → errors if already enabled or prerequisites not met (directs to `setup`) → writes `gpu_passthrough = true` to `spinifex.toml` via `admin.SetGPUPassthrough` → sends SIGHUP to `spinifex-daemon` via `systemctl kill -s HUP` → polls node status for up to 30 s until daemon confirms new state → prints final `gpu status` output. Must be run directly on the target host. |
| `spx admin gpu disable` | — | Checks current passthrough state via NATS → errors if already disabled or if `AllocGPUs > 0` (must terminate GPU instances first) → writes `gpu_passthrough = false` → sends SIGHUP to `spinifex-daemon` → polls for up to 30 s → prints final `gpu status` output. Must be run directly on the target host. |
| `spx admin gpu setup` | — | Idempotent host configuration for GPU passthrough. Steps (skipped if already applied): detect GPUs via `gpu.Discover` → collect PCI IDs for all IOMMU-group siblings → check/enable IOMMU in GRUB (`intel_iommu=on iommu=pt` or `amd_iommu=on iommu=pt`) → write vfio udev rule (`/etc/udev/rules.d/99-spinifex-vfio.rules`) → blacklist nouveau (`/etc/modprobe.d/blacklist-nouveau.conf`) → blacklist amdgpu if AMD GPU present (`/etc/modprobe.d/blacklist-amdgpu.conf`) → write vfio-pci early binding config (`/etc/modprobe.d/vfio-pci.conf`) → add vfio modules to initramfs. If any change requires a reboot: runs `update-initramfs -u` and exits with reboot instructions. After reboot: verifies `vfio_pci` module is loaded → verifies each GPU is bound to `vfio-pci` (binds explicitly via `driver_override` if unbound) → calls `gpu enable` to activate passthrough. |
| `spx admin gpu mig status` | — (must be run on target host) | Runs `gpu.Discover()` locally → for each GPU prints: PCI address, model, MIG capability, MIG mode (enabled/disabled/N/A); for GPUs with MIG enabled lists active MIG slices with GI ID, profile name, and mdev path. |
| `spx admin gpu mig enable` | `--profile <name>` (required, e.g. `1g.10gb`), `--gpu <pci-addr>` (optional, default: all MIG-capable GPUs) | Checks no GPU instances running (NATS) → discovers MIG-capable GPUs via `gpu.Discover()` (filtered by `--gpu` if set) → enables MIG mode on each target (`gpu.EnableMIGMode`) → lists available profiles (`gpu.ListProfiles`) and validates requested profile name → destroys any existing instances (`gpu.DestroyAllInstances`) → creates new instances filling GPU capacity (`gpu.CreateInstances`) → writes `mig_profile` to `spinifex.toml` via `admin.SetMIGProfile` → sends SIGHUP to `spinifex-daemon`. Must be run directly on the target host. |
| `spx admin gpu mig disable` | `--gpu <pci-addr>` (optional, default: all MIG-capable GPUs) | Checks no GPU instances running (NATS) → discovers MIG-capable GPUs via `gpu.Discover()` (filtered by `--gpu` if set) → destroys all GPU instances (`gpu.DestroyAllInstances`) → disables MIG mode (`gpu.DisableMIGMode`) → clears `mig_profile` in `spinifex.toml` via `admin.SetMIGProfile` → sends SIGHUP to `spinifex-daemon`. Must be run directly on the target host. |
## AWS-Compatible API
@@ -295,9 +235,6 @@ by downstream services.
### EC2 — VPC Core
VPC/Subnet/ENI/SG CRUD stores metadata in NATS KV and publishes events to
vpcd for OVN translation. Single AZ for Spinifex v1.
| Command | Implemented Flags | Missing Flags | Status |
|---------|-------------------|---------------|--------|
| `create-vpc` | `--cidr-block`, `--tag-specifications` | `--instance-tenancy`, `--dry-run` | **DONE** |
@@ -393,9 +330,6 @@ EIP commands are only registered when an external IPAM pool is configured.
### EC2 — NAT Gateway
Deleted gateways move to a separate KV bucket with 1-hour TTL for Terraform
polling.
| Command | Implemented Flags | Missing Flags | Status |
|---------|-------------------|---------------|--------|
| `create-nat-gateway` | `--subnet-id`, `--allocation-id` | `--connectivity-type`, `--tag-specifications`, `--dry-run` | **DONE** |
@@ -406,9 +340,6 @@ polling.
### EC2 — Placement Groups
Strategies: `spread` (1 instance per node, strict) and `cluster` (all pinned
to single node). `partition` rejected.
| Command | Implemented Flags | Missing Flags | Status |
|---------|-------------------|---------------|--------|
| `create-placement-group` | `--group-name`, `--strategy` (spread/cluster) | `--partition-count`, `--spread-level`, `--tag-specifications`, `--dry-run` | **DONE** |
@@ -543,9 +474,6 @@ bypasses policy evaluation entirely.
### IAM — Access Keys
Max 2 keys per user. Secrets encrypted with AES-256-GCM using master key,
returned plaintext only at creation.
| Command | Implemented Flags | Missing Flags | Status |
|---------|-------------------|---------------|--------|
| `create-access-key` | `--user-name` | — | **DONE** |
@@ -555,10 +483,6 @@ returned plaintext only at creation.
### IAM — Policies
Policy documents require `Version: "2012-10-17"` and a valid Statement array.
Wildcard action matching supported (`ec2:*`, `ec2:Describe*`, `*`). Evaluation
order: explicit Deny > explicit Allow > implicit Deny.
| Command | Implemented Flags | Missing Flags | Status |
|---------|-------------------|---------------|--------|
| `create-policy` | `--policy-name`, `--policy-document`, `--path`, `--description` | `--tags` | **DONE** |
@@ -570,19 +494,6 @@ order: explicit Deny > explicit Allow > implicit Deny.
| `detach-user-policy` | `--user-name`, `--policy-arn` | — | **DONE** |
| `list-attached-user-policies` | `--user-name` | `--path-prefix`, `--max-items`, `--marker` | **DONE** |
### IAM — Policy Evaluation (internal — not an AWS API)
| Feature | Status |
|---------|--------|
| Root user bypass | **DONE** |
| Default deny | **DONE** |
| Explicit allow | **DONE** |
| Explicit deny | **DONE** |
| Wildcard matching (`*`, `ec2:*`, `ec2:Describe*`) | **DONE** |
| Account-scoped evaluation | **DONE** |
| EC2 action mapping | **DONE** |
| IAM action mapping (16 actions) | **DONE** |
### IAM — Roles
| Command | Implemented Flags | Missing Flags | Status |
@@ -599,9 +510,6 @@ order: explicit Deny > explicit Allow > implicit Deny.
### IAM — Instance Profiles
Containers for IAM roles that allow EC2 instances to assume a role. Required
for `run-instances --iam-instance-profile`. Max 1 role per profile.
| Command | Implemented Flags | Missing Flags | Status |
|---------|-------------------|---------------|--------|
| `create-instance-profile` | `--instance-profile-name`, `--path`, `--tags` | — | **DONE** |
@@ -631,7 +539,7 @@ for `run-instances --iam-instance-profile`. Max 1 role per profile.
## STS
| Command | Implemented Flags | Missing Flags (rejected if supplied) | Status |
| Command | Implemented Flags | Missing Flags | Status |
|---------|-------------------|--------------------------------------|--------|
| `get-caller-identity` | — | — | **DONE** |
| `assume-role` | `--role-arn`, `--role-session-name`, `--duration-seconds` (900–min(role MaxSessionDuration, 43200)) | `--policy`, `--policy-arns` (→ `PackedPolicyTooLarge`); `--tags`, `--transitive-tag-keys` (→ `InvalidParameterValue`); `--serial-number`, `--token-code` (→ `InvalidParameterValue`); `--external-id`, `--source-identity` (accepted and logged, **not enforced** — no Condition evaluator in v1) | **DONE** |
@@ -649,9 +557,52 @@ Condition evaluator so accepting them would silently allow.
---
## IMDS (Instance Metadata Service)
Available at `169.254.169.254` from inside every running guest VM, matching AWS. The endpoint is reached from within a guest over plain HTTP, exactly as on EC2, with no in-VM agent to install. DHCP and fully static guests reach it identically, with no in-guest route configuration.
**IMDSv2-only.** Every read requires a session token. A tokenless (v1-style) `GET` returns `401 Unauthorized` with an empty body. Obtain a token with a `PUT /latest/api/token` carrying `X-aws-ec2-metadata-token-ttl-seconds` (1–21600), then send it back in `X-aws-ec2-metadata-token` on every read.
```bash
# Inside the guest VM:
TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" \
-H "X-aws-ec2-metadata-token-ttl-seconds: 60")
curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \
http://169.254.169.254/latest/meta-data/instance-id
# Tokenless GET → 401 Unauthorized (empty body)
curl -i http://169.254.169.254/latest/meta-data/instance-id
```
### IMDS — Supported Paths
| Path | Method | Source | Status |
|------|--------|--------|--------|
| `/latest/api/token` | PUT | Issues an ENI-bound IMDSv2 token; `X-aws-ec2-metadata-token-ttl-seconds` ∈ [1, 21600] required | **DONE** |
| `/latest/meta-data/` | GET | Directory listing of supported children | **DONE** |
| `/latest/meta-data/instance-id` | GET | `vm.ID` | **DONE** |
| `/latest/meta-data/instance-type` | GET | `vm.InstanceType` | **DONE** |
| `/latest/meta-data/ami-id` | GET | launch `ImageId` | **DONE** |
| `/latest/meta-data/local-ipv4` | GET | `ENIRecord.PrivateIpAddress` (== request source IP) | **DONE** |
| `/latest/meta-data/public-ipv4` | GET | EIP, else instance public IP; empty body if none | **DONE** |
| `/latest/meta-data/mac` | GET | `ENIRecord.MacAddress` | **DONE** |
| `/latest/meta-data/security-groups` | GET | `ENIRecord.SecurityGroupIds`, newline-separated | **DONE** |
| `/latest/meta-data/hostname`, `/local-hostname` | GET | Synthesised `ip-<dashed-ip>.<region>.compute.internal` | **DONE** |
| `/latest/meta-data/placement/availability-zone` | GET | `ENIRecord.AvailabilityZone` | **DONE** |
| `/latest/meta-data/placement/region` | GET | Derived from AZ (trailing letter stripped) | **DONE** |
| `/latest/meta-data/iam/info` | GET | `{InstanceProfileArn, InstanceProfileId}`; 404 if no profile | **DONE** |
| `/latest/meta-data/iam/security-credentials/` | GET | Role name(s) under the profile, one per line; empty body if none | **DONE** |
| `/latest/meta-data/iam/security-credentials/<role>` | GET | STS `AssumeRoleForInstance` → ASIA-prefixed temporary credential JSON | **DONE** |
| `/latest/user-data` | GET | `vm.UserData`; 404 if none | **DONE** |
| `/latest/dynamic/instance-identity/document` | GET | Needs a per-cluster signing key | **NOT STARTED** (404; lands with EKS IRSA) |
| `/latest/meta-data/network/interfaces/...` | GET | Multi-ENI rendering | **NOT STARTED** (404) |
| `/latest/meta-data/tags/...` | GET | Instance-tag metadata | **NOT STARTED** (404) |
---
## ELBv2 (Application & Network Load Balancer)
The data plane uses a system-managed LB VM running HAProxy, launched automatically during `create-load-balancer`. HAProxy config is pushed via NATS on listener/target changes.
The data plane uses a system-managed LB VM running HAProxy, launched automatically during `create-load-balancer`.
### ELBv2 — Load Balancers
spinifex/daemon/daemon.go
+11
View File
@@ -50,6 +50,7 @@ import (
handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2"
handlers_imds "github.com/mulgadc/spinifex/spinifex/handlers/imds"
"github.com/mulgadc/spinifex/spinifex/instancetypes"
"github.com/mulgadc/spinifex/spinifex/network/external/dhcp"
"github.com/mulgadc/spinifex/spinifex/network/host"
@@ -1206,6 +1207,16 @@ func (d *Daemon) startCluster() error {
return fmt.Errorf("failed to initialize VPC service: %w", err)
}
// Wire the eni-by-vpc-ip reverse index so the ENI controller keeps the
// IMDS source-IP→ENI lookup in sync on Create/DeleteNetworkInterface.
if vpcJS, jsErr := d.natsConn.JetStream(); jsErr != nil {
slog.Warn("Failed to get JetStream for eni-by-ip index", "err", jsErr)
} else if eniByIPKV, kvErr := handlers_imds.InitENIByIPBucket(vpcJS, 1); kvErr != nil {
slog.Warn("Failed to init eni-by-ip index bucket", "err", kvErr)
} else {
d.vpcService.SetENIByIPIndex(handlers_ec2_vpc.NewENIByIPIndex(eniByIPKV))
}
d.routeTableService, err = initServiceWithRetry("RouteTable service", func() (*handlers_ec2_routetable.RouteTableServiceImpl, error) {
return handlers_ec2_routetable.NewRouteTableServiceImplWithNATS(d.config, d.natsConn)
})
spinifex/daemon/daemon_handlers_test.go
+5 -2
View File
@@ -1658,7 +1658,10 @@ func TestHandleEC2DescribeInstanceAttribute_UserData(t *testing.T) {
Status: vm.StateStopped,
InstanceType: "t3.micro",
AccountID: testAccountID,
UserData: "#!/bin/bash",
RunInstancesInput: &ec2.RunInstancesInput{
// base64("#!/bin/bash") — DescribeInstanceAttribute returns user-data base64-encoded.
UserData: aws.String("IyEvYmluL2Jhc2g="),
},
Instance: &ec2.Instance{
InstanceId: aws.String(instanceID),
},
@@ -1679,7 +1682,7 @@ func TestHandleEC2DescribeInstanceAttribute_UserData(t *testing.T) {
err = json.Unmarshal(reply.Data, &output)
require.NoError(t, err)
require.NotNil(t, output.UserData)
assert.Equal(t, "#!/bin/bash", *output.UserData.Value)
assert.Equal(t, "IyEvYmluL2Jhc2g=", *output.UserData.Value)
}
func TestHandleEC2DescribeInstanceAttribute_DefaultAttribute_DisableApiTermination(t *testing.T) {
spinifex/gateway/auth.go
+2 -1
View File
@@ -16,6 +16,7 @@ import (
"github.com/mulgadc/predastore/auth"
"github.com/mulgadc/spinifex/spinifex/awserrors"
handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
"github.com/mulgadc/spinifex/spinifex/utils"
)
// Maximum request body size for signature validation (10 MB).
@@ -36,7 +37,7 @@ func (gw *GatewayConfig) SigV4AuthMiddleware() func(http.Handler) http.Handler {
}
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
clientIP := extractClientIP(r)
clientIP := utils.ClientIP(r.RemoteAddr)
if errCode := gw.RateLimiter.CheckIP(clientIP); errCode != "" {
gw.writeSigV4Error(w, r, errCode)
return
spinifex/gateway/auth_test.go
+3
View File
@@ -1561,6 +1561,9 @@ type mockSTSService struct {
func (m *mockSTSService) AssumeRole(_, _, _ string, _ *sts.AssumeRoleInput) (*sts.AssumeRoleOutput, error) {
return nil, nil
}
func (m *mockSTSService) AssumeRoleForInstance(_, _, _ string, _ int64) (*sts.AssumeRoleOutput, error) {
return nil, nil
}
func (m *mockSTSService) GetCallerIdentity(_, _, _ string, _ *sts.GetCallerIdentityInput) (*sts.GetCallerIdentityOutput, error) {
return nil, nil
}
spinifex/gateway/ratelimit.go
-13
View File
@@ -2,8 +2,6 @@ package gateway
import (
"log/slog"
"net"
"net/http"
"sync"
"time"
@@ -187,17 +185,6 @@ func (rl *AuthRateLimiter) cleanup() {
}
}
// extractClientIP returns the IP address from the request's RemoteAddr,
// stripping the port. Handles both IPv4 and IPv6.
func extractClientIP(r *http.Request) string {
host, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
// RemoteAddr might already be bare IP (no port).
return r.RemoteAddr
}
return host
}
// pruneOldFailures returns only the failures within the sliding window.
func pruneOldFailures(failures []time.Time, now time.Time) []time.Time {
cutoff := now.Add(-failureWindow)
spinifex/gateway/ratelimit_test.go
-23
View File
@@ -311,29 +311,6 @@ func TestConcurrentAccess(t *testing.T) {
wg.Wait()
}
func TestExtractClientIP(t *testing.T) {
tests := []struct {
name string
remoteAddr string
expected string
}{
{"IPv4 with port", "192.168.1.1:12345", "192.168.1.1"},
{"IPv6 with port", "[::1]:12345", "::1"},
{"IPv4 bare", "192.168.1.1", "192.168.1.1"},
{"IPv6 full with port", "[2001:db8::1]:443", "2001:db8::1"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
r := &http.Request{RemoteAddr: tt.remoteAddr}
got := extractClientIP(r)
if got != tt.expected {
t.Errorf("extractClientIP(%q) = %q, want %q", tt.remoteAddr, got, tt.expected)
}
})
}
}
// setupTestAppWithRateLimiter creates a test HTTP handler with SigV4 auth and
// the given rate limiter attached. A real NATS test connection is attached so
// the SigV4 middleware's cluster-unavailable short-circuit (mulga-siv-23) does
spinifex/gateway/sts/handler_test.go
+7
View File
@@ -32,6 +32,13 @@ func (s *stubSTSService) AssumeRole(callerAccountID, callerARN, callerIdentity s
return &sts.AssumeRoleOutput{}, nil
}
// AssumeRoleForInstance is the in-process IMDS entry point and is never
// dispatched over the HTTPS gateway; the stub exists only to satisfy the
// interface.
func (s *stubSTSService) AssumeRoleForInstance(_, _, _ string, _ int64) (*sts.AssumeRoleOutput, error) {
return &sts.AssumeRoleOutput{}, nil
}
func (s *stubSTSService) GetCallerIdentity(callerAccountID, callerARN, callerUserID string, input *sts.GetCallerIdentityInput) (*sts.GetCallerIdentityOutput, error) {
if s.getCallerIdentityFn != nil {
return s.getCallerIdentityFn(callerAccountID, callerARN, callerUserID, input)
spinifex/gateway/sts_request_test.go
+6
View File
@@ -37,6 +37,12 @@ func (m *flexMockSTSService) AssumeRole(callerAccountID, callerARN, callerIdenti
return &sts.AssumeRoleOutput{}, nil
}
// AssumeRoleForInstance is the in-process IMDS entry point, never dispatched
// over the HTTPS gateway; the mock exists only to satisfy the interface.
func (m *flexMockSTSService) AssumeRoleForInstance(_, _, _ string, _ int64) (*sts.AssumeRoleOutput, error) {
return &sts.AssumeRoleOutput{}, nil
}
func (m *flexMockSTSService) GetCallerIdentity(callerAccountID, callerARN, callerUserID string, input *sts.GetCallerIdentityInput) (*sts.GetCallerIdentityOutput, error) {
if m.getCallerIdentityFn != nil {
return m.getCallerIdentityFn(callerAccountID, callerARN, callerUserID, input)
spinifex/handlers/ec2/instance/service_impl.go
+9 -4
View File
@@ -2426,10 +2426,10 @@ func (s *InstanceServiceImpl) ModifyInstanceAttribute(input *ec2.ModifyInstanceA
if input.UserData != nil && input.UserData.Value != nil {
slog.Info("ModifyInstanceAttribute: changing user data", "instanceId", instanceID)
instance.UserData = string(input.UserData.Value)
if instance.RunInstancesInput != nil {
instance.RunInstancesInput.UserData = aws.String(base64.StdEncoding.EncodeToString(input.UserData.Value))
if instance.RunInstancesInput == nil {
instance.RunInstancesInput = &ec2.RunInstancesInput{}
}
instance.RunInstancesInput.UserData = aws.String(base64.StdEncoding.EncodeToString(input.UserData.Value))
}
if input.DisableApiTermination != nil && input.DisableApiTermination.Value != nil {
@@ -2803,7 +2803,12 @@ func (s *InstanceServiceImpl) DescribeInstanceAttribute(input *ec2.DescribeInsta
output.InstanceType = &ec2.AttributeValue{Value: &val}
case ec2.InstanceAttributeNameUserData:
val := instance.UserData
// AWS returns user-data base64-encoded; RunInstancesInput.UserData is the
// canonical base64 store, set at launch and kept in sync by Modify.
var val string
if instance.RunInstancesInput != nil && instance.RunInstancesInput.UserData != nil {
val = *instance.RunInstancesInput.UserData
}
output.UserData = &ec2.AttributeValue{Value: &val}
case ec2.InstanceAttributeNameDisableApiTermination:
spinifex/handlers/ec2/instance/service_impl_test.go
+5 -4
View File
@@ -1129,7 +1129,10 @@ func TestDescribeInstanceAttribute_RunningInstance(t *testing.T) {
ID: id,
InstanceType: "t3.large",
AccountID: owner,
UserData: "raw-user-data",
RunInstancesInput: &ec2.RunInstancesInput{
// base64("raw-user-data") — DescribeInstanceAttribute returns user-data base64-encoded.
UserData: aws.String("cmF3LXVzZXItZGF0YQ=="),
},
}
svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{id: v})}
@@ -1151,7 +1154,7 @@ func TestDescribeInstanceAttribute_RunningInstance(t *testing.T) {
attribute: ec2.InstanceAttributeNameUserData,
assertions: func(t *testing.T, out *ec2.DescribeInstanceAttributeOutput) {
require.NotNil(t, out.UserData)
assert.Equal(t, "raw-user-data", *out.UserData.Value)
assert.Equal(t, "cmF3LXVzZXItZGF0YQ==", *out.UserData.Value)
},
},
{
@@ -1530,7 +1533,6 @@ func TestModifyInstanceAttribute_ChangeUserData(t *testing.T) {
ID: id,
Status: vm.StateStopped,
AccountID: "acc",
UserData: "old",
RunInstancesInput: &ec2.RunInstancesInput{
UserData: aws.String("b2xk"),
},
@@ -1548,7 +1550,6 @@ func TestModifyInstanceAttribute_ChangeUserData(t *testing.T) {
updated := store.wroteStopped[id]
require.NotNil(t, updated)
assert.Equal(t, newContent, updated.UserData)
assert.Equal(t, "IyEvYmluL2Jhc2g=", *updated.RunInstancesInput.UserData)
}
spinifex/handlers/ec2/vpc/eni.go
+18
View File
@@ -144,6 +144,15 @@ func (s *VPCServiceImpl) CreateNetworkInterface(input *ec2.CreateNetworkInterfac
return nil, errors.New(awserrors.ErrorServerInternal)
}
// Maintain the eni-by-vpc-ip reverse index after the source-of-truth ENI
// write. A failure here leaves IMDS returning safe 404s for this IP rather
// than phantom permissions, so it is logged, not fatal.
if s.eniIndex != nil {
if err := s.eniIndex.Put(subnet.VpcId, privateIP, eniId, accountID); err != nil {
slog.Warn("CreateNetworkInterface: eni-by-ip index write failed", "eniId", eniId, "vpcId", subnet.VpcId, "ip", privateIP, "err", err)
}
}
slog.Info("CreateNetworkInterface completed", "eniId", eniId, "subnetId", subnetId, "ip", privateIP, "accountID", accountID)
// Send vpc.create-port synchronously so vpcd OVSDB errors surface to the
@@ -213,6 +222,15 @@ func (s *VPCServiceImpl) DeleteNetworkInterface(input *ec2.DeleteNetworkInterfac
return nil, errors.New(awserrors.ErrorServerInternal)
}
// Drop the eni-by-vpc-ip reverse index entry. Idempotent and non-fatal:
// the source-of-truth ENI is already gone, so a stale index entry would
// only resolve to a now-missing ENI and surface as a safe 404.
if s.eniIndex != nil {
if err := s.eniIndex.Delete(record.VpcId, record.PrivateIpAddress); err != nil {
slog.Warn("DeleteNetworkInterface: eni-by-ip index delete failed", "eniId", eniId, "vpcId", record.VpcId, "ip", record.PrivateIpAddress, "err", err)
}
}
slog.Info("DeleteNetworkInterface completed", "eniId", eniId, "accountID", accountID)
// Publish vpc.delete-port event for vpcd topology cleanup. SG IDs are
spinifex/handlers/ec2/vpc/eni_index.go
+52
View File
@@ -0,0 +1,52 @@
package handlers_ec2_vpc
import (
"encoding/json"
"errors"
"fmt"
"github.com/nats-io/nats.go"
)
// eniByIPValue is the stored shape of a spinifex-network-eni-by-vpc-ip row: the ENI
// ID plus its owning account (the ENI bucket is keyed "{accountID}.{eniID}"). Both are
// immutable identity, letting the handler resolve the ENI's account in a single Get.
type eniByIPValue struct {
ENIId string `json:"eni_id"`
AccountID string `json:"account_id"`
}
// ENIByIPIndex maintains the spinifex-network-eni-by-vpc-ip reverse index
// (vpcID/ip → eniID), so the IMDS handler resolves a source IP to an ENI in one
// KV read. Written on CreateNetworkInterface, deleted on DeleteNetworkInterface.
type ENIByIPIndex struct {
kv nats.KeyValue
}
// NewENIByIPIndex wraps an already-opened eni-by-vpc-ip KV bucket.
func NewENIByIPIndex(kv nats.KeyValue) *ENIByIPIndex { return &ENIByIPIndex{kv: kv} }
// eniByVPCIPKey builds the composite vpcID/ip key.
func eniByVPCIPKey(vpcID, ip string) string { return vpcID + "/" + ip }
// Put writes the vpcID/ip → {eniID, accountID} mapping, overwriting any prior
// entry. Called after the source-of-truth ENI record is persisted so that a
// failure here leaves IMDS returning safe 404s rather than phantom permissions.
func (i *ENIByIPIndex) Put(vpcID, ip, eniID, accountID string) error {
data, err := json.Marshal(eniByIPValue{ENIId: eniID, AccountID: accountID})
if err != nil {
return fmt.Errorf("marshal eni-by-ip value: %w", err)
}
if _, err := i.kv.Put(eniByVPCIPKey(vpcID, ip), data); err != nil {
return fmt.Errorf("put eni-by-ip index %s/%s: %w", vpcID, ip, err)
}
return nil
}
// Delete removes the vpcID/ip mapping. Idempotent: a missing key returns nil.
func (i *ENIByIPIndex) Delete(vpcID, ip string) error {
if err := i.kv.Delete(eniByVPCIPKey(vpcID, ip)); err != nil && !errors.Is(err, nats.ErrKeyNotFound) {
return fmt.Errorf("delete eni-by-ip index %s/%s: %w", vpcID, ip, err)
}
return nil
}
spinifex/handlers/ec2/vpc/eni_index_test.go
+147
View File
@@ -0,0 +1,147 @@
package handlers_ec2_vpc
import (
"encoding/json"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
handlers_imds "github.com/mulgadc/spinifex/spinifex/handlers/imds"
"github.com/nats-io/nats.go"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// openTestENIByIPBucket opens the eni-by-vpc-ip reverse-index bucket on the
// test JetStream, exercising the same init path the daemon uses.
func openTestENIByIPBucket(t *testing.T, nc *nats.Conn) nats.KeyValue {
t.Helper()
js, err := nc.JetStream()
require.NoError(t, err)
kv, err := handlers_imds.InitENIByIPBucket(js, 1)
require.NoError(t, err)
return kv
}
// readIndexEntry reads the raw vpcID/ip entry and returns its eni_id and
// account_id, or ("", "") when the key is absent.
func readIndexEntry(t *testing.T, kv nats.KeyValue, vpcID, ip string) (eniID, accountID string) {
t.Helper()
entry, err := kv.Get(vpcID + "/" + ip)
if err != nil {
require.ErrorIs(t, err, nats.ErrKeyNotFound)
return "", ""
}
var v struct {
ENIId string `json:"eni_id"`
AccountID string `json:"account_id"`
}
require.NoError(t, json.Unmarshal(entry.Value(), &v))
return v.ENIId, v.AccountID
}
// readIndexENIID is the eni_id-only convenience wrapper.
func readIndexENIID(t *testing.T, kv nats.KeyValue, vpcID, ip string) string {
t.Helper()
eniID, _ := readIndexEntry(t, kv, vpcID, ip)
return eniID
}
func TestENIByIPIndex_PutGetDelete(t *testing.T) {
_, nc := setupTestVPCServiceWithNC(t)
kv := openTestENIByIPBucket(t, nc)
idx := NewENIByIPIndex(kv)
require.NoError(t, idx.Put("vpc-abc12345", "10.0.1.5", "eni-deadbeef", "111122223333"))
eniID, accountID := readIndexEntry(t, kv, "vpc-abc12345", "10.0.1.5")
assert.Equal(t, "eni-deadbeef", eniID)
assert.Equal(t, "111122223333", accountID)
// Overwrite is allowed (rebind is a Put).
require.NoError(t, idx.Put("vpc-abc12345", "10.0.1.5", "eni-cafef00d", "111122223333"))
assert.Equal(t, "eni-cafef00d", readIndexENIID(t, kv, "vpc-abc12345", "10.0.1.5"))
require.NoError(t, idx.Delete("vpc-abc12345", "10.0.1.5"))
assert.Equal(t, "", readIndexENIID(t, kv, "vpc-abc12345", "10.0.1.5"))
}
func TestENIByIPIndex_DeleteAbsentIsIdempotent(t *testing.T) {
_, nc := setupTestVPCServiceWithNC(t)
idx := NewENIByIPIndex(openTestENIByIPBucket(t, nc))
require.NoError(t, idx.Delete("vpc-nope", "10.0.0.1"))
}
func TestENIByIPIndex_KeysAreVPCScoped(t *testing.T) {
// Same IP in two VPCs must not collide — the key includes the VPC ID.
_, nc := setupTestVPCServiceWithNC(t)
kv := openTestENIByIPBucket(t, nc)
idx := NewENIByIPIndex(kv)
require.NoError(t, idx.Put("vpc-aaaaaaaa", "10.0.1.5", "eni-aaa", "111122223333"))
require.NoError(t, idx.Put("vpc-bbbbbbbb", "10.0.1.5", "eni-bbb", "444455556666"))
assert.Equal(t, "eni-aaa", readIndexENIID(t, kv, "vpc-aaaaaaaa", "10.0.1.5"))
assert.Equal(t, "eni-bbb", readIndexENIID(t, kv, "vpc-bbbbbbbb", "10.0.1.5"))
}
func TestCreateNetworkInterface_WritesENIIndex(t *testing.T) {
svc, nc := setupTestVPCServiceWithNC(t)
kv := openTestENIByIPBucket(t, nc)
svc.SetENIByIPIndex(NewENIByIPIndex(kv))
vpcID := createTestVPC(t, svc, "10.0.0.0/16")
subnetID := createTestSubnet(t, svc, vpcID, "10.0.1.0/24")
out, err := svc.CreateNetworkInterface(&ec2.CreateNetworkInterfaceInput{
SubnetId: aws.String(subnetID),
}, testAccountID)
require.NoError(t, err)
eniID := *out.NetworkInterface.NetworkInterfaceId
ip := *out.NetworkInterface.PrivateIpAddress
gotENI, gotAccount := readIndexEntry(t, kv, vpcID, ip)
assert.Equal(t, eniID, gotENI)
assert.Equal(t, testAccountID, gotAccount)
}
func TestDeleteNetworkInterface_RemovesENIIndex(t *testing.T) {
svc, nc := setupTestVPCServiceWithNC(t)
kv := openTestENIByIPBucket(t, nc)
svc.SetENIByIPIndex(NewENIByIPIndex(kv))
vpcID := createTestVPC(t, svc, "10.0.0.0/16")
subnetID := createTestSubnet(t, svc, vpcID, "10.0.1.0/24")
out, err := svc.CreateNetworkInterface(&ec2.CreateNetworkInterfaceInput{
SubnetId: aws.String(subnetID),
}, testAccountID)
require.NoError(t, err)
eniID := *out.NetworkInterface.NetworkInterfaceId
ip := *out.NetworkInterface.PrivateIpAddress
require.Equal(t, eniID, readIndexENIID(t, kv, vpcID, ip))
_, err = svc.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{
NetworkInterfaceId: aws.String(eniID),
}, testAccountID)
require.NoError(t, err)
assert.Equal(t, "", readIndexENIID(t, kv, vpcID, ip))
}
// Regression: ENI lifecycle must not panic or error when no index is wired
// (IMDS-less deployments, focused tests).
func TestCreateDeleteNetworkInterface_NoIndexWired(t *testing.T) {
svc := setupTestVPCService(t)
vpcID := createTestVPC(t, svc, "10.0.0.0/16")
subnetID := createTestSubnet(t, svc, vpcID, "10.0.1.0/24")
out, err := svc.CreateNetworkInterface(&ec2.CreateNetworkInterfaceInput{
SubnetId: aws.String(subnetID),
}, testAccountID)
require.NoError(t, err)
_, err = svc.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{
NetworkInterfaceId: out.NetworkInterface.NetworkInterfaceId,
}, testAccountID)
require.NoError(t, err)
}
spinifex/handlers/ec2/vpc/invariants_test.go
+72
View File
@@ -0,0 +1,72 @@
package handlers_ec2_vpc
import (
"encoding/json"
"net"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// IMDS-datapath invariant.
// TestI4_ENIIndexBucketShape asserts the spinifex-network-eni-by-vpc-ip reverse
// index keeps its contract: keys parse as "vpcID/ip" with a valid IP, values
// carry a non-empty eni_id, and no mutable/denormalised fields creep back into
// the value. The index is identity-only (eni_id + the immutable account_id) by
// design — every mutable field (IPs, MAC, instance ID, profile ARN) is read
// live off the ENIRecord + instance record at request time. A denormalised
// field here resurrects the staleness class the design deliberately removed:
// an IMDS handler trusting a stale profile ARN would mint credentials for the
// wrong role.
func TestI4_ENIIndexBucketShape(t *testing.T) {
_, nc := setupTestVPCServiceWithNC(t)
kv := openTestENIByIPBucket(t, nc)
idx := NewENIByIPIndex(kv)
// Seed a couple of entries through the real writer.
require.NoError(t, idx.Put("vpc-aaaaaaaa", "10.0.1.5", "eni-aaa", "111122223333"))
require.NoError(t, idx.Put("vpc-bbbbbbbb", "10.0.2.9", "eni-bbb", "444455556666"))
// allowedFields is the closed set the value JSON may contain. Adding a field
// to eniByIPValue without updating this set fails the test by design — a
// reviewer must justify any new field as immutable identity, not cached
// mutable state.
allowedFields := map[string]struct{}{"eni_id": {}, "account_id": {}}
keys, err := kv.Keys()
require.NoError(t, err)
require.NotEmpty(t, keys)
for _, key := range keys {
// Skip migration framework metadata (e.g. _version).
if strings.HasPrefix(key, "_") {
continue
}
vpcID, ip, ok := strings.Cut(key, "/")
require.Truef(t, ok, "key %q does not parse as vpcID/ip", key)
assert.NotEmptyf(t, vpcID, "key %q has empty vpcID", key)
assert.NotNilf(t, net.ParseIP(ip), "key %q has invalid IP %q", key, ip)
entry, err := kv.Get(key)
require.NoError(t, err)
var raw map[string]json.RawMessage
require.NoError(t, json.Unmarshal(entry.Value(), &raw))
var v struct {
ENIId string `json:"eni_id"`
}
require.NoError(t, json.Unmarshal(entry.Value(), &v))
assert.NotEmptyf(t, v.ENIId, "index entry %q has empty eni_id", key)
for field := range raw {
_, allowed := allowedFields[field]
assert.Truef(t, allowed, "index entry %q carries unexpected field %q: the "+
"eni-by-vpc-ip index is IP→ENI identity only — denormalised mutable "+
"fields reintroduce the staleness class the design removed", key, field)
}
}
}
spinifex/handlers/ec2/vpc/service_impl.go
+12
View File
@@ -82,6 +82,11 @@ type VPCServiceImpl struct {
// Optional: injected after construction for public IP cleanup in DeleteNetworkInterface.
externalIPAM *ExternalIPAM
eipKV nats.KeyValue
// Optional: injected after construction. Maintains the eni-by-vpc-ip
// reverse index the IMDS handler reads. Nil-safe — focused tests and
// IMDS-less deployments leave it unset.
eniIndex *ENIByIPIndex
}
// SetExternalIPAM injects external IPAM and EIP KV store references so that
@@ -91,6 +96,13 @@ func (s *VPCServiceImpl) SetExternalIPAM(ipam *ExternalIPAM, eipKV nats.KeyValue
s.eipKV = eipKV
}
// SetENIByIPIndex injects the eni-by-vpc-ip reverse-index writer so
// CreateNetworkInterface / DeleteNetworkInterface keep the IMDS source-IP→ENI
// lookup index in sync. Optional: when unset, the index is simply not written.
func (s *VPCServiceImpl) SetENIByIPIndex(index *ENIByIPIndex) {
s.eniIndex = index
}
// localAZ returns the node's local availability zone, sourced from
// s.config.AZ. Returns "" when no config is wired (mostly test paths),
// which the reconciler treats as a legacy record matching every AZ.
spinifex/handlers/iam/imds_rpc.go
+57
View File
@@ -0,0 +1,57 @@
package handlers_iam
import (
"github.com/aws/aws-sdk-go/service/iam"
"github.com/mulgadc/spinifex/spinifex/utils"
"github.com/nats-io/nats.go"
)
// imdsResponderQueue load-balances the IMDS control-plane RPCs across every
// awsgw in the cluster, matching the cluster-wide worker convention.
const imdsResponderQueue = "spinifex-workers"
// SubjectResolveInstanceProfile and SubjectGetRole are the internal, ACL-gated
// subjects the IMDS handler (in vpcd) calls to resolve an instance profile and its
// role from awsgw's in-process IAM. Only these two read RPCs are surfaced.
const (
SubjectResolveInstanceProfile = "imds.iam.resolve_instance_profile"
SubjectGetRole = "imds.iam.get_role"
)
// ResolveInstanceProfileRequest is the SubjectResolveInstanceProfile payload.
type ResolveInstanceProfileRequest struct {
AccountID string `json:"account_id"`
NameOrARN string `json:"name_or_arn"`
}
// GetRoleRequest is the SubjectGetRole payload.
type GetRoleRequest struct {
AccountID string `json:"account_id"`
Input *iam.GetRoleInput `json:"input"`
}
// SubscribeIMDSResponders wires the profile/role responders onto nc,
// queue-grouped so any awsgw can answer. The returned subscriptions are for
// caller-side cleanup.
func (s *IAMServiceImpl) SubscribeIMDSResponders(nc *nats.Conn) ([]*nats.Subscription, error) {
profileSub, err := nc.QueueSubscribe(SubjectResolveInstanceProfile, imdsResponderQueue, func(msg *nats.Msg) {
utils.ServeNATSRequest(msg, func(req *ResolveInstanceProfileRequest) (*InstanceProfile, error) {
return s.ResolveInstanceProfile(req.AccountID, req.NameOrARN)
})
})
if err != nil {
return nil, err
}
roleSub, err := nc.QueueSubscribe(SubjectGetRole, imdsResponderQueue, func(msg *nats.Msg) {
utils.ServeNATSRequest(msg, func(req *GetRoleRequest) (*iam.GetRoleOutput, error) {
return s.GetRole(req.AccountID, req.Input)
})
})
if err != nil {
_ = profileSub.Unsubscribe()
return nil, err
}
return []*nats.Subscription{profileSub, roleSub}, nil
}
spinifex/handlers/imds/buckets.go
+72
View File
@@ -0,0 +1,72 @@
package handlers_imds
import (
"fmt"
"github.com/mulgadc/spinifex/spinifex/migrate"
"github.com/nats-io/nats.go"
)
const (
// KVBucketIMDSVPCVeth is the canonical record that a VPC has IMDS plumbing
// installed. Keyed by vpcID, the value carries the per-VPC veth/LSP spec the
// BindManager replays to materialise host-side state on every chassis.
KVBucketIMDSVPCVeth = "spinifex-network-imds-vpc-veth"
KVBucketIMDSVPCVethVersion = 1
// KVBucketENIByVPCIP is a pure reverse index (vpcID/ip → eniID) so the IMDS
// handler can resolve a request's datapath-attested source IP to an ENI in
// one KV read instead of an O(N) scan of the ENI bucket.
KVBucketENIByVPCIP = "spinifex-network-eni-by-vpc-ip"
KVBucketENIByVPCIPVersion = 1
)
// InitBuckets opens (or creates) both IMDS KV buckets and runs any pending
// migrations. History is fixed at 1: both are write-once-per-key, so retained
// revisions only waste storage and lengthen tombstone lifetime.
func InitBuckets(js nats.JetStreamContext, replicas int) (vpcVeth, eniByIP nats.KeyValue, err error) {
vpcVeth, err = initIMDSBucket(js, KVBucketIMDSVPCVeth, KVBucketIMDSVPCVethVersion, replicas)
if err != nil {
return nil, nil, err
}
eniByIP, err = initIMDSBucket(js, KVBucketENIByVPCIP, KVBucketENIByVPCIPVersion, replicas)
if err != nil {
return nil, nil, err
}
return vpcVeth, eniByIP, nil
}
// InitENIByIPBucket opens (or creates) just the eni-by-vpc-ip reverse-index
// bucket, for the ENI controller (in the daemon), which may start before the
// IMDS service and so needs an init path independent of InitBuckets. Idempotent.
func InitENIByIPBucket(js nats.JetStreamContext, replicas int) (nats.KeyValue, error) {
return initIMDSBucket(js, KVBucketENIByVPCIP, KVBucketENIByVPCIPVersion, replicas)
}
// initIMDSBucket opens (or creates) a single IMDS KV bucket and runs any
// pending migrations.
func initIMDSBucket(js nats.JetStreamContext, bucket string, version, replicas int) (nats.KeyValue, error) {
if replicas < 1 {
replicas = 1
}
kv, err := js.CreateKeyValue(&nats.KeyValueConfig{
Bucket: bucket,
History: 1,
Replicas: replicas,
})
if err != nil {
kv, err = js.KeyValue(bucket)
if err != nil {
return nil, fmt.Errorf("open %s bucket: %w", bucket, err)
}
}
if err := migrate.DefaultRegistry.RunKV(bucket, kv, version); err != nil {
return nil, fmt.Errorf("migrate %s: %w", bucket, err)
}
return kv, nil
}
spinifex/handlers/imds/buckets_test.go
+31
View File
@@ -0,0 +1,31 @@
package handlers_imds
import (
"testing"
"github.com/mulgadc/spinifex/spinifex/testutil"
"github.com/stretchr/testify/require"
)
func TestInitENIByIPBucket(t *testing.T) {
_, _, js := testutil.StartTestJetStream(t)
kv, err := InitENIByIPBucket(js, 1)
require.NoError(t, err)
require.NotNil(t, kv)
}
// InitBuckets clamps replicas < 1 to 1, and a second call converges on the
// open-existing fallback rather than failing on the already-created bucket.
func TestInitBuckets_ReplicaClampAndIdempotent(t *testing.T) {
_, _, js := testutil.StartTestJetStream(t)
vpcVeth, eniByIP, err := InitBuckets(js, 0)
require.NoError(t, err)
require.NotNil(t, vpcVeth)
require.NotNil(t, eniByIP)
vpcVeth2, eniByIP2, err := InitBuckets(js, 1)
require.NoError(t, err)
require.NotNil(t, vpcVeth2)
require.NotNil(t, eniByIP2)
}
spinifex/handlers/imds/constants.go
+6
View File
@@ -0,0 +1,6 @@
package handlers_imds
// MetaDataServerIP is the link-local address the EC2 Instance Metadata Service is
// served on, identical to AWS. Exported so the network topology layer can set it as
// options:arp_proxy on every subnet router LSP, answering ARP for any guest.
const MetaDataServerIP = "169.254.169.254"
spinifex/handlers/imds/credentials.go
+120
View File
@@ -0,0 +1,120 @@
package handlers_imds
import (
"encoding/json"
"fmt"
"sync"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/sts"
)
const (
// credDurationSeconds is the lifetime of instance-role credentials. 1 hour
// is the AWS default and matches the storage-backed STS v1 design.
credDurationSeconds = 3600
// credRefreshWindow is how far before expiry a cached credential is treated
// as stale and re-minted. AWS SDKs refresh at the same 5-minute mark.
credRefreshWindow = 5 * time.Minute
)
// instanceCredential is the JSON body served at
// /latest/meta-data/iam/security-credentials/<role>, byte-for-byte matching the
// AWS shape so unmodified SDK credential providers parse it.
type instanceCredential struct {
Code string `json:"Code"`
LastUpdated string `json:"LastUpdated"`
Type string `json:"Type"`
AccessKeyId string `json:"AccessKeyId"`
SecretAccessKey string `json:"SecretAccessKey"`
Token string `json:"Token"`
Expiration string `json:"Expiration"`
AccountId string `json:"AccountId"`
}
// stsAssumer is the narrow slice of STSService the IMDS credential path needs:
// the in-process, EC2-service-principal AssumeRole entry point. Narrowed to an
// interface so the cred cache is unit-testable with a fake.
type stsAssumer interface {
AssumeRoleForInstance(accountID, roleARN, instanceID string, durationSeconds int64) (*sts.AssumeRoleOutput, error)
}
// cachedCred holds a rendered credential body and the instant at which it should
// be re-minted (expiry minus the refresh window).
type cachedCred struct {
body []byte
refreshAt time.Time
}
// credCache memoises minted instance-role credentials per (ENI, role) until the
// refresh window. It keeps STS off the hot path: the typical request returns a
// cached body, and only a near-expiry request triggers a fresh assume.
type credCache struct {
sts stsAssumer
mu sync.Mutex
entries map[string]cachedCred
}
func newCredCache(assumer stsAssumer) *credCache {
return &credCache{sts: assumer, entries: make(map[string]cachedCred)}
}
// get returns the credential JSON body for an instance's role, minting fresh
// credentials via STS when the cache is cold or within the refresh window. The
// cache key is (eniID, roleName) so two ENIs sharing a role don't collide.
func (c *credCache) get(eni *eniFacts, roleName, roleARN string, now time.Time) ([]byte, error) {
key := eni.eniID + "\x00" + roleName
c.mu.Lock()
if entry, ok := c.entries[key]; ok && now.Before(entry.refreshAt) {
body := entry.body
c.mu.Unlock()
return body, nil
}
c.mu.Unlock()
out, err := c.sts.AssumeRoleForInstance(eni.accountID, roleARN, eni.instanceID, credDurationSeconds)
if err != nil {
return nil, err
}
if out == nil || out.Credentials == nil {
return nil, fmt.Errorf("assume role for instance %s returned no credentials", eni.instanceID)
}
expiration := aws.TimeValue(out.Credentials.Expiration)
body, err := json.Marshal(instanceCredential{
Code: "Success",
LastUpdated: now.UTC().Format(time.RFC3339),
Type: "AWS-HMAC",
AccessKeyId: aws.StringValue(out.Credentials.AccessKeyId),
SecretAccessKey: aws.StringValue(out.Credentials.SecretAccessKey),
Token: aws.StringValue(out.Credentials.SessionToken),
Expiration: expiration.UTC().Format(time.RFC3339),
AccountId: eni.accountID,
})
if err != nil {
return nil, fmt.Errorf("marshal instance credential: %w", err)
}
c.mu.Lock()
c.entries[key] = cachedCred{body: body, refreshAt: expiration.Add(-credRefreshWindow)}
c.mu.Unlock()
return body, nil
}
// sweep drops fully-expired entries (past refreshAt + the refresh window, i.e.
// past the credential's actual expiry), bounding the cache against ENI churn:
// a terminated instance's entry is never overwritten and would otherwise leak.
// Evicting a live entry would only force a re-mint, so this is always safe.
func (c *credCache) sweep(now time.Time) {
c.mu.Lock()
defer c.mu.Unlock()
for key, entry := range c.entries {
if now.After(entry.refreshAt.Add(credRefreshWindow)) {
delete(c.entries, key)
}
}
}
spinifex/handlers/imds/credentials_test.go
+139
View File
@@ -0,0 +1,139 @@
package handlers_imds
import (
"encoding/json"
"errors"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/sts"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// fakeAssumer is a programmable stsAssumer that records call counts.
type fakeAssumer struct {
out *sts.AssumeRoleOutput
err error
calls int
}
func (f *fakeAssumer) AssumeRoleForInstance(_, _, _ string, _ int64) (*sts.AssumeRoleOutput, error) {
f.calls++
return f.out, f.err
}
func assumeOutput(akid string, exp time.Time) *sts.AssumeRoleOutput {
return &sts.AssumeRoleOutput{
Credentials: &sts.Credentials{
AccessKeyId: aws.String(akid),
SecretAccessKey: aws.String("secret"),
SessionToken: aws.String("token"),
Expiration: aws.Time(exp),
},
}
}
func TestCredCache_MintsAndRendersAWSShape(t *testing.T) {
now := time.Unix(1_700_000_000, 0).UTC()
exp := now.Add(time.Hour)
assumer := &fakeAssumer{out: assumeOutput("ASIAEXAMPLE", exp)}
cache := newCredCache(assumer)
eni := &eniFacts{eniID: "eni-aaa", accountID: "111122223333", instanceID: "i-123"}
body, err := cache.get(eni, "app-role", "arn:aws:iam::111122223333:role/app-role", now)
require.NoError(t, err)
var cred instanceCredential
require.NoError(t, json.Unmarshal(body, &cred))
assert.Equal(t, "Success", cred.Code)
assert.Equal(t, "AWS-HMAC", cred.Type)
assert.Equal(t, "ASIAEXAMPLE", cred.AccessKeyId)
assert.Equal(t, "secret", cred.SecretAccessKey)
assert.Equal(t, "token", cred.Token)
assert.Equal(t, "111122223333", cred.AccountId)
assert.Equal(t, exp.Format(time.RFC3339), cred.Expiration)
}
func TestCredCache_ServesCachedUntilRefreshWindow(t *testing.T) {
now := time.Unix(1_700_000_000, 0).UTC()
exp := now.Add(time.Hour)
assumer := &fakeAssumer{out: assumeOutput("ASIAEXAMPLE", exp)}
cache := newCredCache(assumer)
eni := &eniFacts{eniID: "eni-aaa", accountID: "111122223333", instanceID: "i-123"}
_, err := cache.get(eni, "app-role", "arn:aws:iam::111122223333:role/app-role", now)
require.NoError(t, err)
// Still well outside the 5-minute refresh window → cached, no new mint.
_, err = cache.get(eni, "app-role", "arn:aws:iam::111122223333:role/app-role", now.Add(50*time.Minute))
require.NoError(t, err)
assert.Equal(t, 1, assumer.calls, "second request inside the refresh window must hit the cache")
}
func TestCredCache_RemintsInsideRefreshWindow(t *testing.T) {
now := time.Unix(1_700_000_000, 0).UTC()
exp := now.Add(time.Hour)
assumer := &fakeAssumer{out: assumeOutput("ASIAEXAMPLE", exp)}
cache := newCredCache(assumer)
eni := &eniFacts{eniID: "eni-aaa", accountID: "111122223333", instanceID: "i-123"}
_, err := cache.get(eni, "app-role", "arn:aws:iam::111122223333:role/app-role", now)
require.NoError(t, err)
// 56 minutes in → within 5 min of expiry → re-mint.
assumer.out = assumeOutput("ASIAREFRESHED", now.Add(2*time.Hour))
_, err = cache.get(eni, "app-role", "arn:aws:iam::111122223333:role/app-role", now.Add(56*time.Minute))
require.NoError(t, err)
assert.Equal(t, 2, assumer.calls, "request inside the refresh window must re-mint")
}
func TestCredCache_PropagatesAssumeError(t *testing.T) {
now := time.Unix(1_700_000_000, 0).UTC()
assumer := &fakeAssumer{err: errors.New("AccessDenied")}
cache := newCredCache(assumer)
eni := &eniFacts{eniID: "eni-aaa", accountID: "111122223333", instanceID: "i-123"}
_, err := cache.get(eni, "app-role", "arn:aws:iam::111122223333:role/app-role", now)
require.Error(t, err)
}
func TestCredCache_SweepEvictsExpiredOnly(t *testing.T) {
now := time.Unix(1_700_000_000, 0).UTC()
assumer := &fakeAssumer{out: assumeOutput("ASIAEXAMPLE", now.Add(time.Hour))}
cache := newCredCache(assumer)
stale := &eniFacts{eniID: "eni-gone", accountID: "111122223333", instanceID: "i-old"}
live := &eniFacts{eniID: "eni-live", accountID: "111122223333", instanceID: "i-new"}
_, err := cache.get(stale, "app-role", "arn:aws:iam::111122223333:role/app-role", now)
require.NoError(t, err)
// Mint the live entry an hour later so its expiry is still in the future
// when we sweep at the original expiry boundary.
assumer.out = assumeOutput("ASIALIVE", now.Add(2*time.Hour))
_, err = cache.get(live, "app-role", "arn:aws:iam::111122223333:role/app-role", now.Add(time.Hour))
require.NoError(t, err)
// Sweep just past the stale entry's full expiry (mint + 1h). The live entry
// expires at mint + 2h, so it must survive.
cache.sweep(now.Add(time.Hour).Add(time.Second))
cache.mu.Lock()
_, staleExists := cache.entries[stale.eniID+"\x00"+"app-role"]
_, liveExists := cache.entries[live.eniID+"\x00"+"app-role"]
cache.mu.Unlock()
assert.False(t, staleExists, "expired entry must be swept")
assert.True(t, liveExists, "still-valid entry must survive the sweep")
}
func TestCredCache_PerENIPerRoleKeying(t *testing.T) {
now := time.Unix(1_700_000_000, 0).UTC()
exp := now.Add(time.Hour)
assumer := &fakeAssumer{out: assumeOutput("ASIAEXAMPLE", exp)}
cache := newCredCache(assumer)
a := &eniFacts{eniID: "eni-aaa", accountID: "111122223333", instanceID: "i-1"}
b := &eniFacts{eniID: "eni-bbb", accountID: "111122223333", instanceID: "i-2"}
_, _ = cache.get(a, "app-role", "arn:aws:iam::111122223333:role/app-role", now)
_, _ = cache.get(b, "app-role", "arn:aws:iam::111122223333:role/app-role", now)
assert.Equal(t, 2, assumer.calls, "distinct ENIs must not share a cache entry")
}
spinifex/handlers/imds/instance_lookup.go
+87
View File
@@ -0,0 +1,87 @@
package handlers_imds
import (
"encoding/base64"
"fmt"
"log/slog"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
gateway_ec2_instance "github.com/mulgadc/spinifex/spinifex/gateway/ec2/instance"
"github.com/nats-io/nats.go"
)
// natsInstanceLookup resolves the instance-only metadata fields via the same
// account-scoped NATS fan-out the gateway's DescribeInstances path uses, since
// the instance record lives in each daemon's in-memory manager, not a central KV.
type natsInstanceLookup struct {
nc *nats.Conn
expectedNodes int
}
var _ instanceLookup = (*natsInstanceLookup)(nil)
func (l *natsInstanceLookup) describe(accountID, instanceID string) (*instanceFacts, error) {
out, err := gateway_ec2_instance.DescribeInstances(&ec2.DescribeInstancesInput{
InstanceIds: []*string{aws.String(instanceID)},
}, l.nc, l.expectedNodes, accountID)
if err != nil {
return nil, fmt.Errorf("describe instance %s: %w", instanceID, err)
}
inst := firstInstance(out)
if inst == nil {
// The ENI references an instance the daemons no longer report (mid
// terminate, or not yet visible). Treated as a miss → 404.
return nil, nil
}
facts := &instanceFacts{
instanceType: aws.StringValue(inst.InstanceType),
imageID: aws.StringValue(inst.ImageId),
}
if inst.IamInstanceProfile != nil {
facts.iamInstanceProfileArn = aws.StringValue(inst.IamInstanceProfile.Arn)
}
facts.userData = l.userData(accountID, instanceID)
return facts, nil
}
// userData fetches the base64 user-data blob via DescribeInstanceAttribute and
// decodes it. A miss or decode failure yields nil — /latest/user-data then 404s,
// matching AWS for instances launched without user-data.
func (l *natsInstanceLookup) userData(accountID, instanceID string) []byte {
attr, err := gateway_ec2_instance.DescribeInstanceAttribute(&ec2.DescribeInstanceAttributeInput{
InstanceId: aws.String(instanceID),
Attribute: aws.String("userData"),
}, l.nc, l.expectedNodes, accountID)
if err != nil || attr == nil || attr.UserData == nil || attr.UserData.Value == nil {
return nil
}
decoded, err := base64.StdEncoding.DecodeString(aws.StringValue(attr.UserData.Value))
if err != nil {
slog.Warn("IMDS: failed to decode instance user-data", "instance_id", instanceID, "err", err)
return nil
}
return decoded
}
// firstInstance returns the first instance in a DescribeInstances response, or
// nil when the response carries none.
func firstInstance(out *ec2.DescribeInstancesOutput) *ec2.Instance {
if out == nil {
return nil
}
for _, res := range out.Reservations {
if res == nil {
continue
}
for _, inst := range res.Instances {
if inst != nil {
return inst
}
}
}
return nil
}
spinifex/handlers/imds/instance_lookup_test.go
+33
View File
@@ -0,0 +1,33 @@
package handlers_imds
import (
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestFirstInstance_NilAndEmpty(t *testing.T) {
assert.Nil(t, firstInstance(nil))
assert.Nil(t, firstInstance(&ec2.DescribeInstancesOutput{}))
assert.Nil(t, firstInstance(&ec2.DescribeInstancesOutput{
Reservations: []*ec2.Reservation{nil, {Instances: []*ec2.Instance{nil}}},
}))
}
// firstInstance skips nil reservations and nil instance slots and returns the
// first concrete instance it finds.
func TestFirstInstance_SkipsNilsAndReturnsFirst(t *testing.T) {
out := &ec2.DescribeInstancesOutput{
Reservations: []*ec2.Reservation{
nil,
{Instances: []*ec2.Instance{nil}},
{Instances: []*ec2.Instance{{InstanceId: aws.String("i-first")}, {InstanceId: aws.String("i-second")}}},
},
}
got := firstInstance(out)
require.NotNil(t, got)
assert.Equal(t, "i-first", aws.StringValue(got.InstanceId))
}
spinifex/handlers/imds/metadata.go
+387
View File
@@ -0,0 +1,387 @@
package handlers_imds
import (
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"strconv"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/iam"
"github.com/mulgadc/spinifex/spinifex/awserrors"
"github.com/mulgadc/spinifex/spinifex/utils"
)
// HTTP paths. IMDSv2 token issuance is a PUT; everything else is a token-gated
// GET under /latest/.
const (
pathToken = "/latest/api/token" //nolint:gosec // URL path, not a credential
prefixMetaData = "/latest/meta-data/"
pathMetaDataRoot = "/latest/meta-data"
pathUserData = "/latest/user-data"
prefixSecurityCreds = "/latest/meta-data/iam/security-credentials/" //nolint:gosec // URL path, not a credential
pathSecurityCredsDir = "/latest/meta-data/iam/security-credentials" //nolint:gosec // URL path, not a credential
hdrToken = "X-Aws-Ec2-Metadata-Token" //nolint:gosec // HTTP header name, not a credential
hdrTokenTTL = "X-Aws-Ec2-Metadata-Token-Ttl-Seconds" //nolint:gosec // HTTP header name, not a credential
hdrForwardedFor = "X-Forwarded-For"
)
// rejectForwarded enforces AWS IMDS's SSRF defence: any request carrying an
// X-Forwarded-For header is refused with 403 before token or identity checks,
// blocking the classic "trick a server-side app into relaying metadata" attack.
func rejectForwarded(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get(hdrForwardedFor) != "" {
w.WriteHeader(http.StatusForbidden)
return
}
next.ServeHTTP(w, r)
})
}
// handleToken serves PUT /latest/api/token. It binds a fresh token to the
// requesting ENI (resolved from the datapath-attested source IP) and returns it
// as a text/plain body, matching AWS IMDSv2.
func (s *IMDSServiceImpl) handleToken(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPut {
w.Header().Set("Allow", http.MethodPut)
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
ttl, ok := parseTokenTTL(r.Header.Get(hdrTokenTTL))
if !ok {
w.WriteHeader(http.StatusBadRequest)
return
}
eni := s.resolveCaller(r)
if eni == nil {
w.WriteHeader(http.StatusNotFound)
return
}
token, err := s.tokens.issue(eni.eniID, ttl, s.now())
if err != nil {
slog.Error("IMDS: token issuance failed", "eni_id", eni.eniID, "err", err)
w.WriteHeader(http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "text/plain")
w.Header().Set(hdrTokenTTL, strconv.Itoa(int(ttl.Seconds())))
_, _ = w.Write([]byte(token))
}
// handleMetadata serves every token-gated read path. It resolves the caller's
// ENI, enforces the IMDSv2 token, then dispatches on the request path.
func (s *IMDSServiceImpl) handleMetadata(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
w.Header().Set("Allow", http.MethodGet)
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
eni := s.resolveCaller(r)
if eni == nil {
w.WriteHeader(http.StatusNotFound)
return
}
// IMDSv2: a valid, ENI-bound token is mandatory on every read. A missing,
// unknown, expired, or wrong-ENI token is an indistinguishable 401 so a
// guest cannot probe which tokens exist.
if !s.tokens.validate(r.Header.Get(hdrToken), eni.eniID, s.now()) {
w.WriteHeader(http.StatusUnauthorized)
return
}
s.dispatch(w, r, eni)
}
// resolveCaller maps the request's (vpcID-from-context, source-IP) to the
// owning ENI, or nil when no mapping exists (logged for operator triage). A
// backend error is also surfaced as nil → the caller 404s, never 500s, matching
// AWS's opaque "eventually available" boot posture.
func (s *IMDSServiceImpl) resolveCaller(r *http.Request) *eniFacts {
vpcID, _ := r.Context().Value(ctxKeyVPCID).(string)
srcIP := utils.ClientIP(r.RemoteAddr)
if vpcID == "" || srcIP == "" {
slog.Warn("IMDS: request missing VPC context or source IP", "vpc_id", vpcID, "remote_addr", r.RemoteAddr)
return nil
}
eni, err := s.resolver.resolveENI(vpcID, srcIP)
if err != nil {
slog.Error("IMDS: ENI resolution failed", "vpc_id", vpcID, "src_ip", srcIP, "err", err)
return nil
}
if eni == nil {
slog.Warn("IMDS: no ENI for source IP", "vpc_id", vpcID, "src_ip", srcIP)
return nil
}
return eni
}
// dispatch routes a token-validated GET to the right metadata producer.
func (s *IMDSServiceImpl) dispatch(w http.ResponseWriter, r *http.Request, eni *eniFacts) {
path := r.URL.Path
// Credential fetch for a specific role: /iam/security-credentials/<role>.
if strings.HasPrefix(path, prefixSecurityCreds) && len(path) > len(prefixSecurityCreds) {
s.serveRoleCredentials(w, eni, strings.TrimPrefix(path, prefixSecurityCreds))
return
}
switch path {
case pathMetaDataRoot, prefixMetaData:
writeText(w, "ami-id\nhostname\niam/\ninstance-id\ninstance-type\nlocal-hostname\nlocal-ipv4\nmac\nplacement/\npublic-ipv4\nsecurity-groups")
case prefixMetaData + "instance-id":
writeText(w, eni.instanceID)
case prefixMetaData + "local-ipv4":
writeText(w, eni.privateIP)
case prefixMetaData + "public-ipv4":
writeText(w, eni.publicIP)
case prefixMetaData + "mac":
writeText(w, eni.mac)
case prefixMetaData + "security-groups":
writeText(w, strings.Join(s.resolver.resolveSGNames(eni.accountID, eni.securityGroupIDs), "\n"))
case prefixMetaData + "hostname", prefixMetaData + "local-hostname":
writeText(w, synthHostname(eni.privateIP, regionFromAZ(eni.availabilityZone)))
case prefixMetaData + "placement", prefixMetaData + "placement/":
writeText(w, "availability-zone\nregion")
case prefixMetaData + "placement/availability-zone":
writeText(w, eni.availabilityZone)
case prefixMetaData + "placement/region":
writeText(w, regionFromAZ(eni.availabilityZone))
case prefixMetaData + "instance-type":
s.serveInstanceField(w, eni, func(i *instanceFacts) string { return i.instanceType })
case prefixMetaData + "ami-id":
s.serveInstanceField(w, eni, func(i *instanceFacts) string { return i.imageID })
case prefixMetaData + "iam", prefixMetaData + "iam/":
writeText(w, "info\nsecurity-credentials/")
case prefixMetaData + "iam/info":
s.serveIAMInfo(w, eni)
case pathSecurityCredsDir, prefixSecurityCreds:
s.serveSecurityCredentialsList(w, eni)
case pathUserData:
s.serveUserData(w, eni)
default:
// /latest/dynamic/instance-identity/document and
// /latest/meta-data/network/interfaces/... are out of scope in v1.
w.WriteHeader(http.StatusNotFound)
}
}
// serveInstanceField resolves the instance record and writes one of its
// string fields, 404ing when the instance is no longer visible.
func (s *IMDSServiceImpl) serveInstanceField(w http.ResponseWriter, eni *eniFacts, field func(*instanceFacts) string) {
inst := s.instanceFor(w, eni)
if inst == nil {
return
}
writeText(w, field(inst))
}
// serveUserData writes the instance's user-data, or 404 when there is none —
// matching AWS for instances launched without user-data.
func (s *IMDSServiceImpl) serveUserData(w http.ResponseWriter, eni *eniFacts) {
inst := s.instanceFor(w, eni)
if inst == nil {
return
}
if len(inst.userData) == 0 {
w.WriteHeader(http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "text/plain")
_, _ = w.Write(inst.userData)
}
// serveIAMInfo writes {InstanceProfileArn, InstanceProfileId}, 404 when the
// instance has no instance profile, or 500 when the profile can't be resolved.
func (s *IMDSServiceImpl) serveIAMInfo(w http.ResponseWriter, eni *eniFacts) {
profile, err := s.profileFor(eni)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
if profile == nil {
w.WriteHeader(http.StatusNotFound)
return
}
writeJSON(w, map[string]string{
"InstanceProfileArn": profile.ARN,
"InstanceProfileId": profile.InstanceProfileID,
})
}
// serveSecurityCredentialsList writes the role name(s) under the profile, one
// per line. No profile/role is an empty 200 (matching AWS); a backend failure is
// a 500, never an empty 200 that would silently strip the instance's creds.
func (s *IMDSServiceImpl) serveSecurityCredentialsList(w http.ResponseWriter, eni *eniFacts) {
profile, err := s.profileFor(eni)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
if profile == nil || profile.RoleName == "" {
writeText(w, "")
return
}
writeText(w, profile.RoleName)
}
// serveRoleCredentials mints (or serves cached) credentials for the named role.
// AWS only accepts the actual role name as the path parameter, not the profile
// name, so a mismatch is 404; a backend failure resolving the profile is 500.
func (s *IMDSServiceImpl) serveRoleCredentials(w http.ResponseWriter, eni *eniFacts, roleParam string) {
profile, err := s.profileFor(eni)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
if profile == nil || profile.RoleName == "" || roleParam != profile.RoleName {
w.WriteHeader(http.StatusNotFound)
return
}
role, err := s.iam.GetRole(eni.accountID, &iam.GetRoleInput{RoleName: aws.String(profile.RoleName)})
if err != nil || role == nil || role.Role == nil || role.Role.Arn == nil {
slog.Error("IMDS: resolve role ARN failed", "account_id", eni.accountID, "role", profile.RoleName, "err", err)
w.WriteHeader(http.StatusInternalServerError)
return
}
body, err := s.creds.get(eni, profile.RoleName, *role.Role.Arn, s.now())
if err != nil {
// AWS surfaces a trust-policy failure as a JSON body with Code:"Failed";
// the SDK reports it as "EC2RoleProvider failed". Mirror that rather than
// a bare HTTP error so the SDK error message is AWS-accurate.
slog.Warn("IMDS: AssumeRoleForInstance failed", "account_id", eni.accountID, "role", profile.RoleName, "instance_id", eni.instanceID, "err", err)
writeJSON(w, map[string]string{
"Code": "Failed",
"LastUpdated": s.now().UTC().Format(time.RFC3339),
"Message": "failed to assume instance role",
})
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write(body)
}
// instanceFor resolves the instance record for an ENI, writing the appropriate
// HTTP error and returning nil on miss/failure so callers can early-return.
func (s *IMDSServiceImpl) instanceFor(w http.ResponseWriter, eni *eniFacts) *instanceFacts {
inst, err := s.resolver.resolveInstance(eni)
if err != nil {
slog.Error("IMDS: instance resolution failed", "instance_id", eni.instanceID, "err", err)
w.WriteHeader(http.StatusInternalServerError)
return nil
}
if inst == nil {
w.WriteHeader(http.StatusNotFound)
return nil
}
return inst
}
// profileFor resolves the instance's IAM instance profile. Returns (profile, nil)
// when resolvable, (nil, nil) when genuinely absent (404/empty), (nil, err) on
// backend failure (500) — collapsing err into absent would silently drop creds.
func (s *IMDSServiceImpl) profileFor(eni *eniFacts) (*resolvedProfile, error) {
inst, err := s.resolver.resolveInstance(eni)
if err != nil {
slog.Error("IMDS: instance resolution failed", "instance_id", eni.instanceID, "err", err)
return nil, err
}
if inst == nil || inst.iamInstanceProfileArn == "" {
return nil, nil
}
profile, err := s.iam.ResolveInstanceProfile(eni.accountID, inst.iamInstanceProfileArn)
if err != nil {
if err.Error() == awserrors.ErrorIAMNoSuchEntity {
// The profile was deleted out from under the instance. AWS treats this
// as "no instance role", not a backend fault — NoSuchEntity is raised
// only for a genuinely absent record, never a transient.
return nil, nil
}
slog.Error("IMDS: resolve instance profile failed", "account_id", eni.accountID, "arn", inst.iamInstanceProfileArn, "err", err)
return nil, err
}
if profile == nil {
return nil, nil
}
return &resolvedProfile{ARN: profile.ARN, InstanceProfileID: profile.InstanceProfileID, RoleName: profile.RoleName}, nil
}
// resolvedProfile is the slice of an IAM instance profile the metadata surface
// serves, decoupled from the handlers_iam type so this package needn't import
// it for the field access.
type resolvedProfile struct {
ARN string
InstanceProfileID string
RoleName string
}
// parseTokenTTL validates the X-aws-ec2-metadata-token-ttl-seconds header.
func parseTokenTTL(raw string) (time.Duration, bool) {
if raw == "" {
return 0, false
}
n, err := strconv.Atoi(raw)
if err != nil || n < tokenTTLMin || n > tokenTTLMax {
return 0, false
}
return time.Duration(n) * time.Second, true
}
// regionFromAZ derives the region from an availability-zone name by stripping
// the trailing AZ letter ("ap-southeast-2a" → "ap-southeast-2").
func regionFromAZ(az string) string {
if az == "" {
return ""
}
last := az[len(az)-1]
if last >= 'a' && last <= 'z' {
return az[:len(az)-1]
}
return az
}
// synthHostname builds the AWS-style internal hostname
// "ip-10-0-1-5.<region>.compute.internal" from a private IP and region.
func synthHostname(ip, region string) string {
if ip == "" || region == "" {
return ""
}
return fmt.Sprintf("ip-%s.%s.compute.internal", strings.ReplaceAll(ip, ".", "-"), region)
}
func writeText(w http.ResponseWriter, body string) {
w.Header().Set("Content-Type", "text/plain")
_, _ = io.WriteString(w, body)
}
func writeJSON(w http.ResponseWriter, v any) {
data, err := json.Marshal(v)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write(data)
}
// ctxKey is the unexported context-key type for values the bind manager threads
// into each request (the VPC ID the listener's veth maps to).
type ctxKey int
const ctxKeyVPCID ctxKey = iota
spinifex/handlers/imds/metadata_test.go
+418
View File
@@ -0,0 +1,418 @@
package handlers_imds
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/iam"
"github.com/aws/aws-sdk-go/service/sts"
"github.com/mulgadc/spinifex/spinifex/awserrors"
handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// ----- fakes -------------------------------------------------------------
type fakeResolver struct {
eni *eniFacts
eniErr error
inst *instanceFacts
instErr error
sgNames map[string]string // sg-id → group name; absent IDs fall back to the ID
}
func (f *fakeResolver) resolveENI(_, _ string) (*eniFacts, error) { return f.eni, f.eniErr }
func (f *fakeResolver) resolveInstance(_ *eniFacts) (*instanceFacts, error) {
return f.inst, f.instErr
}
func (f *fakeResolver) resolveSGNames(_ string, sgIDs []string) []string {
out := make([]string, len(sgIDs))
for i, id := range sgIDs {
if name, ok := f.sgNames[id]; ok {
out[i] = name
} else {
out[i] = id
}
}
return out
}
type fakeIAM struct {
profile *handlers_iam.InstanceProfile
profileErr error
roleARN string
roleErr error
}
func (f *fakeIAM) ResolveInstanceProfile(_, _ string) (*handlers_iam.InstanceProfile, error) {
return f.profile, f.profileErr
}
func (f *fakeIAM) GetRole(_ string, _ *iam.GetRoleInput) (*iam.GetRoleOutput, error) {
if f.roleErr != nil {
return nil, f.roleErr
}
return &iam.GetRoleOutput{Role: &iam.Role{Arn: aws.String(f.roleARN)}}, nil
}
const (
testVPC = "vpc-abc12345"
testIP = "10.0.1.5"
)
func testENI() *eniFacts {
return &eniFacts{
eniID: "eni-aaa",
accountID: "111122223333",
instanceID: "i-0123456789",
vpcID: testVPC,
subnetID: "subnet-1",
privateIP: testIP,
publicIP: "203.0.113.7",
mac: "02:11:22:33:44:55",
availabilityZone: "ap-southeast-2a",
securityGroupIDs: []string{"sg-1", "sg-2"},
}
}
// newTestService builds an IMDSServiceImpl with injected fakes and a fixed
// clock. The bind manager is left nil — the HTTP handler is exercised directly.
func newTestService(res eniResolver, fIAM profileLookup, assumer stsAssumer) (*IMDSServiceImpl, time.Time) {
now := time.Unix(1_700_000_000, 0).UTC()
return &IMDSServiceImpl{
resolver: res,
tokens: newTokenStore(),
creds: newCredCache(assumer),
iam: fIAM,
now: func() time.Time { return now },
}, now
}
// get issues a token-gated GET with the VPC context + source IP a real listener
// would have threaded in, plus the supplied token header.
func get(t *testing.T, h http.Handler, path, token string) *httptest.ResponseRecorder {
t.Helper()
req := httptest.NewRequest(http.MethodGet, "http://"+MetaDataServerIP+path, nil)
req.RemoteAddr = testIP + ":50000"
req = req.WithContext(context.WithValue(req.Context(), ctxKeyVPCID, testVPC))
if token != "" {
req.Header.Set(hdrToken, token)
}
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
return rec
}
// issueToken performs the PUT /latest/api/token handshake and returns the token.
func issueToken(t *testing.T, h http.Handler) string {
t.Helper()
req := httptest.NewRequest(http.MethodPut, "http://"+MetaDataServerIP+pathToken, nil)
req.RemoteAddr = testIP + ":50000"
req = req.WithContext(context.WithValue(req.Context(), ctxKeyVPCID, testVPC))
req.Header.Set(hdrTokenTTL, "60")
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
require.Equal(t, http.StatusOK, rec.Code)
require.NotEmpty(t, rec.Body.String())
return rec.Body.String()
}
// ----- token model ------------------------------------------------------
func TestHTTP_TokenlessGETRejected(t *testing.T) {
svc, _ := newTestService(&fakeResolver{eni: testENI()}, &fakeIAM{}, &fakeAssumer{})
h := svc.httpHandler()
rec := get(t, h, prefixMetaData+"instance-id", "")
assert.Equal(t, http.StatusUnauthorized, rec.Code)
assert.Empty(t, rec.Body.String())
}
func TestHTTP_TokenPUTValidation(t *testing.T) {
svc, _ := newTestService(&fakeResolver{eni: testENI()}, &fakeIAM{}, &fakeAssumer{})
h := svc.httpHandler()
for _, ttl := range []string{"", "0", "21601", "notanumber"} {
req := httptest.NewRequest(http.MethodPut, "http://"+MetaDataServerIP+pathToken, nil)
req.RemoteAddr = testIP + ":50000"
req = req.WithContext(context.WithValue(req.Context(), ctxKeyVPCID, testVPC))
if ttl != "" {
req.Header.Set(hdrTokenTTL, ttl)
}
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
assert.Equal(t, http.StatusBadRequest, rec.Code, "ttl=%q", ttl)
}
}
func TestHTTP_TokenGETMethodNotAllowed(t *testing.T) {
svc, _ := newTestService(&fakeResolver{eni: testENI()}, &fakeIAM{}, &fakeAssumer{})
h := svc.httpHandler()
req := httptest.NewRequest(http.MethodGet, "http://"+MetaDataServerIP+pathToken, nil)
req.RemoteAddr = testIP + ":50000"
req = req.WithContext(context.WithValue(req.Context(), ctxKeyVPCID, testVPC))
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
assert.Equal(t, http.StatusMethodNotAllowed, rec.Code)
}
// TestHTTP_ForwardedForRejected pins AWS IMDS's SSRF defence: any request
// carrying X-Forwarded-For is refused with 403 regardless of token or identity,
// on both the read path and token issuance.
func TestHTTP_ForwardedForRejected(t *testing.T) {
svc, _ := newTestService(&fakeResolver{eni: testENI()}, &fakeIAM{}, &fakeAssumer{})
h := svc.httpHandler()
// A valid token + resolvable ENI is still refused once X-Forwarded-For is set.
token := issueToken(t, h)
req := httptest.NewRequest(http.MethodGet, "http://"+MetaDataServerIP+prefixMetaData+"instance-id", nil)
req.RemoteAddr = testIP + ":50000"
req = req.WithContext(context.WithValue(req.Context(), ctxKeyVPCID, testVPC))
req.Header.Set(hdrToken, token)
req.Header.Set(hdrForwardedFor, "10.0.0.99")
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
assert.Equal(t, http.StatusForbidden, rec.Code)
// Token issuance is refused too.
put := httptest.NewRequest(http.MethodPut, "http://"+MetaDataServerIP+pathToken, nil)
put.RemoteAddr = testIP + ":50000"
put = put.WithContext(context.WithValue(put.Context(), ctxKeyVPCID, testVPC))
put.Header.Set(hdrTokenTTL, "60")
put.Header.Set(hdrForwardedFor, "10.0.0.99")
putRec := httptest.NewRecorder()
h.ServeHTTP(putRec, put)
assert.Equal(t, http.StatusForbidden, putRec.Code)
}
func TestHTTP_ENIMissReturns404(t *testing.T) {
svc, _ := newTestService(&fakeResolver{eni: nil}, &fakeIAM{}, &fakeAssumer{})
h := svc.httpHandler()
// Token PUT and metadata GET both 404 when the source IP maps to no ENI.
req := httptest.NewRequest(http.MethodPut, "http://"+MetaDataServerIP+pathToken, nil)
req.RemoteAddr = testIP + ":50000"
req = req.WithContext(context.WithValue(req.Context(), ctxKeyVPCID, testVPC))
req.Header.Set(hdrTokenTTL, "60")
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
assert.Equal(t, http.StatusNotFound, rec.Code)
}
// ----- metadata surface --------------------------------------------------
func TestHTTP_MetadataPaths(t *testing.T) {
res := &fakeResolver{
eni: testENI(),
inst: &instanceFacts{instanceType: "t3.micro", imageID: "ami-12345", userData: []byte("#!/bin/sh\necho hi")},
sgNames: map[string]string{"sg-1": "web-sg", "sg-2": "db-sg"},
}
svc, _ := newTestService(res, &fakeIAM{}, &fakeAssumer{})
h := svc.httpHandler()
token := issueToken(t, h)
cases := []struct{ path, want string }{
{prefixMetaData + "instance-id", "i-0123456789"},
{prefixMetaData + "instance-type", "t3.micro"},
{prefixMetaData + "ami-id", "ami-12345"},
{prefixMetaData + "local-ipv4", "10.0.1.5"},
{prefixMetaData + "public-ipv4", "203.0.113.7"},
{prefixMetaData + "mac", "02:11:22:33:44:55"},
{prefixMetaData + "placement/availability-zone", "ap-southeast-2a"},
{prefixMetaData + "placement/region", "ap-southeast-2"},
{prefixMetaData + "security-groups", "web-sg\ndb-sg"},
{prefixMetaData + "hostname", "ip-10-0-1-5.ap-southeast-2.compute.internal"},
{prefixMetaData + "local-hostname", "ip-10-0-1-5.ap-southeast-2.compute.internal"},
{pathUserData, "#!/bin/sh\necho hi"},
}
for _, c := range cases {
rec := get(t, h, c.path, token)
assert.Equal(t, http.StatusOK, rec.Code, "path=%s", c.path)
assert.Equal(t, c.want, rec.Body.String(), "path=%s", c.path)
}
}
func TestHTTP_DirectoryListing(t *testing.T) {
svc, _ := newTestService(&fakeResolver{eni: testENI()}, &fakeIAM{}, &fakeAssumer{})
h := svc.httpHandler()
token := issueToken(t, h)
rec := get(t, h, pathMetaDataRoot, token)
assert.Equal(t, http.StatusOK, rec.Code)
assert.Contains(t, rec.Body.String(), "instance-id")
assert.Contains(t, rec.Body.String(), "iam/")
}
func TestHTTP_UserDataAbsent404(t *testing.T) {
res := &fakeResolver{eni: testENI(), inst: &instanceFacts{}}
svc, _ := newTestService(res, &fakeIAM{}, &fakeAssumer{})
h := svc.httpHandler()
token := issueToken(t, h)
rec := get(t, h, pathUserData, token)
assert.Equal(t, http.StatusNotFound, rec.Code)
}
func TestHTTP_OutOfScopePaths404(t *testing.T) {
res := &fakeResolver{eni: testENI(), inst: &instanceFacts{}}
svc, _ := newTestService(res, &fakeIAM{}, &fakeAssumer{})
h := svc.httpHandler()
token := issueToken(t, h)
for _, p := range []string{
"/latest/dynamic/instance-identity/document",
prefixMetaData + "network/interfaces/macs",
prefixMetaData + "nonsense",
} {
rec := get(t, h, p, token)
assert.Equal(t, http.StatusNotFound, rec.Code, "path=%s", p)
}
}
// ----- IAM / credentials -------------------------------------------------
func profileFixture() *handlers_iam.InstanceProfile {
return &handlers_iam.InstanceProfile{
InstanceProfileName: "app-profile",
InstanceProfileID: "AIPAEXAMPLE",
AccountID: "111122223333",
ARN: "arn:aws:iam::111122223333:instance-profile/app-profile",
RoleName: "app-role",
}
}
func TestHTTP_IAMInfo(t *testing.T) {
res := &fakeResolver{eni: testENI(), inst: &instanceFacts{iamInstanceProfileArn: "arn:aws:iam::111122223333:instance-profile/app-profile"}}
svc, _ := newTestService(res, &fakeIAM{profile: profileFixture()}, &fakeAssumer{})
h := svc.httpHandler()
token := issueToken(t, h)
rec := get(t, h, prefixMetaData+"iam/info", token)
assert.Equal(t, http.StatusOK, rec.Code)
assert.Contains(t, rec.Body.String(), "arn:aws:iam::111122223333:instance-profile/app-profile")
assert.Contains(t, rec.Body.String(), "AIPAEXAMPLE")
}
func TestHTTP_IAMInfoNoProfile404(t *testing.T) {
res := &fakeResolver{eni: testENI(), inst: &instanceFacts{}} // no profile ARN
svc, _ := newTestService(res, &fakeIAM{}, &fakeAssumer{})
h := svc.httpHandler()
token := issueToken(t, h)
rec := get(t, h, prefixMetaData+"iam/info", token)
assert.Equal(t, http.StatusNotFound, rec.Code)
}
func TestHTTP_SecurityCredentialsList(t *testing.T) {
res := &fakeResolver{eni: testENI(), inst: &instanceFacts{iamInstanceProfileArn: "arn:aws:iam::111122223333:instance-profile/app-profile"}}
svc, _ := newTestService(res, &fakeIAM{profile: profileFixture()}, &fakeAssumer{})
h := svc.httpHandler()
token := issueToken(t, h)
rec := get(t, h, pathSecurityCredsDir, token)
assert.Equal(t, http.StatusOK, rec.Code)
assert.Equal(t, "app-role", rec.Body.String())
}
func TestHTTP_SecurityCredentialsListNoRoleEmpty(t *testing.T) {
res := &fakeResolver{eni: testENI(), inst: &instanceFacts{}}
svc, _ := newTestService(res, &fakeIAM{}, &fakeAssumer{})
h := svc.httpHandler()
token := issueToken(t, h)
rec := get(t, h, pathSecurityCredsDir, token)
assert.Equal(t, http.StatusOK, rec.Code)
assert.Empty(t, rec.Body.String())
}
// A backend failure resolving the instance must surface as 500, never an empty
// 200 — otherwise a transient hiccup is indistinguishable from "instance has no
// role" and the SDK silently proceeds unauthenticated.
func TestHTTP_SecurityCredentialsListBackendError500(t *testing.T) {
res := &fakeResolver{eni: testENI(), instErr: errors.New("backend unavailable")}
svc, _ := newTestService(res, &fakeIAM{}, &fakeAssumer{})
h := svc.httpHandler()
token := issueToken(t, h)
rec := get(t, h, pathSecurityCredsDir, token)
assert.Equal(t, http.StatusInternalServerError, rec.Code)
}
// A profile deleted out from under a running instance (the ARN dereferences to
// NoSuchEntity) is "no instance role", not a backend fault: iam/info 404s, the
// creds list is an empty 200, and the role path 404s — matching AWS. Only a
// genuine backend error (TestHTTP_*BackendError500) is a 500.
func TestHTTP_IAMInfoDeletedProfile404(t *testing.T) {
res := &fakeResolver{eni: testENI(), inst: &instanceFacts{iamInstanceProfileArn: "arn:aws:iam::111122223333:instance-profile/gone"}}
svc, _ := newTestService(res, &fakeIAM{profileErr: errors.New(awserrors.ErrorIAMNoSuchEntity)}, &fakeAssumer{})
h := svc.httpHandler()
token := issueToken(t, h)
rec := get(t, h, prefixMetaData+"iam/info", token)
assert.Equal(t, http.StatusNotFound, rec.Code)
}
func TestHTTP_SecurityCredentialsListDeletedProfileEmpty(t *testing.T) {
res := &fakeResolver{eni: testENI(), inst: &instanceFacts{iamInstanceProfileArn: "arn:aws:iam::111122223333:instance-profile/gone"}}
svc, _ := newTestService(res, &fakeIAM{profileErr: errors.New(awserrors.ErrorIAMNoSuchEntity)}, &fakeAssumer{})
h := svc.httpHandler()
token := issueToken(t, h)
rec := get(t, h, pathSecurityCredsDir, token)
assert.Equal(t, http.StatusOK, rec.Code)
assert.Empty(t, rec.Body.String())
}
func TestHTTP_RoleCredentialsDeletedProfile404(t *testing.T) {
res := &fakeResolver{eni: testENI(), inst: &instanceFacts{iamInstanceProfileArn: "arn:aws:iam::111122223333:instance-profile/gone"}}
svc, _ := newTestService(res, &fakeIAM{profileErr: errors.New(awserrors.ErrorIAMNoSuchEntity)}, &fakeAssumer{})
h := svc.httpHandler()
token := issueToken(t, h)
rec := get(t, h, prefixSecurityCreds+"app-role", token)
assert.Equal(t, http.StatusNotFound, rec.Code)
}
func TestHTTP_IAMInfoBackendError500(t *testing.T) {
res := &fakeResolver{eni: testENI(), inst: &instanceFacts{iamInstanceProfileArn: "arn:aws:iam::111122223333:instance-profile/app-profile"}}
svc, _ := newTestService(res, &fakeIAM{profileErr: errors.New("iam unavailable")}, &fakeAssumer{})
h := svc.httpHandler()
token := issueToken(t, h)
rec := get(t, h, prefixMetaData+"iam/info", token)
assert.Equal(t, http.StatusInternalServerError, rec.Code)
}
func TestHTTP_RoleCredentialsBackendError500(t *testing.T) {
res := &fakeResolver{eni: testENI(), instErr: errors.New("backend unavailable")}
svc, _ := newTestService(res, &fakeIAM{}, &fakeAssumer{})
h := svc.httpHandler()
token := issueToken(t, h)
rec := get(t, h, prefixSecurityCreds+"app-role", token)
assert.Equal(t, http.StatusInternalServerError, rec.Code)
}
func TestHTTP_RoleCredentials(t *testing.T) {
now := time.Unix(1_700_000_000, 0).UTC()
res := &fakeResolver{eni: testENI(), inst: &instanceFacts{iamInstanceProfileArn: "arn:aws:iam::111122223333:instance-profile/app-profile"}}
assumer := &fakeAssumer{out: &sts.AssumeRoleOutput{Credentials: &sts.Credentials{
AccessKeyId: aws.String("ASIAEXAMPLE"),
SecretAccessKey: aws.String("secret"),
SessionToken: aws.String("token"),
Expiration: aws.Time(now.Add(time.Hour)),
}}}
svc, _ := newTestService(res, &fakeIAM{profile: profileFixture(), roleARN: "arn:aws:iam::111122223333:role/app-role"}, assumer)
h := svc.httpHandler()
token := issueToken(t, h)
rec := get(t, h, prefixSecurityCreds+"app-role", token)
assert.Equal(t, http.StatusOK, rec.Code)
assert.Contains(t, rec.Body.String(), "ASIAEXAMPLE")
assert.Contains(t, rec.Body.String(), `"Code":"Success"`)
}
func TestHTTP_RoleCredentialsWrongRole404(t *testing.T) {
res := &fakeResolver{eni: testENI(), inst: &instanceFacts{iamInstanceProfileArn: "arn:aws:iam::111122223333:instance-profile/app-profile"}}
svc, _ := newTestService(res, &fakeIAM{profile: profileFixture(), roleARN: "arn:aws:iam::111122223333:role/app-role"}, &fakeAssumer{})
h := svc.httpHandler()
token := issueToken(t, h)
// AWS only accepts the actual role name, never the profile name.
rec := get(t, h, prefixSecurityCreds+"app-profile", token)
assert.Equal(t, http.StatusNotFound, rec.Code)
}
spinifex/handlers/imds/nats_clients.go
+134
View File
@@ -0,0 +1,134 @@
package handlers_imds
import (
"sync"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/iam"
"github.com/aws/aws-sdk-go/service/sts"
handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
handlers_sts "github.com/mulgadc/spinifex/spinifex/handlers/sts"
"github.com/mulgadc/spinifex/spinifex/utils"
"github.com/nats-io/nats.go"
)
const (
// imdsRPCTimeout bounds each internal control-plane round-trip to awsgw.
imdsRPCTimeout = 10 * time.Second
// profileCacheTTL memoises profile/role lookups. Both mappings are effectively
// static, so a short TTL keeps the iam/* GETs off a NATS round-trip per request.
profileCacheTTL = 5 * time.Minute
)
// NATSSTSAssumer is the NATS-backed stsAssumer. It mints instance-role credentials
// via awsgw's SubjectAssumeRoleForInstance responder, keeping no local cache —
// credCache already memoises minted credentials, so STS stays off the hot path.
type NATSSTSAssumer struct {
nc *nats.Conn
}
var _ stsAssumer = (*NATSSTSAssumer)(nil)
// NewNATSSTSAssumer constructs a NATSSTSAssumer over nc.
func NewNATSSTSAssumer(nc *nats.Conn) *NATSSTSAssumer {
return &NATSSTSAssumer{nc: nc}
}
func (a *NATSSTSAssumer) AssumeRoleForInstance(accountID, roleARN, instanceID string, durationSeconds int64) (*sts.AssumeRoleOutput, error) {
return utils.NATSRequest[sts.AssumeRoleOutput](a.nc, handlers_sts.SubjectAssumeRoleForInstance, handlers_sts.AssumeRoleForInstanceRequest{
AccountID: accountID,
RoleARN: roleARN,
InstanceID: instanceID,
DurationSeconds: durationSeconds,
}, imdsRPCTimeout, accountID)
}
// NATSProfileLookup is the NATS-backed profileLookup. It resolves instance
// profiles and roles via awsgw's IAM responders, fronted by a short-TTL cache
// so repeat iam/* GETs don't take a round-trip each.
type NATSProfileLookup struct {
nc *nats.Conn
profiles *ttlCache[*handlers_iam.InstanceProfile]
roles *ttlCache[*iam.GetRoleOutput]
}
var _ profileLookup = (*NATSProfileLookup)(nil)
// NewNATSProfileLookup constructs a NATSProfileLookup over nc.
func NewNATSProfileLookup(nc *nats.Conn) *NATSProfileLookup {
return &NATSProfileLookup{
nc: nc,
profiles: newTTLCache[*handlers_iam.InstanceProfile](profileCacheTTL),
roles: newTTLCache[*iam.GetRoleOutput](profileCacheTTL),
}
}
func (p *NATSProfileLookup) ResolveInstanceProfile(accountID, nameOrARN string) (*handlers_iam.InstanceProfile, error) {
key := accountID + "\x00" + nameOrARN
if v, ok := p.profiles.get(key); ok {
return v, nil
}
out, err := utils.NATSRequest[handlers_iam.InstanceProfile](p.nc, handlers_iam.SubjectResolveInstanceProfile, handlers_iam.ResolveInstanceProfileRequest{
AccountID: accountID,
NameOrARN: nameOrARN,
}, imdsRPCTimeout, accountID)
if err != nil {
return nil, err
}
p.profiles.put(key, out)
return out, nil
}
func (p *NATSProfileLookup) GetRole(accountID string, input *iam.GetRoleInput) (*iam.GetRoleOutput, error) {
key := accountID + "\x00" + aws.StringValue(input.RoleName)
if v, ok := p.roles.get(key); ok {
return v, nil
}
out, err := utils.NATSRequest[iam.GetRoleOutput](p.nc, handlers_iam.SubjectGetRole, handlers_iam.GetRoleRequest{
AccountID: accountID,
Input: input,
}, imdsRPCTimeout, accountID)
if err != nil {
return nil, err
}
p.roles.put(key, out)
return out, nil
}
// ttlCache is a minimal concurrency-safe map with per-entry expiry, used to
// keep effectively-static IAM lookups off the NATS round-trip. now is a field
// so tests can drive expiry deterministically.
type ttlCache[V any] struct {
mu sync.Mutex
ttl time.Duration
now func() time.Time
m map[string]ttlCacheEntry[V]
}
type ttlCacheEntry[V any] struct {
val V
expiresAt time.Time
}
func newTTLCache[V any](ttl time.Duration) *ttlCache[V] {
return &ttlCache[V]{ttl: ttl, now: time.Now, m: make(map[string]ttlCacheEntry[V])}
}
func (c *ttlCache[V]) get(key string) (V, bool) {
c.mu.Lock()
defer c.mu.Unlock()
entry, ok := c.m[key]
if !ok || c.now().After(entry.expiresAt) {
var zero V
return zero, false
}
return entry.val, true
}
func (c *ttlCache[V]) put(key string, v V) {
c.mu.Lock()
defer c.mu.Unlock()
c.m[key] = ttlCacheEntry[V]{val: v, expiresAt: c.now().Add(c.ttl)}
}
spinifex/handlers/imds/nats_clients_test.go
+136
View File
@@ -0,0 +1,136 @@
package handlers_imds
import (
"sync/atomic"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/iam"
"github.com/aws/aws-sdk-go/service/sts"
handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
handlers_sts "github.com/mulgadc/spinifex/spinifex/handlers/sts"
"github.com/mulgadc/spinifex/spinifex/testutil"
"github.com/mulgadc/spinifex/spinifex/utils"
"github.com/nats-io/nats.go"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const imdsTestAccountID = "123456789012"
// serveCounted subscribes fn to subject and returns a counter of how many
// requests the responder actually handled — the assertion hook for the TTL
// cache (a cache hit must not reach the responder).
func serveCounted[I any, O any](t *testing.T, nc *nats.Conn, subject string, fn func(*I) (*O, error)) *int32 {
t.Helper()
var calls int32
sub, err := nc.Subscribe(subject, func(msg *nats.Msg) {
atomic.AddInt32(&calls, 1)
utils.ServeNATSRequest(msg, fn)
})
require.NoError(t, err)
t.Cleanup(func() { _ = sub.Unsubscribe() })
return &calls
}
func TestNATSSTSAssumer_RoundTrip(t *testing.T) {
_, nc := testutil.StartTestNATS(t)
var gotReq handlers_sts.AssumeRoleForInstanceRequest
calls := serveCounted(t, nc, handlers_sts.SubjectAssumeRoleForInstance,
func(req *handlers_sts.AssumeRoleForInstanceRequest) (*sts.AssumeRoleOutput, error) {
gotReq = *req
return &sts.AssumeRoleOutput{Credentials: &sts.Credentials{
AccessKeyId: aws.String("ASIATESTKEY"),
SecretAccessKey: aws.String("secret"),
SessionToken: aws.String("token"),
Expiration: aws.Time(time.Unix(1700000000, 0).UTC()),
}}, nil
})
assumer := NewNATSSTSAssumer(nc)
out, err := assumer.AssumeRoleForInstance(imdsTestAccountID, "arn:aws:iam::123456789012:role/app", "i-abc", 3600)
require.NoError(t, err)
require.NotNil(t, out.Credentials)
assert.Equal(t, "ASIATESTKEY", aws.StringValue(out.Credentials.AccessKeyId))
assert.Equal(t, int32(1), atomic.LoadInt32(calls))
// The request payload must round-trip every field the responder needs.
assert.Equal(t, handlers_sts.AssumeRoleForInstanceRequest{
AccountID: imdsTestAccountID,
RoleARN: "arn:aws:iam::123456789012:role/app",
InstanceID: "i-abc",
DurationSeconds: 3600,
}, gotReq)
}
func TestNATSProfileLookup_CachesProfile(t *testing.T) {
_, nc := testutil.StartTestNATS(t)
calls := serveCounted(t, nc, handlers_iam.SubjectResolveInstanceProfile,
func(req *handlers_iam.ResolveInstanceProfileRequest) (*handlers_iam.InstanceProfile, error) {
return &handlers_iam.InstanceProfile{
InstanceProfileName: "app",
AccountID: req.AccountID,
RoleName: "app-role",
}, nil
})
lookup := NewNATSProfileLookup(nc)
for range 3 {
prof, err := lookup.ResolveInstanceProfile(imdsTestAccountID, "app")
require.NoError(t, err)
assert.Equal(t, "app-role", prof.RoleName)
}
// Three lookups, one round-trip: the TTL cache suppressed the repeats.
assert.Equal(t, int32(1), atomic.LoadInt32(calls))
}
func TestNATSProfileLookup_CachesRole(t *testing.T) {
_, nc := testutil.StartTestNATS(t)
calls := serveCounted(t, nc, handlers_iam.SubjectGetRole,
func(req *handlers_iam.GetRoleRequest) (*iam.GetRoleOutput, error) {
return &iam.GetRoleOutput{Role: &iam.Role{
RoleName: req.Input.RoleName,
Arn: aws.String("arn:aws:iam::123456789012:role/app-role"),
}}, nil
})
lookup := NewNATSProfileLookup(nc)
for range 3 {
out, err := lookup.GetRole(imdsTestAccountID, &iam.GetRoleInput{RoleName: aws.String("app-role")})
require.NoError(t, err)
assert.Equal(t, "arn:aws:iam::123456789012:role/app-role", aws.StringValue(out.Role.Arn))
}
assert.Equal(t, int32(1), atomic.LoadInt32(calls))
}
func TestNATSProfileLookup_PropagatesError(t *testing.T) {
_, nc := testutil.StartTestNATS(t)
serveCounted(t, nc, handlers_iam.SubjectResolveInstanceProfile,
func(_ *handlers_iam.ResolveInstanceProfileRequest) (*handlers_iam.InstanceProfile, error) {
return nil, assert.AnError
})
lookup := NewNATSProfileLookup(nc)
_, err := lookup.ResolveInstanceProfile(imdsTestAccountID, "missing")
require.Error(t, err)
}
func TestTTLCache_ExpiresEntries(t *testing.T) {
now := time.Unix(1700000000, 0)
c := newTTLCache[string](time.Minute)
c.now = func() time.Time { return now }
c.put("k", "v")
got, ok := c.get("k")
require.True(t, ok)
assert.Equal(t, "v", got)
now = now.Add(2 * time.Minute) // past the TTL
_, ok = c.get("k")
assert.False(t, ok)
}
spinifex/handlers/imds/resolver.go
+189
View File
@@ -0,0 +1,189 @@
package handlers_imds
import (
"encoding/json"
"errors"
"fmt"
"log/slog"
"github.com/nats-io/nats.go"
)
// kvBucketENIs is the ENI source-of-truth bucket (handlers_ec2_vpc.KVBucketENIs),
// duplicated as a literal rather than imported because importing handlers/ec2/vpc
// back would close an import cycle (the veth_store record type does likewise).
const kvBucketENIs = "spinifex-vpc-enis"
// kvBucketSecurityGroups is the security-group source-of-truth bucket
// (handlers_ec2_vpc.KVBucketSecurityGroups). Duplicated as a literal for the
// same import-cycle reason as kvBucketENIs.
const kvBucketSecurityGroups = "spinifex-vpc-security-groups"
// eniFacts is the subset of an ENIRecord the IMDS metadata surface serves
// directly, plus the account ID recovered from the ENI bucket key. Everything
// here is read live off the ENI record, so a change (e.g. a re-associated EIP)
// is reflected on the next request.
type eniFacts struct {
eniID string
accountID string
instanceID string
vpcID string
subnetID string
privateIP string
publicIP string
mac string
availabilityZone string
securityGroupIDs []string
}
// instanceFacts carries the four metadata fields not present on the ENI record.
// They live in the daemon's in-memory instance manager, so they are fetched via
// the account-scoped DescribeInstances fan-out behind instanceLookup.
type instanceFacts struct {
instanceType string
imageID string
iamInstanceProfileArn string
userData []byte
}
// instanceLookup resolves the instance-only metadata fields by instance ID. It
// is an interface so the metadata handlers stay unit-testable without a live
// NATS cluster; the production implementation (natsInstanceLookup) fans out
// DescribeInstances + DescribeInstanceAttribute.
type instanceLookup interface {
describe(accountID, instanceID string) (*instanceFacts, error)
}
// eniIndexValue mirrors the stored shape of a spinifex-network-eni-by-vpc-ip
// row (handlers_ec2_vpc.eniByIPValue). Duplicated for the same cycle reason as
// kvBucketENIs. Both fields are the ENI's immutable identity; everything mutable
// is read live off the ENI record.
type eniIndexValue struct {
ENIId string `json:"eni_id"`
AccountID string `json:"account_id"`
}
// sgNameRecord is the slice of handlers_ec2_vpc.SecurityGroupRecord the metadata
// surface needs: the human-readable group name AWS serves at
// /latest/meta-data/security-groups (not the sg-* ID).
type sgNameRecord struct {
GroupName string `json:"group_name"`
}
// eniRecord is the subset of handlers_ec2_vpc.ENIRecord the resolver reads. The
// full record is owned by the ENI controller; only these fields feed IMDS.
type eniRecord struct {
NetworkInterfaceId string `json:"network_interface_id"`
SubnetId string `json:"subnet_id"`
VpcId string `json:"vpc_id"`
AvailabilityZone string `json:"availability_zone"`
PrivateIpAddress string `json:"private_ip_address"`
MacAddress string `json:"mac_address"`
InstanceId string `json:"instance_id,omitempty"`
PublicIpAddress string `json:"public_ip_address,omitempty"`
SecurityGroupIds []string `json:"security_group_ids,omitempty"`
}
// metadataResolver maps a datapath-attested (vpcID, srcIP) to the ENI + instance
// facts the metadata surface serves. The chain is:
//
// (vpcID, ip) → {eniID, accountID} via the eni-by-vpc-ip reverse index
// (account, eniID) → ENIRecord via a direct KV Get
// (account, instanceID) → facts via the account-scoped DescribeInstances fan-out
//
// The index carries the ENI's immutable identity so the account resolves in one
// Get; every mutable field is read live off the record, so there's no staleness.
type metadataResolver struct {
index nats.KeyValue // spinifex-network-eni-by-vpc-ip
eniKV nats.KeyValue // spinifex-vpc-enis
sgKV nats.KeyValue // spinifex-vpc-security-groups (nil-safe: degrades to IDs)
lookup instanceLookup
}
var _ eniResolver = (*metadataResolver)(nil)
// resolveENI returns the ENI facts for a request's (vpcID, srcIP), or (nil, nil)
// when no mapping exists — the caller maps a miss to 404, matching AWS's
// "eventually available during boot" posture. A non-nil error is reserved for
// genuine backend failures.
func (r *metadataResolver) resolveENI(vpcID, srcIP string) (*eniFacts, error) {
entry, err := r.index.Get(vpcID + "/" + srcIP)
if err != nil {
if errors.Is(err, nats.ErrKeyNotFound) {
return nil, nil
}
return nil, fmt.Errorf("get eni-by-ip index %s/%s: %w", vpcID, srcIP, err)
}
var idx eniIndexValue
if err := json.Unmarshal(entry.Value(), &idx); err != nil {
return nil, fmt.Errorf("unmarshal eni-by-ip index %s/%s: %w", vpcID, srcIP, err)
}
if idx.ENIId == "" || idx.AccountID == "" {
return nil, nil
}
raw, err := r.eniKV.Get(idx.AccountID + "." + idx.ENIId)
if err != nil {
if errors.Is(err, nats.ErrKeyNotFound) {
return nil, nil
}
return nil, fmt.Errorf("get eni record %s.%s: %w", idx.AccountID, idx.ENIId, err)
}
var rec eniRecord
if err := json.Unmarshal(raw.Value(), &rec); err != nil {
return nil, fmt.Errorf("unmarshal eni record %s: %w", idx.ENIId, err)
}
return &eniFacts{
eniID: rec.NetworkInterfaceId,
accountID: idx.AccountID,
instanceID: rec.InstanceId,
vpcID: rec.VpcId,
subnetID: rec.SubnetId,
privateIP: rec.PrivateIpAddress,
publicIP: rec.PublicIpAddress,
mac: rec.MacAddress,
availabilityZone: rec.AvailabilityZone,
securityGroupIDs: rec.SecurityGroupIds,
}, nil
}
// resolveInstance fetches the instance-only metadata fields for an ENI's
// attached instance. Returns (nil, nil) when the ENI has no attached instance.
func (r *metadataResolver) resolveInstance(eni *eniFacts) (*instanceFacts, error) {
if eni.instanceID == "" {
return nil, nil
}
return r.lookup.describe(eni.accountID, eni.instanceID)
}
// resolveSGNames maps security-group IDs to the group names AWS serves at
// /latest/meta-data/security-groups. Best-effort and order-preserving: a missing
// record or nil sgKV falls back to the raw ID rather than failing the request.
func (r *metadataResolver) resolveSGNames(accountID string, sgIDs []string) []string {
names := make([]string, len(sgIDs))
for i, id := range sgIDs {
names[i] = id
if r.sgKV == nil {
continue
}
raw, err := r.sgKV.Get(accountID + "." + id)
if err != nil {
if !errors.Is(err, nats.ErrKeyNotFound) {
slog.Warn("IMDS: security-group name lookup failed", "account_id", accountID, "sg_id", id, "err", err)
}
continue
}
var rec sgNameRecord
if err := json.Unmarshal(raw.Value(), &rec); err != nil {
slog.Warn("IMDS: security-group unmarshal failed", "sg_id", id, "err", err)
continue
}
if rec.GroupName != "" {
names[i] = rec.GroupName
}
}
return names
}
spinifex/handlers/imds/resolver_test.go
+190
View File
@@ -0,0 +1,190 @@
package handlers_imds
import (
"encoding/json"
"errors"
"testing"
"github.com/mulgadc/spinifex/spinifex/testutil"
"github.com/nats-io/nats.go"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// fakeLookup is a programmable instanceLookup so resolver tests never need the
// account-scoped instance fan-out.
type fakeLookup struct {
facts *instanceFacts
err error
calls int
}
func (f *fakeLookup) describe(_, _ string) (*instanceFacts, error) {
f.calls++
return f.facts, f.err
}
// newTestResolver wires a metadataResolver onto a live test JetStream with the
// three buckets it reads (eni-by-vpc-ip index, ENI source-of-truth, SG names)
// created empty, plus a fake instance lookup the caller can program.
func newTestResolver(t *testing.T) (*metadataResolver, *fakeLookup) {
t.Helper()
_, _, js := testutil.StartTestJetStream(t)
index, err := InitENIByIPBucket(js, 1)
require.NoError(t, err)
eniKV, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: kvBucketENIs, History: 1})
require.NoError(t, err)
sgKV, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: kvBucketSecurityGroups, History: 1})
require.NoError(t, err)
lookup := &fakeLookup{}
return &metadataResolver{index: index, eniKV: eniKV, sgKV: sgKV, lookup: lookup}, lookup
}
func putJSON(t *testing.T, kv nats.KeyValue, key string, v any) {
t.Helper()
data, err := json.Marshal(v)
require.NoError(t, err)
_, err = kv.Put(key, data)
require.NoError(t, err)
}
// seedENI writes a complete index → ENI chain for (testVPC, testIP) and returns
// the resolver ready to serve it.
func seedENI(t *testing.T, r *metadataResolver) {
t.Helper()
putJSON(t, r.index, testVPC+"/"+testIP, eniIndexValue{ENIId: "eni-aaa", AccountID: "111122223333"})
putJSON(t, r.eniKV, "111122223333.eni-aaa", eniRecord{
NetworkInterfaceId: "eni-aaa",
SubnetId: "subnet-1",
VpcId: testVPC,
AvailabilityZone: "ap-southeast-2a",
PrivateIpAddress: testIP,
MacAddress: "02:11:22:33:44:55",
InstanceId: "i-0123456789",
PublicIpAddress: "203.0.113.7",
SecurityGroupIds: []string{"sg-1", "sg-2"},
})
}
func TestResolveENI_Hit(t *testing.T) {
r, _ := newTestResolver(t)
seedENI(t, r)
eni, err := r.resolveENI(testVPC, testIP)
require.NoError(t, err)
require.NotNil(t, eni)
assert.Equal(t, "eni-aaa", eni.eniID)
assert.Equal(t, "111122223333", eni.accountID)
assert.Equal(t, "i-0123456789", eni.instanceID)
assert.Equal(t, testVPC, eni.vpcID)
assert.Equal(t, "subnet-1", eni.subnetID)
assert.Equal(t, testIP, eni.privateIP)
assert.Equal(t, "203.0.113.7", eni.publicIP)
assert.Equal(t, "02:11:22:33:44:55", eni.mac)
assert.Equal(t, "ap-southeast-2a", eni.availabilityZone)
assert.Equal(t, []string{"sg-1", "sg-2"}, eni.securityGroupIDs)
}
// A source IP with no reverse-index row is a miss, not an error — the caller
// maps it to 404 per AWS's "eventually available during boot" posture.
func TestResolveENI_IndexMissIsNilNil(t *testing.T) {
r, _ := newTestResolver(t)
eni, err := r.resolveENI(testVPC, "10.9.9.9")
require.NoError(t, err)
assert.Nil(t, eni)
}
func TestResolveENI_IndexBadJSONErrors(t *testing.T) {
r, _ := newTestResolver(t)
_, err := r.index.Put(testVPC+"/"+testIP, []byte("not json"))
require.NoError(t, err)
_, err = r.resolveENI(testVPC, testIP)
require.Error(t, err)
}
// An index row missing either identity field is treated as a miss rather than
// being chased into a malformed ENI Get.
func TestResolveENI_EmptyIndexFieldsIsNilNil(t *testing.T) {
r, _ := newTestResolver(t)
putJSON(t, r.index, testVPC+"/"+testIP, eniIndexValue{ENIId: "eni-aaa"}) // no account
eni, err := r.resolveENI(testVPC, testIP)
require.NoError(t, err)
assert.Nil(t, eni)
}
// A dangling index pointing at an ENI record that no longer exists is a miss,
// matching a mid-teardown race.
func TestResolveENI_ENIRecordMissIsNilNil(t *testing.T) {
r, _ := newTestResolver(t)
putJSON(t, r.index, testVPC+"/"+testIP, eniIndexValue{ENIId: "eni-gone", AccountID: "111122223333"})
eni, err := r.resolveENI(testVPC, testIP)
require.NoError(t, err)
assert.Nil(t, eni)
}
func TestResolveENI_ENIRecordBadJSONErrors(t *testing.T) {
r, _ := newTestResolver(t)
putJSON(t, r.index, testVPC+"/"+testIP, eniIndexValue{ENIId: "eni-aaa", AccountID: "111122223333"})
_, err := r.eniKV.Put("111122223333.eni-aaa", []byte("not json"))
require.NoError(t, err)
_, err = r.resolveENI(testVPC, testIP)
require.Error(t, err)
}
func TestResolveInstance_NoAttachedInstance(t *testing.T) {
r, lookup := newTestResolver(t)
inst, err := r.resolveInstance(&eniFacts{instanceID: ""})
require.NoError(t, err)
assert.Nil(t, inst)
assert.Equal(t, 0, lookup.calls, "lookup must be skipped when no instance is attached")
}
func TestResolveInstance_DelegatesToLookup(t *testing.T) {
r, lookup := newTestResolver(t)
lookup.facts = &instanceFacts{instanceType: "t3.micro", imageID: "ami-1"}
inst, err := r.resolveInstance(&eniFacts{accountID: "111122223333", instanceID: "i-1"})
require.NoError(t, err)
require.NotNil(t, inst)
assert.Equal(t, "t3.micro", inst.instanceType)
assert.Equal(t, 1, lookup.calls)
}
func TestResolveInstance_LookupErrorPropagates(t *testing.T) {
r, lookup := newTestResolver(t)
lookup.err = errors.New("fan-out failed")
_, err := r.resolveInstance(&eniFacts{accountID: "111122223333", instanceID: "i-1"})
require.Error(t, err)
}
// A nil SG bucket (not yet up at start) must degrade the endpoint to raw IDs
// rather than failing the request.
func TestResolveSGNames_NilBucketDegradesToIDs(t *testing.T) {
r, _ := newTestResolver(t)
r.sgKV = nil
got := r.resolveSGNames("111122223333", []string{"sg-1", "sg-2"})
assert.Equal(t, []string{"sg-1", "sg-2"}, got)
}
// Names resolve where a record exists; a missing record or one with an empty
// name falls back to the ID, and order is preserved throughout.
func TestResolveSGNames_ResolvesAndFallsBack(t *testing.T) {
r, _ := newTestResolver(t)
putJSON(t, r.sgKV, "111122223333.sg-1", sgNameRecord{GroupName: "web-sg"})
putJSON(t, r.sgKV, "111122223333.sg-3", sgNameRecord{GroupName: ""}) // empty → ID
got := r.resolveSGNames("111122223333", []string{"sg-1", "sg-2", "sg-3"})
assert.Equal(t, []string{"web-sg", "sg-2", "sg-3"}, got)
}
func TestResolveSGNames_BadJSONFallsBackToID(t *testing.T) {
r, _ := newTestResolver(t)
_, err := r.sgKV.Put("111122223333.sg-1", []byte("not json"))
require.NoError(t, err)
got := r.resolveSGNames("111122223333", []string{"sg-1"})
assert.Equal(t, []string{"sg-1"}, got)
}
spinifex/handlers/imds/server.go
+303
View File
@@ -0,0 +1,303 @@
package handlers_imds
import (
"context"
"errors"
"fmt"
"log/slog"
"net"
"net/http"
"os"
"runtime"
"sync"
"syscall"
"time"
"github.com/mulgadc/spinifex/spinifex/utils"
"github.com/nats-io/nats.go"
"golang.org/x/sys/unix"
)
// ensureVethFunc / removeVethFunc are the per-VPC host veth lifecycle hooks,
// injected (not imported) to avoid an import cycle via network/host. vpcd passes
// host.EnsureIMDSVeth / host.RemoveIMDSVeth; tests pass root-free fakes. ensure
// returns the per-VPC netns the listener must be created in plus the host-end
// veth name (now living inside that netns).
type ensureVethFunc func(ctx context.Context, vpcID string) (netnsName, hostEnd string, err error)
type removeVethFunc func(ctx context.Context, vpcID string) error
// listenFunc binds a TCP listener on 169.254.169.254:80 inside the VPC's netns,
// scoped to its host veth. Swappable in tests so bind-manager logic runs without
// CAP_NET_ADMIN or a real netns. An empty netnsName binds in the current netns.
type listenFunc func(ctx context.Context, netnsName, hostEnd string) (net.Listener, error)
// activeBinding is the per-VPC realised state: a listener bound to the VPC's
// host veth and the http.Server serving it.
type activeBinding struct {
listener net.Listener
server *http.Server
}
// bindManager realises the per-chassis IMDS listener stack. It runs on every
// chassis and eagerly materialises a veth + OVS port + listener for every VPC in
// the vpc-veth bucket — lifecycle is "watch the bucket, ensure local state".
type bindManager struct {
kv nats.KeyValue // spinifex-network-imds-vpc-veth
handler http.Handler
ensureVeth ensureVethFunc
removeVeth removeVethFunc
listen listenFunc
mu sync.Mutex
active map[string]*activeBinding // vpcID → binding
}
func newBindManager(kv nats.KeyValue, handler http.Handler, ensure ensureVethFunc, remove removeVethFunc, listen listenFunc) *bindManager {
return &bindManager{
kv: kv,
handler: handler,
ensureVeth: ensure,
removeVeth: remove,
listen: listen,
active: make(map[string]*activeBinding),
}
}
// sync reads the full vpc-veth bucket and binds a listener for every entry. Run
// once at startup; idempotent, so a re-sync after a transient error is safe.
func (b *bindManager) sync(ctx context.Context) error {
keys, err := b.kv.Keys()
if err != nil {
if errors.Is(err, nats.ErrNoKeysFound) {
return nil
}
return fmt.Errorf("list vpc-veth keys: %w", err)
}
for _, vpcID := range keys {
if vpcID == utils.VersionKey {
continue // schema-version marker written by migrate.RunKV, not a VPC
}
if err := b.bind(ctx, vpcID); err != nil {
slog.Error("IMDS: bind failed during sync", "vpc_id", vpcID, "err", err)
}
}
return nil
}
// watch reacts to vpc-veth bucket changes for the life of ctx: a Put binds the
// VPC (if not already bound), a Delete unbinds it and tears down the veth. Both
// transitions are idempotent.
func (b *bindManager) watch(ctx context.Context) {
watcher, err := b.kv.WatchAll(nats.Context(ctx))
if err != nil {
slog.Error("IMDS: failed to start vpc-veth watcher", "err", err)
return
}
defer func() { _ = watcher.Stop() }()
for {
select {
case <-ctx.Done():
return
case entry, ok := <-watcher.Updates():
if !ok {
return
}
if entry == nil {
// nil marks the end of the initial replay; sync() already
// covered those, so there is nothing extra to do.
continue
}
if entry.Key() == utils.VersionKey {
continue // schema-version marker written by migrate.RunKV, not a VPC
}
switch entry.Operation() {
case nats.KeyValuePut:
if err := b.bind(ctx, entry.Key()); err != nil {
slog.Error("IMDS: bind failed on watch", "vpc_id", entry.Key(), "err", err)
}
case nats.KeyValueDelete, nats.KeyValuePurge:
b.unbind(ctx, entry.Key())
}
}
}
}
// bind ensures the host veth then opens a device-scoped listener for vpcID. A
// no-op if the VPC is already bound.
func (b *bindManager) bind(ctx context.Context, vpcID string) error {
b.mu.Lock()
if _, ok := b.active[vpcID]; ok {
b.mu.Unlock()
return nil
}
b.mu.Unlock()
netnsName, hostEnd, err := b.ensureVeth(ctx, vpcID)
if err != nil {
return fmt.Errorf("ensure veth: %w", err)
}
listener, err := b.listen(ctx, netnsName, hostEnd)
if err != nil {
return fmt.Errorf("listen on %s in netns %s: %w", hostEnd, netnsName, err)
}
server := &http.Server{
Handler: b.handler,
ReadHeaderTimeout: 10 * time.Second,
// Tag every request from this listener with its VPC ID. The handler
// pairs it with the datapath-attested source IP to resolve the ENI.
BaseContext: func(net.Listener) context.Context {
return context.WithValue(ctx, ctxKeyVPCID, vpcID)
},
}
b.mu.Lock()
// Re-check under lock in case a concurrent bind won the race.
if _, ok := b.active[vpcID]; ok {
b.mu.Unlock()
_ = listener.Close()
return nil
}
b.active[vpcID] = &activeBinding{listener: listener, server: server}
b.mu.Unlock()
go func() {
if err := server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) {
slog.Error("IMDS: listener serve exited", "vpc_id", vpcID, "host_end", hostEnd, "err", err)
}
}()
slog.Info("IMDS: listener bound", "vpc_id", vpcID, "netns", netnsName, "host_end", hostEnd, "addr", listener.Addr().String())
return nil
}
// unbind closes the VPC's listener and removes its host veth. Idempotent.
func (b *bindManager) unbind(ctx context.Context, vpcID string) {
b.mu.Lock()
binding := b.active[vpcID]
delete(b.active, vpcID)
b.mu.Unlock()
if binding != nil {
if err := binding.server.Close(); err != nil {
slog.Warn("IMDS: listener close failed", "vpc_id", vpcID, "err", err)
}
}
if err := b.removeVeth(ctx, vpcID); err != nil {
slog.Warn("IMDS: veth removal failed", "vpc_id", vpcID, "err", err)
}
slog.Info("IMDS: listener unbound", "vpc_id", vpcID)
}
// shutdown closes every active listener. The host veths are intentionally left
// in place: they are cheap, idempotent to re-ensure on the next start, and
// tearing them down on a clean vpcd restart would needlessly disrupt any other
// chassis-local consumer.
func (b *bindManager) shutdown() {
b.mu.Lock()
defer b.mu.Unlock()
for vpcID, binding := range b.active {
if err := binding.server.Close(); err != nil {
slog.Warn("IMDS: listener close failed during shutdown", "vpc_id", vpcID, "err", err)
}
delete(b.active, vpcID)
}
}
// bindLocalListener opens a TCP listener on 169.254.169.254:80 inside the VPC's
// netns, where the host-end veth carries 169.254.169.254/30 with a default route
// via the .253 LRP — so the host-served reply has a real L3 next-hop back to the
// guest (SO_BINDTODEVICE alone in the root netns could not route the SYN-ACK).
// The netns also isolates VPCs whose CIDRs overlap, since each is its own routing
// domain. The socket must be *created* in the netns, so we enter it on a locked
// thread for the socket call and restore the root netns before serving.
// SO_BINDTODEVICE (belt-and-braces — there is only one veth in the netns) and
// SO_REUSEADDR (fast rebind on vpcd restart) are still set on the fd.
func bindLocalListener(ctx context.Context, netnsName, hostEnd string) (net.Listener, error) {
lc := net.ListenConfig{
Control: func(_, _ string, c syscall.RawConn) error {
var sockErr error
if err := c.Control(func(fd uintptr) {
if sockErr = unix.SetsockoptString(int(fd), unix.SOL_SOCKET, unix.SO_BINDTODEVICE, hostEnd); sockErr != nil {
return
}
sockErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEADDR, 1)
}); err != nil {
return err
}
return sockErr
},
}
bind := func() (net.Listener, error) {
return lc.Listen(ctx, "tcp4", net.JoinHostPort(MetaDataServerIP, "80"))
}
if netnsName == "" {
// No netns (tests, or a degraded host): bind in the current netns.
return bind()
}
var ln net.Listener
err := inNetns(netnsName, func() error {
var e error
ln, e = bind()
return e
})
if err != nil {
return nil, err
}
return ln, nil
}
// inNetns runs fn on a dedicated OS thread switched into the named network
// namespace, then restores the root netns. fn's socket-creating syscalls happen
// in the target netns, so the listener fd belongs to it; once created, the fd is
// served normally from any thread. If the root netns cannot be restored the
// thread is poisoned, so the goroutine returns while still locked and the Go
// runtime retires the thread rather than returning it to the pool.
func inNetns(netnsName string, fn func() error) error {
const netnsDir = "/var/run/netns/"
errCh := make(chan error, 1)
go func() {
runtime.LockOSThread()
orig, err := os.Open("/proc/thread-self/ns/net")
if err != nil {
runtime.UnlockOSThread()
errCh <- fmt.Errorf("open current netns: %w", err)
return
}
defer func() { _ = orig.Close() }()
target, err := os.Open(netnsDir + netnsName)
if err != nil {
runtime.UnlockOSThread()
errCh <- fmt.Errorf("open netns %s: %w", netnsName, err)
return
}
defer func() { _ = target.Close() }()
if err := unix.Setns(int(target.Fd()), unix.CLONE_NEWNET); err != nil {
runtime.UnlockOSThread()
errCh <- fmt.Errorf("setns into %s: %w", netnsName, err)
return
}
fnErr := fn()
if err := unix.Setns(int(orig.Fd()), unix.CLONE_NEWNET); err != nil {
// Cannot return to the root netns — leave the thread locked so the
// runtime destroys it when this goroutine exits.
errCh <- errors.Join(fnErr, fmt.Errorf("restore root netns: %w", err))
return
}
runtime.UnlockOSThread()
errCh <- fnErr
}()
return <-errCh
}
spinifex/handlers/imds/server_test.go
+122
View File
@@ -0,0 +1,122 @@
package handlers_imds
import (
"context"
"io"
"net"
"net/http"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/mulgadc/spinifex/spinifex/testutil"
"github.com/mulgadc/spinifex/spinifex/utils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// loopbackListen returns a listenFunc backed by a real 127.0.0.1 listener so
// bind-manager logic runs end-to-end (including http.Server.Serve and the
// per-listener BaseContext) without root or the link-local address. It records
// the bound address per host-end so the test can dial it.
func loopbackListen(addrs *sync.Map) listenFunc {
return func(ctx context.Context, netnsName, hostEnd string) (net.Listener, error) {
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
return nil, err
}
addrs.Store(hostEnd, ln.Addr().String())
return ln, nil
}
}
func TestBindManager_BindServesWithVPCContext(t *testing.T) {
const hostEnd = "imds-h-abc12345"
var ensured, removed atomic.Int32
ensure := func(_ context.Context, _ string) (string, string, error) {
ensured.Add(1)
return "", hostEnd, nil
}
remove := func(_ context.Context, _ string) error {
removed.Add(1)
return nil
}
// Echo the VPC ID the bind manager threaded into the request context — this
// is the (VPC-ID, source-IP) → identity path the security model relies on.
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
vpcID, _ := r.Context().Value(ctxKeyVPCID).(string)
_, _ = io.WriteString(w, vpcID)
})
var addrs sync.Map
bm := newBindManager(nil, handler, ensure, remove, loopbackListen(&addrs))
ctx := context.Background()
require.NoError(t, bm.bind(ctx, "vpc-abc12345"))
assert.Equal(t, int32(1), ensured.Load())
// Second bind of the same VPC is a no-op (idempotent), no extra veth ensure.
require.NoError(t, bm.bind(ctx, "vpc-abc12345"))
assert.Equal(t, int32(1), ensured.Load())
raw, ok := addrs.Load(hostEnd)
require.True(t, ok)
addr, ok := raw.(string)
require.True(t, ok)
resp, err := http.Get("http://" + addr + prefixMetaData)
require.NoError(t, err)
body, _ := io.ReadAll(resp.Body)
_ = resp.Body.Close()
assert.Equal(t, "vpc-abc12345", string(body), "handler must see the listener's VPC ID")
// Unbind closes the listener and tears down the veth.
bm.unbind(ctx, "vpc-abc12345")
assert.Equal(t, int32(1), removed.Load())
client := http.Client{Timeout: 500 * time.Millisecond}
_, err = client.Get("http://" + addr + prefixMetaData)
assert.Error(t, err, "listener must be closed after unbind")
}
func TestBindManager_BindPropagatesVethError(t *testing.T) {
ensure := func(_ context.Context, _ string) (string, string, error) {
return "", "", errEnsureFailed
}
bm := newBindManager(nil, http.NewServeMux(), ensure, func(context.Context, string) error { return nil }, loopbackListen(&sync.Map{}))
err := bm.bind(context.Background(), "vpc-x")
require.Error(t, err)
}
var errEnsureFailed = io.ErrUnexpectedEOF
// TestBindManager_SyncSkipsVersionKey guards that the schema-version marker
// migrate.RunKV stamps into the vpc-veth bucket is not mistaken for a VPC ID
// and made to bring up a bogus veth + listener.
func TestBindManager_SyncSkipsVersionKey(t *testing.T) {
_, _, js := testutil.StartTestJetStream(t)
vpcVeth, _, err := InitBuckets(js, 1)
require.NoError(t, err)
// InitBuckets → migrate.RunKV stamps utils.VersionKey; add one real VPC too.
_, err = vpcVeth.PutString("vpc-abcdef12", "{}")
require.NoError(t, err)
var bound sync.Map
ensure := func(_ context.Context, vpcID string) (string, string, error) {
bound.Store(vpcID, true)
return "imds-" + vpcID, "imds-h-" + vpcID, nil
}
remove := func(context.Context, string) error { return nil }
bm := newBindManager(vpcVeth, http.NewServeMux(), ensure, remove, loopbackListen(&sync.Map{}))
require.NoError(t, bm.sync(context.Background()))
defer bm.shutdown()
_, versionBound := bound.Load(utils.VersionKey)
assert.False(t, versionBound, "_version marker must not be bound as a VPC")
_, vpcBound := bound.Load("vpc-abcdef12")
assert.True(t, vpcBound, "real VPC entry must be bound")
}
spinifex/handlers/imds/service.go
+161
View File
@@ -0,0 +1,161 @@
package handlers_imds
import (
"context"
"errors"
"fmt"
"log/slog"
"net/http"
"time"
"github.com/aws/aws-sdk-go/service/iam"
handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
"github.com/nats-io/nats.go"
)
// tokenSweepInterval is how often abandoned/expired IMDSv2 tokens are swept.
const tokenSweepInterval = time.Minute
// profileLookup is the slice of IAMService the IMDS credential + iam/* paths
// need: dereference an instance-profile ARN to its record, and resolve a role
// name to its canonical ARN. Narrowed to an interface for unit testing.
type profileLookup interface {
ResolveInstanceProfile(accountID, nameOrARN string) (*handlers_iam.InstanceProfile, error)
GetRole(accountID string, input *iam.GetRoleInput) (*iam.GetRoleOutput, error)
}
// IMDSService is the host-served EC2 Instance Metadata Service. It runs in
// vpcd on every chassis, serving 169.254.169.254 to local guest VMs over
// per-VPC veths. Run owns the listener lifecycle and blocks until ctx is done.
type IMDSService interface {
Run(ctx context.Context) error
}
var _ IMDSService = (*IMDSServiceImpl)(nil)
// eniResolver maps a datapath-attested (vpcID, srcIP) to ENI + instance facts.
// An interface so the HTTP handlers are unit-testable without a live KV;
// metadataResolver is the production implementation.
type eniResolver interface {
resolveENI(vpcID, srcIP string) (*eniFacts, error)
resolveInstance(eni *eniFacts) (*instanceFacts, error)
resolveSGNames(accountID string, sgIDs []string) []string
}
// IMDSServiceImpl is the in-process IMDS implementation. It is not part of the
// HTTPS gateway dispatch surface — it owns its own per-VPC listener stack.
type IMDSServiceImpl struct {
resolver eniResolver
tokens *tokenStore
creds *credCache
iam profileLookup
bind *bindManager
now func() time.Time
}
// NewIMDSServiceImpl wires the IMDS service. natsConn backs the KV reads and the
// account-scoped instance fan-out (sized by expectedNodes); ensureVeth/removeVeth
// are injected rather than imported to avoid an import cycle via network/host.
func NewIMDSServiceImpl(natsConn *nats.Conn, sts stsAssumer, iamSvc profileLookup, expectedNodes int, ensureVeth ensureVethFunc, removeVeth removeVethFunc) (*IMDSServiceImpl, error) {
if natsConn == nil {
return nil, errors.New("nil NATS connection")
}
if sts == nil {
return nil, errors.New("nil STS service")
}
if iamSvc == nil {
return nil, errors.New("nil IAM service")
}
if ensureVeth == nil || removeVeth == nil {
return nil, errors.New("nil veth lifecycle hooks")
}
js, err := natsConn.JetStream()
if err != nil {
return nil, fmt.Errorf("get JetStream context: %w", err)
}
indexKV, err := js.KeyValue(KVBucketENIByVPCIP)
if err != nil {
return nil, fmt.Errorf("open %s bucket: %w", KVBucketENIByVPCIP, err)
}
eniKV, err := js.KeyValue(kvBucketENIs)
if err != nil {
return nil, fmt.Errorf("open %s bucket: %w", kvBucketENIs, err)
}
vethKV, err := js.KeyValue(KVBucketIMDSVPCVeth)
if err != nil {
return nil, fmt.Errorf("open %s bucket: %w", KVBucketIMDSVPCVeth, err)
}
// The SG bucket is owned by the VPC service and only feeds the non-critical
// security-groups path. Open it best-effort: if it isn't up yet, IMDS still
// starts and resolveSGNames degrades to serving raw IDs.
sgKV, err := js.KeyValue(kvBucketSecurityGroups)
if err != nil {
slog.Warn("IMDS: security-group bucket unavailable, /security-groups will serve IDs", "bucket", kvBucketSecurityGroups, "err", err)
sgKV = nil
}
svc := &IMDSServiceImpl{
resolver: &metadataResolver{
index: indexKV,
eniKV: eniKV,
sgKV: sgKV,
lookup: &natsInstanceLookup{nc: natsConn, expectedNodes: expectedNodes},
},
tokens: newTokenStore(),
creds: newCredCache(sts),
iam: iamSvc,
now: time.Now,
}
svc.bind = newBindManager(vethKV, svc.httpHandler(), ensureVeth, removeVeth, bindLocalListener)
slog.Info("IMDS service initialized",
"index_bucket", KVBucketENIByVPCIP,
"veth_bucket", KVBucketIMDSVPCVeth)
return svc, nil
}
// httpHandler builds the shared mux used by every per-VPC listener. The token
// PUT is the only path reachable without a token; everything else is gated.
func (s *IMDSServiceImpl) httpHandler() http.Handler {
mux := http.NewServeMux()
mux.HandleFunc(pathToken, s.handleToken)
mux.HandleFunc("/", s.handleMetadata)
return rejectForwarded(mux)
}
// Run starts the bind manager (sync + watch) and the token-sweep ticker, then
// blocks until ctx is cancelled. The initial sync failing is logged but not
// fatal — other vpcd services must not have their readiness gated on IMDS
// plumbing, and SDKs retry while it converges.
func (s *IMDSServiceImpl) Run(ctx context.Context) error {
if err := s.bind.sync(ctx); err != nil {
slog.Warn("IMDS: initial bind sync failed, continuing", "err", err)
}
go s.bind.watch(ctx)
go s.sweepExpired(ctx)
<-ctx.Done()
s.bind.shutdown()
return ctx.Err()
}
// sweepExpired evicts expired IMDSv2 tokens and stale credential-cache entries
// on a ticker for the life of ctx. Both stores are in-memory and accrete on VM
// churn, so a single sweep keeps them bounded without a per-store goroutine.
func (s *IMDSServiceImpl) sweepExpired(ctx context.Context) {
ticker := time.NewTicker(tokenSweepInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
now := s.now()
s.tokens.sweep(now)
s.creds.sweep(now)
}
}
}
spinifex/handlers/imds/token.go
+85
View File
@@ -0,0 +1,85 @@
package handlers_imds
import (
"crypto/rand"
"encoding/base64"
"sync"
"time"
)
const (
// tokenTTLMin / tokenTTLMax bound the X-aws-ec2-metadata-token-ttl-seconds
// header, matching AWS (1 second to 6 hours).
tokenTTLMin = 1
tokenTTLMax = 21600
// tokenBytes is the random token length before base64url encoding.
tokenBytes = 32
)
// tokenEntry binds an issued IMDSv2 token to the ENI that requested it (not the
// IP — so the session survives a DHCP-renew IP change but dies on ENI detach)
// and its expiry.
type tokenEntry struct {
eniID string
expiresAt time.Time
}
// tokenStore holds outstanding IMDSv2 tokens in memory, per process. Tokens are
// a CSRF defence, not a security artefact, so they are never persisted; a
// vpcd restart drops them and SDKs reissue transparently.
type tokenStore struct {
mu sync.Mutex
tokens map[string]tokenEntry
}
func newTokenStore() *tokenStore {
return &tokenStore{tokens: make(map[string]tokenEntry)}
}
// issue mints a fresh token bound to eniID, valid for ttl from now.
func (s *tokenStore) issue(eniID string, ttl time.Duration, now time.Time) (string, error) {
buf := make([]byte, tokenBytes)
if _, err := rand.Read(buf); err != nil {
return "", err
}
token := base64.RawURLEncoding.EncodeToString(buf)
s.mu.Lock()
s.tokens[token] = tokenEntry{eniID: eniID, expiresAt: now.Add(ttl)}
s.mu.Unlock()
return token, nil
}
// validate reports whether token is currently valid AND was issued to
// expectedENI. A wrong-ENI token is rejected exactly like an unknown one so the
// caller cannot probe which tokens exist. Expired tokens are evicted lazily.
func (s *tokenStore) validate(token, expectedENI string, now time.Time) bool {
if token == "" {
return false
}
s.mu.Lock()
defer s.mu.Unlock()
entry, ok := s.tokens[token]
if !ok {
return false
}
if now.After(entry.expiresAt) {
delete(s.tokens, token)
return false
}
return entry.eniID == expectedENI
}
// sweep deletes every token that expired at or before now. Called on a 1-minute
// ticker so abandoned tokens (an SDK that issues then never reads) don't leak.
func (s *tokenStore) sweep(now time.Time) {
s.mu.Lock()
defer s.mu.Unlock()
for token, entry := range s.tokens {
if now.After(entry.expiresAt) {
delete(s.tokens, token)
}
}
}
spinifex/handlers/imds/token_test.go
+95
View File
@@ -0,0 +1,95 @@
package handlers_imds
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestTokenStore_IssueAndValidate(t *testing.T) {
s := newTokenStore()
now := time.Unix(1_700_000_000, 0)
token, err := s.issue("eni-aaa", 60*time.Second, now)
require.NoError(t, err)
require.NotEmpty(t, token)
assert.True(t, s.validate(token, "eni-aaa", now), "fresh token for its ENI must validate")
assert.True(t, s.validate(token, "eni-aaa", now.Add(59*time.Second)), "token valid until TTL")
}
func TestTokenStore_WrongENIRejected(t *testing.T) {
s := newTokenStore()
now := time.Unix(1_700_000_000, 0)
token, err := s.issue("eni-aaa", 60*time.Second, now)
require.NoError(t, err)
// A token issued to eni-aaa must not validate for eni-bbb — otherwise a
// guest could replay a peer's token to read the peer's session.
assert.False(t, s.validate(token, "eni-bbb", now))
}
func TestTokenStore_ExpiredRejectedAndEvicted(t *testing.T) {
s := newTokenStore()
now := time.Unix(1_700_000_000, 0)
token, err := s.issue("eni-aaa", 1*time.Second, now)
require.NoError(t, err)
assert.False(t, s.validate(token, "eni-aaa", now.Add(2*time.Second)), "expired token must be rejected")
// Lazy eviction: the failed validate above dropped it, so even at a valid
// time it no longer exists.
assert.False(t, s.validate(token, "eni-aaa", now))
}
func TestTokenStore_UnknownAndEmptyRejected(t *testing.T) {
s := newTokenStore()
now := time.Unix(1_700_000_000, 0)
assert.False(t, s.validate("", "eni-aaa", now))
assert.False(t, s.validate("never-issued", "eni-aaa", now))
}
func TestTokenStore_Sweep(t *testing.T) {
s := newTokenStore()
now := time.Unix(1_700_000_000, 0)
live, _ := s.issue("eni-live", 3600*time.Second, now)
dead, _ := s.issue("eni-dead", 1*time.Second, now)
s.sweep(now.Add(2 * time.Second))
assert.True(t, s.validate(live, "eni-live", now.Add(2*time.Second)), "live token survives sweep")
assert.False(t, s.validate(dead, "eni-dead", now), "expired token swept")
}
func TestTokenStore_IssueUnique(t *testing.T) {
s := newTokenStore()
now := time.Unix(1_700_000_000, 0)
a, _ := s.issue("eni-aaa", 60*time.Second, now)
b, _ := s.issue("eni-aaa", 60*time.Second, now)
assert.NotEqual(t, a, b, "each issued token must be distinct")
}
func TestParseTokenTTL(t *testing.T) {
cases := []struct {
in string
ok bool
secs int
}{
{"", false, 0},
{"abc", false, 0},
{"0", false, 0},
{"21601", false, 0},
{"-5", false, 0},
{"1", true, 1},
{"21600", true, 21600},
{"60", true, 60},
}
for _, c := range cases {
d, ok := parseTokenTTL(c.in)
assert.Equal(t, c.ok, ok, "ttl=%q", c.in)
if ok {
assert.Equal(t, c.secs, int(d.Seconds()), "ttl=%q", c.in)
}
}
}
spinifex/handlers/imds/veth_store.go
+72
View File
@@ -0,0 +1,72 @@
package handlers_imds
import (
"encoding/json"
"errors"
"fmt"
"github.com/nats-io/nats.go"
)
// VPCVethRecord is the persisted per-VPC IMDS plumbing record (KVBucketIMDSVPCVeth,
// keyed by VPC ID): the signal that a VPC's IMDS OVN topology is installed, replayed
// by each chassis's BindManager. It carries only the LSP MAC and LRP /30 the host can't re-derive.
type VPCVethRecord struct {
VPCID string `json:"vpc_id"`
ShortVPCID string `json:"short_vpc_id"`
IMDSPortMAC string `json:"imds_port_mac"`
LRPNetwork string `json:"lrp_network"`
CreatedAt string `json:"created_at"`
}
// VethStore is the persistence surface for VPCVethRecord rows in
// KVBucketIMDSVPCVeth. Backed by a NATS JetStream KV bucket opened by
// InitBuckets in production; tests inject the handle via NewVethStore.
type VethStore struct {
kv nats.KeyValue
}
// NewVethStore wraps an already-opened vpc-veth KV bucket.
func NewVethStore(kv nats.KeyValue) *VethStore { return &VethStore{kv: kv} }
// Get returns the record for vpcID, or (nil, nil) when absent.
func (s *VethStore) Get(vpcID string) (*VPCVethRecord, error) {
raw, err := s.kv.Get(vpcID)
if err != nil {
if errors.Is(err, nats.ErrKeyNotFound) {
return nil, nil
}
return nil, fmt.Errorf("get imds veth record %q: %w", vpcID, err)
}
var rec VPCVethRecord
if err := json.Unmarshal(raw.Value(), &rec); err != nil {
return nil, fmt.Errorf("unmarshal imds veth record %q: %w", vpcID, err)
}
return &rec, nil
}
// Put writes the record keyed by its VPCID, overwriting any prior entry.
func (s *VethStore) Put(rec VPCVethRecord) error {
if rec.VPCID == "" {
return errors.New("imds veth store put: vpc_id is empty")
}
data, err := json.Marshal(rec)
if err != nil {
return fmt.Errorf("marshal imds veth record: %w", err)
}
if _, err := s.kv.Put(rec.VPCID, data); err != nil {
return fmt.Errorf("put imds veth record %q: %w", rec.VPCID, err)
}
return nil
}
// Delete removes the record. Idempotent: a missing key returns nil.
func (s *VethStore) Delete(vpcID string) error {
if err := s.kv.Delete(vpcID); err != nil {
if errors.Is(err, nats.ErrKeyNotFound) {
return nil
}
return fmt.Errorf("delete imds veth record %q: %w", vpcID, err)
}
return nil
}
spinifex/handlers/imds/veth_store_test.go
+57
View File
@@ -0,0 +1,57 @@
package handlers_imds
import (
"testing"
"github.com/mulgadc/spinifex/spinifex/testutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func newTestVethStore(t *testing.T) *VethStore {
t.Helper()
_, _, js := testutil.StartTestJetStream(t)
vpcVeth, _, err := InitBuckets(js, 1)
require.NoError(t, err)
return NewVethStore(vpcVeth)
}
func TestVethStore_PutGetRoundTrip(t *testing.T) {
s := newTestVethStore(t)
rec := VPCVethRecord{
VPCID: "vpc-abc12345",
ShortVPCID: "abc12345",
IMDSPortMAC: "02:aa:bb:cc:dd:ee",
LRPNetwork: "169.254.169.253/30",
CreatedAt: "2026-05-29T00:00:00Z",
}
require.NoError(t, s.Put(rec))
got, err := s.Get("vpc-abc12345")
require.NoError(t, err)
require.NotNil(t, got)
assert.Equal(t, rec, *got)
}
func TestVethStore_GetAbsentReturnsNil(t *testing.T) {
s := newTestVethStore(t)
got, err := s.Get("vpc-nope")
require.NoError(t, err)
assert.Nil(t, got)
}
func TestVethStore_PutRejectsEmptyVPCID(t *testing.T) {
s := newTestVethStore(t)
require.Error(t, s.Put(VPCVethRecord{ShortVPCID: "abc12345"}))
}
func TestVethStore_DeleteIsIdempotent(t *testing.T) {
s := newTestVethStore(t)
require.NoError(t, s.Delete("vpc-missing"))
require.NoError(t, s.Put(VPCVethRecord{VPCID: "vpc-1"}))
require.NoError(t, s.Delete("vpc-1"))
got, err := s.Get("vpc-1")
require.NoError(t, err)
assert.Nil(t, got)
}
spinifex/handlers/sts/assume_role.go
+111 -28
View File
@@ -11,6 +11,7 @@ import (
"fmt"
"log/slog"
"regexp"
"slices"
"strings"
"time"
@@ -28,6 +29,11 @@ const (
stsActionWildcard = "sts:*"
globalWildcard = "*"
// ec2ServicePrincipal is the synthetic caller used by AssumeRoleForInstance — the
// only service principal that matches a trust policy in v1. The HTTPS AssumeRole
// path supplies an empty principalSource, so service principals never match there.
ec2ServicePrincipal = "ec2.amazonaws.com"
minDurationSeconds int64 = 900
maxDurationSeconds int64 = 43200
defaultDurationSeconds int64 = 3600
@@ -72,7 +78,62 @@ func (s *STSServiceImpl) AssumeRole(callerAccountID, callerARN, callerIdentity s
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
}
roleAccountID, roleName, err := parseRoleARN(*input.RoleArn)
duration := int64(0)
if input.DurationSeconds != nil {
duration = *input.DurationSeconds
}
out, err := s.assumeRoleForCaller(callerAccountID, callerARN, "", *input.RoleArn, sessionName, aws.StringValue(input.SourceIdentity), duration)
if err != nil {
return nil, err
}
slog.Info("AssumeRole success",
"caller_arn", callerARN,
"caller_identity", callerIdentity,
"role_arn", aws.StringValue(input.RoleArn),
"session_name", sessionName,
"akid", aws.StringValue(out.Credentials.AccessKeyId),
"expires_at", aws.TimeValue(out.Credentials.Expiration),
"source_identity", aws.StringValue(input.SourceIdentity),
"external_id", aws.StringValue(input.ExternalId),
)
return out, nil
}
// AssumeRoleForInstance is the in-process IMDS entry point, not reachable over HTTPS.
// The caller is synthesised as the EC2 service principal (principalSource =
// ec2.amazonaws.com); the session name is the instanceID, yielding an AWS-style ARN.
func (s *STSServiceImpl) AssumeRoleForInstance(accountID, roleARN, instanceID string, durationSeconds int64) (*sts.AssumeRoleOutput, error) {
if accountID == "" || roleARN == "" || instanceID == "" {
return nil, errors.New(awserrors.ErrorMissingParameter)
}
if !roleSessionNameRegex.MatchString(instanceID) {
return nil, errors.New(awserrors.ErrorValidationError)
}
out, err := s.assumeRoleForCaller(accountID, "", ec2ServicePrincipal, roleARN, instanceID, "", durationSeconds)
if err != nil {
return nil, err
}
slog.Info("AssumeRoleForInstance success",
"account_id", accountID,
"instance_id", instanceID,
"role_arn", roleARN,
"akid", aws.StringValue(out.Credentials.AccessKeyId),
"expires_at", aws.TimeValue(out.Credentials.Expiration),
)
return out, nil
}
// assumeRoleForCaller is the shared core of AssumeRole (HTTPS) and AssumeRoleForInstance
// (IMDS): resolves the role, validates the duration (0 → default), evaluates the trust
// policy, and mints credentials. principalSource is "" for HTTPS, ec2.amazonaws.com for IMDS.
func (s *STSServiceImpl) assumeRoleForCaller(callerAccountID, callerARN, principalSource, roleARN, sessionName, sourceIdentity string, requestedDuration int64) (*sts.AssumeRoleOutput, error) {
roleAccountID, roleName, err := parseRoleARN(roleARN)
if err != nil {
return nil, errors.New(awserrors.ErrorValidationError)
}
@@ -90,9 +151,9 @@ func (s *STSServiceImpl) AssumeRole(callerAccountID, callerARN, callerIdentity s
}
role := roleOut.Role
duration := defaultDurationSeconds
if input.DurationSeconds != nil {
duration = *input.DurationSeconds
duration := requestedDuration
if duration == 0 {
duration = defaultDurationSeconds
}
effectiveMax := aws.Int64Value(role.MaxSessionDuration)
if effectiveMax == 0 {
@@ -105,26 +166,15 @@ func (s *STSServiceImpl) AssumeRole(callerAccountID, callerARN, callerIdentity s
return nil, errors.New(awserrors.ErrorValidationError)
}
if err := evalTrustPolicy(aws.StringValue(role.AssumeRolePolicyDocument), callerARN); err != nil {
if err := evalTrustPolicy(aws.StringValue(role.AssumeRolePolicyDocument), callerARN, principalSource); err != nil {
return nil, err
}
cred, plainSecret, plainToken, err := s.mintSessionCredential(role, roleAccountID, sessionName, aws.StringValue(input.SourceIdentity), duration)
cred, plainSecret, plainToken, err := s.mintSessionCredential(role, roleAccountID, sessionName, sourceIdentity, duration)
if err != nil {
return nil, err
}
slog.Info("AssumeRole success",
"caller_arn", callerARN,
"caller_identity", callerIdentity,
"role_arn", aws.StringValue(role.Arn),
"session_name", sessionName,
"akid", cred.AccessKeyID,
"expires_at", cred.ExpiresAt,
"source_identity", aws.StringValue(input.SourceIdentity),
"external_id", aws.StringValue(input.ExternalId),
)
return &sts.AssumeRoleOutput{
Credentials: &sts.Credentials{
AccessKeyId: aws.String(cred.AccessKeyID),
@@ -172,7 +222,7 @@ func parseRoleARN(arnStr string) (accountID, name string, err error) {
// pass scans every Allow statement (returning nil on the first match). A
// single-pass "return on first Allow" loop would silently skip a later Deny
// and is a real authorisation bug.
func evalTrustPolicy(docJSON, callerARN string) error {
func evalTrustPolicy(docJSON, callerARN, principalSource string) error {
doc, err := handlers_iam.ValidateTrustPolicyDocument(docJSON)
if err != nil {
// Stored docs were validated at CreateRole / UpdateAssumeRolePolicy
@@ -187,7 +237,7 @@ func evalTrustPolicy(docJSON, callerARN string) error {
if !matchTrustAction(stmt.Action) {
continue
}
match, err := matchTrustPrincipal(stmt.Principal, callerARN)
match, err := matchTrustPrincipal(stmt.Principal, callerARN, principalSource)
if err != nil {
return err
}
@@ -203,7 +253,7 @@ func evalTrustPolicy(docJSON, callerARN string) error {
if !matchTrustAction(stmt.Action) {
continue
}
match, err := matchTrustPrincipal(stmt.Principal, callerARN)
match, err := matchTrustPrincipal(stmt.Principal, callerARN, principalSource)
if err != nil {
return err
}
@@ -224,11 +274,10 @@ func matchTrustAction(actions []string) bool {
return false
}
// matchTrustPrincipal evaluates each top-level Principal key independently —
// AWS semantics treat the map as an OR across keys, and unsupported keys
// (Service, Federated) skip at the entry level rather than failing the whole
// statement.
func matchTrustPrincipal(raw json.RawMessage, callerARN string) (bool, error) {
// matchTrustPrincipal evaluates each top-level Principal key independently (AWS
// treats the map as an OR). AWS matches callerARN, Service matches principalSource
// (set only by the IMDS path); unsupported keys (Federated) skip, not fail.
func matchTrustPrincipal(raw json.RawMessage, callerARN, principalSource string) (bool, error) {
if len(raw) == 0 {
return false, nil
}
@@ -242,11 +291,45 @@ func matchTrustPrincipal(raw json.RawMessage, callerARN string) (bool, error) {
if err := json.Unmarshal(raw, &m); err != nil {
return false, fmt.Errorf("unmarshal Principal: %w", err)
}
awsRaw, ok := m["AWS"]
if !ok {
if awsRaw, ok := m["AWS"]; ok {
match, err := matchAWSPrincipal(awsRaw, callerARN)
if err != nil {
return false, err
}
if match {
return true, nil
}
}
if svcRaw, ok := m["Service"]; ok {
match, err := matchServicePrincipal(svcRaw, principalSource)
if err != nil {
return false, err
}
if match {
return true, nil
}
}
return false, nil
}
// matchServicePrincipal matches a Service principal against the synthesised caller.
// v1 trusts only the EC2 service principal: any other principalSource (including
// the empty one used by the HTTPS path) is rejected up front, so unsupported
// service principals are denied even if a future caller synthesised one.
// Expanding the whitelist (ecs-tasks, lambda) is a one-line change to this guard.
func matchServicePrincipal(raw json.RawMessage, principalSource string) (bool, error) {
if principalSource != ec2ServicePrincipal {
return false, nil
}
return matchAWSPrincipal(awsRaw, callerARN)
var single string
if err := json.Unmarshal(raw, &single); err == nil {
return single == principalSource, nil
}
var arr []string
if err := json.Unmarshal(raw, &arr); err != nil {
return false, fmt.Errorf("Principal.Service must be string or array: %w", err)
}
return slices.Contains(arr, principalSource), nil
}
func matchAWSPrincipal(raw json.RawMessage, callerARN string) (bool, error) {
spinifex/handlers/sts/assume_role_test.go
+104 -1
View File
@@ -557,6 +557,109 @@ func TestAssumeRole_ServicePrincipal_NeverMatchesInV1(t *testing.T) {
assert.Equal(t, awserrors.ErrorAccessDenied, err.Error())
}
// ----- AssumeRoleForInstance (IMDS in-process path) ----------------------
const testInstanceID = "i-0123456789abcdef0"
func trustPolicyAllowingEC2Service() string {
return `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"ec2.amazonaws.com"},"Action":"sts:AssumeRole"}]}`
}
func TestAssumeRoleForInstance_EC2ServicePrincipal_Allowed(t *testing.T) {
svc, _ := newTestSetup(t)
role := createRoleInAccount(t, svc, testCallerAccountID, "app-role", trustPolicyAllowingEC2Service())
out, err := svc.AssumeRoleForInstance(testCallerAccountID, *role.Arn, testInstanceID, defaultDurationSeconds)
require.NoError(t, err)
require.NotNil(t, out)
require.NotNil(t, out.Credentials)
akid := aws.StringValue(out.Credentials.AccessKeyId)
assert.True(t, strings.HasPrefix(akid, SessionAccessKeyIDPrefix))
assert.Len(t, akid, 20)
// The session name is the instance ID, matching AWS.
assert.Equal(t, fmt.Sprintf("arn:aws:sts::%s:assumed-role/app-role/%s", testCallerAccountID, testInstanceID),
aws.StringValue(out.AssumedRoleUser.Arn))
stored, err := svc.LookupSessionCredential(akid)
require.NoError(t, err)
require.NotNil(t, stored)
assert.Equal(t, testInstanceID, stored.SessionName)
assert.Equal(t, testCallerAccountID, stored.AccountID)
}
// Service principals other than ec2.amazonaws.com are not whitelisted in v1, so
// even though the caller is synthesised as the EC2 service principal, a
// lambda.amazonaws.com trust statement must not match.
func TestAssumeRoleForInstance_NonWhitelistedService_Denied(t *testing.T) {
svc, _ := newTestSetup(t)
policy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"lambda.amazonaws.com"},"Action":"sts:AssumeRole"}]}`
role := createRoleInAccount(t, svc, testCallerAccountID, "lambda-role", policy)
_, err := svc.AssumeRoleForInstance(testCallerAccountID, *role.Arn, testInstanceID, defaultDurationSeconds)
require.Error(t, err)
assert.Equal(t, awserrors.ErrorAccessDenied, err.Error())
}
// A role that only trusts an AWS principal (no Service entry) must not be
// assumable by an instance — the synthesised EC2 caller has no ARN to match.
func TestAssumeRoleForInstance_AWSOnlyPrincipal_Denied(t *testing.T) {
svc, _ := newTestSetup(t)
role := createRoleInAccount(t, svc, testCallerAccountID, "user-role", trustPolicyAllowingUser(testCallerARN()))
_, err := svc.AssumeRoleForInstance(testCallerAccountID, *role.Arn, testInstanceID, defaultDurationSeconds)
require.Error(t, err)
assert.Equal(t, awserrors.ErrorAccessDenied, err.Error())
}
// Regression: the EC2 service principal must remain unreachable over the HTTPS
// AssumeRole path. A role trusting only ec2.amazonaws.com is never assumable by
// a normal SigV4 caller, no matter what — the security boundary between the two
// entry points.
func TestAssumeRoleForInstance_EC2ServiceUnreachableViaHTTPS(t *testing.T) {
svc, _ := newTestSetup(t)
role := createRoleInAccount(t, svc, testCallerAccountID, "ec2-only", trustPolicyAllowingEC2Service())
_, err := svc.AssumeRole(testCallerAccountID, testCallerARN(), testCallerUserName,
basicAssumeRoleInput(*role.Arn, "sess"))
require.Error(t, err)
assert.Equal(t, awserrors.ErrorAccessDenied, err.Error())
}
func TestAssumeRoleForInstance_RejectsMissingFields(t *testing.T) {
svc, _ := newTestSetup(t)
role := createRoleInAccount(t, svc, testCallerAccountID, "fields", trustPolicyAllowingEC2Service())
cases := []struct {
name string
accountID string
roleARN string
instanceID string
}{
{"missing_account", "", *role.Arn, testInstanceID},
{"missing_role_arn", testCallerAccountID, "", testInstanceID},
{"missing_instance_id", testCallerAccountID, *role.Arn, ""},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
_, err := svc.AssumeRoleForInstance(tc.accountID, tc.roleARN, tc.instanceID, defaultDurationSeconds)
require.Error(t, err)
assert.Equal(t, awserrors.ErrorMissingParameter, err.Error())
})
}
}
func TestAssumeRoleForInstance_CrossAccountRoleMiss_AccessDenied(t *testing.T) {
svc, _ := newTestSetup(t)
// Instance's account differs from the role's account and the role is
// missing — masked to AccessDenied to prevent cross-account enumeration.
_, err := svc.AssumeRoleForInstance(testCallerAccountID,
fmt.Sprintf("arn:aws:iam::%s:role/ghost", testCrossAccountID), testInstanceID, defaultDurationSeconds)
require.Error(t, err)
assert.Equal(t, awserrors.ErrorAccessDenied, err.Error())
}
func TestAssumeRole_RetriesOnAKIDCollision(t *testing.T) {
svc, _ := newTestSetup(t)
caller := testCallerARN()
@@ -643,7 +746,7 @@ func TestEvalTrustPolicy_CorruptDocReturnsError(t *testing.T) {
// Stored docs are validated upstream; reaching evalTrustPolicy with a
// malformed doc indicates corruption — must fail closed and NOT collapse
// to AccessDenied (which would hide the corruption from operators).
err := evalTrustPolicy(`{not json`, testCallerARN())
err := evalTrustPolicy(`{not json`, testCallerARN(), "")
require.Error(t, err)
assert.NotEqual(t, awserrors.ErrorAccessDenied, err.Error(),
"corrupt doc must not be reported as AccessDenied — operators need to see the corruption signal")
spinifex/handlers/sts/imds_rpc.go
+36
View File
@@ -0,0 +1,36 @@
package handlers_sts
import (
"github.com/aws/aws-sdk-go/service/sts"
"github.com/mulgadc/spinifex/spinifex/utils"
"github.com/nats-io/nats.go"
)
// imdsResponderQueue load-balances the IMDS control-plane RPCs across every
// awsgw in the cluster, matching the cluster-wide worker convention.
const imdsResponderQueue = "spinifex-workers"
// SubjectAssumeRoleForInstance is the internal, ACL-gated subject the IMDS handler
// (in vpcd) calls to mint instance-role credentials. STS keeps the IAM master key
// in-process inside awsgw; only this narrow RPC is surfaced, never to a guest.
const SubjectAssumeRoleForInstance = "imds.sts.assume_role_for_instance"
// AssumeRoleForInstanceRequest is the SubjectAssumeRoleForInstance payload. The
// account ID is data here (resolved from the requesting ENI), not an
// authenticated caller identity.
type AssumeRoleForInstanceRequest struct {
AccountID string `json:"account_id"`
RoleARN string `json:"role_arn"`
InstanceID string `json:"instance_id"`
DurationSeconds int64 `json:"duration_seconds"`
}
// SubscribeIMDSResponder wires the AssumeRoleForInstance responder onto nc,
// queue-grouped so any awsgw can answer.
func (s *STSServiceImpl) SubscribeIMDSResponder(nc *nats.Conn) (*nats.Subscription, error) {
return nc.QueueSubscribe(SubjectAssumeRoleForInstance, imdsResponderQueue, func(msg *nats.Msg) {
utils.ServeNATSRequest(msg, func(req *AssumeRoleForInstanceRequest) (*sts.AssumeRoleOutput, error) {
return s.AssumeRoleForInstance(req.AccountID, req.RoleARN, req.InstanceID, req.DurationSeconds)
})
})
}
spinifex/handlers/sts/service.go
+5
View File
@@ -17,6 +17,11 @@ type STSService interface {
// strings to keep the handler unit-testable.
AssumeRole(callerAccountID, callerARN, callerIdentity string, input *sts.AssumeRoleInput) (*sts.AssumeRoleOutput, error)
// AssumeRoleForInstance mints role-bound temporary credentials for an EC2 instance.
// It is the in-process IMDS entry point, NOT reachable over HTTPS: the caller is the
// synthesised EC2 service principal (trust must allow Service ec2.amazonaws.com).
AssumeRoleForInstance(accountID, roleARN, instanceID string, durationSeconds int64) (*sts.AssumeRoleOutput, error)
// AssumeRoleWithWebIdentity exchanges an OIDC ID token (typically a
// projected K8s ServiceAccount token signed by an EKS cluster's
// per-cluster signing key) for short-lived AWS credentials bound to the
spinifex/network/external/igw.go
Vendored
+20 -7
View File
@@ -298,7 +298,7 @@ func (m *igwManager) RemoveSubnetEgress(ctx context.Context, vpcID, subnetID str
if vpcID == "" || subnetID == "" {
return errors.New("RemoveSubnetEgress: vpcID and subnetID required")
}
return m.routes.DeleteSubnetEgress(ctx, vpcID, subnetID, prefix, m.vpcExcludeCIDRs(ctx, vpcID))
return m.routes.DeleteSubnetEgress(ctx, vpcID, subnetID, prefix, m.rerouteExcludeCIDRs(ctx, vpcID))
}
// EnsureNATGatewaySubnetEgress is the NATGW priority sibling of
@@ -315,7 +315,7 @@ func (m *igwManager) RemoveNATGatewaySubnetEgress(ctx context.Context, vpcID, su
if vpcID == "" || subnetID == "" {
return errors.New("RemoveNATGatewaySubnetEgress: vpcID and subnetID required")
}
return m.routes.DeleteSubnetEgress(ctx, vpcID, subnetID, prefix, m.vpcExcludeCIDRs(ctx, vpcID))
return m.routes.DeleteSubnetEgress(ctx, vpcID, subnetID, prefix, m.rerouteExcludeCIDRs(ctx, vpcID))
}
// EnsureSubnetEgressDrop installs a drop policy at SubnetEgressPriorityDrop
@@ -361,7 +361,7 @@ func (m *igwManager) ensureSubnetEgressAtPriority(ctx context.Context, vpcID, su
Nexthop: nexthop,
OutputPort: topology.GatewayRouterPort(vpcID),
Priority: priority,
ExcludeCIDRs: m.vpcExcludeCIDRs(ctx, vpcID),
ExcludeCIDRs: m.rerouteExcludeCIDRs(ctx, vpcID),
})
}
@@ -437,10 +437,9 @@ var (
)
// dropExcludeCIDRs returns the exclusion list for a per-subnet drop policy:
// VPC CIDR (if discoverable) plus link-local and multicast. Reroute policies
// only need the VPC CIDR exclusion because their match scope is narrower; the
// drop policy is a default-deny within the subnet's egress so it must spare
// any class the platform legitimately uses outside the VPC.
// VPC CIDR (if discoverable) plus link-local and multicast. The drop policy is
// a default-deny within the subnet's egress so it must spare any class the
// platform legitimately uses outside the VPC.
func (m *igwManager) dropExcludeCIDRs(ctx context.Context, vpcID string) []netip.Prefix {
excludes := []netip.Prefix{linkLocalCIDR, multicastCIDR}
if vpc := m.vpcExcludeCIDRs(ctx, vpcID); len(vpc) > 0 {
@@ -449,6 +448,20 @@ func (m *igwManager) dropExcludeCIDRs(ctx context.Context, vpcID string) []netip
return excludes
}
// rerouteExcludeCIDRs returns the exclusion list for a per-subnet egress
// reroute policy: VPC CIDR (if discoverable) plus link-local. The reroute
// matches 0.0.0.0/0 — the widest possible scope — so it would otherwise
// hijack 169.254.169.254 (IMDS) out the gateway and defeat the per-VPC IMDS
// /32 static route, which fires earlier in lr_in_ip_routing but is overridden
// by this lr_in_policy reroute. Excluding link-local lets IMDS fall through to
// the /32. Multicast is not excluded here (unlike the drop policy): a reroute
// only diverts traffic that already has somewhere to go, so leaking multicast
// to the gateway is harmless, whereas the drop policy would kill it outright.
func (m *igwManager) rerouteExcludeCIDRs(ctx context.Context, vpcID string) []netip.Prefix {
excludes := m.vpcExcludeCIDRs(ctx, vpcID)
return append(excludes, linkLocalCIDR)
}
// vpcExcludeCIDRs fetches the VPC's primary CIDR from the LR ExternalIDs
// label so per-subnet egress reroute policies skip in-VPC destinations.
// Returns nil on lookup failure or unset label — caller installs a policy
spinifex/network/external/imds_datapath_contract_test.go
Vendored
+252
View File
@@ -0,0 +1,252 @@
package external
import (
"context"
"net/netip"
"sort"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
handlers_imds "github.com/mulgadc/spinifex/spinifex/handlers/imds"
"github.com/mulgadc/spinifex/spinifex/network/ovn/mock"
"github.com/mulgadc/spinifex/spinifex/network/policy"
"github.com/mulgadc/spinifex/spinifex/network/topology"
)
// IMDS datapath contract — desk-speed pipeline oracle.
//
// Every IMDS datapath bug to date (the egress-reroute hijack of the /32, the
// reply-path break) lived inside the VPC logical-router pipeline and was only
// caught by a full VM-boot E2E run, one bug per deploy. This test makes the
// load-bearing half of that pipeline assertable in `go test`: it builds the
// REAL managers (IGW egress reroute + drop, IMDS topology) against the OVN mock
// for two VPCs that share the *identical* CIDR — the overlapping-CIDR case that
// is the whole point of per-VPC IMDS identity — then runs a faithful, minimal
// simulation of OVN's two ingress stages (lr_in_policy, then lr_in_ip_routing)
// and asserts a guest packet to 169.254.169.254 reaches the IMDS LRP rather
// than being rerouted out the WAN or dropped.
//
// A true round-trip oracle is ovn-trace against a live SB DB; that belongs in
// the e2e harness. This is its desk-speed counterpart over the same NB rows —
// it runs on every `make preflight`, so a future broad lr_in_policy rule that
// forgets to spare link-local fails here instead of in a deploy.
const (
imdsContractGuestIP = "10.211.0.5"
imdsContractCIDR = "10.211.0.0/16"
)
func TestIMDSDatapathContract_SurvivesVPCPipeline_OverlappingCIDRs(t *testing.T) {
ctx := context.Background()
m := mock.New()
// Two VPCs on the IDENTICAL CIDR, each with internet egress and a subnet
// whose first guest gets the same private IP. The OVN NB rows for the two
// must each independently route IMDS correctly.
vpcs := []struct{ vpcID, subnetID string }{
{"vpc-aaaa1111", "subnet-aaaa1111"},
{"vpc-bbbb2222", "subnet-bbbb2222"},
}
imdsMgr, _ := newIMDSManager(t, m)
igwMgr, _ := newTestIGWManager(t, m, policy.NATModeDistributed,
&ExternalPoolConfig{Name: "p", Gateway: "192.168.1.1", PrefixLen: 24},
LinkLocalAllocator{}, []string{"chassis-a"})
defaultRoute := netip.MustParsePrefix("0.0.0.0/0")
for _, v := range vpcs {
seedVPCRouter(t, m, v.vpcID, imdsContractCIDR)
require.NoError(t, igwMgr.AttachIGW(ctx, IGWSpec{VPCID: v.vpcID, InternetGatewayID: "igw-" + v.vpcID}))
// The broad 0.0.0.0/0 egress reroute + drop — the pipeline stages that
// previously hijacked / killed the IMDS /32 before #19 excluded link-local.
require.NoError(t, igwMgr.EnsureSubnetEgress(ctx, v.vpcID, v.subnetID, defaultRoute))
require.NoError(t, igwMgr.EnsureSubnetEgressDrop(ctx, v.vpcID, v.subnetID, defaultRoute))
// The IMDS /32 static route + imds-lrp.
_, err := imdsMgr.EnsureForVPC(ctx, v.vpcID)
require.NoError(t, err)
}
for _, v := range vpcs {
assertIMDSDatapathContract(t, m, v.vpcID, v.subnetID, imdsContractGuestIP)
}
}
// TestIMDSDatapathContract_OracleHasTeeth proves the simulator actually detects
// the regression it guards against: a 0.0.0.0/0 egress reroute that does NOT
// exclude link-local (the pre-#19 bug) must be reported as diverting IMDS.
// Without this, a vacuously-passing contract test would give false confidence.
func TestIMDSDatapathContract_OracleHasTeeth(t *testing.T) {
ctx := context.Background()
m := mock.New()
const vpcID, subnetID = "vpc-cccc3333", "subnet-cccc3333"
seedVPCRouter(t, m, vpcID, imdsContractCIDR)
rm := policy.NewRouteManager(m)
gw := "169.254.0.2"
// Reroute excluding ONLY the VPC CIDR — link-local omitted, reproducing the
// bug #19 fixed. IMDS (169.254.169.254) is in 0.0.0.0/0 and not in the VPC
// CIDR, so this policy hijacks it.
require.NoError(t, rm.AddSubnetEgress(ctx, vpcID, policy.SubnetEgressSpec{
SubnetID: subnetID,
Prefix: netip.MustParsePrefix("0.0.0.0/0"),
Nexthop: gw,
OutputPort: topology.GatewayRouterPort(vpcID),
Priority: policy.SubnetEgressPriorityIGW,
ExcludeCIDRs: []netip.Prefix{netip.MustParsePrefix(imdsContractCIDR)},
}))
res := resolveEgress(t, m, vpcID, subnetID, handlers_imds.MetaDataServerIP)
assert.True(t, res.diverted,
"simulator must detect a link-local-unaware reroute as hijacking IMDS; if this fails the oracle is blind")
}
// assertIMDSDatapathContract is the contract: a guest packet to
// 169.254.169.254 must reach the IMDS LRP, and the IMDS reply must not be
// caught by any egress policy.
func assertIMDSDatapathContract(t *testing.T, m *mock.Client, vpcID, subnetID, guestIP string) {
t.Helper()
imdsLRP := topology.IMDSRouterPort(vpcID)
// Request: guest -> 169.254.169.254, ingressing on the subnet LRP.
req := resolveEgress(t, m, vpcID, subnetID, handlers_imds.MetaDataServerIP)
assert.Falsef(t, req.diverted,
"IMDS contract (%s): an lr_in_policy reroute on 0.0.0.0/0 hijacks 169.254.169.254 out the WAN; "+
"every broad egress reroute MUST exclude 169.254.0.0/16 (see decision #19)", vpcID)
assert.Falsef(t, req.dropped,
"IMDS contract (%s): an lr_in_policy drop kills 169.254.169.254; "+
"the per-subnet drop policy MUST exclude 169.254.0.0/16", vpcID)
assert.Equalf(t, "169.254.169.254/32", req.routePrefix,
"IMDS contract (%s): 169.254.169.254 must be resolved by the /32 IMDS static route, not a broader route", vpcID)
assert.Equalf(t, imdsLRP, req.outputPort,
"IMDS contract (%s): the IMDS /32 must egress via %s", vpcID, imdsLRP)
// Reply: 169.254.169.254 -> guest, ingressing on the IMDS LRP. The egress
// policies are inport-scoped to the subnet LRP, so none may catch the reply;
// a future inport-less broad policy would break IMDS replies and fail here.
reply := resolveEgressFrom(t, m, vpcID, imdsLRP, guestIP)
assert.Falsef(t, reply.diverted,
"IMDS contract (%s): an lr_in_policy must not divert the IMDS reply to %s (policies must stay inport-scoped)", vpcID, guestIP)
assert.Falsef(t, reply.dropped,
"IMDS contract (%s): an lr_in_policy must not drop the IMDS reply to %s", vpcID, guestIP)
}
// --- minimal OVN LR ingress-pipeline model ------------------------------------
type egressResult struct {
diverted bool // a reroute lr_in_policy matched
dropped bool // a drop lr_in_policy matched
outputPort string // lr_in_ip_routing winner's output port (empty if a policy already decided)
routePrefix string // lr_in_ip_routing winner's prefix
}
// resolveEgress simulates the VPC LR ingress pipeline for a packet arriving on
// the subnet LRP destined to dstIP.
func resolveEgress(t *testing.T, m *mock.Client, vpcID, subnetID, dstIP string) egressResult {
t.Helper()
return resolveEgressFrom(t, m, vpcID, topology.SubnetRouterPort(subnetID), dstIP)
}
// resolveEgressFrom models OVN's two ingress stages in order:
//
// lr_in_policy — Logical_Router_Policy, highest Priority first, first
// applicable wins (reroute => diverted, drop => dropped).
// lr_in_ip_routing — longest-prefix-match over static routes.
func resolveEgressFrom(t *testing.T, m *mock.Client, vpcID, inport, dstIP string) egressResult {
t.Helper()
dst, err := netip.ParseAddr(dstIP)
require.NoError(t, err)
router := topology.VPCRouter(vpcID)
// Stage 1: lr_in_policy.
policies, err := m.ListLogicalRouterPolicies(context.Background(), router)
require.NoError(t, err)
sort.SliceStable(policies, func(i, j int) bool { return policies[i].Priority > policies[j].Priority })
for _, p := range policies {
mt, ok := parsePolicyMatch(p.Match)
if !ok || !mt.applies(inport, dst) {
continue
}
switch p.Action {
case "reroute":
return egressResult{diverted: true}
case "drop":
return egressResult{dropped: true}
case "allow":
// falls through to routing
}
break
}
// Stage 2: lr_in_ip_routing (longest-prefix-match).
lr, err := m.GetLogicalRouter(context.Background(), router)
require.NoError(t, err)
var best egressResult
bestBits := -1
for _, uuid := range lr.StaticRoutes {
r := m.StaticRoutes[uuid]
if r == nil {
continue
}
pfx, err := netip.ParsePrefix(r.IPPrefix)
if err != nil || !pfx.Contains(dst) {
continue
}
if pfx.Bits() > bestBits {
bestBits = pfx.Bits()
out := ""
if r.OutputPort != nil {
out = *r.OutputPort
}
best = egressResult{outputPort: out, routePrefix: r.IPPrefix}
}
}
return best
}
type policyMatch struct {
inport string // "" => no inport constraint
dst netip.Prefix
excludes []netip.Prefix
}
func (mt policyMatch) applies(inport string, dst netip.Addr) bool {
if mt.inport != "" && mt.inport != inport {
return false
}
if mt.dst.IsValid() && !mt.dst.Contains(dst) {
return false
}
for _, ex := range mt.excludes {
if ex.Contains(dst) {
return false
}
}
return true
}
// parsePolicyMatch parses the match strings produced by subnetEgressMatch:
//
// inport == "rtr-subnet-x" && ip4.dst == 0.0.0.0/0 && ip4.dst != 10.0.0.0/16 && ip4.dst != 169.254.0.0/16
func parsePolicyMatch(match string) (policyMatch, bool) {
var mt policyMatch
for clause := range strings.SplitSeq(match, "&&") {
clause = strings.TrimSpace(clause)
switch {
case strings.HasPrefix(clause, "inport =="):
mt.inport = strings.Trim(strings.TrimSpace(strings.TrimPrefix(clause, "inport ==")), `"`)
case strings.HasPrefix(clause, "ip4.dst =="):
if p, err := netip.ParsePrefix(strings.TrimSpace(strings.TrimPrefix(clause, "ip4.dst =="))); err == nil {
mt.dst = p
}
case strings.HasPrefix(clause, "ip4.dst !="):
if p, err := netip.ParsePrefix(strings.TrimSpace(strings.TrimPrefix(clause, "ip4.dst !="))); err == nil {
mt.excludes = append(mt.excludes, p)
}
}
}
return mt, true
}
spinifex/network/external/imds_topology.go
Vendored
+209
View File
@@ -0,0 +1,209 @@
package external
import (
"context"
"errors"
"fmt"
"log/slog"
"net/netip"
"time"
handlers_imds "github.com/mulgadc/spinifex/spinifex/handlers/imds"
"github.com/mulgadc/spinifex/spinifex/network/host"
"github.com/mulgadc/spinifex/spinifex/network/ovn"
"github.com/mulgadc/spinifex/spinifex/network/ovn/nbdb"
"github.com/mulgadc/spinifex/spinifex/network/policy"
"github.com/mulgadc/spinifex/spinifex/network/topology"
"github.com/mulgadc/spinifex/spinifex/utils"
)
// imdsLRPNetwork is the /30 the IMDS LRP terminates on the VPC router. .253 is
// the LRP, .254 is the host-owned IMDS LSP (MetaDataServerIP); the /30 makes
// the route to .254 directly-connected on the IMDS switch.
const imdsLRPNetwork = "169.254.169.253/30"
// metaDataServerRoute is the /32 the VPC router uses to send IMDS traffic out
// the IMDS LRP instead of the WAN default.
var metaDataServerRoute = netip.PrefixFrom(netip.MustParseAddr(handlers_imds.MetaDataServerIP), 32)
// IMDSVPCSpec is the resolved set of OVN + host names for a VPC's IMDS
// plumbing. Every field is a deterministic function of the VPC ID, so callers
// can rebuild it from the VPC ID alone (BindManager does this per bucket entry).
type IMDSVPCSpec struct {
VPCID string
ShortVPCID string // last 8 chars of VPCID
LSName string // imds-ls-{vpcID}
LRPName string // imds-lrp-{vpcID}
RouterLSPName string // imds-rtr-{vpcID} (type=router)
LSPName string // imds-port-{vpcID} (localport; host veth binds here)
LSPMAC string // 02:.. — HashMAC("imds-"+vpcID)
LRPNetwork string // imdsLRPNetwork
OVSEndName string // imds-o-{shortVpcID}
HostEndName string // imds-h-{shortVpcID}
}
// IMDSTopologyManager installs and removes the per-VPC OVN topology that makes
// 169.254.169.254 routable to the host-owned IMDS localport. Idempotent. Lives in
// external (not policy): it composes L2 creation with L3 routing (ADR-0006 S5).
type IMDSTopologyManager interface {
EnsureForVPC(ctx context.Context, vpcID string) (IMDSVPCSpec, error)
RemoveForVPC(ctx context.Context, vpcID string) error
}
type imdsTopologyManager struct {
ovn ovn.Client
routes policy.RouteManager
store *handlers_imds.VethStore
}
var _ IMDSTopologyManager = (*imdsTopologyManager)(nil)
// NewIMDSTopologyManager constructs an IMDSTopologyManager. store is the
// vpc-veth KV bucket the installer publishes records to.
func NewIMDSTopologyManager(client ovn.Client, routes policy.RouteManager, store *handlers_imds.VethStore) (IMDSTopologyManager, error) {
switch {
case client == nil:
return nil, errors.New("IMDSTopologyManager: OVN client required")
case routes == nil:
return nil, errors.New("IMDSTopologyManager: RouteManager required")
case store == nil:
return nil, errors.New("IMDSTopologyManager: VethStore required")
}
return &imdsTopologyManager{ovn: client, routes: routes, store: store}, nil
}
// IMDSSpecForVPC returns the deterministic name/MAC set for a VPC's IMDS
// plumbing without touching OVN or the bucket.
func IMDSSpecForVPC(vpcID string) IMDSVPCSpec {
return IMDSVPCSpec{
VPCID: vpcID,
ShortVPCID: host.IMDSShortVPCID(vpcID),
LSName: topology.IMDSSwitch(vpcID),
LRPName: topology.IMDSRouterPort(vpcID),
RouterLSPName: topology.IMDSSwitchRouterPort(vpcID),
LSPName: topology.IMDSPort(vpcID),
LSPMAC: utils.HashMAC("imds-" + vpcID),
LRPNetwork: imdsLRPNetwork,
OVSEndName: host.IMDSOVSPortName(vpcID),
HostEndName: host.IMDSHostVethName(vpcID),
}
}
// EnsureForVPC installs the IMDS OVN topology for vpcID and publishes the vpc-veth
// record. Idempotent: a present record short-circuits, and each OVN object is created
// only when absent so a lost record still converges. The VPC router must already exist.
func (m *imdsTopologyManager) EnsureForVPC(ctx context.Context, vpcID string) (IMDSVPCSpec, error) {
if vpcID == "" {
return IMDSVPCSpec{}, errors.New("EnsureForVPC: vpcID required")
}
spec := IMDSSpecForVPC(vpcID)
// The bucket record is the publish gate: present means topology is
// installed and BindManagers have (or will) materialise host state.
rec, err := m.store.Get(vpcID)
if err != nil {
return IMDSVPCSpec{}, fmt.Errorf("read imds veth record for %s: %w", vpcID, err)
}
if rec != nil {
return spec, nil
}
router := topology.VPCRouter(vpcID)
extIDs := map[string]string{"spinifex:vpc_id": vpcID, "spinifex:role": "imds"}
if _, err := m.ovn.EnsureLogicalSwitch(ctx, &nbdb.LogicalSwitch{Name: spec.LSName, ExternalIDs: extIDs}); err != nil {
return IMDSVPCSpec{}, fmt.Errorf("ensure imds switch %s: %w", spec.LSName, err)
}
if _, err := m.ovn.GetLogicalRouterPort(ctx, spec.LRPName); err != nil {
if cErr := m.ovn.CreateLogicalRouterPort(ctx, router, &nbdb.LogicalRouterPort{
Name: spec.LRPName,
MAC: utils.HashMAC(spec.LRPName),
Networks: []string{spec.LRPNetwork},
ExternalIDs: extIDs,
}); cErr != nil {
return IMDSVPCSpec{}, fmt.Errorf("create imds router port %s: %w", spec.LRPName, cErr)
}
}
if _, err := m.ovn.GetLogicalSwitchPort(ctx, spec.RouterLSPName); err != nil {
if cErr := m.ovn.CreateLogicalSwitchPort(ctx, spec.LSName, &nbdb.LogicalSwitchPort{
Name: spec.RouterLSPName,
Type: "router",
Addresses: []string{"router"},
Options: map[string]string{"router-port": spec.LRPName},
ExternalIDs: extIDs,
}); cErr != nil {
return IMDSVPCSpec{}, fmt.Errorf("create imds router LSP %s: %w", spec.RouterLSPName, cErr)
}
}
// Host-owned localport: no port_security (the host is trusted to source
// 169.254.169.254 frames, and port_security would make ovn-controller drop
// reply frames from the host veth's MAC). localport binds on every chassis.
if _, err := m.ovn.GetLogicalSwitchPort(ctx, spec.LSPName); err != nil {
if cErr := m.ovn.CreateLogicalSwitchPort(ctx, spec.LSName, &nbdb.LogicalSwitchPort{
Name: spec.LSPName,
Type: "localport",
Addresses: []string{spec.LSPMAC + " " + handlers_imds.MetaDataServerIP},
ExternalIDs: extIDs,
}); cErr != nil {
return IMDSVPCSpec{}, fmt.Errorf("create imds localport %s: %w", spec.LSPName, cErr)
}
}
if err := m.routes.AddStaticRoute(ctx, vpcID, policy.RouteSpec{
Prefix: metaDataServerRoute,
Nexthop: handlers_imds.MetaDataServerIP,
OutputPort: spec.LRPName,
}); err != nil {
return IMDSVPCSpec{}, fmt.Errorf("add imds static route on %s: %w", router, err)
}
if err := m.store.Put(handlers_imds.VPCVethRecord{
VPCID: vpcID,
ShortVPCID: spec.ShortVPCID,
IMDSPortMAC: spec.LSPMAC,
LRPNetwork: spec.LRPNetwork,
CreatedAt: time.Now().UTC().Format(time.RFC3339),
}); err != nil {
return IMDSVPCSpec{}, fmt.Errorf("publish imds veth record for %s: %w", vpcID, err)
}
slog.Info("external: installed IMDS topology",
"vpc_id", vpcID, "imds_switch", spec.LSName, "imds_lrp", spec.LRPName, "imds_port", spec.LSPName)
return spec, nil
}
// RemoveForVPC tears down the IMDS OVN topology and the vpc-veth record.
// Idempotent and drift-tolerant: missing OVN objects are logged and skipped so the
// record delete always runs. Must run before the VPC router (it holds the LRP/route).
func (m *imdsTopologyManager) RemoveForVPC(ctx context.Context, vpcID string) error {
if vpcID == "" {
return errors.New("RemoveForVPC: vpcID required")
}
spec := IMDSSpecForVPC(vpcID)
router := topology.VPCRouter(vpcID)
if err := m.routes.DeleteStaticRoute(ctx, vpcID, metaDataServerRoute); err != nil {
slog.Warn("external: delete imds static route failed", "vpc_id", vpcID, "err", err)
}
if err := m.ovn.DeleteLogicalSwitchPort(ctx, spec.LSName, spec.LSPName); err != nil {
slog.Warn("external: delete imds localport failed", "port", spec.LSPName, "err", err)
}
if err := m.ovn.DeleteLogicalSwitchPort(ctx, spec.LSName, spec.RouterLSPName); err != nil {
slog.Warn("external: delete imds router LSP failed", "port", spec.RouterLSPName, "err", err)
}
if err := m.ovn.DeleteLogicalRouterPort(ctx, router, spec.LRPName); err != nil {
slog.Warn("external: delete imds router port failed", "port", spec.LRPName, "err", err)
}
if err := m.ovn.DeleteLogicalSwitch(ctx, spec.LSName); err != nil {
slog.Warn("external: delete imds switch failed", "switch", spec.LSName, "err", err)
}
if err := m.store.Delete(vpcID); err != nil {
return fmt.Errorf("delete imds veth record for %s: %w", vpcID, err)
}
slog.Info("external: removed IMDS topology", "vpc_id", vpcID)
return nil
}
spinifex/network/external/imds_topology_test.go
Vendored
+168
View File
@@ -0,0 +1,168 @@
package external
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
handlers_imds "github.com/mulgadc/spinifex/spinifex/handlers/imds"
"github.com/mulgadc/spinifex/spinifex/network/ovn/mock"
"github.com/mulgadc/spinifex/spinifex/network/ovn/nbdb"
"github.com/mulgadc/spinifex/spinifex/network/policy"
"github.com/mulgadc/spinifex/spinifex/testutil"
)
func newIMDSManager(t *testing.T, m *mock.Client) (IMDSTopologyManager, *handlers_imds.VethStore) {
t.Helper()
_, _, js := testutil.StartTestJetStream(t)
vpcVeth, _, err := handlers_imds.InitBuckets(js, 1)
require.NoError(t, err)
store := handlers_imds.NewVethStore(vpcVeth)
mgr, err := NewIMDSTopologyManager(m, policy.NewRouteManager(m), store)
require.NoError(t, err)
return mgr, store
}
func findIMDSRoute(m *mock.Client, prefix string) *nbdb.LogicalRouterStaticRoute {
for _, r := range m.StaticRoutes {
if r.IPPrefix == prefix {
return r
}
}
return nil
}
func TestNewIMDSTopologyManager_RejectsMissingDeps(t *testing.T) {
m := mock.New()
_, _, js := testutil.StartTestJetStream(t)
vpcVeth, _, err := handlers_imds.InitBuckets(js, 1)
require.NoError(t, err)
store := handlers_imds.NewVethStore(vpcVeth)
_, err = NewIMDSTopologyManager(nil, policy.NewRouteManager(m), store)
require.Error(t, err)
_, err = NewIMDSTopologyManager(m, nil, store)
require.Error(t, err)
_, err = NewIMDSTopologyManager(m, policy.NewRouteManager(m), nil)
require.Error(t, err)
}
func TestIMDSEnsureForVPC_InstallsTopology(t *testing.T) {
ctx := context.Background()
m := mock.New()
seedVPCRouter(t, m, "vpc-0a1b2c3d4e5f6789", "10.0.0.0/16")
mgr, store := newIMDSManager(t, m)
spec, err := mgr.EnsureForVPC(ctx, "vpc-0a1b2c3d4e5f6789")
require.NoError(t, err)
// Resolved names are deterministic functions of the VPC ID.
assert.Equal(t, "imds-ls-vpc-0a1b2c3d4e5f6789", spec.LSName)
assert.Equal(t, "imds-lrp-vpc-0a1b2c3d4e5f6789", spec.LRPName)
assert.Equal(t, "imds-port-vpc-0a1b2c3d4e5f6789", spec.LSPName)
assert.Equal(t, "4e5f6789", spec.ShortVPCID)
assert.Equal(t, "imds-o-4e5f6789", spec.OVSEndName)
assert.Equal(t, "imds-h-4e5f6789", spec.HostEndName)
// Logical switch exists, tagged as IMDS.
ls, err := m.GetLogicalSwitch(ctx, spec.LSName)
require.NoError(t, err)
assert.Equal(t, "imds", ls.ExternalIDs["spinifex:role"])
// LRP on the VPC router holds the /30.
lrp, err := m.GetLogicalRouterPort(ctx, spec.LRPName)
require.NoError(t, err)
assert.Equal(t, []string{imdsLRPNetwork}, lrp.Networks)
// type=router LSP peers the switch with the LRP.
rlsp, err := m.GetLogicalSwitchPort(ctx, spec.RouterLSPName)
require.NoError(t, err)
assert.Equal(t, "router", rlsp.Type)
assert.Equal(t, spec.LRPName, rlsp.Options["router-port"])
// Host-owned localport: claims the MetaData IP, no port_security.
lsp, err := m.GetLogicalSwitchPort(ctx, spec.LSPName)
require.NoError(t, err)
assert.Equal(t, "localport", lsp.Type)
assert.Empty(t, lsp.PortSecurity)
assert.Equal(t, []string{spec.LSPMAC + " " + handlers_imds.MetaDataServerIP}, lsp.Addresses)
// Static route on vpc-{vpcID} points 169.254.169.254/32 at the IMDS LRP.
route := findIMDSRoute(m, handlers_imds.MetaDataServerIP+"/32")
require.NotNil(t, route)
assert.Equal(t, handlers_imds.MetaDataServerIP, route.Nexthop)
require.NotNil(t, route.OutputPort)
assert.Equal(t, spec.LRPName, *route.OutputPort)
// Record published to the bucket.
rec, err := store.Get("vpc-0a1b2c3d4e5f6789")
require.NoError(t, err)
require.NotNil(t, rec)
assert.Equal(t, spec.ShortVPCID, rec.ShortVPCID)
assert.Equal(t, spec.LSPMAC, rec.IMDSPortMAC)
assert.Equal(t, imdsLRPNetwork, rec.LRPNetwork)
assert.NotEmpty(t, rec.CreatedAt)
}
func TestIMDSEnsureForVPC_Idempotent(t *testing.T) {
ctx := context.Background()
m := mock.New()
seedVPCRouter(t, m, "vpc-1", "10.0.0.0/16")
mgr, _ := newIMDSManager(t, m)
for range 3 {
_, err := mgr.EnsureForVPC(ctx, "vpc-1")
require.NoError(t, err)
}
// Exactly one IMDS switch and one static route.
assert.Len(t, m.Switches, 1)
count := 0
for _, r := range m.StaticRoutes {
if r.IPPrefix == handlers_imds.MetaDataServerIP+"/32" {
count++
}
}
assert.Equal(t, 1, count)
lsp, err := m.GetLogicalSwitchPort(ctx, "imds-port-vpc-1")
require.NoError(t, err)
assert.Equal(t, "localport", lsp.Type)
}
func TestIMDSEnsureForVPC_MissingRouterErrors(t *testing.T) {
ctx := context.Background()
m := mock.New()
// No router seeded — the LRP create has nowhere to land.
mgr, store := newIMDSManager(t, m)
_, err := mgr.EnsureForVPC(ctx, "vpc-missing")
require.Error(t, err)
// No record published when install fails.
rec, err := store.Get("vpc-missing")
require.NoError(t, err)
assert.Nil(t, rec)
}
func TestIMDSRemoveForVPC_TearsDownAndIsIdempotent(t *testing.T) {
ctx := context.Background()
m := mock.New()
seedVPCRouter(t, m, "vpc-1", "10.0.0.0/16")
mgr, store := newIMDSManager(t, m)
_, err := mgr.EnsureForVPC(ctx, "vpc-1")
require.NoError(t, err)
require.NoError(t, mgr.RemoveForVPC(ctx, "vpc-1"))
assert.Empty(t, m.Switches)
assert.Nil(t, findIMDSRoute(m, handlers_imds.MetaDataServerIP+"/32"))
rec, err := store.Get("vpc-1")
require.NoError(t, err)
assert.Nil(t, rec)
// Second remove is a no-op.
require.NoError(t, mgr.RemoveForVPC(ctx, "vpc-1"))
}
spinifex/network/external/invariants_test.go
Vendored
+122
View File
@@ -0,0 +1,122 @@
package external
import (
"context"
"fmt"
"net/netip"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
handlers_imds "github.com/mulgadc/spinifex/spinifex/handlers/imds"
"github.com/mulgadc/spinifex/spinifex/network/ovn/mock"
"github.com/mulgadc/spinifex/spinifex/network/policy"
"github.com/mulgadc/spinifex/spinifex/network/topology"
)
// IMDS-datapath invariants. These guard the OVN topology
// the IMDS handler trusts as a security + availability boundary. A regression
// in either is a privilege-escalation or single-point-of-failure vector, so the
// failure messages quote the offending design clause verbatim.
// TestI2_IMDSLSPMustBeLocalportNoPortSecurity asserts the host-owned
// imds-port-{vpcID} LSP is a localport with no port_security. The host — not a
// guest — sources 169.254.169.254 frames on this port, so port_security would
// be actively harmful (it pins the allowed src MAC to the LSP's claimed MAC,
// and ovn-controller would drop reply frames egressing from the host veth's
// MAC). A regular (non-localport) LSP would bind to a single chassis, defeating
// the per-chassis self-serve design.
func TestI2_IMDSLSPMustBeLocalportNoPortSecurity(t *testing.T) {
ctx := context.Background()
m := mock.New()
seedVPCRouter(t, m, "vpc-0a1b2c3d4e5f6789", "10.0.0.0/16")
mgr, _ := newIMDSManager(t, m)
spec, err := mgr.EnsureForVPC(ctx, "vpc-0a1b2c3d4e5f6789")
require.NoError(t, err)
lsp, err := m.GetLogicalSwitchPort(ctx, spec.LSPName)
require.NoError(t, err)
if lsp.Type != "localport" {
t.Errorf("imds LSP %s Type = %q, want \"localport\": a regular LSP binds to "+
"one chassis only, forcing IMDS through Geneve and creating a single point "+
"of failure per VPC", spec.LSPName, lsp.Type)
}
if len(lsp.PortSecurity) != 0 {
t.Errorf("imds LSP %s PortSecurity = %v, want empty: port_security set on the "+
"host-owned LSP would cause ovn-controller to drop reply frames from the "+
"host veth's MAC", spec.LSPName, lsp.PortSecurity)
}
}
// TestI3_IMDSStaticRouteShape asserts the static route on vpc-{vpcID} that
// diverts 169.254.169.254 off the WAN default and out the IMDS LRP has the
// exact shape the datapath depends on. A drifted prefix, nexthop, or output
// port silently sends IMDS traffic to the WAN, where it disappears.
func TestI3_IMDSStaticRouteShape(t *testing.T) {
ctx := context.Background()
m := mock.New()
seedVPCRouter(t, m, "vpc-0a1b2c3d4e5f6789", "10.0.0.0/16")
mgr, _ := newIMDSManager(t, m)
spec, err := mgr.EnsureForVPC(ctx, "vpc-0a1b2c3d4e5f6789")
require.NoError(t, err)
route := findIMDSRoute(m, handlers_imds.MetaDataServerIP+"/32")
require.NotNil(t, route, "no static route for %s/32 on the VPC router — IMDS traffic "+
"falls through to the WAN default", handlers_imds.MetaDataServerIP)
assert.Equal(t, handlers_imds.MetaDataServerIP+"/32", route.IPPrefix)
assert.Equal(t, handlers_imds.MetaDataServerIP, route.Nexthop)
require.NotNil(t, route.OutputPort, "IMDS static route has no output port — OVN cannot "+
"resolve which LRP to send 169.254.169.254 traffic out of")
assert.Equal(t, spec.LRPName, *route.OutputPort)
}
// TestI4_SubnetEgressRerouteMustExcludeLinkLocal asserts every per-subnet
// internet-egress reroute policy excludes 169.254.0.0/16. The reroute matches
// 0.0.0.0/0 — the widest possible scope — and fires in lr_in_policy AFTER the
// IMDS /32 static route matches in lr_in_ip_routing, so without this exclusion
// it overrides the /32 and SNAT-reroutes 169.254.169.254 out the WAN, where
// IMDS disappears (the guest's PUT for a token never reaches the per-VPC netns
// listener). Both the IGW- and NATGW-priority reroutes carry the gate, so both
// must spare link-local. A drifted exclude list silently breaks IMDS on any
// VPC with internet egress.
func TestI4_SubnetEgressRerouteMustExcludeLinkLocal(t *testing.T) {
linkLocal := fmt.Sprintf("ip4.dst != %s", linkLocalCIDR.String())
cases := []struct {
name string
install func(mgr IGWManager, ctx context.Context) error
}{
{"IGW", func(mgr IGWManager, ctx context.Context) error {
return mgr.EnsureSubnetEgress(ctx, "vpc-1", "subnet-pub", netip.MustParsePrefix("0.0.0.0/0"))
}},
{"NATGW", func(mgr IGWManager, ctx context.Context) error {
return mgr.EnsureNATGatewaySubnetEgress(ctx, "vpc-1", "subnet-priv", netip.MustParsePrefix("0.0.0.0/0"))
}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
ctx := context.Background()
m := mock.New()
seedVPCRouter(t, m, "vpc-1", "10.0.0.0/16")
pool := &ExternalPoolConfig{Name: "p", Gateway: "192.168.1.1", PrefixLen: 24}
mgr, _ := newTestIGWManager(t, m, policy.NATModeDistributed, pool, LinkLocalAllocator{}, []string{"chassis-a"})
require.NoError(t, mgr.AttachIGW(ctx, IGWSpec{VPCID: "vpc-1", InternetGatewayID: "igw-1"}))
require.NoError(t, tc.install(mgr, ctx))
policies, err := m.ListLogicalRouterPolicies(ctx, topology.VPCRouter("vpc-1"))
require.NoError(t, err)
require.Len(t, policies, 1)
assert.Contains(t, policies[0].Match, linkLocal,
"%s egress reroute %q does not exclude %s — it will hijack 169.254.169.254 "+
"out the WAN and override the IMDS /32 static route, making IMDS unreachable",
tc.name, policies[0].Match, linkLocalCIDR)
})
}
}
spinifex/network/host/imds_names.go
+34
View File
@@ -0,0 +1,34 @@
package host
// IMDS per-VPC veth naming. The names live here (not in network/topology) so
// the IMDS BindManager can derive them without importing topology, which would
// form an import cycle (topology imports handlers/imds for MetaDataServerIP).
// IMDSShortVPCID is the last 8 chars of the VPC ID. Linux IFNAMSIZ caps interface
// names at 15 chars, so the IMDS veth names key off this short form. Deterministic
// and stable: every chassis serving IMDS for VPC X derives the same names.
func IMDSShortVPCID(vpcID string) string {
if len(vpcID) <= 8 {
return vpcID
}
return vpcID[len(vpcID)-8:]
}
// IMDSOVSPortName is the br-int side of the per-VPC IMDS veth pair, bound to the
// imds-port LSP via external_ids:iface-id. The 7-char prefix keeps "imds-o-" + the
// 8-char short VPC ID at exactly IFNAMSIZ-1 (15); longer fails `ip link add`.
func IMDSOVSPortName(vpcID string) string { return "imds-o-" + IMDSShortVPCID(vpcID) }
// IMDSHostVethName is the host-end side of the per-VPC IMDS veth pair — the
// device the IMDS listener binds to via SO_BINDTODEVICE. Same 7-char prefix
// budget as IMDSOVSPortName: "imds-h-" + 8-char short VPC ID = 15 chars. It
// lives inside the VPC's netns (see IMDSNetnsName), not the root netns.
func IMDSHostVethName(vpcID string) string { return "imds-h-" + IMDSShortVPCID(vpcID) }
// IMDSNetnsName is the per-VPC network namespace the IMDS host-end veth and its
// listener live in. The netns gives the reply path a real L3 next-hop — the
// host end carries 169.254.169.254/30 with a default route via the .253 LRP —
// and structurally isolates VPCs with overlapping CIDRs, since each netns is its
// own routing domain. Netns names are filesystem-scoped, not IFNAMSIZ-bound, so
// "imds-" + 8-char short VPC ID (13 chars) is comfortably within budget.
func IMDSNetnsName(vpcID string) string { return "imds-" + IMDSShortVPCID(vpcID) }
spinifex/network/host/imds_netns_unit_test.go
+97
View File
@@ -0,0 +1,97 @@
package host
import (
"os"
"path/filepath"
"runtime"
"strings"
"testing"
)
// repoRoot resolves the spinifex submodule root from this test file's location
// (spinifex/network/host/ → three levels up) so the systemd-unit invariants can
// read build/ artifacts regardless of the test's working directory.
func repoRoot(t *testing.T) string {
t.Helper()
_, file, _, ok := runtime.Caller(0)
if !ok {
t.Fatal("runtime.Caller failed")
}
root := filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", ".."))
if _, err := os.Stat(filepath.Join(root, "build", "systemd")); err != nil {
t.Fatalf("could not locate build/systemd from %s: %v", root, err)
}
return root
}
func readUnit(t *testing.T, root, rel string) string {
t.Helper()
b, err := os.ReadFile(filepath.Join(root, rel))
if err != nil {
t.Fatalf("read %s: %v", rel, err)
}
return string(b)
}
// TestIMDS_NetnsHostMountNSInvariant pins the host-mount-namespace contract for
// per-VPC IMDS netns (see spinifex-vpcd.service + bead js-133): vpcd creates the
// netns via `ip netns add` and binds the listener via in-process setns, so the
// /run/netns/<vpc> bind-mount must live in the HOST mount namespace — host-visible
// for diagnostics and durable across a vpcd restart. Any filesystem-sandbox
// directive forks a PRIVATE mount ns that traps the bind-mount inside vpcd, so the
// host sees only a stale handle that fails setns(2) EINVAL (CI runs 26920490454,
// 26922587471). The earlier MountFlags=shared + spinifex-netns.service shared-mount
// workaround was proven insufficient and removed; vpcd now simply shares the host
// mount ns. Guard that shape: re-adding any mount-ns directive reintroduces the bug.
func TestIMDS_NetnsHostMountNSInvariant(t *testing.T) {
root := repoRoot(t)
vpcd := readUnit(t, root, "build/systemd/spinifex-vpcd.service")
target := readUnit(t, root, "build/systemd/spinifex.target")
// No filesystem-sandbox directive may be set on vpcd: each forks a private
// mount namespace and re-traps the netns bind-mount.
for _, banned := range []string{
"ProtectSystem", "ProtectHome", "PrivateTmp", "ProtectKernelTunables",
"ProtectKernelModules", "ProtectKernelLogs", "ProtectControlGroups",
"ProtectProc", "ReadOnlyPaths", "ReadWritePaths", "MountFlags",
} {
if directiveSet(vpcd, banned) {
t.Errorf("spinifex-vpcd.service must share the HOST mount ns but sets %s= — a private mount ns traps /run/netns/<vpc> and host setns(2) fails EINVAL", banned)
}
}
// vpcd still needs CAP_SYS_ADMIN for setns(CLONE_NEWNET) and the net+mnt
// namespace allowance for the in-netns `ip` plumbing.
mustContain(t, "spinifex-vpcd.service", vpcd, "CAP_SYS_ADMIN")
mustContain(t, "spinifex-vpcd.service", vpcd, "RestrictNamespaces=net mnt")
// The removed shared-mount workaround must not reappear by reference.
if strings.Contains(vpcd, "spinifex-netns.service") && !strings.Contains(vpcd, "# ") {
t.Errorf("spinifex-vpcd.service must not depend on spinifex-netns.service (removed)")
}
if directiveSet(target, "Wants") && strings.Contains(target, "spinifex-netns.service") {
t.Errorf("spinifex.target must not pull in spinifex-netns.service (removed)")
}
if _, err := os.Stat(filepath.Join(root, "build", "systemd", "spinifex-netns.service")); err == nil {
t.Errorf("build/systemd/spinifex-netns.service must be removed — the shared-mount workaround was proven insufficient")
}
}
// directiveSet reports whether key is set as an actual unit directive (a line
// `key=...`), ignoring comment prose that merely names it.
func directiveSet(body, key string) bool {
for line := range strings.SplitSeq(body, "\n") {
if strings.HasPrefix(strings.TrimSpace(line), key+"=") {
return true
}
}
return false
}
func mustContain(t *testing.T, name, body, want string) {
t.Helper()
if !strings.Contains(body, want) {
t.Errorf("%s: missing %q", name, want)
}
}
spinifex/network/host/imds_veth.go
+217
View File
@@ -0,0 +1,217 @@
package host
import (
"context"
"fmt"
"log/slog"
"strings"
"github.com/mulgadc/spinifex/spinifex/utils"
)
const (
// imdsNetnsAddr is the host/listener side of the IMDS /30, assigned to the
// veth host end inside the per-VPC netns. .254 is MetaDataServerIP; the /30
// makes .253 (the LRP) directly connected so the reply path has a next-hop.
imdsNetnsAddr = "169.254.169.254/30"
// imdsNetnsGateway is the IMDS LRP (.253) — the netns default gateway that
// routes guest replies back into OVN. Mirrors external.imdsLRPNetwork's .253.
imdsNetnsGateway = "169.254.169.253"
)
// imdsPortLSPName is the OVN logical-switch-port the OVS-side veth binds to via
// external_ids:iface-id. It mirrors topology.IMDSPort but is duplicated here to
// avoid an import cycle (host ← handlers/imds ← topology).
func imdsPortLSPName(vpcID string) string { return "imds-port-" + vpcID }
// imdsLSPMAC is the logical MAC OVN advertises for 169.254.169.254 (the localport
// address). The netns veth host end MUST carry this exact MAC: OVN resolves .254
// to it and delivers IMDS request frames with dl_dst set to it, so a kernel-random
// veth MAC makes the netns drop every frame not-for-me and the listener never sees
// a SYN. Mirrors external.IMDSSpecForVPC's LSPMAC, duplicated here (like
// imdsPortLSPName) to avoid the import cycle.
func imdsLSPMAC(vpcID string) string { return utils.HashMAC("imds-" + vpcID) }
// EnsureIMDSVeth idempotently creates the per-VPC IMDS veth pair, moves the host end
// into a dedicated netns with 169.254.169.254/30 + a default route via the .253 LRP,
// and attaches the OVS end to br-int so ovn-controller binds the localport here. The
// netns gives the host-served reply a real L3 next-hop (SO_BINDTODEVICE alone cannot
// route the SYN-ACK) and isolates overlapping VPC CIDRs into separate routing domains.
// Returns the netns name and host-end name the listener enters and binds.
func EnsureIMDSVeth(ctx context.Context, vpcID string) (netnsName, hostEndName string, err error) {
ovsEnd := IMDSOVSPortName(vpcID)
hostEnd := IMDSHostVethName(vpcID)
netns := IMDSNetnsName(vpcID)
// Idempotency probe: the full plumbing (veth pair, netns, host-end address +
// route) exists from a prior boot only when the OVS end is a port on br-int
// AND the netns is enterable. A live OVS port over a stale netns (name
// present but setns(2) fails EINVAL — e.g. a crash between umount and unlink,
// or a tmpfs remount across reboot) leaves the listener permanently
// unbindable, so that case must NOT short-circuit: tear the inconsistent
// plumbing down and rebuild it below.
if imdsOVSPortOnBrInt(ovsEnd) {
if netnsEnterable(netns) {
slog.Debug("IMDS veth already present", "vpc", vpcID, "ovs_end", ovsEnd, "host_end", hostEnd, "netns", netns)
return netns, hostEnd, nil
}
slog.Warn("IMDS netns unenterable behind a live OVS port (stale handle), rebuilding plumbing",
"vpc", vpcID, "netns", netns, "ovs_end", ovsEnd)
if err := removeIMDSPlumbing(ovsEnd, hostEnd, netns); err != nil {
slog.Warn("Failed to tear down stale IMDS plumbing before rebuild", "vpc", vpcID, "err", err)
}
}
if err := ensureNetns(netns); err != nil {
return "", "", err
}
if out, err := utils.SudoCommand("ip", "link", "add", ovsEnd, "type", "veth", "peer", "name", hostEnd).CombinedOutput(); err != nil {
if cleanErr := removeIMDSPlumbing(ovsEnd, hostEnd, netns); cleanErr != nil {
slog.Warn("Failed to clean up IMDS plumbing after veth-create failure", "vpc", vpcID, "err", cleanErr)
}
return "", "", fmt.Errorf("create IMDS veth pair %s/%s: %s: %w", ovsEnd, hostEnd, strings.TrimSpace(string(out)), err)
}
if err := configureIMDSNetns(netns, hostEnd, ovsEnd, imdsLSPMAC(vpcID)); err != nil {
if cleanErr := removeIMDSPlumbing(ovsEnd, hostEnd, netns); cleanErr != nil {
slog.Warn("Failed to clean up IMDS plumbing after netns config failure", "vpc", vpcID, "err", cleanErr)
}
return "", "", err
}
ifaceID := imdsPortLSPName(vpcID)
if out, err := utils.SudoCommand("ovs-vsctl", "add-port", "br-int", ovsEnd,
"--", "set", "Interface", ovsEnd, "external_ids:iface-id="+ifaceID).CombinedOutput(); err != nil {
if cleanErr := removeIMDSPlumbing(ovsEnd, hostEnd, netns); cleanErr != nil {
slog.Warn("Failed to clean up IMDS plumbing after OVS failure", "vpc", vpcID, "err", cleanErr)
}
return "", "", fmt.Errorf("add IMDS veth %s to br-int: %s: %w", ovsEnd, strings.TrimSpace(string(out)), err)
}
slog.Info("IMDS veth plumbing complete", "vpc", vpcID, "ovs_end", ovsEnd, "host_end", hostEnd, "netns", netns, "iface_id", ifaceID)
return netns, hostEnd, nil
}
// configureIMDSNetns moves the host end into the netns, sets its MAC to the
// localport's logical MAC, brings it (and lo) up, and assigns the IMDS address +
// default route so the host-served reply has a real next-hop. The OVS end stays in
// the root netns for ovn-controller to bind. The MAC set is the request-path
// contract: OVN delivers frames with dl_dst = the localport MAC, so the veth host
// end must own that MAC or the netns silently drops them. The addr/route adds
// tolerate "File exists" so a re-run after a partial failure converges rather than
// wedging.
func configureIMDSNetns(netns, hostEnd, ovsEnd, hostEndMAC string) error {
if out, err := utils.SudoCommand("ip", "link", "set", ovsEnd, "up").CombinedOutput(); err != nil {
return fmt.Errorf("bring up IMDS OVS end %s: %s: %w", ovsEnd, strings.TrimSpace(string(out)), err)
}
if out, err := utils.SudoCommand("ip", "link", "set", hostEnd, "netns", netns).CombinedOutput(); err != nil {
return fmt.Errorf("move IMDS host end %s into netns %s: %s: %w", hostEnd, netns, strings.TrimSpace(string(out)), err)
}
if out, err := utils.SudoCommand("ip", "-n", netns, "link", "set", hostEnd, "address", hostEndMAC).CombinedOutput(); err != nil {
return fmt.Errorf("set IMDS host end %s MAC %s in netns %s: %s: %w", hostEnd, hostEndMAC, netns, strings.TrimSpace(string(out)), err)
}
for _, dev := range []string{"lo", hostEnd} {
if out, err := utils.SudoCommand("ip", "-n", netns, "link", "set", dev, "up").CombinedOutput(); err != nil {
return fmt.Errorf("bring up %s in netns %s: %s: %w", dev, netns, strings.TrimSpace(string(out)), err)
}
}
if err := ipNetnsTolerate(netns, "File exists", "addr", "add", imdsNetnsAddr, "dev", hostEnd); err != nil {
return err
}
return ipNetnsTolerate(netns, "File exists", "route", "add", "default", "via", imdsNetnsGateway)
}
// ensureNetns creates the netns, treating "already exists" as success — but
// only when the pre-existing handle is actually enterable. A stale
// /run/netns/<ns> bind-mount (name resolves yet setns(2) fails EINVAL) returns
// "File exists" from `ip netns add` while being unusable, which would leave the
// IMDS listener permanently unbindable; that handle is torn down and recreated.
func ensureNetns(netns string) error {
out, err := utils.SudoCommand("ip", "netns", "add", netns).CombinedOutput()
if err == nil {
return nil
}
if !strings.Contains(string(out), "File exists") {
return fmt.Errorf("create IMDS netns %s: %s: %w", netns, strings.TrimSpace(string(out)), err)
}
if netnsEnterable(netns) {
return nil
}
slog.Warn("IMDS netns present but unenterable (stale handle), recreating", "netns", netns)
if out, err := utils.SudoCommand("ip", "netns", "del", netns).CombinedOutput(); err != nil {
if msg := strings.TrimSpace(string(out)); !strings.Contains(msg, "No such file") {
return fmt.Errorf("delete stale IMDS netns %s: %s: %w", netns, msg, err)
}
}
if out, err := utils.SudoCommand("ip", "netns", "add", netns).CombinedOutput(); err != nil {
return fmt.Errorf("recreate IMDS netns %s: %s: %w", netns, strings.TrimSpace(string(out)), err)
}
return nil
}
// imdsOVSPortOnBrInt reports whether the IMDS OVS-end veth is currently a port
// on br-int — the cheap signal that prior plumbing exists.
func imdsOVSPortOnBrInt(ovsEnd string) bool {
out, err := utils.SudoCommand("ovs-vsctl", "port-to-br", ovsEnd).CombinedOutput()
return err == nil && strings.TrimSpace(string(out)) == "br-int"
}
// netnsEnterable reports whether netns can be entered via setns(2). A stale
// bind-mount resolves by name but fails EINVAL; `ip -n <netns> link show lo`
// performs the setns and surfaces it. A truly-absent netns also reports false,
// which is the correct signal for both callers (recreate / rebuild).
func netnsEnterable(netns string) bool {
return utils.SudoCommand("ip", "-n", netns, "link", "show", "lo").Run() == nil
}
// ipNetnsTolerate runs `ip -n <netns> <args...>`, treating output containing
// tolerate as success (for idempotent re-runs of addr/route adds).
func ipNetnsTolerate(netns, tolerate string, args ...string) error {
full := append([]string{"-n", netns}, args...)
if out, err := utils.SudoCommand("ip", full...).CombinedOutput(); err != nil {
if strings.Contains(string(out), tolerate) {
return nil
}
return fmt.Errorf("ip %s: %s: %w", strings.Join(full, " "), strings.TrimSpace(string(out)), err)
}
return nil
}
// RemoveIMDSVeth detaches the OVS port, deletes the netns (which destroys the
// host end and thus the veth pair), and clears any root-netns remnant. Idempotent:
// safe to call for a VPC that never had a veth on this chassis.
func RemoveIMDSVeth(ctx context.Context, vpcID string) error {
return removeIMDSPlumbing(IMDSOVSPortName(vpcID), IMDSHostVethName(vpcID), IMDSNetnsName(vpcID))
}
// removeIMDSPlumbing tears down the OVS port, the netns, and any leftover veth.
// Deleting the netns destroys the host end (and its peer); the trailing link del
// covers the partial state where the host end was never moved into the netns.
func removeIMDSPlumbing(ovsEnd, hostEnd, netns string) error {
if out, err := utils.SudoCommand("ovs-vsctl", "--if-exists", "del-port", ovsEnd).CombinedOutput(); err != nil {
slog.Warn("Failed to remove IMDS veth from OVS", "ovs_end", ovsEnd, "err", err, "out", strings.TrimSpace(string(out)))
}
if out, err := utils.SudoCommand("ip", "netns", "del", netns).CombinedOutput(); err != nil {
msg := strings.TrimSpace(string(out))
// "No such file or directory" — already gone. Anything else is logged
// but not fatal: the link del below still runs.
if !strings.Contains(msg, "No such file") {
slog.Warn("Failed to delete IMDS netns", "netns", netns, "err", err, "out", msg)
}
}
if out, err := utils.SudoCommand("ip", "link", "del", ovsEnd).CombinedOutput(); err != nil {
// "Cannot find device" — the pair was already destroyed with the netns.
msg := strings.TrimSpace(string(out))
if strings.Contains(msg, "Cannot find device") {
slog.Debug("IMDS veth already absent", "ovs_end", ovsEnd)
return nil
}
return fmt.Errorf("delete IMDS veth %s: %s: %w", ovsEnd, msg, err)
}
slog.Info("IMDS veth removed", "ovs_end", ovsEnd, "host_end", hostEnd, "netns", netns)
return nil
}
spinifex/network/host/imds_veth_test.go
+351
View File
@@ -0,0 +1,351 @@
package host
import (
"context"
"os/exec"
"slices"
"strings"
"testing"
"github.com/mulgadc/spinifex/spinifex/utils"
)
const testVPCID = "vpc-0123456789abcdef" // short form: "89abcdef"
func recordSudo(t *testing.T, fail func(name string, args []string) *exec.Cmd) *[][]string {
t.Helper()
var calls [][]string
t.Cleanup(utils.SetSudoCommandForTest(func(name string, args ...string) *exec.Cmd {
calls = append(calls, append([]string{name}, args...))
if fail != nil {
if c := fail(name, args); c != nil {
return c
}
}
return exec.Command("/bin/true")
}))
return &calls
}
func TestEnsureIMDSVeth_HappyPath(t *testing.T) {
// Probe must report "not a port" so creation proceeds.
calls := recordSudo(t, func(name string, args []string) *exec.Cmd {
if name == "ovs-vsctl" && len(args) >= 1 && args[0] == "port-to-br" {
return exec.Command("/bin/false")
}
return nil
})
netns, hostEnd, err := EnsureIMDSVeth(context.Background(), testVPCID)
if err != nil {
t.Fatalf("EnsureIMDSVeth: %v", err)
}
if netns != "imds-89abcdef" {
t.Errorf("netns = %q, want imds-89abcdef", netns)
}
if hostEnd != "imds-h-89abcdef" {
t.Errorf("hostEnd = %q, want imds-h-89abcdef", hostEnd)
}
want := [][]string{
{"ovs-vsctl", "port-to-br", "imds-o-89abcdef"},
{"ip", "netns", "add", "imds-89abcdef"},
{"ip", "link", "add", "imds-o-89abcdef", "type", "veth", "peer", "name", "imds-h-89abcdef"},
{"ip", "link", "set", "imds-o-89abcdef", "up"},
{"ip", "link", "set", "imds-h-89abcdef", "netns", "imds-89abcdef"},
{"ip", "-n", "imds-89abcdef", "link", "set", "imds-h-89abcdef", "address", utils.HashMAC("imds-" + testVPCID)},
{"ip", "-n", "imds-89abcdef", "link", "set", "lo", "up"},
{"ip", "-n", "imds-89abcdef", "link", "set", "imds-h-89abcdef", "up"},
{"ip", "-n", "imds-89abcdef", "addr", "add", "169.254.169.254/30", "dev", "imds-h-89abcdef"},
{"ip", "-n", "imds-89abcdef", "route", "add", "default", "via", "169.254.169.253"},
{"ovs-vsctl", "add-port", "br-int", "imds-o-89abcdef",
"--", "set", "Interface", "imds-o-89abcdef",
"external_ids:iface-id=imds-port-" + testVPCID},
}
if len(*calls) != len(want) {
t.Fatalf("got %d calls, want %d: %v", len(*calls), len(want), *calls)
}
for i := range want {
if !slices.Equal((*calls)[i], want[i]) {
t.Errorf("call %d = %v, want %v", i, (*calls)[i], want[i])
}
}
}
func TestEnsureIMDSVeth_Idempotent(t *testing.T) {
// Probe reports the OVS end is already on br-int and the netns is enterable
// (default mock = /bin/true) → early return, no mutation.
calls := recordSudo(t, func(name string, args []string) *exec.Cmd {
if name == "ovs-vsctl" && len(args) >= 1 && args[0] == "port-to-br" {
return exec.Command("echo", "br-int")
}
return nil
})
netns, hostEnd, err := EnsureIMDSVeth(context.Background(), testVPCID)
if err != nil {
t.Fatalf("EnsureIMDSVeth: %v", err)
}
if netns != "imds-89abcdef" {
t.Errorf("netns = %q, want imds-89abcdef", netns)
}
if hostEnd != "imds-h-89abcdef" {
t.Errorf("hostEnd = %q, want imds-h-89abcdef", hostEnd)
}
want := [][]string{
{"ovs-vsctl", "port-to-br", "imds-o-89abcdef"},
{"ip", "-n", "imds-89abcdef", "link", "show", "lo"},
}
if len(*calls) != len(want) {
t.Fatalf("expected only the two probe calls, got %v", *calls)
}
for i := range want {
if !slices.Equal((*calls)[i], want[i]) {
t.Errorf("call %d = %v, want %v", i, (*calls)[i], want[i])
}
}
}
// TestEnsureIMDSVeth_StaleNetnsBehindLivePort_Rebuilds guards Fix #2: a live
// OVS port over an unenterable netns (stale /run/netns handle, setns EINVAL)
// must NOT short-circuit. The inconsistent plumbing is torn down and rebuilt so
// the IMDS listener becomes bindable again, rather than wedging forever.
func TestEnsureIMDSVeth_StaleNetnsBehindLivePort_Rebuilds(t *testing.T) {
calls := recordSudo(t, func(name string, args []string) *exec.Cmd {
if name == "ovs-vsctl" && len(args) >= 1 && args[0] == "port-to-br" {
return exec.Command("echo", "br-int")
}
// Netns enterability probe fails with the kernel's EINVAL message.
if name == "ip" && len(args) >= 5 && args[0] == "-n" && args[2] == "link" && args[3] == "show" {
return exec.Command("sh", "-c", `echo 'setting the network namespace "imds-89abcdef" failed: Invalid argument' >&2; exit 1`)
}
return nil
})
if _, _, err := EnsureIMDSVeth(context.Background(), testVPCID); err != nil {
t.Fatalf("EnsureIMDSVeth: %v", err)
}
// Teardown (removeIMDSPlumbing) must run before the rebuild, and the rebuild
// must re-add the OVS port — proving we did not short-circuit.
var sawDelPort, sawNetnsDel, sawAddPort bool
for _, c := range *calls {
if c[0] == "ovs-vsctl" && slices.Contains(c, "del-port") {
sawDelPort = true
}
if c[0] == "ip" && len(c) >= 3 && c[1] == "netns" && c[2] == "del" {
sawNetnsDel = true
}
if c[0] == "ovs-vsctl" && slices.Contains(c, "add-port") {
sawAddPort = true
}
}
if !sawDelPort || !sawNetnsDel || !sawAddPort {
t.Errorf("expected stale plumbing torn down and rebuilt (del-port=%v, netns del=%v, add-port=%v); calls=%v",
sawDelPort, sawNetnsDel, sawAddPort, *calls)
}
}
// TestEnsureIMDSVeth_StaleNetnsRecreated guards Fix #1: when `ip netns add`
// reports "File exists" but the handle is unenterable, ensureNetns deletes the
// stale handle and recreates it, then plumbing proceeds to completion.
func TestEnsureIMDSVeth_StaleNetnsRecreated(t *testing.T) {
netnsAdds := 0
calls := recordSudo(t, func(name string, args []string) *exec.Cmd {
if name == "ovs-vsctl" && len(args) >= 1 && args[0] == "port-to-br" {
return exec.Command("/bin/false")
}
if name == "ip" && len(args) >= 2 && args[0] == "netns" && args[1] == "add" {
netnsAdds++
if netnsAdds == 1 {
// First add: name already present as a stale handle.
return exec.Command("sh", "-c", "echo 'Cannot create namespace file: File exists' >&2; exit 1")
}
return nil // recreate succeeds
}
// Enterability probe fails → stale handle detected.
if name == "ip" && len(args) >= 5 && args[0] == "-n" && args[2] == "link" && args[3] == "show" {
return exec.Command("sh", "-c", `echo 'setting the network namespace "imds-89abcdef" failed: Invalid argument' >&2; exit 1`)
}
return nil
})
if _, _, err := EnsureIMDSVeth(context.Background(), testVPCID); err != nil {
t.Fatalf("EnsureIMDSVeth: %v", err)
}
// Stale handle deleted then re-added, and the rebuild ran to the add-port.
var sawNetnsDel, sawAddPort bool
for _, c := range *calls {
if c[0] == "ip" && len(c) >= 3 && c[1] == "netns" && c[2] == "del" {
sawNetnsDel = true
}
if c[0] == "ovs-vsctl" && slices.Contains(c, "add-port") {
sawAddPort = true
}
}
if !sawNetnsDel {
t.Errorf("expected stale netns to be deleted before recreate; calls=%v", *calls)
}
if !sawAddPort {
t.Errorf("expected plumbing to complete after recreate; calls=%v", *calls)
}
if netnsAdds != 2 {
t.Errorf("expected two netns add attempts (stale + recreate), got %d", netnsAdds)
}
}
// TestEnsureIMDSVeth_NetnsExistsTolerated guards idempotent re-runs after a
// partial prior attempt: `ip netns add` reporting "File exists" must not abort.
func TestEnsureIMDSVeth_NetnsExistsTolerated(t *testing.T) {
recordSudo(t, func(name string, args []string) *exec.Cmd {
if name == "ovs-vsctl" && len(args) >= 1 && args[0] == "port-to-br" {
return exec.Command("/bin/false")
}
if name == "ip" && len(args) >= 2 && args[0] == "netns" && args[1] == "add" {
return exec.Command("sh", "-c", "echo 'Cannot create namespace file: File exists' >&2; exit 1")
}
return nil
})
if _, _, err := EnsureIMDSVeth(context.Background(), testVPCID); err != nil {
t.Fatalf("expected nil for pre-existing netns, got %v", err)
}
}
func TestEnsureIMDSVeth_AddPortFailureCleansUp(t *testing.T) {
calls := recordSudo(t, func(name string, args []string) *exec.Cmd {
if name == "ovs-vsctl" && len(args) >= 1 && args[0] == "port-to-br" {
return exec.Command("/bin/false")
}
if name == "ovs-vsctl" && len(args) >= 1 && args[0] == "add-port" {
return exec.Command("/bin/false")
}
return nil
})
_, _, err := EnsureIMDSVeth(context.Background(), testVPCID)
if err == nil || !strings.Contains(err.Error(), "add IMDS veth") {
t.Fatalf("expected add-port error, got %v", err)
}
// Cleanup must have run: del-port, netns del, then link del.
var sawDelPort, sawNetnsDel, sawLinkDel bool
for _, c := range *calls {
if c[0] == "ovs-vsctl" && slices.Contains(c, "del-port") {
sawDelPort = true
}
if c[0] == "ip" && len(c) >= 3 && c[1] == "netns" && c[2] == "del" {
sawNetnsDel = true
}
if c[0] == "ip" && len(c) >= 3 && c[1] == "link" && c[2] == "del" {
sawLinkDel = true
}
}
if !sawDelPort || !sawNetnsDel || !sawLinkDel {
t.Errorf("expected cleanup (del-port=%v, netns del=%v, link del=%v); calls=%v", sawDelPort, sawNetnsDel, sawLinkDel, *calls)
}
}
func TestEnsureIMDSVeth_LinkAddFailureSurfaces(t *testing.T) {
recordSudo(t, func(name string, args []string) *exec.Cmd {
if name == "ovs-vsctl" && len(args) >= 1 && args[0] == "port-to-br" {
return exec.Command("/bin/false")
}
if name == "ip" && len(args) >= 3 && args[0] == "link" && args[1] == "add" {
return exec.Command("/bin/false")
}
return nil
})
_, _, err := EnsureIMDSVeth(context.Background(), testVPCID)
if err == nil || !strings.Contains(err.Error(), "create IMDS veth pair") {
t.Fatalf("expected create error, got %v", err)
}
}
// TestEnsureIMDSVeth_AddrFailureCleansUp asserts a failure assigning the IMDS
// address inside the netns surfaces and tears the half-built plumbing down.
func TestEnsureIMDSVeth_AddrFailureCleansUp(t *testing.T) {
calls := recordSudo(t, func(name string, args []string) *exec.Cmd {
if name == "ovs-vsctl" && len(args) >= 1 && args[0] == "port-to-br" {
return exec.Command("/bin/false")
}
if name == "ip" && len(args) >= 4 && args[0] == "-n" && args[2] == "addr" && args[3] == "add" {
return exec.Command("/bin/false")
}
return nil
})
_, _, err := EnsureIMDSVeth(context.Background(), testVPCID)
if err == nil || !strings.Contains(err.Error(), "addr add") {
t.Fatalf("expected addr-add error, got %v", err)
}
var sawNetnsDel bool
for _, c := range *calls {
if c[0] == "ip" && len(c) >= 3 && c[1] == "netns" && c[2] == "del" {
sawNetnsDel = true
}
}
if !sawNetnsDel {
t.Errorf("expected netns cleanup after addr failure; calls=%v", *calls)
}
}
func TestRemoveIMDSVeth(t *testing.T) {
calls := recordSudo(t, nil)
if err := RemoveIMDSVeth(context.Background(), testVPCID); err != nil {
t.Fatalf("RemoveIMDSVeth: %v", err)
}
want := [][]string{
{"ovs-vsctl", "--if-exists", "del-port", "imds-o-89abcdef"},
{"ip", "netns", "del", "imds-89abcdef"},
{"ip", "link", "del", "imds-o-89abcdef"},
}
if len(*calls) != len(want) {
t.Fatalf("got %d calls, want %d: %v", len(*calls), len(want), *calls)
}
for i := range want {
if !slices.Equal((*calls)[i], want[i]) {
t.Errorf("call %d = %v, want %v", i, (*calls)[i], want[i])
}
}
}
func TestRemoveIMDSVeth_MissingDeviceIsNotError(t *testing.T) {
recordSudo(t, func(name string, args []string) *exec.Cmd {
if name == "ip" && len(args) >= 2 && args[0] == "netns" && args[1] == "del" {
return exec.Command("sh", "-c", "echo 'Cannot remove namespace file \"/var/run/netns/imds-89abcdef\": No such file or directory' >&2; exit 1")
}
if name == "ip" && len(args) >= 2 && args[0] == "link" && args[1] == "del" {
// Emit the kernel's "absent device" message on stderr and fail.
return exec.Command("sh", "-c", "echo 'Cannot find device \"imds-o-89abcdef\"' >&2; exit 1")
}
return nil
})
if err := RemoveIMDSVeth(context.Background(), testVPCID); err != nil {
t.Fatalf("expected nil for absent device, got %v", err)
}
}
// TestIMDSVethNamesWithinIFNAMSIZ guards the load-bearing constraint that both
// veth-end names fit Linux's IFNAMSIZ-1 (15 chars). A longer name makes
// `ip link add` fail with "name too long" for every VPC, silently taking IMDS
// offline cluster-wide — and the SudoCommand mock would never catch it.
func TestIMDSVethNamesWithinIFNAMSIZ(t *testing.T) {
const ifnamsizMax = 15
for _, vpcID := range []string{
"vpc-0123456789abcdef",
"vpc-0a1b2c3d4e5f6a7b8",
"vpc-deadbeef",
"short",
} {
for _, name := range []string{IMDSOVSPortName(vpcID), IMDSHostVethName(vpcID)} {
if len(name) > ifnamsizMax {
t.Errorf("veth name %q (%d chars) exceeds IFNAMSIZ-1 (%d) for vpc %q", name, len(name), ifnamsizMax, vpcID)
}
}
}
}
spinifex/network/reconcile/apply.go
+39 -16
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"log/slog"
handlers_imds "github.com/mulgadc/spinifex/spinifex/handlers/imds"
"github.com/mulgadc/spinifex/spinifex/network/ovn/nbdb"
"github.com/mulgadc/spinifex/spinifex/network/topology"
"github.com/mulgadc/spinifex/spinifex/utils"
@@ -15,22 +16,28 @@ import (
func (r *reconciler) applyVPCs(ctx context.Context, intent IntentState, actual ActualState) {
for vpcID, spec := range intent.VPCs {
routerName := topology.VPCRouter(vpcID)
if _, ok := actual.Routers[routerName]; ok {
continue
if _, ok := actual.Routers[routerName]; !ok {
lr := &nbdb.LogicalRouter{
Name: routerName,
ExternalIDs: map[string]string{
"spinifex:vpc_id": vpcID,
"spinifex:cidr": spec.CIDR.String(),
},
}
if _, err := r.ovn.EnsureLogicalRouter(ctx, lr); err != nil {
slog.Error("reconcile/apply: ensure VPC router failed", "vpc_id", vpcID, "err", err)
continue
}
actual.Routers[routerName] = struct{}{}
slog.Info("reconcile/apply: ensured VPC router", "vpc_id", vpcID, "router", routerName)
}
lr := &nbdb.LogicalRouter{
Name: routerName,
ExternalIDs: map[string]string{
"spinifex:vpc_id": vpcID,
"spinifex:cidr": spec.CIDR.String(),
},
// Install IMDS topology for every intent VPC (idempotent via the
// vpc-veth bucket gate); the router must exist for the IMDS LRP.
if r.imds != nil {
if _, err := r.imds.EnsureForVPC(ctx, vpcID); err != nil {
slog.Error("reconcile/apply: IMDS EnsureForVPC failed", "vpc_id", vpcID, "err", err)
}
}
if _, err := r.ovn.EnsureLogicalRouter(ctx, lr); err != nil {
slog.Error("reconcile/apply: ensure VPC router failed", "vpc_id", vpcID, "err", err)
continue
}
actual.Routers[routerName] = struct{}{}
slog.Info("reconcile/apply: ensured VPC router", "vpc_id", vpcID, "router", routerName)
}
}
@@ -83,12 +90,15 @@ func (r *reconciler) applySubnets(ctx context.Context, intent IntentState, actua
actual.RouterPorts[routerPortName] = struct{}{}
}
if _, err := r.ovn.GetLogicalSwitchPort(ctx, switchRouterPortName); err != nil {
if existing, err := r.ovn.GetLogicalSwitchPort(ctx, switchRouterPortName); err != nil {
lsp := &nbdb.LogicalSwitchPort{
Name: switchRouterPortName,
Type: "router",
Addresses: []string{"router"},
Options: map[string]string{"router-port": routerPortName},
Options: map[string]string{
"router-port": routerPortName,
"arp_proxy": handlers_imds.MetaDataServerIP,
},
ExternalIDs: map[string]string{
"spinifex:subnet_id": subnetID,
"spinifex:vpc_id": spec.VPCID,
@@ -98,6 +108,19 @@ func (r *reconciler) applySubnets(ctx context.Context, intent IntentState, actua
slog.Error("reconcile/apply: create subnet router-LSP failed", "subnet_id", subnetID, "err", err)
continue
}
} else if existing.Options["arp_proxy"] != handlers_imds.MetaDataServerIP {
// Drift convergence: a subnet router LSP created before IMDS proxy-ARP
// landed won't gain arp_proxy from a redeploy alone (the create branch
// only fires when absent), so patch it in place to make it IMDS-reachable.
if existing.Options == nil {
existing.Options = map[string]string{}
}
existing.Options["arp_proxy"] = handlers_imds.MetaDataServerIP
if err := r.ovn.UpdateLogicalSwitchPort(ctx, existing); err != nil {
slog.Error("reconcile/apply: patch subnet router-LSP arp_proxy failed", "subnet_id", subnetID, "err", err)
} else {
slog.Info("reconcile/apply: patched subnet router-LSP with IMDS arp_proxy", "subnet_id", subnetID, "port", switchRouterPortName)
}
}
if existing, err := r.ovn.FindDHCPOptionsByExternalID(ctx, "spinifex:subnet_id", subnetID); err != nil || existing == nil {
spinifex/network/reconcile/invariants_test.go
+135
View File
@@ -0,0 +1,135 @@
package reconcile
import (
"context"
"net"
"net/netip"
"testing"
imds "github.com/mulgadc/spinifex/spinifex/handlers/imds"
"github.com/mulgadc/spinifex/spinifex/network/ovn/mock"
"github.com/mulgadc/spinifex/spinifex/network/topology"
)
// IMDS-datapath invariants. A guest LSP is created by two
// independent paths — the live topology manager (EnsurePort) and the reconciler
// (applyPorts) — and the per-subnet DHCPOptions row likewise. Both paths are
// exercised here so neither can drift away from the contract the IMDS handler
// trusts. These live in the reconcile package because it is the only network
// package that can import both topology (the live path) and itself (the
// reconciler path) without a cycle.
// TestI1_GuestLSPMustHavePortSecurity asserts that every guest-attached LSP,
// created by either path, carries port_security equal to its addresses
// ("<MAC> <IP>"). This is the load-bearing security boundary: ovn-controller
// drops any frame whose eth.src/ip4.src doesn't match port_security at the
// ingress LSP, so a compromised guest cannot forge a peer's source IP. Without
// it the IMDS (VPC-ID, source-IP) → ENI mapping is forgeable.
func TestI1_GuestLSPMustHavePortSecurity(t *testing.T) {
ctx := context.Background()
// Live path: topology.EnsurePort.
t.Run("live", func(t *testing.T) {
m := mock.New()
mgr := topology.NewLiveManager(m)
vpc := topology.VPCSpec{VPCID: "vpc-a", CIDR: netip.MustParsePrefix("10.0.0.0/16")}
sub := topology.SubnetSpec{SubnetID: "subnet-a", VPCID: "vpc-a", CIDR: netip.MustParsePrefix("10.0.1.0/24")}
mac, _ := net.ParseMAC("02:00:00:00:00:01")
port := topology.PortSpec{
PortID: "eni-a", SubnetID: "subnet-a", VPCID: "vpc-a",
PrivateIP: netip.MustParseAddr("10.0.1.5"), MAC: mac,
}
if err := mgr.EnsureVPC(ctx, vpc); err != nil {
t.Fatalf("EnsureVPC: %v", err)
}
if err := mgr.EnsureSubnet(ctx, sub); err != nil {
t.Fatalf("EnsureSubnet: %v", err)
}
if err := mgr.EnsurePort(ctx, port); err != nil {
t.Fatalf("EnsurePort: %v", err)
}
assertPortSecurity(t, m, topology.Port(port.PortID), "02:00:00:00:00:01 10.0.1.5")
})
// Reconciler path: applyPorts.
t.Run("reconciler", func(t *testing.T) {
rec, m := newTestReconciler(t)
intent := freshIntent(t)
if err := rec.Reconcile(ctx, intent); err != nil {
t.Fatalf("Reconcile: %v", err)
}
port := intent.Ports["eni-a"]
want := port.MAC.String() + " " + port.PrivateIP.String()
assertPortSecurity(t, m, topology.Port(port.PortID), want)
})
}
// assertPortSecurity fails with the ADR clause when the LSP lacks port_security
// equal to its addresses.
func assertPortSecurity(t *testing.T, m *mock.Client, portName, want string) {
t.Helper()
lsp, err := m.GetLogicalSwitchPort(context.Background(), portName)
if err != nil {
t.Fatalf("guest LSP %s missing: %v", portName, err)
}
if len(lsp.PortSecurity) != 1 || lsp.PortSecurity[0] != want {
t.Errorf("guest LSP %s PortSecurity = %v, want [%q]: guest LSPs without "+
"port_security allow source-IP spoofing, breaking the IMDS "+
"(VPC-ID, source-IP) → ENI mapping", portName, lsp.PortSecurity, want)
}
// port_security must mirror addresses exactly, or the enforced identity and
// the advertised identity diverge.
if len(lsp.Addresses) != 1 || lsp.Addresses[0] != lsp.PortSecurity[0] {
t.Errorf("guest LSP %s Addresses %v != PortSecurity %v: enforced and advertised "+
"identity must match", portName, lsp.Addresses, lsp.PortSecurity)
}
}
// TestI5_SubnetRouterPortProxiesIMDSARP asserts the subnet router LSP carries
// options:arp_proxy = 169.254.169.254, so the subnet LRP answers ARP/ND for the
// IMDS address link-local. Both emission paths (live EnsureSubnet, reconciler
// applySubnets) are checked so they cannot diverge. This replaces the former
// DHCP option-121 mechanism: proxy-ARP reaches DHCP and fully static guests
// identically (AWS Nitro parity), whereas option 121 never reached a guest with
// no DHCP client.
func TestI5_SubnetRouterPortProxiesIMDSARP(t *testing.T) {
ctx := context.Background()
const cidr = "10.0.1.0/24"
t.Run("live", func(t *testing.T) {
m := mock.New()
mgr := topology.NewLiveManager(m)
if err := mgr.EnsureVPC(ctx, topology.VPCSpec{VPCID: "vpc-a", CIDR: netip.MustParsePrefix("10.0.0.0/16")}); err != nil {
t.Fatalf("EnsureVPC: %v", err)
}
if err := mgr.EnsureSubnet(ctx, topology.SubnetSpec{SubnetID: "subnet-a", VPCID: "vpc-a", CIDR: netip.MustParsePrefix(cidr)}); err != nil {
t.Fatalf("EnsureSubnet: %v", err)
}
assertARPProxy(t, m, topology.SubnetSwitchRouterPort("subnet-a"))
})
t.Run("reconciler", func(t *testing.T) {
rec, m := newTestReconciler(t)
if err := rec.Reconcile(ctx, freshIntent(t)); err != nil {
t.Fatalf("Reconcile: %v", err)
}
assertARPProxy(t, m, topology.SubnetSwitchRouterPort("subnet-a"))
})
}
// assertARPProxy fails with the ADR clause unless the subnet router LSP carries
// options:arp_proxy for the IMDS address.
func assertARPProxy(t *testing.T, m *mock.Client, portName string) {
t.Helper()
lsp, err := m.GetLogicalSwitchPort(context.Background(), portName)
if err != nil {
t.Fatalf("subnet router LSP %s missing: %v", portName, err)
}
if lsp.Options["arp_proxy"] != imds.MetaDataServerIP {
t.Errorf("subnet router LSP %s options:arp_proxy = %q, want %q: subnet router "+
"ports without options:arp_proxy for %s leave IMDS unreachable for any guest "+
"that treats the address as link-local (the common static-IP and "+
"NetworkManager/RHEL/Ubuntu case)",
portName, lsp.Options["arp_proxy"], imds.MetaDataServerIP, imds.MetaDataServerIP)
}
}
spinifex/network/reconcile/reconcile.go
+7 -1
View File
@@ -36,7 +36,11 @@ type Config struct {
Routes policy.RouteManager
IGW external.IGWManager
Topology topology.Manager
LocalAZ string
// IMDS installs per-VPC IMDS OVN topology during the VPC apply pass.
// Optional: nil skips IMDS plumbing (focused reconcile tests leave it
// unset; production always wires it).
IMDS external.IMDSTopologyManager
LocalAZ string
// NodeHostname is the holder identity for leader-election CAS.
NodeHostname string
// Chassis is the SBDB-discovered chassis list for gateway LRP rebinding.
@@ -54,6 +58,7 @@ type reconciler struct {
routes policy.RouteManager
igw external.IGWManager
topology topology.Manager
imds external.IMDSTopologyManager
localAZ string
host string
chassis []string
@@ -92,6 +97,7 @@ func New(cfg Config) (Reconciler, error) {
routes: cfg.Routes,
igw: cfg.IGW,
topology: cfg.Topology,
imds: cfg.IMDS,
localAZ: cfg.LocalAZ,
host: cfg.NodeHostname,
chassis: cfg.Chassis,
spinifex/network/subscribers/subscribers.go
+7 -1
View File
@@ -23,15 +23,20 @@ type Subscriber struct {
eip external.EIPManager
natgw external.NATGWManager
igw external.IGWManager
imds external.IMDSTopologyManager
}
// Config: all fields required.
// Config: all fields required except IMDS.
type Config struct {
Topology topology.Manager
SG policy.SecurityGroupManager
EIP external.EIPManager
NATGW external.NATGWManager
IGW external.IGWManager
// IMDS installs per-VPC IMDS OVN topology on vpc.create / removes it on
// vpc.delete. Optional: nil disables IMDS plumbing (focused topology tests
// leave it unset; production always wires it).
IMDS external.IMDSTopologyManager
}
// New constructs a Subscriber, returning an error when any manager is nil.
@@ -54,6 +59,7 @@ func New(cfg Config) (*Subscriber, error) {
eip: cfg.EIP,
natgw: cfg.NATGW,
igw: cfg.IGW,
imds: cfg.IMDS,
}, nil
}
spinifex/network/subscribers/topology.go
+21 -2
View File
@@ -31,11 +31,20 @@ func (s *Subscriber) handleVPCCreate(msg *nats.Msg) {
}
spec.CIDR = cidr
}
if err := s.topology.EnsureVPC(context.Background(), spec); err != nil {
ctx := context.Background()
if err := s.topology.EnsureVPC(ctx, spec); err != nil {
slog.Error("subscribers: EnsureVPC failed", "vpc_id", evt.VpcId, "err", err)
respond(msg, err)
return
}
// IMDS topology rides on every VPC (private instances still need metadata +
// role creds). Install is best-effort — the reconciler re-ensures it idempotently,
// so a transient OVN failure here must not fail CreateVpc.
if s.imds != nil {
if _, err := s.imds.EnsureForVPC(ctx, evt.VpcId); err != nil {
slog.Warn("subscribers: IMDS EnsureForVPC failed; reconciler will converge", "vpc_id", evt.VpcId, "err", err)
}
}
respond(msg, nil)
}
@@ -46,7 +55,17 @@ func (s *Subscriber) handleVPCDelete(msg *nats.Msg) {
respond(msg, err)
return
}
if err := s.topology.DeleteVPC(context.Background(), evt.VpcId); err != nil {
ctx := context.Background()
// Remove IMDS topology before the router goes away — the IMDS LRP and
// static route live on vpc-{vpcID}.
if s.imds != nil {
if err := s.imds.RemoveForVPC(ctx, evt.VpcId); err != nil {
slog.Error("subscribers: IMDS RemoveForVPC failed", "vpc_id", evt.VpcId, "err", err)
respond(msg, err)
return
}
}
if err := s.topology.DeleteVPC(ctx, evt.VpcId); err != nil {
slog.Error("subscribers: DeleteVPC failed", "vpc_id", evt.VpcId, "err", err)
respond(msg, err)
return
spinifex/network/topology/dhcp_options.go
+4
View File
@@ -4,6 +4,10 @@ package topology
// subnet's DHCPOptions row. Both the live topology manager and the reconciler
// call it so the two paths cannot drift — they previously hard-coded divergent
// dns_server values (live used the configured server, the reconciler "8.8.8.8").
//
// IMDS reachability (169.254.169.254) is no longer steered via DHCP option 121; it
// is answered link-local by the subnet router LSP's options:arp_proxy, so DHCP and
// static guests reach IMDS identically. The default gateway stays option 3 (router).
func BuildSubnetDHCPOptions(gwIP, routerMAC, dnsServer string) map[string]string {
return map[string]string{
"server_id": gwIP,
spinifex/network/topology/dhcp_options_test.go
+24
View File
@@ -0,0 +1,24 @@
package topology
import "testing"
// IMDS reachability moved from DHCP option 121 to the subnet router LSP's
// options:arp_proxy (Step 14), so BuildSubnetDHCPOptions must no longer emit a
// classless_static_route key. The default gateway is still carried by option 3
// (router), and dns_server/server_id remain present.
func TestBuildSubnetDHCPOptions_NoClasslessStaticRoute(t *testing.T) {
opts := BuildSubnetDHCPOptions("10.0.1.1", "02:00:00:00:00:01", "{8.8.8.8, 1.1.1.1}")
if _, ok := opts["classless_static_route"]; ok {
t.Errorf("classless_static_route must be absent (IMDS now uses subnet-router proxy-ARP, not DHCP option 121); got %q", opts["classless_static_route"])
}
if got := opts["router"]; got != "10.0.1.1" {
t.Errorf("router = %q, want %q", got, "10.0.1.1")
}
if got := opts["dns_server"]; got != "{8.8.8.8, 1.1.1.1}" {
t.Errorf("dns_server = %q, want %q", got, "{8.8.8.8, 1.1.1.1}")
}
if got := opts["server_id"]; got != "10.0.1.1" {
t.Errorf("server_id = %q, want %q", got, "10.0.1.1")
}
}
spinifex/network/topology/manager_live.go
+2
View File
@@ -9,6 +9,7 @@ import (
"strconv"
"strings"
handlers_imds "github.com/mulgadc/spinifex/spinifex/handlers/imds"
"github.com/mulgadc/spinifex/spinifex/network/ovn"
"github.com/mulgadc/spinifex/spinifex/network/ovn/nbdb"
"github.com/mulgadc/spinifex/spinifex/utils"
@@ -205,6 +206,7 @@ func (m *liveManager) EnsureSubnet(ctx context.Context, spec SubnetSpec) error {
Addresses: []string{"router"},
Options: map[string]string{
"router-port": routerPortName,
"arp_proxy": handlers_imds.MetaDataServerIP,
},
ExternalIDs: map[string]string{
"spinifex:subnet_id": spec.SubnetID,
spinifex/network/topology/names.go
+17
View File
@@ -36,6 +36,23 @@ func ExternalSwitch(vpcID string) string { return "ext-" + vpcID }
// external switch onto the host uplink.
func ExternalLocalnetPort(vpcID string) string { return "ext-port-" + vpcID }
// IMDSSwitch is the OVN logical switch dedicated to IMDS for a VPC. The
// host-owned 169.254.169.254 LSP lives here, off the subnet switches.
func IMDSSwitch(vpcID string) string { return "imds-ls-" + vpcID }
// IMDSRouterPort is the LRP on the VPC router that terminates the IMDS /30 and
// is the output port for the 169.254.169.254/32 static route.
func IMDSRouterPort(vpcID string) string { return "imds-lrp-" + vpcID }
// IMDSSwitchRouterPort is the type=router LSP on the IMDS switch peered with
// IMDSRouterPort.
func IMDSSwitchRouterPort(vpcID string) string { return "imds-rtr-" + vpcID }
// IMDSPort is the host-owned localport LSP that claims 169.254.169.254 on the
// IMDS switch. ovn-controller binds it on every chassis with a matching
// iface-id OVS port, so each chassis self-serves IMDS for its local VMs.
func IMDSPort(vpcID string) string { return "imds-port-" + vpcID }
// SecurityGroupPortGroup is the OVN port group name for an SG.
// OVN names match [a-zA-Z_][a-zA-Z0-9_]*, so hyphens become underscores.
func SecurityGroupPortGroup(sgID string) string {
spinifex/services/awsgw/awsgw.go
+10
View File
@@ -146,6 +146,16 @@ func launchService(config *config.ClusterConfig) error {
defer cancelJanitor()
go stsService.RunJanitor(janitorCtx)
// IMDS serves 169.254.169.254 to guest VMs from vpcd, which holds the network
// capabilities the hardened awsgw sandbox can't grant. awsgw stays the home of
// STS + IAM, answering the IMDS handler's control-plane RPCs over NATS.
if _, err := stsService.SubscribeIMDSResponder(natsConn); err != nil {
return fmt.Errorf("subscribe IMDS STS responder: %w", err)
}
if _, err := iamService.SubscribeIMDSResponders(natsConn); err != nil {
return fmt.Errorf("subscribe IMDS IAM responders: %w", err)
}
// Expose the in-process STS presigned-URL verify over NATS for the
// in-cluster eks-token-webhook (STS is gateway-local, not otherwise on the
// bus). Bound to the process lifetime via natsConn.Close on return.
spinifex/services/awsgw/imds_absent_test.go
+23
View File
@@ -0,0 +1,23 @@
package awsgw
import (
"os"
"strings"
"testing"
)
// TestIMDSRuntimeAbsentFromAWSGW guards against re-introducing the privileged
// IMDS listener stack into awsgw, whose systemd unit is the most hardened in
// the fleet (NoNewPrivileges=yes, empty CapabilityBoundingSet). The veth +
// privileged-socket bind cannot succeed there; the runtime lives in vpcd, and
// awsgw only answers the IMDS control-plane RPCs (STS + IAM).
func TestIMDSRuntimeAbsentFromAWSGW(t *testing.T) {
src, err := os.ReadFile("awsgw.go")
if err != nil {
t.Fatalf("read awsgw.go: %v", err)
}
if strings.Contains(string(src), "NewIMDSServiceImpl") {
t.Fatal("awsgw must not construct the IMDS runtime (NewIMDSServiceImpl) — " +
"the privileged listener stack belongs in vpcd.")
}
}
spinifex/utils/nats.go
+29
View File
@@ -12,6 +12,7 @@ import (
"time"
"github.com/mulgadc/spinifex/internal/tlsconfig"
"github.com/mulgadc/spinifex/spinifex/awserrors"
"github.com/nats-io/nats.go"
)
@@ -293,6 +294,34 @@ func NATSRequest[Out any](conn *nats.Conn, subject string, input any, timeout ti
return &output, nil
}
// ServeNATSRequest is the responder-side counterpart to NATSRequest: it unmarshals
// the request payload into *I, invokes fn, and replies with the JSON result or an
// awserrors envelope. The payload carries its own context; no X-Account-ID header.
func ServeNATSRequest[I any, O any](msg *nats.Msg, fn func(*I) (*O, error)) {
input := new(I)
if errResp := UnmarshalJsonPayload(input, msg.Data); errResp != nil {
respondNATS(msg, errResp)
return
}
out, err := fn(input)
if err != nil {
respondNATS(msg, GenerateErrorPayload(awserrors.ValidErrorCode(err.Error())))
return
}
data, err := json.Marshal(out)
if err != nil {
respondNATS(msg, GenerateErrorPayload(awserrors.ErrorServerInternal))
return
}
respondNATS(msg, data)
}
func respondNATS(msg *nats.Msg, data []byte) {
if err := msg.Respond(data); err != nil {
slog.Error("failed to respond to NATS request", "subject", msg.Subject, "err", err)
}
}
const (
// maxScatterGatherResponseSize is the maximum allowed size for a single
// scatter-gather response payload (10 MB). Responses exceeding this are
spinifex/utils/utils.go
+10
View File
@@ -326,6 +326,16 @@ func HashMAC(id string) string {
return net.HardwareAddr(b).String()
}
// ClientIP returns the IP from a RemoteAddr, stripping the port. Handles both
// IPv4 and IPv6, and tolerates a RemoteAddr that is already a bare IP (no port).
func ClientIP(remoteAddr string) string {
host, _, err := net.SplitHostPort(remoteAddr)
if err != nil {
return remoteAddr
}
return host
}
func dirExists(path string) bool {
info, err := os.Stat(path)
if os.IsNotExist(err) {
spinifex/utils/utils_test.go
+19
View File
@@ -1686,3 +1686,22 @@ func TestDistroFamily(t *testing.T) {
})
}
}
func TestClientIP(t *testing.T) {
tests := []struct {
name string
remoteAddr string
expected string
}{
{"IPv4 with port", "192.168.1.1:12345", "192.168.1.1"},
{"IPv6 with port", "[::1]:12345", "::1"},
{"IPv4 bare", "192.168.1.1", "192.168.1.1"},
{"IPv6 full with port", "[2001:db8::1]:443", "2001:db8::1"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.expected, ClientIP(tt.remoteAddr))
})
}
}
spinifex/vm/vm.go
-3
View File
@@ -61,9 +61,6 @@ type VM struct {
// Set when ownership is released on stop for shared KV storage.
LastNode string `json:"last_node,omitempty"`
// User data for cloud-init (decoded from base64)
UserData string `json:"user_data,omitempty"`
// Metadata server address (e.g., "127.0.0.1:12345") for EC2 metadata service
MetadataServerAddress string `json:"metadata_server_address,omitempty"`
spinifex/vpcd/vpcd.go
+37
View File
@@ -2,6 +2,7 @@ package vpcd
import (
"context"
"errors"
"fmt"
"log/slog"
"net"
@@ -15,6 +16,7 @@ import (
"time"
"github.com/mulgadc/spinifex/spinifex/admin"
handlers_imds "github.com/mulgadc/spinifex/spinifex/handlers/imds"
"github.com/mulgadc/spinifex/spinifex/network/external"
"github.com/mulgadc/spinifex/spinifex/network/external/dhcp"
"github.com/mulgadc/spinifex/spinifex/network/host"
@@ -523,6 +525,39 @@ func launchService(cfg *Config) error {
return fmt.Errorf("get JetStream context: %w", err)
}
// IMDS per-VPC OVN topology installer. Replicas track the chassis count so
// the vpc-veth bucket survives a node loss (create-or-open: awsgw inits the
// same bucket; first writer wins).
imdsVethKV, _, err := handlers_imds.InitBuckets(js, max(len(chassisNames), 1))
if err != nil {
return fmt.Errorf("init imds buckets: %w", err)
}
imdsTopoMgr, err := external.NewIMDSTopologyManager(liveClient, routeMgr, handlers_imds.NewVethStore(imdsVethKV))
if err != nil {
return fmt.Errorf("construct IMDS topology manager: %w", err)
}
// IMDS host-served datapath. vpcd holds the network capabilities the listener
// stack needs (the awsgw sandbox can't grant them); STS/IAM stay in awsgw over
// NATS RPCs. Run blocks, so it gets its own goroutine.
imdsCtx, cancelIMDS := context.WithCancel(ctx)
defer cancelIMDS()
imdsSvc, err := handlers_imds.NewIMDSServiceImpl(
nc,
handlers_imds.NewNATSSTSAssumer(nc),
handlers_imds.NewNATSProfileLookup(nc),
max(len(chassisNames), 1),
host.EnsureIMDSVeth, host.RemoveIMDSVeth,
)
if err != nil {
return fmt.Errorf("construct IMDS service: %w", err)
}
go func() {
if err := imdsSvc.Run(imdsCtx); err != nil && !errors.Is(err, context.Canceled) {
slog.Error("vpcd: IMDS service exited", "err", err)
}
}()
dhcpMgr, dhcpSubs, err := startDHCPManagerIfNeeded(ctx, nc, js, cfg)
if err != nil {
return fmt.Errorf("start dhcp manager: %w", err)
@@ -574,6 +609,7 @@ func launchService(cfg *Config) error {
EIP: eipMgr,
NATGW: natgwMgr,
IGW: igwMgr,
IMDS: imdsTopoMgr,
})
if err != nil {
return fmt.Errorf("construct subscriber: %w", err)
@@ -596,6 +632,7 @@ func launchService(cfg *Config) error {
Routes: routeMgr,
IGW: igwMgr,
Topology: topoMgr,
IMDS: imdsTopoMgr,
LocalAZ: cfg.AZ,
NodeHostname: holder,
Chassis: chassisNames,
tests/e2e/harness/diag.go
+21 -13
View File
@@ -156,16 +156,7 @@ func dumpDatapathState(t *testing.T, opts VPCDiagnosticsOpts) {
return
}
type capture struct {
filename string
label string
argv []string
// grepFor, when non-empty, post-filters stdout to lines
// containing this substring. Lets us keep ovs-ofctl / conntrack
// output tight without an external pipe.
grepFor []string
}
caps := []capture{
runHostCaptures(t, opts.ArtifactDir, []hostCapture{
{
filename: "ovs-flows-extip.txt",
label: "ovs-ofctl dump-flows br-int (filtered)",
@@ -190,8 +181,25 @@ func dumpDatapathState(t *testing.T, opts VPCDiagnosticsOpts) {
argv: []string{"ovn-nbctl", "--bare", "--columns=_uuid,type,external_ip,logical_ip,external_mac,logical_port",
"find", "NAT", "external_ip=" + opts.ExternalIP},
},
}
})
}
// hostCapture is one `sudo`-run diagnostic command. grepFor, when non-empty,
// post-filters stdout to lines containing any of the substrings (keeps
// ovs-ofctl / conntrack output tight without an external pipe).
type hostCapture struct {
filename string
label string
argv []string
grepFor []string
}
// runHostCaptures runs each capture under `sudo -n`, echoes it to stdout for the
// CI log, and persists it as a separate file under artifactDir (when set) so the
// Stage 2 analyzer bundler picks it up. Assumes the caller already verified
// passwordless sudo is available.
func runHostCaptures(t *testing.T, artifactDir string, caps []hostCapture) {
t.Helper()
for _, cap := range caps {
fmt.Printf(" --- %s ---\n", cap.label)
var stdout, stderr bytes.Buffer
@@ -212,8 +220,8 @@ func dumpDatapathState(t *testing.T, opts VPCDiagnosticsOpts) {
for _, line := range strings.Split(strings.TrimRight(body, "\n"), "\n") {
fmt.Printf(" %s\n", line)
}
if opts.ArtifactDir != "" {
path := filepath.Join(opts.ArtifactDir, cap.filename)
if artifactDir != "" {
path := filepath.Join(artifactDir, cap.filename)
header := fmt.Sprintf("$ sudo %s\n", strings.Join(cap.argv, " "))
if len(cap.grepFor) > 0 {
header += "# filtered to lines matching: " + strings.Join(cap.grepFor, ", ") + "\n"
tests/e2e/harness/imds_diag.go
+129
View File
@@ -0,0 +1,129 @@
//go:build e2e
package harness
import (
"fmt"
"os/exec"
"testing"
"github.com/mulgadc/spinifex/spinifex/network/host"
)
// DumpIMDSDatapathDiagnostics emits a triage bundle for an IMDS reachability
// failure (guest curl to 169.254.169.254 times out). It captures both halves of
// the per-VPC datapath so a triager can tell request-path from reply-path:
//
// - OVN realisation: is imds-port-<vpc> bound/up on this chassis, is the
// 169.254.169.254/32 static route installed on the VPC LR, is the localport
// LSP defined as expected.
// - Host realisation (inside the per-VPC netns imds-<short>): the imds-h-<short>
// veth's address/route/neighbour state and the listener socket — the
// load-bearing question is whether the netns has a real L3 path back to
// guestIP. After the per-VPC-netns fix the veth carries 169.254.169.254/30
// with a default route via the .253 LRP, so a healthy dump shows the IPv4
// address, a neighbour entry for .253, and a reply route via .253; the
// pre-fix failure showed an addressless veth that accepted the SYN but never
// returned the SYN-ACK.
// - In-flight evidence: conntrack (per-netns) for 169.254.169.254 (a stuck
// SYN_RECV is the smoking gun that the host received the SYN and replied but
// the ACK never came back) and the br-int flows touching the IMDS / guest
// addresses.
//
// Non-fatal — runs purely for log/artifact signal so the test's own Fatal still
// wins. Skips silently when OVN tooling / passwordless sudo aren't available, so
// it's safe to call from developer laptops.
func DumpIMDSDatapathDiagnostics(t *testing.T, vpcID, guestIP, artifactDir string) {
t.Helper()
fmt.Printf("\n%s%s── DIAGNOSTICS: IMDS datapath (vpc=%s guest=%s) ──%s\n",
colorBold, colorCyan, vpcID, guestIP, colorReset)
defer fmt.Printf("%s%s── END DIAGNOSTICS ──%s\n\n", colorBold, colorCyan, colorReset)
if _, err := exec.LookPath("ovn-nbctl"); err != nil {
fmt.Printf(" ovn-nbctl unavailable (%v); skipping IMDS datapath dump\n", err)
return
}
if err := exec.Command("sudo", "-n", "true").Run(); err != nil {
fmt.Printf(" passwordless sudo unavailable (%v); skipping IMDS datapath dump\n", err)
return
}
imdsPort := "imds-port-" + vpcID
hostEnd := host.IMDSHostVethName(vpcID)
ovsEnd := host.IMDSOVSPortName(vpcID)
netns := host.IMDSNetnsName(vpcID)
vpcRouter := "vpc-" + vpcID
const imdsIP = "169.254.169.254"
// nsExec wraps an argv to run inside the per-VPC netns, where the host veth
// end and the IMDS listener now live.
nsExec := func(argv ...string) []string {
return append([]string{"ip", "netns", "exec", netns}, argv...)
}
runHostCaptures(t, artifactDir, []hostCapture{
{
filename: "imds-portbinding.txt",
label: "ovn-sbctl Port_Binding imds-port (chassis,up)",
argv: []string{"ovn-sbctl", "--bare", "--columns=logical_port,chassis,up,mac",
"find", "Port_Binding", "logical_port=" + imdsPort},
},
{
filename: "imds-lsp-nb.txt",
label: "ovn-nbctl imds-port LSP (type,addresses,options)",
argv: []string{"ovn-nbctl", "--bare", "--columns=name,type,addresses,options",
"find", "Logical_Switch_Port", "name=" + imdsPort},
},
{
filename: "imds-lr-routes.txt",
label: "ovn-nbctl lr-route-list (169.254.169.254/32 static route?)",
argv: []string{"ovn-nbctl", "lr-route-list", vpcRouter},
grepFor: []string{imdsIP},
},
{
filename: "imds-netns.txt",
label: "ip netns list (per-VPC IMDS netns present?)",
argv: []string{"ip", "netns", "list"},
grepFor: []string{netns},
},
{
filename: "imds-host-veth-addr.txt",
label: "ip -d addr show imds-h in netns (expect 169.254.169.254/30)",
argv: nsExec("ip", "-d", "addr", "show", "dev", hostEnd),
},
{
filename: "imds-host-route-to-guest.txt",
label: "ip route get <guest> in netns (expect via 169.254.169.253)",
argv: nsExec("ip", "route", "get", guestIP),
},
{
filename: "imds-host-neigh.txt",
label: "ip neigh show in netns (expect .253 LRP entry)",
argv: nsExec("ip", "neigh", "show"),
},
{
filename: "imds-listener.txt",
label: "ss -ltnp in netns (IMDS listener bound?)",
argv: nsExec("ss", "-ltnp"),
grepFor: []string{imdsIP, ":80"},
},
{
filename: "imds-conntrack.txt",
label: "conntrack -L in netns (169.254.169.254 — SYN_RECV = reply lost)",
argv: nsExec("conntrack", "-L"),
grepFor: []string{imdsIP},
},
{
filename: "imds-ovs-flows.txt",
label: "ovs-ofctl dump-flows br-int (IMDS + guest, filtered)",
argv: []string{"ovs-ofctl", "dump-flows", "br-int"},
grepFor: []string{imdsIP, guestIP},
},
{
filename: "imds-ovs-iface.txt",
label: "ovs-vsctl Interface imds-o (iface-id binding, ofport)",
argv: []string{"ovs-vsctl", "--columns=name,external_ids,ofport",
"list", "Interface", ovsEnd},
},
})
}
tests/e2e/single/discovery_test.go
+1 -1
View File
@@ -83,7 +83,7 @@ func runEnvironment(t *testing.T, fix *Fixture) {
harness.Detail(t, "az", azName, "region", aws.StringValue(azOut.AvailabilityZones[0].RegionName))
harness.OnFailure(t, func() {
harness.DumpCmd(t, fix.Artifacts, "phase1-az.txt",
harness.DumpCmd(t, fix.ArtifactDir(t), "phase1-az.txt",
"aws", "ec2", "describe-availability-zones")
})
}
tests/e2e/single/imds_test.go
+459
View File
@@ -0,0 +1,459 @@
//go:build e2e
package single
import (
"encoding/base64"
"encoding/json"
"fmt"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/iam"
"github.com/aws/aws-sdk-go/service/sts"
"github.com/mulgadc/spinifex/tests/e2e/harness"
"github.com/stretchr/testify/require"
)
// IMDS E2E identifiers. The role/profile carry an imds-e2e- prefix so they
// never collide with the IAM-roles suite ("app-role"/"app-profile") or the
// STS suite ("sts-e2e-role").
const (
imdsRoleName = "imds-e2e-role"
imdsProfileName = "imds-e2e-profile"
// metaURL is the AWS-compatible link-local IMDS endpoint, served from the
// host per-VPC via SO_BINDTODEVICE on the imds-h-<shortVpcID> veth.
metaURL = "http://169.254.169.254"
// imdsUserData is a no-op cloud-config carrying a unique marker so the
// /latest/user-data round-trip can assert the body the guest sees matches
// what RunInstances was given. A comment-only cloud-config is a valid empty
// config, so it doesn't perturb boot.
imdsUserData = "#cloud-config\n# imds-e2e-userdata-marker-7f3a91\n"
imdsUDMarker = "imds-e2e-userdata-marker-7f3a91"
// imdsProbeVPCCIDR / imdsProbeSubnetCIDR are shared by BOTH isolation VPCs.
// Overlapping CIDRs across VPCs are legal (each VPC is its own routing
// domain); using identical subnets makes IPAM hand the first VM in each the
// SAME private IP (network+4, deterministic per ipam.go), which is exactly
// the cross-VPC source-IP collision the isolation assertion needs.
imdsProbeVPCCIDR = "10.211.0.0/16"
imdsProbeSubnetCIDR = "10.211.7.0/24"
)
// imdsCredDoc mirrors the AWS credential JSON served at
// /latest/meta-data/iam/security-credentials/<role>. Only the fields the test
// asserts on are decoded.
type imdsCredDoc struct {
Code string `json:"Code"`
Type string `json:"Type"`
AccessKeyId string `json:"AccessKeyId"`
SecretAccessKey string `json:"SecretAccessKey"`
Token string `json:"Token"`
Expiration string `json:"Expiration"`
AccountId string `json:"AccountId"`
}
// runIMDS exercises the host-served IMDSv2 surface end-to-end against real guest
// VMs (docs/development/feature/imds-v1.md Step 11). It stands up two VPCs with
// identical subnet CIDRs and one VM in each so the two VMs share a private IP:
//
// - VM X (profile-bound, user-data) drives the full surface: IMDSv2 token
// issuance, the v2-only stance (tokenless / garbage-token GET → 401), the
// metadata fields, the instance-role credential path + a wire round-trip
// proving the ASIA creds resolve to the instance assumed-role ARN, the
// /latest/user-data round-trip, and the DHCP option-121 guest route.
// - The per-VPC datapath invariant: imds-port-<vpcID> is a localport LSP.
// - Cross-VPC isolation: VM X and VM Y share the same source IP in different
// VPCs, yet each IMDS returns its OWN instance-id — the load-bearing
// (VPC-ID, source-IP) → ENI boundary (plan §Datapath-attested identity).
//
// OVN + SSH gated: the IMDS datapath needs ovn-controller to bind the localport
// per chassis, and every assertion runs via SSH into the guest.
func runIMDS(t *testing.T, fix *Fixture) {
harness.Phase(t, "Single — IMDSv2 Host-Served Instance Metadata")
harness.SkipIfNoOVN(t)
requireSSHHealthy(t)
adminAccount := iamEnsureAdminAccountID(t, fix)
_, keyPath := needKeyPair(t, fix)
imdsEnsureRoleProfile(t, fix, adminAccount)
// Two VPCs, identical subnet CIDR → the first VM in each gets the same IP.
vpcX, subX, sgX := imdsEnsureProbeVPC(t, fix, imdsProbeVPCCIDR, imdsProbeSubnetCIDR)
vpcY, subY, sgY := imdsEnsureProbeVPC(t, fix, imdsProbeVPCCIDR, imdsProbeSubnetCIDR)
// VM X — profile-bound + user-data; drives the full surface.
idX, privX, tgtX := imdsProbe(t, fix, keyPath, imdsVMSpec{
subnetID: subX,
sgID: sgX,
profileName: imdsProfileName,
userData: imdsUserData,
})
// VM Y — isolation peer; no profile needed for the instance-id probe.
idY, privY, tgtY := imdsProbe(t, fix, keyPath, imdsVMSpec{
subnetID: subY,
sgID: sgY,
})
require.NotEqual(t, idX, idY, "the two probe VMs must be distinct instances")
require.Equalf(t, privX, privY,
"isolation premise: both first-in-subnet VMs must share a private IP "+
"(IPAM is deterministic at network+4) — got %s vs %s", privX, privY)
harness.Detail(t, "vm_x", idX, "vm_y", idY, "shared_priv", privX)
// --- Per-VPC OVN datapath invariant: imds-port-<vpc> is a localport ------
harness.Step(t, "ovn-nbctl imds-port-%s must be a localport LSP", vpcX)
imdsLSP := "imds-port-" + vpcX // mirrors topology.IMDSPort (inlined, as datapath_test does)
lspType := harness.OvnNbctl(t, "--no-leader-only", "--bare", "--columns=type",
"find", "logical_switch_port", "name="+imdsLSP)
require.Equalf(t, "localport", lspType,
"imds-port LSP %s must be type=localport so every chassis self-serves IMDS "+
"(a regular LSP binds one chassis and forces Geneve tunnelling)", imdsLSP)
// --- Subnet-router proxy-ARP makes 169.254.169.254 reachable link-local --
// IMDS reachability no longer depends on a DHCP option-121 route in the
// guest; the subnet router LSP answers ARP for the IMDS address via OVN
// options:arp_proxy, so DHCP and fully static guests reach it identically.
harness.Step(t, "subnet router LSP rtr-port-%s must carry options:arp_proxy", subX)
rtrLSP := "rtr-port-" + subX // mirrors topology.SubnetSwitchRouterPort
lspOpts := harness.OvnNbctl(t, "--no-leader-only", "--bare", "--columns=options",
"find", "logical_switch_port", "name="+rtrLSP)
require.Containsf(t, lspOpts, "arp_proxy=169.254.169.254",
"subnet router LSP %s must set options:arp_proxy=169.254.169.254 so the subnet LRP "+
"answers ARP for IMDS link-local (static-IP and NetworkManager/RHEL/Ubuntu guests)", rtrLSP)
// --- IMDSv2 token issuance ----------------------------------------------
harness.Step(t, "PUT /latest/api/token (IMDSv2)")
tokenX := imdsAwaitToken(t, fix, tgtX, vpcX, privX)
harness.Detail(t, "token_len", len(tokenX))
// --- v2-only stance: tokenless + garbage-token GET → 401 -----------------
harness.Step(t, "tokenless GET /latest/meta-data/instance-id → 401")
require.Equal(t, "401", imdsCode(tgtX, "", "/latest/meta-data/instance-id"),
"IMDSv1 (tokenless) GET must be rejected with 401")
harness.Step(t, "garbage-token GET → 401")
require.Equal(t, "401",
imdsCode(tgtX, `-H "X-aws-ec2-metadata-token: not-a-real-token"`, "/latest/meta-data/instance-id"),
"unknown token must be rejected with 401 (no token-exists leak)")
// --- Metadata surface ----------------------------------------------------
harness.Step(t, "GET metadata surface")
require.Equal(t, idX, imdsGet(t, tgtX, tokenX, "/latest/meta-data/instance-id"),
"instance-id mismatch")
require.Equal(t, privX, imdsGet(t, tgtX, tokenX, "/latest/meta-data/local-ipv4"),
"local-ipv4 must equal the request source IP")
require.NotEmpty(t, imdsGet(t, tgtX, tokenX, "/latest/meta-data/instance-type"),
"instance-type must be populated")
require.NotEmpty(t, imdsGet(t, tgtX, tokenX, "/latest/meta-data/mac"),
"mac must be populated")
require.NotEmpty(t, imdsGet(t, tgtX, tokenX, "/latest/meta-data/placement/availability-zone"),
"placement/availability-zone must be populated")
// iam/info → InstanceProfileArn ends with the bound profile.
iamInfo := imdsGet(t, tgtX, tokenX, "/latest/meta-data/iam/info")
require.Containsf(t, iamInfo, ":instance-profile/"+imdsProfileName,
"iam/info must carry the bound InstanceProfileArn (got %q)", iamInfo)
// security-credentials/ lists the role name (not the profile name).
require.Equal(t, imdsRoleName, imdsGet(t, tgtX, tokenX, "/latest/meta-data/iam/security-credentials/"),
"security-credentials/ must list the resolved role name")
// --- user-data round-trip ------------------------------------------------
harness.Step(t, "GET /latest/user-data round-trips launch user-data")
require.Contains(t, imdsGet(t, tgtX, tokenX, "/latest/user-data"), imdsUDMarker,
"user-data must round-trip the launch user-data")
// --- Instance-role credentials + wire round-trip -------------------------
harness.Step(t, "GET security-credentials/%s → ASIA creds", imdsRoleName)
credBody := imdsGet(t, tgtX, tokenX, "/latest/meta-data/iam/security-credentials/"+imdsRoleName)
var cred imdsCredDoc
require.NoError(t, json.Unmarshal([]byte(credBody), &cred),
"credential body must be valid JSON: %s", credBody)
require.Equal(t, "Success", cred.Code, "credential Code must be Success (body=%s)", credBody)
require.True(t, strings.HasPrefix(cred.AccessKeyId, "ASIA"),
"instance-role AKID must start with ASIA, got %q", cred.AccessKeyId)
require.NotEmpty(t, cred.SecretAccessKey, "empty SecretAccessKey")
require.NotEmpty(t, cred.Token, "empty session Token")
require.Equal(t, adminAccount, cred.AccountId, "credential AccountId mismatch")
harness.Detail(t, "cred_akid", cred.AccessKeyId)
// Prove the minted creds verify on the gateway and resolve to the instance-
// bound assumed-role identity. The session name is the instance ID, so the
// ARN is .../assumed-role/<role>/<instance-id>.
harness.Step(t, "get-caller-identity with the IMDS-minted creds (ASIA path)")
sessionCli := harness.NewAWSClientWithSessionCreds(t, fix.Env,
cred.AccessKeyId, cred.SecretAccessKey, cred.Token)
who, err := sessionCli.STS.GetCallerIdentity(&sts.GetCallerIdentityInput{})
require.NoError(t, err, "get-caller-identity with instance-role creds")
expectedARN := "arn:aws:sts::" + adminAccount + ":assumed-role/" + imdsRoleName + "/" + idX
require.Equal(t, expectedARN, aws.StringValue(who.Arn),
"IMDS-minted creds must resolve to the instance assumed-role ARN")
// --- Cross-VPC isolation -------------------------------------------------
// VM X and VM Y query 169.254.169.254 from the IDENTICAL source IP, in
// different VPCs. Each must get its own instance-id; a leak here means the
// per-VPC SO_BINDTODEVICE veth / (VPC-ID, source-IP) → ENI mapping is broken.
harness.Step(t, "cross-VPC isolation: shared IP %s, distinct identities", privX)
tokenY := imdsAwaitToken(t, fix, tgtY, vpcY, privY)
gotY := imdsGet(t, tgtY, tokenY, "/latest/meta-data/instance-id")
require.Equalf(t, idY, gotY, "VM Y IMDS must return VM Y's instance-id (got %q)", gotY)
require.NotEqualf(t, idX, gotY,
"cross-VPC leak: VM Y (source IP %s) resolved to VM X — the "+
"(VPC-ID, source-IP) → ENI boundary is broken", privX)
// And VM X still resolves to itself (not Y) after Y came up on the same IP.
require.Equal(t, idX, imdsGet(t, tgtX, tokenX, "/latest/meta-data/instance-id"),
"VM X IMDS must still return VM X's instance-id")
harness.Detail(t, "isolation", "ok")
}
// imdsVMSpec parameterises a launch for the IMDS suite. profileName / userData
// are optional.
type imdsVMSpec struct {
subnetID string
sgID string
profileName string
userData string
}
// imdsProbe launches a VM per spec, waits for it to run + become SSH-reachable,
// and returns (instanceID, privateIP, sshTarget). Registers terminate cleanup
// so the VM is torn down before its VPC/subnet are deleted (LIFO).
func imdsProbe(t *testing.T, fix *Fixture, keyPath string, spec imdsVMSpec) (string, string, harness.SSHTarget) {
t.Helper()
id := imdsRunVM(t, fix, spec)
t.Cleanup(func() {
_, _ = fix.AWS.EC2.TerminateInstances(&ec2.TerminateInstancesInput{
InstanceIds: []*string{aws.String(id)},
})
_ = waitForInstanceStateSoft(fix.AWS, id, "terminated", 5*time.Minute)
})
inst := harness.WaitForInstanceState(t, fix.AWS, id, "running")
priv := aws.StringValue(inst.PrivateIpAddress)
require.NotEmptyf(t, priv, "instance %s has no PrivateIpAddress", id)
host, port := harness.InstancePublicSSHHost(t, inst)
harness.Step(t, "wait for %s SSH at %s:%d", id, host, port)
waitForSSHHandshake(t, host, port, keyPath)
return id, priv, harness.SSHTarget{User: "ec2-user", Host: host, Port: port, KeyPath: keyPath}
}
// imdsRunVM launches a single instance per spec and returns its ID. AMI / type /
// key come from the suite discovery helpers.
func imdsRunVM(t *testing.T, fix *Fixture, spec imdsVMSpec) string {
t.Helper()
amiID := needAMI(t, fix)
instType, _ := needInstanceTypeArch(t, fix)
keyName, _ := needKeyPair(t, fix)
in := &ec2.RunInstancesInput{
ImageId: aws.String(amiID),
InstanceType: aws.String(instType),
KeyName: aws.String(keyName),
SubnetId: aws.String(spec.subnetID),
SecurityGroupIds: []*string{aws.String(spec.sgID)},
MinCount: aws.Int64(1),
MaxCount: aws.Int64(1),
}
if spec.profileName != "" {
in.IamInstanceProfile = &ec2.IamInstanceProfileSpecification{Name: aws.String(spec.profileName)}
}
if spec.userData != "" {
in.UserData = aws.String(base64.StdEncoding.EncodeToString([]byte(spec.userData)))
}
out, err := fix.AWS.EC2.RunInstances(in)
require.NoError(t, err, "run-instances subnet=%s", spec.subnetID)
require.NotEmpty(t, out.Instances, "run-instances returned no Instances")
id := aws.StringValue(out.Instances[0].InstanceId)
require.True(t, strings.HasPrefix(id, "i-"), "unexpected InstanceId %q", id)
return id
}
// imdsEnsureProbeVPC creates a VPC + subnet wired for SSH reachability and
// returns (vpcID, subnetID, sgID). In pool mode it adds the public-IP path
// (MapPublicIpOnLaunch + IGW + 0.0.0.0/0 route) so the runner reaches the guest
// by public IP; in dev_networking mode SSH lands via the per-VM qemu hostfwd, so
// the IGW plumbing is skipped. All resources are torn down LIFO via t.Cleanup.
func imdsEnsureProbeVPC(t *testing.T, fix *Fixture, vpcCIDR, subnetCIDR string) (string, string, string) {
t.Helper()
c := fix.AWS
vpcOut, err := c.EC2.CreateVpc(&ec2.CreateVpcInput{CidrBlock: aws.String(vpcCIDR)})
require.NoError(t, err, "create-vpc %s", vpcCIDR)
vpcID := aws.StringValue(vpcOut.Vpc.VpcId)
require.NotEmpty(t, vpcID, "VpcId empty")
t.Cleanup(func() { _, _ = c.EC2.DeleteVpc(&ec2.DeleteVpcInput{VpcId: aws.String(vpcID)}) })
subOut, err := c.EC2.CreateSubnet(&ec2.CreateSubnetInput{
VpcId: aws.String(vpcID),
CidrBlock: aws.String(subnetCIDR),
})
require.NoError(t, err, "create-subnet %s", subnetCIDR)
subnetID := aws.StringValue(subOut.Subnet.SubnetId)
require.NotEmpty(t, subnetID, "SubnetId empty")
t.Cleanup(func() { _, _ = c.EC2.DeleteSubnet(&ec2.DeleteSubnetInput{SubnetId: aws.String(subnetID)}) })
if fix.PoolMode {
imdsAttachInternet(t, c, vpcID, subnetID)
}
sgID := imdsDefaultSG(t, c, vpcID)
harness.AuthorizeSSHIngress(t, c, sgID)
harness.Detail(t, "probe_vpc", vpcID, "subnet", subnetID, "sg", sgID)
return vpcID, subnetID, sgID
}
// imdsAttachInternet gives subnetID a public-IP egress path (pool mode only):
// MapPublicIpOnLaunch + a fresh IGW + a 0.0.0.0/0 route table. Cleanups unwind
// LIFO before the subnet/VPC are deleted by the caller's earlier-registered
// cleanups.
func imdsAttachInternet(t *testing.T, c *harness.AWSClient, vpcID, subnetID string) {
t.Helper()
_, err := c.EC2.ModifySubnetAttribute(&ec2.ModifySubnetAttributeInput{
SubnetId: aws.String(subnetID),
MapPublicIpOnLaunch: &ec2.AttributeBooleanValue{Value: aws.Bool(true)},
})
require.NoError(t, err, "modify-subnet-attribute MapPublicIpOnLaunch")
igwOut, err := c.EC2.CreateInternetGateway(&ec2.CreateInternetGatewayInput{})
require.NoError(t, err, "create-internet-gateway")
igwID := aws.StringValue(igwOut.InternetGateway.InternetGatewayId)
require.NotEmpty(t, igwID, "InternetGatewayId empty")
t.Cleanup(func() {
_, _ = c.EC2.DetachInternetGateway(&ec2.DetachInternetGatewayInput{
InternetGatewayId: aws.String(igwID), VpcId: aws.String(vpcID),
})
_, _ = c.EC2.DeleteInternetGateway(&ec2.DeleteInternetGatewayInput{InternetGatewayId: aws.String(igwID)})
})
_, err = c.EC2.AttachInternetGateway(&ec2.AttachInternetGatewayInput{
InternetGatewayId: aws.String(igwID), VpcId: aws.String(vpcID),
})
require.NoError(t, err, "attach-internet-gateway")
rtOut, err := c.EC2.CreateRouteTable(&ec2.CreateRouteTableInput{VpcId: aws.String(vpcID)})
require.NoError(t, err, "create-route-table")
rtbID := aws.StringValue(rtOut.RouteTable.RouteTableId)
require.NotEmpty(t, rtbID, "RouteTableId empty")
t.Cleanup(func() { _, _ = c.EC2.DeleteRouteTable(&ec2.DeleteRouteTableInput{RouteTableId: aws.String(rtbID)}) })
assocOut, err := c.EC2.AssociateRouteTable(&ec2.AssociateRouteTableInput{
RouteTableId: aws.String(rtbID), SubnetId: aws.String(subnetID),
})
require.NoError(t, err, "associate-route-table")
assocID := aws.StringValue(assocOut.AssociationId)
t.Cleanup(func() {
_, _ = c.EC2.DisassociateRouteTable(&ec2.DisassociateRouteTableInput{AssociationId: aws.String(assocID)})
})
_, err = c.EC2.CreateRoute(&ec2.CreateRouteInput{
RouteTableId: aws.String(rtbID),
DestinationCidrBlock: aws.String("0.0.0.0/0"),
GatewayId: aws.String(igwID),
})
require.NoError(t, err, "create-route 0.0.0.0/0 -> IGW")
}
// imdsEnsureRoleProfile creates the EC2-trusted role + instance profile the
// IMDS credential path resolves, attaching AdministratorAccess so the minted
// session creds round-trip a real call. Registers fixture-teardown cleanup and
// sweeps any residue from a prior run first.
func imdsEnsureRoleProfile(t *testing.T, fix *Fixture, adminAccount string) {
t.Helper()
adminPolicyARN := iamPolicyARN(adminAccount, iamPolicyAdministrator)
iamDeleteRoleAndProfilesBestEffort(fix, imdsRoleName, []string{imdsProfileName}, adminPolicyARN)
fix.Harness.RegisterCleanup(func() {
iamDeleteRoleAndProfilesBestEffort(fix, imdsRoleName, []string{imdsProfileName}, adminPolicyARN)
})
harness.Step(t, "create-role %q (trust=ec2.amazonaws.com) + profile %q", imdsRoleName, imdsProfileName)
_, err := fix.AWS.IAM.CreateRole(&iam.CreateRoleInput{
RoleName: aws.String(imdsRoleName),
AssumeRolePolicyDocument: aws.String(iamTrustPolicyEC2Standard),
Description: aws.String("E2E IMDS instance-role credentials"),
})
require.NoError(t, err, "create-role")
_, err = fix.AWS.IAM.AttachRolePolicy(&iam.AttachRolePolicyInput{
RoleName: aws.String(imdsRoleName),
PolicyArn: aws.String(adminPolicyARN),
})
require.NoError(t, err, "attach AdministratorAccess to role")
_, err = fix.AWS.IAM.CreateInstanceProfile(&iam.CreateInstanceProfileInput{
InstanceProfileName: aws.String(imdsProfileName),
})
require.NoError(t, err, "create-instance-profile")
_, err = fix.AWS.IAM.AddRoleToInstanceProfile(&iam.AddRoleToInstanceProfileInput{
InstanceProfileName: aws.String(imdsProfileName),
RoleName: aws.String(imdsRoleName),
})
require.NoError(t, err, "add-role-to-instance-profile")
}
// imdsAwaitToken PUTs an IMDSv2 token from inside the guest, retrying to ride
// out the cold-start window before BindManager.Sync + the ENI reverse-index
// land. Returns the token. On timeout it dumps the per-VPC IMDS datapath
// (OVN realisation + host veth route/neigh + conntrack) before failing, so a
// reachability timeout (exit 28) is triaged as request-path vs reply-path
// rather than a bare "condition not met".
func imdsAwaitToken(t *testing.T, fix *Fixture, tgt harness.SSHTarget, vpcID, guestIP string) string {
t.Helper()
deadline := time.Now().Add(90 * time.Second)
var lastErr error
for {
cmd := fmt.Sprintf(
`curl -sf -X PUT "%s/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 120"`, metaURL)
out, err := runSSHCombined(tgt, cmd)
switch {
case err != nil:
lastErr = fmt.Errorf("token PUT failed: %w (out=%q)", err, out)
case strings.TrimSpace(out) == "":
lastErr = fmt.Errorf("token PUT returned empty body")
default:
return strings.TrimSpace(out)
}
if time.Now().After(deadline) {
harness.DumpIMDSDatapathDiagnostics(t, vpcID, guestIP, fix.ArtifactDir(t))
t.Fatalf("imdsAwaitToken: token never minted within 90s (vpc=%s guest=%s): %v "+
"(see IMDS datapath diagnostics above)", vpcID, guestIP, lastErr)
}
time.Sleep(3 * time.Second)
}
}
// imdsGet fetches a token-gated path from inside the guest and returns the
// trimmed body. -sf turns any HTTP >= 400 into a non-zero exit, so a 4xx/5xx
// fails the test loudly with the server status visible via runSSH's stderr.
func imdsGet(t *testing.T, tgt harness.SSHTarget, token, path string) string {
t.Helper()
cmd := fmt.Sprintf(`curl -sf -H "X-aws-ec2-metadata-token: %s" %s%s`, token, metaURL, path)
return strings.TrimSpace(runSSH(t, tgt, cmd))
}
// imdsCode returns the HTTP status code (as a string) the guest sees for a GET,
// without failing on non-2xx. headerArg is an optional pre-formatted curl -H
// argument (empty for the tokenless probe).
func imdsCode(tgt harness.SSHTarget, headerArg, path string) string {
cmd := fmt.Sprintf(`curl -s -o /dev/null -w '%%{http_code}' %s %s%s`, headerArg, metaURL, path)
out, _ := runSSHCombined(tgt, cmd)
return strings.TrimSpace(out)
}
// imdsDefaultSG returns the auto-created default security group ID for a VPC.
func imdsDefaultSG(t *testing.T, c *harness.AWSClient, vpcID string) string {
t.Helper()
out, err := c.EC2.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{
Filters: []*ec2.Filter{
{Name: aws.String("vpc-id"), Values: []*string{aws.String(vpcID)}},
{Name: aws.String("group-name"), Values: []*string{aws.String("default")}},
},
})
require.NoError(t, err, "describe-security-groups default (vpc=%s)", vpcID)
require.NotEmpty(t, out.SecurityGroups, "no default SG for vpc=%s", vpcID)
id := aws.StringValue(out.SecurityGroups[0].GroupId)
require.NotEmpty(t, id, "default SG GroupId empty (vpc=%s)", vpcID)
return id
}
tests/e2e/single/main_test.go
+19 -11
View File
@@ -72,11 +72,10 @@ func requireSingleNodeFixture(t *testing.T) *Fixture {
return
}
fix := &Fixture{
Env: env,
AWS: awsCli,
Harness: h,
Artifacts: harness.ArtifactDir(t, env),
TmpDir: tmpDir,
Env: env,
AWS: awsCli,
Harness: h,
TmpDir: tmpDir,
}
fix.PoolMode = detectPoolMode(env)
// admin init leaves the default SG closed (AWS parity); the e2e suite
@@ -100,12 +99,21 @@ func requireSingleNodeFixture(t *testing.T) *Fixture {
// memoized on Harness (harness.Fixture) and surfaced via the package-local
// need* helpers / harness.Ensure* + harness.Discover*.
type Fixture struct {
Env *harness.Env
AWS *harness.AWSClient
Harness *harness.Fixture // memoized Ensure* fixture; spans the whole process.
Artifacts string
TmpDir string // package-scoped scratch dir; survives every Test* in the package.
PoolMode bool // gates 8b / 8d
Env *harness.Env
AWS *harness.AWSClient
Harness *harness.Fixture // memoized Ensure* fixture; spans the whole process.
TmpDir string // package-scoped scratch dir; survives every Test* in the package.
PoolMode bool // gates 8b / 8d
}
// ArtifactDir returns the artifact directory for the *currently running* test.
// It must be derived per-call from the live t (not memoized on the singleton
// Fixture): the process fixture is built lazily during whichever test runs
// first, so a stored dir would freeze to that test's name and every later test
// would write into a stale — and, once that test passes, pruned — directory.
func (f *Fixture) ArtifactDir(t *testing.T) string {
t.Helper()
return harness.ArtifactDir(t, f.Env)
}
// detectPoolMode reads external_mode from spinifex.toml. Defaults to false
tests/e2e/single/tests_test.go
+10
View File
@@ -225,6 +225,16 @@ func TestSTSAssumeRoleAndGetCallerIdentity(t *testing.T) {
runSTS(t, requireSingleNodeFixture(t))
}
// TestIMDS exercises the host-served IMDSv2 surface end-to-end: token
// issuance, the v2-only stance, the metadata surface, the
// instance-role credential path + wire round-trip, DHCP option 121, the
// per-VPC localport datapath, and cross-VPC source-IP isolation. Owns its own
// role/profile + a second VPC, but sequential: it launches profile-bound VMs
// and a fresh VPC, so it must not race the singleton-mutation tests above.
func TestIMDS(t *testing.T) {
runIMDS(t, requireSingleNodeFixture(t))
}
// TestFinalClusterStats runs as the last sequential test (parallel bucket
// runs afterwards, but those Tests are read-only / own-everything so a
// later cluster-stats snapshot would still be valid).
tests/e2e/single/vpc_test.go
+1 -1
View File
@@ -254,7 +254,7 @@ func runVPCSubnetE2E(t *testing.T, fix *Fixture) {
harness.VPCDiagnosticsOpts{
ExternalIP: pubIP,
LogicalIP: privIP,
ArtifactDir: fix.Artifacts,
ArtifactDir: fix.ArtifactDir(t),
})
t.Fatalf("SSH handshake %s:22 never completed within 3min (see diagnostics above; tracking mulga-siv-111)", pubIP)
}