test: cover the node insight handlers and the facts file mode

Each node collects its hardware facts as root at daemon start (dmidecode,
ethtool, networkctl LLDP where available) into /etc/spinifex/node-facts, and
the daemon serves them with its plane wiring over NATS.

New admin-only actions, refused for every account but the system admin
account regardless of policy:

- GetNodeFacts: hardware, NICs, LLDP neighbours and planes per node
- GetNodeMetrics: live CPU, memory and per-plane traffic for one node
- StartNetworkTest / DescribeNetworkTests: iperf2 between nodes over a
  chosen plane, both directions, one test cluster-wide at a time, recorded
  in KV and converged asynchronously

The iperf listener runs only for the duration of a run, without
capabilities, on TCP 5001, which the peers firewall rule now allows. A run
refuses any target outside the node's own plane subnet.

The console gains a node panel (hardware, network, metrics, network test
with a dial scaled to the port's line rate) and a topology view built with
@xyflow/react. setup.sh installs dmidecode and iperf.

.github/workflows/e2e-nightly.yml

@@ -1203,6 +1203,16 @@ jobs:
           sleep 2
           rm -f "$TMPDIR"/tmp.*.qcow2 "$TMPDIR"/tmp.*.log "$TMPDIR"/tmp.*-node.log
 
+      # The ISO build runs as root inside the workspace, and the next job on
+      # this runner, from any workflow, cannot remove what it leaves there.
+      - name: Release the workspace
+        if: always()
+        run: |
+          sudo umount -R -l "$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/.cache/rootfs 2>/dev/null || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/.cache \
+              "$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/dist
+          sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE"
+
   iso-multi:
     name: cell-22 (iso/dhcp/three-node)
     # After cell-21 rather than beside it: same concurrency group, and running
@@ -1354,6 +1364,16 @@ jobs:
           rm -rf /tmp/cell22-artifacts-$GITHUB_RUN_ID /tmp/cell22-$GITHUB_RUN_ID.log \
                  "$WORK_DIR"
 
+      # The ISO build runs as root inside the workspace, and the next job on
+      # this runner, from any workflow, cannot remove what it leaves there.
+      - name: Release the workspace
+        if: always()
+        run: |
+          sudo umount -R -l "$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/.cache/rootfs 2>/dev/null || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/.cache \
+              "$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/dist
+          sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE"
+
   # Predastore throughput ("cell 23"). Predastore has no runners of its own and
   # spinifex depends on it, so the benchmark lives here and checks predastore
   # out as a sibling.

build/systemd/spinifex-daemon.service

@@ -1,4 +1,4 @@
-# spinifex-unit-version: 5
+# spinifex-unit-version: 6
 [Unit]
 Description=Spinifex Daemon (VM Orchestration)
 PartOf=spinifex.target
@@ -15,6 +15,7 @@ Slice=spinifex-system.slice
 User=spinifex-daemon
 Group=spinifex
 ExecStartPre=/usr/local/bin/spx admin hostnet check
+ExecStartPre=-+/usr/local/bin/spx admin node-facts collect
 ExecStart=/usr/local/bin/spx service spinifex start
 Restart=on-failure
 RestartSec=5

cmd/eks-gateway-fetch/main.go

@@ -6,6 +6,7 @@
 //
 //	eks-gateway-fetch -resource addons   # GET /clusters/{cluster}/internal-addons/{accountId}
 //	eks-gateway-fetch -resource recovery -instance-id i-… # GET /clusters/{cluster}/internal-recovery/{accountId}/{instanceId}
+//	eks-gateway-fetch -resource workers  # GET /clusters/{cluster}/internal-workers/{accountId}
 //
 // Flags default to environment variables seeded by cloud-init:
 // EKS_GATEWAY_URL, EKS_GATEWAY_CA, EKS_ACCESS_KEY, EKS_SECRET_KEY, EKS_REGION,
@@ -33,7 +34,7 @@ func main() {
 	flag.StringVar(&cfg.AccountID, "account-id", os.Getenv("EKS_ACCOUNT_ID"), "Cluster account ID")
 	flag.StringVar(&cfg.ClusterName, "cluster", os.Getenv("EKS_CLUSTER_NAME"), "Cluster name")
 	flag.StringVar(&cfg.InstanceID, "instance-id", os.Getenv("EKS_INSTANCE_ID"), "This member's EC2 instance ID (recovery resource)")
-	flag.StringVar(&cfg.Resource, "resource", "addons", "Resource to fetch: addons | recovery")
+	flag.StringVar(&cfg.Resource, "resource", "addons", "Resource to fetch: addons | recovery | workers")
 	flag.Parse()
 
 	if err := gatewayfetch.Run(context.Background(), cfg); err != nil {

contracts/cluster/v1/facts.go

@@ -0,0 +1,209 @@
+package clusterv1
+
+import "regexp"
+
+const (
+	// NodeFactsSubject fans out a request for each daemon's hardware and
+	// cabling description. Its reply payload is NodeFacts.
+	NodeFactsSubject = "spinifex.node.facts"
+
+	nodeNetTestPrefix = "spinifex.node.nettest."
+)
+
+var nodeIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`)
+
+// ValidNodeID reports whether a node name is safe to place in a subject: no
+// NATS token separator or wildcard can get through.
+func ValidNodeID(node string) bool {
+	return nodeIDPattern.MatchString(node)
+}
+
+// NetTestServeSubject asks one node to start a one-shot throughput server.
+func NetTestServeSubject(node string) string {
+	return nodeNetTestPrefix + node + ".serve"
+}
+
+// NetTestRunSubject asks one node to run the client side of a throughput test.
+func NetTestRunSubject(node string) string {
+	return nodeNetTestPrefix + node + ".run"
+}
+
+// Planes a node's NICs can carry, named as the installer names them.
+const (
+	PlaneWAN = "wan"
+	PlaneLAN = "lan"
+	PlaneVPC = "vpc"
+)
+
+// NetTestPort is the TCP port reserved for node-to-node throughput tests.
+const NetTestPort = 5001
+
+// NodeFacts describes one node's hardware, OS and cabling. Fields a host
+// cannot report are left empty, and Sources says why.
+type NodeFacts struct {
+	Node        string            `json:"node"`
+	CollectedAt string            `json:"collected_at,omitempty"`
+	FormFactor  string            `json:"form_factor"`
+	Virt        string            `json:"virt,omitempty"`
+	System      SystemFacts       `json:"system"`
+	OS          OSFacts           `json:"os"`
+	CPU         CPUFacts          `json:"cpu"`
+	Memory      MemoryFacts       `json:"memory"`
+	NICs        []NICFacts        `json:"nics"`
+	Planes      []PlaneFacts      `json:"planes"`
+	Sources     map[string]string `json:"sources"`
+}
+
+// SystemFacts is the machine's identity from DMI.
+type SystemFacts struct {
+	Vendor       string `json:"vendor,omitempty"`
+	Product      string `json:"product,omitempty"`
+	Family       string `json:"family,omitempty"`
+	Version      string `json:"version,omitempty"`
+	Serial       string `json:"serial,omitempty"`
+	UUID         string `json:"uuid,omitempty"`
+	ChassisType  string `json:"chassis_type,omitempty"`
+	ChassisAsset string `json:"chassis_asset,omitempty"`
+	BoardVendor  string `json:"board_vendor,omitempty"`
+	BoardProduct string `json:"board_product,omitempty"`
+	BoardSerial  string `json:"board_serial,omitempty"`
+	BIOSVendor   string `json:"bios_vendor,omitempty"`
+	BIOSVersion  string `json:"bios_version,omitempty"`
+	BIOSDate     string `json:"bios_date,omitempty"`
+	DeviceModel  string `json:"device_model,omitempty"`
+}
+
+// OSFacts names the running OS and Spinifex build.
+type OSFacts struct {
+	Name          string `json:"name,omitempty"`
+	Kernel        string `json:"kernel,omitempty"`
+	Arch          string `json:"arch,omitempty"`
+	Version       string `json:"spx_version,omitempty"`
+	Commit        string `json:"spx_commit,omitempty"`
+	UptimeSeconds int64  `json:"uptime_seconds,omitempty"`
+	Hostname      string `json:"hostname,omitempty"`
+}
+
+// CPUFacts summarises the sockets. Threads and Cores are totals.
+type CPUFacts struct {
+	Model          string   `json:"model,omitempty"`
+	Vendor         string   `json:"vendor,omitempty"`
+	Sockets        int      `json:"sockets"`
+	CoresPerSocket int      `json:"cores_per_socket,omitempty"`
+	Cores          int      `json:"cores"`
+	Threads        int      `json:"threads"`
+	MaxMHz         int      `json:"max_mhz,omitempty"`
+	CurrentMHz     int      `json:"current_mhz,omitempty"`
+	Virtualization string   `json:"virtualization,omitempty"`
+	Features       []string `json:"features,omitempty"`
+}
+
+// MemoryFacts groups identical DIMMs so a 32-slot board reads as one line.
+type MemoryFacts struct {
+	TotalBytes    uint64        `json:"total_bytes"`
+	SlotsUsed     int           `json:"slots_used"`
+	SlotsTotal    int           `json:"slots_total"`
+	MaxCapacity   string        `json:"max_capacity,omitempty"`
+	ECC           string        `json:"ecc,omitempty"`
+	Groups        []MemoryGroup `json:"groups"`
+	OSTotalBytes  uint64        `json:"os_total_bytes,omitempty"`
+	OSAvailBytes  uint64        `json:"os_available_bytes,omitempty"`
+	HugePageBytes uint64        `json:"huge_page_bytes,omitempty"`
+}
+
+// MemoryGroup is a set of DIMMs alike in every field shown. ConfiguredMTs
+// below SpeedMTs means the board runs them slower than rated.
+type MemoryGroup struct {
+	Count         int      `json:"count"`
+	SizeBytes     uint64   `json:"size_bytes"`
+	Type          string   `json:"type,omitempty"`
+	TypeDetail    string   `json:"type_detail,omitempty"`
+	FormFactor    string   `json:"form_factor,omitempty"`
+	SpeedMTs      int      `json:"speed_mts,omitempty"`
+	ConfiguredMTs int      `json:"configured_mts,omitempty"`
+	Manufacturer  string   `json:"manufacturer,omitempty"`
+	PartNumber    string   `json:"part_number,omitempty"`
+	Rank          string   `json:"rank,omitempty"`
+	Serials       []string `json:"serials,omitempty"`
+}
+
+// NICFacts is one physical network interface and, where LLDP reports it, the
+// switch port it is cabled to.
+type NICFacts struct {
+	Name      string        `json:"name"`
+	MAC       string        `json:"mac,omitempty"`
+	Driver    string        `json:"driver,omitempty"`
+	Firmware  string        `json:"firmware,omitempty"`
+	BusInfo   string        `json:"bus_info,omitempty"`
+	Model     string        `json:"model,omitempty"`
+	SpeedMbps int           `json:"speed_mbps,omitempty"`
+	MaxMbps   int           `json:"max_mbps,omitempty"`
+	Duplex    string        `json:"duplex,omitempty"`
+	Port      string        `json:"port,omitempty"`
+	MTU       int           `json:"mtu,omitempty"`
+	State     string        `json:"state"`
+	Master    string        `json:"master,omitempty"`
+	Planes    []string      `json:"planes,omitempty"`
+	Neighbor  *LLDPNeighbor `json:"neighbor,omitempty"`
+}
+
+// LLDPNeighbor is what the switch on the other end of a NIC advertises.
+type LLDPNeighbor struct {
+	ChassisID         string   `json:"chassis_id"`
+	SystemName        string   `json:"system_name,omitempty"`
+	SystemDescription string   `json:"system_description,omitempty"`
+	PortID            string   `json:"port_id,omitempty"`
+	PortDescription   string   `json:"port_description,omitempty"`
+	Capabilities      []string `json:"capabilities,omitempty"`
+}
+
+// PlaneFacts is the address a plane uses on this node and the NICs under it.
+type PlaneFacts struct {
+	Plane   string   `json:"plane"`
+	Bridge  string   `json:"bridge,omitempty"`
+	Address string   `json:"address,omitempty"`
+	Prefix  int      `json:"prefix,omitempty"`
+	NICs    []string `json:"nics,omitempty"`
+	MTU     int      `json:"mtu,omitempty"`
+}
+
+// NetTestServeRequest starts a server bound to the node's address on Plane.
+type NetTestServeRequest struct {
+	Plane           string `json:"plane"`
+	DurationSeconds int    `json:"duration_seconds"`
+}
+
+// NetTestServeResponse names the address the server listens on.
+type NetTestServeResponse struct {
+	Address string `json:"address,omitempty"`
+	Port    int    `json:"port,omitempty"`
+	Error   string `json:"error,omitempty"`
+}
+
+// NetTestRunRequest runs the client towards Address from the node's own
+// address on Plane.
+type NetTestRunRequest struct {
+	Plane           string `json:"plane"`
+	Address         string `json:"address"`
+	Port            int    `json:"port"`
+	DurationSeconds int    `json:"duration_seconds"`
+	Streams         int    `json:"streams"`
+}
+
+// NetTestRunResponse is one direction of a pair's measurement.
+type NetTestRunResponse struct {
+	SourceAddress string    `json:"source_address,omitempty"`
+	BitsPerSecond float64   `json:"bits_per_second"`
+	Bytes         int64     `json:"bytes"`
+	Seconds       float64   `json:"seconds"`
+	Streams       int       `json:"streams"`
+	StreamBPS     []float64 `json:"stream_bps,omitempty"`
+	RTTMinMillis  float64   `json:"rtt_min_ms,omitempty"`
+	RTTAvgMillis  float64   `json:"rtt_avg_ms,omitempty"`
+	RTTMaxMillis  float64   `json:"rtt_max_ms,omitempty"`
+	PacketLoss    float64   `json:"packet_loss_percent,omitempty"`
+	LinkMbps      int       `json:"link_mbps,omitempty"`
+	MTU           int       `json:"mtu,omitempty"`
+	Tool          string    `json:"tool,omitempty"`
+	Error         string    `json:"error,omitempty"`
+}

contracts/telemetry/v1/node_live.go

@@ -0,0 +1,53 @@
+package telemetryv1
+
+import "regexp"
+
+// Node live subjects mirror the instance ones, keyed by node name. Only the
+// super-admin account's gateway handler subscribes to them.
+const (
+	nodeLiveWatchSubjectPrefix  = "telemetry.watch.node."
+	nodeLiveSampleSubjectPrefix = "telemetry.live.node."
+)
+
+var liveNodePattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`)
+
+// ValidLiveNode reports whether a node name is safe to place in a subject.
+func ValidLiveNode(node string) bool {
+	return liveNodePattern.MatchString(node)
+}
+
+// NodeLiveWatchSubject is where a viewer asks a node to keep sampling itself.
+// A request on it is answered with a NodeLiveRing.
+func NodeLiveWatchSubject(node string) string {
+	return nodeLiveWatchSubjectPrefix + node
+}
+
+// NodeLiveSampleSubject carries one NodeLiveSample per period while watched.
+func NodeLiveSampleSubject(node string) string {
+	return nodeLiveSampleSubjectPrefix + node
+}
+
+// NodeLiveRing answers a watch with the samples newer than its SinceMillis.
+type NodeLiveRing struct {
+	Samples []NodeLiveSample `json:"samples"`
+}
+
+// NodeLiveSample is one host measurement. Rates are per second over the
+// interval ending at TimeMillis.
+type NodeLiveSample struct {
+	TimeMillis int64   `json:"t"`
+	CPUPercent float64 `json:"cpu"`
+	// MemoryUsedPercent is used memory less page cache, as free(1) reports it.
+	MemoryUsedPercent float64 `json:"mem"`
+	Load1             float64 `json:"load1"`
+	// Planes holds the traffic of each plane's physical NICs, keyed wan, lan, vpc.
+	Planes map[string]NodeLivePlane `json:"planes"`
+}
+
+// NodeLivePlane is one plane's traffic from the host's side.
+type NodeLivePlane struct {
+	NetworkIn  float64 `json:"netIn"`
+	NetworkOut float64 `json:"netOut"`
+	PacketsIn  float64 `json:"pktIn"`
+	PacketsOut float64 `json:"pktOut"`
+}

docs/DESIGN.md

@@ -203,12 +203,14 @@ Viperblock responds with an NBD URI that QEMU uses to access the block device.
 
 A direct-boot microVM keeps the launcher's saved `vm.Config` as a clean base, and `startQEMU` adds data volumes only to a launch-only copy of it (`runtime/compute/vm/lifecycle.go`).
 At each start the daemon takes one snapshot of the instance's already-mounted data requests, probes each NBD or host block export from that snapshot, and builds the command from the same snapshot, so the launched endpoint is the one that was checked and is never written back into the saved base.
-Each data volume becomes a `virtio-blk-device` on the MMIO bus with the same id, node name, I/O thread, serial and `werror=stop,rerror=stop` policy as the PCI device, and takes no PCIe hot-plug port.
+The microVM boots with `pcie=on`, so its NICs are `virtio-net-pci` devices with MSI-X interrupts instead of `virtio-net-device` MMIO devices.
+Each data volume becomes a `virtio-blk-pci` device on the default PCIe bus with the same id, node name, I/O thread, serial and `werror=stop,rerror=stop` policy as the q35 device, and takes no PCIe hot-plug port.
+A microVM started without `pcie=on` keeps the `virtio-*-device` MMIO transports.
 
 Boot and EFI requests are rejected on this path, and a direct-boot instance with no data requests launches its base unchanged.
 Disk-boot (q35) instances keep the existing PCI hot-plug path.
 
-Hot-attaching a volume to a running direct-boot microVM is not supported, because the MMIO bus has no PCIe root port to plug into.
+Hot-attaching a volume to a running direct-boot microVM is not supported, because the microVM has no PCIe hot-plug root port to plug into.
 `AttachVolume` refuses it before mounting the volume or changing any instance or volume state, and the EC2 API returns `UnsupportedOperation`.
 
 This covers only requests that are already mounted when the instance starts; nothing yet supplies data volumes to a system instance, and the path is verified with fake-QEMU unit tests, not with a real EBS volume in a guest.

docs/PACKAGE_BOUNDARY_MIGRATION.md

@@ -369,7 +369,7 @@ while the branch is in flight; they do not describe the umbrella repository's
 | `spinifex#1355` | `spinifex/gateway/ec2.go` (`EC2_Request` in production, the legacy service switch, its `supportedServices` and JSON tables) and the `Quota` field of `GatewayConfig` | `spinifex/domains/ec2/awsapi` (`NewRegistration`, `OperationInventory`, `ServiceName`), composed by `spinifex/runtime/roles/awsgw` (`ec2.go`: `newEC2Registration`, `advertisedEndpoint`; `ec2quota`) | Slice 9b of the EC2 move: EC2 registers through the ADR-0002 seam as service `ec2` and the legacy gateway switch is empty. The EC2 query-protocol error envelope is kept through `ErrorEnvelopeXML`, and the policy denial match is now `*dispatch.PolicyDenialError`. The characterisation tests committed first also pin the Authorize service name and action, the trace span and audit action, the throttle key (account plus action) and the cluster-unavailable gate, which runs before policy and action lookup for gateway-local and NATS-backed actions alike. They and the 9a tests pass unchanged apart from setup: STS and EC2 test setups now register the services they sign as, and the two tests that claimed a legacy service name are removed because no legacy name remains. `EC2_Request` survives only as a test shim. Allowlist changes are two deletions (the `gateway` to `domains/ec2/awsapi` and `domains/ec2/awsapi/instance` rows) and one addition, the per-service model-coverage row `cmd/aws-model-coverage` to `domains/ec2/awsapi`. `startInstanceStatus` stays in `awsgw.go` because ADR-0007:S6 records the raw instance-state uses by file. No change to `dispatch.Actor` or `dispatch.Invocation`. Live on env14: DescribeRegions, DescribeAvailabilityZones and DescribeAccountAttributes, an idempotent CreateVolume replay and its parameter mismatch, the DryRun and InvalidVolume.NotFound errors, a no-policy user refused with UnauthorizedOperation (including on the gateway-local DescribeRegions), an assumed-role session allowed and refused per its policy, and one RunInstances to running then terminated. The gateway request logs carry `service=ec2` and the per-action label. Not exercised live: a throttled request (the default limits are not reachable from one client; the throttle key is pinned by the characterisation tests) and an agent-signed EC2 call. |
 | `spinifex#1363` | `spinifex/gateway` context keys (`ctxService`, `ctxRegion`, `ctxAction`, the principal and access key id keys, and the unread `ctxQueryArgs`), the duplicate `callerPrincipal` and `principalContext` types, `gateway/caller_arn.go` (`buildCallerARN`), and the direct `policyauth`, `accountadmission` and NATS-connection reads in the auth pipeline | `spinifex/ingress/aws/reqctx` (the request record), `credverify.CallerARN`, and the gateway-consumed capabilities `PolicyEvaluator`, `AccountAdmission` and `Readiness`, composed by `spinifex/runtime/roles/awsgw` (`clusterready`) | Slice 11a of the gateway removal: the request-context record and the ADR-0002:S8 capabilities, with no behaviour change. The record is a copy-on-write value under one context key; the principal is sealed once and a second `WithPrincipal` keeps the first caller. The layering rule `S7_RequestRecordPrivateToIngress` lets only `gateway` and `ingress/aws/*` import `reqctx`, with a fault-injection test, and slice 11c moves the package to `ingress/aws/internal/reqctx` and drops the rule. The caller ARN is built once in `credverify` (root cases kept) and filled on `Principal.CallerARN`. The policy evaluator, account admission (`credential.ErrAccountNotActive`, with `accountadmission.ErrNotActive` an alias) and cluster readiness (`Available() bool`) are interfaces the gateway consumes and fails closed on when nil. The step 1 characterisation tests (throttle against both readiness gates and the policy gate for ECS and IAM, `/admin` skipping `GetService` and the second gate, the nil-connection difference between the two gates, five error writers each minting a request id that differs from the audit record, and the credential-scope region reaching only the span) pass unchanged apart from setup. No allowlist rows change, and there is no change to `dispatch.Actor` or `dispatch.Invocation`. Debt for slice 11c: the test-only `defaultCapabilities` seam (a package variable set by a test `init` so about 60 gateway test files can build a config from `IAMService` alone), the nil-connection case of the early readiness gate, the remaining `NATSConn` readers (`DiscoverActiveNodes` and the field that `awsgw` and the integration harness set), and the `IAMService`, `STSService`, `Config`, `RootCAs` and `InternalSuffix` fields of `GatewayConfig`, which no production code reads. Live on env14: DescribeRegions, DescribeAvailabilityZones and DescribeAccountAttributes, an idempotent CreateVolume replay and its parameter mismatch, the DryRun and InvalidVolume.NotFound errors, a bad access key refused with InvalidClientTokenId on a Query service and InvalidClientTokenIdException on a JSON service, a no-policy user refused with UnauthorizedOperation, and an assumed-role session allowed and refused per its policy. Not exercised live: a suspended-account refusal, a throttled request and the NATS-down 503, which the step 1 tests pin. |
 | `spinifex#1378` | `spinifex/gateway` `oidc_discovery.go` and `registry.go` (the `/oidc/eks` routes and the `/v2` mount), the `DiscoverActiveNodes` method and its cache, and the `GatewayConfig` fields `Config`, `RootCAs`, `InternalSuffix`, `Version`, `Commit`, `IAMService`, `STSService`, `NATSConn` and `ExpectedNodes` | `spinifex/runtime/roles/awsgw` (`PublicRoutes`, `NodeDiscovery.ActiveNodes`, `composeGateway`); the gateway keeps one `PublicRoutes` mount hook and the `Admin` registration name supplied by the role |
-| `spinifex#TBD` | `spinifex/gateway` (`GatewayConfig`, the SigV4 and policy middleware, `Request`, the admin dispatch, `AWSOperationInventory`) | `spinifex/ingress/aws/pipeline` (`Config`) and `spinifex/ingress/aws/dispatch` (`AWSOperationInventory`); the gateway layering rows are retired and R2 binds the pipeline |
+| `spinifex#1384` | `spinifex/gateway` (`GatewayConfig`, the SigV4 and policy middleware, `Request`, the admin dispatch, `AWSOperationInventory`) | `spinifex/ingress/aws/pipeline` (`Config`) and `spinifex/ingress/aws/dispatch` (`AWSOperationInventory`); the gateway layering rows are retired and R2 binds the pipeline |
 | `spinifex#1334` | `spinifex/gateway/oidc_discovery.go` (`eksAccountKV`, the discovery-document and JWKS read and construction) | `spinifex/domains/eks/cluster` (`OIDCPublisher`, `OIDCDiscoveryDocument`), composed by `spinifex/runtime/roles/awsgw` | The EKS domain now reads its own account bucket and builds the per-cluster discovery document and JWKS responses (ADR-0005:S4), and the gateway only maps the unauthenticated `/oidc/eks/...` routes onto an `OIDCPublisher` interface it is handed. Responses are byte-identical, pinned by `oidc_discovery_current_behaviour_test.go`, whose assertions are unchanged. The gateway no longer imports `domains/eks/cluster` in production code. `STS` keeps its own `ClusterJWKSResolver` (`OIDCJWKSReader`), untouched. Recorded debt, not repaired here: a stored JWKS is served verbatim with 200 even when it is not valid JSON. |
 The EC2 contract row is intentionally different from the directory moves: it creates
 a compatibility boundary. `ec2.cmd.*` retains its deployed one-token NATS
@@ -688,7 +688,7 @@ series above; "needs decision" is open.
 | `handlers/rds` | — | Done. The remaining RDS instance resource moved to `domains/rds/instance` (package `instance`) in `spinifex#1251` with no API, NATS subject, storage or behaviour change, fulfilling ADR-0001's organise-by-resource condition alongside the already-extracted subnet group, parameter group and engine catalogue. The ADR-0004 blocked-dependency imports (quota admission, the duplicated expected-node fan-out, the IAM ensurer and the Bluebottle master key) remain dependency-rule debt, with their allowlist entries now sourced from `domains/rds/instance`. |
 | `handlers/elbv2` | — | Done. The implementation moved to `domains/elbv2` (package `elbv2`) with no API, NATS subject, storage or behaviour change. Q-30's data-plane boundary remains deferred debt; the amended ADR-0001 row makes it not a relocation prerequisite. Its cross-domain, IAM and `agents/elbv2` imports remain dependency-rule debt, with their allowlist entries now sourced from `domains/elbv2`. |
 | `handlers/iam/mock` | — | Done. Moved with the implementation to `domains/iam/mock`. |
-| `gateway` (root) | Generic HTTP route assembly, SigV4 authentication, the policy gate, request-audit middleware and request-context keys. | Done and deleted. The request pipeline moved into `ingress/aws/pipeline` (as `pipeline.Config`) and the operation inventory into `ingress/aws/dispatch` in `spinifex#TBD`; the `/oidc`, `/v2` and node-discovery code had already moved to `runtime/roles/awsgw` in `spinifex#1378`. The layering test binds the pipeline with R2 and `gateway` is a retired legacy root. |
+| `gateway` (root) | Generic HTTP route assembly, SigV4 authentication, the policy gate, request-audit middleware and request-context keys. | Done and deleted. The request pipeline moved into `ingress/aws/pipeline` (as `pipeline.Config`) and the operation inventory into `ingress/aws/dispatch` in `spinifex#1384`; the `/oidc`, `/v2` and node-discovery code had already moved to `runtime/roles/awsgw` in `spinifex#1378`. The layering test binds the pipeline with R2 and `gateway` is a retired legacy root. |
 | `gateway/bedrock` | — | Done. The whole package moved into `domains/ochre` in `spinifex#1266` with its package-private core and tests intact and no behaviour change. Separating the action adapters into `domains/ochre/awsapi` remains deferred, recorded as `domains/ochre` (action adapters) debt above. The action adapters then moved to `domains/ochre/awsapi` behind one ADR-0002 registration in `spinifex#1351`. |
 | `gateway/{iam,sts,eks,rds}` | — | Done. The action adapters moved to `domains/{iam,sts,eks,rds}/awsapi` in `spinifex#1265` with no API, NATS subject, storage or behaviour change. ADR-0001 does not make ADR-0002 registration a prerequisite for relocating a domain's adapters, as the EC2 adapter moves already showed; the root `gateway` dispatch that still imports them is recorded `gateway` (service registration) debt. |
 | `gateway/{ecs,elbv2}` | — | Done. The action adapters moved to `domains/{ecs,elbv2}/awsapi` in `spinifex#1273` with no API, NATS subject, storage or behaviour change; the root `gateway` dispatch that still imports them is recorded `gateway` (service registration) debt. |

docs/install/install-oracle-linux/README.md

@@ -164,7 +164,7 @@ On Oracle Linux the installer makes these changes, which Debian hosts do not nee
 - **Repositories.** It enables `ol10_UEKR8` and `ol10_kvm_utils`, and adds `qemu-kvm*` and `qemu-img*` to the `excludepkgs` of `ol10_appstream`, keeping any excludes already set, so a later AppStream build can never replace the KVM Utilities QEMU.
 - **NFV SIG.** It writes `/etc/yum.repos.d/spinifex-nfv-sig.repo` with an `includepkgs=` list of the OVS and OVN packages, and imports the SIG key to `/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-NFV` only when its fingerprint is `35154228174901BEFA8E69A621465E289D2A76A7`.
 - **OVS and OVN user.** Both run as root, as on Debian, through `OVS_USER_ID` and `OVN_USER_ID` in `/etc/sysconfig/openvswitch` and `/etc/sysconfig/ovn`.
-- **DNS.** Where systemd-resolved is installed but not running, it saves `/etc/resolv.conf` to `/var/lib/spinifex-reset/resolv.conf.orig`, starts resolved behind NetworkManager and links `/etc/resolv.conf` to the resolved stub. It then checks that names still resolve and stops, printing the commands that put DNS back, if they do not.
+- **DNS.** Where systemd-resolved is installed but not running, it saves `/etc/resolv.conf` to `/var/lib/spinifex-reset/resolv.conf.orig`, starts resolved behind NetworkManager, tells NetworkManager in `/etc/NetworkManager/conf.d/90-spinifex-dns.conf` to leave `/etc/resolv.conf` alone, and links `/etc/resolv.conf` to the resolved stub. It then checks that names still resolve and stops, printing the commands that put DNS back, if they do not.
 - **chrony.** It appends `confdir /etc/chrony.d` to `/etc/chrony.conf` once, and writes its step policy to `/etc/chrony.d/spinifex.conf`.
 - **`spx` under `sudo`.** Oracle Linux's `sudo` path leaves out `/usr/local/bin`, so the installer links `/usr/bin/spx` to `/usr/local/bin/spx`.
 

docs/security/network-connections/README.md

@@ -99,6 +99,7 @@ behavior in other words.
 | 4432 | Formation server | HTTPS | Cluster (bootstrap only) | Cluster join coordination; active only while a join token is valid. Binds the node's `--bind` (lan) address. See *Formation port lifecycle* below. | Short-lived bearer token + TLS¹ |
 | 4222 | spinifex-nats (client) | NATS + TLS | Cluster | Internal service bus for EC2/EBS/VPC/S3 handlers | Token + mutual TLS (cluster CA) |
 | 4248 | spinifex-nats (cluster) | NATS + TLS | Cluster | Inter-node NATS federation | Token + mutual TLS (cluster CA) |
+| 5001 | iperf2 (admin network test) | TCP | Cluster, any plane | Throughput test between two nodes, started only by the super-admin account. Listens on one plane address for the length of one test, then exits. Runs with no capabilities. | None — peer-scoped by firewall, and the client may only target its own plane subnet |
 | 5300 | northstar | DNS (UDP + TCP) | Cluster | Forward target for every node's per-instance DNS shim, dialled cross-node. Binds the wildcard by design. | None — restrict by firewall |
 | 6660 | predastore (blob node) | QUIC / UDP | Cluster | Erasure-coded object shard transport between hosts. Multi-node clusters only — see *Predastore ports* below. | TLS 1.3, server certificate verified against the cluster CA |
 | 7660 | predastore (meta node) | QUIC / UDP | Cluster | Raft consensus over global state — buckets and the object index — between hosts. Multi-node clusters only. | TLS 1.3, server certificate verified against the cluster CA |
@@ -221,7 +222,7 @@ table inet spinifex_filter {
 
     # Cluster, from peer nodes only. Replace with your nodes' lan-plane
     # addresses — a CIDR does not generalise to nodes on different subnets.
-    ip saddr { 10.0.1.1, 10.0.1.2, 10.0.1.3 } tcp dport { 4222, 4248, 4432, 5300, 6641, 6642, 6643, 6644, 8660 } accept
+    ip saddr { 10.0.1.1, 10.0.1.2, 10.0.1.3 } tcp dport { 4222, 4248, 4432, 5001, 5300, 6641, 6642, 6643, 6644, 8660 } accept
     ip saddr { 10.0.1.1, 10.0.1.2, 10.0.1.3 } udp dport { 5300, 6660, 7660 } accept
 
     # Encap, from peer chassis only. Replace with your nodes' vpc-plane

docs/testing/e2e-coverage.md

@@ -123,6 +123,10 @@ Internal and internet-facing ALB and NLB, HTTPS listeners, listener rules, `Modi
 Unhealthy-target routing: an NLB with an HTTP health check on a TCP target group stops sending traffic to a target that fails it while still accepting TCP, sends traffic again once it recovers, and fails open when every target is unhealthy.
 An ALB whose targets all fail their health check keeps serving from them instead of answering 503.
 
+`TestInternalALBPrivateSubnet` launches an internal ALB in a private subnet of a non-default VPC with no internet gateway and no public route.
+The load balancer must reach active with a healthy target, which needs the owner stamp on its ENI to resolve, and a client inside the VPC must be served by it on every request.
+The client reports on its serial console because the runner has no route into the VPC.
+
 ### `eks`, `ecs`, `ecr` — container services
 
 `eks`: cluster create/delete, access entries, addon delivery, `GetToken`, kubeconfig artifacts, the EBS CSI volume path, an IRSA web-identity exchange under the stock AWS trust policy (`StringEquals` on the issuer host-path `:sub`/`:aud`, a mismatched `:sub` denied with `AccessDenied`, a `StringLike` `:sub` granted), and an IRSA pod wired with a projected SA token. Worker IMDS is covered too: a managed nodegroup worker reports IMDSv2 required with hop limit 2, a pod with no IRSA resolves to the node role through IMDS, and a nodegroup built from a launch template takes the template's metadata options. Control-plane isolation is covered as well: no control-plane host registers as a Node, a DaemonSet tolerating every taint runs on workers only, metrics-server is Ready, and logs work for every kube-system pod. `ecs`: clusters, container instances, the three network modes (`awsvpc`, `bridge`, `host`), task-role credentials fetched from the container credentials endpoint, a service behind an ELB, and ENI release on terminate. `ecr`: control-plane CRUD plus a real OCI data plane — docker login/build/push/pull round trips, multi-layer images, and raw blob HTTP.

internal/testkit/kvfault.go

@@ -0,0 +1,108 @@
+package testutil
+
+import (
+	"context"
+	"sync"
+
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// StaleReadKV wraps a bucket so that the first Get of a key after a write to it
+// returns the entry as it stood before that write, as a lagging replica does
+// when it answers a direct get. Later Gets see the current entry.
+type StaleReadKV struct {
+	jetstream.KeyValue
+
+	mu    sync.Mutex
+	stale map[string]jetstream.KeyValueEntry
+}
+
+var _ jetstream.KeyValue = (*StaleReadKV)(nil)
+
+// NewStaleReadKV wraps kv.
+func NewStaleReadKV(kv jetstream.KeyValue) *StaleReadKV {
+	return &StaleReadKV{KeyValue: kv, stale: map[string]jetstream.KeyValueEntry{}}
+}
+
+// remember captures the entry about to be overwritten.
+func (k *StaleReadKV) remember(ctx context.Context, key string) jetstream.KeyValueEntry {
+	prev, err := k.KeyValue.Get(ctx, key)
+	if err != nil {
+		return nil
+	}
+	return prev
+}
+
+func (k *StaleReadKV) arm(key string, prev jetstream.KeyValueEntry) {
+	if prev == nil {
+		return
+	}
+	k.mu.Lock()
+	k.stale[key] = prev
+	k.mu.Unlock()
+}
+
+func (k *StaleReadKV) Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error) {
+	k.mu.Lock()
+	prev, ok := k.stale[key]
+	delete(k.stale, key)
+	k.mu.Unlock()
+	if ok {
+		return prev, nil
+	}
+	return k.KeyValue.Get(ctx, key)
+}
+
+func (k *StaleReadKV) Update(ctx context.Context, key string, value []byte, last uint64) (uint64, error) {
+	prev := k.remember(ctx, key)
+	rev, err := k.KeyValue.Update(ctx, key, value, last)
+	if err == nil {
+		k.arm(key, prev)
+	}
+	return rev, err
+}
+
+func (k *StaleReadKV) Put(ctx context.Context, key string, value []byte) (uint64, error) {
+	prev := k.remember(ctx, key)
+	rev, err := k.KeyValue.Put(ctx, key, value)
+	if err == nil {
+		k.arm(key, prev)
+	}
+	return rev, err
+}
+
+// RejectingKV wraps a bucket so that writes for which reject returns an error
+// fail with it, and every other operation passes through.
+type RejectingKV struct {
+	jetstream.KeyValue
+
+	reject func(key string, value []byte) error
+}
+
+var _ jetstream.KeyValue = (*RejectingKV)(nil)
+
+// NewRejectingKV wraps kv. reject runs before each Put, Create and Update.
+func NewRejectingKV(kv jetstream.KeyValue, reject func(key string, value []byte) error) *RejectingKV {
+	return &RejectingKV{KeyValue: kv, reject: reject}
+}
+
+func (k *RejectingKV) Put(ctx context.Context, key string, value []byte) (uint64, error) {
+	if err := k.reject(key, value); err != nil {
+		return 0, err
+	}
+	return k.KeyValue.Put(ctx, key, value)
+}
+
+func (k *RejectingKV) Create(ctx context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error) {
+	if err := k.reject(key, value); err != nil {
+		return 0, err
+	}
+	return k.KeyValue.Create(ctx, key, value, opts...)
+}
+
+func (k *RejectingKV) Update(ctx context.Context, key string, value []byte, last uint64) (uint64, error) {
+	if err := k.reject(key, value); err != nil {
+		return 0, err
+	}
+	return k.KeyValue.Update(ctx, key, value, last)
+}

scripts/README.md

@@ -6,7 +6,7 @@ Node installation, dev-environment lifecycle, guest image builds, and verificati
 
 | Script | What it does |
 |---|---|
-| `setup.sh` | The production binary installer, served from `install.mulgadc.com`. Installs deps, users, sudoers, the host firewall policy, files, systemd units, logrotate and udev rules, and loads `openvswitch`, `geneve`, `vhost_net` and `tun`, listing them in `/etc/modules-load.d/spinifex.conf` for later boots. It masks firewalld wherever it is installed, since the Spinifex nftables policy is the only supported node firewall. It writes a needrestart drop-in so an apt upgrade never restarts the Spinifex, OVS or OVN units; restarting them after an upgrade is left to the operator. An unset `--firewall` keeps the mode recorded in `/etc/spinifex/firewall/mode`, and otherwise arms the policy when firewalld was active. Where systemd-resolved is installed but not running, resolvconf is absent and NetworkManager runs, it hands host DNS to resolved: it saves `/etc/resolv.conf` once to `/var/lib/spinifex-reset/resolv.conf.orig`, links `/etc/resolv.conf` to the resolved stub, and stops unless resolved has upstream servers and the package repository host still resolves. It links `/usr/bin/spx` to `/usr/local/bin/spx` where sudo's PATH leaves out `/usr/local/bin`, and takes the nbdkit plugin directory from `nbdkit --dump-config`, stopping when nbdkit cannot say. `SETUP_STAGES` runs a subset; `ISO_BUILD=1` runs it inside the ISO builder's chroot. |
+| `setup.sh` | The production binary installer, served from `install.mulgadc.com`. Installs deps, users, sudoers, the host firewall policy, files, systemd units, logrotate and udev rules, and loads `openvswitch`, `geneve`, `vhost_net` and `tun`, listing them in `/etc/modules-load.d/spinifex.conf` for later boots. It masks firewalld wherever it is installed, since the Spinifex nftables policy is the only supported node firewall. It writes a needrestart drop-in so an apt upgrade never restarts the Spinifex, OVS or OVN units; restarting them after an upgrade is left to the operator. An unset `--firewall` keeps the mode recorded in `/etc/spinifex/firewall/mode`, and otherwise arms the policy when firewalld was active. Where systemd-resolved is installed but not running, resolvconf is absent and NetworkManager runs, it hands host DNS to resolved: it saves `/etc/resolv.conf` once to `/var/lib/spinifex-reset/resolv.conf.orig`, writes `/etc/NetworkManager/conf.d/90-spinifex-dns.conf` (`dns=systemd-resolved`, `rc-manager=unmanaged`) so NetworkManager never rewrites `/etc/resolv.conf`, links `/etc/resolv.conf` to the resolved stub, and stops unless the link holds, resolved has upstream servers and the package repository host still resolves. A rerun after that handover adds a missing drop-in, reloads NetworkManager and relinks the stub. It links `/usr/bin/spx` to `/usr/local/bin/spx` where sudo's PATH leaves out `/usr/local/bin`, and takes the nbdkit plugin directory from `nbdkit --dump-config`, stopping when nbdkit cannot say. `SETUP_STAGES` runs a subset; `ISO_BUILD=1` runs it inside the ISO builder's chroot. |
 | `setup-ovn.sh` | Bootstraps a compute node for OVN VPC networking: installs OVN/OVS, creates `br-int`, configures the WAN bridge (bridged, DHCP, or routed-NAT uplink), sets the chassis identity, and applies overlay sysctls. It runs OVS and OVN as root, setting `OVS_USER_ID` and `OVN_USER_ID` where `/etc/sysconfig` holds them, and hooks the socket-permission helper onto `ovs-vswitchd`, `ovn-controller`, `openvswitch-ipsec` and, on EL, `ovn-northd`. `--management` also starts the OVN central services, writes their `ovn-ctl` flags to `/etc/default/ovn-central` (Debian) or `/etc/sysconfig/ovn-northd` (EL), and records the role in `/etc/spinifex/ovn-role`. With clustered databases it returns only once this node's NB and SB members are back in their RAFT clusters under a known leader, and fails after `OVN_RAFT_REJOIN_WAIT` seconds (default 60) otherwise, so reruns one node at a time never take two members of a cluster down. A later run without `--management` refuses while that file, or the `OVN_CTL_OPTS` line older nodes carry in `/etc/default/ovn-central`, records the role. It adds `maxsize 100M` to the OVS and OVN logrotate files, plus `rotate 5` where the file sets no count, and rotates as root, matching the daemons. On a host whose IPsec stack is libreswan it enables, stops and masks `openvswitch-ipsec.service` and `ipsec.service` for the daemon to unmask when IPsec is on, unless `openvswitch-ipsec.service` is already running; elsewhere it enables and starts `openvswitch-ipsec.service`, unless the daemon has masked it to keep IPsec off. The links it creates persist through systemd-networkd where that is enabled, and otherwise through `/etc/spinifex/hostnet.json`, which `spinifex-hostnet.service` (`spx admin hostnet apply`) replays at boot and on every OVS start, starting `ovn-controller` again when an OVS stop took it down; the first run records the choice in `/etc/spinifex/hostnet-backend`. Where NetworkManager runs, it leaves the links Spinifex owns unmanaged. |
 | `install-node.sh` | Forms a multi-node cluster across hosts that are already running Spinifex: rebuilds the OVN database in clustered form, runs `spx admin init` on the first host and `spx admin join` on the rest, then verifies. Before touching any host it refuses hosts of different OS families (`distro_os_family`) or `spx` versions, and `--ipsec on` when any host uses libreswan. The scripted form of [install-multi-node](../docs/install/install-multi-node/README.md). Destroys the joining nodes' logical network state and crypto — see its `--help`. |
 | `dev-install.sh` | Full local dev setup through the production installer: builds from source, packs a tarball, runs `setup.sh`, initialises the cluster, and starts services under systemd. |

scripts/images/eks-node/eks-node-role.sh

@@ -12,11 +12,12 @@ set -eu
 # once.
 #
 #   server      → eks-token-webhook, k3s (server), k3s-first-boot (bootstrap
-#                 publish), mulga-eks-state-report
+#                 publish), mulga-eks-state-report, mulga-eks-node-gc
 #   server-join → eks-token-webhook, k3s (server, joins the first server's etcd
-#                 quorum), mulga-eks-state-report. NO k3s-first-boot: the first
-#                 server already publishes the cluster-identical bootstrap
-#                 artifacts; a join re-publish only races the bootstrap bus.
+#                 quorum), mulga-eks-state-report, mulga-eks-node-gc. NO
+#                 k3s-first-boot: the first server already publishes the
+#                 cluster-identical bootstrap artifacts; a join re-publish only
+#                 races the bootstrap bus.
 #   agent       → k3s-agent
 #
 # Paths are overridable via env so the selection logic is unit-testable (bats)
@@ -140,6 +141,7 @@ case "${ROLE}" in
         # writer owns. HA multi-server addon delivery is tracked separately
         # Server-join nodes do not run this setup.
         svc_enable mulga-eks-addon-sync
+        svc_enable mulga-eks-node-gc
         svc_start eks-token-webhook
         svc_start mulga-eks-k3s-recovery
         svc_start k3s
@@ -147,6 +149,7 @@ case "${ROLE}" in
         svc_start k3s-first-boot
         svc_start mulga-eks-state-report
         svc_start mulga-eks-addon-sync
+        svc_start mulga-eks-node-gc
         ;;
     server-join)
         log "configuring server-join role"
@@ -158,11 +161,13 @@ case "${ROLE}" in
         svc_enable k3s
         svc_enable konnectivity-server
         svc_enable mulga-eks-state-report
+        svc_enable mulga-eks-node-gc
         svc_start eks-token-webhook
         svc_start mulga-eks-k3s-recovery
         svc_start k3s
         svc_start konnectivity-server
         svc_start mulga-eks-state-report
+        svc_start mulga-eks-node-gc
         ;;
     agent)
         log "configuring agent role"

scripts/images/eks-node/eks-node-role_test.sh

@@ -141,6 +141,8 @@ for INIT in openrc systemd; do
             END { print (rn > 0 && kn > 0 && rn < kn) ? "yes" : "no" }
         ' "${CALLS}"
     )"
+    check "server(${INIT}): enables node-gc" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-node-gc)")"
+    check "server(${INIT}): starts node-gc" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-node-gc)")"
     check "server(${INIT}): no k3s-agent" "no" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
     check "server(${INIT}): self-disables" "yes" "$(present "${CALLS}" "$(disable_cmd "${INIT}" eks-node-role)")"
 
@@ -153,6 +155,8 @@ for INIT in openrc systemd; do
     check "server-join(${INIT}): enables webhook" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" eks-token-webhook)")"
     check "server-join(${INIT}): enables k3s" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s)")"
     check "server-join(${INIT}): enables state-report" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-state-report)")"
+    check "server-join(${INIT}): enables node-gc" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-node-gc)")"
+    check "server-join(${INIT}): starts node-gc" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-node-gc)")"
     check "server-join(${INIT}): enables konnectivity-server" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" konnectivity-server)")"
     check "server-join(${INIT}): enables k3s-recovery" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-k3s-recovery)")"
     check "server-join(${INIT}): starts k3s-recovery" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-k3s-recovery)")"
@@ -170,7 +174,7 @@ for INIT in openrc systemd; do
     check "agent(${INIT}): exit 0" 0 "${rc}"
     check "agent(${INIT}): role file" "agent" "$(cat "${ROLE_FILE}")"
     check "agent(${INIT}): enables k3s-agent" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
-    check "agent(${INIT}): no server services" "yes" "$(absent_all "${CALLS}" "eks-token-webhook" "k3s-first-boot" "$(enable_cmd "${INIT}" k3s)")"
+    check "agent(${INIT}): no server services" "yes" "$(absent_all "${CALLS}" "eks-token-webhook" "k3s-first-boot" "mulga-eks-node-gc" "$(enable_cmd "${INIT}" k3s)")"
 
     # --- agent (inferred from agent.env, no explicit role) ---
     run_case "${INIT}" agent_inferred

scripts/images/eks-node/manifest.conf

@@ -42,6 +42,8 @@ scripts/images/eks-node/mulga-eks-state-report.initd:/etc/init.d/mulga-eks-state
 scripts/images/eks-node/mulga-eks-state-report.sh:/usr/local/sbin/mulga-eks-state-report \
 scripts/images/eks-node/mulga-eks-addon-sync.initd:/etc/init.d/mulga-eks-addon-sync \
 scripts/images/eks-node/mulga-eks-addon-sync.sh:/usr/local/sbin/mulga-eks-addon-sync \
+scripts/images/eks-node/mulga-eks-node-gc.initd:/etc/init.d/mulga-eks-node-gc \
+scripts/images/eks-node/mulga-eks-node-gc.sh:/usr/local/sbin/mulga-eks-node-gc \
 scripts/images/eks-node/konnectivity-server.initd:/etc/init.d/konnectivity-server \
 scripts/images/eks-node/konnectivity-server-prestart.sh:/usr/local/sbin/konnectivity-server-prestart \
 scripts/images/eks-node/konnectivity/konnectivity-agent.yaml:/usr/share/spinifex-eks/konnectivity/konnectivity-agent.yaml \

scripts/images/eks-node/mulga-eks-node-gc.initd

@@ -0,0 +1,27 @@
+#!/sbin/openrc-run
+# mulga-eks-node-gc — server-role service that deletes the Node of every
+# nodegroup worker the nodegroup has terminated, every NODE_GC_INTERVAL
+# seconds. Enabled on every server role by eks-node-role.sh.
+
+description="Mulga EKS terminated-worker Node cleanup"
+command="/usr/local/sbin/mulga-eks-node-gc"
+command_background="yes"
+pidfile="/run/mulga-eks-node-gc.pid"
+output_log="/var/log/mulga-eks-node-gc.log"
+error_log="/var/log/mulga-eks-node-gc.log"
+
+# A Node goes NotReady about 40s after its worker stops, so 30s deletes it
+# within a couple of minutes of the terminate.
+export NODE_GC_INTERVAL="${NODE_GC_INTERVAL:-30}"
+
+depend() {
+    need net
+    after k3s
+}
+
+start_pre() {
+    if [ ! -f /etc/spinifex-eks/first-boot.env ]; then
+        eerror "/etc/spinifex-eks/first-boot.env not present — cloud-init did not seed gateway creds"
+        return 1
+    fi
+}

scripts/images/eks-node/mulga-eks-node-gc.sh

@@ -0,0 +1,89 @@
+#!/bin/sh
+set -eu
+
+# mulga-eks-node-gc — deletes the Node of a nodegroup worker whose instance the
+# nodegroup has terminated (DeleteNodegroup, scale-down, reclaim), as the AWS
+# cloud controller deletes a Node whose instance no longer exists. K3s runs no
+# AWS cloud provider: its own reports every instance as existing, so without
+# this a terminated worker stays a NotReady Node forever, its pods stay bound to
+# it, and once most Nodes are NotReady the node lifecycle controller stops
+# evicting at all.
+#
+# Pull (host→VM): GET /clusters/{cluster}/internal-workers/{accountId}, the
+# instance ID of every worker the cluster's nodegroups still own.
+#
+# A Node is deleted only when all of these hold, so a live worker is never
+# touched: it carries the eks.amazonaws.com/nodegroup label, its providerID is
+# aws:///<az>/<instance-id>, it is not Ready, and its instance is not in the
+# fetched set. A failed fetch deletes nothing.
+#
+# Runs on every server; deletes are idempotent. Loops every NODE_GC_INTERVAL
+# seconds in service mode, one-shot otherwise.
+
+ENVFILE=${ENVFILE:-/etc/spinifex-eks/first-boot.env}
+[ -f "${ENVFILE}" ] || { logger -t mulga-eks-node-gc "${ENVFILE} missing — exiting"; exit 0; }
+# set -a so the sourced creds are exported to the eks-gateway-fetch child.
+set -a
+# shellcheck disable=SC1090
+. "${ENVFILE}"
+set +a
+
+: "${EKS_GATEWAY_URL:?}"
+: "${EKS_ACCOUNT_ID:?}"
+: "${EKS_CLUSTER_NAME:?}"
+
+KUBECONFIG=${KUBECONFIG:-/etc/rancher/k3s/k3s.yaml}
+export KUBECONFIG
+
+# One "name|providerID|nodegroup|Ready" row per Node.
+NODE_ROWS='jsonpath={range .items[*]}{.metadata.name}{"|"}{.spec.providerID}{"|"}{.metadata.labels.eks\.amazonaws\.com/nodegroup}{"|"}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}'
+
+log() {
+    logger -t mulga-eks-node-gc "$*"
+    echo "[mulga-eks-node-gc] $*" > "${CONSOLE:-/dev/console}" 2>/dev/null || true
+}
+
+# stale_nodes LIVE_FILE — reads Node rows on stdin and prints the name of each
+# Node to delete: a nodegroup worker, not Ready, whose instance is not listed
+# in LIVE_FILE.
+stale_nodes() {
+    awk -F'|' -v live="$1" '
+        BEGIN { while ((getline id < live) > 0) keep[id] = 1 }
+        $3 == "" || $4 == "True" { next }
+        $2 !~ /^aws:\/\/\/[^\/]+\/i-[0-9a-f]+$/ { next }
+        { n = split($2, p, "/"); if (!(p[n] in keep)) print $1 }
+    '
+}
+
+sync_once() {
+    # Nodes are listed before the workers are fetched. A worker is recorded on
+    # its nodegroup before it boots, so every Node listed here is already in a
+    # set fetched after.
+    if ! rows=$(kubectl get nodes -o "${NODE_ROWS}" 2>/dev/null); then
+        log "kubectl get nodes failed; skipping this pass"
+        return 0
+    fi
+    live=$(mktemp)
+    if ! eks-gateway-fetch -resource workers > "${live}" 2>/dev/null; then
+        log "fetch of the cluster's nodegroup workers failed; deleting nothing"
+        rm -f "${live}"
+        return 0
+    fi
+    printf '%s\n' "${rows}" | stale_nodes "${live}" | while read -r node; do
+        if kubectl delete node "${node}" --ignore-not-found --wait=false >/dev/null 2>&1; then
+            log "deleted Node ${node}: its instance is no longer a nodegroup worker"
+        else
+            log "delete of Node ${node} failed; retrying next pass"
+        fi
+    done
+    rm -f "${live}"
+}
+
+if [ -n "${NODE_GC_INTERVAL:-}" ] && [ "${NODE_GC_INTERVAL}" -gt 0 ] 2>/dev/null; then
+    while true; do
+        sync_once
+        sleep "${NODE_GC_INTERVAL}"
+    done
+else
+    sync_once
+fi

scripts/images/eks-node/mulga-eks-node-gc_test.sh

@@ -0,0 +1,107 @@
+#!/bin/sh
+# Self-contained POSIX test for mulga-eks-node-gc.sh. kubectl, eks-gateway-fetch
+# and logger are stubbed on PATH: kubectl serves canned Node rows and records
+# every delete, and eks-gateway-fetch serves a canned worker set or fails.
+#
+# Run: sh scripts/images/eks-node/mulga-eks-node-gc_test.sh
+set -eu
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+AGENT="${SCRIPT_DIR}/mulga-eks-node-gc.sh"
+WORK=$(mktemp -d)
+trap 'rm -rf "${WORK}"' EXIT
+
+STUBBIN="${WORK}/bin"
+mkdir -p "${STUBBIN}"
+
+cat > "${STUBBIN}/kubectl" <<'EOF'
+#!/bin/sh
+case "$1 $2" in
+    "get nodes")
+        [ "${NODES_RC:-0}" -eq 0 ] || exit "${NODES_RC}"
+        printf '%s\n' "${NODES_BODY:-}" ;;
+    "delete node") echo "$3" >> "${DELETED}" ;;
+esac
+EOF
+
+cat > "${STUBBIN}/eks-gateway-fetch" <<'EOF'
+#!/bin/sh
+echo "$*" >> "${FETCHED}"
+[ "${FETCH_RC:-0}" -eq 0 ] || exit "${FETCH_RC}"
+printf '%s' "${LIVE_BODY:-}"
+EOF
+
+cat > "${STUBBIN}/logger" <<'EOF'
+#!/bin/sh
+:
+EOF
+
+chmod +x "${STUBBIN}"/*
+PATH="${STUBBIN}:${PATH}"
+export PATH
+
+ENVFILE="${WORK}/first-boot.env"
+cat > "${ENVFILE}" <<'EOF'
+EKS_GATEWAY_URL=https://gw.invalid:9999
+EKS_ACCOUNT_ID=000000000001
+EKS_CLUSTER_NAME=demo3
+EOF
+CONSOLE="${WORK}/console"
+DELETED="${WORK}/deleted"
+FETCHED="${WORK}/fetched"
+export ENVFILE CONSOLE DELETED FETCHED
+
+PASS=0
+FAIL=0
+check() {
+    if [ "$2" = "$3" ]; then
+        PASS=$((PASS + 1))
+    else
+        FAIL=$((FAIL + 1))
+        printf 'FAIL: %s\n  want: %s\n  got:  %s\n' "$1" "$2" "$3"
+    fi
+}
+
+run_agent() {
+    : > "${DELETED}"
+    : > "${FETCHED}"
+    sh "${AGENT}"
+}
+
+deleted() { tr '\n' ' ' < "${DELETED}" | sed 's/ $//'; }
+
+# Every way a Node can look: only a NotReady nodegroup worker whose instance the
+# cluster's nodegroups no longer own is deleted.
+NODES_BODY='live|aws:///z1/i-0aaa|ng1|True
+gone|aws:///z1/i-0bbb|ng1|Unknown
+gone-never-reported|aws:///z1/i-0ddd|ng2|
+owned-notready|aws:///z1/i-0ccc|ng1|False
+gone-but-ready|aws:///z1/i-0eee|ng1|True
+self-managed|aws:///z1/i-0fff||Unknown
+k3s-provider|k3s://k3s-node|ng1|Unknown'
+LIVE_BODY='i-0aaa
+i-0ccc
+'
+export NODES_BODY LIVE_BODY
+
+run_agent
+check "deletes only NotReady nodegroup Nodes whose instance is gone" "gone gone-never-reported" "$(deleted)"
+check "fetches the workers resource" "-resource workers" "$(cat "${FETCHED}")"
+
+# A cluster with no nodegroups left: every NotReady nodegroup Node goes.
+LIVE_BODY='' run_agent
+check "empty worker set deletes every NotReady nodegroup Node" "gone gone-never-reported owned-notready" "$(deleted)"
+
+# A failed fetch is not an empty set: nothing is deleted.
+FETCH_RC=1 run_agent
+check "failed fetch deletes nothing" "" "$(deleted)"
+
+# Without a Node listing the workers are not fetched, so a set fetched before a
+# Node registered can never be compared against it.
+NODES_RC=1 run_agent
+check "failed node listing deletes nothing" "" "$(deleted)"
+check "failed node listing does not fetch" "" "$(cat "${FETCHED}")"
+
+echo "---"
+echo "PASS=${PASS} FAIL=${FAIL}"
+[ "${FAIL}" -eq 0 ]

scripts/images/eks-node/setup.sh

@@ -44,9 +44,10 @@ ln -sf /usr/local/bin/k3s /usr/local/bin/ctr
 # role's services are baked; the selector enables the right ones at first boot.
 chmod 0755 /etc/init.d/eks-node-role /etc/init.d/k3s /etc/init.d/k3s-agent \
     /etc/init.d/eks-token-webhook /etc/init.d/k3s-first-boot /etc/init.d/mulga-eks-state-report \
-    /etc/init.d/mulga-eks-addon-sync /etc/init.d/konnectivity-server
+    /etc/init.d/mulga-eks-addon-sync /etc/init.d/mulga-eks-node-gc /etc/init.d/konnectivity-server
 chmod 0755 /usr/local/sbin/eks-node-role /usr/local/sbin/k3s-first-boot \
-    /usr/local/sbin/mulga-eks-state-report /usr/local/sbin/mulga-eks-addon-sync
+    /usr/local/sbin/mulga-eks-state-report /usr/local/sbin/mulga-eks-addon-sync \
+    /usr/local/sbin/mulga-eks-node-gc
 # start_pre helpers shared with the systemd wrappers that will eventually
 # replace these runscripts.
 chmod 0755 /usr/local/sbin/k3s-prestart /usr/local/sbin/k3s-agent-prestart \

scripts/setup.sh

@@ -732,7 +732,8 @@ table inet spinifex_filter {
         # inventory and running instances to anyone who asks. 8660 is
         # predastore's unauthenticated /healthz and /readyz, which name the
         # peers a gate cannot reach, so it belongs here for the same reason.
-        ip saddr $spinifex_peers tcp dport { 4222, 4248, 4432, 5300, 6641, 6642, 6643, 6644, 8660 } accept
+        # 5001 is the admin network test, listening only while one runs.
+        ip saddr $spinifex_peers tcp dport { 4222, 4248, 4432, 5001, 5300, 6641, 6642, 6643, 6644, 8660 } accept
         ip saddr $spinifex_peers udp dport { 5300, 6660, 7660 } accept
 
         # Encap plane, peer-scoped: Geneve, IKE, NAT-T and ESP. Geneve is a
@@ -1014,6 +1015,9 @@ RESOLV_CONF="${SPINIFEX_RESOLV_CONF:-/etc/resolv.conf}"
 RESOLVED_STUB=/run/systemd/resolve/stub-resolv.conf
 # node-reset.sh keeps /var/lib/spinifex-reset, so the copy survives until uninstall.
 RESOLV_CONF_SAVED="${SPINIFEX_RESOLV_CONF_SAVED:-/var/lib/spinifex-reset/resolv.conf.orig}"
+# NetworkManager started while resolv.conf is a plain file rewrites it on its
+# next DNS update, replacing the stub link; this drop-in stops that.
+NM_DNS_DROPIN="${SPINIFEX_NM_DNS_DROPIN:-/etc/NetworkManager/conf.d/90-spinifex-dns.conf}"
 
 # systemd-resolved ships resolvconf as a link to resolvectl, which only works
 # while resolved runs, so that one is not a resolvconf backend.
@@ -1066,10 +1070,52 @@ dns_handover_failed() {
     fatal "Host DNS through systemd-resolved does not work: $1.
     To put host DNS back as it was before this installer:
       sudo systemctl disable --now systemd-resolved
+      sudo rm -f $NM_DNS_DROPIN
       sudo cp -a --remove-destination $RESOLV_CONF_SAVED $RESOLV_CONF
       sudo systemctl restart NetworkManager"
 }
 
+# Prints the rc-manager NetworkManager would use if it is not unmanaged, which
+# means a file read after the Spinifex drop-in overrides it.
+nm_rc_manager_override() {
+    local config
+    config=$($SUDO NetworkManager --print-config) || return 1
+    printf '%s\n' "$config" | awk -F= '
+        /^\[/ { main = ($0 == "[main]"); next }
+        main && $1 == "rc-manager" { rc = $2 }
+        END { if (rc != "unmanaged") print "rc-manager=" (rc == "" ? "symlink" : rc) }'
+}
+
+# Writes the NetworkManager drop-in and checks nothing overrides it. Returns 1
+# when it was already in place.
+install_nm_dns_dropin() {
+    local want override changed=1
+    want=$(printf '[main]\ndns=systemd-resolved\nrc-manager=unmanaged')
+    if [ "$(cat "$NM_DNS_DROPIN" 2>/dev/null)" != "$want" ]; then
+        $SUDO mkdir -p "$(dirname "$NM_DNS_DROPIN")"
+        printf '%s\n' "$want" | $SUDO tee "$NM_DNS_DROPIN" > /dev/null || dns_handover_failed "cannot write $NM_DNS_DROPIN"
+        info "  $NM_DNS_DROPIN: NetworkManager leaves $RESOLV_CONF alone"
+        changed=0
+    fi
+    override=$(nm_rc_manager_override) || dns_handover_failed "cannot read NetworkManager's configuration with 'NetworkManager --print-config'"
+    if [ -n "$override" ]; then
+        dns_handover_failed "NetworkManager reads $override from a file after $NM_DNS_DROPIN, so it would replace the $RESOLV_CONF link. Remove that setting ('NetworkManager --print-config' shows the files it reads) and re-run this installer"
+    fi
+    return "$changed"
+}
+
+# Links the stub and checks the link is still there a second later, relinking a
+# bounded number of times in case a write NetworkManager had queued lands first.
+link_resolved_stub() {
+    local i
+    for i in 1 2 3 4 5; do
+        $SUDO ln -sfn "$RESOLVED_STUB" "$RESOLV_CONF"
+        sleep 1
+        [ "$(readlink "$RESOLV_CONF")" = "$RESOLVED_STUB" ] && return 0
+    done
+    dns_handover_failed "$RESOLV_CONF is not a link to $RESOLVED_STUB 5 seconds after linking it, so something keeps rewriting it"
+}
+
 # Checks resolved has upstream servers and still resolves the package
 # repository host, the one name this host is known to have resolved.
 check_resolved_dns() {
@@ -1103,8 +1149,13 @@ hand_dns_to_resolved() {
     [ "${ISO_BUILD:-0}" = "1" ] && return 0
 
     if systemctl is-active --quiet systemd-resolved.service; then
-        # A handover made by an earlier run is checked again on every run.
+        # A handover made by an earlier run is checked again, and repaired, on
+        # every run: installs before the drop-in could lose the link.
         if [ -e "$RESOLV_CONF_SAVED" ] || [ -L "$RESOLV_CONF_SAVED" ]; then
+            if install_nm_dns_dropin && systemctl is-active --quiet NetworkManager.service; then
+                $SUDO systemctl reload NetworkManager.service || dns_handover_failed "NetworkManager did not reload"
+            fi
+            [ "$(readlink "$RESOLV_CONF")" = "$RESOLVED_STUB" ] || link_resolved_stub
             check_resolved_dns
         fi
         return 0
@@ -1131,10 +1182,15 @@ hand_dns_to_resolved() {
         $SUDO cp -a "$RESOLV_CONF" "$RESOLV_CONF_SAVED" || fatal "Cannot save $RESOLV_CONF to $RESOLV_CONF_SAVED"
         info "  saved $RESOLV_CONF to $RESOLV_CONF_SAVED"
     fi
+    install_nm_dns_dropin || true
     $SUDO systemctl enable --now systemd-resolved.service > /dev/null 2>&1 \
         || dns_handover_failed "systemd-resolved did not start"
     $SUDO systemctl restart NetworkManager.service || dns_handover_failed "NetworkManager did not restart"
-    $SUDO ln -sfn "$RESOLVED_STUB" "$RESOLV_CONF"
+    # Best effort: a link with no DHCP server can hold startup past the timeout.
+    if command -v nm-online > /dev/null; then
+        nm-online -s -q --timeout=15 || true
+    fi
+    link_resolved_stub
     check_resolved_dns
     info "  $RESOLV_CONF -> $RESOLVED_STUB"
 }
@@ -1254,6 +1310,7 @@ qemu-utils gdisk ovmf qemu-efi-aarch64 less
 libvirt-daemon-system libvirt-clients
 pciutils
 jq curl iproute2 iputils-arping ethtool netcat-openbsd wget unzip xz-utils file
+dmidecode iperf
 ovn-central ovn-host openvswitch-switch openvswitch-ipsec strongswan-charon dhcpcd-base
 chrony nftables"
 
@@ -1263,7 +1320,7 @@ chrony nftables"
 OL_RUNTIME_PACKAGES="nbdkit nbdkit-basic-filters
 qemu-kvm qemu-img edk2-ovmf seabios-bin less
 pciutils
-jq curl iproute iputils ethtool nmap-ncat wget unzip xz file
+jq curl iproute iputils ethtool nmap-ncat wget unzip xz file dmidecode
 ovn25.03-central ovn25.03-host openvswitch3.5 openvswitch3.5-ipsec libreswan dhcpcd
 chrony nftables iptables-nft systemd-resolved"
 

scripts/setup_host_test.sh

@@ -20,9 +20,10 @@ fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
 echo "-- setup.sh host DNS, chrony, needrestart and nbdkit"
 
 mkdir -p "$TMP/sys"
-for tool in bash awk sed grep readlink basename dirname cp ln mkdir cat head tail wc tee rm chmod install; do
+for tool in bash awk sed grep readlink basename dirname cp ln mkdir cat head tail wc tee rm chmod install mv; do
     ln -s "$(command -v "$tool")" "$TMP/sys/$tool"
 done
+ln -s "$(command -v sleep)" "$TMP/sys/realsleep"
 
 # stub <name> <body> — a stub command; $S is the fixture's state directory.
 stub() {
@@ -30,10 +31,17 @@ stub() {
     chmod +x "$TMP/bin/$1"
 }
 
+# nm_settle — waits for the fixture's NetworkManager to finish writing.
+nm_settle() {
+    n=0
+    while [ -e "$TMP/state/nm-pending" ] && [ $n -lt 500 ]; do "$TMP/sys/realsleep" 0.01; n=$((n + 1)); done
+}
+
 # host — a fresh fixture: resolved installed and inactive, resolvconf the
 # resolvectl link, NetworkManager running with no DNS settings, a dnf host
 # whose first enabled repository is yum.oracle.com.
 host() {
+    nm_settle
     rm -rf "${TMP:?}/state" "${TMP:?}/root" "${TMP:?}/bin"
     mkdir -p "$TMP/state/active" "$TMP/root/etc" "$TMP/root/usr/lib/systemd/system" "$TMP/bin"
     : > "$TMP/root/usr/lib/systemd/system/systemd-resolved.service"
@@ -42,11 +50,28 @@ host() {
     printf '[main]\n# dns=default\nconfigure-and-quit=no\n' > "$TMP/state/nm-config"
     printf 'Global:\nLink 2 (br-wan): 192.168.1.1\n' > "$TMP/state/resolvectl-dns"
     printf '[ol10_UEKR8]\nbaseurl = https://mirror.example/uek/\nenabled = 0\n[ol10_baseos_latest]\nbaseurl = https://yum.oracle.com/repo/OracleLinux/OL10/baseos/latest/x86_64/\nenabled = 1\n' > "$TMP/state/dnf-dump"
+    # A (re)started NetworkManager whose rc-manager is not unmanaged writes
+    # resolv.conf as a plain file on its first DNS update, which can land just
+    # after the installer links the stub.
     stub systemctl '[ "$1" = is-active ] && { [ -e "$S/active/$3" ]; exit; }
 echo "$*" >> "$S/calls"
 [ "$1 $2" = "enable --now" ] && : > "$S/active/$3"
+case "$1 $2" in
+    "restart NetworkManager.service" | "reload NetworkManager.service")
+        NetworkManager --print-config | grep -qx rc-manager=unmanaged && exit 0
+        : > "$S/nm-pending"
+        (
+            n=0
+            while [ ! -L "$S/../root/etc/resolv.conf" ] && [ $n -lt 300 ]; do realsleep 0.01; n=$((n + 1)); done
+            printf "nameserver 192.168.1.1\n" > "$S/nm-resolv.conf"
+            mv -f "$S/nm-resolv.conf" "$S/../root/etc/resolv.conf"
+            rm -f "$S/nm-pending"
+        ) > /dev/null 2>&1 &
+        ;;
+esac
 exit 0'
-    stub NetworkManager 'cat "$S/nm-config"'
+    stub NetworkManager 'cat "$S/nm-config"; cat "$S/../root/nm/90-spinifex-dns.conf" "$S/nm-config-late" 2>/dev/null; :'
+    stub nm-online ':'
     stub resolvectl '[ "$1" = dns ] && cat "$S/resolvectl-dns"'
     stub resolvectl-real ':'
     ln -s resolvectl "$TMP/bin/resolvconf"
@@ -60,10 +85,17 @@ run() {
     RC=0
     OUT=$(PATH="$TMP/bin:$TMP/sys" DISTRO_ROOT="$TMP/root" \
         SPINIFEX_RESOLV_CONF="$TMP/root/etc/resolv.conf" SPINIFEX_RESOLV_CONF_SAVED="$TMP/root/saved/resolv.conf.orig" \
+        SPINIFEX_NM_DNS_DROPIN="$TMP/root/nm/90-spinifex-dns.conf" \
         INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '. "$1"; shift; for f in "$@"; do $f; done' _ "$SETUP" "$@" 2>&1) || RC=$?
 }
 
 calls() { cat "$TMP/state/calls" 2>/dev/null || true; }
+# linked — whether resolv.conf is still the stub link once NetworkManager settles.
+linked() {
+    nm_settle
+    [ "$(readlink "$TMP/root/etc/resolv.conf")" = /run/systemd/resolve/stub-resolv.conf ]
+}
+dropin() { cat "$TMP/root/nm/90-spinifex-dns.conf" 2>/dev/null || echo "<none>"; }
 saved() { cat "$TMP/root/saved/resolv.conf.orig" 2>/dev/null || echo "<none>"; }
 expect_untouched() {
     if [ "$RC" = 0 ] && [ -z "$(calls)" ] && [ ! -L "$TMP/root/etc/resolv.conf" ] && [ "$(saved)" = "<none>" ]; then
@@ -86,7 +118,7 @@ expect_fatal() { # expect_fatal <desc> <text>...
 # --------------------------------------------------------------------------
 host
 run hand_dns_to_resolved
-if [ "$RC" = 0 ] && [ "$(readlink "$TMP/root/etc/resolv.conf")" = /run/systemd/resolve/stub-resolv.conf ] \
+if [ "$RC" = 0 ] && linked \
     && [ "$(saved)" = "$(printf 'nameserver 192.168.1.1\nsearch example.lan')" ] \
     && [ "$(calls)" = "$(printf 'enable --now systemd-resolved.service\nrestart NetworkManager.service')" ] \
     && [ "$(cat "$TMP/state/looked-up")" = yum.oracle.com ]; then
@@ -98,12 +130,44 @@ fi
 : > "$TMP/state/calls"
 rm -f "$TMP/state/looked-up"
 run hand_dns_to_resolved
-if [ "$RC" = 0 ] && [ -z "$(calls)" ] && [ "$(cat "$TMP/state/looked-up")" = yum.oracle.com ]; then
+if [ "$RC" = 0 ] && [ -z "$(calls)" ] && linked && [ "$(cat "$TMP/state/looked-up")" = yum.oracle.com ]; then
     pass "a rerun after a handover checks DNS again and changes nothing"
 else
     fail "rerun after handover: rc=$RC calls='$(calls)': $OUT"
 fi
 
+if [ "$(dropin)" = "$(printf '[main]\ndns=systemd-resolved\nrc-manager=unmanaged')" ]; then
+    pass "the handover tells NetworkManager to send DNS to resolved and leave resolv.conf alone"
+else
+    fail "NetworkManager drop-in: '$(dropin)'"
+fi
+
+# An install made before the drop-in, whose NetworkManager replaced the link.
+host
+mkdir -p "$TMP/root/saved"
+echo "the original" > "$TMP/root/saved/resolv.conf.orig"
+: > "$TMP/state/active/systemd-resolved.service"
+run hand_dns_to_resolved
+if [ "$RC" = 0 ] && linked && [ "$(calls)" = "reload NetworkManager.service" ] \
+    && [ "$(dropin)" = "$(printf '[main]\ndns=systemd-resolved\nrc-manager=unmanaged')" ]; then
+    pass "a rerun after a handover whose link NetworkManager replaced adds the drop-in, reloads NM and relinks the stub"
+else
+    fail "rerun repair: rc=$RC calls='$(calls)' link='$(readlink "$TMP/root/etc/resolv.conf")' dropin='$(dropin)': $OUT"
+fi
+
+host
+printf '[main]\nrc-manager=symlink\n' > "$TMP/state/nm-config-late"
+run hand_dns_to_resolved
+expect_fatal "a NetworkManager file read after the drop-in that sets rc-manager stops the install, naming it" \
+    "rc-manager=symlink" "rm -f $TMP/root/nm/90-spinifex-dns.conf"
+case "$(calls)" in *"restart NetworkManager"*) fail "rc-manager override: restarted NetworkManager first: $(calls)" ;; esac
+
+host
+stub ln ':'
+run hand_dns_to_resolved
+expect_fatal "a resolv.conf that will not stay linked to the stub stops the install" \
+    "is not a link to /run/systemd/resolve/stub-resolv.conf"
+
 host
 mkdir -p "$TMP/root/saved"
 echo "the original" > "$TMP/root/saved/resolv.conf.orig"
@@ -155,7 +219,8 @@ host
 printf 'Global:\nLink 2 (br-wan):\n' > "$TMP/state/resolvectl-dns"
 run hand_dns_to_resolved
 expect_fatal "no upstream servers stops the install, naming the check and printing the restore" \
-    "'resolvectl dns' lists no DNS servers" "systemctl disable --now systemd-resolved" "cp -a --remove-destination $TMP/root/saved/resolv.conf.orig"
+    "'resolvectl dns' lists no DNS servers" "systemctl disable --now systemd-resolved" "cp -a --remove-destination $TMP/root/saved/resolv.conf.orig" \
+    "rm -f $TMP/root/nm/90-spinifex-dns.conf"
 
 host
 : > "$TMP/state/getent-fail"

spinifex/agents/eks/gatewayfetch/fetch.go

@@ -16,6 +16,9 @@
 //
 // The recovery resource returns a single epoch\taction\tsnapshot line the on-VM
 // k3s-recovery agent applies before k3s starts (etcd cluster-reset / wipe-rejoin).
+//
+// The workers resource returns one instance ID per line: every worker the
+// cluster's nodegroups still own, which the on-VM node-gc agent keeps Nodes for.
 package gatewayfetch
 
 import (
@@ -46,7 +49,7 @@ type Config struct {
 	Region      string
 	AccountID   string
 	ClusterName string
-	// Resource is "addons" or "recovery".
+	// Resource is "addons", "recovery" or "workers".
 	Resource   string
 	InstanceID string
 	// Out receives the rendered TSV.
@@ -61,8 +64,8 @@ func Run(ctx context.Context, cfg Config) error {
 		return errors.New("--account-id is required (or set EKS_ACCOUNT_ID)")
 	case cfg.ClusterName == "":
 		return errors.New("--cluster is required (or set EKS_CLUSTER_NAME)")
-	case cfg.Resource != "addons" && cfg.Resource != "recovery":
-		return errors.New("--resource must be addons or recovery")
+	case cfg.Resource != "addons" && cfg.Resource != "recovery" && cfg.Resource != "workers":
+		return errors.New("--resource must be addons, recovery or workers")
 	case cfg.Resource == "recovery" && cfg.InstanceID == "":
 		return errors.New("--instance-id is required for --resource recovery (or set EKS_INSTANCE_ID)")
 	}
@@ -80,6 +83,9 @@ func Run(ctx context.Context, cfg Config) error {
 	case "recovery":
 		path = "/clusters/" + cfg.ClusterName + "/internal-recovery/" + cfg.AccountID + "/" + cfg.InstanceID
 		emit = emitRecoveryTSV
+	case "workers":
+		path = "/clusters/" + cfg.ClusterName + "/internal-workers/" + cfg.AccountID
+		emit = emitWorkers
 	default:
 		path = "/clusters/" + cfg.ClusterName + "/internal-addons/" + cfg.AccountID
 		emit = emitAddonsTSV
@@ -163,3 +169,24 @@ func emitRecoveryTSV(out io.Writer, body []byte) error {
 	}
 	return nil
 }
+
+type internalWorkersResponse struct {
+	InstanceIDs []string `json:"instanceIds"`
+}
+
+// emitWorkers writes one managed-worker instance ID per line. An empty set
+// writes nothing, which the node-gc agent tells apart from a failed fetch by
+// this command's exit status, never by the output being empty.
+func emitWorkers(out io.Writer, body []byte) error {
+	var resp internalWorkersResponse
+	if err := json.Unmarshal(body, &resp); err != nil {
+		return fmt.Errorf("unmarshal internal-workers response: %w", err)
+	}
+	w := bufio.NewWriter(out)
+	for _, id := range resp.InstanceIDs {
+		if _, err := fmt.Fprintln(w, id); err != nil {
+			return err
+		}
+	}
+	return w.Flush()
+}

spinifex/agents/eks/gatewayfetch/fetch_test.go

@@ -49,7 +49,7 @@ func TestRun_RejectsBeforeContactingGateway(t *testing.T) {
 	}{
 		{"account", func(c *Config) { c.AccountID = "" }, "--account-id is required (or set EKS_ACCOUNT_ID)"},
 		{"cluster", func(c *Config) { c.ClusterName = "" }, "--cluster is required (or set EKS_CLUSTER_NAME)"},
-		{"resource", func(c *Config) { c.Resource = "secrets" }, "--resource must be addons or recovery"},
+		{"resource", func(c *Config) { c.Resource = "secrets" }, "--resource must be addons, recovery or workers"},
 		{"recovery instance", func(c *Config) { c.Resource = "recovery" }, "--instance-id is required for --resource recovery (or set EKS_INSTANCE_ID)"},
 		{"gateway client", func(c *Config) { c.GatewayURL = "" }, "build gateway client: eksgw: baseURL is required"},
 	}
@@ -92,6 +92,26 @@ func TestRun_RendersRecoveryDirective(t *testing.T) {
 	}
 }
 
+// The gateway's restjson marshaler names the field InstanceIDs; both that and
+// the json-tag spelling decode, so the agent reads either.
+func TestRun_RendersManagedWorkers(t *testing.T) {
+	for _, body := range []string{
+		`{"InstanceIDs":["i-0aaa","i-0bbb"]}`,
+		`{"instanceIds":["i-0aaa","i-0bbb"]}`,
+	} {
+		var hits atomic.Int32
+		cfg, out := gateway(t, "/clusters/demo/internal-workers/123456789012", http.StatusOK, body, &hits)
+		cfg.Resource = "workers"
+
+		if err := Run(context.Background(), cfg); err != nil {
+			t.Fatalf("Run: %v", err)
+		}
+		if got, want := out.String(), "i-0aaa\ni-0bbb\n"; got != want {
+			t.Fatalf("stdout = %q, want %q", got, want)
+		}
+	}
+}
+
 func TestRun_RenderFailureIsNotRetried(t *testing.T) {
 	var hits atomic.Int32
 	cfg, _ := gateway(t, "/clusters/demo/internal-addons/123456789012", http.StatusOK, `{not-json`, &hits)

spinifex/domains/ec2/vpc/eni.go

@@ -694,43 +694,39 @@ func eniMatchesFilters(record *ENIRecord, filters map[string][]string) bool {
 // AttachENI marks an ENI as attached to an instance (internal use by RunInstances).
 // accountID scopes the lookup to the correct KV key.
 func (s *VPCServiceImpl) AttachENI(accountID, eniId, instanceId string, deviceIndex int64) (string, error) {
-	return s.attachENI(context.Background(), accountID, eniId, instanceId, deviceIndex)
+	return s.attachENI(context.Background(), accountID, eniId, instanceId, deviceIndex, "")
 }
 
-func (s *VPCServiceImpl) attachENI(ctx context.Context, accountID, eniId, instanceId string, deviceIndex int64) (string, error) {
-	key := kvutil.AccountKey(accountID, eniId)
-	entry, err := s.eniKV.Get(ctx, key)
-	if err != nil {
-		return "", eniNotFoundError(eniId)
-	}
-
-	var record ENIRecord
-	if err := json.Unmarshal(entry.Value(), &record); err != nil {
-		return "", errors.New(awserrors.ErrorServerInternal)
-	}
-
-	if record.Status == "in-use" {
-		return "", errors.New(awserrors.ErrorInvalidNetworkInterfaceInUse)
-	}
+// AttachENIWithOwner attaches the ENI and records instanceOwnerID as the owner
+// of the attached instance in the same write, so no later write can lose the
+// stamp. An empty owner leaves the record's owner untouched.
+func (s *VPCServiceImpl) AttachENIWithOwner(accountID, eniId, instanceId string, deviceIndex int64, instanceOwnerID string) (string, error) {
+	return s.attachENI(context.Background(), accountID, eniId, instanceId, deviceIndex, instanceOwnerID)
+}
 
+func (s *VPCServiceImpl) attachENI(ctx context.Context, accountID, eniId, instanceId string, deviceIndex int64, instanceOwnerID string) (string, error) {
 	attachmentId := awsidentifiers.GenerateResourceID("eni-attach")
-	record.Status = "in-use"
-	record.AttachmentId = attachmentId
-	record.InstanceId = instanceId
-	record.DeviceIndex = deviceIndex
-	// Default DeleteOnTermination to true on first attach, matching what
-	// DescribeNetworkInterfaces already advertises; a record carrying an
-	// explicit value from an earlier attach keeps it across re-attach.
-	if record.DeleteOnTermination == nil {
-		record.DeleteOnTermination = aws.Bool(true)
-	}
-
-	data, err := json.Marshal(record)
+	err := s.casENI(ctx, kvutil.AccountKey(accountID, eniId), eniId, func(record *ENIRecord) error {
+		if record.Status == "in-use" {
+			return errors.New(awserrors.ErrorInvalidNetworkInterfaceInUse)
+		}
+		record.Status = "in-use"
+		record.AttachmentId = attachmentId
+		record.InstanceId = instanceId
+		record.DeviceIndex = deviceIndex
+		if instanceOwnerID != "" {
+			record.InstanceOwnerId = instanceOwnerID
+		}
+		// Default DeleteOnTermination to true on first attach, matching what
+		// DescribeNetworkInterfaces already advertises; a record carrying an
+		// explicit value from an earlier attach keeps it across re-attach.
+		if record.DeleteOnTermination == nil {
+			record.DeleteOnTermination = aws.Bool(true)
+		}
+		return nil
+	})
 	if err != nil {
-		return "", fmt.Errorf("failed to marshal ENI record: %w", err)
-	}
-	if _, err := s.eniKV.Update(ctx, key, data, entry.Revision()); err != nil {
-		return "", errors.New(awserrors.ErrorServerInternal)
+		return "", err
 	}
 
 	slog.InfoContext(ctx, "ENI attached", "eniId", eniId, "instanceId", instanceId, "attachmentId", attachmentId)
@@ -796,24 +792,69 @@ func (s *VPCServiceImpl) UpdateENI(accountID, eniId string, fn func(*ENIRecord))
 }
 
 func (s *VPCServiceImpl) updateENI(ctx context.Context, accountID, eniId string, fn func(*ENIRecord)) error {
-	key := kvutil.AccountKey(accountID, eniId)
-	entry, err := s.eniKV.Get(ctx, key)
-	if err != nil {
-		return eniNotFoundError(eniId)
+	return s.casENI(ctx, kvutil.AccountKey(accountID, eniId), eniId, func(r *ENIRecord) error {
+		fn(r)
+		return nil
+	})
+}
+
+const (
+	eniCASAttempts = 3
+	eniCASBackoff  = 10 * time.Millisecond
+)
+
+// storeError reports ServerInternal to API callers while keeping the store
+// failure reachable through errors.Is for the caller's log.
+type storeError struct{ cause error }
+
+func (e *storeError) Error() string { return awserrors.ErrorServerInternal }
+func (e *storeError) Unwrap() error { return e.cause }
+
+// casENI reads the ENI record, applies mutate and writes it back under the
+// revision it read. A direct get can be answered by a replica that has not yet
+// applied the caller's own earlier write, so a revision conflict re-reads and
+// re-applies mutate, bounded. mutate may run more than once and must only
+// depend on the record it is given.
+func (s *VPCServiceImpl) casENI(ctx context.Context, key, eniId string, mutate func(*ENIRecord) error) error {
+	var lastErr error
+	attempts := 0
+	for attempt := range eniCASAttempts {
+		attempts = attempt + 1
+		if attempt > 0 {
+			select {
+			case <-ctx.Done():
+				return &storeError{ctx.Err()}
+			case <-time.After(time.Duration(attempt) * eniCASBackoff):
+			}
+		}
+		entry, err := s.eniKV.Get(ctx, key)
+		if err != nil {
+			return eniNotFoundError(eniId)
+		}
+		var record ENIRecord
+		if err := json.Unmarshal(entry.Value(), &record); err != nil {
+			return errors.New(awserrors.ErrorServerInternal)
+		}
+		if err := mutate(&record); err != nil {
+			return err
+		}
+		data, err := json.Marshal(record)
+		if err != nil {
+			return fmt.Errorf("marshal ENI record: %w", err)
+		}
+		if _, err = s.eniKV.Update(ctx, key, data, entry.Revision()); err == nil {
+			if attempt > 0 {
+				slog.InfoContext(ctx, "ENI write succeeded after revision conflict", "eniId", eniId, "attempt", attempt+1)
+			}
+			return nil
+		}
+		lastErr = err
+		if !errors.Is(err, jetstream.ErrKeyExists) {
+			break
+		}
 	}
-	var record ENIRecord
-	if err := json.Unmarshal(entry.Value(), &record); err != nil {
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	fn(&record)
-	data, err := json.Marshal(record)
-	if err != nil {
-		return fmt.Errorf("marshal ENI record: %w", err)
-	}
-	if _, err := s.eniKV.Update(ctx, key, data, entry.Revision()); err != nil {
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	return nil
+	slog.ErrorContext(ctx, "ENI write failed", "eniId", eniId, "attempts", attempts, "err", lastErr)
+	return &storeError{lastErr}
 }
 
 // ListInstanceENIs returns every ENIRecord in accountID currently attached to

spinifex/domains/ec2/vpc/eni_kv_seam.go

@@ -0,0 +1,9 @@
+package vpc
+
+import "github.com/nats-io/nats.go/jetstream"
+
+// WrapENIKVForTest replaces the ENI bucket handle with wrap(current), so a test
+// can inject the read and write faults a replicated bucket produces.
+func (s *VPCServiceImpl) WrapENIKVForTest(wrap func(jetstream.KeyValue) jetstream.KeyValue) {
+	s.eniKV = wrap(s.eniKV)
+}

spinifex/domains/ec2/vpc/eni_stale_read_test.go

@@ -0,0 +1,125 @@
+package vpc
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"testing"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// A replicated bucket can answer a direct get from a replica that has not yet
+// applied the caller's own previous write. The next compare-and-swap then
+// carries a revision that is already superseded.
+func newStaleReadENI(t *testing.T) (*VPCServiceImpl, string, func() ENIRecord) {
+	t.Helper()
+	svc := setupTestVPCService(t)
+	vpcID := createTestVPC(t, svc, "10.0.0.0/16")
+	subnetID := createTestSubnet(t, svc, vpcID, "10.0.1.0/24")
+	eniID := createTestENI(t, svc, subnetID)
+	var inner jetstream.KeyValue
+	svc.WrapENIKVForTest(func(kv jetstream.KeyValue) jetstream.KeyValue {
+		inner = kv
+		return testutil.NewStaleReadKV(kv)
+	})
+	// Reads the stored record directly: the wrapped bucket would serve the
+	// check itself from a lagging replica.
+	stored := func() ENIRecord {
+		entry, err := inner.Get(context.Background(), kvutil.AccountKey(testAccountID, eniID))
+		require.NoError(t, err)
+		var rec ENIRecord
+		require.NoError(t, json.Unmarshal(entry.Value(), &rec))
+		return rec
+	}
+	return svc, eniID, stored
+}
+
+func TestUpdateENIAfterOwnWriteSurvivesStaleReplicaRead(t *testing.T) {
+	t.Parallel()
+	svc, eniID, stored := newStaleReadENI(t)
+
+	_, err := svc.AttachENI(testAccountID, eniID, "i-test123", 0)
+	require.NoError(t, err)
+
+	err = svc.UpdateENI(testAccountID, eniID, func(r *ENIRecord) { r.InstanceOwnerId = "000000000000" })
+	require.NoError(t, err, "a read served from a lagging replica must not lose the update")
+
+	rec := stored()
+	assert.Equal(t, "000000000000", rec.InstanceOwnerId)
+	assert.Equal(t, "i-test123", rec.InstanceId, "the attach must survive the update")
+}
+
+func TestAttachENIAfterOwnWriteSurvivesStaleReplicaRead(t *testing.T) {
+	t.Parallel()
+	svc, eniID, stored := newStaleReadENI(t)
+
+	require.NoError(t, svc.UpdateENI(testAccountID, eniID, func(r *ENIRecord) { r.Description = "managed" }))
+
+	_, err := svc.AttachENI(testAccountID, eniID, "i-test123", 0)
+	require.NoError(t, err, "a read served from a lagging replica must not fail the attach")
+
+	rec := stored()
+	assert.Equal(t, "in-use", rec.Status)
+	assert.Equal(t, "managed", rec.Description)
+}
+
+func TestUpdateENIReportsTheStoreError(t *testing.T) {
+	t.Parallel()
+	svc := setupTestVPCService(t)
+	vpcID := createTestVPC(t, svc, "10.0.0.0/16")
+	subnetID := createTestSubnet(t, svc, vpcID, "10.0.1.0/24")
+	eniID := createTestENI(t, svc, subnetID)
+
+	cause := errors.New("stream unavailable")
+	svc.WrapENIKVForTest(func(kv jetstream.KeyValue) jetstream.KeyValue {
+		return testutil.NewRejectingKV(kv, func(string, []byte) error { return cause })
+	})
+
+	err := svc.UpdateENI(testAccountID, eniID, func(r *ENIRecord) { r.Description = "x" })
+	require.Error(t, err)
+	assert.ErrorIs(t, err, cause, "the caller logs this error, so it must name the store failure")
+}
+
+func TestAttachENIWithOwnerStampsTheOwnerInTheAttachWrite(t *testing.T) {
+	t.Parallel()
+	svc := setupTestVPCService(t)
+	vpcID := createTestVPC(t, svc, "10.0.0.0/16")
+	subnetID := createTestSubnet(t, svc, vpcID, "10.0.1.0/24")
+	eniID := createTestENI(t, svc, subnetID)
+
+	var writes []ENIRecord
+	svc.WrapENIKVForTest(func(kv jetstream.KeyValue) jetstream.KeyValue {
+		return testutil.NewRejectingKV(kv, func(_ string, value []byte) error {
+			var rec ENIRecord
+			require.NoError(t, json.Unmarshal(value, &rec))
+			writes = append(writes, rec)
+			return nil
+		})
+	})
+
+	_, err := svc.AttachENIWithOwner(testAccountID, eniID, "i-sys", 0, "000000000000")
+	require.NoError(t, err)
+
+	require.Len(t, writes, 1, "attach and owner stamp are one write")
+	assert.Equal(t, "in-use", writes[0].Status)
+	assert.Equal(t, "000000000000", writes[0].InstanceOwnerId)
+}
+
+func TestCASENIGivesUpAfterBoundedConflicts(t *testing.T) {
+	t.Parallel()
+	svc, eniID, _ := newStaleReadENI(t)
+	conflicts := 0
+	err := svc.casENI(t.Context(), kvutil.AccountKey(testAccountID, eniID), eniID, func(*ENIRecord) error {
+		conflicts++
+		// Move the record under the next compare-and-swap, as a competing writer does.
+		return svc.UpdateENI(testAccountID, eniID, func(r *ENIRecord) { r.Description = "other writer" })
+	})
+	require.Error(t, err)
+	assert.ErrorIs(t, err, jetstream.ErrKeyExists, "the conflict is reported, not hidden")
+	assert.Equal(t, eniCASAttempts, conflicts)
+}

spinifex/domains/eks/awsapi/authz.go

@@ -50,10 +50,11 @@ var eksScopes = map[string][]resourceSource{
 	"ListClusters":    {sourceAny},
 
 	// Internal control-plane routes. The cluster's owning account is a path
-	// segment on these two, and the ARN names that account rather than the
+	// segment on these three, and the ARN names that account rather than the
 	// system caller's, because that is the account the handler reads.
 	"ListInternalAddons":   {sourceInternalCluster},
 	"GetRecoveryDirective": {sourceInternalCluster},
+	"ListInternalWorkers":  {sourceInternalCluster},
 	// The owning account arrives in the body on these two, and the ARN names
 	// that account for the same reason: it is the account the handler acts in.
 	"PublishInternal":    {sourceInternalBodyCluster},

spinifex/domains/eks/awsapi/authz_test.go

@@ -179,6 +179,10 @@ func TestResourceARNsInternalRoutesUsePathAccount(t *testing.T) {
 	assert.Equal(t,
 		[]string{"arn:aws:eks:ap-southeast-2:999999999999:cluster/prod"},
 		resolveARNs(t, "GetRecoveryDirective", []string{"prod", "999999999999", "i-abc"}, ""))
+
+	assert.Equal(t,
+		[]string{"arn:aws:eks:ap-southeast-2:999999999999:cluster/prod"},
+		resolveARNs(t, "ListInternalWorkers", []string{"prod", "999999999999"}, ""))
 }
 
 // An identifier the gate cannot read authorizes account-wide, leaving the

spinifex/domains/eks/awsapi/gateway_wire_test.go

@@ -93,6 +93,46 @@ func TestEKSRequest_InternalAddonsWireBody(t *testing.T) {
 		rec.Body.String())
 }
 
+// The node-gc agent reads the same REST-JSON shape: the Go field name, sorted
+// IDs drawn from every nodegroup record of the cluster.
+func TestEKSRequest_InternalWorkersWireBody(t *testing.T) {
+	const cpInstanceID = "i-cp0000000000001"
+	_, nc, js := testutil.StartTestJetStream(t)
+	kv, err := ekscluster.GetOrCreateAccountBucket(t.Context(), js, authzAccountID)
+	require.NoError(t, err)
+	require.NoError(t, ekscluster.PutClusterMeta(t.Context(), kv, &ekscluster.ClusterMeta{
+		Name:              "prod",
+		Status:            ekscluster.ClusterStatusActive,
+		ControlPlaneNodes: []ekscluster.ControlPlaneNode{{InstanceID: cpInstanceID}},
+	}))
+	for _, rec := range []*ekscluster.NodegroupRecord{
+		{ClusterName: "prod", Name: "ng-b", InstanceIDs: []string{"i-0ccc", "i-0aaa"}},
+		{ClusterName: "prod", Name: "ng-a", InstanceIDs: []string{"i-0bbb"}},
+	} {
+		require.NoError(t, ekscluster.PutNodegroupRecord(t.Context(), kv, rec))
+	}
+
+	svc, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Region: authzRegion})
+	require.NoError(t, err)
+	sub, err := nc.Subscribe("eks.ListManagedWorkers", func(m *nats.Msg) {
+		acct := natsmsg.AccountIDFromMsg(m)
+		natsmsg.ServeNATSRequestCtx(m, func(ctx context.Context, in *ekscluster.ListManagedWorkersInput) (*ekscluster.ListManagedWorkersOutput, error) {
+			return svc.ListManagedWorkers(ctx, in, acct)
+		})
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	gw := scopedEKS(gwtest.Statement("Allow", "eks:*", "*"))
+	gw.nc = nc
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequest(http.MethodGet, "/clusters/prod/internal-workers/"+authzAccountID, nil)
+	require.NoError(t, gw.as(cpAgentContext(cpInstanceID)).serve(rec, req))
+
+	assert.Equal(t, http.StatusOK, rec.Code)
+	assert.Equal(t, `{"InstanceIDs":["i-0aaa","i-0bbb","i-0ccc"]}`, rec.Body.String())
+}
+
 // CreateAddon and UpdateAddon hand their service-account role to iam:PassRole,
 // same as CreateCluster's roleArn and CreateNodegroup's nodeRole; an absent
 // role passes nothing.

spinifex/domains/eks/awsapi/internal_authz_test.go

@@ -97,6 +97,8 @@ func TestAuthorizeInternal_RejectsNonCPPrincipals(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			assertDenied(t, eksawsapi.AuthorizeInternal(t.Context(), nil, "ListInternalAddons",
 				tc.caller, []string{"alpha", tenantAccount}, nil))
+			assertDenied(t, eksawsapi.AuthorizeInternal(t.Context(), nil, "ListInternalWorkers",
+				tc.caller, []string{"alpha", tenantAccount}, nil))
 			assertDenied(t, eksawsapi.AuthorizeInternal(t.Context(), nil, "GetRecoveryDirective",
 				tc.caller, []string{"alpha", tenantAccount, cpInstanceID}, nil))
 			assertDenied(t, eksawsapi.AuthorizeInternal(t.Context(), nil, "PublishInternal",
@@ -124,6 +126,8 @@ func TestAuthorizeInternal_AllowsClusterMember(t *testing.T) {
 
 	require.NoError(t, eksawsapi.AuthorizeInternal(t.Context(), nc, "ListInternalAddons",
 		cpAgent(cpInstanceID), []string{"alpha", tenantAccount}, nil))
+	require.NoError(t, eksawsapi.AuthorizeInternal(t.Context(), nc, "ListInternalWorkers",
+		cpAgent(cpInstanceID), []string{"alpha", tenantAccount}, nil))
 	require.NoError(t, eksawsapi.AuthorizeInternal(t.Context(), nc, "GetRecoveryDirective",
 		cpAgent(cpInstanceID), []string{"alpha", tenantAccount, cpInstanceID}, nil))
 	require.NoError(t, eksawsapi.AuthorizeInternal(t.Context(), nc, "PublishInternal",

spinifex/domains/eks/awsapi/internal_workers.go

@@ -0,0 +1,35 @@
+package awsapi
+
+import (
+	"context"
+	"errors"
+
+	ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go"
+)
+
+// internalWorkersOutput is the body returned to the on-VM node-gc agent: the
+// instance ID of every worker the cluster's nodegroups still own.
+type internalWorkersOutput struct {
+	InstanceIDs []string `json:"instanceIds"`
+}
+
+// ListInternalWorkers — GET /clusters/{name}/internal-workers/{acct}.
+// Internal control-plane VM route (not an AWS-SDK action): same carve-out as
+// ListInternalAddons, gated by AuthorizeInternal, which admits only a CP agent
+// serving that account's cluster.
+func ListInternalWorkers(ctx context.Context, natsConn *nats.Conn, clusterName, accountID string) (*internalWorkersOutput, error) {
+	if natsConn == nil {
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if clusterName == "" || accountID == "" {
+		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	out, err := ekscluster.NewNATSEKSService(natsConn).ListManagedWorkers(ctx,
+		&ekscluster.ListManagedWorkersInput{ClusterName: clusterName}, accountID)
+	if err != nil {
+		return nil, err
+	}
+	return &internalWorkersOutput{InstanceIDs: out.InstanceIDs}, nil
+}

spinifex/domains/eks/awsapi/routes.go

@@ -60,6 +60,14 @@ var routes = []route{
 			return GetRecoveryDirective(ctx, nc, p[0], p[1], p[2])
 		}},
 
+	// Internal control-plane route: the on-VM node-gc agent pulls the cluster's
+	// live nodegroup workers to delete the Nodes of terminated ones. Same
+	// system-cred carve-out as internal-addons.
+	{Method: "GET", Pattern: "/clusters/{clusterName}/internal-workers/{accountId}", Action: "ListInternalWorkers",
+		Handler: func(ctx context.Context, nc *nats.Conn, acct, callerARN string, p []string, b []byte) (any, error) {
+			return ListInternalWorkers(ctx, nc, p[0], p[1])
+		}},
+
 	// Nodegroup
 	{Method: "POST", Pattern: "/clusters/{clusterName}/node-groups", Action: "CreateNodegroup",
 		Handler: func(ctx context.Context, nc *nats.Conn, acct, callerARN string, p []string, b []byte) (any, error) {

spinifex/domains/eks/awsapi/routes_test.go

@@ -108,6 +108,7 @@ func TestLookupEKSAction_CoversAllActions(t *testing.T) {
 		"WebhookTokenReview":           false,
 		"ListInternalAddons":           false,
 		"GetRecoveryDirective":         false,
+		"ListInternalWorkers":          false,
 		"DescribeCluster":              false,
 		"ListClusters":                 false,
 		"DeleteCluster":                false,

spinifex/domains/eks/cluster/k3s_server_role.go

@@ -22,9 +22,10 @@ const (
 	// WebhookTokenReview (the eks-token-webhook relays `aws eks get-token` bearer
 	// tokens to the token-review broker for host-side STS verification), and
 	// GetRecoveryDirective (the k3s-recovery agent pulls its per-member etcd
-	// recovery directive at boot). The gateway evaluates these per request against
-	// the role's policies.
-	eksServerInlinePolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["eks:PublishInternal","eks:ListInternalAddons","eks:WebhookTokenReview","eks:GetRecoveryDirective"],"Resource":"*"}]}`
+	// recovery directive at boot), and ListInternalWorkers (the node-gc agent
+	// pulls the live nodegroup workers). The gateway evaluates these per request
+	// against the role's policies.
+	eksServerInlinePolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["eks:PublishInternal","eks:ListInternalAddons","eks:WebhookTokenReview","eks:GetRecoveryDirective","eks:ListInternalWorkers"],"Resource":"*"}]}`
 )
 
 // ensureCPInstanceProfile returns the CP instance-profile ARN, or "" to signal
@@ -35,7 +36,7 @@ func (s *EKSServiceImpl) ensureCPInstanceProfile(accountID string) string {
 	iam := s.iamEnsurer()
 	if iam == nil {
 		slog.Warn("EKS: IAM service unwired; CP VM falls back to baked static gateway creds and "+
-			"will be denied eks:PublishInternal, eks:WebhookTokenReview, eks:ListInternalAddons and eks:GetRecoveryDirective (no instance identity to bind)",
+			"will be denied eks:PublishInternal, eks:WebhookTokenReview, eks:ListInternalAddons, eks:GetRecoveryDirective and eks:ListInternalWorkers (no instance identity to bind)",
 			"accountID", accountID)
 		return ""
 	}
@@ -43,7 +44,7 @@ func (s *EKSServiceImpl) ensureCPInstanceProfile(accountID string) string {
 		CPInstanceRoleName, eksServerInlinePolicyName, eksServerInlinePolicy)
 	if err != nil {
 		slog.Error("EKS: ensure CP instance profile failed; CP VM falls back to baked static gateway creds and "+
-			"will be denied eks:PublishInternal, eks:WebhookTokenReview, eks:ListInternalAddons and eks:GetRecoveryDirective (no instance identity to bind)",
+			"will be denied eks:PublishInternal, eks:WebhookTokenReview, eks:ListInternalAddons, eks:GetRecoveryDirective and eks:ListInternalWorkers (no instance identity to bind)",
 			"accountID", accountID, "role", CPInstanceRoleName, "err", err)
 		return ""
 	}

spinifex/domains/eks/cluster/k3s_server_role_test.go

@@ -38,6 +38,9 @@ func TestEnsureK3sServerInstanceProfile_CreatesRolePolicyProfile(t *testing.T) {
 	// these IMDS creds; without this action the gateway 403s the boot-time fetch
 	// and a wedged control plane can never be auto-reset.
 	assert.Contains(t, policy, "eks:GetRecoveryDirective")
+	// The node-gc agent pulls the live nodegroup workers with these IMDS creds;
+	// without this action a terminated worker's Node is never deleted.
+	assert.Contains(t, policy, "eks:ListInternalWorkers")
 
 	require.NotNil(t, f.Profiles[CPInstanceRoleName])
 	assert.Len(t, f.Profiles[CPInstanceRoleName].Roles, 1)

spinifex/domains/eks/cluster/service.go

@@ -44,6 +44,10 @@ type EKSService interface {
 	// AWS-SDK action): the on-VM addon-sync agent fetches staged manifests for a
 	// cluster via the internal-addons gateway route.
 	ListStagedAddonManifests(ctx context.Context, input *ListStagedAddonManifestsInput, accountID string) (*ListStagedAddonManifestsOutput, error)
+	// ListManagedWorkers is an internal control-plane method (not an AWS-SDK
+	// action): the on-VM node-gc agent fetches the cluster's live nodegroup
+	// workers via the internal-workers gateway route.
+	ListManagedWorkers(ctx context.Context, input *ListManagedWorkersInput, accountID string) (*ListManagedWorkersOutput, error)
 
 	// GetRecoveryDirective / SetRecoveryDirective are internal control-plane
 	// methods (not AWS-SDK actions): the on-VM k3s-recovery agent fetches its

spinifex/domains/eks/cluster/service_nats.go

@@ -129,6 +129,10 @@ func (s *NATSEKSService) ListStagedAddonManifests(ctx context.Context, input *Li
 	return natsmsg.NATSRequest[ListStagedAddonManifestsOutput](ctx, s.natsConn, "eks.ListStagedAddonManifests", input, defaultTimeout, accountID)
 }
 
+func (s *NATSEKSService) ListManagedWorkers(ctx context.Context, input *ListManagedWorkersInput, accountID string) (*ListManagedWorkersOutput, error) {
+	return natsmsg.NATSRequest[ListManagedWorkersOutput](ctx, s.natsConn, "eks.ListManagedWorkers", input, defaultTimeout, accountID)
+}
+
 func (s *NATSEKSService) UpdateAddon(ctx context.Context, input *eks.UpdateAddonInput, accountID string) (*eks.UpdateAddonOutput, error) {
 	return natsmsg.NATSRequest[eks.UpdateAddonOutput](ctx, s.natsConn, "eks.UpdateAddon", input, defaultTimeout, accountID)
 }

spinifex/domains/eks/cluster/worker_membership.go

@@ -0,0 +1,46 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"slices"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// ListManagedWorkersInput names the cluster whose managed-nodegroup workers to
+// list. It is an internal control-plane request (not an AWS-SDK shape), served
+// over NATS for the guest node-gc agent via the internal-workers route.
+type ListManagedWorkersInput struct {
+	ClusterName string `json:"clusterName"`
+}
+
+// ListManagedWorkersOutput carries the instance ID of every worker a managed
+// nodegroup of the cluster still owns, sorted.
+type ListManagedWorkersOutput struct {
+	InstanceIDs []string `json:"instanceIds"`
+}
+
+// ListManagedWorkers returns every worker instance ID recorded on the cluster's
+// nodegroups. A nodegroup drops an ID only after terminating it (DeleteNodegroup,
+// scale-down, reclaim), so the guest deletes a nodegroup Node whose instance is
+// absent here, as the AWS cloud controller deletes a Node whose instance is gone.
+func (s *EKSServiceImpl) ListManagedWorkers(ctx context.Context, input *ListManagedWorkersInput, accountID string) (*ListManagedWorkersOutput, error) {
+	if input == nil || input.ClusterName == "" {
+		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	acctKV, err := s.acctKVForCluster(ctx, accountID, input.ClusterName)
+	if err != nil {
+		return nil, err
+	}
+	recs, err := ListNodegroupRecords(ctx, acctKV, input.ClusterName)
+	if err != nil {
+		return nil, err
+	}
+	ids := make([]string, 0)
+	for _, rec := range recs {
+		ids = append(ids, rec.InstanceIDs...)
+	}
+	slices.Sort(ids)
+	return &ListManagedWorkersOutput{InstanceIDs: slices.Compact(ids)}, nil
+}

spinifex/domains/eks/cluster/worker_membership_test.go

@@ -0,0 +1,74 @@
+package cluster
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func managedWorkers(t *testing.T, f *eksServiceFixture, cluster string) []string {
+	t.Helper()
+	out, err := f.svc.ListManagedWorkers(context.Background(), &ListManagedWorkersInput{ClusterName: cluster}, testAccountID)
+	require.NoError(t, err)
+	return out.InstanceIDs
+}
+
+// A worker leaves the managed set exactly when its nodegroup terminates it, by
+// scale-down or DeleteNodegroup, so the guest node-gc deletes its Node then and
+// leaves every worker still owned by a nodegroup alone.
+func TestListManagedWorkers_DropsWorkersTheNodegroupTerminated(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	for _, ng := range []string{"ng1", "ng2"} {
+		_, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", ng, 2), testAccountID)
+		require.NoError(t, err)
+		markWorkersReady(t, f, "c1", ng, 2)
+		f.svc.WaitLaunches()
+	}
+	ng1, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+	require.NoError(t, err)
+	ng2, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng2")
+	require.NoError(t, err)
+	require.Len(t, ng1.InstanceIDs, 2)
+	assert.ElementsMatch(t, append(append([]string{}, ng1.InstanceIDs...), ng2.InstanceIDs...), managedWorkers(t, f, "c1"))
+
+	_, err = f.svc.UpdateNodegroupConfig(context.Background(), &eks.UpdateNodegroupConfigInput{
+		ClusterName:   aws.String("c1"),
+		NodegroupName: aws.String("ng1"),
+		ScalingConfig: &eks.NodegroupScalingConfig{DesiredSize: aws.Int64(1)},
+	}, testAccountID)
+	require.NoError(t, err)
+	require.Len(t, f.worker.terminateCalls, 1)
+	scaledIn := f.worker.terminateCalls[0]
+	got := managedWorkers(t, f, "c1")
+	assert.Len(t, got, 3)
+	for _, id := range scaledIn {
+		assert.NotContains(t, got, id, "a worker terminated by scale-down must leave the managed set")
+	}
+
+	_, err = f.svc.DeleteNodegroup(context.Background(), &eks.DeleteNodegroupInput{
+		ClusterName: aws.String("c1"), NodegroupName: aws.String("ng2"),
+	}, testAccountID)
+	require.NoError(t, err)
+	got = managedWorkers(t, f, "c1")
+	assert.Len(t, got, 1)
+	for _, id := range ng2.InstanceIDs {
+		assert.NotContains(t, got, id, "a deleted nodegroup's workers must leave the managed set")
+	}
+}
+
+func TestListManagedWorkers_UnknownClusterIsNotFound(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+
+	_, err := f.svc.ListManagedWorkers(context.Background(), &ListManagedWorkersInput{ClusterName: "ghost"}, testAccountID)
+	code, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorEKSResourceNotFound, code)
+}

spinifex/domains/elbv2/reaper_test.go

@@ -2,9 +2,12 @@ package elbv2
 
 import (
 	"context"
+	"errors"
 	"testing"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -80,3 +83,80 @@ func TestStartLifecycleReaper(t *testing.T) {
 
 	cancel()
 }
+
+// A load balancer the reaper gives up on must not leave its managed VM running:
+// the VM holds capacity and no customer data, and nothing else would stop it.
+func TestReapStuckProvisioningLBsTerminatesOnlyTheFailedLBsVM(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	launcher := &mockSystemInstanceLauncher{}
+	svc.InstanceLauncher = launcher
+	now := time.Now().UTC()
+
+	stuck := testLBRecord("stuck0000000000", StateProvisioning, now.Add(-10*time.Minute), time.Time{})
+	stuck.InstanceID = "i-stuck"
+	noVM := testLBRecord("novm0000000000", StateProvisioning, now.Add(-10*time.Minute), time.Time{})
+	fresh := testLBRecord("fresh0000000000", StateProvisioning, now, time.Time{})
+	fresh.InstanceID = "i-fresh"
+	active := testLBRecord("active000000000", StateActive, now.Add(-10*time.Minute), time.Time{})
+	active.InstanceID = "i-active"
+	heartbeating := testLBRecord("heartbeat0000000", StateProvisioning, now.Add(-10*time.Minute), now.Add(-time.Second))
+	heartbeating.InstanceID = "i-heartbeating"
+	for _, lb := range []*LoadBalancerRecord{stuck, noVM, fresh, active, heartbeating} {
+		require.NoError(t, svc.store.PutLoadBalancer(t.Context(), lb))
+	}
+
+	svc.reapStuckProvisioningLBs(t.Context(), now)
+
+	got, err := svc.store.GetLoadBalancer(t.Context(), stuck.LoadBalancerID)
+	require.NoError(t, err)
+	assert.Equal(t, StateFailed, got.State)
+	assert.NotEmpty(t, got.StateReason, "the failure reason stays on the LB")
+
+	terminated := func() []string {
+		launcher.mu.Lock()
+		defer launcher.mu.Unlock()
+		return append([]string(nil), launcher.terminateCalls...)
+	}
+	require.Eventually(t, func() bool { return len(terminated()) > 0 }, 2*time.Second, 10*time.Millisecond,
+		"the reaper must terminate the failed LB's VM")
+	assert.Equal(t, []string{"i-stuck"}, terminated(),
+		"only the failed LB's own VM is terminated, never another LB's")
+}
+
+func TestReapStuckProvisioningLBsDetachesTheFailedLBsENIs(t *testing.T) {
+	t.Parallel()
+	for _, tc := range []struct {
+		name         string
+		terminateErr error
+		wantStatus   string
+	}{
+		{"terminated VM releases its ENI", nil, "available"},
+		{"a VM that cannot be stopped keeps its ENI attached", errors.New("nats timeout"), "in-use"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			svc, vpcSvc, subnetID := setupMicrovmTestService(t)
+			svc.InstanceLauncher = &mockSystemInstanceLauncher{terminateErr: tc.terminateErr}
+
+			eniOut, err := vpcSvc.CreateNetworkInterface(t.Context(), &ec2.CreateNetworkInterfaceInput{SubnetId: aws.String(subnetID)}, testAccountID)
+			require.NoError(t, err)
+			eniID := aws.StringValue(eniOut.NetworkInterface.NetworkInterfaceId)
+			_, err = vpcSvc.AttachENI(testAccountID, eniID, "i-stuck", 0)
+			require.NoError(t, err)
+
+			now := time.Now().UTC()
+			lb := testLBRecord("enirel0000000000", StateProvisioning, now.Add(-10*time.Minute), time.Time{})
+			lb.AccountID = testAccountID
+			lb.InstanceID = "i-stuck"
+			lb.ENIs = []string{eniID}
+			require.NoError(t, svc.store.PutLoadBalancer(t.Context(), lb))
+
+			svc.reapStuckProvisioningLBs(t.Context(), now)
+
+			rec, err := vpcSvc.GetENIRecord(testAccountID, eniID)
+			require.NoError(t, err, "the ENI stays for DeleteLoadBalancer to remove")
+			assert.Equal(t, tc.wantStatus, rec.Status)
+		})
+	}
+}

spinifex/domains/elbv2/service_impl.go

@@ -1772,6 +1772,31 @@ func (s *ELBv2ServiceImpl) reapStuckProvisioningLBs(ctx context.Context, now tim
 			"lbArn", lb.LoadBalancerArn, "lbId", lb.LoadBalancerID,
 			"instanceId", lb.InstanceID, "createdAt", lb.CreatedAt)
 		s.markLBFailed(ctx, lb.LoadBalancerArn, reason)
+		s.releaseFailedLBVM(ctx, lb)
+	}
+}
+
+// releaseFailedLBVM stops the failed LB's own managed VM and detaches its ENIs,
+// so a failed LB holds no running capacity. The ENIs stay on the record for
+// DeleteLoadBalancer to remove. A VM that cannot be stopped keeps its ENIs
+// attached, since detaching under a running guest would only hide the leak.
+func (s *ELBv2ServiceImpl) releaseFailedLBVM(ctx context.Context, lb *LoadBalancerRecord) {
+	if lb.InstanceID == "" || s.InstanceLauncher == nil {
+		return
+	}
+	if err := s.InstanceLauncher.TerminateSystemInstance(lb.InstanceID); err != nil {
+		slog.Error("ELBv2 lifecycle reaper: failed to terminate failed LB VM",
+			"lbId", lb.LoadBalancerID, "instanceId", lb.InstanceID, "err", err)
+		return
+	}
+	if s.VPCService == nil {
+		return
+	}
+	for _, eniID := range lb.ENIs {
+		if err := s.VPCService.DetachENI(ctx, lb.AccountID, eniID); err != nil && !awserrors.IsNotFound(err) {
+			slog.Warn("ELBv2 lifecycle reaper: failed to detach failed LB ENI",
+				"lbId", lb.LoadBalancerID, "eniId", eniID, "err", err)
+		}
 	}
 }
 

spinifex/ingress/aws/pipeline/operations_test.go

@@ -55,6 +55,7 @@ func TestAWSOperationInventoryMeasuresEKSAgainstTheModel(t *testing.T) {
 	require.ElementsMatch(t, []string{
 		"GetRecoveryDirective",
 		"ListInternalAddons",
+		"ListInternalWorkers",
 		"PublishInternal",
 		"WebhookTokenReview",
 	}, eks.Extra)

spinifex/operator/admin/instance_metrics.go

@@ -106,10 +106,33 @@ func flushSupported(w http.ResponseWriter) bool {
 }
 
 func streamInstanceMetrics(w http.ResponseWriter, r *http.Request, nc *nats.Conn, accountID, instanceID string, since int64) error {
+	return streamLive(w, r, nc, liveSubjects{
+		action: GetInstanceMetricsAction,
+		sample: telemetryv1.LiveSampleSubject(accountID, instanceID),
+		watch:  telemetryv1.LiveWatchSubject(accountID, instanceID),
+		absent: awserrors.ErrorInvalidInstanceIDNotFound,
+	}, since, func(data []byte) ([]telemetryv1.LiveSample, error) {
+		var ring telemetryv1.LiveRing
+		err := json.Unmarshal(data, &ring)
+		return ring.Samples, err
+	}, func(s telemetryv1.LiveSample) int64 { return s.TimeMillis })
+}
+
+// liveSubjects names one live source: where its samples are published, where
+// to ask for its ring, and the error for a source nobody answers for.
+type liveSubjects struct {
+	action string
+	sample string
+	watch  string
+	absent string
+}
+
+func streamLive[S any](w http.ResponseWriter, r *http.Request, nc *nats.Conn, src liveSubjects, since int64,
+	decodeRing func([]byte) ([]S, error), stamp func(S) int64) error {
 	samples := make(chan *nats.Msg, 64)
-	sub, err := nc.ChanSubscribe(telemetryv1.LiveSampleSubject(accountID, instanceID), samples)
+	sub, err := nc.ChanSubscribe(src.sample, samples)
 	if err != nil {
-		slog.ErrorContext(r.Context(), "GetInstanceMetrics: subscribe", "err", err)
+		slog.ErrorContext(r.Context(), src.action+": subscribe", "err", err)
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 	defer func() { _ = sub.Unsubscribe() }()
@@ -118,22 +141,21 @@ func streamInstanceMetrics(w http.ResponseWriter, r *http.Request, nc *nats.Conn
 	if err != nil {
 		return errors.New(awserrors.ErrorInternalError)
 	}
-	watchSubject := telemetryv1.LiveWatchSubject(accountID, instanceID)
 
 	// Subscribed first, then asked for the ring, so nothing falls between them.
-	// No responder is an instance that is not running in this account, which
+	// No responder is a source that is not running in this account, which
 	// reads the same as one that does not exist.
-	reply, err := nc.Request(watchSubject, watch, liveWatchTimeout)
+	reply, err := nc.Request(src.watch, watch, liveWatchTimeout)
 	if err != nil {
 		if errors.Is(err, nats.ErrNoResponders) || errors.Is(err, nats.ErrTimeout) {
-			return errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+			return errors.New(src.absent)
 		}
-		slog.ErrorContext(r.Context(), "GetInstanceMetrics: watch request", "err", err)
+		slog.ErrorContext(r.Context(), src.action+": watch request", "err", err)
 		return errors.New(awserrors.ErrorServerInternal)
 	}
-	var ring telemetryv1.LiveRing
-	if err := json.Unmarshal(reply.Data, &ring); err != nil {
-		slog.ErrorContext(r.Context(), "GetInstanceMetrics: malformed ring", "err", err)
+	ring, err := decodeRing(reply.Data)
+	if err != nil {
+		slog.ErrorContext(r.Context(), src.action+": malformed ring", "err", err)
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -146,22 +168,23 @@ func streamInstanceMetrics(w http.ResponseWriter, r *http.Request, nc *nats.Conn
 	// From here the status is committed, so failures end the stream quietly and
 	// the client reconnects.
 	last := since
-	send := func(s telemetryv1.LiveSample) bool {
-		if s.TimeMillis <= last {
+	send := func(s S) bool {
+		t := stamp(s)
+		if t <= last {
 			return true
 		}
 		data, err := json.Marshal(s)
 		if err != nil {
 			return true
 		}
-		last = s.TimeMillis
-		if _, err := fmt.Fprintf(w, "id: %d\nevent: sample\ndata: %s\n\n", s.TimeMillis, data); err != nil {
+		last = t
+		if _, err := fmt.Fprintf(w, "id: %d\nevent: sample\ndata: %s\n\n", t, data); err != nil {
 			return false
 		}
 		return ctrl.Flush() == nil
 	}
 
-	for _, s := range ring.Samples {
+	for _, s := range ring {
 		if !send(s) {
 			return nil
 		}
@@ -184,17 +207,17 @@ func streamInstanceMetrics(w http.ResponseWriter, r *http.Request, nc *nats.Conn
 		case <-end.C:
 			return nil
 		case <-renew.C:
-			if err := nc.Publish(watchSubject, watch); err != nil {
-				slog.DebugContext(r.Context(), "GetInstanceMetrics: renew watch", "err", err)
+			if err := nc.Publish(src.watch, watch); err != nil {
+				slog.DebugContext(r.Context(), src.action+": renew watch", "err", err)
 			}
 		case <-keepAlive.C:
 			if _, err := fmt.Fprint(w, ": keep-alive\n\n"); err != nil || ctrl.Flush() != nil {
 				return nil
 			}
 		case msg := <-samples:
-			var s telemetryv1.LiveSample
+			var s S
 			if err := json.Unmarshal(msg.Data, &s); err != nil {
-				slog.DebugContext(r.Context(), "GetInstanceMetrics: malformed sample", "err", err)
+				slog.DebugContext(r.Context(), src.action+": malformed sample", "err", err)
 				continue
 			}
 			if !send(s) {

spinifex/operator/admin/node_insight.go

@@ -0,0 +1,429 @@
+package admin
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"slices"
+	"strconv"
+	"time"
+
+	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// GetNodeFactsOutput is the response for GetNodeFacts.
+type GetNodeFactsOutput struct {
+	Nodes []clusterv1.NodeFacts `json:"nodes"`
+}
+
+// nodeFactsTimeout allows for a node rebuilding its facts, which reads every NIC.
+const nodeFactsTimeout = 7 * time.Second
+
+// GetNodeFacts gathers every node's hardware and network facts.
+func GetNodeFacts(ctx context.Context, nc *nats.Conn, expectedNodes int) (*GetNodeFactsOutput, error) {
+	frames, _, err := natsmsg.Gather(ctx, nc, clusterv1.NodeFactsSubject, []byte("{}"),
+		natsmsg.GatherOpts{Timeout: nodeFactsTimeout, ExpectedNodes: expectedNodes})
+	if err != nil {
+		return nil, err
+	}
+	out := &GetNodeFactsOutput{Nodes: make([]clusterv1.NodeFacts, 0, len(frames))}
+	for _, frame := range frames {
+		var f clusterv1.NodeFacts
+		if json.Unmarshal(frame.Data, &f) == nil && f.Node != "" {
+			out.Nodes = append(out.Nodes, f)
+		}
+	}
+	slices.SortFunc(out.Nodes, func(a, b clusterv1.NodeFacts) int {
+		switch {
+		case a.Node < b.Node:
+			return -1
+		case a.Node > b.Node:
+			return 1
+		}
+		return 0
+	})
+	return out, nil
+}
+
+// Network test states.
+const (
+	NetworkTestRunning  = "running"
+	NetworkTestComplete = "complete"
+	NetworkTestFailed   = "failed"
+	NetworkTestPending  = "pending"
+)
+
+// Bounds on one network test, so an admin cannot tie the cluster's links up.
+const (
+	defaultNetworkTestSeconds = 10
+	defaultNetworkTestStreams = 4
+	maxNetworkTestRuns        = 24
+	maxNetworkTestListed      = 20
+	networkTestRunSlack       = 20 * time.Second
+	networkTestServeTimeout   = 5 * time.Second
+)
+
+// NetworkTest is one admin-requested test: every ordered pair of nodes on a
+// plane, run one after another so no two share a link.
+type NetworkTest struct {
+	TestID          string              `json:"testId"`
+	Plane           string              `json:"plane"`
+	Status          string              `json:"status"`
+	DurationSeconds int                 `json:"durationSeconds"`
+	Streams         int                 `json:"streams"`
+	StartedAt       time.Time           `json:"startedAt"`
+	CompletedAt     *time.Time          `json:"completedAt,omitempty"`
+	Error           string              `json:"error,omitempty"`
+	Runs            []NetworkTestResult `json:"runs"`
+}
+
+// NetworkTestResult is one direction between two nodes.
+type NetworkTestResult struct {
+	clusterv1.NetTestRunResponse
+
+	From   string `json:"from"`
+	To     string `json:"to"`
+	Status string `json:"status"`
+}
+
+// NetworkTestStore holds tests and the one cluster-wide claim on running one.
+type NetworkTestStore interface {
+	// Claim takes the cluster-wide slot for testID until the given time, and
+	// reports ErrNetworkTestBusy while another unexpired claim holds it.
+	Claim(ctx context.Context, testID string, until time.Time) error
+	Release(ctx context.Context, testID string) error
+	Put(ctx context.Context, t *NetworkTest) error
+	Get(ctx context.Context, testID string) (*NetworkTest, error)
+	List(ctx context.Context) ([]NetworkTest, error)
+}
+
+// ErrNetworkTestBusy is a claim refused because another test is running.
+var ErrNetworkTestBusy = errors.New("a network test is already running")
+
+const (
+	networkTestBucket   = "spinifex-network-tests"
+	networkTestTTL      = 7 * 24 * time.Hour
+	networkTestClaimKey = "active"
+	networkTestPrefix   = "test."
+)
+
+type networkTestClaim struct {
+	TestID string    `json:"testId"`
+	Until  time.Time `json:"until"`
+}
+
+// KVNetworkTests is the NetworkTestStore over a JetStream KV bucket, shared
+// by every gateway so a test started through one is visible through all.
+type KVNetworkTests struct {
+	tests  *kvstore.Store[NetworkTest]
+	claims *kvstore.Store[networkTestClaim]
+}
+
+var _ NetworkTestStore = (*KVNetworkTests)(nil)
+
+// NewKVNetworkTests constructs the store over the cluster's JetStream client.
+func NewKVNetworkTests(js jetstream.JetStream) *KVNetworkTests {
+	b := kvstore.NewBucket(js, kvstore.Config{
+		Name:    networkTestBucket,
+		History: 1,
+		TTL:     networkTestTTL,
+		Missing: "network tests: no JetStream client configured",
+	})
+	return &KVNetworkTests{tests: kvstore.On[NetworkTest](b), claims: kvstore.On[networkTestClaim](b)}
+}
+
+// Claim is a create-only write; an expired claim, left by a gateway that died
+// mid-test, is replaced under its revision so two takers cannot both win.
+func (s *KVNetworkTests) Claim(ctx context.Context, testID string, until time.Time) error {
+	claim := &networkTestClaim{TestID: testID, Until: until}
+	_, err := s.claims.Create(ctx, networkTestClaimKey, claim)
+	if !errors.Is(err, kvstore.ErrExists) {
+		return err
+	}
+	held, rev, err := s.claims.Get(ctx, networkTestClaimKey)
+	if err != nil {
+		return err
+	}
+	if time.Now().Before(held.Until) {
+		return ErrNetworkTestBusy
+	}
+	if _, err := s.claims.CompareAndSet(ctx, networkTestClaimKey, claim, rev); err != nil {
+		if errors.Is(err, kvstore.ErrConflict) {
+			return ErrNetworkTestBusy
+		}
+		return err
+	}
+	return nil
+}
+
+// Release drops the claim only if testID still holds it.
+func (s *KVNetworkTests) Release(ctx context.Context, testID string) error {
+	held, rev, err := s.claims.Get(ctx, networkTestClaimKey)
+	if errors.Is(err, kvstore.ErrNotFound) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	if held.TestID != testID {
+		return nil
+	}
+	return s.claims.CompareAndDelete(ctx, networkTestClaimKey, rev)
+}
+
+func (s *KVNetworkTests) Put(ctx context.Context, t *NetworkTest) error {
+	return s.tests.Set(ctx, networkTestPrefix+t.TestID, t)
+}
+
+func (s *KVNetworkTests) Get(ctx context.Context, testID string) (*NetworkTest, error) {
+	t, _, err := s.tests.Get(ctx, networkTestPrefix+testID)
+	return t, err
+}
+
+func (s *KVNetworkTests) List(ctx context.Context) ([]NetworkTest, error) {
+	return s.tests.List(ctx, networkTestPrefix)
+}
+
+// StartNetworkTest validates the request, claims the cluster-wide slot,
+// records the test and returns at once; the runs converge in the background.
+func (d Deps) StartNetworkTest(ctx context.Context, args map[string]string) (*NetworkTest, error) {
+	if d.NetworkTests == nil || d.NATS == nil {
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	plane := args["Plane"]
+	if plane != clusterv1.PlaneWAN && plane != clusterv1.PlaneLAN && plane != clusterv1.PlaneVPC {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "Plane must be wan, lan or vpc")
+	}
+	duration, err := intArg(args, "DurationSeconds", defaultNetworkTestSeconds, minNetTestSeconds, maxNetTestSeconds)
+	if err != nil {
+		return nil, err
+	}
+	streams, err := intArg(args, "Streams", defaultNetworkTestStreams, 1, maxNetTestStreams)
+	if err != nil {
+		return nil, err
+	}
+	runs, err := networkTestPairs(d.NodeIDs, args["Source"], args["Target"])
+	if err != nil {
+		return nil, err
+	}
+
+	test := &NetworkTest{
+		TestID: newNetworkTestID(), Plane: plane, Status: NetworkTestRunning,
+		DurationSeconds: duration, Streams: streams, StartedAt: time.Now().UTC(), Runs: runs,
+	}
+	perRun := time.Duration(duration)*time.Second + networkTestRunSlack + networkTestServeTimeout
+	until := time.Now().Add(time.Duration(len(runs))*perRun + time.Minute)
+	switch err := d.NetworkTests.Claim(ctx, test.TestID, until); {
+	case errors.Is(err, ErrNetworkTestBusy):
+		return nil, awserrors.Errorf(awserrors.ErrorIncorrectState, "%s", err.Error())
+	case err != nil:
+		slog.ErrorContext(ctx, "StartNetworkTest: claim", "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if err := d.NetworkTests.Put(ctx, test); err != nil {
+		_ = d.NetworkTests.Release(ctx, test.TestID)
+		slog.ErrorContext(ctx, "StartNetworkTest: record", "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	slog.InfoContext(ctx, "network test started", "test_id", test.TestID, "plane", plane, "runs", len(runs))
+
+	started := *test
+	started.Runs = slices.Clone(test.Runs)
+	go d.runNetworkTest(context.WithoutCancel(ctx), test, perRun)
+	return &started, nil
+}
+
+// These mirror the daemon's limits, which refuse anything outside them.
+const (
+	minNetTestSeconds = 5
+	maxNetTestSeconds = 30
+	maxNetTestStreams = 16
+)
+
+func intArg(args map[string]string, name string, def, lo, hi int) (int, error) {
+	raw := args[name]
+	if raw == "" {
+		return def, nil
+	}
+	v, err := strconv.Atoi(raw)
+	if err != nil || v < lo || v > hi {
+		return 0, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "%s must be between %d and %d", name, lo, hi)
+	}
+	return v, nil
+}
+
+// networkTestPairs is both directions between source and target, or with no
+// target every ordered pair of nodes, the source's first.
+func networkTestPairs(nodes []string, source, target string) ([]NetworkTestResult, error) {
+	known := func(n string) bool { return slices.Contains(nodes, n) }
+	if source != "" && !known(source) {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "unknown Source node")
+	}
+	if target != "" && !known(target) {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "unknown Target node")
+	}
+	var runs []NetworkTestResult
+	add := func(from, to string) {
+		runs = append(runs, NetworkTestResult{From: from, To: to, Status: NetworkTestPending})
+	}
+	switch {
+	case source != "" && target != "":
+		if source == target {
+			return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterCombination, "Source and Target must differ")
+		}
+		add(source, target)
+		add(target, source)
+	case target != "":
+		return nil, awserrors.Errorf(awserrors.ErrorMissingParameter, "Target needs a Source")
+	default:
+		order := slices.Clone(nodes)
+		slices.Sort(order)
+		if source != "" {
+			order = append([]string{source}, slices.DeleteFunc(order, func(n string) bool { return n == source })...)
+		}
+		for _, from := range order {
+			for _, to := range order {
+				if from != to {
+					add(from, to)
+				}
+			}
+		}
+	}
+	if len(runs) == 0 {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "a network test needs at least two nodes")
+	}
+	if len(runs) > maxNetworkTestRuns {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "too many nodes for a full mesh; name a Source and Target")
+	}
+	return runs, nil
+}
+
+func newNetworkTestID() string {
+	b := make([]byte, 8)
+	_, _ = rand.Read(b)
+	return "nt-" + hex.EncodeToString(b)
+}
+
+// runNetworkTest drives each run in turn and writes the record after each,
+// so a poller sees results as they land.
+func (d Deps) runNetworkTest(parent context.Context, test *NetworkTest, perRun time.Duration) {
+	ctx, cancel := context.WithTimeout(parent, time.Duration(len(test.Runs))*perRun+time.Minute)
+	defer cancel()
+	defer func() {
+		if err := d.NetworkTests.Release(parent, test.TestID); err != nil {
+			slog.Warn("network test: release claim", "test_id", test.TestID, "err", err)
+		}
+	}()
+
+	failed := 0
+	for i := range test.Runs {
+		run := &test.Runs[i]
+		run.Status = NetworkTestRunning
+		d.saveNetworkTest(ctx, test)
+		res, err := d.runNetworkTestPair(ctx, test, run.From, run.To, perRun)
+		run.NetTestRunResponse = res
+		if err != nil {
+			run.Status, run.Error = NetworkTestFailed, err.Error()
+			failed++
+		} else {
+			run.Status = NetworkTestComplete
+		}
+	}
+	now := time.Now().UTC()
+	test.CompletedAt = &now
+	test.Status = NetworkTestComplete
+	if failed == len(test.Runs) {
+		test.Status, test.Error = NetworkTestFailed, "every run failed"
+	}
+	d.saveNetworkTest(ctx, test)
+	slog.Info("network test finished", "test_id", test.TestID, "status", test.Status, "failed_runs", failed)
+}
+
+func (d Deps) saveNetworkTest(ctx context.Context, test *NetworkTest) {
+	if err := d.NetworkTests.Put(ctx, test); err != nil {
+		slog.Warn("network test: write record", "test_id", test.TestID, "err", err)
+	}
+}
+
+func (d Deps) runNetworkTestPair(ctx context.Context, test *NetworkTest, from, to string, perRun time.Duration) (clusterv1.NetTestRunResponse, error) {
+	var res clusterv1.NetTestRunResponse
+	serveReq, err := json.Marshal(clusterv1.NetTestServeRequest{Plane: test.Plane, DurationSeconds: test.DurationSeconds})
+	if err != nil {
+		return res, fmt.Errorf("encode serve request: %w", err)
+	}
+	msg, err := d.NATS.RequestWithContext(ctx, clusterv1.NetTestServeSubject(to), serveReq)
+	if err != nil {
+		return res, fmt.Errorf("%s did not start its server: %w", to, err)
+	}
+	var served clusterv1.NetTestServeResponse
+	if err := json.Unmarshal(msg.Data, &served); err != nil {
+		return res, fmt.Errorf("%s: malformed serve reply", to)
+	}
+	if served.Error != "" {
+		return res, fmt.Errorf("%s: %s", to, served.Error)
+	}
+
+	runReq, err := json.Marshal(clusterv1.NetTestRunRequest{
+		Plane: test.Plane, Address: served.Address, Port: served.Port,
+		DurationSeconds: test.DurationSeconds, Streams: test.Streams,
+	})
+	if err != nil {
+		return res, fmt.Errorf("encode run request: %w", err)
+	}
+	rctx, cancel := context.WithTimeout(ctx, perRun)
+	defer cancel()
+	msg, err = d.NATS.RequestWithContext(rctx, clusterv1.NetTestRunSubject(from), runReq)
+	if err != nil {
+		return res, fmt.Errorf("%s did not report a result: %w", from, err)
+	}
+	if err := json.Unmarshal(msg.Data, &res); err != nil {
+		return res, fmt.Errorf("%s: malformed result", from)
+	}
+	if res.Error != "" {
+		return res, errors.New(res.Error)
+	}
+	return res, nil
+}
+
+// DescribeNetworkTestsOutput is the response for DescribeNetworkTests.
+type DescribeNetworkTestsOutput struct {
+	Tests []NetworkTest `json:"tests"`
+}
+
+// DescribeNetworkTests returns one test by ID, or the most recent ones.
+func (d Deps) DescribeNetworkTests(ctx context.Context, args map[string]string) (*DescribeNetworkTestsOutput, error) {
+	if d.NetworkTests == nil {
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if id := args["TestId"]; id != "" {
+		t, err := d.NetworkTests.Get(ctx, id)
+		if errors.Is(err, kvstore.ErrNotFound) {
+			return nil, awserrors.Errorf(awserrors.ErrorResourceNotFound, "no network test %s", id)
+		}
+		if err != nil {
+			return nil, err
+		}
+		return &DescribeNetworkTestsOutput{Tests: []NetworkTest{*t}}, nil
+	}
+	tests, err := d.NetworkTests.List(ctx)
+	if err != nil {
+		return nil, err
+	}
+	slices.SortFunc(tests, func(a, b NetworkTest) int { return b.StartedAt.Compare(a.StartedAt) })
+	if len(tests) > maxNetworkTestListed {
+		tests = tests[:maxNetworkTestListed]
+	}
+	if tests == nil {
+		tests = []NetworkTest{}
+	}
+	return &DescribeNetworkTestsOutput{Tests: tests}, nil
+}

spinifex/operator/admin/node_insight_test.go

@@ -0,0 +1,192 @@
+package admin
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/bluebottle/pkg/iampolicy"
+	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetNodeMetricsAsksThePolicyBeforeAnythingElse(t *testing.T) {
+	var asked []string
+	deny := func(_, act string, _ []string, _ iampolicy.ConditionKeys) error {
+		asked = append(asked, act)
+		return errors.New(awserrors.ErrorAccessDenied)
+	}
+	_, err := serve(t, Deps{}, awsidentifiers.DefaultAccountID(), action(GetNodeMetricsAction, "NodeId", "node1"), deny)
+	requireCode(t, err, awserrors.ErrorAccessDenied)
+	assert.Equal(t, []string{GetNodeMetricsAction}, asked)
+}
+
+func TestGetNodeMetricsRefusesANodeOutsideTheCluster(t *testing.T) {
+	d := Deps{NodeIDs: []string{"node1", "node2"}}
+	for _, node := range []string{"node9", "node1.>", "*"} {
+		_, err := serve(t, d, awsidentifiers.DefaultAccountID(), action(GetNodeMetricsAction, "NodeId", node), nil)
+		code, _ := awserrors.ResolveErrorCode(err)
+		assert.Equal(t, awserrors.ErrorInvalidParameterValue, code, node)
+	}
+}
+
+func TestNetworkTestPairs(t *testing.T) {
+	nodes := []string{"c", "a", "b"}
+
+	pair, err := networkTestPairs(nodes, "a", "b")
+	require.NoError(t, err)
+	assert.Equal(t, [][2]string{{"a", "b"}, {"b", "a"}}, directions(pair))
+
+	mesh, err := networkTestPairs(nodes, "b", "")
+	require.NoError(t, err)
+	assert.Equal(t, [][2]string{{"b", "a"}, {"b", "c"}, {"a", "b"}, {"a", "c"}, {"c", "b"}, {"c", "a"}}, directions(mesh))
+
+	for _, c := range []struct{ source, target, code string }{
+		{"a", "a", awserrors.ErrorInvalidParameterCombination},
+		{"", "b", awserrors.ErrorMissingParameter},
+		{"z", "", awserrors.ErrorInvalidParameterValue},
+		{"a", "z", awserrors.ErrorInvalidParameterValue},
+	} {
+		_, err := networkTestPairs(nodes, c.source, c.target)
+		code, _ := awserrors.ResolveErrorCode(err)
+		assert.Equal(t, c.code, code, "%s -> %s", c.source, c.target)
+	}
+
+	_, err = networkTestPairs([]string{"solo"}, "", "")
+	assert.Error(t, err, "one node has nobody to test against")
+}
+
+func directions(runs []NetworkTestResult) [][2]string {
+	out := make([][2]string, len(runs))
+	for i, r := range runs {
+		out[i] = [2]string{r.From, r.To}
+	}
+	return out
+}
+
+func TestStartNetworkTestValidatesBeforeClaiming(t *testing.T) {
+	_, nc, js := testutil.StartTestJetStream(t)
+	d := Deps{NATS: nc, NetworkTests: NewKVNetworkTests(js), NodeIDs: []string{"a", "b"}}
+	for _, args := range []map[string]string{
+		{"Plane": "mgmt"},
+		{"Plane": "lan", "DurationSeconds": "600"},
+		{"Plane": "lan", "Streams": "0"},
+		{"Plane": "lan", "Source": "a", "Target": "a"},
+	} {
+		_, err := d.StartNetworkTest(context.Background(), args)
+		assert.Error(t, err, "%v", args)
+	}
+	tests, err := d.NetworkTests.List(context.Background())
+	require.NoError(t, err)
+	assert.Empty(t, tests, "a refused request records nothing")
+}
+
+func TestNetworkTestClaimIsClusterWide(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	ctx := context.Background()
+	first, second := NewKVNetworkTests(js), NewKVNetworkTests(js)
+
+	require.NoError(t, first.Claim(ctx, "nt-1", time.Now().Add(time.Minute)))
+	assert.ErrorIs(t, second.Claim(ctx, "nt-2", time.Now().Add(time.Minute)), ErrNetworkTestBusy)
+
+	require.NoError(t, second.Release(ctx, "nt-2"), "releasing a claim it does not hold is a no-op")
+	assert.ErrorIs(t, second.Claim(ctx, "nt-2", time.Now().Add(time.Minute)), ErrNetworkTestBusy)
+
+	require.NoError(t, first.Release(ctx, "nt-1"))
+	require.NoError(t, second.Claim(ctx, "nt-2", time.Now().Add(-time.Second)))
+	require.NoError(t, first.Claim(ctx, "nt-3", time.Now().Add(time.Minute)), "an expired claim is taken over")
+}
+
+// fakeNode answers the serve and run halves as a daemon would, reporting a
+// fixed throughput, or failing the serve when broken.
+func fakeNode(t *testing.T, nc *nats.Conn, node, addr string, bps float64, broken bool) {
+	t.Helper()
+	serveSub, err := nc.Subscribe(clusterv1.NetTestServeSubject(node), func(m *nats.Msg) {
+		resp := clusterv1.NetTestServeResponse{Address: addr, Port: clusterv1.NetTestPort}
+		if broken {
+			resp = clusterv1.NetTestServeResponse{Error: "iperf (version 2) is not installed on this node"}
+		}
+		data, _ := json.Marshal(resp)
+		_ = m.Respond(data)
+	})
+	require.NoError(t, err)
+	runSub, err := nc.Subscribe(clusterv1.NetTestRunSubject(node), func(m *nats.Msg) {
+		var req clusterv1.NetTestRunRequest
+		_ = json.Unmarshal(m.Data, &req)
+		data, _ := json.Marshal(clusterv1.NetTestRunResponse{
+			SourceAddress: addr, BitsPerSecond: bps, Streams: req.Streams, LinkMbps: 25000, Tool: "iperf2",
+		})
+		_ = m.Respond(data)
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = serveSub.Unsubscribe(); _ = runSub.Unsubscribe() })
+}
+
+func TestNetworkTestRunsEveryDirectionAndRecordsEachResult(t *testing.T) {
+	_, nc, js := testutil.StartTestJetStream(t)
+	ctx := context.Background()
+	d := Deps{NATS: nc, NetworkTests: NewKVNetworkTests(js), NodeIDs: []string{"a", "b", "c"}}
+	fakeNode(t, nc, "a", "10.10.8.4", 23e9, false)
+	fakeNode(t, nc, "b", "10.10.8.5", 22e9, false)
+	fakeNode(t, nc, "c", "10.10.8.6", 0, true)
+
+	started, err := d.StartNetworkTest(ctx, map[string]string{"Plane": "lan", "DurationSeconds": "5", "Streams": "2"})
+	require.NoError(t, err)
+	assert.Equal(t, NetworkTestRunning, started.Status)
+	assert.Len(t, started.Runs, 6)
+
+	_, err = d.StartNetworkTest(ctx, map[string]string{"Plane": "lan"})
+	code, _ := awserrors.ResolveErrorCode(err)
+	assert.Equal(t, awserrors.ErrorIncorrectState, code, "one test at a time across the cluster")
+
+	var done NetworkTest
+	require.Eventually(t, func() bool {
+		out, err := d.DescribeNetworkTests(ctx, map[string]string{"TestId": started.TestID})
+		if err != nil || len(out.Tests) != 1 {
+			return false
+		}
+		done = out.Tests[0]
+		return done.Status != NetworkTestRunning
+	}, 10*time.Second, 20*time.Millisecond)
+
+	assert.Equal(t, NetworkTestComplete, done.Status, "some runs succeeded")
+	byDirection := map[[2]string]NetworkTestResult{}
+	for _, r := range done.Runs {
+		byDirection[[2]string{r.From, r.To}] = r
+	}
+	ab := byDirection[[2]string{"a", "b"}]
+	assert.Equal(t, NetworkTestComplete, ab.Status)
+	assert.InDelta(t, 23e9, ab.BitsPerSecond, 1)
+	assert.Equal(t, 2, ab.Streams)
+	assert.Equal(t, NetworkTestFailed, byDirection[[2]string{"a", "c"}].Status)
+	assert.Contains(t, byDirection[[2]string{"a", "c"}].Error, "not installed")
+	assert.Equal(t, NetworkTestComplete, byDirection[[2]string{"c", "a"}].Status, "c can still send")
+
+	again, err := d.StartNetworkTest(ctx, map[string]string{"Plane": "lan", "Source": "a", "Target": "b", "DurationSeconds": "5"})
+	require.NoError(t, err, "the claim is released when the test ends")
+
+	list, err := d.DescribeNetworkTests(ctx, map[string]string{})
+	require.NoError(t, err)
+	require.Len(t, list.Tests, 2)
+	assert.Equal(t, again.TestID, list.Tests[0].TestID, "newest first")
+
+	require.Eventually(t, func() bool {
+		out, err := d.DescribeNetworkTests(ctx, map[string]string{"TestId": again.TestID})
+		return err == nil && out.Tests[0].Status != NetworkTestRunning
+	}, 10*time.Second, 20*time.Millisecond)
+}
+
+func TestDescribeNetworkTestsUnknownID(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	d := Deps{NetworkTests: NewKVNetworkTests(js)}
+	_, err := d.DescribeNetworkTests(context.Background(), map[string]string{"TestId": "nt-missing"})
+	code, _ := awserrors.ResolveErrorCode(err)
+	assert.Equal(t, awserrors.ErrorResourceNotFound, code)
+}

spinifex/operator/admin/node_metrics.go

@@ -0,0 +1,63 @@
+package admin
+
+import (
+	"encoding/json"
+	"errors"
+	"log/slog"
+	"net/http"
+	"slices"
+	"strconv"
+
+	telemetryv1 "github.com/mulgadc/spinifex/contracts/telemetry/v1"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
+)
+
+// GetNodeMetricsAction streams one node's host load. The dispatcher has
+// already passed the policy and super-admin gates before it runs.
+const GetNodeMetricsAction = "GetNodeMetrics"
+
+func (d Deps) getNodeMetrics(w http.ResponseWriter, inv dispatch.Invocation, queryArgs map[string]string) error {
+	r := inv.Request
+	node := queryArgs["NodeId"]
+	if node == "" {
+		return errors.New(awserrors.ErrorMissingParameter)
+	}
+	if !telemetryv1.ValidLiveNode(node) || (len(d.NodeIDs) > 0 && !slices.Contains(d.NodeIDs, node)) {
+		return awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "unknown NodeId")
+	}
+	var since int64
+	if raw := queryArgs["Since"]; raw != "" {
+		parsed, err := strconv.ParseInt(raw, 10, 64)
+		if err != nil || parsed < 0 {
+			return errors.New(awserrors.ErrorInvalidParameterValue)
+		}
+		since = parsed
+	}
+	if d.NATS == nil {
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+	if !flushSupported(w) {
+		slog.ErrorContext(r.Context(), "GetNodeMetrics: response writer cannot flush")
+		return errors.New(awserrors.ErrorInternalError)
+	}
+
+	release, ok, _ := liveStreams.acquire(inv.AccountID)
+	if !ok {
+		otelsetup.RecordLiveStream(r.Context(), otelsetup.LiveStreamRefusedTotal)
+		return errors.New(awserrors.ErrorRequestLimitExceeded)
+	}
+	defer release()
+
+	return streamLive(w, r, d.NATS, liveSubjects{
+		action: GetNodeMetricsAction,
+		sample: telemetryv1.NodeLiveSampleSubject(node),
+		watch:  telemetryv1.NodeLiveWatchSubject(node),
+		absent: awserrors.ErrorInvalidParameterValue,
+	}, since, func(data []byte) ([]telemetryv1.NodeLiveSample, error) {
+		var ring telemetryv1.NodeLiveRing
+		err := json.Unmarshal(data, &ring)
+		return ring.Samples, err
+	}, func(s telemetryv1.NodeLiveSample) int64 { return s.TimeMillis })
+}

spinifex/operator/admin/registration.go

@@ -57,21 +57,27 @@ type Deps struct {
 	NodeIDs           []string
 	BucketStore       accountteardown.BucketStore
 	SignupMaxAccounts int
+
+	// NetworkTests holds node-to-node throughput tests; nil refuses them.
+	NetworkTests NetworkTestStore
 }
 
 // adminActions are the actions only the super-admin account may call.
 var adminActions = map[string]bool{
-	"GetVersion":        true,
-	"GetNodes":          true,
-	"GetVMs":            true,
-	"GetStorageStatus":  true,
-	"PromoteImage":      true,
-	"GrantModelAccess":  true,
-	"RevokeModelAccess": true,
-	"ListModelAccess":   true,
-	"ListOchreCatalog":  true,
-	"EnsureSQSWorker":   true,
-	"EnsureSQSStore":    true,
+	"GetVersion":           true,
+	"GetNodes":             true,
+	"GetNodeFacts":         true,
+	"StartNetworkTest":     true,
+	"DescribeNetworkTests": true,
+	"GetVMs":               true,
+	"GetStorageStatus":     true,
+	"PromoteImage":         true,
+	"GrantModelAccess":     true,
+	"RevokeModelAccess":    true,
+	"ListModelAccess":      true,
+	"ListOchreCatalog":     true,
+	"EnsureSQSWorker":      true,
+	"EnsureSQSStore":       true,
 }
 
 // AdminActionNames lists the actions only the super-admin account may call, sorted.
@@ -81,7 +87,7 @@ func AdminActionNames() []string {
 
 // ActionNames lists every action the registration dispatches, sorted.
 func ActionNames() []string {
-	names := append(AdminActionNames(), GetInstanceMetricsAction, GetInstanceTypeCapacityAction)
+	names := append(AdminActionNames(), GetInstanceMetricsAction, GetInstanceTypeCapacityAction, GetNodeMetricsAction)
 	slices.Sort(names)
 	return names
 }
@@ -160,11 +166,16 @@ func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 
-	if adminActions[action] && accountID != awsidentifiers.DefaultAccountID() {
+	if (adminActions[action] || action == GetNodeMetricsAction) && accountID != awsidentifiers.DefaultAccountID() {
 		slog.InfoContext(r.Context(), "Spinifex dispatch: non-admin access denied", "action", action, "accountID", accountID)
 		return errors.New(awserrors.ErrorAccessDenied)
 	}
 
+	// Streams rather than returning one document, and only after both gates.
+	if action == GetNodeMetricsAction {
+		return d.getNodeMetrics(w, inv, args)
+	}
+
 	output, err := d.run(r.Context(), action, args, accountID)
 	if err != nil {
 		return err
@@ -194,6 +205,15 @@ func (d Deps) run(ctx context.Context, action string, args map[string]string, ac
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
 		return GetNodes(ctx, d.NATS, d.activeNodes(ctx))
+	case "GetNodeFacts":
+		if d.NATS == nil {
+			return nil, errors.New(awserrors.ErrorServerInternal)
+		}
+		return GetNodeFacts(ctx, d.NATS, d.activeNodes(ctx))
+	case "StartNetworkTest":
+		return d.StartNetworkTest(ctx, args)
+	case "DescribeNetworkTests":
+		return d.DescribeNetworkTests(ctx, args)
 	case GetInstanceTypeCapacityAction:
 		if d.NATS == nil {
 			return nil, errors.New(awserrors.ErrorServerInternal)

spinifex/operator/admin/registration_test.go

@@ -64,7 +64,7 @@ func TestRegistration_GetVersion_Admin(t *testing.T) {
 }
 
 func TestRegistration_EveryActionDeniesNonAdminAccount(t *testing.T) {
-	for _, name := range AdminActionNames() {
+	for _, name := range append(AdminActionNames(), GetNodeMetricsAction) {
 		t.Run(name, func(t *testing.T) {
 			_, err := serve(t, Deps{}, "000000000002", action(name), nil)
 			requireCode(t, err, awserrors.ErrorAccessDenied)
@@ -243,7 +243,7 @@ func TestNewRegistration_DeclaresTheAdminActionsAsItsInventory(t *testing.T) {
 	assert.Equal(t, ServiceName, reg.Service)
 	assert.Equal(t, dispatch.ErrorEnvelopeXML, reg.Errors)
 	assert.ElementsMatch(t, []string{
-		"GetVersion", "GetNodes", "GetVMs", "GetStorageStatus", "PromoteImage",
+		"GetVersion", "GetNodes", "GetNodeFacts", "GetNodeMetrics", "StartNetworkTest", "DescribeNetworkTests", "GetVMs", "GetStorageStatus", "PromoteImage",
 		"GrantModelAccess", "RevokeModelAccess", "ListModelAccess", "ListOchreCatalog",
 		"EnsureSQSStore",
 		"EnsureSQSWorker",

spinifex/operator/cli/admin_node_facts.go

@@ -0,0 +1,30 @@
+package cli
+
+import (
+	"github.com/mulgadc/spinifex/spinifex/runtime/host/nodefacts"
+	"github.com/spf13/cobra"
+)
+
+var adminNodeFactsCmd = &cobra.Command{
+	Use:   "node-facts",
+	Short: "Record this node's hardware and cabling for the operator console",
+}
+
+var adminNodeFactsCollectCmd = &cobra.Command{
+	Use:   "collect",
+	Short: "Run the root-only hardware probes (run by spinifex-daemon.service as root)",
+	Long: `Runs dmidecode, ethtool, udevadm, networkctl lldp and systemd-detect-virt, and
+writes their raw output to /etc/spinifex/node-facts/ readable by the spinifex group.
+The daemon parses the files on request. A probe that fails leaves no file, and the
+console shows that source as unavailable.`,
+	Args:         cobra.NoArgs,
+	SilenceUsage: true,
+	RunE: func(cmd *cobra.Command, _ []string) error {
+		return nodefacts.Collect(cmd.Context(), nodefacts.DefaultDir, "/sys")
+	},
+}
+
+func init() {
+	adminCmd.AddCommand(adminNodeFactsCmd)
+	adminNodeFactsCmd.AddCommand(adminNodeFactsCollectCmd)
+}

spinifex/operator/cli/service.go

@@ -15,6 +15,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/state"
 	"github.com/mulgadc/spinifex/spinifex/domains/network/external"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs/viperblock"
+	"github.com/mulgadc/spinifex/spinifex/runtime/daemon"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/nats"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/northstar"
@@ -535,6 +536,7 @@ var spinifexStartCmd = &cobra.Command{
 
 		defer initTelemetry("spinifex-daemon", false)()
 
+		daemon.SetBuildInfo(Version, Commit)
 		svc, err := service.New("spinifex", clusterConfig)
 
 		if err != nil {

spinifex/operator/host/systemd/units_gen.go

@@ -8,7 +8,7 @@ package systemd
 var Units = map[string]string{
 	"regenerate-ssh-host-keys.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Regenerate SSH host keys\nConditionPathExists=!/etc/ssh/ssh_host_ed25519_key\nBefore=ssh.service\n\n[Service]\nType=oneshot\nExecStart=/usr/bin/ssh-keygen -A\nRemainAfterExit=yes\n# No ExecStop: the keygen process has already exited by the time this unit\n# is stopped, so there is nothing to signal.\nTimeoutStopSec=5\n\n[Install]\nWantedBy=multi-user.target\n",
 	"spinifex-awsgw.service":           "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex AWS Gateway\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-daemon.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-gw\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service awsgw start\nRestart=on-failure\nRestartSec=5\n# No SIGTERM handler in this process: it exits immediately on signal, so this\n# only bounds an unexpectedly wedged process before SIGKILL.\nTimeoutStopSec=10\n\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/awsgw/\nEnvironment=SPINIFEX_AWSGW_TLS_CERT=/etc/spinifex/server.pem\nEnvironment=SPINIFEX_AWSGW_TLS_KEY=/etc/spinifex/server.key\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/awsgw /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
-	"spinifex-daemon.service":          "# spinifex-unit-version: 5\n[Unit]\nDescription=Spinifex Daemon (VM Orchestration)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-viperblock.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service openvswitch-ipsec.service spinifex-hostnet.service\n# Wants (not Requires): boot without openvswitch-ipsec, survive OVS restarts via hostnet\nWants=openvswitch-ipsec.service spinifex-hostnet.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\n# ZFS root: installer raises SPINIFEX_RESERVED_MEM_GB by the ARC ceiling the admission gate cannot see\nEnvironmentFile=-/etc/spinifex/host.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-daemon\nGroup=spinifex\nExecStartPre=/usr/local/bin/spx admin hostnet check\nExecStart=/usr/local/bin/spx service spinifex start\nRestart=on-failure\nRestartSec=5\n# KillMode=process: guest QEMU survives daemon restart; the new daemon reattaches (DDIL)\nKillMode=process\nTimeoutStopSec=15\n# RG-4: infra tier — reaped only after every guest\nOOMScoreAdjust=-500\n# Guest QEMU shares this cgroup: the ceiling tracks the slice budget, not daemon overhead\nMemoryHigh=75%\nMemoryMax=85%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/\nEnvironment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/\n# Pin to host tmpfs: QEMU pidfiles/QMP sockets must survive PrivateTmp namespace churn (DDIL)\nEnvironment=XDG_RUNTIME_DIR=/run/spinifex\n\n# RG-9: daemon tier — privileged by necessity (GPU vfio). No CAP_NET_ADMIN here,\n# so unlike vpcd it still escalates ip/sysctl and NoNewPrivileges stays omitted.\nAmbientCapabilities=CAP_SYS_ADMIN CAP_DAC_OVERRIDE\nProtectSystem=full\nPrivateDevices=no\nDevicePolicy=closed\nDeviceAllow=/dev/kvm rw\nDeviceAllow=/dev/net/tun rw\nDeviceAllow=/dev/vhost-net rw\nDeviceAllow=char-vfio rw\nDeviceAllow=/dev/vfio/vfio rw\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\n# /proc/sys/net/ipv4/conf: ProtectKernelTunables would keep the per-tap IMDS\n# rp_filter/accept_local read-only; /etc/spinifex/firewall: the root helper writes peers.nft.\nReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf -/etc/spinifex/firewall\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n# LimitMEMLOCK=infinity: VFIO pins all guest RAM\nLimitMEMLOCK=infinity\n\n[Install]\nWantedBy=spinifex.target\n",
+	"spinifex-daemon.service":          "# spinifex-unit-version: 6\n[Unit]\nDescription=Spinifex Daemon (VM Orchestration)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-viperblock.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service openvswitch-ipsec.service spinifex-hostnet.service\n# Wants (not Requires): boot without openvswitch-ipsec, survive OVS restarts via hostnet\nWants=openvswitch-ipsec.service spinifex-hostnet.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\n# ZFS root: installer raises SPINIFEX_RESERVED_MEM_GB by the ARC ceiling the admission gate cannot see\nEnvironmentFile=-/etc/spinifex/host.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-daemon\nGroup=spinifex\nExecStartPre=/usr/local/bin/spx admin hostnet check\nExecStartPre=-+/usr/local/bin/spx admin node-facts collect\nExecStart=/usr/local/bin/spx service spinifex start\nRestart=on-failure\nRestartSec=5\n# KillMode=process: guest QEMU survives daemon restart; the new daemon reattaches (DDIL)\nKillMode=process\nTimeoutStopSec=15\n# RG-4: infra tier — reaped only after every guest\nOOMScoreAdjust=-500\n# Guest QEMU shares this cgroup: the ceiling tracks the slice budget, not daemon overhead\nMemoryHigh=75%\nMemoryMax=85%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/\nEnvironment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/\n# Pin to host tmpfs: QEMU pidfiles/QMP sockets must survive PrivateTmp namespace churn (DDIL)\nEnvironment=XDG_RUNTIME_DIR=/run/spinifex\n\n# RG-9: daemon tier — privileged by necessity (GPU vfio). No CAP_NET_ADMIN here,\n# so unlike vpcd it still escalates ip/sysctl and NoNewPrivileges stays omitted.\nAmbientCapabilities=CAP_SYS_ADMIN CAP_DAC_OVERRIDE\nProtectSystem=full\nPrivateDevices=no\nDevicePolicy=closed\nDeviceAllow=/dev/kvm rw\nDeviceAllow=/dev/net/tun rw\nDeviceAllow=/dev/vhost-net rw\nDeviceAllow=char-vfio rw\nDeviceAllow=/dev/vfio/vfio rw\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\n# /proc/sys/net/ipv4/conf: ProtectKernelTunables would keep the per-tap IMDS\n# rp_filter/accept_local read-only; /etc/spinifex/firewall: the root helper writes peers.nft.\nReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf -/etc/spinifex/firewall\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n# LimitMEMLOCK=infinity: VFIO pins all guest RAM\nLimitMEMLOCK=infinity\n\n[Install]\nWantedBy=spinifex.target\n",
 	"spinifex-firewall.service":        "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex host firewall (nftables input policy)\nDocumentation=file:/etc/spinifex/firewall/spinifex.nft\nWants=network-pre.target\nBefore=network-pre.target shutdown.target\nDefaultDependencies=no\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nStandardInput=null\nProtectSystem=full\nProtectHome=true\n# Unconditional: connection tracking loads on every node, so a flow that opens\n# while the policy is not armed survives the arm. The policy itself still loads\n# only where the daemon has written a peer file.\nExecStart=/usr/local/lib/spinifex/spinifex-firewall-apply boot\nExecReload=/usr/local/lib/spinifex/spinifex-firewall-apply boot\nTimeoutStopSec=10s\n\n# No ExecStop: flushing on stop would take vpcd's MASQUERADE and per-EIP FORWARD\n# rules with it. Remove the policy deliberately with\n# `nft delete table inet spinifex_filter`.\n\n[Install]\nWantedBy=sysinit.target\n",
 	"spinifex-guests.slice":            "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex guest VM tier\nBefore=slices.target\n\n[Slice]\n# RG-3: child of spinifex.slice (spinifex-guests -> spinifex), sibling of\n# spinifex-system. Parent for the per-instance spinifex-instance-<id>.slice\n# leaves (0002 RG-7). Inherits the spinifex.slice ceiling; no extra limit here.\n",
 	"spinifex-hostnet.service":         "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex host network wiring (veth pairs, mgmt address, encap route)\n# PartOf/WantedBy ovs-vswitchd: re-applied whenever OVS starts; daemon and vpcd only Want it\nRequires=ovs-vswitchd.service\nAfter=ovs-vswitchd.service NetworkManager-wait-online.service\nPartOf=ovs-vswitchd.service\n# Stopping OVS stops ovn-controller through its Requires= chain and starting OVS\n# does not bring it back, so the start that re-applies hostnet restarts it too\nWants=ovn-controller.service\nBefore=spinifex.target\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/usr/local/bin/spx admin hostnet apply\n# Covers the 120s wait behind setup-ovn.sh plus the 60s wait for links\nTimeoutStartSec=240\nTimeoutStopSec=10\nProtectSystem=full\nProtectHome=yes\nPrivateTmp=yes\n\n[Install]\nWantedBy=spinifex.target ovs-vswitchd.service\n",

spinifex/operator/host/systemd/units_invariants_test.go

@@ -425,6 +425,7 @@ var unitBodyHashes = map[string]map[int]string{
 		3: "8059fe13a678b8cb2a383ae75c069560cf39adece9a9394e17781998c711cd57",
 		4: "f66e1240cc5655f6a13f3b56ca282c73e2fe289f2e51d0ce624d14b90f12f838",
 		5: "d3d25d7eb0051ccf97f70838ddff763bade38bef934b8f85733e048151ac9319",
+		6: "4db31f63c0b9b297a3ec33c34df99c2ec1424374d39f1752f1fd1dc4ec887316",
 	},
 	"spinifex-firewall.service": {
 		1: "01d0a7a79f47eedca02aaab3ff97a0b6462d61b7834e16c7336a7b96034dc392",

spinifex/providers/ebs/viperblock/provider_handlers.go

@@ -1149,7 +1149,7 @@ func snapshotErrorCode(err error) ebsprovider.ErrorCode {
 		return ebsprovider.ErrorCodeNotFound
 	case errors.Is(err, errSnapshotExistsElsewhere):
 		return ebsprovider.ErrorCodeAlreadyExists
-	case errors.Is(err, errVolumeLeaseHeld), errors.Is(err, errNoVolumeLeaseStore), errors.Is(err, errNoVolumeDirtyStore):
+	case errors.Is(err, errVolumeLeaseHeld), errors.Is(err, errNoVolumeLeaseStore):
 		// Exclusive access could not be established. The holder gives the
 		// volume up on detach, so this is worth repeating rather than the
 		// permanent failure an internal error reads as.

spinifex/providers/ebs/viperblock/recovery_dirty_test.go

@@ -1,90 +0,0 @@
-package viperblock
-
-import (
-	"context"
-	"fmt"
-	"path/filepath"
-	"testing"
-	"time"
-
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
-	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
-	"github.com/nats-io/nats.go"
-	"github.com/nats-io/nats.go/jetstream"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// TestRecoverMountedVolumes_StartupOrderMarksTheAdoptedExport follows Run's
-// order from a cold Config: InitLeases, then recovery. An export adopted
-// before the markers existed ran with no record of its unsealed writes.
-func TestRecoverMountedVolumes_StartupOrderMarksTheAdoptedExport(t *testing.T) {
-	_, natsURL := setupEmbeddedNATS(t)
-	cfg := setupTestConfig(t, natsURL)
-	cfg.leases, cfg.dirty = nil, nil
-	cfg.constructVB = fileBackedConstructVB(t)
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	t.Cleanup(nc.Close)
-	require.NoError(t, cfg.InitLeases(t.Context(), nc))
-
-	const volumeName = "vol-recoverymarked1"
-	procRoot := t.TempDir()
-	socket := filepath.Join(t.TempDir(), "m.sock")
-	listenUnix(t, socket)
-	writeProcFixture(t, procRoot, 434343, "nbdkit", nbdkitArgv(cfg.BaseDir, socket, 0, volumeName))
-
-	recoverMountedVolumes(t.Context(), cfg, nc, procRoot)
-
-	mv, ok := findMountedVolume(cfg, volumeName)
-	require.True(t, ok)
-	t.Cleanup(func() { mv.VB.StopWALSyncer() })
-
-	record, found := cfg.dirty.holder(t.Context(), volumeName)
-	require.True(t, found, "an adopted export holds unsealed writes until it seals, so it must be marked")
-	assert.Equal(t, cfg.leaseOwner(), record.Owner)
-	assert.Equal(t, mv.leaseGeneration(), record.Generation)
-}
-
-// TestRecoverMountedVolumes_UnrecordableAdoptLeavesAnInUseExportRunning covers
-// recovery that cannot record the export's writes. It must not adopt it, and a
-// guest still using the export must keep it: refusing is not a reason to kill.
-func TestRecoverMountedVolumes_UnrecordableAdoptLeavesAnInUseExportRunning(t *testing.T) {
-	cases := map[string]func(t *testing.T, cfg *Config, js jetstream.JetStream){
-		"no marker store": func(_ *testing.T, cfg *Config, _ jetstream.JetStream) { cfg.dirty = nil },
-		"marker write fails": func(t *testing.T, _ *Config, js jetstream.JetStream) {
-			require.NoError(t, js.DeleteKeyValue(t.Context(), viperblocklegacyv1.DirtyBucket))
-		},
-	}
-	for name, breakMarkers := range cases {
-		t.Run(name, func(t *testing.T) {
-			_, natsURL := setupEmbeddedNATS(t)
-			cfg := setupTestConfig(t, natsURL)
-			cfg.constructVB = fileBackedConstructVB(t)
-
-			nc, err := nats.Connect(natsURL)
-			require.NoError(t, err)
-			t.Cleanup(nc.Close)
-			js, err := jetstream.New(nc)
-			require.NoError(t, err)
-			breakMarkers(t, cfg, js)
-
-			const volumeName = "vol-recoveryunmarked1"
-			procRoot := t.TempDir()
-			pid := spawnLongRunningProcess(t)
-			socket := filepath.Join(t.TempDir(), "u.sock")
-			listenUnix(t, socket)
-			writeProcFixture(t, procRoot, pid, "nbdkit", nbdkitArgv(cfg.BaseDir, socket, 0, volumeName))
-			writeReferencerFixture(t, procRoot, 919191,
-				fmt.Sprintf("driver=nbd,node-name=vol0,server.type=unix,server.path=%s,export=", socket))
-
-			recoverMountedVolumes(context.Background(), cfg, nc, procRoot)
-
-			_, ok := findMountedVolume(cfg, volumeName)
-			assert.False(t, ok, "an export whose writes cannot be recorded must not be adopted")
-			assert.Never(t, func() bool { return !hostprocess.ProcessAlive(pid) }, 100*time.Millisecond, 10*time.Millisecond,
-				"an export a guest is still using must never be signalled")
-		})
-	}
-}

spinifex/providers/ebs/viperblock/viperblock.go

@@ -687,20 +687,15 @@ func (cfg *Config) LoadEncryptionKey() error {
 	return nil
 }
 
-// InitLeases binds the volume lease store and dirty markers over nc before
-// recovery or any subject: every engine open refuses without both, and an
-// export adopted before the markers exist would run with no unsealed record.
+// InitLeases binds the cluster-wide volume lease store over nc, bound before
+// recovery: without the store every engine open refuses, and the adapter
+// would come up unable to adopt the exports that outlived it.
 func (cfg *Config) InitLeases(ctx context.Context, nc *nats.Conn) error {
 	leases, err := newVolumeLeasesWaiting(ctx, nc, cfg.leaseOwner())
 	if err != nil {
 		return fmt.Errorf("volume leases: %w", err)
 	}
-	dirty, err := newVolumeDirtyWaiting(ctx, nc, cfg.leaseOwner())
-	if err != nil {
-		return fmt.Errorf("volume dirty markers: %w", err)
-	}
 	cfg.leases = cfg.bindLeaseFence(leases)
-	cfg.dirty = dirty
 	cfg.nc = nc
 	return nil
 }

spinifex/providers/ebs/viperblock/viperblockd_integration_test.go

@@ -46,8 +46,6 @@ func installTestVolumeLeases(t *testing.T, cfg *Config, natsURL string) {
 	leases, err := newVolumeLeases(t.Context(), nc, cfg.leaseOwner())
 	require.NoError(t, err)
 	cfg.leases = leases
-	// An open refuses without dirty markers too, so the fixture needs both.
-	cfg.dirty = newTestDirty(t, natsURL, cfg.leaseOwner())
 }
 
 // startTestService runs the adapter's startup sequence (the same one

spinifex/providers/ebs/viperblock/volume_dirty_test.go

@@ -234,20 +234,18 @@ func TestVolumeDirty_OpenFailsWhenTheMarkerCannotBeWritten(t *testing.T) {
 		"the lease has to be released, or the volume is locked to a node that is not writing it")
 }
 
-// TestVolumeDirty_NoStoreRefusesTheOpen pins that an open this node cannot
-// record is refused: a later takeover could not warn about its writes, and a
-// crash would leave nothing to tell them from a clean seal.
-func TestVolumeDirty_NoStoreRefusesTheOpen(t *testing.T) {
+// TestVolumeDirty_NoStoreDoesNotBlockMounts pins the degraded case. A daemon
+// with no marker store still has the lease for live exclusion, and refusing
+// every mount because the bucket is missing is a worse failure than the one
+// being described.
+func TestVolumeDirty_NoStoreDoesNotBlockMounts(t *testing.T) {
 	_, natsURL := setupEmbeddedNATS(t)
 
 	cfg := &Config{leases: newTestLeases(t, natsURL, "node-b")}
 
 	lease, err := cfg.acquireVolumeLease(t.Context(), "vol-dirtynostore")
-	require.ErrorIs(t, err, errNoVolumeDirtyStore)
-	require.Nil(t, lease)
-
-	_, err = newTestLeases(t, natsURL, "node-c").acquire(t.Context(), "vol-dirtynostore")
-	require.NoError(t, err, "a refused open must not leave the volume leased")
+	require.NoError(t, err)
+	require.NotNil(t, lease)
 }
 
 // TestVolumeDirty_ListReportsTheHolder covers the operator surface behind

spinifex/providers/ebs/viperblock/volume_lease.go

@@ -105,10 +105,6 @@ var errVolumeLeaseHeld = errors.New(ec2v1.VolumeHeldElsewhereText)
 // holds this" from "nobody can say who does".
 var errNoVolumeLeaseStore = errors.New("no volume lease store")
 
-// errNoVolumeDirtyStore reports an open refused because this node cannot record
-// that it holds unsealed writes, so a later takeover could not warn about them.
-var errNoVolumeDirtyStore = errors.New("no volume dirty marker store")
-
 // volumeLeaseRecord is what a holder publishes about itself. Generation is the
 // revision that won the race: monotonic across the bucket, so a later opener
 // always carries a higher one than the writer it displaced.
@@ -683,9 +679,6 @@ func (cfg *Config) acquireVolumeLease(ctx context.Context, volumeName string) (*
 	if cfg.leases == nil {
 		return nil, fmt.Errorf("%w: cannot establish exclusive access to %s", errNoVolumeLeaseStore, volumeName)
 	}
-	if cfg.dirty == nil {
-		return nil, fmt.Errorf("%w: cannot record unsealed writes for %s", errNoVolumeDirtyStore, volumeName)
-	}
 	lease, err := cfg.leases.acquire(ctx, volumeName)
 	if err != nil {
 		return nil, err

spinifex/providers/ebs/viperblock/volume_lease_test.go

@@ -138,8 +138,7 @@ func TestVolumeLease_UnmountedSnapshotIsRefusedByTheHolder(t *testing.T) {
 	require.NoError(t, err)
 
 	// node-b has nothing mounted, so this is the unmounted branch.
-	cfg := &Config{NodeName: "node-b", BaseDir: t.TempDir(), leases: newTestLeases(t, natsURL, "node-b"),
-		dirty: newTestDirty(t, natsURL, "node-b")}
+	cfg := &Config{NodeName: "node-b", BaseDir: t.TempDir(), leases: newTestLeases(t, natsURL, "node-b")}
 	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeName, "snap-excl0000000001")
 
 	require.ErrorIs(t, err, errVolumeLeaseHeld, "a snapshot must not open an engine on a volume another node holds")

spinifex/runtime/compute/vm/devices.go

@@ -3,17 +3,24 @@ package vm
 import (
 	"encoding/json"
 	"fmt"
+	"slices"
 	"strconv"
 	"strings"
 )
 
-// IsMMIO reports whether machineType is a memory-mapped I/O bus machine (e.g.
-// microvm). MMIO machines use virtio-*-device transports instead of
-// virtio-*-pci, because they have no PCI bus.
-func IsMMIO(machineType string) bool {
+// IsMicrovm reports whether machineType is the QEMU microvm machine, with or
+// without a PCIe bus.
+func IsMicrovm(machineType string) bool {
 	return strings.HasPrefix(machineType, "microvm")
 }
 
+// IsMMIO reports whether machineType attaches virtio devices over memory-mapped
+// I/O. A microvm without pcie=on has no PCI bus, so it uses virtio-*-device
+// transports; with pcie=on devices are virtio-*-pci like any other machine.
+func IsMMIO(machineType string) bool {
+	return IsMicrovm(machineType) && !slices.Contains(strings.Split(machineType, ","), "pcie=on")
+}
+
 // MaxNICQueues caps the queue pairs on a guest NIC. vhost-net spawns a kernel
 // thread per queue, so an unbounded count would put 64 of them behind a large
 // instance for throughput a handful already saturates.
@@ -309,6 +316,19 @@ func volumeMMIOBlkDevice(volumeID, nodeName, iothreadID string) Device {
 	return Device{Value: strings.Join(args, ",")}
 }
 
+// volumeMicrovmPCIBlkDevice renders a data volume on a microvm with a PCIe bus
+// as virtio-blk-pci with VolumeBlkDeviceArgs' identity and error policy. The
+// microvm has no hot-plug root ports, so the device sits on the default bus.
+func volumeMicrovmPCIBlkDevice(volumeID, nodeName, iothreadID string) Device {
+	args := []string{"virtio-blk-pci"}
+	for _, arg := range VolumeBlkDeviceArgs(volumeID, nodeName, iothreadID, "") {
+		if !strings.HasPrefix(arg, "bus=") {
+			args = append(args, arg)
+		}
+	}
+	return Device{Value: strings.Join(args, ",")}
+}
+
 // VolumeBlkDeviceQMPArgs renders the same virtio-blk-pci arguments as a
 // device_add argument map for AttachVolume's QMP hot-attach path. See
 // VolumeBlkDeviceArgs for why werror/rerror are set here.

spinifex/runtime/compute/vm/devices_test.go

@@ -314,3 +314,27 @@ func TestBlkDeviceMapped_EveryQueueMappedOnce(t *testing.T) {
 		}
 	}
 }
+
+func TestNetDevice_MicrovmWithPCIEIsPCI(t *testing.T) {
+	mt := "microvm,pic=off,pit=off,rtc=on,acpi=on,isa-serial=on,pcie=on"
+	assert.False(t, IsMMIO(mt), "a microvm with a PCIe bus must not use MMIO transports")
+	d := NetDevice(mt, "net0", "02:00:00:aa:bb:cc", 1, 0)
+	assert.Contains(t, d.Value, "virtio-net-pci,netdev=net0")
+	assert.NotContains(t, d.Value, "virtio-net-device")
+}
+
+func TestNetDevice_PlainMicrovmStaysMMIO(t *testing.T) {
+	assert.True(t, IsMMIO("microvm,pic=off,pit=off,rtc=on,acpi=on,isa-serial=on"))
+	assert.Contains(t, NetDevice("microvm", "net0", "", 0, 0).Value, "virtio-net-device")
+}
+
+func TestNetDevice_Q35Unchanged(t *testing.T) {
+	d := NetDevice("q35", "net0", "02:00:00:aa:bb:cc", 1, 0)
+	assert.Equal(t, "virtio-net-pci,netdev=net0,mac=02:00:00:aa:bb:cc,rx_queue_size=1024", d.Value)
+}
+
+func TestIsMicrovm(t *testing.T) {
+	assert.True(t, IsMicrovm("microvm"))
+	assert.True(t, IsMicrovm("microvm,pcie=on"))
+	assert.False(t, IsMicrovm("q35"))
+}

spinifex/runtime/compute/vm/lifecycle.go

@@ -1376,7 +1376,7 @@ type driveConfig struct {
 // with the mounted data volumes appended after its devices. Neither input is
 // modified, so composing again from the same base never accumulates entries.
 func directBootLaunchConfig(base Config, requests []EBSRequest, cpuCount int) (Config, error) {
-	if !IsMMIO(base.MachineType) {
+	if !IsMicrovm(base.MachineType) {
 		return Config{}, fmt.Errorf("direct-boot launch config needs a microvm base, got machine type %q", base.MachineType)
 	}
 	for _, r := range requests {
@@ -1478,13 +1478,13 @@ func buildDrives(requests []EBSRequest, cpuCount int, machineType string) (drive
 
 		default:
 			// microvm has no PCIe root ports, so its data volumes take none.
-			mmio := IsMMIO(machineType)
+			microvm := IsMicrovm(machineType)
 			// A data (non-boot, non-EFI) volume. Allocate a stable hot-plug
 			// port when the persisted state predates port accounting, or the
 			// volume was attached at RunInstances time without one;
 			// freeHotplugEBSPort re-scans requests as mutated so far in this
 			// loop, so two unset data volumes in one call cannot collide.
-			if v.HotplugPort == 0 && !mmio {
+			if v.HotplugPort == 0 && !microvm {
 				port := freeHotplugEBSPort(requests)
 				if port == 0 {
 					return driveConfig{}, fmt.Errorf("no free EBS hot-plug port for volume %s", v.Name)
@@ -1513,9 +1513,12 @@ func buildDrives(requests []EBSRequest, cpuCount int, machineType string) (drive
 
 			cfg.IOThreads = append(cfg.IOThreads, IOThread{ID: iothreadID})
 			cfg.Blockdevs = append(cfg.Blockdevs, blockdev)
-			if mmio {
+			switch {
+			case IsMMIO(machineType):
 				cfg.Devices = append(cfg.Devices, volumeMMIOBlkDevice(v.Name, nodeName, iothreadID))
-			} else {
+			case microvm:
+				cfg.Devices = append(cfg.Devices, volumeMicrovmPCIBlkDevice(v.Name, nodeName, iothreadID))
+			default:
 				cfg.Devices = append(cfg.Devices, VolumeBlkDevice(v.Name, nodeName, iothreadID, HotplugEBSBus(v.HotplugPort)))
 			}
 			// Deliberately no entry in cfg.Drives: the bare anonymous -drive

spinifex/runtime/compute/vm/lifecycle_test.go

@@ -2175,3 +2175,19 @@ func deviceValues(devs []Device) []string {
 	}
 	return out
 }
+
+func TestDirectBootLaunchConfig_PCIEDataVolumesKeepSerialAndID(t *testing.T) {
+	base := directBootBase()
+	base.MachineType = "microvm,pic=off,pit=off,rtc=on,acpi=on,isa-serial=on,pcie=on"
+	got, err := directBootLaunchConfig(base, directBootRequests(), 2)
+	require.NoError(t, err)
+
+	require.Len(t, got.Devices, 3)
+	assert.Equal(t, base.Devices[0], got.Devices[0])
+	assert.Equal(t, Device{Value: "virtio-blk-pci,id=vdisk-vol-data-a,drive=nbd-vol-data-a,iothread=ioth-vol-data-a,serial=voldataa,werror=stop,rerror=stop"}, got.Devices[1])
+	assert.Equal(t, Device{Value: "virtio-blk-pci,id=vdisk-vol-blk,drive=nbd-vol-blk,iothread=ioth-vol-blk,serial=volblk,werror=stop,rerror=stop"}, got.Devices[2])
+	for _, d := range got.Devices[1:] {
+		assert.NotContains(t, d.Value, "bus=", "microvm has no hot-plug root port to name")
+	}
+	assert.Empty(t, got.Drives)
+}

spinifex/runtime/compute/vm/vm.go

@@ -537,7 +537,7 @@ func (cfg *Config) Execute() (*exec.Cmd, error) {
 		args = append(args, "-display", "none")
 	}
 
-	if IsMMIO(cfg.MachineType) && cfg.ConsoleLogPath != "" {
+	if IsMicrovm(cfg.MachineType) && cfg.ConsoleLogPath != "" {
 		// microvm with isa-serial=on: the machine creates the ISA serial device.
 		// Use -chardev file so output is captured without a socket client, then
 		// -serial chardev:console0 wires it to the existing device (not a new one).

spinifex/runtime/compute/vm/vm_test.go

@@ -675,3 +675,22 @@ func TestExecute_NoNameEmitsNoNameArg(t *testing.T) {
 	assert.NoError(t, err)
 	assert.False(t, argExists(cmd.Args[1:], "-name"))
 }
+
+func TestExecute_MicrovmWithPCIEFileChardev(t *testing.T) {
+	cfg := Config{
+		CPUCount:       1,
+		Memory:         512,
+		Architecture:   "x86_64",
+		MachineType:    "microvm,isa-serial=on,pcie=on",
+		ConsoleLogPath: "/var/log/console.log",
+		SerialSocket:   "/run/console.sock",
+		KernelImage:    "/boot/vmlinuz",
+	}
+
+	cmd, err := cfg.Execute()
+	require.NoError(t, err)
+
+	args := cmd.Args[1:]
+	assert.Equal(t, "file,id=console0,path=/var/log/console.log", argValue(args, "-chardev"))
+	assert.Equal(t, "chardev:console0", argValue(args, "-serial"))
+}

spinifex/runtime/daemon/daemon.go

@@ -40,6 +40,7 @@ import (
 	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
 	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
+	telemetryv1 "github.com/mulgadc/spinifex/contracts/telemetry/v1"
 	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/preflight"
@@ -175,6 +176,7 @@ type Daemon struct {
 	vpcService            *ec2vpc.VPCServiceImpl
 	sqsWorker             *sqsWorkerEnsurer
 	sqsStore              *sqsStoreEnsurer
+	insight               *nodeInsight
 	eipService            ec2eip.EIPService
 	elbv2Service          *elbv2domain.ELBv2ServiceImpl
 	eksService            *ekscluster.EKSServiceImpl
@@ -914,6 +916,10 @@ func (d *Daemon) subscribeAll() error {
 	if d.sqsStore == nil {
 		d.sqsStore = newSQSStoreEnsurer(d.buildSQSStoreLauncher)
 	}
+	if d.insight == nil {
+		d.insight = newNodeInsight(d.node)
+		d.insight.live = &nodeLiveSampler{insight: d.insight, nc: d.natsConn, subject: telemetryv1.NodeLiveSampleSubject(d.node)}
+	}
 	subs := []natsSub{
 		// ec2.RunInstances is handled by dynamic per-instance-type subscriptions
 		// managed by ResourceManager.initSubscriptions()
@@ -1054,6 +1060,7 @@ func (d *Daemon) subscribeAll() error {
 		{clusterv1.NodeHealthSubject(d.node), d.handleHealthCheck, ""},
 		{clusterv1.NodesDiscoverSubject, d.handleNodeDiscover, ""},
 		{clusterv1.NodeStatusSubject, d.handleNodeStatus, ""},
+		{clusterv1.NodeFactsSubject, d.handleNodeFacts, ""},
 		{clusterv1.NodeVMsSubject, d.handleNodeVMs, ""},
 		{operatorv1.StorageConfigSubject, d.handleStorageConfig, ""},
 		{"spinifex.image.promote", d.handleSpinifexPromoteImage, "spinifex-workers"},
@@ -1067,6 +1074,13 @@ func (d *Daemon) subscribeAll() error {
 		{"spinifex.cluster.shutdown.persist", d.handleShutdownPersist, ""},
 		{"spinifex.cluster.shutdown.infra", d.handleShutdownInfra, ""},
 	}
+	if clusterv1.ValidNodeID(d.node) {
+		subs = append(subs,
+			natsSub{clusterv1.NetTestServeSubject(d.node), d.handleNetTestServe, ""},
+			natsSub{clusterv1.NetTestRunSubject(d.node), d.handleNetTestRun, ""},
+			natsSub{telemetryv1.NodeLiveWatchSubject(d.node), d.handleNodeLiveWatch, ""},
+		)
+	}
 
 	// ELBv2 operations require a resolved gateway URL.
 	// Without a subscriber the gateway returns nats.ErrNoResponders → ServiceUnavailable.
@@ -1139,6 +1153,7 @@ func (d *Daemon) subscribeAll() error {
 			natsSub{"eks.UpdateAddon", handleNATSRequest(d.node, d.eksService.UpdateAddon), "spinifex-workers"},
 			natsSub{"eks.ListStagedAddonManifests", handleNATSRequest(d.node, d.eksService.ListStagedAddonManifests), "spinifex-workers"},
 			natsSub{"eks.GetRecoveryDirective", handleNATSRequest(d.node, d.eksService.GetRecoveryDirective), "spinifex-workers"},
+			natsSub{"eks.ListManagedWorkers", handleNATSRequest(d.node, d.eksService.ListManagedWorkers), "spinifex-workers"},
 			natsSub{"eks.SetRecoveryDirective", handleNATSRequest(d.node, d.eksService.SetRecoveryDirective), "spinifex-workers"},
 			natsSub{"eks.RestoreSnapshot", handleNATSRequest(d.node, d.eksService.RestoreSnapshot), "spinifex-workers"},
 			natsSub{"eks.TagResource", handleNATSRequest(d.node, d.eksService.TagResource), "spinifex-workers"},

spinifex/runtime/daemon/daemon_system_instance.go

@@ -39,29 +39,24 @@ func resolveENIAccount(owner, fallback string) string {
 	return fallback
 }
 
-// recordENIInstanceOwner stamps the attached instance's owning account onto the
-// ENI record. System VMs run in the system account but plug into a customer ENI,
-// so without this IMDS would resolve the instance and its IAM role under the ENI
-// account and serve no credentials. Best-effort: a miss leaves IMDS falling back
-// to the ENI account, which only matters for the cross-account system path.
-func (d *Daemon) recordENIInstanceOwner(eniAccountID, eniID, instanceOwnerID string) {
-	if d.vpcService == nil || eniID == "" || instanceOwnerID == eniAccountID {
-		return
-	}
-	if err := d.vpcService.UpdateENI(eniAccountID, eniID, func(r *ec2vpc.ENIRecord) {
-		r.InstanceOwnerId = instanceOwnerID
-	}); err != nil {
-		slog.Warn("LaunchSystemInstance: failed to record ENI instance owner",
-			"eniId", eniID, "instanceOwner", instanceOwnerID, "err", err)
+// eniInstanceOwner is the owner to stamp on an ENI record when a system VM
+// attaches it. System VMs run in the system account but plug into a customer
+// ENI, so without the stamp IMDS resolves the instance under the ENI account
+// and serves no credentials. It is empty when both accounts are the same.
+func eniInstanceOwner(eniAccountID, instanceOwnerID string) string {
+	if instanceOwnerID == eniAccountID {
+		return ""
 	}
+	return instanceOwnerID
 }
 
-// attachExtraENI attaches one pre-created extra ENI and, when the caller asked
-// for an explicit DeleteOnTermination, persists it onto the record. An ENI that
-// has to outlive its VM is only safe once that flag is stored, so a failed stamp
-// fails the attach rather than leaving a disposable NIC behind.
-func (d *Daemon) attachExtraENI(eniAccountID string, extra systeminstance.ExtraENIInput, instanceID string, deviceIndex int64) error {
-	if _, err := d.vpcService.AttachENI(eniAccountID, extra.ENIID, instanceID, deviceIndex); err != nil {
+// attachExtraENI attaches one pre-created extra ENI, stamping the instance
+// owner in the attach write, and, when the caller asked for an explicit
+// DeleteOnTermination, persists it onto the record. An ENI that has to outlive
+// its VM is only safe once that flag is stored, so a failed stamp fails the
+// attach rather than leaving a disposable NIC behind.
+func (d *Daemon) attachExtraENI(eniAccountID string, extra systeminstance.ExtraENIInput, instanceID string, deviceIndex int64, instanceOwnerID string) error {
+	if _, err := d.vpcService.AttachENIWithOwner(eniAccountID, extra.ENIID, instanceID, deviceIndex, eniInstanceOwner(eniAccountID, instanceOwnerID)); err != nil {
 		return err
 	}
 	if extra.DeleteOnTermination == nil {
@@ -188,10 +183,11 @@ func (d *Daemon) launchDirectSystemInstance(input *elbv2domain.SystemInstanceInp
 		}
 		// Mark ENI as attached to this instance
 		if d.vpcService != nil {
-			if _, attachErr := d.vpcService.AttachENI(eniAccountID, instance.ENIId, instance.ID, 0); attachErr != nil {
-				slog.Warn("LaunchSystemInstance: failed to attach ENI", "eniId", instance.ENIId, "instanceId", instance.ID, "err", attachErr)
+			if _, attachErr := d.vpcService.AttachENIWithOwner(eniAccountID, instance.ENIId, instance.ID, 0, eniInstanceOwner(eniAccountID, accountID)); attachErr != nil {
+				slog.Error("LaunchSystemInstance: failed to attach ENI", "eniId", instance.ENIId, "instanceId", instance.ID, "instanceOwner", accountID, "err", attachErr)
+				d.cleanupFailedSystemInstance(instance, instanceType)
+				return nil, fmt.Errorf("attach ENI %s: %w", instance.ENIId, attachErr)
 			}
-			d.recordENIInstanceOwner(eniAccountID, instance.ENIId, accountID)
 		}
 		// Attach any additional pre-created ENIs (multi-subnet ALB VMs).
 		// Use the launcher input slice as the source of truth so the VM
@@ -203,12 +199,11 @@ func (d *Daemon) launchDirectSystemInstance(input *elbv2domain.SystemInstanceInp
 				// customer-VPC ENI) lives in its own account; the ENI record is
 				// account-keyed, so attach under extra.AccountID when set.
 				extraAccount := resolveENIAccount(extra.AccountID, eniAccountID)
-				if attachErr := d.attachExtraENI(extraAccount, extra, instance.ID, int64(idx+1)); attachErr != nil {
+				if attachErr := d.attachExtraENI(extraAccount, extra, instance.ID, int64(idx+1), accountID); attachErr != nil {
 					slog.Error("LaunchSystemInstance: failed to attach extra ENI", "eniId", extra.ENIID, "instanceId", instance.ID, "err", attachErr)
 					d.cleanupFailedSystemInstance(instance, instanceType)
 					return nil, fmt.Errorf("attach extra ENI %s: %w", extra.ENIID, attachErr)
 				}
-				d.recordENIInstanceOwner(extraAccount, extra.ENIID, accountID)
 			}
 			instance.ExtraENIs = append(instance.ExtraENIs, vm.ExtraENI{
 				ENIID:         extra.ENIID,
@@ -240,7 +235,6 @@ func (d *Daemon) launchDirectSystemInstance(input *elbv2domain.SystemInstanceInp
 		if _, attachErr := d.vpcService.AttachENI(accountID, instance.ENIId, instance.ID, 0); attachErr != nil {
 			slog.Warn("LaunchSystemInstance: failed to attach auto-created ENI", "eniId", instance.ENIId, "instanceId", instance.ID, "err", attachErr)
 		}
-		d.recordENIInstanceOwner(accountID, instance.ENIId, accountID)
 	}
 
 	// Allocate public IP for internet-facing ALBs. Route through the EIP
@@ -549,7 +543,7 @@ func (d *Daemon) launchAMISystemInstance(input *systeminstance.SystemInstanceInp
 	for idx, extra := range input.ExtraENIs {
 		if d.vpcService != nil {
 			extraAccount := resolveENIAccount(extra.AccountID, input.AccountID)
-			if attachErr := d.attachExtraENI(extraAccount, extra, inst.ID, int64(idx+1)); attachErr != nil {
+			if attachErr := d.attachExtraENI(extraAccount, extra, inst.ID, int64(idx+1), ""); attachErr != nil {
 				slog.Error("launchAMISystemInstance: failed to attach extra ENI", "eniId", extra.ENIID, "instanceId", inst.ID, "err", attachErr)
 				d.vmMgr.MarkFailed(context.Background(), inst, "extra_eni_attach_failed")
 				return nil, fmt.Errorf("attach extra ENI %s: %w", extra.ENIID, attachErr)
@@ -865,15 +859,14 @@ func (d *Daemon) buildDirectBootConfig(instanceID string, input *elbv2domain.Sys
 		imageDir = guest.imageDir()
 	}
 
-	// Build kernel cmdline.
-	//
-	// QEMU microvm maps virtio-mmio slots at 0xfeb00000 + n*0x200 with IRQ 5+n.
-	// With acpi=on the guest also finds them through ACPI, as data disks are
-	// (they get no parameter); the NIC parameters are kept, matching that map.
+	// Build kernel cmdline. Devices sit on the PCIe bus (pcie=on), so no
+	// virtio_mmio.device= arguments.
 	//
 	// Boot-time performance flags:
-	//   quiet loglevel=3        — suppress printk to ttyS0 (115200 baud serial
-	//                             writes dominate boot time; warnings+errors only)
+	//   quiet                   — suppress info printk to ttyS0 (115200 baud serial
+	//                             writes dominate boot time)
+	//   loglevel                — 3 on SQS guests (errors only); 7 on the LB so
+	//                             kernel warnings reach the console log
 	//   mitigations=off         — skip spectre/meltdown microcode patching at
 	//                             boot. Load balancer only: KVM guest on trusted
 	//                             host CPU, single-tenant LB workload, no untrusted
@@ -885,23 +878,12 @@ func (d *Daemon) buildDirectBootConfig(instanceID string, input *elbv2domain.Sys
 	//   reboot=t                — triple-fault reboot; skip ACPI/keyboard probes
 	//   i8042.no{pnp,aux,kbd}   — microvm has no PS/2; skip i8042 probe timeouts
 	var sb strings.Builder
-	sb.WriteString("console=ttyS0 quiet loglevel=3")
 	if guest == nil {
-		sb.WriteString(" mitigations=off")
+		sb.WriteString("console=ttyS0 quiet loglevel=7 mitigations=off")
+	} else {
+		sb.WriteString("console=ttyS0 quiet loglevel=3")
 	}
 	sb.WriteString(" tsc=reliable no_timer_check reboot=t i8042.nopnp i8042.noaux i8042.nokbd")
-	// Slots must stay packed in the same order QEMU actually creates
-	// -device entries (buildNICNetdevs), so an unprovisioned mgmt NIC
-	// (skipped there) must not reserve a slot here either.
-	slot := 0
-	for i, nic := range input.NICs {
-		if !nicProvisioned(i, nic) {
-			continue
-		}
-		fmt.Fprintf(&sb, " virtio_mmio.device=0x200@0x%x:%d",
-			0xfeb00000+slot*0x200, 5+slot)
-		slot++
-	}
 	cmdline := sb.String()
 
 	machineType := microvmMachineType()
@@ -938,10 +920,10 @@ type nicNetdevResult struct {
 }
 
 // microvmMachineType returns the QEMU -M string for the microvm machine.
-// isa-serial is kept on permanently for per-VM console log access — the
-// boot-time saving from turning it off is not worth losing console output.
+// pcie=on gives NICs a PCI bus with MSI-X; virtio-mmio's shared level-triggered
+// lines stalled a NIC's RX path. isa-serial stays on for per-VM console logs.
 func microvmMachineType() string {
-	return "microvm,pic=off,pit=off,rtc=on,acpi=on,isa-serial=on"
+	return "microvm,pic=off,pit=off,rtc=on,acpi=on,isa-serial=on,pcie=on"
 }
 
 // buildNICNetdevs produces QEMU -netdev and -device entries for each NIC in

spinifex/runtime/daemon/daemon_system_instance_coverage_test.go

@@ -229,16 +229,17 @@ func TestLaunchSystemInstance_PreCreatedENI(t *testing.T) {
 	f := newSysInstFixture(t)
 	d := f.d
 
-	t.Run("made-up ENI still reaches launch", func(t *testing.T) {
+	t.Run("made-up ENI fails the launch at the attach", func(t *testing.T) {
 		sysInstLaunchFails(d)
 		in := f.input(f.newENI(t))
 		in.ENIID = "eni-madeup"
 		in.IamInstanceProfileArn = "arn:aws:iam::123456789012:instance-profile/lb"
 
-		// The attach and owner stamp are best-effort, so only the launch fails.
+		// A VM whose ENI cannot be attached, and so cannot be stamped with
+		// its owner, can never authenticate and must not be launched.
 		_, err := d.LaunchSystemInstance(in)
-		require.ErrorIs(t, err, errSysInstLaunchRefused)
-		assert.Contains(t, err.Error(), "launch instance")
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "attach ENI eni-madeup")
 	})
 
 	t.Run("cross-account ENI and extras are attached and stamped", func(t *testing.T) {

spinifex/runtime/daemon/daemon_system_instance_owner_test.go

@@ -0,0 +1,66 @@
+package daemon
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// A system VM on a customer ENI is only resolvable by IMDS once the ENI record
+// names the system account as the instance owner. Without it the VM never gets
+// role credentials and its load balancer is reaped as failed.
+func TestLaunchSystemInstance_OwnerStampSurvivesStaleReplicaRead(t *testing.T) {
+	f := newSysInstFixture(t)
+	var inner jetstream.KeyValue
+	f.d.vpcService.WrapENIKVForTest(func(kv jetstream.KeyValue) jetstream.KeyValue {
+		inner = kv
+		return testutil.NewStaleReadKV(kv)
+	})
+	sysInstLaunchRuns(f.d)
+
+	eni := f.newENI(t)
+	out, err := f.d.LaunchSystemInstance(f.input(eni))
+	require.NoError(t, err)
+
+	// Read the stored record directly: the wrapped bucket would serve the
+	// check itself from a lagging replica.
+	entry, err := inner.Get(context.Background(), kvutil.AccountKey(testAccountID, aws.StringValue(eni.NetworkInterfaceId)))
+	require.NoError(t, err)
+	var rec ec2vpc.ENIRecord
+	require.NoError(t, json.Unmarshal(entry.Value(), &rec))
+	assert.Equal(t, out.InstanceID, rec.InstanceId)
+	assert.Equal(t, awsidentifiers.GlobalAccountID, rec.InstanceOwnerId,
+		"the owner stamp must land even when the read behind it was served by a lagging replica")
+}
+
+func TestLaunchSystemInstance_UnwritableOwnerStampFailsTheLaunch(t *testing.T) {
+	f := newSysInstFixture(t)
+	f.d.vpcService.WrapENIKVForTest(func(kv jetstream.KeyValue) jetstream.KeyValue {
+		return testutil.NewRejectingKV(kv, func(_ string, value []byte) error {
+			var rec ec2vpc.ENIRecord
+			if json.Unmarshal(value, &rec) == nil && rec.InstanceOwnerId != "" {
+				return errors.New("stream unavailable")
+			}
+			return nil
+		})
+	})
+	sysInstLaunchRuns(f.d)
+
+	eni := f.newENI(t)
+	_, err := f.d.LaunchSystemInstance(f.input(eni))
+	require.Error(t, err, "a VM whose ENI cannot name its owner can never authenticate and must not be launched")
+
+	require.Eventually(t, func() bool { return len(f.d.vmMgr.Snapshot()) == 0 }, 5*time.Second, 20*time.Millisecond,
+		"the failed launch must leave no VM behind")
+}

spinifex/runtime/daemon/daemon_system_instance_pci_test.go

@@ -0,0 +1,66 @@
+package daemon
+
+import (
+	"os"
+	"testing"
+
+	elbv2domain "github.com/mulgadc/spinifex/spinifex/domains/elbv2"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func pciTestInput() *elbv2domain.SystemInstanceInput {
+	return &elbv2domain.SystemInstanceInput{
+		ENIID: "eni-pci0000000000001",
+		NICs: []elbv2domain.NICConfig{
+			{MAC: "02:aa:bb:cc:dd:01", IsDefault: true},
+			{MAC: "02:aa:bb:cc:dd:02"},
+			{MAC: "02:aa:bb:cc:dd:03"},
+		},
+	}
+}
+
+func assertPCINICs(t *testing.T, cfg vm.Config) {
+	t.Helper()
+	assert.Contains(t, cfg.MachineType, "microvm,")
+	assert.Contains(t, cfg.MachineType, "pcie=on")
+	require.Len(t, cfg.Devices, 3)
+	for i, mac := range []string{"02:aa:bb:cc:dd:01", "02:aa:bb:cc:dd:02", "02:aa:bb:cc:dd:03"} {
+		assert.Contains(t, cfg.Devices[i].Value, "virtio-net-pci,netdev=net", "NIC %d must be a PCI device", i)
+		assert.NotContains(t, cfg.Devices[i].Value, "virtio-net-device")
+		assert.Contains(t, cfg.Devices[i].Value, "mac="+mac, "MAC order must be unchanged")
+	}
+	assert.NotContains(t, cfg.KernelCmdline, "virtio_mmio.device=")
+}
+
+func TestBuildDirectBootConfig_LBNICsArePCI(t *testing.T) {
+	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
+	d := &Daemon{resourceMgr: &ResourceManager{}}
+
+	cfg, err := d.buildDirectBootConfig("i-pci-lb", pciTestInput(), nil)
+	require.NoError(t, err)
+	assertPCINICs(t, cfg)
+	assert.Contains(t, cfg.KernelCmdline, "loglevel=7", "LB guest kernel warnings must reach the console log")
+	for _, e := range cfg.FwCfg {
+		assert.NoError(t, os.Remove(e.File))
+	}
+}
+
+func TestBuildDirectBootConfig_SQSWorkerNICsArePCI(t *testing.T) {
+	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
+	d := &Daemon{resourceMgr: &ResourceManager{}}
+	w := &sqsWorkerLaunch{volumeID: "vol-pci", region: "ap-southeast-2", gatewayURL: "https://gw.example:9999", caCert: "ca"}
+
+	cfg, err := d.buildDirectBootConfig("i-pci-sqs", pciTestInput(), w)
+	require.NoError(t, err)
+	assertPCINICs(t, cfg)
+	for _, e := range cfg.FwCfg {
+		assert.NoError(t, os.Remove(e.File))
+	}
+}
+
+func TestMicrovmMachineType_HasPCIE(t *testing.T) {
+	assert.Contains(t, microvmMachineType(), "pcie=on")
+	assert.Contains(t, microvmMachineType(), "isa-serial=on")
+}

spinifex/runtime/daemon/daemon_system_instance_test.go

@@ -294,8 +294,8 @@ func TestBuildNICNetdevs_SingleNIC(t *testing.T) {
 	require.Len(t, res.netdevs, 1)
 	require.Len(t, res.devices, 1)
 	assert.Contains(t, res.netdevs[0].Value, "tap,id=net0,")
-	// microvm machine type → virtio-net-device transport
-	assert.Contains(t, res.devices[0].Value, "virtio-net-device,netdev=net0")
+	// microvm with pcie=on → virtio-net-pci transport
+	assert.Contains(t, res.devices[0].Value, "virtio-net-pci,netdev=net0")
 	assert.Contains(t, res.devices[0].Value, "02:aa:bb:cc:dd:01")
 }
 
@@ -547,7 +547,7 @@ func TestWriteFwCfgBlobs_InvalidNICs_NoDefault(t *testing.T) {
 // buildDirectBootConfig — mmio slot packing
 // ---------------------------------------------------------------------------
 
-func TestBuildDirectBootConfig_MmioSlotsPacked_MgmtSkipped(t *testing.T) {
+func TestBuildDirectBootConfig_PCINICsPacked_MgmtSkipped(t *testing.T) {
 	// NIC[1] (mgmt) unprovisioned (no br-mgmt) must not reserve an mmio slot;
 	// remaining NICs (primary, extra ENI) get packed, contiguous slots.
 	tmpDir := t.TempDir()
@@ -569,10 +569,11 @@ func TestBuildDirectBootConfig_MmioSlotsPacked_MgmtSkipped(t *testing.T) {
 	cfg, err := d.buildDirectBootConfig("i-mmio-test", input, nil)
 	require.NoError(t, err)
 
-	// Only 2 real devices (mgmt skipped) → slots packed at 0 and 1, not 0 and 2.
-	assert.Contains(t, cfg.KernelCmdline, "virtio_mmio.device=0x200@0xfeb00000:5")
-	assert.Contains(t, cfg.KernelCmdline, "virtio_mmio.device=0x200@0xfeb00200:6")
-	assert.NotContains(t, cfg.KernelCmdline, "virtio_mmio.device=0x200@0xfeb00400:7")
+	// Only 2 real devices (mgmt skipped), both PCI and in NIC order.
+	assert.NotContains(t, cfg.KernelCmdline, "virtio_mmio.device=")
+	require.Len(t, cfg.Devices, 2)
+	assert.Contains(t, cfg.Devices[0].Value, "virtio-net-pci,netdev=net0,mac=02:aa:bb:cc:dd:01")
+	assert.Contains(t, cfg.Devices[1].Value, "virtio-net-pci,netdev=net2,mac=02:aa:bb:cc:dd:03")
 
 	require.Len(t, cfg.NetDevs, 2)
 	require.Len(t, cfg.Devices, 2)
@@ -582,7 +583,7 @@ func TestBuildDirectBootConfig_MmioSlotsPacked_MgmtSkipped(t *testing.T) {
 	}
 }
 
-func TestBuildDirectBootConfig_MmioSlotsUnchanged_MgmtProvisioned(t *testing.T) {
+func TestBuildDirectBootConfig_PCINICsUnchanged_MgmtProvisioned(t *testing.T) {
 	// Multi-node host with br-mgmt: all NICs provisioned, slots stay 1:1
 	// with NIC index as before this fix.
 	tmpDir := t.TempDir()
@@ -600,8 +601,10 @@ func TestBuildDirectBootConfig_MmioSlotsUnchanged_MgmtProvisioned(t *testing.T)
 	cfg, err := d.buildDirectBootConfig("i-mmio-test2", input, nil)
 	require.NoError(t, err)
 
-	assert.Contains(t, cfg.KernelCmdline, "virtio_mmio.device=0x200@0xfeb00000:5")
-	assert.Contains(t, cfg.KernelCmdline, "virtio_mmio.device=0x200@0xfeb00200:6")
+	assert.NotContains(t, cfg.KernelCmdline, "virtio_mmio.device=")
+	require.Len(t, cfg.Devices, 2)
+	assert.Contains(t, cfg.Devices[0].Value, "virtio-net-pci,netdev=net0,mac=02:aa:bb:cc:dd:01")
+	assert.Contains(t, cfg.Devices[1].Value, "virtio-net-pci,netdev=net1,mac=02:aa:bb:cc:dd:02")
 	require.Len(t, cfg.NetDevs, 2)
 	require.Len(t, cfg.Devices, 2)
 
@@ -920,7 +923,7 @@ func TestAttachExtraENI_DeleteOnTerminationFalseSurvivesTerminate(t *testing.T)
 	require.NoError(t, f.daemon.attachExtraENI(testAccountID, systeminstance.ExtraENIInput{
 		ENIID:               f.eniID,
 		DeleteOnTermination: aws.Bool(false),
-	}, f.vmInst.ID, 1))
+	}, f.vmInst.ID, 1, ""))
 
 	rec, err := f.daemon.vpcService.GetENIRecord(testAccountID, f.eniID)
 	require.NoError(t, err)
@@ -944,7 +947,7 @@ func TestAttachExtraENI_NilDeleteOnTerminationStaysDisposable(t *testing.T) {
 
 	require.NoError(t, f.daemon.attachExtraENI(testAccountID, systeminstance.ExtraENIInput{
 		ENIID: f.eniID,
-	}, f.vmInst.ID, 1))
+	}, f.vmInst.ID, 1, ""))
 
 	require.NoError(t, newInstanceCleanerAdapter(f.daemon).DetachAndDeleteENI(f.vmInst))
 

spinifex/runtime/daemon/node_insight.go

@@ -0,0 +1,397 @@
+package daemon
+
+import (
+	"context"
+	"encoding/json"
+	"log/slog"
+	"net"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
+	telemetryv1 "github.com/mulgadc/spinifex/contracts/telemetry/v1"
+	"github.com/mulgadc/spinifex/spinifex/runtime/host/nettest"
+	"github.com/mulgadc/spinifex/spinifex/runtime/host/nodefacts"
+	"github.com/nats-io/nats.go"
+)
+
+// Build-time version and commit, set by the cmd package before the daemon starts.
+var (
+	buildVersion = "dev"
+	buildCommit  = "unknown"
+)
+
+// SetBuildInfo records the build-time version and commit the node reports.
+func SetBuildInfo(version, commit string) {
+	buildVersion, buildCommit = version, commit
+}
+
+// nodeFactsTTL bounds how often a request rebuilds the facts; a re-cabled
+// port shows within it.
+const nodeFactsTTL = 15 * time.Second
+
+// nodeInsight serves the operator's view of this host: its facts, a live
+// sampler of its own load, and the two halves of a throughput test.
+type nodeInsight struct {
+	node string
+	env  nodefacts.Env
+
+	mu      sync.Mutex
+	facts   clusterv1.NodeFacts
+	factsAt time.Time
+
+	live   *nodeLiveSampler
+	server nettest.Server
+	// testing serialises the client side, so two tests never share a link.
+	testing sync.Mutex
+}
+
+func newNodeInsight(node string) *nodeInsight {
+	return &nodeInsight{node: node, env: nodefacts.DefaultEnv(node, buildVersion, buildCommit)}
+}
+
+func (n *nodeInsight) currentFacts(ctx context.Context) clusterv1.NodeFacts {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	if time.Since(n.factsAt) < nodeFactsTTL {
+		return n.facts
+	}
+	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+	n.facts = nodefacts.Build(ctx, n.env)
+	n.factsAt = time.Now()
+	return n.facts
+}
+
+func (n *nodeInsight) planeAddress(ctx context.Context, plane string) string {
+	addr, _ := n.planeSubnet(ctx, plane)
+	return addr
+}
+
+// planeSubnet is the node's address on plane and the subnet it sits in. A
+// client may only be pointed inside that subnet, never at an arbitrary host.
+func (n *nodeInsight) planeSubnet(ctx context.Context, plane string) (string, *net.IPNet) {
+	for _, p := range n.currentFacts(ctx).Planes {
+		if p.Plane != plane || p.Address == "" {
+			continue
+		}
+		_, subnet, err := net.ParseCIDR(p.Address + "/" + strconv.Itoa(p.Prefix))
+		if err != nil {
+			return p.Address, nil
+		}
+		return p.Address, subnet
+	}
+	return "", nil
+}
+
+func (n *nodeInsight) planeLink(ctx context.Context, plane string) (speed, mtu int) {
+	f := n.currentFacts(ctx)
+	for _, p := range f.Planes {
+		if p.Plane != plane {
+			continue
+		}
+		mtu = p.MTU
+		for _, nic := range f.NICs {
+			if slices.Contains(p.NICs, nic.Name) {
+				speed = max(speed, nic.SpeedMbps)
+			}
+		}
+	}
+	return speed, mtu
+}
+
+func (d *Daemon) handleNodeFacts(msg *nats.Msg) string {
+	respondWithJSON(d.node, msg, d.insight.currentFacts(context.Background()))
+	return outcomeSuccess
+}
+
+func validPlane(plane string) bool {
+	return plane == clusterv1.PlaneWAN || plane == clusterv1.PlaneLAN || plane == clusterv1.PlaneVPC
+}
+
+func (d *Daemon) handleNetTestServe(msg *nats.Msg) string {
+	var req clusterv1.NetTestServeRequest
+	if err := json.Unmarshal(msg.Data, &req); err != nil || !validPlane(req.Plane) ||
+		req.DurationSeconds < nettest.MinDuration || req.DurationSeconds > nettest.MaxDuration {
+		respondWithJSON(d.node, msg, clusterv1.NetTestServeResponse{Error: "invalid request"})
+		return outcomeError
+	}
+	addr := d.insight.planeAddress(context.Background(), req.Plane)
+	if addr == "" {
+		respondWithJSON(d.node, msg, clusterv1.NetTestServeResponse{Error: "this node has no address on the " + req.Plane + " plane"})
+		return outcomeError
+	}
+	if err := d.insight.server.Serve(addr, req.DurationSeconds); err != nil {
+		respondWithJSON(d.node, msg, clusterv1.NetTestServeResponse{Error: err.Error()})
+		return outcomeError
+	}
+	respondWithJSON(d.node, msg, clusterv1.NetTestServeResponse{Address: addr, Port: clusterv1.NetTestPort})
+	return outcomeSuccess
+}
+
+// handleNetTestRun runs the client in its own goroutine: it takes the test's
+// duration, and the subscription must keep serving other requests meanwhile.
+func (d *Daemon) handleNetTestRun(msg *nats.Msg) string {
+	var req clusterv1.NetTestRunRequest
+	if err := json.Unmarshal(msg.Data, &req); err != nil || !validPlane(req.Plane) ||
+		req.DurationSeconds < nettest.MinDuration || req.DurationSeconds > nettest.MaxDuration ||
+		req.Streams < 1 || req.Streams > nettest.MaxStreams || req.Port != clusterv1.NetTestPort {
+		respondWithJSON(d.node, msg, clusterv1.NetTestRunResponse{Error: "invalid request"})
+		return outcomeError
+	}
+	go func() {
+		ctx := context.Background()
+		addr, subnet := d.insight.planeSubnet(ctx, req.Plane)
+		if addr == "" {
+			respondWithJSON(d.node, msg, clusterv1.NetTestRunResponse{Error: "this node has no address on the " + req.Plane + " plane"})
+			return
+		}
+		if target := net.ParseIP(req.Address); subnet == nil || target == nil || !subnet.Contains(target) {
+			respondWithJSON(d.node, msg, clusterv1.NetTestRunResponse{Error: "target is outside this node's " + req.Plane + " subnet"})
+			return
+		}
+		if !d.insight.testing.TryLock() {
+			respondWithJSON(d.node, msg, clusterv1.NetTestRunResponse{Error: "a test is already running from this node"})
+			return
+		}
+		defer d.insight.testing.Unlock()
+
+		res, err := nettest.Run(ctx, req, addr)
+		res.LinkMbps, res.MTU = d.insight.planeLink(ctx, req.Plane)
+		if err != nil {
+			res.Error = err.Error()
+			slog.Info("node network test failed", "plane", req.Plane, "target", req.Address, "err", err)
+		}
+		respondWithJSON(d.node, msg, res)
+	}()
+	return outcomeSuccess
+}
+
+// nodeLiveSampler samples this host every period while somebody watches, as
+// the instance sampler does, and drops its ring when the lease lapses.
+type nodeLiveSampler struct {
+	insight *nodeInsight
+	nc      *nats.Conn
+	subject string
+
+	mu      sync.Mutex
+	until   time.Time
+	running bool
+	ring    []telemetryv1.NodeLiveSample
+}
+
+func (d *Daemon) handleNodeLiveWatch(msg *nats.Msg) string {
+	var watch telemetryv1.LiveWatch
+	if len(msg.Data) > 0 {
+		if err := json.Unmarshal(msg.Data, &watch); err != nil {
+			return outcomeError
+		}
+	}
+	samples := d.insight.live.watch(watch.SinceMillis)
+	if msg.Reply != "" {
+		data, err := json.Marshal(telemetryv1.NodeLiveRing{Samples: samples})
+		if err == nil {
+			_ = msg.Respond(data)
+		}
+	}
+	return outcomeSuccess
+}
+
+func (l *nodeLiveSampler) watch(sinceMillis int64) []telemetryv1.NodeLiveSample {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.until = time.Now().Add(telemetryv1.LiveWatchLeaseMillis * time.Millisecond)
+	if !l.running {
+		l.running = true
+		go l.run()
+	}
+	out := make([]telemetryv1.NodeLiveSample, 0, len(l.ring))
+	for _, s := range l.ring {
+		if s.TimeMillis > sinceMillis {
+			out = append(out, s)
+		}
+	}
+	return out
+}
+
+func (l *nodeLiveSampler) lapsed() bool {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if time.Now().Before(l.until) {
+		return false
+	}
+	l.running = false
+	l.ring = nil
+	return true
+}
+
+func (l *nodeLiveSampler) record(s telemetryv1.NodeLiveSample) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if len(l.ring) == telemetryv1.LiveRingSamples {
+		l.ring = append(l.ring[:0], l.ring[1:]...)
+	}
+	l.ring = append(l.ring, s)
+}
+
+func (l *nodeLiveSampler) run() {
+	ticker := time.NewTicker(telemetryv1.LiveSamplePeriodMillis * time.Millisecond)
+	defer ticker.Stop()
+	var prev *hostCounters
+	for range ticker.C {
+		if l.lapsed() {
+			return
+		}
+		cur := readHostCounters(l.insight.env.ProcRoot, l.insight.env.SysRoot, planeNICs(l.insight.currentFacts(context.Background())))
+		if prev != nil {
+			if s, ok := hostRates(prev, cur); ok {
+				l.record(s)
+				if data, err := json.Marshal(s); err == nil {
+					_ = l.nc.Publish(l.subject, data)
+				}
+			}
+		}
+		prev = cur
+	}
+}
+
+// planeNICs keys each set of NICs by the planes it carries, so a NIC that
+// carries lan and vpc is counted once, as "lan+vpc".
+func planeNICs(f clusterv1.NodeFacts) map[string][]string {
+	byNICs := map[string][]string{}
+	nics := map[string][]string{}
+	for _, p := range f.Planes {
+		key := strings.Join(p.NICs, ",")
+		if key == "" {
+			key = "bridge:" + p.Bridge
+		}
+		byNICs[key] = append(byNICs[key], p.Plane)
+		if _, ok := nics[key]; !ok {
+			if len(p.NICs) == 0 {
+				nics[key] = []string{p.Bridge}
+			} else {
+				nics[key] = p.NICs
+			}
+		}
+	}
+	out := map[string][]string{}
+	for key, planes := range byNICs {
+		out[strings.Join(planes, "+")] = nics[key]
+	}
+	return out
+}
+
+type nicCounters struct{ rxBytes, txBytes, rxPackets, txPackets uint64 }
+
+type hostCounters struct {
+	at         time.Time
+	busy, idle uint64
+	memPercent float64
+	load1      float64
+	planes     map[string]nicCounters
+}
+
+func readHostCounters(procRoot, sysRoot string, planes map[string][]string) *hostCounters {
+	c := &hostCounters{at: time.Now(), planes: map[string]nicCounters{}}
+	if data, err := os.ReadFile(filepath.Join(procRoot, "stat")); err == nil {
+		line, _, _ := strings.Cut(string(data), "\n")
+		fields := strings.Fields(line)
+		for i, f := range fields[1:] {
+			v, _ := strconv.ParseUint(f, 10, 64)
+			// user nice system idle iowait irq softirq steal; guest time is
+			// already counted in user.
+			switch {
+			case i == 3 || i == 4:
+				c.idle += v
+			case i < 8:
+				c.busy += v
+			}
+		}
+	}
+	if data, err := os.ReadFile(filepath.Join(procRoot, "meminfo")); err == nil {
+		var total, avail float64
+		for line := range strings.SplitSeq(string(data), "\n") {
+			key, value, _ := strings.Cut(line, ":")
+			fields := strings.Fields(value)
+			if len(fields) == 0 {
+				continue
+			}
+			v, _ := strconv.ParseFloat(fields[0], 64)
+			switch key {
+			case "MemTotal":
+				total = v
+			case "MemAvailable":
+				avail = v
+			}
+		}
+		if total > 0 {
+			c.memPercent = (total - avail) / total * 100
+		}
+	}
+	if data, err := os.ReadFile(filepath.Join(procRoot, "loadavg")); err == nil {
+		if fields := strings.Fields(string(data)); len(fields) > 0 {
+			c.load1, _ = strconv.ParseFloat(fields[0], 64)
+		}
+	}
+	for plane, nics := range planes {
+		var sum nicCounters
+		for _, nic := range nics {
+			read := func(name string) uint64 {
+				data, err := os.ReadFile(filepath.Join(sysRoot, "class", "net", nic, "statistics", name))
+				if err != nil {
+					return 0
+				}
+				v, _ := strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64)
+				return v
+			}
+			sum.rxBytes += read("rx_bytes")
+			sum.txBytes += read("tx_bytes")
+			sum.rxPackets += read("rx_packets")
+			sum.txPackets += read("tx_packets")
+		}
+		c.planes[plane] = sum
+	}
+	return c
+}
+
+func hostRates(prev, cur *hostCounters) (telemetryv1.NodeLiveSample, bool) {
+	elapsed := cur.at.Sub(prev.at).Seconds()
+	if elapsed <= 0 || cur.busy < prev.busy || cur.idle < prev.idle {
+		return telemetryv1.NodeLiveSample{}, false
+	}
+	s := telemetryv1.NodeLiveSample{
+		TimeMillis:        cur.at.UnixMilli(),
+		MemoryUsedPercent: round2(cur.memPercent),
+		Load1:             cur.load1,
+		Planes:            map[string]telemetryv1.NodeLivePlane{},
+	}
+	if total := float64((cur.busy - prev.busy) + (cur.idle - prev.idle)); total > 0 {
+		s.CPUPercent = round2(float64(cur.busy-prev.busy) / total * 100)
+	}
+	rate := func(a, b uint64) float64 {
+		if b < a {
+			return 0
+		}
+		return round2(float64(b-a) / elapsed)
+	}
+	for plane, c := range cur.planes {
+		p, ok := prev.planes[plane]
+		if !ok {
+			continue
+		}
+		s.Planes[plane] = telemetryv1.NodeLivePlane{
+			NetworkIn: rate(p.rxBytes, c.rxBytes), NetworkOut: rate(p.txBytes, c.txBytes),
+			PacketsIn: rate(p.rxPackets, c.rxPackets), PacketsOut: rate(p.txPackets, c.txPackets),
+		}
+	}
+	return s, true
+}
+
+func round2(v float64) float64 {
+	return float64(int64(v*100+0.5)) / 100
+}

spinifex/runtime/daemon/node_insight_test.go

@@ -0,0 +1,229 @@
+//test:in-package — the handlers, the plane lookups and the live sampler are
+//unexported, and the fixture seeds the cached facts directly so no test reads
+//this host's hardware.
+
+package daemon
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
+	telemetryv1 "github.com/mulgadc/spinifex/contracts/telemetry/v1"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func insightFacts() clusterv1.NodeFacts {
+	return clusterv1.NodeFacts{
+		NICs: []clusterv1.NICFacts{
+			{Name: "eth0", SpeedMbps: 25000},
+			{Name: "eth1", SpeedMbps: 10000},
+			{Name: "eth2", SpeedMbps: 1000},
+		},
+		Planes: []clusterv1.PlaneFacts{
+			{Plane: clusterv1.PlaneLAN, Bridge: "br-lan", Address: "10.10.8.1", Prefix: 24, NICs: []string{"eth0", "eth1"}, MTU: 9000},
+			{Plane: clusterv1.PlaneVPC, Bridge: "br-lan", Address: "10.10.8.1", Prefix: 24, NICs: []string{"eth0", "eth1"}, MTU: 9000},
+			{Plane: clusterv1.PlaneWAN, Bridge: "br-wan", NICs: []string{"eth2"}},
+		},
+	}
+}
+
+func insightDaemon(t *testing.T, facts clusterv1.NodeFacts) (*Daemon, *nats.Conn) {
+	t.Helper()
+	_, nc, _ := testutil.StartTestJetStream(t)
+	insight := newNodeInsight("node1")
+	insight.facts, insight.factsAt = facts, time.Now()
+	return &Daemon{node: "node1", natsConn: nc, insight: insight}, nc
+}
+
+func callHandler[T any](t *testing.T, nc *nats.Conn, handler func(*nats.Msg) string, req any) T {
+	t.Helper()
+	subject := nats.NewInbox()
+	sub, err := nc.Subscribe(subject, func(msg *nats.Msg) { handler(msg) })
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	data, err := json.Marshal(req)
+	require.NoError(t, err)
+	reply, err := nc.Request(subject, data, 5*time.Second)
+	require.NoError(t, err)
+
+	var out T
+	require.NoError(t, json.Unmarshal(reply.Data, &out))
+	return out
+}
+
+func validRun() clusterv1.NetTestRunRequest {
+	return clusterv1.NetTestRunRequest{
+		Plane: clusterv1.PlaneLAN, Address: "10.10.8.2", Port: clusterv1.NetTestPort,
+		DurationSeconds: 5, Streams: 4,
+	}
+}
+
+func TestHandleNetTestRun_RefusesTargetOutsidePlaneSubnet(t *testing.T) {
+	d, nc := insightDaemon(t, insightFacts())
+
+	for _, target := range []string{"10.10.9.2", "8.8.8.8", "127.0.0.1", "not-an-ip", ""} {
+		req := validRun()
+		req.Address = target
+		res := callHandler[clusterv1.NetTestRunResponse](t, nc, d.handleNetTestRun, req)
+		assert.Contains(t, res.Error, "outside this node's lan subnet", "target %q", target)
+	}
+}
+
+func TestHandleNetTestRun_RefusesInvalidRequests(t *testing.T) {
+	d, nc := insightDaemon(t, insightFacts())
+
+	mutations := map[string]func(*clusterv1.NetTestRunRequest){
+		"unknown plane":  func(r *clusterv1.NetTestRunRequest) { r.Plane = "storage" },
+		"short duration": func(r *clusterv1.NetTestRunRequest) { r.DurationSeconds = 1 },
+		"long duration":  func(r *clusterv1.NetTestRunRequest) { r.DurationSeconds = 600 },
+		"no streams":     func(r *clusterv1.NetTestRunRequest) { r.Streams = 0 },
+		"many streams":   func(r *clusterv1.NetTestRunRequest) { r.Streams = 99 },
+		"other port":     func(r *clusterv1.NetTestRunRequest) { r.Port = 22 },
+	}
+	for name, mutate := range mutations {
+		req := validRun()
+		mutate(&req)
+		res := callHandler[clusterv1.NetTestRunResponse](t, nc, d.handleNetTestRun, req)
+		assert.Equal(t, "invalid request", res.Error, name)
+	}
+}
+
+func TestHandleNetTestRun_RefusesPlaneWithoutAddress(t *testing.T) {
+	d, nc := insightDaemon(t, insightFacts())
+
+	req := validRun()
+	req.Plane = clusterv1.PlaneWAN
+	res := callHandler[clusterv1.NetTestRunResponse](t, nc, d.handleNetTestRun, req)
+	assert.Equal(t, "this node has no address on the wan plane", res.Error)
+}
+
+func TestHandleNetTestRun_OneTestPerNode(t *testing.T) {
+	d, nc := insightDaemon(t, insightFacts())
+	d.insight.testing.Lock()
+	defer d.insight.testing.Unlock()
+
+	res := callHandler[clusterv1.NetTestRunResponse](t, nc, d.handleNetTestRun, validRun())
+	assert.Equal(t, "a test is already running from this node", res.Error)
+}
+
+func TestHandleNetTestServe_RefusesBadRequests(t *testing.T) {
+	d, nc := insightDaemon(t, insightFacts())
+
+	res := callHandler[clusterv1.NetTestServeResponse](t, nc, d.handleNetTestServe,
+		clusterv1.NetTestServeRequest{Plane: clusterv1.PlaneLAN, DurationSeconds: 600})
+	assert.Equal(t, "invalid request", res.Error)
+
+	res = callHandler[clusterv1.NetTestServeResponse](t, nc, d.handleNetTestServe,
+		clusterv1.NetTestServeRequest{Plane: clusterv1.PlaneWAN, DurationSeconds: 5})
+	assert.Equal(t, "this node has no address on the wan plane", res.Error)
+	assert.Zero(t, res.Port)
+}
+
+func TestHandleNodeFacts_ServesCachedFacts(t *testing.T) {
+	d, nc := insightDaemon(t, insightFacts())
+
+	got := callHandler[clusterv1.NodeFacts](t, nc, d.handleNodeFacts, struct{}{})
+	assert.Equal(t, insightFacts().Planes, got.Planes)
+}
+
+func TestPlaneLink_FastestNICAndMTU(t *testing.T) {
+	d, _ := insightDaemon(t, insightFacts())
+
+	speed, mtu := d.insight.planeLink(t.Context(), clusterv1.PlaneLAN)
+	assert.Equal(t, 25000, speed)
+	assert.Equal(t, 9000, mtu)
+
+	speed, mtu = d.insight.planeLink(t.Context(), "storage")
+	assert.Zero(t, speed)
+	assert.Zero(t, mtu)
+}
+
+func TestPlaneNICs_SharedNICsCountedOnce(t *testing.T) {
+	got := planeNICs(insightFacts())
+	assert.Equal(t, map[string][]string{
+		"lan+vpc": {"eth0", "eth1"},
+		"wan":     {"eth2"},
+	}, got)
+
+	got = planeNICs(clusterv1.NodeFacts{Planes: []clusterv1.PlaneFacts{{Plane: clusterv1.PlaneWAN, Bridge: "br-wan"}}})
+	assert.Equal(t, map[string][]string{"wan": {"br-wan"}}, got)
+}
+
+func writeHostTree(t *testing.T, root string, stat, rx string) {
+	t.Helper()
+	proc := filepath.Join(root, "proc")
+	stats := filepath.Join(root, "sys", "class", "net", "eth0", "statistics")
+	require.NoError(t, os.MkdirAll(proc, 0o755))
+	require.NoError(t, os.MkdirAll(stats, 0o755))
+	files := map[string]string{
+		filepath.Join(proc, "stat"):        stat,
+		filepath.Join(proc, "meminfo"):     "MemTotal: 1000 kB\nMemFree: 100 kB\nMemAvailable: 250 kB\n",
+		filepath.Join(proc, "loadavg"):     "1.50 1.00 0.50 1/100 1\n",
+		filepath.Join(stats, "rx_bytes"):   rx,
+		filepath.Join(stats, "tx_bytes"):   "0\n",
+		filepath.Join(stats, "rx_packets"): "0\n",
+		filepath.Join(stats, "tx_packets"): "0\n",
+	}
+	for path, body := range files {
+		require.NoError(t, os.WriteFile(path, []byte(body), 0o600))
+	}
+}
+
+func TestHostRates_FromProcAndSysCounters(t *testing.T) {
+	root := t.TempDir()
+	planes := map[string][]string{"lan": {"eth0", "missing0"}}
+
+	writeHostTree(t, root, "cpu 100 0 100 700 100 0 0 0 0 0\ncpu0 1 1 1 1\n", "1000\n")
+	prev := readHostCounters(filepath.Join(root, "proc"), filepath.Join(root, "sys"), planes)
+	writeHostTree(t, root, "cpu 200 0 200 1300 200 0 0 0 0 0\n", "3000\n")
+	cur := readHostCounters(filepath.Join(root, "proc"), filepath.Join(root, "sys"), planes)
+	cur.at = prev.at.Add(2 * time.Second)
+
+	s, ok := hostRates(prev, cur)
+	require.True(t, ok)
+	assert.InDelta(t, 22.22, s.CPUPercent, 0.01)
+	assert.InDelta(t, 75.0, s.MemoryUsedPercent, 0.01)
+	assert.InDelta(t, 1.5, s.Load1, 0.001)
+	assert.InDelta(t, 1000.0, s.Planes["lan"].NetworkIn, 0.01)
+
+	_, ok = hostRates(cur, prev)
+	assert.False(t, ok, "counters going backwards are a reset, not a rate")
+}
+
+func TestNodeLiveSampler_RingAndLease(t *testing.T) {
+	l := &nodeLiveSampler{running: true}
+	for i := range telemetryv1.LiveRingSamples + 5 {
+		l.record(telemetryv1.NodeLiveSample{TimeMillis: int64(i)})
+	}
+	require.Len(t, l.ring, telemetryv1.LiveRingSamples)
+	assert.Equal(t, int64(5), l.ring[0].TimeMillis)
+
+	l.until = time.Now().Add(time.Minute)
+	got := l.watch(int64(telemetryv1.LiveRingSamples + 2))
+	assert.Len(t, got, 2)
+	assert.False(t, l.lapsed())
+
+	l.until = time.Now().Add(-time.Second)
+	assert.True(t, l.lapsed())
+	assert.Empty(t, l.ring)
+	assert.False(t, l.running)
+}
+
+func TestHandleNodeLiveWatch_ReturnsRing(t *testing.T) {
+	d, nc := insightDaemon(t, insightFacts())
+	d.insight.live = &nodeLiveSampler{insight: d.insight, nc: nc, subject: nats.NewInbox(), running: true}
+	d.insight.live.record(telemetryv1.NodeLiveSample{TimeMillis: 10})
+	d.insight.live.record(telemetryv1.NodeLiveSample{TimeMillis: 20})
+
+	ring := callHandler[telemetryv1.NodeLiveRing](t, nc, d.handleNodeLiveWatch, telemetryv1.LiveWatch{SinceMillis: 10})
+	require.Len(t, ring.Samples, 1)
+	assert.Equal(t, int64(20), ring.Samples[0].TimeMillis)
+}

spinifex/runtime/host/nettest/nettest.go

@@ -0,0 +1,236 @@
+// Package nettest runs one direction of a node-to-node throughput test with
+// iperf2, bound to one plane's address on each end.
+package nettest
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
+)
+
+// Limits on one direction of a test. A request outside them is refused.
+const (
+	MinDuration = 5
+	MaxDuration = 30
+	MaxStreams  = 16
+)
+
+const (
+	serverGrace  = 15 * time.Second
+	readyTimeout = 3 * time.Second
+	pingCount    = 10
+)
+
+var (
+	iperfPaths   = []string{"/usr/bin/iperf", "/usr/local/bin/iperf"}
+	setprivPaths = []string{"/usr/bin/setpriv", "/bin/setpriv"}
+	pingPaths    = []string{"/usr/bin/ping", "/bin/ping"}
+)
+
+func findTool(paths []string) string {
+	for _, p := range paths {
+		if _, err := os.Stat(p); err == nil {
+			return p
+		}
+	}
+	return ""
+}
+
+// ErrNoIperf is returned on a node without iperf2 installed.
+var ErrNoIperf = errors.New("iperf (version 2) is not installed on this node")
+
+// unprivileged wraps a command so it runs with no capabilities. The daemon's
+// ambient capabilities would otherwise pass to a network-facing listener.
+func unprivileged(ctx context.Context, name string, args ...string) (*exec.Cmd, error) {
+	setpriv := findTool(setprivPaths)
+	if setpriv == "" {
+		return nil, errors.New("setpriv is not installed, refusing to run a listener with the daemon's capabilities")
+	}
+	full := append([]string{"--inh-caps=-all", "--ambient-caps=-all", "--no-new-privs", name}, args...)
+	cmd := exec.CommandContext(ctx, setpriv, full...)
+	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+	cmd.Cancel = func() error { return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) }
+	return cmd, nil
+}
+
+// Server is one listener; only one runs on a node at a time.
+type Server struct {
+	mu   sync.Mutex
+	stop context.CancelFunc
+}
+
+// Serve starts a one-shot iperf server on bindIP that exits after the test
+// window, and returns once it accepts connections.
+func (s *Server) Serve(bindIP string, durationSeconds int) error {
+	iperf := findTool(iperfPaths)
+	if iperf == "" {
+		return ErrNoIperf
+	}
+	if net.ParseIP(bindIP) == nil {
+		return fmt.Errorf("invalid bind address %q", bindIP)
+	}
+	window := time.Duration(durationSeconds)*time.Second + serverGrace
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.stop != nil {
+		s.stop()
+		s.stop = nil
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), window)
+	cmd, err := unprivileged(ctx, iperf, "-s", "-B", bindIP, "-p", strconv.Itoa(clusterv1.NetTestPort),
+		"-t", strconv.Itoa(int(window.Seconds())))
+	if err != nil {
+		cancel()
+		return err
+	}
+	if err := cmd.Start(); err != nil {
+		cancel()
+		return fmt.Errorf("start iperf server: %w", err)
+	}
+	s.stop = cancel
+	go func() {
+		_ = cmd.Wait()
+		cancel()
+	}()
+
+	addr := net.JoinHostPort(bindIP, strconv.Itoa(clusterv1.NetTestPort))
+	deadline := time.Now().Add(readyTimeout)
+	for time.Now().Before(deadline) {
+		conn, err := net.DialTimeout("tcp", addr, 200*time.Millisecond)
+		if err == nil {
+			_ = conn.Close()
+			return nil
+		}
+		select {
+		case <-ctx.Done():
+			return errors.New("iperf server exited before it was ready")
+		case <-time.After(100 * time.Millisecond):
+		}
+	}
+	return errors.New("iperf server did not start listening")
+}
+
+// Stop ends a running listener, if any.
+func (s *Server) Stop() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.stop != nil {
+		s.stop()
+		s.stop = nil
+	}
+}
+
+// Run is the client side: it sends to address from bindIP for the duration
+// and measures the round trip with ping beforehand, so the two never overlap.
+func Run(ctx context.Context, req clusterv1.NetTestRunRequest, bindIP string) (clusterv1.NetTestRunResponse, error) {
+	out := clusterv1.NetTestRunResponse{SourceAddress: bindIP, Streams: req.Streams, Tool: "iperf2"}
+	iperf := findTool(iperfPaths)
+	if iperf == "" {
+		return out, ErrNoIperf
+	}
+	if net.ParseIP(req.Address) == nil || net.ParseIP(bindIP) == nil {
+		return out, errors.New("invalid address")
+	}
+
+	if ping := findTool(pingPaths); ping != "" {
+		pctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+		cmd, err := unprivileged(pctx, ping, "-n", "-q", "-c", strconv.Itoa(pingCount), "-i", "0.2", "-W", "1", "-I", bindIP, req.Address)
+		if err == nil {
+			if data, err := cmd.Output(); err == nil || len(data) > 0 {
+				out.RTTMinMillis, out.RTTAvgMillis, out.RTTMaxMillis, out.PacketLoss = parsePing(string(data))
+			}
+		}
+		cancel()
+	}
+
+	cctx, cancel := context.WithTimeout(ctx, time.Duration(req.DurationSeconds)*time.Second+serverGrace)
+	defer cancel()
+	cmd, err := unprivileged(cctx, iperf, "-c", req.Address, "-B", bindIP, "-p", strconv.Itoa(req.Port),
+		"-t", strconv.Itoa(req.DurationSeconds), "-P", strconv.Itoa(req.Streams), "-y", "C")
+	if err != nil {
+		return out, err
+	}
+	data, err := cmd.Output()
+	if err != nil && len(data) == 0 {
+		var exit *exec.ExitError
+		if errors.As(err, &exit) && len(exit.Stderr) > 0 {
+			return out, fmt.Errorf("iperf: %s", strings.TrimSpace(string(exit.Stderr)))
+		}
+		return out, fmt.Errorf("iperf: %w", err)
+	}
+	if err := parseIperfCSV(string(data), &out); err != nil {
+		return out, err
+	}
+	return out, nil
+}
+
+// parseIperfCSV reads iperf2's `-y C` report. With more than one stream the
+// line whose ID is -1 is the sum; with one, the only line is.
+func parseIperfCSV(text string, out *clusterv1.NetTestRunResponse) error {
+	var sum, single []string
+	streams := 0
+	for line := range strings.SplitSeq(strings.TrimSpace(text), "\n") {
+		f := strings.Split(strings.TrimSpace(line), ",")
+		if len(f) < 9 {
+			continue
+		}
+		if f[5] == "-1" {
+			sum = f
+			continue
+		}
+		streams++
+		single = f
+		if bps, err := strconv.ParseFloat(f[8], 64); err == nil {
+			out.StreamBPS = append(out.StreamBPS, bps)
+		}
+	}
+	row := sum
+	if row == nil {
+		row = single
+	}
+	if row == nil {
+		return errors.New("iperf printed no report")
+	}
+	out.Streams = streams
+	out.Bytes, _ = strconv.ParseInt(row[7], 10, 64)
+	out.BitsPerSecond, _ = strconv.ParseFloat(row[8], 64)
+	if start, end, ok := strings.Cut(row[6], "-"); ok {
+		s, _ := strconv.ParseFloat(start, 64)
+		e, _ := strconv.ParseFloat(end, 64)
+		out.Seconds = e - s
+	}
+	return nil
+}
+
+// parsePing reads "x% packet loss" and "rtt min/avg/max/mdev = a/b/c/d ms".
+func parsePing(text string) (minMs, avgMs, maxMs, loss float64) {
+	for line := range strings.SplitSeq(text, "\n") {
+		if i := strings.Index(line, "% packet loss"); i > 0 {
+			fields := strings.Fields(line[:i])
+			if len(fields) > 0 {
+				loss, _ = strconv.ParseFloat(fields[len(fields)-1], 64)
+			}
+		}
+		if _, values, ok := strings.Cut(line, "= "); ok && strings.Contains(line, "min/avg/max") {
+			parts := strings.Split(strings.Fields(values)[0], "/")
+			if len(parts) >= 3 {
+				minMs, _ = strconv.ParseFloat(parts[0], 64)
+				avgMs, _ = strconv.ParseFloat(parts[1], 64)
+				maxMs, _ = strconv.ParseFloat(parts[2], 64)
+			}
+		}
+	}
+	return minMs, avgMs, maxMs, loss
+}

spinifex/runtime/host/nettest/nettest_test.go

@@ -0,0 +1,51 @@
+package nettest
+
+import (
+	"testing"
+
+	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Captured from iperf 2.2.1 with -P 3 -y C.
+const threeStreams = `20261102023725,127.0.0.1,36493,127.0.0.1,5099,1,0.0-2.0,1544421440,6124843671
+20261102023725,127.0.0.1,40019,127.0.0.1,5099,3,0.0-2.0,1603665984,6361182295
+20261102023725,127.0.0.1,35431,127.0.0.1,5099,2,0.0-2.0,1424490560,5647792895
+20261102023725,127.0.0.1,0,127.0.0.1,5099,-1,0.0-2.0,4572577984,18116331744
+`
+
+func TestIperfCSVTakesTheSumLine(t *testing.T) {
+	var out clusterv1.NetTestRunResponse
+	require.NoError(t, parseIperfCSV(threeStreams, &out))
+
+	assert.InDelta(t, 18116331744, out.BitsPerSecond, 1)
+	assert.Equal(t, int64(4572577984), out.Bytes)
+	assert.InDelta(t, 2.0, out.Seconds, 0.001)
+	assert.Equal(t, 3, out.Streams)
+	assert.Len(t, out.StreamBPS, 3)
+}
+
+func TestIperfCSVWithOneStreamHasNoSumLine(t *testing.T) {
+	var out clusterv1.NetTestRunResponse
+	require.NoError(t, parseIperfCSV("20261102023725,10.0.0.1,36493,10.0.0.2,5001,3,0.0-10.0,1180000000,943718400\n", &out))
+
+	assert.InDelta(t, 943718400, out.BitsPerSecond, 1)
+	assert.Equal(t, 1, out.Streams)
+}
+
+func TestIperfCSVRefusesAnEmptyReport(t *testing.T) {
+	var out clusterv1.NetTestRunResponse
+	assert.Error(t, parseIperfCSV("connect failed: Connection refused\n", &out))
+}
+
+func TestPingSummary(t *testing.T) {
+	minMs, avgMs, maxMs, loss := parsePing(`--- 10.2.0.3 ping statistics ---
+10 packets transmitted, 9 received, 10% packet loss, time 1803ms
+rtt min/avg/max/mdev = 0.041/0.063/0.120/0.021 ms
+`)
+	assert.InDelta(t, 0.041, minMs, 1e-9)
+	assert.InDelta(t, 0.063, avgMs, 1e-9)
+	assert.InDelta(t, 0.120, maxMs, 1e-9)
+	assert.InDelta(t, 10, loss, 1e-9)
+}

spinifex/runtime/host/nodefacts/build.go

@@ -0,0 +1,391 @@
+package nodefacts
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"net"
+	"os"
+	"path/filepath"
+	"runtime"
+	"slices"
+	"strconv"
+	"strings"
+
+	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
+)
+
+// Env is where Build reads from. Tests point the roots at fixtures.
+type Env struct {
+	Dir      string
+	SysRoot  string
+	ProcRoot string
+	EtcRoot  string
+	Node     string
+	Version  string
+	Commit   string
+	// LiveLLDP reads neighbours now, so a re-cabled port shows without a
+	// restart. Nil or an error falls back to what Collect saved.
+	LiveLLDP func(context.Context) ([]byte, error)
+	// Addrs returns an interface's IPv4 addresses with their prefix lengths.
+	Addrs func(iface string) []net.IPNet
+}
+
+// DefaultEnv reads the live host.
+func DefaultEnv(node, version, commit string) Env {
+	return Env{
+		Dir: DefaultDir, SysRoot: "/sys", ProcRoot: "/proc", EtcRoot: "/etc",
+		Node: node, Version: version, Commit: commit,
+		LiveLLDP: liveLLDP, Addrs: interfaceAddrs,
+	}
+}
+
+func liveLLDP(ctx context.Context) ([]byte, error) {
+	ctl := findTool(networkctlPaths)
+	if ctl == "" {
+		return nil, errNoFile
+	}
+	return run(ctx, ctl, "lldp", "--json=short")
+}
+
+func interfaceAddrs(name string) []net.IPNet {
+	iface, err := net.InterfaceByName(name)
+	if err != nil {
+		return nil
+	}
+	addrs, err := iface.Addrs()
+	if err != nil {
+		return nil
+	}
+	var out []net.IPNet
+	for _, a := range addrs {
+		if ipn, ok := a.(*net.IPNet); ok && ipn.IP.To4() != nil {
+			out = append(out, *ipn)
+		}
+	}
+	return out
+}
+
+// Source states reported in NodeFacts.Sources.
+const (
+	sourceOK          = "ok"
+	sourceUnavailable = "unavailable"
+)
+
+// Build assembles the facts from the collected probes and live sysfs.
+func Build(ctx context.Context, env Env) clusterv1.NodeFacts {
+	f := clusterv1.NodeFacts{Node: env.Node, Sources: map[string]string{}}
+	if data, err := readFacts(env.Dir, fileCollected); err == nil {
+		f.CollectedAt = strings.TrimSpace(string(data))
+	}
+
+	if data, err := readFacts(env.Dir, fileIdentity); err == nil {
+		f.System = parseIdentity(string(data))
+		f.Sources["dmi"] = sourceOK
+	} else {
+		f.Sources["dmi"] = sourceUnavailable
+	}
+	if data, err := os.ReadFile(filepath.Join(env.ProcRoot, "device-tree", "model")); err == nil {
+		f.System.DeviceModel = strings.TrimRight(string(data), "\x00\n ")
+	}
+	if data, err := readFacts(env.Dir, fileVirt); err == nil {
+		f.Virt = strings.TrimSpace(string(data))
+	}
+	f.FormFactor = formFactor(f.Virt, f.System)
+
+	if data, err := readFacts(env.Dir, fileProcessor); err == nil {
+		f.CPU = parseProcessors(string(data))
+	}
+	addCPUInfo(&f.CPU, env.ProcRoot)
+
+	if data, err := readFacts(env.Dir, fileMemory); err == nil {
+		f.Memory = parseMemory(string(data))
+	}
+	addMeminfo(&f.Memory, env.ProcRoot)
+
+	f.OS = osFacts(env)
+
+	neighbours, lldpState := lldpNeighbours(ctx, env)
+	f.Sources["lldp"] = lldpState
+	f.NICs = nicFacts(env, neighbours)
+	f.Planes = planeFacts(env, f.NICs)
+	for i := range f.NICs {
+		for _, p := range f.Planes {
+			if slices.Contains(p.NICs, f.NICs[i].Name) {
+				f.NICs[i].Planes = append(f.NICs[i].Planes, p.Plane)
+			}
+		}
+	}
+	return f
+}
+
+// formFactor decides the picture the console draws. A hypervisor wins over
+// whatever chassis the virtual firmware claims.
+func formFactor(virt string, s clusterv1.SystemFacts) string {
+	model := strings.ToLower(s.DeviceModel)
+	switch {
+	case virt != "" && virt != "none":
+		return "virtual-machine"
+	case strings.Contains(model, "raspberry pi"):
+		return "sbc"
+	case strings.Contains(model, "jetson"):
+		return "jetson"
+	}
+	switch strings.ToLower(s.ChassisType) {
+	case "rack mount chassis", "blade", "blade enclosure", "multi-system", "main server chassis":
+		return "rack-server"
+	case "tower", "mini tower", "desktop", "low profile desktop", "space-saving", "all in one":
+		return "tower"
+	case "mini pc", "compact pci", "embedded pc", "sub chassis", "stick pc":
+		return "sbc"
+	}
+	if model != "" {
+		return "sbc"
+	}
+	return "unknown"
+}
+
+// addCPUInfo fills what DMI cannot tell a VM, and the features worth naming.
+func addCPUInfo(c *clusterv1.CPUFacts, procRoot string) {
+	data, err := os.ReadFile(filepath.Join(procRoot, "cpuinfo"))
+	if err != nil {
+		return
+	}
+	var (
+		model   string
+		flags   []string
+		threads int
+		sockets = map[string]bool{}
+	)
+	sc := bufio.NewScanner(bytes.NewReader(data))
+	sc.Buffer(make([]byte, 64*1024), 1024*1024)
+	for sc.Scan() {
+		key, value, ok := strings.Cut(sc.Text(), ":")
+		if !ok {
+			continue
+		}
+		key, value = strings.TrimSpace(key), strings.TrimSpace(value)
+		switch key {
+		case "processor":
+			threads++
+		case "model name":
+			model = value
+		case "physical id":
+			sockets[value] = true
+		case "flags":
+			if flags == nil {
+				flags = strings.Fields(value)
+			}
+		}
+	}
+	if model != "" {
+		c.Model = strings.Join(strings.Fields(model), " ")
+	}
+	if c.Threads == 0 {
+		c.Threads = threads
+	}
+	if c.Sockets == 0 {
+		c.Sockets = max(len(sockets), 1)
+	}
+	switch {
+	case slices.Contains(flags, "svm"):
+		c.Virtualization = "AMD-V"
+	case slices.Contains(flags, "vmx"):
+		c.Virtualization = "VT-x"
+	}
+	for _, want := range []string{"avx512f", "avx2", "aes", "sev", "sme"} {
+		if slices.Contains(flags, want) {
+			c.Features = append(c.Features, want)
+		}
+	}
+	if runtime.GOARCH == "arm64" && c.Model == "" {
+		c.Model = "ARM64"
+	}
+}
+
+func addMeminfo(m *clusterv1.MemoryFacts, procRoot string) {
+	data, err := os.ReadFile(filepath.Join(procRoot, "meminfo"))
+	if err != nil {
+		return
+	}
+	for line := range strings.SplitSeq(string(data), "\n") {
+		key, value, ok := strings.Cut(line, ":")
+		if !ok {
+			continue
+		}
+		kb, err := strconv.ParseUint(strings.Fields(value + " 0")[0], 10, 64)
+		if err != nil {
+			continue
+		}
+		switch key {
+		case "MemTotal":
+			m.OSTotalBytes = kb << 10
+		case "MemAvailable":
+			m.OSAvailBytes = kb << 10
+		case "Hugetlb":
+			m.HugePageBytes = kb << 10
+		}
+	}
+}
+
+func osFacts(env Env) clusterv1.OSFacts {
+	o := clusterv1.OSFacts{Arch: runtime.GOARCH, Version: env.Version, Commit: env.Commit}
+	if data, err := os.ReadFile(filepath.Join(env.EtcRoot, "os-release")); err == nil {
+		for line := range strings.SplitSeq(string(data), "\n") {
+			if v, ok := strings.CutPrefix(line, "PRETTY_NAME="); ok {
+				o.Name = strings.Trim(v, `"`)
+			}
+		}
+	}
+	if data, err := os.ReadFile(filepath.Join(env.ProcRoot, "sys", "kernel", "osrelease")); err == nil {
+		o.Kernel = strings.TrimSpace(string(data))
+	}
+	if data, err := os.ReadFile(filepath.Join(env.ProcRoot, "uptime")); err == nil {
+		if fields := strings.Fields(string(data)); len(fields) > 0 {
+			secs, _ := strconv.ParseFloat(fields[0], 64)
+			o.UptimeSeconds = int64(secs)
+		}
+	}
+	if data, err := os.ReadFile(filepath.Join(env.ProcRoot, "sys", "kernel", "hostname")); err == nil {
+		o.Hostname = strings.TrimSpace(string(data))
+	}
+	return o
+}
+
+// lldpNeighbours prefers a live read and falls back to the start-up probe.
+func lldpNeighbours(ctx context.Context, env Env) (map[string]clusterv1.LLDPNeighbor, string) {
+	if env.LiveLLDP != nil {
+		if data, err := env.LiveLLDP(ctx); err == nil {
+			if n, err := parseLLDP(data); err == nil {
+				return n, sourceOK
+			}
+		}
+	}
+	data, err := readFacts(env.Dir, fileLLDP)
+	if err != nil {
+		return nil, sourceUnavailable
+	}
+	n, err := parseLLDP(data)
+	if err != nil {
+		return nil, sourceUnavailable
+	}
+	return n, sourceOK
+}
+
+func readSys(sysRoot, nic, name string) string {
+	data, err := os.ReadFile(filepath.Join(sysRoot, "class", "net", nic, name))
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(string(data))
+}
+
+func nicFacts(env Env, neighbours map[string]clusterv1.LLDPNeighbor) []clusterv1.NICFacts {
+	var out []clusterv1.NICFacts
+	for _, name := range physicalNICs(env.SysRoot) {
+		n := clusterv1.NICFacts{
+			Name:  name,
+			MAC:   readSys(env.SysRoot, name, "address"),
+			State: readSys(env.SysRoot, name, "operstate"),
+		}
+		n.MTU, _ = strconv.Atoi(readSys(env.SysRoot, name, "mtu"))
+		if speed, err := strconv.Atoi(readSys(env.SysRoot, name, "speed")); err == nil && speed > 0 {
+			n.SpeedMbps = speed
+		}
+		if target, err := os.Readlink(filepath.Join(env.SysRoot, "class", "net", name, "master")); err == nil {
+			n.Master = filepath.Base(target)
+		}
+		if data, err := readFacts(env.Dir, nicPrefix+name+".txt"); err == nil {
+			e := parseEthtool(string(data))
+			n.Driver, n.Firmware, n.BusInfo, n.Duplex, n.Port = e.driver, e.firmware, e.busInfo, e.duplex, e.port
+			n.MaxMbps = e.maxMbps
+			if n.SpeedMbps == 0 {
+				n.SpeedMbps = e.speedMbps
+			}
+			n.Model = parseUdevModel(string(data))
+			if br := ovsBridge(string(data)); br != "" && (n.Master == "" || n.Master == "ovs-system") {
+				n.Master = br
+			}
+		}
+		if n.Driver == "" {
+			if target, err := os.Readlink(filepath.Join(env.SysRoot, "class", "net", name, "device", "driver")); err == nil {
+				n.Driver = filepath.Base(target)
+			}
+		}
+		if nb, ok := neighbours[name]; ok {
+			n.Neighbor = &nb
+		}
+		out = append(out, n)
+	}
+	return out
+}
+
+func ovsBridge(text string) string {
+	for line := range strings.SplitSeq(text, "\n") {
+		if v, ok := strings.CutPrefix(strings.TrimSpace(line), "OVS_BRIDGE="); ok {
+			return v
+		}
+	}
+	return ""
+}
+
+// planeFacts finds each plane's bridge and address. A host without br-vpc
+// carries the overlay on whichever bridge holds the encap address.
+func planeFacts(env Env, nics []clusterv1.NICFacts) []clusterv1.PlaneFacts {
+	addrs := env.Addrs
+	if addrs == nil {
+		addrs = func(string) []net.IPNet { return nil }
+	}
+	var encap string
+	if data, err := readFacts(env.Dir, fileEncapIP); err == nil {
+		encap = strings.TrimSpace(string(data))
+	}
+
+	bridges := map[string]string{
+		clusterv1.PlaneWAN: "br-wan",
+		clusterv1.PlaneLAN: "br-lan",
+		clusterv1.PlaneVPC: "br-vpc",
+	}
+	if encap != "" {
+		if br := bridgeHolding(encap, addrs); br != "" {
+			bridges[clusterv1.PlaneVPC] = br
+		}
+	}
+
+	var out []clusterv1.PlaneFacts
+	for _, plane := range []string{clusterv1.PlaneWAN, clusterv1.PlaneLAN, clusterv1.PlaneVPC} {
+		br := bridges[plane]
+		if _, err := os.Stat(filepath.Join(env.SysRoot, "class", "net", br)); err != nil {
+			continue
+		}
+		p := clusterv1.PlaneFacts{Plane: plane, Bridge: br}
+		p.MTU, _ = strconv.Atoi(readSys(env.SysRoot, br, "mtu"))
+		list := addrs(br)
+		for _, a := range list {
+			if plane == clusterv1.PlaneVPC && encap != "" && a.IP.String() != encap {
+				continue
+			}
+			ones, _ := a.Mask.Size()
+			p.Address, p.Prefix = a.IP.String(), ones
+			break
+		}
+		for _, n := range nics {
+			if n.Master == br {
+				p.NICs = append(p.NICs, n.Name)
+			}
+		}
+		out = append(out, p)
+	}
+	return out
+}
+
+func bridgeHolding(ip string, addrs func(string) []net.IPNet) string {
+	for _, br := range []string{"br-vpc", "br-lan", "br-wan"} {
+		for _, a := range addrs(br) {
+			if a.IP.String() == ip {
+				return br
+			}
+		}
+	}
+	return ""
+}

spinifex/runtime/host/nodefacts/collect.go

@@ -0,0 +1,208 @@
+// Package nodefacts describes a node's hardware, OS and cabling for the
+// operator console. The root-only probes run once at daemon start through
+// Collect; Build reads their output and adds what sysfs reports live.
+package nodefacts
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/exec"
+	"os/user"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// DefaultDir is where Collect writes and Build reads.
+const DefaultDir = "/etc/spinifex/node-facts"
+
+const probeTimeout = 10 * time.Second
+
+// Raw probe files under the facts directory.
+const (
+	fileIdentity  = "dmi-identity.txt"
+	fileProcessor = "dmi-processor.txt"
+	fileMemory    = "dmi-memory.txt"
+	fileVirt      = "virt.txt"
+	fileLLDP      = "lldp.json"
+	fileEncapIP   = "ovn-encap-ip.txt"
+	fileCollected = "collected-at.txt"
+	nicPrefix     = "nic-"
+)
+
+// Tools are looked up by absolute path: /usr/sbin is not on a service's PATH.
+var (
+	dmidecodePaths  = []string{"/usr/sbin/dmidecode", "/sbin/dmidecode"}
+	ethtoolPaths    = []string{"/usr/sbin/ethtool", "/sbin/ethtool"}
+	networkctlPaths = []string{"/usr/bin/networkctl", "/bin/networkctl"}
+	ovsVsctlPaths   = []string{"/usr/bin/ovs-vsctl", "/usr/sbin/ovs-vsctl"}
+	udevadmPaths    = []string{"/usr/bin/udevadm", "/bin/udevadm"}
+	detectVirtPaths = []string{"/usr/bin/systemd-detect-virt", "/bin/systemd-detect-virt"}
+)
+
+func findTool(paths []string) string {
+	for _, p := range paths {
+		if info, err := os.Stat(p); err == nil && info.Mode().IsRegular() {
+			return p
+		}
+	}
+	return ""
+}
+
+// Collect runs the probes that need root and writes their raw output under
+// dir, readable by the spinifex group. A probe that fails leaves no file, so
+// Build reports that source as unavailable rather than stopping the daemon.
+func Collect(ctx context.Context, dir, sysRoot string) error {
+	if err := os.MkdirAll(dir, 0o750); err != nil {
+		return fmt.Errorf("create %s: %w", dir, err)
+	}
+	gid := spinifexGID()
+	if gid >= 0 {
+		_ = os.Chown(dir, 0, gid)
+	}
+	write := func(name string, data []byte) {
+		if err := writeFile(dir, name, data, gid); err != nil {
+			slog.Warn("node-facts: write failed", "file", name, "err", err)
+		}
+	}
+
+	if dmi := findTool(dmidecodePaths); dmi != "" {
+		for name, args := range map[string][]string{
+			fileIdentity:  {"-t", "bios", "-t", "system", "-t", "chassis", "-t", "baseboard"},
+			fileProcessor: {"-t", "processor"},
+			fileMemory:    {"-t", "memory"},
+		} {
+			if out, err := run(ctx, dmi, args...); err == nil {
+				write(name, out)
+			} else {
+				slog.Info("node-facts: dmidecode unavailable", "type", name, "err", err)
+			}
+		}
+	}
+
+	if virt := findTool(detectVirtPaths); virt != "" {
+		// Exits 1 on bare metal while printing "none", so the output counts.
+		out, _ := run(ctx, virt)
+		write(fileVirt, out)
+	}
+
+	if ctl := findTool(networkctlPaths); ctl != "" {
+		if out, err := run(ctx, ctl, "lldp", "--json=short"); err == nil {
+			write(fileLLDP, out)
+		}
+	}
+
+	if vsctl := findTool(ovsVsctlPaths); vsctl != "" {
+		if out, err := run(ctx, vsctl, "get", "Open_vSwitch", ".", "external_ids:ovn-encap-ip"); err == nil {
+			write(fileEncapIP, []byte(strings.Trim(strings.TrimSpace(string(out)), `"`)))
+		}
+	}
+
+	ethtool, udevadm, vsctl := findTool(ethtoolPaths), findTool(udevadmPaths), findTool(ovsVsctlPaths)
+	for _, nic := range physicalNICs(sysRoot) {
+		var b strings.Builder
+		if ethtool != "" {
+			out, _ := run(ctx, ethtool, nic)
+			b.Write(out)
+			out, _ = run(ctx, ethtool, "-i", nic)
+			b.Write(out)
+		}
+		if udevadm != "" {
+			out, _ := run(ctx, udevadm, "info", "--query=property", "--path=/sys/class/net/"+nic)
+			b.Write(out)
+		}
+		if vsctl != "" {
+			if out, err := run(ctx, vsctl, "iface-to-br", nic); err == nil {
+				fmt.Fprintf(&b, "OVS_BRIDGE=%s\n", strings.TrimSpace(string(out)))
+			}
+		}
+		write(nicPrefix+nic+".txt", []byte(b.String()))
+	}
+
+	write(fileCollected, []byte(time.Now().UTC().Format(time.RFC3339)))
+	return nil
+}
+
+func run(ctx context.Context, name string, args ...string) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(ctx, probeTimeout)
+	defer cancel()
+	out, err := exec.CommandContext(ctx, name, args...).Output()
+	if err != nil && len(out) == 0 {
+		return nil, err
+	}
+	return out, nil
+}
+
+// writeFile replaces dir/name atomically. The files carry serial numbers, so
+// they are root-owned and readable only by the spinifex group.
+func writeFile(dir, name string, data []byte, gid int) error {
+	tmp, err := os.CreateTemp(dir, "."+name+".*")
+	if err != nil {
+		return err
+	}
+	defer func() { _ = os.Remove(tmp.Name()) }()
+	if _, err := tmp.Write(data); err != nil {
+		_ = tmp.Close()
+		return err
+	}
+	if err := tmp.Chmod(0o640); err != nil {
+		_ = tmp.Close()
+		return err
+	}
+	if gid >= 0 {
+		_ = tmp.Chown(0, gid)
+	}
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+	return os.Rename(tmp.Name(), filepath.Join(dir, name))
+}
+
+func spinifexGID() int {
+	g, err := user.LookupGroup("spinifex")
+	if err != nil {
+		return -1
+	}
+	gid, err := strconv.Atoi(g.Gid)
+	if err != nil {
+		return -1
+	}
+	return gid
+}
+
+// physicalNICs lists interfaces backed by a device, which leaves out bridges,
+// taps, veths and VLANs.
+func physicalNICs(sysRoot string) []string {
+	entries, err := os.ReadDir(filepath.Join(sysRoot, "class", "net"))
+	if err != nil {
+		return nil
+	}
+	var out []string
+	for _, e := range entries {
+		base := filepath.Join(sysRoot, "class", "net", e.Name())
+		if _, err := os.Stat(filepath.Join(base, "device")); err != nil {
+			continue
+		}
+		// A VF or an SR-IOV representor still has a device; its phys_port_name
+		// marks it, and it is not cabled to anything.
+		if name, err := os.ReadFile(filepath.Join(base, "phys_port_name")); err == nil && strings.HasPrefix(strings.TrimSpace(string(name)), "pf") {
+			continue
+		}
+		out = append(out, e.Name())
+	}
+	return out
+}
+
+var errNoFile = errors.New("not collected")
+
+func readFacts(dir, name string) ([]byte, error) {
+	data, err := os.ReadFile(filepath.Join(dir, name))
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, errNoFile
+	}
+	return data, err
+}

spinifex/runtime/host/nodefacts/dmi.go

@@ -0,0 +1,233 @@
+package nodefacts
+
+import (
+	"slices"
+	"strconv"
+	"strings"
+
+	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
+)
+
+// dmiBlock is one structure from dmidecode's text output: its title line and
+// its "Key: value" fields. Indented list items are folded under their key.
+type dmiBlock struct {
+	title  string
+	fields map[string]string
+	lists  map[string][]string
+}
+
+func (b dmiBlock) get(key string) string {
+	return cleanDMI(b.fields[key])
+}
+
+// parseDMI splits dmidecode output into its structures, in order.
+func parseDMI(text string) []dmiBlock {
+	var (
+		blocks  []dmiBlock
+		cur     *dmiBlock
+		listKey string
+	)
+	flush := func() {
+		if cur != nil {
+			blocks = append(blocks, *cur)
+			cur = nil
+		}
+	}
+	for line := range strings.SplitSeq(text, "\n") {
+		line = strings.TrimRight(line, " \r")
+		switch {
+		case line == "" || strings.HasPrefix(line, "Handle ") || strings.HasPrefix(line, "#"):
+			flush()
+			listKey = ""
+		case !strings.HasPrefix(line, "\t"):
+			if strings.HasPrefix(line, "SMBIOS ") || strings.HasPrefix(line, "Getting SMBIOS") {
+				continue
+			}
+			flush()
+			cur = &dmiBlock{title: line, fields: map[string]string{}, lists: map[string][]string{}}
+			listKey = ""
+		case cur == nil:
+		case strings.HasPrefix(line, "\t\t"):
+			if listKey != "" {
+				cur.lists[listKey] = append(cur.lists[listKey], strings.TrimSpace(line))
+			}
+		default:
+			key, value, _ := strings.Cut(strings.TrimSpace(line), ":")
+			key, value = strings.TrimSpace(key), strings.TrimSpace(value)
+			cur.fields[key] = value
+			listKey = ""
+			if value == "" {
+				listKey = key
+			}
+		}
+	}
+	flush()
+	return blocks
+}
+
+func blocksTitled(blocks []dmiBlock, title string) []dmiBlock {
+	var out []dmiBlock
+	for _, b := range blocks {
+		if b.title == title {
+			out = append(out, b)
+		}
+	}
+	return out
+}
+
+// placeholders are the values firmware writes when a field was never filled.
+var placeholders = map[string]bool{
+	"": true, "unknown": true, "not specified": true, "not present": true, "none": true,
+	"to be filled by o.e.m.": true, "default string": true, "system serial number": true,
+	"system product name": true, "system manufacturer": true, "o.e.m.": true, "oem": true,
+	"0123456789": true, "123456789": true, "no asset tag": true, "chassis serial number": true,
+	"base board serial number": true, "not settable": true, "unknown unknown": true,
+	"03000200-0400-0500-0006-000700080009": true,
+}
+
+// cleanDMI trims a DMI value and drops the placeholders firmware ships with.
+func cleanDMI(v string) string {
+	v = strings.TrimSpace(v)
+	lower := strings.ToLower(v)
+	if placeholders[lower] || strings.HasSuffix(lower, "_assettag") || strings.Trim(lower, "0") == "" {
+		return ""
+	}
+	return v
+}
+
+// parseIdentity reads the bios, system, chassis and baseboard structures.
+func parseIdentity(text string) clusterv1.SystemFacts {
+	blocks := parseDMI(text)
+	var s clusterv1.SystemFacts
+	if b := blocksTitled(blocks, "BIOS Information"); len(b) > 0 {
+		s.BIOSVendor, s.BIOSVersion, s.BIOSDate = b[0].get("Vendor"), b[0].get("Version"), b[0].get("Release Date")
+	}
+	if b := blocksTitled(blocks, "System Information"); len(b) > 0 {
+		s.Vendor, s.Product, s.Family = b[0].get("Manufacturer"), b[0].get("Product Name"), b[0].get("Family")
+		s.Version, s.Serial, s.UUID = b[0].get("Version"), b[0].get("Serial Number"), b[0].get("UUID")
+	}
+	if b := blocksTitled(blocks, "Chassis Information"); len(b) > 0 {
+		s.ChassisType, s.ChassisAsset = b[0].get("Type"), b[0].get("Asset Tag")
+	}
+	if b := blocksTitled(blocks, "Base Board Information"); len(b) > 0 {
+		s.BoardVendor, s.BoardProduct, s.BoardSerial = b[0].get("Manufacturer"), b[0].get("Product Name"), b[0].get("Serial Number")
+	}
+	return s
+}
+
+// parseProcessors summarises the populated sockets.
+func parseProcessors(text string) clusterv1.CPUFacts {
+	var c clusterv1.CPUFacts
+	for _, b := range blocksTitled(parseDMI(text), "Processor Information") {
+		if !strings.HasPrefix(b.fields["Status"], "Populated") {
+			continue
+		}
+		c.Sockets++
+		if c.Model == "" {
+			c.Model = strings.Join(strings.Fields(b.get("Version")), " ")
+			c.Vendor = b.get("Manufacturer")
+		}
+		cores, _ := strconv.Atoi(b.fields["Core Count"])
+		threads, _ := strconv.Atoi(b.fields["Thread Count"])
+		c.Cores += cores
+		c.Threads += threads
+		c.CoresPerSocket = max(c.CoresPerSocket, cores)
+		c.MaxMHz = max(c.MaxMHz, atoiPrefix(b.fields["Max Speed"]))
+		c.CurrentMHz = max(c.CurrentMHz, atoiPrefix(b.fields["Current Speed"]))
+	}
+	return c
+}
+
+// memoryKey is everything a DIMM is grouped by.
+type memoryKey struct {
+	size                                 uint64
+	typ, detail, form, maker, part, rank string
+	speed, configured                    int
+}
+
+// parseMemory groups the populated DIMMs and counts the slots of every array.
+func parseMemory(text string) clusterv1.MemoryFacts {
+	var m clusterv1.MemoryFacts
+	blocks := parseDMI(text)
+	for _, b := range blocksTitled(blocks, "Physical Memory Array") {
+		if b.fields["Use"] != "System Memory" {
+			continue
+		}
+		if m.MaxCapacity == "" {
+			m.MaxCapacity = b.get("Maximum Capacity")
+		}
+		if m.ECC == "" {
+			m.ECC = b.get("Error Correction Type")
+		}
+	}
+
+	groups := map[memoryKey]*clusterv1.MemoryGroup{}
+	var order []memoryKey
+	for _, b := range blocksTitled(blocks, "Memory Device") {
+		m.SlotsTotal++
+		size := parseSize(b.fields["Size"])
+		if size == 0 {
+			continue
+		}
+		m.SlotsUsed++
+		m.TotalBytes += size
+		k := memoryKey{
+			size: size, typ: b.get("Type"), detail: b.get("Type Detail"), form: b.get("Form Factor"),
+			maker: b.get("Manufacturer"), part: b.get("Part Number"), rank: b.get("Rank"),
+			speed: atoiPrefix(b.fields["Speed"]), configured: atoiPrefix(b.fields["Configured Memory Speed"]),
+		}
+		g, ok := groups[k]
+		if !ok {
+			g = &clusterv1.MemoryGroup{
+				SizeBytes: size, Type: k.typ, TypeDetail: k.detail, FormFactor: k.form,
+				SpeedMTs: k.speed, ConfiguredMTs: k.configured, Manufacturer: k.maker,
+				PartNumber: k.part, Rank: k.rank,
+			}
+			groups[k] = g
+			order = append(order, k)
+		}
+		g.Count++
+		if serial := b.get("Serial Number"); serial != "" {
+			g.Serials = append(g.Serials, serial)
+		}
+	}
+	for _, k := range order {
+		m.Groups = append(m.Groups, *groups[k])
+	}
+	slices.SortStableFunc(m.Groups, func(a, b clusterv1.MemoryGroup) int { return b.Count - a.Count })
+	return m
+}
+
+// parseSize reads a DMI size such as "16 GB" or "4096 MB"; anything else,
+// including "No Module Installed", is zero.
+func parseSize(v string) uint64 {
+	fields := strings.Fields(v)
+	if len(fields) != 2 {
+		return 0
+	}
+	n, err := strconv.ParseUint(fields[0], 10, 64)
+	if err != nil {
+		return 0
+	}
+	switch fields[1] {
+	case "kB", "KB":
+		return n << 10
+	case "MB":
+		return n << 20
+	case "GB":
+		return n << 30
+	case "TB":
+		return n << 40
+	}
+	return 0
+}
+
+// atoiPrefix reads the leading integer of a value such as "3675 MHz".
+func atoiPrefix(v string) int {
+	fields := strings.Fields(v)
+	if len(fields) == 0 {
+		return 0
+	}
+	n, _ := strconv.Atoi(fields[0])
+	return n
+}

spinifex/runtime/host/nodefacts/network.go

@@ -0,0 +1,132 @@
+package nodefacts
+
+import (
+	"encoding/json"
+	"regexp"
+	"strconv"
+	"strings"
+
+	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
+)
+
+// ethtoolFacts is what `ethtool <if>` and `ethtool -i <if>` add to sysfs.
+type ethtoolFacts struct {
+	speedMbps int
+	maxMbps   int
+	duplex    string
+	port      string
+	driver    string
+	firmware  string
+	busInfo   string
+}
+
+var linkModeSpeed = regexp.MustCompile(`\b(\d+)base`)
+
+// parseEthtool reads both outputs, concatenated in either order.
+func parseEthtool(text string) ethtoolFacts {
+	var e ethtoolFacts
+	inSupported := false
+	for line := range strings.SplitSeq(text, "\n") {
+		trimmed := strings.TrimSpace(line)
+		key, value, hasColon := strings.Cut(trimmed, ":")
+		value = strings.TrimSpace(value)
+		if hasColon && !strings.Contains(key, "base") {
+			inSupported = key == "Supported link modes"
+		}
+		if inSupported {
+			for _, m := range linkModeSpeed.FindAllStringSubmatch(trimmed, -1) {
+				if n, err := strconv.Atoi(m[1]); err == nil {
+					e.maxMbps = max(e.maxMbps, n)
+				}
+			}
+		}
+		if !hasColon {
+			continue
+		}
+		switch key {
+		case "Speed":
+			e.speedMbps = atoiPrefix(strings.TrimSuffix(value, "Mb/s"))
+		case "Duplex":
+			if value != "Unknown! (255)" {
+				e.duplex = value
+			}
+		case "Port":
+			if value != "Other" {
+				e.port = value
+			}
+		case "driver":
+			e.driver = value
+		case "firmware-version":
+			e.firmware = value
+		case "bus-info":
+			e.busInfo = value
+		}
+	}
+	return e
+}
+
+// parseUdevModel reads the hwdb vendor and model from `udevadm info` output.
+func parseUdevModel(text string) string {
+	var vendor, model string
+	for line := range strings.SplitSeq(text, "\n") {
+		key, value, _ := strings.Cut(strings.TrimSpace(line), "=")
+		switch key {
+		case "ID_VENDOR_FROM_DATABASE":
+			vendor = value
+		case "ID_MODEL_FROM_DATABASE":
+			model = value
+		}
+	}
+	vendor = strings.TrimSuffix(strings.TrimSuffix(vendor, " Corporation"), " Technologies")
+	switch {
+	case model == "":
+		return vendor
+	case vendor == "":
+		return model
+	}
+	return vendor + " " + model
+}
+
+// lldpCapabilityNames are the IEEE 802.1AB system capability bits.
+var lldpCapabilityNames = []string{"other", "repeater", "bridge", "wlan-ap", "router", "telephone", "docsis", "station"}
+
+type networkctlLLDP struct {
+	Neighbors []struct {
+		InterfaceName string `json:"InterfaceName"`
+		Neighbors     []struct {
+			ChassisID           string `json:"ChassisID"`
+			PortID              string `json:"PortID"`
+			PortDescription     string `json:"PortDescription"`
+			SystemName          string `json:"SystemName"`
+			SystemDescription   string `json:"SystemDescription"`
+			EnabledCapabilities int    `json:"EnabledCapabilities"`
+		} `json:"Neighbors"`
+	} `json:"Neighbors"`
+}
+
+// parseLLDP reads `networkctl lldp --json=short` into the first neighbour of
+// each interface. A NIC sees one switch port, so a second is a hub or a loop.
+func parseLLDP(data []byte) (map[string]clusterv1.LLDPNeighbor, error) {
+	var doc networkctlLLDP
+	if err := json.Unmarshal(data, &doc); err != nil {
+		return nil, err
+	}
+	out := map[string]clusterv1.LLDPNeighbor{}
+	for _, iface := range doc.Neighbors {
+		if len(iface.Neighbors) == 0 {
+			continue
+		}
+		n := iface.Neighbors[0]
+		var caps []string
+		for i, name := range lldpCapabilityNames {
+			if n.EnabledCapabilities&(1<<i) != 0 {
+				caps = append(caps, name)
+			}
+		}
+		out[iface.InterfaceName] = clusterv1.LLDPNeighbor{
+			ChassisID: n.ChassisID, SystemName: n.SystemName, SystemDescription: n.SystemDescription,
+			PortID: n.PortID, PortDescription: n.PortDescription, Capabilities: caps,
+		}
+	}
+	return out, nil
+}

spinifex/runtime/host/nodefacts/nodefacts_test.go

@@ -0,0 +1,250 @@
+package nodefacts
+
+import (
+	"context"
+	"errors"
+	"net"
+	"os"
+	"path/filepath"
+	"testing"
+
+	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// The fixtures are dmidecode, ethtool and networkctl output captured from
+// prod node1, dev-prod bottlebrush and a QEMU dev box, serials replaced.
+func fixture(t *testing.T, host, name string) string {
+	t.Helper()
+	data, err := os.ReadFile(filepath.Join("testdata", host, name))
+	require.NoError(t, err)
+	return string(data)
+}
+
+func TestMemoryGroupsIdenticalDIMMsAndCountsEmptySlots(t *testing.T) {
+	m := parseMemory(fixture(t, "bottlebrush", "dmi-memory.txt"))
+
+	assert.Equal(t, 32, m.SlotsTotal)
+	assert.Equal(t, 16, m.SlotsUsed)
+	assert.Equal(t, uint64(256)<<30, m.TotalBytes)
+	assert.Equal(t, "8 TB", m.MaxCapacity)
+	assert.Equal(t, "Multi-bit ECC", m.ECC)
+	require.Len(t, m.Groups, 1)
+	g := m.Groups[0]
+	assert.Equal(t, 16, g.Count)
+	assert.Equal(t, uint64(16)<<30, g.SizeBytes)
+	assert.Equal(t, "DDR4", g.Type)
+	assert.Equal(t, 2400, g.SpeedMTs)
+	assert.Equal(t, 2400, g.ConfiguredMTs)
+	assert.Equal(t, "SK Hynix", g.Manufacturer)
+	assert.Equal(t, "HMA82GR7AFR8N-UH", g.PartNumber)
+	assert.Len(t, g.Serials, 16)
+}
+
+func TestMemorySplitsDIMMsThatDifferInAnyShownField(t *testing.T) {
+	dimm := func(size, speed, part string) string {
+		return "Memory Device\n\tSize: " + size + "\n\tType: DDR5\n\tSpeed: " + speed +
+			"\n\tManufacturer: Samsung\n\tPart Number: " + part + "\n\tConfigured Memory Speed: 4800 MT/s\n\n"
+	}
+	m := parseMemory(dimm("32 GB", "5600 MT/s", "A") + dimm("32 GB", "5600 MT/s", "A") +
+		dimm("64 GB", "5600 MT/s", "B") + "Memory Device\n\tSize: No Module Installed\n\n")
+
+	assert.Equal(t, 4, m.SlotsTotal)
+	assert.Equal(t, 3, m.SlotsUsed)
+	require.Len(t, m.Groups, 2)
+	assert.Equal(t, 2, m.Groups[0].Count, "largest group first")
+	assert.Equal(t, 5600, m.Groups[0].SpeedMTs)
+	assert.Equal(t, 4800, m.Groups[0].ConfiguredMTs, "running below rated speed stays visible")
+	assert.Equal(t, uint64(64)<<30, m.Groups[1].SizeBytes)
+}
+
+func TestProcessorsSummariseEverySocket(t *testing.T) {
+	c := parseProcessors(fixture(t, "node1", "dmi-processor.txt"))
+
+	assert.Equal(t, "AMD EPYC 7513 32-Core Processor", c.Model)
+	assert.Equal(t, 2, c.Sockets)
+	assert.Equal(t, 32, c.CoresPerSocket)
+	assert.Equal(t, 64, c.Cores)
+	assert.Equal(t, 128, c.Threads)
+	assert.Equal(t, 3675, c.MaxMHz)
+}
+
+func TestIdentityDropsFirmwarePlaceholders(t *testing.T) {
+	s := parseIdentity(fixture(t, "node1", "dmi-identity.txt"))
+	assert.Equal(t, "GIGABYTE", s.Vendor)
+	assert.Equal(t, "H262-Z62-ZB", s.Product)
+	assert.Equal(t, "Rack Mount Chassis", s.ChassisType)
+	assert.Equal(t, "M30", s.BIOSVersion)
+	assert.Equal(t, "06/23/2026", s.BIOSDate)
+
+	vm := parseIdentity(fixture(t, "devbox", "dmi-identity.txt"))
+	assert.Equal(t, "SeaBIOS", vm.BIOSVendor)
+	assert.Equal(t, "QEMU", vm.Vendor)
+	assert.Empty(t, vm.Serial, "QEMU leaves the serial unset")
+
+	assert.Empty(t, cleanDMI("To Be Filled By O.E.M."))
+	assert.Empty(t, cleanDMI("DIMM_P0_A0_AssetTag"))
+	assert.Empty(t, cleanDMI("00000000"))
+}
+
+func TestFormFactor(t *testing.T) {
+	cases := []struct {
+		name string
+		virt string
+		sys  clusterv1.SystemFacts
+		want string
+	}{
+		{"hypervisor wins over the chassis it fakes", "kvm", clusterv1.SystemFacts{ChassisType: "Rack Mount Chassis"}, "virtual-machine"},
+		{"rack server", "none", clusterv1.SystemFacts{ChassisType: "Rack Mount Chassis"}, "rack-server"},
+		{"tower", "none", clusterv1.SystemFacts{ChassisType: "Tower"}, "tower"},
+		{"raspberry pi from the device tree", "none", clusterv1.SystemFacts{DeviceModel: "Raspberry Pi 5 Model B Rev 1.0"}, "sbc"},
+		{"jetson from the device tree", "none", clusterv1.SystemFacts{DeviceModel: "NVIDIA Jetson AGX Orin Developer Kit"}, "jetson"},
+		{"nothing to go on", "", clusterv1.SystemFacts{}, "unknown"},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			assert.Equal(t, c.want, formFactor(c.virt, c.sys))
+		})
+	}
+}
+
+func TestEthtoolReadsLinkAndDriver(t *testing.T) {
+	text := fixture(t, "bottlebrush", "nic-enp193s0f1np1.txt")
+	e := parseEthtool(text)
+
+	assert.Equal(t, 25000, e.speedMbps)
+	assert.Equal(t, 25000, e.maxMbps)
+	assert.Equal(t, "Full", e.duplex)
+	assert.Equal(t, "Direct Attach Copper", e.port)
+	assert.Equal(t, "mlx5_core", e.driver)
+	assert.Equal(t, "0000:c1:00.1", e.busInfo)
+	assert.Contains(t, parseUdevModel(text), "BlueField-2")
+
+	copper := parseEthtool(fixture(t, "bottlebrush", "nic-eno1.txt"))
+	assert.Equal(t, 1000, copper.maxMbps)
+	assert.Equal(t, "Twisted Pair", copper.port)
+	assert.Equal(t, "igb", copper.driver)
+}
+
+func TestLLDPNamesTheSwitchPortPerNIC(t *testing.T) {
+	n, err := parseLLDP([]byte(fixture(t, "node1", "lldp.json")))
+	require.NoError(t, err)
+
+	require.Contains(t, n, "eno1")
+	assert.Equal(t, "MikroTik 1gbe", n["eno1"].SystemName)
+	assert.Equal(t, "ether18", n["eno1"].PortID)
+	assert.Equal(t, []string{"bridge", "router"}, n["eno1"].Capabilities)
+	assert.Equal(t, n["enp33s0f0np0"].ChassisID, n["enp33s0f1np1"].ChassisID, "both 100G ports land on one switch")
+	assert.Equal(t, "qsfp28-7-1", n["enp33s0f0np0"].PortID)
+}
+
+// fakeHost lays out a sysfs tree with two NICs under bridges, as dev-prod has:
+// no br-vpc, so the overlay rides br-lan.
+func fakeHost(t *testing.T) Env {
+	t.Helper()
+	root := t.TempDir()
+	sys := filepath.Join(root, "sys")
+	nic := func(name, master, speed string) {
+		dir := filepath.Join(sys, "class", "net", name)
+		require.NoError(t, os.MkdirAll(filepath.Join(dir, "device"), 0o755))
+		for file, value := range map[string]string{"speed": speed, "operstate": "up", "mtu": "9000", "address": "02:00:00:00:00:01"} {
+			require.NoError(t, os.WriteFile(filepath.Join(dir, file), []byte(value+"\n"), 0o644))
+		}
+		require.NoError(t, os.Symlink("../"+master, filepath.Join(dir, "master")))
+	}
+	bridge := func(name string) {
+		dir := filepath.Join(sys, "class", "net", name)
+		require.NoError(t, os.MkdirAll(dir, 0o755))
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "mtu"), []byte("9000\n"), 0o644))
+	}
+	bridge("br-wan")
+	bridge("br-lan")
+	nic("eno1", "br-wan", "1000")
+	nic("enp193s0f1np1", "br-lan", "25000")
+
+	dir := filepath.Join(root, "facts")
+	require.NoError(t, os.MkdirAll(dir, 0o755))
+	for name, src := range map[string]string{
+		fileIdentity: "dmi-identity.txt", fileMemory: "dmi-memory.txt", fileProcessor: "dmi-processor.txt",
+		"nic-enp193s0f1np1.txt": "nic-enp193s0f1np1.txt", "nic-eno1.txt": "nic-eno1.txt",
+	} {
+		require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(fixture(t, "bottlebrush", src)), 0o644))
+	}
+	require.NoError(t, os.WriteFile(filepath.Join(dir, fileVirt), []byte("none\n"), 0o644))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, fileEncapIP), []byte("10.10.8.4"), 0o644))
+
+	addrs := map[string][]net.IPNet{
+		"br-wan": {{IP: net.IPv4(192, 168, 1, 14), Mask: net.CIDRMask(23, 32)}},
+		"br-lan": {{IP: net.IPv4(10, 10, 8, 4), Mask: net.CIDRMask(24, 32)}},
+	}
+	return Env{
+		Dir: dir, SysRoot: sys, ProcRoot: filepath.Join(root, "proc"), EtcRoot: filepath.Join(root, "etc"),
+		Node: "bottlebrush",
+		LiveLLDP: func(context.Context) ([]byte, error) {
+			return nil, errors.New("systemd-networkd is not running")
+		},
+		Addrs: func(name string) []net.IPNet { return addrs[name] },
+	}
+}
+
+func TestBuildMapsPlanesToNICsWhenTheOverlayRidesTheLANBridge(t *testing.T) {
+	f := Build(context.Background(), fakeHost(t))
+
+	assert.Equal(t, "rack-server", f.FormFactor)
+	assert.Equal(t, "R182-Z92-00", f.System.Product)
+	assert.Equal(t, sourceUnavailable, f.Sources["lldp"], "no networkd and nothing collected")
+
+	planes := map[string]clusterv1.PlaneFacts{}
+	for _, p := range f.Planes {
+		planes[p.Plane] = p
+	}
+	assert.Equal(t, "192.168.1.14", planes["wan"].Address)
+	assert.Equal(t, []string{"eno1"}, planes["wan"].NICs)
+	assert.Equal(t, "br-lan", planes["vpc"].Bridge, "encap address is on br-lan")
+	assert.Equal(t, "10.10.8.4", planes["vpc"].Address)
+	assert.Equal(t, []string{"enp193s0f1np1"}, planes["lan"].NICs)
+
+	nics := map[string]clusterv1.NICFacts{}
+	for _, n := range f.NICs {
+		nics[n.Name] = n
+	}
+	assert.Equal(t, []string{"lan", "vpc"}, nics["enp193s0f1np1"].Planes)
+	assert.Equal(t, 25000, nics["enp193s0f1np1"].SpeedMbps)
+	assert.Equal(t, "mlx5_core", nics["enp193s0f1np1"].Driver)
+	assert.Nil(t, nics["eno1"].Neighbor)
+}
+
+func TestBuildFallsBackToTheCollectedLLDP(t *testing.T) {
+	env := fakeHost(t)
+	require.NoError(t, os.WriteFile(filepath.Join(env.Dir, fileLLDP),
+		[]byte(`{"Neighbors":[{"InterfaceName":"eno1","Neighbors":[{"ChassisID":"aa:bb","PortID":"ether3","SystemName":"core","EnabledCapabilities":4}]}]}`), 0o644))
+
+	f := Build(context.Background(), env)
+
+	assert.Equal(t, sourceOK, f.Sources["lldp"])
+	for _, n := range f.NICs {
+		if n.Name == "eno1" {
+			require.NotNil(t, n.Neighbor)
+			assert.Equal(t, "ether3", n.Neighbor.PortID)
+		}
+	}
+}
+
+func TestWriteFile_ReplacesAtomicallyGroupReadOnly(t *testing.T) {
+	dir := t.TempDir()
+	require.NoError(t, writeFile(dir, fileIdentity, []byte("first"), -1))
+	require.NoError(t, writeFile(dir, fileIdentity, []byte("second"), -1))
+
+	data, err := readFacts(dir, fileIdentity)
+	require.NoError(t, err)
+	assert.Equal(t, "second", string(data))
+
+	info, err := os.Stat(filepath.Join(dir, fileIdentity))
+	require.NoError(t, err)
+	assert.Equal(t, os.FileMode(0o640), info.Mode().Perm(), "the facts carry serial numbers")
+
+	entries, err := os.ReadDir(dir)
+	require.NoError(t, err)
+	assert.Len(t, entries, 1, "no temporary file is left behind")
+}

spinifex/runtime/host/nodefacts/testdata/bottlebrush/dmi-identity.txt

@@ -0,0 +1,186 @@
+# dmidecode 3.6
+Getting SMBIOS data from sysfs.
+SMBIOS 3.2.0 present.
+
+Handle 0x0000, DMI type 0, 26 bytes
+BIOS Information
+	Vendor: GIGABYTE
+	Version: R23
+	Release Date: 07/10/2021
+	Address: 0xF0000
+	Runtime Size: 64 kB
+	ROM Size: 16 MB
+	Characteristics:
+		PCI is supported
+		BIOS is upgradeable
+		BIOS shadowing is allowed
+		Boot from CD is supported
+		Selectable boot is supported
+		BIOS ROM is socketed
+		EDD is supported
+		5.25"/1.2 MB floppy services are supported (int 13h)
+		3.5"/720 kB floppy services are supported (int 13h)
+		3.5"/2.88 MB floppy services are supported (int 13h)
+		Print screen service is supported (int 5h)
+		Serial services are supported (int 14h)
+		Printer services are supported (int 17h)
+		ACPI is supported
+		USB legacy is supported
+		BIOS boot specification is supported
+		Targeted content distribution is supported
+		UEFI is supported
+	BIOS Revision: 5.14
+
+Handle 0x0001, DMI type 1, 27 bytes
+System Information
+	Manufacturer: GIGABYTE
+	Product Name: R182-Z92-00
+	Version: 0100
+	Serial Number: TEST-0001
+	UUID: TEST-0002
+	Wake-up Type: Power Switch
+	SKU Number: 01234567890123456789AB
+	Family: Server
+
+Handle 0x0002, DMI type 2, 15 bytes
+Base Board Information
+	Manufacturer: GIGABYTE
+	Product Name: MZ92-FS0-00
+	Version: 01010101
+	Serial Number: TEST-0003
+	Asset Tag: TEST-0004
+	Features:
+		Board is a hosting board
+		Board is removable
+		Board is replaceable
+	Location In Chassis: 01234567890123456789AB
+	Chassis Handle: 0x0003
+	Type: Motherboard
+	Contained Object Handles: 0
+
+Handle 0x0003, DMI type 3, 22 bytes
+Chassis Information
+	Manufacturer: GIGABYTE
+	Type: Rack Mount Chassis
+	Lock: Not Present
+	Version: 01234567
+	Serial Number: TEST-0005
+	Asset Tag: TEST-0006
+	Boot-up State: Safe
+	Power Supply State: Safe
+	Thermal State: Safe
+	Security Status: None
+	OEM Information: 0x00000000
+	Height: Unspecified
+	Number Of Power Cords: 1
+	Contained Elements: 0
+	SKU Number: 01234567890123456789AB
+
+Handle 0x0030, DMI type 10, 10 bytes
+On Board Device 1 Information
+	Type: Video
+	Status: Enabled
+	Description: AST2500
+On Board Device 2 Information
+	Type: Ethernet
+	Status: Enabled
+	Description: Intel i350 Lan1
+On Board Device 3 Information
+	Type: Ethernet
+	Status: Enabled
+	Description: Intel i350 Lan2
+
+Handle 0x0031, DMI type 41, 11 bytes
+Onboard Device
+	Reference Designation:  Onboard VGA
+	Type: Video
+	Status: Enabled
+	Type Instance: 1
+	Bus Address: 0000:66:00.0
+
+Handle 0x0032, DMI type 41, 11 bytes
+Onboard Device
+	Reference Designation:  Onboard LAN1
+	Type: Ethernet
+	Status: Enabled
+	Type Instance: 1
+	Bus Address: 0000:64:00.0
+
+Handle 0x0033, DMI type 41, 11 bytes
+Onboard Device
+	Reference Designation:  Onboard LAN2
+	Type: Ethernet
+	Status: Enabled
+	Type Instance: 2
+	Bus Address: 0000:64:00.1
+
+Handle 0x0043, DMI type 15, 73 bytes
+System Event Log
+	Area Length: 0 bytes
+	Header Start Offset: 0x0000
+	Header Length: 16 bytes
+	Data Start Offset: 0x0010
+	Access Method: Memory-mapped physical 32-bit address
+	Access Address: 0xFF4FB000
+	Status: Valid, Not Full
+	Change Token: 0x00000053
+	Header Format: Type 1
+	Supported Log Type Descriptors: 25
+	Descriptor 1: Single-bit ECC memory error
+	Data Format 1: Multiple-event handle
+	Descriptor 2: Multi-bit ECC memory error
+	Data Format 2: Multiple-event handle
+	Descriptor 3: Parity memory error
+	Data Format 3: None
+	Descriptor 4: Bus timeout
+	Data Format 4: None
+	Descriptor 5: I/O channel block
+	Data Format 5: None
+	Descriptor 6: Software NMI
+	Data Format 6: None
+	Descriptor 7: POST memory resize
+	Data Format 7: None
+	Descriptor 8: POST error
+	Data Format 8: POST results bitmap
+	Descriptor 9: PCI parity error
+	Data Format 9: Multiple-event handle
+	Descriptor 10: PCI system error
+	Data Format 10: Multiple-event handle
+	Descriptor 11: CPU failure
+	Data Format 11: None
+	Descriptor 12: EISA failsafe timer timeout
+	Data Format 12: None
+	Descriptor 13: Correctable memory log disabled
+	Data Format 13: None
+	Descriptor 14: Logging disabled
+	Data Format 14: None
+	Descriptor 15: System limit exceeded
+	Data Format 15: None
+	Descriptor 16: Asynchronous hardware timer expired
+	Data Format 16: None
+	Descriptor 17: System configuration information
+	Data Format 17: None
+	Descriptor 18: Hard disk information
+	Data Format 18: None
+	Descriptor 19: System reconfigured
+	Data Format 19: None
+	Descriptor 20: Uncorrectable CPU-complex error
+	Data Format 20: None
+	Descriptor 21: Log area reset/cleared
+	Data Format 21: None
+	Descriptor 22: System boot
+	Data Format 22: None
+	Descriptor 23: End of log
+	Data Format 23: None
+	Descriptor 24: OEM-specific
+	Data Format 24: OEM-specific
+	Descriptor 25: OEM-specific
+	Data Format 25: OEM-specific
+
+Handle 0x00B5, DMI type 13, 22 bytes
+BIOS Language Information
+	Language Description Format: Long
+	Installable Languages: 1
+		en|US|iso8859-1
+	Currently Installed Language: en|US|iso8859-1
+

spinifex/runtime/host/nodefacts/testdata/bottlebrush/dmi-memory.txt

@@ -0,0 +1,797 @@
+# dmidecode 3.6
+Getting SMBIOS data from sysfs.
+SMBIOS 3.2.0 present.
+
+Handle 0x0045, DMI type 16, 23 bytes
+Physical Memory Array
+	Location: System Board Or Motherboard
+	Use: System Memory
+	Error Correction Type: Multi-bit ECC
+	Maximum Capacity: 8 TB
+	Error Information Handle: 0x0044
+	Number Of Devices: 32
+
+Handle 0x004D, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x004C
+	Total Width: Unknown
+	Data Width: Unknown
+	Size: No Module Installed
+	Form Factor: Unknown
+	Set: None
+	Locator: DIMM_P0_A0
+	Bank Locator: BANK 0
+	Type: Unknown
+	Type Detail: Unknown
+
+Handle 0x004F, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x004E
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P0_A1
+	Bank Locator: BANK 1
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2400 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0001
+	Asset Tag: TEST-0002
+	Part Number: HMA82GR7AFR8N-UH    
+	Rank: 2
+	Configured Memory Speed: 2400 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x0052, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x0051
+	Total Width: Unknown
+	Data Width: Unknown
+	Size: No Module Installed
+	Form Factor: Unknown
+	Set: None
+	Locator: DIMM_P0_B0
+	Bank Locator: BANK 0
+	Type: Unknown
+	Type Detail: Unknown
+
+Handle 0x0054, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x0053
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P0_B1
+	Bank Locator: BANK 1
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2400 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0003
+	Asset Tag: TEST-0004
+	Part Number: HMA82GR7AFR8N-UH    
+	Rank: 2
+	Configured Memory Speed: 2400 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x0057, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x0056
+	Total Width: Unknown
+	Data Width: Unknown
+	Size: No Module Installed
+	Form Factor: Unknown
+	Set: None
+	Locator: DIMM_P0_C0
+	Bank Locator: BANK 0
+	Type: Unknown
+	Type Detail: Unknown
+
+Handle 0x0059, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x0058
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P0_C1
+	Bank Locator: BANK 1
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2400 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0005
+	Asset Tag: TEST-0006
+	Part Number: HMA82GR7AFR8N-UH    
+	Rank: 2
+	Configured Memory Speed: 2400 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x005C, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x005B
+	Total Width: Unknown
+	Data Width: Unknown
+	Size: No Module Installed
+	Form Factor: Unknown
+	Set: None
+	Locator: DIMM_P0_D0
+	Bank Locator: BANK 0
+	Type: Unknown
+	Type Detail: Unknown
+
+Handle 0x005E, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x005D
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P0_D1
+	Bank Locator: BANK 1
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2400 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0007
+	Asset Tag: TEST-0008
+	Part Number: HMA82GR7AFR8N-UH    
+	Rank: 2
+	Configured Memory Speed: 2400 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x0061, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x0060
+	Total Width: Unknown
+	Data Width: Unknown
+	Size: No Module Installed
+	Form Factor: Unknown
+	Set: None
+	Locator: DIMM_P0_E0
+	Bank Locator: BANK 0
+	Type: Unknown
+	Type Detail: Unknown
+
+Handle 0x0063, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x0062
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P0_E1
+	Bank Locator: BANK 1
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2400 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0009
+	Asset Tag: TEST-0010
+	Part Number: HMA82GR7AFR8N-UH    
+	Rank: 2
+	Configured Memory Speed: 2400 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x0066, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x0065
+	Total Width: Unknown
+	Data Width: Unknown
+	Size: No Module Installed
+	Form Factor: Unknown
+	Set: None
+	Locator: DIMM_P0_F0
+	Bank Locator: BANK 0
+	Type: Unknown
+	Type Detail: Unknown
+
+Handle 0x0068, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x0067
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P0_F1
+	Bank Locator: BANK 1
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2400 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0011
+	Asset Tag: TEST-0012
+	Part Number: HMA82GR7AFR8N-UH    
+	Rank: 2
+	Configured Memory Speed: 2400 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x006B, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x006A
+	Total Width: Unknown
+	Data Width: Unknown
+	Size: No Module Installed
+	Form Factor: Unknown
+	Set: None
+	Locator: DIMM_P0_G0
+	Bank Locator: BANK 0
+	Type: Unknown
+	Type Detail: Unknown
+
+Handle 0x006D, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x006C
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P0_G1
+	Bank Locator: BANK 1
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2400 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0013
+	Asset Tag: TEST-0014
+	Part Number: HMA82GR7AFR8N-UH    
+	Rank: 2
+	Configured Memory Speed: 2400 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x0070, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x006F
+	Total Width: Unknown
+	Data Width: Unknown
+	Size: No Module Installed
+	Form Factor: Unknown
+	Set: None
+	Locator: DIMM_P0_H0
+	Bank Locator: BANK 0
+	Type: Unknown
+	Type Detail: Unknown
+
+Handle 0x0072, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x0071
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P0_H1
+	Bank Locator: BANK 1
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2400 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0015
+	Asset Tag: TEST-0016
+	Part Number: HMA82GR7AFR8N-UH    
+	Rank: 2
+	Configured Memory Speed: 2400 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x0079, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x0078
+	Total Width: Unknown
+	Data Width: Unknown
+	Size: No Module Installed
+	Form Factor: Unknown
+	Set: None
+	Locator: DIMM_P1_I0
+	Bank Locator: BANK 0
+	Type: Unknown
+	Type Detail: Unknown
+
+Handle 0x007B, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x007A
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P1_I1
+	Bank Locator: BANK 1
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2400 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0017
+	Asset Tag: TEST-0018
+	Part Number: HMA82GR7AFR8N-UH    
+	Rank: 2
+	Configured Memory Speed: 2400 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x007E, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x007D
+	Total Width: Unknown
+	Data Width: Unknown
+	Size: No Module Installed
+	Form Factor: Unknown
+	Set: None
+	Locator: DIMM_P1_J0
+	Bank Locator: BANK 0
+	Type: Unknown
+	Type Detail: Unknown
+
+Handle 0x0080, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x007F
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P1_J1
+	Bank Locator: BANK 1
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2400 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0019
+	Asset Tag: TEST-0020
+	Part Number: HMA82GR7AFR8N-UH    
+	Rank: 2
+	Configured Memory Speed: 2400 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x0083, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x0082
+	Total Width: Unknown
+	Data Width: Unknown
+	Size: No Module Installed
+	Form Factor: Unknown
+	Set: None
+	Locator: DIMM_P1_K0
+	Bank Locator: BANK 0
+	Type: Unknown
+	Type Detail: Unknown
+
+Handle 0x0085, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x0084
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P1_K1
+	Bank Locator: BANK 1
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2400 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0021
+	Asset Tag: TEST-0022
+	Part Number: HMA82GR7AFR8N-UH    
+	Rank: 2
+	Configured Memory Speed: 2400 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x0088, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x0087
+	Total Width: Unknown
+	Data Width: Unknown
+	Size: No Module Installed
+	Form Factor: Unknown
+	Set: None
+	Locator: DIMM_P1_L0
+	Bank Locator: BANK 0
+	Type: Unknown
+	Type Detail: Unknown
+
+Handle 0x008A, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x0089
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P1_L1
+	Bank Locator: BANK 1
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2400 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0023
+	Asset Tag: TEST-0024
+	Part Number: HMA82GR7AFR8N-UH    
+	Rank: 2
+	Configured Memory Speed: 2400 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x008D, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x008C
+	Total Width: Unknown
+	Data Width: Unknown
+	Size: No Module Installed
+	Form Factor: Unknown
+	Set: None
+	Locator: DIMM_P1_M0
+	Bank Locator: BANK 0
+	Type: Unknown
+	Type Detail: Unknown
+
+Handle 0x008F, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x008E
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P1_M1
+	Bank Locator: BANK 1
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2400 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0025
+	Asset Tag: TEST-0026
+	Part Number: HMA82GR7AFR8N-UH    
+	Rank: 2
+	Configured Memory Speed: 2400 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x0092, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x0091
+	Total Width: Unknown
+	Data Width: Unknown
+	Size: No Module Installed
+	Form Factor: Unknown
+	Set: None
+	Locator: DIMM_P1_N0
+	Bank Locator: BANK 0
+	Type: Unknown
+	Type Detail: Unknown
+
+Handle 0x0094, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x0093
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P1_N1
+	Bank Locator: BANK 1
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2400 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0027
+	Asset Tag: TEST-0028
+	Part Number: HMA82GR7AFR8N-UH    
+	Rank: 2
+	Configured Memory Speed: 2400 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x0097, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x0096
+	Total Width: Unknown
+	Data Width: Unknown
+	Size: No Module Installed
+	Form Factor: Unknown
+	Set: None
+	Locator: DIMM_P1_O0
+	Bank Locator: BANK 0
+	Type: Unknown
+	Type Detail: Unknown
+
+Handle 0x0099, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x0098
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P1_O1
+	Bank Locator: BANK 1
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2400 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0029
+	Asset Tag: TEST-0030
+	Part Number: HMA82GR7AFR8N-UH    
+	Rank: 2
+	Configured Memory Speed: 2400 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x009C, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x009B
+	Total Width: Unknown
+	Data Width: Unknown
+	Size: No Module Installed
+	Form Factor: Unknown
+	Set: None
+	Locator: DIMM_P1_P0
+	Bank Locator: BANK 0
+	Type: Unknown
+	Type Detail: Unknown
+
+Handle 0x009E, DMI type 17, 84 bytes
+Memory Device
+	Array Handle: 0x0045
+	Error Information Handle: 0x009D
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P1_P1
+	Bank Locator: BANK 1
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2400 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0031
+	Asset Tag: TEST-0032
+	Part Number: HMA82GR7AFR8N-UH    
+	Rank: 2
+	Configured Memory Speed: 2400 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+

spinifex/runtime/host/nodefacts/testdata/bottlebrush/dmi-processor.txt

@@ -0,0 +1,116 @@
+# dmidecode 3.6
+Getting SMBIOS data from sysfs.
+SMBIOS 3.2.0 present.
+
+Handle 0x004B, DMI type 4, 48 bytes
+Processor Information
+	Socket Designation: P0
+	Type: Central Processor
+	Family: Zen
+	Manufacturer: Advanced Micro Devices, Inc.
+	ID: TEST-0001
+	Signature: Family 23, Model 49, Stepping 0
+	Flags:
+		FPU (Floating-point unit on-chip)
+		VME (Virtual mode extension)
+		DE (Debugging extension)
+		PSE (Page size extension)
+		TSC (Time stamp counter)
+		MSR (Model specific registers)
+		PAE (Physical address extension)
+		MCE (Machine check exception)
+		CX8 (CMPXCHG8 instruction supported)
+		APIC (On-chip APIC hardware supported)
+		SEP (Fast system call)
+		MTRR (Memory type range registers)
+		PGE (Page global enable)
+		MCA (Machine check architecture)
+		CMOV (Conditional move instruction supported)
+		PAT (Page attribute table)
+		PSE-36 (36-bit page size extension)
+		CLFSH (CLFLUSH instruction supported)
+		MMX (MMX technology supported)
+		FXSR (FXSAVE and FXSTOR instructions supported)
+		SSE (Streaming SIMD extensions)
+		SSE2 (Streaming SIMD extensions 2)
+		HTT (Multi-threading)
+	Version: AMD EPYC 7662 64-Core Processor
+	Voltage: 1.1 V
+	External Clock: 100 MHz
+	Max Speed: 3300 MHz
+	Current Speed: 2000 MHz
+	Status: Populated, Enabled
+	Upgrade: Socket SP3
+	L1 Cache Handle: 0x0048
+	L2 Cache Handle: 0x0049
+	L3 Cache Handle: 0x004A
+	Serial Number: Unknown
+	Asset Tag: Unknown
+	Part Number: Unknown
+	Core Count: 64
+	Core Enabled: 64
+	Thread Count: 128
+	Characteristics:
+		64-bit capable
+		Multi-Core
+		Hardware Thread
+		Execute Protection
+		Enhanced Virtualization
+		Power/Performance Control
+
+Handle 0x0077, DMI type 4, 48 bytes
+Processor Information
+	Socket Designation: P1
+	Type: Central Processor
+	Family: Zen
+	Manufacturer: Advanced Micro Devices, Inc.
+	ID: TEST-0004
+	Signature: Family 23, Model 49, Stepping 0
+	Flags:
+		FPU (Floating-point unit on-chip)
+		VME (Virtual mode extension)
+		DE (Debugging extension)
+		PSE (Page size extension)
+		TSC (Time stamp counter)
+		MSR (Model specific registers)
+		PAE (Physical address extension)
+		MCE (Machine check exception)
+		CX8 (CMPXCHG8 instruction supported)
+		APIC (On-chip APIC hardware supported)
+		SEP (Fast system call)
+		MTRR (Memory type range registers)
+		PGE (Page global enable)
+		MCA (Machine check architecture)
+		CMOV (Conditional move instruction supported)
+		PAT (Page attribute table)
+		PSE-36 (36-bit page size extension)
+		CLFSH (CLFLUSH instruction supported)
+		MMX (MMX technology supported)
+		FXSR (FXSAVE and FXSTOR instructions supported)
+		SSE (Streaming SIMD extensions)
+		SSE2 (Streaming SIMD extensions 2)
+		HTT (Multi-threading)
+	Version: AMD EPYC 7662 64-Core Processor
+	Voltage: 1.1 V
+	External Clock: 100 MHz
+	Max Speed: 3300 MHz
+	Current Speed: 2000 MHz
+	Status: Populated, Enabled
+	Upgrade: Socket SP3
+	L1 Cache Handle: 0x0074
+	L2 Cache Handle: 0x0075
+	L3 Cache Handle: 0x0076
+	Serial Number: Unknown
+	Asset Tag: Unknown
+	Part Number: Unknown
+	Core Count: 64
+	Core Enabled: 64
+	Thread Count: 128
+	Characteristics:
+		64-bit capable
+		Multi-Core
+		Hardware Thread
+		Execute Protection
+		Enhanced Virtualization
+		Power/Performance Control
+

spinifex/runtime/host/nodefacts/testdata/bottlebrush/nic-eno1.txt

@@ -0,0 +1,43 @@
+Settings for eno1:
+	Supported ports: [ TP ]
+	Supported link modes:   10baseT/Half 10baseT/Full
+	                        100baseT/Half 100baseT/Full
+	                        1000baseT/Full
+	Supported pause frame use: Symmetric
+	Supports auto-negotiation: Yes
+	Supported FEC modes: Not reported
+	Advertised link modes:  10baseT/Half 10baseT/Full
+	                        100baseT/Half 100baseT/Full
+	                        1000baseT/Full
+	Advertised pause frame use: Symmetric
+	Advertised auto-negotiation: Yes
+	Advertised FEC modes: Not reported
+	Speed: 1000Mb/s
+	Duplex: Full
+	Auto-negotiation: on
+	Port: Twisted Pair
+	PHYAD: 1
+	Transceiver: internal
+	MDI-X: on (auto)
+        Current message level: 0x00000007 (7)
+                               drv probe link
+	Link detected: yes
+netlink error: Operation not permitted
+===
+driver: igb
+version: 6.12.73+deb13-amd64
+firmware-version: 1.63, 0x800009fa
+expansion-rom-version: 
+bus-info: 0000:64:00.0
+supports-statistics: yes
+supports-test: yes
+supports-eeprom-access: yes
+supports-register-dump: yes
+supports-priv-flags: yes
+===
+ID_VENDOR_FROM_DATABASE=Intel Corporation
+ID_MODEL_FROM_DATABASE=I350 Gigabit Network Connection
+ID_NET_NAME_MAC=enx000000000001
+ID_NET_NAME_ONBOARD=eno1
+ID_NET_NAME_PATH=enp100s0f0
+ID_NET_NAME=eno1

spinifex/runtime/host/nodefacts/testdata/bottlebrush/nic-enp193s0f1np1.txt

@@ -0,0 +1,59 @@
+Settings for enp193s0f1np1:
+	Supported ports: [ Backplane ]
+	Supported link modes:   1000baseT/Full
+	                        10000baseT/Full
+	                        1000baseKX/Full
+	                        10000baseKR/Full
+	                        10000baseR_FEC
+	                        25000baseCR/Full
+	                        25000baseKR/Full
+	                        25000baseSR/Full
+	                        1000baseX/Full
+	                        10000baseCR/Full
+	                        10000baseSR/Full
+	                        10000baseLR/Full
+	                        10000baseER/Full
+	Supported pause frame use: Symmetric
+	Supports auto-negotiation: Yes
+	Supported FEC modes: None	 RS	 BASER
+	Advertised link modes:  1000baseT/Full
+	                        10000baseT/Full
+	                        1000baseKX/Full
+	                        10000baseKR/Full
+	                        10000baseR_FEC
+	                        25000baseCR/Full
+	                        25000baseKR/Full
+	                        25000baseSR/Full
+	                        1000baseX/Full
+	                        10000baseCR/Full
+	                        10000baseSR/Full
+	                        10000baseLR/Full
+	                        10000baseER/Full
+	Advertised pause frame use: Symmetric
+	Advertised auto-negotiation: Yes
+	Advertised FEC modes: None
+	Speed: 25000Mb/s
+	Duplex: Full
+	Auto-negotiation: on
+	Port: Direct Attach Copper
+	PHYAD: 0
+	Transceiver: internal
+	Link detected: yes
+netlink error: Operation not permitted
+===
+driver: mlx5_core
+version: 6.12.73+deb13-amd64
+firmware-version: 24.47.1088 (MT_0000000541)
+expansion-rom-version: 
+bus-info: 0000:c1:00.1
+supports-statistics: yes
+supports-test: yes
+supports-eeprom-access: no
+supports-register-dump: no
+supports-priv-flags: yes
+===
+ID_VENDOR_FROM_DATABASE=Mellanox Technologies
+ID_MODEL_FROM_DATABASE=MT42822 BlueField-2 integrated ConnectX-6 Dx network controller
+ID_NET_NAME_MAC=enx000000000002
+ID_NET_NAME_PATH=enp193s0f1np1
+ID_NET_NAME=enp193s0f1np1

spinifex/runtime/host/nodefacts/testdata/devbox/dmi-identity.txt

@@ -0,0 +1,50 @@
+# dmidecode 3.6
+Getting SMBIOS data from sysfs.
+SMBIOS 2.8 present.
+
+Handle 0x0000, DMI type 0, 24 bytes
+BIOS Information
+	Vendor: SeaBIOS
+	Version: 1.16.3-debian-1.16.3-2
+	Release Date: 04/01/2014
+	Address: 0xE8000
+	Runtime Size: 96 kB
+	ROM Size: 64 kB
+	Characteristics:
+		BIOS characteristics not supported
+		Targeted content distribution is supported
+	BIOS Revision: 0.0
+
+Handle 0x0100, DMI type 1, 27 bytes
+System Information
+	Manufacturer: QEMU
+	Product Name: Standard PC (i440FX + PIIX, 1996)
+	Version: pc-i440fx-10.0
+	Serial Number: Not Specified
+	UUID: TEST-0002
+	Wake-up Type: Power Switch
+	SKU Number: Not Specified
+	Family: Not Specified
+
+Handle 0x0300, DMI type 3, 22 bytes
+Chassis Information
+	Manufacturer: QEMU
+	Type: Other
+	Lock: Not Present
+	Version: pc-i440fx-10.0
+	Serial Number: Not Specified
+	Asset Tag: Not Specified
+	Boot-up State: Safe
+	Power Supply State: Safe
+	Thermal State: Safe
+	Security Status: Unknown
+	OEM Information: 0x00000000
+	Height: Unspecified
+	Number Of Power Cords: Unspecified
+	Contained Elements: 0
+	SKU Number: Not Specified
+
+Handle 0x2000, DMI type 32, 11 bytes
+System Boot Information
+	Status: No errors detected
+

spinifex/runtime/host/nodefacts/testdata/devbox/dmi-memory.txt

@@ -0,0 +1,37 @@
+# dmidecode 3.6
+Getting SMBIOS data from sysfs.
+SMBIOS 2.8 present.
+
+Handle 0x1000, DMI type 16, 23 bytes
+Physical Memory Array
+	Location: Other
+	Use: System Memory
+	Error Correction Type: Multi-bit ECC
+	Maximum Capacity: 32 GB
+	Error Information Handle: Not Provided
+	Number Of Devices: 1
+
+Handle 0x1100, DMI type 17, 40 bytes
+Memory Device
+	Array Handle: 0x1000
+	Error Information Handle: Not Provided
+	Total Width: Unknown
+	Data Width: Unknown
+	Size: 32 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM 0
+	Bank Locator: Not Specified
+	Type: RAM
+	Type Detail: Other
+	Speed: Unknown
+	Manufacturer: QEMU
+	Serial Number: Not Specified
+	Asset Tag: Not Specified
+	Part Number: Not Specified
+	Rank: Unknown
+	Configured Memory Speed: Unknown
+	Minimum Voltage: Unknown
+	Maximum Voltage: Unknown
+	Configured Voltage: Unknown
+

spinifex/runtime/host/nodefacts/testdata/devbox/dmi-processor.txt

@@ -0,0 +1,404 @@
+# dmidecode 3.6
+Getting SMBIOS data from sysfs.
+SMBIOS 2.8 present.
+
+Handle 0x0400, DMI type 4, 42 bytes
+Processor Information
+	Socket Designation: CPU 0
+	Type: Central Processor
+	Family: Other
+	Manufacturer: QEMU
+	ID: TEST-0001
+	Version: pc-i440fx-10.0
+	Voltage: Unknown
+	External Clock: Unknown
+	Max Speed: 2000 MHz
+	Current Speed: 2000 MHz
+	Status: Populated, Enabled
+	Upgrade: Other
+	L1 Cache Handle: Not Provided
+	L2 Cache Handle: Not Provided
+	L3 Cache Handle: Not Provided
+	Serial Number: Not Specified
+	Asset Tag: Not Specified
+	Part Number: Not Specified
+	Core Count: 1
+	Core Enabled: 1
+	Thread Count: 1
+	Characteristics: None
+
+Handle 0x0401, DMI type 4, 42 bytes
+Processor Information
+	Socket Designation: CPU 1
+	Type: Central Processor
+	Family: Other
+	Manufacturer: QEMU
+	ID: TEST-0004
+	Version: pc-i440fx-10.0
+	Voltage: Unknown
+	External Clock: Unknown
+	Max Speed: 2000 MHz
+	Current Speed: 2000 MHz
+	Status: Populated, Enabled
+	Upgrade: Other
+	L1 Cache Handle: Not Provided
+	L2 Cache Handle: Not Provided
+	L3 Cache Handle: Not Provided
+	Serial Number: Not Specified
+	Asset Tag: Not Specified
+	Part Number: Not Specified
+	Core Count: 1
+	Core Enabled: 1
+	Thread Count: 1
+	Characteristics: None
+
+Handle 0x0402, DMI type 4, 42 bytes
+Processor Information
+	Socket Designation: CPU 2
+	Type: Central Processor
+	Family: Other
+	Manufacturer: QEMU
+	ID: TEST-0007
+	Version: pc-i440fx-10.0
+	Voltage: Unknown
+	External Clock: Unknown
+	Max Speed: 2000 MHz
+	Current Speed: 2000 MHz
+	Status: Populated, Enabled
+	Upgrade: Other
+	L1 Cache Handle: Not Provided
+	L2 Cache Handle: Not Provided
+	L3 Cache Handle: Not Provided
+	Serial Number: Not Specified
+	Asset Tag: Not Specified
+	Part Number: Not Specified
+	Core Count: 1
+	Core Enabled: 1
+	Thread Count: 1
+	Characteristics: None
+
+Handle 0x0403, DMI type 4, 42 bytes
+Processor Information
+	Socket Designation: CPU 3
+	Type: Central Processor
+	Family: Other
+	Manufacturer: QEMU
+	ID: TEST-0010
+	Version: pc-i440fx-10.0
+	Voltage: Unknown
+	External Clock: Unknown
+	Max Speed: 2000 MHz
+	Current Speed: 2000 MHz
+	Status: Populated, Enabled
+	Upgrade: Other
+	L1 Cache Handle: Not Provided
+	L2 Cache Handle: Not Provided
+	L3 Cache Handle: Not Provided
+	Serial Number: Not Specified
+	Asset Tag: Not Specified
+	Part Number: Not Specified
+	Core Count: 1
+	Core Enabled: 1
+	Thread Count: 1
+	Characteristics: None
+
+Handle 0x0404, DMI type 4, 42 bytes
+Processor Information
+	Socket Designation: CPU 4
+	Type: Central Processor
+	Family: Other
+	Manufacturer: QEMU
+	ID: TEST-0013
+	Version: pc-i440fx-10.0
+	Voltage: Unknown
+	External Clock: Unknown
+	Max Speed: 2000 MHz
+	Current Speed: 2000 MHz
+	Status: Populated, Enabled
+	Upgrade: Other
+	L1 Cache Handle: Not Provided
+	L2 Cache Handle: Not Provided
+	L3 Cache Handle: Not Provided
+	Serial Number: Not Specified
+	Asset Tag: Not Specified
+	Part Number: Not Specified
+	Core Count: 1
+	Core Enabled: 1
+	Thread Count: 1
+	Characteristics: None
+
+Handle 0x0405, DMI type 4, 42 bytes
+Processor Information
+	Socket Designation: CPU 5
+	Type: Central Processor
+	Family: Other
+	Manufacturer: QEMU
+	ID: TEST-0016
+	Version: pc-i440fx-10.0
+	Voltage: Unknown
+	External Clock: Unknown
+	Max Speed: 2000 MHz
+	Current Speed: 2000 MHz
+	Status: Populated, Enabled
+	Upgrade: Other
+	L1 Cache Handle: Not Provided
+	L2 Cache Handle: Not Provided
+	L3 Cache Handle: Not Provided
+	Serial Number: Not Specified
+	Asset Tag: Not Specified
+	Part Number: Not Specified
+	Core Count: 1
+	Core Enabled: 1
+	Thread Count: 1
+	Characteristics: None
+
+Handle 0x0406, DMI type 4, 42 bytes
+Processor Information
+	Socket Designation: CPU 6
+	Type: Central Processor
+	Family: Other
+	Manufacturer: QEMU
+	ID: TEST-0019
+	Version: pc-i440fx-10.0
+	Voltage: Unknown
+	External Clock: Unknown
+	Max Speed: 2000 MHz
+	Current Speed: 2000 MHz
+	Status: Populated, Enabled
+	Upgrade: Other
+	L1 Cache Handle: Not Provided
+	L2 Cache Handle: Not Provided
+	L3 Cache Handle: Not Provided
+	Serial Number: Not Specified
+	Asset Tag: Not Specified
+	Part Number: Not Specified
+	Core Count: 1
+	Core Enabled: 1
+	Thread Count: 1
+	Characteristics: None
+
+Handle 0x0407, DMI type 4, 42 bytes
+Processor Information
+	Socket Designation: CPU 7
+	Type: Central Processor
+	Family: Other
+	Manufacturer: QEMU
+	ID: TEST-0022
+	Version: pc-i440fx-10.0
+	Voltage: Unknown
+	External Clock: Unknown
+	Max Speed: 2000 MHz
+	Current Speed: 2000 MHz
+	Status: Populated, Enabled
+	Upgrade: Other
+	L1 Cache Handle: Not Provided
+	L2 Cache Handle: Not Provided
+	L3 Cache Handle: Not Provided
+	Serial Number: Not Specified
+	Asset Tag: Not Specified
+	Part Number: Not Specified
+	Core Count: 1
+	Core Enabled: 1
+	Thread Count: 1
+	Characteristics: None
+
+Handle 0x0408, DMI type 4, 42 bytes
+Processor Information
+	Socket Designation: CPU 8
+	Type: Central Processor
+	Family: Other
+	Manufacturer: QEMU
+	ID: TEST-0025
+	Version: pc-i440fx-10.0
+	Voltage: Unknown
+	External Clock: Unknown
+	Max Speed: 2000 MHz
+	Current Speed: 2000 MHz
+	Status: Populated, Enabled
+	Upgrade: Other
+	L1 Cache Handle: Not Provided
+	L2 Cache Handle: Not Provided
+	L3 Cache Handle: Not Provided
+	Serial Number: Not Specified
+	Asset Tag: Not Specified
+	Part Number: Not Specified
+	Core Count: 1
+	Core Enabled: 1
+	Thread Count: 1
+	Characteristics: None
+
+Handle 0x0409, DMI type 4, 42 bytes
+Processor Information
+	Socket Designation: CPU 9
+	Type: Central Processor
+	Family: Other
+	Manufacturer: QEMU
+	ID: TEST-0028
+	Version: pc-i440fx-10.0
+	Voltage: Unknown
+	External Clock: Unknown
+	Max Speed: 2000 MHz
+	Current Speed: 2000 MHz
+	Status: Populated, Enabled
+	Upgrade: Other
+	L1 Cache Handle: Not Provided
+	L2 Cache Handle: Not Provided
+	L3 Cache Handle: Not Provided
+	Serial Number: Not Specified
+	Asset Tag: Not Specified
+	Part Number: Not Specified
+	Core Count: 1
+	Core Enabled: 1
+	Thread Count: 1
+	Characteristics: None
+
+Handle 0x040A, DMI type 4, 42 bytes
+Processor Information
+	Socket Designation: CPU a
+	Type: Central Processor
+	Family: Other
+	Manufacturer: QEMU
+	ID: TEST-0031
+	Version: pc-i440fx-10.0
+	Voltage: Unknown
+	External Clock: Unknown
+	Max Speed: 2000 MHz
+	Current Speed: 2000 MHz
+	Status: Populated, Enabled
+	Upgrade: Other
+	L1 Cache Handle: Not Provided
+	L2 Cache Handle: Not Provided
+	L3 Cache Handle: Not Provided
+	Serial Number: Not Specified
+	Asset Tag: Not Specified
+	Part Number: Not Specified
+	Core Count: 1
+	Core Enabled: 1
+	Thread Count: 1
+	Characteristics: None
+
+Handle 0x040B, DMI type 4, 42 bytes
+Processor Information
+	Socket Designation: CPU b
+	Type: Central Processor
+	Family: Other
+	Manufacturer: QEMU
+	ID: TEST-0034
+	Version: pc-i440fx-10.0
+	Voltage: Unknown
+	External Clock: Unknown
+	Max Speed: 2000 MHz
+	Current Speed: 2000 MHz
+	Status: Populated, Enabled
+	Upgrade: Other
+	L1 Cache Handle: Not Provided
+	L2 Cache Handle: Not Provided
+	L3 Cache Handle: Not Provided
+	Serial Number: Not Specified
+	Asset Tag: Not Specified
+	Part Number: Not Specified
+	Core Count: 1
+	Core Enabled: 1
+	Thread Count: 1
+	Characteristics: None
+
+Handle 0x040C, DMI type 4, 42 bytes
+Processor Information
+	Socket Designation: CPU c
+	Type: Central Processor
+	Family: Other
+	Manufacturer: QEMU
+	ID: TEST-0037
+	Version: pc-i440fx-10.0
+	Voltage: Unknown
+	External Clock: Unknown
+	Max Speed: 2000 MHz
+	Current Speed: 2000 MHz
+	Status: Populated, Enabled
+	Upgrade: Other
+	L1 Cache Handle: Not Provided
+	L2 Cache Handle: Not Provided
+	L3 Cache Handle: Not Provided
+	Serial Number: Not Specified
+	Asset Tag: Not Specified
+	Part Number: Not Specified
+	Core Count: 1
+	Core Enabled: 1
+	Thread Count: 1
+	Characteristics: None
+
+Handle 0x040D, DMI type 4, 42 bytes
+Processor Information
+	Socket Designation: CPU d
+	Type: Central Processor
+	Family: Other
+	Manufacturer: QEMU
+	ID: TEST-0040
+	Version: pc-i440fx-10.0
+	Voltage: Unknown
+	External Clock: Unknown
+	Max Speed: 2000 MHz
+	Current Speed: 2000 MHz
+	Status: Populated, Enabled
+	Upgrade: Other
+	L1 Cache Handle: Not Provided
+	L2 Cache Handle: Not Provided
+	L3 Cache Handle: Not Provided
+	Serial Number: Not Specified
+	Asset Tag: Not Specified
+	Part Number: Not Specified
+	Core Count: 1
+	Core Enabled: 1
+	Thread Count: 1
+	Characteristics: None
+
+Handle 0x040E, DMI type 4, 42 bytes
+Processor Information
+	Socket Designation: CPU e
+	Type: Central Processor
+	Family: Other
+	Manufacturer: QEMU
+	ID: TEST-0043
+	Version: pc-i440fx-10.0
+	Voltage: Unknown
+	External Clock: Unknown
+	Max Speed: 2000 MHz
+	Current Speed: 2000 MHz
+	Status: Populated, Enabled
+	Upgrade: Other
+	L1 Cache Handle: Not Provided
+	L2 Cache Handle: Not Provided
+	L3 Cache Handle: Not Provided
+	Serial Number: Not Specified
+	Asset Tag: Not Specified
+	Part Number: Not Specified
+	Core Count: 1
+	Core Enabled: 1
+	Thread Count: 1
+	Characteristics: None
+
+Handle 0x040F, DMI type 4, 42 bytes
+Processor Information
+	Socket Designation: CPU f
+	Type: Central Processor
+	Family: Other
+	Manufacturer: QEMU
+	ID: TEST-0046
+	Version: pc-i440fx-10.0
+	Voltage: Unknown
+	External Clock: Unknown
+	Max Speed: 2000 MHz
+	Current Speed: 2000 MHz
+	Status: Populated, Enabled
+	Upgrade: Other
+	L1 Cache Handle: Not Provided
+	L2 Cache Handle: Not Provided
+	L3 Cache Handle: Not Provided
+	Serial Number: Not Specified
+	Asset Tag: Not Specified
+	Part Number: Not Specified
+	Core Count: 1
+	Core Enabled: 1
+	Thread Count: 1
+	Characteristics: None
+

spinifex/runtime/host/nodefacts/testdata/node1/dmi-identity.txt

@@ -0,0 +1,189 @@
+# dmidecode 3.6
+Getting SMBIOS data from sysfs.
+SMBIOS 3.3.0 present.
+
+Handle 0x0000, DMI type 0, 26 bytes
+BIOS Information
+	Vendor: GIGABYTE
+	Version: M30
+	Release Date: 06/23/2026
+	Address: 0xF0000
+	Runtime Size: 64 kB
+	ROM Size: 16 MB
+	Characteristics:
+		PCI is supported
+		BIOS is upgradeable
+		BIOS shadowing is allowed
+		Boot from CD is supported
+		Selectable boot is supported
+		BIOS ROM is socketed
+		EDD is supported
+		Japanese floppy for NEC 9800 1.2 MB is supported (int 13h)
+		Japanese floppy for Toshiba 1.2 MB is supported (int 13h)
+		5.25"/360 kB floppy services are supported (int 13h)
+		5.25"/1.2 MB floppy services are supported (int 13h)
+		3.5"/720 kB floppy services are supported (int 13h)
+		3.5"/2.88 MB floppy services are supported (int 13h)
+		Print screen service is supported (int 5h)
+		Serial services are supported (int 14h)
+		Printer services are supported (int 17h)
+		CGA/mono video services are supported (int 10h)
+		USB legacy is supported
+		BIOS boot specification is supported
+		Targeted content distribution is supported
+		UEFI is supported
+	BIOS Revision: 5.22
+
+Handle 0x0001, DMI type 1, 27 bytes
+System Information
+	Manufacturer: GIGABYTE
+	Product Name: H262-Z62-ZB
+	Version: 0100
+	Serial Number: TEST-0001
+	UUID: TEST-0002
+	Wake-up Type: Power Switch
+	SKU Number: 01234567890123456789AB
+	Family: Server
+
+Handle 0x0002, DMI type 2, 15 bytes
+Base Board Information
+	Manufacturer: GIGABYTE
+	Product Name: MZ62-HD0-ZB
+	Version: 02000200
+	Serial Number: TEST-0003
+	Asset Tag: TEST-0004
+	Features:
+		Board is a hosting board
+		Board is removable
+		Board is replaceable
+	Location In Chassis: 01234567890123456789AB
+	Chassis Handle: 0x0003
+	Type: Motherboard
+	Contained Object Handles: 0
+
+Handle 0x0003, DMI type 3, 22 bytes
+Chassis Information
+	Manufacturer: GIGABYTE
+	Type: Rack Mount Chassis
+	Lock: Not Present
+	Version: 01234567
+	Serial Number: TEST-0005
+	Asset Tag: TEST-0006
+	Boot-up State: Safe
+	Power Supply State: Safe
+	Thermal State: Safe
+	Security Status: None
+	OEM Information: 0x00000000
+	Height: Unspecified
+	Number Of Power Cords: 1
+	Contained Elements: 0
+	SKU Number: 01234567890123456789AB
+
+Handle 0x0023, DMI type 10, 10 bytes
+On Board Device 1 Information
+	Type: Video
+	Status: Enabled
+	Description: AST2500
+On Board Device 2 Information
+	Type: Ethernet
+	Status: Enabled
+	Description: Intel i350 Lan1
+On Board Device 3 Information
+	Type: Ethernet
+	Status: Enabled
+	Description: Intel i350 Lan2
+
+Handle 0x0024, DMI type 41, 11 bytes
+Onboard Device
+	Reference Designation:  Onboard VGA
+	Type: Video
+	Status: Enabled
+	Type Instance: 1
+	Bus Address: 0000:63:00.0
+
+Handle 0x0025, DMI type 41, 11 bytes
+Onboard Device
+	Reference Designation:  Onboard LAN1
+	Type: Ethernet
+	Status: Enabled
+	Type Instance: 1
+	Bus Address: 0000:61:00.0
+
+Handle 0x0026, DMI type 41, 11 bytes
+Onboard Device
+	Reference Designation:  Onboard LAN2
+	Type: Ethernet
+	Status: Enabled
+	Type Instance: 2
+	Bus Address: 0000:61:00.1
+
+Handle 0x0037, DMI type 15, 73 bytes
+System Event Log
+	Area Length: 0 bytes
+	Header Start Offset: 0x0000
+	Header Length: 16 bytes
+	Data Start Offset: 0x0010
+	Access Method: Memory-mapped physical 32-bit address
+	Access Address: 0xFF536000
+	Status: Valid, Not Full
+	Change Token: 0x00000015
+	Header Format: Type 1
+	Supported Log Type Descriptors: 25
+	Descriptor 1: Single-bit ECC memory error
+	Data Format 1: Multiple-event handle
+	Descriptor 2: Multi-bit ECC memory error
+	Data Format 2: Multiple-event handle
+	Descriptor 3: Parity memory error
+	Data Format 3: None
+	Descriptor 4: Bus timeout
+	Data Format 4: None
+	Descriptor 5: I/O channel block
+	Data Format 5: None
+	Descriptor 6: Software NMI
+	Data Format 6: None
+	Descriptor 7: POST memory resize
+	Data Format 7: None
+	Descriptor 8: POST error
+	Data Format 8: POST results bitmap
+	Descriptor 9: PCI parity error
+	Data Format 9: Multiple-event handle
+	Descriptor 10: PCI system error
+	Data Format 10: Multiple-event handle
+	Descriptor 11: CPU failure
+	Data Format 11: None
+	Descriptor 12: EISA failsafe timer timeout
+	Data Format 12: None
+	Descriptor 13: Correctable memory log disabled
+	Data Format 13: None
+	Descriptor 14: Logging disabled
+	Data Format 14: None
+	Descriptor 15: System limit exceeded
+	Data Format 15: None
+	Descriptor 16: Asynchronous hardware timer expired
+	Data Format 16: None
+	Descriptor 17: System configuration information
+	Data Format 17: None
+	Descriptor 18: Hard disk information
+	Data Format 18: None
+	Descriptor 19: System reconfigured
+	Data Format 19: None
+	Descriptor 20: Uncorrectable CPU-complex error
+	Data Format 20: None
+	Descriptor 21: Log area reset/cleared
+	Data Format 21: None
+	Descriptor 22: System boot
+	Data Format 22: None
+	Descriptor 23: End of log
+	Data Format 23: None
+	Descriptor 24: OEM-specific
+	Data Format 24: OEM-specific
+	Descriptor 25: OEM-specific
+	Data Format 25: OEM-specific
+
+Handle 0x0085, DMI type 13, 22 bytes
+BIOS Language Information
+	Language Description Format: Long
+	Installable Languages: 1
+		en|US|iso8859-1
+	Currently Installed Language: en|US|iso8859-1
+

spinifex/runtime/host/nodefacts/testdata/node1/dmi-memory.txt

@@ -0,0 +1,531 @@
+# dmidecode 3.6
+Getting SMBIOS data from sysfs.
+SMBIOS 3.3.0 present.
+
+Handle 0x0039, DMI type 16, 23 bytes
+Physical Memory Array
+	Location: System Board Or Motherboard
+	Use: System Memory
+	Error Correction Type: Multi-bit ECC
+	Maximum Capacity: 4 TB
+	Error Information Handle: 0x0038
+	Number Of Devices: 16
+
+Handle 0x0041, DMI type 17, 92 bytes
+Memory Device
+	Array Handle: 0x0039
+	Error Information Handle: 0x0040
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P0_A0
+	Bank Locator: BANK 0
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2933 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0001
+	Asset Tag: TEST-0002
+	Part Number: HMA82GR7JJR8N-WM    
+	Rank: 2
+	Configured Memory Speed: 2933 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x0044, DMI type 17, 92 bytes
+Memory Device
+	Array Handle: 0x0039
+	Error Information Handle: 0x0043
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P0_B0
+	Bank Locator: BANK 0
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2933 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0003
+	Asset Tag: TEST-0004
+	Part Number: HMA82GR7JJR8N-WM    
+	Rank: 2
+	Configured Memory Speed: 2933 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x0047, DMI type 17, 92 bytes
+Memory Device
+	Array Handle: 0x0039
+	Error Information Handle: 0x0046
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P0_C0
+	Bank Locator: BANK 0
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2933 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0005
+	Asset Tag: TEST-0006
+	Part Number: HMA82GR7JJR8N-WM    
+	Rank: 2
+	Configured Memory Speed: 2933 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x004A, DMI type 17, 92 bytes
+Memory Device
+	Array Handle: 0x0039
+	Error Information Handle: 0x0049
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P0_D0
+	Bank Locator: BANK 0
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2933 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0007
+	Asset Tag: TEST-0008
+	Part Number: HMA82GR7JJR8N-WM    
+	Rank: 2
+	Configured Memory Speed: 2933 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x004D, DMI type 17, 92 bytes
+Memory Device
+	Array Handle: 0x0039
+	Error Information Handle: 0x004C
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P0_E0
+	Bank Locator: BANK 0
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2933 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0009
+	Asset Tag: TEST-0010
+	Part Number: HMA82GR7JJR8N-WM    
+	Rank: 2
+	Configured Memory Speed: 2933 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x0050, DMI type 17, 92 bytes
+Memory Device
+	Array Handle: 0x0039
+	Error Information Handle: 0x004F
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P0_F0
+	Bank Locator: BANK 0
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2933 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0011
+	Asset Tag: TEST-0012
+	Part Number: HMA82GR7JJR8N-WM    
+	Rank: 2
+	Configured Memory Speed: 2933 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x0053, DMI type 17, 92 bytes
+Memory Device
+	Array Handle: 0x0039
+	Error Information Handle: 0x0052
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P0_G0
+	Bank Locator: BANK 0
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2933 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0013
+	Asset Tag: TEST-0014
+	Part Number: HMA82GR7JJR8N-WM    
+	Rank: 2
+	Configured Memory Speed: 2933 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x0056, DMI type 17, 92 bytes
+Memory Device
+	Array Handle: 0x0039
+	Error Information Handle: 0x0055
+	Total Width: Unknown
+	Data Width: Unknown
+	Size: No Module Installed
+	Form Factor: Unknown
+	Set: None
+	Locator: DIMM_P0_H0
+	Bank Locator: BANK 0
+	Type: Unknown
+	Type Detail: Unknown
+
+Handle 0x005C, DMI type 17, 92 bytes
+Memory Device
+	Array Handle: 0x0039
+	Error Information Handle: 0x005B
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P1_I0
+	Bank Locator: BANK 0
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2933 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0015
+	Asset Tag: TEST-0016
+	Part Number: HMA82GR7JJR8N-WM    
+	Rank: 2
+	Configured Memory Speed: 2933 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x005F, DMI type 17, 92 bytes
+Memory Device
+	Array Handle: 0x0039
+	Error Information Handle: 0x005E
+	Total Width: Unknown
+	Data Width: Unknown
+	Size: No Module Installed
+	Form Factor: Unknown
+	Set: None
+	Locator: DIMM_P1_J0
+	Bank Locator: BANK 0
+	Type: Unknown
+	Type Detail: Unknown
+
+Handle 0x0061, DMI type 17, 92 bytes
+Memory Device
+	Array Handle: 0x0039
+	Error Information Handle: 0x0060
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P1_K0
+	Bank Locator: BANK 0
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2933 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0017
+	Asset Tag: TEST-0018
+	Part Number: HMA82GR7JJR8N-WM    
+	Rank: 2
+	Configured Memory Speed: 2933 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x0064, DMI type 17, 92 bytes
+Memory Device
+	Array Handle: 0x0039
+	Error Information Handle: 0x0063
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P1_L0
+	Bank Locator: BANK 0
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2933 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0019
+	Asset Tag: TEST-0020
+	Part Number: HMA82GR7JJR8N-WM    
+	Rank: 2
+	Configured Memory Speed: 2933 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x0067, DMI type 17, 92 bytes
+Memory Device
+	Array Handle: 0x0039
+	Error Information Handle: 0x0066
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P1_M0
+	Bank Locator: BANK 0
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2933 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0021
+	Asset Tag: TEST-0022
+	Part Number: HMA82GR7JJR8N-WM    
+	Rank: 2
+	Configured Memory Speed: 2933 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x006A, DMI type 17, 92 bytes
+Memory Device
+	Array Handle: 0x0039
+	Error Information Handle: 0x0069
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P1_N0
+	Bank Locator: BANK 0
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2933 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0023
+	Asset Tag: TEST-0024
+	Part Number: HMA82GR7JJR8N-WM    
+	Rank: 2
+	Configured Memory Speed: 2933 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x006D, DMI type 17, 92 bytes
+Memory Device
+	Array Handle: 0x0039
+	Error Information Handle: 0x006C
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P1_O0
+	Bank Locator: BANK 0
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2933 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0025
+	Asset Tag: TEST-0026
+	Part Number: HMA82GR7JJR8N-WM    
+	Rank: 2
+	Configured Memory Speed: 2933 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+
+Handle 0x0070, DMI type 17, 92 bytes
+Memory Device
+	Array Handle: 0x0039
+	Error Information Handle: 0x006F
+	Total Width: 72 bits
+	Data Width: 64 bits
+	Size: 16 GB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM_P1_P0
+	Bank Locator: BANK 0
+	Type: DDR4
+	Type Detail: Synchronous Registered (Buffered)
+	Speed: 2933 MT/s
+	Manufacturer: SK Hynix
+	Serial Number: TEST-0027
+	Asset Tag: TEST-0028
+	Part Number: HMA82GR7JJR8N-WM    
+	Rank: 2
+	Configured Memory Speed: 2933 MT/s
+	Minimum Voltage: 1.2 V
+	Maximum Voltage: 1.2 V
+	Configured Voltage: 1.2 V
+	Memory Technology: DRAM
+	Memory Operating Mode Capability: Volatile memory
+	Firmware Version: Unknown
+	Module Manufacturer ID: Bank 1, Hex 0xAD
+	Module Product ID: Unknown
+	Memory Subsystem Controller Manufacturer ID: Unknown
+	Memory Subsystem Controller Product ID: Unknown
+	Non-Volatile Size: None
+	Volatile Size: 16 GB
+	Cache Size: None
+	Logical Size: None
+

spinifex/runtime/host/nodefacts/testdata/node1/dmi-processor.txt

@@ -0,0 +1,116 @@
+# dmidecode 3.6
+Getting SMBIOS data from sysfs.
+SMBIOS 3.3.0 present.
+
+Handle 0x003F, DMI type 4, 48 bytes
+Processor Information
+	Socket Designation: P0
+	Type: Central Processor
+	Family: Zen
+	Manufacturer: Advanced Micro Devices, Inc.
+	ID: TEST-0001
+	Signature: Family 25, Model 1, Stepping 1
+	Flags:
+		FPU (Floating-point unit on-chip)
+		VME (Virtual mode extension)
+		DE (Debugging extension)
+		PSE (Page size extension)
+		TSC (Time stamp counter)
+		MSR (Model specific registers)
+		PAE (Physical address extension)
+		MCE (Machine check exception)
+		CX8 (CMPXCHG8 instruction supported)
+		APIC (On-chip APIC hardware supported)
+		SEP (Fast system call)
+		MTRR (Memory type range registers)
+		PGE (Page global enable)
+		MCA (Machine check architecture)
+		CMOV (Conditional move instruction supported)
+		PAT (Page attribute table)
+		PSE-36 (36-bit page size extension)
+		CLFSH (CLFLUSH instruction supported)
+		MMX (MMX technology supported)
+		FXSR (FXSAVE and FXSTOR instructions supported)
+		SSE (Streaming SIMD extensions)
+		SSE2 (Streaming SIMD extensions 2)
+		HTT (Multi-threading)
+	Version: AMD EPYC 7513 32-Core Processor
+	Voltage: 1.1 V
+	External Clock: 100 MHz
+	Max Speed: 3675 MHz
+	Current Speed: 2600 MHz
+	Status: Populated, Enabled
+	Upgrade: Socket SP3
+	L1 Cache Handle: 0x003C
+	L2 Cache Handle: 0x003D
+	L3 Cache Handle: 0x003E
+	Serial Number: Unknown
+	Asset Tag: Unknown
+	Part Number: Unknown
+	Core Count: 32
+	Core Enabled: 32
+	Thread Count: 64
+	Characteristics:
+		64-bit capable
+		Multi-Core
+		Hardware Thread
+		Execute Protection
+		Enhanced Virtualization
+		Power/Performance Control
+
+Handle 0x005A, DMI type 4, 48 bytes
+Processor Information
+	Socket Designation: P1
+	Type: Central Processor
+	Family: Zen
+	Manufacturer: Advanced Micro Devices, Inc.
+	ID: TEST-0004
+	Signature: Family 25, Model 1, Stepping 1
+	Flags:
+		FPU (Floating-point unit on-chip)
+		VME (Virtual mode extension)
+		DE (Debugging extension)
+		PSE (Page size extension)
+		TSC (Time stamp counter)
+		MSR (Model specific registers)
+		PAE (Physical address extension)
+		MCE (Machine check exception)
+		CX8 (CMPXCHG8 instruction supported)
+		APIC (On-chip APIC hardware supported)
+		SEP (Fast system call)
+		MTRR (Memory type range registers)
+		PGE (Page global enable)
+		MCA (Machine check architecture)
+		CMOV (Conditional move instruction supported)
+		PAT (Page attribute table)
+		PSE-36 (36-bit page size extension)
+		CLFSH (CLFLUSH instruction supported)
+		MMX (MMX technology supported)
+		FXSR (FXSAVE and FXSTOR instructions supported)
+		SSE (Streaming SIMD extensions)
+		SSE2 (Streaming SIMD extensions 2)
+		HTT (Multi-threading)
+	Version: AMD EPYC 7513 32-Core Processor
+	Voltage: 1.1 V
+	External Clock: 100 MHz
+	Max Speed: 3675 MHz
+	Current Speed: 2600 MHz
+	Status: Populated, Enabled
+	Upgrade: Socket SP3
+	L1 Cache Handle: 0x0057
+	L2 Cache Handle: 0x0058
+	L3 Cache Handle: 0x0059
+	Serial Number: Unknown
+	Asset Tag: Unknown
+	Part Number: Unknown
+	Core Count: 32
+	Core Enabled: 32
+	Thread Count: 64
+	Characteristics:
+		64-bit capable
+		Multi-Core
+		Hardware Thread
+		Execute Protection
+		Enhanced Virtualization
+		Power/Performance Control
+

spinifex/runtime/host/nodefacts/testdata/node1/lldp.json

@@ -0,0 +1 @@
+{"Neighbors":[{"InterfaceIndex":2,"InterfaceName":"eno1","InterfaceAlternativeNames":["enp97s0f0","enx74563c072e4f"],"Neighbors":[{"ChassisID":"04:f4:1c:ec:da:9e","RawChassisID":[4,4,244,28,236,218,158],"PortID":"ether18","RawPortID":[5,101,116,104,101,114,49,56],"PortDescription":"bridge/ether18","SystemName":"MikroTik 1gbe","SystemDescription":"MikroTik RouterOS 7.19.6 (stable) 2025-09-12 09:02:42 CRS326-24G-2S+","EnabledCapabilities":20}]},{"InterfaceIndex":19,"InterfaceName":"enp33s0f0np0","InterfaceAlternativeNames":["enxb8599fe8b142"],"Neighbors":[{"ChassisID":"d0:ea:11:a8:91:4c","RawChassisID":[4,208,234,17,168,145,76],"PortID":"qsfp28-7-1","RawPortID":[5,113,115,102,112,50,56,45,55,45,49],"PortDescription":"bridge/qsfp28-7-1","SystemName":"MikroTik 100gbe","SystemDescription":"MikroTik RouterOS 7.20.8 (long-term) 2026-01-30 09:17:54 CRS520-4XS-16XQ","EnabledCapabilities":20}]},{"InterfaceIndex":20,"InterfaceName":"enp33s0f1np1","InterfaceAlternativeNames":["enxb8599fe8b143"],"Neighbors":[{"ChassisID":"d0:ea:11:a8:91:4c","RawChassisID":[4,208,234,17,168,145,76],"PortID":"qsfp28-5-1","RawPortID":[5,113,115,102,112,50,56,45,53,45,49],"PortDescription":"bridge/qsfp28-5-1","SystemName":"MikroTik 100gbe","SystemDescription":"MikroTik RouterOS 7.20.8 (long-term) 2026-01-30 09:17:54 CRS520-4XS-16XQ","EnabledCapabilities":20}]}]}

spinifex/runtime/roles/awsgw/awsgw.go

@@ -654,6 +654,8 @@ func launchService(config *config.ClusterConfig) error {
 		Credentials: bedrockCredentials,
 		Access:      bedrockAccess,
 
+		NetworkTests: operatoradmin.NewKVNetworkTests(js),
+
 		IAM:               iamService,
 		Quota:             quotaSvc,
 		ExpectedNodes:     len(config.Nodes),

spinifex/runtime/roles/spinifexui/frontend/dist/assets/CompositeItem-CtFYX_Ap.js

@@ -1 +0,0 @@
-import{Mt as e,jt as t,kt as n}from"./schemas-BGv4hnp9.js";import{o as r}from"./tabs-DFnnXw6G.js";function i(i){let{render:a,className:o,style:s,state:c=e,props:l=t,refs:u=t,metadata:d,stateAttributesMapping:f,tag:p=`div`,...m}=i,{compositeProps:h,compositeRef:g}=r({metadata:d});return n(p,i,{state:c,ref:[g,...u],props:[h,...l,m],stateAttributesMapping:f})}export{i as t};

spinifex/runtime/roles/spinifexui/frontend/dist/assets/CompositeItem-Xr7Be9xO.js

@@ -0,0 +1 @@
+import{dt as e,ft as t,lt as n}from"./reason-parts-CkZCsP-9.js";import{o as r}from"./tabs-CqV1jskI.js";function i(i){let{render:a,className:o,style:s,state:c=t,props:l=e,refs:u=e,metadata:d,stateAttributesMapping:f,tag:p=`div`,...m}=i,{compositeProps:h,compositeRef:g}=r({metadata:d});return n(p,i,{state:c,ref:[g,...u],props:[h,...l,m],stateAttributesMapping:f})}export{i as t};

spinifex/runtime/roles/spinifexui/frontend/dist/assets/Match-CqMVQsb0.js

@@ -1 +0,0 @@
-import{r as e}from"./rolldown-runtime-XKyaHtl9.js";import{Ht as t,Ut as n}from"./schemas-BGv4hnp9.js";import{i as r,p as i,r as a}from"./matchContext-DBnoSZp3.js";import{n as o}from"./x-8n_Gyi12.js";import{t as s}from"./useSelector-Cvbg8b7C.js";var c=e(n(),1),l=t(),u=()=>!0,d=()=>!1;function f({children:e,fallback:t=null}){return(0,l.jsx)(c.Fragment,{children:p()?e:t})}function p(e=!0){return c.useSyncExternalStore(m,u,e?d:u)}function m(){return()=>{}}function h(e){let t=r(),n=`not-found-${s(t.stores.location,e=>e.pathname)}-${s(t.stores.status)}`;return(0,l.jsx)(o,{getResetKey:()=>n,onCatch:(t,n)=>{if(i(t))e.onCatch?.(t,n);else throw t},errorComponent:({error:t})=>{if(i(t))return e.fallback?.(t);throw t},children:e.children})}function g(){return(0,l.jsx)(`p`,{children:`Not Found`})}function _(e,t,n){return t.options.notFoundComponent?(0,l.jsx)(t.options.notFoundComponent,{...n}):e.options.defaultNotFoundComponent?(0,l.jsx)(e.options.defaultNotFoundComponent,{...n}):(0,l.jsx)(g,{})}function v(e,t){let n=t?.options.pendingComponent??e.options.defaultPendingComponent;return n?(0,l.jsx)(n,{}):null}var y=(e,t)=>e[0]===t[0]&&e[1]===t[1],b=(e,t,n)=>!t.isRoot||t.options.shellComponent||t.options.wrapInSuspense||n===!1||n===`data-only`||!e.ssr,x=c.memo(function({routeId:e}){let t=r(),n=t.stores.getMatchStore(e);return S(t,s(n))});function S(e,t){let n=e.routesById[t.routeId],r=v(e,n),s=n.options.errorComponent??e.options.defaultErrorComponent,u=n.options.onCatch??e.options.defaultOnCatch,d=n.isRoot?n.options.notFoundComponent??e.options.notFoundRoute?.options.component:n.options.notFoundComponent,p=t.ssr===!1||t.ssr===`data-only`,m=b(e,n,t.ssr)&&(n.options.wrapInSuspense??r??(n.options.errorComponent?.preload||p)),g=(0,l.jsx)(C,{match:t});p&&(g=(0,l.jsx)(f,{fallback:r,children:g})),d&&(g=(0,l.jsx)(h,{fallback:e=>{if(e.routeId??=t.routeId,e.routeId!==t.routeId)throw e;return c.createElement(d,e)},children:g})),s&&(g=(0,l.jsx)(o,{getResetKey:()=>t,errorComponent:s,onCatch:(e,n)=>{if(i(e))throw e.routeId??=t.routeId,e;u?.(e,n)},children:g})),m&&(g=(0,l.jsx)(c.Suspense,{fallback:r,children:g}));let _=n.isRoot&&n.options.shellComponent;return _?(0,l.jsx)(a.Provider,{value:t.routeId,children:(0,l.jsxs)(_,{children:[g,null]})}):(0,l.jsxs)(a.Provider,{value:t.routeId,children:[g,null]})}var C=c.memo(function({match:e}){let t=r(),n=e.routeId,i=t.routesById[n],a=c.useMemo(()=>{let r=(i.options.remountDeps??t.options.defaultRemountDeps)?.({routeId:n,loaderDeps:e.loaderDeps,params:e._strictParams,search:e._strictSearch});return r?JSON.stringify(r):void 0},[n,e.loaderDeps,e._strictParams,e._strictSearch,i.options.remountDeps,t.options.defaultRemountDeps]),o=c.useMemo(()=>{let e=i.options.component??t.options.defaultComponent;return e?(0,l.jsx)(e,{},a):(0,l.jsx)(w,{})},[a,i.options.component,t.options.defaultComponent]);if(e.status===`pending`){if(t.ssr&&!b(t,i,e.ssr))return o;if(t._tx)throw t._tx[5];return v(t,i)}if(e.status===`notFound`)return _(t,i,e.error);if(e.status===`error`)throw e.error;return o}),w=c.memo(function(){let e=r(),t=c.useContext(a),n,i,o;{let r=e.stores.getMatchStore(t);[n,i]=s(r,e=>[!!e._notFound,e.error],{compare:y}),o=s(e.stores.ids,e=>e[e.indexOf(t)+1])}if(n)return _(e,e.routesById[t],i);if(!o)return null;let u=(0,l.jsx)(x,{routeId:o});return t===`__root__`?(0,l.jsx)(c.Suspense,{fallback:v(e),children:u}):u});export{p as i,w as n,v as r,x as t};

spinifex/runtime/roles/spinifexui/frontend/dist/assets/Match-CuyiuHbk.js

@@ -0,0 +1 @@
+import{D as e,O as t,j as n}from"./schemas-4652h2I4.js";import{d as r,r as i}from"./matchContext-aQFkVZMO.js";import{n as a}from"./x-DZTaMNYF.js";import{t as o}from"./useRouter-CN8SiXFR.js";import{t as s}from"./useSelector-IepVhQ_q.js";var c=n(t(),1),l=e(),u=()=>!0,d=()=>!1;function f({children:e,fallback:t=null}){return(0,l.jsx)(c.Fragment,{children:p()?e:t})}function p(e=!0){return c.useSyncExternalStore(m,u,e?d:u)}function m(){return()=>{}}function h(e){let t=o(),n=`not-found-${s(t.stores.location,e=>e.pathname)}-${s(t.stores.status)}`;return(0,l.jsx)(a,{getResetKey:()=>n,onCatch:(t,n)=>{if(r(t))e.onCatch?.(t,n);else throw t},errorComponent:({error:t})=>{if(r(t))return e.fallback?.(t);throw t},children:e.children})}function g(){return(0,l.jsx)(`p`,{children:`Not Found`})}function _(e,t,n){return t.options.notFoundComponent?(0,l.jsx)(t.options.notFoundComponent,{...n}):e.options.defaultNotFoundComponent?(0,l.jsx)(e.options.defaultNotFoundComponent,{...n}):(0,l.jsx)(g,{})}function v(e,t){let n=t?.options.pendingComponent??e.options.defaultPendingComponent;return n?(0,l.jsx)(n,{}):null}var y=(e,t)=>e[0]===t[0]&&e[1]===t[1],b=(e,t,n)=>!t.isRoot||t.options.shellComponent||t.options.wrapInSuspense||n===!1||n===`data-only`||!e.ssr,x=c.memo(function({routeId:e}){let t=o(),n=t.stores.getMatchStore(e);return S(t,s(n))});function S(e,t){let n=e.routesById[t.routeId],o=v(e,n),s=n.options.errorComponent??e.options.defaultErrorComponent,u=n.options.onCatch??e.options.defaultOnCatch,d=n.isRoot?n.options.notFoundComponent??e.options.notFoundRoute?.options.component:n.options.notFoundComponent,p=t.ssr===!1||t.ssr===`data-only`,m=b(e,n,t.ssr)&&(n.options.wrapInSuspense??o??(n.options.errorComponent?.preload||p)),g=(0,l.jsx)(C,{match:t});p&&(g=(0,l.jsx)(f,{fallback:o,children:g})),d&&(g=(0,l.jsx)(h,{fallback:e=>{if(e.routeId??=t.routeId,e.routeId!==t.routeId)throw e;return c.createElement(d,e)},children:g})),s&&(g=(0,l.jsx)(a,{getResetKey:()=>t,errorComponent:s,onCatch:(e,n)=>{if(r(e))throw e.routeId??=t.routeId,e;u?.(e,n)},children:g})),m&&(g=(0,l.jsx)(c.Suspense,{fallback:o,children:g}));let _=n.isRoot&&n.options.shellComponent;return _?(0,l.jsx)(i.Provider,{value:t.routeId,children:(0,l.jsxs)(_,{children:[g,null]})}):(0,l.jsxs)(i.Provider,{value:t.routeId,children:[g,null]})}var C=c.memo(function({match:e}){let t=o(),n=e.routeId,r=t.routesById[n],i=c.useMemo(()=>{let i=(r.options.remountDeps??t.options.defaultRemountDeps)?.({routeId:n,loaderDeps:e.loaderDeps,params:e._strictParams,search:e._strictSearch});return i?JSON.stringify(i):void 0},[n,e.loaderDeps,e._strictParams,e._strictSearch,r.options.remountDeps,t.options.defaultRemountDeps]),a=c.useMemo(()=>{let e=r.options.component??t.options.defaultComponent;return e?(0,l.jsx)(e,{},i):(0,l.jsx)(w,{})},[i,r.options.component,t.options.defaultComponent]);if(e.status===`pending`){if(t.ssr&&!b(t,r,e.ssr))return a;if(t._tx)throw t._tx[5];return v(t,r)}if(e.status===`notFound`)return _(t,r,e.error);if(e.status===`error`)throw e.error;return a}),w=c.memo(function(){let e=o(),t=c.useContext(i),n,r,a;{let i=e.stores.getMatchStore(t);[n,r]=s(i,e=>[!!e._notFound,e.error],{compare:y}),a=s(e.stores.ids,e=>e[e.indexOf(t)+1])}if(n)return _(e,e.routesById[t],r);if(!a)return null;let u=(0,l.jsx)(x,{routeId:a});return t===`__root__`?(0,l.jsx)(c.Suspense,{fallback:v(e),children:u}):u});export{p as i,w as n,v as r,x as t};

spinifex/runtime/roles/spinifexui/frontend/dist/assets/QueryClientProvider-6gGOAVcT.js

@@ -0,0 +1 @@
+import{D as e,O as t,j as n}from"./schemas-4652h2I4.js";var r=n(t(),1),i=e(),a=r.createContext(void 0),o=e=>{let t=r.useContext(a);if(e)return e;if(!t)throw Error(`No QueryClient set, use QueryClientProvider to set one`);return t},s=({client:e,children:t})=>(r.useEffect(()=>(e.mount(),()=>{e.unmount()}),[e]),(0,i.jsx)(a.Provider,{value:e,children:t}));export{o as n,s as t};