fix: bind volume dirty markers before recovery and refuse unmarked opens

Viperblock opened the dirty marker bucket only when registering provider subjects, after recovering surviving exports and after the legacy ebs.mount handler was live. Every export adopted at startup, and any legacy mount served in that window, opened with no marker and no takeover check, so a later crash left nothing to tell its unsealed writes from a clean seal. InitLeases now binds the marker store with the lease store, before recovery or any subject, and Run fails if it cannot. acquireVolumeLease refuses an open when no marker store is bound, the same way it refuses without a lease store.
2026-11-01 16:59:45 +00:00 · 2026-11-01 20:45:25 +10:00
61 changed files with 307 additions and 1609 deletions
@@ -6,7 +6,6 @@
 //
 //	eks-gateway-fetch -resource addons   # GET /clusters/{cluster}/internal-addons/{accountId}
 //	eks-gateway-fetch -resource recovery -instance-id i-… # GET /clusters/{cluster}/internal-recovery/{accountId}/{instanceId}
-//	eks-gateway-fetch -resource workers  # GET /clusters/{cluster}/internal-workers/{accountId}
 //
 // Flags default to environment variables seeded by cloud-init:
 // EKS_GATEWAY_URL, EKS_GATEWAY_CA, EKS_ACCESS_KEY, EKS_SECRET_KEY, EKS_REGION,
@@ -34,7 +33,7 @@ func main() {
 	flag.StringVar(&cfg.AccountID, "account-id", os.Getenv("EKS_ACCOUNT_ID"), "Cluster account ID")
 	flag.StringVar(&cfg.ClusterName, "cluster", os.Getenv("EKS_CLUSTER_NAME"), "Cluster name")
 	flag.StringVar(&cfg.InstanceID, "instance-id", os.Getenv("EKS_INSTANCE_ID"), "This member's EC2 instance ID (recovery resource)")
-	flag.StringVar(&cfg.Resource, "resource", "addons", "Resource to fetch: addons | recovery | workers")
+	flag.StringVar(&cfg.Resource, "resource", "addons", "Resource to fetch: addons | recovery")
 	flag.Parse()

 	if err := gatewayfetch.Run(context.Background(), cfg); err != nil {
@@ -203,14 +203,12 @@ Viperblock responds with an NBD URI that QEMU uses to access the block device.

 A direct-boot microVM keeps the launcher's saved `vm.Config` as a clean base, and `startQEMU` adds data volumes only to a launch-only copy of it (`runtime/compute/vm/lifecycle.go`).
 At each start the daemon takes one snapshot of the instance's already-mounted data requests, probes each NBD or host block export from that snapshot, and builds the command from the same snapshot, so the launched endpoint is the one that was checked and is never written back into the saved base.
-The microVM boots with `pcie=on`, so its NICs are `virtio-net-pci` devices with MSI-X interrupts instead of `virtio-net-device` MMIO devices.
-Each data volume becomes a `virtio-blk-pci` device on the default PCIe bus with the same id, node name, I/O thread, serial and `werror=stop,rerror=stop` policy as the q35 device, and takes no PCIe hot-plug port.
-A microVM started without `pcie=on` keeps the `virtio-*-device` MMIO transports.
+Each data volume becomes a `virtio-blk-device` on the MMIO bus with the same id, node name, I/O thread, serial and `werror=stop,rerror=stop` policy as the PCI device, and takes no PCIe hot-plug port.

 Boot and EFI requests are rejected on this path, and a direct-boot instance with no data requests launches its base unchanged.
 Disk-boot (q35) instances keep the existing PCI hot-plug path.

-Hot-attaching a volume to a running direct-boot microVM is not supported, because the microVM has no PCIe hot-plug root port to plug into.
+Hot-attaching a volume to a running direct-boot microVM is not supported, because the MMIO bus has no PCIe root port to plug into.
 `AttachVolume` refuses it before mounting the volume or changing any instance or volume state, and the EC2 API returns `UnsupportedOperation`.

 This covers only requests that are already mounted when the instance starts; nothing yet supplies data volumes to a system instance, and the path is verified with fake-QEMU unit tests, not with a real EBS volume in a guest.
@@ -369,7 +369,7 @@ while the branch is in flight; they do not describe the umbrella repository's
 | `spinifex#1355` | `spinifex/gateway/ec2.go` (`EC2_Request` in production, the legacy service switch, its `supportedServices` and JSON tables) and the `Quota` field of `GatewayConfig` | `spinifex/domains/ec2/awsapi` (`NewRegistration`, `OperationInventory`, `ServiceName`), composed by `spinifex/runtime/roles/awsgw` (`ec2.go`: `newEC2Registration`, `advertisedEndpoint`; `ec2quota`) | Slice 9b of the EC2 move: EC2 registers through the ADR-0002 seam as service `ec2` and the legacy gateway switch is empty. The EC2 query-protocol error envelope is kept through `ErrorEnvelopeXML`, and the policy denial match is now `*dispatch.PolicyDenialError`. The characterisation tests committed first also pin the Authorize service name and action, the trace span and audit action, the throttle key (account plus action) and the cluster-unavailable gate, which runs before policy and action lookup for gateway-local and NATS-backed actions alike. They and the 9a tests pass unchanged apart from setup: STS and EC2 test setups now register the services they sign as, and the two tests that claimed a legacy service name are removed because no legacy name remains. `EC2_Request` survives only as a test shim. Allowlist changes are two deletions (the `gateway` to `domains/ec2/awsapi` and `domains/ec2/awsapi/instance` rows) and one addition, the per-service model-coverage row `cmd/aws-model-coverage` to `domains/ec2/awsapi`. `startInstanceStatus` stays in `awsgw.go` because ADR-0007:S6 records the raw instance-state uses by file. No change to `dispatch.Actor` or `dispatch.Invocation`. Live on env14: DescribeRegions, DescribeAvailabilityZones and DescribeAccountAttributes, an idempotent CreateVolume replay and its parameter mismatch, the DryRun and InvalidVolume.NotFound errors, a no-policy user refused with UnauthorizedOperation (including on the gateway-local DescribeRegions), an assumed-role session allowed and refused per its policy, and one RunInstances to running then terminated. The gateway request logs carry `service=ec2` and the per-action label. Not exercised live: a throttled request (the default limits are not reachable from one client; the throttle key is pinned by the characterisation tests) and an agent-signed EC2 call. |
 | `spinifex#1363` | `spinifex/gateway` context keys (`ctxService`, `ctxRegion`, `ctxAction`, the principal and access key id keys, and the unread `ctxQueryArgs`), the duplicate `callerPrincipal` and `principalContext` types, `gateway/caller_arn.go` (`buildCallerARN`), and the direct `policyauth`, `accountadmission` and NATS-connection reads in the auth pipeline | `spinifex/ingress/aws/reqctx` (the request record), `credverify.CallerARN`, and the gateway-consumed capabilities `PolicyEvaluator`, `AccountAdmission` and `Readiness`, composed by `spinifex/runtime/roles/awsgw` (`clusterready`) | Slice 11a of the gateway removal: the request-context record and the ADR-0002:S8 capabilities, with no behaviour change. The record is a copy-on-write value under one context key; the principal is sealed once and a second `WithPrincipal` keeps the first caller. The layering rule `S7_RequestRecordPrivateToIngress` lets only `gateway` and `ingress/aws/*` import `reqctx`, with a fault-injection test, and slice 11c moves the package to `ingress/aws/internal/reqctx` and drops the rule. The caller ARN is built once in `credverify` (root cases kept) and filled on `Principal.CallerARN`. The policy evaluator, account admission (`credential.ErrAccountNotActive`, with `accountadmission.ErrNotActive` an alias) and cluster readiness (`Available() bool`) are interfaces the gateway consumes and fails closed on when nil. The step 1 characterisation tests (throttle against both readiness gates and the policy gate for ECS and IAM, `/admin` skipping `GetService` and the second gate, the nil-connection difference between the two gates, five error writers each minting a request id that differs from the audit record, and the credential-scope region reaching only the span) pass unchanged apart from setup. No allowlist rows change, and there is no change to `dispatch.Actor` or `dispatch.Invocation`. Debt for slice 11c: the test-only `defaultCapabilities` seam (a package variable set by a test `init` so about 60 gateway test files can build a config from `IAMService` alone), the nil-connection case of the early readiness gate, the remaining `NATSConn` readers (`DiscoverActiveNodes` and the field that `awsgw` and the integration harness set), and the `IAMService`, `STSService`, `Config`, `RootCAs` and `InternalSuffix` fields of `GatewayConfig`, which no production code reads. Live on env14: DescribeRegions, DescribeAvailabilityZones and DescribeAccountAttributes, an idempotent CreateVolume replay and its parameter mismatch, the DryRun and InvalidVolume.NotFound errors, a bad access key refused with InvalidClientTokenId on a Query service and InvalidClientTokenIdException on a JSON service, a no-policy user refused with UnauthorizedOperation, and an assumed-role session allowed and refused per its policy. Not exercised live: a suspended-account refusal, a throttled request and the NATS-down 503, which the step 1 tests pin. |
 | `spinifex#1378` | `spinifex/gateway` `oidc_discovery.go` and `registry.go` (the `/oidc/eks` routes and the `/v2` mount), the `DiscoverActiveNodes` method and its cache, and the `GatewayConfig` fields `Config`, `RootCAs`, `InternalSuffix`, `Version`, `Commit`, `IAMService`, `STSService`, `NATSConn` and `ExpectedNodes` | `spinifex/runtime/roles/awsgw` (`PublicRoutes`, `NodeDiscovery.ActiveNodes`, `composeGateway`); the gateway keeps one `PublicRoutes` mount hook and the `Admin` registration name supplied by the role |
-| `spinifex#1384` | `spinifex/gateway` (`GatewayConfig`, the SigV4 and policy middleware, `Request`, the admin dispatch, `AWSOperationInventory`) | `spinifex/ingress/aws/pipeline` (`Config`) and `spinifex/ingress/aws/dispatch` (`AWSOperationInventory`); the gateway layering rows are retired and R2 binds the pipeline |
+| `spinifex#TBD` | `spinifex/gateway` (`GatewayConfig`, the SigV4 and policy middleware, `Request`, the admin dispatch, `AWSOperationInventory`) | `spinifex/ingress/aws/pipeline` (`Config`) and `spinifex/ingress/aws/dispatch` (`AWSOperationInventory`); the gateway layering rows are retired and R2 binds the pipeline |
 | `spinifex#1334` | `spinifex/gateway/oidc_discovery.go` (`eksAccountKV`, the discovery-document and JWKS read and construction) | `spinifex/domains/eks/cluster` (`OIDCPublisher`, `OIDCDiscoveryDocument`), composed by `spinifex/runtime/roles/awsgw` | The EKS domain now reads its own account bucket and builds the per-cluster discovery document and JWKS responses (ADR-0005:S4), and the gateway only maps the unauthenticated `/oidc/eks/...` routes onto an `OIDCPublisher` interface it is handed. Responses are byte-identical, pinned by `oidc_discovery_current_behaviour_test.go`, whose assertions are unchanged. The gateway no longer imports `domains/eks/cluster` in production code. `STS` keeps its own `ClusterJWKSResolver` (`OIDCJWKSReader`), untouched. Recorded debt, not repaired here: a stored JWKS is served verbatim with 200 even when it is not valid JSON. |
 The EC2 contract row is intentionally different from the directory moves: it creates
 a compatibility boundary. `ec2.cmd.*` retains its deployed one-token NATS
@@ -688,7 +688,7 @@ series above; "needs decision" is open.
 | `handlers/rds` | — | Done. The remaining RDS instance resource moved to `domains/rds/instance` (package `instance`) in `spinifex#1251` with no API, NATS subject, storage or behaviour change, fulfilling ADR-0001's organise-by-resource condition alongside the already-extracted subnet group, parameter group and engine catalogue. The ADR-0004 blocked-dependency imports (quota admission, the duplicated expected-node fan-out, the IAM ensurer and the Bluebottle master key) remain dependency-rule debt, with their allowlist entries now sourced from `domains/rds/instance`. |
 | `handlers/elbv2` | — | Done. The implementation moved to `domains/elbv2` (package `elbv2`) with no API, NATS subject, storage or behaviour change. Q-30's data-plane boundary remains deferred debt; the amended ADR-0001 row makes it not a relocation prerequisite. Its cross-domain, IAM and `agents/elbv2` imports remain dependency-rule debt, with their allowlist entries now sourced from `domains/elbv2`. |
 | `handlers/iam/mock` | — | Done. Moved with the implementation to `domains/iam/mock`. |
-| `gateway` (root) | Generic HTTP route assembly, SigV4 authentication, the policy gate, request-audit middleware and request-context keys. | Done and deleted. The request pipeline moved into `ingress/aws/pipeline` (as `pipeline.Config`) and the operation inventory into `ingress/aws/dispatch` in `spinifex#1384`; the `/oidc`, `/v2` and node-discovery code had already moved to `runtime/roles/awsgw` in `spinifex#1378`. The layering test binds the pipeline with R2 and `gateway` is a retired legacy root. |
+| `gateway` (root) | Generic HTTP route assembly, SigV4 authentication, the policy gate, request-audit middleware and request-context keys. | Done and deleted. The request pipeline moved into `ingress/aws/pipeline` (as `pipeline.Config`) and the operation inventory into `ingress/aws/dispatch` in `spinifex#TBD`; the `/oidc`, `/v2` and node-discovery code had already moved to `runtime/roles/awsgw` in `spinifex#1378`. The layering test binds the pipeline with R2 and `gateway` is a retired legacy root. |
 | `gateway/bedrock` | — | Done. The whole package moved into `domains/ochre` in `spinifex#1266` with its package-private core and tests intact and no behaviour change. Separating the action adapters into `domains/ochre/awsapi` remains deferred, recorded as `domains/ochre` (action adapters) debt above. The action adapters then moved to `domains/ochre/awsapi` behind one ADR-0002 registration in `spinifex#1351`. |
 | `gateway/{iam,sts,eks,rds}` | — | Done. The action adapters moved to `domains/{iam,sts,eks,rds}/awsapi` in `spinifex#1265` with no API, NATS subject, storage or behaviour change. ADR-0001 does not make ADR-0002 registration a prerequisite for relocating a domain's adapters, as the EC2 adapter moves already showed; the root `gateway` dispatch that still imports them is recorded `gateway` (service registration) debt. |
 | `gateway/{ecs,elbv2}` | — | Done. The action adapters moved to `domains/{ecs,elbv2}/awsapi` in `spinifex#1273` with no API, NATS subject, storage or behaviour change; the root `gateway` dispatch that still imports them is recorded `gateway` (service registration) debt. |
@@ -164,7 +164,7 @@ On Oracle Linux the installer makes these changes, which Debian hosts do not nee
 - **Repositories.** It enables `ol10_UEKR8` and `ol10_kvm_utils`, and adds `qemu-kvm*` and `qemu-img*` to the `excludepkgs` of `ol10_appstream`, keeping any excludes already set, so a later AppStream build can never replace the KVM Utilities QEMU.
 - **NFV SIG.** It writes `/etc/yum.repos.d/spinifex-nfv-sig.repo` with an `includepkgs=` list of the OVS and OVN packages, and imports the SIG key to `/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-NFV` only when its fingerprint is `35154228174901BEFA8E69A621465E289D2A76A7`.
 - **OVS and OVN user.** Both run as root, as on Debian, through `OVS_USER_ID` and `OVN_USER_ID` in `/etc/sysconfig/openvswitch` and `/etc/sysconfig/ovn`.
- **DNS.** Where systemd-resolved is installed but not running, it saves `/etc/resolv.conf` to `/var/lib/spinifex-reset/resolv.conf.orig`, starts resolved behind NetworkManager, tells NetworkManager in `/etc/NetworkManager/conf.d/90-spinifex-dns.conf` to leave `/etc/resolv.conf` alone, and links `/etc/resolv.conf` to the resolved stub. It then checks that names still resolve and stops, printing the commands that put DNS back, if they do not.
+- **DNS.** Where systemd-resolved is installed but not running, it saves `/etc/resolv.conf` to `/var/lib/spinifex-reset/resolv.conf.orig`, starts resolved behind NetworkManager and links `/etc/resolv.conf` to the resolved stub. It then checks that names still resolve and stops, printing the commands that put DNS back, if they do not.
 - **chrony.** It appends `confdir /etc/chrony.d` to `/etc/chrony.conf` once, and writes its step policy to `/etc/chrony.d/spinifex.conf`.
 - **`spx` under `sudo`.** Oracle Linux's `sudo` path leaves out `/usr/local/bin`, so the installer links `/usr/bin/spx` to `/usr/local/bin/spx`.

@@ -123,10 +123,6 @@ Internal and internet-facing ALB and NLB, HTTPS listeners, listener rules, `Modi
 Unhealthy-target routing: an NLB with an HTTP health check on a TCP target group stops sending traffic to a target that fails it while still accepting TCP, sends traffic again once it recovers, and fails open when every target is unhealthy.
 An ALB whose targets all fail their health check keeps serving from them instead of answering 503.

-`TestInternalALBPrivateSubnet` launches an internal ALB in a private subnet of a non-default VPC with no internet gateway and no public route.
-The load balancer must reach active with a healthy target, which needs the owner stamp on its ENI to resolve, and a client inside the VPC must be served by it on every request.
-The client reports on its serial console because the runner has no route into the VPC.
-
 ### `eks`, `ecs`, `ecr` — container services

 `eks`: cluster create/delete, access entries, addon delivery, `GetToken`, kubeconfig artifacts, the EBS CSI volume path, an IRSA web-identity exchange under the stock AWS trust policy (`StringEquals` on the issuer host-path `:sub`/`:aud`, a mismatched `:sub` denied with `AccessDenied`, a `StringLike` `:sub` granted), and an IRSA pod wired with a projected SA token. Worker IMDS is covered too: a managed nodegroup worker reports IMDSv2 required with hop limit 2, a pod with no IRSA resolves to the node role through IMDS, and a nodegroup built from a launch template takes the template's metadata options. Control-plane isolation is covered as well: no control-plane host registers as a Node, a DaemonSet tolerating every taint runs on workers only, metrics-server is Ready, and logs work for every kube-system pod. `ecs`: clusters, container instances, the three network modes (`awsvpc`, `bridge`, `host`), task-role credentials fetched from the container credentials endpoint, a service behind an ELB, and ENI release on terminate. `ecr`: control-plane CRUD plus a real OCI data plane — docker login/build/push/pull round trips, multi-layer images, and raw blob HTTP.
@@ -1,108 +0,0 @@
-package testutil
-
-import (
-	"context"
-	"sync"
-
-	"github.com/nats-io/nats.go/jetstream"
-)
-
-// StaleReadKV wraps a bucket so that the first Get of a key after a write to it
-// returns the entry as it stood before that write, as a lagging replica does
-// when it answers a direct get. Later Gets see the current entry.
-type StaleReadKV struct {
-	jetstream.KeyValue
-
-	mu    sync.Mutex
-	stale map[string]jetstream.KeyValueEntry
-}
-
-var _ jetstream.KeyValue = (*StaleReadKV)(nil)
-
-// NewStaleReadKV wraps kv.
-func NewStaleReadKV(kv jetstream.KeyValue) *StaleReadKV {
-	return &StaleReadKV{KeyValue: kv, stale: map[string]jetstream.KeyValueEntry{}}
-}
-
-// remember captures the entry about to be overwritten.
-func (k *StaleReadKV) remember(ctx context.Context, key string) jetstream.KeyValueEntry {
-	prev, err := k.KeyValue.Get(ctx, key)
-	if err != nil {
-		return nil
-	}
-	return prev
-}
-
-func (k *StaleReadKV) arm(key string, prev jetstream.KeyValueEntry) {
-	if prev == nil {
-		return
-	}
-	k.mu.Lock()
-	k.stale[key] = prev
-	k.mu.Unlock()
-}
-
-func (k *StaleReadKV) Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error) {
-	k.mu.Lock()
-	prev, ok := k.stale[key]
-	delete(k.stale, key)
-	k.mu.Unlock()
-	if ok {
-		return prev, nil
-	}
-	return k.KeyValue.Get(ctx, key)
-}
-
-func (k *StaleReadKV) Update(ctx context.Context, key string, value []byte, last uint64) (uint64, error) {
-	prev := k.remember(ctx, key)
-	rev, err := k.KeyValue.Update(ctx, key, value, last)
-	if err == nil {
-		k.arm(key, prev)
-	}
-	return rev, err
-}
-
-func (k *StaleReadKV) Put(ctx context.Context, key string, value []byte) (uint64, error) {
-	prev := k.remember(ctx, key)
-	rev, err := k.KeyValue.Put(ctx, key, value)
-	if err == nil {
-		k.arm(key, prev)
-	}
-	return rev, err
-}
-
-// RejectingKV wraps a bucket so that writes for which reject returns an error
-// fail with it, and every other operation passes through.
-type RejectingKV struct {
-	jetstream.KeyValue
-
-	reject func(key string, value []byte) error
-}
-
-var _ jetstream.KeyValue = (*RejectingKV)(nil)
-
-// NewRejectingKV wraps kv. reject runs before each Put, Create and Update.
-func NewRejectingKV(kv jetstream.KeyValue, reject func(key string, value []byte) error) *RejectingKV {
-	return &RejectingKV{KeyValue: kv, reject: reject}
-}
-
-func (k *RejectingKV) Put(ctx context.Context, key string, value []byte) (uint64, error) {
-	if err := k.reject(key, value); err != nil {
-		return 0, err
-	}
-	return k.KeyValue.Put(ctx, key, value)
-}
-
-func (k *RejectingKV) Create(ctx context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error) {
-	if err := k.reject(key, value); err != nil {
-		return 0, err
-	}
-	return k.KeyValue.Create(ctx, key, value, opts...)
-}
-
-func (k *RejectingKV) Update(ctx context.Context, key string, value []byte, last uint64) (uint64, error) {
-	if err := k.reject(key, value); err != nil {
-		return 0, err
-	}
-	return k.KeyValue.Update(ctx, key, value, last)
-}
@@ -6,7 +6,7 @@ Node installation, dev-environment lifecycle, guest image builds, and verificati

 | Script | What it does |
 |---|---|
-| `setup.sh` | The production binary installer, served from `install.mulgadc.com`. Installs deps, users, sudoers, the host firewall policy, files, systemd units, logrotate and udev rules, and loads `openvswitch`, `geneve`, `vhost_net` and `tun`, listing them in `/etc/modules-load.d/spinifex.conf` for later boots. It masks firewalld wherever it is installed, since the Spinifex nftables policy is the only supported node firewall. It writes a needrestart drop-in so an apt upgrade never restarts the Spinifex, OVS or OVN units; restarting them after an upgrade is left to the operator. An unset `--firewall` keeps the mode recorded in `/etc/spinifex/firewall/mode`, and otherwise arms the policy when firewalld was active. Where systemd-resolved is installed but not running, resolvconf is absent and NetworkManager runs, it hands host DNS to resolved: it saves `/etc/resolv.conf` once to `/var/lib/spinifex-reset/resolv.conf.orig`, writes `/etc/NetworkManager/conf.d/90-spinifex-dns.conf` (`dns=systemd-resolved`, `rc-manager=unmanaged`) so NetworkManager never rewrites `/etc/resolv.conf`, links `/etc/resolv.conf` to the resolved stub, and stops unless the link holds, resolved has upstream servers and the package repository host still resolves. A rerun after that handover adds a missing drop-in, reloads NetworkManager and relinks the stub. It links `/usr/bin/spx` to `/usr/local/bin/spx` where sudo's PATH leaves out `/usr/local/bin`, and takes the nbdkit plugin directory from `nbdkit --dump-config`, stopping when nbdkit cannot say. `SETUP_STAGES` runs a subset; `ISO_BUILD=1` runs it inside the ISO builder's chroot. |
+| `setup.sh` | The production binary installer, served from `install.mulgadc.com`. Installs deps, users, sudoers, the host firewall policy, files, systemd units, logrotate and udev rules, and loads `openvswitch`, `geneve`, `vhost_net` and `tun`, listing them in `/etc/modules-load.d/spinifex.conf` for later boots. It masks firewalld wherever it is installed, since the Spinifex nftables policy is the only supported node firewall. It writes a needrestart drop-in so an apt upgrade never restarts the Spinifex, OVS or OVN units; restarting them after an upgrade is left to the operator. An unset `--firewall` keeps the mode recorded in `/etc/spinifex/firewall/mode`, and otherwise arms the policy when firewalld was active. Where systemd-resolved is installed but not running, resolvconf is absent and NetworkManager runs, it hands host DNS to resolved: it saves `/etc/resolv.conf` once to `/var/lib/spinifex-reset/resolv.conf.orig`, links `/etc/resolv.conf` to the resolved stub, and stops unless resolved has upstream servers and the package repository host still resolves. It links `/usr/bin/spx` to `/usr/local/bin/spx` where sudo's PATH leaves out `/usr/local/bin`, and takes the nbdkit plugin directory from `nbdkit --dump-config`, stopping when nbdkit cannot say. `SETUP_STAGES` runs a subset; `ISO_BUILD=1` runs it inside the ISO builder's chroot. |
 | `setup-ovn.sh` | Bootstraps a compute node for OVN VPC networking: installs OVN/OVS, creates `br-int`, configures the WAN bridge (bridged, DHCP, or routed-NAT uplink), sets the chassis identity, and applies overlay sysctls. It runs OVS and OVN as root, setting `OVS_USER_ID` and `OVN_USER_ID` where `/etc/sysconfig` holds them, and hooks the socket-permission helper onto `ovs-vswitchd`, `ovn-controller`, `openvswitch-ipsec` and, on EL, `ovn-northd`. `--management` also starts the OVN central services, writes their `ovn-ctl` flags to `/etc/default/ovn-central` (Debian) or `/etc/sysconfig/ovn-northd` (EL), and records the role in `/etc/spinifex/ovn-role`. With clustered databases it returns only once this node's NB and SB members are back in their RAFT clusters under a known leader, and fails after `OVN_RAFT_REJOIN_WAIT` seconds (default 60) otherwise, so reruns one node at a time never take two members of a cluster down. A later run without `--management` refuses while that file, or the `OVN_CTL_OPTS` line older nodes carry in `/etc/default/ovn-central`, records the role. It adds `maxsize 100M` to the OVS and OVN logrotate files, plus `rotate 5` where the file sets no count, and rotates as root, matching the daemons. On a host whose IPsec stack is libreswan it enables, stops and masks `openvswitch-ipsec.service` and `ipsec.service` for the daemon to unmask when IPsec is on, unless `openvswitch-ipsec.service` is already running; elsewhere it enables and starts `openvswitch-ipsec.service`, unless the daemon has masked it to keep IPsec off. The links it creates persist through systemd-networkd where that is enabled, and otherwise through `/etc/spinifex/hostnet.json`, which `spinifex-hostnet.service` (`spx admin hostnet apply`) replays at boot and on every OVS start, starting `ovn-controller` again when an OVS stop took it down; the first run records the choice in `/etc/spinifex/hostnet-backend`. Where NetworkManager runs, it leaves the links Spinifex owns unmanaged. |
 | `install-node.sh` | Forms a multi-node cluster across hosts that are already running Spinifex: rebuilds the OVN database in clustered form, runs `spx admin init` on the first host and `spx admin join` on the rest, then verifies. Before touching any host it refuses hosts of different OS families (`distro_os_family`) or `spx` versions, and `--ipsec on` when any host uses libreswan. The scripted form of [install-multi-node](../docs/install/install-multi-node/README.md). Destroys the joining nodes' logical network state and crypto — see its `--help`. |
 | `dev-install.sh` | Full local dev setup through the production installer: builds from source, packs a tarball, runs `setup.sh`, initialises the cluster, and starts services under systemd. |
@@ -12,12 +12,11 @@ set -eu
 # once.
 #
 #   server      → eks-token-webhook, k3s (server), k3s-first-boot (bootstrap
-#                 publish), mulga-eks-state-report, mulga-eks-node-gc
+#                 publish), mulga-eks-state-report
 #   server-join → eks-token-webhook, k3s (server, joins the first server's etcd
-#                 quorum), mulga-eks-state-report, mulga-eks-node-gc. NO
-#                 k3s-first-boot: the first server already publishes the
-#                 cluster-identical bootstrap artifacts; a join re-publish only
-#                 races the bootstrap bus.
+#                 quorum), mulga-eks-state-report. NO k3s-first-boot: the first
+#                 server already publishes the cluster-identical bootstrap
+#                 artifacts; a join re-publish only races the bootstrap bus.
 #   agent       → k3s-agent
 #
 # Paths are overridable via env so the selection logic is unit-testable (bats)
@@ -141,7 +140,6 @@ case "${ROLE}" in
        # writer owns. HA multi-server addon delivery is tracked separately
        # Server-join nodes do not run this setup.
        svc_enable mulga-eks-addon-sync
-        svc_enable mulga-eks-node-gc
        svc_start eks-token-webhook
        svc_start mulga-eks-k3s-recovery
        svc_start k3s
@@ -149,7 +147,6 @@ case "${ROLE}" in
        svc_start k3s-first-boot
        svc_start mulga-eks-state-report
        svc_start mulga-eks-addon-sync
-        svc_start mulga-eks-node-gc
        ;;
    server-join)
        log "configuring server-join role"
@@ -161,13 +158,11 @@ case "${ROLE}" in
        svc_enable k3s
        svc_enable konnectivity-server
        svc_enable mulga-eks-state-report
-        svc_enable mulga-eks-node-gc
        svc_start eks-token-webhook
        svc_start mulga-eks-k3s-recovery
        svc_start k3s
        svc_start konnectivity-server
        svc_start mulga-eks-state-report
-        svc_start mulga-eks-node-gc
        ;;
    agent)
        log "configuring agent role"
@@ -141,8 +141,6 @@ for INIT in openrc systemd; do
            END { print (rn > 0 && kn > 0 && rn < kn) ? "yes" : "no" }
        ' "${CALLS}"
    )"
-    check "server(${INIT}): enables node-gc" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-node-gc)")"
-    check "server(${INIT}): starts node-gc" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-node-gc)")"
    check "server(${INIT}): no k3s-agent" "no" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
    check "server(${INIT}): self-disables" "yes" "$(present "${CALLS}" "$(disable_cmd "${INIT}" eks-node-role)")"

@@ -155,8 +153,6 @@ for INIT in openrc systemd; do
    check "server-join(${INIT}): enables webhook" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" eks-token-webhook)")"
    check "server-join(${INIT}): enables k3s" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s)")"
    check "server-join(${INIT}): enables state-report" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-state-report)")"
-    check "server-join(${INIT}): enables node-gc" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-node-gc)")"
-    check "server-join(${INIT}): starts node-gc" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-node-gc)")"
    check "server-join(${INIT}): enables konnectivity-server" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" konnectivity-server)")"
    check "server-join(${INIT}): enables k3s-recovery" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-k3s-recovery)")"
    check "server-join(${INIT}): starts k3s-recovery" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-k3s-recovery)")"
@@ -174,7 +170,7 @@ for INIT in openrc systemd; do
    check "agent(${INIT}): exit 0" 0 "${rc}"
    check "agent(${INIT}): role file" "agent" "$(cat "${ROLE_FILE}")"
    check "agent(${INIT}): enables k3s-agent" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
-    check "agent(${INIT}): no server services" "yes" "$(absent_all "${CALLS}" "eks-token-webhook" "k3s-first-boot" "mulga-eks-node-gc" "$(enable_cmd "${INIT}" k3s)")"
+    check "agent(${INIT}): no server services" "yes" "$(absent_all "${CALLS}" "eks-token-webhook" "k3s-first-boot" "$(enable_cmd "${INIT}" k3s)")"

    # --- agent (inferred from agent.env, no explicit role) ---
    run_case "${INIT}" agent_inferred
@@ -42,8 +42,6 @@ scripts/images/eks-node/mulga-eks-state-report.initd:/etc/init.d/mulga-eks-state
 scripts/images/eks-node/mulga-eks-state-report.sh:/usr/local/sbin/mulga-eks-state-report \
 scripts/images/eks-node/mulga-eks-addon-sync.initd:/etc/init.d/mulga-eks-addon-sync \
 scripts/images/eks-node/mulga-eks-addon-sync.sh:/usr/local/sbin/mulga-eks-addon-sync \
-scripts/images/eks-node/mulga-eks-node-gc.initd:/etc/init.d/mulga-eks-node-gc \
-scripts/images/eks-node/mulga-eks-node-gc.sh:/usr/local/sbin/mulga-eks-node-gc \
 scripts/images/eks-node/konnectivity-server.initd:/etc/init.d/konnectivity-server \
 scripts/images/eks-node/konnectivity-server-prestart.sh:/usr/local/sbin/konnectivity-server-prestart \
 scripts/images/eks-node/konnectivity/konnectivity-agent.yaml:/usr/share/spinifex-eks/konnectivity/konnectivity-agent.yaml \
@@ -1,27 +0,0 @@
-#!/sbin/openrc-run
-# mulga-eks-node-gc — server-role service that deletes the Node of every
-# nodegroup worker the nodegroup has terminated, every NODE_GC_INTERVAL
-# seconds. Enabled on every server role by eks-node-role.sh.
-
-description="Mulga EKS terminated-worker Node cleanup"
-command="/usr/local/sbin/mulga-eks-node-gc"
-command_background="yes"
-pidfile="/run/mulga-eks-node-gc.pid"
-output_log="/var/log/mulga-eks-node-gc.log"
-error_log="/var/log/mulga-eks-node-gc.log"
-
-# A Node goes NotReady about 40s after its worker stops, so 30s deletes it
-# within a couple of minutes of the terminate.
-export NODE_GC_INTERVAL="${NODE_GC_INTERVAL:-30}"
-
-depend() {
-    need net
-    after k3s
-}
-
-start_pre() {
-    if [ ! -f /etc/spinifex-eks/first-boot.env ]; then
-        eerror "/etc/spinifex-eks/first-boot.env not present — cloud-init did not seed gateway creds"
-        return 1
-    fi
-}
@@ -1,89 +0,0 @@
-#!/bin/sh
-set -eu
-
-# mulga-eks-node-gc — deletes the Node of a nodegroup worker whose instance the
-# nodegroup has terminated (DeleteNodegroup, scale-down, reclaim), as the AWS
-# cloud controller deletes a Node whose instance no longer exists. K3s runs no
-# AWS cloud provider: its own reports every instance as existing, so without
-# this a terminated worker stays a NotReady Node forever, its pods stay bound to
-# it, and once most Nodes are NotReady the node lifecycle controller stops
-# evicting at all.
-#
-# Pull (host→VM): GET /clusters/{cluster}/internal-workers/{accountId}, the
-# instance ID of every worker the cluster's nodegroups still own.
-#
-# A Node is deleted only when all of these hold, so a live worker is never
-# touched: it carries the eks.amazonaws.com/nodegroup label, its providerID is
-# aws:///<az>/<instance-id>, it is not Ready, and its instance is not in the
-# fetched set. A failed fetch deletes nothing.
-#
-# Runs on every server; deletes are idempotent. Loops every NODE_GC_INTERVAL
-# seconds in service mode, one-shot otherwise.
-
-ENVFILE=${ENVFILE:-/etc/spinifex-eks/first-boot.env}
-[ -f "${ENVFILE}" ] || { logger -t mulga-eks-node-gc "${ENVFILE} missing — exiting"; exit 0; }
-# set -a so the sourced creds are exported to the eks-gateway-fetch child.
-set -a
-# shellcheck disable=SC1090
-. "${ENVFILE}"
-set +a
-
-: "${EKS_GATEWAY_URL:?}"
-: "${EKS_ACCOUNT_ID:?}"
-: "${EKS_CLUSTER_NAME:?}"
-
-KUBECONFIG=${KUBECONFIG:-/etc/rancher/k3s/k3s.yaml}
-export KUBECONFIG
-
-# One "name|providerID|nodegroup|Ready" row per Node.
-NODE_ROWS='jsonpath={range .items[*]}{.metadata.name}{"|"}{.spec.providerID}{"|"}{.metadata.labels.eks\.amazonaws\.com/nodegroup}{"|"}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}'
-
-log() {
-    logger -t mulga-eks-node-gc "$*"
-    echo "[mulga-eks-node-gc] $*" > "${CONSOLE:-/dev/console}" 2>/dev/null || true
-}
-
-# stale_nodes LIVE_FILE — reads Node rows on stdin and prints the name of each
-# Node to delete: a nodegroup worker, not Ready, whose instance is not listed
-# in LIVE_FILE.
-stale_nodes() {
-    awk -F'|' -v live="$1" '
-        BEGIN { while ((getline id < live) > 0) keep[id] = 1 }
-        $3 == "" || $4 == "True" { next }
-        $2 !~ /^aws:\/\/\/[^\/]+\/i-[0-9a-f]+$/ { next }
-        { n = split($2, p, "/"); if (!(p[n] in keep)) print $1 }
-    '
-}
-
-sync_once() {
-    # Nodes are listed before the workers are fetched. A worker is recorded on
-    # its nodegroup before it boots, so every Node listed here is already in a
-    # set fetched after.
-    if ! rows=$(kubectl get nodes -o "${NODE_ROWS}" 2>/dev/null); then
-        log "kubectl get nodes failed; skipping this pass"
-        return 0
-    fi
-    live=$(mktemp)
-    if ! eks-gateway-fetch -resource workers > "${live}" 2>/dev/null; then
-        log "fetch of the cluster's nodegroup workers failed; deleting nothing"
-        rm -f "${live}"
-        return 0
-    fi
-    printf '%s\n' "${rows}" | stale_nodes "${live}" | while read -r node; do
-        if kubectl delete node "${node}" --ignore-not-found --wait=false >/dev/null 2>&1; then
-            log "deleted Node ${node}: its instance is no longer a nodegroup worker"
-        else
-            log "delete of Node ${node} failed; retrying next pass"
-        fi
-    done
-    rm -f "${live}"
-}
-
-if [ -n "${NODE_GC_INTERVAL:-}" ] && [ "${NODE_GC_INTERVAL}" -gt 0 ] 2>/dev/null; then
-    while true; do
-        sync_once
-        sleep "${NODE_GC_INTERVAL}"
-    done
-else
-    sync_once
-fi
@@ -1,107 +0,0 @@
-#!/bin/sh
-# Self-contained POSIX test for mulga-eks-node-gc.sh. kubectl, eks-gateway-fetch
-# and logger are stubbed on PATH: kubectl serves canned Node rows and records
-# every delete, and eks-gateway-fetch serves a canned worker set or fails.
-#
-# Run: sh scripts/images/eks-node/mulga-eks-node-gc_test.sh
-set -eu
-
-SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
-AGENT="${SCRIPT_DIR}/mulga-eks-node-gc.sh"
-WORK=$(mktemp -d)
-trap 'rm -rf "${WORK}"' EXIT
-
-STUBBIN="${WORK}/bin"
-mkdir -p "${STUBBIN}"
-
-cat > "${STUBBIN}/kubectl" <<'EOF'
-#!/bin/sh
-case "$1 $2" in
-    "get nodes")
-        [ "${NODES_RC:-0}" -eq 0 ] || exit "${NODES_RC}"
-        printf '%s\n' "${NODES_BODY:-}" ;;
-    "delete node") echo "$3" >> "${DELETED}" ;;
-esac
-EOF
-
-cat > "${STUBBIN}/eks-gateway-fetch" <<'EOF'
-#!/bin/sh
-echo "$*" >> "${FETCHED}"
-[ "${FETCH_RC:-0}" -eq 0 ] || exit "${FETCH_RC}"
-printf '%s' "${LIVE_BODY:-}"
-EOF
-
-cat > "${STUBBIN}/logger" <<'EOF'
-#!/bin/sh
-:
-EOF
-
-chmod +x "${STUBBIN}"/*
-PATH="${STUBBIN}:${PATH}"
-export PATH
-
-ENVFILE="${WORK}/first-boot.env"
-cat > "${ENVFILE}" <<'EOF'
-EKS_GATEWAY_URL=https://gw.invalid:9999
-EKS_ACCOUNT_ID=000000000001
-EKS_CLUSTER_NAME=demo3
-EOF
-CONSOLE="${WORK}/console"
-DELETED="${WORK}/deleted"
-FETCHED="${WORK}/fetched"
-export ENVFILE CONSOLE DELETED FETCHED
-
-PASS=0
-FAIL=0
-check() {
-    if [ "$2" = "$3" ]; then
-        PASS=$((PASS + 1))
-    else
-        FAIL=$((FAIL + 1))
-        printf 'FAIL: %s\n  want: %s\n  got:  %s\n' "$1" "$2" "$3"
-    fi
-}
-
-run_agent() {
-    : > "${DELETED}"
-    : > "${FETCHED}"
-    sh "${AGENT}"
-}
-
-deleted() { tr '\n' ' ' < "${DELETED}" | sed 's/ $//'; }
-
-# Every way a Node can look: only a NotReady nodegroup worker whose instance the
-# cluster's nodegroups no longer own is deleted.
-NODES_BODY='live|aws:///z1/i-0aaa|ng1|True
-gone|aws:///z1/i-0bbb|ng1|Unknown
-gone-never-reported|aws:///z1/i-0ddd|ng2|
-owned-notready|aws:///z1/i-0ccc|ng1|False
-gone-but-ready|aws:///z1/i-0eee|ng1|True
-self-managed|aws:///z1/i-0fff||Unknown
-k3s-provider|k3s://k3s-node|ng1|Unknown'
-LIVE_BODY='i-0aaa
-i-0ccc
-'
-export NODES_BODY LIVE_BODY
-
-run_agent
-check "deletes only NotReady nodegroup Nodes whose instance is gone" "gone gone-never-reported" "$(deleted)"
-check "fetches the workers resource" "-resource workers" "$(cat "${FETCHED}")"
-
-# A cluster with no nodegroups left: every NotReady nodegroup Node goes.
-LIVE_BODY='' run_agent
-check "empty worker set deletes every NotReady nodegroup Node" "gone gone-never-reported owned-notready" "$(deleted)"
-
-# A failed fetch is not an empty set: nothing is deleted.
-FETCH_RC=1 run_agent
-check "failed fetch deletes nothing" "" "$(deleted)"
-
-# Without a Node listing the workers are not fetched, so a set fetched before a
-# Node registered can never be compared against it.
-NODES_RC=1 run_agent
-check "failed node listing deletes nothing" "" "$(deleted)"
-check "failed node listing does not fetch" "" "$(cat "${FETCHED}")"
-
-echo "---"
-echo "PASS=${PASS} FAIL=${FAIL}"
-[ "${FAIL}" -eq 0 ]
@@ -44,10 +44,9 @@ ln -sf /usr/local/bin/k3s /usr/local/bin/ctr
 # role's services are baked; the selector enables the right ones at first boot.
 chmod 0755 /etc/init.d/eks-node-role /etc/init.d/k3s /etc/init.d/k3s-agent \
    /etc/init.d/eks-token-webhook /etc/init.d/k3s-first-boot /etc/init.d/mulga-eks-state-report \
-    /etc/init.d/mulga-eks-addon-sync /etc/init.d/mulga-eks-node-gc /etc/init.d/konnectivity-server
+    /etc/init.d/mulga-eks-addon-sync /etc/init.d/konnectivity-server
 chmod 0755 /usr/local/sbin/eks-node-role /usr/local/sbin/k3s-first-boot \
-    /usr/local/sbin/mulga-eks-state-report /usr/local/sbin/mulga-eks-addon-sync \
-    /usr/local/sbin/mulga-eks-node-gc
+    /usr/local/sbin/mulga-eks-state-report /usr/local/sbin/mulga-eks-addon-sync
 # start_pre helpers shared with the systemd wrappers that will eventually
 # replace these runscripts.
 chmod 0755 /usr/local/sbin/k3s-prestart /usr/local/sbin/k3s-agent-prestart \
@@ -1014,9 +1014,6 @@ RESOLV_CONF="${SPINIFEX_RESOLV_CONF:-/etc/resolv.conf}"
 RESOLVED_STUB=/run/systemd/resolve/stub-resolv.conf
 # node-reset.sh keeps /var/lib/spinifex-reset, so the copy survives until uninstall.
 RESOLV_CONF_SAVED="${SPINIFEX_RESOLV_CONF_SAVED:-/var/lib/spinifex-reset/resolv.conf.orig}"
-# NetworkManager started while resolv.conf is a plain file rewrites it on its
-# next DNS update, replacing the stub link; this drop-in stops that.
-NM_DNS_DROPIN="${SPINIFEX_NM_DNS_DROPIN:-/etc/NetworkManager/conf.d/90-spinifex-dns.conf}"

 # systemd-resolved ships resolvconf as a link to resolvectl, which only works
 # while resolved runs, so that one is not a resolvconf backend.
@@ -1069,52 +1066,10 @@ dns_handover_failed() {
    fatal "Host DNS through systemd-resolved does not work: $1.
    To put host DNS back as it was before this installer:
      sudo systemctl disable --now systemd-resolved
-      sudo rm -f $NM_DNS_DROPIN
      sudo cp -a --remove-destination $RESOLV_CONF_SAVED $RESOLV_CONF
      sudo systemctl restart NetworkManager"
 }

-# Prints the rc-manager NetworkManager would use if it is not unmanaged, which
-# means a file read after the Spinifex drop-in overrides it.
-nm_rc_manager_override() {
-    local config
-    config=$($SUDO NetworkManager --print-config) || return 1
-    printf '%s\n' "$config" | awk -F= '
-        /^\[/ { main = ($0 == "[main]"); next }
-        main && $1 == "rc-manager" { rc = $2 }
-        END { if (rc != "unmanaged") print "rc-manager=" (rc == "" ? "symlink" : rc) }'
-}
-
-# Writes the NetworkManager drop-in and checks nothing overrides it. Returns 1
-# when it was already in place.
-install_nm_dns_dropin() {
-    local want override changed=1
-    want=$(printf '[main]\ndns=systemd-resolved\nrc-manager=unmanaged')
-    if [ "$(cat "$NM_DNS_DROPIN" 2>/dev/null)" != "$want" ]; then
-        $SUDO mkdir -p "$(dirname "$NM_DNS_DROPIN")"
-        printf '%s\n' "$want" | $SUDO tee "$NM_DNS_DROPIN" > /dev/null || dns_handover_failed "cannot write $NM_DNS_DROPIN"
-        info "  $NM_DNS_DROPIN: NetworkManager leaves $RESOLV_CONF alone"
-        changed=0
-    fi
-    override=$(nm_rc_manager_override) || dns_handover_failed "cannot read NetworkManager's configuration with 'NetworkManager --print-config'"
-    if [ -n "$override" ]; then
-        dns_handover_failed "NetworkManager reads $override from a file after $NM_DNS_DROPIN, so it would replace the $RESOLV_CONF link. Remove that setting ('NetworkManager --print-config' shows the files it reads) and re-run this installer"
-    fi
-    return "$changed"
-}
-
-# Links the stub and checks the link is still there a second later, relinking a
-# bounded number of times in case a write NetworkManager had queued lands first.
-link_resolved_stub() {
-    local i
-    for i in 1 2 3 4 5; do
-        $SUDO ln -sfn "$RESOLVED_STUB" "$RESOLV_CONF"
-        sleep 1
-        [ "$(readlink "$RESOLV_CONF")" = "$RESOLVED_STUB" ] && return 0
-    done
-    dns_handover_failed "$RESOLV_CONF is not a link to $RESOLVED_STUB 5 seconds after linking it, so something keeps rewriting it"
-}
-
 # Checks resolved has upstream servers and still resolves the package
 # repository host, the one name this host is known to have resolved.
 check_resolved_dns() {
@@ -1148,13 +1103,8 @@ hand_dns_to_resolved() {
    [ "${ISO_BUILD:-0}" = "1" ] && return 0

    if systemctl is-active --quiet systemd-resolved.service; then
-        # A handover made by an earlier run is checked again, and repaired, on
-        # every run: installs before the drop-in could lose the link.
+        # A handover made by an earlier run is checked again on every run.
        if [ -e "$RESOLV_CONF_SAVED" ] || [ -L "$RESOLV_CONF_SAVED" ]; then
-            if install_nm_dns_dropin && systemctl is-active --quiet NetworkManager.service; then
-                $SUDO systemctl reload NetworkManager.service || dns_handover_failed "NetworkManager did not reload"
-            fi
-            [ "$(readlink "$RESOLV_CONF")" = "$RESOLVED_STUB" ] || link_resolved_stub
            check_resolved_dns
        fi
        return 0
@@ -1181,15 +1131,10 @@ hand_dns_to_resolved() {
        $SUDO cp -a "$RESOLV_CONF" "$RESOLV_CONF_SAVED" || fatal "Cannot save $RESOLV_CONF to $RESOLV_CONF_SAVED"
        info "  saved $RESOLV_CONF to $RESOLV_CONF_SAVED"
    fi
-    install_nm_dns_dropin || true
    $SUDO systemctl enable --now systemd-resolved.service > /dev/null 2>&1 \
        || dns_handover_failed "systemd-resolved did not start"
    $SUDO systemctl restart NetworkManager.service || dns_handover_failed "NetworkManager did not restart"
-    # Best effort: a link with no DHCP server can hold startup past the timeout.
-    if command -v nm-online > /dev/null; then
-        nm-online -s -q --timeout=15 || true
-    fi
-    link_resolved_stub
+    $SUDO ln -sfn "$RESOLVED_STUB" "$RESOLV_CONF"
    check_resolved_dns
    info "  $RESOLV_CONF -> $RESOLVED_STUB"
 }
@@ -20,10 +20,9 @@ fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
 echo "-- setup.sh host DNS, chrony, needrestart and nbdkit"

 mkdir -p "$TMP/sys"
-for tool in bash awk sed grep readlink basename dirname cp ln mkdir cat head tail wc tee rm chmod install mv; do
+for tool in bash awk sed grep readlink basename dirname cp ln mkdir cat head tail wc tee rm chmod install; do
    ln -s "$(command -v "$tool")" "$TMP/sys/$tool"
 done
-ln -s "$(command -v sleep)" "$TMP/sys/realsleep"

 # stub <name> <body> — a stub command; $S is the fixture's state directory.
 stub() {
@@ -31,17 +30,10 @@ stub() {
    chmod +x "$TMP/bin/$1"
 }

-# nm_settle — waits for the fixture's NetworkManager to finish writing.
-nm_settle() {
-    n=0
-    while [ -e "$TMP/state/nm-pending" ] && [ $n -lt 500 ]; do "$TMP/sys/realsleep" 0.01; n=$((n + 1)); done
-}
-
 # host — a fresh fixture: resolved installed and inactive, resolvconf the
 # resolvectl link, NetworkManager running with no DNS settings, a dnf host
 # whose first enabled repository is yum.oracle.com.
 host() {
-    nm_settle
    rm -rf "${TMP:?}/state" "${TMP:?}/root" "${TMP:?}/bin"
    mkdir -p "$TMP/state/active" "$TMP/root/etc" "$TMP/root/usr/lib/systemd/system" "$TMP/bin"
    : > "$TMP/root/usr/lib/systemd/system/systemd-resolved.service"
@@ -50,28 +42,11 @@ host() {
    printf '[main]\n# dns=default\nconfigure-and-quit=no\n' > "$TMP/state/nm-config"
    printf 'Global:\nLink 2 (br-wan): 192.168.1.1\n' > "$TMP/state/resolvectl-dns"
    printf '[ol10_UEKR8]\nbaseurl = https://mirror.example/uek/\nenabled = 0\n[ol10_baseos_latest]\nbaseurl = https://yum.oracle.com/repo/OracleLinux/OL10/baseos/latest/x86_64/\nenabled = 1\n' > "$TMP/state/dnf-dump"
-    # A (re)started NetworkManager whose rc-manager is not unmanaged writes
-    # resolv.conf as a plain file on its first DNS update, which can land just
-    # after the installer links the stub.
    stub systemctl '[ "$1" = is-active ] && { [ -e "$S/active/$3" ]; exit; }
 echo "$*" >> "$S/calls"
 [ "$1 $2" = "enable --now" ] && : > "$S/active/$3"
-case "$1 $2" in
-    "restart NetworkManager.service" | "reload NetworkManager.service")
-        NetworkManager --print-config | grep -qx rc-manager=unmanaged && exit 0
-        : > "$S/nm-pending"
-        (
-            n=0
-            while [ ! -L "$S/../root/etc/resolv.conf" ] && [ $n -lt 300 ]; do realsleep 0.01; n=$((n + 1)); done
-            printf "nameserver 192.168.1.1\n" > "$S/nm-resolv.conf"
-            mv -f "$S/nm-resolv.conf" "$S/../root/etc/resolv.conf"
-            rm -f "$S/nm-pending"
-        ) > /dev/null 2>&1 &
-        ;;
-esac
 exit 0'
-    stub NetworkManager 'cat "$S/nm-config"; cat "$S/../root/nm/90-spinifex-dns.conf" "$S/nm-config-late" 2>/dev/null; :'
-    stub nm-online ':'
+    stub NetworkManager 'cat "$S/nm-config"'
    stub resolvectl '[ "$1" = dns ] && cat "$S/resolvectl-dns"'
    stub resolvectl-real ':'
    ln -s resolvectl "$TMP/bin/resolvconf"
@@ -85,17 +60,10 @@ run() {
    RC=0
    OUT=$(PATH="$TMP/bin:$TMP/sys" DISTRO_ROOT="$TMP/root" \
        SPINIFEX_RESOLV_CONF="$TMP/root/etc/resolv.conf" SPINIFEX_RESOLV_CONF_SAVED="$TMP/root/saved/resolv.conf.orig" \
-        SPINIFEX_NM_DNS_DROPIN="$TMP/root/nm/90-spinifex-dns.conf" \
        INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '. "$1"; shift; for f in "$@"; do $f; done' _ "$SETUP" "$@" 2>&1) || RC=$?
 }

 calls() { cat "$TMP/state/calls" 2>/dev/null || true; }
-# linked — whether resolv.conf is still the stub link once NetworkManager settles.
-linked() {
-    nm_settle
-    [ "$(readlink "$TMP/root/etc/resolv.conf")" = /run/systemd/resolve/stub-resolv.conf ]
-}
-dropin() { cat "$TMP/root/nm/90-spinifex-dns.conf" 2>/dev/null || echo "<none>"; }
 saved() { cat "$TMP/root/saved/resolv.conf.orig" 2>/dev/null || echo "<none>"; }
 expect_untouched() {
    if [ "$RC" = 0 ] && [ -z "$(calls)" ] && [ ! -L "$TMP/root/etc/resolv.conf" ] && [ "$(saved)" = "<none>" ]; then
@@ -118,7 +86,7 @@ expect_fatal() { # expect_fatal <desc> <text>...
 # --------------------------------------------------------------------------
 host
 run hand_dns_to_resolved
-if [ "$RC" = 0 ] && linked \
+if [ "$RC" = 0 ] && [ "$(readlink "$TMP/root/etc/resolv.conf")" = /run/systemd/resolve/stub-resolv.conf ] \
    && [ "$(saved)" = "$(printf 'nameserver 192.168.1.1\nsearch example.lan')" ] \
    && [ "$(calls)" = "$(printf 'enable --now systemd-resolved.service\nrestart NetworkManager.service')" ] \
    && [ "$(cat "$TMP/state/looked-up")" = yum.oracle.com ]; then
@@ -130,44 +98,12 @@ fi
 : > "$TMP/state/calls"
 rm -f "$TMP/state/looked-up"
 run hand_dns_to_resolved
-if [ "$RC" = 0 ] && [ -z "$(calls)" ] && linked && [ "$(cat "$TMP/state/looked-up")" = yum.oracle.com ]; then
+if [ "$RC" = 0 ] && [ -z "$(calls)" ] && [ "$(cat "$TMP/state/looked-up")" = yum.oracle.com ]; then
    pass "a rerun after a handover checks DNS again and changes nothing"
 else
    fail "rerun after handover: rc=$RC calls='$(calls)': $OUT"
 fi

-if [ "$(dropin)" = "$(printf '[main]\ndns=systemd-resolved\nrc-manager=unmanaged')" ]; then
-    pass "the handover tells NetworkManager to send DNS to resolved and leave resolv.conf alone"
-else
-    fail "NetworkManager drop-in: '$(dropin)'"
-fi
-
-# An install made before the drop-in, whose NetworkManager replaced the link.
-host
-mkdir -p "$TMP/root/saved"
-echo "the original" > "$TMP/root/saved/resolv.conf.orig"
-: > "$TMP/state/active/systemd-resolved.service"
-run hand_dns_to_resolved
-if [ "$RC" = 0 ] && linked && [ "$(calls)" = "reload NetworkManager.service" ] \
-    && [ "$(dropin)" = "$(printf '[main]\ndns=systemd-resolved\nrc-manager=unmanaged')" ]; then
-    pass "a rerun after a handover whose link NetworkManager replaced adds the drop-in, reloads NM and relinks the stub"
-else
-    fail "rerun repair: rc=$RC calls='$(calls)' link='$(readlink "$TMP/root/etc/resolv.conf")' dropin='$(dropin)': $OUT"
-fi
-
-host
-printf '[main]\nrc-manager=symlink\n' > "$TMP/state/nm-config-late"
-run hand_dns_to_resolved
-expect_fatal "a NetworkManager file read after the drop-in that sets rc-manager stops the install, naming it" \
-    "rc-manager=symlink" "rm -f $TMP/root/nm/90-spinifex-dns.conf"
-case "$(calls)" in *"restart NetworkManager"*) fail "rc-manager override: restarted NetworkManager first: $(calls)" ;; esac
-
-host
-stub ln ':'
-run hand_dns_to_resolved
-expect_fatal "a resolv.conf that will not stay linked to the stub stops the install" \
-    "is not a link to /run/systemd/resolve/stub-resolv.conf"
-
 host
 mkdir -p "$TMP/root/saved"
 echo "the original" > "$TMP/root/saved/resolv.conf.orig"
@@ -219,8 +155,7 @@ host
 printf 'Global:\nLink 2 (br-wan):\n' > "$TMP/state/resolvectl-dns"
 run hand_dns_to_resolved
 expect_fatal "no upstream servers stops the install, naming the check and printing the restore" \
-    "'resolvectl dns' lists no DNS servers" "systemctl disable --now systemd-resolved" "cp -a --remove-destination $TMP/root/saved/resolv.conf.orig" \
-    "rm -f $TMP/root/nm/90-spinifex-dns.conf"
+    "'resolvectl dns' lists no DNS servers" "systemctl disable --now systemd-resolved" "cp -a --remove-destination $TMP/root/saved/resolv.conf.orig"

 host
 : > "$TMP/state/getent-fail"
@@ -16,9 +16,6 @@
 //
 // The recovery resource returns a single epoch\taction\tsnapshot line the on-VM
 // k3s-recovery agent applies before k3s starts (etcd cluster-reset / wipe-rejoin).
-//
-// The workers resource returns one instance ID per line: every worker the
-// cluster's nodegroups still own, which the on-VM node-gc agent keeps Nodes for.
 package gatewayfetch

 import (
@@ -49,7 +46,7 @@ type Config struct {
 	Region      string
 	AccountID   string
 	ClusterName string
-	// Resource is "addons", "recovery" or "workers".
+	// Resource is "addons" or "recovery".
 	Resource   string
 	InstanceID string
 	// Out receives the rendered TSV.
@@ -64,8 +61,8 @@ func Run(ctx context.Context, cfg Config) error {
 		return errors.New("--account-id is required (or set EKS_ACCOUNT_ID)")
 	case cfg.ClusterName == "":
 		return errors.New("--cluster is required (or set EKS_CLUSTER_NAME)")
-	case cfg.Resource != "addons" && cfg.Resource != "recovery" && cfg.Resource != "workers":
-		return errors.New("--resource must be addons, recovery or workers")
+	case cfg.Resource != "addons" && cfg.Resource != "recovery":
+		return errors.New("--resource must be addons or recovery")
 	case cfg.Resource == "recovery" && cfg.InstanceID == "":
 		return errors.New("--instance-id is required for --resource recovery (or set EKS_INSTANCE_ID)")
 	}
@@ -83,9 +80,6 @@ func Run(ctx context.Context, cfg Config) error {
 	case "recovery":
 		path = "/clusters/" + cfg.ClusterName + "/internal-recovery/" + cfg.AccountID + "/" + cfg.InstanceID
 		emit = emitRecoveryTSV
-	case "workers":
-		path = "/clusters/" + cfg.ClusterName + "/internal-workers/" + cfg.AccountID
-		emit = emitWorkers
 	default:
 		path = "/clusters/" + cfg.ClusterName + "/internal-addons/" + cfg.AccountID
 		emit = emitAddonsTSV
@@ -169,24 +163,3 @@ func emitRecoveryTSV(out io.Writer, body []byte) error {
 	}
 	return nil
 }
-
-type internalWorkersResponse struct {
-	InstanceIDs []string `json:"instanceIds"`
-}
-
-// emitWorkers writes one managed-worker instance ID per line. An empty set
-// writes nothing, which the node-gc agent tells apart from a failed fetch by
-// this command's exit status, never by the output being empty.
-func emitWorkers(out io.Writer, body []byte) error {
-	var resp internalWorkersResponse
-	if err := json.Unmarshal(body, &resp); err != nil {
-		return fmt.Errorf("unmarshal internal-workers response: %w", err)
-	}
-	w := bufio.NewWriter(out)
-	for _, id := range resp.InstanceIDs {
-		if _, err := fmt.Fprintln(w, id); err != nil {
-			return err
-		}
-	}
-	return w.Flush()
-}
@@ -49,7 +49,7 @@ func TestRun_RejectsBeforeContactingGateway(t *testing.T) {
 	}{
 		{"account", func(c *Config) { c.AccountID = "" }, "--account-id is required (or set EKS_ACCOUNT_ID)"},
 		{"cluster", func(c *Config) { c.ClusterName = "" }, "--cluster is required (or set EKS_CLUSTER_NAME)"},
-		{"resource", func(c *Config) { c.Resource = "secrets" }, "--resource must be addons, recovery or workers"},
+		{"resource", func(c *Config) { c.Resource = "secrets" }, "--resource must be addons or recovery"},
 		{"recovery instance", func(c *Config) { c.Resource = "recovery" }, "--instance-id is required for --resource recovery (or set EKS_INSTANCE_ID)"},
 		{"gateway client", func(c *Config) { c.GatewayURL = "" }, "build gateway client: eksgw: baseURL is required"},
 	}
@@ -92,26 +92,6 @@ func TestRun_RendersRecoveryDirective(t *testing.T) {
 	}
 }

-// The gateway's restjson marshaler names the field InstanceIDs; both that and
-// the json-tag spelling decode, so the agent reads either.
-func TestRun_RendersManagedWorkers(t *testing.T) {
-	for _, body := range []string{
-		`{"InstanceIDs":["i-0aaa","i-0bbb"]}`,
-		`{"instanceIds":["i-0aaa","i-0bbb"]}`,
-	} {
-		var hits atomic.Int32
-		cfg, out := gateway(t, "/clusters/demo/internal-workers/123456789012", http.StatusOK, body, &hits)
-		cfg.Resource = "workers"
-
-		if err := Run(context.Background(), cfg); err != nil {
-			t.Fatalf("Run: %v", err)
-		}
-		if got, want := out.String(), "i-0aaa\ni-0bbb\n"; got != want {
-			t.Fatalf("stdout = %q, want %q", got, want)
-		}
-	}
-}
-
 func TestRun_RenderFailureIsNotRetried(t *testing.T) {
 	var hits atomic.Int32
 	cfg, _ := gateway(t, "/clusters/demo/internal-addons/123456789012", http.StatusOK, `{not-json`, &hits)
@@ -694,39 +694,43 @@ func eniMatchesFilters(record *ENIRecord, filters map[string][]string) bool {
 // AttachENI marks an ENI as attached to an instance (internal use by RunInstances).
 // accountID scopes the lookup to the correct KV key.
 func (s *VPCServiceImpl) AttachENI(accountID, eniId, instanceId string, deviceIndex int64) (string, error) {
-	return s.attachENI(context.Background(), accountID, eniId, instanceId, deviceIndex, "")
+	return s.attachENI(context.Background(), accountID, eniId, instanceId, deviceIndex)
 }

-// AttachENIWithOwner attaches the ENI and records instanceOwnerID as the owner
-// of the attached instance in the same write, so no later write can lose the
-// stamp. An empty owner leaves the record's owner untouched.
-func (s *VPCServiceImpl) AttachENIWithOwner(accountID, eniId, instanceId string, deviceIndex int64, instanceOwnerID string) (string, error) {
-	return s.attachENI(context.Background(), accountID, eniId, instanceId, deviceIndex, instanceOwnerID)
-}
-
-func (s *VPCServiceImpl) attachENI(ctx context.Context, accountID, eniId, instanceId string, deviceIndex int64, instanceOwnerID string) (string, error) {
-	attachmentId := awsidentifiers.GenerateResourceID("eni-attach")
-	err := s.casENI(ctx, kvutil.AccountKey(accountID, eniId), eniId, func(record *ENIRecord) error {
-		if record.Status == "in-use" {
-			return errors.New(awserrors.ErrorInvalidNetworkInterfaceInUse)
-		}
-		record.Status = "in-use"
-		record.AttachmentId = attachmentId
-		record.InstanceId = instanceId
-		record.DeviceIndex = deviceIndex
-		if instanceOwnerID != "" {
-			record.InstanceOwnerId = instanceOwnerID
-		}
-		// Default DeleteOnTermination to true on first attach, matching what
-		// DescribeNetworkInterfaces already advertises; a record carrying an
-		// explicit value from an earlier attach keeps it across re-attach.
-		if record.DeleteOnTermination == nil {
-			record.DeleteOnTermination = aws.Bool(true)
-		}
-		return nil
-	})
+func (s *VPCServiceImpl) attachENI(ctx context.Context, accountID, eniId, instanceId string, deviceIndex int64) (string, error) {
+	key := kvutil.AccountKey(accountID, eniId)
+	entry, err := s.eniKV.Get(ctx, key)
 	if err != nil {
-		return "", err
+		return "", eniNotFoundError(eniId)
+	}
+
+	var record ENIRecord
+	if err := json.Unmarshal(entry.Value(), &record); err != nil {
+		return "", errors.New(awserrors.ErrorServerInternal)
+	}
+
+	if record.Status == "in-use" {
+		return "", errors.New(awserrors.ErrorInvalidNetworkInterfaceInUse)
+	}
+
+	attachmentId := awsidentifiers.GenerateResourceID("eni-attach")
+	record.Status = "in-use"
+	record.AttachmentId = attachmentId
+	record.InstanceId = instanceId
+	record.DeviceIndex = deviceIndex
+	// Default DeleteOnTermination to true on first attach, matching what
+	// DescribeNetworkInterfaces already advertises; a record carrying an
+	// explicit value from an earlier attach keeps it across re-attach.
+	if record.DeleteOnTermination == nil {
+		record.DeleteOnTermination = aws.Bool(true)
+	}
+
+	data, err := json.Marshal(record)
+	if err != nil {
+		return "", fmt.Errorf("failed to marshal ENI record: %w", err)
+	}
+	if _, err := s.eniKV.Update(ctx, key, data, entry.Revision()); err != nil {
+		return "", errors.New(awserrors.ErrorServerInternal)
 	}

 	slog.InfoContext(ctx, "ENI attached", "eniId", eniId, "instanceId", instanceId, "attachmentId", attachmentId)
@@ -792,69 +796,24 @@ func (s *VPCServiceImpl) UpdateENI(accountID, eniId string, fn func(*ENIRecord))
 }

 func (s *VPCServiceImpl) updateENI(ctx context.Context, accountID, eniId string, fn func(*ENIRecord)) error {
-	return s.casENI(ctx, kvutil.AccountKey(accountID, eniId), eniId, func(r *ENIRecord) error {
-		fn(r)
-		return nil
-	})
-}
-
-const (
-	eniCASAttempts = 3
-	eniCASBackoff  = 10 * time.Millisecond
-)
-
-// storeError reports ServerInternal to API callers while keeping the store
-// failure reachable through errors.Is for the caller's log.
-type storeError struct{ cause error }
-
-func (e *storeError) Error() string { return awserrors.ErrorServerInternal }
-func (e *storeError) Unwrap() error { return e.cause }
-
-// casENI reads the ENI record, applies mutate and writes it back under the
-// revision it read. A direct get can be answered by a replica that has not yet
-// applied the caller's own earlier write, so a revision conflict re-reads and
-// re-applies mutate, bounded. mutate may run more than once and must only
-// depend on the record it is given.
-func (s *VPCServiceImpl) casENI(ctx context.Context, key, eniId string, mutate func(*ENIRecord) error) error {
-	var lastErr error
-	attempts := 0
-	for attempt := range eniCASAttempts {
-		attempts = attempt + 1
-		if attempt > 0 {
-			select {
-			case <-ctx.Done():
-				return &storeError{ctx.Err()}
-			case <-time.After(time.Duration(attempt) * eniCASBackoff):
-			}
-		}
-		entry, err := s.eniKV.Get(ctx, key)
-		if err != nil {
-			return eniNotFoundError(eniId)
-		}
-		var record ENIRecord
-		if err := json.Unmarshal(entry.Value(), &record); err != nil {
-			return errors.New(awserrors.ErrorServerInternal)
-		}
-		if err := mutate(&record); err != nil {
-			return err
-		}
-		data, err := json.Marshal(record)
-		if err != nil {
-			return fmt.Errorf("marshal ENI record: %w", err)
-		}
-		if _, err = s.eniKV.Update(ctx, key, data, entry.Revision()); err == nil {
-			if attempt > 0 {
-				slog.InfoContext(ctx, "ENI write succeeded after revision conflict", "eniId", eniId, "attempt", attempt+1)
-			}
-			return nil
-		}
-		lastErr = err
-		if !errors.Is(err, jetstream.ErrKeyExists) {
-			break
-		}
+	key := kvutil.AccountKey(accountID, eniId)
+	entry, err := s.eniKV.Get(ctx, key)
+	if err != nil {
+		return eniNotFoundError(eniId)
 	}
-	slog.ErrorContext(ctx, "ENI write failed", "eniId", eniId, "attempts", attempts, "err", lastErr)
-	return &storeError{lastErr}
+	var record ENIRecord
+	if err := json.Unmarshal(entry.Value(), &record); err != nil {
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+	fn(&record)
+	data, err := json.Marshal(record)
+	if err != nil {
+		return fmt.Errorf("marshal ENI record: %w", err)
+	}
+	if _, err := s.eniKV.Update(ctx, key, data, entry.Revision()); err != nil {
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+	return nil
 }

 // ListInstanceENIs returns every ENIRecord in accountID currently attached to
@@ -1,9 +0,0 @@
-package vpc
-
-import "github.com/nats-io/nats.go/jetstream"
-
-// WrapENIKVForTest replaces the ENI bucket handle with wrap(current), so a test
-// can inject the read and write faults a replicated bucket produces.
-func (s *VPCServiceImpl) WrapENIKVForTest(wrap func(jetstream.KeyValue) jetstream.KeyValue) {
-	s.eniKV = wrap(s.eniKV)
-}
@@ -1,125 +0,0 @@
-package vpc
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"testing"
-
-	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
-	"github.com/nats-io/nats.go/jetstream"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// A replicated bucket can answer a direct get from a replica that has not yet
-// applied the caller's own previous write. The next compare-and-swap then
-// carries a revision that is already superseded.
-func newStaleReadENI(t *testing.T) (*VPCServiceImpl, string, func() ENIRecord) {
-	t.Helper()
-	svc := setupTestVPCService(t)
-	vpcID := createTestVPC(t, svc, "10.0.0.0/16")
-	subnetID := createTestSubnet(t, svc, vpcID, "10.0.1.0/24")
-	eniID := createTestENI(t, svc, subnetID)
-	var inner jetstream.KeyValue
-	svc.WrapENIKVForTest(func(kv jetstream.KeyValue) jetstream.KeyValue {
-		inner = kv
-		return testutil.NewStaleReadKV(kv)
-	})
-	// Reads the stored record directly: the wrapped bucket would serve the
-	// check itself from a lagging replica.
-	stored := func() ENIRecord {
-		entry, err := inner.Get(context.Background(), kvutil.AccountKey(testAccountID, eniID))
-		require.NoError(t, err)
-		var rec ENIRecord
-		require.NoError(t, json.Unmarshal(entry.Value(), &rec))
-		return rec
-	}
-	return svc, eniID, stored
-}
-
-func TestUpdateENIAfterOwnWriteSurvivesStaleReplicaRead(t *testing.T) {
-	t.Parallel()
-	svc, eniID, stored := newStaleReadENI(t)
-
-	_, err := svc.AttachENI(testAccountID, eniID, "i-test123", 0)
-	require.NoError(t, err)
-
-	err = svc.UpdateENI(testAccountID, eniID, func(r *ENIRecord) { r.InstanceOwnerId = "000000000000" })
-	require.NoError(t, err, "a read served from a lagging replica must not lose the update")
-
-	rec := stored()
-	assert.Equal(t, "000000000000", rec.InstanceOwnerId)
-	assert.Equal(t, "i-test123", rec.InstanceId, "the attach must survive the update")
-}
-
-func TestAttachENIAfterOwnWriteSurvivesStaleReplicaRead(t *testing.T) {
-	t.Parallel()
-	svc, eniID, stored := newStaleReadENI(t)
-
-	require.NoError(t, svc.UpdateENI(testAccountID, eniID, func(r *ENIRecord) { r.Description = "managed" }))
-
-	_, err := svc.AttachENI(testAccountID, eniID, "i-test123", 0)
-	require.NoError(t, err, "a read served from a lagging replica must not fail the attach")
-
-	rec := stored()
-	assert.Equal(t, "in-use", rec.Status)
-	assert.Equal(t, "managed", rec.Description)
-}
-
-func TestUpdateENIReportsTheStoreError(t *testing.T) {
-	t.Parallel()
-	svc := setupTestVPCService(t)
-	vpcID := createTestVPC(t, svc, "10.0.0.0/16")
-	subnetID := createTestSubnet(t, svc, vpcID, "10.0.1.0/24")
-	eniID := createTestENI(t, svc, subnetID)
-
-	cause := errors.New("stream unavailable")
-	svc.WrapENIKVForTest(func(kv jetstream.KeyValue) jetstream.KeyValue {
-		return testutil.NewRejectingKV(kv, func(string, []byte) error { return cause })
-	})
-
-	err := svc.UpdateENI(testAccountID, eniID, func(r *ENIRecord) { r.Description = "x" })
-	require.Error(t, err)
-	assert.ErrorIs(t, err, cause, "the caller logs this error, so it must name the store failure")
-}
-
-func TestAttachENIWithOwnerStampsTheOwnerInTheAttachWrite(t *testing.T) {
-	t.Parallel()
-	svc := setupTestVPCService(t)
-	vpcID := createTestVPC(t, svc, "10.0.0.0/16")
-	subnetID := createTestSubnet(t, svc, vpcID, "10.0.1.0/24")
-	eniID := createTestENI(t, svc, subnetID)
-
-	var writes []ENIRecord
-	svc.WrapENIKVForTest(func(kv jetstream.KeyValue) jetstream.KeyValue {
-		return testutil.NewRejectingKV(kv, func(_ string, value []byte) error {
-			var rec ENIRecord
-			require.NoError(t, json.Unmarshal(value, &rec))
-			writes = append(writes, rec)
-			return nil
-		})
-	})
-
-	_, err := svc.AttachENIWithOwner(testAccountID, eniID, "i-sys", 0, "000000000000")
-	require.NoError(t, err)
-
-	require.Len(t, writes, 1, "attach and owner stamp are one write")
-	assert.Equal(t, "in-use", writes[0].Status)
-	assert.Equal(t, "000000000000", writes[0].InstanceOwnerId)
-}
-
-func TestCASENIGivesUpAfterBoundedConflicts(t *testing.T) {
-	t.Parallel()
-	svc, eniID, _ := newStaleReadENI(t)
-	conflicts := 0
-	err := svc.casENI(t.Context(), kvutil.AccountKey(testAccountID, eniID), eniID, func(*ENIRecord) error {
-		conflicts++
-		// Move the record under the next compare-and-swap, as a competing writer does.
-		return svc.UpdateENI(testAccountID, eniID, func(r *ENIRecord) { r.Description = "other writer" })
-	})
-	require.Error(t, err)
-	assert.ErrorIs(t, err, jetstream.ErrKeyExists, "the conflict is reported, not hidden")
-	assert.Equal(t, eniCASAttempts, conflicts)
-}
@@ -50,11 +50,10 @@ var eksScopes = map[string][]resourceSource{
 	"ListClusters":    {sourceAny},

 	// Internal control-plane routes. The cluster's owning account is a path
-	// segment on these three, and the ARN names that account rather than the
+	// segment on these two, and the ARN names that account rather than the
 	// system caller's, because that is the account the handler reads.
 	"ListInternalAddons":   {sourceInternalCluster},
 	"GetRecoveryDirective": {sourceInternalCluster},
-	"ListInternalWorkers":  {sourceInternalCluster},
 	// The owning account arrives in the body on these two, and the ARN names
 	// that account for the same reason: it is the account the handler acts in.
 	"PublishInternal":    {sourceInternalBodyCluster},
@@ -179,10 +179,6 @@ func TestResourceARNsInternalRoutesUsePathAccount(t *testing.T) {
 	assert.Equal(t,
 		[]string{"arn:aws:eks:ap-southeast-2:999999999999:cluster/prod"},
 		resolveARNs(t, "GetRecoveryDirective", []string{"prod", "999999999999", "i-abc"}, ""))
-
-	assert.Equal(t,
-		[]string{"arn:aws:eks:ap-southeast-2:999999999999:cluster/prod"},
-		resolveARNs(t, "ListInternalWorkers", []string{"prod", "999999999999"}, ""))
 }

 // An identifier the gate cannot read authorizes account-wide, leaving the
@@ -93,46 +93,6 @@ func TestEKSRequest_InternalAddonsWireBody(t *testing.T) {
 		rec.Body.String())
 }

-// The node-gc agent reads the same REST-JSON shape: the Go field name, sorted
-// IDs drawn from every nodegroup record of the cluster.
-func TestEKSRequest_InternalWorkersWireBody(t *testing.T) {
-	const cpInstanceID = "i-cp0000000000001"
-	_, nc, js := testutil.StartTestJetStream(t)
-	kv, err := ekscluster.GetOrCreateAccountBucket(t.Context(), js, authzAccountID)
-	require.NoError(t, err)
-	require.NoError(t, ekscluster.PutClusterMeta(t.Context(), kv, &ekscluster.ClusterMeta{
-		Name:              "prod",
-		Status:            ekscluster.ClusterStatusActive,
-		ControlPlaneNodes: []ekscluster.ControlPlaneNode{{InstanceID: cpInstanceID}},
-	}))
-	for _, rec := range []*ekscluster.NodegroupRecord{
-		{ClusterName: "prod", Name: "ng-b", InstanceIDs: []string{"i-0ccc", "i-0aaa"}},
-		{ClusterName: "prod", Name: "ng-a", InstanceIDs: []string{"i-0bbb"}},
-	} {
-		require.NoError(t, ekscluster.PutNodegroupRecord(t.Context(), kv, rec))
-	}
-
-	svc, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Region: authzRegion})
-	require.NoError(t, err)
-	sub, err := nc.Subscribe("eks.ListManagedWorkers", func(m *nats.Msg) {
-		acct := natsmsg.AccountIDFromMsg(m)
-		natsmsg.ServeNATSRequestCtx(m, func(ctx context.Context, in *ekscluster.ListManagedWorkersInput) (*ekscluster.ListManagedWorkersOutput, error) {
-			return svc.ListManagedWorkers(ctx, in, acct)
-		})
-	})
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
-
-	gw := scopedEKS(gwtest.Statement("Allow", "eks:*", "*"))
-	gw.nc = nc
-	rec := httptest.NewRecorder()
-	req := httptest.NewRequest(http.MethodGet, "/clusters/prod/internal-workers/"+authzAccountID, nil)
-	require.NoError(t, gw.as(cpAgentContext(cpInstanceID)).serve(rec, req))
-
-	assert.Equal(t, http.StatusOK, rec.Code)
-	assert.Equal(t, `{"InstanceIDs":["i-0aaa","i-0bbb","i-0ccc"]}`, rec.Body.String())
-}
-
 // CreateAddon and UpdateAddon hand their service-account role to iam:PassRole,
 // same as CreateCluster's roleArn and CreateNodegroup's nodeRole; an absent
 // role passes nothing.
@@ -97,8 +97,6 @@ func TestAuthorizeInternal_RejectsNonCPPrincipals(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			assertDenied(t, eksawsapi.AuthorizeInternal(t.Context(), nil, "ListInternalAddons",
 				tc.caller, []string{"alpha", tenantAccount}, nil))
-			assertDenied(t, eksawsapi.AuthorizeInternal(t.Context(), nil, "ListInternalWorkers",
-				tc.caller, []string{"alpha", tenantAccount}, nil))
 			assertDenied(t, eksawsapi.AuthorizeInternal(t.Context(), nil, "GetRecoveryDirective",
 				tc.caller, []string{"alpha", tenantAccount, cpInstanceID}, nil))
 			assertDenied(t, eksawsapi.AuthorizeInternal(t.Context(), nil, "PublishInternal",
@@ -126,8 +124,6 @@ func TestAuthorizeInternal_AllowsClusterMember(t *testing.T) {

 	require.NoError(t, eksawsapi.AuthorizeInternal(t.Context(), nc, "ListInternalAddons",
 		cpAgent(cpInstanceID), []string{"alpha", tenantAccount}, nil))
-	require.NoError(t, eksawsapi.AuthorizeInternal(t.Context(), nc, "ListInternalWorkers",
-		cpAgent(cpInstanceID), []string{"alpha", tenantAccount}, nil))
 	require.NoError(t, eksawsapi.AuthorizeInternal(t.Context(), nc, "GetRecoveryDirective",
 		cpAgent(cpInstanceID), []string{"alpha", tenantAccount, cpInstanceID}, nil))
 	require.NoError(t, eksawsapi.AuthorizeInternal(t.Context(), nc, "PublishInternal",
@@ -1,35 +0,0 @@
-package awsapi
-
-import (
-	"context"
-	"errors"
-
-	ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/nats-io/nats.go"
-)
-
-// internalWorkersOutput is the body returned to the on-VM node-gc agent: the
-// instance ID of every worker the cluster's nodegroups still own.
-type internalWorkersOutput struct {
-	InstanceIDs []string `json:"instanceIds"`
-}
-
-// ListInternalWorkers — GET /clusters/{name}/internal-workers/{acct}.
-// Internal control-plane VM route (not an AWS-SDK action): same carve-out as
-// ListInternalAddons, gated by AuthorizeInternal, which admits only a CP agent
-// serving that account's cluster.
-func ListInternalWorkers(ctx context.Context, natsConn *nats.Conn, clusterName, accountID string) (*internalWorkersOutput, error) {
-	if natsConn == nil {
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-	if clusterName == "" || accountID == "" {
-		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
-	}
-	out, err := ekscluster.NewNATSEKSService(natsConn).ListManagedWorkers(ctx,
-		&ekscluster.ListManagedWorkersInput{ClusterName: clusterName}, accountID)
-	if err != nil {
-		return nil, err
-	}
-	return &internalWorkersOutput{InstanceIDs: out.InstanceIDs}, nil
-}
@@ -60,14 +60,6 @@ var routes = []route{
 			return GetRecoveryDirective(ctx, nc, p[0], p[1], p[2])
 		}},

-	// Internal control-plane route: the on-VM node-gc agent pulls the cluster's
-	// live nodegroup workers to delete the Nodes of terminated ones. Same
-	// system-cred carve-out as internal-addons.
-	{Method: "GET", Pattern: "/clusters/{clusterName}/internal-workers/{accountId}", Action: "ListInternalWorkers",
-		Handler: func(ctx context.Context, nc *nats.Conn, acct, callerARN string, p []string, b []byte) (any, error) {
-			return ListInternalWorkers(ctx, nc, p[0], p[1])
-		}},
-
 	// Nodegroup
 	{Method: "POST", Pattern: "/clusters/{clusterName}/node-groups", Action: "CreateNodegroup",
 		Handler: func(ctx context.Context, nc *nats.Conn, acct, callerARN string, p []string, b []byte) (any, error) {
@@ -108,7 +108,6 @@ func TestLookupEKSAction_CoversAllActions(t *testing.T) {
 		"WebhookTokenReview":           false,
 		"ListInternalAddons":           false,
 		"GetRecoveryDirective":         false,
-		"ListInternalWorkers":          false,
 		"DescribeCluster":              false,
 		"ListClusters":                 false,
 		"DeleteCluster":                false,
@@ -22,10 +22,9 @@ const (
 	// WebhookTokenReview (the eks-token-webhook relays `aws eks get-token` bearer
 	// tokens to the token-review broker for host-side STS verification), and
 	// GetRecoveryDirective (the k3s-recovery agent pulls its per-member etcd
-	// recovery directive at boot), and ListInternalWorkers (the node-gc agent
-	// pulls the live nodegroup workers). The gateway evaluates these per request
-	// against the role's policies.
-	eksServerInlinePolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["eks:PublishInternal","eks:ListInternalAddons","eks:WebhookTokenReview","eks:GetRecoveryDirective","eks:ListInternalWorkers"],"Resource":"*"}]}`
+	// recovery directive at boot). The gateway evaluates these per request against
+	// the role's policies.
+	eksServerInlinePolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["eks:PublishInternal","eks:ListInternalAddons","eks:WebhookTokenReview","eks:GetRecoveryDirective"],"Resource":"*"}]}`
 )

 // ensureCPInstanceProfile returns the CP instance-profile ARN, or "" to signal
@@ -36,7 +35,7 @@ func (s *EKSServiceImpl) ensureCPInstanceProfile(accountID string) string {
 	iam := s.iamEnsurer()
 	if iam == nil {
 		slog.Warn("EKS: IAM service unwired; CP VM falls back to baked static gateway creds and "+
-			"will be denied eks:PublishInternal, eks:WebhookTokenReview, eks:ListInternalAddons, eks:GetRecoveryDirective and eks:ListInternalWorkers (no instance identity to bind)",
+			"will be denied eks:PublishInternal, eks:WebhookTokenReview, eks:ListInternalAddons and eks:GetRecoveryDirective (no instance identity to bind)",
 			"accountID", accountID)
 		return ""
 	}
@@ -44,7 +43,7 @@ func (s *EKSServiceImpl) ensureCPInstanceProfile(accountID string) string {
 		CPInstanceRoleName, eksServerInlinePolicyName, eksServerInlinePolicy)
 	if err != nil {
 		slog.Error("EKS: ensure CP instance profile failed; CP VM falls back to baked static gateway creds and "+
-			"will be denied eks:PublishInternal, eks:WebhookTokenReview, eks:ListInternalAddons, eks:GetRecoveryDirective and eks:ListInternalWorkers (no instance identity to bind)",
+			"will be denied eks:PublishInternal, eks:WebhookTokenReview, eks:ListInternalAddons and eks:GetRecoveryDirective (no instance identity to bind)",
 			"accountID", accountID, "role", CPInstanceRoleName, "err", err)
 		return ""
 	}
@@ -38,9 +38,6 @@ func TestEnsureK3sServerInstanceProfile_CreatesRolePolicyProfile(t *testing.T) {
 	// these IMDS creds; without this action the gateway 403s the boot-time fetch
 	// and a wedged control plane can never be auto-reset.
 	assert.Contains(t, policy, "eks:GetRecoveryDirective")
-	// The node-gc agent pulls the live nodegroup workers with these IMDS creds;
-	// without this action a terminated worker's Node is never deleted.
-	assert.Contains(t, policy, "eks:ListInternalWorkers")

 	require.NotNil(t, f.Profiles[CPInstanceRoleName])
 	assert.Len(t, f.Profiles[CPInstanceRoleName].Roles, 1)
@@ -44,10 +44,6 @@ type EKSService interface {
 	// AWS-SDK action): the on-VM addon-sync agent fetches staged manifests for a
 	// cluster via the internal-addons gateway route.
 	ListStagedAddonManifests(ctx context.Context, input *ListStagedAddonManifestsInput, accountID string) (*ListStagedAddonManifestsOutput, error)
-	// ListManagedWorkers is an internal control-plane method (not an AWS-SDK
-	// action): the on-VM node-gc agent fetches the cluster's live nodegroup
-	// workers via the internal-workers gateway route.
-	ListManagedWorkers(ctx context.Context, input *ListManagedWorkersInput, accountID string) (*ListManagedWorkersOutput, error)

 	// GetRecoveryDirective / SetRecoveryDirective are internal control-plane
 	// methods (not AWS-SDK actions): the on-VM k3s-recovery agent fetches its
@@ -129,10 +129,6 @@ func (s *NATSEKSService) ListStagedAddonManifests(ctx context.Context, input *Li
 	return natsmsg.NATSRequest[ListStagedAddonManifestsOutput](ctx, s.natsConn, "eks.ListStagedAddonManifests", input, defaultTimeout, accountID)
 }

-func (s *NATSEKSService) ListManagedWorkers(ctx context.Context, input *ListManagedWorkersInput, accountID string) (*ListManagedWorkersOutput, error) {
-	return natsmsg.NATSRequest[ListManagedWorkersOutput](ctx, s.natsConn, "eks.ListManagedWorkers", input, defaultTimeout, accountID)
-}
-
 func (s *NATSEKSService) UpdateAddon(ctx context.Context, input *eks.UpdateAddonInput, accountID string) (*eks.UpdateAddonOutput, error) {
 	return natsmsg.NATSRequest[eks.UpdateAddonOutput](ctx, s.natsConn, "eks.UpdateAddon", input, defaultTimeout, accountID)
 }
@@ -1,46 +0,0 @@
-package cluster
-
-import (
-	"context"
-	"errors"
-	"slices"
-
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-)
-
-// ListManagedWorkersInput names the cluster whose managed-nodegroup workers to
-// list. It is an internal control-plane request (not an AWS-SDK shape), served
-// over NATS for the guest node-gc agent via the internal-workers route.
-type ListManagedWorkersInput struct {
-	ClusterName string `json:"clusterName"`
-}
-
-// ListManagedWorkersOutput carries the instance ID of every worker a managed
-// nodegroup of the cluster still owns, sorted.
-type ListManagedWorkersOutput struct {
-	InstanceIDs []string `json:"instanceIds"`
-}
-
-// ListManagedWorkers returns every worker instance ID recorded on the cluster's
-// nodegroups. A nodegroup drops an ID only after terminating it (DeleteNodegroup,
-// scale-down, reclaim), so the guest deletes a nodegroup Node whose instance is
-// absent here, as the AWS cloud controller deletes a Node whose instance is gone.
-func (s *EKSServiceImpl) ListManagedWorkers(ctx context.Context, input *ListManagedWorkersInput, accountID string) (*ListManagedWorkersOutput, error) {
-	if input == nil || input.ClusterName == "" {
-		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
-	}
-	acctKV, err := s.acctKVForCluster(ctx, accountID, input.ClusterName)
-	if err != nil {
-		return nil, err
-	}
-	recs, err := ListNodegroupRecords(ctx, acctKV, input.ClusterName)
-	if err != nil {
-		return nil, err
-	}
-	ids := make([]string, 0)
-	for _, rec := range recs {
-		ids = append(ids, rec.InstanceIDs...)
-	}
-	slices.Sort(ids)
-	return &ListManagedWorkersOutput{InstanceIDs: slices.Compact(ids)}, nil
-}
@@ -1,74 +0,0 @@
-package cluster
-
-import (
-	"context"
-	"testing"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/eks"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func managedWorkers(t *testing.T, f *eksServiceFixture, cluster string) []string {
-	t.Helper()
-	out, err := f.svc.ListManagedWorkers(context.Background(), &ListManagedWorkersInput{ClusterName: cluster}, testAccountID)
-	require.NoError(t, err)
-	return out.InstanceIDs
-}
-
-// A worker leaves the managed set exactly when its nodegroup terminates it, by
-// scale-down or DeleteNodegroup, so the guest node-gc deletes its Node then and
-// leaves every worker still owned by a nodegroup alone.
-func TestListManagedWorkers_DropsWorkersTheNodegroupTerminated(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	for _, ng := range []string{"ng1", "ng2"} {
-		_, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", ng, 2), testAccountID)
-		require.NoError(t, err)
-		markWorkersReady(t, f, "c1", ng, 2)
-		f.svc.WaitLaunches()
-	}
-	ng1, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
-	require.NoError(t, err)
-	ng2, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng2")
-	require.NoError(t, err)
-	require.Len(t, ng1.InstanceIDs, 2)
-	assert.ElementsMatch(t, append(append([]string{}, ng1.InstanceIDs...), ng2.InstanceIDs...), managedWorkers(t, f, "c1"))
-
-	_, err = f.svc.UpdateNodegroupConfig(context.Background(), &eks.UpdateNodegroupConfigInput{
-		ClusterName:   aws.String("c1"),
-		NodegroupName: aws.String("ng1"),
-		ScalingConfig: &eks.NodegroupScalingConfig{DesiredSize: aws.Int64(1)},
-	}, testAccountID)
-	require.NoError(t, err)
-	require.Len(t, f.worker.terminateCalls, 1)
-	scaledIn := f.worker.terminateCalls[0]
-	got := managedWorkers(t, f, "c1")
-	assert.Len(t, got, 3)
-	for _, id := range scaledIn {
-		assert.NotContains(t, got, id, "a worker terminated by scale-down must leave the managed set")
-	}
-
-	_, err = f.svc.DeleteNodegroup(context.Background(), &eks.DeleteNodegroupInput{
-		ClusterName: aws.String("c1"), NodegroupName: aws.String("ng2"),
-	}, testAccountID)
-	require.NoError(t, err)
-	got = managedWorkers(t, f, "c1")
-	assert.Len(t, got, 1)
-	for _, id := range ng2.InstanceIDs {
-		assert.NotContains(t, got, id, "a deleted nodegroup's workers must leave the managed set")
-	}
-}
-
-func TestListManagedWorkers_UnknownClusterIsNotFound(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-
-	_, err := f.svc.ListManagedWorkers(context.Background(), &ListManagedWorkersInput{ClusterName: "ghost"}, testAccountID)
-	code, ok := awserrors.ResolveErrorCode(err)
-	require.True(t, ok)
-	assert.Equal(t, awserrors.ErrorEKSResourceNotFound, code)
-}
@@ -2,12 +2,9 @@ package elbv2

 import (
 	"context"
-	"errors"
 	"testing"
 	"time"

-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -83,80 +80,3 @@ func TestStartLifecycleReaper(t *testing.T) {

 	cancel()
 }
-
-// A load balancer the reaper gives up on must not leave its managed VM running:
-// the VM holds capacity and no customer data, and nothing else would stop it.
-func TestReapStuckProvisioningLBsTerminatesOnlyTheFailedLBsVM(t *testing.T) {
-	t.Parallel()
-	svc := setupTestService(t)
-	launcher := &mockSystemInstanceLauncher{}
-	svc.InstanceLauncher = launcher
-	now := time.Now().UTC()
-
-	stuck := testLBRecord("stuck0000000000", StateProvisioning, now.Add(-10*time.Minute), time.Time{})
-	stuck.InstanceID = "i-stuck"
-	noVM := testLBRecord("novm0000000000", StateProvisioning, now.Add(-10*time.Minute), time.Time{})
-	fresh := testLBRecord("fresh0000000000", StateProvisioning, now, time.Time{})
-	fresh.InstanceID = "i-fresh"
-	active := testLBRecord("active000000000", StateActive, now.Add(-10*time.Minute), time.Time{})
-	active.InstanceID = "i-active"
-	heartbeating := testLBRecord("heartbeat0000000", StateProvisioning, now.Add(-10*time.Minute), now.Add(-time.Second))
-	heartbeating.InstanceID = "i-heartbeating"
-	for _, lb := range []*LoadBalancerRecord{stuck, noVM, fresh, active, heartbeating} {
-		require.NoError(t, svc.store.PutLoadBalancer(t.Context(), lb))
-	}
-
-	svc.reapStuckProvisioningLBs(t.Context(), now)
-
-	got, err := svc.store.GetLoadBalancer(t.Context(), stuck.LoadBalancerID)
-	require.NoError(t, err)
-	assert.Equal(t, StateFailed, got.State)
-	assert.NotEmpty(t, got.StateReason, "the failure reason stays on the LB")
-
-	terminated := func() []string {
-		launcher.mu.Lock()
-		defer launcher.mu.Unlock()
-		return append([]string(nil), launcher.terminateCalls...)
-	}
-	require.Eventually(t, func() bool { return len(terminated()) > 0 }, 2*time.Second, 10*time.Millisecond,
-		"the reaper must terminate the failed LB's VM")
-	assert.Equal(t, []string{"i-stuck"}, terminated(),
-		"only the failed LB's own VM is terminated, never another LB's")
-}
-
-func TestReapStuckProvisioningLBsDetachesTheFailedLBsENIs(t *testing.T) {
-	t.Parallel()
-	for _, tc := range []struct {
-		name         string
-		terminateErr error
-		wantStatus   string
-	}{
-		{"terminated VM releases its ENI", nil, "available"},
-		{"a VM that cannot be stopped keeps its ENI attached", errors.New("nats timeout"), "in-use"},
-	} {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			svc, vpcSvc, subnetID := setupMicrovmTestService(t)
-			svc.InstanceLauncher = &mockSystemInstanceLauncher{terminateErr: tc.terminateErr}
-
-			eniOut, err := vpcSvc.CreateNetworkInterface(t.Context(), &ec2.CreateNetworkInterfaceInput{SubnetId: aws.String(subnetID)}, testAccountID)
-			require.NoError(t, err)
-			eniID := aws.StringValue(eniOut.NetworkInterface.NetworkInterfaceId)
-			_, err = vpcSvc.AttachENI(testAccountID, eniID, "i-stuck", 0)
-			require.NoError(t, err)
-
-			now := time.Now().UTC()
-			lb := testLBRecord("enirel0000000000", StateProvisioning, now.Add(-10*time.Minute), time.Time{})
-			lb.AccountID = testAccountID
-			lb.InstanceID = "i-stuck"
-			lb.ENIs = []string{eniID}
-			require.NoError(t, svc.store.PutLoadBalancer(t.Context(), lb))
-
-			svc.reapStuckProvisioningLBs(t.Context(), now)
-
-			rec, err := vpcSvc.GetENIRecord(testAccountID, eniID)
-			require.NoError(t, err, "the ENI stays for DeleteLoadBalancer to remove")
-			assert.Equal(t, tc.wantStatus, rec.Status)
-		})
-	}
-}
@@ -1772,31 +1772,6 @@ func (s *ELBv2ServiceImpl) reapStuckProvisioningLBs(ctx context.Context, now tim
 			"lbArn", lb.LoadBalancerArn, "lbId", lb.LoadBalancerID,
 			"instanceId", lb.InstanceID, "createdAt", lb.CreatedAt)
 		s.markLBFailed(ctx, lb.LoadBalancerArn, reason)
-		s.releaseFailedLBVM(ctx, lb)
-	}
-}
-
-// releaseFailedLBVM stops the failed LB's own managed VM and detaches its ENIs,
-// so a failed LB holds no running capacity. The ENIs stay on the record for
-// DeleteLoadBalancer to remove. A VM that cannot be stopped keeps its ENIs
-// attached, since detaching under a running guest would only hide the leak.
-func (s *ELBv2ServiceImpl) releaseFailedLBVM(ctx context.Context, lb *LoadBalancerRecord) {
-	if lb.InstanceID == "" || s.InstanceLauncher == nil {
-		return
-	}
-	if err := s.InstanceLauncher.TerminateSystemInstance(lb.InstanceID); err != nil {
-		slog.Error("ELBv2 lifecycle reaper: failed to terminate failed LB VM",
-			"lbId", lb.LoadBalancerID, "instanceId", lb.InstanceID, "err", err)
-		return
-	}
-	if s.VPCService == nil {
-		return
-	}
-	for _, eniID := range lb.ENIs {
-		if err := s.VPCService.DetachENI(ctx, lb.AccountID, eniID); err != nil && !awserrors.IsNotFound(err) {
-			slog.Warn("ELBv2 lifecycle reaper: failed to detach failed LB ENI",
-				"lbId", lb.LoadBalancerID, "eniId", eniID, "err", err)
-		}
 	}
 }

@@ -55,7 +55,6 @@ func TestAWSOperationInventoryMeasuresEKSAgainstTheModel(t *testing.T) {
 	require.ElementsMatch(t, []string{
 		"GetRecoveryDirective",
 		"ListInternalAddons",
-		"ListInternalWorkers",
 		"PublishInternal",
 		"WebhookTokenReview",
 	}, eks.Extra)
@@ -1149,7 +1149,7 @@ func snapshotErrorCode(err error) ebsprovider.ErrorCode {
 		return ebsprovider.ErrorCodeNotFound
 	case errors.Is(err, errSnapshotExistsElsewhere):
 		return ebsprovider.ErrorCodeAlreadyExists
-	case errors.Is(err, errVolumeLeaseHeld), errors.Is(err, errNoVolumeLeaseStore):
+	case errors.Is(err, errVolumeLeaseHeld), errors.Is(err, errNoVolumeLeaseStore), errors.Is(err, errNoVolumeDirtyStore):
 		// Exclusive access could not be established. The holder gives the
 		// volume up on detach, so this is worth repeating rather than the
 		// permanent failure an internal error reads as.
@@ -0,0 +1,90 @@
+package viperblock
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"testing"
+	"time"
+
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestRecoverMountedVolumes_StartupOrderMarksTheAdoptedExport follows Run's
+// order from a cold Config: InitLeases, then recovery. An export adopted
+// before the markers existed ran with no record of its unsealed writes.
+func TestRecoverMountedVolumes_StartupOrderMarksTheAdoptedExport(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+	cfg := setupTestConfig(t, natsURL)
+	cfg.leases, cfg.dirty = nil, nil
+	cfg.constructVB = fileBackedConstructVB(t)
+
+	nc, err := nats.Connect(natsURL)
+	require.NoError(t, err)
+	t.Cleanup(nc.Close)
+	require.NoError(t, cfg.InitLeases(t.Context(), nc))
+
+	const volumeName = "vol-recoverymarked1"
+	procRoot := t.TempDir()
+	socket := filepath.Join(t.TempDir(), "m.sock")
+	listenUnix(t, socket)
+	writeProcFixture(t, procRoot, 434343, "nbdkit", nbdkitArgv(cfg.BaseDir, socket, 0, volumeName))
+
+	recoverMountedVolumes(t.Context(), cfg, nc, procRoot)
+
+	mv, ok := findMountedVolume(cfg, volumeName)
+	require.True(t, ok)
+	t.Cleanup(func() { mv.VB.StopWALSyncer() })
+
+	record, found := cfg.dirty.holder(t.Context(), volumeName)
+	require.True(t, found, "an adopted export holds unsealed writes until it seals, so it must be marked")
+	assert.Equal(t, cfg.leaseOwner(), record.Owner)
+	assert.Equal(t, mv.leaseGeneration(), record.Generation)
+}
+
+// TestRecoverMountedVolumes_UnrecordableAdoptLeavesAnInUseExportRunning covers
+// recovery that cannot record the export's writes. It must not adopt it, and a
+// guest still using the export must keep it: refusing is not a reason to kill.
+func TestRecoverMountedVolumes_UnrecordableAdoptLeavesAnInUseExportRunning(t *testing.T) {
+	cases := map[string]func(t *testing.T, cfg *Config, js jetstream.JetStream){
+		"no marker store": func(_ *testing.T, cfg *Config, _ jetstream.JetStream) { cfg.dirty = nil },
+		"marker write fails": func(t *testing.T, _ *Config, js jetstream.JetStream) {
+			require.NoError(t, js.DeleteKeyValue(t.Context(), viperblocklegacyv1.DirtyBucket))
+		},
+	}
+	for name, breakMarkers := range cases {
+		t.Run(name, func(t *testing.T) {
+			_, natsURL := setupEmbeddedNATS(t)
+			cfg := setupTestConfig(t, natsURL)
+			cfg.constructVB = fileBackedConstructVB(t)
+
+			nc, err := nats.Connect(natsURL)
+			require.NoError(t, err)
+			t.Cleanup(nc.Close)
+			js, err := jetstream.New(nc)
+			require.NoError(t, err)
+			breakMarkers(t, cfg, js)
+
+			const volumeName = "vol-recoveryunmarked1"
+			procRoot := t.TempDir()
+			pid := spawnLongRunningProcess(t)
+			socket := filepath.Join(t.TempDir(), "u.sock")
+			listenUnix(t, socket)
+			writeProcFixture(t, procRoot, pid, "nbdkit", nbdkitArgv(cfg.BaseDir, socket, 0, volumeName))
+			writeReferencerFixture(t, procRoot, 919191,
+				fmt.Sprintf("driver=nbd,node-name=vol0,server.type=unix,server.path=%s,export=", socket))
+
+			recoverMountedVolumes(context.Background(), cfg, nc, procRoot)
+
+			_, ok := findMountedVolume(cfg, volumeName)
+			assert.False(t, ok, "an export whose writes cannot be recorded must not be adopted")
+			assert.Never(t, func() bool { return !hostprocess.ProcessAlive(pid) }, 100*time.Millisecond, 10*time.Millisecond,
+				"an export a guest is still using must never be signalled")
+		})
+	}
+}
@@ -687,15 +687,20 @@ func (cfg *Config) LoadEncryptionKey() error {
 	return nil
 }

-// InitLeases binds the cluster-wide volume lease store over nc, bound before
-// recovery: without the store every engine open refuses, and the adapter
-// would come up unable to adopt the exports that outlived it.
+// InitLeases binds the volume lease store and dirty markers over nc before
+// recovery or any subject: every engine open refuses without both, and an
+// export adopted before the markers exist would run with no unsealed record.
 func (cfg *Config) InitLeases(ctx context.Context, nc *nats.Conn) error {
 	leases, err := newVolumeLeasesWaiting(ctx, nc, cfg.leaseOwner())
 	if err != nil {
 		return fmt.Errorf("volume leases: %w", err)
 	}
+	dirty, err := newVolumeDirtyWaiting(ctx, nc, cfg.leaseOwner())
+	if err != nil {
+		return fmt.Errorf("volume dirty markers: %w", err)
+	}
 	cfg.leases = cfg.bindLeaseFence(leases)
+	cfg.dirty = dirty
 	cfg.nc = nc
 	return nil
 }
@@ -46,6 +46,8 @@ func installTestVolumeLeases(t *testing.T, cfg *Config, natsURL string) {
 	leases, err := newVolumeLeases(t.Context(), nc, cfg.leaseOwner())
 	require.NoError(t, err)
 	cfg.leases = leases
+	// An open refuses without dirty markers too, so the fixture needs both.
+	cfg.dirty = newTestDirty(t, natsURL, cfg.leaseOwner())
 }

 // startTestService runs the adapter's startup sequence (the same one
@@ -234,18 +234,20 @@ func TestVolumeDirty_OpenFailsWhenTheMarkerCannotBeWritten(t *testing.T) {
 		"the lease has to be released, or the volume is locked to a node that is not writing it")
 }

-// TestVolumeDirty_NoStoreDoesNotBlockMounts pins the degraded case. A daemon
-// with no marker store still has the lease for live exclusion, and refusing
-// every mount because the bucket is missing is a worse failure than the one
-// being described.
-func TestVolumeDirty_NoStoreDoesNotBlockMounts(t *testing.T) {
+// TestVolumeDirty_NoStoreRefusesTheOpen pins that an open this node cannot
+// record is refused: a later takeover could not warn about its writes, and a
+// crash would leave nothing to tell them from a clean seal.
+func TestVolumeDirty_NoStoreRefusesTheOpen(t *testing.T) {
 	_, natsURL := setupEmbeddedNATS(t)

 	cfg := &Config{leases: newTestLeases(t, natsURL, "node-b")}

 	lease, err := cfg.acquireVolumeLease(t.Context(), "vol-dirtynostore")
-	require.NoError(t, err)
-	require.NotNil(t, lease)
+	require.ErrorIs(t, err, errNoVolumeDirtyStore)
+	require.Nil(t, lease)
+
+	_, err = newTestLeases(t, natsURL, "node-c").acquire(t.Context(), "vol-dirtynostore")
+	require.NoError(t, err, "a refused open must not leave the volume leased")
 }

 // TestVolumeDirty_ListReportsTheHolder covers the operator surface behind
@@ -105,6 +105,10 @@ var errVolumeLeaseHeld = errors.New(ec2v1.VolumeHeldElsewhereText)
 // holds this" from "nobody can say who does".
 var errNoVolumeLeaseStore = errors.New("no volume lease store")

+// errNoVolumeDirtyStore reports an open refused because this node cannot record
+// that it holds unsealed writes, so a later takeover could not warn about them.
+var errNoVolumeDirtyStore = errors.New("no volume dirty marker store")
+
 // volumeLeaseRecord is what a holder publishes about itself. Generation is the
 // revision that won the race: monotonic across the bucket, so a later opener
 // always carries a higher one than the writer it displaced.
@@ -679,6 +683,9 @@ func (cfg *Config) acquireVolumeLease(ctx context.Context, volumeName string) (*
 	if cfg.leases == nil {
 		return nil, fmt.Errorf("%w: cannot establish exclusive access to %s", errNoVolumeLeaseStore, volumeName)
 	}
+	if cfg.dirty == nil {
+		return nil, fmt.Errorf("%w: cannot record unsealed writes for %s", errNoVolumeDirtyStore, volumeName)
+	}
 	lease, err := cfg.leases.acquire(ctx, volumeName)
 	if err != nil {
 		return nil, err
@@ -138,7 +138,8 @@ func TestVolumeLease_UnmountedSnapshotIsRefusedByTheHolder(t *testing.T) {
 	require.NoError(t, err)

 	// node-b has nothing mounted, so this is the unmounted branch.
-	cfg := &Config{NodeName: "node-b", BaseDir: t.TempDir(), leases: newTestLeases(t, natsURL, "node-b")}
+	cfg := &Config{NodeName: "node-b", BaseDir: t.TempDir(), leases: newTestLeases(t, natsURL, "node-b"),
+		dirty: newTestDirty(t, natsURL, "node-b")}
 	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeName, "snap-excl0000000001")

 	require.ErrorIs(t, err, errVolumeLeaseHeld, "a snapshot must not open an engine on a volume another node holds")
@@ -3,22 +3,15 @@ package vm
 import (
 	"encoding/json"
 	"fmt"
-	"slices"
 	"strconv"
 	"strings"
 )

-// IsMicrovm reports whether machineType is the QEMU microvm machine, with or
-// without a PCIe bus.
-func IsMicrovm(machineType string) bool {
-	return strings.HasPrefix(machineType, "microvm")
-}
-
-// IsMMIO reports whether machineType attaches virtio devices over memory-mapped
-// I/O. A microvm without pcie=on has no PCI bus, so it uses virtio-*-device
-// transports; with pcie=on devices are virtio-*-pci like any other machine.
+// IsMMIO reports whether machineType is a memory-mapped I/O bus machine (e.g.
+// microvm). MMIO machines use virtio-*-device transports instead of
+// virtio-*-pci, because they have no PCI bus.
 func IsMMIO(machineType string) bool {
-	return IsMicrovm(machineType) && !slices.Contains(strings.Split(machineType, ","), "pcie=on")
+	return strings.HasPrefix(machineType, "microvm")
 }

 // MaxNICQueues caps the queue pairs on a guest NIC. vhost-net spawns a kernel
@@ -316,19 +309,6 @@ func volumeMMIOBlkDevice(volumeID, nodeName, iothreadID string) Device {
 	return Device{Value: strings.Join(args, ",")}
 }

-// volumeMicrovmPCIBlkDevice renders a data volume on a microvm with a PCIe bus
-// as virtio-blk-pci with VolumeBlkDeviceArgs' identity and error policy. The
-// microvm has no hot-plug root ports, so the device sits on the default bus.
-func volumeMicrovmPCIBlkDevice(volumeID, nodeName, iothreadID string) Device {
-	args := []string{"virtio-blk-pci"}
-	for _, arg := range VolumeBlkDeviceArgs(volumeID, nodeName, iothreadID, "") {
-		if !strings.HasPrefix(arg, "bus=") {
-			args = append(args, arg)
-		}
-	}
-	return Device{Value: strings.Join(args, ",")}
-}
-
 // VolumeBlkDeviceQMPArgs renders the same virtio-blk-pci arguments as a
 // device_add argument map for AttachVolume's QMP hot-attach path. See
 // VolumeBlkDeviceArgs for why werror/rerror are set here.
@@ -314,27 +314,3 @@ func TestBlkDeviceMapped_EveryQueueMappedOnce(t *testing.T) {
 		}
 	}
 }
-
-func TestNetDevice_MicrovmWithPCIEIsPCI(t *testing.T) {
-	mt := "microvm,pic=off,pit=off,rtc=on,acpi=on,isa-serial=on,pcie=on"
-	assert.False(t, IsMMIO(mt), "a microvm with a PCIe bus must not use MMIO transports")
-	d := NetDevice(mt, "net0", "02:00:00:aa:bb:cc", 1, 0)
-	assert.Contains(t, d.Value, "virtio-net-pci,netdev=net0")
-	assert.NotContains(t, d.Value, "virtio-net-device")
-}
-
-func TestNetDevice_PlainMicrovmStaysMMIO(t *testing.T) {
-	assert.True(t, IsMMIO("microvm,pic=off,pit=off,rtc=on,acpi=on,isa-serial=on"))
-	assert.Contains(t, NetDevice("microvm", "net0", "", 0, 0).Value, "virtio-net-device")
-}
-
-func TestNetDevice_Q35Unchanged(t *testing.T) {
-	d := NetDevice("q35", "net0", "02:00:00:aa:bb:cc", 1, 0)
-	assert.Equal(t, "virtio-net-pci,netdev=net0,mac=02:00:00:aa:bb:cc,rx_queue_size=1024", d.Value)
-}
-
-func TestIsMicrovm(t *testing.T) {
-	assert.True(t, IsMicrovm("microvm"))
-	assert.True(t, IsMicrovm("microvm,pcie=on"))
-	assert.False(t, IsMicrovm("q35"))
-}
@@ -1376,7 +1376,7 @@ type driveConfig struct {
 // with the mounted data volumes appended after its devices. Neither input is
 // modified, so composing again from the same base never accumulates entries.
 func directBootLaunchConfig(base Config, requests []EBSRequest, cpuCount int) (Config, error) {
-	if !IsMicrovm(base.MachineType) {
+	if !IsMMIO(base.MachineType) {
 		return Config{}, fmt.Errorf("direct-boot launch config needs a microvm base, got machine type %q", base.MachineType)
 	}
 	for _, r := range requests {
@@ -1478,13 +1478,13 @@ func buildDrives(requests []EBSRequest, cpuCount int, machineType string) (drive

 		default:
 			// microvm has no PCIe root ports, so its data volumes take none.
-			microvm := IsMicrovm(machineType)
+			mmio := IsMMIO(machineType)
 			// A data (non-boot, non-EFI) volume. Allocate a stable hot-plug
 			// port when the persisted state predates port accounting, or the
 			// volume was attached at RunInstances time without one;
 			// freeHotplugEBSPort re-scans requests as mutated so far in this
 			// loop, so two unset data volumes in one call cannot collide.
-			if v.HotplugPort == 0 && !microvm {
+			if v.HotplugPort == 0 && !mmio {
 				port := freeHotplugEBSPort(requests)
 				if port == 0 {
 					return driveConfig{}, fmt.Errorf("no free EBS hot-plug port for volume %s", v.Name)
@@ -1513,12 +1513,9 @@ func buildDrives(requests []EBSRequest, cpuCount int, machineType string) (drive

 			cfg.IOThreads = append(cfg.IOThreads, IOThread{ID: iothreadID})
 			cfg.Blockdevs = append(cfg.Blockdevs, blockdev)
-			switch {
-			case IsMMIO(machineType):
+			if mmio {
 				cfg.Devices = append(cfg.Devices, volumeMMIOBlkDevice(v.Name, nodeName, iothreadID))
-			case microvm:
-				cfg.Devices = append(cfg.Devices, volumeMicrovmPCIBlkDevice(v.Name, nodeName, iothreadID))
-			default:
+			} else {
 				cfg.Devices = append(cfg.Devices, VolumeBlkDevice(v.Name, nodeName, iothreadID, HotplugEBSBus(v.HotplugPort)))
 			}
 			// Deliberately no entry in cfg.Drives: the bare anonymous -drive
@@ -2175,19 +2175,3 @@ func deviceValues(devs []Device) []string {
 	}
 	return out
 }
-
-func TestDirectBootLaunchConfig_PCIEDataVolumesKeepSerialAndID(t *testing.T) {
-	base := directBootBase()
-	base.MachineType = "microvm,pic=off,pit=off,rtc=on,acpi=on,isa-serial=on,pcie=on"
-	got, err := directBootLaunchConfig(base, directBootRequests(), 2)
-	require.NoError(t, err)
-
-	require.Len(t, got.Devices, 3)
-	assert.Equal(t, base.Devices[0], got.Devices[0])
-	assert.Equal(t, Device{Value: "virtio-blk-pci,id=vdisk-vol-data-a,drive=nbd-vol-data-a,iothread=ioth-vol-data-a,serial=voldataa,werror=stop,rerror=stop"}, got.Devices[1])
-	assert.Equal(t, Device{Value: "virtio-blk-pci,id=vdisk-vol-blk,drive=nbd-vol-blk,iothread=ioth-vol-blk,serial=volblk,werror=stop,rerror=stop"}, got.Devices[2])
-	for _, d := range got.Devices[1:] {
-		assert.NotContains(t, d.Value, "bus=", "microvm has no hot-plug root port to name")
-	}
-	assert.Empty(t, got.Drives)
-}
@@ -537,7 +537,7 @@ func (cfg *Config) Execute() (*exec.Cmd, error) {
 		args = append(args, "-display", "none")
 	}

-	if IsMicrovm(cfg.MachineType) && cfg.ConsoleLogPath != "" {
+	if IsMMIO(cfg.MachineType) && cfg.ConsoleLogPath != "" {
 		// microvm with isa-serial=on: the machine creates the ISA serial device.
 		// Use -chardev file so output is captured without a socket client, then
 		// -serial chardev:console0 wires it to the existing device (not a new one).
@@ -675,22 +675,3 @@ func TestExecute_NoNameEmitsNoNameArg(t *testing.T) {
 	assert.NoError(t, err)
 	assert.False(t, argExists(cmd.Args[1:], "-name"))
 }
-
-func TestExecute_MicrovmWithPCIEFileChardev(t *testing.T) {
-	cfg := Config{
-		CPUCount:       1,
-		Memory:         512,
-		Architecture:   "x86_64",
-		MachineType:    "microvm,isa-serial=on,pcie=on",
-		ConsoleLogPath: "/var/log/console.log",
-		SerialSocket:   "/run/console.sock",
-		KernelImage:    "/boot/vmlinuz",
-	}
-
-	cmd, err := cfg.Execute()
-	require.NoError(t, err)
-
-	args := cmd.Args[1:]
-	assert.Equal(t, "file,id=console0,path=/var/log/console.log", argValue(args, "-chardev"))
-	assert.Equal(t, "chardev:console0", argValue(args, "-serial"))
-}
@@ -1139,7 +1139,6 @@ func (d *Daemon) subscribeAll() error {
 			natsSub{"eks.UpdateAddon", handleNATSRequest(d.node, d.eksService.UpdateAddon), "spinifex-workers"},
 			natsSub{"eks.ListStagedAddonManifests", handleNATSRequest(d.node, d.eksService.ListStagedAddonManifests), "spinifex-workers"},
 			natsSub{"eks.GetRecoveryDirective", handleNATSRequest(d.node, d.eksService.GetRecoveryDirective), "spinifex-workers"},
-			natsSub{"eks.ListManagedWorkers", handleNATSRequest(d.node, d.eksService.ListManagedWorkers), "spinifex-workers"},
 			natsSub{"eks.SetRecoveryDirective", handleNATSRequest(d.node, d.eksService.SetRecoveryDirective), "spinifex-workers"},
 			natsSub{"eks.RestoreSnapshot", handleNATSRequest(d.node, d.eksService.RestoreSnapshot), "spinifex-workers"},
 			natsSub{"eks.TagResource", handleNATSRequest(d.node, d.eksService.TagResource), "spinifex-workers"},
@@ -39,24 +39,29 @@ func resolveENIAccount(owner, fallback string) string {
 	return fallback
 }

-// eniInstanceOwner is the owner to stamp on an ENI record when a system VM
-// attaches it. System VMs run in the system account but plug into a customer
-// ENI, so without the stamp IMDS resolves the instance under the ENI account
-// and serves no credentials. It is empty when both accounts are the same.
-func eniInstanceOwner(eniAccountID, instanceOwnerID string) string {
-	if instanceOwnerID == eniAccountID {
-		return ""
+// recordENIInstanceOwner stamps the attached instance's owning account onto the
+// ENI record. System VMs run in the system account but plug into a customer ENI,
+// so without this IMDS would resolve the instance and its IAM role under the ENI
+// account and serve no credentials. Best-effort: a miss leaves IMDS falling back
+// to the ENI account, which only matters for the cross-account system path.
+func (d *Daemon) recordENIInstanceOwner(eniAccountID, eniID, instanceOwnerID string) {
+	if d.vpcService == nil || eniID == "" || instanceOwnerID == eniAccountID {
+		return
+	}
+	if err := d.vpcService.UpdateENI(eniAccountID, eniID, func(r *ec2vpc.ENIRecord) {
+		r.InstanceOwnerId = instanceOwnerID
+	}); err != nil {
+		slog.Warn("LaunchSystemInstance: failed to record ENI instance owner",
+			"eniId", eniID, "instanceOwner", instanceOwnerID, "err", err)
 	}
-	return instanceOwnerID
 }

-// attachExtraENI attaches one pre-created extra ENI, stamping the instance
-// owner in the attach write, and, when the caller asked for an explicit
-// DeleteOnTermination, persists it onto the record. An ENI that has to outlive
-// its VM is only safe once that flag is stored, so a failed stamp fails the
-// attach rather than leaving a disposable NIC behind.
-func (d *Daemon) attachExtraENI(eniAccountID string, extra systeminstance.ExtraENIInput, instanceID string, deviceIndex int64, instanceOwnerID string) error {
-	if _, err := d.vpcService.AttachENIWithOwner(eniAccountID, extra.ENIID, instanceID, deviceIndex, eniInstanceOwner(eniAccountID, instanceOwnerID)); err != nil {
+// attachExtraENI attaches one pre-created extra ENI and, when the caller asked
+// for an explicit DeleteOnTermination, persists it onto the record. An ENI that
+// has to outlive its VM is only safe once that flag is stored, so a failed stamp
+// fails the attach rather than leaving a disposable NIC behind.
+func (d *Daemon) attachExtraENI(eniAccountID string, extra systeminstance.ExtraENIInput, instanceID string, deviceIndex int64) error {
+	if _, err := d.vpcService.AttachENI(eniAccountID, extra.ENIID, instanceID, deviceIndex); err != nil {
 		return err
 	}
 	if extra.DeleteOnTermination == nil {
@@ -183,11 +188,10 @@ func (d *Daemon) launchDirectSystemInstance(input *elbv2domain.SystemInstanceInp
 		}
 		// Mark ENI as attached to this instance
 		if d.vpcService != nil {
-			if _, attachErr := d.vpcService.AttachENIWithOwner(eniAccountID, instance.ENIId, instance.ID, 0, eniInstanceOwner(eniAccountID, accountID)); attachErr != nil {
-				slog.Error("LaunchSystemInstance: failed to attach ENI", "eniId", instance.ENIId, "instanceId", instance.ID, "instanceOwner", accountID, "err", attachErr)
-				d.cleanupFailedSystemInstance(instance, instanceType)
-				return nil, fmt.Errorf("attach ENI %s: %w", instance.ENIId, attachErr)
+			if _, attachErr := d.vpcService.AttachENI(eniAccountID, instance.ENIId, instance.ID, 0); attachErr != nil {
+				slog.Warn("LaunchSystemInstance: failed to attach ENI", "eniId", instance.ENIId, "instanceId", instance.ID, "err", attachErr)
 			}
+			d.recordENIInstanceOwner(eniAccountID, instance.ENIId, accountID)
 		}
 		// Attach any additional pre-created ENIs (multi-subnet ALB VMs).
 		// Use the launcher input slice as the source of truth so the VM
@@ -199,11 +203,12 @@ func (d *Daemon) launchDirectSystemInstance(input *elbv2domain.SystemInstanceInp
 				// customer-VPC ENI) lives in its own account; the ENI record is
 				// account-keyed, so attach under extra.AccountID when set.
 				extraAccount := resolveENIAccount(extra.AccountID, eniAccountID)
-				if attachErr := d.attachExtraENI(extraAccount, extra, instance.ID, int64(idx+1), accountID); attachErr != nil {
+				if attachErr := d.attachExtraENI(extraAccount, extra, instance.ID, int64(idx+1)); attachErr != nil {
 					slog.Error("LaunchSystemInstance: failed to attach extra ENI", "eniId", extra.ENIID, "instanceId", instance.ID, "err", attachErr)
 					d.cleanupFailedSystemInstance(instance, instanceType)
 					return nil, fmt.Errorf("attach extra ENI %s: %w", extra.ENIID, attachErr)
 				}
+				d.recordENIInstanceOwner(extraAccount, extra.ENIID, accountID)
 			}
 			instance.ExtraENIs = append(instance.ExtraENIs, vm.ExtraENI{
 				ENIID:         extra.ENIID,
@@ -235,6 +240,7 @@ func (d *Daemon) launchDirectSystemInstance(input *elbv2domain.SystemInstanceInp
 		if _, attachErr := d.vpcService.AttachENI(accountID, instance.ENIId, instance.ID, 0); attachErr != nil {
 			slog.Warn("LaunchSystemInstance: failed to attach auto-created ENI", "eniId", instance.ENIId, "instanceId", instance.ID, "err", attachErr)
 		}
+		d.recordENIInstanceOwner(accountID, instance.ENIId, accountID)
 	}

 	// Allocate public IP for internet-facing ALBs. Route through the EIP
@@ -543,7 +549,7 @@ func (d *Daemon) launchAMISystemInstance(input *systeminstance.SystemInstanceInp
 	for idx, extra := range input.ExtraENIs {
 		if d.vpcService != nil {
 			extraAccount := resolveENIAccount(extra.AccountID, input.AccountID)
-			if attachErr := d.attachExtraENI(extraAccount, extra, inst.ID, int64(idx+1), ""); attachErr != nil {
+			if attachErr := d.attachExtraENI(extraAccount, extra, inst.ID, int64(idx+1)); attachErr != nil {
 				slog.Error("launchAMISystemInstance: failed to attach extra ENI", "eniId", extra.ENIID, "instanceId", inst.ID, "err", attachErr)
 				d.vmMgr.MarkFailed(context.Background(), inst, "extra_eni_attach_failed")
 				return nil, fmt.Errorf("attach extra ENI %s: %w", extra.ENIID, attachErr)
@@ -859,14 +865,15 @@ func (d *Daemon) buildDirectBootConfig(instanceID string, input *elbv2domain.Sys
 		imageDir = guest.imageDir()
 	}

-	// Build kernel cmdline. Devices sit on the PCIe bus (pcie=on), so no
-	// virtio_mmio.device= arguments.
+	// Build kernel cmdline.
+	//
+	// QEMU microvm maps virtio-mmio slots at 0xfeb00000 + n*0x200 with IRQ 5+n.
+	// With acpi=on the guest also finds them through ACPI, as data disks are
+	// (they get no parameter); the NIC parameters are kept, matching that map.
 	//
 	// Boot-time performance flags:
-	//   quiet                   — suppress info printk to ttyS0 (115200 baud serial
-	//                             writes dominate boot time)
-	//   loglevel                — 3 on SQS guests (errors only); 7 on the LB so
-	//                             kernel warnings reach the console log
+	//   quiet loglevel=3        — suppress printk to ttyS0 (115200 baud serial
+	//                             writes dominate boot time; warnings+errors only)
 	//   mitigations=off         — skip spectre/meltdown microcode patching at
 	//                             boot. Load balancer only: KVM guest on trusted
 	//                             host CPU, single-tenant LB workload, no untrusted
@@ -878,12 +885,23 @@ func (d *Daemon) buildDirectBootConfig(instanceID string, input *elbv2domain.Sys
 	//   reboot=t                — triple-fault reboot; skip ACPI/keyboard probes
 	//   i8042.no{pnp,aux,kbd}   — microvm has no PS/2; skip i8042 probe timeouts
 	var sb strings.Builder
+	sb.WriteString("console=ttyS0 quiet loglevel=3")
 	if guest == nil {
-		sb.WriteString("console=ttyS0 quiet loglevel=7 mitigations=off")
-	} else {
-		sb.WriteString("console=ttyS0 quiet loglevel=3")
+		sb.WriteString(" mitigations=off")
 	}
 	sb.WriteString(" tsc=reliable no_timer_check reboot=t i8042.nopnp i8042.noaux i8042.nokbd")
+	// Slots must stay packed in the same order QEMU actually creates
+	// -device entries (buildNICNetdevs), so an unprovisioned mgmt NIC
+	// (skipped there) must not reserve a slot here either.
+	slot := 0
+	for i, nic := range input.NICs {
+		if !nicProvisioned(i, nic) {
+			continue
+		}
+		fmt.Fprintf(&sb, " virtio_mmio.device=0x200@0x%x:%d",
+			0xfeb00000+slot*0x200, 5+slot)
+		slot++
+	}
 	cmdline := sb.String()

 	machineType := microvmMachineType()
@@ -920,10 +938,10 @@ type nicNetdevResult struct {
 }

 // microvmMachineType returns the QEMU -M string for the microvm machine.
-// pcie=on gives NICs a PCI bus with MSI-X; virtio-mmio's shared level-triggered
-// lines stalled a NIC's RX path. isa-serial stays on for per-VM console logs.
+// isa-serial is kept on permanently for per-VM console log access — the
+// boot-time saving from turning it off is not worth losing console output.
 func microvmMachineType() string {
-	return "microvm,pic=off,pit=off,rtc=on,acpi=on,isa-serial=on,pcie=on"
+	return "microvm,pic=off,pit=off,rtc=on,acpi=on,isa-serial=on"
 }

 // buildNICNetdevs produces QEMU -netdev and -device entries for each NIC in
@@ -229,17 +229,16 @@ func TestLaunchSystemInstance_PreCreatedENI(t *testing.T) {
 	f := newSysInstFixture(t)
 	d := f.d

-	t.Run("made-up ENI fails the launch at the attach", func(t *testing.T) {
+	t.Run("made-up ENI still reaches launch", func(t *testing.T) {
 		sysInstLaunchFails(d)
 		in := f.input(f.newENI(t))
 		in.ENIID = "eni-madeup"
 		in.IamInstanceProfileArn = "arn:aws:iam::123456789012:instance-profile/lb"

-		// A VM whose ENI cannot be attached, and so cannot be stamped with
-		// its owner, can never authenticate and must not be launched.
+		// The attach and owner stamp are best-effort, so only the launch fails.
 		_, err := d.LaunchSystemInstance(in)
-		require.Error(t, err)
-		assert.Contains(t, err.Error(), "attach ENI eni-madeup")
+		require.ErrorIs(t, err, errSysInstLaunchRefused)
+		assert.Contains(t, err.Error(), "launch instance")
 	})

 	t.Run("cross-account ENI and extras are attached and stamped", func(t *testing.T) {
@@ -1,66 +0,0 @@
-package daemon
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/mulgadc/spinifex/internal/testkit"
-	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
-	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
-	"github.com/nats-io/nats.go/jetstream"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// A system VM on a customer ENI is only resolvable by IMDS once the ENI record
-// names the system account as the instance owner. Without it the VM never gets
-// role credentials and its load balancer is reaped as failed.
-func TestLaunchSystemInstance_OwnerStampSurvivesStaleReplicaRead(t *testing.T) {
-	f := newSysInstFixture(t)
-	var inner jetstream.KeyValue
-	f.d.vpcService.WrapENIKVForTest(func(kv jetstream.KeyValue) jetstream.KeyValue {
-		inner = kv
-		return testutil.NewStaleReadKV(kv)
-	})
-	sysInstLaunchRuns(f.d)
-
-	eni := f.newENI(t)
-	out, err := f.d.LaunchSystemInstance(f.input(eni))
-	require.NoError(t, err)
-
-	// Read the stored record directly: the wrapped bucket would serve the
-	// check itself from a lagging replica.
-	entry, err := inner.Get(context.Background(), kvutil.AccountKey(testAccountID, aws.StringValue(eni.NetworkInterfaceId)))
-	require.NoError(t, err)
-	var rec ec2vpc.ENIRecord
-	require.NoError(t, json.Unmarshal(entry.Value(), &rec))
-	assert.Equal(t, out.InstanceID, rec.InstanceId)
-	assert.Equal(t, awsidentifiers.GlobalAccountID, rec.InstanceOwnerId,
-		"the owner stamp must land even when the read behind it was served by a lagging replica")
-}
-
-func TestLaunchSystemInstance_UnwritableOwnerStampFailsTheLaunch(t *testing.T) {
-	f := newSysInstFixture(t)
-	f.d.vpcService.WrapENIKVForTest(func(kv jetstream.KeyValue) jetstream.KeyValue {
-		return testutil.NewRejectingKV(kv, func(_ string, value []byte) error {
-			var rec ec2vpc.ENIRecord
-			if json.Unmarshal(value, &rec) == nil && rec.InstanceOwnerId != "" {
-				return errors.New("stream unavailable")
-			}
-			return nil
-		})
-	})
-	sysInstLaunchRuns(f.d)
-
-	eni := f.newENI(t)
-	_, err := f.d.LaunchSystemInstance(f.input(eni))
-	require.Error(t, err, "a VM whose ENI cannot name its owner can never authenticate and must not be launched")
-
-	require.Eventually(t, func() bool { return len(f.d.vmMgr.Snapshot()) == 0 }, 5*time.Second, 20*time.Millisecond,
-		"the failed launch must leave no VM behind")
-}
@@ -1,66 +0,0 @@
-package daemon
-
-import (
-	"os"
-	"testing"
-
-	elbv2domain "github.com/mulgadc/spinifex/spinifex/domains/elbv2"
-	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func pciTestInput() *elbv2domain.SystemInstanceInput {
-	return &elbv2domain.SystemInstanceInput{
-		ENIID: "eni-pci0000000000001",
-		NICs: []elbv2domain.NICConfig{
-			{MAC: "02:aa:bb:cc:dd:01", IsDefault: true},
-			{MAC: "02:aa:bb:cc:dd:02"},
-			{MAC: "02:aa:bb:cc:dd:03"},
-		},
-	}
-}
-
-func assertPCINICs(t *testing.T, cfg vm.Config) {
-	t.Helper()
-	assert.Contains(t, cfg.MachineType, "microvm,")
-	assert.Contains(t, cfg.MachineType, "pcie=on")
-	require.Len(t, cfg.Devices, 3)
-	for i, mac := range []string{"02:aa:bb:cc:dd:01", "02:aa:bb:cc:dd:02", "02:aa:bb:cc:dd:03"} {
-		assert.Contains(t, cfg.Devices[i].Value, "virtio-net-pci,netdev=net", "NIC %d must be a PCI device", i)
-		assert.NotContains(t, cfg.Devices[i].Value, "virtio-net-device")
-		assert.Contains(t, cfg.Devices[i].Value, "mac="+mac, "MAC order must be unchanged")
-	}
-	assert.NotContains(t, cfg.KernelCmdline, "virtio_mmio.device=")
-}
-
-func TestBuildDirectBootConfig_LBNICsArePCI(t *testing.T) {
-	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
-	d := &Daemon{resourceMgr: &ResourceManager{}}
-
-	cfg, err := d.buildDirectBootConfig("i-pci-lb", pciTestInput(), nil)
-	require.NoError(t, err)
-	assertPCINICs(t, cfg)
-	assert.Contains(t, cfg.KernelCmdline, "loglevel=7", "LB guest kernel warnings must reach the console log")
-	for _, e := range cfg.FwCfg {
-		assert.NoError(t, os.Remove(e.File))
-	}
-}
-
-func TestBuildDirectBootConfig_SQSWorkerNICsArePCI(t *testing.T) {
-	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
-	d := &Daemon{resourceMgr: &ResourceManager{}}
-	w := &sqsWorkerLaunch{volumeID: "vol-pci", region: "ap-southeast-2", gatewayURL: "https://gw.example:9999", caCert: "ca"}
-
-	cfg, err := d.buildDirectBootConfig("i-pci-sqs", pciTestInput(), w)
-	require.NoError(t, err)
-	assertPCINICs(t, cfg)
-	for _, e := range cfg.FwCfg {
-		assert.NoError(t, os.Remove(e.File))
-	}
-}
-
-func TestMicrovmMachineType_HasPCIE(t *testing.T) {
-	assert.Contains(t, microvmMachineType(), "pcie=on")
-	assert.Contains(t, microvmMachineType(), "isa-serial=on")
-}
@@ -294,8 +294,8 @@ func TestBuildNICNetdevs_SingleNIC(t *testing.T) {
 	require.Len(t, res.netdevs, 1)
 	require.Len(t, res.devices, 1)
 	assert.Contains(t, res.netdevs[0].Value, "tap,id=net0,")
-	// microvm with pcie=on → virtio-net-pci transport
-	assert.Contains(t, res.devices[0].Value, "virtio-net-pci,netdev=net0")
+	// microvm machine type → virtio-net-device transport
+	assert.Contains(t, res.devices[0].Value, "virtio-net-device,netdev=net0")
 	assert.Contains(t, res.devices[0].Value, "02:aa:bb:cc:dd:01")
 }

@@ -547,7 +547,7 @@ func TestWriteFwCfgBlobs_InvalidNICs_NoDefault(t *testing.T) {
 // buildDirectBootConfig — mmio slot packing
 // ---------------------------------------------------------------------------

-func TestBuildDirectBootConfig_PCINICsPacked_MgmtSkipped(t *testing.T) {
+func TestBuildDirectBootConfig_MmioSlotsPacked_MgmtSkipped(t *testing.T) {
 	// NIC[1] (mgmt) unprovisioned (no br-mgmt) must not reserve an mmio slot;
 	// remaining NICs (primary, extra ENI) get packed, contiguous slots.
 	tmpDir := t.TempDir()
@@ -569,11 +569,10 @@ func TestBuildDirectBootConfig_PCINICsPacked_MgmtSkipped(t *testing.T) {
 	cfg, err := d.buildDirectBootConfig("i-mmio-test", input, nil)
 	require.NoError(t, err)

-	// Only 2 real devices (mgmt skipped), both PCI and in NIC order.
-	assert.NotContains(t, cfg.KernelCmdline, "virtio_mmio.device=")
-	require.Len(t, cfg.Devices, 2)
-	assert.Contains(t, cfg.Devices[0].Value, "virtio-net-pci,netdev=net0,mac=02:aa:bb:cc:dd:01")
-	assert.Contains(t, cfg.Devices[1].Value, "virtio-net-pci,netdev=net2,mac=02:aa:bb:cc:dd:03")
+	// Only 2 real devices (mgmt skipped) → slots packed at 0 and 1, not 0 and 2.
+	assert.Contains(t, cfg.KernelCmdline, "virtio_mmio.device=0x200@0xfeb00000:5")
+	assert.Contains(t, cfg.KernelCmdline, "virtio_mmio.device=0x200@0xfeb00200:6")
+	assert.NotContains(t, cfg.KernelCmdline, "virtio_mmio.device=0x200@0xfeb00400:7")

 	require.Len(t, cfg.NetDevs, 2)
 	require.Len(t, cfg.Devices, 2)
@@ -583,7 +582,7 @@ func TestBuildDirectBootConfig_PCINICsPacked_MgmtSkipped(t *testing.T) {
 	}
 }

-func TestBuildDirectBootConfig_PCINICsUnchanged_MgmtProvisioned(t *testing.T) {
+func TestBuildDirectBootConfig_MmioSlotsUnchanged_MgmtProvisioned(t *testing.T) {
 	// Multi-node host with br-mgmt: all NICs provisioned, slots stay 1:1
 	// with NIC index as before this fix.
 	tmpDir := t.TempDir()
@@ -601,10 +600,8 @@ func TestBuildDirectBootConfig_PCINICsUnchanged_MgmtProvisioned(t *testing.T) {
 	cfg, err := d.buildDirectBootConfig("i-mmio-test2", input, nil)
 	require.NoError(t, err)

-	assert.NotContains(t, cfg.KernelCmdline, "virtio_mmio.device=")
-	require.Len(t, cfg.Devices, 2)
-	assert.Contains(t, cfg.Devices[0].Value, "virtio-net-pci,netdev=net0,mac=02:aa:bb:cc:dd:01")
-	assert.Contains(t, cfg.Devices[1].Value, "virtio-net-pci,netdev=net1,mac=02:aa:bb:cc:dd:02")
+	assert.Contains(t, cfg.KernelCmdline, "virtio_mmio.device=0x200@0xfeb00000:5")
+	assert.Contains(t, cfg.KernelCmdline, "virtio_mmio.device=0x200@0xfeb00200:6")
 	require.Len(t, cfg.NetDevs, 2)
 	require.Len(t, cfg.Devices, 2)

@@ -923,7 +920,7 @@ func TestAttachExtraENI_DeleteOnTerminationFalseSurvivesTerminate(t *testing.T)
 	require.NoError(t, f.daemon.attachExtraENI(testAccountID, systeminstance.ExtraENIInput{
 		ENIID:               f.eniID,
 		DeleteOnTermination: aws.Bool(false),
-	}, f.vmInst.ID, 1, ""))
+	}, f.vmInst.ID, 1))

 	rec, err := f.daemon.vpcService.GetENIRecord(testAccountID, f.eniID)
 	require.NoError(t, err)
@@ -947,7 +944,7 @@ func TestAttachExtraENI_NilDeleteOnTerminationStaysDisposable(t *testing.T) {

 	require.NoError(t, f.daemon.attachExtraENI(testAccountID, systeminstance.ExtraENIInput{
 		ENIID: f.eniID,
-	}, f.vmInst.ID, 1, ""))
+	}, f.vmInst.ID, 1))

 	require.NoError(t, newInstanceCleanerAdapter(f.daemon).DetachAndDeleteENI(f.vmInst))

@@ -27,9 +27,9 @@ const nodeGroupLabelRows = `jsonpath={range .items[*]}{.metadata.name} {.metadat
 const nodeReadyRows = `jsonpath={range .items[*]}{.metadata.name}{"="}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}`

 // runCPIsolation proves control-plane hosts never register as Nodes, as on AWS:
-// only nodegroup workers are listed, one becomes Ready, deleted nodegroups leave
-// no Node behind, a DaemonSet tolerating every taint runs on workers only,
-// metrics-server and its APIService become available, and kube-system logs work.
+// only nodegroup workers are listed and one becomes Ready, a DaemonSet tolerating
+// every taint runs on workers only, metrics-server and its APIService become
+// available, and logs work for every kube-system pod on a Ready node.
 func runCPIsolation(t *testing.T, c *harness.AWSClient, env *harness.Env, artifacts string, fx *clusterFixture) {
 	nodeRoleArn, _ := ensureNodeRole(t, c, fx)
 	createWorkerNodegroup(t, c, fx, cpIsolationNodegroup, nodeRoleArn, nil)
@@ -37,39 +37,19 @@ func runCPIsolation(t *testing.T, c *harness.AWSClient, env *harness.Env, artifa
 	kcPath := writeKubeconfig(t, artifacts, fx.Cluster)
 	kc := harness.NewKubectl(t, kcPath, getTokenEnv(t, env))

-	// Earlier subtests' nodegroups are deleted by now, so a Ready Node from one
-	// of them cannot stand in for this nodegroup's worker.
 	t.Run("WorkerNodeReady", func(t *testing.T) {
 		harness.EventuallyErr(t, func() error {
-			out, err := kc.Run(30*time.Second, "get", "nodes",
-				"-l", "eks.amazonaws.com/nodegroup="+cpIsolationNodegroup, "-o", nodeReadyRows)
+			out, err := kc.Run(30*time.Second, "get", "nodes", "-o", nodeReadyRows)
 			if err != nil {
 				return fmt.Errorf("kubectl get nodes: %v\n%s", err, out)
 			}
 			if !strings.Contains(out, "=True") {
-				return fmt.Errorf("no Ready %s Node yet:\n%s", cpIsolationNodegroup, out)
+				return fmt.Errorf("no Ready node yet:\n%s", out)
 			}
 			return nil
 		}, 6*time.Minute, 5*time.Second)
 	})

-	// As on AWS, a terminated worker's Node is deleted: once every other
-	// nodegroup is gone, the only Nodes left are this nodegroup's.
-	t.Run("TerminatedWorkersLeaveNoNodes", func(t *testing.T) {
-		harness.EventuallyErr(t, func() error {
-			out, err := kc.Run(30*time.Second, "get", "nodes", "-o", nodeGroupLabelRows)
-			if err != nil {
-				return fmt.Errorf("kubectl get nodes: %v\n%s", err, out)
-			}
-			for _, row := range strings.Split(strings.TrimSpace(out), "\n") {
-				if f := strings.Fields(row); len(f) == 2 && f[1] != cpIsolationNodegroup {
-					return fmt.Errorf("node %s of deleted nodegroup %s is still registered:\n%s", f[0], f[1], out)
-				}
-			}
-			return nil
-		}, 4*time.Minute, 10*time.Second)
-	})
-
 	t.Run("OnlyWorkersAreNodes", func(t *testing.T) {
 		out, err := kc.Run(30*time.Second, "get", "nodes", "-o", nodeGroupLabelRows)
 		require.NoErrorf(t, err, "get nodes:\n%s", out)
@@ -91,21 +71,19 @@ func runCPIsolation(t *testing.T, c *harness.AWSClient, env *harness.Env, artifa
 			_, _ = kc.Run(60*time.Second, "delete", "daemonset", ds, "-n", "default", "--ignore-not-found")
 		})

-		// One "<pod>=<Ready status>" row per pod: a pod bound to a Node that never
-		// reported has no Ready condition, and its row must count as not Ready.
 		harness.EventuallyErr(t, func() error {
 			ready, err := kc.Run(30*time.Second, "get", "pods", "-l", "app="+ds,
-				"-o", `jsonpath={range .items[*]}{.metadata.name}{"="}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}`)
+				"-o", `jsonpath={range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}`)
 			if err != nil {
 				return fmt.Errorf("get daemonset pods: %v\n%s", err, ready)
 			}
-			rows := strings.Fields(ready)
-			if len(rows) == 0 {
+			states := strings.Fields(ready)
+			if len(states) == 0 {
 				return fmt.Errorf("daemonset has no pods yet")
 			}
-			for _, row := range rows {
-				if _, status, _ := strings.Cut(row, "="); status != "True" {
-					return fmt.Errorf("daemonset pods not all Ready: %v", rows)
+			for _, s := range states {
+				if s != "True" {
+					return fmt.Errorf("daemonset pods not all Ready: %v", states)
 				}
 			}
 			return nil
@@ -536,7 +536,12 @@ func runIRSAPod(t *testing.T, c *harness.AWSClient, env *harness.Env, artifacts
 		},
 	})
 	require.NoError(t, err, "create-nodegroup")
-	t.Cleanup(func() { harness.DeleteEKSNodegroupAndWait(t, c, fx.ClusterName, nodegroup) })
+	t.Cleanup(func() {
+		_, _ = c.EKS.DeleteNodegroup(&eks.DeleteNodegroupInput{
+			ClusterName:   aws.String(fx.ClusterName),
+			NodegroupName: aws.String(nodegroup),
+		})
+	})
 	harness.EventuallyErr(t, func() error {
 		out, derr := c.EKS.DescribeNodegroup(&eks.DescribeNodegroupInput{
 			ClusterName:   aws.String(fx.ClusterName),
@@ -784,7 +789,12 @@ func runEBSCSIVolume(t *testing.T, c *harness.AWSClient, env *harness.Env, artif
 		},
 	})
 	require.NoError(t, err, "create-nodegroup")
-	t.Cleanup(func() { harness.DeleteEKSNodegroupAndWait(t, c, fx.ClusterName, nodegroup) })
+	t.Cleanup(func() {
+		_, _ = c.EKS.DeleteNodegroup(&eks.DeleteNodegroupInput{
+			ClusterName:   aws.String(fx.ClusterName),
+			NodegroupName: aws.String(nodegroup),
+		})
+	})
 	harness.EventuallyErr(t, func() error {
 		out, derr := c.EKS.DescribeNodegroup(&eks.DescribeNodegroupInput{
 			ClusterName:   aws.String(fx.ClusterName),
@@ -37,7 +37,12 @@ func createWorkerNodegroup(t *testing.T, c *harness.AWSClient, fx *clusterFixtur
 		},
 	})
 	require.NoError(t, err, "create-nodegroup")
-	t.Cleanup(func() { harness.DeleteEKSNodegroupAndWait(t, c, fx.ClusterName, nodegroup) })
+	t.Cleanup(func() {
+		_, _ = c.EKS.DeleteNodegroup(&eks.DeleteNodegroupInput{
+			ClusterName:   aws.String(fx.ClusterName),
+			NodegroupName: aws.String(nodegroup),
+		})
+	})
 	harness.EventuallyErr(t, func() error {
 		out, derr := c.EKS.DescribeNodegroup(&eks.DescribeNodegroupInput{
 			ClusterName:   aws.String(fx.ClusterName),
@@ -71,31 +71,3 @@ func isClusterNotFound(err error) bool {
 	}
 	return false
 }
-
-// DeleteEKSNodegroupAndWait deletes a nodegroup and polls DescribeNodegroup
-// until it returns ResourceNotFoundException, so a later step never runs
-// beside a worker that is still going away.
-func DeleteEKSNodegroupAndWait(t *testing.T, c *AWSClient, cluster, nodegroup string) {
-	t.Helper()
-	_, err := c.EKS.DeleteNodegroup(&eks.DeleteNodegroupInput{
-		ClusterName:   aws.String(cluster),
-		NodegroupName: aws.String(nodegroup),
-	})
-	if err != nil && !isClusterNotFound(err) {
-		t.Errorf("delete-nodegroup %s: %v", nodegroup, err)
-		return
-	}
-	EventuallyErr(t, func() error {
-		_, err := c.EKS.DescribeNodegroup(&eks.DescribeNodegroupInput{
-			ClusterName:   aws.String(cluster),
-			NodegroupName: aws.String(nodegroup),
-		})
-		if err == nil {
-			return fmt.Errorf("nodegroup %s still exists", nodegroup)
-		}
-		if isClusterNotFound(err) {
-			return nil
-		}
-		return fmt.Errorf("describe-nodegroup %s: %w", nodegroup, err)
-	}, 5*time.Minute, 5*time.Second)
-}
@@ -1,150 +0,0 @@
-//go:build e2e
-
-package lb
-
-import (
-	"fmt"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/tests/e2e/harness"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const (
-	privVPCCIDR    = "10.201.0.0/16"
-	privSubnetCIDR = "10.201.1.0/24"
-	privProbes     = 10
-)
-
-// TestInternalALBPrivateSubnet launches an internal ALB in a private subnet of a
-// non-default VPC that has no internet gateway and no public route. The LB must reach
-// active with a healthy target, which needs the LB VM's owner stamp to resolve, and an
-// instance inside the VPC must be served by it. The runner cannot reach the VPC, so the
-// in-VPC client reports its results on its serial console.
-func TestInternalALBPrivateSubnet(t *testing.T) {
-	env := harness.LoadEnv(t)
-	skipIfDevNetworking(t, env)
-
-	c := harness.NewAWSClient(t, env)
-	artifacts := harness.ArtifactDir(t, env)
-	f := &sharedFixture{}
-
-	harness.Phase(t, "Discovering cluster capacity")
-	f.InstanceType = discoverNanoInstanceType(t, c)
-	f.AMIID = discoverAMI(t, c)
-	ensureKeyPair(t, c)
-	t.Cleanup(func() { deleteKeyPair(t, c) })
-
-	harness.Phase(t, "Creating private VPC (%s), no internet gateway", privVPCCIDR)
-	vpc, err := c.EC2.CreateVpc(&ec2.CreateVpcInput{CidrBlock: aws.String(privVPCCIDR)})
-	require.NoError(t, err)
-	f.VPCID = aws.StringValue(vpc.Vpc.VpcId)
-	t.Cleanup(func() { deleteVPC(t, c, f) })
-
-	sn, err := c.EC2.CreateSubnet(&ec2.CreateSubnetInput{
-		VpcId:     aws.String(f.VPCID),
-		CidrBlock: aws.String(privSubnetCIDR),
-	})
-	require.NoError(t, err)
-	f.SubnetID = aws.StringValue(sn.Subnet.SubnetId)
-	t.Cleanup(func() { deleteSubnet(t, c, f) })
-
-	sgOut, err := c.EC2.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{
-		Filters: []*ec2.Filter{
-			{Name: aws.String("vpc-id"), Values: []*string{aws.String(f.VPCID)}},
-			{Name: aws.String("group-name"), Values: []*string{aws.String("default")}},
-		},
-	})
-	require.NoError(t, err)
-	require.NotEmpty(t, sgOut.SecurityGroups, "VPC default SG missing")
-	f.SecurityGroup = aws.StringValue(sgOut.SecurityGroups[0].GroupId)
-	_, err = c.EC2.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{
-		GroupId: aws.String(f.SecurityGroup),
-		IpPermissions: []*ec2.IpPermission{{
-			IpProtocol: aws.String("tcp"),
-			FromPort:   aws.Int64(httpPort),
-			ToPort:     aws.Int64(httpPort),
-			IpRanges:   []*ec2.IpRange{{CidrIp: aws.String(privVPCCIDR)}},
-		}},
-	})
-	require.NoError(t, err)
-
-	harness.OnFailure(t, func() { dumpDaemonLogs(t, artifacts, "private-subnet") })
-
-	harness.Phase(t, "Launching backend in the private subnet")
-	backend := runPrivateInstance(t, c, f, appUserData)
-	t.Cleanup(func() { terminateInstances(t, c, []string{backend}) })
-
-	tgArn := createTargetGroup(t, c, f, "lb-e2e-priv-tg", "HTTP", httpPort, "/index.html")
-	t.Cleanup(func() { deleteTargetGroup(t, c, tgArn) })
-	registerTargets(t, c, tgArn, []string{backend})
-	t.Cleanup(func() { deregisterTargets(t, c, tgArn, []string{backend}) })
-
-	label := "ALB internal private subnet"
-	lb, _ := createActiveLB(t, c, f, "lb-e2e-priv", "application", "internal", "HTTP", httpPort, tgArn, label, []string{f.SecurityGroup})
-	harness.WaitForTargetsHealthy(t, c, tgArn, 1, label, 2*time.Minute)
-
-	eni := lbENI(t, c, "app", lb)
-	assert.Empty(t, publicIP(eni), "%s must not have a public IP", label)
-	lbIP := privateIP(eni)
-	require.NotEmpty(t, lbIP, "%s needs a private IP", label)
-
-	harness.Phase(t, "Probing %s from a client inside the VPC", lbIP)
-	client := runPrivateInstance(t, c, f, privateProbeUserData(lbIP))
-	t.Cleanup(func() { terminateInstances(t, c, []string{client}) })
-
-	var served int
-	harness.OnFailure(t, func() {
-		harness.DumpInstanceConsole(t, c, client, artifacts, "private-client-console.log")
-	})
-	harness.Eventually(t, func() bool {
-		out, err := harness.InstanceConsole(c, client)
-		if err != nil {
-			return false
-		}
-		served = strings.Count(out, "LBPROBE ok")
-		return strings.Contains(out, "LBPROBE done")
-	}, 5*time.Minute, 5*time.Second, "client never reported its probe results on the console")
-	assert.Equal(t, privProbes, served, "%s: every in-VPC request must be served", label)
-}
-
-func runPrivateInstance(t *testing.T, c *harness.AWSClient, f *sharedFixture, userData string) string {
-	t.Helper()
-	out, err := c.EC2.RunInstances(&ec2.RunInstancesInput{
-		ImageId:      aws.String(f.AMIID),
-		InstanceType: aws.String(f.InstanceType),
-		KeyName:      aws.String(lbKeyName),
-		SubnetId:     aws.String(f.SubnetID),
-		MinCount:     aws.Int64(1),
-		MaxCount:     aws.Int64(1),
-		UserData:     aws.String(base64Encode(userData)),
-	})
-	require.NoError(t, err, "run-instances in private subnet")
-	require.NotEmpty(t, out.Instances)
-	id := aws.StringValue(out.Instances[0].InstanceId)
-	harness.WaitForInstanceRunning(t, c, id, 120*time.Second)
-	t.Logf("private instance: %s", id)
-	return id
-}
-
-// privateProbeUserData returns a boot script that requests the LB privProbes times and
-// writes one console line per served request, then a terminating line.
-func privateProbeUserData(lbIP string) string {
-	return fmt.Sprintf(`#!/bin/bash
-for i in $(seq 1 %d); do
-  for try in $(seq 1 30); do
-    if curl -sf --max-time 5 http://%s/index.html | grep -q instance_id; then
-      echo "LBPROBE ok $i" > /dev/console
-      break
-    fi
-    sleep 5
-  done
-done
-echo "LBPROBE done" > /dev/console
-`, privProbes, lbIP)
-}