gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-22 04:07:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' of github.com:mulgadc/spinifex into dev

Browse Source
This commit is contained in:
tomnewton-mulga
2026-06-17 16:45:25 +10:00
parent a396e1bd30 d88783d5bf
commit 42db84184d
110 changed files with 31384 additions and 1127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/e2e-suites.yml
+3 -3
View File
@@ -3,7 +3,7 @@
# Restructures the looped e2e.yml into env-grouped job blocks so each suite
# surfaces as its own node in the Actions graph (mulga-siv-283):
#
# single_provision ─┬─ single_suite (matrix: single, cert, lb, eks)
# single_provision ─┬─ single_suite (matrix: single, cert, lb, eks, iam)
# └─ single_teardown (if: always)
# multi_provision ─┬─ multi_suite (matrix: multinode)
# └─ multi_teardown (if: always)
@@ -27,7 +27,7 @@ on:
single_suites:
description: "Single-node suites to run as matrix legs"
required: false
default: "single cert lb eks"
default: "single cert lb eks iam"
run_multinode:
description: "Also run the multinode env block"
type: boolean
@@ -114,7 +114,7 @@ jobs:
- name: Compute suite matrix
id: suites
run: |
LIST="${{ inputs.single_suites || 'single cert lb eks' }}"
LIST="${{ inputs.single_suites || 'single cert lb eks iam' }}"
echo "list=${LIST}" >> "$GITHUB_OUTPUT"
JSON=$(printf '%s\n' $LIST | jq -R . | jq -cs .)
echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
.github/workflows/e2e.yml
+10 -14
View File
@@ -14,13 +14,10 @@ on:
- .github/workflows/e2e.yml
workflow_dispatch:
inputs:
sha:
description: "PR head SHA to report status on"
required: false
suites:
description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks')"
description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks e2e-iam')"
required: false
default: "e2e-single e2e-multinode e2e-cert e2e-lb"
default: "e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam"
test_run:
description: "Optional regex for -test.run (e.g. '^TestMultinodeIPSec$'). Empty = run all tests in suite."
required: false
@@ -32,7 +29,7 @@ permissions:
env:
GIT_BRANCH: ${{ github.ref_name }}
SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb' }}
SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam' }}
TEST_RUN: ${{ inputs.test_run || '' }}
jobs:
@@ -78,7 +75,7 @@ jobs:
# suites='e2e-multinode' only). Push triggers and the default SUITES list
# both include single/cert/lb so the inputs.suites null-check keeps the
# job running on plain PR pushes.
if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-lb') || contains(github.event.inputs.suites, 'e2e-eks')
if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-lb') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-iam')
runs-on: [self-hosted, ci-single]
timeout-minutes: 45
env:
@@ -593,12 +590,15 @@ jobs:
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
# single + iam are single-node-only suites — their fixtures skip
# unless SPINIFEX_MODE=single, so building/running them on multinode
# is wasted work.
# eks is excluded on multinode: EKS KV is a clustered (Raft) JetStream
# stream and create-cluster needs an elected leader, but multinode
# JetStream clustering isn't solid yet (mulga-omq/z2z/5yg). EKS is
# validated single-node only for now; re-enable here once clustered
# JetStream + multinode EKS are a deliberate target.
SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(single|eks)$' | tr '\n' ' ')
SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(single|eks|iam)$' | tr '\n' ' ')
if [ -z "$(echo "${SUITES_BUILD}" | tr -d ' ')" ]; then
echo "no multinode-eligible suites requested (SUITES=${SUITES}); nothing to build"
exit 0
@@ -681,7 +681,7 @@ jobs:
id: e2e
run: |
set +e
SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(single|eks)$' | tr '\n' ' ')
SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(single|eks|iam)$' | tr '\n' ' ')
if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
echo "no multinode-eligible suites requested (SUITES=${SUITES}); skipping"
echo "rc=0" >> "$GITHUB_OUTPUT"
@@ -892,12 +892,8 @@ jobs:
- name: Report Status to PR
env:
GH_TOKEN: ${{ github.token }}
REPORT_SHA: ${{ inputs.sha || github.sha }}
REPORT_SHA: ${{ github.sha }}
run: |
if [[ -z "${REPORT_SHA}" ]]; then
echo "::error::no SHA available (inputs.sha and github.sha both empty)"
exit 1
fi
single_ok=true; multi_ok=true
case "${{ needs.e2e_single.result }}" in success|skipped) ;; *) single_ok=false ;; esac
case "${{ needs.e2e_real_multi.result }}" in success|skipped) ;; *) multi_ok=false ;; esac
docs/COMMANDS.md
+1 -1
View File
@@ -113,7 +113,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
| Command | Implemented Flags | Missing Flags | Status |
|---------|-------------------|---------------|--------|
| `run-instances` | `--image-id`, `--instance-type`, `--count`, `--key-name`, `--user-data`, `--subnet-id`, `--block-device-mappings` (DeviceName, VolumeSize, VolumeType, Iops, DeleteOnTermination), `--placement` (GroupName), `--iam-instance-profile` (Name/Arn) | `--security-group-ids`, `--tag-specifications`, `--dry-run`, `--client-token`, `--disable-api-termination`, `--ebs-optimized`, `--network-interfaces`, `--private-ip-address`, `--monitoring`, `--credit-specification`, `--cpu-options`, `--metadata-options`, `--launch-template`, `--hibernate-options` | **DONE** |
| `run-instances` | `--image-id`, `--instance-type`, `--count`, `--key-name`, `--user-data`, `--subnet-id`, `--security-group-ids`, `--tag-specifications` (instance-scoped), `--block-device-mappings` (DeviceName, VolumeSize, VolumeType, Iops, DeleteOnTermination), `--placement` (GroupName), `--iam-instance-profile` (Name/Arn) | `--dry-run`, `--client-token`, `--disable-api-termination`, `--ebs-optimized`, `--network-interfaces`, `--private-ip-address`, `--monitoring`, `--credit-specification`, `--cpu-options`, `--metadata-options`, `--launch-template`, `--hibernate-options` | **DONE** |
| `describe-instances` | `--instance-ids`, `--filters` (instance-state-name, instance-id, instance-type, vpc-id, subnet-id, tag:*, tag-key, tag-value) | `--max-results`, `--next-token`, `--dry-run` | **DONE** |
| `start-instances` | `--instance-ids` | `--dry-run`, `--force` | **DONE** |
| `stop-instances` | `--instance-ids` | `--force`, `--hibernate`, `--dry-run` | **DONE** |
docs/service-interfaces.yaml
+8
View File
@@ -284,6 +284,14 @@ suites:
# into per-test fixture-driven coverage so the selector can pick
# subtests, not just whole suites.
e2e-iam:
path: tests/e2e/iam
covers: [awsgw, spinifex_daemon, vpcd, viperblockd, predastore]
# "Everything IAM": the IAM/STS control plane plus the instance-identity
# tests (instance-profile association + IMDS), which boot guest VMs — OVN
# via vpcd, AMI via predastore, root volume via viperblockd, lifecycle via
# spinifex_daemon. Single-mode, VM-booting leg.
e2e-ddil:
path: tests/e2e/ddil
covers: [awsgw, spinifex_daemon, vpcd, viperblockd, predastore]
docs/terraform-workbooks/eks-addons-access/README.md
+37 -13
View File
@@ -1,6 +1,6 @@
---
title: "EKS with Addons + Access-Entry RBAC"
description: "Extend a single-node EKS cluster with a managed CoreDNS addon, access-entry RBAC, and a browsable load-balanced demo app, using Terraform on Spinifex."
description: "Extend a single-node EKS cluster with a managed Argo CD addon, access-entry RBAC, and a browsable load-balanced demo app, using Terraform on Spinifex."
category: "Terraform Workbooks"
tags:
- terraform
@@ -23,7 +23,7 @@ resources:
# Terraform: EKS with Addons + Access-Entry RBAC
> Builds on the quickstart by installing a managed addon (CoreDNS) and granting a second IAM principal read-only cluster access — then deploys the same browsable, load-balanced demo app so you can see the result.
> Builds on the quickstart by installing a managed addon (Argo CD) and granting a second IAM principal read-only cluster access — then deploys the same browsable, load-balanced demo app so you can see the result.
## Table of Contents
@@ -37,7 +37,7 @@ resources:
This workbook takes the same single-node cluster (and the same browsable demo app) from [EKS Quickstart](../eks-quickstart) and adds the two things you reach for on almost any real cluster:
1. **A managed addon.** CoreDNS is installed through the EKS API (`aws_eks_addon`) rather than `kubectl apply`, so its lifecycle is managed by the cluster. The demo app is deployed *after* CoreDNS, so a healthy page also confirms in-cluster DNS is working.
1. **A managed addon.** Argo CD is installed through the EKS API (`aws_eks_addon`) rather than `kubectl apply`, so its lifecycle is managed by the cluster. Spinifex stages the addon's manifests host-side, then the worker renders them into the K3s auto-deploy dir. The demo app is deployed *after* the addon reaches `ACTIVE`, so a healthy page also confirms the addon delivery pipeline worked end to end.
2. **Access-entry RBAC.** A second IAM role is granted read-only access via an `aws_eks_access_entry` plus an `aws_eks_access_policy_association` bound to the AWS-managed `AmazonEKSViewPolicy`. This is the modern access-entry path (`authentication_mode = "API"`) — no `aws-auth` ConfigMap.
When `apply` finishes, open the **`demo_url`** output and refresh — the `nginxdemos/hello` page reports which pod served each request.
@@ -60,15 +60,15 @@ When `apply` finishes, open the **`demo_url`** output and refresh — the `nginx
| IAM Role | `eks-addons-viewer` | Stand-in principal granted read-only access |
| EKS Cluster | `eks-addons` | Public endpoint, API auth mode |
| Node Group | `default` | One `t3.medium` worker |
| Addon | `coredns` | Cluster DNS, managed via the EKS API |
| Addon | `argocd` | GitOps CD, managed via the EKS API |
| Access Entry + Assoc. | `eks-addons-viewer` → `AmazonEKSViewPolicy` | Read-only, cluster scope |
| SG Ingress Rule | `eks-addons-demo-nodeport` | Opens the demo NodePort on the worker SG |
| K8s Deployment + Service | `hello` | `nginxdemos/hello`, 2 replicas, NodePort |
**Spinifex specifics**
- Addon catalog is limited. Spinifex currently serves `coredns`, `aws-load-balancer-controller` and `spinifex-noop`. There is no `vpc-cni`, `kube-proxy`, `ebs-csi` or `metrics-server`.
- **Leave `addon_version` unset.** Spinifex defaults an unset version to its catalog default (`coredns` 1.11.1). Pinning is a dead end: the AWS provider rejects a bare `1.11.1` (it requires the `v`-prefixed `v1.11.1-eksbuild.N` form), while Spinifex's catalog only accepts the bare `1.11.1` — so no single string satisfies both.
- Addon catalog is limited. Spinifex currently serves `argocd`, `coredns`, `aws-load-balancer-controller` and `spinifex-noop`. There is no `vpc-cni`, `kube-proxy`, `ebs-csi` or `metrics-server`. Only addons with a baked bundle shipped in the `eks-node` AMI install successfully — `argocd` and `spinifex-noop` are the bundles shipped today.
- **Leave `addon_version` unset.** Spinifex defaults an unset version to its catalog default (`argocd` 3.0.23). Pinning is a dead end: the AWS provider rejects a bare `3.0.23` (it requires the `v`-prefixed `v3.0.23-eksbuild.N` form), while Spinifex's catalog only accepts the bare `3.0.23` — so no single string satisfies both.
- Cluster access policies are the four AWS-managed `cluster-access-policy/Amazon EKS{ClusterAdmin,Admin,Edit,View}Policy` ARNs.
- The worker SG is auto-managed and the worker AMI is always the `eks-node` image; the NodePort is exposed by adding one rule to the looked-up worker SG.
@@ -95,18 +95,32 @@ Or create a `main.tf` file and paste the full configuration below.
### Step 2. Deploy
The cluster and the demo app are **two root modules with separate state**. Apply the cluster first:
```bash
export AWS_PROFILE=spinifex
tofu init
tofu apply
```
Allow a few minutes for the cluster to reach `ACTIVE` and the worker to register. CoreDNS installs onto the worker, then Terraform deploys the demo app.
Allow a few minutes for the cluster to reach `ACTIVE`, the worker to register, and Argo CD to install. Then deploy the demo app from the nested `workloads/` module:
> Keep `AWS_PROFILE=spinifex` exported for the whole `apply` — the Kubernetes provider runs `aws eks get-token` against the Spinifex STS endpoint.
```bash
cd workloads
tofu init
tofu apply
```
> **Why two modules?** The Kubernetes provider in `workloads/` reads the cluster endpoint from a live `data "aws_eks_cluster"` source, so it's only ever configured while the cluster exists. Keeping it out of the cluster module means `destroy` never tries to refresh a workload against a cluster that's already gone — the failure mode where the provider falls back to `http://localhost:80` and reports `connection refused`. Always destroy `workloads/` before the cluster.
> Keep `AWS_PROFILE=spinifex` exported for both applies — the Kubernetes provider runs `aws eks get-token` against the Spinifex STS endpoint.
<!-- INCLUDE: workloads/main.tf lang:hcl -->
### Step 3. Open the Demo
Run from the cluster module directory (`cd ..` if you're still in `workloads/`):
```bash
tofu output demo_url
```
@@ -117,7 +131,7 @@ Open it in a browser and refresh — the `nginxdemos/hello` page shows the pod t
```bash
aws eks list-addons --cluster-name eks-addons
aws eks describe-addon --cluster-name eks-addons --addon-name coredns --query 'addon.status'
aws eks describe-addon --cluster-name eks-addons --addon-name argocd --query 'addon.status'
aws eks list-access-entries --cluster-name eks-addons
aws eks list-associated-access-policies \
@@ -129,7 +143,12 @@ The viewer principal is bound to `AmazonEKSViewPolicy` at cluster scope — it c
### Cleanup
Destroy in reverse — the demo app first (while the cluster is still up), then the cluster:
```bash
cd workloads
tofu destroy
cd ..
tofu destroy
```
@@ -145,15 +164,16 @@ aws ec2 describe-security-groups --filters "Name=group-name,Values=eks-cluster-e
--query 'SecurityGroups[0].IpPermissions'
```
If the demo pods are stuck `ContainerCreating` with DNS errors, confirm CoreDNS is healthy first:
If the addon never reaches `ACTIVE`, confirm Argo CD's pods came up on the worker:
```bash
kubectl -n kube-system get deploy coredns
kubectl -n argocd get pods
kubectl -n argocd get deploy
```
### Addon Stuck in CREATING / DEGRADED
CoreDNS needs a schedulable worker. Confirm the node group is `ACTIVE` and the node is `Ready`:
Argo CD needs a schedulable worker with headroom (it runs ~6 pods). Confirm the node group is `ACTIVE` and the node is `Ready`:
```bash
aws eks describe-nodegroup --cluster-name eks-addons --nodegroup-name default --query 'nodegroup.status'
@@ -162,7 +182,11 @@ kubectl get nodes
### Addon Version Rejected
This workbook leaves `addon_version` unset so Spinifex picks its catalog default. If you add an explicit `addon_version`, expect a wall: the AWS provider requires the `v`-prefixed `v1.11.1-eksbuild.N` form, but Spinifex's catalog only knows the bare `1.11.1` — a value that passes one is rejected by the other. Prefer omitting the field.
This workbook leaves `addon_version` unset so Spinifex picks its catalog default. If you add an explicit `addon_version`, expect a wall: the AWS provider requires the `v`-prefixed `v3.0.23-eksbuild.N` form, but Spinifex's catalog only knows the bare `3.0.23` — a value that passes one is rejected by the other. Prefer omitting the field.
### Addon Fails With "no baked bundle"
Spinifex only installs addons whose manifests are baked into the `eks-node` AMI. If `describe-addon` reports `CREATE_FAILED` with `no baked bundle for version <v>`, the running worker image predates that addon. Rebuild and republish the `eks-node` image so the bundle ships, then recreate the addon.
### Access Entry Not Taking Effect
docs/terraform-workbooks/eks-addons-access/main.tf
+16 -87
View File
@@ -3,13 +3,13 @@
# Builds on eks-quickstart: same single-worker cluster and browsable demo app,
# but adds two things you reach for on a real cluster:
#
# * a managed addon (CoreDNS) installed through the EKS API, and
# * a managed addon (Argo CD) installed through the EKS API, and
# * a second IAM principal granted scoped, read-only access via an EKS access
# entry + access-policy association (authentication_mode = "API").
#
# After `apply`, open the demo_url output — the page reports the pod that
# served it, so a refresh shows requests landing on different replicas. CoreDNS
# being healthy is what lets in-cluster name resolution work.
# served it, so a refresh shows requests landing on different replicas. The Argo
# CD addon reaching ACTIVE is what gates the demo app rollout below.
#
# Usage:
# cd spinifex/docs/terraform-workbooks/eks-addons-access
@@ -25,10 +25,6 @@ terraform {
source = "hashicorp/aws"
version = ">= 5.40, < 6.0"
}
kubernetes = {
source = "hashicorp/kubernetes"
version = ">= 2.20"
}
}
}
@@ -61,11 +57,6 @@ variable "node_port" {
default = 30080
}
variable "replicas" {
type = number
default = 2
}
variable "browse_cidr" {
type = string
default = "0.0.0.0/0"
@@ -97,17 +88,6 @@ provider "aws" {
skip_region_validation = true
}
provider "kubernetes" {
host = aws_eks_cluster.this.endpoint
cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority[0].data)
exec {
api_version = "client.authentication.k8s.io/v1beta1"
command = "aws"
args = ["eks", "get-token", "--cluster-name", aws_eks_cluster.this.name, "--region", var.region]
}
}
data "aws_availability_zones" "available" {
state = "available"
}
@@ -308,22 +288,22 @@ resource "aws_eks_node_group" "default" {
}
# ---------------------------------------------------------------------------
# Addon — CoreDNS
# Addon — Argo CD
# ---------------------------------------------------------------------------
resource "aws_eks_addon" "coredns" {
resource "aws_eks_addon" "argocd" {
cluster_name = aws_eks_cluster.this.name
addon_name = "coredns"
addon_name = "argocd"
resolve_conflicts_on_create = "OVERWRITE"
# addon_version omitted on purpose. Spinifex defaults an unset version to its
# catalog default (coredns 1.11.1). The AWS provider rejects a bare "1.11.1"
# catalog default (argocd 3.0.23). The AWS provider rejects a bare "3.0.23"
# (it demands a v-prefixed form) which Spinifex's catalog would in turn
# reject — so pinning is a dead end here; let the server choose.
depends_on = [aws_eks_node_group.default]
tags = {
Name = "${var.cluster_name}-coredns"
Name = "${var.cluster_name}-argocd"
}
}
@@ -379,65 +359,6 @@ resource "aws_vpc_security_group_ingress_rule" "nodeport" {
}
}
# ---------------------------------------------------------------------------
# Demo workload — deployed after CoreDNS so name resolution is up
# ---------------------------------------------------------------------------
resource "kubernetes_deployment_v1" "hello" {
metadata {
name = "hello"
namespace = "default"
labels = { app = "hello" }
}
spec {
replicas = var.replicas
selector {
match_labels = { app = "hello" }
}
template {
metadata {
labels = { app = "hello" }
}
spec {
container {
name = "hello"
image = "nginxdemos/hello"
port {
container_port = 80
}
}
}
}
}
depends_on = [aws_eks_addon.coredns]
}
resource "kubernetes_service_v1" "hello" {
metadata {
name = "hello"
namespace = "default"
}
spec {
selector = { app = "hello" }
type = "NodePort"
port {
port = 80
target_port = 80
node_port = var.node_port
}
}
depends_on = [kubernetes_deployment_v1.hello]
}
data "aws_instances" "workers" {
instance_tags = {
"spinifex:eks-cluster" = aws_eks_cluster.this.name
@@ -454,6 +375,14 @@ output "cluster_name" {
value = aws_eks_cluster.this.name
}
output "region" {
value = var.region
}
output "node_port" {
value = var.node_port
}
output "viewer_principal_arn" {
value = aws_iam_role.viewer.arn
}
docs/terraform-workbooks/eks-addons-access/workloads/main.tf
+138
View File
@@ -0,0 +1,138 @@
# Demo workload for eks-addons-access — separate root module / state.
#
# Kept apart from the cluster root module on purpose: the kubernetes provider
# below reads the cluster endpoint from a *live* data source, so it is only ever
# configured while the cluster exists. Destroy this module before the parent and
# the provider never falls back to localhost.
#
# Usage:
# cd spinifex/docs/terraform-workbooks/eks-addons-access
# tofu init && tofu apply # parent: cluster + infra
# cd workloads && tofu init && tofu apply # this module: demo app
# # teardown is the reverse — destroy here first, then the parent.
terraform {
required_version = ">= 1.6.0"
required_providers {
aws = {
source = "hashicorp/aws"
version = ">= 5.40, < 6.0"
}
kubernetes = {
source = "hashicorp/kubernetes"
version = ">= 2.20"
}
}
}
variable "spinifex_endpoint" {
type = string
default = "https://127.0.0.1:9999"
}
variable "replicas" {
type = number
default = 2
}
provider "aws" {
region = data.terraform_remote_state.infra.outputs.region
endpoints {
ec2 = var.spinifex_endpoint
iam = var.spinifex_endpoint
sts = var.spinifex_endpoint
eks = var.spinifex_endpoint
}
skip_credentials_validation = true
skip_metadata_api_check = true
skip_requesting_account_id = true
skip_region_validation = true
}
# Cluster identity comes from the parent module's state; the live endpoint/CA
# come from a data source so the provider is only configured while the cluster
# is up.
data "terraform_remote_state" "infra" {
backend = "local"
config = {
path = "../terraform.tfstate"
}
}
locals {
cluster_name = data.terraform_remote_state.infra.outputs.cluster_name
region = data.terraform_remote_state.infra.outputs.region
node_port = data.terraform_remote_state.infra.outputs.node_port
}
data "aws_eks_cluster" "this" {
name = local.cluster_name
}
provider "kubernetes" {
host = data.aws_eks_cluster.this.endpoint
cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data)
exec {
api_version = "client.authentication.k8s.io/v1beta1"
command = "aws"
args = ["eks", "get-token", "--cluster-name", local.cluster_name, "--region", local.region]
}
}
resource "kubernetes_deployment_v1" "hello" {
metadata {
name = "hello"
namespace = "default"
labels = { app = "hello" }
}
spec {
replicas = var.replicas
selector {
match_labels = { app = "hello" }
}
template {
metadata {
labels = { app = "hello" }
}
spec {
container {
name = "hello"
image = "nginxdemos/hello"
port {
container_port = 80
}
}
}
}
}
}
resource "kubernetes_service_v1" "hello" {
metadata {
name = "hello"
namespace = "default"
}
spec {
selector = { app = "hello" }
type = "NodePort"
port {
port = 80
target_port = 80
node_port = local.node_port
}
}
depends_on = [kubernetes_deployment_v1.hello]
}
docs/terraform-workbooks/eks-https-ingress/README.md
+21 -2
View File
@@ -108,15 +108,29 @@ Or create a `main.tf` file and paste the full configuration below.
### Step 2. Deploy
The cluster (with its ALB and ACM cert) and the demo app are **two root modules with separate state**. Apply the cluster first:
```bash
export AWS_PROFILE=spinifex
tofu init
tofu apply
```
Expect several minutes: the control plane bootstraps, three workers launch and join over the private endpoint, the NAT gateway comes up, CoreDNS installs, the ALB is provisioned, and the Kubernetes provider rolls out the demo app.
Expect several minutes: the control plane bootstraps, three workers launch and join over the private endpoint, the NAT gateway comes up, CoreDNS installs, and the ALB is provisioned. Then deploy the demo app from the nested `workloads/` module:
> **Keep `AWS_PROFILE=spinifex` exported** for the whole `apply` — the Kubernetes provider authenticates with `aws eks get-token` against the Spinifex STS endpoint.
```bash
cd workloads
tofu init
tofu apply
```
The ALB target group already points at the workers' NodePort; once these pods are running the targets turn healthy.
> **Why two modules?** The Kubernetes provider in `workloads/` reads the cluster endpoint from a live `data "aws_eks_cluster"` source, so it's only ever configured while the cluster exists. Keeping it out of the cluster module means `destroy` never tries to refresh a workload against a cluster that's already gone — the failure mode where the provider falls back to `http://localhost:80` and reports `connection refused`. Always destroy `workloads/` before the cluster.
> **Keep `AWS_PROFILE=spinifex` exported** for both applies — the Kubernetes provider authenticates with `aws eks get-token` against the Spinifex STS endpoint.
<!-- INCLUDE: workloads/main.tf lang:hcl -->
> **Tighten access in production.** `api_public_access_cidr` and `alb_ingress_cidr` both default to `0.0.0.0/0`. Set them to your own CIDR:
>
@@ -158,7 +172,12 @@ aws elbv2 describe-target-health --target-group-arn "$TG_ARN"
### Cleanup
Destroy in reverse — the demo app first (while the cluster is still up), then the cluster:
```bash
cd workloads
tofu destroy
cd ..
tofu destroy
```
docs/terraform-workbooks/eks-https-ingress/main.tf
+12 -85
View File
@@ -40,10 +40,6 @@ terraform {
source = "hashicorp/tls"
version = ">= 4.0"
}
kubernetes = {
source = "hashicorp/kubernetes"
version = ">= 2.20"
}
}
}
@@ -127,17 +123,6 @@ provider "aws" {
skip_region_validation = true
}
provider "kubernetes" {
host = aws_eks_cluster.this.endpoint
cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority[0].data)
exec {
api_version = "client.authentication.k8s.io/v1beta1"
command = "aws"
args = ["eks", "get-token", "--cluster-name", aws_eks_cluster.this.name, "--region", var.region]
}
}
data "aws_availability_zones" "available" {
state = "available"
}
@@ -572,76 +557,6 @@ resource "aws_vpc_security_group_ingress_rule" "nodeport_from_alb" {
}
}
# ---------------------------------------------------------------------------
# Demo workload — three replicas spread across the workers
# ---------------------------------------------------------------------------
resource "kubernetes_deployment_v1" "hello" {
metadata {
name = "hello"
namespace = "default"
labels = { app = "hello" }
}
spec {
replicas = var.node_desired_size
selector {
match_labels = { app = "hello" }
}
template {
metadata {
labels = { app = "hello" }
}
spec {
container {
name = "hello"
image = "nginxdemos/hello"
port {
container_port = 80
}
}
# Spread one replica per node so a refresh visibly hits different nodes.
topology_spread_constraint {
max_skew = 1
topology_key = "kubernetes.io/hostname"
when_unsatisfiable = "ScheduleAnyway"
label_selector {
match_labels = { app = "hello" }
}
}
}
}
}
depends_on = [aws_eks_addon.coredns]
}
resource "kubernetes_service_v1" "hello" {
metadata {
name = "hello"
namespace = "default"
}
spec {
selector = { app = "hello" }
type = "NodePort"
port {
port = 80
target_port = 80
node_port = var.node_port
}
}
depends_on = [kubernetes_deployment_v1.hello]
}
# ---------------------------------------------------------------------------
# Outputs
# ---------------------------------------------------------------------------
@@ -650,6 +565,18 @@ output "cluster_name" {
value = aws_eks_cluster.this.name
}
output "region" {
value = var.region
}
output "node_port" {
value = var.node_port
}
output "node_desired_size" {
value = var.node_desired_size
}
output "certificate_arn" {
value = aws_acm_certificate.ingress.arn
}
docs/terraform-workbooks/eks-https-ingress/workloads/main.tf
+148
View File
@@ -0,0 +1,148 @@
# Demo workload for eks-https-ingress — separate root module / state.
#
# Kept apart from the cluster root module on purpose: the kubernetes provider
# below reads the cluster endpoint from a *live* data source, so it is only ever
# configured while the cluster exists. Destroy this module before the parent and
# the provider never falls back to localhost.
#
# The ALB target group in the parent forwards to the workers' NodePort; this
# module is what actually answers on that port.
#
# Usage:
# cd spinifex/docs/terraform-workbooks/eks-https-ingress
# tofu init && tofu apply # parent: cluster + ALB + infra
# cd workloads && tofu init && tofu apply # this module: demo app
# # teardown is the reverse — destroy here first, then the parent.
terraform {
required_version = ">= 1.6.0"
required_providers {
aws = {
source = "hashicorp/aws"
version = ">= 5.40, < 6.0"
}
kubernetes = {
source = "hashicorp/kubernetes"
version = ">= 2.20"
}
}
}
variable "spinifex_endpoint" {
type = string
default = "https://127.0.0.1:9999"
}
provider "aws" {
region = data.terraform_remote_state.infra.outputs.region
endpoints {
ec2 = var.spinifex_endpoint
iam = var.spinifex_endpoint
sts = var.spinifex_endpoint
eks = var.spinifex_endpoint
}
skip_credentials_validation = true
skip_metadata_api_check = true
skip_requesting_account_id = true
skip_region_validation = true
}
# Cluster identity comes from the parent module's state; the live endpoint/CA
# come from a data source so the provider is only configured while the cluster
# is up.
data "terraform_remote_state" "infra" {
backend = "local"
config = {
path = "../terraform.tfstate"
}
}
locals {
cluster_name = data.terraform_remote_state.infra.outputs.cluster_name
region = data.terraform_remote_state.infra.outputs.region
node_port = data.terraform_remote_state.infra.outputs.node_port
replicas = data.terraform_remote_state.infra.outputs.node_desired_size
}
data "aws_eks_cluster" "this" {
name = local.cluster_name
}
provider "kubernetes" {
host = data.aws_eks_cluster.this.endpoint
cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data)
exec {
api_version = "client.authentication.k8s.io/v1beta1"
command = "aws"
args = ["eks", "get-token", "--cluster-name", local.cluster_name, "--region", local.region]
}
}
resource "kubernetes_deployment_v1" "hello" {
metadata {
name = "hello"
namespace = "default"
labels = { app = "hello" }
}
spec {
replicas = local.replicas
selector {
match_labels = { app = "hello" }
}
template {
metadata {
labels = { app = "hello" }
}
spec {
container {
name = "hello"
image = "nginxdemos/hello"
port {
container_port = 80
}
}
# Spread one replica per node so a refresh visibly hits different nodes.
topology_spread_constraint {
max_skew = 1
topology_key = "kubernetes.io/hostname"
when_unsatisfiable = "ScheduleAnyway"
label_selector {
match_labels = { app = "hello" }
}
}
}
}
}
}
resource "kubernetes_service_v1" "hello" {
metadata {
name = "hello"
namespace = "default"
}
spec {
selector = { app = "hello" }
type = "NodePort"
port {
port = 80
target_port = 80
node_port = local.node_port
}
}
depends_on = [kubernetes_deployment_v1.hello]
}
docs/terraform-workbooks/eks-quickstart/README.md
+21 -2
View File
@@ -90,18 +90,32 @@ Or create a `main.tf` file and paste the full configuration below.
### Step 2. Deploy
The cluster and the demo app are **two root modules with separate state**. Apply the cluster first:
```bash
export AWS_PROFILE=spinifex
tofu init
tofu apply
```
This single `apply` does the whole job: it creates the cluster (which bootstraps a control-plane VM and brings up k3s — a few minutes in `CREATING`), launches the worker, opens the NodePort, then deploys the demo app and waits for the pods to roll out.
This creates the cluster (which bootstraps a control-plane VM and brings up k3s — a few minutes in `CREATING`), launches the worker, and opens the NodePort. Once it's `ACTIVE`, deploy the demo app from the nested `workloads/` module:
> **Same profile for the Kubernetes provider.** The Kubernetes provider authenticates by shelling out to `aws eks get-token`, which has to reach the Spinifex STS endpoint. Keep `AWS_PROFILE=spinifex` exported for the whole `apply`.
```bash
cd workloads
tofu init
tofu apply
```
> **Why two modules?** The Kubernetes provider in `workloads/` reads the cluster endpoint from a live `data "aws_eks_cluster"` source, so it's only ever configured while the cluster exists. Keeping it out of the cluster module means `destroy` never tries to refresh a workload against a cluster that's already gone — the failure mode where the provider falls back to `http://localhost:80` and reports `connection refused`. Always destroy `workloads/` before the cluster.
> **Same profile for the Kubernetes provider.** The Kubernetes provider authenticates by shelling out to `aws eks get-token`, which has to reach the Spinifex STS endpoint. Keep `AWS_PROFILE=spinifex` exported for both applies.
<!-- INCLUDE: workloads/main.tf lang:hcl -->
### Step 3. Open the Demo
Run from the cluster module directory (`cd ..` if you're still in `workloads/`):
```bash
tofu output demo_url
```
@@ -120,7 +134,12 @@ kubectl get pods -o wide
### Cleanup
Destroy in reverse — the demo app first (while the cluster is still up), then the cluster:
```bash
cd workloads
tofu destroy
cd ..
tofu destroy
```
docs/terraform-workbooks/eks-quickstart/main.tf
+8 -85
View File
@@ -27,10 +27,6 @@ terraform {
source = "hashicorp/aws"
version = ">= 5.40, < 6.0"
}
kubernetes = {
source = "hashicorp/kubernetes"
version = ">= 2.20"
}
}
}
@@ -65,12 +61,6 @@ variable "node_port" {
description = "NodePort the demo Service is published on"
}
variable "replicas" {
type = number
default = 2
description = "Demo app replicas; refresh the page to see requests land on different pods"
}
variable "browse_cidr" {
type = string
default = "0.0.0.0/0"
@@ -103,19 +93,6 @@ provider "aws" {
skip_region_validation = true
}
# Authenticates to the cluster with the same `aws eks get-token` exec flow the
# generated kubeconfig uses, so the Kubernetes provider can deploy the demo app.
provider "kubernetes" {
host = aws_eks_cluster.this.endpoint
cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority[0].data)
exec {
api_version = "client.authentication.k8s.io/v1beta1"
command = "aws"
args = ["eks", "get-token", "--cluster-name", aws_eks_cluster.this.name, "--region", var.region]
}
}
# ---------------------------------------------------------------------------
# Data sources
# ---------------------------------------------------------------------------
@@ -336,68 +313,6 @@ resource "aws_vpc_security_group_ingress_rule" "nodeport" {
}
}
# ---------------------------------------------------------------------------
# Demo workload — deployed onto the cluster by Terraform
#
# nginxdemos/hello renders a page showing which pod served the request; with
# multiple replicas, refreshing the demo_url alternates between them.
# ---------------------------------------------------------------------------
resource "kubernetes_deployment_v1" "hello" {
metadata {
name = "hello"
namespace = "default"
labels = { app = "hello" }
}
spec {
replicas = var.replicas
selector {
match_labels = { app = "hello" }
}
template {
metadata {
labels = { app = "hello" }
}
spec {
container {
name = "hello"
image = "nginxdemos/hello"
port {
container_port = 80
}
}
}
}
}
depends_on = [aws_eks_node_group.default]
}
resource "kubernetes_service_v1" "hello" {
metadata {
name = "hello"
namespace = "default"
}
spec {
selector = { app = "hello" }
type = "NodePort"
port {
port = 80
target_port = 80
node_port = var.node_port
}
}
depends_on = [kubernetes_deployment_v1.hello]
}
# ---------------------------------------------------------------------------
# Discover the worker's public IP for the demo URL
# ---------------------------------------------------------------------------
@@ -418,6 +333,14 @@ output "cluster_name" {
value = aws_eks_cluster.this.name
}
output "region" {
value = var.region
}
output "node_port" {
value = var.node_port
}
output "demo_url" {
value = "http://${data.aws_instances.workers.public_ips[0]}:${var.node_port}"
description = "Open in a browser; refresh to see different pods answer"
docs/terraform-workbooks/eks-quickstart/workloads/main.tf
+144
View File
@@ -0,0 +1,144 @@
# Demo workload for eks-quickstart — separate root module / state.
#
# Kept apart from the cluster root module on purpose: the kubernetes provider
# below reads the cluster endpoint from a *live* data source, so it is only ever
# configured while the cluster exists. Destroy this module before the parent and
# the provider never falls back to localhost.
#
# Usage:
# cd spinifex/docs/terraform-workbooks/eks-quickstart
# tofu init && tofu apply # parent: cluster + infra
# cd workloads && tofu init && tofu apply # this module: demo app
# # teardown is the reverse — destroy here first, then the parent.
terraform {
required_version = ">= 1.6.0"
required_providers {
aws = {
source = "hashicorp/aws"
version = ">= 5.40, < 6.0"
}
kubernetes = {
source = "hashicorp/kubernetes"
version = ">= 2.20"
}
}
}
variable "spinifex_endpoint" {
type = string
default = "https://127.0.0.1:9999"
description = "Spinifex AWS gateway endpoint"
}
variable "replicas" {
type = number
default = 2
description = "Demo app replicas; refresh the page to see requests land on different pods"
}
provider "aws" {
region = data.terraform_remote_state.infra.outputs.region
endpoints {
ec2 = var.spinifex_endpoint
iam = var.spinifex_endpoint
sts = var.spinifex_endpoint
eks = var.spinifex_endpoint
}
skip_credentials_validation = true
skip_metadata_api_check = true
skip_requesting_account_id = true
skip_region_validation = true
}
# Cluster identity comes from the parent module's state; the live endpoint/CA
# come from a data source so the provider is only configured while the cluster
# is up.
data "terraform_remote_state" "infra" {
backend = "local"
config = {
path = "../terraform.tfstate"
}
}
locals {
cluster_name = data.terraform_remote_state.infra.outputs.cluster_name
region = data.terraform_remote_state.infra.outputs.region
node_port = data.terraform_remote_state.infra.outputs.node_port
}
data "aws_eks_cluster" "this" {
name = local.cluster_name
}
# Authenticates with the same `aws eks get-token` exec flow the generated
# kubeconfig uses, so the Kubernetes provider can deploy the demo app.
provider "kubernetes" {
host = data.aws_eks_cluster.this.endpoint
cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data)
exec {
api_version = "client.authentication.k8s.io/v1beta1"
command = "aws"
args = ["eks", "get-token", "--cluster-name", local.cluster_name, "--region", local.region]
}
}
# nginxdemos/hello renders a page showing which pod served the request; with
# multiple replicas, refreshing the demo_url alternates between them.
resource "kubernetes_deployment_v1" "hello" {
metadata {
name = "hello"
namespace = "default"
labels = { app = "hello" }
}
spec {
replicas = var.replicas
selector {
match_labels = { app = "hello" }
}
template {
metadata {
labels = { app = "hello" }
}
spec {
container {
name = "hello"
image = "nginxdemos/hello"
port {
container_port = 80
}
}
}
}
}
}
resource "kubernetes_service_v1" "hello" {
metadata {
name = "hello"
namespace = "default"
}
spec {
selector = { app = "hello" }
type = "NodePort"
port {
port = 80
target_port = 80
node_port = local.node_port
}
}
depends_on = [kubernetes_deployment_v1.hello]
}
scripts/images/eks-node/addons/argocd/3.0.23/argocd.yaml
+26742
View File
File diff suppressed because it is too large Load Diff
scripts/images/eks-node/coredns/coredns-mulga.yaml
+232
View File
@@ -0,0 +1,232 @@
# Mulga-owned CoreDNS for the eks-node AMI. Replaces k3s's bundled CoreDNS
# (suppressed via a coredns.yaml.skip marker staged by k3s.initd) so DNS is
# node-local and never depends on the cross-node pod overlay: a DaemonSet runs
# CoreDNS on every node and kube-dns uses internalTrafficPolicy=Local, so each
# node resolves through its own CoreDNS. In the managed-control-plane topology
# the worker<->CP overlay cannot cross the VPC boundary, so a CP-resident
# CoreDNS is unreachable from worker pods.
#
# Faithful to k3s v1.32.5+k3s1 (image, Corefile, RBAC, Service) except the two
# locality changes above. Refresh image/Corefile when bumping K3S_VERSION in
# setup.sh.
apiVersion: v1
kind: ServiceAccount
metadata:
name: coredns
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
kubernetes.io/bootstrapping: rbac-defaults
addonmanager.kubernetes.io/mode: Reconcile
name: system:coredns
rules:
- apiGroups:
- ""
resources:
- endpoints
- services
- pods
- namespaces
verbs:
- list
- watch
- apiGroups:
- discovery.k8s.io
resources:
- endpointslices
verbs:
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
labels:
kubernetes.io/bootstrapping: rbac-defaults
addonmanager.kubernetes.io/mode: EnsureExists
name: system:coredns
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:coredns
subjects:
- kind: ServiceAccount
name: coredns
namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
name: coredns
namespace: kube-system
labels:
addonmanager.kubernetes.io/mode: EnsureExists
data:
Corefile: |
.:53 {
errors
health
ready
kubernetes cluster.local in-addr.arpa ip6.arpa {
pods insecure
fallthrough in-addr.arpa ip6.arpa
}
hosts /etc/coredns/NodeHosts {
ttl 60
reload 15s
fallthrough
}
prometheus :9153
forward . /etc/resolv.conf
cache 30
loop
reload
loadbalance
import /etc/coredns/custom/*.override
}
import /etc/coredns/custom/*.server
NodeHosts: ""
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: coredns
namespace: kube-system
labels:
k8s-app: kube-dns
kubernetes.io/name: "CoreDNS"
spec:
selector:
matchLabels:
k8s-app: kube-dns
updateStrategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
template:
metadata:
labels:
k8s-app: kube-dns
spec:
priorityClassName: system-cluster-critical
serviceAccountName: coredns
# CriticalAddonsOnly (Exists, all effects) tolerates the CP's
# CriticalAddonsOnly=true:NoExecute taint so a CoreDNS pod also runs on
# the control-plane node; the control-plane/master tolerations cover any
# role taints. Required for full per-node coverage with internalTrafficPolicy=Local.
tolerations:
- key: CriticalAddonsOnly
operator: Exists
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
nodeSelector:
kubernetes.io/os: linux
containers:
- name: coredns
image: rancher/mirrored-coredns-coredns:1.12.1
imagePullPolicy: IfNotPresent
resources:
limits:
memory: 170Mi
requests:
cpu: 100m
memory: 70Mi
args: ["-conf", "/etc/coredns/Corefile"]
volumeMounts:
- name: config-volume
mountPath: /etc/coredns
readOnly: true
- name: custom-config-volume
mountPath: /etc/coredns/custom
readOnly: true
ports:
- containerPort: 53
name: dns
protocol: UDP
- containerPort: 53
name: dns-tcp
protocol: TCP
- containerPort: 9153
name: metrics
protocol: TCP
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- NET_BIND_SERVICE
drop:
- all
readOnlyRootFilesystem: true
livenessProbe:
httpGet:
path: /health
port: 8080
scheme: HTTP
initialDelaySeconds: 60
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
readinessProbe:
httpGet:
path: /ready
port: 8181
scheme: HTTP
dnsPolicy: Default
volumes:
- name: config-volume
configMap:
name: coredns
items:
- key: Corefile
path: Corefile
- key: NodeHosts
path: NodeHosts
- name: custom-config-volume
configMap:
name: coredns-custom
optional: true
---
apiVersion: v1
kind: Service
metadata:
name: kube-dns
namespace: kube-system
annotations:
prometheus.io/port: "9153"
prometheus.io/scrape: "true"
labels:
k8s-app: kube-dns
kubernetes.io/cluster-service: "true"
kubernetes.io/name: "CoreDNS"
spec:
selector:
k8s-app: kube-dns
clusterIP: 10.43.0.10
# Local: kube-proxy routes each node's 10.43.0.10 queries only to that node's
# CoreDNS DaemonSet pod, so DNS never crosses the worker<->CP boundary.
internalTrafficPolicy: Local
ports:
- name: dns
port: 53
protocol: UDP
targetPort: 53
- name: dns-tcp
port: 53
protocol: TCP
targetPort: 53
- name: metrics
port: 9153
protocol: TCP
targetPort: 9153
scripts/images/eks-node/k3s.initd
+14
View File
@@ -50,6 +50,20 @@ start_pre() {
fi
fi
# Node-local CoreDNS: suppress k3s's bundled CoreDNS (a single Deployment that
# tolerates the CP taint and strands itself on the control-plane node) and
# deploy the Mulga DaemonSet variant so every node resolves via its own
# CoreDNS. Worker pods cannot reach a CP-resident CoreDNS — the worker<->CP
# overlay does not cross the VPC boundary. Staged pre-start so the deploy
# controller never applies the bundled manifest. Guarded on the baked bundle:
# an older image without it keeps k3s's CoreDNS rather than losing DNS.
if [ -f /usr/share/spinifex-eks/coredns/coredns-mulga.yaml ]; then
mkdir -p /var/lib/rancher/k3s/server/manifests
: > /var/lib/rancher/k3s/server/manifests/coredns.yaml.skip
cp /usr/share/spinifex-eks/coredns/coredns-mulga.yaml \
/var/lib/rancher/k3s/server/manifests/coredns-mulga.yaml
fi
# The apiserver is configured with --authentication-token-webhook-config-file
# pointing at the kubeconfig eks-token-webhook writes at startup. `before k3s`
# orders the webhook's start but it backgrounds, so wait (bounded) for the
scripts/images/eks-node/manifest.conf
+2
View File
@@ -37,6 +37,8 @@ scripts/images/eks-node/mulga-eks-state-report.sh:/usr/local/sbin/mulga-eks-stat
scripts/images/eks-node/mulga-eks-addon-sync.initd:/etc/init.d/mulga-eks-addon-sync \
scripts/images/eks-node/mulga-eks-addon-sync.sh:/usr/local/sbin/mulga-eks-addon-sync \
scripts/images/eks-node/addons/spinifex-noop/0.1.0/noop.yaml:/usr/share/spinifex-eks/addons/spinifex-noop/0.1.0/noop.yaml \
scripts/images/eks-node/addons/argocd/3.0.23/argocd.yaml:/usr/share/spinifex-eks/addons/argocd/3.0.23/argocd.yaml \
scripts/images/eks-node/coredns/coredns-mulga.yaml:/usr/share/spinifex-eks/coredns/coredns-mulga.yaml \
scripts/images/eks-node/mulga-eks-etcd-snapshot.sh:/etc/periodic/daily/mulga-eks-etcd-snapshot"
# Only crond (etcd snapshot) + the role selector at bake. The selector adds the
scripts/images/eks-node/setup.sh
+8
View File
@@ -94,4 +94,12 @@ sed -i \
's|console=ttyS0,115200n8 console=ttyAMA0,115200n8 console=tty0|console=tty0 console=ttyAMA0,115200n8 console=ttyS0,115200n8|' \
/etc/update-extlinux.conf /boot/extlinux.conf
# Cut the boot-menu countdown from 10s to ~1s. The stock cloud image waits 10s at
# the SYSLINUX menu before auto-booting — a fixed, network-independent tax on every
# VM start. Patch the generator config (timeout in seconds) and the rendered output
# (TIMEOUT in 1/10s) so a regenerate keeps the short value. A small nonzero value is
# kept so the menu stays interruptible over serial; TIMEOUT 0 would wait forever.
sed -i 's/^timeout=.*/timeout=1/' /etc/update-extlinux.conf
sed -i 's/^TIMEOUT[[:space:]].*/TIMEOUT 10/' /boot/extlinux.conf
echo "[eks-node-setup] done"
scripts/setup.sh
+1
View File
@@ -221,6 +221,7 @@ install_apt_deps() {
pciutils \
jq curl iproute2 netcat-openbsd wget unzip xz-utils file \
ovn-central ovn-host openvswitch-switch openvswitch-ipsec strongswan-charon dhcpcd-base \
chrony \
> /dev/null
info "System dependencies installed"
spinifex/awserrors/awserrors.go
+8
View File
@@ -491,6 +491,14 @@ func ValidErrorCode(code string) string {
return ErrorServerInternal
}
// HasErrorCode reports whether s is exactly a registered AWS error code.
// Use it to decide whether an error string may be surfaced to clients verbatim
// (mapped to its real HTTP status) rather than wrapped/sanitized to 500.
func HasErrorCode(s string) bool {
_, ok := ErrorLookup[s]
return ok
}
// IsErrorCode reports whether err carries the named AWS error code.
// Uses substring matching because handlers return codes as plain strings, sometimes wrapped with %w.
func IsErrorCode(err error, code string) bool {
spinifex/awserrors/awserrors_test.go
+22
View File
@@ -70,6 +70,28 @@ func TestValidErrorCode(t *testing.T) {
}
}
func TestHasErrorCode(t *testing.T) {
tests := []struct {
name string
code string
want bool
}{
{"known code", ErrorInsufficientInstanceCapacity, true},
{"another known code", ErrorAuthFailure, true},
{"wrapped code is not exact", "launch workers: " + ErrorInsufficientInstanceCapacity, false},
{"unknown code", "CompletelyMadeUp", false},
{"empty string", "", false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := HasErrorCode(tt.code); got != tt.want {
t.Errorf("HasErrorCode(%q) = %v, want %v", tt.code, got, tt.want)
}
})
}
}
// TestIsNotFound covers the destroy-orchestration tolerance helper: it matches
// any AWS canonical InvalidX.NotFound (so teardown converges on already-gone
// resources) and only those — a nil error or a non-NotFound failure is not
spinifex/daemon/daemon_handlers_test.go
-16
View File
@@ -16,7 +16,6 @@ import (
awss3 "github.com/aws/aws-sdk-go/service/s3"
"github.com/mulgadc/spinifex/spinifex/awserrors"
"github.com/mulgadc/spinifex/spinifex/config"
handlers_ec2_account "github.com/mulgadc/spinifex/spinifex/handlers/ec2/account"
handlers_ec2_eigw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eigw"
handlers_ec2_eip "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eip"
@@ -1125,21 +1124,6 @@ func TestHandleEC2CreateImage_MalformedJSON(t *testing.T) {
assert.Equal(t, awserrors.ErrorValidationError, errResp["Code"])
}
// --- SetConfigPath test ---
func TestSetConfigPath(t *testing.T) {
clusterCfg := &config.ClusterConfig{
Node: "node-1",
Nodes: map[string]config.Config{"node-1": {BaseDir: "/tmp"}},
}
daemon, err := NewDaemon(clusterCfg)
require.NoError(t, err)
assert.Empty(t, daemon.configPath)
daemon.SetConfigPath("/etc/spinifex/config.toml")
assert.Equal(t, "/etc/spinifex/config.toml", daemon.configPath)
}
// --- handleEC2StartStoppedInstance tests ---
func TestHandleEC2StartStoppedInstance_MissingInstance(t *testing.T) {
spinifex/daemon/daemon_test.go
+98 -117
View File
@@ -759,40 +759,6 @@ func TestDaemon_BootAllocation(t *testing.T) {
assert.Equal(t, expectedMem, daemon.resourceMgr.allocatedMem)
}
// TestStopInstance_Deallocation verifies that stopping an instance deallocates resources
func TestStopInstance_Deallocation(t *testing.T) {
clusterCfg := &config.ClusterConfig{
Node: "node-1",
Nodes: map[string]config.Config{"node-1": {BaseDir: "/tmp"}},
}
daemon, err := NewDaemon(clusterCfg)
require.NoError(t, err)
// Setup a running instance with allocated resources
instanceId := "i-test-stop"
instanceTypeStr := getTestInstanceType(t)
instanceType := daemon.resourceMgr.instanceTypes[instanceTypeStr]
daemon.vmMgr.Insert(&vm.VM{
ID: instanceId,
InstanceType: instanceTypeStr,
Status: vm.StateRunning,
AccountID: testAccountID,
})
err = daemon.resourceMgr.allocate(instanceType)
require.NoError(t, err)
assert.Greater(t, daemon.resourceMgr.allocatedVCPU, 0)
// Call stopInstance (we can't easily wait for QMP/PID here, so we just want to see deallocate call)
// Actually stopInstance runs in goroutines and waits for PID removal.
// This might be tricky to test without heavy mocking.
// Let's test the ResourceManager deallocate directly since we've already verified
// that stopInstance calls it in the code.
daemon.resourceMgr.deallocate(instanceType)
assert.Equal(t, 0, daemon.resourceMgr.allocatedVCPU)
}
// TestCanAllocate_CountEdgeCases tests edge cases for canAllocate with count parameter
func TestCanAllocate_CountEdgeCases(t *testing.T) {
t.Run("MinCount_equals_MaxCount", func(t *testing.T) {
@@ -1627,78 +1593,6 @@ func TestResourceManager_ConcurrentAccess(t *testing.T) {
assert.Equal(t, float64(0), rm.allocatedMem, "All memory should be deallocated")
}
// TestGenerateVolumes_DeleteOnTermination_FromBlockDeviceMapping verifies that
// the deleteOnTermination flag from RunInstancesInput.BlockDeviceMappings is
// propagated to the EBSRequest on the instance's volume list.
func TestGenerateVolumes_DeleteOnTermination_FromBlockDeviceMapping(t *testing.T) {
tests := []struct {
name string
deleteOnTerminationFlag *bool
expectedFlag bool
}{
{
name: "DeleteOnTermination=true",
deleteOnTerminationFlag: aws.Bool(true),
expectedFlag: true,
},
{
name: "DeleteOnTermination=false",
deleteOnTerminationFlag: aws.Bool(false),
expectedFlag: false,
},
{
name: "DeleteOnTermination=nil (defaults to true)",
deleteOnTerminationFlag: nil,
expectedFlag: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Build a RunInstancesInput with BlockDeviceMappings
input := &ec2.RunInstancesInput{
ImageId: aws.String("vol-existing-volume"),
InstanceType: aws.String("t3.micro"),
MinCount: aws.Int64(1),
MaxCount: aws.Int64(1),
}
if tt.deleteOnTerminationFlag != nil {
input.BlockDeviceMappings = []*ec2.BlockDeviceMapping{
{
DeviceName: aws.String("/dev/vda"),
Ebs: &ec2.EbsBlockDevice{
VolumeSize: aws.Int64(8),
DeleteOnTermination: tt.deleteOnTerminationFlag,
},
},
}
}
// Exercise the parsing logic that GenerateVolumes uses
// Default is true (matches AWS RunInstances behavior for root volumes)
deleteOnTermination := true
if len(input.BlockDeviceMappings) > 0 {
bdm := input.BlockDeviceMappings[0]
if bdm.Ebs != nil && bdm.Ebs.DeleteOnTermination != nil {
deleteOnTermination = *bdm.Ebs.DeleteOnTermination
}
}
assert.Equal(t, tt.expectedFlag, deleteOnTermination,
"deleteOnTermination should match expected value")
// Verify the flag is correctly assigned to an EBSRequest
ebsReq := types.EBSRequest{
Name: "vol-test",
Boot: true,
DeleteOnTermination: deleteOnTermination,
}
assert.Equal(t, tt.expectedFlag, ebsReq.DeleteOnTermination)
})
}
}
// TestInstanceCleanerAdapter_DeleteVolumes_DeleteOnTermination tests that
// the instanceCleanerAdapter correctly handles DeleteOnTermination for each
// volume type: internal volumes (EFI, cloud-init) always go through
@@ -3279,7 +3173,8 @@ func TestVolumeMounterAdapter_UnmountOne_Success(t *testing.T) {
}
// TestVolumeMounterAdapter_UnmountOne_UnmountError verifies that UnmountOne
// tolerates an error response from ebs.unmount (logs only, no propagation).
// propagates an error response from ebs.unmount. DetachVolume gates the volume's
// available transition on this error, so it must not be swallowed.
func TestVolumeMounterAdapter_UnmountOne_UnmountError(t *testing.T) {
natsURL := sharedNATSURL
@@ -3294,11 +3189,12 @@ func TestVolumeMounterAdapter_UnmountOne_UnmountError(t *testing.T) {
require.NoError(t, err)
defer sub.Unsubscribe()
adapter.UnmountOne(types.EBSRequest{Name: "vol-rollback-err"})
require.Error(t, adapter.UnmountOne(types.EBSRequest{Name: "vol-rollback-err"}),
"an ebs.unmount error response must propagate so DetachVolume can keep the volume attached")
}
// TestVolumeMounterAdapter_UnmountOne_StillMounted verifies that UnmountOne
// tolerates an unmount response that reports the volume still mounted.
// returns an error when the unmount response reports the volume still mounted.
func TestVolumeMounterAdapter_UnmountOne_StillMounted(t *testing.T) {
natsURL := sharedNATSURL
@@ -3313,18 +3209,103 @@ func TestVolumeMounterAdapter_UnmountOne_StillMounted(t *testing.T) {
require.NoError(t, err)
defer sub.Unsubscribe()
adapter.UnmountOne(types.EBSRequest{Name: "vol-still-mounted"})
require.Error(t, adapter.UnmountOne(types.EBSRequest{Name: "vol-still-mounted"}),
"a still-mounted response must propagate as an error")
}
// TestVolumeMounterAdapter_UnmountOne_NATSTimeout verifies that UnmountOne
// tolerates NATS request timeout (no subscriber on ebs.unmount).
func TestVolumeMounterAdapter_UnmountOne_NATSTimeout(t *testing.T) {
natsURL := sharedNATSURL
// TestVolumeMounterAdapter_UnmountOne_RequestFailure verifies that UnmountOne
// surfaces a failed ebs.unmount NATS request as an error. A closed connection
// fails immediately, rather than blocking on the full unmountSealTimeout (120s)
// that a no-subscriber timeout would incur in every preflight run.
func TestVolumeMounterAdapter_UnmountOne_RequestFailure(t *testing.T) {
nc, err := nats.Connect(sharedNATSURL)
require.NoError(t, err)
nc.Close()
daemon := createTestDaemon(t, natsURL)
adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
adapter := newVolumeMounterAdapter(nc, "node-1", nil)
require.Error(t, adapter.UnmountOne(types.EBSRequest{Name: "vol-timeout"}),
"a failed ebs.unmount request must propagate as an error")
}
adapter.UnmountOne(types.EBSRequest{Name: "vol-timeout"})
// recordingVolState records UpdateVolumeState calls so the bulk-unmount test can
// assert which volumes transitioned to available.
type recordingVolState struct {
mu sync.Mutex
calls []string
}
func (r *recordingVolState) UpdateVolumeState(volumeID, state, instanceID, attachmentDevice string) error {
r.mu.Lock()
defer r.mu.Unlock()
r.calls = append(r.calls, volumeID+":"+state)
return nil
}
func (r *recordingVolState) snapshot() []string {
r.mu.Lock()
defer r.mu.Unlock()
return append([]string(nil), r.calls...)
}
var _ vm.VolumeStateUpdater = (*recordingVolState)(nil)
// TestVolumeMounterAdapter_Unmount_SealFailureSkipsAvailable verifies the
// teardown-path durability gate: a per-volume seal failure during the bulk
// Unmount (stop/terminate) must NOT transition that volume to available — a
// reattach on a WAL-less node would find no checkpoint (bad superblock) — while
// other volumes still seal and the loop completes (terminate stays idempotent).
func TestVolumeMounterAdapter_Unmount_SealFailureSkipsAvailable(t *testing.T) {
daemon := createTestDaemon(t, sharedNATSURL)
volState := &recordingVolState{}
adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, volState)
sub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
var req types.EBSRequest
json.Unmarshal(msg.Data, &req)
resp := types.EBSUnMountResponse{Volume: req.Name, Mounted: false}
if req.Name == "vol-fail" {
resp.Error = "seal volume: predastore unreachable"
}
data, _ := json.Marshal(resp)
msg.Respond(data)
})
require.NoError(t, err)
defer sub.Unsubscribe()
inst := &vm.VM{
ID: "i-1",
EBSRequests: types.EBSRequests{
Requests: []types.EBSRequest{{Name: "vol-fail"}, {Name: "vol-ok"}},
},
}
require.NoError(t, adapter.Unmount(inst),
"bulk Unmount must tolerate a per-volume seal failure so terminate stays idempotent")
calls := volState.snapshot()
assert.NotContains(t, calls, "vol-fail:available",
"a failed seal must not transition the volume to available")
assert.Contains(t, calls, "vol-ok:available",
"a successful seal must still transition the volume to available")
}
// TestUnmountResponseError covers the shared seal-result parser used by both the
// gated DetachVolume path (unmountOne) and the tolerated teardown path (Unmount).
func TestUnmountResponseError(t *testing.T) {
t.Run("success (not mounted, no error)", func(t *testing.T) {
data, _ := json.Marshal(types.EBSUnMountResponse{Volume: "v", Mounted: false})
require.NoError(t, unmountResponseError(data))
})
t.Run("seal error propagates", func(t *testing.T) {
data, _ := json.Marshal(types.EBSUnMountResponse{Volume: "v", Error: "seal volume: boom"})
require.Error(t, unmountResponseError(data))
})
t.Run("still mounted propagates", func(t *testing.T) {
data, _ := json.Marshal(types.EBSUnMountResponse{Volume: "v", Mounted: true})
require.Error(t, unmountResponseError(data))
})
t.Run("malformed payload propagates", func(t *testing.T) {
require.Error(t, unmountResponseError([]byte("not json")))
})
}
// TestVolumeMounterAdapter_Mount_PartialFailureRollback verifies that when
spinifex/daemon/vm_adapters.go
+33 -7
View File
@@ -77,6 +77,11 @@ type volumeMounterAdapter struct {
var _ vm.VolumeMounter = (*volumeMounterAdapter)(nil)
// unmountSealTimeout bounds the ebs.unmount NATS request. The handler now drives
// a synchronous block-map seal to predastore (S3 I/O), so it matches
// viperblockd's 120s KillProcess wait rather than a bare RPC budget.
const unmountSealTimeout = 120 * time.Second
func newVolumeMounterAdapter(nc *nats.Conn, node string, volState vm.VolumeStateUpdater) *volumeMounterAdapter {
return &volumeMounterAdapter{nc: nc, node: node, volState: volState}
}
@@ -152,16 +157,27 @@ func (a *volumeMounterAdapter) Unmount(instance *vm.VM) error {
continue
}
msg, err := a.nc.Request(a.topic("unmount"), ebsUnMountRequest, 30*time.Second)
// ebs.unmount seals the block map to predastore. Teardown tolerates a
// failed seal (log + continue) so terminate stays idempotent, but the
// volume must NOT then go available: a reattach on a node without the
// local WAL would find no checkpoint (bad superblock). On terminate the
// volume is deleted regardless; on stop it stays attached/retryable.
sealed := true
msg, err := a.nc.Request(a.topic("unmount"), ebsUnMountRequest, unmountSealTimeout)
if err != nil {
slog.Error("Failed to unmount volume",
"name", ebsRequest.Name, "instance", instance.ID, "err", err)
sealed = false
} else if sealErr := unmountResponseError(msg.Data); sealErr != nil {
slog.Error("Volume unmount seal failed, leaving volume non-available",
"instance", instance.ID, "volume", ebsRequest.Name, "err", sealErr)
sealed = false
} else {
slog.Info("Unmounted volume",
"instance", instance.ID, "volume", ebsRequest.Name, "data", string(msg.Data))
}
if !ebsRequest.EFI && !ebsRequest.CloudInit && a.volState != nil {
if sealed && !ebsRequest.EFI && !ebsRequest.CloudInit && a.volState != nil {
if err := a.volState.UpdateVolumeState(ebsRequest.Name, "available", "", ""); err != nil {
slog.Error("Failed to update volume state to available after unmount",
"volumeId", ebsRequest.Name, "err", err)
@@ -200,13 +216,16 @@ func (a *volumeMounterAdapter) MountOne(req *types.EBSRequest) error {
return nil
}
// UnmountOne sends ebs.unmount; errors are logged and swallowed.
func (a *volumeMounterAdapter) UnmountOne(req types.EBSRequest) {
// UnmountOne sends ebs.unmount and returns any error. The handler seals the
// volume's block map to predastore, so the caller decides whether a failure
// blocks the volume's available transition.
func (a *volumeMounterAdapter) UnmountOne(req types.EBSRequest) error {
if err := a.unmountOne(req); err != nil {
slog.Error("UnmountOne failed", "volume", req.Name, "err", err)
return
return err
}
slog.Info("UnmountOne: volume unmounted successfully", "volume", req.Name)
return nil
}
// unmountOne sends ebs.unmount and returns any error.
@@ -215,12 +234,19 @@ func (a *volumeMounterAdapter) unmountOne(req types.EBSRequest) error {
if err != nil {
return fmt.Errorf("marshal unmount request: %w", err)
}
msg, err := a.nc.Request(a.topic("unmount"), payload, 10*time.Second)
msg, err := a.nc.Request(a.topic("unmount"), payload, unmountSealTimeout)
if err != nil {
return fmt.Errorf("ebs.unmount NATS request: %w", err)
}
return unmountResponseError(msg.Data)
}
// unmountResponseError reports a seal/unmount failure from an ebs.unmount
// response payload: a non-empty Error, or a volume still reported mounted.
// Returns nil when the unmount and its block-map seal succeeded.
func unmountResponseError(data []byte) error {
var resp types.EBSUnMountResponse
if err := json.Unmarshal(msg.Data, &resp); err != nil {
if err := json.Unmarshal(data, &resp); err != nil {
return fmt.Errorf("unmarshal unmount response: %w", err)
}
if resp.Error != "" {
spinifex/gateway/eks.go
+2 -2
View File
@@ -112,9 +112,9 @@ var eksRoutes = []eksRoute{
func(gw *GatewayConfig, acct, callerARN string, p []string, b []byte) (any, error) {
return gateway_eks.AssociateAccessPolicy(gw.NATSConn, acct, p[0], p[1], b)
}},
{"DELETE", regexp.MustCompile(`^/clusters/([^/]+)/access-entries/([^/]+)/access-policies$`), "DisassociateAccessPolicy",
{"DELETE", regexp.MustCompile(`^/clusters/([^/]+)/access-entries/([^/]+)/access-policies/([^/]+)$`), "DisassociateAccessPolicy",
func(gw *GatewayConfig, acct, callerARN string, p []string, b []byte) (any, error) {
return gateway_eks.DisassociateAccessPolicy(gw.NATSConn, acct, p[0], p[1], b)
return gateway_eks.DisassociateAccessPolicy(gw.NATSConn, acct, p[0], p[1], p[2])
}},
{"GET", regexp.MustCompile(`^/clusters/([^/]+)/access-entries/([^/]+)/access-policies$`), "ListAssociatedAccessPolicies",
func(gw *GatewayConfig, acct, callerARN string, p []string, b []byte) (any, error) {
spinifex/gateway/eks/access.go
+6 -7
View File
@@ -63,14 +63,13 @@ func AssociateAccessPolicy(natsConn *nats.Conn, accountID, cluster, principalARN
return handlers_eks.NewNATSEKSService(natsConn).AssociateAccessPolicy(input, accountID)
}
// DisassociateAccessPolicy — DELETE /clusters/{name}/access-entries/{arn}/access-policies
func DisassociateAccessPolicy(natsConn *nats.Conn, accountID, cluster, principalARN string, body []byte) (*eks.DisassociateAccessPolicyOutput, error) {
input := new(eks.DisassociateAccessPolicyInput)
if err := unmarshalIfBody(body, input); err != nil {
return nil, err
// DisassociateAccessPolicy — DELETE /clusters/{name}/access-entries/{arn}/access-policies/{policyArn}
func DisassociateAccessPolicy(natsConn *nats.Conn, accountID, cluster, principalARN, policyARN string) (*eks.DisassociateAccessPolicyOutput, error) {
input := &eks.DisassociateAccessPolicyInput{
ClusterName: aws.String(cluster),
PrincipalArn: aws.String(principalARN),
PolicyArn: aws.String(policyARN),
}
input.ClusterName = aws.String(cluster)
input.PrincipalArn = aws.String(principalARN)
return handlers_eks.NewNATSEKSService(natsConn).DisassociateAccessPolicy(input, accountID)
}
spinifex/gateway/eks/wrappers_test.go
+1 -7
View File
@@ -172,14 +172,10 @@ func TestGatewayWrappers_Access(t *testing.T) {
require.NoError(t, err)
assert.NotNil(t, out9)
out10, err := DisassociateAccessPolicy(nc, acct, "alpha", "arn-user", []byte(`{}`))
out10, err := DisassociateAccessPolicy(nc, acct, "alpha", "arn-user", "policy-arn")
require.NoError(t, err)
assert.NotNil(t, out10)
out11, err := DisassociateAccessPolicy(nc, acct, "alpha", "arn-user", nil)
require.NoError(t, err)
assert.NotNil(t, out11)
out12, err := ListAssociatedAccessPolicies(nc, acct, "alpha", "arn-user")
require.NoError(t, err)
assert.NotNil(t, out12)
@@ -199,8 +195,6 @@ func TestGatewayWrappers_Access_BadJSON(t *testing.T) {
require.Error(t, err)
_, err = AssociateAccessPolicy(nc, acct, "alpha", "arn-user", []byte(`{x`))
require.Error(t, err)
_, err = DisassociateAccessPolicy(nc, acct, "alpha", "arn-user", []byte(`{x`))
require.Error(t, err)
}
func TestGatewayWrappers_Addons(t *testing.T) {
spinifex/gateway/eks_test.go
+19 -2
View File
@@ -38,7 +38,7 @@ func TestLookupEKSAction_ResolvesKnownRoutes(t *testing.T) {
{"POST", "/clusters/alpha/access-entries/arn-user", "UpdateAccessEntry", []string{"alpha", "arn-user"}},
{"DELETE", "/clusters/alpha/access-entries/arn-user", "DeleteAccessEntry", []string{"alpha", "arn-user"}},
{"POST", "/clusters/alpha/access-entries/arn-user/access-policies", "AssociateAccessPolicy", []string{"alpha", "arn-user"}},
{"DELETE", "/clusters/alpha/access-entries/arn-user/access-policies", "DisassociateAccessPolicy", []string{"alpha", "arn-user"}},
{"DELETE", "/clusters/alpha/access-entries/arn-user/access-policies/policy-arn", "DisassociateAccessPolicy", []string{"alpha", "arn-user", "policy-arn"}},
{"GET", "/clusters/alpha/access-entries/arn-user/access-policies", "ListAssociatedAccessPolicies", []string{"alpha", "arn-user"}},
{"GET", "/access-policies", "ListAccessPolicies", nil},
{"GET", "/clusters/alpha/addons", "ListAddons", []string{"alpha"}},
@@ -81,7 +81,6 @@ func TestLookupEKSAction_EncodedPrincipalARN(t *testing.T) {
{"POST", "/clusters/alpha/access-entries/" + escaped, "UpdateAccessEntry"},
{"DELETE", "/clusters/alpha/access-entries/" + escaped, "DeleteAccessEntry"},
{"POST", "/clusters/alpha/access-entries/" + escaped + "/access-policies", "AssociateAccessPolicy"},
{"DELETE", "/clusters/alpha/access-entries/" + escaped + "/access-policies", "DisassociateAccessPolicy"},
{"GET", "/clusters/alpha/access-entries/" + escaped + "/access-policies", "ListAssociatedAccessPolicies"},
}
for _, tc := range cases {
@@ -95,6 +94,24 @@ func TestLookupEKSAction_EncodedPrincipalARN(t *testing.T) {
}
}
// DisassociateAccessPolicy carries the policy ARN as the final path segment
// (DELETE .../access-policies/{policyArn}); both principal and policy ARNs are
// percent-encoded and must survive as separate single segments.
func TestLookupEKSAction_DisassociateEncodedARNs(t *testing.T) {
const (
principal = "arn:aws:iam::000000000001:user/admin"
principalEscaped = "arn%3Aaws%3Aiam%3A%3A000000000001%3Auser%2Fadmin"
policy = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy"
policyEscaped = "arn%3Aaws%3Aeks%3A%3Aaws%3Acluster-access-policy%2FAmazonEKSViewPolicy"
)
path := "/clusters/alpha/access-entries/" + principalEscaped + "/access-policies/" + policyEscaped
action, params, handler, ok := lookupEKSAction("DELETE", path)
require.True(t, ok, "encoded disassociate path should match: %s", path)
require.NotNil(t, handler)
assert.Equal(t, "DisassociateAccessPolicy", action)
assert.Equal(t, []string{"alpha", principal, policy}, params)
}
func TestLookupEKSAction_CoversAllActions(t *testing.T) {
expected := map[string]bool{
"CreateCluster": false,
spinifex/handlers/ec2/instance/service_impl.go
+24
View File
@@ -523,6 +523,11 @@ func (s *InstanceServiceImpl) RunInstance(input *ec2.RunInstancesInput) (*vm.VM,
}
}
// Apply instance-scoped tags from TagSpecifications so DescribeInstances
// returns them and tag filters match. Node groups discover their workers by
// the spinifex:eks-cluster tag; scale-up convergence depends on it.
ec2Instance.Tags = instanceTagsFromSpec(input.TagSpecifications)
// Store EC2 API metadata in VM for DescribeInstances compatibility
instance.RunInstancesInput = input
instance.Instance = ec2Instance
@@ -530,6 +535,25 @@ func (s *InstanceServiceImpl) RunInstance(input *ec2.RunInstancesInput) (*vm.VM,
return instance, ec2Instance, nil
}
// instanceTagsFromSpec extracts the instance-scoped tags from a RunInstances
// TagSpecifications list. Only ResourceType "instance" applies to the launched
// instance; volume/network-interface specs are handled elsewhere.
func instanceTagsFromSpec(specs []*ec2.TagSpecification) []*ec2.Tag {
var tags []*ec2.Tag
for _, spec := range specs {
if spec == nil || aws.StringValue(spec.ResourceType) != "instance" {
continue
}
for _, t := range spec.Tags {
if t == nil || t.Key == nil {
continue
}
tags = append(tags, &ec2.Tag{Key: t.Key, Value: t.Value})
}
}
return tags
}
// PrepareRunInstances validates input, allocates capacity, creates VM metadata,
// auto-creates the primary ENI, and auto-assigns a public IP when needed.
// Does NOT touch vmMgr or NATS — callers insert VMs then call LaunchRunInstances.
spinifex/handlers/ec2/instance/service_impl_test.go
+51
View File
@@ -138,6 +138,57 @@ func TestRunInstance_WithIamInstanceProfile(t *testing.T) {
"daemon must not emit InstanceProfileID — that is the gateway's responsibility")
}
func TestRunInstance_AppliesInstanceTags(t *testing.T) {
svc := &InstanceServiceImpl{
instanceTypes: map[string]*ec2.InstanceTypeInfo{"t3.micro": {InstanceType: aws.String("t3.micro")}},
}
input := &ec2.RunInstancesInput{
ImageId: aws.String("ami-012345"),
InstanceType: aws.String("t3.micro"),
TagSpecifications: []*ec2.TagSpecification{
{
ResourceType: aws.String("instance"),
Tags: []*ec2.Tag{
{Key: aws.String("spinifex:eks-cluster"), Value: aws.String("eks-quickstart")},
{Key: aws.String("spinifex:eks-nodegroup"), Value: aws.String("default")},
},
},
// A non-instance spec must not leak onto the instance.
{
ResourceType: aws.String("volume"),
Tags: []*ec2.Tag{{Key: aws.String("ignored"), Value: aws.String("vol")}},
},
},
}
_, ec2Instance, err := svc.RunInstance(input)
require.NoError(t, err)
// Tags must surface on the instance metadata so DescribeInstances returns
// them and the node group can discover its workers by tag.
got := map[string]string{}
for _, tg := range ec2Instance.Tags {
got[aws.StringValue(tg.Key)] = aws.StringValue(tg.Value)
}
assert.Equal(t, "eks-quickstart", got["spinifex:eks-cluster"])
assert.Equal(t, "default", got["spinifex:eks-nodegroup"])
assert.NotContains(t, got, "ignored", "volume-scoped tags must not land on the instance")
}
func TestRunInstance_NoTagSpecifications(t *testing.T) {
svc := &InstanceServiceImpl{
instanceTypes: map[string]*ec2.InstanceTypeInfo{"t3.micro": {InstanceType: aws.String("t3.micro")}},
}
input := &ec2.RunInstancesInput{
ImageId: aws.String("ami-012345"),
InstanceType: aws.String("t3.micro"),
}
_, ec2Instance, err := svc.RunInstance(input)
require.NoError(t, err)
assert.Empty(t, ec2Instance.Tags)
}
func TestRunInstance_IamInstanceProfileEmptyARNIgnored(t *testing.T) {
// AWS SDKs sometimes round-trip an IamInstanceProfile with both fields
// empty. Treat that as "no profile attached" rather than persisting a
spinifex/handlers/ec2/vpc/security_group.go
+12 -2
View File
@@ -696,8 +696,13 @@ func (s *VPCServiceImpl) AuthorizeSecurityGroupIngress(input *ec2.AuthorizeSecur
return nil, err
}
ruleSet := make([]*ec2.SecurityGroupRule, 0, len(newRules))
for _, nr := range newRules {
ruleSet = append(ruleSet, sgRuleToSecurityGroupRule(&record, nr, false, accountID))
}
return &ec2.AuthorizeSecurityGroupIngressOutput{
Return: aws.Bool(true),
Return: aws.Bool(true),
SecurityGroupRules: ruleSet,
}, nil
}
@@ -766,8 +771,13 @@ func (s *VPCServiceImpl) AuthorizeSecurityGroupEgress(input *ec2.AuthorizeSecuri
return nil, err
}
ruleSet := make([]*ec2.SecurityGroupRule, 0, len(newRules))
for _, nr := range newRules {
ruleSet = append(ruleSet, sgRuleToSecurityGroupRule(&record, nr, true, accountID))
}
return &ec2.AuthorizeSecurityGroupEgressOutput{
Return: aws.Bool(true),
Return: aws.Bool(true),
SecurityGroupRules: ruleSet,
}, nil
}
spinifex/handlers/ec2/vpc/security_group_test.go
+52
View File
@@ -2,6 +2,7 @@ package handlers_ec2_vpc
import (
"encoding/json"
"strings"
"testing"
"github.com/aws/aws-sdk-go/aws"
@@ -245,6 +246,57 @@ func TestAuthorizeSecurityGroupEgress_NotFound(t *testing.T) {
assert.Error(t, err)
}
// The AWS provider reads SecurityGroupRules[0].SecurityGroupRuleId on create;
// an empty ruleset panics the plugin. Authorize must echo the created rules.
func TestAuthorizeSecurityGroupIngress_ReturnsRuleSet(t *testing.T) {
svc := setupTestVPCService(t)
vpcID := createTestVPC(t, svc, "10.0.0.0/16")
sgID := createTestSG(t, svc, vpcID, "ingress-ruleset-sg")
out, err := svc.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{
GroupId: aws.String(sgID),
IpPermissions: []*ec2.IpPermission{{
IpProtocol: aws.String("tcp"),
FromPort: aws.Int64(30000),
ToPort: aws.Int64(32767),
IpRanges: []*ec2.IpRange{{CidrIp: aws.String("0.0.0.0/0")}},
}},
}, testAccountID)
require.NoError(t, err)
require.Len(t, out.SecurityGroupRules, 1)
r := out.SecurityGroupRules[0]
assert.NotEmpty(t, aws.StringValue(r.SecurityGroupRuleId))
assert.True(t, strings.HasPrefix(aws.StringValue(r.SecurityGroupRuleId), "sgr-"))
assert.Equal(t, sgID, aws.StringValue(r.GroupId))
assert.False(t, aws.BoolValue(r.IsEgress))
assert.Equal(t, "tcp", aws.StringValue(r.IpProtocol))
assert.Equal(t, int64(30000), aws.Int64Value(r.FromPort))
assert.Equal(t, int64(32767), aws.Int64Value(r.ToPort))
assert.Equal(t, "0.0.0.0/0", aws.StringValue(r.CidrIpv4))
}
func TestAuthorizeSecurityGroupEgress_ReturnsRuleSet(t *testing.T) {
svc := setupTestVPCService(t)
vpcID := createTestVPC(t, svc, "10.0.0.0/16")
sgID := createTestSG(t, svc, vpcID, "egress-ruleset-sg")
out, err := svc.AuthorizeSecurityGroupEgress(&ec2.AuthorizeSecurityGroupEgressInput{
GroupId: aws.String(sgID),
IpPermissions: []*ec2.IpPermission{{
IpProtocol: aws.String("tcp"),
FromPort: aws.Int64(443),
ToPort: aws.Int64(443),
IpRanges: []*ec2.IpRange{{CidrIp: aws.String("10.0.0.0/8")}},
}},
}, testAccountID)
require.NoError(t, err)
require.Len(t, out.SecurityGroupRules, 1)
r := out.SecurityGroupRules[0]
assert.NotEmpty(t, aws.StringValue(r.SecurityGroupRuleId))
assert.True(t, aws.BoolValue(r.IsEgress))
assert.Equal(t, "10.0.0.0/8", aws.StringValue(r.CidrIpv4))
}
// --- RevokeSecurityGroupIngress ---
func TestRevokeSecurityGroupIngress_Success(t *testing.T) {
spinifex/handlers/ec2/vpc/service_nats_test.go
-12
View File
@@ -3,7 +3,6 @@ package handlers_ec2_vpc
import (
"encoding/json"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
@@ -201,14 +200,3 @@ func TestNATSVPCService_DescribeNetworkInterfaces(t *testing.T) {
require.NoError(t, err)
assert.GreaterOrEqual(t, len(out.NetworkInterfaces), 1)
}
func TestNATSVPCService_Timeout(t *testing.T) {
// Create NATS connection without any subscribers — should timeout
_, nc := setupTestVPCServiceWithNC(t)
client := &NATSVPCService{natsConn: nc}
// Production code uses a 30s timeout; just verify the constructor works
// without exercising the full round-trip.
_ = client
_ = time.Second // reference time package
}
spinifex/handlers/eks/addon_catalog.go
+3
View File
@@ -23,6 +23,9 @@ var addonCatalog = buildAddonCatalog(
newAddonSpec("coredns", false,
"Cluster DNS server.",
"1.11.1"),
newAddonSpec("argocd", false,
"Declarative GitOps continuous delivery for Kubernetes.",
"3.0.23"),
// spinifex-noop is the delivery-transport fixture: a trivial bundle
// (Namespace + ConfigMap) used by the addon e2e to prove stage → render →
// auto-deploy → ACTIVE → delete round-trips end-to-end without depending on
spinifex/handlers/eks/delete_cluster_test.go
+27 -1
View File
@@ -109,6 +109,7 @@ type deleteClusterFixture struct {
inst *fakeK3sInst
vpc *fakeK3sVPC
eip *fakeEIPProvisioner
sg *fakeSGProvisioner
}
func newDeleteClusterFixture(t *testing.T, clusterName string) *deleteClusterFixture {
@@ -131,7 +132,32 @@ func newDeleteClusterFixture(t *testing.T, clusterName string) *deleteClusterFix
_, _, err := GenerateClusterOIDCKeypair(kv, clusterName, bootstrapTestMasterKey)
require.NoError(t, err)
return &deleteClusterFixture{svc: f.svc, kv: kv, nlb: f.nlb, inst: f.inst, vpc: f.vpc, eip: f.eip}
return &deleteClusterFixture{svc: f.svc, kv: kv, nlb: f.nlb, inst: f.inst, vpc: f.vpc, eip: f.eip, sg: f.sg}
}
// TestDeleteCluster_LeakedSGKeepsClusterDeleting guards the customer-VPC SG
// no-orphan contract (mulga-siv-340): the cluster SGs live in the customer VPC,
// so a DeleteClusterSGs failure must surface and leave the cluster DELETING — its
// meta intact for a retry — never complete deletion with an SG stranded that pins
// the VPC on DependencyViolation. All other teardown steps succeed, isolating the
// SG failure as the sole cause.
func TestDeleteCluster_LeakedSGKeepsClusterDeleting(t *testing.T) {
origBudget, origInterval := sgDeleteWaitBudget, sgDeleteWaitInterval
sgDeleteWaitBudget, sgDeleteWaitInterval = 50*time.Millisecond, time.Millisecond
defer func() { sgDeleteWaitBudget, sgDeleteWaitInterval = origBudget, origInterval }()
f := newDeleteClusterFixture(t, "alpha")
f.sg.existing["eks-cluster-alpha-control-plane-sg|vpc-aaa"] = "sg-cp"
f.sg.existing["eks-cluster-alpha-nodegroup-sg|vpc-aaa"] = "sg-ng"
// A worker ENI never detaches — the SG delete keeps returning DependencyViolation.
f.sg.deleteErr = errors.New("DependencyViolation: resource has a dependent object")
_, err := f.svc.DeleteCluster(deleteInput("alpha"), testAccountID)
require.Error(t, err, "a leaked customer-VPC SG must surface, not be swallowed")
meta, getErr := GetClusterMeta(f.kv, "alpha")
require.NoError(t, getErr, "the KV sweep must NOT run while an SG is still leaked — meta must survive for retry")
assert.Equal(t, ClusterStatusDeleting, meta.Status, "a cluster with a leaked SG must stay DELETING")
}
// TestRLC5_DeleteClusterCPVPCReleasesNATGWEIPAfterRoutes locks the mulga-siv-303
spinifex/handlers/eks/k3s_server_vm.go
+14 -2
View File
@@ -475,6 +475,16 @@ func buildK3sUserData(in K3sServerInput) string {
if in.PrivateEndpointIP != "" && in.PrivateEndpointIP != in.EndpointIP {
configLines = append(configLines, " - "+in.PrivateEndpointIP)
}
// advertise-address lands in the in-cluster `kubernetes` Endpoints. The default
// CP node-ip sits in the unpeered managed-CP VPC, unreachable from worker pods;
// advertise the NLB front-end (Set A private-endpoint, else public) — both SANed.
advertiseIP := in.PrivateEndpointIP
if advertiseIP == "" {
advertiseIP = in.EndpointIP
}
if advertiseIP != "" {
configLines = append(configLines, "advertise-address: "+advertiseIP)
}
configLines = append(configLines,
"kube-apiserver-arg:",
" - service-account-key-file="+k3sOIDCPublicKeyPath,
@@ -530,10 +540,12 @@ func buildK3sUserData(in K3sServerInput) string {
}
}
// Enable OpenRC `local` so the IMDS route script runs on every boot.
// Enable OpenRC `local` for reboot persistence, then run the IMDS route script
// directly. Starting the service here would deadlock: runcmd runs inside
// cloud-final, but `local` is ordered after it — blocking until OpenRC times out.
buf.WriteString("runcmd:\n")
buf.WriteString(" - [ rc-update, add, local, default ]\n")
buf.WriteString(" - [ rc-service, local, start ]\n")
buf.WriteString(" - [ /etc/local.d/imds-onlink-route.start ]\n")
return buf.String()
}
spinifex/handlers/eks/k3s_server_vm_test.go
+28
View File
@@ -4,6 +4,7 @@ import (
"errors"
"fmt"
"strings"
"sync"
"testing"
"time"
@@ -137,6 +138,7 @@ func (f *fakeK3sInst) TerminateSystemInstance(instanceID string) error {
}
type fakeK3sAMI struct {
mu sync.Mutex
describeCalls []*ec2.DescribeImagesInput
describeOut *ec2.DescribeImagesOutput
@@ -146,6 +148,8 @@ type fakeK3sAMI struct {
var _ k3sAMIResolver = (*fakeK3sAMI)(nil)
func (f *fakeK3sAMI) DescribeImages(input *ec2.DescribeImagesInput, _ string) (*ec2.DescribeImagesOutput, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.describeCalls = append(f.describeCalls, input)
if f.describeErr != nil {
return nil, f.describeErr
@@ -359,6 +363,26 @@ func TestBuildK3sUserData_TLSSANDedupsWhenPrivateEqualsEndpoint(t *testing.T) {
assert.Equal(t, 1, strings.Count(ud, " - 10.20.0.5"), "must not emit a duplicate SAN")
}
func TestBuildK3sUserData_AdvertiseAddressPrefersPrivateEndpoint(t *testing.T) {
in := validK3sInput()
in.EndpointIP = "203.0.113.9"
in.PrivateEndpointIP = "10.20.0.5"
ud := buildK3sUserData(in)
assert.Contains(t, ud, "advertise-address: 10.20.0.5",
"apiserver must advertise the worker-reachable Set A NLB front-end, not the CP node-ip")
assert.NotContains(t, ud, "advertise-address: 203.0.113.9")
}
func TestBuildK3sUserData_AdvertiseAddressFallsBackToEndpoint(t *testing.T) {
in := validK3sInput()
in.EndpointIP = "203.0.113.9"
in.PrivateEndpointIP = ""
ud := buildK3sUserData(in)
assert.Contains(t, ud, "advertise-address: 203.0.113.9")
}
func TestLaunchK3sServerVM_UserDataContainsAllArtifacts(t *testing.T) {
vpc, inst, ami := &fakeK3sVPC{}, &fakeK3sInst{}, &fakeK3sAMI{}
@@ -447,6 +471,10 @@ func TestLaunchK3sServerVM_UserDataContainsAllArtifacts(t *testing.T) {
assert.Contains(t, udata, `ip route replace 169.254.169.254/32 dev "$dev" scope link`)
assert.Contains(t, udata, "runcmd:")
assert.Contains(t, udata, "[ rc-update, add, local, default ]")
// Route script is run directly, not via `rc-service local start` — that would
// deadlock against cloud-final and stall the boot ~50s on the OpenRC timeout.
assert.Contains(t, udata, "[ /etc/local.d/imds-onlink-route.start ]")
assert.NotContains(t, udata, "rc-service, local, start")
// Exactly one write_files / runcmd key — a second would collide and silently
// drop a block under yaml.safe_load (last key wins).
spinifex/handlers/eks/nlb.go
+32 -20
View File
@@ -67,9 +67,11 @@ func ClusterTargetGroupName(clusterName string) string {
}
// EnsureClusterNLB provisions (or returns) the cluster's NLB + target group +
// :443→:6443 TCP listener. Idempotent on clusterName: existing resources are
// reused so DeleteCluster failure-retry and reconciler crash-recovery both
// converge without duplicate creates.
// two TCP listeners (:443 and :6443), both forwarding to the CP target group.
// :443 serves kubectl/SDK; :6443 serves the in-cluster `kubernetes` Service
// Endpoints path (worker pods reach PrivateEndpointIP:6443). Idempotent on
// clusterName: existing resources are reused so DeleteCluster failure-retry and
// reconciler crash-recovery both converge without duplicate creates.
//
// The K3s server VM ENI IP is NOT registered here — Stage 4
// (k3s_server_vm.go) calls RegisterClusterTarget once the VM has its ENI IP.
@@ -111,7 +113,14 @@ func EnsureClusterNLB(nlbp nlbProvisioner, accountID, clusterName string, subnet
if err := ensureClusterTG(nlbp, accountID, clusterName, tgName, out); err != nil {
return nil, err
}
if err := ensureClusterListener(nlbp, accountID, lbName, out); err != nil {
var err error
if out.ListenerArn, err = ensureClusterListener(nlbp, accountID, lbName, out.LoadBalancerArn, out.TargetGroupArn, clusterNLBListenPort); err != nil {
return nil, err
}
// Second listener :6443 → same CP target group. The apiserver advertises itself
// on :6443 in the in-cluster `kubernetes` Endpoints; worker pods reach it only if
// the NLB serves :6443. Arn not persisted — teardown cascades via DeleteLoadBalancer.
if _, err = ensureClusterListener(nlbp, accountID, lbName, out.LoadBalancerArn, out.TargetGroupArn, k3sAPIServerPort); err != nil {
return nil, err
}
if internetFacing && narrowsPublicAccess(publicAccessCidrs) {
@@ -364,37 +373,40 @@ func ensureClusterTG(nlbp nlbProvisioner, accountID, clusterName, tgName string,
return nil
}
func ensureClusterListener(nlbp nlbProvisioner, accountID, lbName string, out *ClusterNLB) error {
if l, err := lookupListenerByPort(nlbp, accountID, out.LoadBalancerArn, clusterNLBListenPort); err != nil {
return err
// ensureClusterListener creates (or reuses) one TCP listener on the LB at the
// given port forwarding to tgArn, returning its ARN. Idempotent: an existing
// listener on the port is reused.
func ensureClusterListener(nlbp nlbProvisioner, accountID, lbName, lbArn, tgArn string, port int64) (string, error) {
if l, err := lookupListenerByPort(nlbp, accountID, lbArn, port); err != nil {
return "", err
} else if l != nil {
out.ListenerArn = aws.StringValue(l.ListenerArn)
if out.ListenerArn == "" {
return fmt.Errorf("eks: existing listener for %s:%d missing arn", lbName, clusterNLBListenPort)
arn := aws.StringValue(l.ListenerArn)
if arn == "" {
return "", fmt.Errorf("eks: existing listener for %s:%d missing arn", lbName, port)
}
return nil
return arn, nil
}
created, err := nlbp.CreateListener(&elbv2.CreateListenerInput{
LoadBalancerArn: aws.String(out.LoadBalancerArn),
LoadBalancerArn: aws.String(lbArn),
Protocol: aws.String(elbv2.ProtocolEnumTcp),
Port: aws.Int64(clusterNLBListenPort),
Port: aws.Int64(port),
DefaultActions: []*elbv2.Action{{
Type: aws.String(elbv2.ActionTypeEnumForward),
TargetGroupArn: aws.String(out.TargetGroupArn),
TargetGroupArn: aws.String(tgArn),
}},
}, accountID)
if err != nil {
return fmt.Errorf("create listener %s:%d: %w", lbName, clusterNLBListenPort, err)
return "", fmt.Errorf("create listener %s:%d: %w", lbName, port, err)
}
if created == nil || len(created.Listeners) == 0 || created.Listeners[0] == nil {
return fmt.Errorf("eks: CreateListener returned no listener for %s:%d", lbName, clusterNLBListenPort)
return "", fmt.Errorf("eks: CreateListener returned no listener for %s:%d", lbName, port)
}
out.ListenerArn = aws.StringValue(created.Listeners[0].ListenerArn)
if out.ListenerArn == "" {
return fmt.Errorf("eks: CreateListener returned empty arn for %s:%d", lbName, clusterNLBListenPort)
arn := aws.StringValue(created.Listeners[0].ListenerArn)
if arn == "" {
return "", fmt.Errorf("eks: CreateListener returned empty arn for %s:%d", lbName, port)
}
return nil
return arn, nil
}
func lookupLBByName(nlbp nlbProvisioner, accountID, name string) (*elbv2.LoadBalancer, error) {
spinifex/handlers/eks/nlb_test.go
+21 -20
View File
@@ -2,6 +2,7 @@ package handlers_eks
import (
"errors"
"fmt"
"strings"
"testing"
@@ -39,7 +40,6 @@ type fakeNLBProvisioner struct {
createLBOut *elbv2.CreateLoadBalancerOutput
createTGOut *elbv2.CreateTargetGroupOutput
createListenerOut *elbv2.CreateListenerOutput
createLBErr error
createTGErr error
createListenerErr error
@@ -210,25 +210,20 @@ func (f *fakeNLBProvisioner) CreateListener(input *elbv2.CreateListenerInput, _
if f.createListenerErr != nil {
return nil, f.createListenerErr
}
if f.createListenerOut == nil {
lbArn := aws.StringValue(input.LoadBalancerArn)
port := aws.Int64Value(input.Port)
listenerArn := lbArn + "/listener/lst-001"
f.createListenerOut = &elbv2.CreateListenerOutput{
Listeners: []*elbv2.Listener{{
ListenerArn: aws.String(listenerArn),
LoadBalancerArn: input.LoadBalancerArn,
Port: input.Port,
Protocol: input.Protocol,
DefaultActions: input.DefaultActions,
}},
}
if f.listenerByPort[lbArn] == nil {
f.listenerByPort[lbArn] = map[int64]*elbv2.Listener{}
}
f.listenerByPort[lbArn][port] = f.createListenerOut.Listeners[0]
lbArn := aws.StringValue(input.LoadBalancerArn)
port := aws.Int64Value(input.Port)
listener := &elbv2.Listener{
ListenerArn: aws.String(fmt.Sprintf("%s/listener/lst-%d", lbArn, port)),
LoadBalancerArn: input.LoadBalancerArn,
Port: input.Port,
Protocol: input.Protocol,
DefaultActions: input.DefaultActions,
}
return f.createListenerOut, nil
if f.listenerByPort[lbArn] == nil {
f.listenerByPort[lbArn] = map[int64]*elbv2.Listener{}
}
f.listenerByPort[lbArn][port] = listener
return &elbv2.CreateListenerOutput{Listeners: []*elbv2.Listener{listener}}, nil
}
func (f *fakeNLBProvisioner) DescribeListeners(input *elbv2.DescribeListenersInput, _ string) (*elbv2.DescribeListenersOutput, error) {
@@ -315,7 +310,7 @@ func TestEnsureClusterNLB_FreshCreatesAllThree(t *testing.T) {
assert.Equal(t, elbv2.TargetTypeEnumIp, aws.StringValue(tgIn.TargetType))
assertELBv2TaggedAsEKS(t, tgIn.Tags, "alpha")
require.Len(t, nlbp.createListenerCalls, 1)
require.Len(t, nlbp.createListenerCalls, 2)
lstIn := nlbp.createListenerCalls[0]
assert.Equal(t, out.LoadBalancerArn, aws.StringValue(lstIn.LoadBalancerArn))
assert.Equal(t, elbv2.ProtocolEnumTcp, aws.StringValue(lstIn.Protocol))
@@ -323,6 +318,12 @@ func TestEnsureClusterNLB_FreshCreatesAllThree(t *testing.T) {
require.Len(t, lstIn.DefaultActions, 1)
assert.Equal(t, elbv2.ActionTypeEnumForward, aws.StringValue(lstIn.DefaultActions[0].Type))
assert.Equal(t, out.TargetGroupArn, aws.StringValue(lstIn.DefaultActions[0].TargetGroupArn))
// Second listener serves :6443 to the same TG — the in-cluster kubernetes Endpoints path.
apiLstIn := nlbp.createListenerCalls[1]
assert.Equal(t, k3sAPIServerPort, aws.Int64Value(apiLstIn.Port))
assert.Equal(t, elbv2.ProtocolEnumTcp, aws.StringValue(apiLstIn.Protocol))
require.Len(t, apiLstIn.DefaultActions, 1)
assert.Equal(t, out.TargetGroupArn, aws.StringValue(apiLstIn.DefaultActions[0].TargetGroupArn))
}
func TestEnsureClusterNLB_NoFrontendIPFailsLoud(t *testing.T) {
spinifex/handlers/eks/nodegroup.go
+250 -106
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"log/slog"
"maps"
"slices"
"sort"
"strings"
"time"
@@ -32,6 +33,10 @@ const (
defaultNodegroupReadyPoll = 15 * time.Second
)
// ngCASMaxRetries bounds the compare-and-swap retry loop that serializes
// concurrent nodegroup-record mutations (overlapping UpdateNodegroupConfig).
const ngCASMaxRetries = 16
// ErrNodegroupNotFound is returned by GetNodegroupRecord when the record key is
// absent. Callers translate to the AWS shape (ResourceNotFoundException) at the
// service boundary.
@@ -382,57 +387,17 @@ func (s *EKSServiceImpl) waitWorkersReady(acctKV nats.KeyValue, cluster string,
// the launched instance IDs. On a partial failure it returns the IDs that did
// launch plus the error so the caller can persist them for teardown.
func (s *EKSServiceImpl) launchWorkers(acctKV nats.KeyValue, accountID string, rec *NodegroupRecord, meta *ClusterMeta, ngSGID, amiID, token string, count int) ([]string, error) {
instanceType := defaultNodegroupInstanceType
if len(rec.InstanceTypes) > 0 {
instanceType = rec.InstanceTypes[0]
}
subnet := rec.Subnets[0]
// base is the worker set already on the record (non-empty on a scale-up).
// Each incremental persist below writes base+newly-launched so the durable
// record always reflects every live worker, never just this call's additions.
base := append([]string(nil), rec.InstanceIDs...)
ids := make([]string, 0, count)
for i := range count {
shortID := uuid.NewString()[:8]
userData := buildAgentUserData(agentUserDataInput{
ClusterName: rec.ClusterName,
NodegroupName: rec.Name,
ServerURL: clusterJoinEndpoint(meta),
JoinToken: token,
NodeName: fmt.Sprintf("%s-%s-%s", rec.ClusterName, rec.Name, shortID),
})
runInput := &ec2.RunInstancesInput{
ImageId: aws.String(amiID),
InstanceType: aws.String(instanceType),
MinCount: aws.Int64(1),
MaxCount: aws.Int64(1),
SubnetId: aws.String(subnet),
SecurityGroupIds: aws.StringSlice([]string{ngSGID}),
UserData: aws.String(userData),
TagSpecifications: []*ec2.TagSpecification{{
ResourceType: aws.String("instance"),
Tags: []*ec2.Tag{
{Key: aws.String(clusterEKSClusterTagKey), Value: aws.String(rec.ClusterName)},
{Key: aws.String(clusterEKSNodegroupTagKey), Value: aws.String(rec.Name)},
},
}},
}
if rec.DiskSize > 0 {
runInput.BlockDeviceMappings = []*ec2.BlockDeviceMapping{{
DeviceName: aws.String("/dev/vda"),
Ebs: &ec2.EbsBlockDevice{VolumeSize: aws.Int64(rec.DiskSize)},
}}
}
res, err := s.deps.Worker.RunWorkerInstance(runInput, accountID)
id, err := s.launchOneWorker(rec, meta, ngSGID, amiID, token, accountID)
if err != nil {
return ids, fmt.Errorf("run worker %d/%d: %w", i+1, count, err)
}
for _, inst := range res.Instances {
if id := aws.StringValue(inst.InstanceId); id != "" {
ids = append(ids, id)
}
}
ids = append(ids, id)
// Persist the launched worker IDs before issuing the next RunInstances so
// a crash mid-loop leaves every live worker recorded and reclaimable (by
// DeleteNodegroup or the boot reclaim sweep). Without this the record keeps
@@ -446,6 +411,57 @@ func (s *EKSServiceImpl) launchWorkers(acctKV nats.KeyValue, accountID string, r
return ids, nil
}
// launchOneWorker provisions a single tagged worker VM and returns its instance
// ID. It performs no durable record write; the caller owns persistence (plain
// for the single-owner create path, CAS-append for concurrent scale-up).
func (s *EKSServiceImpl) launchOneWorker(rec *NodegroupRecord, meta *ClusterMeta, ngSGID, amiID, token, accountID string) (string, error) {
instanceType := defaultNodegroupInstanceType
if len(rec.InstanceTypes) > 0 {
instanceType = rec.InstanceTypes[0]
}
subnet := rec.Subnets[0]
shortID := uuid.NewString()[:8]
userData := buildAgentUserData(agentUserDataInput{
ClusterName: rec.ClusterName,
NodegroupName: rec.Name,
ServerURL: clusterJoinEndpoint(meta),
JoinToken: token,
NodeName: fmt.Sprintf("%s-%s-%s", rec.ClusterName, rec.Name, shortID),
})
runInput := &ec2.RunInstancesInput{
ImageId: aws.String(amiID),
InstanceType: aws.String(instanceType),
MinCount: aws.Int64(1),
MaxCount: aws.Int64(1),
SubnetId: aws.String(subnet),
SecurityGroupIds: aws.StringSlice([]string{ngSGID}),
UserData: aws.String(userData),
TagSpecifications: []*ec2.TagSpecification{{
ResourceType: aws.String("instance"),
Tags: []*ec2.Tag{
{Key: aws.String(clusterEKSClusterTagKey), Value: aws.String(rec.ClusterName)},
{Key: aws.String(clusterEKSNodegroupTagKey), Value: aws.String(rec.Name)},
},
}},
}
if rec.DiskSize > 0 {
runInput.BlockDeviceMappings = []*ec2.BlockDeviceMapping{{
DeviceName: aws.String("/dev/vda"),
Ebs: &ec2.EbsBlockDevice{VolumeSize: aws.Int64(rec.DiskSize)},
}}
}
res, err := s.deps.Worker.RunWorkerInstance(runInput, accountID)
if err != nil {
return "", err
}
for _, inst := range res.Instances {
if id := aws.StringValue(inst.InstanceId); id != "" {
return id, nil
}
}
return "", errors.New("run worker returned no instance id")
}
func (s *EKSServiceImpl) describeNodegroup(acctKV nats.KeyValue, input *eks.DescribeNodegroupInput) (*eks.DescribeNodegroupOutput, error) {
cluster := aws.StringValue(input.ClusterName)
ng := aws.StringValue(input.NodegroupName)
@@ -500,41 +516,9 @@ func (s *EKSServiceImpl) updateNodegroupConfig(acctKV nats.KeyValue, input *eks.
}
return nil, err
}
rec, err := GetNodegroupRecord(acctKV, cluster, ng)
rec, err := s.reconcileNodegroup(acctKV, accountID, cluster, ng, meta, input.ScalingConfig, input.Labels)
if err != nil {
if errors.Is(err, ErrNodegroupNotFound) {
return nil, errors.New(awserrors.ErrorEKSResourceNotFound)
}
return nil, err
}
if input.ScalingConfig != nil {
if v := input.ScalingConfig.MinSize; v != nil {
rec.ScalingMin = *v
}
if v := input.ScalingConfig.MaxSize; v != nil {
rec.ScalingMax = *v
}
if v := input.ScalingConfig.DesiredSize; v != nil {
rec.ScalingDesired = *v
}
}
if input.Labels != nil {
rec.Labels = applyLabelUpdate(rec.Labels, input.Labels)
}
if err := s.reconcileWorkerCount(acctKV, accountID, rec, meta); err != nil {
// Persist partial IDs so DeleteNodegroup can reclaim a failed scale-up.
rec.ModifiedAt = time.Now().UTC()
if perr := PutNodegroupRecord(acctKV, rec); perr != nil {
slog.Error("updateNodegroupConfig: persist after reconcile failure",
"cluster", cluster, "nodegroup", ng, "err", perr)
}
return nil, err
}
rec.ModifiedAt = time.Now().UTC()
if err := PutNodegroupRecord(acctKV, rec); err != nil {
return nil, err
}
@@ -546,42 +530,202 @@ func (s *EKSServiceImpl) updateNodegroupConfig(acctKV nats.KeyValue, input *eks.
}}, nil
}
// reconcileWorkerCount launches or terminates workers so len(InstanceIDs)
// matches ScalingDesired. Surplus removal terminates the last (highest-index)
// instance IDs so scale-down is deterministic.
func (s *EKSServiceImpl) reconcileWorkerCount(acctKV nats.KeyValue, accountID string, rec *NodegroupRecord, meta *ClusterMeta) error {
current := len(rec.InstanceIDs)
desired := int(rec.ScalingDesired)
switch {
case desired > current:
token, err := s.decryptNodeToken(acctKV, rec.ClusterName)
// reconcileNodegroup applies the scaling/label deltas and converges the worker
// count to ScalingDesired under compare-and-swap. Every durable mutation is a
// CAS on the record revision, so two overlapping UpdateNodegroupConfig calls can
// never both launch the same delta — the lost-update that scaled 1 worker to 5
// (two reconciles each reading current=1 and each launching desired-current=2).
func (s *EKSServiceImpl) reconcileNodegroup(acctKV nats.KeyValue, accountID, cluster, ng string, meta *ClusterMeta, scaling *eks.NodegroupScalingConfig, labels *eks.UpdateLabelsPayload) (*NodegroupRecord, error) {
for range ngCASMaxRetries {
rec, rev, err := getNodegroupEntry(acctKV, cluster, ng)
if err != nil {
return fmt.Errorf("decrypt node token: %w", err)
}
amiID, err := lookupEKSServerAMI(s.deps.Image, accountID)
if err != nil {
if errors.Is(err, ErrEKSServerAMINotFound) {
return errors.New(awserrors.ErrorServiceUnavailable)
if errors.Is(err, ErrNodegroupNotFound) {
return nil, errors.New(awserrors.ErrorEKSResourceNotFound)
}
return fmt.Errorf("resolve eks-node AMI: %w", err)
return nil, err
}
_, ngSGID, err := EnsureClusterSGs(s.deps.VPCSG, accountID, rec.ClusterName, meta.ResourcesVpcConfig.VpcId)
if err != nil {
return fmt.Errorf("ensure cluster SGs: %w", err)
applyScalingUpdate(rec, scaling)
if labels != nil {
rec.Labels = applyLabelUpdate(rec.Labels, labels)
}
// launchWorkers appends to and persists rec.InstanceIDs incrementally.
if _, err := s.launchWorkers(acctKV, accountID, rec, meta, ngSGID, amiID, token, desired-current); err != nil {
return fmt.Errorf("launch workers: %w", err)
desired := int(rec.ScalingDesired)
current := len(rec.InstanceIDs)
rec.ModifiedAt = time.Now().UTC()
switch {
case desired < current:
// Terminate surplus before shrinking the record. A crash or CAS retry
// between the two leaves dead IDs in the record (a stale over-count the
// next scale prunes), never a running VM absent from every record — an
// orphan that record-driven reclaim cannot reach. Terminate is
// idempotent, so re-terminating on a CAS retry is safe.
surplus := rec.InstanceIDs[desired:]
if err := s.deps.Worker.TerminateWorkerInstances(surplus, accountID); err != nil {
return nil, fmt.Errorf("terminate surplus workers: %w", err)
}
rec.InstanceIDs = rec.InstanceIDs[:desired]
if ok, err := s.casPutNodegroup(acctKV, rec, rev); err != nil {
return nil, err
} else if !ok {
continue
}
return rec, nil
case desired > current:
// Commit the deltas + new desired first, then launch the gap one worker
// at a time with a per-VM CAS-append (launchWorkersCAS).
if ok, err := s.casPutNodegroup(acctKV, rec, rev); err != nil {
return nil, err
} else if !ok {
continue
}
return s.launchWorkersCAS(acctKV, accountID, cluster, ng, meta, desired)
default:
if ok, err := s.casPutNodegroup(acctKV, rec, rev); err != nil {
return nil, err
} else if !ok {
continue
}
return rec, nil
}
case desired < current:
surplus := rec.InstanceIDs[desired:]
if err := s.deps.Worker.TerminateWorkerInstances(surplus, accountID); err != nil {
return fmt.Errorf("terminate surplus workers: %w", err)
}
rec.InstanceIDs = rec.InstanceIDs[:desired]
}
return nil
return nil, errors.New(awserrors.ErrorServerInternal)
}
// launchWorkersCAS converges the worker count up to desired, launching one
// worker at a time and CAS-appending its ID only while len(InstanceIDs) <
// desired. A worker launched into an already-full record (a concurrent reconcile
// got there first) is terminated, so the count never overshoots desired.
func (s *EKSServiceImpl) launchWorkersCAS(acctKV nats.KeyValue, accountID, cluster, ng string, meta *ClusterMeta, desired int) (*NodegroupRecord, error) {
token, err := s.decryptNodeToken(acctKV, cluster)
if err != nil {
return nil, fmt.Errorf("decrypt node token: %w", err)
}
amiID, err := lookupEKSServerAMI(s.deps.Image, accountID)
if err != nil {
if errors.Is(err, ErrEKSServerAMINotFound) {
return nil, errors.New(awserrors.ErrorServiceUnavailable)
}
return nil, fmt.Errorf("resolve eks-node AMI: %w", err)
}
_, ngSGID, err := EnsureClusterSGs(s.deps.VPCSG, accountID, cluster, meta.ResourcesVpcConfig.VpcId)
if err != nil {
return nil, fmt.Errorf("ensure cluster SGs: %w", err)
}
// Each iteration records at most one worker, so bound by the gap plus CAS slack.
// The target is re-read from the record each pass (not the entry-time desired),
// so a concurrent rescale is honored rather than overrun.
for range desired + ngCASMaxRetries {
rec, _, err := getNodegroupEntry(acctKV, cluster, ng)
if err != nil {
return nil, err
}
target := int(rec.ScalingDesired)
if len(rec.InstanceIDs) >= target {
return rec, nil
}
id, err := s.launchOneWorker(rec, meta, ngSGID, amiID, token, accountID)
if err != nil {
// Surface a client-facing code (e.g. InsufficientInstanceCapacity)
// verbatim so the gateway maps its real status; wrap only opaque
// internal failures, which stay 500.
if awserrors.HasErrorCode(err.Error()) {
return nil, err
}
return nil, fmt.Errorf("launch workers: %w", err)
}
if _, err := s.recordLaunchedWorker(acctKV, accountID, cluster, ng, id); err != nil {
return nil, err
}
}
return nil, errors.New(awserrors.ErrorServerInternal)
}
// recordLaunchedWorker CAS-appends id while the record is below its live
// ScalingDesired, or terminates the worker when a concurrent reconcile already
// filled the gap (or lowered the target). Returns true when id was recorded.
func (s *EKSServiceImpl) recordLaunchedWorker(kv nats.KeyValue, accountID, cluster, ng, id string) (bool, error) {
for range ngCASMaxRetries {
rec, rev, err := getNodegroupEntry(kv, cluster, ng)
if err != nil {
return false, err
}
if slices.Contains(rec.InstanceIDs, id) {
return true, nil
}
if len(rec.InstanceIDs) >= int(rec.ScalingDesired) {
if terr := s.deps.Worker.TerminateWorkerInstances([]string{id}, accountID); terr != nil {
return false, fmt.Errorf("terminate surplus worker: %w", terr)
}
return false, nil
}
rec.InstanceIDs = append(rec.InstanceIDs, id)
rec.ModifiedAt = time.Now().UTC()
if ok, err := s.casPutNodegroup(kv, rec, rev); err != nil {
return false, err
} else if ok {
return true, nil
}
}
// Could not record within the CAS budget; terminate so the VM is not orphaned.
if terr := s.deps.Worker.TerminateWorkerInstances([]string{id}, accountID); terr != nil {
return false, fmt.Errorf("terminate unrecorded worker: %w", terr)
}
return false, errors.New(awserrors.ErrorServerInternal)
}
// applyScalingUpdate copies the non-nil min/max/desired fields of an
// UpdateNodegroupConfig scaling delta onto the record.
func applyScalingUpdate(rec *NodegroupRecord, scaling *eks.NodegroupScalingConfig) {
if scaling == nil {
return
}
if v := scaling.MinSize; v != nil {
rec.ScalingMin = *v
}
if v := scaling.MaxSize; v != nil {
rec.ScalingMax = *v
}
if v := scaling.DesiredSize; v != nil {
rec.ScalingDesired = *v
}
}
// getNodegroupEntry reads one record with its KV revision for CAS updates.
func getNodegroupEntry(kv nats.KeyValue, cluster, ng string) (*NodegroupRecord, uint64, error) {
if cluster == "" || ng == "" {
return nil, 0, errors.New("eks: getNodegroupEntry empty cluster or name")
}
entry, err := kv.Get(NodegroupKey(cluster, ng))
if err != nil {
if errors.Is(err, nats.ErrKeyNotFound) {
return nil, 0, ErrNodegroupNotFound
}
return nil, 0, fmt.Errorf("kv get nodegroup: %w", err)
}
var rec NodegroupRecord
if err := json.Unmarshal(entry.Value(), &rec); err != nil {
return nil, 0, fmt.Errorf("unmarshal nodegroup %s: %w", ng, err)
}
return &rec, entry.Revision(), nil
}
// casPutNodegroup writes rec only if the durable revision still matches rev.
// ok=false signals a concurrent writer won; the caller re-reads and retries.
func (s *EKSServiceImpl) casPutNodegroup(kv nats.KeyValue, rec *NodegroupRecord, rev uint64) (bool, error) {
data, err := json.Marshal(rec)
if err != nil {
return false, fmt.Errorf("marshal nodegroup %s: %w", rec.Name, err)
}
if _, err := kv.Update(NodegroupKey(rec.ClusterName, rec.Name), data, rev); err != nil {
if errors.Is(err, nats.ErrKeyExists) {
return false, nil
}
return false, fmt.Errorf("kv update nodegroup %s: %w", rec.Name, err)
}
return true, nil
}
func (s *EKSServiceImpl) updateNodegroupVersion(input *eks.UpdateNodegroupVersionInput) (*eks.UpdateNodegroupVersionOutput, error) {
spinifex/handlers/eks/nodegroup_test.go
+111 -1
View File
@@ -4,6 +4,7 @@ import (
"errors"
"strconv"
"strings"
"sync"
"testing"
"github.com/aws/aws-sdk-go/aws"
@@ -18,8 +19,10 @@ import (
// fakeWorkerLauncher records RunWorkerInstance / TerminateWorkerInstances calls
// and hands back sequential instance IDs so tests can assert on the IDs a
// nodegroup tracks.
// nodegroup tracks. It is mutex-guarded so concurrent-reconcile tests exercise
// the production CAS path, not a race in the fake.
type fakeWorkerLauncher struct {
mu sync.Mutex
runCalls []*ec2.RunInstancesInput
terminateCalls [][]string
@@ -35,6 +38,8 @@ func newFakeWorkerLauncher() *fakeWorkerLauncher {
}
func (f *fakeWorkerLauncher) RunWorkerInstance(input *ec2.RunInstancesInput, _ string) (*ec2.Reservation, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.runCalls = append(f.runCalls, input)
if f.runErr != nil {
return nil, f.runErr
@@ -45,10 +50,30 @@ func (f *fakeWorkerLauncher) RunWorkerInstance(input *ec2.RunInstancesInput, _ s
}
func (f *fakeWorkerLauncher) TerminateWorkerInstances(ids []string, _ string) error {
f.mu.Lock()
defer f.mu.Unlock()
f.terminateCalls = append(f.terminateCalls, ids)
return f.termErr
}
// runCount / terminatedCount return the recorded call totals under the lock so
// concurrent tests can read them safely after joining.
func (f *fakeWorkerLauncher) runCount() int {
f.mu.Lock()
defer f.mu.Unlock()
return len(f.runCalls)
}
func (f *fakeWorkerLauncher) terminatedCount() int {
f.mu.Lock()
defer f.mu.Unlock()
var n int
for _, ids := range f.terminateCalls {
n += len(ids)
}
return n
}
// seedActiveClusterWithToken lays down an ACTIVE cluster meta with the control-
// plane ENI IP + VPC populated and an encrypted K3s join token, so the
// nodegroup path can run end-to-end.
@@ -196,6 +221,10 @@ func TestBuildAgentUserData_Shape(t *testing.T) {
assert.Contains(t, ud, agentEnvPath)
// IMDS on-link route line present.
assert.Contains(t, ud, "ip route replace "+imdsServerIP+"/32")
// Route script is run directly, not via `rc-service local start` — that would
// deadlock against cloud-final and stall the boot ~50s on the OpenRC timeout.
assert.Contains(t, ud, "[ /etc/local.d/imds-onlink-route.start ]")
assert.NotContains(t, ud, "rc-service, local, start")
// Exactly one write_files block (single "write_files:" key).
assert.Equal(t, 1, strings.Count(ud, "write_files:"))
}
@@ -412,6 +441,87 @@ func TestUpdateNodegroupConfig_ScaleUpAndDown(t *testing.T) {
assert.Len(t, rec.InstanceIDs, 1)
}
// TestUpdateNodegroupConfig_ConcurrentScaleUpConverges proves the CAS reconcile
// fixes the lost-update overshoot: two overlapping scale-up requests (UI +
// terraform retry) must converge to exactly desired, not double-launch. Pre-fix
// each read current=1 and launched desired-current=2, yielding 5 live workers.
func TestUpdateNodegroupConfig_ConcurrentScaleUpConverges(t *testing.T) {
f := newEKSServiceFixture(t)
seedActiveClusterWithToken(t, f, "c1")
_, err := f.svc.CreateNodegroup(createNGInput("c1", "ng1", 1), testAccountID)
require.NoError(t, err)
markWorkersReady(t, f, "c1", 1)
f.svc.WaitLaunches()
require.Len(t, f.worker.runCalls, 1)
const goroutines = 2
var wg sync.WaitGroup
start := make(chan struct{})
errs := make([]error, goroutines)
for i := range goroutines {
wg.Add(1)
go func(idx int) {
defer wg.Done()
<-start
_, errs[idx] = f.svc.UpdateNodegroupConfig(&eks.UpdateNodegroupConfigInput{
ClusterName: aws.String("c1"),
NodegroupName: aws.String("ng1"),
ScalingConfig: &eks.NodegroupScalingConfig{DesiredSize: aws.Int64(3)},
}, testAccountID)
}(i)
}
close(start)
wg.Wait()
for _, e := range errs {
require.NoError(t, e)
}
rec, err := GetNodegroupRecord(f.kv, "c1", "ng1")
require.NoError(t, err)
assert.Len(t, rec.InstanceIDs, 3, "record must converge to desired=3, not overshoot")
// Net live workers (Run − Terminate) must equal desired: a surplus VM launched
// under the exact-boundary race is terminated, never left running.
assert.Equal(t, 3, f.worker.runCount()-f.worker.terminatedCount(),
"net live workers must equal desired=3")
}
// TestUpdateNodegroupConfig_CapacityErrorSurfacesCode proves an out-of-capacity
// scale returns the bare InsufficientInstanceCapacity code (gateway maps to 400),
// not a wrapped string the gateway sanitizes to 500. A generic launcher failure
// still wraps, staying an opaque 500.
func TestUpdateNodegroupConfig_CapacityErrorSurfacesCode(t *testing.T) {
scaleUpWithRunErr := func(t *testing.T, runErr error) error {
f := newEKSServiceFixture(t)
seedActiveClusterWithToken(t, f, "c1")
_, err := f.svc.CreateNodegroup(createNGInput("c1", "ng1", 1), testAccountID)
require.NoError(t, err)
markWorkersReady(t, f, "c1", 1)
f.svc.WaitLaunches()
f.worker.runErr = runErr
_, err = f.svc.UpdateNodegroupConfig(&eks.UpdateNodegroupConfigInput{
ClusterName: aws.String("c1"),
NodegroupName: aws.String("ng1"),
ScalingConfig: &eks.NodegroupScalingConfig{DesiredSize: aws.Int64(3)},
}, testAccountID)
return err
}
t.Run("capacity code preserved", func(t *testing.T) {
err := scaleUpWithRunErr(t, errors.New(awserrors.ErrorInsufficientInstanceCapacity))
require.EqualError(t, err, awserrors.ErrorInsufficientInstanceCapacity)
assert.True(t, awserrors.HasErrorCode(err.Error()))
})
t.Run("opaque error wrapped", func(t *testing.T) {
err := scaleUpWithRunErr(t, errors.New("kvm: out of host memory"))
require.Error(t, err)
assert.Contains(t, err.Error(), "launch workers:")
assert.False(t, awserrors.HasErrorCode(err.Error()))
})
}
func TestDeleteNodegroup_TerminatesAndIsIdempotent(t *testing.T) {
f := newEKSServiceFixture(t)
seedActiveClusterWithToken(t, f, "c1")
spinifex/handlers/eks/nodegroup_userdata.go
+4 -1
View File
@@ -69,9 +69,12 @@ func buildAgentUserData(in agentUserDataInput) string {
}
}
// Enable OpenRC `local` for reboot persistence, then run the IMDS route script
// directly. Starting the service here would deadlock: runcmd runs inside
// cloud-final, but `local` is ordered after it — blocking until OpenRC times out.
buf.WriteString("runcmd:\n")
buf.WriteString(" - [ rc-update, add, local, default ]\n")
buf.WriteString(" - [ rc-service, local, start ]\n")
buf.WriteString(" - [ /etc/local.d/imds-onlink-route.start ]\n")
return buf.String()
}
spinifex/handlers/eks/private_endpoint_test.go
+16 -11
View File
@@ -49,7 +49,7 @@ func TestClusterJoinEndpoint_PublicOnlyUsesPublished(t *testing.T) {
assert.Equal(t, "https://203.0.113.9:443", clusterJoinEndpoint(meta))
}
func TestEnsurePrivateEndpointSG_AuthorizesVPCCIDROn443(t *testing.T) {
func TestEnsurePrivateEndpointSG_AuthorizesVPCCIDROnAPIServerPorts(t *testing.T) {
sgp := newFakeSGProvisioner()
sgp.createIDs = []string{"sg-pe-001"}
@@ -57,16 +57,21 @@ func TestEnsurePrivateEndpointSG_AuthorizesVPCCIDROn443(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, "sg-pe-001", sgID)
require.Len(t, sgp.authorizeCalls, 1)
in := sgp.authorizeCalls[0]
assert.Equal(t, "sg-pe-001", aws.StringValue(in.GroupId))
require.Len(t, in.IpPermissions, 1)
perm := in.IpPermissions[0]
assert.Equal(t, "tcp", aws.StringValue(perm.IpProtocol))
assert.Equal(t, clusterNLBListenPort, aws.Int64Value(perm.FromPort))
assert.Equal(t, clusterNLBListenPort, aws.Int64Value(perm.ToPort))
require.Len(t, perm.IpRanges, 1)
assert.Equal(t, "10.0.0.0/16", aws.StringValue(perm.IpRanges[0].CidrIp))
// :443 for kubectl/SDK; :6443 for worker pods hitting the in-cluster kubernetes Endpoints.
require.Len(t, sgp.authorizeCalls, 2)
ports := map[int64]bool{}
for _, in := range sgp.authorizeCalls {
assert.Equal(t, "sg-pe-001", aws.StringValue(in.GroupId))
require.Len(t, in.IpPermissions, 1)
perm := in.IpPermissions[0]
assert.Equal(t, "tcp", aws.StringValue(perm.IpProtocol))
assert.Equal(t, aws.Int64Value(perm.FromPort), aws.Int64Value(perm.ToPort))
require.Len(t, perm.IpRanges, 1)
assert.Equal(t, "10.0.0.0/16", aws.StringValue(perm.IpRanges[0].CidrIp))
ports[aws.Int64Value(perm.FromPort)] = true
}
assert.True(t, ports[clusterNLBListenPort], "admits :443")
assert.True(t, ports[k3sAPIServerPort], "admits :6443")
}
func TestEnsurePrivateEndpointSG_EmptyInputsRejected(t *testing.T) {
spinifex/handlers/eks/security_groups.go
+116 -29
View File
@@ -4,6 +4,7 @@ import (
"errors"
"fmt"
"log/slog"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
@@ -17,6 +18,7 @@ type sgProvisioner interface {
DescribeSecurityGroups(input *ec2.DescribeSecurityGroupsInput, accountID string) (*ec2.DescribeSecurityGroupsOutput, error)
DeleteSecurityGroup(input *ec2.DeleteSecurityGroupInput, accountID string) (*ec2.DeleteSecurityGroupOutput, error)
AuthorizeSecurityGroupIngress(input *ec2.AuthorizeSecurityGroupIngressInput, accountID string) (*ec2.AuthorizeSecurityGroupIngressOutput, error)
RevokeSecurityGroupIngress(input *ec2.RevokeSecurityGroupIngressInput, accountID string) (*ec2.RevokeSecurityGroupIngressOutput, error)
}
// clusterEKSClusterTagKey groups all cluster-scoped resources for DeleteCluster sweeps.
@@ -86,7 +88,12 @@ func EnsureClusterSGs(sgp sgProvisioner, accountID, clusterName, vpcID string) (
return controlPlaneSGID, nodegroupSGID, nil
}
// DeleteClusterSGs removes both cluster SGs. Missing SGs are no-ops; first error is returned.
// DeleteClusterSGs removes both cluster SGs. The control-plane and nodegroup SGs
// cross-reference each other (EnsureNodegroupSGRules), and AWS refuses to delete a
// security group still referenced by another SG's rule — so every ingress rule on
// both is revoked first to break the cycle, then both are deleted. Skipping the
// revoke leaks the SGs and pins the VPC on DependencyViolation. Missing SGs are
// no-ops; the first error is returned after the full sweep.
func DeleteClusterSGs(sgp sgProvisioner, accountID, clusterName, vpcID string) error {
if clusterName == "" {
return errors.New("eks: DeleteClusterSGs empty cluster name")
@@ -96,32 +103,110 @@ func DeleteClusterSGs(sgp sgProvisioner, accountID, clusterName, vpcID string) e
}
var firstErr error
record := func(err error) {
if err != nil && firstErr == nil {
firstErr = err
}
}
// Resolve both IDs up front so all cross-references can be revoked before any
// delete is attempted.
var ids []string
for _, name := range []string{ClusterControlPlaneSGName(clusterName), ClusterNodegroupSGName(clusterName)} {
id, err := lookupSGByName(sgp, accountID, vpcID, name)
if err != nil {
slog.Warn("DeleteClusterSGs: SG lookup failed", "sg", name, "err", err)
if firstErr == nil {
firstErr = err
}
record(err)
continue
}
if id == "" {
continue
if id != "" {
ids = append(ids, id)
}
if _, err := sgp.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{GroupId: aws.String(id)}, accountID); err != nil && !awserrors.IsNotFound(err) {
slog.Warn("DeleteClusterSGs: SG delete failed", "sg", id, "name", name, "err", err)
if firstErr == nil {
firstErr = fmt.Errorf("delete SG %s: %w", id, err)
}
}
// Break the cp<->ng cross-references before deleting either SG.
for _, id := range ids {
if err := revokeAllSGIngress(sgp, accountID, id); err != nil {
slog.Warn("DeleteClusterSGs: revoke ingress failed", "sg", id, "err", err)
record(err)
}
}
for _, id := range ids {
if err := deleteSGAwaitingDetach(sgp, accountID, id); err != nil {
slog.Warn("DeleteClusterSGs: SG delete failed", "sg", id, "err", err)
record(fmt.Errorf("delete SG %s: %w", id, err))
}
}
return firstErr
}
var (
// sgDeleteWaitBudget bounds how long deleteSGAwaitingDetach retries a
// DeleteSecurityGroup returning DependencyViolation while a terminating
// worker's ENI still references the nodegroup SG. The instance-terminate
// cascade releases the ENI asynchronously, so the immediate delete can lose
// the race. Vars (not consts) so tests can shrink them.
sgDeleteWaitBudget = 45 * time.Second
sgDeleteWaitInterval = 1 * time.Second
)
// deleteSGAwaitingDetach deletes a cluster SG, tolerating the DependencyViolation
// window while the async instance-terminate cascade releases the last worker ENI
// still referencing it. A missing SG is idempotent success. A persistent
// DependencyViolation past the budget is surfaced so the teardown backstop retries
// rather than leaking the SG — a leaked cluster SG sits in the customer VPC and
// pins it on DependencyViolation.
func deleteSGAwaitingDetach(sgp sgProvisioner, accountID, sgID string) error {
deadline := time.Now().Add(sgDeleteWaitBudget)
for {
_, err := sgp.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{GroupId: aws.String(sgID)}, accountID)
switch {
case err == nil, awserrors.IsNotFound(err):
return nil
case awserrors.IsErrorCode(err, awserrors.ErrorDependencyViolation):
if time.Now().After(deadline) {
return err
}
time.Sleep(sgDeleteWaitInterval)
default:
return err
}
}
}
// revokeAllSGIngress clears every ingress rule on an SG so cross-references from a
// sibling SG no longer block its deletion. A missing SG or one with no ingress is
// a no-op.
func revokeAllSGIngress(sgp sgProvisioner, accountID, sgID string) error {
out, err := sgp.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{
GroupIds: aws.StringSlice([]string{sgID}),
}, accountID)
if err != nil {
if awserrors.IsNotFound(err) {
return nil
}
return fmt.Errorf("describe SG %s: %w", sgID, err)
}
for _, g := range out.SecurityGroups {
if g == nil || len(g.IpPermissions) == 0 {
continue
}
if _, err := sgp.RevokeSecurityGroupIngress(&ec2.RevokeSecurityGroupIngressInput{
GroupId: aws.String(sgID),
IpPermissions: g.IpPermissions,
}, accountID); err != nil && !awserrors.IsNotFound(err) {
return fmt.Errorf("revoke ingress on %s: %w", sgID, err)
}
}
return nil
}
// EnsurePrivateEndpointSG creates (or reuses) the customer-VPC SG for the
// private-endpoint ENI and admits the customer VPC CIDR on the NLB listen port
// (:443), so in-VPC workers + kubectl reach the apiserver via the Set A NIC.
// Idempotent: SG lookup-or-create + duplicate-tolerant ingress authorize.
// private-endpoint ENI and admits the customer VPC CIDR on :443 (kubectl/SDK) and
// :6443, so in-VPC workers + kubectl reach the apiserver via the Set A NIC. :6443
// carries worker pods to the in-cluster `kubernetes` Endpoints (apiserver
// advertised on :6443). Idempotent: SG lookup-or-create + duplicate-tolerant authorize.
func EnsurePrivateEndpointSG(sgp sgProvisioner, accountID, clusterName, vpcID, vpcCIDR string) (string, error) {
if clusterName == "" {
return "", errors.New("eks: EnsurePrivateEndpointSG empty cluster name")
@@ -141,21 +226,23 @@ func EnsurePrivateEndpointSG(sgp sgProvisioner, accountID, clusterName, vpcID, v
return "", err
}
perm := &ec2.IpPermission{
IpProtocol: aws.String("tcp"),
FromPort: aws.Int64(clusterNLBListenPort),
ToPort: aws.Int64(clusterNLBListenPort),
IpRanges: []*ec2.IpRange{{
CidrIp: aws.String(vpcCIDR),
Description: aws.String("EKS in-VPC clients to private apiserver endpoint"),
}},
}
_, err = sgp.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{
GroupId: aws.String(sgID),
IpPermissions: []*ec2.IpPermission{perm},
}, accountID)
if err != nil && !awserrors.IsErrorCode(err, awserrors.ErrorInvalidPermissionDuplicate) {
return "", fmt.Errorf("authorize private-endpoint ingress on %s: %w", sgID, err)
for _, port := range []int64{clusterNLBListenPort, k3sAPIServerPort} {
perm := &ec2.IpPermission{
IpProtocol: aws.String("tcp"),
FromPort: aws.Int64(port),
ToPort: aws.Int64(port),
IpRanges: []*ec2.IpRange{{
CidrIp: aws.String(vpcCIDR),
Description: aws.String("EKS in-VPC clients to private apiserver endpoint"),
}},
}
_, err = sgp.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{
GroupId: aws.String(sgID),
IpPermissions: []*ec2.IpPermission{perm},
}, accountID)
if err != nil && !awserrors.IsErrorCode(err, awserrors.ErrorInvalidPermissionDuplicate) {
return "", fmt.Errorf("authorize private-endpoint :%d ingress on %s: %w", port, sgID, err)
}
}
return sgID, nil
}
spinifex/handlers/eks/security_groups_test.go
+128 -1
View File
@@ -2,7 +2,9 @@ package handlers_eks
import (
"errors"
"sync"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
@@ -13,14 +15,30 @@ import (
)
type fakeSGProvisioner struct {
mu sync.Mutex
createCalls []*ec2.CreateSecurityGroupInput
describeCalls []*ec2.DescribeSecurityGroupsInput
deleteCalls []*ec2.DeleteSecurityGroupInput
authorizeCalls []*ec2.AuthorizeSecurityGroupIngressInput
revokeCalls []*ec2.RevokeSecurityGroupIngressInput
// existing maps "name|vpcId" → groupId for DescribeSecurityGroups lookup.
existing map[string]string
// perms maps groupId → its current ingress rules, served by a GroupIds
// DescribeSecurityGroups and cleared by RevokeSecurityGroupIngress.
perms map[string][]*ec2.IpPermission
// enforceDepViolation models AWS refusing to delete an SG that still has
// ingress rules (e.g. a sibling cross-reference), so a test proves the
// revoke-before-delete ordering.
enforceDepViolation bool
// deleteFailsBefore maps groupId → the number of DeleteSecurityGroup calls
// that return DependencyViolation before succeeding, modelling a worker ENI
// detaching after a few retries.
deleteFailsBefore map[string]int
// nextCreateID is returned by the next CreateSecurityGroup call. Tests can
// pre-seed a queue via createIDs to differentiate the control-plane vs
// nodegroup SG IDs.
@@ -39,6 +57,8 @@ func newFakeSGProvisioner() *fakeSGProvisioner {
}
func (f *fakeSGProvisioner) CreateSecurityGroup(input *ec2.CreateSecurityGroupInput, _ string) (*ec2.CreateSecurityGroupOutput, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.createCalls = append(f.createCalls, input)
if f.createErr != nil {
return nil, f.createErr
@@ -55,10 +75,25 @@ func (f *fakeSGProvisioner) CreateSecurityGroup(input *ec2.CreateSecurityGroupIn
}
func (f *fakeSGProvisioner) DescribeSecurityGroups(input *ec2.DescribeSecurityGroupsInput, _ string) (*ec2.DescribeSecurityGroupsOutput, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.describeCalls = append(f.describeCalls, input)
if f.describeErr != nil {
return nil, f.describeErr
}
if len(input.GroupIds) > 0 {
out := &ec2.DescribeSecurityGroupsOutput{}
for _, gid := range input.GroupIds {
if gid == nil {
continue
}
out.SecurityGroups = append(out.SecurityGroups, &ec2.SecurityGroup{
GroupId: gid,
IpPermissions: f.perms[*gid],
})
}
return out, nil
}
var name, vpc string
for _, filt := range input.Filters {
if filt == nil || filt.Name == nil || len(filt.Values) == 0 || filt.Values[0] == nil {
@@ -80,14 +115,35 @@ func (f *fakeSGProvisioner) DescribeSecurityGroups(input *ec2.DescribeSecurityGr
}
func (f *fakeSGProvisioner) DeleteSecurityGroup(input *ec2.DeleteSecurityGroupInput, _ string) (*ec2.DeleteSecurityGroupOutput, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.deleteCalls = append(f.deleteCalls, input)
if f.deleteErr != nil {
return nil, f.deleteErr
}
if input.GroupId != nil && f.deleteFailsBefore[*input.GroupId] > 0 {
f.deleteFailsBefore[*input.GroupId]--
return nil, errors.New("DependencyViolation: resource has a dependent object")
}
if f.enforceDepViolation && input.GroupId != nil && len(f.perms[*input.GroupId]) > 0 {
return nil, errors.New("DependencyViolation: resource has a dependent object")
}
return &ec2.DeleteSecurityGroupOutput{}, nil
}
func (f *fakeSGProvisioner) RevokeSecurityGroupIngress(input *ec2.RevokeSecurityGroupIngressInput, _ string) (*ec2.RevokeSecurityGroupIngressOutput, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.revokeCalls = append(f.revokeCalls, input)
if input.GroupId != nil {
delete(f.perms, *input.GroupId)
}
return &ec2.RevokeSecurityGroupIngressOutput{}, nil
}
func (f *fakeSGProvisioner) AuthorizeSecurityGroupIngress(input *ec2.AuthorizeSecurityGroupIngressInput, _ string) (*ec2.AuthorizeSecurityGroupIngressOutput, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.authorizeCalls = append(f.authorizeCalls, input)
if f.authorizeErr != nil {
return nil, f.authorizeErr
@@ -183,7 +239,9 @@ func TestDeleteClusterSGs_FirstErrorSurfacedSweepContinues(t *testing.T) {
sgp := newFakeSGProvisioner()
sgp.existing["eks-cluster-alpha-control-plane-sg|vpc-aaa"] = "sg-existing-cp"
sgp.existing["eks-cluster-alpha-nodegroup-sg|vpc-aaa"] = "sg-existing-ng"
sgp.deleteErr = errors.New("DependencyViolation")
// A non-DependencyViolation error is not retried, so the sweep surfaces it
// after a single attempt per SG (the retry path is covered separately).
sgp.deleteErr = errors.New("vpcd unavailable")
err := DeleteClusterSGs(sgp, "111122223333", "alpha", "vpc-aaa")
require.Error(t, err)
@@ -191,6 +249,75 @@ func TestDeleteClusterSGs_FirstErrorSurfacedSweepContinues(t *testing.T) {
assert.Len(t, sgp.deleteCalls, 2, "both delete calls should be attempted despite first error")
}
// TestDeleteClusterSGs_RetriesDependencyViolation guards the worker-ENI-detach
// race: a terminating worker's ENI can still reference the nodegroup SG when the
// delete runs. DeleteSecurityGroup must retry the DependencyViolation until the
// instance-terminate cascade releases the ENI (delete succeeds) and surface a
// persistent one past the budget so the SG is never silently leaked — a leaked
// cluster SG pins the customer VPC.
func TestDeleteClusterSGs_RetriesDependencyViolation(t *testing.T) {
origBudget, origInterval := sgDeleteWaitBudget, sgDeleteWaitInterval
sgDeleteWaitBudget, sgDeleteWaitInterval = 200*time.Millisecond, time.Millisecond
defer func() { sgDeleteWaitBudget, sgDeleteWaitInterval = origBudget, origInterval }()
t.Run("transient then succeeds", func(t *testing.T) {
sgp := newFakeSGProvisioner()
sgp.existing["eks-cluster-alpha-nodegroup-sg|vpc-aaa"] = "sg-ng"
sgp.deleteFailsBefore = map[string]int{"sg-ng": 2} // 2 DepViolations, then ok
err := DeleteClusterSGs(sgp, "111122223333", "alpha", "vpc-aaa")
require.NoError(t, err, "delete must succeed once the ENI detaches")
assert.GreaterOrEqual(t, len(sgp.deleteCalls), 3, "retried past the transient violations")
})
t.Run("persistent surfaced past budget", func(t *testing.T) {
sgp := newFakeSGProvisioner()
sgp.existing["eks-cluster-alpha-nodegroup-sg|vpc-aaa"] = "sg-ng"
sgp.deleteErr = errors.New("DependencyViolation: still referenced")
err := DeleteClusterSGs(sgp, "111122223333", "alpha", "vpc-aaa")
require.Error(t, err, "a persistent DependencyViolation must surface, not leak the SG")
assert.Contains(t, err.Error(), "sg-ng")
})
}
// TestDeleteClusterSGs_RevokesCrossRefsBeforeDelete guards the no-orphan
// teardown contract: the control-plane and nodegroup SGs reference each other, so
// AWS rejects deleting either while the other's rule points at it. DeleteClusterSGs
// must revoke all ingress on both before deleting, or the SGs leak and pin the VPC.
func TestDeleteClusterSGs_RevokesCrossRefsBeforeDelete(t *testing.T) {
sgp := newFakeSGProvisioner()
sgp.existing["eks-cluster-alpha-control-plane-sg|vpc-aaa"] = "sg-cp"
sgp.existing["eks-cluster-alpha-nodegroup-sg|vpc-aaa"] = "sg-ng"
// cp<->ng cross-references, as EnsureNodegroupSGRules creates them.
sgp.perms = map[string][]*ec2.IpPermission{
"sg-cp": {groupRefPerm("tcp", 6443, "sg-ng"), groupRefPerm("udp", 8472, "sg-ng")},
"sg-ng": {groupRefPerm("udp", 8472, "sg-cp"), groupRefPerm("tcp", 10250, "sg-cp")},
}
// AWS refuses to delete an SG still referenced by a sibling's rule.
sgp.enforceDepViolation = true
err := DeleteClusterSGs(sgp, "111122223333", "alpha", "vpc-aaa")
require.NoError(t, err, "revoke-then-delete must clear the cp<->ng cycle so both SGs delete")
require.Len(t, sgp.revokeCalls, 2, "ingress on both SGs must be revoked before delete")
require.Len(t, sgp.deleteCalls, 2, "both SGs deleted once cross-refs are cleared")
deleted := map[string]bool{}
for _, d := range sgp.deleteCalls {
deleted[aws.StringValue(d.GroupId)] = true
}
assert.True(t, deleted["sg-cp"] && deleted["sg-ng"], "both cluster SGs removed")
}
func groupRefPerm(proto string, port int64, refSG string) *ec2.IpPermission {
return &ec2.IpPermission{
IpProtocol: aws.String(proto),
FromPort: aws.Int64(port),
ToPort: aws.Int64(port),
UserIdGroupPairs: []*ec2.UserIdGroupPair{{GroupId: aws.String(refSG)}},
}
}
func TestEnsureControlPlaneIngress_AuthorizesAPIServerFromVPCCIDR(t *testing.T) {
sgp := newFakeSGProvisioner()
spinifex/handlers/eks/service_impl.go
+20 -16
View File
@@ -867,11 +867,12 @@ func (s *EKSServiceImpl) claimClusterName(accountID string, acctKV nats.KeyValue
// purgeClusterInfra tears down a cluster's live AWS resources and erases its
// per-cluster KV state. Zeroize (a security guarantee — no recoverable key
// material) plus NLB and VM teardown are blocking: their failures are joined
// and returned BEFORE the KV sweep, so a billable resource is never orphaned
// without the meta record that owns its ARNs/IDs. SG cleanup is best-effort
// (a leaked SG strands no owning record). Shared by DeleteCluster and the
// FAILED-cluster reclaim path in CreateCluster.
// material), NLB, VM, SG, IGW, and CP-VPC teardown are all blocking: their
// failures are joined and returned BEFORE the KV sweep, so neither a billable
// resource nor a VPC-pinning SG is orphaned without the meta record that owns
// its IDs. The cluster SGs sit in the customer VPC, so a leaked SG pins the VPC
// on DependencyViolation — leaving the cluster DELETING for a retry is correct.
// Shared by DeleteCluster and the FAILED-cluster reclaim path in CreateCluster.
func (s *EKSServiceImpl) purgeClusterInfra(accountID, name string, meta *ClusterMeta, acctKV nats.KeyValue, deleteMeta bool) error {
s.registry.Stop(accountID, name)
@@ -961,10 +962,6 @@ func (s *EKSServiceImpl) purgeClusterInfra(accountID, name string, meta *Cluster
}
}
if len(teardownErrs) > 0 {
return errors.Join(teardownErrs...)
}
// Private endpoint (301): reclaim the customer-VPC SG now its ENI is gone.
// Best-effort — its only billable dependant (the ENI) is already deleted.
if meta.ResourcesVpcConfig != nil && meta.ResourcesVpcConfig.VpcId != "" {
@@ -973,29 +970,29 @@ func (s *EKSServiceImpl) purgeClusterInfra(accountID, name string, meta *Cluster
}
}
// SG / IGW / CP-VPC cleanup runs even when an earlier step failed: the cluster
// SGs sit in the customer VPC, so skipping them would pin the VPC. Failures are
// joined into teardownErrs (not swallowed) so the meta survives for a retry.
if meta.ManagedCPVPC != nil && meta.ManagedCPVPC.VpcId != "" {
// New topology: SGs + IGW + subnets + route tables + NAT GW + VPC all live
// in the managed CP VPC under the system account. DeleteClusterCPVPC removes
// the IGW + VPC after the SGs, so SG cleanup must precede it.
if err := DeleteClusterSGs(s.deps.VPCSG, infraAcct, name, meta.ManagedCPVPC.VpcId); err != nil {
slog.Warn("purgeClusterInfra: DeleteClusterSGs failed", "cluster", name, "err", err)
teardownErrs = append(teardownErrs, fmt.Errorf("delete cluster SGs: %w", err))
}
// The CP VPC holds a billable NAT-GW EIP, so a teardown failure is surfaced
// (not best-effort): returning before the KV sweep keeps the meta refs alive
// for a delete retry rather than orphaning the EIP.
if err := DeleteClusterCPVPC(s.cpVPCDeps(), infraAcct, name); err != nil {
return fmt.Errorf("delete managed CP VPC: %w", err)
teardownErrs = append(teardownErrs, fmt.Errorf("delete managed CP VPC: %w", err))
}
} else if meta.ResourcesVpcConfig != nil && meta.ResourcesVpcConfig.VpcId != "" {
// Legacy topology: CP lived in the customer VPC; reclaim its SGs + the
// cluster-owned IGW (ownership-scoped — a reused customer IGW is left
// intact). IGW delete must precede any VPC delete to avoid DependencyViolation.
if err := DeleteClusterSGs(s.deps.VPCSG, accountID, name, meta.ResourcesVpcConfig.VpcId); err != nil {
slog.Warn("purgeClusterInfra: DeleteClusterSGs failed", "cluster", name, "err", err)
teardownErrs = append(teardownErrs, fmt.Errorf("delete cluster SGs: %w", err))
}
if s.deps.IGW != nil {
if err := DeleteClusterIGW(s.deps.IGW, accountID, meta.ResourcesVpcConfig.VpcId, name); err != nil {
slog.Warn("purgeClusterInfra: DeleteClusterIGW failed", "cluster", name, "err", err)
teardownErrs = append(teardownErrs, fmt.Errorf("delete cluster IGW: %w", err))
}
}
}
@@ -1005,6 +1002,13 @@ func (s *EKSServiceImpl) purgeClusterInfra(accountID, name string, meta *Cluster
// group strands nothing.
s.teardownSpreadGroup(meta)
// Any blocking-teardown failure (billable infra or a VPC-pinning SG) keeps the
// meta — and the IDs that own the stranded resources — alive for a delete retry
// or the DELETING backstop, rather than completing deletion with infra leaked.
if len(teardownErrs) > 0 {
return errors.Join(teardownErrs...)
}
// Only now, with NLB + VM confirmed gone, erase the meta + per-cluster KV.
// The reclaim path passes deleteMeta=false: it has already CAS-claimed the
// meta key (status CREATING) as its lock for the recreate, so the key must
spinifex/services/spinifexui/frontend/dist/assets/_clusterName.js
Vendored
+1 -1
View File
File diff suppressed because one or more lines are too long
spinifex/services/spinifexui/frontend/dist/assets/create-cluster.js
Vendored
+1 -1
View File
File diff suppressed because one or more lines are too long
spinifex/services/spinifexui/frontend/dist/assets/create-policy.js
Vendored
+1 -1
View File
@@ -1,4 +1,4 @@
import{n as e,t}from"./compiler-runtime.js";import{t as n}from"./error-banner.js";import{t as r}from"./back-link.js";import{t as i}from"./cli-command-panel.js";import{o as a}from"./iam.js";import{Ir as o,Rn as s}from"./index.js";import{a as ee,i as te,t as ne}from"./zod.js";import{t as re}from"./form-actions.js";import{t as ie}from"./page-heading.js";import{a as c,r as l,t as u}from"./field.js";import{t as d}from"./textarea.js";import{r as f}from"./iam2.js";var ae=t(),p=e(),oe=JSON.stringify({Version:`2012-10-17`,Statement:[{Effect:`Allow`,Action:`*`,Resource:`*`}]},null,2);function m(){let e=(0,ae.c)(67),t=o(),m=a(),h;e[0]===Symbol.for(`react.memo_cache_sentinel`)?(h={resolver:ne(f),defaultValues:{policyDocument:oe}},e[0]=h):h=e[0];let{handleSubmit:g,register:_,control:v,formState:ce}=te(h),{errors:y,isSubmitting:b}=ce,x;e[1]===v?x=e[2]:(x={control:v},e[1]=v,e[2]=x);let S=ee(x),C;e[3]===S?C=e[4]:(C=e=>e?S[e]:void 0,e[3]=S,e[4]=C);let w=C,T;e[5]!==m||e[6]!==t?(T=async e=>{await m.mutateAsync(e),t({to:`/iam/list-policies`})},e[5]=m,e[6]=t,e[7]=T):T=e[7];let E=T,D,O;e[8]===Symbol.for(`react.memo_cache_sentinel`)?(D=(0,p.jsx)(r,{to:`/iam/list-policies`,children:`Back to policies`}),O=(0,p.jsx)(ie,{title:`Create Policy`}),e[8]=D,e[9]=O):(D=e[8],O=e[9]);let k;e[10]===m.error?k=e[11]:(k=m.error&&(0,p.jsx)(n,{error:m.error,msg:`Failed to create policy`}),e[10]=m.error,e[11]=k);let A;e[12]!==g||e[13]!==E?(A=g(E),e[12]=g,e[13]=E,e[14]=A):A=e[14];let j;e[15]===Symbol.for(`react.memo_cache_sentinel`)?(j=(0,p.jsx)(c,{children:(0,p.jsx)(`label`,{htmlFor:`policyName`,children:`Policy Name`})}),e[15]=j):j=e[15];let M=!!y.policyName,N;e[16]===_?N=e[17]:(N=_(`policyName`),e[16]=_,e[17]=N);let P;e[18]!==M||e[19]!==N?(P=(0,p.jsx)(s,{"aria-invalid":M,id:`policyName`,placeholder:`my-policy...`,...N}),e[18]=M,e[19]=N,e[20]=P):P=e[20];let F;e[21]===y.policyName?F=e[22]:(F=(0,p.jsx)(l,{errors:[y.policyName]}),e[21]=y.policyName,e[22]=F);let I;e[23]!==P||e[24]!==F?(I=(0,p.jsxs)(u,{children:[j,P,F]}),e[23]=P,e[24]=F,e[25]=I):I=e[25];let L;e[26]===Symbol.for(`react.memo_cache_sentinel`)?(L=(0,p.jsx)(c,{children:(0,p.jsx)(`label`,{htmlFor:`description`,children:`Description (optional)`})}),e[26]=L):L=e[26];let R;e[27]===_?R=e[28]:(R=_(`description`),e[27]=_,e[28]=R);let z;e[29]===R?z=e[30]:(z=(0,p.jsx)(s,{id:`description`,placeholder:`A brief description of this policy...`,...R}),e[29]=R,e[30]=z);let B;e[31]===y.description?B=e[32]:(B=(0,p.jsx)(l,{errors:[y.description]}),e[31]=y.description,e[32]=B);let V;e[33]!==z||e[34]!==B?(V=(0,p.jsxs)(u,{children:[L,z,B]}),e[33]=z,e[34]=B,e[35]=V):V=e[35];let H;e[36]===Symbol.for(`react.memo_cache_sentinel`)?(H=(0,p.jsx)(c,{children:(0,p.jsx)(`label`,{htmlFor:`policyDocument`,children:`Policy Document (JSON)`})}),e[36]=H):H=e[36];let U=!!y.policyDocument,W;e[37]===_?W=e[38]:(W=_(`policyDocument`),e[37]=_,e[38]=W);let G;e[39]!==U||e[40]!==W?(G=(0,p.jsx)(d,{"aria-invalid":U,className:`font-mono text-sm`,id:`policyDocument`,rows:15,...W}),e[39]=U,e[40]=W,e[41]=G):G=e[41];let K;e[42]===y.policyDocument?K=e[43]:(K=(0,p.jsx)(l,{errors:[y.policyDocument]}),e[42]=y.policyDocument,e[43]=K);let q;e[44]!==G||e[45]!==K?(q=(0,p.jsxs)(u,{children:[H,G,K]}),e[44]=G,e[45]=K,e[46]=q):q=e[46];let J;e[47]===w?J=e[48]:(J=se(w),e[47]=w,e[48]=J);let Y;e[49]===J?Y=e[50]:(Y=(0,p.jsx)(i,{commands:J}),e[49]=J,e[50]=Y);let X;e[51]===t?X=e[52]:(X=async()=>await t({to:`/iam/list-policies`}),e[51]=t,e[52]=X);let Z;e[53]!==m.isPending||e[54]!==b||e[55]!==X?(Z=(0,p.jsx)(re,{isPending:m.isPending,isSubmitting:b,onCancel:X,pendingLabel:`Creating...`,submitLabel:`Create Policy`}),e[53]=m.isPending,e[54]=b,e[55]=X,e[56]=Z):Z=e[56];let Q;e[57]!==I||e[58]!==V||e[59]!==q||e[60]!==Y||e[61]!==Z||e[62]!==A?(Q=(0,p.jsxs)(`form`,{className:`max-w-4xl space-y-6`,onSubmit:A,children:[I,V,q,Y,Z]}),e[57]=I,e[58]=V,e[59]=q,e[60]=Y,e[61]=Z,e[62]=A,e[63]=Q):Q=e[63];let $;return e[64]!==Q||e[65]!==k?($=(0,p.jsxs)(p.Fragment,{children:[D,O,k,Q]}),e[64]=Q,e[65]=k,e[66]=$):$=e[66],$}function se(e){let t=e(`policyName`),n=typeof t==`string`?t:``,r=e(`description`),i=typeof r==`string`?r:``,a=e(`policyDocument`),o=typeof a==`string`?a:``,s=[{type:`bin`,value:`AWS_PROFILE=spinifex aws iam create-policy`},{type:`flag`,value:` \\
import{n as e,t}from"./compiler-runtime.js";import{t as n}from"./error-banner.js";import{t as r}from"./back-link.js";import{t as i}from"./cli-command-panel.js";import{o as a}from"./iam.js";import{Ir as o,Rn as s}from"./index.js";import{a as c,i as ee,n as te,t as ne}from"./zod.js";import{t as re}from"./form-actions.js";import{t as ie}from"./page-heading.js";import{a as l,r as u,t as d}from"./field.js";import{t as ae}from"./json-editor.js";import{r as oe}from"./iam2.js";var se=t(),f=e(),p=JSON.stringify({Version:`2012-10-17`,Statement:[{Effect:`Allow`,Action:`*`,Resource:`*`}]},null,2);function m(){let e=(0,se.c)(67),t=o(),m=a(),g;e[0]===Symbol.for(`react.memo_cache_sentinel`)?(g={resolver:ne(oe),defaultValues:{policyDocument:p}},e[0]=g):g=e[0];let{handleSubmit:_,register:v,control:y,formState:ce}=ee(g),{errors:b,isSubmitting:x}=ce,S;e[1]===y?S=e[2]:(S={control:y},e[1]=y,e[2]=S);let C=c(S),w;e[3]===C?w=e[4]:(w=e=>e?C[e]:void 0,e[3]=C,e[4]=w);let T=w,E;e[5]!==m||e[6]!==t?(E=async e=>{await m.mutateAsync(e),t({to:`/iam/list-policies`})},e[5]=m,e[6]=t,e[7]=E):E=e[7];let D=E,O,k;e[8]===Symbol.for(`react.memo_cache_sentinel`)?(O=(0,f.jsx)(r,{to:`/iam/list-policies`,children:`Back to policies`}),k=(0,f.jsx)(ie,{title:`Create Policy`}),e[8]=O,e[9]=k):(O=e[8],k=e[9]);let A;e[10]===m.error?A=e[11]:(A=m.error&&(0,f.jsx)(n,{error:m.error,msg:`Failed to create policy`}),e[10]=m.error,e[11]=A);let j;e[12]!==_||e[13]!==D?(j=_(D),e[12]=_,e[13]=D,e[14]=j):j=e[14];let M;e[15]===Symbol.for(`react.memo_cache_sentinel`)?(M=(0,f.jsx)(l,{children:(0,f.jsx)(`label`,{htmlFor:`policyName`,children:`Policy Name`})}),e[15]=M):M=e[15];let N=!!b.policyName,P;e[16]===v?P=e[17]:(P=v(`policyName`),e[16]=v,e[17]=P);let F;e[18]!==N||e[19]!==P?(F=(0,f.jsx)(s,{"aria-invalid":N,id:`policyName`,placeholder:`my-policy...`,...P}),e[18]=N,e[19]=P,e[20]=F):F=e[20];let I;e[21]===b.policyName?I=e[22]:(I=(0,f.jsx)(u,{errors:[b.policyName]}),e[21]=b.policyName,e[22]=I);let L;e[23]!==F||e[24]!==I?(L=(0,f.jsxs)(d,{children:[M,F,I]}),e[23]=F,e[24]=I,e[25]=L):L=e[25];let R;e[26]===Symbol.for(`react.memo_cache_sentinel`)?(R=(0,f.jsx)(l,{children:(0,f.jsx)(`label`,{htmlFor:`description`,children:`Description (optional)`})}),e[26]=R):R=e[26];let z;e[27]===v?z=e[28]:(z=v(`description`),e[27]=v,e[28]=z);let B;e[29]===z?B=e[30]:(B=(0,f.jsx)(s,{id:`description`,placeholder:`A brief description of this policy...`,...z}),e[29]=z,e[30]=B);let V;e[31]===b.description?V=e[32]:(V=(0,f.jsx)(u,{errors:[b.description]}),e[31]=b.description,e[32]=V);let H;e[33]!==B||e[34]!==V?(H=(0,f.jsxs)(d,{children:[R,B,V]}),e[33]=B,e[34]=V,e[35]=H):H=e[35];let U;e[36]===Symbol.for(`react.memo_cache_sentinel`)?(U=(0,f.jsx)(l,{children:(0,f.jsx)(`label`,{htmlFor:`policyDocument`,children:`Policy Document (JSON)`})}),e[36]=U):U=e[36];let W;e[37]===b.policyDocument?W=e[38]:(W=e=>{let{field:t}=e,{onBlur:n,onChange:r,ref:i,value:a}=t;return(0,f.jsx)(ae,{error:!!b.policyDocument,id:`policyDocument`,onBlur:n,onChange:r,ref:i,rows:15,value:a})},e[37]=b.policyDocument,e[38]=W);let G;e[39]!==y||e[40]!==W?(G=(0,f.jsx)(te,{control:y,name:`policyDocument`,render:W}),e[39]=y,e[40]=W,e[41]=G):G=e[41];let K;e[42]===b.policyDocument?K=e[43]:(K=(0,f.jsx)(u,{errors:[b.policyDocument]}),e[42]=b.policyDocument,e[43]=K);let q;e[44]!==G||e[45]!==K?(q=(0,f.jsxs)(d,{children:[U,G,K]}),e[44]=G,e[45]=K,e[46]=q):q=e[46];let J;e[47]===T?J=e[48]:(J=h(T),e[47]=T,e[48]=J);let Y;e[49]===J?Y=e[50]:(Y=(0,f.jsx)(i,{commands:J}),e[49]=J,e[50]=Y);let X;e[51]===t?X=e[52]:(X=async()=>await t({to:`/iam/list-policies`}),e[51]=t,e[52]=X);let Z;e[53]!==m.isPending||e[54]!==x||e[55]!==X?(Z=(0,f.jsx)(re,{isPending:m.isPending,isSubmitting:x,onCancel:X,pendingLabel:`Creating...`,submitLabel:`Create Policy`}),e[53]=m.isPending,e[54]=x,e[55]=X,e[56]=Z):Z=e[56];let Q;e[57]!==L||e[58]!==H||e[59]!==q||e[60]!==Y||e[61]!==Z||e[62]!==j?(Q=(0,f.jsxs)(`form`,{className:`max-w-4xl space-y-6`,onSubmit:j,children:[L,H,q,Y,Z]}),e[57]=L,e[58]=H,e[59]=q,e[60]=Y,e[61]=Z,e[62]=j,e[63]=Q):Q=e[63];let $;return e[64]!==Q||e[65]!==A?($=(0,f.jsxs)(f.Fragment,{children:[O,k,A,Q]}),e[64]=Q,e[65]=A,e[66]=$):$=e[66],$}function h(e){let t=e(`policyName`),n=typeof t==`string`?t:``,r=e(`description`),i=typeof r==`string`?r:``,a=e(`policyDocument`),o=typeof a==`string`?a:``,s=[{type:`bin`,value:`AWS_PROFILE=spinifex aws iam create-policy`},{type:`flag`,value:` \\
--policy-name`},{type:`value`,value:` ${n||`<PolicyName>`}`}];return i&&s.push({type:`flag`,value:` \\
--description`},{type:`value`,value:` "${i}"`}),s.push({type:`flag`,value:` \\
--policy-document`},{type:`value`,value:` '${o||`<PolicyDocument>`}'`}),[{label:`Create Policy`,parts:s}]}export{m as component};
spinifex/services/spinifexui/frontend/dist/assets/create-role.js
Vendored
+1 -1
View File
@@ -1,4 +1,4 @@
import{n as e,t}from"./compiler-runtime.js";import{t as n}from"./error-banner.js";import{t as r}from"./back-link.js";import{t as i}from"./cli-command-panel.js";import{s as a}from"./iam.js";import{Ir as o,Rn as s}from"./index.js";import{a as ee,i as te,t as ne}from"./zod.js";import{t as re}from"./form-actions.js";import{t as ie}from"./page-heading.js";import{a as c,r as l,t as u}from"./field.js";import{t as d}from"./textarea.js";import{i as f,t as ae}from"./iam2.js";var oe=t(),p=e();function m(){let e=(0,oe.c)(67),t=o(),m=a(),h;e[0]===Symbol.for(`react.memo_cache_sentinel`)?(h={resolver:ne(f),defaultValues:{assumeRolePolicyDocument:ae}},e[0]=h):h=e[0];let{handleSubmit:g,register:_,control:v,formState:ce}=te(h),{errors:y,isSubmitting:b}=ce,x;e[1]===v?x=e[2]:(x={control:v},e[1]=v,e[2]=x);let S=ee(x),C;e[3]===S?C=e[4]:(C=e=>e?S[e]:void 0,e[3]=S,e[4]=C);let w=C,T;e[5]!==m||e[6]!==t?(T=async e=>{await m.mutateAsync(e),t({to:`/iam/list-roles`})},e[5]=m,e[6]=t,e[7]=T):T=e[7];let E=T,D,O;e[8]===Symbol.for(`react.memo_cache_sentinel`)?(D=(0,p.jsx)(r,{to:`/iam/list-roles`,children:`Back to roles`}),O=(0,p.jsx)(ie,{title:`Create Role`}),e[8]=D,e[9]=O):(D=e[8],O=e[9]);let k;e[10]===m.error?k=e[11]:(k=m.error&&(0,p.jsx)(n,{error:m.error,msg:`Failed to create role`}),e[10]=m.error,e[11]=k);let A;e[12]!==g||e[13]!==E?(A=g(E),e[12]=g,e[13]=E,e[14]=A):A=e[14];let j;e[15]===Symbol.for(`react.memo_cache_sentinel`)?(j=(0,p.jsx)(c,{children:(0,p.jsx)(`label`,{htmlFor:`roleName`,children:`Role Name`})}),e[15]=j):j=e[15];let M=!!y.roleName,N;e[16]===_?N=e[17]:(N=_(`roleName`),e[16]=_,e[17]=N);let P;e[18]!==M||e[19]!==N?(P=(0,p.jsx)(s,{"aria-invalid":M,id:`roleName`,placeholder:`my-role...`,...N}),e[18]=M,e[19]=N,e[20]=P):P=e[20];let F;e[21]===y.roleName?F=e[22]:(F=(0,p.jsx)(l,{errors:[y.roleName]}),e[21]=y.roleName,e[22]=F);let I;e[23]!==P||e[24]!==F?(I=(0,p.jsxs)(u,{children:[j,P,F]}),e[23]=P,e[24]=F,e[25]=I):I=e[25];let L;e[26]===Symbol.for(`react.memo_cache_sentinel`)?(L=(0,p.jsx)(c,{children:(0,p.jsx)(`label`,{htmlFor:`description`,children:`Description (optional)`})}),e[26]=L):L=e[26];let R;e[27]===_?R=e[28]:(R=_(`description`),e[27]=_,e[28]=R);let z;e[29]===R?z=e[30]:(z=(0,p.jsx)(s,{id:`description`,placeholder:`A brief description of this role...`,...R}),e[29]=R,e[30]=z);let B;e[31]===y.description?B=e[32]:(B=(0,p.jsx)(l,{errors:[y.description]}),e[31]=y.description,e[32]=B);let V;e[33]!==z||e[34]!==B?(V=(0,p.jsxs)(u,{children:[L,z,B]}),e[33]=z,e[34]=B,e[35]=V):V=e[35];let H;e[36]===Symbol.for(`react.memo_cache_sentinel`)?(H=(0,p.jsx)(c,{children:(0,p.jsx)(`label`,{htmlFor:`assumeRolePolicyDocument`,children:`Trust Policy (JSON)`})}),e[36]=H):H=e[36];let U=!!y.assumeRolePolicyDocument,W;e[37]===_?W=e[38]:(W=_(`assumeRolePolicyDocument`),e[37]=_,e[38]=W);let G;e[39]!==U||e[40]!==W?(G=(0,p.jsx)(d,{"aria-invalid":U,className:`font-mono text-sm`,id:`assumeRolePolicyDocument`,rows:15,...W}),e[39]=U,e[40]=W,e[41]=G):G=e[41];let K;e[42]===y.assumeRolePolicyDocument?K=e[43]:(K=(0,p.jsx)(l,{errors:[y.assumeRolePolicyDocument]}),e[42]=y.assumeRolePolicyDocument,e[43]=K);let q;e[44]!==G||e[45]!==K?(q=(0,p.jsxs)(u,{children:[H,G,K]}),e[44]=G,e[45]=K,e[46]=q):q=e[46];let J;e[47]===w?J=e[48]:(J=se(w),e[47]=w,e[48]=J);let Y;e[49]===J?Y=e[50]:(Y=(0,p.jsx)(i,{commands:J}),e[49]=J,e[50]=Y);let X;e[51]===t?X=e[52]:(X=async()=>await t({to:`/iam/list-roles`}),e[51]=t,e[52]=X);let Z;e[53]!==m.isPending||e[54]!==b||e[55]!==X?(Z=(0,p.jsx)(re,{isPending:m.isPending,isSubmitting:b,onCancel:X,pendingLabel:`Creating...`,submitLabel:`Create Role`}),e[53]=m.isPending,e[54]=b,e[55]=X,e[56]=Z):Z=e[56];let Q;e[57]!==I||e[58]!==V||e[59]!==q||e[60]!==Y||e[61]!==Z||e[62]!==A?(Q=(0,p.jsxs)(`form`,{className:`max-w-4xl space-y-6`,onSubmit:A,children:[I,V,q,Y,Z]}),e[57]=I,e[58]=V,e[59]=q,e[60]=Y,e[61]=Z,e[62]=A,e[63]=Q):Q=e[63];let $;return e[64]!==Q||e[65]!==k?($=(0,p.jsxs)(p.Fragment,{children:[D,O,k,Q]}),e[64]=Q,e[65]=k,e[66]=$):$=e[66],$}function se(e){let t=e(`roleName`),n=typeof t==`string`?t:``,r=e(`description`),i=typeof r==`string`?r:``,a=e(`assumeRolePolicyDocument`),o=typeof a==`string`?a:``,s=[{type:`bin`,value:`AWS_PROFILE=spinifex aws iam create-role`},{type:`flag`,value:` \\
import{n as e,t}from"./compiler-runtime.js";import{t as n}from"./error-banner.js";import{t as r}from"./back-link.js";import{t as i}from"./cli-command-panel.js";import{s as a}from"./iam.js";import{Ir as o,Rn as s}from"./index.js";import{a as c,i as ee,n as te,t as ne}from"./zod.js";import{t as re}from"./form-actions.js";import{t as ie}from"./page-heading.js";import{a as l,r as u,t as d}from"./field.js";import{t as ae}from"./json-editor.js";import{i as oe,t as se}from"./iam2.js";var f=t(),p=e();function m(){let e=(0,f.c)(67),t=o(),m=a(),g;e[0]===Symbol.for(`react.memo_cache_sentinel`)?(g={resolver:ne(oe),defaultValues:{assumeRolePolicyDocument:se}},e[0]=g):g=e[0];let{handleSubmit:_,register:v,control:y,formState:ce}=ee(g),{errors:b,isSubmitting:x}=ce,S;e[1]===y?S=e[2]:(S={control:y},e[1]=y,e[2]=S);let C=c(S),w;e[3]===C?w=e[4]:(w=e=>e?C[e]:void 0,e[3]=C,e[4]=w);let T=w,E;e[5]!==m||e[6]!==t?(E=async e=>{await m.mutateAsync(e),t({to:`/iam/list-roles`})},e[5]=m,e[6]=t,e[7]=E):E=e[7];let D=E,O,k;e[8]===Symbol.for(`react.memo_cache_sentinel`)?(O=(0,p.jsx)(r,{to:`/iam/list-roles`,children:`Back to roles`}),k=(0,p.jsx)(ie,{title:`Create Role`}),e[8]=O,e[9]=k):(O=e[8],k=e[9]);let A;e[10]===m.error?A=e[11]:(A=m.error&&(0,p.jsx)(n,{error:m.error,msg:`Failed to create role`}),e[10]=m.error,e[11]=A);let j;e[12]!==_||e[13]!==D?(j=_(D),e[12]=_,e[13]=D,e[14]=j):j=e[14];let M;e[15]===Symbol.for(`react.memo_cache_sentinel`)?(M=(0,p.jsx)(l,{children:(0,p.jsx)(`label`,{htmlFor:`roleName`,children:`Role Name`})}),e[15]=M):M=e[15];let N=!!b.roleName,P;e[16]===v?P=e[17]:(P=v(`roleName`),e[16]=v,e[17]=P);let F;e[18]!==N||e[19]!==P?(F=(0,p.jsx)(s,{"aria-invalid":N,id:`roleName`,placeholder:`my-role...`,...P}),e[18]=N,e[19]=P,e[20]=F):F=e[20];let I;e[21]===b.roleName?I=e[22]:(I=(0,p.jsx)(u,{errors:[b.roleName]}),e[21]=b.roleName,e[22]=I);let L;e[23]!==F||e[24]!==I?(L=(0,p.jsxs)(d,{children:[M,F,I]}),e[23]=F,e[24]=I,e[25]=L):L=e[25];let R;e[26]===Symbol.for(`react.memo_cache_sentinel`)?(R=(0,p.jsx)(l,{children:(0,p.jsx)(`label`,{htmlFor:`description`,children:`Description (optional)`})}),e[26]=R):R=e[26];let z;e[27]===v?z=e[28]:(z=v(`description`),e[27]=v,e[28]=z);let B;e[29]===z?B=e[30]:(B=(0,p.jsx)(s,{id:`description`,placeholder:`A brief description of this role...`,...z}),e[29]=z,e[30]=B);let V;e[31]===b.description?V=e[32]:(V=(0,p.jsx)(u,{errors:[b.description]}),e[31]=b.description,e[32]=V);let H;e[33]!==B||e[34]!==V?(H=(0,p.jsxs)(d,{children:[R,B,V]}),e[33]=B,e[34]=V,e[35]=H):H=e[35];let U;e[36]===Symbol.for(`react.memo_cache_sentinel`)?(U=(0,p.jsx)(l,{children:(0,p.jsx)(`label`,{htmlFor:`assumeRolePolicyDocument`,children:`Trust Policy (JSON)`})}),e[36]=U):U=e[36];let W;e[37]===b.assumeRolePolicyDocument?W=e[38]:(W=e=>{let{field:t}=e,{onBlur:n,onChange:r,ref:i,value:a}=t;return(0,p.jsx)(ae,{error:!!b.assumeRolePolicyDocument,id:`assumeRolePolicyDocument`,onBlur:n,onChange:r,ref:i,rows:15,value:a})},e[37]=b.assumeRolePolicyDocument,e[38]=W);let G;e[39]!==y||e[40]!==W?(G=(0,p.jsx)(te,{control:y,name:`assumeRolePolicyDocument`,render:W}),e[39]=y,e[40]=W,e[41]=G):G=e[41];let K;e[42]===b.assumeRolePolicyDocument?K=e[43]:(K=(0,p.jsx)(u,{errors:[b.assumeRolePolicyDocument]}),e[42]=b.assumeRolePolicyDocument,e[43]=K);let q;e[44]!==G||e[45]!==K?(q=(0,p.jsxs)(d,{children:[U,G,K]}),e[44]=G,e[45]=K,e[46]=q):q=e[46];let J;e[47]===T?J=e[48]:(J=h(T),e[47]=T,e[48]=J);let Y;e[49]===J?Y=e[50]:(Y=(0,p.jsx)(i,{commands:J}),e[49]=J,e[50]=Y);let X;e[51]===t?X=e[52]:(X=async()=>await t({to:`/iam/list-roles`}),e[51]=t,e[52]=X);let Z;e[53]!==m.isPending||e[54]!==x||e[55]!==X?(Z=(0,p.jsx)(re,{isPending:m.isPending,isSubmitting:x,onCancel:X,pendingLabel:`Creating...`,submitLabel:`Create Role`}),e[53]=m.isPending,e[54]=x,e[55]=X,e[56]=Z):Z=e[56];let Q;e[57]!==L||e[58]!==H||e[59]!==q||e[60]!==Y||e[61]!==Z||e[62]!==j?(Q=(0,p.jsxs)(`form`,{className:`max-w-4xl space-y-6`,onSubmit:j,children:[L,H,q,Y,Z]}),e[57]=L,e[58]=H,e[59]=q,e[60]=Y,e[61]=Z,e[62]=j,e[63]=Q):Q=e[63];let $;return e[64]!==Q||e[65]!==A?($=(0,p.jsxs)(p.Fragment,{children:[O,k,A,Q]}),e[64]=Q,e[65]=A,e[66]=$):$=e[66],$}function h(e){let t=e(`roleName`),n=typeof t==`string`?t:``,r=e(`description`),i=typeof r==`string`?r:``,a=e(`assumeRolePolicyDocument`),o=typeof a==`string`?a:``,s=[{type:`bin`,value:`AWS_PROFILE=spinifex aws iam create-role`},{type:`flag`,value:` \\
--role-name`},{type:`value`,value:` ${n||`<RoleName>`}`}];return i&&s.push({type:`flag`,value:` \\
--description`},{type:`value`,value:` "${i}"`}),s.push({type:`flag`,value:` \\
--assume-role-policy-document`},{type:`value`,value:` '${o||`<TrustPolicy>`}'`}),[{label:`Create Role`,parts:s}]}export{m as component};
spinifex/services/spinifexui/frontend/dist/assets/eks.js
Vendored
+1 -1
View File
File diff suppressed because one or more lines are too long
spinifex/services/spinifexui/frontend/dist/assets/iam2.js
Vendored
+1 -1
View File
@@ -1 +1 @@
import{f as e,u as t}from"./auth.js";var n=t({userName:e().min(1,`User name is required`).max(64,`User name must be 64 characters or less`).regex(/^[\w+=,.@-]+$/,`User name contains invalid characters`),path:e().optional()}),r=t({policyName:e().min(1,`Policy name is required`).max(128,`Policy name must be 128 characters or less`).regex(/^[\w+=,.@-]+$/,`Policy name contains invalid characters`),description:e().optional(),policyDocument:e().min(1,`Policy document is required`).refine(e=>{try{return JSON.parse(e),!0}catch{return!1}},{message:`Policy document must be valid JSON`})}),i=JSON.stringify({Version:`2012-10-17`,Statement:[{Effect:`Allow`,Principal:{Service:`ec2.amazonaws.com`},Action:`sts:AssumeRole`}]},null,2),a=t({roleName:e().min(1,`Role name is required`).max(64,`Role name must be 64 characters or less`).regex(/^[\w+=,.@-]+$/,`Role name contains invalid characters`),path:e().optional(),description:e().optional(),assumeRolePolicyDocument:e().min(1,`Trust policy is required`).refine(e=>{try{return JSON.parse(e),!0}catch{return!1}},{message:`Trust policy must be valid JSON`})}),o=t({instanceProfileName:e().min(1,`Instance profile name is required`).max(128,`Instance profile name must be 128 characters or less`).regex(/^[\w+=,.@-]+$/,`Instance profile name contains invalid characters`),path:e().optional()});export{n as a,a as i,o as n,r,i as t};
import{f as e,u as t}from"./auth.js";import{n}from"./json.js";var r=t({userName:e().min(1,`User name is required`).max(64,`User name must be 64 characters or less`).regex(/^[\w+=,.@-]+$/,`User name contains invalid characters`),path:e().optional()}),i=t({policyName:e().min(1,`Policy name is required`).max(128,`Policy name must be 128 characters or less`).regex(/^[\w+=,.@-]+$/,`Policy name contains invalid characters`),description:e().optional(),policyDocument:n({label:`Policy document`})}),a=JSON.stringify({Version:`2012-10-17`,Statement:[{Effect:`Allow`,Principal:{Service:`ec2.amazonaws.com`},Action:`sts:AssumeRole`}]},null,2),o=t({roleName:e().min(1,`Role name is required`).max(64,`Role name must be 64 characters or less`).regex(/^[\w+=,.@-]+$/,`Role name contains invalid characters`),path:e().optional(),description:e().optional(),assumeRolePolicyDocument:n({label:`Trust policy`})}),s=t({instanceProfileName:e().min(1,`Instance profile name is required`).max(128,`Instance profile name must be 128 characters or less`).regex(/^[\w+=,.@-]+$/,`Instance profile name contains invalid characters`),path:e().optional()});export{r as a,o as i,s as n,i as r,a as t};
spinifex/services/spinifexui/frontend/dist/assets/index.js
Vendored
+2 -2
View File
File diff suppressed because one or more lines are too long
spinifex/services/spinifexui/frontend/dist/assets/json-editor.js
Vendored
+1
View File
@@ -0,0 +1 @@
import{n as e,t}from"./compiler-runtime.js";import{Lt as n}from"./auth.js";import{t as r}from"./textarea.js";import{t as i}from"./json.js";var a=t(),o=e();function s(e){let t=(0,a.c)(22),s,c,l,u,d,f;t[0]===e?(s=t[1],c=t[2],l=t[3],u=t[4],d=t[5],f=t[6]):({value:f,onChange:u,error:c,onBlur:l,className:s,...d}=e,t[0]=e,t[1]=s,t[2]=c,t[3]=l,t[4]=u,t[5]=d,t[6]=f);let p;t[7]!==l||t[8]!==u||t[9]!==f?(p=e=>{let t=i(f);t!==null&&t!==f&&u(t),l?.(e)},t[7]=l,t[8]=u,t[9]=f,t[10]=p):p=t[10];let m=p,h;t[11]===s?h=t[12]:(h=n(`font-mono text-sm`,s),t[11]=s,t[12]=h);let g;t[13]===u?g=t[14]:(g=e=>u(e.target.value),t[13]=u,t[14]=g);let _;return t[15]!==c||t[16]!==m||t[17]!==d||t[18]!==h||t[19]!==g||t[20]!==f?(_=(0,o.jsx)(r,{"aria-invalid":c,className:h,onBlur:m,onChange:g,value:f,...d}),t[15]=c,t[16]=m,t[17]=d,t[18]=h,t[19]=g,t[20]=f,t[21]=_):_=t[21],_}export{s as t};
spinifex/services/spinifexui/frontend/dist/assets/json.js
Vendored
+1
View File
@@ -0,0 +1 @@
import{f as e}from"./auth.js";function t(e){try{return JSON.parse(e),!0}catch{return!1}}function n(e){try{return JSON.stringify(JSON.parse(e),null,2)}catch{return null}}function r(n){let{label:r,allowEmpty:i=!1}=n;return(i?e():e().min(1,`${r} is required`)).refine(e=>i&&e.trim()===``?!0:t(e),{message:`${r} must be valid JSON`})}export{r as n,n as t};
spinifex/services/spinifexui/frontend/dist/favicon.ico
Vendored
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 6.4 KiB

spinifex/services/spinifexui/frontend/dist/favicon.svg
Vendored
+3
View File
@@ -0,0 +1,3 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="-75 -25.5 514 514" fill="#fff">
<path d="M272.523 171.205C247.773 197.595 211.323 215.815 182.103 237.425C165.663 249.575 122.633 281.165 151.093 301.825C185.533 326.835 271.333 304.985 301.013 277.585C317.193 262.645 316.693 242.485 294.703 234.075C270.913 224.975 243.483 231.925 219.353 235.755C218.543 235.885 216.823 236.805 216.993 235.345C227.053 228.885 236.103 220.925 245.933 214.135C255.403 207.585 267.053 198.225 278.643 198.015C314.693 197.345 351.803 215.135 361.203 252.215C376.833 313.885 303.333 362.255 249.833 366.635L280.913 352.235C292.563 345.305 304.353 338.105 314.653 329.215C317.213 327.005 319.073 324.145 321.243 322.035C321.753 321.535 327.053 319.265 324.003 319.125C296.253 337.335 263.793 349.415 231.323 356.335C220.173 358.715 208.323 360.995 196.963 361.285C197.243 362.845 198.643 362.525 199.663 362.845C208.823 365.795 219.313 367.545 228.843 369.005C230.163 370.005 228.733 370.415 227.683 370.555C219.063 371.665 209.923 373.305 201.293 373.745C176.843 374.985 152.203 372.435 128.573 366.425L115.293 328.755C114.153 328.865 114.073 329.895 113.783 330.725C112.613 334.015 107.893 358.025 106.313 358.365C93.6234 354.515 82.2334 345.915 72.9034 336.575L95.0434 296.915L59.7134 316.375C54.7434 305.145 53.8334 292.435 55.4934 280.355L101.213 257.645C86.9934 256.705 72.6134 258.955 58.4734 260.425C60.8734 253.495 63.8034 246.615 67.3634 240.195C68.6634 237.855 75.5634 226.305 77.3834 226.245C93.9334 229.415 111.773 230.895 128.363 227.675L92.3534 212.825L91.0334 211.045C98.3834 204.125 105.843 197.235 114.113 191.325C129.603 195.825 145.523 199.455 161.733 199.655L130.633 179.885C134.133 177.125 137.813 174.575 141.553 172.165C144.643 170.175 153.283 164.425 156.423 164.835C168.933 169.105 181.923 172.125 195.123 173.185C189.093 166.935 177.633 162.325 171.833 156.465C171.203 155.825 170.713 155.785 171.003 154.585C186.153 145.885 203.003 137.235 216.053 125.375C229.363 113.265 243.423 93.4848 233.853 75.2048C244.893 78.7148 254.393 92.6648 254.153 104.325C253.493 136.595 219.073 165.375 197.483 185.615C230.833 171.615 268.993 142.685 268.923 102.735C268.863 70.2248 239.263 57.1948 211.283 54.2348C210.103 61.3848 207.573 68.9448 203.863 75.1748C195.623 89.0148 178.803 97.6648 170.013 112.055C164.023 121.865 162.883 135.065 150.163 138.825C144.973 140.365 138.293 140.595 132.903 140.615L144.743 134.365C158.003 124.625 157.773 107.025 165.893 93.9248C164.953 92.5948 155.413 90.0748 153.183 89.5248C143.363 87.1048 126.713 84.0248 116.883 84.6948C113.843 84.9048 100.143 87.0348 98.9634 89.3748C97.2634 92.7648 105.533 107.255 107.973 110.575C108.463 111.245 109.463 111.075 109.513 111.145C109.973 111.745 109.663 113.165 108.373 112.675C107.443 112.315 101.853 108.205 100.593 107.235C90.8934 99.7648 79.8234 88.4748 71.7234 79.3648C70.4034 76.1748 68.6334 72.6148 71.7634 69.9848C88.8634 59.4048 105.793 48.3148 123.163 38.1948C174.603 8.20482 233.143 -25.6952 282.353 29.2448C323.153 74.8048 311.503 129.615 272.493 171.195L272.523 171.205ZM114.153 67.5948C127.613 65.2648 153.013 78.6248 157.193 58.8348C157.603 56.9048 156.493 55.7948 157.583 53.9148C158.403 52.5048 163.083 49.1048 164.713 47.7848C169.903 43.6048 175.823 40.3348 180.923 36.0148C161.113 39.2048 141.163 50.5648 123.853 60.5948C120.983 62.2548 117.873 63.9948 115.153 65.8548C114.453 66.3348 113.713 65.7348 114.143 67.5848L114.153 67.5948ZM21.1634 276.415C-11.8666 312.635 -4.35661 364.395 30.6734 396.135C65.5434 427.725 120.403 438.925 166.373 436.725C207.813 434.735 249.893 421.075 290.773 434.965C306.663 440.365 320.533 450.465 332.363 462.135C330.773 454.415 326.123 446.365 321.623 439.825C280.593 380.225 200.893 402.565 140.123 390.665C107.163 384.215 72.8934 369.135 52.8234 341.225C35.4734 317.105 32.7734 288.665 41.3634 260.475C39.6134 258.725 23.0334 274.355 21.1634 276.415ZM143.743 109.535C136.373 116.045 128.573 122.235 120.403 127.775L97.1634 139.135C116.843 139.885 137.193 128.635 143.753 109.545L143.743 109.535Z"/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 3.9 KiB

spinifex/services/spinifexui/frontend/dist/index.html
Vendored
+1 -1
View File
@@ -4,7 +4,7 @@
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="icon" href="/mulga-logo.svg" type="image/svg+xml" />
<link rel="icon" href="/favicon.ico" sizes="32x32" />
<link rel="icon" href="/favicon.svg" type="image/svg+xml" />
<meta name="theme-color" content="#000000" />
<meta name="description" content="Spinifex Control Panel" />
<script type="module" crossorigin src="/assets/index.js"></script>
spinifex/services/spinifexui/frontend/index.html
+1 -1
View File
@@ -4,7 +4,7 @@
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="icon" href="/mulga-logo.svg" type="image/svg+xml" />
<link rel="icon" href="/favicon.ico" sizes="32x32" />
<link rel="icon" href="/favicon.svg" type="image/svg+xml" />
<meta name="theme-color" content="#000000" />
<meta name="description" content="Spinifex Control Panel" />
</head>
spinifex/services/spinifexui/frontend/public/favicon.ico
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 6.4 KiB

spinifex/services/spinifexui/frontend/public/favicon.svg
+3
View File
@@ -0,0 +1,3 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="-75 -25.5 514 514" fill="#fff">
<path d="M272.523 171.205C247.773 197.595 211.323 215.815 182.103 237.425C165.663 249.575 122.633 281.165 151.093 301.825C185.533 326.835 271.333 304.985 301.013 277.585C317.193 262.645 316.693 242.485 294.703 234.075C270.913 224.975 243.483 231.925 219.353 235.755C218.543 235.885 216.823 236.805 216.993 235.345C227.053 228.885 236.103 220.925 245.933 214.135C255.403 207.585 267.053 198.225 278.643 198.015C314.693 197.345 351.803 215.135 361.203 252.215C376.833 313.885 303.333 362.255 249.833 366.635L280.913 352.235C292.563 345.305 304.353 338.105 314.653 329.215C317.213 327.005 319.073 324.145 321.243 322.035C321.753 321.535 327.053 319.265 324.003 319.125C296.253 337.335 263.793 349.415 231.323 356.335C220.173 358.715 208.323 360.995 196.963 361.285C197.243 362.845 198.643 362.525 199.663 362.845C208.823 365.795 219.313 367.545 228.843 369.005C230.163 370.005 228.733 370.415 227.683 370.555C219.063 371.665 209.923 373.305 201.293 373.745C176.843 374.985 152.203 372.435 128.573 366.425L115.293 328.755C114.153 328.865 114.073 329.895 113.783 330.725C112.613 334.015 107.893 358.025 106.313 358.365C93.6234 354.515 82.2334 345.915 72.9034 336.575L95.0434 296.915L59.7134 316.375C54.7434 305.145 53.8334 292.435 55.4934 280.355L101.213 257.645C86.9934 256.705 72.6134 258.955 58.4734 260.425C60.8734 253.495 63.8034 246.615 67.3634 240.195C68.6634 237.855 75.5634 226.305 77.3834 226.245C93.9334 229.415 111.773 230.895 128.363 227.675L92.3534 212.825L91.0334 211.045C98.3834 204.125 105.843 197.235 114.113 191.325C129.603 195.825 145.523 199.455 161.733 199.655L130.633 179.885C134.133 177.125 137.813 174.575 141.553 172.165C144.643 170.175 153.283 164.425 156.423 164.835C168.933 169.105 181.923 172.125 195.123 173.185C189.093 166.935 177.633 162.325 171.833 156.465C171.203 155.825 170.713 155.785 171.003 154.585C186.153 145.885 203.003 137.235 216.053 125.375C229.363 113.265 243.423 93.4848 233.853 75.2048C244.893 78.7148 254.393 92.6648 254.153 104.325C253.493 136.595 219.073 165.375 197.483 185.615C230.833 171.615 268.993 142.685 268.923 102.735C268.863 70.2248 239.263 57.1948 211.283 54.2348C210.103 61.3848 207.573 68.9448 203.863 75.1748C195.623 89.0148 178.803 97.6648 170.013 112.055C164.023 121.865 162.883 135.065 150.163 138.825C144.973 140.365 138.293 140.595 132.903 140.615L144.743 134.365C158.003 124.625 157.773 107.025 165.893 93.9248C164.953 92.5948 155.413 90.0748 153.183 89.5248C143.363 87.1048 126.713 84.0248 116.883 84.6948C113.843 84.9048 100.143 87.0348 98.9634 89.3748C97.2634 92.7648 105.533 107.255 107.973 110.575C108.463 111.245 109.463 111.075 109.513 111.145C109.973 111.745 109.663 113.165 108.373 112.675C107.443 112.315 101.853 108.205 100.593 107.235C90.8934 99.7648 79.8234 88.4748 71.7234 79.3648C70.4034 76.1748 68.6334 72.6148 71.7634 69.9848C88.8634 59.4048 105.793 48.3148 123.163 38.1948C174.603 8.20482 233.143 -25.6952 282.353 29.2448C323.153 74.8048 311.503 129.615 272.493 171.195L272.523 171.205ZM114.153 67.5948C127.613 65.2648 153.013 78.6248 157.193 58.8348C157.603 56.9048 156.493 55.7948 157.583 53.9148C158.403 52.5048 163.083 49.1048 164.713 47.7848C169.903 43.6048 175.823 40.3348 180.923 36.0148C161.113 39.2048 141.163 50.5648 123.853 60.5948C120.983 62.2548 117.873 63.9948 115.153 65.8548C114.453 66.3348 113.713 65.7348 114.143 67.5848L114.153 67.5948ZM21.1634 276.415C-11.8666 312.635 -4.35661 364.395 30.6734 396.135C65.5434 427.725 120.403 438.925 166.373 436.725C207.813 434.735 249.893 421.075 290.773 434.965C306.663 440.365 320.533 450.465 332.363 462.135C330.773 454.415 326.123 446.365 321.623 439.825C280.593 380.225 200.893 402.565 140.123 390.665C107.163 384.215 72.8934 369.135 52.8234 341.225C35.4734 317.105 32.7734 288.665 41.3634 260.475C39.6134 258.725 23.0334 274.355 21.1634 276.415ZM143.743 109.535C136.373 116.045 128.573 122.235 120.403 127.775L97.1634 139.135C116.843 139.885 137.193 128.635 143.753 109.545L143.743 109.535Z"/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 3.9 KiB

spinifex/services/spinifexui/frontend/src/components/ui/json-editor.test.tsx
+62
View File
@@ -0,0 +1,62 @@
import { render, screen } from "@testing-library/react"
import userEvent from "@testing-library/user-event"
import { useState } from "react"
import { describe, expect, it, vi } from "vitest"
import { JsonEditor } from "./json-editor"
function Harness({ initial }: { initial: string }) {
const [value, setValue] = useState(initial)
return (
<>
<JsonEditor aria-label="json" onChange={setValue} value={value} />
<button type="button">elsewhere</button>
</>
)
}
const blurAway = async (user: ReturnType<typeof userEvent.setup>) =>
await user.click(screen.getByRole("button", { name: "elsewhere" }))
describe("JsonEditor", () => {
it("pretty-prints valid JSON on blur", async () => {
const user = userEvent.setup()
render(<Harness initial='{"a":1}' />)
const editor = screen.getByLabelText("json")
await user.click(editor)
await blurAway(user)
expect(editor).toHaveValue('{\n "a": 1\n}')
})
it("leaves malformed JSON untouched on blur", async () => {
const user = userEvent.setup()
render(<Harness initial="{ not json" />)
const editor = screen.getByLabelText("json")
await user.click(editor)
await blurAway(user)
expect(editor).toHaveValue("{ not json")
})
it("propagates raw input through onChange", async () => {
const user = userEvent.setup()
const onChange = vi.fn()
render(<JsonEditor aria-label="json" onChange={onChange} value="" />)
await user.type(screen.getByLabelText("json"), "x")
expect(onChange).toHaveBeenCalledWith("x")
})
it("marks the field invalid when error is set", () => {
render(<JsonEditor aria-label="json" error onChange={vi.fn()} value="" />)
expect(screen.getByLabelText("json")).toHaveAttribute(
"aria-invalid",
"true",
)
})
})
spinifex/services/spinifexui/frontend/src/components/ui/json-editor.tsx
+46
View File
@@ -0,0 +1,46 @@
import type * as React from "react"
import { Textarea } from "@/components/ui/textarea"
import { formatJson } from "@/lib/json"
import { cn } from "@/lib/utils"
interface JsonEditorProps extends Omit<
React.ComponentProps<typeof Textarea>,
"onChange" | "value"
> {
value: string
onChange: (value: string) => void
error?: boolean
}
// Controlled JSON textarea. On blur it pretty-prints valid JSON; malformed
// input is left untouched so the field's validation error stays visible. The
// plain value/onChange/error contract keeps it RHF-agnostic, so it works with
// Controller or local state and is testable in isolation.
export function JsonEditor({
value,
onChange,
error,
onBlur,
className,
...props
}: JsonEditorProps) {
const handleBlur = (event: React.FocusEvent<HTMLTextAreaElement>) => {
const formatted = formatJson(value)
if (formatted !== null && formatted !== value) {
onChange(formatted)
}
onBlur?.(event)
}
return (
<Textarea
aria-invalid={error}
className={cn("font-mono text-sm", className)}
onBlur={handleBlur}
onChange={(event) => onChange(event.target.value)}
value={value}
{...props}
/>
)
}
spinifex/services/spinifexui/frontend/src/contexts/admin-context.test.tsx
+1 -1
View File
@@ -17,7 +17,7 @@ vi.mock("@/lib/sts", () => ({
}))
vi.mock("@/lib/signed-fetch", () => ({
signedFetch: async (opts: unknown) => mockSignedFetch(opts),
signedFetch: async (opts: unknown) => await mockSignedFetch(opts),
}))
vi.mock("@aws-sdk/client-sts", () => ({
spinifex/services/spinifexui/frontend/src/lib/json.test.ts
+66
View File
@@ -0,0 +1,66 @@
import { describe, expect, it } from "vitest"
import { formatJson, isValidJson, jsonStringSchema } from "./json"
type JsonSchema = ReturnType<typeof jsonStringSchema>
function issueMessages(schema: JsonSchema, value: string): string[] {
const result = schema.safeParse(value)
return result.success ? [] : result.error.issues.map((issue) => issue.message)
}
describe("isValidJson", () => {
it("accepts valid JSON", () => {
expect(isValidJson('{"a":1}')).toBeTruthy()
expect(isValidJson("[]")).toBeTruthy()
expect(isValidJson('"str"')).toBeTruthy()
})
it("rejects malformed JSON and empty strings", () => {
expect(isValidJson("{")).toBeFalsy()
expect(isValidJson("{a:1}")).toBeFalsy()
expect(isValidJson("")).toBeFalsy()
})
})
describe("formatJson", () => {
it("pretty-prints valid JSON with a 2-space indent", () => {
expect(formatJson('{"a":1}')).toBe('{\n "a": 1\n}')
})
it("returns null for malformed JSON", () => {
expect(formatJson("{ not json")).toBeNull()
expect(formatJson("")).toBeNull()
})
})
describe("jsonStringSchema", () => {
const required = jsonStringSchema({ label: "Policy document" })
const optional = jsonStringSchema({
label: "Configuration",
allowEmpty: true,
})
it("accepts valid JSON", () => {
expect(required.safeParse('{"Version":"2012-10-17"}').success).toBeTruthy()
})
it("rejects malformed JSON with a derived message", () => {
expect(issueMessages(required, "{ not json")).toContain(
"Policy document must be valid JSON",
)
})
it("rejects empty input when required", () => {
expect(issueMessages(required, "")).toContain("Policy document is required")
})
it("allows empty input when allowEmpty is set", () => {
expect(optional.safeParse("").success).toBeTruthy()
expect(optional.safeParse(" ").success).toBeTruthy()
})
it("still rejects malformed input when allowEmpty is set", () => {
expect(optional.safeParse("{bad").success).toBeFalsy()
})
})
spinifex/services/spinifexui/frontend/src/lib/json.ts
+37
View File
@@ -0,0 +1,37 @@
import { z } from "zod"
export function isValidJson(value: string): boolean {
try {
JSON.parse(value)
return true
} catch {
return false
}
}
// Pretty-print JSON with a 2-space indent. Returns null when the input is not
// valid JSON so callers can leave the original text untouched.
export function formatJson(value: string): string | null {
try {
return JSON.stringify(JSON.parse(value), null, 2)
} catch {
return null
}
}
// Zod field for a JSON document entered as a string. Messages derive from
// `label` so they read naturally per field. When `allowEmpty` is true an
// empty/whitespace value passes (optional field); otherwise it is required.
export function jsonStringSchema(opts: {
label: string
allowEmpty?: boolean
}) {
const { label, allowEmpty = false } = opts
const base = allowEmpty
? z.string()
: z.string().min(1, `${label} is required`)
return base.refine(
(value) => (allowEmpty && value.trim() === "" ? true : isValidJson(value)),
{ message: `${label} must be valid JSON` },
)
}
spinifex/services/spinifexui/frontend/src/routes/_auth/eks/(clusters)/-components/addons-tab.tsx
+28 -24
View File
@@ -3,7 +3,7 @@ import { zodResolver } from "@hookform/resolvers/zod"
import { useQuery, useSuspenseQuery } from "@tanstack/react-query"
import { ArrowUpCircle, Trash2 } from "lucide-react"
import { useEffect, useState } from "react"
import { useForm } from "react-hook-form"
import { Controller, useForm } from "react-hook-form"
import { DeleteConfirmationDialog } from "@/components/delete-confirmation-dialog"
import { DetailCard } from "@/components/detail-card"
@@ -21,7 +21,7 @@ import { Badge } from "@/components/ui/badge"
import { Button } from "@/components/ui/button"
import { Field, FieldError, FieldTitle } from "@/components/ui/field"
import { Input } from "@/components/ui/input"
import { Textarea } from "@/components/ui/textarea"
import { JsonEditor } from "@/components/ui/json-editor"
import { useCreateAddon, useDeleteAddon, useUpdateAddon } from "@/mutations/eks"
import {
eksAddonQueryOptions,
@@ -29,6 +29,7 @@ import {
eksAddonVersionsQueryOptions,
} from "@/queries/eks"
import {
addonConfigurationSchema,
addonStatusVariant,
type CreateAddonFormValues,
createAddonSchema,
@@ -40,18 +41,6 @@ const STATUS_POLL_MS = 5000
const selectClassName =
"h-7 w-full rounded-md border border-input bg-input/20 px-2 text-sm"
function isValidJson(value: string): boolean {
if (value.trim() === "") {
return true
}
try {
JSON.parse(value)
return true
} catch {
return false
}
}
function AddAddonDialog({
clusterName,
catalog,
@@ -71,6 +60,7 @@ function AddAddonDialog({
)
const {
control,
formState: { errors },
handleSubmit,
register,
@@ -189,12 +179,21 @@ function AddAddonDialog({
<FieldTitle>
<label htmlFor="addon-config">Configuration values (JSON)</label>
</FieldTitle>
<Textarea
aria-invalid={!!errors.configurationValues}
id="addon-config"
placeholder='{"replicaCount": 2}'
rows={4}
{...register("configurationValues")}
<Controller
control={control}
name="configurationValues"
render={({ field: { onBlur, onChange, ref, value } }) => (
<JsonEditor
error={!!errors.configurationValues}
id="addon-config"
onBlur={onBlur}
onChange={onChange}
placeholder='{"replicaCount": 2}'
ref={ref}
rows={4}
value={value}
/>
)}
/>
<FieldError errors={[errors.configurationValues]} />
</Field>
@@ -257,10 +256,14 @@ function UpdateAddonDialog({
])
const handleConfirm = () => {
if (!isValidJson(config)) {
setError("Configuration must be valid JSON")
const result = addonConfigurationSchema.safeParse(config)
if (!result.success) {
setError(
result.error.issues[0]?.message ?? "Configuration must be valid JSON",
)
return
}
setError(undefined)
updateAddon.mutate(
{
clusterName,
@@ -320,9 +323,10 @@ function UpdateAddonDialog({
Configuration values (JSON)
</label>
</FieldTitle>
<Textarea
<JsonEditor
error={!!error}
id="update-addon-config"
onChange={(e) => setConfig(e.target.value)}
onChange={setConfig}
rows={4}
value={config}
/>
spinifex/services/spinifexui/frontend/src/routes/_auth/iam/(policies)/create-policy.tsx
+16 -8
View File
@@ -1,6 +1,6 @@
import { zodResolver } from "@hookform/resolvers/zod"
import { createFileRoute, useNavigate } from "@tanstack/react-router"
import { useForm, useWatch } from "react-hook-form"
import { Controller, useForm, useWatch } from "react-hook-form"
import { BackLink } from "@/components/back-link"
import {
@@ -12,7 +12,7 @@ import { FormActions } from "@/components/form-actions"
import { PageHeading } from "@/components/page-heading"
import { Field, FieldError, FieldTitle } from "@/components/ui/field"
import { Input } from "@/components/ui/input"
import { Textarea } from "@/components/ui/textarea"
import { JsonEditor } from "@/components/ui/json-editor"
import { useCreatePolicy } from "@/mutations/iam"
import { type CreatePolicyFormData, createPolicySchema } from "@/types/iam"
@@ -105,12 +105,20 @@ function CreatePolicy() {
<FieldTitle>
<label htmlFor="policyDocument">Policy Document (JSON)</label>
</FieldTitle>
<Textarea
aria-invalid={!!errors.policyDocument}
className="font-mono text-sm"
id="policyDocument"
rows={15}
{...register("policyDocument")}
<Controller
control={control}
name="policyDocument"
render={({ field: { onBlur, onChange, ref, value } }) => (
<JsonEditor
error={!!errors.policyDocument}
id="policyDocument"
onBlur={onBlur}
onChange={onChange}
ref={ref}
rows={15}
value={value}
/>
)}
/>
<FieldError errors={[errors.policyDocument]} />
</Field>
spinifex/services/spinifexui/frontend/src/routes/_auth/iam/(roles)/create-role.tsx
+16 -8
View File
@@ -1,6 +1,6 @@
import { zodResolver } from "@hookform/resolvers/zod"
import { createFileRoute, useNavigate } from "@tanstack/react-router"
import { useForm, useWatch } from "react-hook-form"
import { Controller, useForm, useWatch } from "react-hook-form"
import { BackLink } from "@/components/back-link"
import {
@@ -12,7 +12,7 @@ import { FormActions } from "@/components/form-actions"
import { PageHeading } from "@/components/page-heading"
import { Field, FieldError, FieldTitle } from "@/components/ui/field"
import { Input } from "@/components/ui/input"
import { Textarea } from "@/components/ui/textarea"
import { JsonEditor } from "@/components/ui/json-editor"
import { useCreateRole } from "@/mutations/iam"
import {
type CreateRoleFormData,
@@ -93,12 +93,20 @@ function CreateRole() {
Trust Policy (JSON)
</label>
</FieldTitle>
<Textarea
aria-invalid={!!errors.assumeRolePolicyDocument}
className="font-mono text-sm"
id="assumeRolePolicyDocument"
rows={15}
{...register("assumeRolePolicyDocument")}
<Controller
control={control}
name="assumeRolePolicyDocument"
render={({ field: { onBlur, onChange, ref, value } }) => (
<JsonEditor
error={!!errors.assumeRolePolicyDocument}
id="assumeRolePolicyDocument"
onBlur={onBlur}
onChange={onChange}
ref={ref}
rows={15}
value={value}
/>
)}
/>
<FieldError errors={[errors.assumeRolePolicyDocument]} />
</Field>
spinifex/services/spinifexui/frontend/src/types/eks.ts
+7 -14
View File
@@ -1,6 +1,7 @@
import type { AMITypes, CapacityTypes } from "@aws-sdk/client-eks"
import { z } from "zod"
import { jsonStringSchema } from "@/lib/json"
import { isValidCidr } from "@/lib/subnet-calculator"
// K3s control plane is pinned per Mulga release; expose the supported
@@ -89,24 +90,16 @@ export const createNodegroupSchema = z
export type CreateNodegroupFormValues = z.infer<typeof createNodegroupSchema>
export const addonConfigurationSchema = jsonStringSchema({
label: "Configuration",
allowEmpty: true,
})
export const createAddonSchema = z.object({
addonName: z.string().min(1, "Add-on is required"),
addonVersion: z.string().min(1, "Version is required"),
serviceAccountRoleArn: z.string(),
configurationValues: z.string().refine(
(v) => {
if (v.trim() === "") {
return true
}
try {
JSON.parse(v)
return true
} catch {
return false
}
},
{ message: "Configuration must be valid JSON" },
),
configurationValues: addonConfigurationSchema,
})
export type CreateAddonFormValues = z.infer<typeof createAddonSchema>
spinifex/services/spinifexui/frontend/src/types/iam.ts
+4 -28
View File
@@ -1,5 +1,7 @@
import { z } from "zod"
import { jsonStringSchema } from "@/lib/json"
export const createUserSchema = z.object({
userName: z
.string()
@@ -18,20 +20,7 @@ export const createPolicySchema = z.object({
.max(128, "Policy name must be 128 characters or less")
.regex(/^[\w+=,.@-]+$/, "Policy name contains invalid characters"),
description: z.string().optional(),
policyDocument: z
.string()
.min(1, "Policy document is required")
.refine(
(val) => {
try {
JSON.parse(val)
return true
} catch {
return false
}
},
{ message: "Policy document must be valid JSON" },
),
policyDocument: jsonStringSchema({ label: "Policy document" }),
})
export type CreatePolicyFormData = z.infer<typeof createPolicySchema>
@@ -59,20 +48,7 @@ export const createRoleSchema = z.object({
.regex(/^[\w+=,.@-]+$/, "Role name contains invalid characters"),
path: z.string().optional(),
description: z.string().optional(),
assumeRolePolicyDocument: z
.string()
.min(1, "Trust policy is required")
.refine(
(val) => {
try {
JSON.parse(val)
return true
} catch {
return false
}
},
{ message: "Trust policy must be valid JSON" },
),
assumeRolePolicyDocument: jsonStringSchema({ label: "Trust policy" }),
})
export type CreateRoleFormData = z.infer<typeof createRoleSchema>
spinifex/services/viperblockd/viperblockd.go
+90 -5
View File
@@ -7,6 +7,7 @@ import (
"log/slog"
"os"
"os/signal"
"path/filepath"
"strconv"
"strings"
"sync"
@@ -197,10 +198,12 @@ func makeConfigUpdateHandler(vb *viperblock.VB, volumeName string) nats.MsgHandl
}
}
// openVolumeVB opens an existing viperblock volume for a short-lived, exclusive
// state update (detached-volume config reseal). The caller MUST Close the
// returned VB. Construction mirrors the ebs.mount path so encrypted volumes open
// with the master key and matching encryption state.
// openVolumeVB constructs and opens an existing viperblock volume with its
// config state loaded (LoadState) but NOT its block map. Construction mirrors
// the ebs.mount path so encrypted volumes open with the master key and matching
// encryption state. Callers that Close() the VB MUST go through
// openLoadedVolumeVB instead, so the block map is restored before Close()
// flushes it back to predastore.
func openVolumeVB(cfg *Config, volumeName string) (*viperblock.VB, error) {
s3cfg := s3.S3Config{
VolumeName: volumeName,
@@ -231,6 +234,70 @@ func openVolumeVB(cfg *Config, volumeName string) (*viperblock.VB, error) {
return vb, nil
}
// isAuxVolume reports whether a volume is an -efi/-cloudinit auxiliary volume.
// Auxiliary volumes are recreated on launch and carry no durable guest data, so
// they never need sealing to predastore.
func isAuxVolume(volumeName string) bool {
return strings.HasSuffix(volumeName, "-efi") || strings.HasSuffix(volumeName, "-cloudinit")
}
// volumeNeedsSeal reports whether an unmounted volume must be sealed to
// predastore on this node: it carries durable guest data (not an auxiliary
// volume) and has local viperblock state under baseDir/<volume> to flush. A
// node that never held the local WAL has nothing to seal.
func volumeNeedsSeal(volumeName, baseDir string) bool {
if isAuxVolume(volumeName) {
return false
}
_, err := os.Stat(filepath.Join(baseDir, volumeName))
return err == nil
}
// openLoadedVolumeVB opens a detached volume and fully restores its state for a
// short-lived operation that ends in Close(): LoadState + LoadBlockState +
// RecoverLocalWALs. Skipping LoadBlockState would leave an empty in-memory block
// map that Close() then flushes over the good checkpoint in predastore — silent
// data loss (a reattach then finds an empty map, bad superblock). The caller
// MUST Close the returned VB; on error the WAL syncer is stopped and no VB is
// returned. The caller MUST ensure no nbdkit process is writing the shared
// BaseDir first (post-KillProcess, or volume detached).
func openLoadedVolumeVB(cfg *Config, volumeName string) (*viperblock.VB, error) {
vb, err := openVolumeVB(cfg, volumeName)
if err != nil {
return nil, err
}
if err := vb.LoadBlockState(); err != nil {
vb.StopWALSyncer()
return nil, fmt.Errorf("load block state: %w", err)
}
// RecoverLocalWALs is fail-closed on integrity errors and persists recovered
// state itself; on failure retain the local WAL (no Close) for retry.
if err := vb.RecoverLocalWALs(); err != nil {
vb.StopWALSyncer()
return nil, fmt.Errorf("recover local WALs: %w", err)
}
return vb, nil
}
// sealVolumeVB persists a detached volume's block->object map to predastore.
// The runtime nbdkit plugin is the only path that flushes the map on close and
// it does not reliably fire on detach, so without this seal a reattach on a
// node lacking the local WAL finds no checkpoint (bad superblock). It mirrors
// the plugin's recover sequence (LoadBlockState + RecoverLocalWALs replay
// un-sealed chunk WALs) then Close()s to flush the map.
func sealVolumeVB(cfg *Config, volumeName string) error {
vb, err := openLoadedVolumeVB(cfg, volumeName)
if err != nil {
return err
}
// Close removes local files only after the predastore writes succeed, so a
// failed seal leaves the WAL intact rather than losing data.
if err := vb.Close(); err != nil {
return fmt.Errorf("seal close: %w", err)
}
return nil
}
// respondJSON marshals data and sends it as a NATS response. On marshal
// failure a raw JSON error string is sent instead.
func respondJSON(msg *nats.Msg, data any) {
@@ -452,6 +519,24 @@ func launchService(cfg *Config) (err error) {
slog.Error("Failed to kill nbdkit process", "pid", matched.PID, "err", err)
}
// nbdkit is now dead, so no process writes the shared BaseDir: seal
// the block map to predastore for volumes that hold local state to
// flush (see volumeNeedsSeal).
if volumeNeedsSeal(matched.Name, cfg.BaseDir) {
if err := sealVolumeVB(cfg, matched.Name); err != nil {
slog.Error("ebs.unmount: failed to seal volume to predastore", "volume", matched.Name, "err", err)
ebsResponse.Error = fmt.Sprintf("seal volume: %v", err)
} else {
slog.Info("ebs.unmount: volume sealed to predastore", "volume", matched.Name)
}
} else if !isAuxVolume(matched.Name) {
// A durable volume reached unmount with no local WAL under
// BaseDir: this node never held its state, so there is nothing to
// seal. WARN since a missing local WAL for a volume we expected to
// seal can mask the durability gap the seal closes.
slog.Warn("ebs.unmount: no local viperblock state for volume, skipping seal", "volume", matched.Name, "baseDir", cfg.BaseDir)
}
// Remove the socket file if using socket transport
if matched.Socket != "" {
slog.Info("Removing socket file", "socket", matched.Socket)
@@ -547,7 +632,7 @@ func launchService(cfg *Config) (err error) {
return
}
vb, err := openVolumeVB(cfg, req.Volume)
vb, err := openLoadedVolumeVB(cfg, req.Volume)
if err != nil {
slog.Error("ebs.config: failed to open detached volume", "volume", req.Volume, "err", err)
respondJSON(msg, types.EBSConfigUpdateResponse{Volume: req.Volume, Error: fmt.Sprintf("open volume: %v", err)})
spinifex/utils/utils_test.go
-17
View File
@@ -1182,23 +1182,6 @@ func TestHashMAC_IdSeparates(t *testing.T) {
assert.NotEqual(t, a, b)
}
func TestHashMAC_Distribution(t *testing.T) {
// 100k random ids — 0 collisions expected. 46-bit space, birthday-
// paradox 1% at ~1.2M; 100k is well below.
if testing.Short() {
t.Skip("skipping distribution test in -short mode")
}
seen := make(map[string]string, 100_000)
for range 100_000 {
id := GenerateResourceID("res")
mac := HashMAC(id)
if prev, ok := seen[mac]; ok {
t.Fatalf("collision on %s: id=%s prev=%s", mac, id, prev)
}
seen[mac] = id
}
}
// writeTempImage writes content to a file named name under t.TempDir() and
// returns the absolute path.
func writeTempImage(t *testing.T, name string, content []byte) string {
spinifex/vm/manager_deps_test.go
+4 -1
View File
@@ -116,6 +116,7 @@ type fakeVolumeMounter struct {
mountedOne, unmountedOne []string
mountErr error
mountOneErr error
unmountOneErr error
mountOneURI string
// onMount fires synchronously inside Mount before the configured
// mountErr is returned. Used by lifecycle tests to simulate a
@@ -157,10 +158,12 @@ func (f *fakeVolumeMounter) MountOne(req *types.EBSRequest) error {
return nil
}
func (f *fakeVolumeMounter) UnmountOne(req types.EBSRequest) {
func (f *fakeVolumeMounter) UnmountOne(req types.EBSRequest) error {
f.mu.Lock()
f.unmountedOne = append(f.unmountedOne, req.Name)
err := f.unmountOneErr
f.mu.Unlock()
return err
}
var _ VolumeMounter = (*fakeVolumeMounter)(nil)
spinifex/vm/restore_test.go
+3 -3
View File
@@ -486,9 +486,9 @@ func (f *recoveryMounter) Mount(v *VM) error {
return nil
}
func (f *recoveryMounter) Unmount(*VM) error { return nil }
func (f *recoveryMounter) MountOne(*types.EBSRequest) error { return nil }
func (f *recoveryMounter) UnmountOne(types.EBSRequest) {}
func (f *recoveryMounter) Unmount(*VM) error { return nil }
func (f *recoveryMounter) MountOne(*types.EBSRequest) error { return nil }
func (f *recoveryMounter) UnmountOne(types.EBSRequest) error { return nil }
var _ VolumeMounter = (*recoveryMounter)(nil)
spinifex/vm/volume_mounter.go
+4 -3
View File
@@ -16,7 +16,8 @@ type VolumeMounter interface {
// MountOne sends ebs.mount for a single request and writes the resolved
// NBDURI back into req.NBDURI on success. Used by hot-attach.
MountOne(req *types.EBSRequest) error
// UnmountOne sends ebs.unmount for a single request. Best-effort: errors
// are logged and tolerated.
UnmountOne(req types.EBSRequest)
// UnmountOne sends ebs.unmount for a single request and returns any error.
// ebs.unmount drives the synchronous block-map seal to predastore, so hot
// detach gates the volume's available transition on the returned error.
UnmountOne(req types.EBSRequest) error
}
spinifex/vm/volumes.go
+47 -7
View File
@@ -27,6 +27,18 @@ type AttachVolumeResult struct {
GuestDevice string
}
// rollbackUnmount best-effort unmounts a volume while unwinding a failed
// AttachVolume. The volume never took guest writes, so the unmount seal is a
// no-op; a failure is logged and tolerated rather than masking the attach error.
func (m *Manager) rollbackUnmount(req types.EBSRequest) {
if m.deps.VolumeMounter == nil {
return
}
if err := m.deps.VolumeMounter.UnmountOne(req); err != nil {
slog.Warn("AttachVolume: rollback unmount failed", "volume", req.Name, "err", err)
}
}
// AttachVolume hot-plugs a volume via the QMP pipeline (mount → blockdev-add →
// device_add). Partial state is rolled back on failure. If device is empty, the
// next free /dev/sd[f-p] slot is allocated. Instance must be in StateRunning.
@@ -61,7 +73,7 @@ func (m *Manager) AttachVolume(id, volumeID, device string) (AttachVolumeResult,
// Empty-URI response leaves backend NBD state ambiguous; unmount
// defensively to avoid orphaning a half-started mount.
if errors.Is(err, ErrMountAmbiguous) {
m.deps.VolumeMounter.UnmountOne(ebsRequest)
m.rollbackUnmount(ebsRequest)
}
return AttachVolumeResult{}, fmt.Errorf("mount volume %s: %w", volumeID, err)
}
@@ -69,7 +81,7 @@ func (m *Manager) AttachVolume(id, volumeID, device string) (AttachVolumeResult,
serverType, socketPath, nbdHost, nbdPort, err := utils.ParseNBDURI(ebsRequest.NBDURI)
if err != nil {
slog.Error("AttachVolume: failed to parse NBDURI", "uri", ebsRequest.NBDURI, "err", err)
m.deps.VolumeMounter.UnmountOne(ebsRequest)
m.rollbackUnmount(ebsRequest)
return AttachVolumeResult{}, fmt.Errorf("parse NBDURI: %w", err)
}
@@ -92,7 +104,7 @@ func (m *Manager) AttachVolume(id, volumeID, device string) (AttachVolumeResult,
},
}, instance.ID); err != nil {
slog.Error("AttachVolume: QMP object-add iothread failed", "volumeId", volumeID, "err", err)
m.deps.VolumeMounter.UnmountOne(ebsRequest)
m.rollbackUnmount(ebsRequest)
return AttachVolumeResult{}, fmt.Errorf("QMP object-add iothread: %w", err)
}
@@ -107,7 +119,7 @@ func (m *Manager) AttachVolume(id, volumeID, device string) (AttachVolumeResult,
},
}, instance.ID); err != nil {
slog.Error("AttachVolume: QMP blockdev-add failed", "volumeId", volumeID, "err", err)
m.deps.VolumeMounter.UnmountOne(ebsRequest)
m.rollbackUnmount(ebsRequest)
return AttachVolumeResult{}, fmt.Errorf("QMP blockdev-add: %w", err)
}
@@ -146,7 +158,7 @@ func (m *Manager) AttachVolume(id, volumeID, device string) (AttachVolumeResult,
slog.Error("AttachVolume: rollback blockdev-del failed, skipping EBS unmount",
"volumeId", volumeID, "err", delErr)
} else {
m.deps.VolumeMounter.UnmountOne(ebsRequest)
m.rollbackUnmount(ebsRequest)
}
return AttachVolumeResult{}, fmt.Errorf("QMP device_add: %w", err)
}
@@ -313,9 +325,16 @@ func (m *Manager) DetachVolume(id, volumeID, device string, force bool) (string,
"volumeId", volumeID, "err", iothreadErr)
}
// ebs.unmount (best-effort).
// ebs.unmount drives the synchronous block-map seal to predastore. On
// failure the volume's local WAL is retained, so keep the volume attached
// and return the error; an AWS-CLI retry re-drives the seal and same-node
// reattach still works meanwhile.
if m.deps.VolumeMounter != nil {
m.deps.VolumeMounter.UnmountOne(ebsReq)
if err := m.deps.VolumeMounter.UnmountOne(ebsReq); err != nil {
slog.Error("DetachVolume: ebs.unmount seal failed, leaving volume attached",
"volumeId", volumeID, "err", err)
return "", fmt.Errorf("ebs.unmount seal: %w", err)
}
}
// State cleanup.
@@ -373,6 +392,15 @@ func (m *Manager) tryBlockdevDel(instance *VM, nodeName string) error {
}
return nil
}
// A prior detach already removed the block node (e.g. an AWS-CLI retry
// after the ebs.unmount seal failed): treat as success so the retry
// resumes through object-del to the seal instead of wedging on a
// now-absent node.
if isQMPNodeNotFound(err) {
slog.Info("DetachVolume: block node already removed (resuming detach)",
"nodeName", nodeName, "err", err)
return nil
}
lastErr = err
if !isQMPNodeInUse(err) {
return err
@@ -406,3 +434,15 @@ func isQMPNodeInUse(err error) bool {
}
return strings.Contains(err.Error(), "in use")
}
// isQMPNodeNotFound reports whether err is a QMP error for a block node that no
// longer exists. blockdev-del on an already-removed node returns this (QEMU:
// "Failed to find node with node-name=..."), making blockdev-del idempotent
// across detach retries — mirroring isQMPDeviceNotFound for device_del.
func isQMPNodeNotFound(err error) bool {
if err == nil {
return false
}
msg := err.Error()
return strings.Contains(msg, "Failed to find node") || strings.Contains(msg, "Cannot find device")
}
spinifex/vm/volumes_test.go
+87 -1
View File
@@ -231,6 +231,24 @@ func TestIsQMPNodeInUse(t *testing.T) {
}
}
func TestIsQMPNodeNotFound(t *testing.T) {
tests := []struct {
name string
err error
want bool
}{
{name: "nil", err: nil, want: false},
{name: "matches 'Failed to find node'", err: errors.New("QMP error: GenericError: Failed to find node with node-name='nbd-x'"), want: true},
{name: "matches 'Cannot find device'", err: errors.New("QMP error: GenericError: Cannot find device='' nor node-name='nbd-x'"), want: true},
{name: "in-use is not not-found", err: errors.New("QMP error: GenericError: Node 'nbd-x' is in use"), want: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, isQMPNodeNotFound(tt.err))
})
}
}
func TestAttachVolume_InstanceNotFound(t *testing.T) {
m := NewManager()
_, err := m.AttachVolume("i-missing", "vol-1", "")
@@ -386,11 +404,63 @@ func TestDetachVolume_DeviceGuards(t *testing.T) {
assert.Contains(t, executed, "blockdev-del",
"proceed path must issue blockdev-del")
assert.Equal(t, []string{tt.req.Name}, mounter.unmountedOne,
"proceed path must invoke UnmountOne best-effort")
"proceed path must invoke UnmountOne and, on success, complete the detach")
})
}
}
// TestDetachVolume_SealFailureGatesAvailable covers the durability gate in the
// detach-seal fix: when ebs.unmount (the synchronous block-map seal) fails,
// DetachVolume must propagate the error, must NOT flip the volume to "available",
// and must leave it in EBSRequests so the volume stays attached/retryable and its
// local WAL is retained. A regression back to fire-and-forget unmount would let a
// volume go available with an unsealed block map — the bad-superblock bug.
func TestDetachVolume_SealFailureGatesAvailable(t *testing.T) {
recorder := &qmpRecorder{}
qmpClient, cancel := newMockQMPClient(t, func(cmd qmp.QMPCommand) map[string]any {
recorder.record(cmd)
return nil
})
defer cancel()
sealErr := errors.New("seal volume: predastore unreachable")
mounter := &fakeVolumeMounter{unmountOneErr: sealErr}
stateUpdater := &fakeVolumeStateUpdater{}
m := NewManagerWithDeps(Deps{VolumeMounter: mounter, VolumeStateUpdater: stateUpdater})
m.Insert(&VM{
ID: "i-1",
Status: StateRunning,
Instance: &ec2.Instance{},
QMPClient: qmpClient,
EBSRequests: types.EBSRequests{
Requests: []types.EBSRequest{{Name: "vol-1", DeviceName: "/dev/sdf"}},
},
})
_, err := m.DetachVolume("i-1", "vol-1", "", false)
require.Error(t, err)
assert.ErrorIs(t, err, sealErr,
"seal failure must propagate so the caller keeps the volume attached/retryable")
assert.Equal(t, []string{"vol-1"}, mounter.unmountedOne,
"DetachVolume must still attempt the unmount/seal before gating")
assert.Empty(t, stateUpdater.snapshot(),
"a failed seal must not drive any volume-state update — least of all the available transition")
instance, ok := m.Get("i-1")
require.True(t, ok)
instance.EBSRequests.Mu.Lock()
defer instance.EBSRequests.Mu.Unlock()
var retained bool
for _, r := range instance.EBSRequests.Requests {
if r.Name == "vol-1" {
retained = true
}
}
assert.True(t, retained,
"volume must stay in EBSRequests after a seal failure so reattach/retry can recover it")
}
func TestReboot_InstanceNotFound(t *testing.T) {
m := NewManager()
err := m.Reboot("i-missing")
@@ -805,6 +875,22 @@ func TestTryBlockdevDel(t *testing.T) {
"non-retryable error must stop the loop on the attempt that produced it")
})
t.Run("node-not-found returns nil so a detach retry resumes to the seal", func(t *testing.T) {
var attempts atomic.Int32
qmpClient, cancel := newMockQMPClient(t, func(cmd qmp.QMPCommand) map[string]any {
attempts.Add(1)
return map[string]any{"error": map[string]any{"class": "GenericError", "desc": "Failed to find node with node-name='nbd-vol-1'"}}
})
defer cancel()
m := NewManagerWithDeps(Deps{})
err := m.tryBlockdevDel(&VM{ID: "i-1", QMPClient: qmpClient}, "nbd-vol-1")
require.NoError(t, err,
"an already-removed block node must be idempotent so a detach retry can re-drive the unmount seal")
assert.Equal(t, int32(1), attempts.Load(),
"node-not-found must not be retried")
})
t.Run("twenty in-use errors return the last error", func(t *testing.T) {
var attempts atomic.Int32
qmpClient, cancel := newMockQMPClient(t, func(cmd qmp.QMPCommand) map[string]any {
tests/e2e/Makefile
+4 -3
View File
@@ -1,4 +1,4 @@
PKG ?= ./cert/... ./lb/... ./eks/... ./multinode/... ./baremetal/... ./reboot/... ./tofu/...
PKG ?= ./cert/... ./lb/... ./eks/... ./iam/... ./multinode/... ./baremetal/... ./reboot/... ./tofu/...
ARTIFACT_DIR ?= /tmp/spinifex-e2e-artifacts
TIMEOUT ?= 30m
MODE ?= single
@@ -11,9 +11,9 @@ SUITE_TAG ?= $(notdir $(patsubst %/...,%,$(firstword $(PKG))))
# named suite into BIN_DIR/<suite>.test so CI can ship binaries (not source +
# go toolchain) to the remote VM.
BIN_DIR ?= $(CURDIR)/_bin
SUITES_BUILD ?= cert lb eks single gpu
SUITES_BUILD ?= cert lb eks single iam gpu
.PHONY: e2e e2e-cert e2e-lb e2e-eks e2e-single e2e-multinode e2e-bm e2e-reboot e2e-tofu e2e-gpu e2e-all clean e2e-build
.PHONY: e2e e2e-cert e2e-lb e2e-eks e2e-single e2e-iam e2e-multinode e2e-bm e2e-reboot e2e-tofu e2e-gpu e2e-all clean e2e-build
ifeq ($(GOTESTSUM),1)
E2E_CMD = gotestsum \
@@ -36,6 +36,7 @@ e2e-cert: ; $(MAKE) e2e PKG=./cert/...
e2e-lb: ; $(MAKE) e2e PKG=./lb/...
e2e-eks: ; $(MAKE) e2e PKG=./eks/...
e2e-single: ; $(MAKE) e2e PKG=./single/...
e2e-iam: ; $(MAKE) e2e PKG=./iam/...
e2e-multinode: ; $(MAKE) e2e PKG=./multinode/... MODE=multinode
e2e-bm: ; $(MAKE) e2e PKG=./baremetal/... MODE=baremetal
e2e-reboot: ; $(MAKE) e2e PKG=./reboot/...
tests/e2e/eks/eks_test.go
+103
View File
@@ -17,6 +17,7 @@ import (
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/eks"
"github.com/aws/aws-sdk-go/service/iam"
"github.com/aws/aws-sdk-go/service/sts"
"github.com/mulgadc/spinifex/tests/e2e/harness"
"github.com/stretchr/testify/assert"
@@ -208,6 +209,11 @@ func TestEKS(t *testing.T) {
}, 3*time.Minute, 10*time.Second)
})
t.Run("IRSAWebIdentity", func(t *testing.T) {
requireClusterReady(t, fx)
runIRSAWebIdentity(t, c, env, artifacts, fx)
})
t.Run("DeleteCluster", func(t *testing.T) {
requireClusterReady(t, fx)
_, err := c.EKS.DeleteCluster(&eks.DeleteClusterInput{Name: aws.String(fx.ClusterName)})
@@ -217,6 +223,103 @@ func TestEKS(t *testing.T) {
})
}
// runIRSAWebIdentity exercises the IRSA token-exchange path at the API level
// (no pod, no nodegroup): register the cluster's OIDC provider, create a role
// trusting it, mint a real ServiceAccount token via `kubectl create token`
// (signed by the cluster's OIDC key, JWKS published at CreateCluster), exchange
// it through AssumeRoleWithWebIdentity, and prove the returned credentials are
// usable via GetCallerIdentity.
func runIRSAWebIdentity(t *testing.T, c *harness.AWSClient, env *harness.Env, artifacts string, fx *clusterFixture) {
require.NotNil(t, fx.Cluster.Identity, "ACTIVE cluster must expose Identity")
require.NotNil(t, fx.Cluster.Identity.Oidc, "ACTIVE cluster must expose Identity.Oidc")
issuer := aws.StringValue(fx.Cluster.Identity.Oidc.Issuer)
require.NotEmpty(t, issuer, "cluster OIDC issuer must be published")
t.Logf("OIDC issuer: %s", issuer)
// 1) Register the cluster's OIDC provider in the caller account so the STS
// handler will accept tokens carrying this issuer.
oidcOut, err := c.IAM.CreateOpenIDConnectProvider(&iam.CreateOpenIDConnectProviderInput{
Url: aws.String(issuer),
ClientIDList: aws.StringSlice([]string{"sts.amazonaws.com"}),
ThumbprintList: aws.StringSlice([]string{"0000000000000000000000000000000000000000"}),
})
require.NoError(t, err, "create-open-id-connect-provider")
providerArn := aws.StringValue(oidcOut.OpenIDConnectProviderArn)
require.NotEmpty(t, providerArn, "OIDC provider ARN empty")
t.Cleanup(func() {
_, _ = c.IAM.DeleteOpenIDConnectProvider(&iam.DeleteOpenIDConnectProviderInput{
OpenIDConnectProviderArn: aws.String(providerArn),
})
})
t.Logf("OIDC provider: %s", providerArn)
// 2) Create a role whose trust policy federates the OIDC provider. No
// Condition block — the Federated principal + AssumeRoleWithWebIdentity
// action are sufficient to grant, which keeps the test independent of the
// condition-key issuer-prefix format.
roleName := fmt.Sprintf("%s-irsa", fx.ClusterName)
trustPolicy := fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Federated":%q},"Action":"sts:AssumeRoleWithWebIdentity"}]}`, providerArn)
roleOut, err := c.IAM.CreateRole(&iam.CreateRoleInput{
RoleName: aws.String(roleName),
AssumeRolePolicyDocument: aws.String(trustPolicy),
Description: aws.String("E2E IRSA web-identity role"),
})
require.NoError(t, err, "create-role")
roleArn := aws.StringValue(roleOut.Role.Arn)
require.NotEmpty(t, roleArn, "role ARN empty")
t.Cleanup(func() {
_, _ = c.IAM.DeleteRole(&iam.DeleteRoleInput{RoleName: aws.String(roleName)})
})
t.Logf("role: %s", roleArn)
// 3) Mint a ServiceAccount token bound to sts.amazonaws.com. `kubectl create
// token` uses the TokenRequest API — no pod required. The token's iss is
// the cluster OIDC issuer and aud includes sts.amazonaws.com (k3s is wired
// with --service-account-issuer / --api-audiences at CreateCluster).
kcPath := writeKubeconfig(t, artifacts, fx.Cluster)
kc := harness.NewKubectl(t, kcPath, getTokenEnv(t, env))
tokenOut, err := kc.Run(30*time.Second, "create", "token", "default",
"--namespace", "default", "--audience", "sts.amazonaws.com")
require.NoErrorf(t, err, "kubectl create token:\n%s", tokenOut)
token := strings.TrimSpace(tokenOut)
require.Equal(t, 2, strings.Count(token, "."), "web-identity token must be a JWT (3 dot-separated parts)")
// 4) Exchange the token. AssumeRoleWithWebIdentity is anonymous (the SDK
// strips SigV4 for this op); the JWT is the identity.
const sessionName = "e2e-irsa"
var assumeOut *sts.AssumeRoleWithWebIdentityOutput
harness.EventuallyErr(t, func() error {
out, aerr := c.STS.AssumeRoleWithWebIdentity(&sts.AssumeRoleWithWebIdentityInput{
RoleArn: aws.String(roleArn),
RoleSessionName: aws.String(sessionName),
WebIdentityToken: aws.String(token),
})
if aerr != nil {
return fmt.Errorf("assume-role-with-web-identity: %w", aerr)
}
assumeOut = out
return nil
}, 60*time.Second, 5*time.Second)
require.NotNil(t, assumeOut.Credentials, "AssumeRoleWithWebIdentity returned no credentials")
assert.Equal(t, "system:serviceaccount:default:default",
aws.StringValue(assumeOut.SubjectFromWebIdentityToken), "subject must be the default SA")
require.True(t, strings.HasPrefix(aws.StringValue(assumeOut.Credentials.AccessKeyId), "ASIA"),
"web-identity credentials must be temporary (ASIA…)")
// 5) The returned temporary credentials must be usable: GetCallerIdentity
// must resolve to the assumed-role principal.
sessClient := harness.NewAWSClientWithSessionCreds(t, env,
aws.StringValue(assumeOut.Credentials.AccessKeyId),
aws.StringValue(assumeOut.Credentials.SecretAccessKey),
aws.StringValue(assumeOut.Credentials.SessionToken))
ident, err := sessClient.STS.GetCallerIdentity(&sts.GetCallerIdentityInput{})
require.NoError(t, err, "get-caller-identity with web-identity creds")
assert.Equal(t, fx.AccountID, aws.StringValue(ident.Account), "caller account must match")
assert.Contains(t, aws.StringValue(ident.Arn), "assumed-role/"+roleName,
"caller ARN must reflect the assumed IRSA role")
t.Logf("GetCallerIdentity (web-identity creds): %s", aws.StringValue(ident.Arn))
}
// --- Fixture --------------------------------------------------------------
type clusterFixture struct {
tests/e2e/harness/aws.go
+4
View File
@@ -13,6 +13,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/acm"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
"github.com/aws/aws-sdk-go/service/eks"
@@ -33,6 +34,7 @@ type AWSClient struct {
IAM *iam.IAM
STS *sts.STS
EKS *eks.EKS
ACM *acm.ACM
}
// NewAWSClient builds clients pointed at the spinifex gateway using the
@@ -128,6 +130,7 @@ func newAWSClient(t *testing.T, env *Env, accessKey, secretKey, sessionToken str
IAM: iam.New(sess),
STS: sts.New(sess),
EKS: eks.New(sess),
ACM: acm.New(sess),
}
}
@@ -141,5 +144,6 @@ func (c *AWSClient) IgnoreCertErrors() {
c.EKS.Config.HTTPClient = hc
c.IAM.Config.HTTPClient = hc
c.STS.Config.HTTPClient = hc
c.ACM.Config.HTTPClient = hc
_ = (*x509.CertPool)(nil) // silence unused-import on hardened paths
}
tests/e2e/harness/cert.go
+50
View File
@@ -3,15 +3,21 @@
package harness
import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"fmt"
"math/big"
"net"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
)
// ResolveCACert walks the standard config locations for the Spinifex CA cert
@@ -121,6 +127,50 @@ func OpenSSLVerify(t *testing.T, caFile, host string, port int) error {
return fmt.Errorf("openssl verify %s failed: %v\noutput:\n%s", target, err, out)
}
// GenerateSelfSignedCertPEM mints an ECDSA P-256 self-signed leaf for host
// (an IP or DNS name) and returns the cert PEM, the PKCS#8 key PEM, and the
// parsed leaf. Used by the ACM ImportCertificate → HTTPS listener e2e to feed
// a real keypair through the import → HAProxy termination path.
func GenerateSelfSignedCertPEM(host string) (certPEM, keyPEM []byte, leaf *x509.Certificate, err error) {
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return nil, nil, nil, fmt.Errorf("generate key: %w", err)
}
serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
if err != nil {
return nil, nil, nil, fmt.Errorf("serial: %w", err)
}
tmpl := &x509.Certificate{
SerialNumber: serial,
Subject: pkix.Name{CommonName: host},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(24 * time.Hour),
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
BasicConstraintsValid: true,
}
if ip := net.ParseIP(host); ip != nil {
tmpl.IPAddresses = []net.IP{ip}
} else {
tmpl.DNSNames = []string{host}
}
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
if err != nil {
return nil, nil, nil, fmt.Errorf("create certificate: %w", err)
}
keyDER, err := x509.MarshalPKCS8PrivateKey(key)
if err != nil {
return nil, nil, nil, fmt.Errorf("marshal key: %w", err)
}
certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
keyPEM = pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER})
leaf, err = x509.ParseCertificate(der)
if err != nil {
return nil, nil, nil, fmt.Errorf("parse leaf: %w", err)
}
return certPEM, keyPEM, leaf, nil
}
// FingerprintMatches returns true if the two PEM certs share the SHA-256
// fingerprint that openssl x509 -fingerprint prints.
func FingerprintMatches(a, b *x509.Certificate) bool {
tests/e2e/harness/ec2helpers.go
+54
View File
@@ -8,6 +8,7 @@ import (
"strconv"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
@@ -213,6 +214,59 @@ func EnsureDefaultSGOpen(t *testing.T, c *AWSClient) {
AuthorizeICMPIngress(t, c, sgID)
}
// AttachVolumeWait attaches volID to instanceID at device and blocks until the
// volume reaches "in-use". Used by the data-durability tests where the attach
// must complete before the guest can format the disk.
func AttachVolumeWait(t *testing.T, c *AWSClient, volID, instanceID, device string) {
t.Helper()
_, err := c.EC2.AttachVolume(&ec2.AttachVolumeInput{
VolumeId: aws.String(volID),
InstanceId: aws.String(instanceID),
Device: aws.String(device),
})
if err != nil {
t.Fatalf("attach-volume %s -> %s as %s: %v", volID, instanceID, device, err)
}
WaitForVolumeState(t, c, volID, ec2.VolumeStateInUse, WithPoll(500*time.Millisecond))
}
// DetachVolumeWait detaches volID and blocks until it reaches "available".
func DetachVolumeWait(t *testing.T, c *AWSClient, volID string) {
t.Helper()
_, err := c.EC2.DetachVolume(&ec2.DetachVolumeInput{VolumeId: aws.String(volID)})
if err != nil {
t.Fatalf("detach-volume %s: %v", volID, err)
}
WaitForVolumeState(t, c, volID, "available", WithPoll(500*time.Millisecond))
}
// RegisterVolumeTeardown best-effort force-detaches then deletes volID at test
// end. Non-fatal — it polls for "available" within a bounded window and never
// calls t.Fatal so a cleanup hiccup doesn't mask the test's real result.
func RegisterVolumeTeardown(t *testing.T, c *AWSClient, volID string) {
t.Helper()
t.Cleanup(func() {
_, _ = c.EC2.DetachVolume(&ec2.DetachVolumeInput{
VolumeId: aws.String(volID),
Force: aws.Bool(true),
})
deadline := time.Now().Add(90 * time.Second)
for time.Now().Before(deadline) {
out, err := c.EC2.DescribeVolumes(&ec2.DescribeVolumesInput{
VolumeIds: []*string{aws.String(volID)},
})
if err != nil || len(out.Volumes) == 0 {
return
}
if aws.StringValue(out.Volumes[0].State) == "available" {
break
}
time.Sleep(2 * time.Second)
}
_, _ = c.EC2.DeleteVolume(&ec2.DeleteVolumeInput{VolumeId: aws.String(volID)})
})
}
// asErr is a thin errors.As that lets the helper stay testify-free.
func asErr(err error, target *awserr.Error) bool {
if a, ok := err.(awserr.Error); ok {
tests/e2e/harness/eks_mgmt.go
+19 -1
View File
@@ -11,6 +11,7 @@ import (
"github.com/mulgadc/spinifex/spinifex/config"
handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
"github.com/mulgadc/spinifex/spinifex/utils"
"github.com/spf13/viper"
)
// ControlPlaneMgmtIP resolves the EKS control-plane VM's br-mgmt address —
@@ -56,7 +57,7 @@ func natsConn(t *testing.T, env *Env) (host, token, ca string) {
}
cfgPath := filepath.Join(env.ConfigDir, "spinifex.toml")
cc, err := config.LoadConfig(cfgPath)
cc, err := loadClusterConfig(cfgPath)
if err != nil {
t.Fatalf("load %s for NATS settings: %v", cfgPath, err)
}
@@ -67,6 +68,23 @@ func natsConn(t *testing.T, env *Env) (host, token, ca string) {
return dialableNATSHost(node.NATS.Host), node.NATS.ACL.Token, node.NATS.CACert
}
// loadClusterConfig reads spinifex.toml via a private viper instance with no
// AutomaticEnv, so the harness's own SPINIFEX_* vars (the multinode workflow
// exports SPINIFEX_NODES) cannot shadow the [nodes.*] table and blank the parse.
func loadClusterConfig(path string) (*config.ClusterConfig, error) {
v := viper.New()
v.SetConfigFile(path)
v.SetConfigType("toml")
if err := v.ReadInConfig(); err != nil {
return nil, err
}
var cc config.ClusterConfig
if err := v.Unmarshal(&cc); err != nil {
return nil, err
}
return &cc, nil
}
// dialableNATSHost rewrites a wildcard bind address to loopback. NATS.Host is a
// bind address (commonly 0.0.0.0:4222); the TLS serving cert SANs the loopback
// and real node IPs but not 0.0.0.0, so dial 127.0.0.1 instead.
tests/e2e/harness/guest_data.go
+159
View File
@@ -0,0 +1,159 @@
//go:build e2e
package harness
import (
"context"
"fmt"
"os/exec"
"regexp"
"strconv"
"strings"
"testing"
"time"
)
// Guest-side data-durability helpers shared by the volume-durability,
// snapshot-restore, and CreateImage e2e tests. They drive a real guest over SSH
// to format, write, sync, and checksum bytes so the assembled
// QEMU↔viperblock↔predastore I/O path is exercised end-to-end — something the
// per-layer unit tests cannot cover.
// guestSentinelFile is the file written into a freshly formatted data volume.
const guestSentinelFile = "e2e-sentinel.bin"
// guestExecTimeout bounds a single guest command (mkfs/dd are sub-second on the
// small payloads used here; the ceiling only guards against a hung SSH).
const guestExecTimeout = 2 * time.Minute
// sha256RE matches a bare sha256 digest so the checksum can be lifted out of
// command output that may also carry the file path (`<sha>␠␠<path>`).
var sha256RE = regexp.MustCompile(`\b[0-9a-f]{64}\b`)
// GuestExec runs cmd over SSH against tgt and returns combined stdout+stderr
// plus the run error. It never calls t.Fatal, so callers can branch on an
// expected non-zero exit.
func GuestExec(tgt SSHTarget, cmd string) (string, error) {
ctx, cancel := context.WithTimeout(context.Background(), guestExecTimeout)
defer cancel()
args := []string{
"-o", "StrictHostKeyChecking=no",
"-o", "UserKnownHostsFile=/dev/null",
"-o", "LogLevel=ERROR",
"-o", "ConnectTimeout=5",
"-o", "BatchMode=yes",
"-p", strconv.Itoa(tgt.Port),
"-i", tgt.KeyPath,
tgt.User + "@" + tgt.Host,
cmd,
}
out, err := exec.CommandContext(ctx, "ssh", args...).CombinedOutput()
return string(out), err
}
// guestDiskSet returns the set of whole-disk device names (lsblk TYPE=disk)
// currently visible in the guest, keyed by bare name (e.g. "vda", "vdc").
func guestDiskSet(tgt SSHTarget) (map[string]struct{}, error) {
out, err := GuestExec(tgt, "lsblk -dn -o NAME,TYPE")
if err != nil {
return nil, fmt.Errorf("lsblk: %w\n%s", err, out)
}
set := map[string]struct{}{}
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
f := strings.Fields(line)
if len(f) >= 2 && f[1] == "disk" {
set[f[0]] = struct{}{}
}
}
return set, nil
}
// GuestDiskSet is the t.Fatal-on-error wrapper around guestDiskSet, used to
// snapshot the guest's disk inventory before an AttachVolume.
func GuestDiskSet(t *testing.T, tgt SSHTarget) map[string]struct{} {
t.Helper()
set, err := guestDiskSet(tgt)
if err != nil {
t.Fatalf("GuestDiskSet: %v", err)
}
return set
}
// WaitForNewGuestDisk polls the guest until a whole-disk device absent from
// `before` appears, returning its bare name (e.g. "vdc"). Proves a hotplugged
// volume actually reached the guest kernel, not just the EC2 control plane.
func WaitForNewGuestDisk(t *testing.T, tgt SSHTarget, before map[string]struct{}, timeout time.Duration) string {
t.Helper()
var found string
EventuallyErr(t, func() error {
now, err := guestDiskSet(tgt)
if err != nil {
return err
}
for name := range now {
if _, ok := before[name]; !ok {
found = name
return nil
}
}
names := make([]string, 0, len(now))
for name := range now {
names = append(names, name)
}
return fmt.Errorf("no new disk yet (visible: %s)", strings.Join(names, ","))
}, timeout, 2*time.Second)
return found
}
// GuestFormatWriteSentinel formats /dev/<dev> as ext4 with the given label,
// mounts it, writes sizeMiB of random data to the sentinel file, fsyncs, and
// returns the file's sha256. Unmounts before returning so the volume can be
// detached cleanly.
func GuestFormatWriteSentinel(t *testing.T, tgt SSHTarget, dev, label string, sizeMiB int) string {
t.Helper()
mnt := "/mnt/" + label
script := strings.Join([]string{
fmt.Sprintf("sudo mkfs.ext4 -F -L %s /dev/%s >/dev/null 2>&1", label, dev),
fmt.Sprintf("sudo mkdir -p %s", mnt),
fmt.Sprintf("sudo mount /dev/%s %s", dev, mnt),
fmt.Sprintf("sudo dd if=/dev/urandom of=%s/%s bs=1M count=%d conv=fsync status=none", mnt, guestSentinelFile, sizeMiB),
"sync",
fmt.Sprintf("sudo sha256sum %s/%s", mnt, guestSentinelFile),
fmt.Sprintf("sudo umount %s", mnt),
}, " && ")
out, err := GuestExec(tgt, script)
if err != nil {
t.Fatalf("GuestFormatWriteSentinel(%s,%s): %v\n%s", dev, label, err, out)
}
return mustSha(t, out)
}
// GuestReadSentinelSha mounts source (a device path such as "/dev/vdc" or a
// "/dev/disk/by-label/<label>" path) at a temp mountpoint, returns the
// sentinel file's sha256, and unmounts.
func GuestReadSentinelSha(t *testing.T, tgt SSHTarget, source, label string) string {
t.Helper()
mnt := "/mnt/" + label
script := strings.Join([]string{
fmt.Sprintf("sudo mkdir -p %s", mnt),
fmt.Sprintf("sudo mount %s %s", source, mnt),
fmt.Sprintf("sudo sha256sum %s/%s", mnt, guestSentinelFile),
fmt.Sprintf("sudo umount %s", mnt),
}, " && ")
out, err := GuestExec(tgt, script)
if err != nil {
t.Fatalf("GuestReadSentinelSha(%s): %v\n%s", source, err, out)
}
return mustSha(t, out)
}
// mustSha extracts the single sha256 digest from command output, failing the
// test if none is present.
func mustSha(t *testing.T, out string) string {
t.Helper()
sha := sha256RE.FindString(out)
if sha == "" {
t.Fatalf("no sha256 digest in guest output:\n%s", out)
}
return sha
}
tests/e2e/harness/iam.go
+70
View File
@@ -0,0 +1,70 @@
//go:build e2e
package harness
import (
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/iam"
"github.com/aws/aws-sdk-go/service/sts"
"github.com/stretchr/testify/require"
)
// IAMRoleARN constructs arn:aws:iam::<account>:role/<name>.
func IAMRoleARN(account, name string) string {
return "arn:aws:iam::" + account + ":role/" + name
}
// IAMPolicyARN builds the canonical policy ARN for a given account + policy
// "key" (a bare name, or "path/name" without the leading slash).
func IAMPolicyARN(account, key string) string {
return "arn:aws:iam::" + account + ":policy/" + key
}
// IAMAccountID returns the active account ID via STS GetCallerIdentity. Unlike
// deriving it from a created IAM user's ARN, this is side-effect-free, so suites
// sharing one account can call it concurrently without minting colliding users.
func IAMAccountID(t *testing.T, c *AWSClient) string {
t.Helper()
out, err := c.STS.GetCallerIdentity(&sts.GetCallerIdentityInput{})
require.NoError(t, err, "sts:GetCallerIdentity")
require.NotEmpty(t, aws.StringValue(out.Account), "empty account in GetCallerIdentity response")
return aws.StringValue(out.Account)
}
// IAMDeleteRoleAndProfilesBestEffort tears down every fragment of a role +
// instance-profile graph a suite might have left behind. Each step swallows
// errors so a missing fragment never cascades; usable both as a pre-test sweep
// and as a fixture-teardown cleanup. Order matters: unbind role-from-profile
// before deleting the profile, detach policies before deleting the role.
func IAMDeleteRoleAndProfilesBestEffort(c *AWSClient, roleName string, profileNames []string, policyARNs ...string) {
for _, p := range profileNames {
_, _ = c.IAM.RemoveRoleFromInstanceProfile(&iam.RemoveRoleFromInstanceProfileInput{
InstanceProfileName: aws.String(p),
RoleName: aws.String(roleName),
})
_, _ = c.IAM.DeleteInstanceProfile(&iam.DeleteInstanceProfileInput{
InstanceProfileName: aws.String(p),
})
}
for _, arn := range policyARNs {
_, _ = c.IAM.DetachRolePolicy(&iam.DetachRolePolicyInput{
RoleName: aws.String(roleName),
PolicyArn: aws.String(arn),
})
}
// Defensive: pull any other attached policies so the final DeleteRole isn't
// blocked by a stray attach from a partial run.
if attached, err := c.IAM.ListAttachedRolePolicies(&iam.ListAttachedRolePoliciesInput{
RoleName: aws.String(roleName),
}); err == nil {
for _, p := range attached.AttachedPolicies {
_, _ = c.IAM.DetachRolePolicy(&iam.DetachRolePolicyInput{
RoleName: aws.String(roleName),
PolicyArn: p.PolicyArn,
})
}
}
_, _ = c.IAM.DeleteRole(&iam.DeleteRoleInput{RoleName: aws.String(roleName)})
}
tests/e2e/harness/jetstream.go
+39
View File
@@ -0,0 +1,39 @@
//go:build e2e
package harness
import (
"testing"
"github.com/mulgadc/spinifex/spinifex/utils"
)
// KVReplicaFactors connects to the cluster NATS and returns the configured
// replica factor for each named KV bucket, read from JetStream stream metadata
// (the "KV_" stream prefix is applied internally). The replica count lives in
// cluster-global Raft metadata, so any node's NATS reports the same value.
// Fatals if NATS is unreachable or a bucket's stream cannot be introspected.
func KVReplicaFactors(t *testing.T, env *Env, buckets ...string) map[string]int {
t.Helper()
host, token, ca := natsConn(t, env)
nc, err := utils.ConnectNATS(host, token, ca)
if err != nil {
t.Fatalf("connect NATS %s: %v", host, err)
}
defer nc.Close()
js, err := nc.JetStream()
if err != nil {
t.Fatalf("jetstream context: %v", err)
}
out := make(map[string]int, len(buckets))
for _, b := range buckets {
info, err := js.StreamInfo("KV_" + b)
if err != nil {
t.Fatalf("stream info for KV bucket %q: %v", b, err)
}
out[b] = info.Config.Replicas
}
return out
}
tests/e2e/single/assumed_role_enforcement_test.go → tests/e2e/iam/assumed_role_enforcement_test.go
+5 -5
View File
@@ -1,6 +1,6 @@
//go:build e2e
package single
package iam
import (
"testing"
@@ -28,13 +28,13 @@ const (
// live session is allowed once a policy is attached (policies resolved per request).
func runAssumedRoleControlPlaneEnforcement(t *testing.T, fix *Fixture) {
harness.Phase(t, "Single — Assumed-role control-plane enforcement")
adminAccount := iamEnsureAdminAccountID(t, fix)
roleARN := iamRoleARN(adminAccount, areRoleName)
policyARN := iamPolicyARN(adminAccount, arePolicyName)
adminAccount := harness.IAMAccountID(t, fix.AWS)
roleARN := harness.IAMRoleARN(adminAccount, areRoleName)
policyARN := harness.IAMPolicyARN(adminAccount, arePolicyName)
// Pre-sweep in case a previous run left fragments.
sweep := func() {
iamDeleteRoleAndProfilesBestEffort(fix, areRoleName, nil, policyARN)
harness.IAMDeleteRoleAndProfilesBestEffort(fix.AWS, areRoleName, nil, policyARN)
_, _ = fix.AWS.IAM.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: aws.String(policyARN)})
}
sweep()
tests/e2e/iam/datapath_helpers_test.go
+230
View File
@@ -0,0 +1,230 @@
//go:build e2e
package iam
import (
"bufio"
"bytes"
"fmt"
"net"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/mulgadc/spinifex/tests/e2e/harness"
)
// SSH / datapath helpers ported from the single suite so the IMDS and
// instance-profile tests run self-contained inside iam. The single suite keeps
// its own copy — other single tests still use these. They are separate test
// binaries (separate processes), so the sticky-skip state below is per-suite.
// sshDatapathBroken is set the first time an SSH probe in this suite times out.
// Downstream SSH-dependent assertions call requireSSHHealthy(t) and skip rather
// than re-running the same multi-minute timeout against the same broken datapath.
var sshDatapathBroken atomic.Bool
// sshReadyBudget bounds the SSH-handshake probes. Widen via
// SPINIFEX_SSH_READY_TIMEOUT (Go duration); default 3m bounds shared runners.
var sshReadyBudget = resolveSSHReadyBudget(3 * time.Minute)
func resolveSSHReadyBudget(def time.Duration) time.Duration {
if v := os.Getenv("SPINIFEX_SSH_READY_TIMEOUT"); v != "" {
if d, err := time.ParseDuration(v); err == nil && d > 0 {
return d
}
}
return def
}
// requireSSHHealthy skips the calling test if a prior SSH probe in this suite
// has already failed, cutting the cascade cost of a broken datapath.
func requireSSHHealthy(t *testing.T) {
t.Helper()
if sshDatapathBroken.Load() {
t.Skipf("skipping: earlier SSH probe failed; " +
"not retrying to keep suite time bounded")
}
}
// waitForSSHReady probes a full SSH handshake (BatchMode + ConnectTimeout)
// against host:port, retrying until the daemon completes banner exchange.
// TCP-port reachability alone is insufficient — sshd accepts the connect
// while pam/cloud-init are still finishing, and the first real runSSH then
// hits "Connection timed out during banner exchange".
func waitForSSHReady(t *testing.T, host string, port int, keyPath string) {
t.Helper()
requireSSHHealthy(t)
addr := net.JoinHostPort(host, strconv.Itoa(port))
harness.Step(t, "waiting for SSH handshake %s", addr)
if !trySSHReady(host, port, keyPath, sshReadyBudget) {
sshDatapathBroken.Store(true)
t.Fatalf("Eventually: condition not met within %s: "+
"[SSH handshake %s never completed] "+
"(sticky-skip enabled for downstream)", sshReadyBudget, addr)
}
}
// waitForSSHHandshake is an alias kept for the imds_test.go call site.
func waitForSSHHandshake(t *testing.T, host string, port int, keyPath string) {
waitForSSHReady(t, host, port, keyPath)
}
// trySSHReady mirrors waitForSSHReady but returns reachability as a bool
// instead of t.Fatal-ing on timeout.
func trySSHReady(host string, port int, keyPath string, timeout time.Duration) bool {
if sshDatapathBroken.Load() {
return false
}
deadline := time.Now().Add(timeout)
for {
cmd := exec.Command("ssh",
"-o", "StrictHostKeyChecking=no",
"-o", "UserKnownHostsFile=/dev/null",
"-o", "LogLevel=ERROR",
"-o", "ConnectTimeout=3",
"-o", "BatchMode=yes",
"-p", strconv.Itoa(port),
"-i", keyPath,
"ec2-user@"+host,
"true")
if cmd.Run() == nil {
return true
}
if time.Now().After(deadline) {
return false
}
time.Sleep(2 * time.Second)
}
}
// runSSH is a thin wrapper around `ssh` matching the option set used by
// harness.LsblkRootGiB. Returns stdout. t.Fatal on non-zero exit so callers
// can chain assertions on the output without nil-checking err.
func runSSH(t *testing.T, tgt harness.SSHTarget, command string) string {
t.Helper()
args := []string{
"-o", "StrictHostKeyChecking=no",
"-o", "UserKnownHostsFile=/dev/null",
"-o", "LogLevel=ERROR",
"-o", "ConnectTimeout=5",
"-o", "BatchMode=yes",
"-p", strconv.Itoa(tgt.Port),
"-i", tgt.KeyPath,
tgt.User + "@" + tgt.Host,
command,
}
var stdout, stderr bytes.Buffer
cmd := exec.Command("ssh", args...)
cmd.Stdout = &stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
t.Fatalf("ssh %s@%s:%d %q failed: %v\nstderr: %s",
tgt.User, tgt.Host, tgt.Port, command, err, stderr.String())
}
return stdout.String()
}
// runSSHCombined runs `command` over SSH against tgt and returns combined
// stdout+stderr regardless of exit status. Unlike runSSH it does not t.Fatal
// on non-zero exit — needed for probes (e.g. curl status codes) where a
// non-zero exit is an expected outcome.
func runSSHCombined(tgt harness.SSHTarget, command string) (string, error) {
args := []string{
"-o", "StrictHostKeyChecking=no",
"-o", "UserKnownHostsFile=/dev/null",
"-o", "LogLevel=ERROR",
"-o", "ConnectTimeout=5",
"-o", "BatchMode=yes",
"-p", strconv.Itoa(tgt.Port),
"-i", tgt.KeyPath,
tgt.User + "@" + tgt.Host,
command,
}
var combined bytes.Buffer
cmd := exec.Command("ssh", args...)
cmd.Stdout = &combined
cmd.Stderr = &combined
err := cmd.Run()
return combined.String(), err
}
// primaryENI returns the NetworkInterfaceId of an instance's first ENI.
// t.Fatal if the instance has no ENI — every running EC2 instance must.
func primaryENI(t *testing.T, inst *ec2.Instance) string {
t.Helper()
if len(inst.NetworkInterfaces) == 0 {
t.Fatalf("instance %s has no NetworkInterfaces", aws.StringValue(inst.InstanceId))
}
eni := aws.StringValue(inst.NetworkInterfaces[0].NetworkInterfaceId)
if eni == "" {
t.Fatalf("instance %s primary ENI has empty NetworkInterfaceId", aws.StringValue(inst.InstanceId))
}
return eni
}
// waitForInstanceStateSoft is the cleanup-time analogue of
// harness.WaitForInstanceState — no t.Fatal, just polls and returns.
func waitForInstanceStateSoft(c *harness.AWSClient, id, target string, timeout time.Duration) error {
deadline := time.Now().Add(timeout)
for {
out, err := c.EC2.DescribeInstances(&ec2.DescribeInstancesInput{
InstanceIds: []*string{aws.String(id)},
})
if err == nil && len(out.Reservations) > 0 && len(out.Reservations[0].Instances) > 0 {
if aws.StringValue(out.Reservations[0].Instances[0].State.Name) == target {
return nil
}
}
if time.Now().After(deadline) {
return fmt.Errorf("instance %s did not reach %s within %s", id, target, timeout)
}
time.Sleep(2 * time.Second)
}
}
// detectPoolMode reads external_mode from spinifex.toml. Defaults to false
// (dev_networking) which is the single-node CI fixture. Any non-empty
// external_mode value ("pool" / "nat") means external IPAM is in play, which
// gates the IMDS probe-VPC public-IP path.
func detectPoolMode(env *harness.Env) bool {
cfg := os.ExpandEnv("$HOME/spinifex/config/spinifex.toml")
if env.ConfigDir != "" {
cfg = filepath.Join(env.ConfigDir, "spinifex.toml")
}
f, err := os.Open(cfg)
if err != nil {
return false
}
defer f.Close()
scanner := bufio.NewScanner(f)
inNetwork := false
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if strings.HasPrefix(line, "[") {
inNetwork = line == "[network]"
continue
}
if !inNetwork {
continue
}
if !strings.HasPrefix(line, "external_mode") {
continue
}
// external_mode = "pool" — quoted value, anything non-empty == pool mode.
if i := bytes.IndexByte([]byte(line), '='); i >= 0 {
val := strings.TrimSpace(line[i+1:])
val = strings.Trim(val, "\"'")
return val != ""
}
}
return false
}
tests/e2e/iam/helpers_test.go
+82
View File
@@ -0,0 +1,82 @@
//go:build e2e
package iam
import (
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/mulgadc/spinifex/tests/e2e/harness"
"github.com/stretchr/testify/require"
)
// Instance-bootstrap helpers wrapping harness.Discover*/Ensure* so the IMDS and
// instance-profile tests can self-bootstrap their VM prerequisites. Results are
// memoized by harness.Fixture. The single suite keeps its own copy of these —
// other single tests still use them.
// needInstanceTypeArch returns the nano instance type and its architecture.
func needInstanceTypeArch(t *testing.T, fix *Fixture) (instanceType, arch string) {
t.Helper()
return harness.DiscoverNanoInstanceType(t, fix.Harness)
}
// needAMI returns the discovered architecture-appropriate Ubuntu AMI ID.
func needAMI(t *testing.T, fix *Fixture) string {
t.Helper()
_, arch := needInstanceTypeArch(t, fix)
return harness.DiscoverUbuntuAMI(t, fix.Harness, arch)
}
// needKeyPair returns the memoized primary EC2 key pair name + PEM path.
func needKeyPair(t *testing.T, fix *Fixture) (name, pemPath string) {
t.Helper()
return harness.EnsureKeyPair(t, fix.Harness, fix.TmpDir)
}
// needInstance bootstraps the suite's primary running instance and returns
// the *ec2.Instance plus its root volume ID. Idempotent via harness.EnsureInstance.
func needInstance(t *testing.T, fix *Fixture) (inst *ec2.Instance, rootVolumeID string) {
t.Helper()
instType, _ := needInstanceTypeArch(t, fix)
keyName, _ := needKeyPair(t, fix)
ami := needAMI(t, fix)
vpc := harness.EnsureDefaultVPC(t, fix.Harness)
require.NotEmpty(t, vpc.SGID, "default SG ID required")
harness.AuthorizeSSHIngress(t, fix.AWS, vpc.SGID)
instanceID := harness.EnsureInstance(t, fix.Harness, harness.InstanceSpec{
AMIID: ami,
InstanceType: instType,
KeyName: keyName,
SubnetID: vpc.SubnetID,
SGID: vpc.SGID,
})
require.NotEmpty(t, instanceID, "EnsureInstance returned empty InstanceId")
descOut, err := fix.AWS.EC2.DescribeInstances(&ec2.DescribeInstancesInput{
InstanceIds: []*string{aws.String(instanceID)},
})
require.NoError(t, err, "describe-instances %s", instanceID)
require.NotEmpty(t, descOut.Reservations, "no reservations for %s", instanceID)
require.NotEmpty(t, descOut.Reservations[0].Instances, "no instances for %s", instanceID)
inst = descOut.Reservations[0].Instances[0]
// Match BDM by RootDeviceName; fall back to first BDM if empty.
rootDev := aws.StringValue(inst.RootDeviceName)
for _, bdm := range inst.BlockDeviceMappings {
if rootDev != "" && aws.StringValue(bdm.DeviceName) != rootDev {
continue
}
if bdm.Ebs != nil {
rootVolumeID = aws.StringValue(bdm.Ebs.VolumeId)
break
}
}
if rootVolumeID == "" && len(inst.BlockDeviceMappings) > 0 && inst.BlockDeviceMappings[0].Ebs != nil {
rootVolumeID = aws.StringValue(inst.BlockDeviceMappings[0].Ebs.VolumeId)
}
require.NotEmpty(t, rootVolumeID, "could not resolve root volume from BlockDeviceMappings")
return inst, rootVolumeID
}
tests/e2e/single/iam_helpers_test.go → tests/e2e/iam/iam_helpers_test.go
+1 -25
View File
@@ -1,6 +1,6 @@
//go:build e2e
package single
package iam
import (
"sync"
@@ -36,10 +36,6 @@ var (
iamCharlieKeyID string
iamCharlieKeySecret string
iamCharlieKeyErr error
iamAdminAccountOnce sync.Once
iamAdminAccountID string
iamAdminAccountErr error
)
// iamEnsureAlice ensures the canonical "alice" IAM user exists. The user is
@@ -129,26 +125,6 @@ func iamEnsureCharlieKey(t *testing.T, fix *Fixture) (keyID, secret string) {
return iamCharlieKeyID, iamCharlieKeySecret
}
// iamEnsureAdminAccountID returns the active account ID, derived from the
// ARN of a probe IAM resource. Uses GetUser on the canonical alice user so
// no policy churn is needed; alice is created on first call.
func iamEnsureAdminAccountID(t *testing.T, fix *Fixture) string {
t.Helper()
iamEnsureAlice(t, fix)
iamAdminAccountOnce.Do(func() {
out, err := fix.AWS.IAM.GetUser(&iam.GetUserInput{UserName: aws.String(iamUserAlice)})
if err != nil {
iamAdminAccountErr = err
return
}
iamAdminAccountID = iamAccountFromARN(t, aws.StringValue(out.User.Arn))
})
if iamAdminAccountErr != nil {
t.Fatalf("iamEnsureAdminAccountID: %v", iamAdminAccountErr)
}
return iamAdminAccountID
}
// iamCreateUserIdempotent creates a user, treating EntityAlreadyExists as success.
func iamCreateUserIdempotent(fix *Fixture, name, path string) error {
input := &iam.CreateUserInput{UserName: aws.String(name)}
tests/e2e/single/iam_roles_test.go → tests/e2e/iam/iam_roles_test.go
+7 -48
View File
@@ -1,6 +1,6 @@
//go:build e2e
package single
package iam
import (
"testing"
@@ -32,17 +32,17 @@ const (
// which recreates the same names.
func runIAMRolesAndProfiles(t *testing.T, fix *Fixture) {
harness.Phase(t, "Single — IAM Roles & Instance Profiles")
adminAccount := iamEnsureAdminAccountID(t, fix)
roleARN := iamRoleARN(adminAccount, iamRoleAppName)
adminPolicyARN := iamPolicyARN(adminAccount, iamPolicyAdministrator)
adminAccount := harness.IAMAccountID(t, fix.AWS)
roleARN := harness.IAMRoleARN(adminAccount, iamRoleAppName)
adminPolicyARN := harness.IAMPolicyARN(adminAccount, iamPolicyAdministrator)
// Defensive sweep — previous failed run may have left fragments. Order
// matters: detach role from profile before delete-instance-profile,
// detach policy from role before delete-role.
iamDeleteRoleAndProfilesBestEffort(fix, iamRoleAppName,
harness.IAMDeleteRoleAndProfilesBestEffort(fix.AWS, iamRoleAppName,
[]string{iamProfileAppName, iamProfileOtherName, iamProfileEngName}, adminPolicyARN)
fix.Harness.RegisterCleanup(func() {
iamDeleteRoleAndProfilesBestEffort(fix, iamRoleAppName,
harness.IAMDeleteRoleAndProfilesBestEffort(fix.AWS, iamRoleAppName,
[]string{iamProfileAppName, iamProfileOtherName, iamProfileEngName}, adminPolicyARN)
})
@@ -159,7 +159,7 @@ func runIAMRolesAndProfiles(t *testing.T, fix *Fixture) {
harness.ExpectError(t, "NoSuchEntity", func() error {
_, e := fix.AWS.IAM.AttachRolePolicy(&iam.AttachRolePolicyInput{
RoleName: aws.String(iamRoleAppName),
PolicyArn: aws.String(iamPolicyARN(adminAccount, "Ghost")),
PolicyArn: aws.String(harness.IAMPolicyARN(adminAccount, "Ghost")),
})
return e
})
@@ -329,44 +329,3 @@ func runIAMRolesAndProfiles(t *testing.T, fix *Fixture) {
return e
})
}
// iamRoleARN constructs arn:aws:iam::<acct>:role/<name>.
func iamRoleARN(account, name string) string {
return "arn:aws:iam::" + account + ":role/" + name
}
// iamDeleteRoleAndProfilesBestEffort tears down every fragment of a role +
// profile graph the suite might have left behind. Each step swallows errors
// so a missing fragment doesn't cascade. Used both as a pre-test sweep and
// as a fixture-teardown cleanup.
func iamDeleteRoleAndProfilesBestEffort(fix *Fixture, roleName string, profileNames []string, policyARNs ...string) {
for _, p := range profileNames {
// Drop role-from-profile binding (idempotent).
_, _ = fix.AWS.IAM.RemoveRoleFromInstanceProfile(&iam.RemoveRoleFromInstanceProfileInput{
InstanceProfileName: aws.String(p),
RoleName: aws.String(roleName),
})
_, _ = fix.AWS.IAM.DeleteInstanceProfile(&iam.DeleteInstanceProfileInput{
InstanceProfileName: aws.String(p),
})
}
for _, arn := range policyARNs {
_, _ = fix.AWS.IAM.DetachRolePolicy(&iam.DetachRolePolicyInput{
RoleName: aws.String(roleName),
PolicyArn: aws.String(arn),
})
}
// Defensive: pull any other attached policies we don't know about so the
// final DeleteRole isn't blocked by a stray attach from a partial run.
if attached, err := fix.AWS.IAM.ListAttachedRolePolicies(&iam.ListAttachedRolePoliciesInput{
RoleName: aws.String(roleName),
}); err == nil {
for _, p := range attached.AttachedPolicies {
_, _ = fix.AWS.IAM.DetachRolePolicy(&iam.DetachRolePolicyInput{
RoleName: aws.String(roleName),
PolicyArn: p.PolicyArn,
})
}
}
_, _ = fix.AWS.IAM.DeleteRole(&iam.DeleteRoleInput{RoleName: aws.String(roleName)})
}
tests/e2e/single/iam_test.go → tests/e2e/iam/iam_test.go
+39 -130
View File
@@ -1,6 +1,6 @@
//go:build e2e
package single
package iam
import (
"strings"
@@ -81,24 +81,13 @@ const (
"Resource": "*"
}]
}`
iamDocEC2ReadOnlyV2 = `{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": ["ec2:DescribeInstances", "ec2:DescribeVolumes", "ec2:DescribeVpcs", "ec2:DescribeImages"],
"Resource": "*"
}]
}`
)
// runIAMUserCRUD ports run-e2e.sh IAM Phase 1 (~1405–1459):
// CreateUser, GetUser, ListUsers, UpdateUser (rename+path), DeleteUser
// plus the EntityAlreadyExists / NoSuchEntity negative paths. Each
// created user is scheduled for cleanup so a mid-phase failure can't
// poison later runs. UpdateUser exercises an out-of-band rename +
// rename-back so Phase 2+ continue to find "alice" by the well-known
// name.
// CreateUser, GetUser, ListUsers, DeleteUser plus the
// EntityAlreadyExists / NoSuchEntity negative paths. Each created user
// is scheduled for cleanup so a mid-phase failure can't poison later
// runs.
func runIAMUserCRUD(t *testing.T, fix *Fixture) {
harness.Phase(t, "Single — IAM User CRUD")
@@ -169,34 +158,6 @@ func runIAMUserCRUD(t *testing.T, fix *Fixture) {
require.Len(t, eng.Users, 1,
"expected 1 user under %s, got %d (names=%v)", iamUserBobPath, len(eng.Users), iamUserNames(eng.Users))
// UpdateUser rename+path round-trip: daemon does not implement the
// UpdateUser action yet (returns InvalidAction). Skip-gate the block
// so the rest of Phase 1 still runs; flip to a real assertion once
// the daemon handler lands.
t.Run("UpdateUser", func(t *testing.T) {
t.Skip("daemon UpdateUser handler not implemented — InvalidAction")
const renamed = "alice-renamed"
const renamedPath = "/staff/"
harness.Step(t, "update-user %q -> %q path=%q", iamUserAlice, renamed, renamedPath)
_, err := fix.AWS.IAM.UpdateUser(&iam.UpdateUserInput{
UserName: aws.String(iamUserAlice),
NewUserName: aws.String(renamed),
NewPath: aws.String(renamedPath),
})
require.NoError(t, err, "update-user rename+path")
check, err := fix.AWS.IAM.GetUser(&iam.GetUserInput{UserName: aws.String(renamed)})
require.NoError(t, err, "get-user after rename")
assert.Equal(t, renamedPath, aws.StringValue(check.User.Path), "path not updated")
harness.Step(t, "update-user %q -> %q (restore)", renamed, iamUserAlice)
_, err = fix.AWS.IAM.UpdateUser(&iam.UpdateUserInput{
UserName: aws.String(renamed),
NewUserName: aws.String(iamUserAlice),
NewPath: aws.String("/"),
})
require.NoError(t, err, "update-user restore")
})
// DeleteUser idempotency: create a throwaway user, delete it, then
// delete it again and assert the second delete surfaces NoSuchEntity
// rather than silently succeeding (matches AWS semantics).
@@ -315,25 +276,28 @@ func runIAMAccessKeyLifecycle(t *testing.T, fix *Fixture) {
require.Len(t, after.AccessKeyMetadata, 1, "alice should have 1 key after delete")
}
// runIAMUserAuthentication ports IAM Phase 3 (~1538–1579):
// build a scoped AWS client with alice's key and confirm signing
// works (active key) / fails (deactivated key, bad secret, bogus ID).
// Also creates bob's key so Phase 5 enforcement / Phase 7 cleanup
// can use it.
// runIAMUserAuthentication ports IAM Phase 3 (~1538–1579): build a scoped
// AWS client with alice's key and confirm signing is honoured (active key)
// / rejected (deactivated key, bad secret, bogus ID). Alice has no policy in
// this phase, so the authenticated call is authz-denied — AccessDenied (not
// an authn error) proves the active key signed and was accepted. Also creates
// bob's key so Phase 5 enforcement / Phase 7 cleanup can use it.
func runIAMUserAuthentication(t *testing.T, fix *Fixture) {
harness.Phase(t, "Single — IAM User Authentication")
// Daemon does not yet honour active-key signatures created via the IAM
// API — every scoped DescribeInstances returns 403.
t.Skip("daemon IAM signature/principal lookup not implemented")
aliceKeyID, aliceSecret := iamEnsureAliceKey(t, fix)
harness.Step(t, "scoped client (alice) describe-instances — active key OK")
// Active, correctly-signed key with no policy: the request authenticates
// then default-denies. AccessDenied (vs the InvalidClientTokenId /
// SignatureDoesNotMatch below) is the signal that authn succeeded.
harness.Step(t, "scoped client (alice) describe-instances — active key authenticates (AccessDenied, no policy)")
aliceCli := harness.NewAWSClientWithCreds(t, fix.Env, aliceKeyID, aliceSecret)
_, err := aliceCli.EC2.DescribeInstances(&ec2.DescribeInstancesInput{})
require.NoError(t, err, "alice describe-instances with active key")
harness.ExpectError(t, "AccessDenied", func() error {
_, e := aliceCli.EC2.DescribeInstances(&ec2.DescribeInstancesInput{})
return e
})
harness.Step(t, "deactivate alice key, expect InvalidClientTokenId")
_, err = fix.AWS.IAM.UpdateAccessKey(&iam.UpdateAccessKeyInput{
_, err := fix.AWS.IAM.UpdateAccessKey(&iam.UpdateAccessKeyInput{
UserName: aws.String(iamUserAlice),
AccessKeyId: aws.String(aliceKeyID),
Status: aws.String(iam.StatusTypeInactive),
@@ -375,11 +339,9 @@ func runIAMUserAuthentication(t *testing.T, fix *Fixture) {
}
// runIAMPolicyCRUD ports IAM Phase 4 (~1583–1698):
// CreatePolicy (5 variants), CreatePolicyVersion / ListPolicyVersions /
// SetDefaultPolicyVersion / DeletePolicyVersion (extends bash, per
// task spec), GetPolicy, GetPolicyVersion, ListPolicies plus
// EntityAlreadyExists / MalformedPolicyDocument / NoSuchEntity errors.
// Captures the admin account ID into the Fixture for Phase 5–7.
// CreatePolicy (5 variants), GetPolicy, GetPolicyVersion, ListPolicies
// plus EntityAlreadyExists / MalformedPolicyDocument / NoSuchEntity
// errors. Captures the admin account ID into the Fixture for Phase 5–7.
func runIAMPolicyCRUD(t *testing.T, fix *Fixture) {
harness.Phase(t, "Single — IAM Policy CRUD")
@@ -416,7 +378,7 @@ func runIAMPolicyCRUD(t *testing.T, fix *Fixture) {
if p.path != "" {
key = p.path[1:] + p.name
}
iamDeletePolicyBestEffort(fix, iamPolicyARN(adminAccount, key))
iamDeletePolicyBestEffort(fix, harness.IAMPolicyARN(adminAccount, key))
}
})
harness.Detail(t, "policy", iamPolicyEC2ReadOnly, "arn", ec2roArn, "account", adminAccount)
@@ -490,7 +452,7 @@ func runIAMPolicyCRUD(t *testing.T, fix *Fixture) {
require.Equal(t, iamPolicyEC2ReadOnly, aws.StringValue(got.Policy.PolicyName))
harness.Step(t, "get-policy nonexistent (expect NoSuchEntity)")
ghostArn := iamPolicyARN(adminAccount, "Ghost")
ghostArn := harness.IAMPolicyARN(adminAccount, "Ghost")
harness.ExpectError(t, "NoSuchEntity", func() error {
_, e := fix.AWS.IAM.GetPolicy(&iam.GetPolicyInput{PolicyArn: aws.String(ghostArn)})
return e
@@ -504,47 +466,6 @@ func runIAMPolicyCRUD(t *testing.T, fix *Fixture) {
require.NoError(t, err, "get-policy-version v1")
require.Equal(t, "v1", aws.StringValue(pv.PolicyVersion.VersionId))
// Policy version lifecycle — extends the bash driver (task spec).
// Round-trip: create v2 (as default), list, set v1 default, delete v2.
// Daemon does not implement CreatePolicyVersion / ListPolicyVersions /
// SetDefaultPolicyVersion / DeletePolicyVersion yet (InvalidAction).
// Skip-gate until the handlers land.
t.Run("PolicyVersionLifecycle", func(t *testing.T) {
t.Skip("daemon CreatePolicyVersion handler not implemented — InvalidAction")
harness.Step(t, "create-policy-version %s v2 (default)", iamPolicyEC2ReadOnly)
cv, err := fix.AWS.IAM.CreatePolicyVersion(&iam.CreatePolicyVersionInput{
PolicyArn: aws.String(ec2roArn),
PolicyDocument: aws.String(iamDocEC2ReadOnlyV2),
SetAsDefault: aws.Bool(true),
})
require.NoError(t, err, "create-policy-version v2")
v2 := aws.StringValue(cv.PolicyVersion.VersionId)
require.NotEmpty(t, v2, "empty new VersionId")
harness.Detail(t, "new_version", v2)
harness.Step(t, "list-policy-versions %s", iamPolicyEC2ReadOnly)
versions, err := fix.AWS.IAM.ListPolicyVersions(&iam.ListPolicyVersionsInput{
PolicyArn: aws.String(ec2roArn),
})
require.NoError(t, err, "list-policy-versions")
require.GreaterOrEqual(t, len(versions.Versions), 2,
"expected >=2 versions, got %d", len(versions.Versions))
harness.Step(t, "set-default-policy-version %s v1", iamPolicyEC2ReadOnly)
_, err = fix.AWS.IAM.SetDefaultPolicyVersion(&iam.SetDefaultPolicyVersionInput{
PolicyArn: aws.String(ec2roArn),
VersionId: aws.String("v1"),
})
require.NoError(t, err, "set-default-policy-version v1")
harness.Step(t, "delete-policy-version %s %s", iamPolicyEC2ReadOnly, v2)
_, err = fix.AWS.IAM.DeletePolicyVersion(&iam.DeletePolicyVersionInput{
PolicyArn: aws.String(ec2roArn),
VersionId: aws.String(v2),
})
require.NoError(t, err, "delete-policy-version %s", v2)
})
harness.Step(t, "list-policies (>=5)")
all, err := fix.AWS.IAM.ListPolicies(&iam.ListPoliciesInput{})
require.NoError(t, err, "list-policies")
@@ -557,20 +478,17 @@ func runIAMPolicyCRUD(t *testing.T, fix *Fixture) {
// (default deny / explicit allow / wildcard / explicit deny / root
// bypass / prefix wildcard / FullAdmin), DetachUserPolicy.
func runIAMPolicyAttachmentEnforcement(t *testing.T, fix *Fixture) {
harness.Phase(t, "Single — IAM Policy Attachment IAM Phase 5 — Policy Attachment & Enforcement Enforcement")
// Enforcement depends on scoped-credential signing which the daemon
// doesn't honour yet.
t.Skip("daemon IAM scoped-credential enforcement not implemented")
adminAccount := iamEnsureAdminAccountID(t, fix)
harness.Phase(t, "Single — IAM Policy Attachment & Enforcement")
adminAccount := harness.IAMAccountID(t, fix.AWS)
aliceKeyID, aliceSecret := iamEnsureAliceKey(t, fix)
bobKeyID, bobSecret := iamEnsureBobKey(t, fix)
charlieKeyID, charlieSecret := iamEnsureCharlieKey(t, fix)
ec2roArn := iamPolicyARN(adminAccount, iamPolicyEC2ReadOnly)
iamroArn := iamPolicyARN(adminAccount, iamPolicyIAMReadOnly)
denyArn := iamPolicyARN(adminAccount, iamPolicyDenyTerminate)
descAllArn := iamPolicyARN(adminAccount, iamPolicyEC2DescribeAll)
fullAdminArn := iamPolicyARN(adminAccount, iamPolicyFullAdminPath[1:]+iamPolicyFullAdmin)
ec2roArn := harness.IAMPolicyARN(adminAccount, iamPolicyEC2ReadOnly)
iamroArn := harness.IAMPolicyARN(adminAccount, iamPolicyIAMReadOnly)
denyArn := harness.IAMPolicyARN(adminAccount, iamPolicyDenyTerminate)
descAllArn := harness.IAMPolicyARN(adminAccount, iamPolicyEC2DescribeAll)
fullAdminArn := harness.IAMPolicyARN(adminAccount, iamPolicyFullAdminPath[1:]+iamPolicyFullAdmin)
harness.Detail(t, "charlie_key", charlieKeyID)
aliceCli := harness.NewAWSClientWithCreds(t, fix.Env, aliceKeyID, aliceSecret)
@@ -617,7 +535,7 @@ func runIAMPolicyAttachmentEnforcement(t *testing.T, fix *Fixture) {
harness.ExpectError(t, "NoSuchEntity", func() error {
_, e := fix.AWS.IAM.AttachUserPolicy(&iam.AttachUserPolicyInput{
UserName: aws.String(iamUserAlice),
PolicyArn: aws.String(iamPolicyARN(adminAccount, "Ghost")),
PolicyArn: aws.String(harness.IAMPolicyARN(adminAccount, "Ghost")),
})
return e
})
@@ -730,15 +648,12 @@ func runIAMPolicyAttachmentEnforcement(t *testing.T, fix *Fixture) {
// still-attached policy must surface DeleteConflict; after detach +
// delete the policy is gone (NoSuchEntity).
func runIAMPolicyLifecycle(t *testing.T, fix *Fixture) {
harness.Phase(t, "Single — IAM Policy Lifecycle (Detach IAM Phase 6 — Policy Lifecycle (Detach & Delete) Delete)")
// DetachUserPolicy returns 404 NoSuchEntity even for confirmed attachments —
// the daemon's attachment ledger isn't persisted across the API boundary.
t.Skip("daemon DetachUserPolicy not implemented")
adminAccount := iamEnsureAdminAccountID(t, fix)
harness.Phase(t, "Single — IAM Policy Lifecycle (Detach & Delete)")
adminAccount := harness.IAMAccountID(t, fix.AWS)
aliceKeyID, aliceSecret := iamEnsureAliceKey(t, fix)
descAllArn := iamPolicyARN(adminAccount, iamPolicyEC2DescribeAll)
denyArn := iamPolicyARN(adminAccount, iamPolicyDenyTerminate)
descAllArn := harness.IAMPolicyARN(adminAccount, iamPolicyEC2DescribeAll)
denyArn := harness.IAMPolicyARN(adminAccount, iamPolicyDenyTerminate)
aliceCli := harness.NewAWSClientWithCreds(t, fix.Env, aliceKeyID, aliceSecret)
@@ -785,7 +700,7 @@ func runIAMCleanup(t *testing.T, fix *Fixture) {
// Resolve account ID before deleting users — iamEnsureAdminAccountID
// probes via GetUser(alice), and the user is gone after the loop below.
adminAccount := iamEnsureAdminAccountID(t, fix)
adminAccount := harness.IAMAccountID(t, fix.AWS)
// Users: delete their keys first, then the user. iamDeleteUserBestEffort
// handles already-deleted resources gracefully.
@@ -819,7 +734,7 @@ func runIAMCleanup(t *testing.T, fix *Fixture) {
// stripping the leading '/' matches iamPolicyARN's expectation.
key = p.path[1:] + p.name
}
arn := iamPolicyARN(adminAccount, key)
arn := harness.IAMPolicyARN(adminAccount, key)
harness.Step(t, "cleanup policy %s", arn)
iamDeletePolicyBestEffort(fix, arn)
}
@@ -945,9 +860,3 @@ func iamAccountFromARN(t *testing.T, arn string) string {
require.NotEmpty(t, parts[4], "empty account-id in ARN %q", arn)
return parts[4]
}
// iamPolicyARN builds the canonical policy ARN for a given account +
// policy "key" (name or "path/name" without the leading slash).
func iamPolicyARN(account, key string) string {
return "arn:aws:iam::" + account + ":policy/" + key
}
tests/e2e/single/imds_test.go → tests/e2e/iam/imds_test.go
+12 -6
View File
@@ -1,6 +1,6 @@
//go:build e2e
package single
package iam
import (
"encoding/base64"
@@ -27,6 +27,12 @@ const (
imdsRoleName = "imds-e2e-role"
imdsProfileName = "imds-e2e-profile"
// imdsAdminPolicyName is the bootstrap managed policy attached to the IMDS
// role so the minted instance-role creds round-trip a real API call.
imdsAdminPolicyName = "AdministratorAccess"
// imdsTrustPolicyEC2 lets ec2.amazonaws.com assume the IMDS instance role.
imdsTrustPolicyEC2 = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"ec2.amazonaws.com"},"Action":"sts:AssumeRole"}]}`
// metaURL is the AWS-compatible link-local IMDS endpoint, answered L2 on each
// subnet's own switch from the host via SO_BINDTODEVICE on the
// imds-h-<shortSubnetID> veth (no router, no proxy-ARP, no /32 route).
@@ -87,7 +93,7 @@ func runIMDS(t *testing.T, fix *Fixture) {
harness.SkipIfNoOVN(t)
requireSSHHealthy(t)
adminAccount := iamEnsureAdminAccountID(t, fix)
adminAccount := harness.IAMAccountID(t, fix.AWS)
keyName, keyPath := needKeyPair(t, fix)
imdsEnsureRoleProfile(t, fix, adminAccount)
@@ -525,16 +531,16 @@ func imdsAttachInternet(t *testing.T, c *harness.AWSClient, vpcID, subnetID stri
// sweeps any residue from a prior run first.
func imdsEnsureRoleProfile(t *testing.T, fix *Fixture, adminAccount string) {
t.Helper()
adminPolicyARN := iamPolicyARN(adminAccount, iamPolicyAdministrator)
iamDeleteRoleAndProfilesBestEffort(fix, imdsRoleName, []string{imdsProfileName}, adminPolicyARN)
adminPolicyARN := harness.IAMPolicyARN(adminAccount, imdsAdminPolicyName)
harness.IAMDeleteRoleAndProfilesBestEffort(fix.AWS, imdsRoleName, []string{imdsProfileName}, adminPolicyARN)
fix.Harness.RegisterCleanup(func() {
iamDeleteRoleAndProfilesBestEffort(fix, imdsRoleName, []string{imdsProfileName}, adminPolicyARN)
harness.IAMDeleteRoleAndProfilesBestEffort(fix.AWS, imdsRoleName, []string{imdsProfileName}, adminPolicyARN)
})
harness.Step(t, "create-role %q (trust=ec2.amazonaws.com) + profile %q", imdsRoleName, imdsProfileName)
_, err := fix.AWS.IAM.CreateRole(&iam.CreateRoleInput{
RoleName: aws.String(imdsRoleName),
AssumeRolePolicyDocument: aws.String(iamTrustPolicyEC2Standard),
AssumeRolePolicyDocument: aws.String(imdsTrustPolicyEC2),
Description: aws.String("E2E IMDS instance-role credentials"),
})
require.NoError(t, err, "create-role")
tests/e2e/single/instance_iam_profile_test.go → tests/e2e/iam/instance_iam_profile_test.go
+36 -24
View File
@@ -1,6 +1,6 @@
//go:build e2e
package single
package iam
import (
"strings"
@@ -14,24 +14,36 @@ import (
"github.com/stretchr/testify/require"
)
// IAM identifiers owned by this test. They use a single-iamprofile- namespace
// distinct from the control-plane iam suite's app-role/app-profile so the two
// suites can run concurrently against the same account without colliding on
// account-flat IAM names.
const (
singleIAMProfileRole = "single-iamprofile-role"
singleIAMProfileApp = "single-iamprofile-profile"
singleIAMProfileOther = "single-iamprofile-other-profile"
singleIAMProfileAdminPolicy = "AdministratorAccess"
singleIAMProfileTrustPolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"ec2.amazonaws.com"},"Action":"sts:AssumeRole"}]}`
)
// runIAMInstanceProfileAssociation ports run-e2e.sh IAM Phase 9: EC2 IAM
// instance-profile association lifecycle. Reuses the singleton VM for the
// post-launch Associate/Replace/Disassociate cycle, then launches a single
// short-lived VM whose sole purpose is to exercise (a) RunInstances with
// --iam-instance-profile and (b) auto-disassociate on terminate.
//
// Pre-state: TestIAMRolesAndProfiles already tore everything down — this
// test recreates the role + two profiles, asserts the EC2 surface, then
// cleans up so subsequent tests see no residue.
// Self-bootstrapping: it creates its own role + two profiles under the
// single-iamprofile- namespace, asserts the EC2 surface, then cleans up so
// subsequent tests see no residue.
func runIAMInstanceProfileAssociation(t *testing.T, fix *Fixture) {
harness.Phase(t, "Single — EC2 IAM Instance Profile Association")
adminAccount := iamEnsureAdminAccountID(t, fix)
adminPolicyARN := iamPolicyARN(adminAccount, iamPolicyAdministrator)
adminAccount := harness.IAMAccountID(t, fix.AWS)
adminPolicyARN := harness.IAMPolicyARN(adminAccount, singleIAMProfileAdminPolicy)
// Defensive cleanup of stale state from a prior run.
iamDeleteRoleAndProfilesBestEffort(fix, iamRoleAppName,
[]string{iamProfileAppName, iamProfileOtherName}, adminPolicyARN)
harness.IAMDeleteRoleAndProfilesBestEffort(fix.AWS, singleIAMProfileRole,
[]string{singleIAMProfileApp, singleIAMProfileOther}, adminPolicyARN)
// Singleton instance — reused for the post-launch Associate/Replace
// cycle. Must be running before we proceed.
@@ -43,27 +55,27 @@ func runIAMInstanceProfileAssociation(t *testing.T, fix *Fixture) {
// Build role + two profiles. The second profile is the Replace target.
harness.Step(t, "setup: create-role + 2 instance profiles")
_, err := fix.AWS.IAM.CreateRole(&iam.CreateRoleInput{
RoleName: aws.String(iamRoleAppName),
AssumeRolePolicyDocument: aws.String(iamTrustPolicyEC2Standard),
RoleName: aws.String(singleIAMProfileRole),
AssumeRolePolicyDocument: aws.String(singleIAMProfileTrustPolicy),
})
require.NoError(t, err, "create-role")
// Even though root bypasses iam:PassRole, attach AdministratorAccess
// to match production usage and exercise the PassRole code path.
_, err = fix.AWS.IAM.AttachRolePolicy(&iam.AttachRolePolicyInput{
RoleName: aws.String(iamRoleAppName),
RoleName: aws.String(singleIAMProfileRole),
PolicyArn: aws.String(adminPolicyARN),
})
require.NoError(t, err, "attach AdministratorAccess to role")
for _, p := range []string{iamProfileAppName, iamProfileOtherName} {
for _, p := range []string{singleIAMProfileApp, singleIAMProfileOther} {
_, err = fix.AWS.IAM.CreateInstanceProfile(&iam.CreateInstanceProfileInput{
InstanceProfileName: aws.String(p),
})
require.NoError(t, err, "create-instance-profile %s", p)
_, err = fix.AWS.IAM.AddRoleToInstanceProfile(&iam.AddRoleToInstanceProfileInput{
InstanceProfileName: aws.String(p),
RoleName: aws.String(iamRoleAppName),
RoleName: aws.String(singleIAMProfileRole),
})
require.NoError(t, err, "add-role-to-instance-profile %s", p)
}
@@ -86,15 +98,15 @@ func runIAMInstanceProfileAssociation(t *testing.T, fix *Fixture) {
&ec2.DisassociateIamInstanceProfileInput{AssociationId: a.AssociationId})
}
}
iamDeleteRoleAndProfilesBestEffort(fix, iamRoleAppName,
[]string{iamProfileAppName, iamProfileOtherName}, adminPolicyARN)
harness.IAMDeleteRoleAndProfilesBestEffort(fix.AWS, singleIAMProfileRole,
[]string{singleIAMProfileApp, singleIAMProfileOther}, adminPolicyARN)
})
// ---- Associate (post-launch) on singleton ----
harness.Step(t, "associate-iam-instance-profile %s -> %s", iamProfileAppName, singletonID)
harness.Step(t, "associate-iam-instance-profile %s -> %s", singleIAMProfileApp, singletonID)
assocOut, err := fix.AWS.EC2.AssociateIamInstanceProfile(&ec2.AssociateIamInstanceProfileInput{
InstanceId: aws.String(singletonID),
IamInstanceProfile: &ec2.IamInstanceProfileSpecification{Name: aws.String(iamProfileAppName)},
IamInstanceProfile: &ec2.IamInstanceProfileSpecification{Name: aws.String(singleIAMProfileApp)},
})
require.NoError(t, err, "associate-iam-instance-profile")
require.NotNil(t, assocOut.IamInstanceProfileAssociation, "missing association in response")
@@ -108,7 +120,7 @@ func runIAMInstanceProfileAssociation(t *testing.T, fix *Fixture) {
harness.ExpectError(t, "IamInstanceProfileAlreadyAssociated", func() error {
_, e := fix.AWS.EC2.AssociateIamInstanceProfile(&ec2.AssociateIamInstanceProfileInput{
InstanceId: aws.String(singletonID),
IamInstanceProfile: &ec2.IamInstanceProfileSpecification{Name: aws.String(iamProfileAppName)},
IamInstanceProfile: &ec2.IamInstanceProfileSpecification{Name: aws.String(singleIAMProfileApp)},
})
return e
})
@@ -124,7 +136,7 @@ func runIAMInstanceProfileAssociation(t *testing.T, fix *Fixture) {
ip := desc.Reservations[0].Instances[0].IamInstanceProfile
require.NotNil(t, ip, "DescribeInstances must populate IamInstanceProfile when bound")
require.True(t,
strings.HasSuffix(aws.StringValue(ip.Arn), ":instance-profile/"+iamProfileAppName),
strings.HasSuffix(aws.StringValue(ip.Arn), ":instance-profile/"+singleIAMProfileApp),
"unexpected IamInstanceProfile.Arn %q", aws.StringValue(ip.Arn))
// DescribeIamInstanceProfileAssociations by id.
@@ -143,17 +155,17 @@ func runIAMInstanceProfileAssociation(t *testing.T, fix *Fixture) {
harness.Step(t, "delete-instance-profile while bound (expect DeleteConflict)")
harness.ExpectError(t, "DeleteConflict", func() error {
_, e := fix.AWS.IAM.DeleteInstanceProfile(&iam.DeleteInstanceProfileInput{
InstanceProfileName: aws.String(iamProfileAppName),
InstanceProfileName: aws.String(singleIAMProfileApp),
})
return e
})
// Replace → other-profile, must mint a new assoc id.
harness.Step(t, "replace-iam-instance-profile-association -> %s", iamProfileOtherName)
harness.Step(t, "replace-iam-instance-profile-association -> %s", singleIAMProfileOther)
replaceOut, err := fix.AWS.EC2.ReplaceIamInstanceProfileAssociation(
&ec2.ReplaceIamInstanceProfileAssociationInput{
AssociationId: aws.String(assocID),
IamInstanceProfile: &ec2.IamInstanceProfileSpecification{Name: aws.String(iamProfileOtherName)},
IamInstanceProfile: &ec2.IamInstanceProfileSpecification{Name: aws.String(singleIAMProfileOther)},
})
require.NoError(t, err, "replace-iam-instance-profile-association")
newAssocID := aws.StringValue(replaceOut.IamInstanceProfileAssociation.AssociationId)
@@ -166,7 +178,7 @@ func runIAMInstanceProfileAssociation(t *testing.T, fix *Fixture) {
_, e := fix.AWS.EC2.ReplaceIamInstanceProfileAssociation(
&ec2.ReplaceIamInstanceProfileAssociationInput{
AssociationId: aws.String(assocID),
IamInstanceProfile: &ec2.IamInstanceProfileSpecification{Name: aws.String(iamProfileAppName)},
IamInstanceProfile: &ec2.IamInstanceProfileSpecification{Name: aws.String(singleIAMProfileApp)},
})
return e
})
@@ -200,7 +212,7 @@ func runIAMInstanceProfileAssociation(t *testing.T, fix *Fixture) {
// Reuse the singleton's launch attributes so we don't need to rediscover
// AMI / type / key / subnet / SG — guarantees the ephemeral VM lands on a
// known-good combination on the local single-node fixture.
ephID := runInstanceWithProfile(t, fix, inst, iamProfileAppName)
ephID := runInstanceWithProfile(t, fix, inst, singleIAMProfileApp)
defer func() {
// Belt-and-braces: even if the auto-disassociate assertion fails, we
// must not leave a VM running. Terminate is idempotent.
tests/e2e/iam/main_test.go
+108
View File
@@ -0,0 +1,108 @@
//go:build e2e
// Package iam is the standalone "everything IAM" E2E suite, split out of the
// single-node suite so it builds and runs as its own iam.test binary. Every
// entry point is a top-level Test* in tests_test.go. The control-plane phases
// (IAM/STS CRUD) are zero-VM; the instance-identity tail (instance-profile
// association + IMDS) boots guest VMs, so the suite is a single-mode,
// VM-booting matrix leg covering the awsgw IAM/STS surface plus the
// instance-profile + IMDS datapath.
package iam
import (
"os"
"sync"
"testing"
"github.com/mulgadc/spinifex/tests/e2e/harness"
)
// Package-scoped singleton fixture. Initialised lazily by the first call to
// requireIAMFixture. TestMain drains its cleanup chain after m.Run() so every
// resource ensured during the run is torn down at process exit, regardless of
// which Test* created it.
var (
pkgFixOnce sync.Once
pkgFix *Fixture
pkgFixErr error
)
// TestMain owns process-level lifecycle for the singleton fixture. The
// singleton itself is built lazily by requireIAMFixture so a test run with
// SPINIFEX_E2E unset stays cheap (no AWS dial, no temp dir).
func TestMain(m *testing.M) {
code := m.Run()
if pkgFix != nil {
if pkgFix.Harness != nil {
pkgFix.Harness.Close()
}
if pkgFix.TmpDir != "" {
_ = os.RemoveAll(pkgFix.TmpDir)
}
}
os.Exit(code)
}
// requireIAMFixture returns the package-scoped Fixture singleton, building it on
// first call. Skips the calling test if SPINIFEX_E2E is unset or the cluster
// mode is not single-node. Fails the test if init itself errors (e.g. AWS
// client construction). PoolMode is detected so the instance-identity tests can
// gate their probe-VPC public-IP path.
func requireIAMFixture(t *testing.T) *Fixture {
t.Helper()
pkgFixOnce.Do(func() {
if os.Getenv("SPINIFEX_E2E") == "" {
return
}
env := harness.LoadEnv(t)
if env.Mode != harness.ModeSingle {
return
}
awsCli := harness.NewAWSClient(t, env)
h, herr := harness.NewProcessFixture(awsCli)
if herr != nil {
pkgFixErr = herr
return
}
tmpDir, terr := os.MkdirTemp("", "iam-pkgfix-*")
if terr != nil {
pkgFixErr = terr
return
}
pkgFix = &Fixture{
Env: env,
AWS: awsCli,
Harness: h,
TmpDir: tmpDir,
PoolMode: detectPoolMode(env),
}
})
if pkgFixErr != nil {
t.Fatalf("iam fixture init failed: %v", pkgFixErr)
}
if pkgFix == nil {
t.Skip("iam fixture unavailable (SPINIFEX_E2E unset or mode != single)")
}
return pkgFix
}
// Fixture carries the per-process state shared across every Test* in this
// package. Only environment-level slots — every per-phase resource ID is
// memoized on Harness (harness.Fixture) or the package-local iamEnsure* helpers.
type Fixture struct {
Env *harness.Env
AWS *harness.AWSClient
Harness *harness.Fixture // memoized Ensure* fixture; spans the whole process.
TmpDir string // package-scoped scratch dir; survives every Test* in the package.
PoolMode bool // external IPAM in play; gates the IMDS probe-VPC public-IP path.
}
// ArtifactDir returns the artifact directory for the *currently running* test.
// It must be derived per-call from the live t (not memoized on the singleton
// Fixture): the process fixture is built lazily during whichever test runs
// first, so a stored dir would freeze to that test's name and every later test
// would write into a stale — and, once that test passes, pruned — directory.
func (f *Fixture) ArtifactDir(t *testing.T) string {
t.Helper()
return harness.ArtifactDir(t, f.Env)
}
tests/e2e/single/sts_test.go → tests/e2e/iam/sts_test.go
+7 -7
View File
@@ -1,6 +1,6 @@
//go:build e2e
package single
package iam
import (
"fmt"
@@ -54,7 +54,7 @@ const (
// blocks (Condition / NotPrincipal / NotAction) via the SDK.
func runSTS(t *testing.T, fix *Fixture) {
harness.Phase(t, "Single — STS AssumeRole + GetCallerIdentity")
adminAccount := iamEnsureAdminAccountID(t, fix)
adminAccount := harness.IAMAccountID(t, fix.AWS)
// Bootstrap-creds GetCallerIdentity — sanity check the live endpoint and
// capture the active ARN for the detail log so failures elsewhere are
@@ -151,11 +151,11 @@ func runSTS(t *testing.T, fix *Fixture) {
stsRoleNameForbidNotAction,
}
for _, name := range stsRoles {
iamDeleteRoleAndProfilesBestEffort(fix, name, nil)
harness.IAMDeleteRoleAndProfilesBestEffort(fix.AWS, name, nil)
}
fix.Harness.RegisterCleanup(func() {
for _, name := range stsRoles {
iamDeleteRoleAndProfilesBestEffort(fix, name, nil)
harness.IAMDeleteRoleAndProfilesBestEffort(fix.AWS, name, nil)
}
})
@@ -167,7 +167,7 @@ func runSTS(t *testing.T, fix *Fixture) {
})
require.NoError(t, err, "create-role")
roleARN := aws.StringValue(createOut.Role.Arn)
require.Equal(t, iamRoleARN(adminAccount, stsRoleName), roleARN,
require.Equal(t, harness.IAMRoleARN(adminAccount, stsRoleName), roleARN,
"role ARN must follow arn:aws:iam::<acct>:role/<name>")
// Happy path. Verify the wire-format invariants: ASIA prefix, non-empty
@@ -375,7 +375,7 @@ func runSTS(t *testing.T, fix *Fixture) {
// → trust-policy match → new ASIA mint) is not.
chainedTrustPolicy := fmt.Sprintf(
`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":%q},"Action":"sts:AssumeRole"}]}`,
iamRoleARN(adminAccount, stsRoleName),
harness.IAMRoleARN(adminAccount, stsRoleName),
)
harness.Step(t, "create-role %q (trust=role/%s for chained assume)",
stsRoleNameChain, stsRoleName)
@@ -445,7 +445,7 @@ func runSTS(t *testing.T, fix *Fixture) {
harness.Step(t, "assume-role missing role (expect NoSuchEntity)")
harness.ExpectError(t, "NoSuchEntity", func() error {
_, e := fix.AWS.STS.AssumeRole(&sts.AssumeRoleInput{
RoleArn: aws.String(iamRoleARN(adminAccount,
RoleArn: aws.String(harness.IAMRoleARN(adminAccount,
"sts-e2e-no-such-role")),
RoleSessionName: aws.String("ghost"),
})
tests/e2e/iam/tests_test.go
+82
View File
@@ -0,0 +1,82 @@
//go:build e2e
package iam
import "testing"
// Top-level Test* entry points for the IAM/STS + instance-identity suite. Each
// delegates to a runX function in the matching <name>_test.go file. Kept as
// separate top-level Test* (not one inline Test) so `go test -run TestX` and the
// suite selector can address each individually.
//
// Execution is sequential in source order: these tests share the package-scoped
// IAM state (users/policies/roles are account-flat), and TestIAMCleanup tears
// down the user/policy graph the earlier phases build. The parallelism win of
// the split is the suite-as-matrix-leg, not intra-suite concurrency — sub-tests
// inside each runIAM* still fan out with t.Parallel where it is safe.
//
// The final two entries (TestIAMInstanceProfileAssociation, TestIMDS) boot guest
// VMs. They self-bootstrap their own roles/profiles/VPCs under collision-safe
// namespaces and run last, after the control-plane phases.
func TestIAMUserCRUD(t *testing.T) {
runIAMUserCRUD(t, requireIAMFixture(t))
}
func TestIAMAccessKeyLifecycle(t *testing.T) {
runIAMAccessKeyLifecycle(t, requireIAMFixture(t))
}
func TestIAMUserAuthentication(t *testing.T) {
runIAMUserAuthentication(t, requireIAMFixture(t))
}
func TestIAMPolicyCRUD(t *testing.T) {
runIAMPolicyCRUD(t, requireIAMFixture(t))
}
func TestIAMPolicyAttachmentEnforcement(t *testing.T) {
runIAMPolicyAttachmentEnforcement(t, requireIAMFixture(t))
}
func TestIAMPolicyLifecycle(t *testing.T) {
runIAMPolicyLifecycle(t, requireIAMFixture(t))
}
func TestIAMCleanup(t *testing.T) {
runIAMCleanup(t, requireIAMFixture(t))
}
func TestIAMRolesAndProfiles(t *testing.T) {
runIAMRolesAndProfiles(t, requireIAMFixture(t))
}
// TestSTSAssumeRoleAndGetCallerIdentity is sequential to avoid racing trust-policy
// mutations against a parallel AssumeRole.
func TestSTSAssumeRoleAndGetCallerIdentity(t *testing.T) {
runSTS(t, requireIAMFixture(t))
}
// TestAssumedRoleControlPlaneEnforcement verifies a zero-policy assumed-role is
// denied and permitted once a policy is attached. Sequential to avoid racing
// the mid-test grant.
func TestAssumedRoleControlPlaneEnforcement(t *testing.T) {
runAssumedRoleControlPlaneEnforcement(t, requireIAMFixture(t))
}
// TestIAMInstanceProfileAssociation exercises the EC2 IAM instance-profile
// association lifecycle (associate/replace/disassociate, RunInstances
// --iam-instance-profile, auto-disassociate on terminate). It boots its own
// instance and self-bootstraps a role + two profiles under the
// single-iamprofile- namespace, distinct from this suite's app-role/app-profile.
// Sequential: it boots VMs and mutates instance state.
func TestIAMInstanceProfileAssociation(t *testing.T) {
runIAMInstanceProfileAssociation(t, requireIAMFixture(t))
}
// TestIMDS exercises IMDSv2 end-to-end: token issuance, metadata surface,
// instance-role credentials, OVN datapath, and cross-VPC isolation. Sequential
// because it launches profile-bound VMs and creates fresh VPCs (imds-e2e-*).
func TestIMDS(t *testing.T) {
runIMDS(t, requireIAMFixture(t))
}
tests/e2e/lb/lb_test.go
+147
View File
@@ -5,6 +5,7 @@ package lb
import (
"bytes"
"context"
"crypto/x509"
_ "embed"
"encoding/base64"
"encoding/json"
@@ -18,6 +19,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/acm"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/elbv2"
"github.com/mulgadc/spinifex/tests/e2e/harness"
@@ -88,6 +90,15 @@ func TestLoadBalancer(t *testing.T) {
}
runLBSuite(t, client, fixture, kindNLB, "internet-facing", ssh, peer)
})
t.Run("InternetFacing_ALB_HTTPS", func(t *testing.T) {
if peer == "" {
// HTTPS handshake is driven from the test runner against the LB
// public IP; gate on the same peer-available signal the sibling
// internet-facing subtests use, where driver→LB reachability holds.
t.Skip("no peer node available")
}
runHTTPSCertSuite(t, client, fixture)
})
// Gate remaining internal subtests on Internal_ALB: if the LB never reaches active,
// the rest will time out identically — fail fast instead of burning ~5min each.
albOK := t.Run("Internal_ALB", func(t *testing.T) {
@@ -527,6 +538,142 @@ func runLBSuite(t *testing.T, c *harness.AWSClient, f *sharedFixture, kind lbKin
runInternalTrafficViaClient(t, c, f, kind, priv)
}
// runHTTPSCertSuite exercises the ACM → HTTPS listener path end-to-end:
// ImportCertificate a self-signed leaf → create an internet-facing ALB HTTPS
// listener referencing the cert ARN → complete a TLS handshake against the LB
// and assert HAProxy serves exactly the imported cert → drive one HTTPS GET
// that terminates TLS at the LB and forwards to a healthy backend →
// DeleteCertificate. The cert is minted for the LB's public IP (known only
// after the LB is active) so the end-to-end GET can verify the chain by IP SAN.
func runHTTPSCertSuite(t *testing.T, c *harness.AWSClient, f *sharedFixture) {
t.Helper()
const label = "ALB internet-facing HTTPS (ACM)"
const httpsPort int64 = 443
authorizeSGPort(t, c, f, "tcp", httpsPort)
tgArn := createTargetGroup(t, c, f, "lb-e2e-https-tg", "HTTP", httpPort, "/index.html")
t.Cleanup(func() { deleteTargetGroup(t, c, tgArn) })
registerTargets(t, c, tgArn, f.AppInstanceIDs)
t.Cleanup(func() { deregisterTargets(t, c, tgArn, f.AppInstanceIDs) })
lb := createLB(t, c, f, "lb-e2e-https", "application", "internet-facing")
t.Cleanup(func() { deleteLB(t, c, lb) })
harness.WaitForLBActive(t, c, lb.ARN, label, 5*time.Minute)
eni := lbENI(t, c, "app", lb)
pubIP := publicIP(eni)
require.NotEmpty(t, pubIP, label+" needs a public IP for the TLS handshake")
// Mint a self-signed leaf for the LB's public IP and import it into ACM.
certPEM, keyPEM, leaf, err := harness.GenerateSelfSignedCertPEM(pubIP)
require.NoError(t, err, "generate self-signed cert")
imp, err := c.ACM.ImportCertificate(&acm.ImportCertificateInput{
Certificate: certPEM,
PrivateKey: keyPEM,
})
require.NoError(t, err, "acm import-certificate")
certArn := aws.StringValue(imp.CertificateArn)
require.NotEmpty(t, certArn, "ImportCertificate returned empty CertificateArn")
require.Contains(t, certArn, ":certificate/", "cert ARN must be an ACM certificate ARN")
t.Cleanup(func() {
if _, derr := c.ACM.DeleteCertificate(&acm.DeleteCertificateInput{
CertificateArn: aws.String(certArn),
}); derr != nil {
t.Logf("delete certificate %s: %v", certArn, derr)
}
})
t.Logf("imported cert: %s", certArn)
listener := createHTTPSListener(t, c, lb.ARN, httpsPort, tgArn, certArn)
t.Cleanup(func() { deleteListener(t, c, listener) })
harness.WaitForTargetsHealthy(t, c, tgArn, 2, label, 2*time.Minute)
// TLS handshake: HAProxy must present exactly the imported leaf. The
// lb-agent picks up the new listener cert on its reconcile tick, so poll
// the handshake until the served cert matches before asserting.
var served *x509.Certificate
harness.EventuallyErr(t, func() error {
got, ferr := harness.FetchServedCert(pubIP, int(httpsPort), 5*time.Second)
if ferr != nil {
return ferr
}
if !harness.FingerprintMatches(got, leaf) {
return fmt.Errorf("served cert does not match imported cert yet")
}
served = got
return nil
}, 90*time.Second, 3*time.Second)
require.NotNil(t, served, label+" never served the imported cert")
t.Logf("TLS handshake ok — HAProxy served the imported cert (CN=%s)", served.Subject.CommonName)
version, _, err := harness.FetchTLSPosture(pubIP, int(httpsPort), 5*time.Second)
require.NoError(t, err, "fetch TLS posture")
assert.GreaterOrEqualf(t, version, uint16(0x0303), "negotiated TLS version must be >= 1.2, got 0x%04x", version)
// End-to-end: one HTTPS GET that terminates TLS at the LB and forwards to a
// backend. Verify the chain against the imported cert (IP SAN matches pubIP).
pool := x509.NewCertPool()
require.True(t, pool.AppendCertsFromPEM(certPEM), "build CA pool from imported cert")
url := fmt.Sprintf("https://%s:%d/index.html", pubIP, httpsPort)
harness.EventuallyErr(t, func() error {
status, body, gerr := harness.HTTPSGet(url, pool, 10*time.Second)
if gerr != nil {
return gerr
}
if status != http.StatusOK {
return fmt.Errorf("status %d (body=%q)", status, string(body))
}
return nil
}, 90*time.Second, 3*time.Second)
t.Logf("HTTPS GET %s -> 200 (TLS terminated at LB, forwarded to backend)", url)
}
// createHTTPSListener creates an HTTPS listener referencing certArn. Distinct
// from createListener because the ELBv2 handler requires Certificates for HTTPS
// and rejects them for HTTP.
func createHTTPSListener(t *testing.T, c *harness.AWSClient, lbArn string, port int64, tgArn, certArn string) string {
t.Helper()
// e2e:allow-create — the HTTPS listener is the subject under test (ACM cert -> termination).
out, err := c.ELBv2.CreateListener(&elbv2.CreateListenerInput{
LoadBalancerArn: aws.String(lbArn),
Protocol: aws.String("HTTPS"),
Port: aws.Int64(port),
Certificates: []*elbv2.Certificate{{CertificateArn: aws.String(certArn)}},
DefaultActions: []*elbv2.Action{{
Type: aws.String("forward"),
TargetGroupArn: aws.String(tgArn),
}},
})
require.NoError(t, err, "create-listener HTTPS")
require.NotEmpty(t, out.Listeners, "create-listener HTTPS returned no listeners")
arn := aws.StringValue(out.Listeners[0].ListenerArn)
t.Logf("listener HTTPS:%d -> %s (cert %s)", port, arn, certArn)
return arn
}
// authorizeSGPort idempotently opens proto/port from 0.0.0.0/0 on the shared
// default SG, tolerating the Duplicate code on re-runs.
func authorizeSGPort(t *testing.T, c *harness.AWSClient, f *sharedFixture, proto string, port int64) {
t.Helper()
_, err := c.EC2.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{
GroupId: aws.String(f.SecurityGroup),
IpPermissions: []*ec2.IpPermission{{
IpProtocol: aws.String(proto),
FromPort: aws.Int64(port),
ToPort: aws.Int64(port),
IpRanges: []*ec2.IpRange{{CidrIp: aws.String("0.0.0.0/0")}},
}},
})
if err != nil {
var aerr awserr.Error
if !errors.As(err, &aerr) || aerr.Code() != "InvalidPermission.Duplicate" {
t.Fatalf("authorize %s/%d: %v", proto, port, err)
}
}
}
// runUDPNLBSuite exercises an internal NLB with a UDP listener. The lb-agent probes
// targets over TCP/9000 (nginx stream has no active health-check); once healthy, a
// UDP datagram round-trip through the VIP proves the L4 data plane.
tests/e2e/manifest-lint/baseline.txt
+9 -7
View File
@@ -7,6 +7,13 @@ create tests/e2e/eks/eks_test.go CreateVpc #1
create tests/e2e/gpu/gpu_test.go RunInstances #1
create tests/e2e/gpu/gpu_test.go RunInstances #2
create tests/e2e/gpu/helpers_test.go RunInstances #1
create tests/e2e/iam/iam_test.go CreateKeyPair #1
create tests/e2e/iam/imds_test.go CreateInternetGateway #1
create tests/e2e/iam/imds_test.go CreateRouteTable #1
create tests/e2e/iam/imds_test.go CreateSubnet #1
create tests/e2e/iam/imds_test.go CreateVpc #1
create tests/e2e/iam/imds_test.go RunInstances #1
create tests/e2e/iam/instance_iam_profile_test.go RunInstances #1
create tests/e2e/lb/lb_test.go CreateInternetGateway #1
create tests/e2e/lb/lb_test.go CreateKeyPair #1
create tests/e2e/lb/lb_test.go CreateListener #1
@@ -30,6 +37,7 @@ create tests/e2e/multinode/placement_nat_test.go CreateSubnet #2
create tests/e2e/multinode/placement_nat_test.go CreateVpc #1
create tests/e2e/multinode/placement_nat_test.go RunInstances #1
create tests/e2e/multinode/placement_nat_test.go RunInstances #2
create tests/e2e/multinode/volume_durability_test.go CreateVolume #1
create tests/e2e/multinode/volume_test.go CreateVolume #1
create tests/e2e/multinode/vpc_test.go CreateSubnet #1
create tests/e2e/multinode/vpc_test.go CreateVpc #1
@@ -71,13 +79,6 @@ create tests/e2e/single/baseline_test.go CreateVpc #1
create tests/e2e/single/baseline_test.go RunInstances #1
create tests/e2e/single/datapath_test.go CreateSecurityGroup #1
create tests/e2e/single/datapath_test.go RunInstances #1
create tests/e2e/single/iam_test.go CreateKeyPair #1
create tests/e2e/single/imds_test.go CreateInternetGateway #1
create tests/e2e/single/imds_test.go CreateRouteTable #1
create tests/e2e/single/imds_test.go CreateSubnet #1
create tests/e2e/single/imds_test.go CreateVpc #1
create tests/e2e/single/imds_test.go RunInstances #1
create tests/e2e/single/instance_iam_profile_test.go RunInstances #1
create tests/e2e/single/keypair_test.go ImportKeyPair #1
create tests/e2e/single/lifecycle_test.go CreateVolume #1
create tests/e2e/single/lifecycle_test.go RunInstances #1
@@ -96,6 +97,7 @@ create tests/e2e/single/negative_test.go RunInstances #2
create tests/e2e/single/negative_test.go RunInstances #3
create tests/e2e/single/negative_test.go RunInstances #4
create tests/e2e/single/snapshot_test.go CreateSnapshot #1
create tests/e2e/single/volume_durability_test.go CreateVolume #1
create tests/e2e/single/volume_test.go CreateVolume #1
create tests/e2e/single/vpc_test.go CreateInternetGateway #1
create tests/e2e/single/vpc_test.go CreateRouteTable #1

Some files were not shown because too many files have changed in this diff Show More