Merge commit '02931567ac207bd074d8b2efda6e523ab36540d9' into vpc-public-subnet

.github/workflows/e2e.yml

@@ -28,7 +28,7 @@ jobs:
       - name: Clean workspace
         run: rm -rf "$GITHUB_WORKSPACE"/*
 
-      - uses: actions/create-github-app-token@v2
+      - uses: actions/create-github-app-token@v3
         id: app-token
         with:
           app-id: ${{ secrets.APP_ID }}
@@ -91,7 +91,7 @@ jobs:
       - name: Clean workspace
         run: rm -rf "$GITHUB_WORKSPACE"/*
 
-      - uses: actions/create-github-app-token@v2
+      - uses: actions/create-github-app-token@v3
         id: app-token
         with:
           app-id: ${{ secrets.APP_ID }}
@@ -154,7 +154,7 @@ jobs:
       - name: Clean workspace
         run: rm -rf "$GITHUB_WORKSPACE"/*
 
-      - uses: actions/create-github-app-token@v2
+      - uses: actions/create-github-app-token@v3
         id: app-token
         with:
           app-id: ${{ secrets.APP_ID }}

.github/workflows/release.yml

@@ -8,7 +8,7 @@ permissions:
   contents: write
 
 jobs:
-  distro:
+  distro-amd64:
     runs-on: ubuntu-24.04
     steps:
       - name: Checkout Spinifex
@@ -29,19 +29,176 @@ jobs:
           path: viperblock
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
-      - name: Set up QEMU (for arm64 cross-build)
-        uses: docker/setup-qemu-action@v3
+      - name: Build amd64 Tarball
+        env:
+          VERSION: ${{ github.ref_name }}
+        run: cd spinifex && make distro-amd64 VERSION="$VERSION"
 
-      - name: Build distribution tarballs
-        run: cd spinifex && cd spinifex && make distro VERSION=${{ github.ref_name }}
+      - name: Upload amd64 Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: distro-amd64
+          path: |
+            spinifex/dist/spinifex-${{ github.ref_name }}-linux-amd64.tar.gz
+            spinifex/dist/spinifex-${{ github.ref_name }}-linux-amd64.tar.gz.sha256
+            spinifex/scripts/setup.sh
 
-      - name: Create GitHub Release
+  distro-arm64:
+    runs-on: ubuntu-24.04-arm
+    steps:
+      - name: Checkout Spinifex
+        uses: actions/checkout@v6
+        with:
+          path: spinifex
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v6
+        with:
+          repository: mulgadc/predastore
+          path: predastore
+
+      - name: Checkout Viperblock
+        uses: actions/checkout@v6
+        with:
+          repository: mulgadc/viperblock
+          path: viperblock
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Build arm64 Tarball
+        env:
+          VERSION: ${{ github.ref_name }}
+        run: cd spinifex && make distro-arm64 VERSION="$VERSION"
+
+      - name: Upload arm64 Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: distro-arm64
+          path: |
+            spinifex/dist/spinifex-${{ github.ref_name }}-linux-arm64.tar.gz
+            spinifex/dist/spinifex-${{ github.ref_name }}-linux-arm64.tar.gz.sha256
+
+  release:
+    needs: [distro-amd64, distro-arm64]
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Download amd64 Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: distro-amd64
+          path: artifacts
+
+      - name: Download arm64 Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: distro-arm64
+          path: artifacts
+
+      - name: Create Draft GitHub Release
         uses: softprops/action-gh-release@v2
         with:
+          draft: true
           files: |
-            spinifex/dist/*.tar.gz
-            spinifex/dist/*.sha256
-            spinifex/scripts/setup.sh
+            artifacts/**/*.tar.gz
+            artifacts/**/*.sha256
+            artifacts/**/setup.sh
           generate_release_notes: true
+
+      - name: Upload Tarball for Verify Job
+        uses: actions/upload-artifact@v4
+        with:
+          name: release-assets
+          path: artifacts/
+
+  verify:
+    needs: release
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 25
+    env:
+      TOFU_LOG: /tmp/release-verify-${{ github.run_id }}.log
+      TAG: ${{ github.ref_name }}
+      REPO: ${{ github.repository }}
+    steps:
+      - name: Clean Workspace
+        run: rm -rf "$GITHUB_WORKSPACE"/*
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          app-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v6
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/tofu-cluster
+          path: mulga
+
+      - name: Download Release Assets
+        uses: actions/download-artifact@v4
+        with:
+          name: release-assets
+          path: release-assets
+
+      - name: Provision VM
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          tofu init -input=false >> "$TOFU_LOG" 2>&1
+          tofu workspace select -or-create env3 >> "$TOFU_LOG" 2>&1
+          tofu destroy -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
+          ./scrub-env.sh env3 >> "$TOFU_LOG" 2>&1 || true
+          tofu apply -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
+
+      - name: Bootstrap via Binary Install
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          SETUP_SH=$(find "${GITHUB_WORKSPACE}/release-assets" -name "setup.sh" | head -1)
+          TARBALL=$(find "${GITHUB_WORKSPACE}/release-assets" -name "*-linux-amd64.tar.gz" | head -1)
+          ./bootstrap-install.sh env3 "$SETUP_SH" "$TARBALL"
+
+      - name: Run Smoke Test
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
+            -i "$SSH_KEY" \
+            smoke-test-install.sh \
+            "tf-user@${WAN_IP}:/tmp/smoke-test.sh"
+          ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
+            -i "$SSH_KEY" "tf-user@${WAN_IP}" \
+            'bash /tmp/smoke-test.sh'
+
+      - name: Promote Release
+        if: success()
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh release edit "$TAG" --repo "$REPO" --draft=false
+          echo "Release ${TAG} promoted to latest"
+
+      - name: Mark Release Failed
+        if: failure()
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          EXISTING_BODY=$(gh release view "$TAG" --repo "$REPO" --json body -q .body)
+          NOTES_FILE=$(mktemp)
+          printf 'Verification failed — see workflow run for details.\n\n%s' "$EXISTING_BODY" > "$NOTES_FILE"
+          gh release edit "$TAG" --repo "$REPO" --notes-file "$NOTES_FILE"
+          rm -f "$NOTES_FILE"
+
+      - name: Teardown
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          tofu workspace select -or-create env3 >> "$TOFU_LOG" 2>&1
+          tofu destroy -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
+          ./scrub-env.sh env3 >> "$TOFU_LOG" 2>&1 || true

.github/workflows/spinifex.yml

@@ -120,14 +120,11 @@ jobs:
       - name: Setup Go Workspace
         run: go work init ./spinifex ./viperblock ./predastore
 
-      - name: Check Format
-        run: make -C spinifex check-format
+      - name: Lint
+        uses: golangci/golangci-lint-action@v7
+        with:
+          version: v2.11.3
+          working-directory: spinifex
 
-      - name: Check Modernize
-        run: make -C spinifex check-modernize
-
-      - name: Go Vet
-        run: make -C spinifex vet
-
-      - name: Security Checks
-        run: make -C spinifex security-check
+      - name: Govulncheck
+        run: make -C spinifex govulncheck

.github/workflows/ui.yml

@@ -25,7 +25,7 @@ jobs:
           sparse-checkout: spinifex/services/spinifexui/frontend
 
       - name: Setup pnpm
-        uses: pnpm/action-setup@v4
+        uses: pnpm/action-setup@v5
         with:
           package_json_file: spinifex/services/spinifexui/frontend/package.json
 
@@ -39,7 +39,7 @@ jobs:
       - name: Install Dependencies
         run: pnpm install --frozen-lockfile
 
-      - name: Run Biome Check
+      - name: Run Check
         run: pnpm check
 
       - name: Run Tests with Coverage

.golangci.yml

@@ -0,0 +1,128 @@
+version: "2"
+
+formatters:
+  enable:
+    - gofmt
+  settings:
+    gofmt:
+      simplify: false # modernize linter covers gofmt -s simplifications (Go 1.23+)
+
+linters:
+  enable:
+    - asasalint # catches pass []any as any in variadic func(...any)
+    - asciicheck # checks that code does not contain non-ASCII identifiers
+    - bidichk # checks for dangerous unicode bidi character sequences
+    - bodyclose # checks whether HTTP response body is closed successfully
+    - canonicalheader # checks whether net/http.Header uses canonical header keys
+    - decorder # checks declaration order of types, consts, vars, and funcs
+    - durationcheck # checks for two durations multiplied together
+    - embeddedstructfieldcheck # checks embedded sync.Mutex/RWMutex in structs
+    - errcheck # checks for unchecked errors in type assertions and I/O
+    - errchkjson # checks types passed to json encoding functions, flags omittable error checks
+    - errname # checks that sentinel errors are prefixed with Err, error types suffixed with Error
+    - exptostd # detects golang.org/x/exp/ functions replaceable by stdlib
+    - fatcontext # detects nested contexts in loops
+    - gocheckcompilerdirectives # validates go compiler directive comments (//go:)
+    - goprintffuncname # checks that printf-like functions are named with f at the end
+    - gosec # inspects source code for security problems
+    - govet # reports suspicious constructs (Printf alignment, shadow, etc.)
+    - ineffassign # detects assignments to existing variables that are not used
+    - makezero # finds slice declarations with non-zero initial length followed by append
+    - mirror # catches wrong mirror patterns in bytes/strings usage
+    - modernize # modernizes code to use newer Go idioms (go fix analyzers)
+    - nakedret # finds naked returns in functions
+    - nilerr # catches code that returns nil even after checking err != nil
+    - nilnesserr # catches returning a different nil error variable after err != nil check
+    - nolintlint # reports ill-formed or insufficient nolint directives
+    - predeclared # catches shadowing of Go builtins (len, cap, error, etc.)
+    - reassign # checks that package variables are not reassigned
+    - recvcheck # checks for receiver type consistency (pointer vs value)
+    - sloglint # enforces consistent log/slog usage (key-value pairs, no globals)
+    - staticcheck # extensive static analysis checks
+    - unconvert # removes unnecessary type conversions
+    - usestdlibvars # detects hardcoded values that should use stdlib constants
+    - usetesting # catches os.TempDir() etc. in tests — should use t.TempDir()
+    - wastedassign # finds assignments to variables that are never subsequently read
+    - whitespace # detects leading and trailing whitespace in functions
+
+  settings:
+    embeddedstructfieldcheck:
+      forbid-mutex: true
+
+    errcheck:
+      check-type-assertions: true
+      exclude-functions:
+        - (*github.com/nats-io/nats.go.Subscription).Unsubscribe # cleanup-only, errors not actionable
+        - (*github.com/kdomanski/iso9660.ImageWriter).Cleanup # deferred cleanup, errors not actionable
+
+    govet:
+      enable-all: true
+      disable:
+        - fieldalignment
+        - shadow # too noisy to retrofit — all 50 findings are err re-declarations
+        - unusedwrite # false positives on struct field assignments before marshal/send
+
+    staticcheck:
+      checks:
+        - all
+        - -ST1000 # package comment
+        - -ST1003 # naming conventions (Id→ID etc.) — too noisy to retrofit
+        - -ST1012 # error var naming — errname covers this + error type suffixes
+        - -ST1013 # HTTP status constants — usestdlibvars is broader
+        - -ST1016 # receiver name consistency (too noisy to retrofit)
+        - -ST1020 # godoc on exported types — not enforcing yet
+        - -ST1021 # godoc on exported type groups — not enforcing yet
+        - -ST1022 # godoc on exported vars/consts — not enforcing yet
+        - -SA1019 # deprecated usage (handle separately)
+        - -SA4006 # value overwritten — ineffassign + wastedassign cover this better
+        - -SA9005 # ineffective break (staticcheck false positives)
+
+    gosec:
+      excludes:
+        - G104 # unchecked errors — errcheck is stricter
+        - G116 # bidi chars — bidichk is more thorough
+        - G117 # use of net/http serve functions (we handle TLS separately)
+        - G703 # path traversal taint analysis — false positives on config file loading
+        - G705 # XSS taint analysis — false positives on internal error responses
+        - G706 # log injection taint analysis — false positives on structured slog usage
+
+    modernize:
+      disable:
+        - newexpr # don't replace aws.String() with new() — AWS SDK idiom
+        - stringsbuilder # don't replace small string concatenations with strings.Builder
+
+    nakedret:
+      max-func-lines: 0
+
+    nolintlint:
+      require-explanation: true
+      require-specific: true
+
+    reassign:
+      patterns:
+        - ".*"
+
+  exclusions:
+    presets:
+      - std-error-handling
+      - common-false-positives
+    rules:
+      - path: '_test\.go'
+        linters:
+          - bodyclose
+          - errcheck
+          - errchkjson
+          - gosec
+
+      # spinifex-specific: gosec exclusions
+      - path: "cmd/"
+        linters:
+          - errcheck # CLI glue code — viper/cobra/pterm errors not actionable
+          - gosec # matches current -exclude-dir=cmd
+
+      - text: "G204"
+        linters: [gosec] # subprocess execution — legitimate in VM orchestration
+      - text: "G304"
+        linters: [gosec] # file path from variable — necessary for config loading
+      - text: "G402"
+        linters: [gosec] # TLS config — we handle TLS ourselves

Makefile

@@ -36,12 +36,12 @@ endif
 # Quiet-mode filters (active when QUIET=1, set by preflight via recursive make)
 ifdef QUIET
   _Q     = @
-  _COVQ  = 2>&1 | grep -Ev '^\s*(ok|PASS|\?|=== RUN|--- PASS:)\s' | grep -v 'coverage: 0\.0%' || true
+  _COVQ  = 2>&1 | { grep -Ev '^\s*(ok|PASS|\?|=== RUN|--- PASS:)\s' | grep -v 'coverage: 0\.0%' || true; }; exit $${PIPESTATUS[0]}
   _RACEQ = 2>&1 | { grep -Ev '^\s*(ok|PASS|\?|=== RUN|--- PASS:)\s' || true; }; exit $${PIPESTATUS[0]}
   _SECQ  = >
 else
   _Q     =
-  _COVQ  = || true
+  _COVQ  =
   _RACEQ =
   _SECQ  = 2>&1 | tee
 endif
@@ -67,10 +67,10 @@ go_run:
 	@echo -e "\n....Running $(GO_PROJECT_NAME)...."
 	$(GOPATH)/bin/$(GO_PROJECT_NAME)
 
-# Preflight — runs the same checks as GitHub Actions (format + lint + security + tests).
+# Preflight — runs the same checks as GitHub Actions (lint + vuln + tests).
 # Use this before committing to catch CI failures locally.
 preflight:
-	@$(MAKE) --no-print-directory QUIET=1 check-format check-modernize vet security-check test-cover diff-coverage test-race
+	@$(MAKE) --no-print-directory QUIET=1 lint govulncheck test-cover diff-coverage test-race
 	@echo -e "\n ✅ Preflight passed — safe to commit."
 
 # Run unit tests
@@ -79,8 +79,6 @@ test:
 	LOG_IGNORE=1 go test -timeout 120s ./spinifex/...
 
 # Run unit tests with coverage profile
-# Note: go test may exit non-zero due to Go version mismatch in coverage instrumentation
-# for packages without test files. We check actual test results + coverage threshold instead.
 COVERPROFILE ?= coverage.out
 test-cover:
 	@echo -e "\n....Running tests with coverage for $(GO_PROJECT_NAME)...."
@@ -145,65 +143,33 @@ quickinstall: install-system install-go install-aws
 	@echo "   Please ensure /usr/local/go/bin is in your PATH."
 	@echo "   Installed: Go ($(GO_ARCH)), AWS CLI ($(AWS_ARCH)), QEMU ($(QEMU_PACKAGES))"
 
-# Format all Go files in place
-format:
-	gofmt -w .
+# Lint all Go code via golangci-lint (replaces check-format, vet, gosec, staticcheck)
+lint:
+	@echo "Running golangci-lint..."
+	$(_Q)golangci-lint run ./...
+	@echo "  golangci-lint ok"
 
-# Check that all Go files are formatted (CI-compatible, fails on diff)
-check-format:
-	@echo "Checking gofmt..."
-	@UNFORMATTED=$$(gofmt -l .); \
-	if [ -n "$$UNFORMATTED" ]; then \
-		echo "Files not formatted:"; \
-		echo "$$UNFORMATTED"; \
-		echo "Run 'make format' to fix."; \
-		exit 1; \
-	fi
-	@echo "  gofmt ok"
+# Auto-fix all linter issues that have fixers
+fix:
+	golangci-lint run --fix ./...
 
-# Go vet (fails on issues, matches CI)
-vet:
-	@echo "Running go vet..."
-	$(_Q)go vet ./...
-	@echo "  go vet ok"
-
-# Excluded: newexpr (replaces aws.String with new, not idiomatic for AWS SDK)
-# Excluded: stringsbuilder (replaces string += in loops with strings.Builder, not worth the complexity for small loops)
-GOFIX_EXCLUDE := -newexpr=false -stringsbuilder=false
-
-# Apply go fix modernizations
-modernize:
-	@echo "Applying go fix modernizations..."
-	go fix $(GOFIX_EXCLUDE) ./...
-	@echo "  go fix applied"
-
-# Check that code is modernized (CI-compatible, fails on diff)
-check-modernize:
-	@echo "Checking go fix modernizations..."
-	@DIFF=$$(go fix $(GOFIX_EXCLUDE) -diff ./...); \
-	if [ -n "$$DIFF" ]; then \
-		echo "$$DIFF"; \
-		echo "Run 'make modernize' to fix."; \
-		exit 1; \
-	fi
-	@echo "  go fix ok"
-
-# Security checks — each tool fails the build on findings (matches CI).
-# Reports are also saved to tests/ for review.
-security-check:
-	@echo -e "\n....Running security checks for $(GO_PROJECT_NAME)...."
-	$(_Q)set -o pipefail && go tool govulncheck ./... $(_SECQ) tests/govulncheck-report.txt $(if $(QUIET),|| { cat tests/govulncheck-report.txt; exit 1; })
+# Govulncheck — dependency vulnerability scanning (not covered by golangci-lint)
+govulncheck:
+	@echo "Running govulncheck..."
+	$(_Q)go tool govulncheck ./...
 	@echo "  govulncheck ok"
-	$(_Q)set -o pipefail && go tool gosec -quiet -exclude=G204,G304,G402,G117,G703,G705,G706 -exclude-generated -exclude-dir=cmd ./... $(_SECQ) tests/gosec-report.txt $(if $(QUIET),|| { cat tests/gosec-report.txt; exit 1; })
-	@echo "  gosec ok"
-	$(_Q)set -o pipefail && go tool staticcheck -checks="all,-ST1000,-ST1003,-ST1016,-ST1020,-ST1021,-ST1022,-SA1019,-SA9005" ./... $(_SECQ) tests/staticcheck-report.txt $(if $(QUIET),|| { cat tests/staticcheck-report.txt; exit 1; })
-	@echo "  staticcheck ok"
 
-# Build release tarballs for both architectures via Docker
-distro:
-	@echo "Building spinifex $(VERSION) distribution tarballs..."
+# Build release tarballs — use distro-ARCH for single arch, distro for both
+distro: distro-amd64 distro-arm64
+	@echo ""
+	@echo "Distribution tarballs:"
+	@ls -lh dist/*.tar.gz
+	@echo ""
+	@cat dist/*.sha256
+
+distro-amd64:
+	@echo "Building spinifex $(VERSION) linux/amd64..."
 	@mkdir -p dist/
-	@echo "Building linux/amd64..."
 	docker buildx build \
 		--platform linux/amd64 \
 		--build-arg VERSION=$(VERSION) \
@@ -212,7 +178,10 @@ distro:
 		../
 	tar -czf dist/spinifex-$(VERSION)-linux-amd64.tar.gz -C dist/amd64 .
 	sha256sum dist/spinifex-$(VERSION)-linux-amd64.tar.gz > dist/spinifex-$(VERSION)-linux-amd64.tar.gz.sha256
-	@echo "Building linux/arm64..."
+
+distro-arm64:
+	@echo "Building spinifex $(VERSION) linux/arm64..."
+	@mkdir -p dist/
 	docker buildx build \
 		--platform linux/arm64 \
 		--build-arg VERSION=$(VERSION) \
@@ -221,16 +190,11 @@ distro:
 		../
 	tar -czf dist/spinifex-$(VERSION)-linux-arm64.tar.gz -C dist/arm64 .
 	sha256sum dist/spinifex-$(VERSION)-linux-arm64.tar.gz > dist/spinifex-$(VERSION)-linux-arm64.tar.gz.sha256
-	@echo ""
-	@echo "Distribution tarballs:"
-	@ls -lh dist/*.tar.gz
-	@echo ""
-	@cat dist/*.sha256
 
 distro-clean:
 	rm -rf dist/
 
 .PHONY: build build-ui go_build go_run preflight test test-cover test-race diff-coverage bench run clean \
 	install-system install-go install-aws quickinstall \
-	format check-format modernize check-modernize vet security-check \
-	distro distro-clean
+	lint fix govulncheck \
+	distro distro-amd64 distro-arm64 distro-clean

cmd/spinifex/cmd/admin.go

@@ -238,11 +238,9 @@ func init() {
 	imagesImportCmd.Flags().String("arch", "", "Specified distro arch (e.g aarch64, arm64, x86_64)")
 	imagesImportCmd.Flags().String("platform", "Linux/UNIX", "Specified platform (e.g Linux/UNIX, Windows)")
 	imagesImportCmd.Flags().Bool("force", false, "Force command execution (overwrites existing files)")
-
 }
 
 func runimagesImportCmd(cmd *cobra.Command, args []string) {
-
 	var image utils.Images
 
 	var imageFile string
@@ -268,7 +266,8 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	imageDir := fmt.Sprintf("%s/images", baseDir)
 
 	if !admin.FileExists(imageDir) {
-		fmt.Fprintf(os.Stderr, "Image directory does not exist. Base path specified correctly? %s", imageDir)
+		fmt.Fprintf(os.Stderr, "Error: image directory does not exist: %s\n\n", imageDir)
+		fmt.Fprintf(os.Stderr, "Run 'spx admin init' first to initialize the Spinifex platform.\n")
 		os.Exit(1)
 	}
 
@@ -276,7 +275,6 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	imageName, _ := cmd.Flags().GetString("name")
 
 	if imageName != "" {
-
 		var exists bool
 		// Confirm the image exists
 		image, exists = utils.AvailableImages[imageName]
@@ -285,9 +283,7 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 			fmt.Fprintf(os.Stderr, "Image name not found in available images")
 			os.Exit(1)
 		}
-
 	} else {
-
 		imageFile, _ = cmd.Flags().GetString("file")
 
 		if imageFile == "" {
@@ -295,10 +291,8 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 			os.Exit(1)
 		}
 
-		//imageStat, err = os.Stat(imageFile)
-
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "File could not be found %s", err)
+		if _, err := os.Stat(imageFile); err != nil {
+			fmt.Fprintf(os.Stderr, "File could not be found: %s", err)
 			os.Exit(1)
 		}
 
@@ -306,7 +300,6 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		image.Version, _ = cmd.Flags().GetString("version")
 		image.Arch, _ = cmd.Flags().GetString("arch")
 		image.Platform, _ = cmd.Flags().GetString("platform")
-
 	}
 
 	if image.Distro == "" {
@@ -353,14 +346,12 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		if admin.FileExists(imageFile) && !forceCmd {
 			fmt.Printf("Image file already exists, skipping download, use --force to overwrite: %s\n", imageFile)
 		} else {
-
 			err := utils.DownloadFileWithProgress(image.URL, image.Name, imageFile, 0)
 
 			if err != nil {
 				fmt.Printf("Download failed: %v\n", err)
 				os.Exit(1)
 			}
-
 		}
 
 		// Update image file path for later extraction
@@ -477,12 +468,10 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	defer os.RemoveAll(tmpDir)
 
 	fmt.Printf("✅ Image import complete. Image-ID (AMI): %s\n", volumeId)
-
 }
 
 // List remote images available
 func runimagesListCmd(cmd *cobra.Command, args []string) {
-
 	//fmt.Println(availableImages)
 
 	tableData := pterm.TableData{
@@ -501,7 +490,6 @@ func runimagesListCmd(cmd *cobra.Command, args []string) {
 
 	// 3. Iterate in sorted order
 	for _, k := range keys {
-
 		img := utils.AvailableImages[k]
 
 		//for _, img := range utils.AvailableImages {
@@ -726,6 +714,9 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 
 		PredastoreNodeID: predastoreNodeID,
 		Services:         services,
+
+		OVNNBAddr: "tcp:127.0.0.1:6641",
+		OVNSBAddr: "tcp:127.0.0.1:6642",
 	}
 
 	// Generate config files
@@ -788,7 +779,6 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, natsToken, clusterName,
 	configDir, spxRoot, certPath, region, az, node, bindIP, clusterBind string,
 	port, expectedNodes int, formationTimeoutStr string, services []string) {
-
 	formationTimeout, err := time.ParseDuration(formationTimeoutStr)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "❌ Error: Invalid --formation-timeout: %v\n", err)
@@ -934,6 +924,10 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		PredastoreNodeID: predastoreNodeID,
 		Services:         services,
 		RemoteNodes:      buildRemoteNodes(allNodes, node),
+
+		// Init node runs ovn-central locally
+		OVNNBAddr: "tcp:127.0.0.1:6641",
+		OVNSBAddr: "tcp:127.0.0.1:6642",
 	}
 
 	// Generate config files
@@ -1007,6 +1001,13 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
+	// Extract leader IP for OVN NB/SB DB address (strip port from host:port)
+	leaderIP, _, err := net.SplitHostPort(leaderHost)
+	if err != nil {
+		// leaderHost might be an IP without port
+		leaderIP = leaderHost
+	}
+
 	// Validate IP address format
 	if net.ParseIP(bindIP) == nil {
 		fmt.Fprintf(os.Stderr, "❌ Error: Invalid IP address for --bind: %s\n", bindIP)
@@ -1246,6 +1247,10 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		PredastoreNodeID: predastoreNodeID,
 		Services:         services,
 		RemoteNodes:      buildRemoteNodes(statusResp.Nodes, node),
+
+		// Joining nodes connect to the init node's OVN NB/SB DB
+		OVNNBAddr: fmt.Sprintf("tcp:%s:6641", leaderIP),
+		OVNSBAddr: fmt.Sprintf("tcp:%s:6642", leaderIP),
 	}
 
 	// Generate config files
@@ -1530,6 +1535,7 @@ func writeBootstrapFilesWithAdmin(configDir string, masterKey []byte, accessKey,
 	}
 
 	bd := &handlers_iam.BootstrapData{
+		Version:         handlers_iam.BootstrapVersion,
 		AccessKeyID:     accessKey,
 		EncryptedSecret: encryptedSecret,
 		AccountID:       accountID,

cmd/spinifex/cmd/get.go

@@ -236,6 +236,7 @@ func runGetVMs(cmd *cobra.Command, args []string) {
 
 	type vmRow struct {
 		types.VMInfo
+
 		Node string
 		Host string
 	}

cmd/spinifex/cmd/root.go

@@ -102,7 +102,6 @@ func init() {
 	//viper.BindPFlag("nats.host", rootCmd.PersistentFlags().Lookup("nats-host"))
 	//viper.BindPFlag("nats.acl.token", rootCmd.PersistentFlags().Lookup("nats-token"))
 	//viper.BindPFlag("nats.sub.subject", rootCmd.PersistentFlags().Lookup("nats-subject"))
-
 }
 
 // initConfig reads in config file and ENV variables if set.

cmd/spinifex/cmd/service.go

@@ -761,7 +761,6 @@ var vpcdStatusCmd = &cobra.Command{
 }
 
 func init() {
-
 	viper.SetEnvPrefix("SPINIFEX") // Prefix for environment variables
 	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
 
@@ -945,5 +944,4 @@ func init() {
 	vpcdCmd.AddCommand(vpcdStartCmd)
 	vpcdCmd.AddCommand(vpcdStopCmd)
 	vpcdCmd.AddCommand(vpcdStatusCmd)
-
 }

cmd/spinifex/cmd/templates/spinifex.toml

@@ -39,8 +39,8 @@ token = "{{.NatsToken}}"
 shardwal = false
 
 [nodes.{{.Node}}.vpcd]
-ovn_nb_addr = "tcp:127.0.0.1:6641"
-ovn_sb_addr = "tcp:127.0.0.1:6642"
+ovn_nb_addr = "{{.OVNNBAddr}}"
+ovn_sb_addr = "{{.OVNSBAddr}}"
 
 [nodes.{{.Node}}.predastore]
 host = "{{.BindIP}}:8443"

go.mod

@@ -8,8 +8,8 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/kdomanski/iso9660 v0.4.0
 	github.com/klauspost/cpuid/v2 v2.3.0
-	github.com/mulgadc/predastore v1.0.3
-	github.com/mulgadc/viperblock v1.0.17
+	github.com/mulgadc/predastore v1.1.1
+	github.com/mulgadc/viperblock v1.1.1
 	github.com/nats-io/nats-server/v2 v2.12.4
 	github.com/nats-io/nats.go v1.49.0
 	github.com/ovn-kubernetes/libovsdb v0.8.1
@@ -25,21 +25,14 @@ require (
 	atomicgo.dev/cursor v0.2.0 // indirect
 	atomicgo.dev/keyboard v0.2.9 // indirect
 	atomicgo.dev/schedule v0.1.0 // indirect
-	cloud.google.com/go v0.123.0 // indirect
-	cloud.google.com/go/auth v0.18.2 // indirect
-	cloud.google.com/go/compute/metadata v0.9.0 // indirect
-	github.com/BurntSushi/toml v1.6.0 // indirect
-	github.com/anthropics/anthropic-sdk-go v1.26.0 // indirect
 	github.com/antithesishq/antithesis-sdk-go v0.6.0 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/buraksezer/consistent v0.10.0 // indirect
-	github.com/ccojocar/zxcvbn-go v1.0.4 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cenkalti/hub v1.0.2 // indirect
 	github.com/cenkalti/rpc2 v1.0.5 // indirect
-	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
 	github.com/containerd/console v1.0.5 // indirect
@@ -48,7 +41,6 @@ require (
 	github.com/dgraph-io/ristretto/v2 v2.4.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/fatih/color v1.18.0 // indirect
-	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.13 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
@@ -58,13 +50,8 @@ require (
 	github.com/go-playground/validator/v10 v10.30.1 // indirect
 	github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
 	github.com/google/flatbuffers v25.12.19+incompatible // indirect
-	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/go-tpm v0.9.8 // indirect
-	github.com/google/s2a-go v0.1.9 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.13 // indirect
-	github.com/googleapis/gax-go/v2 v2.17.0 // indirect
 	github.com/gookit/color v1.6.0 // indirect
-	github.com/gorilla/websocket v1.5.3 // indirect
 	github.com/hashicorp/go-hclog v1.6.3 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/go-metrics v0.5.4 // indirect
@@ -88,7 +75,6 @@ require (
 	github.com/nats-io/jwt/v2 v2.8.0 // indirect
 	github.com/nats-io/nkeys v0.4.15 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/openai/openai-go/v3 v3.24.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.23.2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
@@ -96,27 +82,20 @@ require (
 	github.com/prometheus/procfs v0.20.1 // indirect
 	github.com/quic-go/quic-go v0.59.0 // indirect
 	github.com/sagikazarmark/locafero v0.12.0 // indirect
-	github.com/securego/gosec/v2 v2.24.7 // indirect
 	github.com/spf13/afero v1.15.0 // indirect
 	github.com/spf13/cast v1.10.0 // indirect
 	github.com/spf13/pflag v1.0.10 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
-	github.com/tidwall/gjson v1.18.0 // indirect
-	github.com/tidwall/match v1.2.0 // indirect
-	github.com/tidwall/pretty v1.2.1 // indirect
-	github.com/tidwall/sjson v1.2.5 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	go.etcd.io/bbolt v1.4.3 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.66.0 // indirect
 	go.opentelemetry.io/otel v1.41.0 // indirect
 	go.opentelemetry.io/otel/metric v1.41.0 // indirect
 	go.opentelemetry.io/otel/trace v1.41.0 // indirect
-	go.uber.org/automaxprocs v1.6.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
 	golang.org/x/crypto v0.48.0 // indirect
-	golang.org/x/exp/typeparams v0.0.0-20260218203240-3dfff04db8fa // indirect
+	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
 	golang.org/x/mod v0.33.0 // indirect
 	golang.org/x/net v0.51.0 // indirect
 	golang.org/x/sync v0.19.0 // indirect
@@ -127,16 +106,8 @@ require (
 	golang.org/x/time v0.14.0 // indirect
 	golang.org/x/tools v0.42.0 // indirect
 	golang.org/x/vuln v1.1.4 // indirect
-	google.golang.org/genai v1.49.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect
-	google.golang.org/grpc v1.79.1 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	honnef.co/go/tools v0.7.0 // indirect
 )
 
-tool (
-	github.com/securego/gosec/v2/cmd/gosec
-	golang.org/x/vuln/cmd/govulncheck
-	honnef.co/go/tools/cmd/staticcheck
-)
+tool golang.org/x/vuln/cmd/govulncheck

go.sum

@@ -7,14 +7,6 @@ atomicgo.dev/keyboard v0.2.9/go.mod h1:BC4w9g00XkxH/f1HXhW2sXmJFOCWbKn9xrOunSFtE
 atomicgo.dev/schedule v0.1.0 h1:nTthAbhZS5YZmgYbb2+DH8uQIZcTlIrd4eYr3UQxEjs=
 atomicgo.dev/schedule v0.1.0/go.mod h1:xeUa3oAkiuHYh8bKiQBRojqAMq3PXXbJujjb0hw8pEU=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE=
-cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU=
-cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM=
-cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M=
-cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
-cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
-github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk=
-github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs=
 github.com/MarvinJWendt/testza v0.2.1/go.mod h1:God7bhG8n6uQxwdScay+gjm9/LnO4D3kkcZX4hv9Rp8=
@@ -25,17 +17,11 @@ github.com/MarvinJWendt/testza v0.3.0/go.mod h1:eFcL4I0idjtIx8P9C6KkAuLgATNKpX4/
 github.com/MarvinJWendt/testza v0.4.2/go.mod h1:mSdhXiKH8sg/gQehJ63bINcCKp7RtYewEjXsvsVUPbE=
 github.com/MarvinJWendt/testza v0.5.2 h1:53KDo64C1z/h/d/stCYCPY69bt/OSwjq5KpFNwi+zB4=
 github.com/MarvinJWendt/testza v0.5.2/go.mod h1:xu53QFE5sCdjtMCKk8YMQ2MnymimEctc4n3EjyIYvEY=
-github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
-github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
-github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/anthropics/anthropic-sdk-go v1.26.0 h1:oUTzFaUpAevfuELAP1sjL6CQJ9HHAfT7CoSYSac11PY=
-github.com/anthropics/anthropic-sdk-go v1.26.0/go.mod h1:qUKmaW+uuPB64iy1l+4kOSvaLqPXnHTTBKH6RVZ7q5Q=
 github.com/antithesishq/antithesis-sdk-go v0.6.0 h1:v/YViLhFYkZOEEof4AXjD5AgGnGM84YHF4RqEwp6I2g=
 github.com/antithesishq/antithesis-sdk-go v0.6.0/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
 github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
@@ -89,16 +75,12 @@ github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/buraksezer/consistent v0.10.0 h1:hqBgz1PvNLC5rkWcEBVAL9dFMBWz6I0VgUCW25rrZlU=
 github.com/buraksezer/consistent v0.10.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw=
-github.com/ccojocar/zxcvbn-go v1.0.4 h1:FWnCIRMXPj43ukfX000kvBZvV6raSxakYr1nzyNrUcc=
-github.com/ccojocar/zxcvbn-go v1.0.4/go.mod h1:3GxGX+rHmueTUMvm5ium7irpyjmm7ikxYFOSJB21Das=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/cenkalti/hub v1.0.2 h1:Nqv9TNaA9boeO2wQFW8o87BY3zKthtnzXmWGmJqhAV8=
 github.com/cenkalti/hub v1.0.2/go.mod h1:8LAFAZcCasb83vfxatMUnZHRoQcffho2ELpHb+kaTJU=
 github.com/cenkalti/rpc2 v1.0.5 h1:T6l4SS3ja3eaJfRyZrn7Oco/PSx/pr3YK5cjCgLVLTk=
 github.com/cenkalti/rpc2 v1.0.5/go.mod h1:2yfU5b86vOr16+iY1jN3MvT6Kxc9Nf8j5iZWwUf7iaw=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
-github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -119,15 +101,11 @@ github.com/dgraph-io/ristretto/v2 v2.4.0 h1:I/w09yLjhdcVD2QV192UJcq8dPBaAJb9pOuM
 github.com/dgraph-io/ristretto/v2 v2.4.0/go.mod h1:0KsrXtXvnv0EqnzyowllbVJB8yBonswa2lTCK2gGo9E=
 github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38=
 github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
-github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
-github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
 github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
 github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
-github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
-github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
@@ -156,8 +134,6 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91
 github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w=
 github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
-github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
 github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
 github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -171,8 +147,6 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
-github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs=
 github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786 h1:rcv+Ippz6RAtvaGgKxc+8FQIpxHgsF+HBzPyYL2cyVU=
@@ -187,26 +161,16 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX
 github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
 github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc=
-github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI=
 github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
-github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/enterprise-certificate-proxy v0.3.13 h1:hSPAhW3NX+7HNlTsmrvU0jL75cIzxFktheceg95Nq14=
-github.com/googleapis/enterprise-certificate-proxy v0.3.13/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
-github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc=
-github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
 github.com/gookit/assert v0.1.1 h1:lh3GcawXe/p+cU7ESTZ5Ui3Sm/x8JWpIis4/1aF0mY0=
 github.com/gookit/assert v0.1.1/go.mod h1:jS5bmIVQZTIwk42uXl4lyj4iaaxx32tqH16CFj0VX2E=
 github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ=
 github.com/gookit/color v1.5.0/go.mod h1:43aQb+Zerm/BWh2GnrgOQm7ffz7tvQXEKV6BFMl7wAo=
 github.com/gookit/color v1.6.0 h1:JjJXBTk1ETNyqyilJhkTXJYYigHG24TM9Xa2M1xAhRA=
 github.com/gookit/color v1.6.0/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs=
-github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
-github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
 github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
@@ -291,10 +255,10 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/mulgadc/predastore v1.0.3 h1:qkuOEUH9B98QzVByn5osKeXm7SSPgk+ardQ3MibPSok=
-github.com/mulgadc/predastore v1.0.3/go.mod h1:x3Z6Ts/PONpmVx/R6t/hkHDJ3+0xRieYS3BuuVaNgyA=
-github.com/mulgadc/viperblock v1.0.17 h1:bGRVlXgcqnmH+Xxon03W/ZsbXqu/EE5+/VG3J4E3EBU=
-github.com/mulgadc/viperblock v1.0.17/go.mod h1:w9V0DjUH5hMXbXsiLbDr9lpqzXBGhEiYB1BeV6TYBkw=
+github.com/mulgadc/predastore v1.1.1 h1:rqpvA/qmewzJBHT2r2N9gNuUaLUKJrONFAyVu2zsRbY=
+github.com/mulgadc/predastore v1.1.1/go.mod h1:WFls3cw+0g0wYBDyKNjEvTihJxuMjIWe2lVNWcYtY7A=
+github.com/mulgadc/viperblock v1.1.1 h1:Z+G6JtR03lhWUGp39iC6pMOjBzYQfLEjyk8c3jcqVCQ=
+github.com/mulgadc/viperblock v1.1.1/go.mod h1:wlLRcHc/28+deiWWXLw+mXEDcmTv9WK2V3SVKkpoyp0=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -309,12 +273,6 @@ github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=
 github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
-github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI=
-github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE=
-github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28=
-github.com/onsi/gomega v1.39.1/go.mod h1:hL6yVALoTOxeWudERyfppUcZXjMwIMLnuSfruD2lcfg=
-github.com/openai/openai-go/v3 v3.24.0 h1:08x6GnYiB+AAejTo6yzPY8RkZMJQ8NpreiOyM5QfyYU=
-github.com/openai/openai-go/v3 v3.24.0/go.mod h1:cdufnVK14cWcT9qA1rRtrXx4FTRsgbDPW7Ia7SS5cZo=
 github.com/ovn-kubernetes/libovsdb v0.8.1 h1:M2J8bcJt5mXCom0HqzfEtuHkT80CTSQRcYG7acT8gf4=
 github.com/ovn-kubernetes/libovsdb v0.8.1/go.mod h1:ZlnHLzagmLOSvyd9qfxBIZp6wOSOw0IsRsc+6lNUGbU=
 github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
@@ -326,8 +284,6 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
-github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
@@ -370,17 +326,11 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4=
 github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI=
-github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
-github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
-github.com/securego/gosec/v2 v2.24.7 h1:3k5yJnrhT1TTdsG0ZsnenlfCcT+7Y/+zeCPHbL7QAn8=
-github.com/securego/gosec/v2 v2.24.7/go.mod h1:AdDJbjcG/XxFgVv7pW19vMNYlFM6+Q6Qy3t6lWAUcEY=
 github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
 github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
 github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
 github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
@@ -396,7 +346,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
 github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -411,17 +360,6 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
-github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
-github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
-github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM=
-github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
-github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
-github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
-github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
-github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
-github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
@@ -431,20 +369,12 @@ go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
 go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.66.0 h1:PnV4kVnw0zOmwwFkAzCN5O07fw1YOIQor120zrh0AVo=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.66.0/go.mod h1:ofAwF4uinaf8SXdVzzbL4OsxJ3VfeEg3f/F6CeF49/Y=
 go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
 go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
 go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
 go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
-go.opentelemetry.io/otel/sdk v1.41.0 h1:YPIEXKmiAwkGl3Gu1huk1aYWwtpRLeskpV+wPisxBp8=
-go.opentelemetry.io/otel/sdk v1.41.0/go.mod h1:ahFdU0G5y8IxglBf0QBJXgSe7agzjE4GiTJ6HT9ud90=
-go.opentelemetry.io/otel/sdk/metric v1.41.0 h1:siZQIYBAUd1rlIWQT2uCxWJxcCO7q3TriaMlf08rXw8=
-go.opentelemetry.io/otel/sdk/metric v1.41.0/go.mod h1:HNBuSvT7ROaGtGI50ArdRLUnvRTRGniSUZbxiWxSO8Y=
 go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
 go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
-go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
-go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko=
@@ -461,8 +391,6 @@ golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
 golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
-golang.org/x/exp/typeparams v0.0.0-20260218203240-3dfff04db8fa h1:6Wi43P0isP8Nl8D4qJmA3VC4FswVH0RJkr5cauo67SQ=
-golang.org/x/exp/typeparams v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:PqrXSW65cXDZH0k4IeUbhmg/bcAZDbzNz3byBpKCsXo=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
@@ -552,15 +480,7 @@ golang.org/x/vuln v1.1.4 h1:Ju8QsuyhX3Hk8ma3CesTbO8vfJD9EvUBgHvkxHBzj0I=
 golang.org/x/vuln v1.1.4/go.mod h1:F+45wmU18ym/ca5PLTPLsSzr2KppzswxPP603ldA67s=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
-gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/genai v1.49.0 h1:Se+QJaH2GYK1aaR1o5S38mlU2GD5FnVvP76nfkV7LH0=
-google.golang.org/genai v1.49.0/go.mod h1:A3kkl0nyBjyFlNjgxIwKq70julKbIxpSxqKO5gw/gmk=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
-google.golang.org/grpc v1.79.1 h1:zGhSi45ODB9/p3VAawt9a+O/MULLl9dpizzNNpq7flY=
-google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -589,5 +509,3 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-honnef.co/go/tools v0.7.0 h1:w6WUp1VbkqPEgLz4rkBzH/CSU6HkoqNLp6GstyTx3lU=
-honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc=

scripts/dev-setup.sh

@@ -3,7 +3,7 @@
 # Spinifex Platform Development Environment Setup
 # This script sets up a complete development environment for the Spinifex platform
 
-set -e
+set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
@@ -13,17 +13,9 @@ MULGA_ROOT="$(cd "$PROJECT_ROOT/.." && pwd)"
 NATS_PORT=4222
 PREDASTORE_PORT=8443
 SPINIFEX_GATEWAY_PORT=9999
-DATA_DIR="$HOME/spinifex"
-# Use CONFIG_DIR environment variable if set, otherwise default to ~/spinifex/config
-CONFIG_DIR="${CONFIG_DIR:-$HOME/spinifex/config}"
 
 echo "🚀 Setting up Spinifex development environment..."
 echo "Project root: $PROJECT_ROOT"
-echo "Data directory: $DATA_DIR"
-
-# Create necessary directories
-mkdir -p "$DATA_DIR"/{nats,predastore,viperblock,logs,spinifex}
-mkdir -p "$CONFIG_DIR"
 
 # Function to check if command exists
 command_exists() {
@@ -40,11 +32,13 @@ echo "🔍 Checking dependencies..."
 
 # Check for required commands
 required_commands=("go" "make")
+missing_required=false
 for cmd in "${required_commands[@]}"; do
     if command_exists "$cmd"; then
         echo "✅ $cmd found"
     else
         echo "❌ $cmd not found. Please install $cmd"
+        missing_required=true
         case "$cmd" in
             "go")
                 echo "   Install Go: https://golang.org/dl/"
@@ -52,6 +46,10 @@ for cmd in "${required_commands[@]}"; do
         esac
     fi
 done
+if [ "$missing_required" = true ]; then
+    echo "❌ Missing required dependencies. Cannot continue."
+    exit 1
+fi
 
 # Check optional commands
 optional_commands=("air" "nbdkit")
@@ -142,19 +140,6 @@ cd "$PROJECT_ROOT"
 make build
 echo "✅ Spinifex built successfully"
 
-# Initialize Spinifex configuration using admin init
-#echo ""
-#echo "🔐 Initializing Spinifex configuration..."
-
-#if [[ ! -f "$CONFIG_DIR/spinifex.toml" ]]; then
-#    echo "📋 Running spx admin init..."
-#    ./bin/spx admin init --config-dir "$CONFIG_DIR"
-#    echo "✅ Spinifex configuration initialized"
-#else
-#    echo "✅ Spinifex configuration already exists"
-#    echo "   To re-initialize, run: ./bin/spx admin init --force"
-#fi
-
 # Build components
 echo ""
 echo "🔨 Building components..."
@@ -182,23 +167,26 @@ fi
 echo ""
 echo "🎉 Development environment setup complete!"
 echo ""
-echo "When running Spinifex for the first time, run the init function to create the"
-echo "default directories for data, config files and layout required."
-echo "./bin/spx admin init"
+echo "📋 Next steps:"
 echo ""
-echo "🚀 To start the development environment:"
-echo "   ./scripts/start-dev.sh"
+echo "   1. Setup OVN networking:"
+echo "      ./scripts/setup-ovn.sh --management"
 echo ""
-echo "🛑 To stop the development environment:"
-echo "   ./scripts/stop-dev.sh"
+echo "   2. Initialize Spinifex (creates ~/spinifex/ directories, config, and certs):"
+echo "      ./bin/spx admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1"
+echo ""
+echo "   3. Trust the CA certificate:"
+echo "      sudo cp ~/spinifex/config/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt"
+echo "      sudo update-ca-certificates"
+echo ""
+echo "   4. Start services:"
+echo "      ./scripts/start-dev.sh"
 echo ""
 echo "🔧 Development endpoints:"
 echo "   - Spinifex Gateway:  https://localhost:$SPINIFEX_GATEWAY_PORT"
 echo "   - Predastore S3: https://localhost:$PREDASTORE_PORT"
 echo "   - NATS:          nats://localhost:$NATS_PORT"
 echo ""
-echo "📊 Monitor logs:"
-echo "   tail -f $DATA_DIR/logs/*.log"
-echo ""
 echo "🧪 Test with AWS CLI:"
-echo "   aws --endpoint-url https://localhost:$SPINIFEX_GATEWAY_PORT --no-verify-ssl ec2 describe-instances"
+echo "   export AWS_PROFILE=spinifex"
+echo "   aws ec2 describe-instances"

scripts/diff-coverage.sh

@@ -13,7 +13,7 @@ set -euo pipefail
 
 PROFILE=""
 BASE_REF=""
-THRESHOLD=65
+THRESHOLD=60
 QUIET="${QUIET:-}"
 
 while [[ $# -gt 0 ]]; do

scripts/reset-dev-env.sh

@@ -1,10 +1,15 @@
-#!/bin/sh
+#!/bin/bash
 
 # Reset dev environment (will purge all data)
 
+set -euo pipefail
+
 echo "Shutting down services"
 
-./scripts/stop-dev.sh
+if ! ./scripts/stop-dev.sh; then
+    echo "❌ Failed to stop services. Aborting reset to prevent data loss."
+    exit 1
+fi
 
 echo "Removing data"
 

scripts/start-dev.sh

@@ -9,7 +9,7 @@
 # Environment variables:
 #   UI=false              Skip starting Spinifex UI (e.g., UI=false ./scripts/start-dev.sh)
 
-set -e
+set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
@@ -114,8 +114,14 @@ start_service() {
 
     echo "   PID: $pid, Log: $logfile"
 
-    # Brief pause to let service start
+    # Brief pause to let service start, then verify it's still alive
     sleep 1
+    if ! kill -0 "$pid" 2>/dev/null; then
+        echo "   ❌ $name failed to start (exited immediately)"
+        echo "   Check log: $logfile"
+        tail -5 "$logfile" 2>/dev/null || true
+        exit 1
+    fi
 }
 
 # Function to start service in foreground (for final daemon)
@@ -302,7 +308,7 @@ if has_service "predastore"; then
     export SPINIFEX_PREDASTORE_BACKEND=distributed
 
     # Auto-detect Predastore NODE_ID from spinifex.toml if not already set
-    if [ -z "$SPINIFEX_PREDASTORE_NODE_ID" ]; then
+    if [ -z "${SPINIFEX_PREDASTORE_NODE_ID:-}" ]; then
         if [ -f "$CONFIG_DIR/spinifex.toml" ]; then
             DETECTED_NODE_ID=$(awk -F'= *' '/node_id/{gsub(/ /,"",$2); print $2; exit}' "$CONFIG_DIR/spinifex.toml")
             if [ -n "$DETECTED_NODE_ID" ] && [ "$DETECTED_NODE_ID" != "0" ]; then
@@ -407,7 +413,7 @@ else
 fi
 
 # 7️⃣ Start Spinifex UI (skip with UI=false or if not a local service)
-if [ "${UI}" != "false" ] && has_service "ui"; then
+if [ "${UI:-}" != "false" ] && has_service "ui"; then
     echo ""
     echo "7️⃣. Starting Spinifex UI..."
 
@@ -422,7 +428,7 @@ fi
 
 echo ""
 echo "🔗 Service endpoints will be:"
-if [ "${UI}" != "false" ]; then
+if [ "${UI:-}" != "false" ]; then
     echo "   - Spinifex UI:       https://localhost:3000"
 fi
 echo "   - NATS:          nats://localhost:4222"

scripts/start-multi-node.sh

@@ -1,12 +1,12 @@
-#!/bin/sh
+#!/bin/bash
+
+set -euo pipefail
 
 # Start node1
 ./scripts/start-dev.sh ~/node1/
 
-
 # Start node2
 ./scripts/start-dev.sh ~/node2/
 
-
 # Start node3
 ./scripts/start-dev.sh ~/node3/

scripts/stop-dev.sh

@@ -5,6 +5,8 @@
 # Usage: ./scripts/stop-dev.sh
 # Note: Services are stopped using PID files, so data-dir is not required
 
+set -uo pipefail
+
 # Accept optional data directory argument
 DATA_DIR="${1:-$HOME/spinifex}"
 
@@ -65,37 +67,41 @@ is_multinode() {
 # For multi-node clusters, delegate to coordinated shutdown via NATS.
 # Only when called without arguments (default path). When a data-dir is
 # explicitly provided (e.g., E2E per-node cleanup), use per-service stop.
-if is_multinode && [ -z "$SPINIFEX_FORCE_LOCAL_STOP" ] && [ -z "$1" ]; then
+if is_multinode && [ -z "${SPINIFEX_FORCE_LOCAL_STOP:-}" ] && [ -z "${1:-}" ]; then
     echo "Multi-node cluster detected. Using coordinated shutdown..."
     echo "  (Set SPINIFEX_FORCE_LOCAL_STOP=1 to force per-service stop)"
     exec $PROJECT_ROOT/bin/spx admin cluster shutdown
 fi
 
-# Function to stop service by PID file
+# Function to stop a service. Sends SIGTERM and waits for the process to exit
+# so resources (ports, DB locks) are fully released before returning.
 stop_service() {
     local name="$1"
     local pidpath="$2"
 
-    echo "🛑 Stopping $name $pidpath..."
+    echo "🛑 Stopping $name..."
 
-    local rc=0
-    # Workaround for local development for multi-node config on single instance
-    if [ -n "$pidpath" ]; then
-        kill -SIGTERM $(cat "$pidpath/$name.pid" 2>/dev/null) 2>/dev/null || rc=$?
+    local pid
+    pid=$(cat "$pidpath/$name.pid" 2>/dev/null) || { echo "⚠️  No PID file for $name"; return 1; }
+
+    kill -SIGTERM "$pid" 2>/dev/null || { rm -f "$pidpath/$name.pid"; echo "✅ $name already stopped"; echo ""; return 0; }
+
+    # Wait for process to fully exit (up to 15s) so ports/locks are released
+    local checks=0
+    while kill -0 "$pid" 2>/dev/null; do
         sleep 1
-    else
-    # Correct graceful shutdown via spx binary, waits for clean exit
-        $PROJECT_ROOT/bin/spx service $name stop || rc=$?
-    fi
-
-    if [[ $rc -ne 0 ]]; then
-        echo "⚠️  Failed to stop $name (exit code $rc)"
-        return 1
-    else
-        echo "✅ $name stopped"
-        echo ""
-    fi
+        checks=$((checks + 1))
+        if [ "$checks" -ge 15 ]; then
+            echo "  Force killing $name..."
+            kill -9 "$pid" 2>/dev/null || true
+            sleep 1
+            break
+        fi
+    done
 
+    rm -f "$pidpath/$name.pid"
+    echo "✅ $name stopped"
+    echo ""
 }
 
 # Stop services in reverse order

spinifex/admin/admin.go

@@ -59,6 +59,11 @@ type ConfigSettings struct {
 	// Node capabilities
 	Services []string
 
+	// OVN Northbound DB address — "tcp:127.0.0.1:6641" on the primary node,
+	// "tcp:<primary-mgmt-ip>:6641" on joining nodes.
+	OVNNBAddr string
+	OVNSBAddr string
+
 	// Other nodes in the cluster (for config source of truth)
 	RemoteNodes []RemoteNode
 }
@@ -76,7 +81,6 @@ type ConfigFile struct {
 }
 
 func GenerateConfigFiles(configs []ConfigFile, configSettings ConfigSettings) error {
-
 	for _, cfg := range configs {
 		if err := GenerateConfigFile(cfg.Path, cfg.Template, configSettings); err != nil {
 			return fmt.Errorf("error creating %s: %v", cfg.Name, err)
@@ -173,7 +177,6 @@ func GenerateServerCertOnly(configDir string, bindIP string) error {
 }
 
 func CreateServiceDirectories(spxRoot string) {
-
 	// Create additional directories
 	dirs := []string{
 		filepath.Join(spxRoot, "images"),
@@ -189,7 +192,6 @@ func CreateServiceDirectories(spxRoot string) {
 
 	fmt.Println("\n📁 Creating directory structure...")
 	for _, dir := range dirs {
-
 		// Check if directory exists
 		if _, err := os.Stat(dir); os.IsNotExist(err) {
 			if err := os.MkdirAll(dir, 0750); err != nil {
@@ -198,7 +200,6 @@ func CreateServiceDirectories(spxRoot string) {
 		}
 	}
 	fmt.Printf("✅ Directory structure created in %s\n", spxRoot)
-
 }
 
 // Helper functions
@@ -234,7 +235,8 @@ func ChownRecursive(path, username string) {
 
 	_ = fs.WalkDir(root.FS(), ".", func(p string, d fs.DirEntry, err error) error {
 		if err != nil {
-			return nil
+			slog.Debug("chown walk: skipping inaccessible entry", "path", p, "err", err)
+			return nil // best-effort: continue past inaccessible entries
 		}
 		fullPath := filepath.Join(path, p)
 		if chownErr := os.Lchown(fullPath, uid, gid); chownErr != nil {
@@ -586,12 +588,22 @@ func GenerateSelfSignedCert(certPath, keyPath string) error {
 
 // SetupAWSCredentials updates ~/.aws/credentials and ~/.aws/config.
 // bindIP is the IP the AWS gateway listens on. If empty or "0.0.0.0", defaults to "localhost".
+// When running under sudo, writes to SUDO_USER's home instead of root's.
 func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP string) error {
 	homeDir, err := os.UserHomeDir()
 	if err != nil {
 		return err
 	}
 
+	// When running under sudo, write to the invoking user's home directory
+	// so the operator can use AWS_PROFILE=spinifex without sudo.
+	sudoUser := os.Getenv("SUDO_USER")
+	if os.Getuid() == 0 && sudoUser != "" {
+		if u, err := user.Lookup(sudoUser); err == nil {
+			homeDir = u.HomeDir
+		}
+	}
+
 	awsDir := filepath.Join(homeDir, ".aws")
 	if err := os.MkdirAll(awsDir, 0700); err != nil {
 		return err
@@ -642,6 +654,11 @@ func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP string)
 		return err
 	}
 
+	// Fix ownership so the sudo invoking user can read the files
+	if os.Getuid() == 0 && sudoUser != "" {
+		ChownRecursive(awsDir, sudoUser)
+	}
+
 	fmt.Printf("   Profile: %s\n", profileName)
 	if profileName != "default" {
 		fmt.Printf("   Use: export AWS_PROFILE=%s\n", profileName)

spinifex/config/config.go

@@ -168,7 +168,6 @@ func LoadConfig(configPath string) (*ClusterConfig, error) {
 				return nil, fmt.Errorf("error reading config file: %w", err)
 			}
 			//fmt.Fprintf(os.Stderr, "Using config file: %s\n", viper.ConfigFileUsed())
-
 		} else {
 			fmt.Fprintf(os.Stderr, "Config file not found: %s, using environment variables and defaults\n", configPath)
 		}

spinifex/daemon/daemon.go

@@ -188,16 +188,17 @@ func getSystemMemory() (float64, error) {
 	}
 }
 
-// NewResourceManager creates a new resource manager with system capabilities
-func NewResourceManager() *ResourceManager {
+// NewResourceManager creates a new resource manager with system capabilities.
+// Returns an error if system memory cannot be detected, since an incorrect
+// default would either under-provision (large servers) or over-commit (small devices).
+func NewResourceManager() (*ResourceManager, error) {
 	// Get system CPU cores
 	numCPU := runtime.NumCPU()
 
 	// Get system memory (in GB)
 	totalMemGB, err := getSystemMemory()
 	if err != nil {
-		slog.Warn("Failed to get system memory, using default of 8GB", "err", err)
-		totalMemGB = 8.0 // Default to 8GB if we can't get the actual memory
+		return nil, fmt.Errorf("detect system memory: %w", err)
 	}
 
 	// Determine architecture
@@ -217,7 +218,7 @@ func NewResourceManager() *ResourceManager {
 		availableVCPU: numCPU,
 		availableMem:  totalMemGB,
 		instanceTypes: instanceTypes,
-	}
+	}, nil
 }
 
 // instanceTypeVCPUs returns the default vCPU count for an instance type, or 0 if unavailable.
@@ -301,13 +302,13 @@ func (rm *ResourceManager) GetResourceStats() (totalVCPU int, totalMemGB float64
 	remainingMem := rm.availableMem - rm.allocatedMem
 
 	for _, it := range rm.instanceTypes {
-		cap := resourceStatsForType(remainingVCPU, remainingMem, it)
-		if cap.VCPU == 0 || cap.MemoryGB == 0 {
+		typeCap := resourceStatsForType(remainingVCPU, remainingMem, it)
+		if typeCap.VCPU == 0 || typeCap.MemoryGB == 0 {
 			continue
 		}
-		caps = append(caps, cap)
+		caps = append(caps, typeCap)
 	}
-	return
+	return totalVCPU, totalMemGB, allocVCPU, allocMemGB, caps
 }
 
 // SetConfigPath sets the configuration file path for cluster management
@@ -316,7 +317,7 @@ func (d *Daemon) SetConfigPath(path string) {
 }
 
 // NewDaemon creates a new daemon instance
-func NewDaemon(cfg *config.ClusterConfig) *Daemon {
+func NewDaemon(cfg *config.ClusterConfig) (*Daemon, error) {
 	ctx, cancel := context.WithCancel(context.Background())
 
 	// If WalDir is not set, use BaseDir
@@ -327,18 +328,24 @@ func NewDaemon(cfg *config.ClusterConfig) *Daemon {
 		cfg.Nodes[cfg.Node] = config
 	}
 
+	rm, err := NewResourceManager()
+	if err != nil {
+		cancel()
+		return nil, fmt.Errorf("initialize resource manager: %w", err)
+	}
+
 	return &Daemon{
 		node:              cfg.Node,
 		clusterConfig:     cfg,
 		config:            &config,
-		resourceMgr:       NewResourceManager(),
+		resourceMgr:       rm,
 		ctx:               ctx,
 		cancel:            cancel,
 		Instances:         vm.Instances{VMS: make(map[string]*vm.VM)},
 		natsSubscriptions: make(map[string]*nats.Subscription),
 		startTime:         time.Now(),
 		detachDelay:       1 * time.Second,
-	}
+	}, nil
 }
 
 // natsSub defines a single NATS subscription entry for the table-driven setup.
@@ -845,17 +852,57 @@ func (d *Daemon) restoreInstances() {
 		if ok {
 			slog.Info("Re-allocating resources for instance", "instanceId", instance.ID, "type", instance.InstanceType)
 			if err := d.resourceMgr.allocate(instanceType); err != nil {
-				slog.Error("Failed to re-allocate resources for instance on startup", "instanceId", instance.ID, "err", err)
+				slog.Error("Failed to re-allocate resources for instance on startup, moving to stopped",
+					"instanceId", instance.ID, "err", err)
+				instance.Status = vm.StateStopped
+				if instance.Instance != nil {
+					instance.Instance.StateReason = &ec2.StateReason{}
+					instance.Instance.StateReason.SetCode("Server.InsufficientInstanceCapacity")
+					instance.Instance.StateReason.SetMessage(
+						fmt.Sprintf("insufficient resources to restore instance: %v", err))
+				}
+				d.migrateStoppedToSharedKV(instance)
+				continue
 			}
 		}
 
 		// Check if QEMU process is still alive from before the restart
 		if d.isInstanceProcessRunning(instance) {
-			slog.Info("Instance QEMU process still alive, reconnecting", "instance", instance.ID)
-			if err := d.reconnectInstance(instance); err != nil {
-				slog.Error("Failed to reconnect to running instance", "instanceId", instance.ID, "err", err)
+			// Verify NBD sockets are still valid. After a viperblock restart,
+			// old sockets are gone and QEMU's block devices are broken. Kill
+			// the orphaned QEMU and relaunch from scratch instead of
+			// reconnecting to an instance with dead storage.
+			if !d.areVolumeSocketsValid(instance) {
+				slog.Warn("QEMU alive but NBD sockets are stale, killing orphaned process for relaunch",
+					"instance", instance.ID)
+				pid, pidErr := utils.ReadPidFile(instance.ID)
+				if pidErr != nil || pid <= 0 {
+					slog.Error("Cannot read PID for orphaned QEMU, skipping relaunch",
+						"instanceId", instance.ID, "err", pidErr)
+					continue
+				}
+				// SIGKILL directly — orphaned QEMU with dead storage has no
+				// state worth a graceful shutdown, and KillProcess's 120s
+				// SIGTERM timeout would block daemon startup.
+				if proc, err := os.FindProcess(pid); err == nil {
+					_ = proc.Signal(syscall.SIGKILL)
+				}
+				// Wait for the process to actually die, then remove the PID
+				// file ourselves. SIGKILL cannot be caught, so QEMU never
+				// runs its cleanup handler and the PID file stays on disk.
+				if err := utils.WaitForProcessExit(pid, 10*time.Second); err != nil {
+					slog.Error("Orphaned QEMU did not exit after SIGKILL, skipping relaunch",
+						"instanceId", instance.ID, "pid", pid, "err", err)
+					continue
+				}
+				_ = utils.RemovePidFile(instance.ID)
+			} else {
+				slog.Info("Instance QEMU process still alive, reconnecting", "instance", instance.ID)
+				if err := d.reconnectInstance(instance); err != nil {
+					slog.Error("Failed to reconnect to running instance", "instanceId", instance.ID, "err", err)
+				}
+				continue
 			}
-			continue
 		}
 
 		// QEMU is not running -- resolve transitional states from interrupted operations
@@ -887,21 +934,36 @@ func (d *Daemon) restoreInstances() {
 		case vm.StateRunning:
 			// Was running but QEMU died - reset to pending so LaunchInstance can transition to running
 			instance.Status = vm.StatePending
-			// Reset LaunchTime so the pending watchdog gives a fresh timeout window.
-			// Without this, the stale LaunchTime from the original launch causes the
-			// watchdog to immediately mark the instance as failed.
-			now := time.Now()
-			if instance.Instance != nil {
-				instance.Instance.LaunchTime = &now
-			}
 			slog.Info("Instance was running but QEMU exited, relaunching", "instance", instance.ID)
 		}
 
+		// Reset LaunchTime so the pending watchdog gives a fresh timeout window.
+		// Without this, the stale LaunchTime from the original launch causes the
+		// watchdog to immediately mark the instance as failed after a prolonged outage.
+		now := time.Now()
+		if instance.Instance != nil {
+			instance.Instance.LaunchTime = &now
+		}
 		toLaunch = append(toLaunch, instance)
 	}
 
 	// Phase 2: Relaunch crashed VMs with semaphore-based throttling
 	if len(toLaunch) > 0 {
+		// Subscribe to per-instance NATS topics before launching so that
+		// terminate/stop commands can reach this daemon while instances are
+		// still being relaunched. Without this, pending instances are
+		// unreachable via ec2.cmd.<id> and TerminateInstances fails.
+		d.mu.Lock()
+		for _, instance := range toLaunch {
+			sub, subErr := d.natsConn.Subscribe(fmt.Sprintf("ec2.cmd.%s", instance.ID), d.handleEC2Events)
+			if subErr != nil {
+				slog.Error("Failed to early-subscribe during recovery", "instanceId", instance.ID, "err", subErr)
+			} else {
+				d.natsSubscriptions[instance.ID] = sub
+			}
+		}
+		d.mu.Unlock()
+
 		slog.Info("Launching instances (recovery)", "count", len(toLaunch), "maxConcurrent", maxConcurrentRecovery)
 		sem := make(chan struct{}, maxConcurrentRecovery)
 		var wg sync.WaitGroup
@@ -917,9 +979,19 @@ func (d *Daemon) restoreInstances() {
 						slog.Error("Panic during instance recovery", "instanceId", inst.ID, "panic", r, "stack", string(debug.Stack()))
 					}
 				}()
+				// Skip if instance was terminated while waiting for semaphore
+				d.Instances.Mu.Lock()
+				status := inst.Status
+				d.Instances.Mu.Unlock()
+				if status != vm.StatePending && status != vm.StateProvisioning {
+					slog.Info("Instance state changed during recovery, skipping launch",
+						"instanceId", inst.ID, "status", string(status))
+					return
+				}
 				slog.Info("Launching instance (recovery)", "instance", inst.ID)
 				if err := d.LaunchInstance(inst); err != nil {
-					slog.Error("Failed to launch instance", "instanceId", inst.ID, "err", err)
+					slog.Error("Failed to launch instance during recovery", "instanceId", inst.ID, "err", err)
+					d.markInstanceFailed(inst, "recovery_launch_failed")
 				}
 			}(instance)
 		}
@@ -945,6 +1017,32 @@ func (d *Daemon) isInstanceProcessRunning(instance *vm.VM) bool {
 	return process.Signal(syscall.Signal(0)) == nil
 }
 
+// areVolumeSocketsValid checks whether the NBD Unix sockets backing an
+// instance's volumes are reachable. A dial probe (not just os.Stat) is used
+// because viperblock may restart with sockets at the same paths — the file
+// exists but no process is listening on the old fd that QEMU holds.
+func (d *Daemon) areVolumeSocketsValid(instance *vm.VM) bool {
+	instance.EBSRequests.Mu.Lock()
+	defer instance.EBSRequests.Mu.Unlock()
+
+	for _, req := range instance.EBSRequests.Requests {
+		if req.NBDURI == "" {
+			continue
+		}
+		serverType, sockPath, _, _, err := utils.ParseNBDURI(req.NBDURI)
+		if err != nil || serverType != "unix" {
+			continue // TCP or unparseable — can't validate locally
+		}
+		conn, err := net.DialTimeout("unix", sockPath, 2*time.Second)
+		if err != nil {
+			slog.Debug("NBD socket unreachable", "volume", req.Name, "socket", sockPath, "err", err)
+			return false
+		}
+		_ = conn.Close()
+	}
+	return true
+}
+
 // reconnectInstance re-establishes QMP and NATS connections to a running QEMU instance
 // after a daemon restart. This bypasses the state machine since recovery is not a
 // normal state transition.
@@ -1033,7 +1131,6 @@ func (d *Daemon) saveClusterConfig() error {
 
 // ClusterManager starts the HTTP cluster management server
 func (d *Daemon) ClusterManager() error {
-
 	// Get daemon host from config
 	daemonHost := d.config.Daemon.Host
 	if daemonHost == "" {
@@ -1047,7 +1144,7 @@ func (d *Daemon) ClusterManager() error {
 		configHash, err := d.computeConfigHash()
 		if err != nil {
 			w.Header().Set("Content-Type", "application/json")
-			w.WriteHeader(500)
+			w.WriteHeader(http.StatusInternalServerError)
 			_, _ = w.Write([]byte(`{"error":"failed to compute config hash"}`))
 			return
 		}
@@ -1104,7 +1201,7 @@ func (d *Daemon) ClusterManager() error {
 		var req types.NodeJoinRequest
 		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
 			w.Header().Set("Content-Type", "application/json")
-			w.WriteHeader(400)
+			w.WriteHeader(http.StatusBadRequest)
 			if err := json.NewEncoder(w).Encode(types.NodeJoinResponse{
 				Success: false,
 				Message: "invalid request body",
@@ -1119,7 +1216,7 @@ func (d *Daemon) ClusterManager() error {
 		// Validate request
 		if req.Node == "" || req.Region == "" || req.AZ == "" {
 			w.Header().Set("Content-Type", "application/json")
-			w.WriteHeader(400)
+			w.WriteHeader(http.StatusBadRequest)
 			if err := json.NewEncoder(w).Encode(types.NodeJoinResponse{
 				Success: false,
 				Message: "node, region, and az are required",
@@ -1132,7 +1229,7 @@ func (d *Daemon) ClusterManager() error {
 		// Check if node already exists
 		if _, exists := d.clusterConfig.Nodes[req.Node]; exists {
 			w.Header().Set("Content-Type", "application/json")
-			w.WriteHeader(409)
+			w.WriteHeader(http.StatusConflict)
 			if err := json.NewEncoder(w).Encode(types.NodeJoinResponse{
 				Success: false,
 				Message: fmt.Sprintf("node %s already exists in cluster", req.Node),
@@ -1167,7 +1264,7 @@ func (d *Daemon) ClusterManager() error {
 		if err := d.saveClusterConfig(); err != nil {
 			slog.Error("Failed to save cluster config", "error", err)
 			w.Header().Set("Content-Type", "application/json")
-			w.WriteHeader(500)
+			w.WriteHeader(http.StatusInternalServerError)
 			if err := json.NewEncoder(w).Encode(types.NodeJoinResponse{
 				Success: false,
 				Message: "failed to save cluster config",
@@ -1253,16 +1350,23 @@ func (d *Daemon) ClusterManager() error {
 		}
 	})
 
-	// Start HTTP server in goroutine
+	// Start HTTP server in goroutine — use a channel to propagate bind errors
+	// back to the caller so Start() fails visibly instead of silently.
 	d.clusterServer = &http.Server{
 		Addr:              daemonHost,
 		Handler:           r,
 		ReadHeaderTimeout: 10 * time.Second,
 	}
+
+	ln, err := net.Listen("tcp", daemonHost)
+	if err != nil {
+		return fmt.Errorf("cluster manager listen on %s: %w", daemonHost, err)
+	}
+
 	go func() {
 		slog.Info("Starting cluster manager", "host", daemonHost)
-		if err := d.clusterServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
-			slog.Error("Cluster manager failed to start", "error", err)
+		if err := d.clusterServer.Serve(ln); err != nil && err != http.ErrServerClosed {
+			slog.Error("Cluster manager failed", "error", err)
 		}
 	}()
 
@@ -1300,7 +1404,6 @@ func (d *Daemon) LoadState() error {
 }
 
 func (d *Daemon) SendQMPCommand(q *qmp.QMPClient, cmd qmp.QMPCommand, instanceId string) (*qmp.QMPResponse, error) {
-
 	// Confirm QMP client is initialized
 	if q == nil || q.Encoder == nil || q.Decoder == nil {
 		return nil, fmt.Errorf("QMP client is not initialized")
@@ -1314,7 +1417,7 @@ func (d *Daemon) SendQMPCommand(q *qmp.QMPClient, cmd qmp.QMPCommand, instanceId
 	if err := q.Conn.SetReadDeadline(time.Now().Add(30 * time.Second)); err != nil {
 		return nil, fmt.Errorf("set read deadline: %w", err)
 	}
-	defer q.Conn.SetReadDeadline(time.Time{}) // clear deadline after command
+	defer func() { _ = q.Conn.SetReadDeadline(time.Time{}) }() // clear deadline after command
 
 	if err := q.Encoder.Encode(cmd); err != nil {
 		return nil, fmt.Errorf("encode error: %w", err)
@@ -1341,7 +1444,10 @@ func (d *Daemon) SendQMPCommand(q *qmp.QMPClient, cmd qmp.QMPCommand, instanceId
 			return nil, fmt.Errorf("QMP error: %s: %s", errObj["class"], errObj["desc"])
 		}
 		if _, ok := msg["return"]; ok {
-			respBytes, _ := json.Marshal(msg)
+			respBytes, err := json.Marshal(msg)
+			if err != nil {
+				return nil, fmt.Errorf("marshal QMP response: %w", err)
+			}
 			var resp qmp.QMPResponse
 			if err := json.Unmarshal(respBytes, &resp); err != nil {
 				return nil, fmt.Errorf("unmarshal error: %w", err)
@@ -1352,15 +1458,12 @@ func (d *Daemon) SendQMPCommand(q *qmp.QMPClient, cmd qmp.QMPCommand, instanceId
 }
 
 func (d *Daemon) stopInstance(instances map[string]*vm.VM, deleteVolume bool) error {
-
 	// Signal to shutdown each VM
 	var wg sync.WaitGroup
 
 	// Run asynchronously within a worker group
 	for _, instance := range instances {
-
 		wg.Go(func() {
-
 			// Send shutdown command - if it fails, VM may already be dead, continue with cleanup
 			_, err := d.SendQMPCommand(instance.QMPClient, qmp.QMPCommand{Execute: "system_powerdown"}, instance.ID)
 			if err != nil {
@@ -1390,7 +1493,6 @@ func (d *Daemon) stopInstance(instances map[string]*vm.VM, deleteVolume bool) er
 			defer instance.EBSRequests.Mu.Unlock()
 
 			for _, ebsRequest := range instance.EBSRequests.Requests {
-
 				// Send the volume payload as JSON
 				ebsUnMountRequest, err := json.Marshal(ebsRequest)
 
@@ -1510,12 +1612,10 @@ func (d *Daemon) stopInstance(instances map[string]*vm.VM, deleteVolume bool) er
 		}
 	}
 	return nil
-
 }
 
 func (d *Daemon) setupShutdown() {
 	d.shutdownWg.Go(func() {
-
 		sigChan := make(chan os.Signal, 1)
 		signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
 
@@ -1545,7 +1645,6 @@ func (d *Daemon) setupShutdown() {
 			if err := sub.Unsubscribe(); err != nil {
 				slog.Error("Error unsubscribing from NATS", "err", err)
 			}
-
 		}
 
 		// Write shutdown marker to cluster state KV
@@ -1577,7 +1676,6 @@ func (d *Daemon) setupShutdown() {
 }
 
 func (d *Daemon) CreateQMPClient(instance *vm.VM) (err error) {
-
 	// Create a new QMP client to communicate with the instance
 	instance.QMPClient, err = qmp.NewQMPClient(instance.Config.QMPSocket)
 
@@ -1629,13 +1727,20 @@ func (d *Daemon) CreateQMPClient(instance *vm.VM) (err error) {
 	}()
 
 	return nil
-
 }
 
 func (d *Daemon) LaunchInstance(instance *vm.VM) (err error) {
+	// Abort if instance is no longer in a launchable state (e.g., terminated
+	// by a concurrent request while waiting in the launch queue).
+	d.Instances.Mu.Lock()
+	status := instance.Status
+	d.Instances.Mu.Unlock()
+	if status != vm.StatePending && status != vm.StateStopped && status != vm.StateProvisioning {
+		return fmt.Errorf("instance %s in %s state, not launchable", instance.ID, status)
+	}
 
 	// First, confirm if the instance is already running
-	pid, err := utils.ReadPidFile(instance.ID)
+	pid, _ := utils.ReadPidFile(instance.ID)
 
 	if pid > 0 {
 		process, err := os.FindProcess(pid)
@@ -1728,8 +1833,19 @@ func (d *Daemon) LaunchInstance(instance *vm.VM) (err error) {
 // then completes the termination lifecycle in the background so the instance
 // reaches terminated state and doesn't get stuck in shutting-down.
 func (d *Daemon) markInstanceFailed(instance *vm.VM, reason string) {
-	// Set state reason before transition (requires lock)
+	// If the instance is already being cleaned up (e.g., a concurrent terminate
+	// request transitioned it to shutting-down while LaunchInstance was running),
+	// don't spawn a second finalizeTermination goroutine — the existing cleanup
+	// handler owns the lifecycle from here.
 	d.Instances.Mu.Lock()
+	if instance.Status == vm.StateShuttingDown || instance.Status == vm.StateTerminated {
+		d.Instances.Mu.Unlock()
+		slog.Info("markInstanceFailed: instance already in cleanup state, skipping",
+			"instanceId", instance.ID, "status", string(instance.Status), "reason", reason)
+		return
+	}
+
+	// Set state reason before transition
 	if instance.Instance != nil {
 		instance.Instance.StateReason = &ec2.StateReason{}
 		instance.Instance.StateReason.SetCode("Server.InternalError")
@@ -1849,7 +1965,6 @@ func (d *Daemon) startPendingWatchdog() {
 }
 
 func (d *Daemon) StartInstance(instance *vm.VM) error {
-
 	pidFile, err := utils.GeneratePidFile(instance.ID)
 
 	if err != nil {
@@ -1900,7 +2015,6 @@ func (d *Daemon) StartInstance(instance *vm.VM) error {
 	instance.EBSRequests.Mu.Lock()
 
 	for _, v := range instance.EBSRequests.Requests {
-
 		drive := vm.Drive{}
 
 		// Use the NBDURI from mount response - contains socket path or TCP address
@@ -1964,34 +2078,42 @@ func (d *Daemon) StartInstance(instance *vm.VM) error {
 
 		// DEV_NETWORKING: add a second NIC with hostfwd for SSH dev access
 		if d.config.Daemon.DevNetworking {
-			sshDebugPort, err := viperblock.FindFreePort()
+			sshDebugAddr, err := viperblock.FindFreePort()
 			if err != nil {
 				slog.Warn("DEV_NETWORKING: failed to find free port for dev NIC", "err", err)
 			} else {
-				_, sshDebugPort, _ = net.SplitHostPort(sshDebugPort)
-				bindIP := d.config.Host
-				if bindIP == "" || bindIP == "0.0.0.0" {
-					bindIP = "127.0.0.1"
+				_, sshDebugPort, err := net.SplitHostPort(sshDebugAddr)
+				if err != nil {
+					slog.Warn("DEV_NETWORKING: failed to parse port from address", "addr", sshDebugAddr, "err", err)
+				} else {
+					bindIP := d.config.Host
+					if bindIP == "" || bindIP == "0.0.0.0" {
+						bindIP = "127.0.0.1"
+					}
+					instance.Config.NetDevs = append(instance.Config.NetDevs, vm.NetDev{
+						Value: fmt.Sprintf("user,id=dev0,hostfwd=tcp:%s:%s-:22", bindIP, sshDebugPort),
+					})
+					devMac := generateDevMAC(instance.ID)
+					instance.Config.Devices = append(instance.Config.Devices, vm.Device{
+						Value: fmt.Sprintf("virtio-net-pci,netdev=dev0,mac=%s", devMac),
+					})
+					slog.Info("DEV_NETWORKING: added dev NIC with SSH hostfwd",
+						"bindIP", bindIP, "port", sshDebugPort, "mac", devMac, "instanceId", instance.ID)
 				}
-				instance.Config.NetDevs = append(instance.Config.NetDevs, vm.NetDev{
-					Value: fmt.Sprintf("user,id=dev0,hostfwd=tcp:%s:%s-:22", bindIP, sshDebugPort),
-				})
-				devMac := generateDevMAC(instance.ID)
-				instance.Config.Devices = append(instance.Config.Devices, vm.Device{
-					Value: fmt.Sprintf("virtio-net-pci,netdev=dev0,mac=%s", devMac),
-				})
-				slog.Info("DEV_NETWORKING: added dev NIC with SSH hostfwd",
-					"bindIP", bindIP, "port", sshDebugPort, "mac", devMac, "instanceId", instance.ID)
 			}
 		}
 	} else {
 		// Non-VPC fallback: user-mode networking with SSH port forwarding
-		sshDebugPort, err := viperblock.FindFreePort()
+		sshDebugAddr, err := viperblock.FindFreePort()
 		if err != nil {
 			slog.Error("Failed to find free port", "err", err)
 			return err
 		}
-		_, sshDebugPort, _ = net.SplitHostPort(sshDebugPort)
+		_, sshDebugPort, err := net.SplitHostPort(sshDebugAddr)
+		if err != nil {
+			slog.Error("Failed to parse port from address", "addr", sshDebugAddr, "err", err)
+			return fmt.Errorf("parse port from %s: %w", sshDebugAddr, err)
+		}
 
 		bindIP := d.config.Host
 		if bindIP == "" || bindIP == "0.0.0.0" {
@@ -2160,12 +2282,10 @@ func (d *Daemon) ebsTopic(action string) string {
 
 // MountVolumes mounts the volumes for an instance
 func (d *Daemon) MountVolumes(instance *vm.VM) error {
-
 	instance.EBSRequests.Mu.Lock()
 	defer instance.EBSRequests.Mu.Unlock()
 
 	for k, v := range instance.EBSRequests.Requests {
-
 		// Send the volume payload as JSON
 		ebsMountRequest, err := json.Marshal(v)
 
@@ -2194,21 +2314,17 @@ func (d *Daemon) MountVolumes(instance *vm.VM) error {
 		}
 
 		if ebsMountResponse.Error == "" {
-
 			slog.Debug("Mounted volume successfully", "response", ebsMountResponse.URI)
 
 			// Append the NBD URI to the request
 			instance.EBSRequests.Requests[k].NBDURI = ebsMountResponse.URI
-
 		} else {
 			slog.Error("Failed to mount volume", "error", ebsMountResponse.Error)
 			return fmt.Errorf("failed to mount volume: %s", ebsMountResponse.Error)
 		}
-
 	}
 
 	return nil
-
 }
 
 // rollbackEBSMount sends an ebs.unmount request to undo a previously successful ebs.mount.

spinifex/daemon/daemon_handlers_instance.go

@@ -262,9 +262,33 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) {
 
 	slog.Info("Instances added to state with pending status", "count", len(instances))
 
+	// Subscribe to per-instance NATS topics early so terminate/stop commands
+	// can reach this daemon while volumes are being prepared. LaunchInstance
+	// will replace these subscriptions when it completes.
+	d.mu.Lock()
+	for _, instance := range instances {
+		sub, subErr := d.natsConn.Subscribe(fmt.Sprintf("ec2.cmd.%s", instance.ID), d.handleEC2Events)
+		if subErr != nil {
+			slog.Error("Failed to early-subscribe to per-instance topic", "instanceId", instance.ID, "err", subErr)
+		} else {
+			d.natsSubscriptions[instance.ID] = sub
+		}
+	}
+	d.mu.Unlock()
+
 	// Launch all instances (volumes and VMs)
 	var successCount int
 	for _, instance := range instances {
+		// Skip if instance was terminated by a concurrent request
+		d.Instances.Mu.Lock()
+		status := instance.Status
+		d.Instances.Mu.Unlock()
+		if status != vm.StatePending && status != vm.StateProvisioning {
+			slog.Info("Instance state changed during provisioning, skipping launch",
+				"instanceId", instance.ID, "status", string(status))
+			continue
+		}
+
 		// Pre-compute dev MAC so cloud-init can generate per-interface netplan
 		// that suppresses the default route on the dev/hostfwd NIC.
 		if d.config.Daemon.DevNetworking && instance.ENIId != "" {

spinifex/daemon/daemon_handlers_test.go

@@ -386,7 +386,7 @@ func TestHandleEC2RunInstances_InvalidAMI(t *testing.T) {
 
 	input := &ec2.RunInstancesInput{
 		ImageId:      aws.String("ami-nonexistent"),
-		InstanceType: aws.String(getTestInstanceType()),
+		InstanceType: aws.String(getTestInstanceType(t)),
 		MinCount:     aws.Int64(1),
 		MaxCount:     aws.Int64(1),
 	}
@@ -412,7 +412,7 @@ func TestHandleEC2RunInstances_InvalidKeyPair(t *testing.T) {
 
 	input := &ec2.RunInstancesInput{
 		ImageId:      aws.String("ami-test123"),
-		InstanceType: aws.String(getTestInstanceType()),
+		InstanceType: aws.String(getTestInstanceType(t)),
 		KeyName:      aws.String("nonexistent-keypair"),
 		MinCount:     aws.Int64(1),
 		MaxCount:     aws.Int64(1),
@@ -456,7 +456,7 @@ func TestHandleEC2RunInstances_ValidKeyPairPassesValidation(t *testing.T) {
 
 	input := &ec2.RunInstancesInput{
 		ImageId:      aws.String("ami-test456"),
-		InstanceType: aws.String(getTestInstanceType()),
+		InstanceType: aws.String(getTestInstanceType(t)),
 		KeyName:      aws.String("my-key"),
 		MinCount:     aws.Int64(1),
 		MaxCount:     aws.Int64(1),
@@ -482,7 +482,7 @@ func TestHandleEC2RunInstances_EmptyKeyNameSkipsValidation(t *testing.T) {
 	// No KeyName at all — should skip validation
 	input := &ec2.RunInstancesInput{
 		ImageId:      aws.String("ami-test789"),
-		InstanceType: aws.String(getTestInstanceType()),
+		InstanceType: aws.String(getTestInstanceType(t)),
 		MinCount:     aws.Int64(1),
 		MaxCount:     aws.Int64(1),
 	}
@@ -517,7 +517,7 @@ func TestHandleEC2RunInstances_ServiceErrorPropagated(t *testing.T) {
 
 	input := &ec2.RunInstancesInput{
 		ImageId:      aws.String("ami-propatest"),
-		InstanceType: aws.String(getTestInstanceType()),
+		InstanceType: aws.String(getTestInstanceType(t)),
 		MinCount:     aws.Int64(1),
 		MaxCount:     aws.Int64(1),
 	}
@@ -541,7 +541,7 @@ func TestHandleEC2Events_StopInstance(t *testing.T) {
 	instanceID := "i-test-stop-001"
 	daemon.Instances.VMS[instanceID] = &vm.VM{
 		ID:           instanceID,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		Status:       vm.StateRunning,
 		Instance:     &ec2.Instance{},
 		QMPClient:    &qmp.QMPClient{},
@@ -587,7 +587,7 @@ func TestHandleEC2Events_TerminateInstance(t *testing.T) {
 	instanceID := "i-test-term-001"
 	daemon.Instances.VMS[instanceID] = &vm.VM{
 		ID:           instanceID,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		Status:       vm.StateRunning,
 		Instance:     &ec2.Instance{},
 		QMPClient:    &qmp.QMPClient{},
@@ -631,7 +631,7 @@ func TestHandleEC2Events_RebootRunningInstance(t *testing.T) {
 	instanceID := "i-test-reboot-001"
 	daemon.Instances.VMS[instanceID] = &vm.VM{
 		ID:           instanceID,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		Status:       vm.StateRunning,
 		Instance:     &ec2.Instance{},
 		QMPClient:    &qmp.QMPClient{},
@@ -681,7 +681,7 @@ func TestHandleEC2Events_RebootStoppedInstance(t *testing.T) {
 	instanceID := "i-test-reboot-stopped"
 	daemon.Instances.VMS[instanceID] = &vm.VM{
 		ID:           instanceID,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		Status:       vm.StateStopped,
 		Instance:     &ec2.Instance{},
 		QMPClient:    &qmp.QMPClient{},
@@ -724,7 +724,7 @@ func TestHandleEC2Events_RebootTerminatedInstance(t *testing.T) {
 	instanceID := "i-test-reboot-terminated"
 	daemon.Instances.VMS[instanceID] = &vm.VM{
 		ID:           instanceID,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		Status:       vm.StateTerminated,
 		Instance:     &ec2.Instance{},
 		QMPClient:    &qmp.QMPClient{},
@@ -1150,7 +1150,8 @@ func TestSetConfigPath(t *testing.T) {
 		Node:  "node-1",
 		Nodes: map[string]config.Config{"node-1": {BaseDir: "/tmp"}},
 	}
-	daemon := NewDaemon(clusterCfg)
+	daemon, err := NewDaemon(clusterCfg)
+	require.NoError(t, err)
 
 	assert.Empty(t, daemon.configPath)
 	daemon.SetConfigPath("/etc/spinifex/config.toml")
@@ -1208,7 +1209,7 @@ func TestHandleEC2StartStoppedInstance_NotStoppedState(t *testing.T) {
 	runningVM := &vm.VM{
 		ID:           "i-running-shared",
 		Status:       vm.StateRunning,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		AccountID:    testAccountID,
 	}
 	err := daemon.jsManager.WriteStoppedInstance(runningVM.ID, runningVM)
@@ -1242,7 +1243,7 @@ func TestHandleEC2DescribeStoppedInstances_ReturnsStoppedInstances(t *testing.T)
 	stoppedVM := &vm.VM{
 		ID:           "i-describe-stopped-001",
 		Status:       vm.StateStopped,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		LastNode:     "node-1",
 		AccountID:    testAccountID,
 		Reservation: &ec2.Reservation{
@@ -1251,7 +1252,7 @@ func TestHandleEC2DescribeStoppedInstances_ReturnsStoppedInstances(t *testing.T)
 		},
 		Instance: &ec2.Instance{
 			InstanceId:   aws.String("i-describe-stopped-001"),
-			InstanceType: aws.String(getTestInstanceType()),
+			InstanceType: aws.String(getTestInstanceType(t)),
 		},
 	}
 	err := daemon.jsManager.WriteStoppedInstance(stoppedVM.ID, stoppedVM)
@@ -1396,7 +1397,7 @@ func TestHandleEC2TerminateStoppedInstance_NotStoppedState(t *testing.T) {
 	runningVM := &vm.VM{
 		ID:           "i-term-running",
 		Status:       vm.StateRunning,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		AccountID:    testAccountID,
 	}
 	err := daemon.jsManager.WriteStoppedInstance(runningVM.ID, runningVM)
@@ -1428,7 +1429,7 @@ func TestHandleEC2TerminateStoppedInstance_Success(t *testing.T) {
 	stoppedVM := &vm.VM{
 		ID:           "i-term-stopped-001",
 		Status:       vm.StateStopped,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		LastNode:     "node-1",
 		AccountID:    testAccountID,
 		Reservation: &ec2.Reservation{
@@ -1437,7 +1438,7 @@ func TestHandleEC2TerminateStoppedInstance_Success(t *testing.T) {
 		},
 		Instance: &ec2.Instance{
 			InstanceId:   aws.String("i-term-stopped-001"),
-			InstanceType: aws.String(getTestInstanceType()),
+			InstanceType: aws.String(getTestInstanceType(t)),
 		},
 	}
 	err := daemon.jsManager.WriteStoppedInstance(stoppedVM.ID, stoppedVM)
@@ -1586,7 +1587,7 @@ func TestAttachVolume_ZoneMismatch(t *testing.T) {
 	// Create a running instance
 	instance := &vm.VM{
 		ID:           instanceID,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		Status:       vm.StateRunning,
 		AccountID:    testAccountID,
 		Instance:     &ec2.Instance{},
@@ -2119,7 +2120,7 @@ func TestHandleNodeVMs(t *testing.T) {
 	daemon := createTestDaemon(t, sharedNATSURL)
 	daemon.config.Daemon.Host = "10.0.0.5:4432"
 
-	instanceType := getTestInstanceType()
+	instanceType := getTestInstanceType(t)
 	launchTime := time.Date(2025, 1, 15, 10, 0, 0, 0, time.UTC)
 
 	daemon.Instances.Mu.Lock()
@@ -2674,7 +2675,7 @@ func TestHandleEC2TerminateStoppedInstance_WithVolumes(t *testing.T) {
 		ID:           "i-term-vol-001",
 		Status:       vm.StateStopped,
 		AccountID:    testAccountID,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		LastNode:     "node-1",
 		Reservation: &ec2.Reservation{
 			ReservationId: aws.String("r-term-vol-001"),
@@ -2682,7 +2683,7 @@ func TestHandleEC2TerminateStoppedInstance_WithVolumes(t *testing.T) {
 		},
 		Instance: &ec2.Instance{
 			InstanceId:   aws.String("i-term-vol-001"),
-			InstanceType: aws.String(getTestInstanceType()),
+			InstanceType: aws.String(getTestInstanceType(t)),
 		},
 	}
 	// Add EFI, CloudInit, and a user volume with DeleteOnTermination
@@ -2828,7 +2829,7 @@ func TestHandleEC2CreateImage_RunningInstanceReachesService(t *testing.T) {
 	daemon.Instances.VMS[instanceID] = &vm.VM{
 		ID:           instanceID,
 		Status:       vm.StateRunning,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		AccountID:    testAccountID,
 		Instance: &ec2.Instance{
 			InstanceId: aws.String(instanceID),
@@ -2879,7 +2880,7 @@ func TestAttachVolume_MissingVolumeData(t *testing.T) {
 		ID:           instanceID,
 		Status:       vm.StateRunning,
 		AccountID:    testAccountID,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		Instance:     &ec2.Instance{},
 		QMPClient:    &qmp.QMPClient{},
 	}
@@ -2919,7 +2920,7 @@ func TestAttachVolume_InstanceNotRunning(t *testing.T) {
 		ID:           instanceID,
 		Status:       vm.StateStopped,
 		AccountID:    testAccountID,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		Instance:     &ec2.Instance{},
 		QMPClient:    &qmp.QMPClient{},
 	}
@@ -2960,7 +2961,7 @@ func TestAttachVolume_VolumeNotFound(t *testing.T) {
 		ID:           instanceID,
 		Status:       vm.StateRunning,
 		AccountID:    testAccountID,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		Instance:     &ec2.Instance{},
 		QMPClient:    &qmp.QMPClient{},
 	}
@@ -3003,7 +3004,7 @@ func TestAttachVolume_VolumeInUse(t *testing.T) {
 		ID:           instanceID,
 		Status:       vm.StateRunning,
 		AccountID:    testAccountID,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		Instance:     &ec2.Instance{},
 		QMPClient:    &qmp.QMPClient{},
 	}
@@ -3065,7 +3066,7 @@ func TestDetachVolume_MissingVolumeData(t *testing.T) {
 		ID:           instanceID,
 		Status:       vm.StateRunning,
 		AccountID:    testAccountID,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		Instance:     &ec2.Instance{},
 		QMPClient:    &qmp.QMPClient{},
 	}
@@ -3104,7 +3105,7 @@ func TestDetachVolume_InstanceNotRunning(t *testing.T) {
 		ID:           instanceID,
 		Status:       vm.StateStopped,
 		AccountID:    testAccountID,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		Instance:     &ec2.Instance{},
 		QMPClient:    &qmp.QMPClient{},
 	}
@@ -3145,7 +3146,7 @@ func TestDetachVolume_VolumeNotAttached(t *testing.T) {
 		ID:           instanceID,
 		Status:       vm.StateRunning,
 		AccountID:    testAccountID,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		Instance:     &ec2.Instance{},
 		QMPClient:    &qmp.QMPClient{},
 	}
@@ -3186,7 +3187,7 @@ func TestDetachVolume_BootVolumeRejected(t *testing.T) {
 		ID:           instanceID,
 		Status:       vm.StateRunning,
 		AccountID:    testAccountID,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		Instance:     &ec2.Instance{},
 		QMPClient:    &qmp.QMPClient{},
 	}
@@ -3230,7 +3231,7 @@ func TestDetachVolume_DeviceMismatch(t *testing.T) {
 		ID:           instanceID,
 		Status:       vm.StateRunning,
 		AccountID:    testAccountID,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		Instance:     &ec2.Instance{},
 		QMPClient:    &qmp.QMPClient{},
 	}
@@ -3279,7 +3280,7 @@ func TestHandleEC2RunInstances_InsufficientCapacity(t *testing.T) {
 	// Request a very large instance count that can't be satisfied
 	input := &ec2.RunInstancesInput{
 		ImageId:      aws.String("ami-test"),
-		InstanceType: aws.String(getTestInstanceType()),
+		InstanceType: aws.String(getTestInstanceType(t)),
 		MinCount:     aws.Int64(9999),
 		MaxCount:     aws.Int64(9999),
 	}
@@ -3359,7 +3360,7 @@ func TestHandleEC2DescribeTerminatedInstances_ReturnsTerminatedInstances(t *test
 	terminatedVM := &vm.VM{
 		ID:           "i-describe-term-001",
 		Status:       vm.StateTerminated,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		LastNode:     "node-1",
 		AccountID:    testAccountID,
 		Reservation: &ec2.Reservation{
@@ -3368,7 +3369,7 @@ func TestHandleEC2DescribeTerminatedInstances_ReturnsTerminatedInstances(t *test
 		},
 		Instance: &ec2.Instance{
 			InstanceId:   aws.String("i-describe-term-001"),
-			InstanceType: aws.String(getTestInstanceType()),
+			InstanceType: aws.String(getTestInstanceType(t)),
 		},
 	}
 	err := daemon.jsManager.WriteTerminatedInstance(terminatedVM.ID, terminatedVM)
@@ -3417,7 +3418,7 @@ func TestHandleEC2DescribeTerminatedInstances_WithFilter(t *testing.T) {
 			},
 			Instance: &ec2.Instance{
 				InstanceId:   aws.String(id),
-				InstanceType: aws.String(getTestInstanceType()),
+				InstanceType: aws.String(getTestInstanceType(t)),
 			},
 		}
 		require.NoError(t, daemon.jsManager.WriteTerminatedInstance(v.ID, v))
@@ -3454,7 +3455,7 @@ func TestHandleEC2TerminateStoppedInstance_WritesToTerminatedKV(t *testing.T) {
 	stoppedVM := &vm.VM{
 		ID:           "i-stop-to-term-001",
 		Status:       vm.StateStopped,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		AccountID:    testAccountID,
 	}
 	require.NoError(t, daemon.jsManager.WriteStoppedInstance(stoppedVM.ID, stoppedVM))

spinifex/daemon/daemon_test.go

@@ -31,12 +31,7 @@ import (
 // createTestDaemon creates a test daemon instance with minimal configuration
 func createTestDaemon(t *testing.T, natsURL string) *Daemon {
 	// Create a temporary directory for test data
-	tmpDir, err := os.MkdirTemp("", "spx-daemon-test-*")
-	require.NoError(t, err, "Failed to create temp directory")
-
-	t.Cleanup(func() {
-		os.RemoveAll(tmpDir)
-	})
+	tmpDir := t.TempDir()
 
 	// New cluster config
 	clusterCfg := &config.ClusterConfig{
@@ -65,7 +60,8 @@ func createTestDaemon(t *testing.T, natsURL string) *Daemon {
 
 	clusterCfg.Nodes["node-1"] = *cfg
 
-	daemon := NewDaemon(clusterCfg)
+	daemon, err := NewDaemon(clusterCfg)
+	require.NoError(t, err)
 
 	// Connect to NATS
 	nc, err := nats.Connect(natsURL)
@@ -88,8 +84,10 @@ func createTestDaemon(t *testing.T, natsURL string) *Daemon {
 }
 
 // getTestInstanceType returns a valid instance type for testing based on the system's CPU
-func getTestInstanceType() string {
-	rm := NewResourceManager()
+func getTestInstanceType(t *testing.T) string {
+	t.Helper()
+	rm, err := NewResourceManager()
+	require.NoError(t, err)
 	// Find any .micro instance type
 	for key := range rm.instanceTypes {
 		if strings.HasSuffix(key, ".micro") {
@@ -133,7 +131,7 @@ func TestHandleEC2RunInstances_MessageParsing(t *testing.T) {
 	}{
 		{
 			name:           "Valid RunInstancesInput",
-			input:          createValidRunInstancesInput(),
+			input:          createValidRunInstancesInput(t),
 			expectError:    false,
 			errorInPayload: false,
 			validate: func(t *testing.T, reply *nats.Msg) {
@@ -342,9 +340,7 @@ func TestHandleEC2RunInstances_ResourceManagement(t *testing.T) {
 
 // TestDaemon_Initialization tests daemon initialization
 func TestDaemon_Initialization(t *testing.T) {
-	tmpDir, err := os.MkdirTemp("", "spx-daemon-init-test-*")
-	require.NoError(t, err)
-	defer os.RemoveAll(tmpDir)
+	tmpDir := t.TempDir()
 
 	// New cluster config
 	clusterCfg := &config.ClusterConfig{
@@ -362,7 +358,8 @@ func TestDaemon_Initialization(t *testing.T) {
 
 	clusterCfg.Nodes["node-1"] = *cfg
 
-	daemon := NewDaemon(clusterCfg)
+	daemon, err := NewDaemon(clusterCfg)
+	require.NoError(t, err)
 
 	assert.NotNil(t, daemon)
 	assert.NotNil(t, daemon.resourceMgr)
@@ -372,7 +369,8 @@ func TestDaemon_Initialization(t *testing.T) {
 
 // TestResourceManager tests resource manager functionality
 func TestResourceManager(t *testing.T) {
-	rm := NewResourceManager()
+	rm, err := NewResourceManager()
+	require.NoError(t, err)
 
 	require.NotNil(t, rm)
 	assert.Greater(t, rm.availableVCPU, 0)
@@ -398,7 +396,7 @@ func TestResourceManager(t *testing.T) {
 	assert.Equal(t, 1, canAlloc)
 
 	// Allocate
-	err := rm.allocate(instanceType)
+	err = rm.allocate(instanceType)
 	assert.NoError(t, err)
 
 	// Check resources were allocated
@@ -421,7 +419,8 @@ func TestResourceManager(t *testing.T) {
 	// Test canAllocate with count parameter
 	t.Run("canAllocate_with_count", func(t *testing.T) {
 		// Fresh resource manager for predictable testing
-		rm := NewResourceManager()
+		rm, err := NewResourceManager()
+		require.NoError(t, err)
 
 		// Find a .micro instance type
 		var microType *ec2.InstanceTypeInfo
@@ -457,7 +456,8 @@ func TestResourceManager(t *testing.T) {
 
 // TestGetInstanceTypeInfos tests the GetInstanceTypeInfos method
 func TestGetInstanceTypeInfos(t *testing.T) {
-	rm := NewResourceManager()
+	rm, err := NewResourceManager()
+	require.NoError(t, err)
 
 	infos := rm.GetInstanceTypeInfos()
 
@@ -484,7 +484,8 @@ func TestGetInstanceTypeInfos(t *testing.T) {
 
 // TestGetAvailableInstanceTypeInfos_ResourceFiltering tests that instance types are filtered by available resources
 func TestGetAvailableInstanceTypeInfos_ResourceFiltering(t *testing.T) {
-	rm := NewResourceManager()
+	rm, err := NewResourceManager()
+	require.NoError(t, err)
 
 	// Get initial count of all available types
 	allTypes := rm.GetInstanceTypeInfos()
@@ -524,7 +525,7 @@ func TestGetAvailableInstanceTypeInfos_ResourceFiltering(t *testing.T) {
 	}
 	require.True(t, exists, "Should have at least one .nano instance type")
 
-	err := rm.allocate(nanoType)
+	err = rm.allocate(nanoType)
 	require.NoError(t, err, "Should be able to allocate %s", nanoKey)
 
 	t.Logf("After allocating %s: allocated %d vCPUs, %.2f GB RAM",
@@ -851,29 +852,27 @@ func TestDaemon_BootAllocation(t *testing.T) {
 	natsURL := sharedJSNATSURL
 
 	// Create daemon temp directory
-	tmpDir, err := os.MkdirTemp("", "spx-daemon-boot-test-*")
-	require.NoError(t, err)
-	defer os.RemoveAll(tmpDir)
+	tmpDir := t.TempDir()
 
 	// Create test VMs with one running and one stopped instance
 	vms := map[string]*vm.VM{
 		"i-running": {
 			ID:           "i-running",
-			InstanceType: getTestInstanceType(),
+			InstanceType: getTestInstanceType(t),
 			Status:       vm.StateRunning,
 			AccountID:    testAccountID,
 			Attributes:   types.EC2CommandAttributes{StopInstance: false},
 		},
 		"i-stopped": {
 			ID:           "i-stopped",
-			InstanceType: getTestInstanceType(),
+			InstanceType: getTestInstanceType(t),
 			Status:       vm.StateStopped,
 			AccountID:    testAccountID,
 			Attributes:   types.EC2CommandAttributes{StopInstance: true},
 		},
 		"i-terminated": {
 			ID:           "i-terminated",
-			InstanceType: getTestInstanceType(),
+			InstanceType: getTestInstanceType(t),
 			Status:       vm.StateTerminated,
 			Attributes:   types.EC2CommandAttributes{StopInstance: false},
 		},
@@ -884,7 +883,8 @@ func TestDaemon_BootAllocation(t *testing.T) {
 		Node:  "node-1",
 		Nodes: map[string]config.Config{"node-1": {BaseDir: tmpDir}},
 	}
-	daemon := NewDaemon(clusterCfg)
+	daemon, err := NewDaemon(clusterCfg)
+	require.NoError(t, err)
 	daemon.config = &config.Config{BaseDir: tmpDir}
 
 	// Connect to NATS and initialize JetStream
@@ -933,11 +933,12 @@ func TestStopInstance_Deallocation(t *testing.T) {
 		Node:  "node-1",
 		Nodes: map[string]config.Config{"node-1": {BaseDir: "/tmp"}},
 	}
-	daemon := NewDaemon(clusterCfg)
+	daemon, err := NewDaemon(clusterCfg)
+	require.NoError(t, err)
 
 	// Setup a running instance with allocated resources
 	instanceId := "i-test-stop"
-	instanceTypeStr := getTestInstanceType()
+	instanceTypeStr := getTestInstanceType(t)
 	instanceType := daemon.resourceMgr.instanceTypes[instanceTypeStr]
 	daemon.Instances.VMS[instanceId] = &vm.VM{
 		ID:           instanceId,
@@ -946,7 +947,7 @@ func TestStopInstance_Deallocation(t *testing.T) {
 		AccountID:    testAccountID,
 	}
 
-	err := daemon.resourceMgr.allocate(instanceType)
+	err = daemon.resourceMgr.allocate(instanceType)
 	require.NoError(t, err)
 	assert.Greater(t, daemon.resourceMgr.allocatedVCPU, 0)
 
@@ -961,10 +962,11 @@ func TestStopInstance_Deallocation(t *testing.T) {
 }
 
 // createValidRunInstancesInput creates a valid RunInstancesInput for testing
-func createValidRunInstancesInput() *ec2.RunInstancesInput {
+func createValidRunInstancesInput(t *testing.T) *ec2.RunInstancesInput {
+	t.Helper()
 	return &ec2.RunInstancesInput{
 		ImageId:      aws.String("ami-0abcdef1234567890"),
-		InstanceType: aws.String(getTestInstanceType()),
+		InstanceType: aws.String(getTestInstanceType(t)),
 		MinCount:     aws.Int64(1),
 		MaxCount:     aws.Int64(1),
 		KeyName:      aws.String("test-key"),
@@ -976,7 +978,8 @@ func createValidRunInstancesInput() *ec2.RunInstancesInput {
 // TestCanAllocate_CountEdgeCases tests edge cases for canAllocate with count parameter
 func TestCanAllocate_CountEdgeCases(t *testing.T) {
 	t.Run("MinCount_equals_MaxCount", func(t *testing.T) {
-		rm := NewResourceManager()
+		rm, err := NewResourceManager()
+		require.NoError(t, err)
 
 		var microType *ec2.InstanceTypeInfo
 		for key, it := range rm.instanceTypes {
@@ -994,7 +997,8 @@ func TestCanAllocate_CountEdgeCases(t *testing.T) {
 	})
 
 	t.Run("Request_exceeds_capacity", func(t *testing.T) {
-		rm := NewResourceManager()
+		rm, err := NewResourceManager()
+		require.NoError(t, err)
 
 		// Find the largest instance type to exhaust resources faster
 		var largeType *ec2.InstanceTypeInfo
@@ -1016,7 +1020,8 @@ func TestCanAllocate_CountEdgeCases(t *testing.T) {
 	})
 
 	t.Run("Capacity_decreases_after_allocation", func(t *testing.T) {
-		rm := NewResourceManager()
+		rm, err := NewResourceManager()
+		require.NoError(t, err)
 
 		var microType *ec2.InstanceTypeInfo
 		for key, it := range rm.instanceTypes {
@@ -1031,7 +1036,7 @@ func TestCanAllocate_CountEdgeCases(t *testing.T) {
 		t.Logf("Initial capacity: %d micro instances", initial)
 
 		// Allocate one
-		err := rm.allocate(microType)
+		err = rm.allocate(microType)
 		require.NoError(t, err)
 
 		afterOne := rm.canAllocate(microType, 100)
@@ -1053,7 +1058,8 @@ func TestCanAllocate_CountEdgeCases(t *testing.T) {
 	})
 
 	t.Run("Mixed_instance_types", func(t *testing.T) {
-		rm := NewResourceManager()
+		rm, err := NewResourceManager()
+		require.NoError(t, err)
 
 		var microType, mediumType *ec2.InstanceTypeInfo
 		for key, it := range rm.instanceTypes {
@@ -1071,7 +1077,7 @@ func TestCanAllocate_CountEdgeCases(t *testing.T) {
 		initialMedium := rm.canAllocate(mediumType, 100)
 
 		// Allocate a medium (uses more resources)
-		err := rm.allocate(mediumType)
+		err = rm.allocate(mediumType)
 		require.NoError(t, err)
 
 		// Both capacities should decrease
@@ -1085,7 +1091,8 @@ func TestCanAllocate_CountEdgeCases(t *testing.T) {
 	})
 
 	t.Run("Zero_and_negative_counts", func(t *testing.T) {
-		rm := NewResourceManager()
+		rm, err := NewResourceManager()
+		require.NoError(t, err)
 
 		var microType *ec2.InstanceTypeInfo
 		for key, it := range rm.instanceTypes {
@@ -1300,7 +1307,7 @@ func TestDescribeInstances_ReservationGrouping(t *testing.T) {
 // TestRunInstances_CountValidation tests MinCount/MaxCount validation scenarios
 func TestRunInstances_CountValidation(t *testing.T) {
 	natsURL := sharedNATSURL
-	instanceType := getTestInstanceType()
+	instanceType := getTestInstanceType(t)
 	topic := fmt.Sprintf("ec2.RunInstances.%s", instanceType)
 
 	daemon, memStore := createFullTestDaemonWithStore(t, natsURL)
@@ -1380,7 +1387,8 @@ func TestInstanceTypeSubscriptions(t *testing.T) {
 
 	t.Run("InitialSubscriptions", func(t *testing.T) {
 		// A fresh ResourceManager should subscribe to all instance types that fit
-		rm := NewResourceManager()
+		rm, err := NewResourceManager()
+		require.NoError(t, err)
 		nc, err := nats.Connect(natsURL)
 		require.NoError(t, err)
 		defer nc.Close()
@@ -1409,7 +1417,8 @@ func TestInstanceTypeSubscriptions(t *testing.T) {
 	})
 
 	t.Run("UnsubscribesWhenFull", func(t *testing.T) {
-		rm := NewResourceManager()
+		rm, err := NewResourceManager()
+		require.NoError(t, err)
 		nc, err := nats.Connect(natsURL)
 		require.NoError(t, err)
 		defer nc.Close()
@@ -1433,7 +1442,8 @@ func TestInstanceTypeSubscriptions(t *testing.T) {
 	})
 
 	t.Run("ResubscribesWhenFreed", func(t *testing.T) {
-		rm := NewResourceManager()
+		rm, err := NewResourceManager()
+		require.NoError(t, err)
 		nc, err := nats.Connect(natsURL)
 		require.NoError(t, err)
 		defer nc.Close()
@@ -1463,7 +1473,8 @@ func TestInstanceTypeSubscriptions(t *testing.T) {
 	})
 
 	t.Run("PartialCapacity", func(t *testing.T) {
-		rm := NewResourceManager()
+		rm, err := NewResourceManager()
+		require.NoError(t, err)
 		nc, err := nats.Connect(natsURL)
 		require.NoError(t, err)
 		defer nc.Close()
@@ -1491,7 +1502,8 @@ func TestInstanceTypeSubscriptions(t *testing.T) {
 	})
 
 	t.Run("AllocateTriggersSubs", func(t *testing.T) {
-		rm := NewResourceManager()
+		rm, err := NewResourceManager()
+		require.NoError(t, err)
 		nc, err := nats.Connect(natsURL)
 		require.NoError(t, err)
 		defer nc.Close()
@@ -1534,7 +1546,8 @@ func TestInstanceTypeSubscriptions(t *testing.T) {
 	})
 
 	t.Run("NoRespondersWhenFull", func(t *testing.T) {
-		rm := NewResourceManager()
+		rm, err := NewResourceManager()
+		require.NoError(t, err)
 		nc, err := nats.Connect(natsURL)
 		require.NoError(t, err)
 		defer nc.Close()
@@ -1551,7 +1564,7 @@ func TestInstanceTypeSubscriptions(t *testing.T) {
 		assert.Equal(t, 0, len(rm.instanceSubs))
 
 		// Publishing to an instance type topic should get no responders
-		instanceType := getTestInstanceType()
+		instanceType := getTestInstanceType(t)
 		topic := fmt.Sprintf("ec2.RunInstances.%s", instanceType)
 
 		_, err = nc.Request(topic, []byte("{}"), 500*time.Millisecond)
@@ -1562,7 +1575,8 @@ func TestInstanceTypeSubscriptions(t *testing.T) {
 
 // TestResourceManager_ConcurrentAccess tests thread safety of resource manager
 func TestResourceManager_ConcurrentAccess(t *testing.T) {
-	rm := NewResourceManager()
+	rm, err := NewResourceManager()
+	require.NoError(t, err)
 
 	var microType *ec2.InstanceTypeInfo
 	for key, it := range rm.instanceTypes {
@@ -1757,7 +1771,7 @@ func TestStopInstance_DeleteOnTermination_VolumeDeletion(t *testing.T) {
 
 	instance := &vm.VM{
 		ID:           "i-test-dot",
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		Status:       vm.StateRunning,
 		AccountID:    testAccountID,
 		QMPClient:    &qmp.QMPClient{}, // nil encoder/decoder => QMP will fail, which is fine
@@ -1855,7 +1869,7 @@ func TestStopInstance_DeleteOnTermination_False_SkipsVolumeDeletion(t *testing.T
 
 	instance := &vm.VM{
 		ID:           "i-test-no-delete",
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		Status:       vm.StateRunning,
 		AccountID:    testAccountID,
 		QMPClient:    &qmp.QMPClient{},
@@ -1942,7 +1956,7 @@ func TestStopInstance_NoDelete_OnStop(t *testing.T) {
 
 	instance := &vm.VM{
 		ID:           "i-test-stop-only",
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		Status:       vm.StateRunning,
 		AccountID:    testAccountID,
 		QMPClient:    &qmp.QMPClient{},
@@ -1989,7 +2003,7 @@ func TestHandleEC2Events_AttachVolume(t *testing.T) {
 
 	instanceID := "i-test-attach"
 	volumeID := "vol-test-attach"
-	instanceType := getTestInstanceType()
+	instanceType := getTestInstanceType(t)
 
 	// Create a running instance (no actual QMP client - will fail at QMP step)
 	instance := &vm.VM{
@@ -2118,7 +2132,7 @@ func TestHandleEC2Events_DetachVolume(t *testing.T) {
 
 	instanceID := "i-test-detach"
 	volumeID := "vol-test-detach"
-	instanceType := getTestInstanceType()
+	instanceType := getTestInstanceType(t)
 
 	// Create a running instance with an attached volume
 	instance := &vm.VM{
@@ -2444,7 +2458,7 @@ func TestDetachVolume_SuccessPath(t *testing.T) {
 
 	instanceID := "i-test-detach-success"
 	volumeID := "vol-detach-success"
-	instanceType := getTestInstanceType()
+	instanceType := getTestInstanceType(t)
 
 	// Track QMP commands issued
 	var mu sync.Mutex
@@ -2585,7 +2599,7 @@ func TestDetachVolume_ForceFlag(t *testing.T) {
 
 	instanceID := "i-test-detach-force"
 	volumeID := "vol-detach-force"
-	instanceType := getTestInstanceType()
+	instanceType := getTestInstanceType(t)
 
 	var mu sync.Mutex
 	var qmpCommands []string
@@ -2701,7 +2715,7 @@ func TestDetachVolume_BlockdevDelFailure(t *testing.T) {
 
 	instanceID := "i-test-blockdev-fail"
 	volumeID := "vol-blockdev-fail"
-	instanceType := getTestInstanceType()
+	instanceType := getTestInstanceType(t)
 
 	callCount := 0
 	var mu sync.Mutex
@@ -2813,7 +2827,7 @@ func TestDetachVolume_SuccessWithDeviceMatch(t *testing.T) {
 
 	instanceID := "i-test-device-match"
 	volumeID := "vol-device-match"
-	instanceType := getTestInstanceType()
+	instanceType := getTestInstanceType(t)
 
 	qmpClient, cancelQMP := newMockQMPClient(t, nil)
 	defer cancelQMP()
@@ -2897,7 +2911,7 @@ func TestAttachVolume_ReplacesStaleEBSRequest(t *testing.T) {
 
 	instanceID := "i-test-stale-replace"
 	volumeID := "vol-stale-replace"
-	instanceType := getTestInstanceType()
+	instanceType := getTestInstanceType(t)
 
 	qmpClient, cancelQMP := newMockQMPClient(t, nil)
 	defer cancelQMP()
@@ -3219,7 +3233,8 @@ func TestNewDaemon_WalDirDefaultsToBaseDir(t *testing.T) {
 		},
 	}
 
-	d := NewDaemon(cfg)
+	d, err := NewDaemon(cfg)
+	require.NoError(t, err)
 	assert.Equal(t, "/data/spinifex", d.config.WalDir)
 }
 
@@ -3234,7 +3249,8 @@ func TestNewDaemon_WalDirPreservedIfSet(t *testing.T) {
 		},
 	}
 
-	d := NewDaemon(cfg)
+	d, err := NewDaemon(cfg)
+	require.NoError(t, err)
 	assert.Equal(t, "/fast-ssd/wal", d.config.WalDir)
 }
 
@@ -3251,7 +3267,7 @@ func TestMarkInstanceFailed(t *testing.T) {
 
 	instance := &vm.VM{
 		ID:           instanceID,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		Status:       vm.StatePending,
 		AccountID:    testAccountID,
 		Instance:     ec2Instance,
@@ -3283,7 +3299,7 @@ func TestMarkInstanceFailed_NilInstance(t *testing.T) {
 	instanceID := "i-test-mark-failed-nil"
 	instance := &vm.VM{
 		ID:           instanceID,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		Status:       vm.StatePending,
 		AccountID:    testAccountID,
 		Instance:     nil, // no ec2.Instance
@@ -3436,7 +3452,7 @@ func TestStopTerminate_IncorrectInstanceState(t *testing.T) {
 	instanceID := "i-test-state-check"
 	instance := &vm.VM{
 		ID:           instanceID,
-		InstanceType: getTestInstanceType(),
+		InstanceType: getTestInstanceType(t),
 		Status:       vm.StateStopped,
 		AccountID:    testAccountID,
 		Instance:     &ec2.Instance{},
@@ -3764,12 +3780,13 @@ func TestConnectNATS_RetriesOnFailure(t *testing.T) {
 		},
 	}
 	clusterCfg.Nodes["node-1"] = cfg
-	daemon := NewDaemon(clusterCfg)
+	daemon, err := NewDaemon(clusterCfg)
+	require.NoError(t, err)
 	daemon.natsMaxWait = 500 * time.Millisecond
 	daemon.natsRetryDelay = 50 * time.Millisecond
 
 	start := time.Now()
-	err := daemon.connectNATS()
+	err = daemon.connectNATS()
 	elapsed := time.Since(start)
 
 	require.Error(t, err)
@@ -3790,10 +3807,11 @@ func TestConnectNATS_SucceedsImmediately(t *testing.T) {
 		},
 	}
 	clusterCfg.Nodes["node-1"] = cfg
-	daemon := NewDaemon(clusterCfg)
+	daemon, err := NewDaemon(clusterCfg)
+	require.NoError(t, err)
 
 	start := time.Now()
-	err := daemon.connectNATS()
+	err = daemon.connectNATS()
 	elapsed := time.Since(start)
 
 	require.NoError(t, err)
@@ -3811,7 +3829,8 @@ func TestDaemonReadyFlag(t *testing.T) {
 	}
 	cfg := config.Config{}
 	clusterCfg.Nodes["node-1"] = cfg
-	daemon := NewDaemon(clusterCfg)
+	daemon, err := NewDaemon(clusterCfg)
+	require.NoError(t, err)
 
 	assert.False(t, daemon.ready.Load(), "daemon should not be ready before Start()")
 

spinifex/daemon/health.go

@@ -259,10 +259,25 @@ func (d *Daemon) restartCrashedInstance(instance *vm.VM) {
 		"instance", instance.ID,
 		"restartCount", instance.Health.RestartCount)
 
+	// Re-allocate resources before relaunch — handleCrashedInstance deallocates
+	// them, so we must re-reserve before starting the VM again.
+	instanceType, ok := d.resourceMgr.instanceTypes[instance.InstanceType]
+	if !ok || instanceType == nil {
+		slog.Error("Unknown instance type during restart, cannot re-allocate resources",
+			"instance", instance.ID, "type", instance.InstanceType)
+		return
+	}
+	if err := d.resourceMgr.allocate(instanceType); err != nil {
+		slog.Error("Insufficient resources to restart crashed instance",
+			"instance", instance.ID, "type", instance.InstanceType, "err", err)
+		return
+	}
+
 	// Transition Error → Pending (valid now that we added it to the transitions map)
 	if err := d.TransitionState(instance, vm.StatePending); err != nil {
 		slog.Error("Failed to transition instance to pending for restart",
 			"instance", instance.ID, "err", err)
+		d.resourceMgr.deallocate(instanceType)
 		return
 	}
 
@@ -271,6 +286,7 @@ func (d *Daemon) restartCrashedInstance(instance *vm.VM) {
 	if err := d.LaunchInstance(instance); err != nil {
 		slog.Error("Failed to restart crashed instance",
 			"instance", instance.ID, "err", err)
+		d.resourceMgr.deallocate(instanceType)
 		if err := d.TransitionState(instance, vm.StateError); err != nil {
 			slog.Error("Failed to transition instance back to error after restart failure",
 				"instance", instance.ID, "err", err)

spinifex/daemon/health_test.go

@@ -51,9 +51,12 @@ func TestHandleInstanceCrash_SkipsNonRunning(t *testing.T) {
 	require.NoError(t, err)
 	defer nc.Close()
 
+	rm, err := NewResourceManager()
+	require.NoError(t, err)
+
 	d := &Daemon{
 		natsConn:    nc,
-		resourceMgr: NewResourceManager(),
+		resourceMgr: rm,
 		Instances:   vm.Instances{VMS: make(map[string]*vm.VM)},
 	}
 
@@ -73,9 +76,12 @@ func TestHandleInstanceCrash_SkipsShuttingDown(t *testing.T) {
 	require.NoError(t, err)
 	defer nc.Close()
 
+	rm, err := NewResourceManager()
+	require.NoError(t, err)
+
 	d := &Daemon{
 		natsConn:    nc,
-		resourceMgr: NewResourceManager(),
+		resourceMgr: rm,
 		Instances:   vm.Instances{VMS: make(map[string]*vm.VM)},
 	}
 	d.shuttingDown.Store(true)
@@ -97,13 +103,16 @@ func TestMaybeRestart_ExceedsMaxInWindow(t *testing.T) {
 	require.NoError(t, err)
 	defer nc.Close()
 
+	rm, err := NewResourceManager()
+	require.NoError(t, err)
+
 	d := &Daemon{
 		node:     "test-node",
 		natsConn: nc,
 		config: &config.Config{
 			Services: []string{"daemon"},
 		},
-		resourceMgr: NewResourceManager(),
+		resourceMgr: rm,
 		Instances:   vm.Instances{VMS: make(map[string]*vm.VM)},
 	}
 
@@ -139,13 +148,16 @@ func TestMaybeRestart_ResetsAfterWindow(t *testing.T) {
 	require.NoError(t, err)
 	defer nc.Close()
 
+	rm, err := NewResourceManager()
+	require.NoError(t, err)
+
 	d := &Daemon{
 		node:     "test-node",
 		natsConn: nc,
 		config: &config.Config{
 			Services: []string{"daemon"},
 		},
-		resourceMgr: NewResourceManager(),
+		resourceMgr: rm,
 		Instances:   vm.Instances{VMS: make(map[string]*vm.VM)},
 	}
 
@@ -206,13 +218,16 @@ func newTestDaemon(t *testing.T) (*Daemon, func()) {
 	nc, err := nats.Connect(sharedNATSURL)
 	require.NoError(t, err)
 
+	rm, err := NewResourceManager()
+	require.NoError(t, err)
+
 	d := &Daemon{
 		node:     "test-node",
 		natsConn: nc,
 		config: &config.Config{
 			Services: []string{"daemon"},
 		},
-		resourceMgr: NewResourceManager(),
+		resourceMgr: rm,
 		Instances:   vm.Instances{VMS: make(map[string]*vm.VM)},
 	}
 	return d, func() { nc.Close() }

spinifex/daemon/heartbeat_test.go

@@ -12,6 +12,9 @@ import (
 
 // TestBuildHeartbeat verifies that buildHeartbeat populates the struct from daemon state.
 func TestBuildHeartbeat(t *testing.T) {
+	rm, err := NewResourceManager()
+	require.NoError(t, err)
+
 	d := &Daemon{
 		node: "test-node",
 		clusterConfig: &config.ClusterConfig{
@@ -20,7 +23,7 @@ func TestBuildHeartbeat(t *testing.T) {
 		config: &config.Config{
 			Services: []string{"daemon", "nats", "viperblock"},
 		},
-		resourceMgr: NewResourceManager(),
+		resourceMgr: rm,
 		Instances:   vm.Instances{VMS: make(map[string]*vm.VM)},
 	}
 
@@ -38,6 +41,9 @@ func TestBuildHeartbeat(t *testing.T) {
 
 // TestHeartbeatReflectsAllocation verifies that allocating resources changes the heartbeat values.
 func TestHeartbeatReflectsAllocation(t *testing.T) {
+	rm, err := NewResourceManager()
+	require.NoError(t, err)
+
 	d := &Daemon{
 		node: "alloc-node",
 		clusterConfig: &config.ClusterConfig{
@@ -46,7 +52,7 @@ func TestHeartbeatReflectsAllocation(t *testing.T) {
 		config: &config.Config{
 			Services: []string{"daemon"},
 		},
-		resourceMgr: NewResourceManager(),
+		resourceMgr: rm,
 		Instances:   vm.Instances{VMS: make(map[string]*vm.VM)},
 	}
 
@@ -64,7 +70,7 @@ func TestHeartbeatReflectsAllocation(t *testing.T) {
 	require.NotEmpty(t, allocType, "Should have at least one allocatable instance type")
 
 	// Allocate resources
-	err := d.resourceMgr.allocate(d.resourceMgr.instanceTypes[allocType])
+	err = d.resourceMgr.allocate(d.resourceMgr.instanceTypes[allocType])
 	require.NoError(t, err)
 
 	// Add a VM to the instance map

spinifex/daemon/jetstream.go

@@ -9,6 +9,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/mulgadc/spinifex/spinifex/vm"
 	"github.com/nats-io/nats.go"
 )
@@ -26,6 +27,11 @@ const (
 	TerminatedInstanceBucket = "spinifex-terminated-instances"
 	// TerminatedInstancePrefix is the key prefix for terminated instances
 	TerminatedInstancePrefix = "terminated."
+
+	// Schema versions for daemon KV buckets
+	InstanceStateBucketVersion      = 1
+	ClusterStateBucketVersion       = 1
+	TerminatedInstanceBucketVersion = 1
 )
 
 // JetStreamManager manages JetStream KV store operations for instance state
@@ -77,6 +83,9 @@ func (m *JetStreamManager) InitKVBucket() error {
 	}
 
 	m.kv = kv
+	if err := utils.WriteVersion(kv, InstanceStateBucketVersion); err != nil {
+		return fmt.Errorf("write version to %s: %w", InstanceStateBucket, err)
+	}
 	return nil
 }
 
@@ -104,6 +113,9 @@ func (m *JetStreamManager) InitClusterStateBucket() error {
 	}
 
 	m.clusterKV = kv
+	if err := utils.WriteVersion(kv, ClusterStateBucketVersion); err != nil {
+		return fmt.Errorf("write version to %s: %w", ClusterStateBucket, err)
+	}
 	return nil
 }
 
@@ -132,6 +144,9 @@ func (m *JetStreamManager) InitTerminatedInstanceBucket() error {
 	}
 
 	m.terminatedKV = kv
+	if err := utils.WriteVersion(kv, TerminatedInstanceBucketVersion); err != nil {
+		return fmt.Errorf("write version to %s: %w", TerminatedInstanceBucket, err)
+	}
 	return nil
 }
 
@@ -157,7 +172,8 @@ func isStreamUnavailable(err error) bool {
 // recoverBucket attempts to reconnect to or re-create a KV bucket after the
 // underlying JetStream stream was lost during cluster formation.
 // Returns the recovered KV handle directly so callers avoid a racy re-read.
-func (m *JetStreamManager) recoverBucket(cfg *nats.KeyValueConfig, field *nats.KeyValue) (nats.KeyValue, error) {
+// When a bucket is recreated, the schema version is re-stamped.
+func (m *JetStreamManager) recoverBucket(cfg *nats.KeyValueConfig, field *nats.KeyValue, version int) (nats.KeyValue, error) {
 	m.kvMu.Lock()
 	defer m.kvMu.Unlock()
 
@@ -183,6 +199,10 @@ func (m *JetStreamManager) recoverBucket(cfg *nats.KeyValueConfig, field *nats.K
 		return nil, err
 	}
 
+	if err := utils.WriteVersion(kv, version); err != nil {
+		slog.Error("Failed to write version to recreated bucket", "bucket", cfg.Bucket, "err", err)
+	}
+
 	*field = kv
 	slog.Info("KV bucket recreated successfully", "bucket", cfg.Bucket)
 	return kv, nil
@@ -192,7 +212,7 @@ func (m *JetStreamManager) recoverKVBucket() (nats.KeyValue, error) {
 	return m.recoverBucket(&nats.KeyValueConfig{
 		Bucket:      InstanceStateBucket,
 		Description: "Spinifex instance state storage",
-	}, &m.kv)
+	}, &m.kv, InstanceStateBucketVersion)
 }
 
 func (m *JetStreamManager) recoverTerminatedKVBucket() (nats.KeyValue, error) {
@@ -200,7 +220,7 @@ func (m *JetStreamManager) recoverTerminatedKVBucket() (nats.KeyValue, error) {
 		Bucket:      TerminatedInstanceBucket,
 		Description: "Terminated instances (auto-expire after 1 hour)",
 		TTL:         1 * time.Hour,
-	}, &m.terminatedKV)
+	}, &m.terminatedKV, TerminatedInstanceBucketVersion)
 }
 
 // Heartbeat represents a daemon's periodic health status published to cluster KV.
@@ -662,6 +682,9 @@ func (m *JetStreamManager) ListStoppedInstances() ([]*vm.VM, error) {
 
 	var instances []*vm.VM
 	for _, key := range keys {
+		if key == utils.VersionKey {
+			continue
+		}
 		if !strings.HasPrefix(key, StoppedInstancePrefix) {
 			continue
 		}
@@ -751,6 +774,9 @@ func (m *JetStreamManager) ListTerminatedInstances() ([]*vm.VM, error) {
 
 	var instances []*vm.VM
 	for _, key := range keys {
+		if key == utils.VersionKey {
+			continue
+		}
 		if !strings.HasPrefix(key, TerminatedInstancePrefix) {
 			continue
 		}

spinifex/daemon/jetstream_test.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"testing"
 
+	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/mulgadc/spinifex/spinifex/vm"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
@@ -1208,3 +1209,58 @@ func TestJetStreamManager_ListTerminatedInstances_RecoverAfterStreamLost(t *test
 	require.NoError(t, err, "ListTerminatedInstances should recover after stream loss")
 	assert.Empty(t, instances)
 }
+
+func TestJetStreamManager_InitBuckets_WritesVersion(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+
+	require.NoError(t, jsm.InitKVBucket())
+	require.NoError(t, jsm.InitClusterStateBucket())
+	require.NoError(t, jsm.InitTerminatedInstanceBucket())
+
+	v, err := utils.ReadVersion(jsm.kv)
+	require.NoError(t, err)
+	assert.Equal(t, InstanceStateBucketVersion, v)
+
+	v, err = utils.ReadVersion(jsm.clusterKV)
+	require.NoError(t, err)
+	assert.Equal(t, ClusterStateBucketVersion, v)
+
+	v, err = utils.ReadVersion(jsm.terminatedKV)
+	require.NoError(t, err)
+	assert.Equal(t, TerminatedInstanceBucketVersion, v)
+}
+
+func TestJetStreamManager_ListStoppedInstances_SkipsVersionKey(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	require.NoError(t, jsm.InitKVBucket())
+
+	// _version key is written by InitKVBucket; listing should not include it
+	instances, err := jsm.ListStoppedInstances()
+	require.NoError(t, err)
+	assert.Empty(t, instances)
+}
+
+func TestJetStreamManager_ListTerminatedInstances_SkipsVersionKey(t *testing.T) {
+	nc, err := nats.Connect(sharedJSNATSURL)
+	require.NoError(t, err)
+	defer nc.Close()
+
+	jsm, err := NewJetStreamManager(nc, 1)
+	require.NoError(t, err)
+	require.NoError(t, jsm.InitTerminatedInstanceBucket())
+
+	// _version key is written by InitTerminatedInstanceBucket; listing should not include it
+	instances, err := jsm.ListTerminatedInstances()
+	require.NoError(t, err)
+	assert.Empty(t, instances)
+}

spinifex/daemon/network.go

@@ -40,8 +40,12 @@ func (p *OVSNetworkPlumber) SetupTapDevice(eniId, mac string) error {
 	// 0. If tap already exists (e.g. unclean shutdown), clean it up first
 	if _, err := os.Stat("/sys/class/net/" + tapName); err == nil {
 		slog.Warn("Stale tap device found, cleaning up before recreate", "tap", tapName)
-		_ = sudoCommand("ovs-vsctl", "--if-exists", "del-port", "br-int", tapName).Run()
-		_ = sudoCommand("ip", "tuntap", "del", "dev", tapName, "mode", "tap").Run()
+		if err := sudoCommand("ovs-vsctl", "--if-exists", "del-port", "br-int", tapName).Run(); err != nil {
+			slog.Warn("Failed to remove stale tap from br-int", "tap", tapName, "err", err)
+		}
+		if err := sudoCommand("ip", "tuntap", "del", "dev", tapName, "mode", "tap").Run(); err != nil {
+			slog.Warn("Failed to delete stale tap device", "tap", tapName, "err", err)
+		}
 	}
 
 	// 1. Create tap device
@@ -51,7 +55,9 @@ func (p *OVSNetworkPlumber) SetupTapDevice(eniId, mac string) error {
 
 	// 2. Bring tap up
 	if out, err := sudoCommand("ip", "link", "set", tapName, "up").CombinedOutput(); err != nil {
-		_ = sudoCommand("ip", "tuntap", "del", "dev", tapName, "mode", "tap").Run()
+		if cleanErr := sudoCommand("ip", "tuntap", "del", "dev", tapName, "mode", "tap").Run(); cleanErr != nil {
+			slog.Warn("Failed to clean up tap device after bring-up failure", "tap", tapName, "err", cleanErr)
+		}
 		return fmt.Errorf("bring up tap %s: %s: %w", tapName, strings.TrimSpace(string(out)), err)
 	}
 
@@ -62,7 +68,9 @@ func (p *OVSNetworkPlumber) SetupTapDevice(eniId, mac string) error {
 		fmt.Sprintf("external_ids:iface-id=%s", ifaceID),
 		fmt.Sprintf("external_ids:attached-mac=%s", mac),
 	).CombinedOutput(); err != nil {
-		_ = sudoCommand("ip", "tuntap", "del", "dev", tapName, "mode", "tap").Run()
+		if cleanErr := sudoCommand("ip", "tuntap", "del", "dev", tapName, "mode", "tap").Run(); cleanErr != nil {
+			slog.Warn("Failed to clean up tap device after OVS failure", "tap", tapName, "err", cleanErr)
+		}
 		return fmt.Errorf("add tap to br-int: %s: %w", strings.TrimSpace(string(out)), err)
 	}
 

spinifex/daemon/network_test.go

@@ -403,7 +403,7 @@ func TestGenerateDevMAC_Format(t *testing.T) {
 					t.Errorf("generateDevMAC(%q) = %q, expected ':' at pos %d", id, mac, i)
 				}
 			} else {
-				if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')) {
+				if (c < '0' || c > '9') && (c < 'a' || c > 'f') {
 					t.Errorf("generateDevMAC(%q) = %q, invalid hex char '%c' at pos %d", id, mac, c, i)
 				}
 			}

spinifex/daemon/ownership_test.go

@@ -0,0 +1,63 @@
+package daemon
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/utils"
+)
+
+func TestIsInstanceVisible(t *testing.T) {
+	tests := []struct {
+		name            string
+		callerAccountID string
+		ownerAccountID  string
+		expected        bool
+	}{
+		{
+			name:            "same account",
+			callerAccountID: "123456789012",
+			ownerAccountID:  "123456789012",
+			expected:        true,
+		},
+		{
+			name:            "different account",
+			callerAccountID: "123456789012",
+			ownerAccountID:  "999999999999",
+			expected:        false,
+		},
+		{
+			name:            "pre-Phase4 instance visible to root",
+			callerAccountID: utils.GlobalAccountID,
+			ownerAccountID:  "",
+			expected:        true,
+		},
+		{
+			name:            "pre-Phase4 instance hidden from non-root",
+			callerAccountID: "123456789012",
+			ownerAccountID:  "",
+			expected:        false,
+		},
+		{
+			name:            "root accessing owned instance",
+			callerAccountID: utils.GlobalAccountID,
+			ownerAccountID:  utils.GlobalAccountID,
+			expected:        true,
+		},
+		{
+			name:            "root accessing other account instance",
+			callerAccountID: utils.GlobalAccountID,
+			ownerAccountID:  "123456789012",
+			expected:        false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := isInstanceVisible(tt.callerAccountID, tt.ownerAccountID)
+			if got != tt.expected {
+				t.Errorf("isInstanceVisible(%q, %q) = %v, want %v",
+					tt.callerAccountID, tt.ownerAccountID, got, tt.expected)
+			}
+		})
+	}
+}

spinifex/daemon/resource.go

@@ -16,7 +16,6 @@ var errInsufficientCapacity = errors.New("insufficient capacity to satisfy MinCo
 // or side effects.
 func canAllocateCount(availVCPU, allocVCPU int, availMem, allocMem float64,
 	vCPUs int64, memMiB int64, maxCount int) int {
-
 	remainingVCPU := availVCPU - allocVCPU
 	remainingMem := availMem - allocMem
 	memoryGB := float64(memMiB) / 1024.0

spinifex/daemon/resource_test.go

@@ -201,11 +201,11 @@ func TestResourceStatsForType(t *testing.T) {
 
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
-			cap := resourceStatsForType(tc.remainVCPU, tc.remainMem, tc.it)
-			assert.Equal(t, tc.wantName, cap.Name)
-			assert.Equal(t, tc.wantVCPU, cap.VCPU)
-			assert.InDelta(t, tc.wantMemGB, cap.MemoryGB, 0.001)
-			assert.Equal(t, tc.wantAvail, cap.Available)
+			typeCap := resourceStatsForType(tc.remainVCPU, tc.remainMem, tc.it)
+			assert.Equal(t, tc.wantName, typeCap.Name)
+			assert.Equal(t, tc.wantVCPU, typeCap.VCPU)
+			assert.InDelta(t, tc.wantMemGB, typeCap.MemoryGB, 0.001)
+			assert.Equal(t, tc.wantAvail, typeCap.Available)
 		})
 	}
 }

spinifex/daemon/state_test.go

@@ -2,7 +2,8 @@ package daemon
 
 import (
 	"fmt"
-	"os"
+	"net"
+	"path/filepath"
 	"sync"
 	"testing"
 	"time"
@@ -22,9 +23,7 @@ import (
 func createDaemonWithJetStream(t *testing.T) *Daemon {
 	t.Helper()
 
-	jsTmpDir, err := os.MkdirTemp("", "nats-js-state-test-*")
-	require.NoError(t, err)
-	t.Cleanup(func() { os.RemoveAll(jsTmpDir) })
+	jsTmpDir := t.TempDir()
 
 	opts := &server.Options{
 		Host:      "127.0.0.1",
@@ -42,15 +41,14 @@ func createDaemonWithJetStream(t *testing.T) *Daemon {
 
 	natsURL := ns.ClientURL()
 
-	tmpDir, err := os.MkdirTemp("", "spx-state-test-*")
-	require.NoError(t, err)
-	t.Cleanup(func() { os.RemoveAll(tmpDir) })
+	tmpDir := t.TempDir()
 
 	clusterCfg := &config.ClusterConfig{
 		Node:  "node-1",
 		Nodes: map[string]config.Config{"node-1": {BaseDir: tmpDir}},
 	}
-	daemon := NewDaemon(clusterCfg)
+	daemon, err := NewDaemon(clusterCfg)
+	require.NoError(t, err)
 	daemon.config = &config.Config{BaseDir: tmpDir}
 
 	nc, err := nats.Connect(natsURL)
@@ -257,7 +255,7 @@ func TestEveryState_CanReachTerminated(t *testing.T) {
 			continue
 		}
 
-		visited := map[vm.InstanceState]bool{start: true}
+		visited := map[vm.InstanceState]bool{start: true} //nolint:exhaustive // BFS seed — populated dynamically
 		queue := []vm.InstanceState{start}
 		found := false
 
@@ -403,7 +401,7 @@ func TestTransitionState_ConcurrentTransitions(t *testing.T) {
 
 // reachableStates returns all states reachable from start via BFS.
 func reachableStates(start vm.InstanceState) map[vm.InstanceState]bool {
-	visited := map[vm.InstanceState]bool{start: true}
+	visited := map[vm.InstanceState]bool{start: true} //nolint:exhaustive // BFS seed — populated dynamically
 	queue := []vm.InstanceState{start}
 	for len(queue) > 0 {
 		current := queue[0]
@@ -1043,3 +1041,152 @@ func TestPendingWatchdog_MarksStuckInstanceFailed(t *testing.T) {
 	freshInst := daemon.Instances.VMS["i-fresh"]
 	assert.Equal(t, vm.StatePending, freshInst.Status)
 }
+
+func TestAreVolumeSocketsValid_ValidSocket(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	// Create a real Unix socket listener
+	sockPath := filepath.Join(t.TempDir(), "test.sock")
+	listener, err := net.Listen("unix", sockPath)
+	require.NoError(t, err)
+	defer listener.Close()
+
+	instance := &vm.VM{
+		ID: "i-valid-sock",
+		EBSRequests: types.EBSRequests{
+			Requests: []types.EBSRequest{
+				{Name: "vol-1", NBDURI: "nbd:unix:" + sockPath},
+			},
+		},
+	}
+	assert.True(t, daemon.areVolumeSocketsValid(instance))
+}
+
+func TestAreVolumeSocketsValid_MissingSocket(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	instance := &vm.VM{
+		ID: "i-missing-sock",
+		EBSRequests: types.EBSRequests{
+			Requests: []types.EBSRequest{
+				{Name: "vol-1", NBDURI: "nbd:unix:/nonexistent/path.sock"},
+			},
+		},
+	}
+	assert.False(t, daemon.areVolumeSocketsValid(instance))
+}
+
+func TestAreVolumeSocketsValid_StaleSocketFile(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	// Create a socket file with no listener — simulates viperblock crash
+	// leaving a stale socket on disk
+	sockPath := filepath.Join(t.TempDir(), "stale.sock")
+	listener, err := net.Listen("unix", sockPath)
+	require.NoError(t, err)
+	listener.Close() // close listener but leave file on disk
+
+	instance := &vm.VM{
+		ID: "i-stale-sock",
+		EBSRequests: types.EBSRequests{
+			Requests: []types.EBSRequest{
+				{Name: "vol-1", NBDURI: "nbd:unix:" + sockPath},
+			},
+		},
+	}
+	// os.Stat would return true here, but dial should fail
+	assert.False(t, daemon.areVolumeSocketsValid(instance))
+}
+
+func TestAreVolumeSocketsValid_EmptyRequests(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	instance := &vm.VM{
+		ID:          "i-no-vols",
+		EBSRequests: types.EBSRequests{},
+	}
+	assert.True(t, daemon.areVolumeSocketsValid(instance))
+}
+
+func TestAreVolumeSocketsValid_TCPTransport(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	instance := &vm.VM{
+		ID: "i-tcp",
+		EBSRequests: types.EBSRequests{
+			Requests: []types.EBSRequest{
+				{Name: "vol-tcp", NBDURI: "nbd://192.168.1.1:10809"},
+			},
+		},
+	}
+	// TCP transport can't be validated locally — should return true
+	assert.True(t, daemon.areVolumeSocketsValid(instance))
+}
+
+func TestMarkInstanceFailed_AlreadyShuttingDown(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	instance := &vm.VM{
+		ID:       "i-already-shutting",
+		Status:   vm.StateShuttingDown,
+		Instance: &ec2.Instance{},
+	}
+	daemon.Instances.VMS[instance.ID] = instance
+
+	// Should be a no-op — instance is already being cleaned up
+	daemon.markInstanceFailed(instance, "test_reason")
+
+	// Status should not change
+	assert.Equal(t, vm.StateShuttingDown, instance.Status)
+}
+
+func TestMarkInstanceFailed_AlreadyTerminated(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	instance := &vm.VM{
+		ID:       "i-already-term",
+		Status:   vm.StateTerminated,
+		Instance: &ec2.Instance{},
+	}
+	daemon.Instances.VMS[instance.ID] = instance
+
+	daemon.markInstanceFailed(instance, "test_reason")
+
+	// Status should not change
+	assert.Equal(t, vm.StateTerminated, instance.Status)
+}
+
+func TestRestoreInstances_ProvisioningInstanceLaunchTimeReset(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	// Provisioning instances (daemon crashed during first launch) should get
+	// their LaunchTime reset so the pending watchdog doesn't immediately kill
+	// them after a prolonged outage.
+	staleTime := time.Now().Add(-30 * time.Minute)
+	daemon.Instances.VMS["i-provisioning"] = &vm.VM{
+		ID:       "i-provisioning",
+		Status:   vm.StateProvisioning,
+		Instance: &ec2.Instance{LaunchTime: &staleTime},
+	}
+	require.NoError(t, daemon.WriteState())
+
+	daemon.Instances.VMS = make(map[string]*vm.VM)
+	before := time.Now()
+	simulateCleanRestore(t, daemon)
+
+	// LoadState inside restoreInstances creates fresh objects, so look up
+	// from the map. The instance may have been removed by markInstanceFailed
+	// (launch fails in test env), so load directly from JetStream state.
+	require.NoError(t, daemon.LoadState())
+	instance := daemon.Instances.VMS["i-provisioning"]
+	if instance == nil {
+		// Instance was cleaned up by markInstanceFailed — that's fine.
+		// The important thing is that the LaunchTime WAS reset before
+		// the launch attempt. We can't verify it post-cleanup, so skip.
+		t.Skip("Instance was cleaned up during recovery — LaunchTime was reset before launch attempt")
+	}
+	require.NotNil(t, instance.Instance)
+	require.NotNil(t, instance.Instance.LaunchTime)
+	assert.True(t, instance.Instance.LaunchTime.After(before) || instance.Instance.LaunchTime.Equal(before),
+		"LaunchTime should be reset for provisioning instances, got %v", *instance.Instance.LaunchTime)
+}

spinifex/gateway/account.go

@@ -3,6 +3,5 @@ package gateway
 import "net/http"
 
 func (gw *GatewayConfig) Account_Request(w http.ResponseWriter, r *http.Request) error {
-
 	return nil
 }

spinifex/gateway/auth.go

@@ -28,7 +28,7 @@ func (gw *GatewayConfig) SigV4AuthMiddleware() func(http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			// Skip OPTIONS requests (CORS preflight)
-			if r.Method == "OPTIONS" {
+			if r.Method == http.MethodOptions {
 				next.ServeHTTP(w, r)
 				return
 			}
@@ -154,8 +154,7 @@ func (gw *GatewayConfig) SigV4AuthMiddleware() func(http.Handler) http.Handler {
 			// Compare signatures using constant-time comparison to prevent timing attacks
 			if subtle.ConstantTimeCompare([]byte(expectedSignature), []byte(providedSignature)) != 1 {
 				slog.Debug("Signature mismatch",
-					"expected", expectedSignature,
-					"provided", providedSignature,
+					"accessKeyId", accessKey,
 				)
 				gw.writeSigV4Error(w, r, awserrors.ErrorSignatureDoesNotMatch)
 				return
@@ -302,7 +301,7 @@ func canonicalHeaderName(header string) string {
 
 // writeSigV4Error writes an EC2-compatible XML error response for authentication failures.
 func (gw *GatewayConfig) writeSigV4Error(w http.ResponseWriter, r *http.Request, errorCode string) {
-	requestID := r.Header.Get("x-amz-request-id")
+	requestID := r.Header.Get("X-Amz-Request-Id")
 	if requestID == "" {
 		requestID = uuid.NewString()
 	}

spinifex/gateway/auth_test.go

@@ -211,12 +211,12 @@ func setupTestApp(accessKey, secretKey string) http.Handler {
 func TestSigV4Auth_NoAuthorizationHeader(t *testing.T) {
 	handler := setupTestApp(testAccessKey, testSecretKey)
 
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
 
 	resp := doRequest(handler, req)
 
-	if resp.StatusCode != 403 {
+	if resp.StatusCode != http.StatusForbidden {
 		t.Errorf("Expected status 403, got %d", resp.StatusCode)
 	}
 
@@ -242,14 +242,14 @@ func TestSigV4Auth_MalformedHeader(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			req := httptest.NewRequest("GET", "/", nil)
+			req := httptest.NewRequest(http.MethodGet, "/", nil)
 			req.Host = "localhost:9999"
 			req.Header.Set("Authorization", tc.authHeader)
 			req.Header.Set("X-Amz-Date", time.Now().UTC().Format(auth.TimeFormat))
 
 			resp := doRequest(handler, req)
 
-			if resp.StatusCode != 400 {
+			if resp.StatusCode != http.StatusBadRequest {
 				t.Errorf("Expected status 400, got %d", resp.StatusCode)
 			}
 
@@ -269,14 +269,14 @@ func TestSigV4Auth_InvalidAccessKey(t *testing.T) {
 		"INVALID_ACCESS_KEY", testSecretKey, testRegion, testService,
 	)
 
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
 
 	resp := doRequest(handler, req)
 
-	if resp.StatusCode != 403 {
+	if resp.StatusCode != http.StatusForbidden {
 		t.Errorf("Expected status 403, got %d", resp.StatusCode)
 	}
 
@@ -295,14 +295,14 @@ func TestSigV4Auth_InvalidSignature(t *testing.T) {
 		testAccessKey, "WRONG_SECRET_KEY", testRegion, testService,
 	)
 
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
 
 	resp := doRequest(handler, req)
 
-	if resp.StatusCode != 403 {
+	if resp.StatusCode != http.StatusForbidden {
 		t.Errorf("Expected status 403, got %d", resp.StatusCode)
 	}
 
@@ -320,14 +320,14 @@ func TestSigV4Auth_ValidSignature(t *testing.T) {
 		testAccessKey, testSecretKey, testRegion, testService,
 	)
 
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
 
 	resp := doRequest(handler, req)
 
-	if resp.StatusCode != 200 {
+	if resp.StatusCode != http.StatusOK {
 		body, _ := io.ReadAll(resp.Body)
 		t.Errorf("Expected status 200, got %d, body: %s", resp.StatusCode, string(body))
 	}
@@ -336,13 +336,13 @@ func TestSigV4Auth_ValidSignature(t *testing.T) {
 func TestSigV4Auth_OptionsSkipsAuth(t *testing.T) {
 	handler := setupTestApp(testAccessKey, testSecretKey)
 
-	req := httptest.NewRequest("OPTIONS", "/", nil)
+	req := httptest.NewRequest(http.MethodOptions, "/", nil)
 	req.Host = "localhost:9999"
 	// No Authorization header
 
 	resp := doRequest(handler, req)
 
-	if resp.StatusCode != 200 {
+	if resp.StatusCode != http.StatusOK {
 		t.Errorf("Expected status 200 for OPTIONS, got %d", resp.StatusCode)
 	}
 }
@@ -356,7 +356,7 @@ func TestSigV4Auth_ValidSignatureWithBody(t *testing.T) {
 		testAccessKey, testSecretKey, testRegion, testService,
 	)
 
-	req := httptest.NewRequest("POST", "/", strings.NewReader(body))
+	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
@@ -364,7 +364,7 @@ func TestSigV4Auth_ValidSignatureWithBody(t *testing.T) {
 
 	resp := doRequest(handler, req)
 
-	if resp.StatusCode != 200 {
+	if resp.StatusCode != http.StatusOK {
 		respBody, _ := io.ReadAll(resp.Body)
 		t.Errorf("Expected status 200, got %d, body: %s", resp.StatusCode, string(respBody))
 	}
@@ -379,14 +379,14 @@ func TestSigV4Auth_ValidSignatureWithQueryString(t *testing.T) {
 		testAccessKey, testSecretKey, testRegion, testService,
 	)
 
-	req := httptest.NewRequest("GET", "/?"+queryString, nil)
+	req := httptest.NewRequest(http.MethodGet, "/?"+queryString, nil)
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
 
 	resp := doRequest(handler, req)
 
-	if resp.StatusCode != 200 {
+	if resp.StatusCode != http.StatusOK {
 		respBody, _ := io.ReadAll(resp.Body)
 		t.Errorf("Expected status 200, got %d, body: %s", resp.StatusCode, string(respBody))
 	}
@@ -427,14 +427,14 @@ func TestSigV4Auth_InactiveAccessKey(t *testing.T) {
 		testAccessKey, testSecretKey, testRegion, testService,
 	)
 
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
 
 	resp := doRequest(r, req)
 
-	if resp.StatusCode != 403 {
+	if resp.StatusCode != http.StatusForbidden {
 		t.Errorf("Expected status 403 for inactive key, got %d", resp.StatusCode)
 	}
 
@@ -462,14 +462,14 @@ func TestSigV4Auth_NilIAMService(t *testing.T) {
 		testAccessKey, testSecretKey, testRegion, testService,
 	)
 
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
 
 	resp := doRequest(r, req)
 
-	if resp.StatusCode != 500 {
+	if resp.StatusCode != http.StatusInternalServerError {
 		t.Errorf("Expected status 500 for nil IAM service, got %d", resp.StatusCode)
 	}
 
@@ -515,14 +515,14 @@ func TestSigV4Auth_CallerIdentity(t *testing.T) {
 		testAccessKey, testSecretKey, testRegion, testService,
 	)
 
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
 
 	resp := doRequest(r, req)
 
-	if resp.StatusCode != 200 {
+	if resp.StatusCode != http.StatusOK {
 		body, _ := io.ReadAll(resp.Body)
 		t.Fatalf("Expected status 200, got %d, body: %s", resp.StatusCode, string(body))
 	}
@@ -650,14 +650,14 @@ func TestSigV4Auth_DecryptFailure(t *testing.T) {
 		testAccessKey, testSecretKey, testRegion, testService,
 	)
 
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
 
 	resp := doRequest(r, req)
 
-	if resp.StatusCode != 500 {
+	if resp.StatusCode != http.StatusInternalServerError {
 		t.Errorf("Expected status 500 for decrypt failure, got %d", resp.StatusCode)
 	}
 
@@ -786,14 +786,14 @@ func TestSigV4Auth_ExpiredTimestamp(t *testing.T) {
 		testAccessKey, testSecretKey, testRegion, testService, past,
 	)
 
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
 
 	resp := doRequest(handler, req)
 
-	if resp.StatusCode != 403 {
+	if resp.StatusCode != http.StatusForbidden {
 		t.Errorf("Expected status 403, got %d", resp.StatusCode)
 	}
 
@@ -813,14 +813,14 @@ func TestSigV4Auth_FutureTimestamp(t *testing.T) {
 		testAccessKey, testSecretKey, testRegion, testService, future,
 	)
 
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
 
 	resp := doRequest(handler, req)
 
-	if resp.StatusCode != 403 {
+	if resp.StatusCode != http.StatusForbidden {
 		t.Errorf("Expected status 403, got %d", resp.StatusCode)
 	}
 
@@ -840,14 +840,14 @@ func TestSigV4Auth_TimestampWithinSkew(t *testing.T) {
 		testAccessKey, testSecretKey, testRegion, testService, recent,
 	)
 
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
 
 	resp := doRequest(handler, req)
 
-	if resp.StatusCode != 200 {
+	if resp.StatusCode != http.StatusOK {
 		body, _ := io.ReadAll(resp.Body)
 		t.Errorf("Expected status 200, got %d, body: %s", resp.StatusCode, string(body))
 	}
@@ -878,18 +878,18 @@ func TestSigV4Auth_ClockSkewBoundary(t *testing.T) {
 				testAccessKey, testSecretKey, testRegion, testService, at,
 			)
 
-			req := httptest.NewRequest("GET", "/", nil)
+			req := httptest.NewRequest(http.MethodGet, "/", nil)
 			req.Host = "localhost:9999"
 			req.Header.Set("Authorization", authHeader)
 			req.Header.Set("X-Amz-Date", timestamp)
 
 			resp := doRequest(handler, req)
 
-			if tc.expectPass && resp.StatusCode != 200 {
+			if tc.expectPass && resp.StatusCode != http.StatusOK {
 				body, _ := io.ReadAll(resp.Body)
 				t.Errorf("Expected 200, got %d, body: %s", resp.StatusCode, string(body))
 			}
-			if !tc.expectPass && resp.StatusCode != 403 {
+			if !tc.expectPass && resp.StatusCode != http.StatusForbidden {
 				t.Errorf("Expected 403, got %d", resp.StatusCode)
 			}
 		})
@@ -904,14 +904,14 @@ func TestSigV4Auth_MissingXAmzDate(t *testing.T) {
 		testAccessKey, testSecretKey, testRegion, testService,
 	)
 
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	// Deliberately omit X-Amz-Date
 
 	resp := doRequest(handler, req)
 
-	if resp.StatusCode != 400 {
+	if resp.StatusCode != http.StatusBadRequest {
 		t.Errorf("Expected status 400, got %d", resp.StatusCode)
 	}
 
@@ -929,14 +929,14 @@ func TestSigV4Auth_MalformedTimestamp(t *testing.T) {
 		testAccessKey, testSecretKey, testRegion, testService,
 	)
 
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", "not-a-valid-date")
 
 	resp := doRequest(handler, req)
 
-	if resp.StatusCode != 400 {
+	if resp.StatusCode != http.StatusBadRequest {
 		t.Errorf("Expected status 400, got %d", resp.StatusCode)
 	}
 
@@ -965,7 +965,7 @@ func TestWriteSigV4Error_ResponseFormat(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.errorCode, func(t *testing.T) {
 			w := httptest.NewRecorder()
-			req := httptest.NewRequest("GET", "/", nil)
+			req := httptest.NewRequest(http.MethodGet, "/", nil)
 
 			gw.writeSigV4Error(w, req, tc.errorCode)
 			resp := w.Result()
@@ -1002,8 +1002,8 @@ func TestWriteSigV4Error_PreservesRequestID(t *testing.T) {
 	gw := &GatewayConfig{DisableLogging: true, Region: testRegion}
 
 	w := httptest.NewRecorder()
-	req := httptest.NewRequest("GET", "/", nil)
-	req.Header.Set("x-amz-request-id", "test-request-id-123")
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
+	req.Header.Set("X-Amz-Request-Id", "test-request-id-123")
 
 	gw.writeSigV4Error(w, req, awserrors.ErrorIncompleteSignature)
 
@@ -1059,14 +1059,14 @@ func TestSigV4Auth_ContextPropagation(t *testing.T) {
 		testAccessKey, testSecretKey, testRegion, testService,
 	)
 
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
 
 	resp := doRequest(r, req)
 
-	if resp.StatusCode != 200 {
+	if resp.StatusCode != http.StatusOK {
 		body, _ := io.ReadAll(resp.Body)
 		t.Fatalf("Expected status 200, got %d, body: %s", resp.StatusCode, string(body))
 	}
@@ -1142,7 +1142,7 @@ func TestSigV4Auth_ContextDoesNotLeakBetweenRequests(t *testing.T) {
 		"GET", "/", "", "",
 		testAccessKey, testSecretKey, testRegion, testService,
 	)
-	req1 := httptest.NewRequest("GET", "/", nil)
+	req1 := httptest.NewRequest(http.MethodGet, "/", nil)
 	req1.Host = "localhost:9999"
 	req1.Header.Set("Authorization", authHeader1)
 	req1.Header.Set("X-Amz-Date", timestamp1)
@@ -1158,7 +1158,7 @@ func TestSigV4Auth_ContextDoesNotLeakBetweenRequests(t *testing.T) {
 		"GET", "/", "", "",
 		secondKey, testSecretKey, testRegion, testService,
 	)
-	req2 := httptest.NewRequest("GET", "/", nil)
+	req2 := httptest.NewRequest(http.MethodGet, "/", nil)
 	req2.Host = "localhost:9999"
 	req2.Header.Set("Authorization", authHeader2)
 	req2.Header.Set("X-Amz-Date", timestamp2)
@@ -1191,14 +1191,14 @@ func TestSigV4Auth_LookupAccessKeyUnexpectedError(t *testing.T) {
 		testAccessKey, testSecretKey, testRegion, testService,
 	)
 
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
 
 	resp := doRequest(r, req)
 
-	if resp.StatusCode != 500 {
+	if resp.StatusCode != http.StatusInternalServerError {
 		t.Errorf("Expected status 500 for unexpected lookup error, got %d", resp.StatusCode)
 	}
 
@@ -1230,14 +1230,14 @@ func TestSigV4Auth_PathWithSpecialCharacters(t *testing.T) {
 				testAccessKey, testSecretKey, testRegion, testService,
 			)
 
-			req := httptest.NewRequest("GET", tc.reqPath, nil)
+			req := httptest.NewRequest(http.MethodGet, tc.reqPath, nil)
 			req.Host = "localhost:9999"
 			req.Header.Set("Authorization", authHeader)
 			req.Header.Set("X-Amz-Date", timestamp)
 
 			resp := doRequest(handler, req)
 
-			if resp.StatusCode != 200 {
+			if resp.StatusCode != http.StatusOK {
 				body, _ := io.ReadAll(resp.Body)
 				t.Errorf("Expected status 200, got %d, body: %s", resp.StatusCode, string(body))
 			}
@@ -1291,7 +1291,7 @@ func TestSigV4Auth_SignedContentType(t *testing.T) {
 		headers, "content-type;host;x-amz-date",
 	)
 
-	req := httptest.NewRequest("POST", "/", strings.NewReader(body))
+	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
@@ -1299,7 +1299,7 @@ func TestSigV4Auth_SignedContentType(t *testing.T) {
 
 	resp := doRequest(r, req)
 
-	if resp.StatusCode != 200 {
+	if resp.StatusCode != http.StatusOK {
 		respBody, _ := io.ReadAll(resp.Body)
 		t.Errorf("Expected status 200, got %d, body: %s", resp.StatusCode, string(respBody))
 	}
@@ -1314,14 +1314,14 @@ func TestSigV4Auth_EmptyBodyPOST(t *testing.T) {
 		testAccessKey, testSecretKey, testRegion, testService,
 	)
 
-	req := httptest.NewRequest("POST", "/", strings.NewReader(""))
+	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(""))
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
 
 	resp := doRequest(handler, req)
 
-	if resp.StatusCode != 200 {
+	if resp.StatusCode != http.StatusOK {
 		body, _ := io.ReadAll(resp.Body)
 		t.Errorf("Expected status 200, got %d, body: %s", resp.StatusCode, string(body))
 	}
@@ -1388,7 +1388,7 @@ func TestComputeSignatureWithSecret_SpecialURIPaths(t *testing.T) {
 			if reqPath == "" {
 				reqPath = "/"
 			}
-			req := httptest.NewRequest("GET", reqPath, nil)
+			req := httptest.NewRequest(http.MethodGet, reqPath, nil)
 			req.Host = "localhost:9999"
 			req.Header.Set("X-Amz-Date", timestamp)
 
@@ -1412,7 +1412,7 @@ func TestComputeSignatureWithSecret_Determinism(t *testing.T) {
 
 	var first string
 	for i := range 10 {
-		req := httptest.NewRequest("POST", "/", nil)
+		req := httptest.NewRequest(http.MethodPost, "/", nil)
 		req.Host = "localhost:9999"
 		req.Header.Set("X-Amz-Date", timestamp)
 
@@ -1442,7 +1442,7 @@ func TestComputeSignatureWithSecret_MultipartContentType(t *testing.T) {
 		headers, "content-type;host;x-amz-date",
 	)
 
-	req := httptest.NewRequest("POST", "/", strings.NewReader(body))
+	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
@@ -1450,7 +1450,7 @@ func TestComputeSignatureWithSecret_MultipartContentType(t *testing.T) {
 
 	resp := doRequest(handler, req)
 
-	if resp.StatusCode != 200 {
+	if resp.StatusCode != http.StatusOK {
 		respBody, _ := io.ReadAll(resp.Body)
 		t.Errorf("Expected status 200 for multipart content-type, got %d, body: %s", resp.StatusCode, string(respBody))
 	}
@@ -1475,14 +1475,14 @@ func TestSigV4Auth_QueryStringEdgeCases(t *testing.T) {
 				testAccessKey, testSecretKey, testRegion, testService,
 			)
 
-			req := httptest.NewRequest("GET", "/?"+tc.queryString, nil)
+			req := httptest.NewRequest(http.MethodGet, "/?"+tc.queryString, nil)
 			req.Host = "localhost:9999"
 			req.Header.Set("Authorization", authHeader)
 			req.Header.Set("X-Amz-Date", timestamp)
 
 			resp := doRequest(handler, req)
 
-			if resp.StatusCode != 200 {
+			if resp.StatusCode != http.StatusOK {
 				body, _ := io.ReadAll(resp.Body)
 				t.Errorf("Expected status 200, got %d, body: %s", resp.StatusCode, string(body))
 			}
@@ -1495,6 +1495,7 @@ func TestSigV4Auth_QueryStringEdgeCases(t *testing.T) {
 // policyMockIAMService extends mockIAMService with configurable GetUserPolicies.
 type policyMockIAMService struct {
 	mockIAMService
+
 	getUserPoliciesFn func(accountID, userName string) ([]handlers_iam.PolicyDocument, error)
 }
 
@@ -1508,6 +1509,7 @@ func (m *policyMockIAMService) GetUserPolicies(accountID, userName string) ([]ha
 // errorLookupMockIAMService returns a configurable error from LookupAccessKey.
 type errorLookupMockIAMService struct {
 	mockIAMService
+
 	err error
 }
 
@@ -1566,14 +1568,14 @@ func TestCheckPolicy_NonRootNoPolicies_Denied(t *testing.T) {
 		testAccessKey, testSecretKey, testRegion, testService,
 	)
 
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
 
 	resp := doRequest(handler, req)
 
-	if resp.StatusCode != 403 {
+	if resp.StatusCode != http.StatusForbidden {
 		body, _ := io.ReadAll(resp.Body)
 		t.Errorf("Expected status 403, got %d, body: %s", resp.StatusCode, string(body))
 	}
@@ -1632,14 +1634,14 @@ func TestCheckPolicy_NonRootWithAllow_Passes(t *testing.T) {
 		testAccessKey, testSecretKey, testRegion, testService,
 	)
 
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
 
 	resp := doRequest(handler, req)
 
-	if resp.StatusCode != 200 {
+	if resp.StatusCode != http.StatusOK {
 		body, _ := io.ReadAll(resp.Body)
 		t.Errorf("Expected status 200, got %d, body: %s", resp.StatusCode, string(body))
 	}
@@ -1698,14 +1700,14 @@ func TestCheckPolicy_NonRootWithExplicitDeny_Denied(t *testing.T) {
 		testAccessKey, testSecretKey, testRegion, testService,
 	)
 
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
 
 	resp := doRequest(handler, req)
 
-	if resp.StatusCode != 403 {
+	if resp.StatusCode != http.StatusForbidden {
 		body, _ := io.ReadAll(resp.Body)
 		t.Errorf("Expected status 403, got %d, body: %s", resp.StatusCode, string(body))
 	}
@@ -1765,14 +1767,14 @@ func TestCheckPolicy_RootGlobalAccount_Bypasses(t *testing.T) {
 		testAccessKey, testSecretKey, testRegion, testService,
 	)
 
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
 
 	resp := doRequest(handler, req)
 
-	if resp.StatusCode != 200 {
+	if resp.StatusCode != http.StatusOK {
 		body, _ := io.ReadAll(resp.Body)
 		t.Errorf("Expected status 200 (root bypass), got %d, body: %s", resp.StatusCode, string(body))
 	}
@@ -1813,14 +1815,14 @@ func TestCheckPolicy_MissingAccountID_InternalError(t *testing.T) {
 		testAccessKey, testSecretKey, testRegion, testService,
 	)
 
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
 
 	resp := doRequest(handler, req)
 
-	if resp.StatusCode != 500 {
+	if resp.StatusCode != http.StatusInternalServerError {
 		body, _ := io.ReadAll(resp.Body)
 		t.Errorf("Expected status 500, got %d, body: %s", resp.StatusCode, string(body))
 	}
@@ -1863,14 +1865,14 @@ func TestCheckPolicy_GetUserPoliciesError_InternalError(t *testing.T) {
 		testAccessKey, testSecretKey, testRegion, testService,
 	)
 
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
 
 	resp := doRequest(handler, req)
 
-	if resp.StatusCode != 500 {
+	if resp.StatusCode != http.StatusInternalServerError {
 		body, _ := io.ReadAll(resp.Body)
 		t.Errorf("Expected status 500, got %d, body: %s", resp.StatusCode, string(body))
 	}
@@ -1915,7 +1917,7 @@ func TestCheckPolicy_RootNonGlobalAccount_StillEvaluated(t *testing.T) {
 		testAccessKey, testSecretKey, testRegion, testService,
 	)
 
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
 	req.Header.Set("Authorization", authHeader)
 	req.Header.Set("X-Amz-Date", timestamp)
@@ -1924,7 +1926,7 @@ func TestCheckPolicy_RootNonGlobalAccount_StillEvaluated(t *testing.T) {
 
 	// root on non-global account is evaluated by the policy engine like any
 	// other user. With no policies attached, the default deny applies.
-	if resp.StatusCode != 403 {
+	if resp.StatusCode != http.StatusForbidden {
 		body, _ := io.ReadAll(resp.Body)
 		t.Errorf("Expected status 403, got %d, body: %s", resp.StatusCode, string(body))
 	}

spinifex/gateway/ec2.go

@@ -269,6 +269,8 @@ func (gw *GatewayConfig) EC2_Request(w http.ResponseWriter, r *http.Request) err
 
 	w.Header().Set("Content-Type", "text/xml")
 	w.WriteHeader(http.StatusOK)
-	_, _ = w.Write(xmlOutput)
+	if _, err := w.Write(xmlOutput); err != nil {
+		slog.Error("Failed to write EC2 response", "err", err)
+	}
 	return nil
 }

spinifex/gateway/ec2/image/DescribeImages.go

@@ -24,11 +24,10 @@ func ValidateDescribeImagesInput(input *ec2.DescribeImagesInput) (err error) {
 		}
 	}
 
-	return
+	return err
 }
 
 func DescribeImages(input *ec2.DescribeImagesInput, natsConn *nats.Conn, accountID string) (output ec2.DescribeImagesOutput, err error) {
-
 	// Validate input
 	err = ValidateDescribeImagesInput(input)
 

spinifex/gateway/ec2/instance/RebootInstances.go

@@ -57,7 +57,10 @@ func RebootInstances(input *ec2.RebootInstancesInput, natsConn *nats.Conn, accou
 			describeInput := &ec2.DescribeInstancesInput{
 				InstanceIds: []*string{&instanceID},
 			}
-			describeData, _ := json.Marshal(describeInput)
+			describeData, err := json.Marshal(describeInput)
+			if err != nil {
+				return nil, fmt.Errorf("marshal describe input: %w", err)
+			}
 			if reservations := queryInstanceBucket(natsConn, "ec2.DescribeStoppedInstances", describeData, accountID); len(reservations) > 0 {
 				return nil, errors.New(awserrors.ErrorIncorrectInstanceState)
 			}

spinifex/gateway/ec2/instance/RunInstances.go

@@ -53,15 +53,12 @@ func ValidateRunInstancesInput(input *ec2.RunInstancesInput) (err error) {
 
 	if !strings.HasPrefix(*input.ImageId, "ami-") {
 		return errors.New(awserrors.ErrorInvalidAMIIDMalformed)
-
 	}
 
-	return
-
+	return err
 }
 
 func RunInstances(input *ec2.RunInstancesInput, natsConn *nats.Conn, accountID string) (reservation ec2.Reservation, err error) {
-
 	// Validate input
 	err = ValidateRunInstancesInput(input)
 

spinifex/gateway/ec2/instance/RunInstances_test.go

@@ -23,7 +23,6 @@ var defaults = ec2.RunInstancesInput{
 }
 
 func TestParseRunInstances(t *testing.T) {
-
 	// Group multiple tests
 	tests := []struct {
 		name  string
@@ -214,5 +213,4 @@ func TestParseRunInstances(t *testing.T) {
 	}
 
 	// Additional test
-
 }

spinifex/gateway/ec2/key/CreateKeyPair.go

@@ -19,11 +19,10 @@ func ValidateCreateKeyPairInput(input *ec2.CreateKeyPairInput) (err error) {
 		return errors.New(awserrors.ErrorMissingParameter)
 	}
 
-	return
+	return err
 }
 
 func CreateKeyPair(input *ec2.CreateKeyPairInput, natsConn *nats.Conn, accountID string) (output ec2.CreateKeyPairOutput, err error) {
-
 	// Validate input
 	err = ValidateCreateKeyPairInput(input)
 

spinifex/gateway/ec2/key/DeleteKeyPair.go

@@ -20,11 +20,10 @@ func ValidateDeleteKeyPairInput(input *ec2.DeleteKeyPairInput) (err error) {
 		return errors.New(awserrors.ErrorMissingParameter)
 	}
 
-	return
+	return err
 }
 
 func DeleteKeyPair(input *ec2.DeleteKeyPairInput, natsConn *nats.Conn, accountID string) (output ec2.DeleteKeyPairOutput, err error) {
-
 	// Validate input
 	err = ValidateDeleteKeyPairInput(input)
 

spinifex/gateway/ec2/key/ImportKeyPair.go

@@ -22,7 +22,7 @@ func ValidateImportKeyPairInput(input *ec2.ImportKeyPairInput) (err error) {
 		return errors.New(awserrors.ErrorMissingParameter)
 	}
 
-	return
+	return err
 }
 
 func ImportKeyPair(input *ec2.ImportKeyPairInput, natsConn *nats.Conn, accountID string) (output ec2.ImportKeyPairOutput, err error) {

spinifex/gateway/ec2/volume/DetachVolume.go

@@ -39,7 +39,7 @@ func DetachVolume(input *ec2.DetachVolumeInput, natsConn *nats.Conn, accountID s
 	volumeID := *input.VolumeId
 
 	// Resolve InstanceId if not provided by looking up volume metadata
-	instanceID := ""
+	var instanceID string
 	if input.InstanceId != nil && *input.InstanceId != "" {
 		instanceID = *input.InstanceId
 	} else {

spinifex/gateway/ec2/zone/AvailabilityZone.go

@@ -6,7 +6,6 @@ import (
 )
 
 func DescribeAvailabilityZones(input *ec2.DescribeAvailabilityZonesInput, region string, az string) (output *ec2.DescribeAvailabilityZonesOutput, err error) {
-
 	output = &ec2.DescribeAvailabilityZonesOutput{
 		AvailabilityZones: []*ec2.AvailabilityZone{
 			{
@@ -27,7 +26,6 @@ func DescribeAvailabilityZones(input *ec2.DescribeAvailabilityZonesInput, region
 }
 
 func DescribeRegions(input *ec2.DescribeRegionsInput, region string) (output *ec2.DescribeRegionsOutput, err error) {
-
 	output = &ec2.DescribeRegionsOutput{
 		Regions: []*ec2.Region{
 			{

spinifex/gateway/gateway.go

@@ -68,7 +68,6 @@ type ErrorDetail struct {
 }
 
 func (gw *GatewayConfig) SetupRoutes() http.Handler {
-
 	var logLevel slog.Level
 
 	if gw.Debug {
@@ -159,7 +158,7 @@ func corsMiddleware(next http.Handler) http.Handler {
 		w.Header().Set("Access-Control-Allow-Methods", "GET,POST,PUT,DELETE,HEAD,OPTIONS")
 		w.Header().Set("Access-Control-Allow-Headers", "*")
 		w.Header().Set("Access-Control-Allow-Credentials", "true")
-		if r.Method == "OPTIONS" {
+		if r.Method == http.MethodOptions {
 			w.WriteHeader(http.StatusNoContent)
 			return
 		}
@@ -173,7 +172,6 @@ func corsMiddleware(next http.Handler) http.Handler {
 // AWS_ENDPOINT_URL=https://localhost:9999/ aws  --no-verify-ssl ec2 describe-instances
 
 func (gw *GatewayConfig) Request(w http.ResponseWriter, r *http.Request) {
-
 	// Route the request to the appropriate endpoint (e.g EC2, IAM, etc)
 	svc, err := gw.GetService(r)
 	slog.Info("Request", "service", svc, "method", r.Method, "path", r.URL.Path)
@@ -291,7 +289,7 @@ func (gw *GatewayConfig) ErrorHandler(w http.ResponseWriter, r *http.Request, er
 
 	// Get the request ID
 	var requestId = uuid.NewString()
-	if rid := r.Header.Get("x-amz-request-id"); rid != "" {
+	if rid := r.Header.Get("X-Amz-Request-Id"); rid != "" {
 		requestId = rid
 	}
 
@@ -321,7 +319,9 @@ func (gw *GatewayConfig) ErrorHandler(w http.ResponseWriter, r *http.Request, er
 
 	w.Header().Set("Content-Type", "application/xml")
 	w.WriteHeader(errorMsg.HTTPCode)
-	_, _ = w.Write(xmlError)
+	if _, err := w.Write(xmlError); err != nil {
+		slog.Error("Failed to write error response", "err", err)
+	}
 }
 
 // Parse AWS query arguments (used by some services like EC2/S3)
@@ -344,7 +344,6 @@ func ParseAWSQueryArgs(query string) map[string]string {
 }
 
 func GenerateEC2ErrorResponse(code, message, requestID string) (output []byte) {
-
 	errorXml := ErrorResponse{
 		Errors: Errors{
 			Error: ErrorDetail{
@@ -403,7 +402,6 @@ func GenerateIAMErrorResponse(code, message, requestID string) (output []byte) {
 }
 
 func ParseArgsToStruct(input *any, args map[string]string) (err error) {
-
 	// Generated from input shape: RunInstancesRequest
 	err = awsec2query.QueryParamsToStruct(args, input)
 
@@ -412,7 +410,6 @@ func ParseArgsToStruct(input *any, args map[string]string) (err error) {
 	}
 
 	return nil
-
 }
 
 // DiscoverActiveNodes discovers the number of active spinifex daemon nodes in the cluster
@@ -485,6 +482,7 @@ func (gw *GatewayConfig) DiscoverActiveNodes() int {
 // statusWriter wraps http.ResponseWriter to capture the status code.
 type statusWriter struct {
 	http.ResponseWriter
+
 	status int
 }
 

spinifex/gateway/gateway_test.go

@@ -175,7 +175,7 @@ func TestErrorHandler_IAMService(t *testing.T) {
 		gw.ErrorHandler(w, r, errors.New(awserrors.ErrorIAMNoSuchEntity))
 	})
 
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	resp := doRequest(handler, req)
 	assert.Equal(t, 404, resp.StatusCode)
 
@@ -197,7 +197,7 @@ func TestErrorHandler_UnknownError(t *testing.T) {
 		gw.ErrorHandler(w, r, errors.New("SomeCompletelyBogusError"))
 	})
 
-	req := httptest.NewRequest("POST", "/", nil)
+	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	resp := doRequest(handler, req)
 	assert.Equal(t, 500, resp.StatusCode)
 
@@ -218,7 +218,7 @@ func TestErrorHandler_EC2Service(t *testing.T) {
 		gw.ErrorHandler(w, r, errors.New(awserrors.ErrorInvalidParameterValue))
 	})
 
-	req := httptest.NewRequest("POST", "/", nil)
+	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	resp := doRequest(handler, req)
 	assert.Equal(t, 400, resp.StatusCode)
 
@@ -240,8 +240,8 @@ func TestErrorHandler_UsesRequestIDHeader(t *testing.T) {
 		gw.ErrorHandler(w, r, errors.New(awserrors.ErrorInternalError))
 	})
 
-	req := httptest.NewRequest("POST", "/", nil)
-	req.Header.Set("x-amz-request-id", "custom-req-id-123")
+	req := httptest.NewRequest(http.MethodPost, "/", nil)
+	req.Header.Set("X-Amz-Request-Id", "custom-req-id-123")
 	resp := doRequest(handler, req)
 
 	body, _ := io.ReadAll(resp.Body)
@@ -257,7 +257,7 @@ func TestErrorHandler_ContentTypeXML(t *testing.T) {
 		gw.ErrorHandler(w, r, errors.New(awserrors.ErrorInternalError))
 	})
 
-	req := httptest.NewRequest("POST", "/", nil)
+	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	resp := doRequest(handler, req)
 	assert.Equal(t, "application/xml", resp.Header.Get("Content-Type"))
 }
@@ -493,7 +493,7 @@ func TestGetService(t *testing.T) {
 
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
-			req := httptest.NewRequest("POST", "/", nil)
+			req := httptest.NewRequest(http.MethodPost, "/", nil)
 			if tc.ctxVal != nil {
 				ctx := context.WithValue(req.Context(), ctxService, tc.ctxVal)
 				req = req.WithContext(ctx)
@@ -514,7 +514,7 @@ func TestGetService(t *testing.T) {
 
 func TestRequest_NoServiceContext(t *testing.T) {
 	gw := &GatewayConfig{DisableLogging: true}
-	req := httptest.NewRequest("POST", "/", nil)
+	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	w := httptest.NewRecorder()
 
 	gw.Request(w, req)
@@ -527,7 +527,7 @@ func TestRequest_NoServiceContext(t *testing.T) {
 
 func TestRequest_UnsupportedService(t *testing.T) {
 	gw := &GatewayConfig{DisableLogging: true}
-	req := httptest.NewRequest("POST", "/", nil)
+	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	ctx := context.WithValue(req.Context(), ctxService, "s3")
 	req = req.WithContext(ctx)
 	w := httptest.NewRecorder()
@@ -542,7 +542,7 @@ func TestRequest_UnsupportedService(t *testing.T) {
 
 func TestRequest_EC2MissingAction(t *testing.T) {
 	gw := &GatewayConfig{DisableLogging: true}
-	req := httptest.NewRequest("POST", "/", strings.NewReader(""))
+	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(""))
 	ctx := context.WithValue(req.Context(), ctxService, "ec2")
 	ctx = context.WithValue(ctx, ctxAccountID, "123456789012")
 	req = req.WithContext(ctx)
@@ -558,7 +558,7 @@ func TestRequest_EC2MissingAction(t *testing.T) {
 
 func TestRequest_IAMNilService(t *testing.T) {
 	gw := &GatewayConfig{DisableLogging: true, IAMService: nil}
-	req := httptest.NewRequest("POST", "/", strings.NewReader("Action=CreateUser&UserName=test"))
+	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("Action=CreateUser&UserName=test"))
 	ctx := context.WithValue(req.Context(), ctxService, "iam")
 	ctx = context.WithValue(ctx, ctxAccountID, "123456789012")
 	req = req.WithContext(ctx)
@@ -574,7 +574,7 @@ func TestRequest_IAMNilService(t *testing.T) {
 
 func TestRequest_AccountReturns200(t *testing.T) {
 	gw := &GatewayConfig{DisableLogging: true}
-	req := httptest.NewRequest("POST", "/", nil)
+	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	ctx := context.WithValue(req.Context(), ctxService, "account")
 	req = req.WithContext(ctx)
 	w := httptest.NewRecorder()
@@ -587,7 +587,7 @@ func TestRequest_AccountReturns200(t *testing.T) {
 
 // setupEC2Request creates an http.Request with EC2 service context and optional account ID.
 func setupEC2Request(body string, accountID string) *http.Request {
-	req := httptest.NewRequest("POST", "/", strings.NewReader(body))
+	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
 	ctx := context.WithValue(req.Context(), ctxService, "ec2")
 	if accountID != "" {
 		ctx = context.WithValue(ctx, ctxAccountID, accountID)
@@ -697,7 +697,7 @@ func TestEC2Request_DescribeAvailabilityZones(t *testing.T) {
 
 func TestCheckPolicy_NilIAMService(t *testing.T) {
 	gw := &GatewayConfig{DisableLogging: true, IAMService: nil}
-	req := httptest.NewRequest("POST", "/", nil)
+	req := httptest.NewRequest(http.MethodPost, "/", nil)
 
 	err := gw.checkPolicy(req, "ec2", "DescribeInstances")
 	assert.NoError(t, err)
@@ -705,7 +705,7 @@ func TestCheckPolicy_NilIAMService(t *testing.T) {
 
 func TestCheckPolicy_NoIdentityInContext(t *testing.T) {
 	gw := &GatewayConfig{DisableLogging: true, IAMService: &mockIAMService{}}
-	req := httptest.NewRequest("POST", "/", nil)
+	req := httptest.NewRequest(http.MethodPost, "/", nil)
 
 	err := gw.checkPolicy(req, "ec2", "DescribeInstances")
 	assert.NoError(t, err)
@@ -713,7 +713,7 @@ func TestCheckPolicy_NoIdentityInContext(t *testing.T) {
 
 func TestCheckPolicy_RootUserGlobalAccount(t *testing.T) {
 	gw := &GatewayConfig{DisableLogging: true, IAMService: &mockIAMService{}}
-	req := httptest.NewRequest("POST", "/", nil)
+	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	ctx := context.WithValue(req.Context(), ctxIdentity, "root")
 	ctx = context.WithValue(ctx, ctxAccountID, "000000000000") // GlobalAccountID
 	req = req.WithContext(ctx)
@@ -736,7 +736,7 @@ func TestCheckPolicy_NonRootAllowPolicy(t *testing.T) {
 		},
 	}
 	gw := &GatewayConfig{DisableLogging: true, IAMService: mock}
-	req := httptest.NewRequest("POST", "/", nil)
+	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	ctx := context.WithValue(req.Context(), ctxIdentity, "alice")
 	ctx = context.WithValue(ctx, ctxAccountID, "123456789012")
 	req = req.WithContext(ctx)
@@ -759,7 +759,7 @@ func TestCheckPolicy_NonRootDenyPolicy(t *testing.T) {
 		},
 	}
 	gw := &GatewayConfig{DisableLogging: true, IAMService: mock}
-	req := httptest.NewRequest("POST", "/", nil)
+	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	ctx := context.WithValue(req.Context(), ctxIdentity, "alice")
 	ctx = context.WithValue(ctx, ctxAccountID, "123456789012")
 	req = req.WithContext(ctx)
@@ -776,7 +776,7 @@ func TestCheckPolicy_NonRootNoPolicies(t *testing.T) {
 		},
 	}
 	gw := &GatewayConfig{DisableLogging: true, IAMService: mock}
-	req := httptest.NewRequest("POST", "/", nil)
+	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	ctx := context.WithValue(req.Context(), ctxIdentity, "alice")
 	ctx = context.WithValue(ctx, ctxAccountID, "123456789012")
 	req = req.WithContext(ctx)
@@ -793,7 +793,7 @@ func TestCheckPolicy_GetUserPoliciesError(t *testing.T) {
 		},
 	}
 	gw := &GatewayConfig{DisableLogging: true, IAMService: mock}
-	req := httptest.NewRequest("POST", "/", nil)
+	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	ctx := context.WithValue(req.Context(), ctxIdentity, "alice")
 	ctx = context.WithValue(ctx, ctxAccountID, "123456789012")
 	req = req.WithContext(ctx)
@@ -805,7 +805,7 @@ func TestCheckPolicy_GetUserPoliciesError(t *testing.T) {
 
 func TestCheckPolicy_EmptyIdentity(t *testing.T) {
 	gw := &GatewayConfig{DisableLogging: true, IAMService: &mockIAMService{}}
-	req := httptest.NewRequest("POST", "/", nil)
+	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	ctx := context.WithValue(req.Context(), ctxIdentity, "")
 	ctx = context.WithValue(ctx, ctxAccountID, "123456789012")
 	req = req.WithContext(ctx)
@@ -816,7 +816,7 @@ func TestCheckPolicy_EmptyIdentity(t *testing.T) {
 
 func TestCheckPolicy_MissingAccountID(t *testing.T) {
 	gw := &GatewayConfig{DisableLogging: true, IAMService: &mockIAMService{}}
-	req := httptest.NewRequest("POST", "/", nil)
+	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	ctx := context.WithValue(req.Context(), ctxIdentity, "alice")
 	// No account ID
 	req = req.WithContext(ctx)
@@ -835,7 +835,7 @@ func TestCheckPolicy_NATSTransientRetriesAllAttempts(t *testing.T) {
 		},
 	}
 	gw := &GatewayConfig{DisableLogging: true, IAMService: mock}
-	req := httptest.NewRequest("POST", "/", nil)
+	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	ctx := context.WithValue(req.Context(), ctxIdentity, "alice")
 	ctx = context.WithValue(ctx, ctxAccountID, "123456789012")
 	req = req.WithContext(ctx)
@@ -865,7 +865,7 @@ func TestCheckPolicy_NATSTransientRetriesThenSucceeds(t *testing.T) {
 		},
 	}
 	gw := &GatewayConfig{DisableLogging: true, IAMService: mock}
-	req := httptest.NewRequest("POST", "/", nil)
+	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	ctx := context.WithValue(req.Context(), ctxIdentity, "alice")
 	ctx = context.WithValue(ctx, ctxAccountID, "123456789012")
 	req = req.WithContext(ctx)
@@ -882,7 +882,7 @@ func TestCheckPolicy_NonTransientErrorStillFails(t *testing.T) {
 		},
 	}
 	gw := &GatewayConfig{DisableLogging: true, IAMService: mock}
-	req := httptest.NewRequest("POST", "/", nil)
+	req := httptest.NewRequest(http.MethodPost, "/", nil)
 	ctx := context.WithValue(req.Context(), ctxIdentity, "alice")
 	ctx = context.WithValue(ctx, ctxAccountID, "123456789012")
 	req = req.WithContext(ctx)
@@ -904,11 +904,11 @@ func TestCorsMiddleware_GETRequest(t *testing.T) {
 	nextCalled := false
 	next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		nextCalled = true
-		w.WriteHeader(200)
+		w.WriteHeader(http.StatusOK)
 	})
 
 	handler := corsMiddleware(next)
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	w := httptest.NewRecorder()
 	handler.ServeHTTP(w, req)
 
@@ -926,7 +926,7 @@ func TestCorsMiddleware_OPTIONSRequest(t *testing.T) {
 	})
 
 	handler := corsMiddleware(next)
-	req := httptest.NewRequest("OPTIONS", "/", nil)
+	req := httptest.NewRequest(http.MethodOptions, "/", nil)
 	w := httptest.NewRecorder()
 	handler.ServeHTTP(w, req)
 
@@ -937,7 +937,7 @@ func TestCorsMiddleware_OPTIONSRequest(t *testing.T) {
 
 func TestCorsMiddleware_LocalIPOrigin(t *testing.T) {
 	next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(200)
+		w.WriteHeader(http.StatusOK)
 	})
 
 	handler := corsMiddleware(next)
@@ -948,7 +948,7 @@ func TestCorsMiddleware_LocalIPOrigin(t *testing.T) {
 		if origin == "https://localhost:3000" {
 			continue
 		}
-		req := httptest.NewRequest("GET", "/", nil)
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
 		req.Header.Set("Origin", origin)
 		w := httptest.NewRecorder()
 		handler.ServeHTTP(w, req)
@@ -959,11 +959,11 @@ func TestCorsMiddleware_LocalIPOrigin(t *testing.T) {
 
 func TestCorsMiddleware_UnknownOrigin(t *testing.T) {
 	next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(200)
+		w.WriteHeader(http.StatusOK)
 	})
 
 	handler := corsMiddleware(next)
-	req := httptest.NewRequest("GET", "/", nil)
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Header.Set("Origin", "https://evil.example.com")
 	w := httptest.NewRecorder()
 	handler.ServeHTTP(w, req)
@@ -979,7 +979,7 @@ func TestSlogRequestLogger_CallsNext(t *testing.T) {
 	})
 
 	handler := slogRequestLogger(next)
-	req := httptest.NewRequest("GET", "/test", nil)
+	req := httptest.NewRequest(http.MethodGet, "/test", nil)
 	w := httptest.NewRecorder()
 	handler.ServeHTTP(w, req)
 
@@ -993,7 +993,7 @@ func TestSlogRequestLogger_CapturesStatusCode(t *testing.T) {
 	})
 
 	handler := slogRequestLogger(next)
-	req := httptest.NewRequest("GET", "/missing", nil)
+	req := httptest.NewRequest(http.MethodGet, "/missing", nil)
 	w := httptest.NewRecorder()
 	handler.ServeHTTP(w, req)
 

spinifex/gateway/iam.go

@@ -131,6 +131,8 @@ func (gw *GatewayConfig) IAM_Request(w http.ResponseWriter, r *http.Request) err
 
 	w.Header().Set("Content-Type", "text/xml")
 	w.WriteHeader(http.StatusOK)
-	_, _ = w.Write(xmlOutput)
+	if _, err := w.Write(xmlOutput); err != nil {
+		slog.Error("Failed to write IAM response", "err", err)
+	}
 	return nil
 }

spinifex/gateway/iam_request_test.go

@@ -167,7 +167,7 @@ func TestIAMRequest_CreateUser_Success(t *testing.T) {
 	}
 	handler := setupIAMRequestHandler(svc)
 
-	req := httptest.NewRequest("POST", "/", strings.NewReader("Action=CreateUser&UserName=alice"))
+	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("Action=CreateUser&UserName=alice"))
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 
 	resp := doRequest(handler, req)
@@ -193,7 +193,7 @@ func TestIAMRequest_ListUsers_Success(t *testing.T) {
 	}
 	handler := setupIAMRequestHandler(svc)
 
-	req := httptest.NewRequest("POST", "/", strings.NewReader("Action=ListUsers"))
+	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("Action=ListUsers"))
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 
 	resp := doRequest(handler, req)
@@ -209,7 +209,7 @@ func TestIAMRequest_ListUsers_Success(t *testing.T) {
 func TestIAMRequest_UnknownAction(t *testing.T) {
 	handler := setupIAMRequestHandler(&flexMockIAMService{})
 
-	req := httptest.NewRequest("POST", "/", strings.NewReader("Action=DoesNotExist"))
+	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("Action=DoesNotExist"))
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 
 	resp := doRequest(handler, req)
@@ -222,7 +222,7 @@ func TestIAMRequest_UnknownAction(t *testing.T) {
 func TestIAMRequest_EmptyAction(t *testing.T) {
 	handler := setupIAMRequestHandler(&flexMockIAMService{})
 
-	req := httptest.NewRequest("POST", "/", strings.NewReader("UserName=alice"))
+	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("UserName=alice"))
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 
 	resp := doRequest(handler, req)
@@ -247,7 +247,7 @@ func TestIAMRequest_NilService(t *testing.T) {
 		}
 	})
 
-	req := httptest.NewRequest("POST", "/", strings.NewReader("Action=CreateUser&UserName=alice"))
+	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("Action=CreateUser&UserName=alice"))
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 
 	resp := doRequest(handler, req)
@@ -265,7 +265,7 @@ func TestIAMRequest_ServiceError(t *testing.T) {
 	}
 	handler := setupIAMRequestHandler(svc)
 
-	req := httptest.NewRequest("POST", "/", strings.NewReader("Action=CreateUser&UserName=alice"))
+	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("Action=CreateUser&UserName=alice"))
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 
 	resp := doRequest(handler, req)
@@ -291,7 +291,7 @@ func TestIAMRequest_MissingAccountID(t *testing.T) {
 		}
 	})
 
-	req := httptest.NewRequest("POST", "/", strings.NewReader("Action=CreateUser&UserName=alice"))
+	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("Action=CreateUser&UserName=alice"))
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 
 	resp := doRequest(handler, req)
@@ -305,7 +305,7 @@ func TestIAMRequest_ValidationError(t *testing.T) {
 	// CreateUser with missing UserName should return MissingParameter
 	handler := setupIAMRequestHandler(&flexMockIAMService{})
 
-	req := httptest.NewRequest("POST", "/", strings.NewReader("Action=CreateUser"))
+	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("Action=CreateUser"))
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 
 	resp := doRequest(handler, req)

spinifex/gateway/policy/evaluator.go

@@ -31,7 +31,7 @@ func EvaluateAccess(identity, action, resource string, policies []handlers_iam.P
 	hasAllow := false
 	for i := range policies {
 		for j := range policies[i].Statement {
-			stmt := &policies[i].Statement[j]
+			stmt := &policies[i].Statement[j] //nolint:gosec // G602 false positive: j is bounded by range
 
 			if !matchesAny(stmt.Action, action) {
 				continue

spinifex/handlers/ec2/account/service_impl.go

@@ -14,9 +14,10 @@ import (
 )
 
 const (
-	KVBucketAccountSettings = "spinifex-ec2-account-settings"
-	KeyEbsEncryptionDefault = "ebs-encryption-default"
-	KeySerialConsoleAccess  = "serial-console-access"
+	KVBucketAccountSettings        = "spinifex-ec2-account-settings"
+	KVBucketAccountSettingsVersion = 1
+	KeyEbsEncryptionDefault        = "ebs-encryption-default"
+	KeySerialConsoleAccess         = "serial-console-access"
 )
 
 // AccountSettingsRecord represents stored account settings
@@ -43,6 +44,9 @@ func NewAccountSettingsServiceImplWithNATS(cfg *config.Config, natsConn *nats.Co
 	if err != nil {
 		return nil, fmt.Errorf("failed to create account settings KV bucket: %w", err)
 	}
+	if err := utils.WriteVersion(settingsKV, KVBucketAccountSettingsVersion); err != nil {
+		return nil, fmt.Errorf("write version to %s: %w", KVBucketAccountSettings, err)
+	}
 
 	slog.Info("Account settings service initialized with JetStream KV", "bucket", KVBucketAccountSettings)
 

spinifex/handlers/ec2/eigw/service_impl.go

@@ -20,7 +20,10 @@ import (
 // Ensure EgressOnlyIGWServiceImpl implements EgressOnlyIGWService
 var _ EgressOnlyIGWService = (*EgressOnlyIGWServiceImpl)(nil)
 
-const KVBucketEgressOnlyIGW = "spinifex-eigw"
+const (
+	KVBucketEgressOnlyIGW        = "spinifex-eigw"
+	KVBucketEgressOnlyIGWVersion = 1
+)
 
 // EgressOnlyIGWRecord represents a stored Egress-only Internet Gateway
 type EgressOnlyIGWRecord struct {
@@ -49,6 +52,9 @@ func NewEgressOnlyIGWServiceImplWithNATS(cfg *config.Config, natsConn *nats.Conn
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketEgressOnlyIGW, err)
 	}
+	if err := utils.WriteVersion(eigwKV, KVBucketEgressOnlyIGWVersion); err != nil {
+		return nil, fmt.Errorf("write version to %s: %w", KVBucketEgressOnlyIGW, err)
+	}
 
 	// Get or create VPC KV bucket for cross-resource ownership validation
 	vpcKV, err := utils.GetOrCreateKVBucket(js, handlers_ec2_vpc.KVBucketVPCs, 10)
@@ -161,6 +167,9 @@ func (s *EgressOnlyIGWServiceImpl) DescribeEgressOnlyInternetGateways(input *ec2
 	}
 
 	for _, key := range keys {
+		if key == utils.VersionKey {
+			continue
+		}
 		if !strings.HasPrefix(key, prefix) {
 			continue
 		}

spinifex/handlers/ec2/igw/service_impl.go

@@ -21,7 +21,10 @@ import (
 // Ensure IGWServiceImpl implements IGWService
 var _ IGWService = (*IGWServiceImpl)(nil)
 
-const KVBucketIGW = "spinifex-igw"
+const (
+	KVBucketIGW        = "spinifex-igw"
+	KVBucketIGWVersion = 1
+)
 
 // IGWRecord represents a stored Internet Gateway
 type IGWRecord struct {
@@ -51,6 +54,9 @@ func NewIGWServiceImplWithNATS(cfg *config.Config, natsConn *nats.Conn) (*IGWSer
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketIGW, err)
 	}
+	if err := utils.WriteVersion(igwKV, KVBucketIGWVersion); err != nil {
+		return nil, fmt.Errorf("write version to %s: %w", KVBucketIGW, err)
+	}
 
 	// Get or create VPC KV bucket for cross-resource ownership validation
 	vpcKV, err := utils.GetOrCreateKVBucket(js, handlers_ec2_vpc.KVBucketVPCs, 10)
@@ -157,6 +163,9 @@ func (s *IGWServiceImpl) DescribeInternetGateways(input *ec2.DescribeInternetGat
 	foundIDs := make(map[string]bool)
 
 	for _, key := range keys {
+		if key == utils.VersionKey {
+			continue
+		}
 		if !strings.HasPrefix(key, prefix) {
 			continue
 		}
@@ -250,8 +259,10 @@ func (s *IGWServiceImpl) AttachInternetGateway(input *ec2.AttachInternetGatewayI
 			InternetGatewayId: igwID,
 			VpcId:             vpcID,
 		}
-		eventData, _ := json.Marshal(event)
-		if err := s.natsConn.Publish("vpc.igw-attach", eventData); err != nil {
+		eventData, err := json.Marshal(event)
+		if err != nil {
+			slog.Warn("Failed to marshal IGW attach event", "error", err)
+		} else if err := s.natsConn.Publish("vpc.igw-attach", eventData); err != nil {
 			slog.Warn("Failed to publish IGW attach event", "error", err)
 		}
 	}
@@ -306,8 +317,10 @@ func (s *IGWServiceImpl) DetachInternetGateway(input *ec2.DetachInternetGatewayI
 			InternetGatewayId: igwID,
 			VpcId:             vpcID,
 		}
-		eventData, _ := json.Marshal(event)
-		if err := s.natsConn.Publish("vpc.igw-detach", eventData); err != nil {
+		eventData, err := json.Marshal(event)
+		if err != nil {
+			slog.Warn("Failed to marshal IGW detach event", "error", err)
+		} else if err := s.natsConn.Publish("vpc.igw-detach", eventData); err != nil {
 			slog.Warn("Failed to publish IGW detach event", "error", err)
 		}
 	}

spinifex/handlers/ec2/instance/service_impl.go

@@ -297,7 +297,6 @@ func (s *InstanceServiceImpl) GenerateVolumes(input *ec2.RunInstancesInput, inst
 	}
 
 	return volumeInfos, nil
-
 }
 
 // newViperblock creates a viperblock instance with the service's S3/Predastore credentials.

spinifex/handlers/ec2/key/service_impl_test.go

@@ -536,6 +536,73 @@ func TestValidateKeyPairExists_NotFound(t *testing.T) {
 // formatFingerprint Tests
 // ============================================================
 
+// ============================================================
+// getKeyNameFromKeyPairId Tests
+// ============================================================
+
+func TestGetKeyNameFromKeyPairId_NotFound(t *testing.T) {
+	svc, _ := newTestKeyService()
+
+	_, err := svc.getKeyNameFromKeyPairId(testAccountID, "key-nonexistent")
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidKeyPairNotFound, err.Error())
+}
+
+func TestGetKeyNameFromKeyPairId_InvalidJSON(t *testing.T) {
+	svc, store := newTestKeyService()
+
+	// Seed store with garbage data at the metadata path
+	metadataPath := "keys/" + testAccountID + "/key-badjson.json"
+	_, err := store.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(testBucket),
+		Key:    aws.String(metadataPath),
+		Body:   strings.NewReader("this is not valid json{{{"),
+	})
+	require.NoError(t, err)
+
+	_, err = svc.getKeyNameFromKeyPairId(testAccountID, "key-badjson")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "failed to unmarshal metadata")
+}
+
+func TestGetKeyNameFromKeyPairId_MissingKeyName(t *testing.T) {
+	svc, store := newTestKeyService()
+
+	// Seed store with valid JSON but missing KeyName field
+	metadataPath := "keys/" + testAccountID + "/key-noname.json"
+	jsonData, err := json.Marshal(map[string]string{
+		"KeyPairId":      "key-noname",
+		"KeyFingerprint": "SHA256:abc123",
+	})
+	require.NoError(t, err)
+
+	_, err = store.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(testBucket),
+		Key:    aws.String(metadataPath),
+		Body:   strings.NewReader(string(jsonData)),
+	})
+	require.NoError(t, err)
+
+	_, err = svc.getKeyNameFromKeyPairId(testAccountID, "key-noname")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "invalid metadata: missing KeyName")
+}
+
+func TestGetKeyNameFromKeyPairId_Success(t *testing.T) {
+	svc, _ := newTestKeyService()
+
+	// Import a key, then look it up by keyPairId
+	imported := importTestKey(t, svc, "lookup-test")
+
+	keyName, err := svc.getKeyNameFromKeyPairId(testAccountID, *imported.KeyPairId)
+	require.NoError(t, err)
+	assert.Equal(t, "lookup-test", keyName)
+}
+
+// ============================================================
+// formatFingerprint Tests
+// ============================================================
+
 func TestFormatFingerprint_MD5(t *testing.T) {
 	hash := []byte{0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99}
 	result := formatFingerprint(hash, "MD5")

spinifex/handlers/ec2/snapshot/service_impl.go

@@ -26,7 +26,10 @@ import (
 // Ensure SnapshotServiceImpl implements SnapshotService
 var _ SnapshotService = (*SnapshotServiceImpl)(nil)
 
-const KVBucketVolumeSnapshots = "spinifex-volume-snapshots"
+const (
+	KVBucketVolumeSnapshots        = "spinifex-volume-snapshots"
+	KVBucketVolumeSnapshotsVersion = 1
+)
 
 // SnapshotServiceImpl implements SnapshotService with S3-backed storage
 type SnapshotServiceImpl struct {
@@ -70,6 +73,9 @@ func NewSnapshotServiceImplWithNATS(cfg *config.Config, natsConn *nats.Conn) (*S
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketVolumeSnapshots, err)
 	}
+	if err := utils.WriteVersion(kv, KVBucketVolumeSnapshotsVersion); err != nil {
+		return nil, nil, fmt.Errorf("write version to %s: %w", KVBucketVolumeSnapshots, err)
+	}
 
 	slog.Info("Snapshot service initialized with JetStream KV", "bucket", KVBucketVolumeSnapshots)
 

spinifex/handlers/ec2/volume/service_impl.go

@@ -620,7 +620,7 @@ func (s *VolumeServiceImpl) mergeVolumeConfig(configKey string, cfg *viperblock.
 
 	// Full VBState exists -- update VolumeConfig and reconcile VolumeSize
 	state.VolumeConfig = *cfg
-	configSizeBytes := uint64(cfg.VolumeMetadata.SizeGiB) * 1024 * 1024 * 1024
+	configSizeBytes := cfg.VolumeMetadata.SizeGiB * 1024 * 1024 * 1024
 	if configSizeBytes > 0 && configSizeBytes > state.VolumeSize {
 		state.VolumeSize = configSizeBytes
 	}

spinifex/handlers/ec2/volume/service_impl_test.go

@@ -569,7 +569,7 @@ func createVolumeInStoreWithVBState(t *testing.T, store *objectstore.MemoryObjec
 	t.Helper()
 	state := viperblock.VBState{
 		VolumeName: volumeID,
-		VolumeSize: uint64(meta.SizeGiB) * 1024 * 1024 * 1024,
+		VolumeSize: meta.SizeGiB * 1024 * 1024 * 1024,
 		BlockSize:  blockSize,
 		SeqNum:     seqNum,
 		VolumeConfig: viperblock.VolumeConfig{

spinifex/handlers/ec2/vpc/eni.go

@@ -16,7 +16,8 @@ import (
 )
 
 const (
-	KVBucketENIs = "spinifex-vpc-enis"
+	KVBucketENIs        = "spinifex-vpc-enis"
+	KVBucketENIsVersion = 1
 )
 
 // ENIRecord represents a stored Elastic Network Interface
@@ -197,6 +198,9 @@ func (s *VPCServiceImpl) DescribeNetworkInterfaces(input *ec2.DescribeNetworkInt
 	}
 
 	for _, key := range keys {
+		if key == utils.VersionKey {
+			continue
+		}
 		if !strings.HasPrefix(key, prefix) {
 			continue
 		}

spinifex/handlers/ec2/vpc/ipam.go

@@ -13,7 +13,8 @@ import (
 )
 
 const (
-	KVBucketIPAM = "spinifex-vpc-ipam"
+	KVBucketIPAM        = "spinifex-vpc-ipam"
+	KVBucketIPAMVersion = 1
 )
 
 // IPAMRecord tracks allocated IPs for a subnet.
@@ -34,6 +35,9 @@ func NewIPAM(js nats.JetStreamContext) (*IPAM, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to create IPAM KV bucket: %w", err)
 	}
+	if err := utils.WriteVersion(kv, KVBucketIPAMVersion); err != nil {
+		return nil, fmt.Errorf("write version to %s: %w", KVBucketIPAM, err)
+	}
 	return &IPAM{kv: kv}, nil
 }
 

spinifex/handlers/ec2/vpc/service_impl.go

@@ -26,6 +26,10 @@ const (
 	KVBucketVNICounter = "spinifex-vpc-vni-counter"
 	vniCounterKey      = "counter"
 	vniStart           = 100 // Starting VNI value (avoid 0 and low numbers)
+
+	KVBucketVPCsVersion       = 1
+	KVBucketSubnetsVersion    = 1
+	KVBucketVNICounterVersion = 1
 )
 
 // VPCRecord represents a stored VPC
@@ -73,21 +77,33 @@ func NewVPCServiceImplWithNATS(cfg *config.Config, natsConn *nats.Conn) (*VPCSer
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketVPCs, err)
 	}
+	if err := utils.WriteVersion(vpcKV, KVBucketVPCsVersion); err != nil {
+		return nil, fmt.Errorf("write version to %s: %w", KVBucketVPCs, err)
+	}
 
 	subnetKV, err := utils.GetOrCreateKVBucket(js, KVBucketSubnets, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketSubnets, err)
 	}
+	if err := utils.WriteVersion(subnetKV, KVBucketSubnetsVersion); err != nil {
+		return nil, fmt.Errorf("write version to %s: %w", KVBucketSubnets, err)
+	}
 
 	vniKV, err := utils.GetOrCreateKVBucket(js, KVBucketVNICounter, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketVNICounter, err)
 	}
+	if err := utils.WriteVersion(vniKV, KVBucketVNICounterVersion); err != nil {
+		return nil, fmt.Errorf("write version to %s: %w", KVBucketVNICounter, err)
+	}
 
 	eniKV, err := utils.GetOrCreateKVBucket(js, KVBucketENIs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketENIs, err)
 	}
+	if err := utils.WriteVersion(eniKV, KVBucketENIsVersion); err != nil {
+		return nil, fmt.Errorf("write version to %s: %w", KVBucketENIs, err)
+	}
 
 	ipam, err := NewIPAM(js)
 	if err != nil {
@@ -118,7 +134,10 @@ func (s *VPCServiceImpl) nextVNI() (int64, error) {
 		if errors.Is(err, nats.ErrKeyNotFound) {
 			// First VNI allocation — initialize counter
 			vni := int64(vniStart)
-			data, _ := json.Marshal(vni + 1)
+			data, marshalErr := json.Marshal(vni + 1)
+			if marshalErr != nil {
+				return 0, fmt.Errorf("failed to marshal VNI counter: %w", marshalErr)
+			}
 			if _, err := s.vniKV.Create(vniCounterKey, data); err != nil {
 				return 0, fmt.Errorf("failed to initialize VNI counter: %w", err)
 			}
@@ -133,7 +152,10 @@ func (s *VPCServiceImpl) nextVNI() (int64, error) {
 	}
 
 	next := current + 1
-	data, _ := json.Marshal(next)
+	data, marshalErr := json.Marshal(next)
+	if marshalErr != nil {
+		return 0, fmt.Errorf("failed to marshal VNI counter: %w", marshalErr)
+	}
 	if _, err := s.vniKV.Update(vniCounterKey, data, entry.Revision()); err != nil {
 		return 0, fmt.Errorf("failed to update VNI counter (CAS conflict): %w", err)
 	}
@@ -225,6 +247,9 @@ func (s *VPCServiceImpl) DeleteVpc(input *ec2.DeleteVpcInput, accountID string)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 	for _, k := range subnetKeys {
+		if k == utils.VersionKey {
+			continue
+		}
 		if !strings.HasPrefix(k, prefix) {
 			continue
 		}
@@ -271,6 +296,9 @@ func (s *VPCServiceImpl) DescribeVpcs(input *ec2.DescribeVpcsInput, accountID st
 	}
 
 	for _, key := range keys {
+		if key == utils.VersionKey {
+			continue
+		}
 		if !strings.HasPrefix(key, prefix) {
 			continue
 		}
@@ -368,6 +396,9 @@ func (s *VPCServiceImpl) CreateSubnet(input *ec2.CreateSubnetInput, accountID st
 	}
 
 	for _, k := range subnetKeys {
+		if k == utils.VersionKey {
+			continue
+		}
 		if !strings.HasPrefix(k, prefix) {
 			continue
 		}
@@ -499,6 +530,9 @@ func (s *VPCServiceImpl) DescribeSubnets(input *ec2.DescribeSubnetsInput, accoun
 	}
 
 	for _, key := range keys {
+		if key == utils.VersionKey {
+			continue
+		}
 		if !strings.HasPrefix(key, prefix) {
 			continue
 		}
@@ -634,6 +668,9 @@ func (s *VPCServiceImpl) EnsureDefaultVPC(accountID string) error {
 	}
 
 	for _, key := range keys {
+		if key == utils.VersionKey {
+			continue
+		}
 		if !strings.HasPrefix(key, prefix) {
 			continue
 		}
@@ -730,6 +767,9 @@ func (s *VPCServiceImpl) GetDefaultSubnet(accountID string) (*SubnetRecord, erro
 	}
 
 	for _, key := range keys {
+		if key == utils.VersionKey {
+			continue
+		}
 		if !strings.HasPrefix(key, prefix) {
 			continue
 		}

spinifex/handlers/iam/crypto.go

@@ -12,6 +12,9 @@ import (
 
 const masterKeySize = 32 // AES-256
 
+// BootstrapVersion is the current bootstrap.json config version.
+const BootstrapVersion = "1.0"
+
 // GenerateMasterKey returns 32 cryptographically random bytes suitable
 // for use as an AES-256-GCM key.
 func GenerateMasterKey() ([]byte, error) {
@@ -117,7 +120,9 @@ func DecryptSecret(ciphertext string, key []byte) (string, error) {
 }
 
 // SaveBootstrapData writes bootstrap data as JSON to disk with 0600 permissions.
+// It always sets the version to BootstrapVersion.
 func SaveBootstrapData(path string, data *BootstrapData) error {
+	data.Version = BootstrapVersion
 	b, err := json.Marshal(data)
 	if err != nil {
 		return fmt.Errorf("marshal bootstrap data: %w", err)
@@ -138,5 +143,8 @@ func LoadBootstrapData(path string) (*BootstrapData, error) {
 	if err := json.Unmarshal(data, &bd); err != nil {
 		return nil, fmt.Errorf("parse bootstrap data: %w", err)
 	}
+	if bd.Version == "" {
+		return nil, fmt.Errorf("bootstrap data missing version field")
+	}
 	return &bd, nil
 }

spinifex/handlers/iam/crypto_test.go

@@ -185,6 +185,7 @@ func TestLoadBootstrapData(t *testing.T) {
 	path := filepath.Join(dir, "bootstrap.json")
 
 	bd := BootstrapData{
+		Version:         BootstrapVersion,
 		AccessKeyID:     "AKIAIOSFODNN7EXAMPLE",
 		EncryptedSecret: "dGVzdC1lbmNyeXB0ZWQ=",
 		AccountID:       "000000000000",
@@ -199,6 +200,9 @@ func TestLoadBootstrapData(t *testing.T) {
 		t.Fatalf("LoadBootstrapData() error: %v", err)
 	}
 
+	if loaded.Version != BootstrapVersion {
+		t.Fatalf("Version: expected %q, got %q", BootstrapVersion, loaded.Version)
+	}
 	if loaded.AccessKeyID != bd.AccessKeyID {
 		t.Fatalf("AccessKeyID: expected %q, got %q", bd.AccessKeyID, loaded.AccessKeyID)
 	}
@@ -259,6 +263,9 @@ func TestSaveBootstrapData_RoundTrip(t *testing.T) {
 	if err != nil {
 		t.Fatalf("LoadBootstrapData() error: %v", err)
 	}
+	if loaded.Version != BootstrapVersion {
+		t.Fatalf("Version: expected %q, got %q", BootstrapVersion, loaded.Version)
+	}
 	if loaded.AccessKeyID != data.AccessKeyID {
 		t.Fatalf("AccessKeyID: expected %q, got %q", data.AccessKeyID, loaded.AccessKeyID)
 	}
@@ -325,13 +332,12 @@ func TestBootstrapData_ExtendedRoundTrip(t *testing.T) {
 	}
 }
 
-func TestBootstrapData_NoAdmin_BackwardCompat(t *testing.T) {
+func TestBootstrapData_NoAdmin(t *testing.T) {
 	dir := t.TempDir()
 	path := filepath.Join(dir, "bootstrap.json")
 
-	// Write bootstrap data without admin field (old format)
-	oldJSON := `{"access_key_id":"AKIAOLD123","encrypted_secret":"b2xk","account_id":"000000000000"}`
-	if err := os.WriteFile(path, []byte(oldJSON), 0600); err != nil {
+	noAdminJSON := `{"version":"1.0","access_key_id":"AKIAOLD123","encrypted_secret":"b2xk","account_id":"000000000000"}`
+	if err := os.WriteFile(path, []byte(noAdminJSON), 0600); err != nil {
 		t.Fatal(err)
 	}
 
@@ -344,7 +350,22 @@ func TestBootstrapData_NoAdmin_BackwardCompat(t *testing.T) {
 		t.Fatalf("AccessKeyID: expected %q, got %q", "AKIAOLD123", loaded.AccessKeyID)
 	}
 	if loaded.Admin != nil {
-		t.Fatal("Admin field should be nil for old-format bootstrap data")
+		t.Fatal("Admin field should be nil")
+	}
+}
+
+func TestLoadBootstrapData_MissingVersion(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "bootstrap.json")
+
+	noVersion := `{"access_key_id":"AKIA123","encrypted_secret":"dGVzdA==","account_id":"000000000000"}`
+	if err := os.WriteFile(path, []byte(noVersion), 0600); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err := LoadBootstrapData(path)
+	if err == nil {
+		t.Fatal("LoadBootstrapData() should fail for missing version")
 	}
 }
 

spinifex/handlers/iam/service_impl.go

@@ -27,6 +27,12 @@ const (
 	KVBucketAccounts       = "spinifex-accounts"
 	KVBucketAccountCounter = "spinifex-account-counter"
 
+	KVBucketUsersVersion          = 1
+	KVBucketAccessKeysVersion     = 1
+	KVBucketPoliciesVersion       = 1
+	KVBucketAccountsVersion       = 1
+	KVBucketAccountCounterVersion = 1
+
 	maxAccessKeysPerUser = 2
 )
 
@@ -63,26 +69,41 @@ func NewIAMServiceImpl(natsConn *nats.Conn, masterKey []byte, clusterSize int) (
 	if err != nil {
 		return nil, fmt.Errorf("init users bucket: %w", err)
 	}
+	if err := utils.WriteVersion(usersBucket, KVBucketUsersVersion); err != nil {
+		return nil, fmt.Errorf("write version to %s: %w", KVBucketUsers, err)
+	}
 
 	accessKeysBucket, err := getOrCreateBucket(js, KVBucketAccessKeys, 5, replicas)
 	if err != nil {
 		return nil, fmt.Errorf("init access keys bucket: %w", err)
 	}
+	if err := utils.WriteVersion(accessKeysBucket, KVBucketAccessKeysVersion); err != nil {
+		return nil, fmt.Errorf("write version to %s: %w", KVBucketAccessKeys, err)
+	}
 
 	policiesBucket, err := getOrCreateBucket(js, KVBucketPolicies, 10, replicas)
 	if err != nil {
 		return nil, fmt.Errorf("init policies bucket: %w", err)
 	}
+	if err := utils.WriteVersion(policiesBucket, KVBucketPoliciesVersion); err != nil {
+		return nil, fmt.Errorf("write version to %s: %w", KVBucketPolicies, err)
+	}
 
 	accountsBucket, err := getOrCreateBucket(js, KVBucketAccounts, 5, replicas)
 	if err != nil {
 		return nil, fmt.Errorf("init accounts bucket: %w", err)
 	}
+	if err := utils.WriteVersion(accountsBucket, KVBucketAccountsVersion); err != nil {
+		return nil, fmt.Errorf("write version to %s: %w", KVBucketAccounts, err)
+	}
 
 	accountCounterBucket, err := getOrCreateBucket(js, KVBucketAccountCounter, 5, replicas)
 	if err != nil {
 		return nil, fmt.Errorf("init account counter bucket: %w", err)
 	}
+	if err := utils.WriteVersion(accountCounterBucket, KVBucketAccountCounterVersion); err != nil {
+		return nil, fmt.Errorf("write version to %s: %w", KVBucketAccountCounter, err)
+	}
 
 	decrypter, err := NewDecrypter(masterKey)
 	if err != nil {
@@ -231,6 +252,9 @@ func (s *IAMServiceImpl) ListUsers(accountID string, input *iam.ListUsersInput)
 	keyPrefix := accountID + "."
 	var users []*iam.User
 	for _, key := range keys {
+		if key == utils.VersionKey {
+			continue
+		}
 		if !strings.HasPrefix(key, keyPrefix) {
 			continue
 		}
@@ -714,7 +738,12 @@ func (s *IAMServiceImpl) IsEmpty() (bool, error) {
 		}
 		return false, fmt.Errorf("check users bucket: %w", err)
 	}
-	return len(keys) == 0, nil
+	for _, key := range keys {
+		if key != utils.VersionKey {
+			return false, nil
+		}
+	}
+	return true, nil
 }
 
 // ---------------------------------------------------------------------------
@@ -826,6 +855,9 @@ func (s *IAMServiceImpl) ListAccounts() ([]*Account, error) {
 
 	accounts := make([]*Account, 0, len(keys))
 	for _, key := range keys {
+		if key == utils.VersionKey {
+			continue
+		}
 		entry, err := s.accountsBucket.Get(key)
 		if err != nil {
 			if errors.Is(err, nats.ErrKeyNotFound) {
@@ -984,6 +1016,9 @@ func (s *IAMServiceImpl) ListPolicies(accountID string, input *iam.ListPoliciesI
 	keyPrefix := accountID + "."
 	var policies []*iam.Policy
 	for _, key := range keys {
+		if key == utils.VersionKey {
+			continue
+		}
 		if !strings.HasPrefix(key, keyPrefix) {
 			continue
 		}
@@ -1229,6 +1264,9 @@ func (s *IAMServiceImpl) buildAttachmentCounts(accountID string) (map[string]int
 	counts := make(map[string]int64)
 	keyPrefix := accountID + "."
 	for _, key := range keys {
+		if key == utils.VersionKey {
+			continue
+		}
 		if !strings.HasPrefix(key, keyPrefix) {
 			continue
 		}
@@ -1267,6 +1305,9 @@ func (s *IAMServiceImpl) countPolicyAttachments(accountID, policyARN string) (in
 	keyPrefix := accountID + "."
 	var count int64
 	for _, key := range keys {
+		if key == utils.VersionKey {
+			continue
+		}
 		if !strings.HasPrefix(key, keyPrefix) {
 			continue
 		}

spinifex/handlers/iam/service_impl_test.go

@@ -2,7 +2,6 @@ package handlers_iam
 
 import (
 	"fmt"
-	"io"
 	"log/slog"
 	"strings"
 	"testing"
@@ -790,7 +789,7 @@ func TestSeedBootstrap_WithAdmin(t *testing.T) {
 	policies, err := svc.GetUserPolicies("000000000001", "admin")
 	require.NoError(t, err)
 	require.Len(t, policies, 1)
-	assert.Equal(t, "Allow", string(policies[0].Statement[0].Effect))
+	assert.Equal(t, "Allow", policies[0].Statement[0].Effect)
 	assert.Equal(t, StringOrArr{"*"}, policies[0].Statement[0].Action)
 	assert.Equal(t, StringOrArr{"*"}, policies[0].Statement[0].Resource)
 
@@ -1556,7 +1555,7 @@ func TestSensitiveDataNotLogged_CreateAccessKey(t *testing.T) {
 	logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
 	slog.SetDefault(logger)
 	t.Cleanup(func() {
-		slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, nil)))
+		slog.SetDefault(slog.New(slog.DiscardHandler))
 	})
 
 	akOut, err := svc.CreateAccessKey(testAccountID, &iam.CreateAccessKeyInput{
@@ -1584,7 +1583,7 @@ func TestSensitiveDataNotLogged_MasterKey(t *testing.T) {
 	logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
 	slog.SetDefault(logger)
 	t.Cleanup(func() {
-		slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, nil)))
+		slog.SetDefault(slog.New(slog.DiscardHandler))
 	})
 
 	svc := setupTestIAMService(t)

spinifex/handlers/iam/types.go

@@ -103,6 +103,7 @@ func (s StringOrArr) MarshalJSON() ([]byte, error) {
 // BootstrapData is the on-disk JSON file consumed on first gateway start
 // to seed the root IAM user into NATS KV.
 type BootstrapData struct {
+	Version         string              `json:"version"`
 	AccessKeyID     string              `json:"access_key_id"`
 	EncryptedSecret string              `json:"encrypted_secret"`
 	AccountID       string              `json:"account_id"`

spinifex/service/service.go

@@ -21,7 +21,6 @@ type Service interface {
 }
 
 func New(btype string, config any) (Service, error) {
-
 	switch btype {
 	case "nats":
 		return nats.New(config)
@@ -43,7 +42,6 @@ func New(btype string, config any) (Service, error) {
 
 	case "vpcd":
 		return vpcd.New(config)
-
 	}
 
 	return nil, fmt.Errorf("unknown service type: %s", btype)

spinifex/service/service_test.go

@@ -15,12 +15,10 @@ func TestNew(t *testing.T) {
 	services := []string{"nats", "predastore", "viperblock", "spinifex", "awsgw"}
 
 	for _, s := range services {
-
 		var svc Service
 		var err error
 
 		switch s {
-
 		// TODO: Standardize service config handling (use config.Config for all?)
 		case "nats":
 			svc, err = New(s, &nats.Config{})
@@ -34,7 +32,6 @@ func TestNew(t *testing.T) {
 		case "awsgw":
 			svc, err = New(s, &config.ClusterConfig{})
 			// No special setup needed
-
 		}
 
 		assert.NoError(t, err)

spinifex/services/awsgw/awsgw.go

@@ -36,7 +36,7 @@ func New(cfg any) (svc *Service, err error) {
 
 func (svc *Service) Start() (int, error) {
 	if err := utils.WritePidFileTo(svc.Config.NodeBaseDir(), serviceName, os.Getpid()); err != nil {
-		slog.Error("Failed to write pid file", "err", err)
+		return 0, fmt.Errorf("write pid file: %w", err)
 	}
 	err := launchService(svc.Config)
 	if err != nil {
@@ -51,7 +51,14 @@ func (svc *Service) Stop() (err error) {
 }
 
 func (svc *Service) Status() (string, error) {
-	return "", nil
+	pid, err := utils.ReadPidFileFrom(svc.Config.NodeBaseDir(), serviceName)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return "stopped", nil
+		}
+		return "", fmt.Errorf("read pid file: %w", err)
+	}
+	return fmt.Sprintf("running (pid: %d)", pid), nil
 }
 
 func (svc *Service) Shutdown() (err error) {
@@ -63,7 +70,6 @@ func (svc *Service) Reload() (err error) {
 }
 
 func launchService(config *config.ClusterConfig) error {
-
 	nodeConfig := config.Nodes[config.Node]
 
 	// Connect to NATS for service communication. On concurrent startup the

spinifex/services/nats/nats.go

@@ -38,7 +38,7 @@ func New(config any) (svc *Service, err error) {
 
 func (svc *Service) Start() (int, error) {
 	if err := utils.WritePidFileTo(svc.Config.DataDir, serviceName, os.Getpid()); err != nil {
-		slog.Error("Failed to write pid file", "err", err)
+		return 0, fmt.Errorf("write pid file: %w", err)
 	}
 	err := launchService(svc.Config)
 	if err != nil {
@@ -52,7 +52,14 @@ func (svc *Service) Stop() (err error) {
 }
 
 func (svc *Service) Status() (string, error) {
-	return "", nil
+	pid, err := utils.ReadPidFileFrom(svc.Config.DataDir, serviceName)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return "stopped", nil
+		}
+		return "", fmt.Errorf("read pid file: %w", err)
+	}
+	return fmt.Sprintf("running (pid: %d)", pid), nil
 }
 
 func (svc *Service) Shutdown() (err error) {
@@ -65,20 +72,17 @@ func (svc *Service) Reload() (err error) {
 
 func launchService(config *Config) (err error) {
 	// Create proper server options
-	opts := &server.Options{}
+	var opts *server.Options
 
 	// If configFile set use, otherwise set defaults
 	if config.ConfigFile != "" {
-
 		opts, err = server.ProcessConfigFile(config.ConfigFile)
 
 		if err != nil {
 			slog.Error("Failed to process NATS config file", "err", err)
 			return err
 		}
-
 	} else {
-
 		opts = &server.Options{
 			ConfigFile: config.ConfigFile,
 			Port:       config.Port,
@@ -96,10 +100,9 @@ func launchService(config *Config) (err error) {
 		if opts.Host == "" {
 			opts.Host = "0.0.0.0"
 		}
-
 	}
 
-	fmt.Println(opts)
+	slog.Debug("NATS server options", "opts", opts)
 
 	// Initialize new server with options
 	ns, err := server.NewServer(opts)
@@ -120,5 +123,4 @@ func launchService(config *Config) (err error) {
 	ns.WaitForShutdown()
 
 	return nil
-
 }

spinifex/services/predastore/predastore.go

@@ -57,7 +57,7 @@ func New(config any) (svc *Service, err error) {
 // Start starts the predastore service
 func (svc *Service) Start() (int, error) {
 	if err := utils.WritePidFileTo(svc.Config.BasePath, serviceName, os.Getpid()); err != nil {
-		slog.Error("Failed to write pid file", "err", err)
+		return 0, fmt.Errorf("write pid file: %w", err)
 	}
 
 	server, err := s3.NewServer(
@@ -106,7 +106,14 @@ func (svc *Service) Stop() error {
 
 // Status returns the status of the predastore service
 func (svc *Service) Status() (string, error) {
-	return "", nil
+	pid, err := utils.ReadPidFileFrom(svc.Config.BasePath, serviceName)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return "stopped", nil
+		}
+		return "", fmt.Errorf("read pid file: %w", err)
+	}
+	return fmt.Sprintf("running (pid: %d)", pid), nil
 }
 
 // Shutdown gracefully shuts down the predastore service

spinifex/services/predastore/predastore_integration_test.go

@@ -125,7 +125,7 @@ func startPredastoreServer(t *testing.T) *Config {
 	}
 
 	// Create persistent test environment (not using t.TempDir() for shared server)
-	testDir, err := os.MkdirTemp("", "predastore-integration-*")
+	testDir, err := os.MkdirTemp("", "predastore-integration-*") //nolint:usetesting // shared server outlives individual test
 	if err != nil {
 		t.Fatalf("Failed to create temp dir: %v", err)
 	}
@@ -666,7 +666,7 @@ func TestIntegration_LargeFile(t *testing.T) {
 	downloadedData, err := io.ReadAll(result.Body)
 	result.Body.Close()
 	assert.NoError(t, err, "Reading large file should succeed")
-	assert.Equal(t, size, len(downloadedData), "Downloaded size should match")
+	assert.Len(t, downloadedData, size, "Downloaded size should match")
 	assert.Equal(t, largeData, downloadedData, "Downloaded content should match")
 
 	// Cleanup

spinifex/services/predastore/predastore_test.go

@@ -100,10 +100,10 @@ func TestServiceMethods(t *testing.T) {
 	svc, err := New(cfg)
 	assert.NoError(t, err)
 
-	// Test Status - returns empty string and no error
+	// Test Status - returns "stopped" when no PID file exists
 	status, err := svc.Status()
 	assert.NoError(t, err)
-	assert.Empty(t, status)
+	assert.Equal(t, "stopped", status)
 
 	// Test Reload - returns no error
 	err = svc.Reload()

spinifex/services/spinifex/spinifex.go

@@ -34,7 +34,7 @@ func (svc *Service) SetConfigPath(path string) {
 
 func (svc *Service) Start() (int, error) {
 	if err := utils.WritePidFileTo(svc.Config.NodeBaseDir(), serviceName, os.Getpid()); err != nil {
-		slog.Error("Failed to write pid file", "err", err)
+		return 0, fmt.Errorf("write pid file: %w", err)
 	}
 	err := launchService(svc.Config, svc.ConfigPath)
 	if err != nil {
@@ -49,7 +49,14 @@ func (svc *Service) Stop() (err error) {
 }
 
 func (svc *Service) Status() (string, error) {
-	return "", nil
+	pid, err := utils.ReadPidFileFrom(svc.Config.NodeBaseDir(), serviceName)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return "stopped", nil
+		}
+		return "", fmt.Errorf("read pid file: %w", err)
+	}
+	return fmt.Sprintf("running (pid: %d)", pid), nil
 }
 
 func (svc *Service) Shutdown() (err error) {
@@ -61,8 +68,10 @@ func (svc *Service) Reload() (err error) {
 }
 
 func launchService(config *config.ClusterConfig, configPath string) (err error) {
-
-	d := daemon.NewDaemon(config)
+	d, err := daemon.NewDaemon(config)
+	if err != nil {
+		return fmt.Errorf("create daemon: %w", err)
+	}
 	d.SetConfigPath(configPath)
 	slog.Info("Starting Spinifex daemon ...")
 	err = d.Start()
@@ -73,5 +82,4 @@ func launchService(config *config.ClusterConfig, configPath string) (err error)
 	}
 
 	return nil
-
 }

spinifex/services/spinifexui/frontend/pnpm-lock.yaml

@@ -2087,8 +2087,8 @@ packages:
   fast-xml-builder@1.1.4:
     resolution: {integrity: sha512-f2jhpN4Eccy0/Uz9csxh3Nu6q4ErKxf0XIsasomfOihuSUa3/xw6w8dnOtCDgEItQFJG8KyXPzQXzcODDrrbOg==}
 
-  fast-xml-parser@5.5.6:
-    resolution: {integrity: sha512-3+fdZyBRVg29n4rXP0joHthhcHdPUHaIC16cuyyd1iLsuaO6Vea36MPrxgAzbZna8lhvZeRL8Bc9GP56/J9xEw==}
+  fast-xml-parser@5.5.7:
+    resolution: {integrity: sha512-LteOsISQ2GEiDHZch6L9hB0+MLoYVLToR7xotrzU0opCICBkxOPgHAy1HxAvtxfJNXDJpgAsQN30mkrfpO2Prg==}
     hasBin: true
 
   fdir@6.5.0:
@@ -2829,8 +2829,8 @@ packages:
   undici-types@7.16.0:
     resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==}
 
-  undici@7.24.4:
-    resolution: {integrity: sha512-BM/JzwwaRXxrLdElV2Uo6cTLEjhSb3WXboncJamZ15NgUURmvlXvxa6xkwIOILIjPNo9i8ku136ZvWV0Uly8+w==}
+  undici@7.24.5:
+    resolution: {integrity: sha512-3IWdCpjgxp15CbJnsi/Y9TCDE7HWVN19j1hmzVhoAkY/+CJx449tVxT5wZc1Gwg8J+P0LWvzlBzxYRnHJ+1i7Q==}
     engines: {node: '>=20.18.1'}
 
   unplugin@2.3.11:
@@ -3588,7 +3588,7 @@ snapshots:
   '@aws-sdk/xml-builder@3.972.11':
     dependencies:
       '@smithy/types': 4.13.1
-      fast-xml-parser: 5.5.6
+      fast-xml-parser: 5.5.7
       tslib: 2.8.1
 
   '@aws/lambda-invoke-store@0.2.4': {}
@@ -5257,7 +5257,7 @@ snapshots:
     dependencies:
       path-expression-matcher: 1.1.3
 
-  fast-xml-parser@5.5.6:
+  fast-xml-parser@5.5.7:
     dependencies:
       fast-xml-builder: 1.1.4
       path-expression-matcher: 1.1.3
@@ -5427,7 +5427,7 @@ snapshots:
       saxes: 6.0.0
       symbol-tree: 3.2.4
       tough-cookie: 6.0.1
-      undici: 7.24.4
+      undici: 7.24.5
       w3c-xmlserializer: 5.0.0
       webidl-conversions: 8.0.1
       whatwg-mimetype: 5.0.0
@@ -5949,7 +5949,7 @@ snapshots:
 
   undici-types@7.16.0: {}
 
-  undici@7.24.4:
+  undici@7.24.5:
     optional: true
 
   unplugin@2.3.11:

spinifex/services/spinifexui/frontend/src/lib/auth.test.ts

@@ -45,7 +45,10 @@ describe("getCredentials", () => {
   })
 
   it("reads from localStorage on first call", () => {
-    localStorage.setItem("spinifex:v1:aws-credentials", JSON.stringify(validCreds))
+    localStorage.setItem(
+      "spinifex:v1:aws-credentials",
+      JSON.stringify(validCreds),
+    )
     expect(getCredentials()).toEqual(validCreds)
   })
 

spinifex/services/spinifexui/spinifexui.go

@@ -100,7 +100,10 @@ func (svc *Service) Stop() error {
 func (svc *Service) Status() (string, error) {
 	pid, err := utils.ReadPidFile(serviceName)
 	if err != nil {
-		return "stopped", nil
+		if os.IsNotExist(err) {
+			return "stopped", nil
+		}
+		return "", fmt.Errorf("read pid file: %w", err)
 	}
 	return fmt.Sprintf("running (pid: %d)", pid), nil
 }
@@ -311,6 +314,7 @@ var gzipContentTypes = map[string]bool{
 // gzipResponseWriter wraps http.ResponseWriter to compress eligible responses.
 type gzipResponseWriter struct {
 	http.ResponseWriter
+
 	gw          *gzip.Writer
 	wroteHeader bool
 	compress    bool
@@ -367,6 +371,7 @@ const tlsRecordTypeHandshake = 0x16
 // a plain-HTTP redirect to HTTPS is sent and the connection is closed.
 type tlsSplitListener struct {
 	net.Listener
+
 	port   int
 	tlsCfg *tls.Config
 }
@@ -442,6 +447,7 @@ func (ln *tlsSplitListener) redirectHTTP(conn net.Conn, firstByte byte) {
 // prefixConn wraps a net.Conn with a reader that replays prefixed bytes.
 type prefixConn struct {
 	net.Conn
+
 	r io.Reader
 }
 

spinifex/services/viperblockd/viperblockd.go

@@ -121,9 +121,8 @@ func makeSnapshotHandler(vb *viperblock.VB, volumeName string) nats.MsgHandler {
 }
 
 func (svc *Service) Start() (int, error) {
-
 	if err := utils.WritePidFileTo(svc.Config.BaseDir, serviceName, os.Getpid()); err != nil {
-		slog.Error("Failed to write pid file", "err", err)
+		return 0, fmt.Errorf("write pid file: %w", err)
 	}
 	err := launchService(svc.Config)
 
@@ -140,7 +139,14 @@ func (svc *Service) Stop() (err error) {
 }
 
 func (svc *Service) Status() (string, error) {
-	return "", nil
+	pid, err := utils.ReadPidFileFrom(svc.Config.BaseDir, serviceName)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return "stopped", nil
+		}
+		return "", fmt.Errorf("read pid file: %w", err)
+	}
+	return fmt.Sprintf("running (pid: %d)", pid), nil
 }
 
 func (svc *Service) Shutdown() (err error) {
@@ -152,7 +158,6 @@ func (svc *Service) Reload() (err error) {
 }
 
 func launchService(cfg *Config) (err error) {
-
 	// Connect to NATS
 	nc, err := utils.ConnectNATS(cfg.NatsHost, cfg.NatsToken)
 	if err != nil {
@@ -262,7 +267,12 @@ func launchService(cfg *Config) (err error) {
 		err := json.Unmarshal(msg.Data, &ebsRequest)
 		if err != nil {
 			slog.Error("Failed to unmarshal ebs.unmount message", "err", err)
-			errResp, _ := json.Marshal(types.EBSUnMountResponse{Error: fmt.Sprintf("bad request: %v", err)})
+			errResp, marshalErr := json.Marshal(types.EBSUnMountResponse{Error: fmt.Sprintf("bad request: %v", err)})
+			if marshalErr != nil {
+				slog.Error("Failed to marshal ebs.unmount error response", "err", marshalErr)
+				_ = msg.Respond([]byte(`{"Error":"internal marshal failure"}`))
+				return
+			}
 			if err := msg.Respond(errResp); err != nil {
 				slog.Error("Failed to respond to ebs.unmount request", "err", err)
 			}
@@ -273,7 +283,7 @@ func launchService(cfg *Config) (err error) {
 		// then release before calling VB.Close() (which does heavy S3 I/O).
 		var ebsResponse types.EBSUnMountResponse
 		var matched MountedVolume
-		var matchIdx int = -1
+		var matchIdx = -1
 		cfg.mu.Lock()
 		for i, volume := range cfg.MountedVolumes {
 			if volume.Name == ebsRequest.Name {
@@ -329,6 +339,7 @@ func launchService(cfg *Config) (err error) {
 		response, err := json.Marshal(ebsResponse)
 		if err != nil {
 			slog.Error("Failed to marshal response", "err", err)
+			_ = msg.Respond([]byte(`{"Error":"internal marshal failure"}`))
 			return
 		}
 
@@ -348,7 +359,12 @@ func launchService(cfg *Config) (err error) {
 		var syncRequest types.EBSSyncRequest
 		if err := json.Unmarshal(msg.Data, &syncRequest); err != nil {
 			slog.Error("Failed to unmarshal ebs.sync message", "err", err)
-			errResp, _ := json.Marshal(types.EBSSyncResponse{Error: fmt.Sprintf("bad request: %v", err)})
+			errResp, marshalErr := json.Marshal(types.EBSSyncResponse{Error: fmt.Sprintf("bad request: %v", err)})
+			if marshalErr != nil {
+				slog.Error("Failed to marshal ebs.sync error response", "err", marshalErr)
+				_ = msg.Respond([]byte(`{"Error":"internal marshal failure"}`))
+				return
+			}
 			if err := msg.Respond(errResp); err != nil {
 				slog.Error("Failed to respond to ebs.sync request", "err", err)
 			}
@@ -419,7 +435,12 @@ func launchService(cfg *Config) (err error) {
 		err := json.Unmarshal(msg.Data, &ebsRequest)
 		if err != nil {
 			slog.Error("Failed to unmarshal ebs.mount message", "err", err)
-			errResp, _ := json.Marshal(types.EBSMountResponse{Error: fmt.Sprintf("bad request: %v", err)})
+			errResp, marshalErr := json.Marshal(types.EBSMountResponse{Error: fmt.Sprintf("bad request: %v", err)})
+			if marshalErr != nil {
+				slog.Error("Failed to marshal ebs.mount error response", "err", marshalErr)
+				_ = msg.Respond([]byte(`{"Error":"internal marshal failure"}`))
+				return
+			}
 			if err := msg.Respond(errResp); err != nil {
 				slog.Error("Failed to respond to ebs.mount request", "err", err)
 			}
@@ -478,7 +499,12 @@ func launchService(cfg *Config) (err error) {
 		if err != nil {
 			ebsResponse.Error = fmt.Sprintf("Failed to connect to Viperblock store: %v", err)
 			// Marshal and send error response immediately
-			response, _ := json.Marshal(ebsResponse)
+			response, marshalErr := json.Marshal(ebsResponse)
+			if marshalErr != nil {
+				slog.Error("Failed to marshal ebs.mount error response", "err", marshalErr)
+				_ = msg.Respond([]byte(`{"Error":"internal marshal failure"}`))
+				return
+			}
 			if err := msg.Respond(response); err != nil {
 				slog.Error("Failed to respond to ebs.mount request", "err", err)
 			}
@@ -498,7 +524,12 @@ func launchService(cfg *Config) (err error) {
 		if err != nil {
 			ebsResponse.Error = err.Error()
 			// Marshal and send error response immediately
-			response, _ := json.Marshal(ebsResponse)
+			response, marshalErr := json.Marshal(ebsResponse)
+			if marshalErr != nil {
+				slog.Error("Failed to marshal ebs.mount error response", "err", marshalErr)
+				_ = msg.Respond([]byte(`{"Error":"internal marshal failure"}`))
+				return
+			}
 			if err := msg.Respond(response); err != nil {
 				slog.Error("Failed to respond to ebs.mount request", "err", err)
 			}
@@ -515,7 +546,12 @@ func launchService(cfg *Config) (err error) {
 		if err != nil {
 			ebsResponse.Error = err.Error()
 			// Marshal and send error response immediately
-			response, _ := json.Marshal(ebsResponse)
+			response, marshalErr := json.Marshal(ebsResponse)
+			if marshalErr != nil {
+				slog.Error("Failed to marshal ebs.mount error response", "err", marshalErr)
+				_ = msg.Respond([]byte(`{"Error":"internal marshal failure"}`))
+				return
+			}
 			if err := msg.Respond(response); err != nil {
 				slog.Error("Failed to respond to ebs.mount request", "err", err)
 			}
@@ -539,7 +575,12 @@ func launchService(cfg *Config) (err error) {
 			portStr, err := viperblock.FindFreePort()
 			if err != nil {
 				ebsResponse.Error = err.Error()
-				response, _ := json.Marshal(ebsResponse)
+				response, marshalErr := json.Marshal(ebsResponse)
+				if marshalErr != nil {
+					slog.Error("Failed to marshal ebs.mount error response", "err", marshalErr)
+					_ = msg.Respond([]byte(`{"Error":"internal marshal failure"}`))
+					return
+				}
 				if err := msg.Respond(response); err != nil {
 					slog.Error("Failed to respond to ebs.mount request", "err", err)
 				}
@@ -555,7 +596,12 @@ func launchService(cfg *Config) (err error) {
 			if err != nil {
 				slog.Error("Failed to convert port to int", "err", err)
 				ebsResponse.Error = fmt.Sprintf("failed to parse port: %v", err)
-				response, _ := json.Marshal(ebsResponse)
+				response, marshalErr := json.Marshal(ebsResponse)
+				if marshalErr != nil {
+					slog.Error("Failed to marshal ebs.mount error response", "err", marshalErr)
+					_ = msg.Respond([]byte(`{"Error":"internal marshal failure"}`))
+					return
+				}
 				if err := msg.Respond(response); err != nil {
 					slog.Error("Failed to respond to ebs.mount request", "err", err)
 				}
@@ -572,7 +618,12 @@ func launchService(cfg *Config) (err error) {
 			nbdSocket, err = utils.GenerateUniqueSocketFile(ebsRequest.Name)
 			if err != nil {
 				ebsResponse.Error = err.Error()
-				response, _ := json.Marshal(ebsResponse)
+				response, marshalErr := json.Marshal(ebsResponse)
+				if marshalErr != nil {
+					slog.Error("Failed to marshal ebs.mount error response", "err", marshalErr)
+					_ = msg.Respond([]byte(`{"Error":"internal marshal failure"}`))
+					return
+				}
 				if err := msg.Respond(response); err != nil {
 					slog.Error("Failed to respond to ebs.mount request", "err", err)
 				}
@@ -591,7 +642,12 @@ func launchService(cfg *Config) (err error) {
 		if err != nil {
 			slog.Error("Failed to generate nbdkit pid file", "err", err)
 			ebsResponse.Error = fmt.Sprintf("failed to generate pid file: %v", err)
-			response, _ := json.Marshal(ebsResponse)
+			response, marshalErr := json.Marshal(ebsResponse)
+			if marshalErr != nil {
+				slog.Error("Failed to marshal ebs.mount error response", "err", marshalErr)
+				_ = msg.Respond([]byte(`{"Error":"internal marshal failure"}`))
+				return
+			}
 			if err := msg.Respond(response); err != nil {
 				slog.Error("Failed to respond to ebs.mount request", "err", err)
 			}
@@ -609,7 +665,7 @@ func launchService(cfg *Config) (err error) {
 			PluginPath: cfg.PluginPath,
 			BaseDir:    cfg.BaseDir,
 			Host:       cfg.S3Host,
-			Verbose:    true,
+			Verbose:    false,
 			Size:       utils.SafeUint64ToInt64(vb.GetVolumeSize()),
 			Volume:     ebsRequest.Name,
 			Bucket:     cfg.Bucket,
@@ -661,7 +717,12 @@ func launchService(cfg *Config) (err error) {
 		if pid == 0 {
 			ebsResponse.Error = "Failed to start nbdkit"
 			// Marshal and send error response immediately
-			response, _ := json.Marshal(ebsResponse)
+			response, marshalErr := json.Marshal(ebsResponse)
+			if marshalErr != nil {
+				slog.Error("Failed to marshal ebs.mount error response", "err", marshalErr)
+				_ = msg.Respond([]byte(`{"Error":"internal marshal failure"}`))
+				return
+			}
 			if err := msg.Respond(response); err != nil {
 				slog.Error("Failed to respond to ebs.mount request", "err", err)
 			}
@@ -679,7 +740,12 @@ func launchService(cfg *Config) (err error) {
 		case exitErr := <-exitChan:
 			if exitErr != 0 {
 				ebsResponse.Error = fmt.Sprintf("nbdkit failed: %v", exitErr)
-				response, _ := json.Marshal(ebsResponse)
+				response, marshalErr := json.Marshal(ebsResponse)
+				if marshalErr != nil {
+					slog.Error("Failed to marshal ebs.mount error response", "err", marshalErr)
+					_ = msg.Respond([]byte(`{"Error":"internal marshal failure"}`))
+					return
+				}
 				if err := msg.Respond(response); err != nil {
 					slog.Error("Failed to respond to ebs.mount request", "err", err)
 				}
@@ -718,6 +784,7 @@ func launchService(cfg *Config) (err error) {
 		response, err := json.Marshal(ebsResponse)
 		if err != nil {
 			slog.Error("Failed to marshal response", "err", err)
+			_ = msg.Respond([]byte(`{"Error":"internal marshal failure"}`))
 			return
 		}
 
@@ -765,5 +832,4 @@ func launchService(cfg *Config) (err error) {
 	}
 
 	return nil
-
 }

spinifex/services/viperblockd/viperblockd_test.go

@@ -155,10 +155,10 @@ func TestServiceMethods(t *testing.T) {
 	svc, err := New(cfg)
 	assert.NoError(t, err)
 
-	// Test Status - returns empty string and no error
+	// Test Status - returns "stopped" when no PID file exists
 	status, err := svc.Status()
 	assert.NoError(t, err)
-	assert.Empty(t, status)
+	assert.Equal(t, "stopped", status)
 
 	// Test Reload - returns no error
 	err = svc.Reload()

spinifex/services/vpcd/ovn_test.go

@@ -7,6 +7,33 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/services/vpcd/nbdb"
 )
 
+func TestNamedUUID(t *testing.T) {
+	tests := []struct {
+		name     string
+		prefix   string
+		input    string
+		expected string
+	}{
+		{"hyphen replacement", "sw_", "vpc-abc123", "sw_vpc_abc123"},
+		{"slash and dot replacement", "rp_", "10.0.0.1/24", "rp_10_0_0_1_24"},
+		{"no special characters", "lr_", "vpcabc", "lr_vpcabc"},
+		{"empty name", "sw_", "", "sw_"},
+		{"empty prefix", "", "vpc-1", "vpc_1"},
+		{"both empty", "", "", ""},
+		{"multiple consecutive hyphens", "x_", "a--b", "x_a__b"},
+		{"mixed special chars", "p_", "a.b-c/d", "p_a_b_c_d"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := namedUUID(tt.prefix, tt.input)
+			if got != tt.expected {
+				t.Errorf("namedUUID(%q, %q) = %q, want %q", tt.prefix, tt.input, got, tt.expected)
+			}
+		})
+	}
+}
+
 func TestMockOVNClient_Connect(t *testing.T) {
 	mock := NewMockOVNClient()
 	if mock.Connected() {

spinifex/services/vpcd/topology.go

@@ -653,6 +653,8 @@ func (h *TopologyHandler) handleIGWAttach(msg *nats.Msg) {
 		vpcCIDR := router.ExternalIDs["spinifex:cidr"]
 		if vpcCIDR == "" {
 			vpcCIDR = "10.0.0.0/8"
+			slog.Warn("vpcd: VPC CIDR missing from router metadata, using overbroad fallback",
+				"router", routerName, "fallbackCIDR", vpcCIDR, "vpcId", evt.VpcId)
 		}
 		snatRule := &nbdb.NAT{
 			Type:       "snat",
@@ -726,6 +728,8 @@ func (h *TopologyHandler) handleIGWDetach(msg *nats.Msg) {
 		vpcCIDR := router.ExternalIDs["spinifex:cidr"]
 		if vpcCIDR == "" {
 			vpcCIDR = "10.0.0.0/8"
+			slog.Warn("vpcd: VPC CIDR missing from router metadata, using overbroad fallback for NAT cleanup",
+				"router", routerName, "fallbackCIDR", vpcCIDR)
 		}
 		if err := h.ovn.DeleteNAT(ctx, routerName, "snat", vpcCIDR); err != nil {
 			slog.Warn("vpcd: failed to delete SNAT rule", "router", routerName, "err", err)
@@ -812,7 +816,11 @@ func respond(msg *nats.Msg, err error) {
 		resp.Error = err.Error()
 	}
 
-	data, _ := json.Marshal(resp)
+	data, marshalErr := json.Marshal(resp)
+	if marshalErr != nil {
+		slog.Error("vpcd: failed to marshal NATS response", "err", marshalErr)
+		data = []byte(`{"success":false,"error":"internal marshal failure"}`)
+	}
 	if err := msg.Respond(data); err != nil {
 		slog.Error("vpcd: failed to respond to NATS request", "err", err)
 	}

spinifex/services/vpcd/vpcd.go

@@ -76,7 +76,7 @@ func New(config any) (*Service, error) {
 // Start starts the vpcd service.
 func (svc *Service) Start() (int, error) {
 	if err := utils.WritePidFileTo(svc.Config.BaseDir, serviceName, os.Getpid()); err != nil {
-		slog.Error("Failed to write pid file", "err", err)
+		return 0, fmt.Errorf("write pid file: %w", err)
 	}
 
 	err := launchService(svc.Config)
@@ -95,7 +95,14 @@ func (svc *Service) Stop() error {
 
 // Status returns the vpcd service status.
 func (svc *Service) Status() (string, error) {
-	return "", nil
+	pid, err := utils.ReadPidFileFrom(svc.Config.BaseDir, serviceName)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return "stopped", nil
+		}
+		return "", fmt.Errorf("read pid file: %w", err)
+	}
+	return fmt.Sprintf("running (pid: %d)", pid), nil
 }
 
 // Shutdown gracefully shuts down the vpcd service.

spinifex/utils/account.go

@@ -1,6 +1,54 @@
 package utils
 
-import "github.com/nats-io/nats.go"
+import (
+	"errors"
+	"fmt"
+	"strconv"
+
+	"github.com/nats-io/nats.go"
+)
+
+// VersionKey is the well-known KV key used to store a bucket's schema version.
+const VersionKey = "_version"
+
+// WriteVersion writes the schema version to a bucket.
+// Only writes if the key doesn't exist or the stored version is older.
+// Returns an error if the stored value is corrupt (non-integer).
+func WriteVersion(kv nats.KeyValue, version int) error {
+	entry, err := kv.Get(VersionKey)
+	if err != nil && !errors.Is(err, nats.ErrKeyNotFound) {
+		return fmt.Errorf("read current version: %w", err)
+	}
+	if err == nil {
+		stored, parseErr := strconv.Atoi(string(entry.Value()))
+		if parseErr != nil {
+			return fmt.Errorf("corrupted _version key (raw=%q): %w", string(entry.Value()), parseErr)
+		}
+		if stored >= version {
+			return nil
+		}
+	}
+	_, err = kv.PutString(VersionKey, strconv.Itoa(version))
+	return err
+}
+
+// ReadVersion reads the schema version from a bucket.
+// Returns 0 if the key is not set. Returns an error on network failures
+// or corrupt values so callers can distinguish "not set" from "unreachable".
+func ReadVersion(kv nats.KeyValue) (int, error) {
+	entry, err := kv.Get(VersionKey)
+	if err != nil {
+		if errors.Is(err, nats.ErrKeyNotFound) {
+			return 0, nil
+		}
+		return 0, fmt.Errorf("read version: %w", err)
+	}
+	v, parseErr := strconv.Atoi(string(entry.Value()))
+	if parseErr != nil {
+		return 0, fmt.Errorf("corrupted _version key (raw=%q): %w", string(entry.Value()), parseErr)
+	}
+	return v, nil
+}
 
 // GlobalAccountID is the root/system account. Pre-Phase4 resources are owned by this account.
 const GlobalAccountID = "000000000000"

spinifex/utils/account_test.go

@@ -1,6 +1,14 @@
 package utils
 
-import "testing"
+import (
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
 
 func TestAccountKey(t *testing.T) {
 	tests := []struct {
@@ -52,3 +60,120 @@ func TestGlobalAccountID(t *testing.T) {
 		t.Error("GlobalAccountID should be a valid account ID")
 	}
 }
+
+// startJSNATSServer starts an embedded JetStream-enabled NATS server for testing.
+func startJSNATSServer(t *testing.T) *server.Server {
+	t.Helper()
+	tmpDir := t.TempDir()
+
+	ns, err := server.NewServer(&server.Options{
+		Host:      "127.0.0.1",
+		Port:      -1,
+		JetStream: true,
+		StoreDir:  tmpDir,
+		NoLog:     true,
+		NoSigs:    true,
+	})
+	require.NoError(t, err)
+	go ns.Start()
+	require.True(t, ns.ReadyForConnections(5*time.Second))
+	t.Cleanup(func() { ns.Shutdown() })
+	return ns
+}
+
+func TestWriteVersion_WritesKey(t *testing.T) {
+	ns := startJSNATSServer(t)
+	nc, err := nats.Connect(ns.ClientURL())
+	require.NoError(t, err)
+	defer nc.Close()
+
+	js, err := nc.JetStream()
+	require.NoError(t, err)
+
+	kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "test-write-version"})
+	require.NoError(t, err)
+
+	err = WriteVersion(kv, 1)
+	require.NoError(t, err)
+
+	entry, err := kv.Get(VersionKey)
+	require.NoError(t, err)
+	assert.Equal(t, "1", string(entry.Value()))
+}
+
+func TestReadVersion_ReturnsZeroWhenUnset(t *testing.T) {
+	ns := startJSNATSServer(t)
+	nc, err := nats.Connect(ns.ClientURL())
+	require.NoError(t, err)
+	defer nc.Close()
+
+	js, err := nc.JetStream()
+	require.NoError(t, err)
+
+	kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "test-read-unset"})
+	require.NoError(t, err)
+
+	v, err := ReadVersion(kv)
+	require.NoError(t, err)
+	assert.Equal(t, 0, v)
+}
+
+func TestWriteVersion_IdempotentOnSameVersion(t *testing.T) {
+	ns := startJSNATSServer(t)
+	nc, err := nats.Connect(ns.ClientURL())
+	require.NoError(t, err)
+	defer nc.Close()
+
+	js, err := nc.JetStream()
+	require.NoError(t, err)
+
+	kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "test-idempotent"})
+	require.NoError(t, err)
+
+	require.NoError(t, WriteVersion(kv, 1))
+	require.NoError(t, WriteVersion(kv, 1))
+
+	v, err := ReadVersion(kv)
+	require.NoError(t, err)
+	assert.Equal(t, 1, v)
+}
+
+func TestWriteVersion_UpdatesOnHigherVersion(t *testing.T) {
+	ns := startJSNATSServer(t)
+	nc, err := nats.Connect(ns.ClientURL())
+	require.NoError(t, err)
+	defer nc.Close()
+
+	js, err := nc.JetStream()
+	require.NoError(t, err)
+
+	kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "test-upgrade"})
+	require.NoError(t, err)
+
+	require.NoError(t, WriteVersion(kv, 1))
+	require.NoError(t, WriteVersion(kv, 2))
+
+	v, err := ReadVersion(kv)
+	require.NoError(t, err)
+	assert.Equal(t, 2, v)
+}
+
+func TestWriteVersion_NoOpOnLowerVersion(t *testing.T) {
+	ns := startJSNATSServer(t)
+	nc, err := nats.Connect(ns.ClientURL())
+	require.NoError(t, err)
+	defer nc.Close()
+
+	js, err := nc.JetStream()
+	require.NoError(t, err)
+
+	kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "test-downgrade"})
+	require.NoError(t, err)
+
+	require.NoError(t, WriteVersion(kv, 3))
+	require.NoError(t, WriteVersion(kv, 1))
+
+	v, err := ReadVersion(kv)
+	require.NoError(t, err)
+	assert.Equal(t, 3, v)
+}

spinifex/utils/images.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -137,7 +138,6 @@ var AvailableImages = map[string]Images{
 
 // AMI / image extraction utils
 func ExtractDiskImageFromFile(imagepath string, tmpdir string) (diskimage string, err error) {
-
 	var args []string
 	var execCmd string
 
@@ -145,7 +145,7 @@ func ExtractDiskImageFromFile(imagepath string, tmpdir string) (diskimage string
 	_, err = os.Stat(imagepath)
 
 	if err != nil {
-		return
+		return diskimage, err
 	}
 
 	// Extract the filepath
@@ -153,7 +153,6 @@ func ExtractDiskImageFromFile(imagepath string, tmpdir string) (diskimage string
 
 	// Already in raw/image formt, confirm the file contains a valid disk image/MBR
 	if strings.HasSuffix(imagefile, ".raw") || strings.HasSuffix(imagefile, ".img") || strings.HasSuffix(imagefile, ".qcow2") || strings.HasSuffix(imagefile, ".qcow") {
-
 		path, err := filepath.Abs(imagepath)
 
 		if err != nil {
@@ -166,7 +165,6 @@ func ExtractDiskImageFromFile(imagepath string, tmpdir string) (diskimage string
 		// Check error response
 
 		if errors.Is(err, ErrQCOWDetected) {
-
 			extractpath := fmt.Sprintf("%s/%s", tmpdir, imagefile)
 			extractpath = strings.TrimSuffix(extractpath, ".qcow2") + ".raw"
 
@@ -191,13 +189,10 @@ func ExtractDiskImageFromFile(imagepath string, tmpdir string) (diskimage string
 			}
 
 			return extractpath, nil
-
 		}
 
 		return path, err
-
 	} else if strings.HasSuffix(imagefile, ".tar.xz") {
-
 		args = []string{
 			"xfvJ",
 			imagepath,
@@ -206,9 +201,7 @@ func ExtractDiskImageFromFile(imagepath string, tmpdir string) (diskimage string
 		}
 
 		execCmd = "tar"
-
 	} else if strings.HasSuffix(imagefile, ".tar.gz") || strings.HasSuffix(imagefile, ".tgz") {
-
 		args = []string{
 			"xfvz",
 			imagepath,
@@ -217,9 +210,7 @@ func ExtractDiskImageFromFile(imagepath string, tmpdir string) (diskimage string
 		}
 
 		execCmd = "tar"
-
 	} else if strings.HasSuffix(imagefile, ".tar") {
-
 		args = []string{
 			"xfv",
 			imagepath,
@@ -228,88 +219,81 @@ func ExtractDiskImageFromFile(imagepath string, tmpdir string) (diskimage string
 		}
 
 		execCmd = "tar"
-
 	} else if strings.HasSuffix(imagefile, ".xz") {
-
 		args = []string{
 			"-dk",
 			imagepath,
 		}
 
 		execCmd = "xz"
-
 	} else {
 		err = errors.New("unsupported filetype")
-		return
+		return diskimage, err
 	}
 
 	cmd := exec.Command(execCmd, args...)
 	output, err := cmd.CombinedOutput()
 
 	if err != nil {
-		return
+		return diskimage, err
 	}
 
 	diskimage, err = extractDiskImagePath(tmpdir, output)
 
-	return
-
+	return diskimage, err
 }
 
 func extractDiskImagePath(imagedir string, output []byte) (diskimage string, err error) {
-
 	reader := bytes.NewReader(output)
 
 	r := bufio.NewReader(reader)
 
 	for {
-		line, err := r.ReadString('\n')
-		line = strings.Replace(line, "\n", "", 1)
+		line, readErr := r.ReadString('\n')
+		line = strings.TrimRight(line, "\n")
 
 		// MacOS tar, filenames begin with `x FILE` (to STDERR)
 		if runtime.GOOS == "darwin" && strings.HasPrefix(line, "x ") {
 			line = strings.Replace(line, "x ", "", 1)
-
 		}
 
 		if strings.HasSuffix(line, ".raw") || strings.HasSuffix(line, ".img") {
 			diskimage := fmt.Sprintf("%s/%s", imagedir, line)
-
 			err = validateDiskImagePath(diskimage)
-
 			return diskimage, err
 		}
 
-		if err != nil && err.Error() == "EOF" {
-			break
+		if readErr != nil {
+			if errors.Is(readErr, io.EOF) {
+				break
+			}
+			return "", fmt.Errorf("read tar output: %w", readErr)
 		}
 	}
 
-	return
-
+	return diskimage, err
 }
 
 func validateDiskImagePath(diskimage string) (err error) {
-
 	args := []string{
 		diskimage,
 	}
 
 	cmd := exec.Command("file", args...)
-	output, _ := cmd.CombinedOutput()
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("run file command on %s: %w", diskimage, err)
+	}
 
 	filetype := strings.Split(string(output), ":")
 
 	if len(filetype) > 1 {
-
 		if strings.Contains(filetype[1], "DOS/MBR boot sector") || strings.Contains(filetype[1], "Linux ") {
 			return nil
 		} else if strings.Contains(filetype[1], "QEMU QCOW") {
 			return ErrQCOWDetected
 		}
-
 	}
 
 	return errors.New("no valid disk image found")
-
 }

spinifex/utils/nats.go

@@ -17,10 +17,10 @@ func ConnectNATS(host, token string) (*nats.Conn, error) {
 		nats.ReconnectWait(time.Second),
 		nats.MaxReconnects(-1),
 		nats.DisconnectErrHandler(func(nc *nats.Conn, err error) {
-			slog.Debug("NATS disconnected", "err", err)
+			slog.Warn("NATS disconnected", "err", err)
 		}),
 		nats.ReconnectHandler(func(nc *nats.Conn) {
-			slog.Debug("NATS reconnected", "url", nc.ConnectedUrl())
+			slog.Warn("NATS reconnected", "url", nc.ConnectedUrl())
 		}),
 	}
 

spinifex/utils/nbd.go

@@ -12,7 +12,6 @@ import (
 // GenerateSocketFile generates a socket file path for the given name.
 // Deprecated: Use GenerateUniqueSocketFile for new code to ensure uniqueness.
 func GenerateSocketFile(name string) (string, error) {
-
 	if name == "" {
 		return "", errors.New("name is required")
 	}

spinifex/utils/pidfile.go

@@ -7,11 +7,11 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"syscall"
 	"time"
 )
 
 func ReadPidFile(name string) (int, error) {
-
 	pidPath := pidPath()
 
 	pidFile, err := os.ReadFile(filepath.Join(pidPath, fmt.Sprintf("%s.pid", name)))
@@ -27,7 +27,6 @@ func ReadPidFile(name string) (int, error) {
 }
 
 func GeneratePidFile(name string) (string, error) {
-
 	if name == "" {
 		return "", errors.New("name is required")
 	}
@@ -42,7 +41,6 @@ func GeneratePidFile(name string) (string, error) {
 }
 
 func WritePidFile(name string, pid int) error {
-
 	// Write PID to file, check XDG, otherwise user home directory ~/spinifex/
 	pidFilename, err := GeneratePidFile(name)
 
@@ -57,7 +55,7 @@ func WritePidFile(name string, pid int) error {
 	}
 
 	defer pidFile.Close()
-	_, err = pidFile.WriteString(fmt.Sprintf("%d", pid))
+	_, err = fmt.Fprintf(pidFile, "%d", pid)
 	if err != nil {
 		return err
 	}
@@ -82,7 +80,7 @@ func WritePidFileTo(dir string, name string, pid int) error {
 	}
 
 	defer pidFile.Close()
-	_, err = pidFile.WriteString(fmt.Sprintf("%d", pid))
+	_, err = fmt.Fprintf(pidFile, "%d", pid)
 	return err
 }
 
@@ -133,7 +131,6 @@ func StopProcessAt(dir string, name string) error {
 }
 
 func RemovePidFile(serviceName string) error {
-
 	pidPath := pidPath()
 
 	err := os.Remove(filepath.Join(pidPath, fmt.Sprintf("%s.pid", serviceName)))
@@ -159,6 +156,32 @@ func pidPath() string {
 	return os.TempDir()
 }
 
+// WaitForProcessExit polls until the given PID is no longer alive or the
+// timeout expires. Unlike WaitForPidFileRemoval, this checks the process
+// itself via kill(pid, 0), so it works after SIGKILL where the target
+// cannot clean up its own PID file.
+func WaitForProcessExit(pid int, timeout time.Duration) error {
+	timer := time.NewTimer(timeout)
+	defer timer.Stop()
+	ticker := time.NewTicker(100 * time.Millisecond)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-timer.C:
+			return fmt.Errorf("timeout waiting for process %d to exit", pid)
+		case <-ticker.C:
+			proc, err := os.FindProcess(pid)
+			if err != nil {
+				return nil // process gone
+			}
+			if proc.Signal(syscall.Signal(0)) != nil {
+				return nil // process no longer alive
+			}
+		}
+	}
+}
+
 func WaitForPidFileRemoval(instanceID string, timeout time.Duration) error {
 	timeoutCh := time.After(timeout)
 	ticker := time.NewTicker(100 * time.Millisecond)

spinifex/utils/process.go

@@ -41,7 +41,6 @@ func StopProcess(serviceName string) error {
 }
 
 func KillProcess(pid int) error {
-
 	process, err := os.FindProcess(pid)
 
 	if err != nil {
@@ -68,7 +67,7 @@ func KillProcess(pid int) error {
 		err = process.Signal(syscall.Signal(0))
 
 		if err != nil {
-			// Process terminated, break
+			// Signal(0) error means the process has terminated — this is the expected outcome.
 			break
 		}
 
@@ -86,6 +85,5 @@ func KillProcess(pid int) error {
 		}
 	}
 
-	return nil
-
+	return nil //nolint:nilerr // Signal(0) error means process exited — that's success
 }

spinifex/utils/utils.go

@@ -41,7 +41,6 @@ func GenerateResourceID(prefix string) string {
 
 // Convert interface to XML
 func MarshalToXML(payload any) ([]byte, error) {
-
 	var buf bytes.Buffer
 	enc := xml.NewEncoder(&buf)
 
@@ -56,7 +55,6 @@ func MarshalToXML(payload any) ([]byte, error) {
 	}
 
 	return buf.Bytes(), nil
-
 }
 
 // wrapWithLocation decorates payload with the requested locationName tag.
@@ -106,7 +104,6 @@ func GenerateIAMXMLPayload(action string, payload any) any {
 
 // Generate JSON Error Payload
 func GenerateErrorPayload(code string) (jsonResponse []byte) {
-
 	var responseError ec2.ResponseError
 	responseError.Code = aws.String(code)
 
@@ -117,13 +114,11 @@ func GenerateErrorPayload(code string) (jsonResponse []byte) {
 		return nil
 	}
 
-	return
-
+	return jsonResponse
 }
 
 // Validate the payload is an ec2.ResponseError
 func ValidateErrorPayload(payload []byte) (responseError ec2.ResponseError, err error) {
-
 	decoder := json.NewDecoder(bytes.NewReader(payload))
 	decoder.DisallowUnknownFields()
 
@@ -137,13 +132,11 @@ func ValidateErrorPayload(payload []byte) (responseError ec2.ResponseError, err
 
 	// Either failed to decode (not an error structure) or Code is nil (empty valid response)
 	return responseError, nil
-
 }
 
 // Unmarshal payload
 
 func UnmarshalJsonPayload(input any, jsonData []byte) []byte {
-
 	decoder := json.NewDecoder(bytes.NewReader(jsonData))
 	decoder.DisallowUnknownFields()
 
@@ -154,11 +147,9 @@ func UnmarshalJsonPayload(input any, jsonData []byte) []byte {
 	}
 
 	return nil
-
 }
 
 func MarshalJsonPayload(input any, jsonData []byte) []byte {
-
 	decoder := json.NewDecoder(bytes.NewReader(jsonData))
 	decoder.DisallowUnknownFields()
 
@@ -169,7 +160,6 @@ func MarshalJsonPayload(input any, jsonData []byte) []byte {
 	}
 
 	return nil
-
 }
 
 // ValidateKeyPairName validates that a key pair name only contains allowed characters:
@@ -204,7 +194,6 @@ func ValidateKeyPairName(name string) error {
 }
 
 func CreateS3Client(cfg *config.Config) *s3.S3 {
-
 	tr := &http.Transport{
 		TLSClientConfig: &tls.Config{
 			InsecureSkipVerify: true,
@@ -225,13 +214,11 @@ func CreateS3Client(cfg *config.Config) *s3.S3 {
 	}))
 
 	return s3.New(sess)
-
 }
 
 // Download helper
 
 func DownloadFileWithProgress(url string, name string, filename string, timeout time.Duration) (err error) {
-
 	// Context with optional timeout and Ctrl+C cancel
 	ctx, cancel := context.WithCancel(context.Background())
 	if timeout > 0 {
@@ -288,10 +275,8 @@ func DownloadFileWithProgress(url string, name string, filename string, timeout
 		}
 
 		pterm.Printf("Saved %s (%s)\n", filename, humanBytes(SafeInt64ToUint64(written)))
-		return
-
+		return err
 	} else {
-
 		// Unknown size: spinner that shows bytes downloaded
 		spin, _ := pterm.DefaultSpinner.
 			WithText("Downloading (size unknown)...").
@@ -309,7 +294,6 @@ func DownloadFileWithProgress(url string, name string, filename string, timeout
 		}
 
 		pterm.Printf("Saved %s (%s)\n", filename, humanBytes(SafeInt64ToUint64(written)))
-
 	}
 
 	return nil

spinifex/utils/utils_test.go

@@ -43,7 +43,6 @@ func TestGenerateResourceID(t *testing.T) {
 }
 
 func TestGeneratePidFile(t *testing.T) {
-
 	// Simulate a sample process running (e.g cat)
 	cmd := exec.Command("cat")
 	cmd.Start()
@@ -70,12 +69,10 @@ func TestGeneratePidFile(t *testing.T) {
 	//time.Sleep(2 * time.Second)
 
 	// Simulate process ending
-
 }
 
 func TestGenerateSocketFile(t *testing.T) {
-
-	socketPath := fmt.Sprintf("%s/%s", os.TempDir(), "utilsunittest")
+	socketPath := fmt.Sprintf("%s/%s", t.TempDir(), "utilsunittest")
 
 	name, err := GenerateSocketFile(socketPath)
 
@@ -87,11 +84,9 @@ func TestGenerateSocketFile(t *testing.T) {
 	_, err = GenerateSocketFile("")
 
 	assert.Error(t, err)
-
 }
 
 func TestExecProcessAndKill(t *testing.T) {
-
 	// Simulate a sample process running (e.g sleep, 30 secs)
 	cmd := exec.Command("sleep", "30")
 
@@ -142,7 +137,6 @@ func TestExecProcessAndKill(t *testing.T) {
 	// Verify process is killed
 	err = cmd.Process.Signal(syscall.Signal(0))
 	assert.Error(t, err) // Should return an error since process is killed
-
 }
 
 func TestUnmarshalJsonPayload(t *testing.T) {
@@ -593,13 +587,10 @@ func TestStopProcess(t *testing.T) {
 // Test file extraction process
 
 func TestExtractDiskImageFromFile(t *testing.T) {
-
-	tmpDir, err := os.MkdirTemp("", "spx-utils-test-*")
+	tmpDir := t.TempDir()
 
 	t.Log("Temp dir:", tmpDir)
 
-	assert.NoError(t, err, "Temp dir should be created")
-
 	// Sample .xz (fail)
 	imagePath, err := ExtractDiskImageFromFile("/tmp/file.xz", tmpDir)
 
@@ -631,7 +622,6 @@ func TestExtractDiskImageFromFile(t *testing.T) {
 	_, err = exec.LookPath("tar")
 
 	if err == nil {
-
 		// Sample .tgz
 		imagePath, err = ExtractDiskImageFromFile("../../tests/unit-test-disk-image2.tgz", tmpDir)
 
@@ -671,14 +661,12 @@ func TestExtractDiskImageFromFile(t *testing.T) {
 		assert.Contains(t, imagePath, ".raw")
 
 		assert.NoError(t, err, "Should not error")
-
 	} else {
 		t.Skip("tar command not found, skipping archive extraction tests")
 	}
 
 	//err = os.RemoveAll(tmpDir)
 	//assert.NoError(t, err, "Could not remove temp dir")
-
 }
 
 func TestIsSocketURI(t *testing.T) {
@@ -774,9 +762,8 @@ func TestDirExists(t *testing.T) {
 	assert.False(t, dirExists("/nonexistent/path/should/not/exist"))
 
 	// File (not a directory)
-	tmpFile, err := os.CreateTemp("", "direxists-test-*")
+	tmpFile, err := os.CreateTemp(t.TempDir(), "direxists-test-*")
 	require.NoError(t, err)
-	defer os.Remove(tmpFile.Name())
 	tmpFile.Close()
 	assert.False(t, dirExists(tmpFile.Name()))
 }

spinifex/vm/vm.go

@@ -135,7 +135,6 @@ type Config struct {
 }
 
 func (cfg *Config) Execute() (*exec.Cmd, error) {
-
 	args := []string{}
 
 	if cfg.PIDFile != "" {
@@ -154,9 +153,7 @@ func (cfg *Config) Execute() (*exec.Cmd, error) {
 		//slog.Warn("Setting -cpu max, `host` CPU type unavailable")
 		// Use qemu defaults
 		//args = append(args, "-cpu", "max")
-
 	} else {
-
 		if cfg.EnableKVM {
 			args = append(args, "-enable-kvm")
 		}
@@ -164,7 +161,6 @@ func (cfg *Config) Execute() (*exec.Cmd, error) {
 		if cfg.CPUType != "" {
 			args = append(args, "-cpu", cfg.CPUType)
 		}
-
 	}
 
 	if cfg.NoGraphic {
@@ -198,7 +194,6 @@ func (cfg *Config) Execute() (*exec.Cmd, error) {
 	}
 
 	for _, drive := range cfg.Drives {
-
 		var opts []string
 
 		//args = append(args, "-drive", fmt.Sprintf("file=%s", drive.File)
@@ -228,7 +223,6 @@ func (cfg *Config) Execute() (*exec.Cmd, error) {
 		}
 
 		args = append(args, "-drive", strings.Join(opts, ","))
-
 	}
 
 	for _, device := range cfg.Devices {
@@ -267,7 +261,6 @@ func (cfg *Config) Execute() (*exec.Cmd, error) {
 			slog.Warn("UEFI firmware file not found for ARM virt machine. Ensure qemu-efi-aarch64 package is installed.", "path", uefiPath)
 			return nil, fmt.Errorf("UEFI firmware file not found for ARM virt machine")
 		}
-
 	} else if cfg.MachineType != "" {
 		args = append(args, "-M", cfg.MachineType)
 	}

spinifex/vm/vm_test.go

@@ -9,7 +9,6 @@ import (
 )
 
 func TestExecute(t *testing.T) {
-
 	cfg := Config{
 		Name: "test-vm",
 	}
@@ -74,7 +73,6 @@ func TestExecute(t *testing.T) {
 
 	assert.Contains(t, cmd.Path, "qemu-system-aarch64")
 	assert.Equal(t, expectedArgs, cmd.Args[1:])
-
 }
 
 func TestExecute_IOThreadAndCache(t *testing.T) {

tests/e2e/lib/multinode-helpers.sh

@@ -76,7 +76,8 @@ start_node_services() {
     echo "Starting services for node$node_num at $node_ip..."
 
     # Start all services - each node's config binds to its specific IP
-    ./scripts/start-dev.sh "$data_dir"
+    # UI is not needed for E2E tests and fails in pseudo multi-node (wrong cert path)
+    UI=false ./scripts/start-dev.sh "$data_dir"
 
     echo "Node$node_num services started"
 }
@@ -552,7 +553,7 @@ wait_for_instance_recovery() {
 
     while [ $attempt -lt $max_attempts ]; do
         local state
-        state=$(aws --endpoint-url https://${NODE1_IP}:${AWSGW_PORT} ec2 describe-instances \
+        state=$(aws ec2 describe-instances \
             --instance-ids "$instance_id" \
             --query 'Reservations[0].Instances[0].State.Name' \
             --output text 2>/dev/null) || {

tests/e2e/run-multinode-e2e.sh

@@ -910,9 +910,223 @@ fi
 echo ""
 
 # ==========================================================================
-# Phase 10: Cleanup
+# Phase 10: VPC Networking
 # ==========================================================================
-echo "Phase 10: Cleanup"
+echo "Phase 10: VPC Networking"
+echo "========================================"
+echo "Testing VPC instance launch, PrivateIpAddress, and stop/start IP persistence..."
+
+# Create a fresh VPC + subnet (don't reuse bootstrap's to avoid interference)
+echo ""
+echo "Creating test VPC + subnet..."
+VPC_OUTPUT=$($AWS_EC2 create-vpc --cidr-block 10.200.0.0/16)
+VPC_ID=$(echo "$VPC_OUTPUT" | jq -r '.Vpc.VpcId')
+if [ -z "$VPC_ID" ] || [ "$VPC_ID" == "null" ]; then
+    echo "  ERROR: Failed to create VPC"
+    fail_test "VPC create"
+else
+    echo "  Created VPC: $VPC_ID (10.200.0.0/16)"
+    pass_test "VPC create"
+
+    SUBNET_OUTPUT=$($AWS_EC2 create-subnet --vpc-id "$VPC_ID" --cidr-block 10.200.1.0/24)
+    SUBNET_ID=$(echo "$SUBNET_OUTPUT" | jq -r '.Subnet.SubnetId')
+    if [ -z "$SUBNET_ID" ] || [ "$SUBNET_ID" == "null" ]; then
+        echo "  ERROR: Failed to create subnet"
+        fail_test "Subnet create"
+    else
+        echo "  Created Subnet: $SUBNET_ID (10.200.1.0/24)"
+        pass_test "Subnet create"
+
+        # Brief pause for OVN topology programming
+        sleep 2
+
+        # Launch 3 instances with subnet
+        echo ""
+        echo "Launching 3 VPC instances..."
+        VPC_INSTANCE_IDS=()
+        VPC_LAUNCH_OK=true
+        for i in 1 2 3; do
+            echo "  Launching VPC instance $i with subnet $SUBNET_ID..."
+            RUN_OUTPUT=$($AWS_EC2 run-instances \
+                --image-id "$AMI_ID" \
+                --instance-type "$INSTANCE_TYPE" \
+                --key-name spinifex-key \
+                --subnet-id "$SUBNET_ID")
+
+            VPC_INST_ID=$(echo "$RUN_OUTPUT" | jq -r '.Instances[0].InstanceId')
+            VPC_INST_IP=$(echo "$RUN_OUTPUT" | jq -r '.Instances[0].PrivateIpAddress // empty')
+
+            if [ -z "$VPC_INST_ID" ] || [ "$VPC_INST_ID" == "null" ]; then
+                echo "  ERROR: Failed to launch VPC instance $i"
+                VPC_LAUNCH_OK=false
+                break
+            fi
+            echo "  Launched: $VPC_INST_ID (PrivateIpAddress: ${VPC_INST_IP:-not yet assigned})"
+            VPC_INSTANCE_IDS+=("$VPC_INST_ID")
+            sleep 1
+        done
+
+        if [ "$VPC_LAUNCH_OK" = true ] && [ ${#VPC_INSTANCE_IDS[@]} -eq 3 ]; then
+            pass_test "VPC instance launch"
+
+            # Wait for all VPC instances to be running
+            echo ""
+            echo "Waiting for VPC instances to reach running state..."
+            VPC_ALL_RUNNING=true
+            for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do
+                wait_for_instance_state "$vpc_inst" "running" 60 || {
+                    echo "  ERROR: VPC instance $vpc_inst failed to start"
+                    VPC_ALL_RUNNING=false
+                }
+            done
+
+            if [ "$VPC_ALL_RUNNING" = true ]; then
+                # Verify PrivateIpAddress in DescribeInstances
+                echo ""
+                echo "Verifying PrivateIpAddress in DescribeInstances..."
+                VPC_PRIVATE_IPS=()
+                VPC_IP_OK=true
+                for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do
+                    DESCRIBE_OUT=$($AWS_EC2 describe-instances --instance-ids "$vpc_inst")
+                    PRIVATE_IP=$(echo "$DESCRIBE_OUT" | jq -r '.Reservations[0].Instances[0].PrivateIpAddress // empty')
+                    INST_SUBNET=$(echo "$DESCRIBE_OUT" | jq -r '.Reservations[0].Instances[0].SubnetId // empty')
+                    INST_VPC=$(echo "$DESCRIBE_OUT" | jq -r '.Reservations[0].Instances[0].VpcId // empty')
+
+                    if [ -z "$PRIVATE_IP" ]; then
+                        echo "  ERROR: $vpc_inst has no PrivateIpAddress"
+                        VPC_IP_OK=false
+                        continue
+                    fi
+
+                    echo "  $vpc_inst: IP=$PRIVATE_IP, Subnet=$INST_SUBNET, VPC=$INST_VPC"
+
+                    if [ "$INST_SUBNET" != "$SUBNET_ID" ]; then
+                        echo "  ERROR: SubnetId mismatch (expected $SUBNET_ID, got $INST_SUBNET)"
+                        VPC_IP_OK=false
+                    fi
+                    if [ "$INST_VPC" != "$VPC_ID" ]; then
+                        echo "  ERROR: VpcId mismatch (expected $VPC_ID, got $INST_VPC)"
+                        VPC_IP_OK=false
+                    fi
+
+                    VPC_PRIVATE_IPS+=("$PRIVATE_IP")
+                done
+
+                if [ "$VPC_IP_OK" = true ]; then
+                    # Verify all IPs are unique and in the subnet range (10.200.1.x)
+                    UNIQUE_IPS=$(printf '%s\n' "${VPC_PRIVATE_IPS[@]}" | sort -u | wc -l)
+                    IP_RANGE_OK=true
+                    if [ "$UNIQUE_IPS" -ne "${#VPC_PRIVATE_IPS[@]}" ]; then
+                        echo "  ERROR: Duplicate IPs detected: ${VPC_PRIVATE_IPS[*]}"
+                        IP_RANGE_OK=false
+                    fi
+                    for ip in "${VPC_PRIVATE_IPS[@]}"; do
+                        if ! echo "$ip" | grep -qE '^10\.200\.1\.[0-9]+$'; then
+                            echo "  ERROR: IP $ip not in expected subnet 10.200.1.0/24"
+                            IP_RANGE_OK=false
+                        fi
+                    done
+
+                    if [ "$IP_RANGE_OK" = true ]; then
+                        echo "  All IPs unique and in correct subnet: ${VPC_PRIVATE_IPS[*]}"
+                        pass_test "VPC IP allocation"
+                    else
+                        fail_test "VPC IP allocation"
+                    fi
+
+                    # Stop/Start IP persistence
+                    echo ""
+                    echo "Testing stop/start IP persistence..."
+                    echo "  IPs before stop: ${VPC_PRIVATE_IPS[*]}"
+
+                    # Stop all VPC instances
+                    for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do
+                        $AWS_EC2 stop-instances --instance-ids "$vpc_inst" > /dev/null
+                    done
+                    VPC_ALL_STOPPED=true
+                    for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do
+                        wait_for_instance_state "$vpc_inst" "stopped" 60 || {
+                            echo "  ERROR: VPC instance $vpc_inst failed to stop"
+                            VPC_ALL_STOPPED=false
+                        }
+                    done
+
+                    if [ "$VPC_ALL_STOPPED" = true ]; then
+                        # Verify IPs persist while stopped
+                        IP_PERSIST_OK=true
+                        for idx in "${!VPC_INSTANCE_IDS[@]}"; do
+                            vpc_inst="${VPC_INSTANCE_IDS[$idx]}"
+                            expected_ip="${VPC_PRIVATE_IPS[$idx]}"
+                            STOPPED_IP=$($AWS_EC2 describe-instances --instance-ids "$vpc_inst" \
+                                --query 'Reservations[0].Instances[0].PrivateIpAddress' --output text)
+                            if [ "$STOPPED_IP" != "$expected_ip" ]; then
+                                echo "  ERROR: $vpc_inst IP changed while stopped (expected $expected_ip, got $STOPPED_IP)"
+                                IP_PERSIST_OK=false
+                            fi
+                        done
+
+                        # Restart all VPC instances
+                        for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do
+                            $AWS_EC2 start-instances --instance-ids "$vpc_inst" > /dev/null
+                        done
+                        for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do
+                            wait_for_instance_state "$vpc_inst" "running" 60 || true
+                        done
+
+                        # Verify IPs persist after restart
+                        for idx in "${!VPC_INSTANCE_IDS[@]}"; do
+                            vpc_inst="${VPC_INSTANCE_IDS[$idx]}"
+                            expected_ip="${VPC_PRIVATE_IPS[$idx]}"
+                            RESTARTED_IP=$($AWS_EC2 describe-instances --instance-ids "$vpc_inst" \
+                                --query 'Reservations[0].Instances[0].PrivateIpAddress' --output text)
+                            if [ "$RESTARTED_IP" != "$expected_ip" ]; then
+                                echo "  ERROR: $vpc_inst IP changed after restart (expected $expected_ip, got $RESTARTED_IP)"
+                                IP_PERSIST_OK=false
+                            else
+                                echo "  $vpc_inst: IP=$RESTARTED_IP (matches pre-stop)"
+                            fi
+                        done
+
+                        if [ "$IP_PERSIST_OK" = true ]; then
+                            pass_test "VPC stop/start IP persistence"
+                        else
+                            fail_test "VPC stop/start IP persistence"
+                        fi
+                    fi
+                else
+                    fail_test "VPC IP verification"
+                fi
+            fi
+
+            # Cleanup VPC instances
+            echo ""
+            echo "Cleaning up VPC instances..."
+            terminate_and_wait "${VPC_INSTANCE_IDS[@]}" || true
+        else
+            fail_test "VPC instance launch"
+            # Cleanup any partially-launched VPC instances
+            if [ ${#VPC_INSTANCE_IDS[@]} -gt 0 ]; then
+                echo "  Cleaning up partially-launched VPC instances..."
+                terminate_and_wait "${VPC_INSTANCE_IDS[@]}" || true
+            fi
+        fi
+
+        # Cleanup subnet
+        echo "  Deleting subnet $SUBNET_ID..."
+        $AWS_EC2 delete-subnet --subnet-id "$SUBNET_ID" 2>/dev/null || true
+    fi
+
+    # Cleanup VPC
+    echo "  Deleting VPC $VPC_ID..."
+    $AWS_EC2 delete-vpc --vpc-id "$VPC_ID" 2>/dev/null || true
+fi
+
+echo ""
+
+# ==========================================================================
+# Phase 11: Cleanup
+# ==========================================================================
+echo "Phase 11: Cleanup"
 echo "========================================"
 
 echo "Terminating all instances..."