diff --git a/Makefile b/Makefile index 8b885f372..f86cfae05 100644 --- a/Makefile +++ b/Makefile @@ -82,6 +82,10 @@ build-ecs-agent: ## Build the ecs-agent (ships in the ECS-AMI guest; not in host @echo -e "\n....Building ecs-agent (static)" CGO_ENABLED=0 go build -ldflags "-s -w -X main.version=$(VERSION)" -o ./bin/ecs-agent ./cmd/ecs-agent +build-loadgen: ## Build spx-loadgen (drives the stress harness's measured load; not shipped to nodes) + @echo -e "\n....Building spx-loadgen" + GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/spx-loadgen ./cmd/spx-loadgen + build-system-image: ## Build a system image from manifest (use IMAGE=lb or IMAGE=eks-node) ifndef IMAGE $(error IMAGE is required. Usage: make build-system-image IMAGE=lb) diff --git a/build/scripts/nats-js-watchdog.sh b/build/scripts/nats-js-watchdog.sh index 28f518720..71e75c889 100755 --- a/build/scripts/nats-js-watchdog.sh +++ b/build/scripts/nats-js-watchdog.sh @@ -58,6 +58,9 @@ RESTART_ESCALATE_AT="${RESTART_ESCALATE_AT:-3}" STATE_DIR="${STATE_DIR:-/run/spinifex}" COOLDOWN_STAMP="$STATE_DIR/nats-js-watchdog.last-restart" RESTART_COUNT_FILE="$STATE_DIR/nats-js-watchdog.restart-count" +# Marks that the escalation message has already been printed for the current +# latch, so a node that stays broken says so once instead of once per tick. +ESCALATED_STAMP="$STATE_DIR/nats-js-watchdog.escalated" # Monotonic clock source, so a wall-clock step during boot (NTP correcting an # unset RTC) cannot manufacture a false age or cooldown. Overridable so tests @@ -101,10 +104,10 @@ fi restart_count=$(cat "$RESTART_COUNT_FILE" 2>/dev/null) || restart_count="" case "$restart_count" in ''|*[!0-9]*) restart_count=0 ;; esac -if [ "$restart_count" -ge "$RESTART_ESCALATE_AT" ]; then - echo "nats-js-watchdog: already restarted $SERVICE $restart_count time(s) without recovery; giving up rather than looping (clear $RESTART_COUNT_FILE to re-arm)" >&2 - exit 0 -fi + +# The escalation count is read here but acted on only after the probes, below. +# Checking it up front would return before the one line that clears it, so a +# latched watchdog could never observe recovery and never re-arm itself. # curl is the probe transport; if it is unavailable, do nothing rather than # false-restart a healthy server. @@ -145,7 +148,9 @@ while [ "$i" -le "$SAMPLES" ]; do wrc=$? fi if [ "$wrc" -eq 0 ]; then - rm -f "$RESTART_COUNT_FILE" + # Recovery: drop the history so a later failure gets the full + # restart budget again, and re-arm the escalation message. + rm -f "$RESTART_COUNT_FILE" "$ESCALATED_STAMP" exit 0 fi if [ "$wrc" -eq 2 ]; then @@ -162,6 +167,18 @@ while [ "$i" -le "$SAMPLES" ]; do i=$((i + 1)) done +# The failure is real and current, so the restart history now means what it +# says. Above the threshold another restart will not help, so stop acting — +# but keep probing on later runs, because that is what notices a recovery. +if [ "$restart_count" -ge "$RESTART_ESCALATE_AT" ]; then + if [ ! -f "$ESCALATED_STAMP" ]; then + mkdir -p "$STATE_DIR" + : > "$ESCALATED_STAMP" + echo "nats-js-watchdog: already restarted $SERVICE $restart_count time(s) without recovery; giving up rather than looping (clear $RESTART_COUNT_FILE to re-arm)" >&2 + fi + exit 0 +fi + echo "nats-js-watchdog: $failure on $SAMPLES consecutive probes; restarting $SERVICE" >&2 mkdir -p "$STATE_DIR" uptime_seconds > "$COOLDOWN_STAMP" diff --git a/build/scripts/nats-js-watchdog_test.sh b/build/scripts/nats-js-watchdog_test.sh index 68fb38422..65a63df2b 100755 --- a/build/scripts/nats-js-watchdog_test.sh +++ b/build/scripts/nats-js-watchdog_test.sh @@ -100,8 +100,12 @@ invoke [ "${rc}" -eq 0 ] && pass "cooldown: exit 0" || fail "cooldown: expected exit 0 (rc=${rc}): $(cat "${STDERR}")" restarted && fail "cooldown: must not restart again inside the cooldown window" || pass "cooldown: no restart" -# --- Case 3: escalation threshold reached -> no restart, loud log --- +# --- Case 3: escalation threshold reached while still failing -> no restart, +# loud log. The probe must fail: a healthy probe at the threshold is case 3b, +# and it re-arms rather than giving up. --- reset_env escalated +FAKE_SPX_EXIT=1 +export FAKE_SPX_EXIT echo 3 > "${STATE_DIR}/nats-js-watchdog.restart-count" invoke [ "${rc}" -eq 0 ] && pass "escalated: exit 0" || fail "escalated: expected exit 0 (rc=${rc}): $(cat "${STDERR}")" @@ -109,6 +113,49 @@ restarted && fail "escalated: must not restart past the escalation threshold" || grep -q 'giving up rather than looping' "${STDERR}" \ && pass "escalated: loud log emitted" || fail "escalated: escalation log missing: $(cat "${STDERR}")" +# Second identical run: the node is still broken, so the decision stands, but +# repeating the message every timer tick is what buried the real signal for 42 +# hours on node1. +invoke +grep -q 'giving up rather than looping' "${STDERR}" \ + && fail "escalated: message repeated on every run instead of once per latch" \ + || pass "escalated: message not repeated" + +# --- Case 3b: escalation threshold reached but JetStream has recovered -> +# the watchdog must re-arm itself. This is the production bug: the escalation +# check used to run before the probe, so the clear below was unreachable and +# the node stayed disarmed until reboot. --- +reset_env escalated_recovered +echo 3 > "${STATE_DIR}/nats-js-watchdog.restart-count" +: > "${STATE_DIR}/nats-js-watchdog.escalated" +invoke +[ "${rc}" -eq 0 ] && pass "escalated-recovered: exit 0" || fail "escalated-recovered: expected exit 0 (rc=${rc}): $(cat "${STDERR}")" +restarted && fail "escalated-recovered: must not restart a recovered server" || pass "escalated-recovered: no restart" +[ -f "${STATE_DIR}/nats-js-watchdog.restart-count" ] \ + && fail "escalated-recovered: restart count must clear once JetStream is writable again" \ + || pass "escalated-recovered: restart count cleared" +[ -f "${STATE_DIR}/nats-js-watchdog.escalated" ] \ + && fail "escalated-recovered: escalation marker must clear so a later failure is reported" \ + || pass "escalated-recovered: escalation marker cleared" +[ -s "${STDERR}" ] \ + && fail "escalated-recovered: a recovered node must be silent: $(cat "${STDERR}")" \ + || pass "escalated-recovered: silent" + +# --- Case 3c: re-armed watchdog acts again. One healthy run clears the latch, +# and the next genuine failure must restart -- proving it is armed, not just +# quiet. --- +reset_env escalated_rearm +echo 3 > "${STATE_DIR}/nats-js-watchdog.restart-count" +invoke +FAKE_SPX_EXIT=1 +export FAKE_SPX_EXIT +invoke +restarted && pass "rearm: restarts again after recovery cleared the latch" \ + || fail "rearm: expected a restart once re-armed: $(cat "${STDERR}")" +[ "$(cat "${STATE_DIR}/nats-js-watchdog.restart-count" 2>/dev/null)" = "1" ] \ + && pass "rearm: restart count restarts from 1" \ + || fail "rearm: expected count 1, got $(cat "${STATE_DIR}/nats-js-watchdog.restart-count" 2>/dev/null)" + # --- Case 4: js-probe exits 2 (inconclusive) -> no restart --- reset_env inconclusive FAKE_SPX_EXIT=2 diff --git a/cmd/spinifex/cmd/admin.go b/cmd/spinifex/cmd/admin.go index e1ccdb478..fb0eb1bd4 100644 --- a/cmd/spinifex/cmd/admin.go +++ b/cmd/spinifex/cmd/admin.go @@ -2743,6 +2743,14 @@ func runAccountCreate(cmd *cobra.Command, args []string) { } func runAccountList(cmd *cobra.Command, args []string) { + if remote, _ := cmd.Flags().GetBool("remote"); remote { + if err := runAccountListRemote(cmd); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + return + } + svc, _, _, cleanup, err := initIAMServiceFromConfig() if err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) @@ -2756,20 +2764,7 @@ func runAccountList(cmd *cobra.Command, args []string) { os.Exit(1) } - if len(accounts) == 0 { - fmt.Println("No accounts found.") - return - } - - fmt.Printf("%-14s %-20s %-10s %s\n", "ACCOUNT ID", "NAME", "STATUS", "CREATED") - fmt.Printf("%-14s %-20s %-10s %s\n", "----------", "----", "------", "-------") - for _, a := range accounts { - created := a.CreatedAt - if t, err := time.Parse(time.RFC3339, a.CreatedAt); err == nil { - created = t.Format("2006-01-02 15:04") - } - fmt.Printf("%-14s %-20s %-10s %s\n", a.AccountID, a.AccountName, a.Status, created) - } + printAccountTable(accountSummaries(accounts)) } func runCertRenew(cmd *cobra.Command, _ []string) { diff --git a/cmd/spinifex/cmd/admin_account_delete.go b/cmd/spinifex/cmd/admin_account_delete.go new file mode 100644 index 000000000..dd725de4c --- /dev/null +++ b/cmd/spinifex/cmd/admin_account_delete.go @@ -0,0 +1,200 @@ +package cmd + +import ( + "bufio" + "context" + "errors" + "fmt" + "os" + "strings" + "time" + + "github.com/mulgadc/spinifex/spinifex/accountteardown" + "github.com/mulgadc/spinifex/spinifex/admin" + "github.com/mulgadc/spinifex/spinifex/objectstore" + "github.com/spf13/cobra" +) + +// accountDeleteTimeout is the outer bound on the whole teardown. It must stay +// well above the sum of the per-stage drain budgets, which are what actually +// decide when to give up on a resource; cutting in before them would abandon a +// large account midway with no stuck report to show for it. +const accountDeleteTimeout = 2 * time.Hour + +var accountDeleteCmd = &cobra.Command{ + Use: "delete ", + Short: "Tear down an account's infrastructure and remove the account", + Long: `Remove an account and everything it owns. + +The account is marked TERMINATING first, which blocks its credentials, and then +its resources are deleted in dependency order: compute, attachments, storage, +network, platform, identity, account. Nothing is removed until the account can +no longer create more. + +The account name must be given to confirm, so a mistyped account id fails +rather than emptying a live tenant. The system account (000000000000) and the +super admin (000000000001) can never be deleted. + + spx admin account delete 000000000042 --dry-run + spx admin account delete 000000000042 --name tenant@example.com + +--force is for resources the ordinary API refuses to delete: a volume attached +to an instance that will not terminate leaves both stranded and the account can +never be emptied. It clears the attachment in the control plane without the +guest's cooperation, so use it only once the guest is being destroyed.`, + Args: cobra.ExactArgs(1), + RunE: runAccountDelete, + + SilenceUsage: true, +} + +func init() { + accountCmd.AddCommand(accountDeleteCmd) + + accountDeleteCmd.Flags().Bool("dry-run", false, "Report what would be deleted and change nothing") + accountDeleteCmd.Flags().Bool("force", false, "Escalate past state guards for resources that will not delete") + accountDeleteCmd.Flags().Bool("yes", false, "Skip the interactive confirmation (requires --name)") + accountDeleteCmd.Flags().String("name", "", "Account name, which must match the stored record") + accountDeleteCmd.Flags().Bool("remote", false, "Delete over POST /admin/DeleteAccount instead of connecting to NATS") + accountDeleteCmd.Flags().String("endpoint", "", "Gateway endpoint for --remote (default: this node's AWS gateway)") + accountDeleteCmd.Flags().String("region", "", "SigV4 region for --remote (default: this node's region)") + accountDeleteCmd.Flags().String("ca-bundle", "", "CA certificate for --remote (default: this node's CA)") + accountDeleteCmd.Flags().String("client-token", "", "Idempotency token for --remote (default: generated; reuse it to follow the same teardown)") +} + +func runAccountDelete(cmd *cobra.Command, args []string) error { + accountID := args[0] + if remote, _ := cmd.Flags().GetBool("remote"); remote { + return runAccountDeleteRemote(cmd, accountID) + } + + dryRun, _ := cmd.Flags().GetBool("dry-run") + force, _ := cmd.Flags().GetBool("force") + assumeYes, _ := cmd.Flags().GetBool("yes") + name, _ := cmd.Flags().GetString("name") + + svc, cfg, nc, cleanup, err := initIAMServiceFromConfig() + if err != nil { + return err + } + defer cleanup() + + ctx, cancel := context.WithTimeout(cmd.Context(), accountDeleteTimeout) + defer cancel() + + node := cfg.Nodes[cfg.Node] + buckets := objectstore.NewS3ObjectStoreFromConfig( + admin.DialTarget(node.Predastore.Host), + node.Predastore.Region, + node.Predastore.AccessKey, + node.Predastore.SecretKey, + ) + + engine, err := accountteardown.NewClusterEngine(ctx, nc, len(cfg.Nodes), svc, buckets) + if err != nil { + return err + } + + if dryRun { + result, err := engine.Inventory(ctx, accountID) + if err != nil { + return err + } + printTeardownPlan(result) + fmt.Printf("\nDry run: nothing was deleted. %d resources would be removed.\n", result.DeletedCount()) + return nil + } + + // Always show the inventory before destroying it. An operator who has not + // seen what is about to go has not really confirmed anything. + plan, err := engine.Inventory(ctx, accountID) + if err != nil { + return err + } + printTeardownPlan(plan) + + if name == "" { + if assumeYes { + return errors.New("--yes requires --name so the account is still confirmed") + } + name, err = promptAccountName(accountID) + if err != nil { + return err + } + } + + result, err := engine.Teardown(ctx, accountteardown.Request{ + AccountID: accountID, AccountName: name, Force: force, + }) + if result != nil { + printTeardownResult(result) + } + if errors.Is(err, accountteardown.ErrResourcesStuck) { + fmt.Fprintf(os.Stderr, "\nThe account is still TERMINATING so its leftovers keep an owner.\n") + if !force { + fmt.Fprintf(os.Stderr, "Re-run with --force to clear resources the ordinary API refuses to delete.\n") + } + return err + } + if err != nil { + return err + } + + fmt.Printf("\nAccount %s deleted. %d resources removed.\n", accountID, result.DeletedCount()) + return nil +} + +// promptAccountName asks for the confirmation name on a terminal. +func promptAccountName(accountID string) (string, error) { + fmt.Printf("\nThis permanently deletes account %s and everything above.\n", accountID) + fmt.Print("Type the account name to confirm: ") + + reader := bufio.NewReader(os.Stdin) + entered, err := reader.ReadString('\n') + if err != nil { + return "", fmt.Errorf("read confirmation: %w", err) + } + entered = strings.TrimSpace(entered) + if entered == "" { + return "", errors.New("cancelled: no account name given") + } + return entered, nil +} + +func printTeardownPlan(result *accountteardown.Result) { + fmt.Printf("Account %s (%s)\n\n", result.AccountID, result.AccountName) + if result.DeletedCount() == 0 { + fmt.Println("The account holds no resources.") + return + } + for _, stage := range result.Stages { + if len(stage.Deleted) == 0 { + continue + } + fmt.Printf(" %s\n", stage.Stage) + for _, resource := range stage.Deleted { + fmt.Printf(" %s\n", resource) + } + } +} + +func printTeardownResult(result *accountteardown.Result) { + fmt.Println() + for _, stage := range result.Stages { + printTeardownStage(stage) + } +} + +// printTeardownStage reports one finished stage. A stage that removed nothing +// and left nothing behind is not printed: an account with no ECS clusters +// should not read as seven lines of activity. +func printTeardownStage(stage accountteardown.StageResult) { + if len(stage.Deleted) == 0 && len(stage.Stuck) == 0 { + return + } + fmt.Printf(" %-12s %d deleted, %d stuck (%s)\n", + stage.Stage, len(stage.Deleted), len(stage.Stuck), stage.Elapsed) + for _, stuck := range stage.Stuck { + fmt.Printf(" STUCK %s: %s\n", stuck.Resource, stuck.Reason) + } +} diff --git a/cmd/spinifex/cmd/admin_account_delete_output_test.go b/cmd/spinifex/cmd/admin_account_delete_output_test.go new file mode 100644 index 000000000..af7ea0db5 --- /dev/null +++ b/cmd/spinifex/cmd/admin_account_delete_output_test.go @@ -0,0 +1,162 @@ +package cmd_test + +import ( + "bytes" + "io" + "os" + "testing" + + "github.com/mulgadc/spinifex/cmd/spinifex/cmd" + "github.com/mulgadc/spinifex/spinifex/accountteardown" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// captureOutput redirects os.Stdout for the duration of fn and returns what was +// written. The teardown printers are the operator's only view of what is about +// to be destroyed, so what they print is the thing under test. +func captureOutput(t *testing.T, fn func()) string { + t.Helper() + r, w, err := os.Pipe() + require.NoError(t, err) + orig := os.Stdout + os.Stdout = w //nolint:reassign // test-local stdout capture, restored below + + fn() + + require.NoError(t, w.Close()) + os.Stdout = orig //nolint:reassign // restoring the real os.Stdout captured above + + var buf bytes.Buffer + _, err = io.Copy(&buf, r) + require.NoError(t, err) + return buf.String() +} + +// feedStdin swaps os.Stdin for a pipe carrying content, mirroring the stdout +// capture above. +func feedStdin(t *testing.T, content string, fn func()) { + t.Helper() + r, w, err := os.Pipe() + require.NoError(t, err) + orig := os.Stdin + os.Stdin = r //nolint:reassign // test-local stdin swap, restored below + t.Cleanup(func() { os.Stdin = orig }) //nolint:reassign // restoring the real os.Stdin captured above + + go func() { + _, _ = io.WriteString(w, content) + _ = w.Close() + }() + + fn() +} + +func teardownPlan() *accountteardown.Result { + return &accountteardown.Result{ + AccountID: "000000000042", + AccountName: "tenant@example.com", + DryRun: true, + Stages: []accountteardown.StageResult{ + { + Stage: accountteardown.StageCompute, + Deleted: []accountteardown.Resource{{Kind: "instance", ID: "i-1", Detail: "running"}}, + Elapsed: "0s", + }, + {Stage: accountteardown.StageAttachments, Elapsed: "0s"}, + { + Stage: accountteardown.StageStorage, + Deleted: []accountteardown.Resource{{Kind: "volume", ID: "vol-1"}}, + Elapsed: "0s", + }, + }, + } +} + +// The plan is what the operator confirms against, so every resource has to +// appear under its stage — and an empty stage must not pad the list. +func TestPrintTeardownPlanListsEveryResource(t *testing.T) { + output := captureOutput(t, func() { cmd.PrintTeardownPlan(teardownPlan()) }) + + assert.Contains(t, output, "000000000042") + assert.Contains(t, output, "tenant@example.com") + assert.Contains(t, output, "i-1") + assert.Contains(t, output, "vol-1") + assert.Contains(t, output, "compute") + assert.Contains(t, output, "storage") + assert.NotContains(t, output, "attachments", "an empty stage is noise, not information") +} + +// An account with nothing in it must say so. A bare header reads as truncated +// output, which is exactly when an operator stops and asks. +func TestPrintTeardownPlanSaysWhenTheAccountIsEmpty(t *testing.T) { + output := captureOutput(t, func() { + cmd.PrintTeardownPlan(&accountteardown.Result{ + AccountID: "000000000042", + AccountName: "tenant@example.com", + Stages: []accountteardown.StageResult{{Stage: accountteardown.StageCompute, Elapsed: "0s"}}, + }) + }) + + assert.Contains(t, output, "holds no resources") +} + +// A stuck resource is the most valuable thing a teardown reports: each one is a +// delete path that does not work. It must be named, with the reason. +func TestPrintTeardownResultNamesStuckResources(t *testing.T) { + result := &accountteardown.Result{ + AccountID: "000000000042", + Stages: []accountteardown.StageResult{ + { + Stage: accountteardown.StageCompute, + Deleted: []accountteardown.Resource{{Kind: "instance", ID: "i-1"}}, + Elapsed: "5s", + }, + {Stage: accountteardown.StageAttachments, Elapsed: "0s"}, + { + Stage: accountteardown.StageStorage, + Stuck: []accountteardown.Stuck{{ + Resource: accountteardown.Resource{Kind: "volume", ID: "vol-1"}, + Reason: "still attached to i-1", + }}, + Elapsed: "5m0s", + }, + }, + } + + output := captureOutput(t, func() { cmd.PrintTeardownResult(result) }) + + assert.Contains(t, output, "1 deleted, 0 stuck") + assert.Contains(t, output, "STUCK") + assert.Contains(t, output, "vol-1") + assert.Contains(t, output, "still attached to i-1") + assert.NotContains(t, output, "attachments") +} + +// The typed name is the confirmation. Surrounding whitespace is a paste +// artefact rather than a mismatch, but an empty line is a cancellation. +func TestPromptAccountNameReadsTheConfirmation(t *testing.T) { + var name string + var err error + + captureOutput(t, func() { + feedStdin(t, " tenant@example.com \n", func() { + name, err = cmd.PromptAccountName("000000000042") + }) + }) + + require.NoError(t, err) + assert.Equal(t, "tenant@example.com", name) +} + +func TestPromptAccountNameTreatsAnEmptyReplyAsCancelled(t *testing.T) { + var err error + + captureOutput(t, func() { + feedStdin(t, "\n", func() { + _, err = cmd.PromptAccountName("000000000042") + }) + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "cancelled") +} diff --git a/cmd/spinifex/cmd/admin_account_delete_remote.go b/cmd/spinifex/cmd/admin_account_delete_remote.go new file mode 100644 index 000000000..1290c2eab --- /dev/null +++ b/cmd/spinifex/cmd/admin_account_delete_remote.go @@ -0,0 +1,142 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "os" + "time" + + "github.com/mulgadc/spinifex/spinifex/accountteardown" + "github.com/mulgadc/spinifex/spinifex/gateway" + "github.com/spf13/cobra" +) + +// accountDeletePollInterval is how often --remote asks the gateway how far the +// teardown has got. The work is minutes long, so a tighter poll would only add +// load without telling the operator anything new. +var accountDeletePollInterval = 10 * time.Second + +// runAccountDeleteRemote drives POST /admin/DeleteAccount and then follows the +// job to completion, so the remote path reports the same outcome the local one +// prints rather than only that the teardown started. +func runAccountDeleteRemote(cmd *cobra.Command, accountID string) error { + dryRun, _ := cmd.Flags().GetBool("dry-run") + force, _ := cmd.Flags().GetBool("force") + assumeYes, _ := cmd.Flags().GetBool("yes") + name, _ := cmd.Flags().GetString("name") + clientToken, _ := cmd.Flags().GetString("client-token") + + target, err := resolveAdminTarget(cmd) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(cmd.Context(), accountDeleteTimeout) + defer cancel() + + // The inventory is fetched first even for a real deletion: an operator who + // has not seen what is about to go has not really confirmed anything. + plan, err := deleteAccountRemote(ctx, target, gateway.DeleteAccountRequest{ + AccountID: accountID, DryRun: true, + }) + if err != nil { + return err + } + if plan.Inventory != nil { + printTeardownPlan(plan.Inventory) + } + if dryRun { + fmt.Printf("\nDry run: nothing was deleted. %d resources would be removed.\n", plan.Inventory.DeletedCount()) + return nil + } + + if name == "" { + if assumeYes { + return errors.New("--yes requires --name so the account is still confirmed") + } + if name, err = promptAccountName(accountID); err != nil { + return err + } + } + if clientToken == "" { + clientToken = newClientToken() + } + + started, err := deleteAccountRemote(ctx, target, gateway.DeleteAccountRequest{ + AccountID: accountID, AccountName: name, ClientToken: clientToken, Force: force, + }) + if err != nil { + var adminErr *adminError + if errors.As(err, &adminErr) && retryableAdminErrors[adminErr.Code] { + fmt.Fprintf(os.Stderr, "Retry with --client-token %s to follow the same teardown.\n", clientToken) + } + return err + } + fmt.Printf("\nTeardown started: %s\n", started.DeletionID) + + return followAccountDeletion(ctx, target, accountID) +} + +// followAccountDeletion polls until the job reaches a terminal state, printing +// each stage as it lands. +func followAccountDeletion(ctx context.Context, target adminTarget, accountID string) error { + reported := 0 + for { + job, err := describeAccountDeletionRemote(ctx, target, accountID) + if err != nil { + return err + } + + for ; reported < len(job.Stages); reported++ { + printTeardownStage(job.Stages[reported]) + } + + switch job.State { + case gateway.DeletionStateCompleted: + fmt.Printf("\nAccount %s deleted.\n", accountID) + return nil + case gateway.DeletionStateFailed: + fmt.Fprintf(os.Stderr, "\nThe account is still TERMINATING so its leftovers keep an owner.\n") + return fmt.Errorf("teardown failed: %s", job.Error) + } + + select { + case <-ctx.Done(): + return fmt.Errorf("stopped following teardown of %s: %w", accountID, ctx.Err()) + case <-time.After(accountDeletePollInterval): + } + } +} + +func deleteAccountRemote( + ctx context.Context, + target adminTarget, + req gateway.DeleteAccountRequest, +) (*gateway.DeleteAccountResponse, error) { + var out gateway.DeleteAccountResponse + if err := callAdmin(ctx, target, "DeleteAccount", req, &out); err != nil { + return nil, err + } + return &out, nil +} + +func describeAccountDeletionRemote(ctx context.Context, target adminTarget, accountID string) (*accountDeletionStatus, error) { + var out accountDeletionStatus + err := callAdmin(ctx, target, "DescribeAccountDeletion", + gateway.DescribeAccountDeletionRequest{AccountID: accountID}, &out) + if err != nil { + return nil, err + } + return &out, nil +} + +// accountDeletionStatus is the subset of the stored job this command reports. +// The gateway's own record type is unexported, and the CLI needs no more than +// the state, the stages and why it stopped. +type accountDeletionStatus struct { + DeletionID string `json:"deletionId"` + State string `json:"state"` + Stages []accountteardown.StageResult `json:"stages"` + Error string `json:"error"` +} diff --git a/cmd/spinifex/cmd/admin_account_delete_remote_test.go b/cmd/spinifex/cmd/admin_account_delete_remote_test.go new file mode 100644 index 000000000..9f89a3483 --- /dev/null +++ b/cmd/spinifex/cmd/admin_account_delete_remote_test.go @@ -0,0 +1,199 @@ +package cmd_test + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/mulgadc/spinifex/cmd/spinifex/cmd" + "github.com/mulgadc/spinifex/spinifex/gateway" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// adminTestCredentials puts a static key in the environment so the SigV4 signer +// resolves without reaching for IMDS. +func adminTestCredentials(t *testing.T) { + t.Helper() + t.Setenv("AWS_ACCESS_KEY_ID", "AKIAEXAMPLE") + t.Setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY") +} + +// newScriptedAdminServer answers each successive request with the next body, +// repeating the last one, which is how a job that changes state between polls +// is reproduced. +func newScriptedAdminServer(t *testing.T, bodies ...string) (*httptest.Server, *atomic.Int32) { + t.Helper() + var calls atomic.Int32 + + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + index := int(calls.Add(1)) - 1 + if index >= len(bodies) { + index = len(bodies) - 1 + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(bodies[index])) + })) + t.Cleanup(srv.Close) + return srv, &calls +} + +// The delete request must reach /admin/DeleteAccount signed for the spinifex +// service: a key signed for another service is treated as probing and denied. +func TestDeleteAccountRemoteSignsAndPosts(t *testing.T) { + adminTestCredentials(t) + + var got http.Request + srv := newAdminTestServer(t, http.StatusOK, + `{"deletionId":"d-1","accountId":"000000000042","state":"RUNNING"}`, &got) + + out, err := cmd.DeleteAccountRemote(t.Context(), + cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), + gateway.DeleteAccountRequest{ + AccountID: "000000000042", + AccountName: "tenant@example.com", + ClientToken: strings.Repeat("a", 32), + }) + require.NoError(t, err) + + assert.Equal(t, "d-1", out.DeletionID) + assert.Equal(t, gateway.DeletionStateRunning, out.State) + + assert.Equal(t, http.MethodPost, got.Method) + assert.Equal(t, "/admin/DeleteAccount", got.URL.Path) + assert.Contains(t, got.Header.Get("Authorization"), "/us-west-1/spinifex/aws4_request") +} + +// A dry run answers inline with the inventory, which is what the operator +// confirms against before the real call. +func TestDeleteAccountRemoteDryRunCarriesTheInventory(t *testing.T) { + adminTestCredentials(t) + + var got http.Request + srv := newAdminTestServer(t, http.StatusOK, + `{"accountId":"000000000042","state":"DRY_RUN","dryRun":true,"inventory":{"accountId":"000000000042",`+ + `"dryRun":true,"stages":[{"stage":1,"deleted":[{"kind":"instance","id":"i-1"}],"elapsed":"0s"}]}}`, &got) + + out, err := cmd.DeleteAccountRemote(t.Context(), + cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), + gateway.DeleteAccountRequest{AccountID: "000000000042", DryRun: true}) + require.NoError(t, err) + + assert.True(t, out.DryRun) + require.NotNil(t, out.Inventory) + assert.Equal(t, 1, out.Inventory.DeletedCount()) +} + +// A refusal must surface as the gateway's code, not as a decode failure, so the +// operator can tell a protected account from an unreachable cluster. +func TestDeleteAccountRemoteSurfacesGatewayError(t *testing.T) { + adminTestCredentials(t) + + var got http.Request + srv := newAdminTestServer(t, http.StatusForbidden, + `{"error":{"code":"AccessDenied","message":"denied"},"requestId":"req-9"}`, &got) + + _, err := cmd.DeleteAccountRemote(t.Context(), + cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), + gateway.DeleteAccountRequest{ + AccountID: "000000000001", + AccountName: "spinifex", + ClientToken: strings.Repeat("a", 32), + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "AccessDenied") + assert.Contains(t, err.Error(), "req-9") + assert.False(t, cmd.RetryableAdminError(err), "a protected account is not fixed by retrying") +} + +func TestDescribeAccountDeletionRemoteReportsProgress(t *testing.T) { + adminTestCredentials(t) + + var got http.Request + srv := newAdminTestServer(t, http.StatusOK, + `{"deletionId":"d-1","state":"RUNNING","stages":[{"stage":1,"elapsed":"5s"}]}`, &got) + + out, err := cmd.DescribeAccountDeletionRemote(t.Context(), + cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), "000000000042") + require.NoError(t, err) + + assert.Equal(t, "/admin/DescribeAccountDeletion", got.URL.Path) + assert.Equal(t, gateway.DeletionStateRunning, out.State) + assert.Len(t, out.Stages, 1) +} + +// The follow loop is what makes --remote report the same outcome the local +// path prints. It must keep polling while the job runs and stop on the first +// terminal state. +func TestFollowAccountDeletionPollsUntilComplete(t *testing.T) { + adminTestCredentials(t) + defer cmd.SetAccountDeletePollInterval(time.Millisecond)() + + srv, calls := newScriptedAdminServer(t, + `{"deletionId":"d-1","state":"RUNNING","stages":[{"stage":1,"elapsed":"5s"}]}`, + `{"deletionId":"d-1","state":"RUNNING","stages":[{"stage":1,"elapsed":"5s"},{"stage":2,"elapsed":"1s"}]}`, + `{"deletionId":"d-1","state":"COMPLETED","stages":[{"stage":1,"elapsed":"5s"},{"stage":2,"elapsed":"1s"}]}`, + ) + + err := cmd.FollowAccountDeletion(t.Context(), + cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), "000000000042") + + require.NoError(t, err) + assert.Equal(t, int32(3), calls.Load(), "the loop must keep polling while the job runs") +} + +// A failed teardown must exit non-zero with the stored reason: the account is +// left TERMINATING, and an operator who reads "done" would not go looking. +func TestFollowAccountDeletionReportsFailure(t *testing.T) { + adminTestCredentials(t) + defer cmd.SetAccountDeletePollInterval(time.Millisecond)() + + srv, _ := newScriptedAdminServer(t, + `{"deletionId":"d-1","state":"FAILED","stages":[],"error":"2 resources left"}`) + + err := cmd.FollowAccountDeletion(t.Context(), + cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), "000000000042") + + require.Error(t, err) + assert.Contains(t, err.Error(), "2 resources left") +} + +// Giving up on following is not the same as the teardown stopping, so the +// message has to name the account the caller must go back to. +func TestFollowAccountDeletionStopsWhenTheContextEnds(t *testing.T) { + adminTestCredentials(t) + defer cmd.SetAccountDeletePollInterval(50 * time.Millisecond)() + + srv, _ := newScriptedAdminServer(t, `{"deletionId":"d-1","state":"RUNNING","stages":[]}`) + + ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) + defer cancel() + + err := cmd.FollowAccountDeletion(ctx, + cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), "000000000042") + + require.Error(t, err) + assert.Contains(t, err.Error(), "000000000042") +} + +// An unreadable job is not a finished one. Reporting it as complete would hide +// a teardown still holding an account's resources. +func TestFollowAccountDeletionFailsOnAnUnreadableJob(t *testing.T) { + adminTestCredentials(t) + defer cmd.SetAccountDeletePollInterval(time.Millisecond)() + + var got http.Request + srv := newAdminTestServer(t, http.StatusNotFound, + `{"error":{"code":"NoSuchEntity","message":"no job"},"requestId":"req-2"}`, &got) + + err := cmd.FollowAccountDeletion(t.Context(), + cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), "000000000042") + + require.Error(t, err) + assert.Contains(t, err.Error(), "NoSuchEntity") +} diff --git a/cmd/spinifex/cmd/admin_account_list_remote.go b/cmd/spinifex/cmd/admin_account_list_remote.go new file mode 100644 index 000000000..52ea5cee8 --- /dev/null +++ b/cmd/spinifex/cmd/admin_account_list_remote.go @@ -0,0 +1,67 @@ +package cmd + +import ( + "context" + "fmt" + "time" + + "github.com/mulgadc/spinifex/spinifex/gateway" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + "github.com/spf13/cobra" +) + +// runAccountListRemote reads the listing over POST /admin/ListAccounts, which +// is how an off-cluster operator or the load-test harness sees the tenants it +// created without SSH access to a node. +func runAccountListRemote(cmd *cobra.Command) error { + target, err := resolveAdminTarget(cmd) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(cmd.Context(), adminRequestTimeout) + defer cancel() + + var out gateway.ListAccountsResponse + if err := callAdmin(ctx, target, "ListAccounts", struct{}{}, &out); err != nil { + return err + } + + printAccountTable(out.Accounts) + return nil +} + +// accountSummaries adapts the local listing to the shape the remote one +// returns, so both paths print through one function and cannot drift. +func accountSummaries(accounts []*handlers_iam.Account) []gateway.AccountSummary { + summaries := make([]gateway.AccountSummary, 0, len(accounts)) + for _, account := range accounts { + if account == nil { + continue + } + summaries = append(summaries, gateway.AccountSummary{ + AccountID: account.AccountID, + AccountName: account.AccountName, + Status: account.Status, + CreatedAt: account.CreatedAt, + }) + } + return summaries +} + +func printAccountTable(accounts []gateway.AccountSummary) { + if len(accounts) == 0 { + fmt.Println("No accounts found.") + return + } + + fmt.Printf("%-14s %-30s %-12s %s\n", "ACCOUNT ID", "NAME", "STATUS", "CREATED") + fmt.Printf("%-14s %-30s %-12s %s\n", "----------", "----", "------", "-------") + for _, account := range accounts { + created := account.CreatedAt + if parsed, err := time.Parse(time.RFC3339, account.CreatedAt); err == nil { + created = parsed.Format("2006-01-02 15:04") + } + fmt.Printf("%-14s %-30s %-12s %s\n", account.AccountID, account.AccountName, account.Status, created) + } +} diff --git a/cmd/spinifex/cmd/admin_ochre.go b/cmd/spinifex/cmd/admin_ochre.go index 22e468edc..2b6aa8788 100644 --- a/cmd/spinifex/cmd/admin_ochre.go +++ b/cmd/spinifex/cmd/admin_ochre.go @@ -774,8 +774,8 @@ func materializeWeightsVolume(ctx context.Context, provider ebsprovider.EBSProvi // stage time -- and treats a missing object as gone. Any other read failure // is a real error, not a signal to guess the snapshot's existence. func newWeightsSnapshotChecker(store objectstore.ObjectStore, bucket string) weightsSnapshotChecker { - return func(_ context.Context, snapshotID string) (bool, error) { - if _, err := handlers_ec2_snapshot.ReadSnapshotConfig(store, bucket, snapshotID); err != nil { + return func(ctx context.Context, snapshotID string) (bool, error) { + if _, err := handlers_ec2_snapshot.ReadSnapshotConfig(ctx, store, bucket, snapshotID); err != nil { if objectstore.IsNoSuchKeyError(err) { return false, nil } diff --git a/cmd/spinifex/cmd/admin_ochre_test.go b/cmd/spinifex/cmd/admin_ochre_test.go index 7bba9892e..1c0fec5b8 100644 --- a/cmd/spinifex/cmd/admin_ochre_test.go +++ b/cmd/spinifex/cmd/admin_ochre_test.go @@ -1100,7 +1100,7 @@ func TestRegisterWeightsSnapshot_WritesEC2ReadableMetadata(t *testing.T) { require.NoError(t, registerWeightsSnapshot(store, bucket, "snap-vol-abc", "vol-abc", 12*bytesPerGiB, "ap-southeast-2a", true)) - cfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(store, bucket, "snap-vol-abc") + cfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(context.Background(), store, bucket, "snap-vol-abc") require.NoError(t, err) assert.Equal(t, "snap-vol-abc", cfg.SnapshotID) assert.Equal(t, "vol-abc", cfg.VolumeID) @@ -1117,7 +1117,7 @@ func TestRegisterWeightsSnapshot_UnencryptedVolume(t *testing.T) { require.NoError(t, registerWeightsSnapshot(store, "predastore", "snap-vol-plain", "vol-plain", bytesPerGiB, "ap-southeast-2a", false)) - cfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(store, "predastore", "snap-vol-plain") + cfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(context.Background(), store, "predastore", "snap-vol-plain") require.NoError(t, err) assert.False(t, cfg.Encrypted) } diff --git a/cmd/spinifex/cmd/admin_principal.go b/cmd/spinifex/cmd/admin_principal.go new file mode 100644 index 000000000..1cad21c82 --- /dev/null +++ b/cmd/spinifex/cmd/admin_principal.go @@ -0,0 +1,628 @@ +package cmd + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "sort" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/mulgadc/spinifex/spinifex/admin" + "github.com/mulgadc/spinifex/spinifex/gateway" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + "github.com/spf13/cobra" +) + +// adminPrincipalPolicyName is the inline policy every admin principal carries. +// One name for every principal is what makes the listing readable: a user with +// this policy is an admin-API caller and its actions are the whole grant. +const adminPrincipalPolicyName = "spx-admin-methods" + +// adminPrincipalService is the SigV4 credential scope the admin surface +// requires, and the prefix each granted action carries. +const adminPrincipalService = "spinifex" + +// runAdminPrincipalCreate provisions a named principal in the super-admin +// account. Every step is create-if-absent except the access key, which is +// replaced so exactly one key is ever live. +func runAdminPrincipalCreate(cmd *cobra.Command, args []string) error { + userName := args[0] + if err := validatePrincipalName(userName); err != nil { + return err + } + + requested, _ := cmd.Flags().GetStringSlice("grant") + grants, err := resolvePrincipalGrants(requested) + if err != nil { + return err + } + + svc, _, _, cleanup, err := initIAMServiceFromConfig() + if err != nil { + return err + } + defer cleanup() + + accountID := admin.DefaultAccountID() + + if _, err := svc.CreateUser(accountID, &iam.CreateUserInput{ + UserName: aws.String(userName), + }); err != nil && !strings.Contains(err.Error(), "EntityAlreadyExists") { + return fmt.Errorf("create user %s: %w", userName, err) + } + + if _, err := svc.PutUserPolicy(accountID, &iam.PutUserPolicyInput{ + UserName: aws.String(userName), + PolicyName: aws.String(adminPrincipalPolicyName), + PolicyDocument: aws.String(principalPolicyDocument(grants)), + }); err != nil { + return fmt.Errorf("attach policy to %s: %w", userName, err) + } + + dropped, err := dropStalePrincipalPolicies(svc, accountID, userName) + if err != nil { + return err + } + for _, name := range dropped { + fmt.Printf("Removed stale inline policy %s\n", name) + } + + revoked, err := revokePrincipalKeys(svc, accountID, userName) + if err != nil { + return err + } + for _, keyID := range revoked { + fmt.Printf("Revoked previous access key %s\n", keyID) + } + + akOut, err := svc.CreateAccessKey(accountID, &iam.CreateAccessKeyInput{ + UserName: aws.String(userName), + }) + if err != nil { + return fmt.Errorf("create access key for %s: %w", userName, err) + } + if akOut == nil || akOut.AccessKey == nil { + return fmt.Errorf("create access key for %s: no key returned", userName) + } + + printPrincipalCredential(accountID, userName, grants, + aws.StringValue(akOut.AccessKey.AccessKeyId), + aws.StringValue(akOut.AccessKey.SecretAccessKey)) + return nil +} + +// printPrincipalCredential writes the one and only view of the secret, then the +// two things a caller still needs: the SigV4 scope and the region the key is +// useless without. +func printPrincipalCredential(accountID, userName string, grants []string, keyID, secret string) { + fmt.Printf("\nPrincipal %q ready.\n", userName) + fmt.Printf(" Account ID: %s\n", accountID) + fmt.Printf(" User: %s\n", userName) + fmt.Printf(" Permitted actions: %s\n", strings.Join(grants, ", ")) + fmt.Printf(" Access Key ID: %s\n", keyID) + fmt.Printf(" Secret Access Key: %s\n", secret) + fmt.Println("\nThe secret is shown once and is not recoverable. Store it in a secret manager") + fmt.Println("or an AWS profile; never commit it and never place it in a Spinifex config file.") + fmt.Printf("\nRequests must be SigV4-signed with service=%q and this cluster's region.\n", adminPrincipalService) +} + +// runAdminPrincipalList reports every principal in the super-admin account and +// what it may call, so an operator can answer "who can delete an account" from +// the cluster rather than from memory. +func runAdminPrincipalList(_ *cobra.Command, _ []string) error { + svc, _, _, cleanup, err := initIAMServiceFromConfig() + if err != nil { + return err + } + defer cleanup() + + accountID := admin.DefaultAccountID() + users, err := svc.ListUsers(accountID, &iam.ListUsersInput{}) + if err != nil { + return fmt.Errorf("list users: %w", err) + } + + rows := make([]principalRow, 0, len(users.Users)) + for _, user := range users.Users { + if user == nil || user.UserName == nil { + continue + } + row, err := describePrincipal(svc, accountID, aws.StringValue(user.UserName)) + if err != nil { + return err + } + rows = append(rows, row) + } + + printPrincipalTable(accountID, rows) + return nil +} + +// principalRow is one line of the principal listing. +type principalRow struct { + UserName string + Grants []string + Keys int +} + +// describePrincipal resolves what a user in the super-admin account may call. +// An attached wildcard policy is reported as such rather than expanded: the +// point of the listing is to show which principals are unscoped. +func describePrincipal(svc handlers_iam.IAMService, accountID, userName string) (principalRow, error) { + row := principalRow{UserName: userName} + + attached, err := svc.ListAttachedUserPolicies(accountID, &iam.ListAttachedUserPoliciesInput{ + UserName: aws.String(userName), + }) + if err != nil { + return row, fmt.Errorf("list attached policies for %s: %w", userName, err) + } + if attached != nil { + for _, policy := range attached.AttachedPolicies { + if policy == nil { + continue + } + row.Grants = append(row.Grants, aws.StringValue(policy.PolicyName)+" (attached)") + } + } + + inline, err := svc.GetUserPolicy(accountID, &iam.GetUserPolicyInput{ + UserName: aws.String(userName), + PolicyName: aws.String(adminPrincipalPolicyName), + }) + if err == nil && inline != nil { + row.Grants = append(row.Grants, principalPolicyActions(aws.StringValue(inline.PolicyDocument))...) + } + + keys, err := svc.ListAccessKeys(accountID, &iam.ListAccessKeysInput{UserName: aws.String(userName)}) + if err != nil { + return row, fmt.Errorf("list access keys for %s: %w", userName, err) + } + if keys != nil { + row.Keys = len(keys.AccessKeyMetadata) + } + return row, nil +} + +// principalPolicyActions returns the actions an inline policy allows, shortened +// to the method name. An undecodable document reports nothing rather than +// guessing, and the listing shows the user with no grants. +func principalPolicyActions(document string) []string { + var doc struct { + Statement []struct { + Effect string `json:"Effect"` + Action handlers_iam.StringOrArr `json:"Action"` + } `json:"Statement"` + } + if err := json.Unmarshal([]byte(document), &doc); err != nil { + return nil + } + + var actions []string + for _, stmt := range doc.Statement { + if stmt.Effect != "Allow" { + continue + } + for _, action := range stmt.Action { + actions = append(actions, strings.TrimPrefix(action, adminPrincipalService+":")) + } + } + sort.Strings(actions) + return actions +} + +// printPrincipalTable renders the listing. +func printPrincipalTable(accountID string, rows []principalRow) { + fmt.Printf("\nPrincipals in the super-admin account (%s):\n\n", accountID) + fmt.Printf("%-24s %-6s %s\n", "USER", "KEYS", "GRANTS") + fmt.Println(strings.Repeat("-", 96)) + for _, row := range rows { + grants := strings.Join(row.Grants, ", ") + if grants == "" { + grants = "-" + } + fmt.Printf("%-24s %-6d %s\n", row.UserName, row.Keys, grants) + } + fmt.Printf("\n%d principal(s)\n", len(rows)) +} + +// runAdminPrincipalRevoke removes every access key a principal holds, leaving +// the user and its policy in place. This is the response to a leaked secret: +// it takes effect cluster-wide immediately and needs no restart. +func runAdminPrincipalRevoke(_ *cobra.Command, args []string) error { + userName := args[0] + + svc, _, _, cleanup, err := initIAMServiceFromConfig() + if err != nil { + return err + } + defer cleanup() + + accountID := admin.DefaultAccountID() + if _, err := svc.GetUser(accountID, &iam.GetUserInput{UserName: aws.String(userName)}); err != nil { + return fmt.Errorf("no principal %q in account %s", userName, accountID) + } + + revoked, err := revokePrincipalKeys(svc, accountID, userName) + if err != nil { + return err + } + if len(revoked) == 0 { + fmt.Printf("Principal %q holds no access keys.\n", userName) + return nil + } + for _, keyID := range revoked { + fmt.Printf("Revoked access key %s\n", keyID) + } + fmt.Printf("\nPrincipal %q can no longer sign requests. Re-issue with:\n", userName) + fmt.Printf(" spx admin principal create %s\n", userName) + return nil +} + +// dropStalePrincipalPolicies removes every inline policy on the principal that +// this command did not write. A principal's grants are exactly one policy, so +// re-creating one with narrower grants cannot leave a wider policy behind. +func dropStalePrincipalPolicies(svc handlers_iam.IAMService, accountID, userName string) ([]string, error) { + listed, err := svc.ListUserPolicies(accountID, &iam.ListUserPoliciesInput{ + UserName: aws.String(userName), + }) + if err != nil { + return nil, fmt.Errorf("list inline policies for %s: %w", userName, err) + } + if listed == nil { + return nil, nil + } + + var dropped []string + for _, name := range listed.PolicyNames { + policyName := aws.StringValue(name) + if policyName == "" || policyName == adminPrincipalPolicyName { + continue + } + if _, err := svc.DeleteUserPolicy(accountID, &iam.DeleteUserPolicyInput{ + UserName: aws.String(userName), + PolicyName: name, + }); err != nil { + return nil, fmt.Errorf("remove inline policy %s from %s: %w", policyName, userName, err) + } + dropped = append(dropped, policyName) + } + return dropped, nil +} + +// runAdminPrincipalDelete removes a principal entirely: its keys, its policies +// and the user. Revoking leaves a principal that can be re-issued; this is for +// one that should not exist at all. +func runAdminPrincipalDelete(_ *cobra.Command, args []string) error { + userName := args[0] + if err := validatePrincipalName(userName); err != nil { + return err + } + + svc, _, _, cleanup, err := initIAMServiceFromConfig() + if err != nil { + return err + } + defer cleanup() + + accountID := admin.DefaultAccountID() + if _, err := svc.GetUser(accountID, &iam.GetUserInput{UserName: aws.String(userName)}); err != nil { + return fmt.Errorf("no principal %q in account %s", userName, accountID) + } + + revoked, err := revokePrincipalKeys(svc, accountID, userName) + if err != nil { + return err + } + for _, keyID := range revoked { + fmt.Printf("Revoked access key %s\n", keyID) + } + + policies, err := svc.ListUserPolicies(accountID, &iam.ListUserPoliciesInput{ + UserName: aws.String(userName), + }) + if err != nil { + return fmt.Errorf("list inline policies for %s: %w", userName, err) + } + if policies != nil { + for _, name := range policies.PolicyNames { + if _, err := svc.DeleteUserPolicy(accountID, &iam.DeleteUserPolicyInput{ + UserName: aws.String(userName), + PolicyName: name, + }); err != nil { + return fmt.Errorf("remove inline policy %s from %s: %w", + aws.StringValue(name), userName, err) + } + fmt.Printf("Removed inline policy %s\n", aws.StringValue(name)) + } + } + + if _, err := svc.DeleteUser(accountID, &iam.DeleteUserInput{ + UserName: aws.String(userName), + }); err != nil { + return fmt.Errorf("delete user %s: %w", userName, err) + } + + fmt.Printf("\nPrincipal %q removed from account %s.\n", userName, accountID) + return nil +} + +// revokePrincipalKeys deletes every access key the user holds and returns the +// IDs. A key from an earlier run is unrecoverable to its holder but still +// authenticates, so it is removed rather than left live alongside a new one. +func revokePrincipalKeys(svc handlers_iam.IAMService, accountID, userName string) ([]string, error) { + listed, err := svc.ListAccessKeys(accountID, &iam.ListAccessKeysInput{ + UserName: aws.String(userName), + }) + if err != nil { + return nil, fmt.Errorf("list access keys for %s: %w", userName, err) + } + if listed == nil { + return nil, nil + } + + var revoked []string + for _, meta := range listed.AccessKeyMetadata { + if meta == nil || meta.AccessKeyId == nil { + continue + } + if _, err := svc.DeleteAccessKey(accountID, &iam.DeleteAccessKeyInput{ + UserName: aws.String(userName), + AccessKeyId: meta.AccessKeyId, + }); err != nil { + return nil, fmt.Errorf("remove access key for %s: %w", userName, err) + } + revoked = append(revoked, aws.StringValue(meta.AccessKeyId)) + } + return revoked, nil +} + +// resolvePrincipalGrants turns the requested method names into the actions the +// policy allows. An empty request means every admin method: a principal minted +// without a --grant is the operator credential. +func resolvePrincipalGrants(requested []string) ([]string, error) { + known := gateway.AdminMethodNames() + if len(requested) == 0 { + return prefixGrants(known), nil + } + + byLower := make(map[string]string, len(known)) + for _, method := range known { + byLower[strings.ToLower(method)] = method + } + + seen := make(map[string]bool, len(requested)) + methods := make([]string, 0, len(requested)) + for _, entry := range requested { + name := strings.TrimSpace(entry) + name = strings.TrimPrefix(name, adminPrincipalService+":") + method, ok := byLower[strings.ToLower(name)] + if !ok { + return nil, fmt.Errorf("unknown admin method %q — the surface serves: %s", + entry, strings.Join(known, ", ")) + } + if seen[method] { + continue + } + seen[method] = true + methods = append(methods, method) + } + + sort.Strings(methods) + return prefixGrants(methods), nil +} + +// prefixGrants qualifies method names with the service scope the gateway +// evaluates them under. +func prefixGrants(methods []string) []string { + grants := make([]string, 0, len(methods)) + for _, method := range methods { + grants = append(grants, adminPrincipalService+":"+method) + } + return grants +} + +// principalPolicyDocument builds the inline policy. Every method is named: +// granting spinifex:* would authorise whatever is added to the admin surface +// next with a key minted before it existed. +func principalPolicyDocument(grants []string) string { + quoted := make([]string, 0, len(grants)) + for _, grant := range grants { + quoted = append(quoted, `"`+grant+`"`) + } + return `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":[` + + strings.Join(quoted, ",") + `],"Resource":"*"}]}` +} + +// validatePrincipalName rejects names that would collide with the account's own +// administrator or that IAM would not accept. +func validatePrincipalName(name string) error { + if name == "" { + return errors.New("principal name is required") + } + if name == handlers_iam.AdminUserName { + return fmt.Errorf("%q is the super-admin account's own administrator — "+ + "rotating its key would lock out the bootstrap credential; choose another name", + handlers_iam.AdminUserName) + } + if len(name) > 64 { + return errors.New("principal name must be 64 characters or fewer") + } + for _, r := range name { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + case r == '-', r == '_', r == '.', r == '+', r == '=', r == ',', r == '@': + default: + return fmt.Errorf("principal name %q contains an unsupported character %q", name, r) + } + } + return nil +} + +// runAdminPrincipalAudit reports roles in the super-admin account that any +// principal in the account can assume, which includes every admin principal. +func runAdminPrincipalAudit(_ *cobra.Command, _ []string) error { + svc, _, _, cleanup, err := initIAMServiceFromConfig() + if err != nil { + return err + } + defer cleanup() + + accountID := admin.DefaultAccountID() + roles, err := svc.ListRoles(accountID, &iam.ListRolesInput{}) + if err != nil { + return fmt.Errorf("list roles: %w", err) + } + + var flagged []string + for _, role := range roles.Roles { + if role == nil || role.AssumeRolePolicyDocument == nil { + continue + } + if trustsWholeAccount(aws.StringValue(role.AssumeRolePolicyDocument), accountID) { + flagged = append(flagged, aws.StringValue(role.RoleName)) + } + } + + if len(flagged) == 0 { + fmt.Printf("No roles in %s trust the account as a whole.\n", accountID) + return nil + } + + fmt.Fprintf(os.Stderr, "Roles in %s assumable by any principal in the account, including every admin principal:\n", accountID) + for _, name := range flagged { + fmt.Fprintf(os.Stderr, " %s\n", name) + } + return fmt.Errorf("%d role(s) trust the account as a whole — scope their trust policies to a specific principal", len(flagged)) +} + +// trustsWholeAccount reports whether an AssumeRolePolicyDocument admits every +// principal in accountID. A malformed document is reported as trusting, since +// a document nobody can parse is not a document anybody has verified. +func trustsWholeAccount(document, accountID string) bool { + doc, err := handlers_iam.ValidateTrustPolicyDocument(document) + if err != nil { + return true + } + for _, stmt := range doc.Statement { + if stmt.Effect != "Allow" { + continue + } + var principal struct { + AWS handlers_iam.StringOrArr `json:"AWS"` + } + if err := json.Unmarshal(stmt.Principal, &principal); err != nil { + continue + } + for _, entry := range principal.AWS { + switch entry { + case "*", accountID, "arn:aws:iam::" + accountID + ":root": + return true + } + } + } + return false +} + +var adminPrincipalCmd = &cobra.Command{ + Use: "principal", + Short: "Manage the credentials that call the private admin API", + Long: `Manage the IAM users in the super-admin account that sign POST /admin/ +requests. + +The private admin API creates, lists and deletes tenant accounts. It is not an +AWS API and is unreachable until a principal exists — there is no config toggle +and no default credential for it.`, +} + +var adminPrincipalCreateCmd = &cobra.Command{ + Use: "create ", + Short: "Create an admin principal and print its access key once", + Long: `Create an IAM user in the super-admin account with an inline policy allowing the +requested admin methods, then mint an access key. + +Without --grant the principal may call every admin method, which is the +credential an operator or a test harness uses. Pass --grant to scope it: a +self-service signup form, for example, needs only CreateAccount, and a key with +that grant cannot delete anything if the form's host is compromised. + +Each method is granted by name rather than with a wildcard, so a later addition +to the admin surface is not authorised by an existing key. + +The principal's grants are exactly one inline policy: re-running with fewer +--grant values narrows it, and any other inline policy on the user is removed. + +The secret is printed once and never recoverable. Re-running replaces the access +key, which revokes the previous one. + + spx admin principal create operator + spx admin principal create signup --grant CreateAccount + spx admin principal create auditor --grant ListAccounts,DescribeAccountDeletion`, + Args: cobra.ExactArgs(1), + RunE: runAdminPrincipalCreate, + SilenceUsage: true, +} + +var adminPrincipalListCmd = &cobra.Command{ + Use: "list", + Short: "List the principals in the super-admin account and what they may call", + Args: cobra.NoArgs, + RunE: runAdminPrincipalList, + SilenceUsage: true, +} + +var adminPrincipalRevokeCmd = &cobra.Command{ + Use: "revoke ", + Short: "Revoke a principal's access keys without removing the principal", + Long: `Delete every access key the named principal holds. The user and its policy stay, +so re-issuing is one "principal create" away. + +Revocation is immediate and cluster-wide.`, + Args: cobra.ExactArgs(1), + RunE: runAdminPrincipalRevoke, + SilenceUsage: true, +} + +var adminPrincipalDeleteCmd = &cobra.Command{ + Use: "delete ", + Short: "Remove a principal, its policies and its access keys", + Long: `Delete the named principal from the super-admin account entirely. + +Use "principal revoke" instead to stop a credential working while keeping the +principal, which is what a rotation after a leak needs.`, + Args: cobra.ExactArgs(1), + RunE: runAdminPrincipalDelete, + SilenceUsage: true, +} + +var adminPrincipalAuditCmd = &cobra.Command{ + Use: "audit", + Short: "Flag roles an admin principal could assume", + Long: `Report roles in the super-admin account whose trust policy names the account, +its root ARN, or a wildcard. + +STS does not evaluate the caller's identity policy on AssumeRole, so any such +role is assumable by a scoped admin principal, which then inherits its +permissions. Roles trusting a service principal are unaffected. + +Exits non-zero if any role is flagged.`, + Args: cobra.NoArgs, + RunE: runAdminPrincipalAudit, + SilenceUsage: true, +} + +func init() { + adminCmd.AddCommand(adminPrincipalCmd) + adminPrincipalCmd.AddCommand(adminPrincipalCreateCmd) + adminPrincipalCmd.AddCommand(adminPrincipalListCmd) + adminPrincipalCmd.AddCommand(adminPrincipalRevokeCmd) + adminPrincipalCmd.AddCommand(adminPrincipalDeleteCmd) + adminPrincipalCmd.AddCommand(adminPrincipalAuditCmd) + + adminPrincipalCreateCmd.Flags().StringSlice("grant", nil, + "Admin methods this principal may call (default: all of "+ + strings.Join(gateway.AdminMethodNames(), ", ")+")") +} diff --git a/cmd/spinifex/cmd/admin_principal_iam_test.go b/cmd/spinifex/cmd/admin_principal_iam_test.go new file mode 100644 index 000000000..08fdc0334 --- /dev/null +++ b/cmd/spinifex/cmd/admin_principal_iam_test.go @@ -0,0 +1,143 @@ +package cmd_test + +import ( + "errors" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/mulgadc/spinifex/cmd/spinifex/cmd" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const superAdminAccountID = "000000000001" + +// principalIAMStub embeds the IAMService interface so only the calls the +// principal commands make are implemented. Anything else nil-panics, which is +// the point: an unexpected call is a change in behaviour, not a passing test. +type principalIAMStub struct { + handlers_iam.IAMService + + inlinePolicies []string + inlineDocument string + attachedPolicies []string + accessKeys []string + + deletedPolicies []string + deletedKeys []string + listKeysErr error +} + +func (s *principalIAMStub) ListUserPolicies(_ string, _ *iam.ListUserPoliciesInput) (*iam.ListUserPoliciesOutput, error) { + return &iam.ListUserPoliciesOutput{PolicyNames: aws.StringSlice(s.inlinePolicies)}, nil +} + +func (s *principalIAMStub) DeleteUserPolicy(_ string, input *iam.DeleteUserPolicyInput) (*iam.DeleteUserPolicyOutput, error) { + s.deletedPolicies = append(s.deletedPolicies, aws.StringValue(input.PolicyName)) + return &iam.DeleteUserPolicyOutput{}, nil +} + +func (s *principalIAMStub) GetUserPolicy(_ string, input *iam.GetUserPolicyInput) (*iam.GetUserPolicyOutput, error) { + if s.inlineDocument == "" { + return nil, errors.New("NoSuchEntity") + } + return &iam.GetUserPolicyOutput{ + PolicyName: input.PolicyName, + PolicyDocument: aws.String(s.inlineDocument), + }, nil +} + +func (s *principalIAMStub) ListAttachedUserPolicies(_ string, _ *iam.ListAttachedUserPoliciesInput) (*iam.ListAttachedUserPoliciesOutput, error) { + out := &iam.ListAttachedUserPoliciesOutput{} + for _, name := range s.attachedPolicies { + out.AttachedPolicies = append(out.AttachedPolicies, + &iam.AttachedPolicy{PolicyName: aws.String(name)}) + } + return out, nil +} + +func (s *principalIAMStub) ListAccessKeys(_ string, _ *iam.ListAccessKeysInput) (*iam.ListAccessKeysOutput, error) { + if s.listKeysErr != nil { + return nil, s.listKeysErr + } + out := &iam.ListAccessKeysOutput{} + for _, id := range s.accessKeys { + out.AccessKeyMetadata = append(out.AccessKeyMetadata, + &iam.AccessKeyMetadata{AccessKeyId: aws.String(id)}) + } + return out, nil +} + +func (s *principalIAMStub) DeleteAccessKey(_ string, input *iam.DeleteAccessKeyInput) (*iam.DeleteAccessKeyOutput, error) { + s.deletedKeys = append(s.deletedKeys, aws.StringValue(input.AccessKeyId)) + return &iam.DeleteAccessKeyOutput{}, nil +} + +// Re-creating a principal with fewer grants has to narrow it. A policy left +// from an earlier name would keep authorising what the operator just removed. +func TestCreateDropsAnInlinePolicyItDidNotWrite(t *testing.T) { + svc := &principalIAMStub{inlinePolicies: []string{ + "operator-admin-methods", cmd.AdminPrincipalPolicyName, "signup-createaccount", + }} + + dropped, err := cmd.DropStalePrincipalPolicies(svc, superAdminAccountID, "operator") + require.NoError(t, err) + + assert.Equal(t, []string{"operator-admin-methods", "signup-createaccount"}, dropped) + assert.NotContains(t, svc.deletedPolicies, cmd.AdminPrincipalPolicyName, + "the policy the command owns must survive its own rewrite") +} + +// Every key goes, not just the first: two live keys means two holders, and a +// rotation that leaves one behind has revoked nothing. +func TestRevokeRemovesEveryAccessKey(t *testing.T) { + svc := &principalIAMStub{accessKeys: []string{"AKIAONE", "AKIATWO"}} + + revoked, err := cmd.RevokePrincipalKeys(svc, superAdminAccountID, "operator") + require.NoError(t, err) + + assert.Equal(t, []string{"AKIAONE", "AKIATWO"}, revoked) + assert.Equal(t, []string{"AKIAONE", "AKIATWO"}, svc.deletedKeys) +} + +// A listing failure must stop the rotation. Minting a second key while an +// unknown number are live is the opposite of what create promises. +func TestRevokeFailsRatherThanGuessing(t *testing.T) { + svc := &principalIAMStub{listKeysErr: errors.New("kv unavailable")} + + _, err := cmd.RevokePrincipalKeys(svc, superAdminAccountID, "operator") + + require.Error(t, err) + assert.Empty(t, svc.deletedKeys) +} + +// The listing is how an operator answers "who can delete an account", so it +// has to report the scoped grants by method name. +func TestDescribePrincipalReportsScopedGrants(t *testing.T) { + svc := &principalIAMStub{ + inlineDocument: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow",` + + `"Action":["spinifex:ListAccounts","spinifex:CreateAccount"],"Resource":"*"}]}`, + accessKeys: []string{"AKIAONE"}, + } + + row, err := cmd.DescribePrincipal(svc, superAdminAccountID, "signup") + require.NoError(t, err) + + assert.Equal(t, "signup", row.UserName) + assert.Equal(t, []string{"CreateAccount", "ListAccounts"}, row.Grants) + assert.Equal(t, 1, row.Keys) +} + +// An attached wildcard policy is reported rather than expanded: an unscoped +// principal is exactly what the listing exists to surface. +func TestDescribePrincipalMarksAnAttachedPolicy(t *testing.T) { + svc := &principalIAMStub{attachedPolicies: []string{handlers_iam.AdminPolicyName}} + + row, err := cmd.DescribePrincipal(svc, superAdminAccountID, handlers_iam.AdminUserName) + require.NoError(t, err) + + assert.Equal(t, []string{handlers_iam.AdminPolicyName + " (attached)"}, row.Grants) + assert.Zero(t, row.Keys) +} diff --git a/cmd/spinifex/cmd/admin_principal_test.go b/cmd/spinifex/cmd/admin_principal_test.go new file mode 100644 index 000000000..83261d5a1 --- /dev/null +++ b/cmd/spinifex/cmd/admin_principal_test.go @@ -0,0 +1,193 @@ +package cmd_test + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/mulgadc/spinifex/cmd/spinifex/cmd" + "github.com/mulgadc/spinifex/spinifex/gateway" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// policyActions returns the actions an inline policy document allows. +func policyActions(t *testing.T, document string) []string { + t.Helper() + + var doc struct { + Statement []struct { + Effect string `json:"Effect"` + Action handlers_iam.StringOrArr `json:"Action"` + } `json:"Statement"` + } + require.NoError(t, json.Unmarshal([]byte(document), &doc)) + + var actions []string + for _, statement := range doc.Statement { + if statement.Effect != "Allow" { + continue + } + actions = append(actions, statement.Action...) + } + return actions +} + +// A principal minted without --grant is the operator credential, so it has to +// cover the surface the gateway actually serves rather than a list that can +// fall behind it. +func TestPrincipalGrantsDefaultToEveryAdminMethod(t *testing.T) { + grants, err := cmd.ResolvePrincipalGrants(nil) + require.NoError(t, err) + + var want []string + for _, method := range gateway.AdminMethodNames() { + want = append(want, "spinifex:"+method) + } + assert.ElementsMatch(t, want, grants) +} + +// Scoping is the whole point of the flag: a signup credential holds exactly +// CreateAccount, so a leaked key cannot remove a tenant. +func TestPrincipalGrantsScopeToTheRequestedMethods(t *testing.T) { + grants, err := cmd.ResolvePrincipalGrants([]string{"CreateAccount"}) + require.NoError(t, err) + + assert.Equal(t, []string{"spinifex:CreateAccount"}, grants) +} + +// Operators write the method either way, and a duplicate is a typo rather than +// a reason to refuse. +func TestPrincipalGrantsAcceptQualifiedAndRepeatedNames(t *testing.T) { + grants, err := cmd.ResolvePrincipalGrants([]string{ + "spinifex:ListAccounts", "listaccounts", " DeleteAccount ", + }) + require.NoError(t, err) + + assert.Equal(t, []string{"spinifex:DeleteAccount", "spinifex:ListAccounts"}, grants) +} + +// A misspelt method must fail rather than mint a key that silently authorises +// nothing — the caller would only find out at the first AccessDenied. +func TestPrincipalGrantsRejectAnUnknownMethod(t *testing.T) { + _, err := cmd.ResolvePrincipalGrants([]string{"DeleteEverything"}) + + require.Error(t, err) + assert.Contains(t, err.Error(), "DeleteEverything") + assert.Contains(t, err.Error(), "CreateAccount") +} + +// A wildcard here would authorise whatever is added to the surface next with a +// key minted before it existed. +func TestPrincipalPolicyNamesEveryMethod(t *testing.T) { + grants, err := cmd.ResolvePrincipalGrants(nil) + require.NoError(t, err) + + actions := policyActions(t, cmd.PrincipalPolicyDocument(grants)) + + assert.ElementsMatch(t, grants, actions) + for _, action := range actions { + assert.NotContains(t, action, "*", "an admin grant must name its method") + } +} + +// The listing reads back what the create path wrote, so a principal's grants +// can be audited without decoding the document by hand. +func TestPrincipalPolicyActionsRoundTrip(t *testing.T) { + grants, err := cmd.ResolvePrincipalGrants([]string{"CreateAccount", "ListAccounts"}) + require.NoError(t, err) + + assert.Equal(t, []string{"CreateAccount", "ListAccounts"}, + cmd.PrincipalPolicyActions(cmd.PrincipalPolicyDocument(grants))) +} + +// A document nobody can parse is reported as no grants rather than as a guess. +func TestPrincipalPolicyActionsIgnoreAnUndecodableDocument(t *testing.T) { + assert.Empty(t, cmd.PrincipalPolicyActions("{not json")) +} + +// Rotating the super-admin account's own administrator would revoke the +// bootstrap credential the operator is holding. +func TestPrincipalNameRefusesTheAccountAdministrator(t *testing.T) { + err := cmd.ValidatePrincipalName(handlers_iam.AdminUserName) + + require.Error(t, err) + assert.Contains(t, err.Error(), handlers_iam.AdminUserName) +} + +func TestPrincipalNameRejectsUnusableNames(t *testing.T) { + assert.Error(t, cmd.ValidatePrincipalName("")) + assert.Error(t, cmd.ValidatePrincipalName("has space")) + assert.Error(t, cmd.ValidatePrincipalName(strings.Repeat("a", 65))) + + assert.NoError(t, cmd.ValidatePrincipalName("signup")) + assert.NoError(t, cmd.ValidatePrincipalName("ops-team_1@example.com")) +} + +// The listing prints one line per principal, including a revoked credential +// that holds no key — an operator has to see it to know it is still there. +func TestPrintPrincipalTableRendersEveryPrincipal(t *testing.T) { + output := captureOutput(t, func() { + cmd.PrintPrincipalTable("000000000001", []cmd.PrincipalRowSpec{ + {UserName: "admin", Grants: []string{"AdministratorAccess (attached)"}, Keys: 1}, + {UserName: "signup", Grants: []string{"CreateAccount"}, Keys: 1}, + {UserName: "retired", Keys: 0}, + }) + }) + + assert.Contains(t, output, "000000000001") + assert.Contains(t, output, "AdministratorAccess (attached)") + assert.Contains(t, output, "signup") + assert.Contains(t, output, "retired") + assert.Contains(t, output, "3 principal(s)") +} + +// A principal with nothing granted must read as a dash. A blank cell reads as +// a rendering fault rather than as an answer. +func TestPrintPrincipalTableMarksAnEmptyGrant(t *testing.T) { + output := captureOutput(t, func() { + cmd.PrintPrincipalTable("000000000001", []cmd.PrincipalRowSpec{{UserName: "retired"}}) + }) + + assert.Contains(t, output, "-") +} + +// The local and remote listings print through one function, so a tenant that is +// TERMINATING reads the same either way — which is how a stuck teardown is +// noticed. +func TestAccountSummariesCarryStatus(t *testing.T) { + summaries := cmd.AccountSummaries([]*handlers_iam.Account{ + {AccountID: "000000000042", AccountName: "t@example.com", Status: handlers_iam.AccountStatusActive}, + nil, + {AccountID: "000000000043", AccountName: "u@example.com", Status: handlers_iam.AccountStatusTerminating}, + }) + + require.Len(t, summaries, 2) + assert.Equal(t, handlers_iam.AccountStatusTerminating, summaries[1].Status) +} + +func TestPrintAccountTableRendersEveryAccount(t *testing.T) { + output := captureOutput(t, func() { + cmd.PrintAccountTable([]gateway.AccountSummary{ + { + AccountID: "000000000042", + AccountName: "tenant@example.com", + Status: handlers_iam.AccountStatusTerminating, + CreatedAt: "2026-08-16T07:00:00Z", + }, + }) + }) + + assert.Contains(t, output, "000000000042") + assert.Contains(t, output, "tenant@example.com") + assert.Contains(t, output, handlers_iam.AccountStatusTerminating) + assert.Contains(t, output, "2026-08-16 07:00") +} + +// An empty listing must say so. A bare header reads as truncated output. +func TestPrintAccountTableSaysWhenThereAreNone(t *testing.T) { + output := captureOutput(t, func() { cmd.PrintAccountTable(nil) }) + + assert.Contains(t, output, "No accounts found") +} diff --git a/cmd/spinifex/cmd/admin_remote.go b/cmd/spinifex/cmd/admin_remote.go new file mode 100644 index 000000000..ebfa4755e --- /dev/null +++ b/cmd/spinifex/cmd/admin_remote.go @@ -0,0 +1,299 @@ +package cmd + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/mulgadc/spinifex/internal/gwsign" + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/gateway" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +// adminRequestTimeout bounds a remote /admin call. Provisioning waits on the +// default VPC, so it is comfortably longer than the gateway's own wait. +const adminRequestTimeout = 90 * time.Second + +// adminMaxResponseBytes caps the response read. Most answers are a few hundred +// bytes; a teardown inventory is one line per resource, so the cap is sized for +// a large account rather than for the common case. +const adminMaxResponseBytes = 1 << 20 + +// retryableAdminErrors are the only codes worth retrying, and only with the +// same client token. Suggesting a retry for the rest invites a caller to send +// a fresh token, which is what produces a duplicate account. +var retryableAdminErrors = map[string]bool{ + awserrors.ErrorOperationInProgress: true, + awserrors.ErrorServiceUnavailable: true, + awserrors.ErrorInternalError: true, +} + +func init() { + accountCreateCmd.Flags().Bool("remote", false, "Create the account over POST /admin/CreateAccount instead of connecting to NATS") + accountCreateCmd.Flags().String("endpoint", "", "Gateway endpoint for --remote (default: this node's AWS gateway)") + accountCreateCmd.Flags().String("region", "", "SigV4 region for --remote (default: this node's region)") + accountCreateCmd.Flags().String("ca-bundle", "", "CA certificate for --remote (default: this node's CA)") + accountCreateCmd.Flags().String("client-token", "", "Idempotency token for --remote (default: generated; reuse it to retry)") + accountCreateCmd.Flags().String("source", "spx-cli", "Provenance tag recorded in the gateway log for --remote") + + accountListCmd.Flags().Bool("remote", false, "List over POST /admin/ListAccounts instead of connecting to NATS") + accountListCmd.Flags().String("endpoint", "", "Gateway endpoint for --remote (default: this node's AWS gateway)") + accountListCmd.Flags().String("region", "", "SigV4 region for --remote (default: this node's region)") + accountListCmd.Flags().String("ca-bundle", "", "CA certificate for --remote (default: this node's CA)") +} + +// runAccountCreateRemote drives POST /admin/CreateAccount, exercising the same +// path a self-service signup form uses. Credentials come from the standard AWS +// chain (env vars or AWS_PROFILE), so this never reads the cluster master key. +func runAccountCreateRemote(cmd *cobra.Command, name string) { + ctx, cancel := context.WithTimeout(context.Background(), adminRequestTimeout) + defer cancel() + + clientToken, _ := cmd.Flags().GetString("client-token") + source, _ := cmd.Flags().GetString("source") + + target, err := resolveAdminTarget(cmd) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + if clientToken == "" { + clientToken = newClientToken() + } + + out, err := createAccountRemote(ctx, target, + gateway.CreateAccountRequest{Name: name, ClientToken: clientToken, Source: source}) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + var adminErr *adminError + if errors.As(err, &adminErr) && retryableAdminErrors[adminErr.Code] { + fmt.Fprintf(os.Stderr, "Retry with --client-token %s to resume; a new token would create a second account.\n", clientToken) + } + os.Exit(1) + } + + fmt.Println("\nAccount created successfully!") + fmt.Printf(" Account ID: %s\n", out.AccountID) + fmt.Printf(" Account Name: %s\n", out.AccountName) + fmt.Printf(" Admin User: %s\n", out.AdminUser) + fmt.Printf(" Access Key ID: %s\n", out.AccessKeyID) + fmt.Printf(" Secret Access Key: %s\n", out.SecretAccessKey) + fmt.Printf(" Default VPC: %s\n", out.DefaultVpcID) + fmt.Printf(" Client Token: %s\n", clientToken) +} + +// adminTarget is where a remote /admin call goes and how it is signed. +type adminTarget struct { + endpoint string + region string + caBundle string +} + +// resolveAdminTarget fills the endpoint, region and CA from this node's config +// when the command runs on a cluster member. Off-cluster callers pass flags, +// which always win over what the local config says. +func resolveAdminTarget(cmd *cobra.Command) (adminTarget, error) { + endpoint, _ := cmd.Flags().GetString("endpoint") + region, _ := cmd.Flags().GetString("region") + caBundle, _ := cmd.Flags().GetString("ca-bundle") + + if endpoint == "" || region == "" || caBundle == "" { + if cfg, err := loadLocalConfig(); err == nil { + node := cfg.Nodes[cfg.Node] + if endpoint == "" { + endpoint = localGatewayEndpoint(node) + } + if region == "" { + region = node.Region + } + if caBundle == "" { + caBundle = filepath.Join(cfg.NodeBaseDir(), "config", "ca.pem") + } + } + } + if endpoint == "" || region == "" { + return adminTarget{}, errors.New("--endpoint and --region are required when no local node config is available") + } + return adminTarget{endpoint: endpoint, region: region, caBundle: caBundle}, nil +} + +// adminError is a structured error from the /admin surface. The code is +// carried separately from the message so the caller can decide whether a +// retry is safe without matching on text. +type adminError struct { + Code string + Message string + RequestID string + StatusCode int +} + +func (e *adminError) Error() string { + return fmt.Sprintf("%s: %s (HTTP %d, requestId %s)", e.Code, e.Message, e.StatusCode, e.RequestID) +} + +// createAccountRemote signs and sends POST /admin/CreateAccount. Credentials +// come from the standard AWS chain, so this never reads the cluster master +// key — it is the same request a self-service signup form makes. +func createAccountRemote(ctx context.Context, target adminTarget, req gateway.CreateAccountRequest) (*gateway.CreateAccountResponse, error) { + var out gateway.CreateAccountResponse + if err := callAdmin(ctx, target, "CreateAccount", req, &out); err != nil { + return nil, err + } + return &out, nil +} + +// callAdmin signs and sends one POST /admin/, decoding the success body +// into out. Credentials come from the standard AWS chain, so this never reads +// the cluster master key. +func callAdmin(ctx context.Context, target adminTarget, method string, req, out any) error { + body, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("marshal request: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, + strings.TrimRight(target.endpoint, "/")+"/admin/"+method, bytes.NewReader(body)) + if err != nil { + return err + } + httpReq.Header.Set("Content-Type", "application/json") + + signer, err := gwsign.NewIMDS(ctx, target.region) + if err != nil { + return fmt.Errorf("resolve AWS credentials: %w", err) + } + sum := sha256.Sum256(body) + if err := signer.Sign(httpReq, hex.EncodeToString(sum[:]), "spinifex", target.region); err != nil { + return fmt.Errorf("sign request: %w", err) + } + + client, err := adminHTTPClient(target.caBundle) + if err != nil { + return err + } + resp, err := client.Do(httpReq) + if err != nil { + return fmt.Errorf("call %s: %w", target.endpoint, err) + } + defer resp.Body.Close() + + payload, err := io.ReadAll(io.LimitReader(resp.Body, adminMaxResponseBytes)) + if err != nil { + return fmt.Errorf("gateway returned HTTP %d with an unreadable body: %w", resp.StatusCode, err) + } + if resp.StatusCode != http.StatusOK { + return decodeAdminError(resp.StatusCode, payload) + } + if err := json.Unmarshal(payload, out); err != nil { + return fmt.Errorf("decode response: %w", err) + } + return nil +} + +// decodeAdminError turns a non-200 into an adminError. A body that is not the +// JSON envelope is reported verbatim rather than as a decode failure, since +// then the response came from something other than the gateway. +func decodeAdminError(statusCode int, payload []byte) error { + var body struct { + Error struct { + Code string `json:"code"` + Message string `json:"message"` + } `json:"error"` + RequestID string `json:"requestId"` + } + if err := json.Unmarshal(payload, &body); err != nil || body.Error.Code == "" { + return fmt.Errorf("gateway returned HTTP %d: %s", statusCode, payload) + } + return &adminError{ + Code: body.Error.Code, + Message: body.Error.Message, + RequestID: body.RequestID, + StatusCode: statusCode, + } +} + +// newClientToken returns a fresh idempotency token in the character set the +// endpoint accepts. +func newClientToken() string { + var buf [24]byte + if _, err := rand.Read(buf[:]); err != nil { + // crypto/rand does not fail on any supported platform; if it does, a + // weaker token would silently break idempotency. + fmt.Fprintf(os.Stderr, "Error generating client token: %v\n", err) + os.Exit(1) + } + return hex.EncodeToString(buf[:]) +} + +// adminHTTPClient trusts caBundle in addition to the system roots so the same +// command works against a cluster's self-signed CA and against a public +// certificate. An unreadable bundle is an error, never a silent downgrade. +func adminHTTPClient(caBundle string) (*http.Client, error) { + transport := http.DefaultTransport.(*http.Transport).Clone() + if caBundle != "" { + pem, err := os.ReadFile(caBundle) + if err != nil { + return nil, fmt.Errorf("read CA bundle %s: %w", caBundle, err) + } + pool, err := x509.SystemCertPool() + if err != nil || pool == nil { + pool = x509.NewCertPool() + } + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("CA bundle %s contains no certificates", caBundle) + } + transport.TLSClientConfig = &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12} + } + return &http.Client{Transport: transport}, nil +} + +// loadLocalConfig reads this node's cluster config without connecting to NATS. +func loadLocalConfig() (*config.ClusterConfig, error) { + cfgPath := viper.GetString("config") + if cfgPath == "" { + cfgPath = DefaultConfigFile() + } + cfg, err := config.LoadConfig(cfgPath) + if err != nil { + return nil, fmt.Errorf("load config: %w", err) + } + if nodeConfig, ok := cfg.Nodes[cfg.Node]; ok && nodeConfig.BaseDir == "" { + if isProductionLayout() { + nodeConfig.BaseDir = DefaultDataDir() + } else { + nodeConfig.BaseDir = filepath.Dir(filepath.Dir(cfgPath)) + } + cfg.Nodes[cfg.Node] = nodeConfig + } + return cfg, nil +} + +// localGatewayEndpoint is this node's AWS gateway URL. A wildcard bind address +// is not dialable, so it resolves to localhost. +func localGatewayEndpoint(node config.Config) string { + host, port, err := net.SplitHostPort(node.AWSGW.Host) + if err != nil { + return "" + } + if host == "" || host == "0.0.0.0" || host == "::" { + host = "localhost" + } + return "https://" + net.JoinHostPort(host, port) +} diff --git a/cmd/spinifex/cmd/admin_signup_test.go b/cmd/spinifex/cmd/admin_remote_test.go similarity index 100% rename from cmd/spinifex/cmd/admin_signup_test.go rename to cmd/spinifex/cmd/admin_remote_test.go diff --git a/cmd/spinifex/cmd/admin_signup.go b/cmd/spinifex/cmd/admin_signup.go deleted file mode 100644 index e105e9fb3..000000000 --- a/cmd/spinifex/cmd/admin_signup.go +++ /dev/null @@ -1,469 +0,0 @@ -package cmd - -import ( - "bytes" - "context" - "crypto/rand" - "crypto/sha256" - "crypto/tls" - "crypto/x509" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "net" - "net/http" - "os" - "path/filepath" - "strings" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/iam" - "github.com/mulgadc/spinifex/internal/gwsign" - "github.com/mulgadc/spinifex/spinifex/admin" - "github.com/mulgadc/spinifex/spinifex/awserrors" - "github.com/mulgadc/spinifex/spinifex/config" - "github.com/mulgadc/spinifex/spinifex/gateway" - handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -// SignupPrincipalUserName is the IAM user in the super-admin account that the -// website's signup Worker signs with. It is deliberately separate from the -// operator's own admin key so revoking it costs nothing else. -const SignupPrincipalUserName = "signup" - -// signupPrincipalPolicyName is the inline policy attached to that user. -const signupPrincipalPolicyName = "signup-createaccount" - -// signupPrincipalPolicyDocument grants exactly one action. Widening it turns -// the credential into an account-existence oracle for arbitrary email -// addresses, so it is written out here rather than composed from a list. -const signupPrincipalPolicyDocument = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"spinifex:CreateAccount","Resource":"*"}]}` - -// adminRequestTimeout bounds a remote /admin call. Provisioning waits on the -// default VPC, so it is comfortably longer than the gateway's own wait. -const adminRequestTimeout = 90 * time.Second - -// adminMaxResponseBytes caps the response read. The endpoint answers in a few -// hundred bytes; the cap keeps a misdirected endpoint from streaming into memory. -const adminMaxResponseBytes = 64 << 10 - -// retryableAdminErrors are the only codes worth retrying, and only with the -// same client token. Suggesting a retry for the rest invites a caller to send -// a fresh token, which is what produces a duplicate account. -var retryableAdminErrors = map[string]bool{ - awserrors.ErrorOperationInProgress: true, - awserrors.ErrorServiceUnavailable: true, - awserrors.ErrorInternalError: true, -} - -var signupPrincipalCmd = &cobra.Command{ - Use: "signup-principal", - Short: "Manage the credential the public signup form uses", - Long: `Manage the IAM user in the super-admin account that mulgadc.com/signup signs -its POST /admin/CreateAccount requests with.`, -} - -var signupPrincipalCreateCmd = &cobra.Command{ - Use: "create", - Short: "Create the signup principal and print its access key once", - Long: `Create the "signup" IAM user in the super-admin account with an inline policy -allowing exactly spinifex:CreateAccount, then mint an access key. - -The secret is printed once and never recoverable. Store it as a Cloudflare -Worker secret; never commit it and never place it in a Spinifex config file. - -Re-running replaces the access key, which revokes the previous one.`, - Run: runSignupPrincipalCreate, -} - -var signupPrincipalAuditCmd = &cobra.Command{ - Use: "audit", - Short: "Flag roles the signup credential could assume", - Long: `Report roles in the super-admin account whose trust policy names the account, -its root ARN, or a wildcard. - -STS does not evaluate the caller's identity policy on AssumeRole, so any such -role is assumable by the signup credential, which then inherits its -permissions. Roles trusting a service principal are unaffected. - -Exits non-zero if any role is flagged.`, - RunE: runSignupPrincipalAudit, - SilenceUsage: true, -} - -func init() { - adminCmd.AddCommand(signupPrincipalCmd) - signupPrincipalCmd.AddCommand(signupPrincipalCreateCmd) - signupPrincipalCmd.AddCommand(signupPrincipalAuditCmd) - - accountCreateCmd.Flags().Bool("remote", false, "Create the account over POST /admin/CreateAccount instead of connecting to NATS") - accountCreateCmd.Flags().String("endpoint", "", "Gateway endpoint for --remote (default: this node's AWS gateway)") - accountCreateCmd.Flags().String("region", "", "SigV4 region for --remote (default: this node's region)") - accountCreateCmd.Flags().String("ca-bundle", "", "CA certificate for --remote (default: this node's CA)") - accountCreateCmd.Flags().String("client-token", "", "Idempotency token for --remote (default: generated; reuse it to retry)") - accountCreateCmd.Flags().String("source", "spx-cli", "Provenance tag recorded in the gateway log for --remote") -} - -// runSignupPrincipalCreate provisions the signup principal in the super-admin -// account. Every step is create-if-absent except the access key, which is -// replaced so exactly one key is ever live. -func runSignupPrincipalCreate(cmd *cobra.Command, args []string) { - svc, _, _, cleanup, err := initIAMServiceFromConfig() - if err != nil { - fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(1) - } - defer cleanup() - - accountID := admin.DefaultAccountID() - - if _, err := svc.CreateUser(accountID, &iam.CreateUserInput{ - UserName: aws.String(SignupPrincipalUserName), - }); err != nil && !strings.Contains(err.Error(), "EntityAlreadyExists") { - fmt.Fprintf(os.Stderr, "Error creating signup user: %v\n", err) - os.Exit(1) - } - - if _, err := svc.PutUserPolicy(accountID, &iam.PutUserPolicyInput{ - UserName: aws.String(SignupPrincipalUserName), - PolicyName: aws.String(signupPrincipalPolicyName), - PolicyDocument: aws.String(signupPrincipalPolicyDocument), - }); err != nil { - fmt.Fprintf(os.Stderr, "Error attaching signup policy: %v\n", err) - os.Exit(1) - } - - // Any key from a previous run is unrecoverable to its holder but still - // authenticates, so it is removed rather than left live alongside the new one. - listed, err := svc.ListAccessKeys(accountID, &iam.ListAccessKeysInput{ - UserName: aws.String(SignupPrincipalUserName), - }) - if err != nil { - fmt.Fprintf(os.Stderr, "Error listing signup access keys: %v\n", err) - os.Exit(1) - } - if listed != nil { - for _, meta := range listed.AccessKeyMetadata { - if meta == nil || meta.AccessKeyId == nil { - continue - } - if _, err := svc.DeleteAccessKey(accountID, &iam.DeleteAccessKeyInput{ - UserName: aws.String(SignupPrincipalUserName), - AccessKeyId: meta.AccessKeyId, - }); err != nil { - fmt.Fprintf(os.Stderr, "Error removing previous signup access key: %v\n", err) - os.Exit(1) - } - fmt.Printf("Revoked previous access key %s\n", aws.StringValue(meta.AccessKeyId)) - } - } - - akOut, err := svc.CreateAccessKey(accountID, &iam.CreateAccessKeyInput{ - UserName: aws.String(SignupPrincipalUserName), - }) - if err != nil || akOut == nil || akOut.AccessKey == nil { - fmt.Fprintf(os.Stderr, "Error creating signup access key: %v\n", err) - os.Exit(1) - } - - fmt.Println("\nSignup principal ready.") - fmt.Printf(" Account ID: %s\n", accountID) - fmt.Printf(" User: %s\n", SignupPrincipalUserName) - fmt.Printf(" Permitted action: spinifex:CreateAccount\n") - fmt.Printf(" Access Key ID: %s\n", aws.StringValue(akOut.AccessKey.AccessKeyId)) - fmt.Printf(" Secret Access Key: %s\n", aws.StringValue(akOut.AccessKey.SecretAccessKey)) - fmt.Println("\nThe secret is shown once. Store it as a Cloudflare Worker secret:") - fmt.Println(" wrangler secret put SPX_SIGNUP_ACCESS_KEY_ID") - fmt.Println(" wrangler secret put SPX_SIGNUP_SECRET_ACCESS_KEY") -} - -// runSignupPrincipalAudit reports roles in the super-admin account that any -// principal in the account can assume, which today includes the signup user. -func runSignupPrincipalAudit(_ *cobra.Command, _ []string) error { - svc, _, _, cleanup, err := initIAMServiceFromConfig() - if err != nil { - return err - } - defer cleanup() - - accountID := admin.DefaultAccountID() - roles, err := svc.ListRoles(accountID, &iam.ListRolesInput{}) - if err != nil { - return fmt.Errorf("list roles: %w", err) - } - - var flagged []string - for _, role := range roles.Roles { - if role == nil || role.AssumeRolePolicyDocument == nil { - continue - } - if trustsWholeAccount(aws.StringValue(role.AssumeRolePolicyDocument), accountID) { - flagged = append(flagged, aws.StringValue(role.RoleName)) - } - } - - if len(flagged) == 0 { - fmt.Printf("No roles in %s trust the account as a whole.\n", accountID) - return nil - } - - fmt.Fprintf(os.Stderr, "Roles in %s assumable by any principal in the account, including the signup credential:\n", accountID) - for _, name := range flagged { - fmt.Fprintf(os.Stderr, " %s\n", name) - } - return fmt.Errorf("%d role(s) trust the account as a whole — scope their trust policies to a specific principal", len(flagged)) -} - -// trustsWholeAccount reports whether an AssumeRolePolicyDocument admits every -// principal in accountID. A malformed document is reported as trusting, since -// a document nobody can parse is not a document anybody has verified. -func trustsWholeAccount(document, accountID string) bool { - doc, err := handlers_iam.ValidateTrustPolicyDocument(document) - if err != nil { - return true - } - for _, stmt := range doc.Statement { - if stmt.Effect != "Allow" { - continue - } - var principal struct { - AWS handlers_iam.StringOrArr `json:"AWS"` - } - if err := json.Unmarshal(stmt.Principal, &principal); err != nil { - continue - } - for _, entry := range principal.AWS { - switch entry { - case "*", accountID, "arn:aws:iam::" + accountID + ":root": - return true - } - } - } - return false -} - -// runAccountCreateRemote drives POST /admin/CreateAccount, exercising the same -// path the signup Worker uses. Credentials come from the standard AWS chain -// (env vars or AWS_PROFILE), so this never reads the cluster master key. -func runAccountCreateRemote(cmd *cobra.Command, name string) { - ctx, cancel := context.WithTimeout(context.Background(), adminRequestTimeout) - defer cancel() - - endpoint, _ := cmd.Flags().GetString("endpoint") - region, _ := cmd.Flags().GetString("region") - caBundle, _ := cmd.Flags().GetString("ca-bundle") - clientToken, _ := cmd.Flags().GetString("client-token") - source, _ := cmd.Flags().GetString("source") - - // The local node's config supplies defaults when the command runs on a - // cluster member; off-cluster callers pass the flags instead. - if endpoint == "" || region == "" || caBundle == "" { - if cfg, err := loadLocalConfig(); err == nil { - node := cfg.Nodes[cfg.Node] - if endpoint == "" { - endpoint = localGatewayEndpoint(node) - } - if region == "" { - region = node.Region - } - if caBundle == "" { - caBundle = filepath.Join(cfg.NodeBaseDir(), "config", "ca.pem") - } - } - } - if endpoint == "" || region == "" { - fmt.Fprintln(os.Stderr, "Error: --endpoint and --region are required when no local node config is available") - os.Exit(1) - } - if clientToken == "" { - clientToken = newClientToken() - } - - out, err := createAccountRemote(ctx, adminTarget{endpoint: endpoint, region: region, caBundle: caBundle}, - gateway.CreateAccountRequest{Name: name, ClientToken: clientToken, Source: source}) - if err != nil { - fmt.Fprintf(os.Stderr, "Error: %v\n", err) - var adminErr *adminError - if errors.As(err, &adminErr) && retryableAdminErrors[adminErr.Code] { - fmt.Fprintf(os.Stderr, "Retry with --client-token %s to resume; a new token would create a second account.\n", clientToken) - } - os.Exit(1) - } - - fmt.Println("\nAccount created successfully!") - fmt.Printf(" Account ID: %s\n", out.AccountID) - fmt.Printf(" Account Name: %s\n", out.AccountName) - fmt.Printf(" Admin User: %s\n", out.AdminUser) - fmt.Printf(" Access Key ID: %s\n", out.AccessKeyID) - fmt.Printf(" Secret Access Key: %s\n", out.SecretAccessKey) - fmt.Printf(" Default VPC: %s\n", out.DefaultVpcID) - fmt.Printf(" Client Token: %s\n", clientToken) -} - -// adminTarget is where a remote /admin call goes and how it is signed. -type adminTarget struct { - endpoint string - region string - caBundle string -} - -// adminError is a structured error from the /admin surface. The code is -// carried separately from the message so the caller can decide whether a -// retry is safe without matching on text. -type adminError struct { - Code string - Message string - RequestID string - StatusCode int -} - -func (e *adminError) Error() string { - return fmt.Sprintf("%s: %s (HTTP %d, requestId %s)", e.Code, e.Message, e.StatusCode, e.RequestID) -} - -// createAccountRemote signs and sends POST /admin/CreateAccount. Credentials -// come from the standard AWS chain, so this never reads the cluster master -// key — it is the same request the signup Worker makes. -func createAccountRemote(ctx context.Context, target adminTarget, req gateway.CreateAccountRequest) (*gateway.CreateAccountResponse, error) { - body, err := json.Marshal(req) - if err != nil { - return nil, fmt.Errorf("marshal request: %w", err) - } - - httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, - strings.TrimRight(target.endpoint, "/")+"/admin/CreateAccount", bytes.NewReader(body)) - if err != nil { - return nil, err - } - httpReq.Header.Set("Content-Type", "application/json") - - signer, err := gwsign.NewIMDS(ctx, target.region) - if err != nil { - return nil, fmt.Errorf("resolve AWS credentials: %w", err) - } - sum := sha256.Sum256(body) - if err := signer.Sign(httpReq, hex.EncodeToString(sum[:]), "spinifex", target.region); err != nil { - return nil, fmt.Errorf("sign request: %w", err) - } - - client, err := adminHTTPClient(target.caBundle) - if err != nil { - return nil, err - } - resp, err := client.Do(httpReq) - if err != nil { - return nil, fmt.Errorf("call %s: %w", target.endpoint, err) - } - defer resp.Body.Close() - - payload, err := io.ReadAll(io.LimitReader(resp.Body, adminMaxResponseBytes)) - if err != nil { - return nil, fmt.Errorf("gateway returned HTTP %d with an unreadable body: %w", resp.StatusCode, err) - } - - if resp.StatusCode != http.StatusOK { - return nil, decodeAdminError(resp.StatusCode, payload) - } - - var out gateway.CreateAccountResponse - if err := json.Unmarshal(payload, &out); err != nil { - return nil, fmt.Errorf("decode response: %w", err) - } - return &out, nil -} - -// decodeAdminError turns a non-200 into an adminError. A body that is not the -// JSON envelope is reported verbatim rather than as a decode failure, since -// then the response came from something other than the gateway. -func decodeAdminError(statusCode int, payload []byte) error { - var body struct { - Error struct { - Code string `json:"code"` - Message string `json:"message"` - } `json:"error"` - RequestID string `json:"requestId"` - } - if err := json.Unmarshal(payload, &body); err != nil || body.Error.Code == "" { - return fmt.Errorf("gateway returned HTTP %d: %s", statusCode, payload) - } - return &adminError{ - Code: body.Error.Code, - Message: body.Error.Message, - RequestID: body.RequestID, - StatusCode: statusCode, - } -} - -// newClientToken returns a fresh idempotency token in the character set the -// endpoint accepts. -func newClientToken() string { - var buf [24]byte - if _, err := rand.Read(buf[:]); err != nil { - // crypto/rand does not fail on any supported platform; if it does, a - // weaker token would silently break idempotency. - fmt.Fprintf(os.Stderr, "Error generating client token: %v\n", err) - os.Exit(1) - } - return hex.EncodeToString(buf[:]) -} - -// adminHTTPClient trusts caBundle in addition to the system roots so the same -// command works against a cluster's self-signed CA and against a public -// certificate. An unreadable bundle is an error, never a silent downgrade. -func adminHTTPClient(caBundle string) (*http.Client, error) { - transport := http.DefaultTransport.(*http.Transport).Clone() - if caBundle != "" { - pem, err := os.ReadFile(caBundle) - if err != nil { - return nil, fmt.Errorf("read CA bundle %s: %w", caBundle, err) - } - pool, err := x509.SystemCertPool() - if err != nil || pool == nil { - pool = x509.NewCertPool() - } - if !pool.AppendCertsFromPEM(pem) { - return nil, fmt.Errorf("CA bundle %s contains no certificates", caBundle) - } - transport.TLSClientConfig = &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12} - } - return &http.Client{Transport: transport}, nil -} - -// loadLocalConfig reads this node's cluster config without connecting to NATS. -func loadLocalConfig() (*config.ClusterConfig, error) { - cfgPath := viper.GetString("config") - if cfgPath == "" { - cfgPath = DefaultConfigFile() - } - cfg, err := config.LoadConfig(cfgPath) - if err != nil { - return nil, fmt.Errorf("load config: %w", err) - } - if nodeConfig, ok := cfg.Nodes[cfg.Node]; ok && nodeConfig.BaseDir == "" { - if isProductionLayout() { - nodeConfig.BaseDir = DefaultDataDir() - } else { - nodeConfig.BaseDir = filepath.Dir(filepath.Dir(cfgPath)) - } - cfg.Nodes[cfg.Node] = nodeConfig - } - return cfg, nil -} - -// localGatewayEndpoint is this node's AWS gateway URL. A wildcard bind address -// is not dialable, so it resolves to localhost. -func localGatewayEndpoint(node config.Config) string { - host, port, err := net.SplitHostPort(node.AWSGW.Host) - if err != nil { - return "" - } - if host == "" || host == "0.0.0.0" || host == "::" { - host = "localhost" - } - return "https://" + net.JoinHostPort(host, port) -} diff --git a/cmd/spinifex/cmd/export_test.go b/cmd/spinifex/cmd/export_test.go index 64f7e4787..490374a3a 100644 --- a/cmd/spinifex/cmd/export_test.go +++ b/cmd/spinifex/cmd/export_test.go @@ -6,6 +6,9 @@ package cmd import ( "errors" "strconv" + "time" + + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" ) // Test hooks for the external cmd_test package. @@ -67,3 +70,79 @@ func RetryableAdminError(err error) bool { // ConsoleRegion exposes consoleRegion for testing. var ConsoleRegion = consoleRegion + +// DeleteAccountRemote exposes deleteAccountRemote for testing. +var DeleteAccountRemote = deleteAccountRemote + +// DescribeAccountDeletionRemote exposes describeAccountDeletionRemote for testing. +var DescribeAccountDeletionRemote = describeAccountDeletionRemote + +// FollowAccountDeletion exposes followAccountDeletion for testing. +var FollowAccountDeletion = followAccountDeletion + +// SetAccountDeletePollInterval shortens the follow loop's poll and returns the +// function that restores it, so a test does not wait an operator's interval. +func SetAccountDeletePollInterval(d time.Duration) func() { + orig := accountDeletePollInterval + accountDeletePollInterval = d + return func() { accountDeletePollInterval = orig } +} + +// PrintTeardownPlan exposes printTeardownPlan for testing. +var PrintTeardownPlan = printTeardownPlan + +// PrintTeardownResult exposes printTeardownResult for testing. +var PrintTeardownResult = printTeardownResult + +// PromptAccountName exposes promptAccountName for testing. +var PromptAccountName = promptAccountName + +// ResolvePrincipalGrants exposes resolvePrincipalGrants for testing. +var ResolvePrincipalGrants = resolvePrincipalGrants + +// PrincipalPolicyDocument exposes principalPolicyDocument for testing. +var PrincipalPolicyDocument = principalPolicyDocument + +// PrincipalPolicyActions exposes principalPolicyActions for testing. +var PrincipalPolicyActions = principalPolicyActions + +// ValidatePrincipalName exposes validatePrincipalName for testing. +var ValidatePrincipalName = validatePrincipalName + +// AdminPrincipalPolicyName exposes the inline policy name for testing. +const AdminPrincipalPolicyName = adminPrincipalPolicyName + +// PrincipalRowSpec mirrors the unexported listing row so the external test +// package can build one. +type PrincipalRowSpec struct { + UserName string + Grants []string + Keys int +} + +// DropStalePrincipalPolicies exposes dropStalePrincipalPolicies for testing. +var DropStalePrincipalPolicies = dropStalePrincipalPolicies + +// RevokePrincipalKeys exposes revokePrincipalKeys for testing. +var RevokePrincipalKeys = revokePrincipalKeys + +// DescribePrincipal exposes describePrincipal for testing. +func DescribePrincipal(svc handlers_iam.IAMService, accountID, userName string) (PrincipalRowSpec, error) { + row, err := describePrincipal(svc, accountID, userName) + return PrincipalRowSpec(row), err +} + +// PrintPrincipalTable exposes printPrincipalTable for testing. +func PrintPrincipalTable(accountID string, specs []PrincipalRowSpec) { + rows := make([]principalRow, 0, len(specs)) + for _, spec := range specs { + rows = append(rows, principalRow(spec)) + } + printPrincipalTable(accountID, rows) +} + +// PrintAccountTable exposes printAccountTable for testing. +var PrintAccountTable = printAccountTable + +// AccountSummaries exposes accountSummaries for testing. +var AccountSummaries = accountSummaries diff --git a/cmd/spinifex/cmd/register_imported_ami_snapshot_test.go b/cmd/spinifex/cmd/register_imported_ami_snapshot_test.go index b1c5ec12d..cc7f60804 100644 --- a/cmd/spinifex/cmd/register_imported_ami_snapshot_test.go +++ b/cmd/spinifex/cmd/register_imported_ami_snapshot_test.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "testing" "github.com/mulgadc/spinifex/spinifex/ebsmetadata" @@ -28,7 +29,7 @@ func TestRegisterImportedAMISnapshot_WritesEC2ReadableMetadata(t *testing.T) { require.NoError(t, registerImportedAMISnapshot(store, bucket, ami, "ap-southeast-2a", true)) - cfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(store, bucket, "snap-ami-import01") + cfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(context.Background(), store, bucket, "snap-ami-import01") require.NoError(t, err) assert.Equal(t, "snap-ami-import01", cfg.SnapshotID) assert.Equal(t, "ami-import01", cfg.VolumeID, "the import path creates the volume under the AMI's own ID") @@ -49,7 +50,7 @@ func TestRegisterImportedAMISnapshot_UnencryptedVolume(t *testing.T) { require.NoError(t, registerImportedAMISnapshot(store, "predastore", ami, "ap-southeast-2a", false)) - cfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(store, "predastore", "snap-ami-plain01") + cfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(context.Background(), store, "predastore", "snap-ami-plain01") require.NoError(t, err) assert.False(t, cfg.Encrypted) } diff --git a/cmd/spinifex/cmd/service_exit_test.go b/cmd/spinifex/cmd/service_exit_test.go index 20714fea0..0303e4ae9 100644 --- a/cmd/spinifex/cmd/service_exit_test.go +++ b/cmd/spinifex/cmd/service_exit_test.go @@ -43,6 +43,18 @@ region = "us-east-1" az = "us-east-1a" ` +// regionlessClusterToml is a valid cluster whose node carries no region, the +// one thing the console needs to sign requests. +const regionlessClusterToml = ` +version = "1.0" +epoch = 1 +node = "node1" + +[nodes.node1] +node = "node1" +az = "us-east-1a" +` + // writeMalformedToml writes a syntactically invalid TOML file so // config.LoadConfig fails at viper.ReadInConfig rather than at validation. func writeMalformedToml(t *testing.T) string { @@ -254,6 +266,9 @@ func TestServiceStartCmdsReturnErrorOnMissingConfig(t *testing.T) { wantErr: "start awsgw service", }, { + // The config is pinned so the node resolves a region. Without it the + // command reads whatever config the host happens to have and fails at + // the region check instead, which is not what this case is about. name: "spinifex-ui start (start failure via missing tls cert)", cmd: spinifexUIStartCmd, setup: func(t *testing.T) { @@ -264,6 +279,16 @@ func TestServiceStartCmdsReturnErrorOnMissingConfig(t *testing.T) { }, wantErr: "start spinifex-ui service", }, + { + name: "spinifex-ui start (node has no region)", + cmd: spinifexUIStartCmd, + setup: func(t *testing.T) { + resetGlobalViper(t) + viper.Set("config", writeSpinifexToml(t, regionlessClusterToml)) + viper.Set("spinifex-ui-base-dir", t.TempDir()) + }, + wantErr: "has no region configured", + }, {name: "vpcd start (missing config)", cmd: vpcdStartCmd}, { name: "vpcd start (config file load failure)", diff --git a/cmd/spx-loadgen/main.go b/cmd/spx-loadgen/main.go new file mode 100644 index 000000000..5acce4d07 --- /dev/null +++ b/cmd/spx-loadgen/main.go @@ -0,0 +1,249 @@ +// spx-loadgen drives a measured request load at a Spinifex control plane and +// reports per-operation latency distributions as JSON and as a summary. +// +// It is the measuring half of the stress harness: scripts/stress in the mulga +// repo owns tenants, workloads and teardown, and spawns this to produce the +// numbers. Driving the AWS CLI instead costs over a second of interpreter +// start-up per call, which is larger than the latency being measured. +// +// Two pacing modes, reported separately because they answer different +// questions. Closed loop holds N callers in flight and slows down when the +// cluster does, which is what a fixed pool of console users looks like. Open +// loop issues a fixed rate regardless and times each request from when it was +// due, which is the only way to watch a queue form. +// +// Usage: +// +// spx-loadgen -endpoint https://api.spx3.com -region us-west-1 \ +// -tenants run/tenants.txt -closed 1,2,4,8 -open 10,20,40 \ +// -duration 30s -out run/loadgen.json +// +// Tenants are read from the harness's tenants.txt: " " +// per line, profile naming a shared-config AWS profile. -profile runs against +// one profile without a file. +package main + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "os/signal" + "strconv" + "strings" + "syscall" + "time" + + "github.com/mulgadc/spinifex/spinifex/loadgen" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "spx-loadgen: %v\n", err) + os.Exit(1) + } +} + +type options struct { + endpoint string + region string + caBundle string + tenants string + profile string + ops string + closed string + open string + duration time.Duration + timeout time.Duration + readP99 time.Duration + writeP99 time.Duration + expected string + out string + failOnSLO bool +} + +func run() error { + var opt options + flag.StringVar(&opt.endpoint, "endpoint", "", "control-plane endpoint, e.g. https://api.spx3.com") + flag.StringVar(&opt.region, "region", "us-west-1", "region") + flag.StringVar(&opt.caBundle, "ca-bundle", "", "CA bundle, only for an endpoint serving the cluster's own certificate") + flag.StringVar(&opt.tenants, "tenants", "", "tenants file: ' ' per line") + flag.StringVar(&opt.profile, "profile", "", "single AWS profile to run as, instead of -tenants") + flag.StringVar(&opt.ops, "ops", strings.Join(loadgen.DefaultOps, ","), + "operations to drive, comma separated (known: "+strings.Join(loadgen.OpNames(), ", ")+")") + flag.StringVar(&opt.closed, "closed", "", "closed-loop concurrency stages, e.g. 1,2,4,8") + flag.StringVar(&opt.open, "open", "", "open-loop rate stages in requests/second, e.g. 10,20,40") + flag.DurationVar(&opt.duration, "duration", 30*time.Second, "duration of each stage") + flag.DurationVar(&opt.timeout, "timeout", 30*time.Second, "per-request timeout") + flag.DurationVar(&opt.readP99, "slo-read-p99", 500*time.Millisecond, "p99 SLO for read operations") + flag.DurationVar(&opt.writeP99, "slo-write-p99", 2*time.Second, "p99 SLO for write operations") + flag.StringVar(&opt.expected, "expected-errors", "", + "error codes that are not failures, comma separated (quota rejections belong here)") + flag.StringVar(&opt.out, "out", "", "write the JSON report here (default stdout only)") + flag.BoolVar(&opt.failOnSLO, "fail-on-breach", false, "exit non-zero if any stage breaches the SLO") + flag.Parse() + + if opt.endpoint == "" { + return errors.New("-endpoint is required") + } + if opt.closed == "" && opt.open == "" { + return errors.New("nothing to run: give -closed, -open, or both") + } + + ops, err := loadgen.ResolveOps(strings.Split(opt.ops, ",")) + if err != nil { + return err + } + stages, err := buildStages(opt) + if err != nil { + return err + } + profiles, accounts, err := readTenants(opt) + if err != nil { + return err + } + + // The transport must hold at least as many connections as the widest stage + // or the run measures TLS handshakes rather than the control plane. + dial := loadgen.Dial{ + Endpoint: opt.endpoint, Region: opt.region, CABundle: opt.caBundle, + MaxIdleConnsPerHost: maxConcurrency(stages), Timeout: opt.timeout, + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + targets := make([]*loadgen.Target, 0, len(profiles)) + for index, profile := range profiles { + clients, err := loadgen.NewClients(dial, profile) + if err != nil { + return err + } + target := &loadgen.Target{Account: accounts[index], Profile: profile, Clients: clients} + if loadgen.NeedsVPC(ops) { + if err := loadgen.ResolveVPC(ctx, target); err != nil { + return err + } + } + if loadgen.NeedsVolume(ops) { + if err := loadgen.ResolveVolume(ctx, target); err != nil { + return err + } + } + targets = append(targets, target) + } + + slo := loadgen.SLO{ + ReadP99: opt.readP99, WriteP99: opt.writeP99, + ReadP99MS: float64(opt.readP99.Milliseconds()), WriteP99MS: float64(opt.writeP99.Milliseconds()), + ExpectedErrorCodes: splitNonEmpty(opt.expected), + } + + report, err := loadgen.Run(ctx, targets, ops, stages, slo) + if err != nil { + return err + } + report.Endpoint = opt.endpoint + report.Region = opt.region + + encoded, err := json.MarshalIndent(report, "", " ") + if err != nil { + return fmt.Errorf("encode report: %w", err) + } + if opt.out != "" { + if err := os.WriteFile(opt.out, append(encoded, '\n'), 0o644); err != nil { + return fmt.Errorf("write report: %w", err) + } + } + fmt.Print(loadgen.Summary(report)) + + if opt.failOnSLO && len(report.FirstBreach) > 0 { + return errors.New("SLO breached") + } + return nil +} + +// buildStages runs every closed-loop stage before any open-loop one. Mixing +// them would leave each mode measuring the queue the other left behind. +func buildStages(opt options) ([]loadgen.Stage, error) { + var stages []loadgen.Stage + for _, value := range splitNonEmpty(opt.closed) { + concurrency, err := strconv.Atoi(value) + if err != nil || concurrency < 1 { + return nil, fmt.Errorf("-closed: %q is not a positive integer", value) + } + stages = append(stages, loadgen.Stage{ + Mode: loadgen.ModeClosed, Concurrency: concurrency, Duration: opt.duration, + }) + } + for _, value := range splitNonEmpty(opt.open) { + rps, err := strconv.ParseFloat(value, 64) + if err != nil || rps <= 0 { + return nil, fmt.Errorf("-open: %q is not a positive rate", value) + } + stages = append(stages, loadgen.Stage{ + Mode: loadgen.ModeOpen, RPS: rps, Duration: opt.duration, + }) + } + return stages, nil +} + +// readTenants returns the profiles to run as and the account each belongs to. +func readTenants(opt options) (profiles, accounts []string, err error) { + if opt.profile != "" { + return []string{opt.profile}, []string{opt.profile}, nil + } + if opt.tenants == "" { + return nil, nil, errors.New("give -tenants or -profile") + } + + file, err := os.Open(opt.tenants) + if err != nil { + return nil, nil, fmt.Errorf("read tenants: %w", err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) < 3 || strings.HasPrefix(fields[0], "#") { + continue + } + accounts = append(accounts, fields[0]) + profiles = append(profiles, fields[2]) + } + if err := scanner.Err(); err != nil { + return nil, nil, fmt.Errorf("read tenants: %w", err) + } + if len(profiles) == 0 { + return nil, nil, fmt.Errorf("no tenants in %s", opt.tenants) + } + return profiles, accounts, nil +} + +func maxConcurrency(stages []loadgen.Stage) int { + widest := 16 + for _, stage := range stages { + if stage.Concurrency > widest { + widest = stage.Concurrency + } + if int(stage.RPS) > widest { + widest = int(stage.RPS) + } + } + return widest +} + +func splitNonEmpty(value string) []string { + var out []string + for part := range strings.SplitSeq(value, ",") { + part = strings.TrimSpace(part) + if part != "" { + out = append(out, part) + } + } + return out +} diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index 886dbcff2..8e6fe6287 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -94,7 +94,15 @@ Operational commands for inspecting cluster state. These fan out NATS requests t | `spx admin account create` | `--name` (required) | Connects to NATS → refuses if an account already holds the name (case- and whitespace-insensitive) → reserves the name in `spinifex-account-names` KV → CAS loop on `spinifex-account-counter:next_id` for sequential 12-digit ID → creates Account record in `spinifex-accounts` KV → creates default VPC → creates `admin` user, AdministratorAccess policy (Action:*, Resource:*) and one access key → prints credentials and writes a `spinifex-` AWS CLI profile | | `spx admin account create --remote` | `--name` (required), `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA), `--client-token` (default: generated), `--source` (default: `spx-cli`) | Same account, created through `POST /admin/CreateAccount` over SigV4 instead of NATS — the path the public signup form uses, so a shell can exercise it. Credentials come from the standard AWS chain (env vars or `AWS_PROFILE`), never from the cluster master key; the caller must be an IAM user in the super-admin account holding `spinifex:CreateAccount`. Reuse the printed `--client-token` to retry: a fresh token would create a second account. | | `spx admin account list` | — | Connects to NATS → IAMService.ListAccounts() → prints table with Account ID, Name, Status, Created | -| `spx admin signup-principal create` | — | Creates the `signup` IAM user in the super-admin account with an inline policy allowing exactly `spinifex:CreateAccount`, revokes any access key left from a previous run, and mints one new key. The secret is printed once — store it as a Cloudflare Worker secret, never in a config file. Revoke with `aws iam delete-access-key`. | +| `spx admin account list --remote` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Same listing read through `POST /admin/ListAccounts` over SigV4 instead of NATS, so an off-cluster operator or the load-test harness sees its tenants without SSH access to a node. Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:ListAccounts`. | +| `spx admin account delete ` | `--dry-run`, `--force`, `--yes` (requires `--name`), `--name` (account name, must match the stored record) | Prints the inventory it is about to remove → confirms the account name (prompted unless `--name` is given) → marks the account `TERMINATING`, which blocks its credentials cluster-wide → deletes its resources in dependency order (compute, attachments, storage, network, platform, identity, account), waiting for each stage to actually drain before starting the next → releases the name reservation and quota counter → deletes the account record. Anything that will not delete is reported as **stuck** with the reason, and the account stays `TERMINATING` so the residue keeps an owner. `000000000000` and `000000000001` are refused. `--dry-run` lists and changes nothing. The compute stage reaps ECS clusters, EKS nodegroups and clusters, and RDS instances before it reaps plain instances, because their backing VMs belong to the system account and a tenant-scoped instance listing cannot see them. The storage stage reaps the account's S3 buckets: uploads in flight are aborted, then the objects are removed a page at a time, then the bucket. Only buckets the account owns are ever listed, so the shared platform buckets are out of reach rather than filtered out. Bedrock model endpoints are reaped ahead of instances, because only their own delete releases the ENI and weights volume alongside the VM, and spot instance requests are cancelled first so a live request cannot launch a replacement for something already terminated. The network stage reaps load balancers, then target groups, then NAT gateways and egress-only internet gateways before the subnets, security groups and addresses they hold. The platform stage reaps ACM certificates. | +| `spx admin account delete --force` | as above | Escalates only for resources the ordinary API refuses: it clears a volume attachment in the control plane without the guest's cooperation (`ec2.ForceDetachVolume`), hard-destroys after a graceful attempt times out, and treats an already-missing resource as deleted. It never reorders the stages and never widens scope beyond the account. Use it once the guest is being destroyed — this is the escape from a volume stuck attached to an instance that will not terminate, where neither can otherwise be deleted. | +| `spx admin account delete --remote` | as above, plus `--endpoint`, `--region`, `--ca-bundle`, `--client-token` | Same teardown driven through `POST /admin/DeleteAccount` over SigV4 instead of NATS, then follows `DescribeAccountDeletion` to completion, printing each stage as it lands. Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:DeleteAccount` and `spinifex:DescribeAccountDeletion`. | +| `spx admin principal create ` | `--grant` (repeatable or comma-separated; default: every admin method) | Creates an IAM user in the super-admin account with an inline `spx-admin-methods` policy allowing the requested methods, revokes any access key left from a previous run, and mints one new key. Each method is named rather than granted as `spinifex:*`, so a later addition to the admin surface is not authorised by an existing key. The principal's grants are exactly that one policy: re-running with fewer `--grant` values narrows it, and any other inline policy on the user is removed, so a rename cannot leave a wider grant behind. The secret is printed once; store it in a secret manager or an AWS profile, never in a config file. `admin` is refused as a name — it is the account's own administrator and rotating its key would revoke the bootstrap credential. | +| `spx admin principal list` | — | Lists every principal in the super-admin account with its access-key count and what it may call. An attached policy is reported by name (`AdministratorAccess (attached)`) rather than expanded, which is how an unscoped principal is spotted. | +| `spx admin principal revoke ` | — | Deletes every access key the named principal holds, leaving the user and its policy in place. Immediate and cluster-wide; re-issue with `principal create`. | +| `spx admin principal delete ` | — | Removes the principal entirely: its access keys, its inline policies and the user. `admin` is refused, as it is for `create`. Use `revoke` instead when the principal should stay and only its credential should stop working. | +| `spx admin principal audit` | — | Reports roles in the super-admin account whose trust policy names the account, its root ARN, or a wildcard. STS does not evaluate the caller's identity policy on `AssumeRole`, so any such role is assumable by a scoped admin principal, which then inherits its permissions. Exits non-zero if any role is flagged. | ### Image Management @@ -146,17 +154,22 @@ Self-hosted models are served from the `ubuntu-26.04-vllm-serving-x86_64` system ## Private Admin API -Not an AWS API: a Spinifex-internal surface on the AWS gateway, JSON in and JSON out, one method per path segment at `POST /admin/`. It exists so the public signup form can create a tenant account without an operator shell. +Not an AWS API: a Spinifex-internal surface on the AWS gateway, JSON in and JSON out, one method per path segment at `POST /admin/`. It exists so a self-service signup form, a provisioning system or a load-test harness can create and remove tenant accounts without an operator shell on a node. Requests are SigV4-signed with `service=spinifex` and the gateway's configured region (`region` in `awsgw.toml`; production is `us-west-1`). The caller must be an IAM **user** in the super-admin account (`000000000001`) holding `spinifex:` — assumed-role sessions are refused outright. Every gate denies with the same `AccessDenied`, so a caller cannot learn which one it failed. Every response carries `X-Amzn-RequestId`, repeated as `requestId` in an error body. | Method | Request | Response | Notes | |--------|---------|----------|-------| | `POST /admin/CreateAccount` | `name` (email address, required), `clientToken` (32–128 chars of `[A-Za-z0-9_-]`, required), `source` (free-form provenance tag, max 64 chars) | `accountId`, `accountName`, `adminUser`, `accessKeyId`, `secretAccessKey`, `defaultVpcId` | Creates a tenant account with an `admin` user, an AdministratorAccess policy, one access key and a default VPC. The secret is returned **once**; replaying the same `clientToken` within 24 hours returns the identical response and is the only way to re-obtain it. | +| `POST /admin/DeleteAccount` | `accountId` (12 digits, required), `accountName` (must match the stored record), `clientToken` (32–128 chars), `force`, `dryRun` | `deletionId`, `accountId`, `state`; plus `inventory` when `dryRun` | Marks the account `TERMINATING` and tears down everything it owns in dependency order, then deletes the account record. Returns as soon as the teardown starts — it takes minutes, so progress comes from `DescribeAccountDeletion`. `dryRun` needs neither the name nor a token, answers inline and changes nothing. One job per account: a second call with a different token gets `OperationInProgress` while one is running. `000000000000` and `000000000001` return `AccessDenied`, the same answer an unauthorized caller gets. | +| `POST /admin/DescribeAccountDeletion` | `accountId` (12 digits, required) | `deletionId`, `state` (`RUNNING`, `COMPLETED`, `FAILED`), `startedAt`, `updatedAt`, `finishedAt`, `stages[]` with per-stage `deleted[]`, `stuck[]` and `elapsed`, `error` | The record outlives the account, so this stays answerable after the deletion completes. `NoSuchEntity` means no teardown was ever started for that account. A `RUNNING` job whose `updatedAt` has not advanced for five minutes is treated as abandoned — the gateway running it restarted — and a fresh `DeleteAccount` takes it over. | +| `POST /admin/ListAccounts` | — | `accounts[]` of `accountId`, `accountName`, `status`, `createdAt`; `count` | Enumerates every account. No credential material, and no paging: `[signup] max_accounts` bounds the listing. `status` is how a teardown stuck in `TERMINATING` is noticed. | Errors are `{"error":{"code":…,"message":…},"requestId":…}`. `OperationInProgress` (409), `ServiceUnavailable` (503) and `InternalError` (500) are retryable with backoff, **always reusing the same `clientToken`** — a fresh token after a failure creates a duplicate account. `AccountAlreadyExists` (409), `IdempotentParameterMismatch` (400), `LimitExceeded` (400, `[signup] max_accounts` reached), `InvalidParameterValue`, `MissingParameter`, `InvalidRequest`, `InvalidAction`, `MethodNotAllowed` (405) and `AccessDenied` (403) are not. -The endpoint is unreachable until an operator runs `spx admin signup-principal create`; there is no config toggle. Revocation is `aws iam delete-access-key`, which is immediate and cluster-wide. `[signup] max_accounts` in `awsgw.toml` caps how many accounts may exist; it defaults to 128 when the section is absent, and 0 means uncapped. +Each method is a separate grant, and `spx admin principal create` decides which a credential holds. `spx admin principal create operator` grants all four — the operator and load-test credential. `spx admin principal create signup --grant CreateAccount` grants one, which is what an internet-facing signup form should hold: that key cannot reach deletion, so a compromise of the form's host cannot remove a tenant. Separate principals are the point rather than an accident: revoking either costs nothing else. + +The surface is unreachable until an operator runs `spx admin principal create`; there is no config toggle and no default credential for it. The super-admin account's own `admin` user does hold these grants already, through the `AdministratorAccess` policy every account's administrator carries (`Action: "*"`), so its bootstrap key can call the surface without any further setup — but it is unscoped and unrotatable without locking out the operator, which is why automation should use a named principal. Revocation is `spx admin principal revoke ` (or `aws iam delete-access-key`), which is immediate and cluster-wide. `[signup] max_accounts` in `awsgw.toml` caps how many accounts may exist; it defaults to 128 when the section is absent, and 0 means uncapped. ## AWS-Compatible API diff --git a/docs/admin/account-management/README.md b/docs/admin/account-management/README.md new file mode 100644 index 000000000..dc33fc93f --- /dev/null +++ b/docs/admin/account-management/README.md @@ -0,0 +1,392 @@ +--- +title: "Account Management" +description: "Create, list and delete Spinifex tenant accounts from the CLI or over the private admin API." +category: "Admin" +tags: + - accounts + - iam + - admin api + - multi-tenancy +resources: + - title: "Spinifex Repository" + url: "https://github.com/mulgadc/spinifex" + - title: "Spinifex Admin CLI" + url: "/docs/spinifex-admin-cli" +--- + +# Account Management + +> Create, list and delete tenant accounts — from an operator shell, or from your own signup and provisioning systems. + +## Table of Contents + +- [Overview](#overview) +- [1. The accounts a cluster starts with](#1-the-accounts-a-cluster-starts-with) +- [2. Create an admin principal](#2-create-an-admin-principal) +- [3. Create an account](#3-create-an-account) +- [4. List accounts](#4-list-accounts) +- [5. Delete an account](#5-delete-an-account) +- [6. Calling the admin API from your own service](#6-calling-the-admin-api-from-your-own-service) +- [7. Operating the credential](#7-operating-the-credential) +- [Troubleshooting](#troubleshooting) + +--- + +## Overview + +A Spinifex cluster is multi-tenant. Each tenant is an **account**: a 12-digit ID that owns its own VPCs, instances, volumes, buckets, IAM users and keys. Resources in one account are invisible to every other account, including to the super-admin account. + +There are two ways to manage accounts, and they do the same work: + +| | Where it runs | What it needs | +|---|---|---| +| **`spx admin account …`** | On a cluster node | A shell on the node, and the cluster's own NATS credentials | +| **`POST /admin/`** | Anywhere that can reach the gateway | A SigV4 credential from an **admin principal** | + +The API exists so a self-service signup page, a provisioning system, a billing job or a load-test harness can create and remove tenants without an operator shell on a node. It is a Spinifex-internal surface, not an AWS API — the AWS CLI cannot call it. + +The same `spx` commands drive both: add `--remote` and they sign an HTTPS request to the gateway instead of connecting to NATS. That makes `spx` the reference client for the API as well as the operator tool. + +## 1. The accounts a cluster starts with + +`spx admin init` seeds two accounts before any tenant exists: + +| Account ID | Purpose | +|---|---| +| `000000000000` | The system account. Internal platform resources — system AMIs and the cluster's own objects — live here. | +| `000000000001` | The **super-admin** account. Its `admin` user is the credential printed at the end of `spx admin init`. | + +Both are refused by `DeleteAccount`. Tenants are numbered from `000000000002` upward. + +The private admin API is served **only** for principals in `000000000001`. A key from a tenant account is denied even if that tenant's own policy grants the action, and so is an assumed-role session from the super-admin account itself. That restriction is deliberate: creating and deleting tenants is a platform operation, not something a tenant can be delegated. + +Because of it, `spx admin principal create` always works in `000000000001`. There is no `--account-id`: granting these actions to a principal anywhere else would produce a credential the gateway refuses. + +### What the bootstrap admin can already do + +The `admin` user in `000000000001` is created with the `AdministratorAccess` policy, whose action is `*`. The gateway evaluates the caller's policy for `spinifex:`, and `*` matches — so **the bootstrap credential can already call every admin method**, with no extra setup. + +That is fine for a first look from a shell. It is a poor credential to give a service: + +- it is unscoped — the same key that reads a listing can delete every tenant; +- it is the account's administrator, so rotating it to contain a leak locks the operator out of everything else. + +For anything long-lived, mint a named principal instead. + +## 2. Create an admin principal + +An **admin principal** is an IAM user in the super-admin account with an inline policy naming the admin methods it may call. Create one per consumer. + +```bash +spx admin principal create operator +``` + +``` +Principal "operator" ready. + Account ID: 000000000001 + User: operator + Permitted actions: spinifex:CreateAccount, spinifex:DeleteAccount, spinifex:DescribeAccountDeletion, spinifex:ListAccounts + Access Key ID: AKIA... + Secret Access Key: ... + +The secret is shown once and is not recoverable. Store it in a secret manager +or an AWS profile; never commit it and never place it in a Spinifex config file. + +Requests must be SigV4-signed with service="spinifex" and this cluster's region. +``` + +Without `--grant` the principal may call every admin method. Scope it with `--grant` when the consumer needs less: + +```bash +# A public signup form: it creates accounts and can do nothing else. +spx admin principal create signup --grant CreateAccount + +# A billing or inventory job: read-only. +spx admin principal create billing --grant ListAccounts,DescribeAccountDeletion +``` + +Scoping matters most for anything internet-facing. A signup page's credential lives wherever that page runs; holding only `CreateAccount`, a leak of it costs capped junk accounts, not every tenant on the cluster. + +Each method is granted by name rather than as `spinifex:*`, so a key minted today is not silently authorised for a method added to the surface tomorrow. + +A principal's grants are exactly one inline policy. Re-running `create` with fewer `--grant` values narrows the principal, and any other inline policy on the user is removed — reducing a credential's reach never leaves the old grant behind. + +Store the secret as an AWS profile if you will use `spx --remote` or the AWS SDKs: + +```ini +# ~/.aws/credentials +[spinifex-operator] +aws_access_key_id = AKIA... +aws_secret_access_key = ... +``` + +Check what exists at any time: + +```bash +spx admin principal list +``` + +``` +Principals in the super-admin account (000000000001): + +USER KEYS GRANTS +------------------------------------------------------------------------------------------------ +admin 1 AdministratorAccess (attached) +operator 1 CreateAccount, DeleteAccount, DescribeAccountDeletion, ListAccounts +signup 1 CreateAccount + +3 principal(s) +``` + +An attached policy is reported by name rather than expanded — `AdministratorAccess (attached)` is how an unscoped principal shows up in the listing. + +## 3. Create an account + +On a node: + +```bash +spx admin account create --name customer@example.com +``` + +Over the API, from anywhere: + +```bash +AWS_PROFILE=spinifex-operator spx admin account create --remote \ + --name customer@example.com \ + --endpoint https://node1.example.com:9999 \ + --region us-west-1 \ + --ca-bundle ~/.spinifex/ca.pem +``` + +Either way the cluster creates, in one call: + +- the account record and its 12-digit ID; +- an `admin` IAM user in it, with `AdministratorAccess` and one access key; +- a default VPC. + +``` +Account created successfully! + Account ID: 000000000042 + Account Name: customer@example.com + Admin User: admin + Access Key ID: AKIA... + Secret Access Key: ... + Default VPC: vpc-... + Client Token: 3f9c... +``` + +**The secret is returned once.** Hand it to the customer, or store it, at the moment you receive it. + +### Client tokens + +`CreateAccount` is idempotent on `clientToken`. Replaying the same token within 24 hours returns the identical response — including the secret — rather than creating a second account. That is the only way to re-obtain a secret you lost to a dropped connection. + +The rule that follows: **retry with the same token, never a fresh one.** A fresh token after a timeout is how a duplicate account gets made. `spx` prints the token it used for exactly this reason, and only suggests a retry for the error codes that are safe to retry (`OperationInProgress`, `ServiceUnavailable`, `InternalError`). + +Account names are unique cluster-wide, case- and whitespace-insensitive. `[signup] max_accounts` in `awsgw.toml` caps how many accounts may exist — 128 by default, 0 for uncapped. + +## 4. List accounts + +```bash +spx admin account list # on a node +AWS_PROFILE=spinifex-operator spx admin account list --remote \ + --endpoint https://node1.example.com:9999 --region us-west-1 +``` + +``` +ACCOUNT ID NAME STATUS CREATED +000000000001 admin ACTIVE 2026-06-01 09:14 +000000000042 customer@example.com ACTIVE 2026-08-16 07:00 +``` + +`STATUS` is worth watching. An account left in `TERMINATING` is a teardown that did not finish — see [Troubleshooting](#troubleshooting). + +## 5. Delete an account + +Deleting an account removes everything it owns. Look before you leap: + +```bash +spx admin account delete 000000000042 --dry-run +``` + +`--dry-run` lists the inventory and changes nothing. It needs neither the account name nor a client token. + +Then delete for real: + +```bash +spx admin account delete 000000000042 +``` + +The command prints the inventory, asks for the account **name** as confirmation (type it, or pass `--name` with `--yes` for automation), then works through seven stages in dependency order: + +| Stage | What it removes | +|---|---| +| compute | Instances, and anything running on them | +| attachments | Volume attachments, ENI attachments, address associations | +| storage | Volumes, snapshots, buckets and their objects | +| network | Subnets, route tables, gateways, security groups, VPCs | +| platform | Higher-level services built on the above | +| identity | IAM users, keys, policies and roles | +| account | The name reservation, the quota counter and the account record | + +Each stage runs to empty before the next begins, so nothing is deleted while something still depends on it. The account is marked `TERMINATING` first, which blocks its credentials cluster-wide — a tenant cannot create new resources into a teardown that is already walking past them. + +### It returns before it finishes + +A large account takes minutes to tear down. The API does not hold the connection open for it: `DeleteAccount` starts the job, returns a `deletionId` immediately, and progress is read from `DescribeAccountDeletion`. + +```bash +AWS_PROFILE=spinifex-operator spx admin account delete 000000000042 --remote \ + --name customer@example.com --yes \ + --endpoint https://node1.example.com:9999 --region us-west-1 +``` + +`spx --remote` starts the job and then polls, printing each stage as it lands. Your own client can do the same, or start the job and check back later — the deletion record outlives the account, so `DescribeAccountDeletion` stays answerable after the account is gone. + +One job runs per account. A second `DeleteAccount` with a different token while one is running returns `OperationInProgress`; with the *same* token it returns the running job, which is what makes a retry safe. + +### When something will not delete + +Anything that refuses to go is reported as **stuck**, with the reason, and the account stays `TERMINATING` so the residue keeps an owner. Re-running the delete resumes from where it stopped. + +For the case where two resources hold each other — a volume attached to an instance that will not terminate, so neither can be deleted — use `--force`: + +```bash +spx admin account delete 000000000042 --force --name customer@example.com --yes +``` + +`--force` escalates only where the ordinary API refuses: it clears an attachment in the control plane without the guest's cooperation, hard-destroys after a graceful attempt times out, and treats an already-missing resource as deleted. It never reorders the stages and never touches anything outside the account. + +## 6. Calling the admin API from your own service + +The surface is `POST https://:9999/admin/`, JSON in and JSON out. Requests are SigV4-signed with: + +- **service** — `spinifex`. Not `ec2`, not `iam`. A key signed for another service is treated as probing and denied. +- **region** — the cluster's configured region (`region` in `awsgw.toml`). +- **credentials** — an admin principal's access key. + +| Method | Request | Response | +|---|---|---| +| `CreateAccount` | `name`, `clientToken` (32–128 chars of `[A-Za-z0-9_-]`), `source` | `accountId`, `accountName`, `adminUser`, `accessKeyId`, `secretAccessKey`, `defaultVpcId` | +| `DeleteAccount` | `accountId`, `accountName`, `clientToken`, `force`, `dryRun` | `deletionId`, `accountId`, `state`; plus `inventory` when `dryRun` | +| `DescribeAccountDeletion` | `accountId` | `deletionId`, `state`, `startedAt`, `updatedAt`, `finishedAt`, `stages[]`, `error` | +| `ListAccounts` | — | `accounts[]`, `count` | + +Errors are `{"error":{"code":…,"message":…},"requestId":…}`. Every response carries `X-Amzn-RequestId`, repeated as `requestId` in an error body — quote it when reporting a problem. + +Retry `OperationInProgress` (409), `ServiceUnavailable` (503) and `InternalError` (500) with backoff, **always reusing the same `clientToken`**. Do not retry `AccountAlreadyExists`, `IdempotentParameterMismatch`, `LimitExceeded`, `InvalidParameterValue`, `MissingParameter`, `InvalidRequest`, `InvalidAction`, `MethodNotAllowed` or `AccessDenied`. + +Any SigV4 library will sign these. With `botocore` in Python: + +```python +import json, requests +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.session import Session + +ENDPOINT = "https://node1.example.com:9999" +REGION = "us-west-1" + +def call_admin(method, body, profile="spinifex-operator"): + url = f"{ENDPOINT}/admin/{method}" + payload = json.dumps(body) + request = AWSRequest(method="POST", url=url, data=payload, + headers={"Content-Type": "application/json"}) + credentials = Session(profile=profile).get_credentials() + # The credential scope is "spinifex" — the gateway denies any other service. + SigV4Auth(credentials, "spinifex", REGION).add_auth(request) + + response = requests.post(url, data=payload, headers=dict(request.headers), + verify="/path/to/cluster-ca.pem", timeout=90) + response.raise_for_status() + return response.json() + +account = call_admin("CreateAccount", { + "name": "customer@example.com", + "clientToken": "signup-9f2c1e7b4a0d8356", # store this before you send it + "source": "signup-web", +}) +``` + +To confirm the route is served after a deployment, an unauthenticated request is enough: + +```bash +curl -sk -X POST https://node1.example.com:9999/admin/ListAccounts +``` + +`403 MissingAuthenticationToken` means the surface is up. `404` means it is not. + +## 7. Operating the credential + +**Rotate** by re-running create. It replaces the access key, which revokes the previous one: + +```bash +spx admin principal create operator +``` + +**Revoke** without removing the principal — the response to a leaked secret: + +```bash +spx admin principal revoke signup +``` + +**Remove** a principal that should not exist at all — its keys, its policies and the user: + +```bash +spx admin principal delete probe +``` + +Revocation and deletion take effect immediately and cluster-wide. No restart, no config change. + +**Audit** what an admin principal could escalate into: + +```bash +spx admin principal audit +``` + +STS does not evaluate the caller's identity policy on `AssumeRole`. A role in the super-admin account whose trust policy names the account, its root ARN, or a wildcard is therefore assumable by *any* principal in that account — including one scoped to `CreateAccount` — which then inherits the role's permissions. `audit` lists those roles and exits non-zero if it finds any. Scope their trust policies to a specific principal. + +Rules worth keeping: + +- One principal per consumer. Shared credentials cannot be revoked independently. +- Grant the least the consumer needs. A signup form needs `CreateAccount`. +- Never put an admin secret in a Spinifex config file, an AMI, or a repository. +- The `admin` user of `000000000001` is a break-glass credential, not a service credential. + +## Troubleshooting + +### `AccessDenied` on every admin call + +The surface returns the same `AccessDenied` for every failed gate, so it cannot be used to work out which one you failed. Check each: + +- Is the request signed with **`service=spinifex`**? A key signed for `ec2` or `iam` is denied here. +- Is the region the cluster's configured region? +- Does the credential belong to a user in **`000000000001`**? A tenant account's key is denied. Confirm with `spx admin principal list` on a node. +- Is it a **user's** long-lived key, not an assumed-role session? +- Does the principal hold `spinifex:` for the method you called? `spx admin principal list` shows the grants. + +The gateway's log names the failed gate, so `journalctl -u spinifex-awsgw` on the node that served the request is the fastest answer. + +### A principal's key stopped working + +`spx admin principal create ` revokes the previous key by design — someone re-running it to mint a key for another machine invalidates yours. Re-run it and distribute the new secret, or create a second principal so each holder has its own. + +### An account is stuck in `TERMINATING` + +The teardown could not remove something. Ask what it stopped on: + +```bash +spx admin account delete 000000000042 --dry-run +``` + +The inventory lists what remains. Re-run the delete to resume; if two resources hold each other, add `--force`. + +A teardown whose gateway restarted mid-job stalls with a `RUNNING` record. After five minutes without progress the job is treated as abandoned, and a fresh `DeleteAccount` takes it over. + +### `IdempotentParameterMismatch` + +The client token has been used before with different parameters — usually a token reused across two different account names. Tokens are per-request, not per-caller. Generate one at the point you build the request and store it with the pending record. + +### A duplicate account appeared + +A create that timed out was retried with a **new** client token. The first call had succeeded. Delete the extra account, and change the caller to persist its token before sending and reuse it on every retry. diff --git a/docs/admin/spinifex-admin-cli/README.md b/docs/admin/spinifex-admin-cli/README.md index b231a32ca..2ac576eff 100644 --- a/docs/admin/spinifex-admin-cli/README.md +++ b/docs/admin/spinifex-admin-cli/README.md @@ -76,6 +76,8 @@ ACCOUNT ID NAME STATUS CREATED 000000000002 myteam ACTIVE 2026-07-03 03:32 ``` +Deleting an account, and creating or removing accounts from your own signup or provisioning system over the private admin API, are covered in [Account Management](/docs/admin/account-management). + ## Node Management List nodes in the cluster: diff --git a/go.mod b/go.mod index e37d40fc3..f61c83a37 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/klauspost/cpuid/v2 v2.4.0 github.com/miekg/dns v1.1.72 github.com/mulgadc/bluebottle v1.16.1-0.20260812012137-683e88e465e3 - github.com/mulgadc/northstar v1.16.1-0.20260812012604-5d7a622ad059 + github.com/mulgadc/northstar v1.16.1-0.20260815211305-45682dc038c2 github.com/mulgadc/predastore v1.16.1-0.20260813022738-0b4392b43cae github.com/mulgadc/viperblock v1.16.1-0.20260812124023-7e0df2323c75 github.com/nats-io/nats-server/v2 v2.14.5 diff --git a/go.sum b/go.sum index 6f0afed73..8467eb1e3 100644 --- a/go.sum +++ b/go.sum @@ -1329,8 +1329,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/mulgadc/bluebottle v1.16.1-0.20260812012137-683e88e465e3 h1:EJ8JMQB9YQmtTDIrnPpbeti7U42MiWfwUhse6m2pJqM= github.com/mulgadc/bluebottle v1.16.1-0.20260812012137-683e88e465e3/go.mod h1:bhJitsX6VKkbmQ1dVh3vPUmoSeQsvtn5Jm4D8hUyq2s= -github.com/mulgadc/northstar v1.16.1-0.20260812012604-5d7a622ad059 h1:CK3Ed0Rg5dGvpSeMFTGpE1nf+390Rbf5hePOcnm1+kM= -github.com/mulgadc/northstar v1.16.1-0.20260812012604-5d7a622ad059/go.mod h1:vyfhxWlBlAnJfUeSEt+z/Z6KrfReC/uzZM3WTllorYM= +github.com/mulgadc/northstar v1.16.1-0.20260815211305-45682dc038c2 h1:Jhja5eM0ESlYMn7FrcqjhPujNdeUZdOxIXL5xJ4CM2g= +github.com/mulgadc/northstar v1.16.1-0.20260815211305-45682dc038c2/go.mod h1:6PoTvvzGwSmXq+VnkbXuLj43wvoJN9y/LURY4suk7cc= github.com/mulgadc/predastore v1.16.1-0.20260813022738-0b4392b43cae h1:itP5arN88Xv/xe4kI/5TWC/fkawnOeIR8MF1AL7aC48= github.com/mulgadc/predastore v1.16.1-0.20260813022738-0b4392b43cae/go.mod h1:S5vcni7R3VYi3nccelMagKbWdX58WtbKf0I1R/7MmNI= github.com/mulgadc/viperblock v1.16.1-0.20260812124023-7e0df2323c75 h1:eqB0VOuFPldBHuQ7ELYZHSRxq+3MXbMMi9D5bfVUwe8= diff --git a/scripts/build-system-image.sh b/scripts/build-system-image.sh index 65ef074f3..e2c882fa7 100755 --- a/scripts/build-system-image.sh +++ b/scripts/build-system-image.sh @@ -398,7 +398,7 @@ else echo " --file $OUTPUT_RAW \\" NAME_HINT="" if [[ -n "${AMI_NAME:-}" ]]; then - NAME_HINT=" \\\n --ami-name ${AMI_NAME}" + NAME_HINT=" \\\\\n --ami-name ${AMI_NAME}" fi if [[ -n "${SYSTEM_TAG:-}" ]]; then echo -e " --distro ${DISTRO} --version ${DISTRO_VERSION} --arch x86_64 --boot-mode ${BOOT_MODE}${NAME_HINT} \\" diff --git a/scripts/setup-ovn.sh b/scripts/setup-ovn.sh index 271548794..50c1f607d 100755 --- a/scripts/setup-ovn.sh +++ b/scripts/setup-ovn.sh @@ -572,6 +572,25 @@ EOF echo " OVN SB DB listening on tcp:127.0.0.1:6642" fi fi +else + # A compute node must not run a database. The ovn-central package starts a + # standalone one on install, and nothing here used to stop it, so every + # node past the quorum kept an unclustered NB/SB that no chassis reports + # into. It serves nothing, but it answers on the local socket, which is + # enough for any ovn-nbctl without an explicit --db to confirm against the + # wrong database — a `--wait=hv` there can never be acknowledged and burns + # its whole timeout instead. + if systemctl is-enabled ovn-central >/dev/null 2>&1 || + systemctl is-active ovn-central >/dev/null 2>&1; then + sudo systemctl stop ovn-central ovn-northd ovn-ovsdb-server-nb ovn-ovsdb-server-sb 2>/dev/null || true + sudo systemctl disable ovn-central ovn-northd ovn-ovsdb-server-nb ovn-ovsdb-server-sb 2>/dev/null || true + echo " ovn-central: stopped and disabled (compute node runs no database)" + fi + for db in "$OVN_DBDIR/ovnnb_db.db" "$OVN_DBDIR/ovnsb_db.db"; do + [ -f "$db" ] || continue + sudo mv "$db" "$db.standalone-disabled" + echo " moved aside $db (left by the package's standalone ovn-central)" + done fi # --- Step 3: Create and configure br-int --- diff --git a/spinifex/accountteardown/cluster.go b/spinifex/accountteardown/cluster.go new file mode 100644 index 000000000..c8a424655 --- /dev/null +++ b/spinifex/accountteardown/cluster.go @@ -0,0 +1,75 @@ +package accountteardown + +import ( + "context" + "fmt" + "log/slog" + + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + handlers_quota "github.com/mulgadc/spinifex/spinifex/handlers/quota" + "github.com/mulgadc/spinifex/spinifex/kvutil" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +// NewClusterEngine wires every reaper against a live cluster. +// +// The CLI and the admin API both build their engine here rather than each +// assembling its own list: a reaper registered on one surface and not the other +// would leave resources behind depending on how the teardown was started. +func NewClusterEngine( + ctx context.Context, + nc *nats.Conn, + expectedNodes int, + svc handlers_iam.IAMService, + buckets BucketStore, +) (*Engine, error) { + // Refused rather than skipped. A teardown missing its bucket reaper still + // reports every stage drained and still deletes the account record, leaving + // the tenant's objects behind with nothing able to attribute them to anyone. + if buckets == nil { + return nil, fmt.Errorf("teardown needs an object store to reap buckets with") + } + + js, err := jetstream.New(nc) + if err != nil { + return nil, fmt.Errorf("get JetStream context: %w", err) + } + + names, err := handlers_iam.NewAccountNameIndex(ctx, js) + if err != nil { + return nil, fmt.Errorf("open account name index: %w", err) + } + + // Absent when the cluster never enabled quotas, which is not a reason to + // refuse a teardown: there is simply no counter to remove. + usage, err := kvutil.GetOrCreateBucket(ctx, js, handlers_quota.KVBucketAccountUsage, 1) + if err != nil { + slog.WarnContext(ctx, "Teardown: quota usage bucket unavailable, its counter will not be removed", "err", err) + usage = nil + } + + // Managed services first: their teardown stops the tasks and releases the + // ENIs riding on instances the EC2 reaper is about to terminate, and their + // compute does not always belong to the tenant, so the instance reaper + // cannot see it. + reapers := BedrockReapers(nc) + reapers = append(reapers, ECSReapers(nc)...) + reapers = append(reapers, EKSReapers(nc)...) + reapers = append(reapers, RDSReapers(nc)...) + + // Before EC2, and the order matters within each stage because the sort that + // groups reapers by stage is stable. Load balancers and NAT gateways hold + // the subnets, security groups and addresses EC2's reapers delete, and a + // live spot request launches a replacement for anything already terminated. + reapers = append(reapers, ELBv2Reapers(nc)...) + reapers = append(reapers, EC2ExtraReapers(nc)...) + reapers = append(reapers, EC2Reapers(nc, expectedNodes)...) + + reapers = append(reapers, S3Reapers(buckets)...) + reapers = append(reapers, ACMReapers(nc)...) + reapers = append(reapers, IAMReapers(svc)...) + reapers = append(reapers, AccountReapers(svc, names, usage)...) + + return NewEngine(NewIAMAccounts(svc), reapers...), nil +} diff --git a/spinifex/accountteardown/cluster_test.go b/spinifex/accountteardown/cluster_test.go new file mode 100644 index 000000000..397ba15b7 --- /dev/null +++ b/spinifex/accountteardown/cluster_test.go @@ -0,0 +1,68 @@ +package accountteardown + +//test:in-package — the reaper types the wiring registers are unexported, and +// what this asserts is that both surfaces get the full set of them. + +import ( + "testing" + + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The CLI and the admin API share this wiring precisely so neither can be +// missing a reaper. A stage with none registered would silently skip whatever +// it owns, leaving resources behind that nothing else deletes. +func TestNewClusterEngineCoversEveryStage(t *testing.T) { + _, nc, _ := testutil.StartTestJetStream(t) + + masterKey, err := handlers_iam.GenerateMasterKey() + require.NoError(t, err) + svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey, 1) + require.NoError(t, err) + + engine, err := NewClusterEngine(t.Context(), nc, 1, svc, newFakeBuckets()) + require.NoError(t, err) + require.NotNil(t, engine.Accounts) + + // A teardown with no object store would report every stage drained and + // still delete the account record, leaving the tenant's objects behind. + _, err = NewClusterEngine(t.Context(), nc, 1, svc, nil) + require.Error(t, err) + + for _, stage := range Stages() { + if stage == StageAttachments { + // Attachments has no reaper of its own: it re-checks what stage 1 left. + continue + } + assert.NotEmpty(t, engine.reapersFor(stage), "stage %s has no reaper", stage.String()) + } +} + +// A teardown must refuse the accounts no credential may delete before it reads +// anything, and refuse a name confirmation that does not match the record. +func TestClusterEnginePrechecks(t *testing.T) { + _, nc, _ := testutil.StartTestJetStream(t) + + masterKey, err := handlers_iam.GenerateMasterKey() + require.NoError(t, err) + svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey, 1) + require.NoError(t, err) + + engine, err := NewClusterEngine(t.Context(), nc, 1, svc, newFakeBuckets()) + require.NoError(t, err) + + // Account ids are sequential and the first is the protected super admin. + _, err = svc.CreateAccount("super-admin@example.com") + require.NoError(t, err) + tenant, err := svc.CreateAccount("tenant@example.com") + require.NoError(t, err) + + assert.ErrorIs(t, engine.Precheck("000000000000", ""), ErrProtectedAccount) + assert.ErrorIs(t, engine.Precheck("000000000001", ""), ErrProtectedAccount) + assert.ErrorIs(t, engine.Precheck(tenant.AccountID, "wrong@example.com"), ErrAccountNameMismatch) + assert.NoError(t, engine.Precheck(tenant.AccountID, "tenant@example.com")) + assert.Error(t, engine.Precheck("000000009999", "")) +} diff --git a/spinifex/accountteardown/engine.go b/spinifex/accountteardown/engine.go new file mode 100644 index 000000000..ac5e8533a --- /dev/null +++ b/spinifex/accountteardown/engine.go @@ -0,0 +1,286 @@ +package accountteardown + +import ( + "context" + "errors" + "fmt" + "log/slog" + "sort" + "time" +) + +// AccountStatusTerminating mirrors handlers_iam.AccountStatusTerminating. +// Redeclared so this package does not depend on the IAM service to be tested. +const AccountStatusTerminating = "TERMINATING" + +// Engine tears an account down. Reapers are consulted in the order given +// within a stage, so a caller registering snapshots before volumes gets +// snapshots deleted first — which is required, because a volume with a live +// snapshot refuses to delete. +type Engine struct { + Accounts AccountStore + Reapers []Reaper + Timeouts Timeouts + Logger *slog.Logger + + // OnStage reports each finished stage as it happens, so a caller running a + // teardown in the background can publish progress. A teardown takes minutes + // and this is the only view into it while it runs. + OnStage func(ctx context.Context, stage StageResult) +} + +// NewEngine builds an engine with the default timeouts. +func NewEngine(accounts AccountStore, reapers ...Reaper) *Engine { + return &Engine{ + Accounts: accounts, + Reapers: reapers, + Timeouts: DefaultTimeouts(), + Logger: slog.Default(), + } +} + +func (e *Engine) logger() *slog.Logger { + if e.Logger != nil { + return e.Logger + } + return slog.Default() +} + +// Precheck applies the rules that decide whether a teardown may start at all, +// without listing anything. A caller that runs the teardown in the background +// needs these answered synchronously: a protected account, an unknown id or a +// mistyped name confirmation must fail the request, not the job. +func (e *Engine) Precheck(accountID, accountName string) error { + _, err := e.checkDeletable(accountID, accountName) + return err +} + +// Inventory reports what a teardown would remove, grouped by stage, without +// deleting anything or changing the account's status. +func (e *Engine) Inventory(ctx context.Context, accountID string) (*Result, error) { + account, err := e.checkDeletable(accountID, "") + if err != nil { + return nil, err + } + + result := &Result{ + AccountID: account.AccountID, + AccountName: account.AccountName, + DryRun: true, + } + for _, stage := range Stages() { + stageResult := StageResult{Stage: stage, Elapsed: "0s"} + for _, reaper := range e.reapersFor(stage) { + found, err := reaper.List(ctx, accountID) + if err != nil { + return nil, fmt.Errorf("inventory %s: %w", reaper.Kind(), err) + } + stageResult.Deleted = append(stageResult.Deleted, found...) + } + result.Stages = append(result.Stages, stageResult) + } + return result, nil +} + +// Teardown empties the account and then removes it. +// +// The account is marked TERMINATING before anything is deleted. Without that +// the customer can keep creating resources behind the cascade and it never +// converges; with it, a stage that has drained stays drained. +func (e *Engine) Teardown(ctx context.Context, req Request) (*Result, error) { + account, err := e.checkDeletable(req.AccountID, req.AccountName) + if err != nil { + return nil, err + } + if req.DryRun { + return e.Inventory(ctx, req.AccountID) + } + + if _, err := e.Accounts.SetAccountStatus(req.AccountID, AccountStatusTerminating); err != nil { + return nil, fmt.Errorf("mark account terminating: %w", err) + } + e.logger().InfoContext(ctx, "Account teardown started", + "accountID", account.AccountID, "accountName", account.AccountName, "force", req.Force) + + result := &Result{ + AccountID: account.AccountID, + AccountName: account.AccountName, + Forced: req.Force, + } + for _, stage := range Stages() { + stageResult, err := e.runStage(ctx, stage, req) + if err != nil { + return result, err + } + result.Stages = append(result.Stages, stageResult) + if e.OnStage != nil { + e.OnStage(ctx, stageResult) + } + } + + if stuck := result.StuckCount(); stuck > 0 { + // The account stays TERMINATING on purpose. Removing the record now + // would leave the residue with no owner, and an orphan nobody can + // attribute is worse than an account that visibly failed to empty. + e.logger().ErrorContext(ctx, "Account teardown incomplete — account record kept", + "accountID", account.AccountID, "stuck", stuck) + return result, fmt.Errorf("%d resources left: %w", stuck, ErrResourcesStuck) + } + + if err := e.Accounts.DeleteAccount(req.AccountID); err != nil { + return result, fmt.Errorf("delete account record: %w", err) + } + result.AccountDeleted = true + e.logger().InfoContext(ctx, "Account teardown complete", + "accountID", account.AccountID, "deleted", result.DeletedCount()) + return result, nil +} + +// runStage deletes everything in one stage and then waits for it to actually +// drain. Deletes here are mostly asynchronous — a terminate returns long +// before the guest is gone — so a stage is finished when a fresh listing comes +// back empty, not when the delete calls returned. +func (e *Engine) runStage(ctx context.Context, stage Stage, req Request) (StageResult, error) { + started := time.Now() + result := StageResult{Stage: stage} + reapers := e.reapersFor(stage) + if len(reapers) == 0 { + result.Elapsed = time.Since(started).String() + return result, nil + } + + deadline := started.Add(e.stageDrain()) + // reasons keeps the last failure per resource so a resource that is still + // listed at the deadline is reported with why it would not go, rather than + // only that it is still there. + reasons := map[string]string{} + + for { + remaining, err := e.sweep(ctx, reapers, req, &result, reasons) + if err != nil { + return result, err + } + if len(remaining) == 0 { + result.Elapsed = time.Since(started).String() + return result, nil + } + if !time.Now().Before(deadline) { + for _, resource := range remaining { + reason := reasons[resource.Kind+"/"+resource.ID] + if reason == "" { + reason = "still present after the stage drain timeout" + } + result.Stuck = append(result.Stuck, Stuck{Resource: resource, Reason: reason}) + } + result.Elapsed = time.Since(started).String() + e.logger().WarnContext(ctx, "Teardown stage did not drain", + "accountID", req.AccountID, "stage", stage.String(), "stuck", len(result.Stuck)) + return result, nil + } + + select { + case <-ctx.Done(): + return result, ctx.Err() + case <-time.After(e.drainPoll()): + } + } +} + +// sweep lists every reaper in the stage and issues a delete for whatever it +// finds, returning what was still listed at the start of this pass. +func (e *Engine) sweep( + ctx context.Context, + reapers []Reaper, + req Request, + result *StageResult, + reasons map[string]string, +) ([]Resource, error) { + var remaining []Resource + for _, reaper := range reapers { + found, err := reaper.List(ctx, req.AccountID) + if err != nil { + return nil, fmt.Errorf("list %s: %w", reaper.Kind(), err) + } + remaining = append(remaining, found...) + + for _, resource := range found { + if err := reaper.Delete(ctx, req.AccountID, resource, req.Force); err != nil { + reasons[resource.Kind+"/"+resource.ID] = err.Error() + e.logger().WarnContext(ctx, "Teardown delete failed", + "accountID", req.AccountID, "kind", resource.Kind, "id", resource.ID, "err", err) + continue + } + delete(reasons, resource.Kind+"/"+resource.ID) + if !containsResource(result.Deleted, resource) { + result.Deleted = append(result.Deleted, resource) + e.logger().InfoContext(ctx, "Teardown deleted resource", + "accountID", req.AccountID, "kind", resource.Kind, "id", resource.ID) + } + } + } + return remaining, nil +} + +// checkDeletable applies the two rules that no caller may skip: the protected +// accounts, and the name confirmation. An empty wantName skips the second, +// which is what makes Inventory usable as a read-only probe. +func (e *Engine) checkDeletable(accountID, wantName string) (*Account, error) { + if reason, protected := protectedAccountIDs[accountID]; protected { + return nil, fmt.Errorf("%s is the %s account: %w", accountID, reason, ErrProtectedAccount) + } + if e.Accounts == nil { + return nil, errors.New("teardown engine has no account store") + } + + account, err := e.Accounts.GetAccount(accountID) + if err != nil { + return nil, fmt.Errorf("read account %s: %w", accountID, err) + } + if wantName != "" && account.AccountName != wantName { + // Never echo the stored name: a caller who guessed the id would learn + // the tenant's email address by probing this. + return nil, fmt.Errorf("%q does not name account %s: %w", wantName, accountID, ErrAccountNameMismatch) + } + return account, nil +} + +// reapersFor returns the stage's reapers in registration order. +func (e *Engine) reapersFor(stage Stage) []Reaper { + var stageReapers []Reaper + for _, reaper := range e.Reapers { + if reaper.Stage() == stage { + stageReapers = append(stageReapers, reaper) + } + } + return stageReapers +} + +func (e *Engine) stageDrain() time.Duration { + if e.Timeouts.StageDrain > 0 { + return e.Timeouts.StageDrain + } + return DefaultTimeouts().StageDrain +} + +func (e *Engine) drainPoll() time.Duration { + if e.Timeouts.DrainPoll > 0 { + return e.Timeouts.DrainPoll + } + return DefaultTimeouts().DrainPoll +} + +func containsResource(resources []Resource, want Resource) bool { + for _, resource := range resources { + if resource.Kind == want.Kind && resource.ID == want.ID { + return true + } + } + return false +} + +// SortReapers orders reapers by stage while preserving the caller's order +// within each stage, which is where the snapshots-before-volumes dependency +// lives. +func SortReapers(reapers []Reaper) { + sort.SliceStable(reapers, func(i, j int) bool { return reapers[i].Stage() < reapers[j].Stage() }) +} diff --git a/spinifex/accountteardown/engine_test.go b/spinifex/accountteardown/engine_test.go new file mode 100644 index 000000000..ff23678c1 --- /dev/null +++ b/spinifex/accountteardown/engine_test.go @@ -0,0 +1,475 @@ +package accountteardown + +//test:in-package — the engine drives unexported reapers and the protected +// account list, and the stage ordering it guarantees is only observable from +// inside the package. + +import ( + "context" + "errors" + "log/slog" + "strings" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/admin" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" +) + +const ( + testAccountID = "000000000042" + testAccountName = "tenant@example.com" +) + +// fakeAccounts is an in-memory AccountStore that records what happened to it. +type fakeAccounts struct { + account *Account + statuses []string + deleted bool + getErr error + statusErr error + deleteErr error +} + +func newFakeAccounts() *fakeAccounts { + return &fakeAccounts{account: &Account{ + AccountID: testAccountID, + AccountName: testAccountName, + Status: "ACTIVE", + }} +} + +func (f *fakeAccounts) GetAccount(string) (*Account, error) { + if f.getErr != nil { + return nil, f.getErr + } + return f.account, nil +} + +func (f *fakeAccounts) SetAccountStatus(_, status string) (*Account, error) { + if f.statusErr != nil { + return nil, f.statusErr + } + f.statuses = append(f.statuses, status) + f.account.Status = status + return f.account, nil +} + +func (f *fakeAccounts) DeleteAccount(string) error { + if f.deleteErr != nil { + return f.deleteErr + } + f.deleted = true + return nil +} + +// fakeReaper holds a set of resources and removes them when deleted. order is +// shared between reapers so a test can assert the sequence across kinds. +type fakeReaper struct { + kind string + stage Stage + resources []Resource + order *[]string + + // blockedUntilForce refuses to delete anything unless force is set, which + // is how the attached-volume deadlock is modelled. + blockedUntilForce bool + + // listErr, when set, fails enumeration. + listErr error + + // deletesAttempted counts every Delete call, successful or not. + deletesAttempted int +} + +func (r *fakeReaper) Kind() string { return r.kind } +func (r *fakeReaper) Stage() Stage { return r.stage } + +func (r *fakeReaper) List(context.Context, string) ([]Resource, error) { + if r.listErr != nil { + return nil, r.listErr + } + return append([]Resource(nil), r.resources...), nil +} + +func (r *fakeReaper) Delete(_ context.Context, _ string, resource Resource, force bool) error { + r.deletesAttempted++ + if r.blockedUntilForce && !force { + return errors.New("resource is in use") + } + if r.order != nil { + *r.order = append(*r.order, resource.Kind+"/"+resource.ID) + } + remaining := r.resources[:0] + for _, held := range r.resources { + if held.ID != resource.ID { + remaining = append(remaining, held) + } + } + r.resources = remaining + return nil +} + +func quietEngine(accounts AccountStore, reapers ...Reaper) *Engine { + engine := NewEngine(accounts, reapers...) + engine.Logger = slog.New(slog.DiscardHandler) + engine.Timeouts = Timeouts{StageDrain: 200 * time.Millisecond, DrainPoll: 5 * time.Millisecond} + return engine +} + +func request() Request { + return Request{AccountID: testAccountID, AccountName: testAccountName} +} + +// The protected-account list must be the same two accounts the rest of the +// system treats as undeletable. Two hard-coded lists that drift is exactly the +// failure this guards against. +func TestProtectedAccountsMatchTheRestOfTheSystem(t *testing.T) { + for _, id := range []string{admin.SystemAccountID(), admin.DefaultAccountID()} { + if _, protected := protectedAccountIDs[id]; !protected { + t.Fatalf("account %s is not protected by teardown", id) + } + if _, protected := handlers_iam.UndeletableAccountIDs[id]; !protected { + t.Fatalf("account %s is not protected by the IAM service", id) + } + } + if len(protectedAccountIDs) != len(handlers_iam.UndeletableAccountIDs) { + t.Fatal("teardown and the IAM service disagree on which accounts are protected") + } +} + +func TestTeardownRefusesTheSuperAdminAccount(t *testing.T) { + accounts := newFakeAccounts() + engine := quietEngine(accounts) + + _, err := engine.Teardown(context.Background(), Request{ + AccountID: admin.DefaultAccountID(), AccountName: "spinifex", + }) + if !errors.Is(err, ErrProtectedAccount) { + t.Fatalf("expected ErrProtectedAccount, got %v", err) + } + if len(accounts.statuses) != 0 { + t.Fatal("a protected account must not even be marked terminating") + } +} + +func TestTeardownRefusesTheSystemAccount(t *testing.T) { + engine := quietEngine(newFakeAccounts()) + + _, err := engine.Teardown(context.Background(), Request{ + AccountID: admin.SystemAccountID(), AccountName: "system", + }) + if !errors.Is(err, ErrProtectedAccount) { + t.Fatalf("expected ErrProtectedAccount, got %v", err) + } +} + +// A mistyped account id must fail closed rather than empty a live tenant. +func TestTeardownRefusesAMismatchedName(t *testing.T) { + accounts := newFakeAccounts() + reaper := &fakeReaper{kind: "volume", stage: StageStorage, resources: []Resource{{Kind: "volume", ID: "vol-1"}}} + engine := quietEngine(accounts, reaper) + + _, err := engine.Teardown(context.Background(), Request{ + AccountID: testAccountID, AccountName: "someone-else@example.com", + }) + if !errors.Is(err, ErrAccountNameMismatch) { + t.Fatalf("expected ErrAccountNameMismatch, got %v", err) + } + if reaper.deletesAttempted != 0 { + t.Fatal("nothing may be deleted when the name does not match") + } + if accounts.deleted { + t.Fatal("the account must survive a name mismatch") + } +} + +// The error must not disclose the stored name: it is a customer email address +// and a caller who guessed an id would otherwise learn it by probing. +func TestNameMismatchDoesNotDiscloseTheStoredName(t *testing.T) { + engine := quietEngine(newFakeAccounts()) + + _, err := engine.Teardown(context.Background(), Request{ + AccountID: testAccountID, AccountName: "guess@example.com", + }) + if err == nil { + t.Fatal("expected an error") + } + if strings.Contains(err.Error(), testAccountName) { + t.Fatalf("error disclosed the stored account name: %v", err) + } +} + +// The account is quiesced before anything is deleted, otherwise the customer +// keeps creating resources behind the cascade and it never converges. +func TestTeardownMarksTerminatingBeforeDeletingAnything(t *testing.T) { + accounts := newFakeAccounts() + var order []string + reaper := &fakeReaper{ + kind: "volume", stage: StageStorage, order: &order, + resources: []Resource{{Kind: "volume", ID: "vol-1"}}, + } + engine := quietEngine(accounts, reaper) + + if _, err := engine.Teardown(context.Background(), request()); err != nil { + t.Fatalf("teardown: %v", err) + } + if len(accounts.statuses) == 0 || accounts.statuses[0] != AccountStatusTerminating { + t.Fatalf("expected TERMINATING to be set first, got %v", accounts.statuses) + } + if !accounts.deleted { + t.Fatal("a fully drained account must be deleted") + } +} + +// Stage order is the reason teardown converges at all: an instance holding a +// volume has to go first, or the volume is undeletable. +func TestTeardownRunsStagesInDependencyOrder(t *testing.T) { + var order []string + volumes := &fakeReaper{kind: "volume", stage: StageStorage, order: &order, + resources: []Resource{{Kind: "volume", ID: "vol-1"}}} + instances := &fakeReaper{kind: "instance", stage: StageCompute, order: &order, + resources: []Resource{{Kind: "instance", ID: "i-1"}}} + vpcs := &fakeReaper{kind: "vpc", stage: StageNetwork, order: &order, + resources: []Resource{{Kind: "vpc", ID: "vpc-1"}}} + + // Registered out of order on purpose: the stage decides, not the caller. + engine := quietEngine(newFakeAccounts(), vpcs, volumes, instances) + if _, err := engine.Teardown(context.Background(), request()); err != nil { + t.Fatalf("teardown: %v", err) + } + + want := []string{"instance/i-1", "volume/vol-1", "vpc/vpc-1"} + if len(order) != len(want) { + t.Fatalf("expected %v, got %v", want, order) + } + for i := range want { + if order[i] != want[i] { + t.Fatalf("expected %v, got %v", want, order) + } + } +} + +// Within one stage the registration order carries the dependency, which is +// where snapshots-before-volumes lives. +func TestTeardownPreservesRegistrationOrderWithinAStage(t *testing.T) { + var order []string + snapshots := &fakeReaper{kind: "snapshot", stage: StageStorage, order: &order, + resources: []Resource{{Kind: "snapshot", ID: "snap-1"}}} + volumes := &fakeReaper{kind: "volume", stage: StageStorage, order: &order, + resources: []Resource{{Kind: "volume", ID: "vol-1"}}} + + engine := quietEngine(newFakeAccounts(), snapshots, volumes) + if _, err := engine.Teardown(context.Background(), request()); err != nil { + t.Fatalf("teardown: %v", err) + } + if len(order) != 2 || order[0] != "snapshot/snap-1" { + t.Fatalf("snapshots must be deleted before volumes, got %v", order) + } +} + +// The deadlock the whole --force flag exists for: a volume an instance still +// holds. Without force it must be reported, not silently skipped. +func TestStuckResourceIsReportedAndTheAccountSurvives(t *testing.T) { + accounts := newFakeAccounts() + volumes := &fakeReaper{ + kind: "volume", stage: StageStorage, blockedUntilForce: true, + resources: []Resource{{Kind: "volume", ID: "vol-stuck", Detail: "attached to i-1"}}, + } + engine := quietEngine(accounts, volumes) + + result, err := engine.Teardown(context.Background(), request()) + if !errors.Is(err, ErrResourcesStuck) { + t.Fatalf("expected ErrResourcesStuck, got %v", err) + } + if result.StuckCount() != 1 { + t.Fatalf("expected 1 stuck resource, got %d", result.StuckCount()) + } + if accounts.deleted { + t.Fatal("the account must stay so the residue keeps an owner") + } + if accounts.account.Status != AccountStatusTerminating { + t.Fatalf("a failed teardown must leave the account TERMINATING, got %q", accounts.account.Status) + } + if result.AccountDeleted { + t.Fatal("AccountDeleted must be false when anything was left behind") + } +} + +// The reason a resource would not go is the useful half of the report. +func TestStuckResourceCarriesTheDeleteFailureReason(t *testing.T) { + volumes := &fakeReaper{ + kind: "volume", stage: StageStorage, blockedUntilForce: true, + resources: []Resource{{Kind: "volume", ID: "vol-stuck"}}, + } + engine := quietEngine(newFakeAccounts(), volumes) + + result, _ := engine.Teardown(context.Background(), request()) + if result.StuckCount() != 1 { + t.Fatalf("expected 1 stuck resource, got %d", result.StuckCount()) + } + for _, stage := range result.Stages { + for _, stuck := range stage.Stuck { + if !strings.Contains(stuck.Reason, "in use") { + t.Fatalf("expected the delete failure as the reason, got %q", stuck.Reason) + } + } + } +} + +func TestForceClearsAResourceThatRefusesTheOrdinaryDelete(t *testing.T) { + accounts := newFakeAccounts() + volumes := &fakeReaper{ + kind: "volume", stage: StageStorage, blockedUntilForce: true, + resources: []Resource{{Kind: "volume", ID: "vol-stuck", Detail: "attached to i-1"}}, + } + engine := quietEngine(accounts, volumes) + + req := request() + req.Force = true + result, err := engine.Teardown(context.Background(), req) + if err != nil { + t.Fatalf("force teardown: %v", err) + } + if result.StuckCount() != 0 { + t.Fatalf("force must clear the stuck volume, got %d stuck", result.StuckCount()) + } + if !accounts.deleted { + t.Fatal("a fully drained account must be deleted") + } +} + +// A dry run is the pre-flight for every runbook step and for the load-test +// harness, so it must be provably read-only. +func TestDryRunDeletesNothing(t *testing.T) { + accounts := newFakeAccounts() + volumes := &fakeReaper{kind: "volume", stage: StageStorage, + resources: []Resource{{Kind: "volume", ID: "vol-1"}}} + instances := &fakeReaper{kind: "instance", stage: StageCompute, + resources: []Resource{{Kind: "instance", ID: "i-1"}}} + engine := quietEngine(accounts, volumes, instances) + + req := request() + req.DryRun = true + result, err := engine.Teardown(context.Background(), req) + if err != nil { + t.Fatalf("dry run: %v", err) + } + if volumes.deletesAttempted != 0 || instances.deletesAttempted != 0 { + t.Fatal("a dry run must not attempt any delete") + } + if len(accounts.statuses) != 0 { + t.Fatal("a dry run must not change the account status") + } + if accounts.deleted { + t.Fatal("a dry run must not delete the account") + } + if result.DeletedCount() != 2 { + t.Fatalf("expected both resources in the inventory, got %d", result.DeletedCount()) + } +} + +// Inventory is the read-only probe, so it must not require the confirmation +// name and must not touch the account. +func TestInventoryIsReadOnlyAndNeedsNoConfirmation(t *testing.T) { + accounts := newFakeAccounts() + volumes := &fakeReaper{kind: "volume", stage: StageStorage, + resources: []Resource{{Kind: "volume", ID: "vol-1"}}} + engine := quietEngine(accounts, volumes) + + result, err := engine.Inventory(context.Background(), testAccountID) + if err != nil { + t.Fatalf("inventory: %v", err) + } + if result.DeletedCount() != 1 { + t.Fatalf("expected 1 resource, got %d", result.DeletedCount()) + } + if volumes.deletesAttempted != 0 || len(accounts.statuses) != 0 || accounts.deleted { + t.Fatal("inventory must not change anything") + } +} + +func TestInventoryRefusesAProtectedAccount(t *testing.T) { + engine := quietEngine(newFakeAccounts()) + + if _, err := engine.Inventory(context.Background(), admin.SystemAccountID()); !errors.Is(err, ErrProtectedAccount) { + t.Fatalf("expected ErrProtectedAccount, got %v", err) + } +} + +// A listing that fails says nothing about whether the account is empty, so it +// must stop the teardown rather than let it read as drained. +func TestAFailedListingAbortsTheTeardown(t *testing.T) { + accounts := newFakeAccounts() + volumes := &fakeReaper{kind: "volume", stage: StageStorage, listErr: errors.New("store unavailable")} + engine := quietEngine(accounts, volumes) + + if _, err := engine.Teardown(context.Background(), request()); err == nil { + t.Fatal("expected the failed listing to abort the teardown") + } + if accounts.deleted { + t.Fatal("the account must not be deleted when a listing failed") + } +} + +// Teardown re-runs after a crash, so a second pass over an empty account has +// to succeed rather than trip over resources that are already gone. +func TestTeardownIsIdempotentOverAnAlreadyEmptyAccount(t *testing.T) { + accounts := newFakeAccounts() + volumes := &fakeReaper{kind: "volume", stage: StageStorage} + engine := quietEngine(accounts, volumes) + + result, err := engine.Teardown(context.Background(), request()) + if err != nil { + t.Fatalf("teardown: %v", err) + } + if result.DeletedCount() != 0 || !result.AccountDeleted { + t.Fatalf("expected an empty successful teardown, got %+v", result) + } +} + +// A stage whose deletes are asynchronous is finished when a fresh listing is +// empty, not when the delete calls returned. +func TestAStageWaitsForItsResourcesToActuallyDisappear(t *testing.T) { + slow := &slowReaper{kind: "instance", stage: StageCompute, listsBeforeGone: 3, + resource: Resource{Kind: "instance", ID: "i-1"}} + engine := quietEngine(newFakeAccounts(), slow) + + result, err := engine.Teardown(context.Background(), request()) + if err != nil { + t.Fatalf("teardown: %v", err) + } + if result.StuckCount() != 0 { + t.Fatalf("expected the instance to drain, got %d stuck", result.StuckCount()) + } + if slow.lists < 2 { + t.Fatalf("expected the stage to re-list until empty, listed %d times", slow.lists) + } +} + +// slowReaper keeps reporting its resource for a few listings after the delete +// is accepted, the way a terminate does. +type slowReaper struct { + kind string + stage Stage + resource Resource + listsBeforeGone int + lists int + deleted bool +} + +func (r *slowReaper) Kind() string { return r.kind } +func (r *slowReaper) Stage() Stage { return r.stage } + +func (r *slowReaper) List(context.Context, string) ([]Resource, error) { + r.lists++ + if r.deleted && r.lists > r.listsBeforeGone { + return nil, nil + } + return []Resource{r.resource}, nil +} + +func (r *slowReaper) Delete(context.Context, string, Resource, bool) error { + r.deleted = true + return nil +} diff --git a/spinifex/accountteardown/reapers_account.go b/spinifex/accountteardown/reapers_account.go new file mode 100644 index 000000000..93ef7a49a --- /dev/null +++ b/spinifex/accountteardown/reapers_account.go @@ -0,0 +1,125 @@ +package accountteardown + +import ( + "context" + "errors" + "fmt" + + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + "github.com/nats-io/nats.go/jetstream" +) + +// AccountReapers returns the final-stage reapers: the per-account counters and +// reservations that outlive every resource but must not outlive the account. +func AccountReapers(svc handlers_iam.IAMService, names *handlers_iam.AccountNameIndex, usage jetstream.KeyValue) []Reaper { + return []Reaper{ + "aUsageReaper{usage: usage}, + &nameReservationReaper{svc: svc, names: names}, + } +} + +// quotaUsageReaper removes the account's vCPU counter. Left behind it is a +// small leak that also makes the account id look live to anything reading the +// bucket to decide what exists. +type quotaUsageReaper struct{ usage jetstream.KeyValue } + +func (r *quotaUsageReaper) Kind() string { return "quota-usage" } +func (r *quotaUsageReaper) Stage() Stage { return StageAccount } + +func (r *quotaUsageReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + if r.usage == nil { + return nil, nil + } + if _, err := r.usage.Get(ctx, accountID); err != nil { + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, nil + } + return nil, fmt.Errorf("read quota usage: %w", err) + } + return []Resource{{Kind: r.Kind(), ID: accountID}}, nil +} + +func (r *quotaUsageReaper) Delete(ctx context.Context, _ string, resource Resource, _ bool) error { + if r.usage == nil { + return nil + } + if err := r.usage.Delete(ctx, resource.ID); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + return err + } + return nil +} + +// nameReservationReaper releases the account's name so the same email can sign +// up again. A customer who deletes an account and returns must not be told the +// address is taken by a tenant that no longer exists. +type nameReservationReaper struct { + svc handlers_iam.IAMService + names *handlers_iam.AccountNameIndex +} + +func (r *nameReservationReaper) Kind() string { return "name-reservation" } +func (r *nameReservationReaper) Stage() Stage { return StageAccount } + +func (r *nameReservationReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + if r.names == nil { + return nil, nil + } + account, err := r.svc.GetAccount(accountID) + if err != nil { + // The account record is deleted after this stage, so it must still be + // here. Its absence means someone else removed it and the reservation + // can no longer be attributed — report rather than guess. + return nil, fmt.Errorf("read account for its name reservation: %w", err) + } + + reserved, found, err := r.names.Lookup(ctx, account.AccountName) + if err != nil { + return nil, fmt.Errorf("look up name reservation: %w", err) + } + if !found || reserved != accountID { + return nil, nil + } + return []Resource{{Kind: r.Kind(), ID: account.AccountName}}, nil +} + +func (r *nameReservationReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + if r.names == nil { + return nil + } + return r.names.ReleaseDeleted(ctx, resource.ID, accountID) +} + +// IAMAccounts adapts the IAM service to the engine's AccountStore. The engine +// declares its own Account type so its tests need no cluster; this is the one +// place the two meet. +type IAMAccounts struct{ svc handlers_iam.IAMService } + +var _ AccountStore = (*IAMAccounts)(nil) + +func NewIAMAccounts(svc handlers_iam.IAMService) *IAMAccounts { + return &IAMAccounts{svc: svc} +} + +func (a *IAMAccounts) GetAccount(accountID string) (*Account, error) { + account, err := a.svc.GetAccount(accountID) + if err != nil { + return nil, err + } + return &Account{ + AccountID: account.AccountID, AccountName: account.AccountName, Status: account.Status, + }, nil +} + +func (a *IAMAccounts) SetAccountStatus(accountID, status string) (*Account, error) { + account, err := a.svc.SetAccountStatus(accountID, status) + if err != nil { + return nil, err + } + return &Account{ + AccountID: account.AccountID, AccountName: account.AccountName, Status: account.Status, + }, nil +} + +func (a *IAMAccounts) DeleteAccount(accountID string) error { + return a.svc.DeleteAccount(accountID) +} diff --git a/spinifex/accountteardown/reapers_account_test.go b/spinifex/accountteardown/reapers_account_test.go new file mode 100644 index 000000000..c83ddd98c --- /dev/null +++ b/spinifex/accountteardown/reapers_account_test.go @@ -0,0 +1,207 @@ +package accountteardown + +//test:in-package — the final-stage reapers are unexported, and what they +// guarantee (the name is released, and only to its own account) is only +// observable from inside the package. + +import ( + "context" + "errors" + "testing" + + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + "github.com/mulgadc/spinifex/spinifex/kvutil" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/nats-io/nats.go/jetstream" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// accountIAM adds the account-record calls the final stage needs to the IAM +// double the identity reapers use. +type accountIAM struct { + fakeIAM + + account *handlers_iam.Account + getErr error + deleted bool + setStatus string +} + +func (f *accountIAM) GetAccount(string) (*handlers_iam.Account, error) { + if f.getErr != nil { + return nil, f.getErr + } + return f.account, nil +} + +func (f *accountIAM) SetAccountStatus(_, status string) (*handlers_iam.Account, error) { + f.setStatus = status + updated := *f.account + updated.Status = status + f.account = &updated + return f.account, nil +} + +func (f *accountIAM) DeleteAccount(string) error { + f.deleted = true + return nil +} + +func testKV(t *testing.T, bucket string) (context.Context, jetstream.JetStream, jetstream.KeyValue) { + t.Helper() + _, _, js := testutil.StartTestJetStream(t) + ctx := testCtx(t) + kv, err := kvutil.GetOrCreateBucket(ctx, js, bucket, 1) + require.NoError(t, err) + return ctx, js, kv +} + +// Left behind, the counter is a small leak that also makes a deleted account +// id look live to anything reading the bucket to decide what exists. +func TestQuotaUsageReaperRemovesTheCounter(t *testing.T) { + ctx, _, usage := testKV(t, "spinifex-account-usage-test") + _, err := usage.Put(ctx, "000000000042", []byte(`{"vcpus":4}`)) + require.NoError(t, err) + + reaper := "aUsageReaper{usage: usage} + assert.Equal(t, StageAccount, reaper.Stage()) + + found, err := reaper.List(ctx, "000000000042") + require.NoError(t, err) + require.Len(t, found, 1) + + require.NoError(t, reaper.Delete(ctx, "000000000042", found[0], false)) + + found, err = reaper.List(ctx, "000000000042") + require.NoError(t, err) + assert.Empty(t, found) +} + +// A cluster that never enabled quotas has no counter to remove, which is not a +// reason to refuse a teardown. +func TestQuotaUsageReaperToleratesAnAbsentBucket(t *testing.T) { + ctx, _, usage := testKV(t, "spinifex-account-usage-absent") + + reaper := "aUsageReaper{usage: usage} + found, err := reaper.List(ctx, "000000000042") + require.NoError(t, err) + assert.Empty(t, found) + assert.NoError(t, reaper.Delete(ctx, "000000000042", Resource{ID: "000000000042"}, false)) + + nilReaper := "aUsageReaper{} + found, err = nilReaper.List(ctx, "000000000042") + require.NoError(t, err) + assert.Empty(t, found) + assert.NoError(t, nilReaper.Delete(ctx, "000000000042", Resource{ID: "000000000042"}, false)) +} + +// A customer who deletes an account and comes back must not be told their own +// address is taken by a tenant that no longer exists. +func TestNameReservationReaperReleasesTheName(t *testing.T) { + _, js, _ := testKV(t, "spinifex-name-reaper-warmup") + ctx := testCtx(t) + + names, err := handlers_iam.NewAccountNameIndex(ctx, js) + require.NoError(t, err) + require.NoError(t, names.Reserve(ctx, "tenant@example.com", "token-1")) + require.NoError(t, names.Commit(ctx, "tenant@example.com", "000000000042", "token-1")) + + svc := &accountIAM{account: &handlers_iam.Account{ + AccountID: "000000000042", AccountName: "tenant@example.com", + }} + reaper := &nameReservationReaper{svc: svc, names: names} + + found, err := reaper.List(ctx, "000000000042") + require.NoError(t, err) + require.Len(t, found, 1) + assert.Equal(t, "tenant@example.com", found[0].ID) + + require.NoError(t, reaper.Delete(ctx, "000000000042", found[0], false)) + + _, stillReserved, err := names.Lookup(ctx, "tenant@example.com") + require.NoError(t, err) + assert.False(t, stillReserved) +} + +// A reservation held by a different account is not this teardown's to release: +// a stale name must not take another tenant's reservation with it. +func TestNameReservationReaperIgnoresAnotherAccountsName(t *testing.T) { + _, js, _ := testKV(t, "spinifex-name-reaper-other") + ctx := testCtx(t) + + names, err := handlers_iam.NewAccountNameIndex(ctx, js) + require.NoError(t, err) + require.NoError(t, names.Reserve(ctx, "tenant@example.com", "token-1")) + require.NoError(t, names.Commit(ctx, "tenant@example.com", "000000000099", "token-1")) + + svc := &accountIAM{account: &handlers_iam.Account{ + AccountID: "000000000042", AccountName: "tenant@example.com", + }} + + found, err := (&nameReservationReaper{svc: svc, names: names}).List(ctx, "000000000042") + require.NoError(t, err) + assert.Empty(t, found) + + _, stillReserved, err := names.Lookup(ctx, "tenant@example.com") + require.NoError(t, err) + assert.True(t, stillReserved) +} + +// The account record is deleted after this stage, so its absence means someone +// else removed it and the reservation can no longer be attributed. +func TestNameReservationReaperReportsAMissingAccount(t *testing.T) { + _, js, _ := testKV(t, "spinifex-name-reaper-missing") + ctx := testCtx(t) + + names, err := handlers_iam.NewAccountNameIndex(ctx, js) + require.NoError(t, err) + + svc := &accountIAM{getErr: errors.New("account not found")} + _, err = (&nameReservationReaper{svc: svc, names: names}).List(ctx, "000000000042") + + assert.Error(t, err) +} + +func TestAccountReapersRunLast(t *testing.T) { + reapers := AccountReapers(&accountIAM{}, nil, nil) + + var kinds []string + for _, reaper := range reapers { + kinds = append(kinds, reaper.Kind()) + assert.Equal(t, StageAccount, reaper.Stage()) + } + assert.Equal(t, []string{"quota-usage", "name-reservation"}, kinds) +} + +// The adapter is the one place the engine's Account and the IAM record meet, +// so a field dropped in translation would silently break the name confirmation. +func TestIAMAccountsAdapterCarriesEveryField(t *testing.T) { + svc := &accountIAM{account: &handlers_iam.Account{ + AccountID: "000000000042", + AccountName: "tenant@example.com", + Status: handlers_iam.AccountStatusActive, + }} + accounts := NewIAMAccounts(svc) + + account, err := accounts.GetAccount("000000000042") + require.NoError(t, err) + assert.Equal(t, "000000000042", account.AccountID) + assert.Equal(t, "tenant@example.com", account.AccountName) + assert.Equal(t, handlers_iam.AccountStatusActive, account.Status) + + updated, err := accounts.SetAccountStatus("000000000042", AccountStatusTerminating) + require.NoError(t, err) + assert.Equal(t, AccountStatusTerminating, updated.Status) + assert.Equal(t, handlers_iam.AccountStatusTerminating, svc.setStatus) + + require.NoError(t, accounts.DeleteAccount("000000000042")) + assert.True(t, svc.deleted) +} + +// The engine declares TERMINATING itself so its tests need no IAM service. The +// two must not drift: a mismatch would mark accounts with a status the auth +// gate does not recognise, and they would keep authenticating. +func TestTerminatingStatusMatchesTheIAMRecord(t *testing.T) { + assert.Equal(t, handlers_iam.AccountStatusTerminating, AccountStatusTerminating) +} diff --git a/spinifex/accountteardown/reapers_acm.go b/spinifex/accountteardown/reapers_acm.go new file mode 100644 index 000000000..f7b7a9a75 --- /dev/null +++ b/spinifex/accountteardown/reapers_acm.go @@ -0,0 +1,57 @@ +package accountteardown + +import ( + "context" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/acm" + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/nats-io/nats.go" +) + +// acmNATSTimeout matches the gateway's own certificate timeout. +const acmNATSTimeout = 30 * time.Second + +// ACMReapers returns the certificate reaper. +// +// ACM has no service adapter, so this talks to the same subjects the gateway +// does. Platform stage: a certificate outlives the listener that referenced it, +// and nothing in the network stage blocks on one. +func ACMReapers(nc *nats.Conn) []Reaper { + return []Reaper{&certificateReaper{nc: nc}} +} + +type certificateReaper struct { + nc *nats.Conn +} + +func (r *certificateReaper) Kind() string { return "acm-certificate" } +func (r *certificateReaper) Stage() Stage { return StagePlatform } + +func (r *certificateReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + out, err := utils.NATSRequest[acm.ListCertificatesOutput](ctx, r.nc, "acm.ListCertificates", + &acm.ListCertificatesInput{}, acmNATSTimeout, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, summary := range out.CertificateSummaryList { + if summary == nil || summary.CertificateArn == nil { + continue + } + found = append(found, Resource{ + Kind: r.Kind(), + ID: aws.StringValue(summary.CertificateArn), + Detail: aws.StringValue(summary.DomainName), + }) + } + return found, nil +} + +func (r *certificateReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + _, err := utils.NATSRequest[acm.DeleteCertificateOutput](ctx, r.nc, "acm.DeleteCertificate", + &acm.DeleteCertificateInput{CertificateArn: aws.String(resource.ID)}, acmNATSTimeout, accountID) + return ignoreAlreadyGone(err) +} diff --git a/spinifex/accountteardown/reapers_acm_test.go b/spinifex/accountteardown/reapers_acm_test.go new file mode 100644 index 000000000..7556b3dde --- /dev/null +++ b/spinifex/accountteardown/reapers_acm_test.go @@ -0,0 +1,69 @@ +package accountteardown + +//test:in-package — the reaper is unexported, and ACM has no service adapter to +// fake, so the subjects it speaks are what has to be asserted. + +import ( + "encoding/json" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/acm" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// replyWith answers subject with payload once subscribed, recording the +// account header every request carried. +func replyWith(t *testing.T, nc *nats.Conn, subject string, payload []byte, accounts *[]string) { + t.Helper() + sub, err := nc.Subscribe(subject, func(msg *nats.Msg) { + *accounts = append(*accounts, msg.Header.Get(utils.AccountIDHeader)) + require.NoError(t, msg.Respond(payload)) + }) + require.NoError(t, err) + require.NoError(t, nc.Flush()) + t.Cleanup(func() { _ = sub.Unsubscribe() }) +} + +func TestCertificateReaperListsAndDeletesByARN(t *testing.T) { + _, nc, _ := testutil.StartTestJetStream(t) + + listing, err := json.Marshal(acm.ListCertificatesOutput{CertificateSummaryList: []*acm.CertificateSummary{ + {CertificateArn: aws.String("arn:acm/one"), DomainName: aws.String("api.example.com")}, + {DomainName: aws.String("no-arn.example.com")}, + }}) + require.NoError(t, err) + + var accounts []string + replyWith(t, nc, "acm.ListCertificates", listing, &accounts) + replyWith(t, nc, "acm.DeleteCertificate", []byte(`{}`), &accounts) + + reaper := &certificateReaper{nc: nc} + assert.Equal(t, StagePlatform, reaper.Stage()) + + found, err := reaper.List(t.Context(), "000000000002") + require.NoError(t, err) + require.Len(t, found, 1) + assert.Equal(t, "arn:acm/one", found[0].ID) + assert.Equal(t, "api.example.com", found[0].Detail) + + require.NoError(t, reaper.Delete(t.Context(), "000000000002", found[0], false)) + + // Both calls have to carry the tenant, or they read another account's + // certificates and delete from it. + assert.Equal(t, []string{"000000000002", "000000000002"}, accounts) +} + +func TestCertificateReaperTreatsAMissingCertificateAsDeleted(t *testing.T) { + _, nc, _ := testutil.StartTestJetStream(t) + + var accounts []string + replyWith(t, nc, "acm.DeleteCertificate", utils.GenerateErrorPayload("ResourceNotFound"), &accounts) + + reaper := &certificateReaper{nc: nc} + assert.NoError(t, reaper.Delete(t.Context(), "000000000002", Resource{ID: "arn:acm/gone"}, false)) +} diff --git a/spinifex/accountteardown/reapers_bedrock.go b/spinifex/accountteardown/reapers_bedrock.go new file mode 100644 index 000000000..6820b7df7 --- /dev/null +++ b/spinifex/accountteardown/reapers_bedrock.go @@ -0,0 +1,58 @@ +package accountteardown + +import ( + "context" + + handlers_bedrock "github.com/mulgadc/spinifex/spinifex/handlers/bedrock" + "github.com/nats-io/nats.go" +) + +// BedrockReapers returns the model endpoint reaper. +// +// Compute stage, ahead of the instance reaper: an endpoint owns a VM, an ENI +// and a weights volume, and only its own delete releases all three. Terminating +// its instance first would strand the other two with no record naming them. +func BedrockReapers(nc *nats.Conn) []Reaper { + return []Reaper{&bedrockEndpointReaper{svc: handlers_bedrock.NewNATSEndpointService(nc)}} +} + +type bedrockEndpointReaper struct { + svc handlers_bedrock.EndpointService +} + +func (r *bedrockEndpointReaper) Kind() string { return "bedrock-endpoint" } +func (r *bedrockEndpointReaper) Stage() Stage { return StageCompute } + +// List filters by account itself. Bedrock's List returns every endpoint in the +// cluster whatever account it is called for, so an unfiltered reaper would +// report — and then delete — endpoints belonging to other tenants and to the +// shared global account. +func (r *bedrockEndpointReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + out, err := r.svc.List(ctx, &handlers_bedrock.ListEndpointsInput{}, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, record := range out.Endpoints { + if record.AccountID != accountID || record.ModelID == "" { + continue + } + found = append(found, Resource{ + Kind: r.Kind(), + ID: record.ModelID, + Detail: string(record.State), + }) + } + return found, nil +} + +// Delete names the account explicitly: an empty AccountID resolves to the +// global one, which would delete the shared endpoint instead of the tenant's. +func (r *bedrockEndpointReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + _, err := r.svc.Delete(ctx, &handlers_bedrock.DeleteEndpointInput{ + ModelID: resource.ID, + AccountID: accountID, + }, accountID) + return ignoreAlreadyGone(err) +} diff --git a/spinifex/accountteardown/reapers_bedrock_test.go b/spinifex/accountteardown/reapers_bedrock_test.go new file mode 100644 index 000000000..7354c23bc --- /dev/null +++ b/spinifex/accountteardown/reapers_bedrock_test.go @@ -0,0 +1,74 @@ +package accountteardown + +//test:in-package — the reaper is unexported, and the account filter it applies +// to a cluster-wide listing is the substance of it. + +import ( + "context" + "errors" + "testing" + + handlers_bedrock "github.com/mulgadc/spinifex/spinifex/handlers/bedrock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeEndpoints struct { + handlers_bedrock.EndpointService + + records []handlers_bedrock.EndpointRecord + deleted []handlers_bedrock.DeleteEndpointInput + err error +} + +func (f *fakeEndpoints) List(_ context.Context, _ *handlers_bedrock.ListEndpointsInput, _ string) (*handlers_bedrock.ListEndpointsOutput, error) { + return &handlers_bedrock.ListEndpointsOutput{Endpoints: f.records}, nil +} + +func (f *fakeEndpoints) Delete(_ context.Context, in *handlers_bedrock.DeleteEndpointInput, _ string) (*handlers_bedrock.DeleteEndpointOutput, error) { + if f.err != nil { + return nil, f.err + } + f.deleted = append(f.deleted, *in) + return &handlers_bedrock.DeleteEndpointOutput{}, nil +} + +// Bedrock's List answers with every endpoint in the cluster whatever account +// it is called for, so the filter is the reaper's own responsibility: without +// it a tenant teardown would delete the shared endpoint and other tenants'. +func TestBedrockEndpointReaperListsOnlyItsOwnAccount(t *testing.T) { + svc := &fakeEndpoints{records: []handlers_bedrock.EndpointRecord{ + {AccountID: "000000000002", ModelID: "mistral-7b", State: handlers_bedrock.StateReady}, + {AccountID: "000000000003", ModelID: "llama-3", State: handlers_bedrock.StateReady}, + {AccountID: "global", ModelID: "shared", State: handlers_bedrock.StateReady}, + {AccountID: "000000000002", ModelID: "", State: handlers_bedrock.StateStarting}, + }} + reaper := &bedrockEndpointReaper{svc: svc} + + assert.Equal(t, StageCompute, reaper.Stage()) + + found, err := reaper.List(t.Context(), "000000000002") + require.NoError(t, err) + require.Len(t, found, 1) + assert.Equal(t, "mistral-7b", found[0].ID) + assert.Equal(t, "READY", found[0].Detail) +} + +// An empty AccountID resolves to the global account on the far side, so a +// delete that leaves it unset removes the shared endpoint instead. +func TestBedrockEndpointReaperNamesTheAccountItDeletesFor(t *testing.T) { + svc := &fakeEndpoints{} + reaper := &bedrockEndpointReaper{svc: svc} + + require.NoError(t, reaper.Delete(t.Context(), "000000000002", Resource{ID: "mistral-7b"}, false)) + require.Len(t, svc.deleted, 1) + assert.Equal(t, "000000000002", svc.deleted[0].AccountID) + assert.Equal(t, "mistral-7b", svc.deleted[0].ModelID) +} + +func TestBedrockEndpointReaperTreatsAMissingEndpointAsDeleted(t *testing.T) { + svc := &fakeEndpoints{err: errors.New("ResourceNotFoundException: no such endpoint")} + reaper := &bedrockEndpointReaper{svc: svc} + + assert.NoError(t, reaper.Delete(t.Context(), "000000000002", Resource{ID: "mistral-7b"}, false)) +} diff --git a/spinifex/accountteardown/reapers_ec2.go b/spinifex/accountteardown/reapers_ec2.go new file mode 100644 index 000000000..4c730bdb4 --- /dev/null +++ b/spinifex/accountteardown/reapers_ec2.go @@ -0,0 +1,712 @@ +package accountteardown + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/mulgadc/spinifex/spinifex/awserrors" + gateway_ec2_eip "github.com/mulgadc/spinifex/spinifex/gateway/ec2/eip" + gateway_ec2_igw "github.com/mulgadc/spinifex/spinifex/gateway/ec2/igw" + gateway_ec2_image "github.com/mulgadc/spinifex/spinifex/gateway/ec2/image" + gateway_ec2_instance "github.com/mulgadc/spinifex/spinifex/gateway/ec2/instance" + gateway_ec2_key "github.com/mulgadc/spinifex/spinifex/gateway/ec2/key" + gateway_ec2_launchtemplate "github.com/mulgadc/spinifex/spinifex/gateway/ec2/launchtemplate" + gateway_ec2_placementgroup "github.com/mulgadc/spinifex/spinifex/gateway/ec2/placementgroup" + gateway_ec2_routetable "github.com/mulgadc/spinifex/spinifex/gateway/ec2/routetable" + gateway_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/gateway/ec2/snapshot" + gateway_ec2_volume "github.com/mulgadc/spinifex/spinifex/gateway/ec2/volume" + gateway_ec2_vpc "github.com/mulgadc/spinifex/spinifex/gateway/ec2/vpc" + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/nats-io/nats.go" +) + +// EC2Reapers returns every EC2-backed reaper in teardown order. +// +// They drive the same NATS-backed operations the gateway serves a customer +// request with, which is what keeps OVN teardown, IPAM release and quota +// accounting correct without teardown having to reimplement any of it. +func EC2Reapers(nc *nats.Conn, expectedNodes int) []Reaper { + return []Reaper{ + &instanceReaper{nc: nc, expectedNodes: expectedNodes}, + + // Snapshots before volumes: a volume with a live snapshot refuses to + // delete, because snapshot-backed clones read from its chunk prefix. + &snapshotReaper{nc: nc}, + &imageReaper{nc: nc}, + &volumeReaper{nc: nc, expectedNodes: expectedNodes}, + + // Reverse of creation order. The VPC cannot go until everything + // inside it has, and the gateway enforces that. + &addressReaper{nc: nc}, + + // After addresses so a released EIP is already off the interface, and + // before subnets because an interface still in a subnet refuses to let + // that subnet go — which wedged the whole account, not just the ENI. + &networkInterfaceReaper{nc: nc}, + + &igwReaper{nc: nc}, + &routeTableReaper{nc: nc}, + &subnetReaper{nc: nc}, + &securityGroupReaper{nc: nc}, + &vpcReaper{nc: nc}, + + &keyPairReaper{nc: nc}, + &placementGroupReaper{nc: nc}, + &launchTemplateReaper{nc: nc}, + } +} + +// terminalInstanceStates are the states an instance no longer holds anything +// in. Anything else is still occupying a volume, an address and a host. +var terminalInstanceStates = map[string]bool{"terminated": true, "shutting-down": true} + +type instanceReaper struct { + nc *nats.Conn + expectedNodes int +} + +func (r *instanceReaper) Kind() string { return "instance" } +func (r *instanceReaper) Stage() Stage { return StageCompute } + +func (r *instanceReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + // Checked: a partial answer here would read as "no instances left" and let + // storage deletion start while a guest still holds a volume. + out, err := gateway_ec2_instance.DescribeInstancesChecked(ctx, &ec2.DescribeInstancesInput{}, r.nc, r.expectedNodes, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, reservation := range out.Reservations { + for _, instance := range reservation.Instances { + if instance == nil || instance.InstanceId == nil { + continue + } + state := "" + if instance.State != nil && instance.State.Name != nil { + state = *instance.State.Name + } + if terminalInstanceStates[state] { + continue + } + found = append(found, Resource{Kind: r.Kind(), ID: *instance.InstanceId, Detail: state}) + } + } + return found, nil +} + +func (r *instanceReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + _, err := gateway_ec2_instance.TerminateInstances(ctx, &ec2.TerminateInstancesInput{ + InstanceIds: []*string{aws.String(resource.ID)}, + }, r.nc, accountID) + return ignoreAlreadyGone(err) +} + +type volumeReaper struct { + nc *nats.Conn + expectedNodes int +} + +func (r *volumeReaper) Kind() string { return "volume" } +func (r *volumeReaper) Stage() Stage { return StageStorage } + +func (r *volumeReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + out, err := gateway_ec2_volume.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{}, r.nc, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, volume := range out.Volumes { + if volume == nil || volume.VolumeId == nil { + continue + } + resource := Resource{Kind: r.Kind(), ID: *volume.VolumeId} + // The attachment is the reason a volume will not go, so carry it: it + // is the difference between "stuck" and "stuck on i-abc". + for _, attachment := range volume.Attachments { + if attachment != nil && attachment.InstanceId != nil { + resource.Detail = "attached to " + *attachment.InstanceId + break + } + } + found = append(found, resource) + } + return found, nil +} + +func (r *volumeReaper) Delete(ctx context.Context, accountID string, resource Resource, force bool) error { + _, err := gateway_ec2_volume.DeleteVolume(ctx, &ec2.DeleteVolumeInput{ + VolumeId: aws.String(resource.ID), + }, r.nc, r.expectedNodes, accountID) + if isAlreadyGone(err) { + return nil + } + if !force { + return err + } + // Force path: the ordinary delete refuses an attached volume, and an + // instance that will not terminate would otherwise strand it forever. + return forceDeleteVolume(ctx, r.nc, r.expectedNodes, accountID, resource.ID) +} + +type snapshotReaper struct{ nc *nats.Conn } + +func (r *snapshotReaper) Kind() string { return "snapshot" } +func (r *snapshotReaper) Stage() Stage { return StageStorage } + +func (r *snapshotReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + out, err := gateway_ec2_snapshot.DescribeSnapshots(ctx, &ec2.DescribeSnapshotsInput{ + OwnerIds: []*string{aws.String(accountID)}, + }, r.nc, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, snapshot := range out.Snapshots { + if snapshot == nil || snapshot.SnapshotId == nil { + continue + } + found = append(found, Resource{Kind: r.Kind(), ID: *snapshot.SnapshotId}) + } + return found, nil +} + +func (r *snapshotReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + _, err := gateway_ec2_snapshot.DeleteSnapshot(ctx, &ec2.DeleteSnapshotInput{ + SnapshotId: aws.String(resource.ID), + }, r.nc, accountID) + return ignoreAlreadyGone(err) +} + +type imageReaper struct{ nc *nats.Conn } + +func (r *imageReaper) Kind() string { return "image" } +func (r *imageReaper) Stage() Stage { return StageStorage } + +func (r *imageReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + // Owner-scoped: a public or shared image the account merely sees is not + // its property and must survive its deletion. + out, err := gateway_ec2_image.DescribeImages(ctx, &ec2.DescribeImagesInput{ + Owners: []*string{aws.String(accountID)}, + }, r.nc, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, image := range out.Images { + if image == nil || image.ImageId == nil { + continue + } + found = append(found, Resource{Kind: r.Kind(), ID: *image.ImageId}) + } + return found, nil +} + +func (r *imageReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + _, err := gateway_ec2_image.DeregisterImage(ctx, &ec2.DeregisterImageInput{ + ImageId: aws.String(resource.ID), + }, r.nc, accountID) + return ignoreAlreadyGone(err) +} + +type addressReaper struct{ nc *nats.Conn } + +func (r *addressReaper) Kind() string { return "address" } +func (r *addressReaper) Stage() Stage { return StageNetwork } + +func (r *addressReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + out, err := gateway_ec2_eip.DescribeAddresses(ctx, &ec2.DescribeAddressesInput{}, r.nc, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, address := range out.Addresses { + if address == nil || address.AllocationId == nil { + continue + } + resource := Resource{Kind: r.Kind(), ID: *address.AllocationId} + if address.PublicIp != nil { + resource.Detail = *address.PublicIp + } + found = append(found, resource) + } + return found, nil +} + +// Delete disassociates before releasing. An associated address cannot be +// released, and the association usually outlives the instance by a moment. +func (r *addressReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + associationID, err := r.associationID(ctx, accountID, resource.ID) + if err != nil && !isAlreadyGone(err) { + return err + } + if associationID != "" { + if _, err := gateway_ec2_eip.DisassociateAddress(ctx, &ec2.DisassociateAddressInput{ + AssociationId: aws.String(associationID), + }, r.nc, accountID); err != nil && !isAlreadyGone(err) && !isNotAssociated(err) { + return err + } + } + _, err = gateway_ec2_eip.ReleaseAddress(ctx, &ec2.ReleaseAddressInput{ + AllocationId: aws.String(resource.ID), + }, r.nc, accountID) + return ignoreAlreadyGone(err) +} + +// associationID reads the address's current association rather than carrying it +// from the listing, because a delete may also be driven from an id an operator +// supplied. An empty answer means unattached, which is not an error. +func (r *addressReaper) associationID(ctx context.Context, accountID, allocationID string) (string, error) { + out, err := gateway_ec2_eip.DescribeAddresses(ctx, &ec2.DescribeAddressesInput{ + AllocationIds: []*string{aws.String(allocationID)}, + }, r.nc, accountID) + if err != nil { + return "", err + } + for _, address := range out.Addresses { + if address == nil || address.AllocationId == nil || *address.AllocationId != allocationID { + continue + } + return aws.StringValue(address.AssociationId), nil + } + return "", nil +} + +// networkInterfaceReaper removes the interfaces nothing else will. A running +// instance takes its own interfaces with it, so anything still listed by the +// time the network stage runs was either created by hand or left behind by a +// launch that failed after the interface was made. +// +// Either way it holds a subnet, and a subnet that will not delete stops the VPC +// and leaves the account in TERMINATING for good. +type networkInterfaceReaper struct{ nc *nats.Conn } + +func (r *networkInterfaceReaper) Kind() string { return "network-interface" } +func (r *networkInterfaceReaper) Stage() Stage { return StageNetwork } + +func (r *networkInterfaceReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + out, err := gateway_ec2_vpc.DescribeNetworkInterfaces(ctx, &ec2.DescribeNetworkInterfacesInput{}, r.nc, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, eni := range out.NetworkInterfaces { + if eni == nil || eni.NetworkInterfaceId == nil { + continue + } + resource := Resource{Kind: r.Kind(), ID: *eni.NetworkInterfaceId} + if eni.SubnetId != nil { + resource.Detail = *eni.SubnetId + } + found = append(found, resource) + } + return found, nil +} + +// Delete detaches first when the interface is attached, because an in-use +// interface cannot be deleted. The attachment is read fresh rather than carried +// from the listing, so a delete driven from an id an operator supplied behaves +// the same as one driven from a listing. +func (r *networkInterfaceReaper) Delete(ctx context.Context, accountID string, resource Resource, force bool) error { + attachmentID, err := r.attachmentID(ctx, accountID, resource.ID) + if err != nil && !isAlreadyGone(err) { + return err + } + if attachmentID != "" { + if _, err := gateway_ec2_vpc.DetachNetworkInterface(ctx, &ec2.DetachNetworkInterfaceInput{ + AttachmentId: aws.String(attachmentID), + Force: aws.Bool(force), + }, r.nc, accountID); err != nil && !isAlreadyGone(err) { + return err + } + } + _, err = gateway_ec2_vpc.DeleteNetworkInterface(ctx, &ec2.DeleteNetworkInterfaceInput{ + NetworkInterfaceId: aws.String(resource.ID), + }, r.nc, accountID) + return ignoreAlreadyGone(err) +} + +// attachmentID reports the interface's current attachment, or empty when it is +// unattached. Unattached is the ordinary case here and is not an error. +func (r *networkInterfaceReaper) attachmentID(ctx context.Context, accountID, interfaceID string) (string, error) { + out, err := gateway_ec2_vpc.DescribeNetworkInterfaces(ctx, &ec2.DescribeNetworkInterfacesInput{ + NetworkInterfaceIds: []*string{aws.String(interfaceID)}, + }, r.nc, accountID) + if err != nil { + return "", err + } + for _, eni := range out.NetworkInterfaces { + if eni == nil || eni.NetworkInterfaceId == nil || *eni.NetworkInterfaceId != interfaceID { + continue + } + if eni.Attachment == nil { + return "", nil + } + return aws.StringValue(eni.Attachment.AttachmentId), nil + } + return "", nil +} + +type igwReaper struct{ nc *nats.Conn } + +func (r *igwReaper) Kind() string { return "internet-gateway" } +func (r *igwReaper) Stage() Stage { return StageNetwork } + +func (r *igwReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + out, err := gateway_ec2_igw.DescribeInternetGateways(ctx, &ec2.DescribeInternetGatewaysInput{}, r.nc, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, gateway := range out.InternetGateways { + if gateway == nil || gateway.InternetGatewayId == nil { + continue + } + resource := Resource{Kind: r.Kind(), ID: *gateway.InternetGatewayId} + for _, attachment := range gateway.Attachments { + if attachment != nil && attachment.VpcId != nil { + resource.Detail = *attachment.VpcId + break + } + } + found = append(found, resource) + } + return found, nil +} + +// Delete detaches first: an attached gateway cannot be deleted, and its VPC +// cannot be deleted while it is attached. +func (r *igwReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + if resource.Detail != "" { + if _, err := gateway_ec2_igw.DetachInternetGateway(ctx, &ec2.DetachInternetGatewayInput{ + InternetGatewayId: aws.String(resource.ID), + VpcId: aws.String(resource.Detail), + }, r.nc, accountID); err != nil && !isAlreadyGone(err) { + return err + } + } + _, err := gateway_ec2_igw.DeleteInternetGateway(ctx, &ec2.DeleteInternetGatewayInput{ + InternetGatewayId: aws.String(resource.ID), + }, r.nc, accountID) + return ignoreAlreadyGone(err) +} + +type routeTableReaper struct{ nc *nats.Conn } + +func (r *routeTableReaper) Kind() string { return "route-table" } +func (r *routeTableReaper) Stage() Stage { return StageNetwork } + +// List skips main route tables. A VPC's main table is deleted with the VPC and +// cannot be deleted on its own, so listing it would guarantee a stuck report. +func (r *routeTableReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + out, err := gateway_ec2_routetable.DescribeRouteTables(ctx, &ec2.DescribeRouteTablesInput{}, r.nc, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, table := range out.RouteTables { + if table == nil || table.RouteTableId == nil { + continue + } + var associations []string + main := false + for _, association := range table.Associations { + if association == nil { + continue + } + if association.Main != nil && *association.Main { + main = true + } + if association.RouteTableAssociationId != nil { + associations = append(associations, *association.RouteTableAssociationId) + } + } + if main { + continue + } + found = append(found, Resource{ + Kind: r.Kind(), ID: *table.RouteTableId, Detail: strings.Join(associations, ","), + }) + } + return found, nil +} + +func (r *routeTableReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + for association := range strings.SplitSeq(resource.Detail, ",") { + if association == "" { + continue + } + if _, err := gateway_ec2_routetable.DisassociateRouteTable(ctx, &ec2.DisassociateRouteTableInput{ + AssociationId: aws.String(association), + }, r.nc, accountID); err != nil && !isAlreadyGone(err) { + return err + } + } + _, err := gateway_ec2_routetable.DeleteRouteTable(ctx, &ec2.DeleteRouteTableInput{ + RouteTableId: aws.String(resource.ID), + }, r.nc, accountID) + return ignoreAlreadyGone(err) +} + +type subnetReaper struct{ nc *nats.Conn } + +func (r *subnetReaper) Kind() string { return "subnet" } +func (r *subnetReaper) Stage() Stage { return StageNetwork } + +func (r *subnetReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + out, err := gateway_ec2_vpc.DescribeSubnets(ctx, &ec2.DescribeSubnetsInput{}, r.nc, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, subnet := range out.Subnets { + if subnet == nil || subnet.SubnetId == nil { + continue + } + resource := Resource{Kind: r.Kind(), ID: *subnet.SubnetId} + if subnet.VpcId != nil { + resource.Detail = *subnet.VpcId + } + found = append(found, resource) + } + return found, nil +} + +func (r *subnetReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + _, err := gateway_ec2_vpc.DeleteSubnet(ctx, &ec2.DeleteSubnetInput{ + SubnetId: aws.String(resource.ID), + }, r.nc, accountID) + return ignoreAlreadyGone(err) +} + +type securityGroupReaper struct{ nc *nats.Conn } + +func (r *securityGroupReaper) Kind() string { return "security-group" } +func (r *securityGroupReaper) Stage() Stage { return StageNetwork } + +// List skips the default group of each VPC: it is created and destroyed with +// the VPC and cannot be deleted on its own. +func (r *securityGroupReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + out, err := gateway_ec2_vpc.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{}, r.nc, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, group := range out.SecurityGroups { + if group == nil || group.GroupId == nil { + continue + } + if group.GroupName != nil && *group.GroupName == "default" { + continue + } + found = append(found, Resource{Kind: r.Kind(), ID: *group.GroupId}) + } + return found, nil +} + +func (r *securityGroupReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + _, err := gateway_ec2_vpc.DeleteSecurityGroup(ctx, &ec2.DeleteSecurityGroupInput{ + GroupId: aws.String(resource.ID), + }, r.nc, accountID) + return ignoreAlreadyGone(err) +} + +type vpcReaper struct{ nc *nats.Conn } + +func (r *vpcReaper) Kind() string { return "vpc" } +func (r *vpcReaper) Stage() Stage { return StageNetwork } + +func (r *vpcReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + out, err := gateway_ec2_vpc.DescribeVpcs(ctx, &ec2.DescribeVpcsInput{}, r.nc, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, vpc := range out.Vpcs { + if vpc == nil || vpc.VpcId == nil { + continue + } + resource := Resource{Kind: r.Kind(), ID: *vpc.VpcId} + if vpc.IsDefault != nil && *vpc.IsDefault { + resource.Detail = "default" + } + found = append(found, resource) + } + return found, nil +} + +func (r *vpcReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + _, err := gateway_ec2_vpc.DeleteVpc(ctx, &ec2.DeleteVpcInput{ + VpcId: aws.String(resource.ID), + }, r.nc, accountID) + return ignoreAlreadyGone(err) +} + +type keyPairReaper struct{ nc *nats.Conn } + +func (r *keyPairReaper) Kind() string { return "key-pair" } +func (r *keyPairReaper) Stage() Stage { return StagePlatform } + +func (r *keyPairReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + out, err := gateway_ec2_key.DescribeKeyPairs(ctx, &ec2.DescribeKeyPairsInput{}, r.nc, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, pair := range out.KeyPairs { + if pair == nil || pair.KeyName == nil { + continue + } + found = append(found, Resource{Kind: r.Kind(), ID: *pair.KeyName}) + } + return found, nil +} + +func (r *keyPairReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + _, err := gateway_ec2_key.DeleteKeyPair(ctx, &ec2.DeleteKeyPairInput{ + KeyName: aws.String(resource.ID), + }, r.nc, accountID) + return ignoreAlreadyGone(err) +} + +type placementGroupReaper struct{ nc *nats.Conn } + +func (r *placementGroupReaper) Kind() string { return "placement-group" } +func (r *placementGroupReaper) Stage() Stage { return StagePlatform } + +func (r *placementGroupReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + out, err := gateway_ec2_placementgroup.DescribePlacementGroups(ctx, &ec2.DescribePlacementGroupsInput{}, r.nc, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, group := range out.PlacementGroups { + if group == nil || group.GroupName == nil { + continue + } + found = append(found, Resource{Kind: r.Kind(), ID: *group.GroupName}) + } + return found, nil +} + +func (r *placementGroupReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + _, err := gateway_ec2_placementgroup.DeletePlacementGroup(ctx, &ec2.DeletePlacementGroupInput{ + GroupName: aws.String(resource.ID), + }, r.nc, accountID) + return ignoreAlreadyGone(err) +} + +type launchTemplateReaper struct{ nc *nats.Conn } + +func (r *launchTemplateReaper) Kind() string { return "launch-template" } +func (r *launchTemplateReaper) Stage() Stage { return StagePlatform } + +func (r *launchTemplateReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + out, err := gateway_ec2_launchtemplate.DescribeLaunchTemplates(ctx, &ec2.DescribeLaunchTemplatesInput{}, r.nc, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, template := range out.LaunchTemplates { + if template == nil || template.LaunchTemplateId == nil { + continue + } + found = append(found, Resource{Kind: r.Kind(), ID: *template.LaunchTemplateId}) + } + return found, nil +} + +func (r *launchTemplateReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + _, err := gateway_ec2_launchtemplate.DeleteLaunchTemplate(ctx, &ec2.DeleteLaunchTemplateInput{ + LaunchTemplateId: aws.String(resource.ID), + }, r.nc, accountID) + return ignoreAlreadyGone(err) +} + +// notFoundMarkers are the AWS error-code fragments that mean the resource is +// already gone. Teardown re-runs after a crash, so treating these as failures +// would make a second pass unable to finish what the first one started. +var notFoundMarkers = []string{ + ".NotFound", "NoSuchEntity", "InvalidAllocationID.NotFound", + awserrors.ErrorECSClusterNotFound, awserrors.ErrorECSServiceNotFound, + awserrors.ErrorEKSResourceNotFound, + awserrors.ErrorDBInstanceNotFound, + awserrors.ErrorDBSubnetGroupNotFound, awserrors.ErrorDBParameterGroupNotFound, + awserrors.ErrorELBv2LoadBalancerNotFound, awserrors.ErrorELBv2TargetGroupNotFound, + awserrors.ErrorResourceNotFound, + s3.ErrCodeNoSuchBucket, s3.ErrCodeNoSuchKey, s3.ErrCodeNoSuchUpload, +} + +func isAlreadyGone(err error) bool { + if err == nil { + return true + } + message := err.Error() + for _, marker := range notFoundMarkers { + if strings.Contains(message, marker) { + return true + } + } + return false +} + +// isNotAssociated reports the benign case of disassociating something that was +// never associated, which is the normal state of an unattached address. +func isNotAssociated(err error) bool { + return err != nil && strings.Contains(err.Error(), "InvalidAssociationID") +} + +func ignoreAlreadyGone(err error) error { + if isAlreadyGone(err) { + return nil + } + return err +} + +// forceDetachSubject clears a volume's attachment in the control plane only. +// It is answered by any node, unlike the ordinary detach, which routes to the +// host running the instance and so cannot help when that host is the problem. +const forceDetachSubject = "ec2.ForceDetachVolume" + +// forceDetachTimeout bounds the metadata-only detach. It touches one document +// and needs nothing from the guest, so a slow answer means the cluster is +// unwell rather than that the work is large. +const forceDetachTimeout = 30 * time.Second + +// forceDeleteVolume clears the attachment the ordinary delete refuses to +// override, then deletes. +// +// This is the deadlock the force path exists for: a volume attached to an +// instance that will not terminate leaves both undeletable, and the account +// can never be emptied. +func forceDeleteVolume(ctx context.Context, nc *nats.Conn, expectedNodes int, accountID, volumeID string) error { + _, err := utils.NATSRequest[ec2.VolumeAttachment](ctx, nc, forceDetachSubject, + &ec2.DetachVolumeInput{VolumeId: aws.String(volumeID), Force: aws.Bool(true)}, + forceDetachTimeout, accountID) + if err != nil && !isAlreadyGone(err) { + return fmt.Errorf("force detach %s: %w", volumeID, err) + } + + _, err = gateway_ec2_volume.DeleteVolume(ctx, &ec2.DeleteVolumeInput{ + VolumeId: aws.String(volumeID), + }, nc, expectedNodes, accountID) + return ignoreAlreadyGone(err) +} diff --git a/spinifex/accountteardown/reapers_ec2_extra.go b/spinifex/accountteardown/reapers_ec2_extra.go new file mode 100644 index 000000000..dc46db2b0 --- /dev/null +++ b/spinifex/accountteardown/reapers_ec2_extra.go @@ -0,0 +1,160 @@ +package accountteardown + +import ( + "context" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eigw" + "github.com/mulgadc/spinifex/spinifex/handlers/ec2/natgw" + "github.com/mulgadc/spinifex/spinifex/handlers/ec2/spotinstance" + "github.com/nats-io/nats.go" +) + +// EC2ExtraReapers returns the EC2 reapers that hang off the VPC rather than off +// an instance. +// +// Spot requests come first and in the compute stage: a live request can launch +// a replacement after the instance reaper has drained, so leaving one running +// makes teardown race its own fulfilment rather than merely leak. +func EC2ExtraReapers(nc *nats.Conn) []Reaper { + return []Reaper{ + &spotRequestReaper{svc: handlers_ec2_spotinstance.NewNATSSpotInstanceService(nc)}, + &natGatewayReaper{svc: handlers_ec2_natgw.NewNATSNatGatewayService(nc)}, + &egressOnlyIGWReaper{svc: handlers_ec2_eigw.NewNATSEgressOnlyIGWService(nc)}, + } +} + +// terminalSpotStates are the states a request can no longer launch from. +// Anything else may still produce an instance. +var terminalSpotStates = map[string]bool{"cancelled": true, "closed": true, "failed": true} + +type spotRequestReaper struct { + svc handlers_ec2_spotinstance.SpotInstanceService +} + +func (r *spotRequestReaper) Kind() string { return "spot-request" } +func (r *spotRequestReaper) Stage() Stage { return StageCompute } + +func (r *spotRequestReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + out, err := r.svc.DescribeSpotInstanceRequests(ctx, &ec2.DescribeSpotInstanceRequestsInput{}, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, request := range out.SpotInstanceRequests { + if request == nil || request.SpotInstanceRequestId == nil { + continue + } + state := aws.StringValue(request.State) + if terminalSpotStates[state] { + continue + } + found = append(found, Resource{ + Kind: r.Kind(), + ID: aws.StringValue(request.SpotInstanceRequestId), + Detail: state, + }) + } + return found, nil +} + +func (r *spotRequestReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + _, err := r.svc.CancelSpotInstanceRequests(ctx, &ec2.CancelSpotInstanceRequestsInput{ + SpotInstanceRequestIds: []*string{aws.String(resource.ID)}, + }, accountID) + return ignoreAlreadyGone(err) +} + +// deletedNatGatewayStates are the states a NAT gateway has already released its +// address and its subnet placement in. +var deletedNatGatewayStates = map[string]bool{"deleted": true} + +type natGatewayReaper struct { + svc handlers_ec2_natgw.NatGatewayService +} + +func (r *natGatewayReaper) Kind() string { return "nat-gateway" } +func (r *natGatewayReaper) Stage() Stage { return StageNetwork } + +// List keeps a gateway that is merely deleting, because it still holds its EIP +// and its place in the subnet until it is actually gone — and those are the two +// things the reapers after it are waiting on. +func (r *natGatewayReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + out, err := r.svc.DescribeNatGateways(ctx, &ec2.DescribeNatGatewaysInput{}, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, gateway := range out.NatGateways { + if gateway == nil || gateway.NatGatewayId == nil { + continue + } + state := aws.StringValue(gateway.State) + if deletedNatGatewayStates[state] { + continue + } + found = append(found, Resource{ + Kind: r.Kind(), + ID: aws.StringValue(gateway.NatGatewayId), + Detail: natGatewayDetail(gateway, state), + }) + } + return found, nil +} + +// natGatewayDetail names the subnet, because a subnet that will not delete with +// a NAT gateway still in it is the case an operator has to be able to read. +func natGatewayDetail(gateway *ec2.NatGateway, state string) string { + subnet := aws.StringValue(gateway.SubnetId) + if subnet == "" { + return state + } + return state + " in " + subnet +} + +func (r *natGatewayReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + _, err := r.svc.DeleteNatGateway(ctx, &ec2.DeleteNatGatewayInput{ + NatGatewayId: aws.String(resource.ID), + }, accountID) + return ignoreAlreadyGone(err) +} + +type egressOnlyIGWReaper struct { + svc handlers_ec2_eigw.EgressOnlyIGWService +} + +func (r *egressOnlyIGWReaper) Kind() string { return "egress-only-internet-gateway" } +func (r *egressOnlyIGWReaper) Stage() Stage { return StageNetwork } + +func (r *egressOnlyIGWReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + out, err := r.svc.DescribeEgressOnlyInternetGateways(ctx, &ec2.DescribeEgressOnlyInternetGatewaysInput{}, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, gateway := range out.EgressOnlyInternetGateways { + if gateway == nil || gateway.EgressOnlyInternetGatewayId == nil { + continue + } + resource := Resource{Kind: r.Kind(), ID: aws.StringValue(gateway.EgressOnlyInternetGatewayId)} + for _, attachment := range gateway.Attachments { + if attachment != nil && aws.StringValue(attachment.VpcId) != "" { + resource.Detail = aws.StringValue(attachment.VpcId) + break + } + } + found = append(found, resource) + } + return found, nil +} + +func (r *egressOnlyIGWReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + _, err := r.svc.DeleteEgressOnlyInternetGateway(ctx, &ec2.DeleteEgressOnlyInternetGatewayInput{ + EgressOnlyInternetGatewayId: aws.String(resource.ID), + }, accountID) + return ignoreAlreadyGone(err) +} diff --git a/spinifex/accountteardown/reapers_ec2_extra_test.go b/spinifex/accountteardown/reapers_ec2_extra_test.go new file mode 100644 index 000000000..cb93cb107 --- /dev/null +++ b/spinifex/accountteardown/reapers_ec2_extra_test.go @@ -0,0 +1,177 @@ +package accountteardown + +//test:in-package — the reapers and the states they choose to skip are +// unexported, and which states they skip is the substance of them. + +import ( + "context" + "errors" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + handlers_ec2_eigw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eigw" + handlers_ec2_natgw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/natgw" + handlers_ec2_spotinstance "github.com/mulgadc/spinifex/spinifex/handlers/ec2/spotinstance" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The embedded interfaces are deliberately nil: a reaper that calls anything +// beyond the two methods it is supposed to use panics rather than passing. +type fakeSpot struct { + handlers_ec2_spotinstance.SpotInstanceService + + requests []*ec2.SpotInstanceRequest + cancelled []string + err error +} + +func (f *fakeSpot) DescribeSpotInstanceRequests(_ context.Context, _ *ec2.DescribeSpotInstanceRequestsInput, _ string) (*ec2.DescribeSpotInstanceRequestsOutput, error) { + return &ec2.DescribeSpotInstanceRequestsOutput{SpotInstanceRequests: f.requests}, nil +} + +func (f *fakeSpot) CancelSpotInstanceRequests(_ context.Context, in *ec2.CancelSpotInstanceRequestsInput, _ string) (*ec2.CancelSpotInstanceRequestsOutput, error) { + if f.err != nil { + return nil, f.err + } + for _, id := range in.SpotInstanceRequestIds { + f.cancelled = append(f.cancelled, aws.StringValue(id)) + } + return &ec2.CancelSpotInstanceRequestsOutput{}, nil +} + +type fakeNatGateways struct { + handlers_ec2_natgw.NatGatewayService + + gateways []*ec2.NatGateway + deleted []string + err error +} + +func (f *fakeNatGateways) DescribeNatGateways(_ context.Context, _ *ec2.DescribeNatGatewaysInput, _ string) (*ec2.DescribeNatGatewaysOutput, error) { + return &ec2.DescribeNatGatewaysOutput{NatGateways: f.gateways}, nil +} + +func (f *fakeNatGateways) DeleteNatGateway(_ context.Context, in *ec2.DeleteNatGatewayInput, _ string) (*ec2.DeleteNatGatewayOutput, error) { + if f.err != nil { + return nil, f.err + } + f.deleted = append(f.deleted, aws.StringValue(in.NatGatewayId)) + return &ec2.DeleteNatGatewayOutput{}, nil +} + +type fakeEgressGateways struct { + handlers_ec2_eigw.EgressOnlyIGWService + + gateways []*ec2.EgressOnlyInternetGateway + deleted []string + err error +} + +func (f *fakeEgressGateways) DescribeEgressOnlyInternetGateways(_ context.Context, _ *ec2.DescribeEgressOnlyInternetGatewaysInput, _ string) (*ec2.DescribeEgressOnlyInternetGatewaysOutput, error) { + return &ec2.DescribeEgressOnlyInternetGatewaysOutput{EgressOnlyInternetGateways: f.gateways}, nil +} + +func (f *fakeEgressGateways) DeleteEgressOnlyInternetGateway(_ context.Context, in *ec2.DeleteEgressOnlyInternetGatewayInput, _ string) (*ec2.DeleteEgressOnlyInternetGatewayOutput, error) { + if f.err != nil { + return nil, f.err + } + f.deleted = append(f.deleted, aws.StringValue(in.EgressOnlyInternetGatewayId)) + return &ec2.DeleteEgressOnlyInternetGatewayOutput{}, nil +} + +// A cancelled request cannot launch anything, so listing it would leave the +// stage waiting on a row that never clears — spot requests are never removed +// from the account, only moved to a terminal state. +func TestSpotRequestReaperSkipsTerminalRequests(t *testing.T) { + svc := &fakeSpot{requests: []*ec2.SpotInstanceRequest{ + {SpotInstanceRequestId: aws.String("sir-open"), State: aws.String("open")}, + {SpotInstanceRequestId: aws.String("sir-active"), State: aws.String("active")}, + {SpotInstanceRequestId: aws.String("sir-cancelled"), State: aws.String("cancelled")}, + {SpotInstanceRequestId: aws.String("sir-closed"), State: aws.String("closed")}, + {SpotInstanceRequestId: aws.String("sir-failed"), State: aws.String("failed")}, + }} + reaper := &spotRequestReaper{svc: svc} + + assert.Equal(t, StageCompute, reaper.Stage()) + + found, err := reaper.List(t.Context(), "000000000002") + require.NoError(t, err) + require.Len(t, found, 2) + assert.Equal(t, "sir-open", found[0].ID) + assert.Equal(t, "open", found[0].Detail) + assert.Equal(t, "sir-active", found[1].ID) + + require.NoError(t, reaper.Delete(t.Context(), "000000000002", found[0], false)) + assert.Equal(t, []string{"sir-open"}, svc.cancelled) +} + +// A request cancelled by someone else between the listing and the delete is +// the ordinary second-pass case, not a failure. +func TestSpotRequestReaperTreatsAMissingRequestAsCancelled(t *testing.T) { + svc := &fakeSpot{err: errors.New("InvalidSpotInstanceRequestID.NotFound: no such request")} + reaper := &spotRequestReaper{svc: svc} + + err := reaper.Delete(t.Context(), "000000000002", Resource{ID: "sir-gone"}, false) + assert.NoError(t, err) +} + +// A deleting gateway still holds its address and its subnet, so it has to keep +// listing until it is actually gone or the address reaper runs too early. +func TestNatGatewayReaperKeepsDeletingGateways(t *testing.T) { + svc := &fakeNatGateways{gateways: []*ec2.NatGateway{ + {NatGatewayId: aws.String("nat-live"), State: aws.String("available"), SubnetId: aws.String("subnet-a")}, + {NatGatewayId: aws.String("nat-going"), State: aws.String("deleting"), SubnetId: aws.String("subnet-b")}, + {NatGatewayId: aws.String("nat-done"), State: aws.String("deleted"), SubnetId: aws.String("subnet-c")}, + }} + reaper := &natGatewayReaper{svc: svc} + + assert.Equal(t, StageNetwork, reaper.Stage()) + + found, err := reaper.List(t.Context(), "000000000002") + require.NoError(t, err) + require.Len(t, found, 2) + assert.Equal(t, "nat-live", found[0].ID) + assert.Equal(t, "available in subnet-a", found[0].Detail) + assert.Equal(t, "nat-going", found[1].ID) + + require.NoError(t, reaper.Delete(t.Context(), "000000000002", found[0], false)) + assert.Equal(t, []string{"nat-live"}, svc.deleted) +} + +func TestNatGatewayReaperTreatsAMissingGatewayAsDeleted(t *testing.T) { + svc := &fakeNatGateways{err: errors.New("InvalidNatGatewayID.NotFound: no such gateway")} + reaper := &natGatewayReaper{svc: svc} + + assert.NoError(t, reaper.Delete(t.Context(), "000000000002", Resource{ID: "nat-gone"}, false)) +} + +func TestEgressOnlyIGWReaperCarriesItsVPC(t *testing.T) { + svc := &fakeEgressGateways{gateways: []*ec2.EgressOnlyInternetGateway{ + { + EgressOnlyInternetGatewayId: aws.String("eigw-a"), + Attachments: []*ec2.InternetGatewayAttachment{{VpcId: aws.String("vpc-1")}}, + }, + {EgressOnlyInternetGatewayId: aws.String("eigw-detached")}, + }} + reaper := &egressOnlyIGWReaper{svc: svc} + + assert.Equal(t, StageNetwork, reaper.Stage()) + + found, err := reaper.List(t.Context(), "000000000002") + require.NoError(t, err) + require.Len(t, found, 2) + assert.Equal(t, "vpc-1", found[0].Detail) + assert.Empty(t, found[1].Detail) + + require.NoError(t, reaper.Delete(t.Context(), "000000000002", found[0], false)) + assert.Equal(t, []string{"eigw-a"}, svc.deleted) +} + +func TestEgressOnlyIGWReaperTreatsAMissingGatewayAsDeleted(t *testing.T) { + svc := &fakeEgressGateways{err: errors.New("InvalidGatewayID.NotFound: no such gateway")} + reaper := &egressOnlyIGWReaper{svc: svc} + + assert.NoError(t, reaper.Delete(t.Context(), "000000000002", Resource{ID: "eigw-gone"}, false)) +} diff --git a/spinifex/accountteardown/reapers_ec2_test.go b/spinifex/accountteardown/reapers_ec2_test.go new file mode 100644 index 000000000..ada376331 --- /dev/null +++ b/spinifex/accountteardown/reapers_ec2_test.go @@ -0,0 +1,674 @@ +package accountteardown + +//test:in-package — the reapers are deliberately unexported so no ordinary +// request path can reach a force delete, and their listing filters and delete +// ordering are the whole substance of the package. + +import ( + "context" + "encoding/json" + "sync" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeCluster answers the EC2 subjects the reapers drive, recording the order +// they arrive in. Detach-before-delete and disassociate-before-release are +// ordering rules, so a double that only recorded a set would not test them. +type fakeCluster struct { + nc *nats.Conn + + mu sync.Mutex + calls []string + requests map[string][]byte + replies map[string]any + errorCode map[string]string +} + +func newFakeCluster(t *testing.T) *fakeCluster { + t.Helper() + _, nc := testutil.StartTestNATS(t) + + cluster := &fakeCluster{ + nc: nc, + requests: map[string][]byte{}, + replies: map[string]any{}, + errorCode: map[string]string{}, + } + + sub, err := nc.Subscribe("ec2.>", cluster.serve) + require.NoError(t, err) + t.Cleanup(func() { _ = sub.Unsubscribe() }) + require.NoError(t, nc.Flush()) + + return cluster +} + +func (c *fakeCluster) serve(msg *nats.Msg) { + c.mu.Lock() + c.calls = append(c.calls, msg.Subject) + c.requests[msg.Subject] = msg.Data + code, isError := c.errorCode[msg.Subject] + reply, hasReply := c.replies[msg.Subject] + c.mu.Unlock() + + switch { + case isError: + _ = msg.Respond(utils.GenerateErrorPayload(code)) + case hasReply: + payload, err := json.Marshal(reply) + if err != nil { + _ = msg.Respond(utils.GenerateErrorPayload("InternalError")) + return + } + _ = msg.Respond(payload) + default: + _ = msg.Respond([]byte(`{}`)) + } +} + +func (c *fakeCluster) reply(subject string, output any) { + c.mu.Lock() + defer c.mu.Unlock() + c.replies[subject] = output +} + +func (c *fakeCluster) fail(subject, code string) { + c.mu.Lock() + defer c.mu.Unlock() + c.errorCode[subject] = code +} + +func (c *fakeCluster) called() []string { + c.mu.Lock() + defer c.mu.Unlock() + return append([]string(nil), c.calls...) +} + +func (c *fakeCluster) request(t *testing.T, subject string, into any) { + t.Helper() + c.mu.Lock() + payload, ok := c.requests[subject] + c.mu.Unlock() + require.True(t, ok, "subject %s was never called", subject) + require.NoError(t, json.Unmarshal(payload, into)) +} + +func testCtx(t *testing.T) context.Context { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + t.Cleanup(cancel) + return ctx +} + +// The stage graph is what keeps teardown converging, so it is asserted rather +// than left to the order the constructor happens to list. +func TestEC2ReapersAreOrderedByDependency(t *testing.T) { + reapers := EC2Reapers(nil, 1) + + var kinds []string + for _, reaper := range reapers { + kinds = append(kinds, reaper.Kind()) + } + + assert.Equal(t, []string{ + "instance", + "snapshot", "image", "volume", + "address", "network-interface", "internet-gateway", "route-table", "subnet", "security-group", "vpc", + "key-pair", "placement-group", "launch-template", + }, kinds) + + // A volume with a live snapshot refuses to delete, so the reverse order + // would strand every snapshot-backed volume in the account. + assert.Less(t, indexOfKind(reapers, "snapshot"), indexOfKind(reapers, "volume")) + assert.Equal(t, StageCompute, reapers[indexOfKind(reapers, "instance")].Stage()) + assert.Equal(t, StageStorage, reapers[indexOfKind(reapers, "volume")].Stage()) + assert.Equal(t, StageNetwork, reapers[indexOfKind(reapers, "vpc")].Stage()) + assert.Equal(t, StagePlatform, reapers[indexOfKind(reapers, "key-pair")].Stage()) +} + +func indexOfKind(reapers []Reaper, kind string) int { + for i, reaper := range reapers { + if reaper.Kind() == kind { + return i + } + } + return -1 +} + +// A terminated instance holds nothing. Listing it would keep the compute stage +// polling until its timeout and then report it stuck, for no reason. +func TestInstanceReaperSkipsTerminalStates(t *testing.T) { + cluster := newFakeCluster(t) + cluster.reply("ec2.DescribeInstances", ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{{Instances: []*ec2.Instance{ + {InstanceId: aws.String("i-running"), State: &ec2.InstanceState{Name: aws.String("running")}}, + {InstanceId: aws.String("i-gone"), State: &ec2.InstanceState{Name: aws.String("terminated")}}, + {InstanceId: aws.String("i-going"), State: &ec2.InstanceState{Name: aws.String("shutting-down")}}, + {State: &ec2.InstanceState{Name: aws.String("running")}}, + }}}, + }) + + reaper := &instanceReaper{nc: cluster.nc, expectedNodes: 1} + found, err := reaper.List(testCtx(t), "000000000042") + require.NoError(t, err) + + require.Len(t, found, 1) + assert.Equal(t, "i-running", found[0].ID) + assert.Equal(t, "running", found[0].Detail) +} + +// Termination is addressed to the node hosting the instance, which is what +// makes a stuck host the case --force exists for. +func TestInstanceReaperTerminates(t *testing.T) { + cluster := newFakeCluster(t) + reaper := &instanceReaper{nc: cluster.nc, expectedNodes: 1} + + require.NoError(t, reaper.Delete(testCtx(t), "000000000042", + Resource{Kind: "instance", ID: "i-abc"}, false)) + + assert.Contains(t, cluster.called(), "ec2.cmd.i-abc") +} + +// The attachment is the reason a volume will not go, so it has to reach the +// stuck report — "stuck" and "stuck on i-abc" are different bug reports. +func TestVolumeReaperCarriesItsAttachment(t *testing.T) { + cluster := newFakeCluster(t) + cluster.reply("ec2.DescribeVolumes", ec2.DescribeVolumesOutput{ + Volumes: []*ec2.Volume{ + {VolumeId: aws.String("vol-free")}, + {VolumeId: aws.String("vol-held"), Attachments: []*ec2.VolumeAttachment{ + {InstanceId: aws.String("i-abc")}, + }}, + }, + }) + + reaper := &volumeReaper{nc: cluster.nc, expectedNodes: 1} + found, err := reaper.List(testCtx(t), "000000000042") + require.NoError(t, err) + + require.Len(t, found, 2) + assert.Empty(t, found[0].Detail) + assert.Equal(t, "attached to i-abc", found[1].Detail) +} + +// Without --force an attached volume must surface the refusal, not swallow it: +// a stuck resource is a bug we want reported rather than papered over. +func TestVolumeReaperReportsRefusalWithoutForce(t *testing.T) { + cluster := newFakeCluster(t) + cluster.fail("ec2.DeleteVolume", "VolumeInUse") + + reaper := &volumeReaper{nc: cluster.nc, expectedNodes: 1} + err := reaper.Delete(testCtx(t), "000000000042", Resource{Kind: "volume", ID: "vol-held"}, false) + + require.Error(t, err) + assert.NotContains(t, cluster.called(), forceDetachSubject) +} + +// The deadlock the force path exists for: the ordinary delete refuses, so the +// attachment is cleared cluster-wide and the delete retried. +func TestVolumeReaperForceClearsTheAttachment(t *testing.T) { + cluster := newFakeCluster(t) + cluster.fail("ec2.DeleteVolume", "VolumeInUse") + + reaper := &volumeReaper{nc: cluster.nc, expectedNodes: 1} + err := reaper.Delete(testCtx(t), "000000000042", Resource{Kind: "volume", ID: "vol-held"}, true) + + // The delete still fails here because the double refuses every attempt; + // what matters is that force reached for the detach at all. + require.Error(t, err) + assert.Contains(t, cluster.called(), forceDetachSubject) + + var input ec2.DetachVolumeInput + cluster.request(t, forceDetachSubject, &input) + assert.Equal(t, "vol-held", aws.StringValue(input.VolumeId)) + assert.True(t, aws.BoolValue(input.Force)) +} + +// A volume that is already gone is a success. Teardown re-runs after a crash +// and would otherwise never finish what the first pass started. +func TestVolumeReaperTreatsAMissingVolumeAsDeleted(t *testing.T) { + cluster := newFakeCluster(t) + cluster.fail("ec2.DeleteVolume", "InvalidVolume.NotFound") + + reaper := &volumeReaper{nc: cluster.nc, expectedNodes: 1} + + assert.NoError(t, reaper.Delete(testCtx(t), "000000000042", + Resource{Kind: "volume", ID: "vol-gone"}, false)) +} + +// Owner-scoped: an image the account can merely see is not its property and +// must survive its deletion. +func TestImageReaperListsOnlyOwnedImages(t *testing.T) { + cluster := newFakeCluster(t) + cluster.reply("ec2.DescribeImages", ec2.DescribeImagesOutput{ + Images: []*ec2.Image{{ImageId: aws.String("ami-owned")}, {}}, + }) + + reaper := &imageReaper{nc: cluster.nc} + found, err := reaper.List(testCtx(t), "000000000042") + require.NoError(t, err) + + require.Len(t, found, 1) + assert.Equal(t, "ami-owned", found[0].ID) + + var input ec2.DescribeImagesInput + cluster.request(t, "ec2.DescribeImages", &input) + require.Len(t, input.Owners, 1) + assert.Equal(t, "000000000042", aws.StringValue(input.Owners[0])) +} + +func TestSnapshotReaperListsOnlyOwnedSnapshots(t *testing.T) { + cluster := newFakeCluster(t) + cluster.reply("ec2.DescribeSnapshots", ec2.DescribeSnapshotsOutput{ + Snapshots: []*ec2.Snapshot{{SnapshotId: aws.String("snap-1")}, {}}, + }) + + reaper := &snapshotReaper{nc: cluster.nc} + found, err := reaper.List(testCtx(t), "000000000042") + require.NoError(t, err) + + require.Len(t, found, 1) + assert.Equal(t, "snap-1", found[0].ID) + + var input ec2.DescribeSnapshotsInput + cluster.request(t, "ec2.DescribeSnapshots", &input) + require.Len(t, input.OwnerIds, 1) + assert.Equal(t, "000000000042", aws.StringValue(input.OwnerIds[0])) +} + +// An associated address cannot be released, and the association routinely +// outlives the instance by a moment. +func TestAddressReaperDisassociatesBeforeReleasing(t *testing.T) { + cluster := newFakeCluster(t) + cluster.reply("ec2.DescribeAddresses", ec2.DescribeAddressesOutput{ + Addresses: []*ec2.Address{ + { + AllocationId: aws.String("eipalloc-1"), + AssociationId: aws.String("eipassoc-1"), + PublicIp: aws.String("203.0.113.7"), + }, + {PublicIp: aws.String("203.0.113.8")}, + }, + }) + + reaper := &addressReaper{nc: cluster.nc} + found, err := reaper.List(testCtx(t), "000000000042") + require.NoError(t, err) + require.Len(t, found, 1) + assert.Equal(t, "203.0.113.7", found[0].Detail) + + require.NoError(t, reaper.Delete(testCtx(t), "000000000042", found[0], false)) + assert.Equal(t, []string{ + "ec2.DescribeAddresses", "ec2.DescribeAddresses", + "ec2.DisassociateAddress", "ec2.ReleaseAddress", + }, cluster.called()) +} + +// The allocation id is not the association id. Sending the former is answered +// InvalidAssociationID.NotFound, which reads as never-associated, so the +// address stays associated and the release is refused. +func TestAddressReaperDisassociatesByAssociationID(t *testing.T) { + cluster := newFakeCluster(t) + cluster.reply("ec2.DescribeAddresses", ec2.DescribeAddressesOutput{ + Addresses: []*ec2.Address{{ + AllocationId: aws.String("eipalloc-1"), + AssociationId: aws.String("eipassoc-1"), + PublicIp: aws.String("203.0.113.7"), + }}, + }) + + reaper := &addressReaper{nc: cluster.nc} + require.NoError(t, reaper.Delete(testCtx(t), "000000000042", + Resource{Kind: "address", ID: "eipalloc-1", Detail: "203.0.113.7"}, false)) + + var disassociate ec2.DisassociateAddressInput + cluster.request(t, "ec2.DisassociateAddress", &disassociate) + assert.Equal(t, "eipassoc-1", aws.StringValue(disassociate.AssociationId)) + + var describe ec2.DescribeAddressesInput + cluster.request(t, "ec2.DescribeAddresses", &describe) + require.Len(t, describe.AllocationIds, 1) + assert.Equal(t, "eipalloc-1", aws.StringValue(describe.AllocationIds[0])) +} + +// An address that was never associated is the normal case, not a failure. It +// has nothing to disassociate, so the call is skipped rather than made and +// forgiven — a real error from it should still stop the release. +func TestAddressReaperReleasesAnUnassociatedAddress(t *testing.T) { + cluster := newFakeCluster(t) + cluster.reply("ec2.DescribeAddresses", ec2.DescribeAddressesOutput{ + Addresses: []*ec2.Address{{AllocationId: aws.String("eipalloc-1"), PublicIp: aws.String("203.0.113.7")}}, + }) + + reaper := &addressReaper{nc: cluster.nc} + err := reaper.Delete(testCtx(t), "000000000042", Resource{Kind: "address", ID: "eipalloc-1"}, false) + + require.NoError(t, err) + assert.Equal(t, []string{"ec2.DescribeAddresses", "ec2.ReleaseAddress"}, cluster.called()) +} + +// An address already gone by the time its stage runs is a success: teardown +// re-runs after a crash and would otherwise never finish. +func TestAddressReaperTreatsAMissingAddressAsDeleted(t *testing.T) { + cluster := newFakeCluster(t) + cluster.fail("ec2.DescribeAddresses", "InvalidAllocationID.NotFound") + cluster.fail("ec2.ReleaseAddress", "InvalidAllocationID.NotFound") + + reaper := &addressReaper{nc: cluster.nc} + err := reaper.Delete(testCtx(t), "000000000042", Resource{Kind: "address", ID: "eipalloc-1"}, false) + + require.NoError(t, err) + assert.Equal(t, []string{"ec2.DescribeAddresses", "ec2.ReleaseAddress"}, cluster.called()) +} + +// An attached gateway cannot be deleted, and its VPC cannot be deleted while +// it is attached — so the attachment is carried from the listing to the delete. +func TestIGWReaperDetachesBeforeDeleting(t *testing.T) { + cluster := newFakeCluster(t) + cluster.reply("ec2.DescribeInternetGateways", ec2.DescribeInternetGatewaysOutput{ + InternetGateways: []*ec2.InternetGateway{{ + InternetGatewayId: aws.String("igw-1"), + Attachments: []*ec2.InternetGatewayAttachment{{VpcId: aws.String("vpc-1")}}, + }}, + }) + + reaper := &igwReaper{nc: cluster.nc} + found, err := reaper.List(testCtx(t), "000000000042") + require.NoError(t, err) + require.Len(t, found, 1) + assert.Equal(t, "vpc-1", found[0].Detail) + + require.NoError(t, reaper.Delete(testCtx(t), "000000000042", found[0], false)) + assert.Equal(t, []string{ + "ec2.DescribeInternetGateways", "ec2.DetachInternetGateway", "ec2.DeleteInternetGateway", + }, cluster.called()) +} + +// A detached gateway needs no detach call, and issuing one would fail. +func TestIGWReaperDeletesADetachedGatewayDirectly(t *testing.T) { + cluster := newFakeCluster(t) + + reaper := &igwReaper{nc: cluster.nc} + require.NoError(t, reaper.Delete(testCtx(t), "000000000042", + Resource{Kind: "internet-gateway", ID: "igw-1"}, false)) + + assert.Equal(t, []string{"ec2.DeleteInternetGateway"}, cluster.called()) +} + +// A VPC's main route table is deleted with the VPC and cannot be deleted on +// its own, so listing it would guarantee a stuck report every time. +func TestRouteTableReaperSkipsTheMainTable(t *testing.T) { + cluster := newFakeCluster(t) + cluster.reply("ec2.DescribeRouteTables", ec2.DescribeRouteTablesOutput{ + RouteTables: []*ec2.RouteTable{ + { + RouteTableId: aws.String("rtb-main"), + Associations: []*ec2.RouteTableAssociation{{Main: aws.Bool(true)}}, + }, + { + RouteTableId: aws.String("rtb-custom"), + Associations: []*ec2.RouteTableAssociation{ + {RouteTableAssociationId: aws.String("rtbassoc-1")}, + {RouteTableAssociationId: aws.String("rtbassoc-2")}, + nil, + }, + }, + }, + }) + + reaper := &routeTableReaper{nc: cluster.nc} + found, err := reaper.List(testCtx(t), "000000000042") + require.NoError(t, err) + + require.Len(t, found, 1) + assert.Equal(t, "rtb-custom", found[0].ID) + assert.Equal(t, "rtbassoc-1,rtbassoc-2", found[0].Detail) + + require.NoError(t, reaper.Delete(testCtx(t), "000000000042", found[0], false)) + assert.Equal(t, []string{ + "ec2.DescribeRouteTables", + "ec2.DisassociateRouteTable", "ec2.DisassociateRouteTable", + "ec2.DeleteRouteTable", + }, cluster.called()) +} + +// The default group is created and destroyed with its VPC and cannot be +// deleted on its own. +func TestSecurityGroupReaperSkipsTheDefaultGroup(t *testing.T) { + cluster := newFakeCluster(t) + cluster.reply("ec2.DescribeSecurityGroups", ec2.DescribeSecurityGroupsOutput{ + SecurityGroups: []*ec2.SecurityGroup{ + {GroupId: aws.String("sg-default"), GroupName: aws.String("default")}, + {GroupId: aws.String("sg-app"), GroupName: aws.String("app")}, + {GroupName: aws.String("no-id")}, + }, + }) + + reaper := &securityGroupReaper{nc: cluster.nc} + found, err := reaper.List(testCtx(t), "000000000042") + require.NoError(t, err) + + require.Len(t, found, 1) + assert.Equal(t, "sg-app", found[0].ID) +} + +func TestSubnetAndVPCReapersReportTheirVPC(t *testing.T) { + cluster := newFakeCluster(t) + cluster.reply("ec2.DescribeSubnets", ec2.DescribeSubnetsOutput{ + Subnets: []*ec2.Subnet{{SubnetId: aws.String("subnet-1"), VpcId: aws.String("vpc-1")}, {}}, + }) + cluster.reply("ec2.DescribeVpcs", ec2.DescribeVpcsOutput{ + Vpcs: []*ec2.Vpc{ + {VpcId: aws.String("vpc-1"), IsDefault: aws.Bool(true)}, + {VpcId: aws.String("vpc-2")}, + {}, + }, + }) + + subnets, err := (&subnetReaper{nc: cluster.nc}).List(testCtx(t), "000000000042") + require.NoError(t, err) + require.Len(t, subnets, 1) + assert.Equal(t, "vpc-1", subnets[0].Detail) + + vpcs, err := (&vpcReaper{nc: cluster.nc}).List(testCtx(t), "000000000042") + require.NoError(t, err) + require.Len(t, vpcs, 2) + // The default VPC is deleted with the account like any other: an account + // being torn down has no use for one. + assert.Equal(t, "default", vpcs[0].Detail) + assert.Empty(t, vpcs[1].Detail) +} + +func TestPlatformReapersListAndDelete(t *testing.T) { + cluster := newFakeCluster(t) + cluster.reply("ec2.DescribeKeyPairs", ec2.DescribeKeyPairsOutput{ + KeyPairs: []*ec2.KeyPairInfo{{KeyName: aws.String("deploy")}, {}}, + }) + cluster.reply("ec2.DescribePlacementGroups", ec2.DescribePlacementGroupsOutput{ + PlacementGroups: []*ec2.PlacementGroup{{GroupName: aws.String("spread")}, {}}, + }) + cluster.reply("ec2.DescribeLaunchTemplates", ec2.DescribeLaunchTemplatesOutput{ + LaunchTemplates: []*ec2.LaunchTemplate{{LaunchTemplateId: aws.String("lt-1")}, {}}, + }) + + ctx := testCtx(t) + for _, tc := range []struct { + reaper Reaper + wantID string + }{ + {&keyPairReaper{nc: cluster.nc}, "deploy"}, + {&placementGroupReaper{nc: cluster.nc}, "spread"}, + {&launchTemplateReaper{nc: cluster.nc}, "lt-1"}, + } { + t.Run(tc.reaper.Kind(), func(t *testing.T) { + found, err := tc.reaper.List(ctx, "000000000042") + require.NoError(t, err) + require.Len(t, found, 1) + assert.Equal(t, tc.wantID, found[0].ID) + assert.NoError(t, tc.reaper.Delete(ctx, "000000000042", found[0], false)) + }) + } +} + +// A listing that fails must abort the teardown. Reading a failed listing as an +// empty account is how a stage reports itself drained while resources remain. +func TestAListingFailureIsReported(t *testing.T) { + cluster := newFakeCluster(t) + cluster.fail("ec2.DescribeVolumes", "InternalError") + + _, err := (&volumeReaper{nc: cluster.nc, expectedNodes: 1}).List(testCtx(t), "000000000042") + + assert.Error(t, err) +} + +// Teardown re-runs after a crash, so an already-missing resource is a success. +// The customer API fails closed on these, which is right for a customer and +// wrong for a reaper. +func TestAlreadyGoneRecognisesMissingResources(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + {"nil is gone", nil, true}, + {"ec2 not found", assertError("InvalidVolume.NotFound"), true}, + {"iam not found", assertError("NoSuchEntity: no such user"), true}, + {"allocation not found", assertError("InvalidAllocationID.NotFound"), true}, + {"still in use", assertError("VolumeInUse"), false}, + {"internal error", assertError("InternalError"), false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, isAlreadyGone(tc.err)) + if tc.want { + assert.NoError(t, ignoreAlreadyGone(tc.err)) + } else { + assert.Error(t, ignoreAlreadyGone(tc.err)) + } + }) + } +} + +func TestNotAssociatedIsBenign(t *testing.T) { + assert.True(t, isNotAssociated(assertError("InvalidAssociationID.NotFound"))) + assert.False(t, isNotAssociated(assertError("AuthFailure"))) + assert.False(t, isNotAssociated(nil)) +} + +type stringError string + +func (e stringError) Error() string { return string(e) } + +func assertError(message string) error { return stringError(message) } + +// An interface still attached cannot be deleted, and an interface that will not +// delete holds its subnet, which holds the VPC, which leaves the account in +// TERMINATING for good. +// +// The detach is dispatched to the owning instance as a command, not as its own +// EC2 call, so ec2.cmd. is what proves it was driven. +func TestNetworkInterfaceReaperDetachesBeforeDeleting(t *testing.T) { + cluster := newFakeCluster(t) + cluster.reply("ec2.DescribeNetworkInterfaces", ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []*ec2.NetworkInterface{{ + NetworkInterfaceId: aws.String("eni-1"), + SubnetId: aws.String("subnet-1"), + Attachment: &ec2.NetworkInterfaceAttachment{ + AttachmentId: aws.String("eni-attach-1"), + InstanceId: aws.String("i-1"), + }, + }}, + }) + + reaper := &networkInterfaceReaper{nc: cluster.nc} + found, err := reaper.List(testCtx(t), "000000000042") + require.NoError(t, err) + require.Len(t, found, 1) + assert.Equal(t, "subnet-1", found[0].Detail) + + require.NoError(t, reaper.Delete(testCtx(t), "000000000042", found[0], false)) + assert.Contains(t, cluster.called(), "ec2.cmd.i-1", "the detach was never dispatched") + assert.Contains(t, cluster.called(), "ec2.DeleteNetworkInterface") +} + +// An interface whose instance is already gone is the wedged case: the detach +// cannot resolve an owner and answers NotFound, which must not stop the delete +// — the interface is exactly what has to go. +func TestNetworkInterfaceReaperDeletesWhenTheOwnerIsGone(t *testing.T) { + cluster := newFakeCluster(t) + cluster.reply("ec2.DescribeNetworkInterfaces", ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []*ec2.NetworkInterface{{ + NetworkInterfaceId: aws.String("eni-1"), + SubnetId: aws.String("subnet-1"), + Attachment: &ec2.NetworkInterfaceAttachment{AttachmentId: aws.String("eni-attach-1")}, + }}, + }) + + reaper := &networkInterfaceReaper{nc: cluster.nc} + err := reaper.Delete(testCtx(t), "000000000042", + Resource{Kind: "network-interface", ID: "eni-1"}, false) + + require.NoError(t, err) + assert.Contains(t, cluster.called(), "ec2.DeleteNetworkInterface") +} + +// The interface left behind by a half-finished launch is the case this reaper +// exists for, and it is unattached: there is nothing to detach, so the detach +// is skipped rather than made and forgiven. +func TestNetworkInterfaceReaperDeletesAnUnattachedInterface(t *testing.T) { + cluster := newFakeCluster(t) + cluster.reply("ec2.DescribeNetworkInterfaces", ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []*ec2.NetworkInterface{{ + NetworkInterfaceId: aws.String("eni-1"), SubnetId: aws.String("subnet-1"), + }}, + }) + + reaper := &networkInterfaceReaper{nc: cluster.nc} + err := reaper.Delete(testCtx(t), "000000000042", Resource{Kind: "network-interface", ID: "eni-1"}, false) + + require.NoError(t, err) + assert.Equal(t, []string{ + "ec2.DescribeNetworkInterfaces", "ec2.DeleteNetworkInterface", + }, cluster.called()) +} + +// An interface already gone by the time its stage runs is a success: teardown +// re-runs after a crash and would otherwise never finish. +func TestNetworkInterfaceReaperTreatsAMissingInterfaceAsDeleted(t *testing.T) { + cluster := newFakeCluster(t) + cluster.fail("ec2.DescribeNetworkInterfaces", "InvalidNetworkInterfaceID.NotFound") + cluster.fail("ec2.DeleteNetworkInterface", "InvalidNetworkInterfaceID.NotFound") + + reaper := &networkInterfaceReaper{nc: cluster.nc} + err := reaper.Delete(testCtx(t), "000000000042", Resource{Kind: "network-interface", ID: "eni-1"}, false) + + require.NoError(t, err) + assert.Equal(t, []string{ + "ec2.DescribeNetworkInterfaces", "ec2.DeleteNetworkInterface", + }, cluster.called()) +} + +// Interfaces are reaped before subnets. The reverse order cannot work: the +// subnet delete is refused while an interface is still in it. +func TestEC2ReapersRemoveInterfacesBeforeSubnets(t *testing.T) { + var eni, subnet = -1, -1 + for i, reaper := range EC2Reapers(nil, 1) { + switch reaper.Kind() { + case "network-interface": + eni = i + case "subnet": + subnet = i + } + } + require.NotEqual(t, -1, eni, "there is no network-interface reaper") + require.Less(t, eni, subnet, "interfaces must be reaped before the subnets holding them") +} diff --git a/spinifex/accountteardown/reapers_ecs.go b/spinifex/accountteardown/reapers_ecs.go new file mode 100644 index 000000000..34236795a --- /dev/null +++ b/spinifex/accountteardown/reapers_ecs.go @@ -0,0 +1,69 @@ +package accountteardown + +import ( + "context" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ecs" + handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs" + "github.com/nats-io/nats.go" +) + +// ECSReapers returns the ECS-backed reapers. +// +// One reaper, not two. DeleteCluster already force-stops every task, marks +// every service INACTIVE and sweeps the cluster's whole KV subtree, so a +// separate service reaper would duplicate that — and worse, DeleteService +// leaves an INACTIVE record that still lists, so a stage waiting for services +// to disappear would never drain. +func ECSReapers(nc *nats.Conn) []Reaper { + return []Reaper{&ecsClusterReaper{svc: handlers_ecs.NewNATSECSService(nc)}} +} + +// ecsClusterReaper removes the account's ECS clusters. +// +// Its container instances are ordinary EC2 instances in the tenant's account, +// so the instance reaper takes them — but only after this one has stopped the +// tasks riding on them, which is why this is registered first. +type ecsClusterReaper struct { + svc handlers_ecs.ECSService +} + +func (r *ecsClusterReaper) Kind() string { return "ecs-cluster" } +func (r *ecsClusterReaper) Stage() Stage { return StageCompute } + +func (r *ecsClusterReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + out, err := r.svc.ListClusters(ctx, &ecs.ListClustersInput{}, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, arn := range out.ClusterArns { + name := ecsClusterName(aws.StringValue(arn)) + if name == "" { + continue + } + found = append(found, Resource{Kind: r.Kind(), ID: name}) + } + return found, nil +} + +func (r *ecsClusterReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + _, err := r.svc.DeleteCluster(ctx, &ecs.DeleteClusterInput{ + Cluster: aws.String(resource.ID), + }, accountID) + return ignoreAlreadyGone(err) +} + +// ecsClusterName reduces a cluster ARN to the name the API takes back. The +// name is the cluster's identity in ECS, and it is what an operator reading a +// stuck line needs to see. +func ecsClusterName(ref string) string { + ref = strings.TrimSpace(ref) + if i := strings.LastIndex(ref, "cluster/"); i >= 0 { + return ref[i+len("cluster/"):] + } + return ref +} diff --git a/spinifex/accountteardown/reapers_ecs_test.go b/spinifex/accountteardown/reapers_ecs_test.go new file mode 100644 index 000000000..12e345433 --- /dev/null +++ b/spinifex/accountteardown/reapers_ecs_test.go @@ -0,0 +1,204 @@ +package accountteardown + +//test:in-package — the ECS reaper is unexported for the same reason the EC2 +// ones are, and what matters here is its listing shape and the position it is +// registered in. + +import ( + "encoding/json" + "sync" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ecs" + handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeECS answers the ecs.* subjects the reaper drives. +type fakeECS struct { + nc *nats.Conn + + mu sync.Mutex + calls []string + requests map[string][]byte + replies map[string]any + errorCode map[string]string +} + +func newFakeECS(t *testing.T) *fakeECS { + t.Helper() + _, nc := testutil.StartTestNATS(t) + + cluster := &fakeECS{ + nc: nc, + requests: map[string][]byte{}, + replies: map[string]any{}, + errorCode: map[string]string{}, + } + + sub, err := nc.Subscribe("ecs.>", cluster.serve) + require.NoError(t, err) + t.Cleanup(func() { _ = sub.Unsubscribe() }) + require.NoError(t, nc.Flush()) + + return cluster +} + +func (c *fakeECS) serve(msg *nats.Msg) { + c.mu.Lock() + c.calls = append(c.calls, msg.Subject) + c.requests[msg.Subject] = msg.Data + code, isError := c.errorCode[msg.Subject] + reply, hasReply := c.replies[msg.Subject] + c.mu.Unlock() + + switch { + case isError: + _ = msg.Respond(utils.GenerateErrorPayload(code)) + case hasReply: + payload, err := json.Marshal(reply) + if err != nil { + _ = msg.Respond(utils.GenerateErrorPayload("InternalError")) + return + } + _ = msg.Respond(payload) + default: + _ = msg.Respond([]byte(`{}`)) + } +} + +func (c *fakeECS) reply(subject string, output any) { + c.mu.Lock() + defer c.mu.Unlock() + c.replies[subject] = output +} + +func (c *fakeECS) fail(subject, code string) { + c.mu.Lock() + defer c.mu.Unlock() + c.errorCode[subject] = code +} + +func (c *fakeECS) request(t *testing.T, subject string, into any) { + t.Helper() + c.mu.Lock() + payload, ok := c.requests[subject] + c.mu.Unlock() + require.True(t, ok, "subject %s was never called", subject) + require.NoError(t, json.Unmarshal(payload, into)) +} + +func newECSReaper(nc *nats.Conn) *ecsClusterReaper { + return &ecsClusterReaper{svc: handlers_ecs.NewNATSECSService(nc)} +} + +// The listing is keyed by cluster name, which is the identity ECS takes back +// and the only part of the ARN an operator reading a stuck line can act on. +func TestECSClusterReaperListsByName(t *testing.T) { + cluster := newFakeECS(t) + cluster.reply("ecs.ListClusters", ecs.ListClustersOutput{ + ClusterArns: []*string{ + aws.String("arn:aws:ecs:us-west-1:000000000042:cluster/web"), + aws.String("arn:aws:ecs:us-west-1:000000000042:cluster/batch"), + }, + }) + + found, err := newECSReaper(cluster.nc).List(testCtx(t), "000000000042") + require.NoError(t, err) + + assert.Equal(t, []Resource{ + {Kind: "ecs-cluster", ID: "web"}, + {Kind: "ecs-cluster", ID: "batch"}, + }, found) +} + +// An account with no ECS at all must list empty rather than error, or the +// compute stage would never drain for the overwhelming majority of tenants. +func TestECSClusterReaperListsNothingForAnAccountWithNoClusters(t *testing.T) { + cluster := newFakeECS(t) + cluster.reply("ecs.ListClusters", ecs.ListClustersOutput{}) + + found, err := newECSReaper(cluster.nc).List(testCtx(t), "000000000042") + require.NoError(t, err) + + assert.Empty(t, found) +} + +func TestECSClusterReaperDeletesByName(t *testing.T) { + cluster := newFakeECS(t) + cluster.reply("ecs.DeleteCluster", ecs.DeleteClusterOutput{}) + + err := newECSReaper(cluster.nc).Delete(testCtx(t), "000000000042", + Resource{Kind: "ecs-cluster", ID: "web"}, false) + require.NoError(t, err) + + var input ecs.DeleteClusterInput + cluster.request(t, "ecs.DeleteCluster", &input) + assert.Equal(t, "web", aws.StringValue(input.Cluster)) +} + +// Teardown re-runs after a crash, so a cluster a previous pass already removed +// is a success. Treating it as a failure would leave the account undeletable. +func TestECSClusterReaperTreatsAMissingClusterAsDeleted(t *testing.T) { + cluster := newFakeECS(t) + cluster.fail("ecs.DeleteCluster", "ClusterNotFoundException") + + err := newECSReaper(cluster.nc).Delete(testCtx(t), "000000000042", + Resource{Kind: "ecs-cluster", ID: "gone"}, false) + + assert.NoError(t, err) +} + +// A real failure must surface. Swallowing it would report the stage drained +// while the cluster is still holding tasks, ENIs and capacity. +func TestECSClusterReaperReportsARealFailure(t *testing.T) { + cluster := newFakeECS(t) + cluster.fail("ecs.DeleteCluster", "InternalError") + + err := newECSReaper(cluster.nc).Delete(testCtx(t), "000000000042", + Resource{Kind: "ecs-cluster", ID: "web"}, false) + + assert.Error(t, err) +} + +func TestECSClusterNameFromARN(t *testing.T) { + assert.Equal(t, "web", ecsClusterName("arn:aws:ecs:us-west-1:000000000042:cluster/web")) + assert.Equal(t, "web", ecsClusterName("web")) + assert.Equal(t, "web", ecsClusterName(" web ")) + assert.Empty(t, ecsClusterName("")) +} + +// Container instances are ordinary EC2 instances in the tenant's account, so +// the instance reaper takes them — but the tasks riding on them have to be +// stopped first, which is what puts ECS ahead of it in the compute stage. +func TestECSReapersRunBeforeTheInstanceReaper(t *testing.T) { + reapers := ECSReapers(nil) + reapers = append(reapers, EC2Reapers(nil, 1)...) + + ecsIndex := indexOfKind(reapers, "ecs-cluster") + instanceIndex := indexOfKind(reapers, "instance") + + require.NotEqual(t, -1, ecsIndex) + require.NotEqual(t, -1, instanceIndex) + assert.Less(t, ecsIndex, instanceIndex) + assert.Equal(t, StageCompute, reapers[ecsIndex].Stage()) + + // Both are in the same stage, so the engine's within-stage ordering is + // what enforces this rather than the stage graph. + assert.Equal(t, reapers[instanceIndex].Stage(), reapers[ecsIndex].Stage()) +} + +// SortReapers is what the engine applies, and a stable sort is the only reason +// the registration order above survives it. +func TestSortReapersKeepsECSAheadOfInstances(t *testing.T) { + reapers := ECSReapers(nil) + reapers = append(reapers, EC2Reapers(nil, 1)...) + SortReapers(reapers) + + assert.Less(t, indexOfKind(reapers, "ecs-cluster"), indexOfKind(reapers, "instance")) +} diff --git a/spinifex/accountteardown/reapers_eks.go b/spinifex/accountteardown/reapers_eks.go new file mode 100644 index 000000000..f6ad9fae8 --- /dev/null +++ b/spinifex/accountteardown/reapers_eks.go @@ -0,0 +1,138 @@ +package accountteardown + +import ( + "context" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/eks" + handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks" + "github.com/nats-io/nats.go" +) + +// EKSReapers returns the EKS-backed reapers, nodegroups before clusters. +// +// DeleteCluster tears down the NLB, the control-plane VMs, the security groups +// and the managed control-plane VPC, but it does not touch nodegroups. Deleting +// the cluster first would strand every worker node with nothing left to +// address it by. +func EKSReapers(nc *nats.Conn) []Reaper { + svc := handlers_eks.NewNATSEKSService(nc) + return []Reaper{ + &eksNodegroupReaper{svc: svc}, + &eksClusterReaper{svc: svc}, + } +} + +// eksNodegroupSeparator joins a nodegroup's cluster to its name. A nodegroup +// name is only unique within its cluster, so the pair is the identity. +const eksNodegroupSeparator = "/" + +type eksNodegroupReaper struct { + svc handlers_eks.EKSService +} + +func (r *eksNodegroupReaper) Kind() string { return "eks-nodegroup" } +func (r *eksNodegroupReaper) Stage() Stage { return StageCompute } + +func (r *eksNodegroupReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + clusters, err := listEKSClusterNames(ctx, r.svc, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, cluster := range clusters { + out, err := r.svc.ListNodegroups(ctx, &eks.ListNodegroupsInput{ + ClusterName: aws.String(cluster), + }, accountID) + if err != nil { + // The cluster went between the two calls, which is the ordinary + // result of a concurrent delete rather than a reason to fail the + // whole listing. + if isAlreadyGone(err) { + continue + } + return nil, err + } + for _, name := range out.Nodegroups { + if aws.StringValue(name) == "" { + continue + } + found = append(found, Resource{ + Kind: r.Kind(), + ID: cluster + eksNodegroupSeparator + aws.StringValue(name), + }) + } + } + return found, nil +} + +func (r *eksNodegroupReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + cluster, nodegroup, ok := strings.Cut(resource.ID, eksNodegroupSeparator) + if !ok { + return nil + } + _, err := r.svc.DeleteNodegroup(ctx, &eks.DeleteNodegroupInput{ + ClusterName: aws.String(cluster), + NodegroupName: aws.String(nodegroup), + }, accountID) + return ignoreAlreadyGone(err) +} + +type eksClusterReaper struct { + svc handlers_eks.EKSService +} + +func (r *eksClusterReaper) Kind() string { return "eks-cluster" } +func (r *eksClusterReaper) Stage() Stage { return StageCompute } + +func (r *eksClusterReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + names, err := listEKSClusterNames(ctx, r.svc, accountID) + if err != nil { + return nil, err + } + + found := make([]Resource, 0, len(names)) + for _, name := range names { + found = append(found, Resource{Kind: r.Kind(), ID: name}) + } + return found, nil +} + +func (r *eksClusterReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + _, err := r.svc.DeleteCluster(ctx, &eks.DeleteClusterInput{ + Name: aws.String(resource.ID), + }, accountID) + return ignoreAlreadyGone(err) +} + +// eksListPageLimit bounds a runaway pagination loop. A tenant is nowhere near +// this many clusters; the cap exists so a NextToken that never advances cannot +// spin the drain loop forever. +const eksListPageLimit = 100 + +// listEKSClusterNames pages through ListClusters. The listing is capped at 100 +// per page, so taking only the first page would silently leave every cluster +// past it behind — and a teardown that misses one never converges. +func listEKSClusterNames(ctx context.Context, svc handlers_eks.EKSService, accountID string) ([]string, error) { + var names []string + var token *string + + for range eksListPageLimit { + out, err := svc.ListClusters(ctx, &eks.ListClustersInput{NextToken: token}, accountID) + if err != nil { + return nil, err + } + for _, name := range out.Clusters { + if aws.StringValue(name) != "" { + names = append(names, aws.StringValue(name)) + } + } + if out.NextToken == nil || aws.StringValue(out.NextToken) == "" { + break + } + token = out.NextToken + } + return names, nil +} diff --git a/spinifex/accountteardown/reapers_eks_test.go b/spinifex/accountteardown/reapers_eks_test.go new file mode 100644 index 000000000..cb368509e --- /dev/null +++ b/spinifex/accountteardown/reapers_eks_test.go @@ -0,0 +1,227 @@ +package accountteardown + +//test:in-package — the reapers are unexported, and the pagination and ordering +// rules asserted here are the substance of them. + +import ( + "encoding/json" + "sync" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/eks" + handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeEKS answers the eks.* subjects. Replies can be queued per subject so a +// paginated listing can be scripted across successive calls. +type fakeEKS struct { + nc *nats.Conn + + mu sync.Mutex + requests map[string][][]byte + queued map[string][]any + errorCode map[string]string +} + +func newFakeEKS(t *testing.T) *fakeEKS { + t.Helper() + _, nc := testutil.StartTestNATS(t) + + fake := &fakeEKS{ + nc: nc, + requests: map[string][][]byte{}, + queued: map[string][]any{}, + errorCode: map[string]string{}, + } + + sub, err := nc.Subscribe("eks.>", fake.serve) + require.NoError(t, err) + t.Cleanup(func() { _ = sub.Unsubscribe() }) + require.NoError(t, nc.Flush()) + + return fake +} + +func (f *fakeEKS) serve(msg *nats.Msg) { + f.mu.Lock() + f.requests[msg.Subject] = append(f.requests[msg.Subject], msg.Data) + code, isError := f.errorCode[msg.Subject] + + var reply any + hasReply := false + if queue := f.queued[msg.Subject]; len(queue) > 0 { + reply = queue[0] + hasReply = true + // The last queued reply repeats, so a drain loop that lists once more + // than the test scripted does not fall off the end. + if len(queue) > 1 { + f.queued[msg.Subject] = queue[1:] + } + } + f.mu.Unlock() + + switch { + case isError: + _ = msg.Respond(utils.GenerateErrorPayload(code)) + case hasReply: + payload, err := json.Marshal(reply) + if err != nil { + _ = msg.Respond(utils.GenerateErrorPayload("InternalError")) + return + } + _ = msg.Respond(payload) + default: + _ = msg.Respond([]byte(`{}`)) + } +} + +func (f *fakeEKS) reply(subject string, outputs ...any) { + f.mu.Lock() + defer f.mu.Unlock() + f.queued[subject] = append(f.queued[subject], outputs...) +} + +func (f *fakeEKS) fail(subject, code string) { + f.mu.Lock() + defer f.mu.Unlock() + f.errorCode[subject] = code +} + +func (f *fakeEKS) requestCount(subject string) int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.requests[subject]) +} + +func (f *fakeEKS) requestAt(t *testing.T, subject string, index int, into any) { + t.Helper() + f.mu.Lock() + payloads := f.requests[subject] + f.mu.Unlock() + require.Greater(t, len(payloads), index, "subject %s was called %d times", subject, len(payloads)) + require.NoError(t, json.Unmarshal(payloads[index], into)) +} + +func newEKSReapers(nc *nats.Conn) (*eksNodegroupReaper, *eksClusterReaper) { + svc := handlers_eks.NewNATSEKSService(nc) + return &eksNodegroupReaper{svc: svc}, &eksClusterReaper{svc: svc} +} + +func TestEKSClusterReaperListsEveryCluster(t *testing.T) { + fake := newFakeEKS(t) + fake.reply("eks.ListClusters", eks.ListClustersOutput{ + Clusters: aws.StringSlice([]string{"alpha", "beta"}), + }) + + _, clusters := newEKSReapers(fake.nc) + found, err := clusters.List(testCtx(t), "000000000042") + require.NoError(t, err) + + assert.Equal(t, []Resource{ + {Kind: "eks-cluster", ID: "alpha"}, + {Kind: "eks-cluster", ID: "beta"}, + }, found) +} + +// ListClusters caps a page at 100. Reading only the first page would report +// the stage drained with clusters still standing, and a teardown that misses +// one never converges. +func TestEKSClusterReaperFollowsPagination(t *testing.T) { + fake := newFakeEKS(t) + fake.reply("eks.ListClusters", + eks.ListClustersOutput{Clusters: aws.StringSlice([]string{"alpha"}), NextToken: aws.String("beta")}, + eks.ListClustersOutput{Clusters: aws.StringSlice([]string{"beta"})}, + ) + + _, clusters := newEKSReapers(fake.nc) + found, err := clusters.List(testCtx(t), "000000000042") + require.NoError(t, err) + + require.Len(t, found, 2) + assert.Equal(t, "beta", found[1].ID) + + var second eks.ListClustersInput + fake.requestAt(t, "eks.ListClusters", 1, &second) + assert.Equal(t, "beta", aws.StringValue(second.NextToken)) +} + +// A NextToken that never advances would otherwise spin the drain loop for the +// whole stage budget without listing anything new. +func TestEKSClusterReaperStopsOnARepeatingPageToken(t *testing.T) { + fake := newFakeEKS(t) + fake.reply("eks.ListClusters", eks.ListClustersOutput{ + Clusters: aws.StringSlice([]string{"alpha"}), + NextToken: aws.String("alpha"), + }) + + _, clusters := newEKSReapers(fake.nc) + found, err := clusters.List(testCtx(t), "000000000042") + require.NoError(t, err) + + assert.Len(t, found, eksListPageLimit) + assert.Equal(t, eksListPageLimit, fake.requestCount("eks.ListClusters")) +} + +// A nodegroup name is unique only within its cluster, so the pair is carried +// through the listing and split again on delete. +func TestEKSNodegroupReaperCarriesTheClusterWithTheName(t *testing.T) { + fake := newFakeEKS(t) + fake.reply("eks.ListClusters", eks.ListClustersOutput{Clusters: aws.StringSlice([]string{"alpha"})}) + fake.reply("eks.ListNodegroups", eks.ListNodegroupsOutput{ + Nodegroups: aws.StringSlice([]string{"workers"}), + }) + + nodegroups, _ := newEKSReapers(fake.nc) + found, err := nodegroups.List(testCtx(t), "000000000042") + require.NoError(t, err) + require.Equal(t, []Resource{{Kind: "eks-nodegroup", ID: "alpha/workers"}}, found) + + require.NoError(t, nodegroups.Delete(testCtx(t), "000000000042", found[0], false)) + + var input eks.DeleteNodegroupInput + fake.requestAt(t, "eks.DeleteNodegroup", 0, &input) + assert.Equal(t, "alpha", aws.StringValue(input.ClusterName)) + assert.Equal(t, "workers", aws.StringValue(input.NodegroupName)) +} + +// A cluster deleted between the two calls is the ordinary result of a +// concurrent teardown, not a reason to fail the whole listing. +func TestEKSNodegroupReaperSkipsAClusterThatWentAway(t *testing.T) { + fake := newFakeEKS(t) + fake.reply("eks.ListClusters", eks.ListClustersOutput{Clusters: aws.StringSlice([]string{"alpha"})}) + fake.fail("eks.ListNodegroups", "ResourceNotFoundException") + + nodegroups, _ := newEKSReapers(fake.nc) + found, err := nodegroups.List(testCtx(t), "000000000042") + + require.NoError(t, err) + assert.Empty(t, found) +} + +func TestEKSClusterReaperTreatsAMissingClusterAsDeleted(t *testing.T) { + fake := newFakeEKS(t) + fake.fail("eks.DeleteCluster", "ResourceNotFoundException") + + _, clusters := newEKSReapers(fake.nc) + err := clusters.Delete(testCtx(t), "000000000042", Resource{Kind: "eks-cluster", ID: "gone"}, false) + + assert.NoError(t, err) +} + +// DeleteCluster tears down the NLB, the control-plane VMs and the managed CP +// VPC but never touches nodegroups. Deleting the cluster first would strand +// every worker node with nothing left to address it by. +func TestEKSNodegroupsAreReapedBeforeClusters(t *testing.T) { + reapers := EKSReapers(nil) + + assert.Less(t, indexOfKind(reapers, "eks-nodegroup"), indexOfKind(reapers, "eks-cluster")) + for _, reaper := range reapers { + assert.Equal(t, StageCompute, reaper.Stage(), "%s runs in the compute stage", reaper.Kind()) + } +} diff --git a/spinifex/accountteardown/reapers_elbv2.go b/spinifex/accountteardown/reapers_elbv2.go new file mode 100644 index 000000000..332d9138c --- /dev/null +++ b/spinifex/accountteardown/reapers_elbv2.go @@ -0,0 +1,92 @@ +package accountteardown + +import ( + "context" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/elbv2" + handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2" + "github.com/nats-io/nats.go" +) + +// ELBv2Reapers returns the load balancer reapers in teardown order. +// +// Both sit in the network stage: a load balancer holds subnets and security +// groups, so it has to go before either can be deleted. Target groups follow +// their balancers because a group still referenced by a listener refuses. +func ELBv2Reapers(nc *nats.Conn) []Reaper { + svc := handlers_elbv2.NewNATSELBv2Service(nc) + return []Reaper{ + &loadBalancerReaper{svc: svc}, + &targetGroupReaper{svc: svc}, + } +} + +type loadBalancerReaper struct { + svc handlers_elbv2.ELBv2Service +} + +func (r *loadBalancerReaper) Kind() string { return "load-balancer" } +func (r *loadBalancerReaper) Stage() Stage { return StageNetwork } + +func (r *loadBalancerReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + out, err := r.svc.DescribeLoadBalancers(ctx, &elbv2.DescribeLoadBalancersInput{}, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, lb := range out.LoadBalancers { + if lb == nil || lb.LoadBalancerArn == nil { + continue + } + // The ARN is what deletes; the name is what an operator recognises. + found = append(found, Resource{ + Kind: r.Kind(), + ID: aws.StringValue(lb.LoadBalancerArn), + Detail: aws.StringValue(lb.LoadBalancerName), + }) + } + return found, nil +} + +func (r *loadBalancerReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + _, err := r.svc.DeleteLoadBalancer(ctx, &elbv2.DeleteLoadBalancerInput{ + LoadBalancerArn: aws.String(resource.ID), + }, accountID) + return ignoreAlreadyGone(err) +} + +type targetGroupReaper struct { + svc handlers_elbv2.ELBv2Service +} + +func (r *targetGroupReaper) Kind() string { return "target-group" } +func (r *targetGroupReaper) Stage() Stage { return StageNetwork } + +func (r *targetGroupReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + out, err := r.svc.DescribeTargetGroups(ctx, &elbv2.DescribeTargetGroupsInput{}, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, group := range out.TargetGroups { + if group == nil || group.TargetGroupArn == nil { + continue + } + found = append(found, Resource{ + Kind: r.Kind(), + ID: aws.StringValue(group.TargetGroupArn), + Detail: aws.StringValue(group.TargetGroupName), + }) + } + return found, nil +} + +func (r *targetGroupReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + _, err := r.svc.DeleteTargetGroup(ctx, &elbv2.DeleteTargetGroupInput{ + TargetGroupArn: aws.String(resource.ID), + }, accountID) + return ignoreAlreadyGone(err) +} diff --git a/spinifex/accountteardown/reapers_elbv2_test.go b/spinifex/accountteardown/reapers_elbv2_test.go new file mode 100644 index 000000000..f3b851826 --- /dev/null +++ b/spinifex/accountteardown/reapers_elbv2_test.go @@ -0,0 +1,105 @@ +package accountteardown + +//test:in-package — both reapers are unexported, and the order they are +// registered in is what keeps a target group from outliving its listener. + +import ( + "context" + "errors" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/elbv2" + handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The embedded interface is nil, so a reaper reaching for anything beyond the +// four methods below panics rather than passing. +type fakeELBv2 struct { + handlers_elbv2.ELBv2Service + + loadBalancers []*elbv2.LoadBalancer + targetGroups []*elbv2.TargetGroup + deleted []string + err error +} + +func (f *fakeELBv2) DescribeLoadBalancers(_ context.Context, _ *elbv2.DescribeLoadBalancersInput, _ string) (*elbv2.DescribeLoadBalancersOutput, error) { + return &elbv2.DescribeLoadBalancersOutput{LoadBalancers: f.loadBalancers}, nil +} + +func (f *fakeELBv2) DeleteLoadBalancer(_ context.Context, in *elbv2.DeleteLoadBalancerInput, _ string) (*elbv2.DeleteLoadBalancerOutput, error) { + if f.err != nil { + return nil, f.err + } + f.deleted = append(f.deleted, aws.StringValue(in.LoadBalancerArn)) + return &elbv2.DeleteLoadBalancerOutput{}, nil +} + +func (f *fakeELBv2) DescribeTargetGroups(_ context.Context, _ *elbv2.DescribeTargetGroupsInput, _ string) (*elbv2.DescribeTargetGroupsOutput, error) { + return &elbv2.DescribeTargetGroupsOutput{TargetGroups: f.targetGroups}, nil +} + +func (f *fakeELBv2) DeleteTargetGroup(_ context.Context, in *elbv2.DeleteTargetGroupInput, _ string) (*elbv2.DeleteTargetGroupOutput, error) { + if f.err != nil { + return nil, f.err + } + f.deleted = append(f.deleted, aws.StringValue(in.TargetGroupArn)) + return &elbv2.DeleteTargetGroupOutput{}, nil +} + +func TestLoadBalancerReaperListsByARNAndNamesTheBalancer(t *testing.T) { + svc := &fakeELBv2{loadBalancers: []*elbv2.LoadBalancer{ + {LoadBalancerArn: aws.String("arn:lb/web"), LoadBalancerName: aws.String("web")}, + {LoadBalancerName: aws.String("no-arn")}, + }} + reaper := &loadBalancerReaper{svc: svc} + + assert.Equal(t, StageNetwork, reaper.Stage()) + + found, err := reaper.List(t.Context(), "000000000002") + require.NoError(t, err) + require.Len(t, found, 1) + assert.Equal(t, "arn:lb/web", found[0].ID) + assert.Equal(t, "web", found[0].Detail) + + require.NoError(t, reaper.Delete(t.Context(), "000000000002", found[0], false)) + assert.Equal(t, []string{"arn:lb/web"}, svc.deleted) +} + +func TestTargetGroupReaperListsAndDeletesByARN(t *testing.T) { + svc := &fakeELBv2{targetGroups: []*elbv2.TargetGroup{ + {TargetGroupArn: aws.String("arn:tg/web"), TargetGroupName: aws.String("web")}, + }} + reaper := &targetGroupReaper{svc: svc} + + assert.Equal(t, StageNetwork, reaper.Stage()) + + found, err := reaper.List(t.Context(), "000000000002") + require.NoError(t, err) + require.Len(t, found, 1) + assert.Equal(t, "arn:tg/web", found[0].ID) + + require.NoError(t, reaper.Delete(t.Context(), "000000000002", found[0], false)) + assert.Equal(t, []string{"arn:tg/web"}, svc.deleted) +} + +func TestELBv2ReapersTreatAMissingResourceAsDeleted(t *testing.T) { + lbGone := &loadBalancerReaper{svc: &fakeELBv2{err: errors.New("LoadBalancerNotFound: no such balancer")}} + assert.NoError(t, lbGone.Delete(t.Context(), "000000000002", Resource{ID: "arn:lb/web"}, false)) + + tgGone := &targetGroupReaper{svc: &fakeELBv2{err: errors.New("TargetGroupNotFound: no such group")}} + assert.NoError(t, tgGone.Delete(t.Context(), "000000000002", Resource{ID: "arn:tg/web"}, false)) +} + +// A target group still referenced by a listener refuses to delete, which is +// why the balancer has to be reaped first — and the sort that groups reapers +// by stage is stable, so registration order is what guarantees it. +func TestELBv2ReapersDeleteBalancersBeforeTargetGroups(t *testing.T) { + reapers := ELBv2Reapers(nil) + require.Len(t, reapers, 2) + assert.Equal(t, "load-balancer", reapers[0].Kind()) + assert.Equal(t, "target-group", reapers[1].Kind()) +} diff --git a/spinifex/accountteardown/reapers_iam.go b/spinifex/accountteardown/reapers_iam.go new file mode 100644 index 000000000..57417e2ed --- /dev/null +++ b/spinifex/accountteardown/reapers_iam.go @@ -0,0 +1,332 @@ +package accountteardown + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/iam" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" +) + +// IAMReapers returns the identity-stage reapers in teardown order. +// +// Access keys go first and deliberately: it is the second quiesce, and unlike +// the account-status gate it holds even against a caller that reached the +// cluster some other way. +func IAMReapers(svc handlers_iam.IAMService) []Reaper { + return []Reaper{ + &accessKeyReaper{svc: svc}, + &instanceProfileReaper{svc: svc}, + &iamUserReaper{svc: svc}, + &iamRoleReaper{svc: svc}, + &iamGroupReaper{svc: svc}, + &iamPolicyReaper{svc: svc}, + + // Platform stage: an OIDC provider is what an EKS cluster's service + // accounts trust, so it outlives the cluster and is not an identity + // the account authenticates with. + &oidcProviderReaper{svc: svc}, + } +} + +// oidcProviderReaper removes the IAM OIDC providers EKS clusters register. +// They survive their cluster's deletion, so nothing else in teardown reaches +// them and an account would be left holding a trust anchor for a cluster that +// no longer exists. +type oidcProviderReaper struct{ svc handlers_iam.IAMService } + +func (r *oidcProviderReaper) Kind() string { return "oidc-provider" } +func (r *oidcProviderReaper) Stage() Stage { return StagePlatform } + +func (r *oidcProviderReaper) List(_ context.Context, accountID string) ([]Resource, error) { + providers, err := r.svc.ListOpenIDConnectProviders(accountID, &iam.ListOpenIDConnectProvidersInput{}) + if err != nil { + return nil, fmt.Errorf("list OIDC providers: %w", err) + } + + var found []Resource + for _, provider := range providers.OpenIDConnectProviderList { + if provider == nil || provider.Arn == nil { + continue + } + found = append(found, Resource{Kind: r.Kind(), ID: *provider.Arn}) + } + return found, nil +} + +func (r *oidcProviderReaper) Delete(_ context.Context, accountID string, resource Resource, _ bool) error { + _, err := r.svc.DeleteOpenIDConnectProvider(accountID, &iam.DeleteOpenIDConnectProviderInput{ + OpenIDConnectProviderArn: aws.String(resource.ID), + }) + return ignoreAlreadyGone(err) +} + +type accessKeyReaper struct{ svc handlers_iam.IAMService } + +func (r *accessKeyReaper) Kind() string { return "access-key" } +func (r *accessKeyReaper) Stage() Stage { return StageIdentity } + +// List walks every user because access keys are addressed by user, and a key +// whose user is missed stays valid. +func (r *accessKeyReaper) List(_ context.Context, accountID string) ([]Resource, error) { + users, err := r.svc.ListUsers(accountID, &iam.ListUsersInput{}) + if err != nil { + return nil, fmt.Errorf("list users: %w", err) + } + + var found []Resource + for _, user := range users.Users { + if user == nil || user.UserName == nil { + continue + } + keys, err := r.svc.ListAccessKeys(accountID, &iam.ListAccessKeysInput{UserName: user.UserName}) + if err != nil { + return nil, fmt.Errorf("list access keys for %s: %w", *user.UserName, err) + } + for _, key := range keys.AccessKeyMetadata { + if key == nil || key.AccessKeyId == nil { + continue + } + found = append(found, Resource{Kind: r.Kind(), ID: *key.AccessKeyId, Detail: *user.UserName}) + } + } + return found, nil +} + +func (r *accessKeyReaper) Delete(_ context.Context, accountID string, resource Resource, _ bool) error { + _, err := r.svc.DeleteAccessKey(accountID, &iam.DeleteAccessKeyInput{ + AccessKeyId: aws.String(resource.ID), + UserName: aws.String(resource.Detail), + }) + return ignoreAlreadyGone(err) +} + +type iamUserReaper struct{ svc handlers_iam.IAMService } + +func (r *iamUserReaper) Kind() string { return "iam-user" } +func (r *iamUserReaper) Stage() Stage { return StageIdentity } + +func (r *iamUserReaper) List(_ context.Context, accountID string) ([]Resource, error) { + users, err := r.svc.ListUsers(accountID, &iam.ListUsersInput{}) + if err != nil { + return nil, err + } + + var found []Resource + for _, user := range users.Users { + if user == nil || user.UserName == nil { + continue + } + found = append(found, Resource{Kind: r.Kind(), ID: *user.UserName}) + } + return found, nil +} + +// Delete strips the user's policies first: AWS refuses to delete a user that +// still has attachments, and so does this implementation. +func (r *iamUserReaper) Delete(_ context.Context, accountID string, resource Resource, _ bool) error { + attached, err := r.svc.ListAttachedUserPolicies(accountID, &iam.ListAttachedUserPoliciesInput{ + UserName: aws.String(resource.ID), + }) + if err != nil && !isAlreadyGone(err) { + return err + } + if attached != nil { + for _, policy := range attached.AttachedPolicies { + if policy == nil || policy.PolicyArn == nil { + continue + } + if _, err := r.svc.DetachUserPolicy(accountID, &iam.DetachUserPolicyInput{ + UserName: aws.String(resource.ID), PolicyArn: policy.PolicyArn, + }); err != nil && !isAlreadyGone(err) { + return err + } + } + } + + inline, err := r.svc.ListUserPolicies(accountID, &iam.ListUserPoliciesInput{UserName: aws.String(resource.ID)}) + if err != nil && !isAlreadyGone(err) { + return err + } + if inline != nil { + for _, name := range inline.PolicyNames { + if name == nil { + continue + } + if _, err := r.svc.DeleteUserPolicy(accountID, &iam.DeleteUserPolicyInput{ + UserName: aws.String(resource.ID), PolicyName: name, + }); err != nil && !isAlreadyGone(err) { + return err + } + } + } + + _, err = r.svc.DeleteUser(accountID, &iam.DeleteUserInput{UserName: aws.String(resource.ID)}) + return ignoreAlreadyGone(err) +} + +type iamRoleReaper struct{ svc handlers_iam.IAMService } + +func (r *iamRoleReaper) Kind() string { return "iam-role" } +func (r *iamRoleReaper) Stage() Stage { return StageIdentity } + +func (r *iamRoleReaper) List(_ context.Context, accountID string) ([]Resource, error) { + roles, err := r.svc.ListRoles(accountID, &iam.ListRolesInput{}) + if err != nil { + return nil, err + } + + var found []Resource + for _, role := range roles.Roles { + if role == nil || role.RoleName == nil { + continue + } + found = append(found, Resource{Kind: r.Kind(), ID: *role.RoleName}) + } + return found, nil +} + +func (r *iamRoleReaper) Delete(_ context.Context, accountID string, resource Resource, _ bool) error { + attached, err := r.svc.ListAttachedRolePolicies(accountID, &iam.ListAttachedRolePoliciesInput{ + RoleName: aws.String(resource.ID), + }) + if err != nil && !isAlreadyGone(err) { + return err + } + if attached != nil { + for _, policy := range attached.AttachedPolicies { + if policy == nil || policy.PolicyArn == nil { + continue + } + if _, err := r.svc.DetachRolePolicy(accountID, &iam.DetachRolePolicyInput{ + RoleName: aws.String(resource.ID), PolicyArn: policy.PolicyArn, + }); err != nil && !isAlreadyGone(err) { + return err + } + } + } + + inline, err := r.svc.ListRolePolicies(accountID, &iam.ListRolePoliciesInput{RoleName: aws.String(resource.ID)}) + if err != nil && !isAlreadyGone(err) { + return err + } + if inline != nil { + for _, name := range inline.PolicyNames { + if name == nil { + continue + } + if _, err := r.svc.DeleteRolePolicy(accountID, &iam.DeleteRolePolicyInput{ + RoleName: aws.String(resource.ID), PolicyName: name, + }); err != nil && !isAlreadyGone(err) { + return err + } + } + } + + _, err = r.svc.DeleteRole(accountID, &iam.DeleteRoleInput{RoleName: aws.String(resource.ID)}) + return ignoreAlreadyGone(err) +} + +type iamGroupReaper struct{ svc handlers_iam.IAMService } + +func (r *iamGroupReaper) Kind() string { return "iam-group" } +func (r *iamGroupReaper) Stage() Stage { return StageIdentity } + +func (r *iamGroupReaper) List(_ context.Context, accountID string) ([]Resource, error) { + groups, err := r.svc.ListGroups(accountID, &iam.ListGroupsInput{}) + if err != nil { + return nil, err + } + + var found []Resource + for _, group := range groups.Groups { + if group == nil || group.GroupName == nil { + continue + } + found = append(found, Resource{Kind: r.Kind(), ID: *group.GroupName}) + } + return found, nil +} + +func (r *iamGroupReaper) Delete(_ context.Context, accountID string, resource Resource, _ bool) error { + _, err := r.svc.DeleteGroup(accountID, &iam.DeleteGroupInput{GroupName: aws.String(resource.ID)}) + return ignoreAlreadyGone(err) +} + +type iamPolicyReaper struct{ svc handlers_iam.IAMService } + +func (r *iamPolicyReaper) Kind() string { return "iam-policy" } +func (r *iamPolicyReaper) Stage() Stage { return StageIdentity } + +// List is scoped to the account's own policies. AWS-managed policies are not +// the account's to delete and are shared with every other tenant. +func (r *iamPolicyReaper) List(_ context.Context, accountID string) ([]Resource, error) { + policies, err := r.svc.ListPolicies(accountID, &iam.ListPoliciesInput{Scope: aws.String("Local")}) + if err != nil { + return nil, err + } + + var found []Resource + for _, policy := range policies.Policies { + if policy == nil || policy.Arn == nil { + continue + } + found = append(found, Resource{Kind: r.Kind(), ID: *policy.Arn}) + } + return found, nil +} + +func (r *iamPolicyReaper) Delete(_ context.Context, accountID string, resource Resource, _ bool) error { + _, err := r.svc.DeletePolicy(accountID, &iam.DeletePolicyInput{PolicyArn: aws.String(resource.ID)}) + return ignoreAlreadyGone(err) +} + +type instanceProfileReaper struct{ svc handlers_iam.IAMService } + +func (r *instanceProfileReaper) Kind() string { return "instance-profile" } +func (r *instanceProfileReaper) Stage() Stage { return StageIdentity } + +func (r *instanceProfileReaper) List(_ context.Context, accountID string) ([]Resource, error) { + profiles, err := r.svc.ListInstanceProfiles(accountID, &iam.ListInstanceProfilesInput{}) + if err != nil { + return nil, err + } + + var found []Resource + for _, profile := range profiles.InstanceProfiles { + if profile == nil || profile.InstanceProfileName == nil { + continue + } + found = append(found, Resource{Kind: r.Kind(), ID: *profile.InstanceProfileName}) + } + return found, nil +} + +// Delete unbinds the profile's roles first, otherwise the role reaper later +// finds a role it cannot delete because a profile still references it. +func (r *instanceProfileReaper) Delete(_ context.Context, accountID string, resource Resource, _ bool) error { + profile, err := r.svc.GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{ + InstanceProfileName: aws.String(resource.ID), + }) + if err != nil && !isAlreadyGone(err) { + return err + } + if profile != nil && profile.InstanceProfile != nil { + for _, role := range profile.InstanceProfile.Roles { + if role == nil || role.RoleName == nil { + continue + } + if _, err := r.svc.RemoveRoleFromInstanceProfile(accountID, &iam.RemoveRoleFromInstanceProfileInput{ + InstanceProfileName: aws.String(resource.ID), RoleName: role.RoleName, + }); err != nil && !isAlreadyGone(err) { + return err + } + } + } + + _, err = r.svc.DeleteInstanceProfile(accountID, &iam.DeleteInstanceProfileInput{ + InstanceProfileName: aws.String(resource.ID), + }) + return ignoreAlreadyGone(err) +} diff --git a/spinifex/accountteardown/reapers_iam_test.go b/spinifex/accountteardown/reapers_iam_test.go new file mode 100644 index 000000000..96837f9aa --- /dev/null +++ b/spinifex/accountteardown/reapers_iam_test.go @@ -0,0 +1,416 @@ +package accountteardown + +//test:in-package — the IAM reapers are unexported, and the rules worth testing +// are their delete ordering (detach before delete) and the scoping that keeps +// teardown off resources the account does not own. + +import ( + "errors" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/iam" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeIAM answers the calls the identity reapers make and records them in +// order. It embeds the interface so the many methods teardown never touches +// stay unimplemented — calling one is a test failure by panic, which is the +// honest outcome for a reaper reaching somewhere it was not meant to. +type fakeIAM struct { + handlers_iam.IAMService + + calls []string + + users []string + roles []string + groups []string + policies []string + instanceProfiles []string + accessKeys map[string][]string + profileRoles map[string][]string + attachedUser []string + inlineUser []string + attachedRole []string + inlineRole []string + + oidcProviders []string + + listUsersErr error + listKeysErr error + scopeAsked string +} + +func newFakeIAM() *fakeIAM { + return &fakeIAM{accessKeys: map[string][]string{}, profileRoles: map[string][]string{}} +} + +func (f *fakeIAM) record(call string) { f.calls = append(f.calls, call) } + +func (f *fakeIAM) ListOpenIDConnectProviders(_ string, _ *iam.ListOpenIDConnectProvidersInput) (*iam.ListOpenIDConnectProvidersOutput, error) { + f.record("ListOpenIDConnectProviders") + out := &iam.ListOpenIDConnectProvidersOutput{} + for _, arn := range f.oidcProviders { + out.OpenIDConnectProviderList = append(out.OpenIDConnectProviderList, + &iam.OpenIDConnectProviderListEntry{Arn: aws.String(arn)}) + } + out.OpenIDConnectProviderList = append(out.OpenIDConnectProviderList, nil) + return out, nil +} + +func (f *fakeIAM) DeleteOpenIDConnectProvider(_ string, input *iam.DeleteOpenIDConnectProviderInput) (*iam.DeleteOpenIDConnectProviderOutput, error) { + f.record("DeleteOpenIDConnectProvider " + aws.StringValue(input.OpenIDConnectProviderArn)) + return &iam.DeleteOpenIDConnectProviderOutput{}, nil +} + +func (f *fakeIAM) ListUsers(_ string, _ *iam.ListUsersInput) (*iam.ListUsersOutput, error) { + f.record("ListUsers") + if f.listUsersErr != nil { + return nil, f.listUsersErr + } + out := &iam.ListUsersOutput{} + for _, name := range f.users { + out.Users = append(out.Users, &iam.User{UserName: aws.String(name)}) + } + out.Users = append(out.Users, nil) + return out, nil +} + +func (f *fakeIAM) ListAccessKeys(_ string, input *iam.ListAccessKeysInput) (*iam.ListAccessKeysOutput, error) { + f.record("ListAccessKeys") + if f.listKeysErr != nil { + return nil, f.listKeysErr + } + out := &iam.ListAccessKeysOutput{} + for _, key := range f.accessKeys[aws.StringValue(input.UserName)] { + out.AccessKeyMetadata = append(out.AccessKeyMetadata, &iam.AccessKeyMetadata{ + AccessKeyId: aws.String(key), + }) + } + out.AccessKeyMetadata = append(out.AccessKeyMetadata, nil) + return out, nil +} + +func (f *fakeIAM) DeleteAccessKey(_ string, _ *iam.DeleteAccessKeyInput) (*iam.DeleteAccessKeyOutput, error) { + f.record("DeleteAccessKey") + return &iam.DeleteAccessKeyOutput{}, nil +} + +func (f *fakeIAM) ListAttachedUserPolicies(_ string, _ *iam.ListAttachedUserPoliciesInput) (*iam.ListAttachedUserPoliciesOutput, error) { + f.record("ListAttachedUserPolicies") + out := &iam.ListAttachedUserPoliciesOutput{} + for _, arn := range f.attachedUser { + out.AttachedPolicies = append(out.AttachedPolicies, &iam.AttachedPolicy{PolicyArn: aws.String(arn)}) + } + return out, nil +} + +func (f *fakeIAM) DetachUserPolicy(_ string, _ *iam.DetachUserPolicyInput) (*iam.DetachUserPolicyOutput, error) { + f.record("DetachUserPolicy") + return &iam.DetachUserPolicyOutput{}, nil +} + +func (f *fakeIAM) ListUserPolicies(_ string, _ *iam.ListUserPoliciesInput) (*iam.ListUserPoliciesOutput, error) { + f.record("ListUserPolicies") + out := &iam.ListUserPoliciesOutput{} + for _, name := range f.inlineUser { + out.PolicyNames = append(out.PolicyNames, aws.String(name)) + } + return out, nil +} + +func (f *fakeIAM) DeleteUserPolicy(_ string, _ *iam.DeleteUserPolicyInput) (*iam.DeleteUserPolicyOutput, error) { + f.record("DeleteUserPolicy") + return &iam.DeleteUserPolicyOutput{}, nil +} + +func (f *fakeIAM) DeleteUser(_ string, _ *iam.DeleteUserInput) (*iam.DeleteUserOutput, error) { + f.record("DeleteUser") + return &iam.DeleteUserOutput{}, nil +} + +func (f *fakeIAM) ListRoles(_ string, _ *iam.ListRolesInput) (*iam.ListRolesOutput, error) { + f.record("ListRoles") + out := &iam.ListRolesOutput{} + for _, name := range f.roles { + out.Roles = append(out.Roles, &iam.Role{RoleName: aws.String(name)}) + } + out.Roles = append(out.Roles, nil) + return out, nil +} + +func (f *fakeIAM) ListAttachedRolePolicies(_ string, _ *iam.ListAttachedRolePoliciesInput) (*iam.ListAttachedRolePoliciesOutput, error) { + f.record("ListAttachedRolePolicies") + out := &iam.ListAttachedRolePoliciesOutput{} + for _, arn := range f.attachedRole { + out.AttachedPolicies = append(out.AttachedPolicies, &iam.AttachedPolicy{PolicyArn: aws.String(arn)}) + } + return out, nil +} + +func (f *fakeIAM) DetachRolePolicy(_ string, _ *iam.DetachRolePolicyInput) (*iam.DetachRolePolicyOutput, error) { + f.record("DetachRolePolicy") + return &iam.DetachRolePolicyOutput{}, nil +} + +func (f *fakeIAM) ListRolePolicies(_ string, _ *iam.ListRolePoliciesInput) (*iam.ListRolePoliciesOutput, error) { + f.record("ListRolePolicies") + out := &iam.ListRolePoliciesOutput{} + for _, name := range f.inlineRole { + out.PolicyNames = append(out.PolicyNames, aws.String(name)) + } + return out, nil +} + +func (f *fakeIAM) DeleteRolePolicy(_ string, _ *iam.DeleteRolePolicyInput) (*iam.DeleteRolePolicyOutput, error) { + f.record("DeleteRolePolicy") + return &iam.DeleteRolePolicyOutput{}, nil +} + +func (f *fakeIAM) DeleteRole(_ string, _ *iam.DeleteRoleInput) (*iam.DeleteRoleOutput, error) { + f.record("DeleteRole") + return &iam.DeleteRoleOutput{}, nil +} + +func (f *fakeIAM) ListGroups(_ string, _ *iam.ListGroupsInput) (*iam.ListGroupsOutput, error) { + f.record("ListGroups") + out := &iam.ListGroupsOutput{} + for _, name := range f.groups { + out.Groups = append(out.Groups, &iam.Group{GroupName: aws.String(name)}) + } + out.Groups = append(out.Groups, nil) + return out, nil +} + +func (f *fakeIAM) DeleteGroup(_ string, _ *iam.DeleteGroupInput) (*iam.DeleteGroupOutput, error) { + f.record("DeleteGroup") + return &iam.DeleteGroupOutput{}, nil +} + +func (f *fakeIAM) ListPolicies(_ string, input *iam.ListPoliciesInput) (*iam.ListPoliciesOutput, error) { + f.record("ListPolicies") + f.scopeAsked = aws.StringValue(input.Scope) + out := &iam.ListPoliciesOutput{} + for _, arn := range f.policies { + out.Policies = append(out.Policies, &iam.Policy{Arn: aws.String(arn)}) + } + out.Policies = append(out.Policies, nil) + return out, nil +} + +func (f *fakeIAM) DeletePolicy(_ string, _ *iam.DeletePolicyInput) (*iam.DeletePolicyOutput, error) { + f.record("DeletePolicy") + return &iam.DeletePolicyOutput{}, nil +} + +func (f *fakeIAM) ListInstanceProfiles(_ string, _ *iam.ListInstanceProfilesInput) (*iam.ListInstanceProfilesOutput, error) { + f.record("ListInstanceProfiles") + out := &iam.ListInstanceProfilesOutput{} + for _, name := range f.instanceProfiles { + out.InstanceProfiles = append(out.InstanceProfiles, &iam.InstanceProfile{ + InstanceProfileName: aws.String(name), + }) + } + out.InstanceProfiles = append(out.InstanceProfiles, nil) + return out, nil +} + +func (f *fakeIAM) GetInstanceProfile(_ string, input *iam.GetInstanceProfileInput) (*iam.GetInstanceProfileOutput, error) { + f.record("GetInstanceProfile") + profile := &iam.InstanceProfile{InstanceProfileName: input.InstanceProfileName} + for _, role := range f.profileRoles[aws.StringValue(input.InstanceProfileName)] { + profile.Roles = append(profile.Roles, &iam.Role{RoleName: aws.String(role)}) + } + return &iam.GetInstanceProfileOutput{InstanceProfile: profile}, nil +} + +func (f *fakeIAM) RemoveRoleFromInstanceProfile(_ string, _ *iam.RemoveRoleFromInstanceProfileInput) (*iam.RemoveRoleFromInstanceProfileOutput, error) { + f.record("RemoveRoleFromInstanceProfile") + return &iam.RemoveRoleFromInstanceProfileOutput{}, nil +} + +func (f *fakeIAM) DeleteInstanceProfile(_ string, _ *iam.DeleteInstanceProfileInput) (*iam.DeleteInstanceProfileOutput, error) { + f.record("DeleteInstanceProfile") + return &iam.DeleteInstanceProfileOutput{}, nil +} + +// Access keys are the second quiesce, so they must be removed before the users +// and roles that own them; instance profiles must be unbound before the roles +// they reference. +func TestIAMReapersAreOrderedForRemoval(t *testing.T) { + reapers := IAMReapers(newFakeIAM()) + + var kinds []string + for _, reaper := range reapers { + kinds = append(kinds, reaper.Kind()) + } + + assert.Equal(t, []string{ + "access-key", "instance-profile", "iam-user", "iam-role", "iam-group", "iam-policy", + "oidc-provider", + }, kinds) + + // Everything the account authenticates with goes in the identity stage, + // which is the second quiesce. + for _, reaper := range reapers[:indexOfKind(reapers, "oidc-provider")] { + assert.Equal(t, StageIdentity, reaper.Stage(), "%s is a credential", reaper.Kind()) + } + + // An OIDC provider is a trust anchor for an EKS cluster's service + // accounts, not a credential of this account's, and it outlives the + // cluster — so it is removed with the other platform leftovers. + assert.Equal(t, StagePlatform, reapers[indexOfKind(reapers, "oidc-provider")].Stage()) +} + +// Nothing else in teardown reaches an OIDC provider: EKS DeleteCluster does not +// remove it, so an account would be left holding a trust anchor for a cluster +// that no longer exists. +func TestOIDCProviderReaperListsAndDeletesByARN(t *testing.T) { + svc := newFakeIAM() + svc.oidcProviders = []string{ + "arn:aws:iam::000000000042:oidc-provider/oidc.spx.local/id/alpha", + } + + reaper := &oidcProviderReaper{svc: svc} + found, err := reaper.List(testCtx(t), "000000000042") + require.NoError(t, err) + require.Len(t, found, 1) + assert.Equal(t, "arn:aws:iam::000000000042:oidc-provider/oidc.spx.local/id/alpha", found[0].ID) + + require.NoError(t, reaper.Delete(testCtx(t), "000000000042", found[0], false)) + assert.Contains(t, svc.calls, + "DeleteOpenIDConnectProvider arn:aws:iam::000000000042:oidc-provider/oidc.spx.local/id/alpha") +} + +// Keys are addressed by user, so a key whose user is missed stays valid — the +// listing has to walk every user rather than a single one. +func TestAccessKeyReaperWalksEveryUser(t *testing.T) { + svc := newFakeIAM() + svc.users = []string{"admin", "deploy"} + svc.accessKeys = map[string][]string{"admin": {"AKIA1"}, "deploy": {"AKIA2", "AKIA3"}} + + found, err := (&accessKeyReaper{svc: svc}).List(testCtx(t), "000000000042") + require.NoError(t, err) + + require.Len(t, found, 3) + assert.Equal(t, "AKIA1", found[0].ID) + // The owning user has to travel with the key: DeleteAccessKey needs both. + assert.Equal(t, "admin", found[0].Detail) + assert.Equal(t, "deploy", found[2].Detail) +} + +// Reading a failed listing as an empty account is how credentials survive a +// teardown, so both listings fail loudly. +func TestAccessKeyReaperFailsClosed(t *testing.T) { + svc := newFakeIAM() + svc.listUsersErr = errors.New("kv unavailable") + _, err := (&accessKeyReaper{svc: svc}).List(testCtx(t), "000000000042") + assert.Error(t, err) + + svc = newFakeIAM() + svc.users = []string{"admin"} + svc.listKeysErr = errors.New("kv unavailable") + _, err = (&accessKeyReaper{svc: svc}).List(testCtx(t), "000000000042") + assert.Error(t, err) +} + +// A user with attachments cannot be deleted, so the attachments go first. +func TestUserReaperStripsPoliciesBeforeDeleting(t *testing.T) { + svc := newFakeIAM() + svc.attachedUser = []string{"arn:aws:iam::000000000042:policy/admin"} + svc.inlineUser = []string{"inline-1"} + + err := (&iamUserReaper{svc: svc}).Delete(testCtx(t), "000000000042", + Resource{Kind: "iam-user", ID: "admin"}, false) + require.NoError(t, err) + + assert.Equal(t, []string{ + "ListAttachedUserPolicies", "DetachUserPolicy", + "ListUserPolicies", "DeleteUserPolicy", + "DeleteUser", + }, svc.calls) +} + +func TestRoleReaperStripsPoliciesBeforeDeleting(t *testing.T) { + svc := newFakeIAM() + svc.attachedRole = []string{"arn:aws:iam::000000000042:policy/task"} + svc.inlineRole = []string{"inline-1"} + + err := (&iamRoleReaper{svc: svc}).Delete(testCtx(t), "000000000042", + Resource{Kind: "iam-role", ID: "task-role"}, false) + require.NoError(t, err) + + assert.Equal(t, []string{ + "ListAttachedRolePolicies", "DetachRolePolicy", + "ListRolePolicies", "DeleteRolePolicy", + "DeleteRole", + }, svc.calls) +} + +// A profile still referencing a role leaves the role reaper with a role it +// cannot delete, which reads as a stuck resource for an unrelated reason. +func TestInstanceProfileReaperUnbindsItsRoles(t *testing.T) { + svc := newFakeIAM() + svc.instanceProfiles = []string{"web"} + svc.profileRoles = map[string][]string{"web": {"web-role"}} + + reaper := &instanceProfileReaper{svc: svc} + found, err := reaper.List(testCtx(t), "000000000042") + require.NoError(t, err) + require.Len(t, found, 1) + + require.NoError(t, reaper.Delete(testCtx(t), "000000000042", found[0], false)) + assert.Equal(t, []string{ + "ListInstanceProfiles", "GetInstanceProfile", + "RemoveRoleFromInstanceProfile", "DeleteInstanceProfile", + }, svc.calls) +} + +// AWS-managed policies are shared with every other tenant and are not the +// account's to delete. +func TestPolicyReaperAsksOnlyForLocalPolicies(t *testing.T) { + svc := newFakeIAM() + svc.policies = []string{"arn:aws:iam::000000000042:policy/app"} + + reaper := &iamPolicyReaper{svc: svc} + found, err := reaper.List(testCtx(t), "000000000042") + require.NoError(t, err) + + assert.Equal(t, "Local", svc.scopeAsked) + require.Len(t, found, 1) + assert.Equal(t, "arn:aws:iam::000000000042:policy/app", found[0].ID) + assert.NoError(t, reaper.Delete(testCtx(t), "000000000042", found[0], false)) +} + +func TestUserRoleAndGroupListingsSkipEmptyRecords(t *testing.T) { + svc := newFakeIAM() + svc.users = []string{"admin"} + svc.roles = []string{"task-role"} + svc.groups = []string{"engineers"} + + ctx := testCtx(t) + users, err := (&iamUserReaper{svc: svc}).List(ctx, "000000000042") + require.NoError(t, err) + assert.Len(t, users, 1) + + roles, err := (&iamRoleReaper{svc: svc}).List(ctx, "000000000042") + require.NoError(t, err) + assert.Len(t, roles, 1) + + groupReaper := &iamGroupReaper{svc: svc} + groups, err := groupReaper.List(ctx, "000000000042") + require.NoError(t, err) + require.Len(t, groups, 1) + assert.NoError(t, groupReaper.Delete(ctx, "000000000042", groups[0], false)) +} + +// A key that is already gone is a success: teardown re-runs after a crash and +// would otherwise never finish what the first pass started. +func TestIdentityDeletesTolerateMissingRecords(t *testing.T) { + svc := newFakeIAM() + + assert.NoError(t, (&accessKeyReaper{svc: svc}).Delete(testCtx(t), "000000000042", + Resource{Kind: "access-key", ID: "AKIA1", Detail: "admin"}, false)) +} diff --git a/spinifex/accountteardown/reapers_rds.go b/spinifex/accountteardown/reapers_rds.go new file mode 100644 index 000000000..ae30956d4 --- /dev/null +++ b/spinifex/accountteardown/reapers_rds.go @@ -0,0 +1,167 @@ +package accountteardown + +import ( + "context" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/rds" + handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds" + "github.com/nats-io/nats.go" +) + +// RDSReapers returns the RDS-backed reapers. +// +// Instances go in the compute stage alongside everything else holding a volume +// and an address. The configuration groups outlive them and cannot be removed +// while an instance still references one, so they wait for the platform stage. +func RDSReapers(nc *nats.Conn) []Reaper { + svc := handlers_rds.NewNATSService(nc) + return []Reaper{ + &rdsInstanceReaper{svc: svc}, + &rdsSubnetGroupReaper{svc: svc}, + &rdsParameterGroupReaper{svc: svc}, + } +} + +// rdsDefaultParameterGroupPrefix marks the groups RDS synthesises per engine. +// They appear in every listing whether or not anything created them, and +// DeleteDBParameterGroup refuses them — so a reaper that offered them up would +// leave the platform stage unable to drain for any account at all. +const rdsDefaultParameterGroupPrefix = "default." + +type rdsInstanceReaper struct { + svc *handlers_rds.NATSService +} + +func (r *rdsInstanceReaper) Kind() string { return "rds-instance" } +func (r *rdsInstanceReaper) Stage() Stage { return StageCompute } + +func (r *rdsInstanceReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + out, err := r.svc.DescribeDBInstances(ctx, &rds.DescribeDBInstancesInput{}, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, instance := range out.DBInstances { + if instance == nil || instance.DBInstanceIdentifier == nil { + continue + } + // A deleting instance still holds its VM, volume and ENI, so it stays + // in the listing until it is actually gone rather than being counted + // as drained the moment it was asked to go. + found = append(found, Resource{ + Kind: r.Kind(), + ID: aws.StringValue(instance.DBInstanceIdentifier), + Detail: aws.StringValue(instance.DBInstanceStatus), + }) + } + return found, nil +} + +// Delete always skips the final snapshot. The AWS-faithful default is to take +// one, which inside an account being deleted either forces the storage stage to +// remove it again or, if it lands after that stage, leaks a snapshot with no +// owner to attribute it to. +func (r *rdsInstanceReaper) Delete(ctx context.Context, accountID string, resource Resource, force bool) error { + err := r.deleteInstance(ctx, accountID, resource.ID) + if err == nil || !force || !isDeletionProtected(err) { + return err + } + + // Force path: deletion protection is RDS's version of the deadlock the + // force flag exists for — the tenant is gone and nobody is left to clear + // the flag through the ordinary API. + if _, mErr := r.svc.ModifyDBInstance(ctx, &rds.ModifyDBInstanceInput{ + DBInstanceIdentifier: aws.String(resource.ID), + DeletionProtection: aws.Bool(false), + ApplyImmediately: aws.Bool(true), + }, accountID); mErr != nil { + return ignoreAlreadyGone(mErr) + } + return r.deleteInstance(ctx, accountID, resource.ID) +} + +func (r *rdsInstanceReaper) deleteInstance(ctx context.Context, accountID, id string) error { + _, err := r.svc.DeleteDBInstance(ctx, &rds.DeleteDBInstanceInput{ + DBInstanceIdentifier: aws.String(id), + SkipFinalSnapshot: aws.Bool(true), + }, accountID) + return ignoreAlreadyGone(err) +} + +// isDeletionProtected reports the one refusal the force path can do something +// about. Every other invalid-state answer means the instance is busy, and +// retrying the delete is what the drain loop already does. +func isDeletionProtected(err error) bool { + return err != nil && strings.Contains(err.Error(), "deletion protection is enabled") +} + +type rdsSubnetGroupReaper struct { + svc *handlers_rds.NATSService +} + +func (r *rdsSubnetGroupReaper) Kind() string { return "rds-subnet-group" } +func (r *rdsSubnetGroupReaper) Stage() Stage { return StagePlatform } + +func (r *rdsSubnetGroupReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + out, err := r.svc.DescribeDBSubnetGroups(ctx, &rds.DescribeDBSubnetGroupsInput{}, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, group := range out.DBSubnetGroups { + if group == nil || group.DBSubnetGroupName == nil { + continue + } + found = append(found, Resource{Kind: r.Kind(), ID: aws.StringValue(group.DBSubnetGroupName)}) + } + return found, nil +} + +func (r *rdsSubnetGroupReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + _, err := r.svc.DeleteDBSubnetGroup(ctx, &rds.DeleteDBSubnetGroupInput{ + DBSubnetGroupName: aws.String(resource.ID), + }, accountID) + return ignoreAlreadyGone(err) +} + +type rdsParameterGroupReaper struct { + svc *handlers_rds.NATSService +} + +func (r *rdsParameterGroupReaper) Kind() string { return "rds-parameter-group" } +func (r *rdsParameterGroupReaper) Stage() Stage { return StagePlatform } + +func (r *rdsParameterGroupReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + out, err := r.svc.DescribeDBParameterGroups(ctx, &rds.DescribeDBParameterGroupsInput{}, accountID) + if err != nil { + return nil, err + } + + var found []Resource + for _, group := range out.DBParameterGroups { + if group == nil || group.DBParameterGroupName == nil { + continue + } + name := aws.StringValue(group.DBParameterGroupName) + if isRDSDefaultParameterGroup(name) { + continue + } + found = append(found, Resource{Kind: r.Kind(), ID: name}) + } + return found, nil +} + +func (r *rdsParameterGroupReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error { + _, err := r.svc.DeleteDBParameterGroup(ctx, &rds.DeleteDBParameterGroupInput{ + DBParameterGroupName: aws.String(resource.ID), + }, accountID) + return ignoreAlreadyGone(err) +} + +func isRDSDefaultParameterGroup(name string) bool { + return strings.HasPrefix(strings.ToLower(name), rdsDefaultParameterGroupPrefix) +} diff --git a/spinifex/accountteardown/reapers_rds_test.go b/spinifex/accountteardown/reapers_rds_test.go new file mode 100644 index 000000000..ca4567164 --- /dev/null +++ b/spinifex/accountteardown/reapers_rds_test.go @@ -0,0 +1,272 @@ +package accountteardown + +//test:in-package — the reapers are unexported, and the snapshot choice and +// default-group filter asserted here are the substance of them. + +import ( + "encoding/json" + "sync" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/rds" + handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeRDS answers the rds.* subjects, with per-subject reply queues so a +// delete that succeeds only after a modify can be scripted. +type fakeRDS struct { + nc *nats.Conn + + mu sync.Mutex + calls []string + requests map[string][][]byte + queued map[string][]any + errorCode map[string][]string +} + +func newFakeRDS(t *testing.T) *fakeRDS { + t.Helper() + _, nc := testutil.StartTestNATS(t) + + fake := &fakeRDS{ + nc: nc, + requests: map[string][][]byte{}, + queued: map[string][]any{}, + errorCode: map[string][]string{}, + } + + sub, err := nc.Subscribe("rds.>", fake.serve) + require.NoError(t, err) + t.Cleanup(func() { _ = sub.Unsubscribe() }) + require.NoError(t, nc.Flush()) + + return fake +} + +func (f *fakeRDS) serve(msg *nats.Msg) { + f.mu.Lock() + f.calls = append(f.calls, msg.Subject) + f.requests[msg.Subject] = append(f.requests[msg.Subject], msg.Data) + + code := "" + if queue := f.errorCode[msg.Subject]; len(queue) > 0 { + code = queue[0] + f.errorCode[msg.Subject] = queue[1:] + } + + var reply any + hasReply := false + if queue := f.queued[msg.Subject]; len(queue) > 0 { + reply = queue[0] + hasReply = true + if len(queue) > 1 { + f.queued[msg.Subject] = queue[1:] + } + } + f.mu.Unlock() + + switch { + case code != "": + _ = msg.Respond(utils.GenerateErrorPayload(code)) + case hasReply: + payload, err := json.Marshal(reply) + if err != nil { + _ = msg.Respond(utils.GenerateErrorPayload("InternalError")) + return + } + _ = msg.Respond(payload) + default: + _ = msg.Respond([]byte(`{}`)) + } +} + +func (f *fakeRDS) reply(subject string, outputs ...any) { + f.mu.Lock() + defer f.mu.Unlock() + f.queued[subject] = append(f.queued[subject], outputs...) +} + +// failNext queues one error per call, in order, so a first attempt can fail +// and a retry succeed. +func (f *fakeRDS) failNext(subject string, codes ...string) { + f.mu.Lock() + defer f.mu.Unlock() + f.errorCode[subject] = append(f.errorCode[subject], codes...) +} + +func (f *fakeRDS) requestAt(t *testing.T, subject string, index int, into any) { + t.Helper() + f.mu.Lock() + payloads := f.requests[subject] + f.mu.Unlock() + require.Greater(t, len(payloads), index, "subject %s was called %d times", subject, len(payloads)) + require.NoError(t, json.Unmarshal(payloads[index], into)) +} + +func (f *fakeRDS) called() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.calls...) +} + +func newRDSReapers(nc *nats.Conn) (*rdsInstanceReaper, *rdsSubnetGroupReaper, *rdsParameterGroupReaper) { + svc := handlers_rds.NewNATSService(nc) + return &rdsInstanceReaper{svc: svc}, &rdsSubnetGroupReaper{svc: svc}, &rdsParameterGroupReaper{svc: svc} +} + +// A deleting instance still holds its VM, volume and ENI, so it stays listed +// until it is actually gone rather than counting as drained when asked to go. +func TestRDSInstanceReaperListsInstancesThatAreAlreadyDeleting(t *testing.T) { + fake := newFakeRDS(t) + fake.reply("rds.DescribeDBInstances", rds.DescribeDBInstancesOutput{ + DBInstances: []*rds.DBInstance{ + {DBInstanceIdentifier: aws.String("orders"), DBInstanceStatus: aws.String("available")}, + {DBInstanceIdentifier: aws.String("reports"), DBInstanceStatus: aws.String("deleting")}, + }, + }) + + instances, _, _ := newRDSReapers(fake.nc) + found, err := instances.List(testCtx(t), "000000000042") + require.NoError(t, err) + + assert.Equal(t, []Resource{ + {Kind: "rds-instance", ID: "orders", Detail: "available"}, + {Kind: "rds-instance", ID: "reports", Detail: "deleting"}, + }, found) +} + +// The AWS-faithful default takes a final snapshot. Inside an account being +// deleted that either forces the storage stage to remove it again or leaks a +// snapshot with no owner, so teardown must always skip it. +func TestRDSInstanceReaperAlwaysSkipsTheFinalSnapshot(t *testing.T) { + fake := newFakeRDS(t) + fake.reply("rds.DeleteDBInstance", rds.DeleteDBInstanceOutput{}) + + instances, _, _ := newRDSReapers(fake.nc) + require.NoError(t, instances.Delete(testCtx(t), "000000000042", + Resource{Kind: "rds-instance", ID: "orders"}, false)) + + var input rds.DeleteDBInstanceInput + fake.requestAt(t, "rds.DeleteDBInstance", 0, &input) + assert.True(t, aws.BoolValue(input.SkipFinalSnapshot)) + assert.Empty(t, aws.StringValue(input.FinalDBSnapshotIdentifier)) +} + +// Deletion protection is RDS's version of the deadlock --force exists for: the +// tenant is gone and nobody is left to clear the flag through the ordinary API. +func TestRDSInstanceReaperForceClearsDeletionProtection(t *testing.T) { + fake := newFakeRDS(t) + fake.failNext("rds.DeleteDBInstance", + "InvalidParameterCombination: DB instance orders cannot be deleted because deletion protection is enabled") + fake.reply("rds.ModifyDBInstance", rds.ModifyDBInstanceOutput{}) + fake.reply("rds.DeleteDBInstance", rds.DeleteDBInstanceOutput{}) + + instances, _, _ := newRDSReapers(fake.nc) + require.NoError(t, instances.Delete(testCtx(t), "000000000042", + Resource{Kind: "rds-instance", ID: "orders"}, true)) + + var modify rds.ModifyDBInstanceInput + fake.requestAt(t, "rds.ModifyDBInstance", 0, &modify) + assert.False(t, aws.BoolValue(modify.DeletionProtection)) + assert.True(t, aws.BoolValue(modify.ApplyImmediately)) + + assert.Equal(t, []string{ + "rds.DeleteDBInstance", "rds.ModifyDBInstance", "rds.DeleteDBInstance", + }, fake.called(), "the flag is cleared only after the ordinary delete refuses") +} + +// Without --force the refusal stands. Silently disarming a protection the +// operator did not ask to override is not teardown's call to make. +func TestRDSInstanceReaperLeavesDeletionProtectionAloneWithoutForce(t *testing.T) { + fake := newFakeRDS(t) + fake.failNext("rds.DeleteDBInstance", + "InvalidParameterCombination: DB instance orders cannot be deleted because deletion protection is enabled") + + instances, _, _ := newRDSReapers(fake.nc) + err := instances.Delete(testCtx(t), "000000000042", + Resource{Kind: "rds-instance", ID: "orders"}, false) + + require.Error(t, err) + assert.NotContains(t, fake.called(), "rds.ModifyDBInstance") +} + +// Force is not a licence to escalate past every refusal — only past the one it +// can do something about. A busy instance is retried by the drain loop instead. +func TestRDSInstanceReaperForceDoesNotEscalatePastOtherFailures(t *testing.T) { + fake := newFakeRDS(t) + fake.failNext("rds.DeleteDBInstance", "InternalError") + + instances, _, _ := newRDSReapers(fake.nc) + err := instances.Delete(testCtx(t), "000000000042", + Resource{Kind: "rds-instance", ID: "orders"}, true) + + require.Error(t, err) + assert.NotContains(t, fake.called(), "rds.ModifyDBInstance") +} + +// RDS synthesises a default parameter group per engine into every listing, +// whether or not anything created one, and refuses to delete it. Offering them +// up would leave the platform stage unable to drain for any account at all. +func TestRDSParameterGroupReaperSkipsTheSynthesisedDefaults(t *testing.T) { + fake := newFakeRDS(t) + fake.reply("rds.DescribeDBParameterGroups", rds.DescribeDBParameterGroupsOutput{ + DBParameterGroups: []*rds.DBParameterGroup{ + {DBParameterGroupName: aws.String("default.postgres16")}, + {DBParameterGroupName: aws.String("Default.MariaDB11")}, + {DBParameterGroupName: aws.String("app-tuned")}, + }, + }) + + _, _, groups := newRDSReapers(fake.nc) + found, err := groups.List(testCtx(t), "000000000042") + require.NoError(t, err) + + assert.Equal(t, []Resource{{Kind: "rds-parameter-group", ID: "app-tuned"}}, found) +} + +func TestRDSSubnetGroupReaperListsAndDeletes(t *testing.T) { + fake := newFakeRDS(t) + fake.reply("rds.DescribeDBSubnetGroups", rds.DescribeDBSubnetGroupsOutput{ + DBSubnetGroups: []*rds.DBSubnetGroup{{DBSubnetGroupName: aws.String("db-private")}}, + }) + fake.reply("rds.DeleteDBSubnetGroup", rds.DeleteDBSubnetGroupOutput{}) + + _, subnets, _ := newRDSReapers(fake.nc) + found, err := subnets.List(testCtx(t), "000000000042") + require.NoError(t, err) + require.Equal(t, []Resource{{Kind: "rds-subnet-group", ID: "db-private"}}, found) + + require.NoError(t, subnets.Delete(testCtx(t), "000000000042", found[0], false)) + + var input rds.DeleteDBSubnetGroupInput + fake.requestAt(t, "rds.DeleteDBSubnetGroup", 0, &input) + assert.Equal(t, "db-private", aws.StringValue(input.DBSubnetGroupName)) +} + +func TestRDSReaperTreatsAMissingResourceAsDeleted(t *testing.T) { + fake := newFakeRDS(t) + fake.failNext("rds.DeleteDBInstance", "DBInstanceNotFound") + fake.failNext("rds.DeleteDBSubnetGroup", "DBSubnetGroupNotFoundFault") + fake.failNext("rds.DeleteDBParameterGroup", "DBParameterGroupNotFound") + + instances, subnets, groups := newRDSReapers(fake.nc) + assert.NoError(t, instances.Delete(testCtx(t), "000000000042", Resource{ID: "gone"}, false)) + assert.NoError(t, subnets.Delete(testCtx(t), "000000000042", Resource{ID: "gone"}, false)) + assert.NoError(t, groups.Delete(testCtx(t), "000000000042", Resource{ID: "gone"}, false)) +} + +// A configuration group cannot be removed while an instance still references +// it, so the groups wait for a later stage than the instances do. +func TestRDSGroupsAreReapedAfterInstances(t *testing.T) { + reapers := RDSReapers(nil) + + assert.Equal(t, StageCompute, reapers[indexOfKind(reapers, "rds-instance")].Stage()) + assert.Equal(t, StagePlatform, reapers[indexOfKind(reapers, "rds-subnet-group")].Stage()) + assert.Equal(t, StagePlatform, reapers[indexOfKind(reapers, "rds-parameter-group")].Stage()) +} diff --git a/spinifex/accountteardown/reapers_s3.go b/spinifex/accountteardown/reapers_s3.go new file mode 100644 index 000000000..f953c571b --- /dev/null +++ b/spinifex/accountteardown/reapers_s3.go @@ -0,0 +1,139 @@ +package accountteardown + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/mulgadc/spinifex/spinifex/objectstore" +) + +// BucketStore is the slice of the object store the bucket reaper needs. +// +// Narrow on purpose, and read from the tenant's side only: nothing here can +// create a bucket or write an object, so a teardown bug cannot leave new state +// behind in an account it is emptying. +type BucketStore interface { + ListBucketsForOwner(ctx context.Context, accountID string) ([]string, error) + ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) + DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) + ListMultipartUploads(ctx context.Context, bucket string) ([]objectstore.MultipartUploadRef, error) + AbortMultipartUpload(ctx context.Context, upload objectstore.MultipartUploadRef) error + DeleteBucket(ctx context.Context, bucket string) error +} + +var _ BucketStore = (*objectstore.S3ObjectStore)(nil) + +// S3Reapers returns the S3-backed reapers. +// +// One reaper whose unit is a bucket, rather than separate object and bucket +// reapers. Objects are only interesting as a reason a bucket will not delete, +// and listing every object of every bucket on each sweep — which is what a +// stage's drain check does — would make emptying a large bucket quadratic. +func S3Reapers(store BucketStore) []Reaper { + return []Reaper{&bucketReaper{store: store}} +} + +// bucketPageLimit bounds a runaway pagination loop. It exists so a listing that +// never advances cannot spin the drain loop for the whole stage budget; a real +// bucket empties long before this. +const bucketPageLimit = 10000 + +type bucketReaper struct { + store BucketStore +} + +func (r *bucketReaper) Kind() string { return "s3-bucket" } +func (r *bucketReaper) Stage() Stage { return StageStorage } + +// List returns only the buckets the account owns. The platform buckets — the +// config bucket and the system bucket holding volume metadata, AMIs and key +// material — belong to the system account and are shared by every tenant, so +// the owner scoping is the whole safety property rather than a filter applied +// afterwards. +func (r *bucketReaper) List(ctx context.Context, accountID string) ([]Resource, error) { + names, err := r.store.ListBucketsForOwner(ctx, accountID) + if err != nil { + return nil, err + } + + found := make([]Resource, 0, len(names)) + for _, name := range names { + found = append(found, Resource{Kind: r.Kind(), ID: name}) + } + return found, nil +} + +// Delete empties the bucket and then removes it. All three steps run on every +// call: a bucket that could not be deleted last time is still listed, and the +// retry has to re-check what is in it rather than assume the first pass emptied +// it. +func (r *bucketReaper) Delete(ctx context.Context, _ string, resource Resource, _ bool) error { + bucket := resource.ID + + if err := r.abortUploads(ctx, bucket); err != nil { + return err + } + if err := r.deleteObjects(ctx, bucket); err != nil { + return err + } + + return ignoreAlreadyGone(r.store.DeleteBucket(ctx, bucket)) +} + +// abortUploads discards uploads in flight. They are invisible to an object +// listing and hold real storage, so a bucket can list as empty and still have +// parts in it — and predastore's bucket delete only checks for objects, so +// nothing else would ever notice them. +func (r *bucketReaper) abortUploads(ctx context.Context, bucket string) error { + uploads, err := r.store.ListMultipartUploads(ctx, bucket) + if err != nil { + return ignoreAlreadyGone(err) + } + + for _, upload := range uploads { + if err := r.store.AbortMultipartUpload(ctx, upload); err != nil && !isAlreadyGone(err) { + return fmt.Errorf("abort upload %s in %s: %w", upload.UploadID, bucket, err) + } + } + return nil +} + +// deleteObjects empties the bucket a page at a time, re-listing rather than +// trusting one page. Objects go one at a time because predastore serves no +// batch delete. +func (r *bucketReaper) deleteObjects(ctx context.Context, bucket string) error { + var token *string + + for range bucketPageLimit { + out, err := r.store.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + ContinuationToken: token, + }) + if err != nil { + return ignoreAlreadyGone(err) + } + + for _, object := range out.Contents { + if object == nil || aws.StringValue(object.Key) == "" { + continue + } + _, err := r.store.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: object.Key, + }) + // A key that is already gone is a success: teardown re-runs after a + // crash and would otherwise never get past the first object. + if err != nil && !objectstore.IsNoSuchKeyError(err) && !isAlreadyGone(err) { + return fmt.Errorf("delete %s/%s: %w", bucket, aws.StringValue(object.Key), err) + } + } + + if !aws.BoolValue(out.IsTruncated) || aws.StringValue(out.NextContinuationToken) == "" { + return nil + } + token = out.NextContinuationToken + } + return fmt.Errorf("bucket %s still listing objects after %d pages", bucket, bucketPageLimit) +} diff --git a/spinifex/accountteardown/reapers_s3_test.go b/spinifex/accountteardown/reapers_s3_test.go new file mode 100644 index 000000000..d7e144ee7 --- /dev/null +++ b/spinifex/accountteardown/reapers_s3_test.go @@ -0,0 +1,293 @@ +package accountteardown + +//test:in-package — the reaper is unexported, and the order it empties a bucket +// in is the substance of it. + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/mulgadc/spinifex/spinifex/objectstore" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeBuckets is an in-memory object store keyed by owner, so a reaper that +// reaches past the account it was given is visible rather than merely unproven. +type fakeBuckets struct { + mu sync.Mutex + + // owners maps account id to the buckets it owns. + owners map[string][]string + // objects maps bucket to its keys, in the order they list. + objects map[string][]string + // uploads maps bucket to the uploads in flight against it. + uploads map[string][]objectstore.MultipartUploadRef + // pageSize truncates object listings so pagination is exercised. + pageSize int + + calls []string + deleted []string + failWith map[string]error +} + +func newFakeBuckets() *fakeBuckets { + return &fakeBuckets{ + owners: map[string][]string{}, + objects: map[string][]string{}, + uploads: map[string][]objectstore.MultipartUploadRef{}, + failWith: map[string]error{}, + } +} + +func (f *fakeBuckets) record(call string) { + f.calls = append(f.calls, call) +} + +func (f *fakeBuckets) ListBucketsForOwner(_ context.Context, accountID string) ([]string, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.record("ListBuckets") + if err := f.failWith["ListBuckets"]; err != nil { + return nil, err + } + return append([]string(nil), f.owners[accountID]...), nil +} + +func (f *fakeBuckets) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.record("ListObjects") + if err := f.failWith["ListObjects"]; err != nil { + return nil, err + } + + bucket := aws.StringValue(input.Bucket) + keys := f.objects[bucket] + + start := 0 + if token := aws.StringValue(input.ContinuationToken); token != "" { + for i, key := range keys { + if key == token { + start = i + break + } + } + } + + end := len(keys) + if f.pageSize > 0 && start+f.pageSize < end { + end = start + f.pageSize + } + + out := &s3.ListObjectsV2Output{IsTruncated: aws.Bool(end < len(keys))} + if end < len(keys) { + out.NextContinuationToken = aws.String(keys[end]) + } + for _, key := range keys[start:end] { + out.Contents = append(out.Contents, &s3.Object{Key: aws.String(key)}) + } + return out, nil +} + +func (f *fakeBuckets) DeleteObject(_ context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.record("DeleteObject") + if err := f.failWith["DeleteObject"]; err != nil { + return nil, err + } + + bucket, key := aws.StringValue(input.Bucket), aws.StringValue(input.Key) + f.deleted = append(f.deleted, bucket+"/"+key) + + kept := make([]string, 0, len(f.objects[bucket])) + for _, existing := range f.objects[bucket] { + if existing != key { + kept = append(kept, existing) + } + } + f.objects[bucket] = kept + return &s3.DeleteObjectOutput{}, nil +} + +func (f *fakeBuckets) ListMultipartUploads(_ context.Context, bucket string) ([]objectstore.MultipartUploadRef, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.record("ListUploads") + if err := f.failWith["ListUploads"]; err != nil { + return nil, err + } + return append([]objectstore.MultipartUploadRef(nil), f.uploads[bucket]...), nil +} + +func (f *fakeBuckets) AbortMultipartUpload(_ context.Context, upload objectstore.MultipartUploadRef) error { + f.mu.Lock() + defer f.mu.Unlock() + f.record("AbortUpload") + if err := f.failWith["AbortUpload"]; err != nil { + return err + } + f.deleted = append(f.deleted, "upload:"+upload.Bucket+"/"+upload.UploadID) + f.uploads[upload.Bucket] = nil + return nil +} + +func (f *fakeBuckets) DeleteBucket(_ context.Context, bucket string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.record("DeleteBucket") + if err := f.failWith["DeleteBucket"]; err != nil { + return err + } + if len(f.objects[bucket]) > 0 { + return awserr.New("BucketNotEmpty", "The bucket you tried to delete is not empty", nil) + } + f.deleted = append(f.deleted, "bucket:"+bucket) + delete(f.owners, bucket) + return nil +} + +var _ BucketStore = (*fakeBuckets)(nil) + +func newBucketReaper(store BucketStore) *bucketReaper { + return &bucketReaper{store: store} +} + +// The platform buckets belong to the system account and are shared by every +// tenant. Deleting one takes the cluster down, so the reaper must never so +// much as return one. +func TestBucketReaperListsOnlyTheAccountsOwnBuckets(t *testing.T) { + fake := newFakeBuckets() + fake.owners["000000000042"] = []string{"tenant-logs", "tenant-data"} + fake.owners["000000000000"] = []string{"predastore", "spinifex-system"} + + found, err := newBucketReaper(fake).List(testCtx(t), "000000000042") + require.NoError(t, err) + + assert.Equal(t, []Resource{ + {Kind: "s3-bucket", ID: "tenant-logs"}, + {Kind: "s3-bucket", ID: "tenant-data"}, + }, found) +} + +// Uploads are aborted before the objects go, because an abort that ran after +// the bucket delete would have nothing left to address. +func TestBucketReaperAbortsUploadsThenEmptiesThenDeletes(t *testing.T) { + fake := newFakeBuckets() + fake.objects["tenant-data"] = []string{"a.txt", "b.txt"} + fake.uploads["tenant-data"] = []objectstore.MultipartUploadRef{ + {Bucket: "tenant-data", Key: "big.img", UploadID: "u-1"}, + } + + require.NoError(t, newBucketReaper(fake).Delete(testCtx(t), "000000000042", + Resource{Kind: "s3-bucket", ID: "tenant-data"}, false)) + + assert.Equal(t, []string{ + "upload:tenant-data/u-1", "tenant-data/a.txt", "tenant-data/b.txt", "bucket:tenant-data", + }, fake.deleted) +} + +// An incomplete upload is invisible to an object listing and predastore's +// bucket delete only checks for objects, so nothing else would ever notice the +// parts it holds. +func TestBucketReaperAbortsUploadsInAnOtherwiseEmptyBucket(t *testing.T) { + fake := newFakeBuckets() + fake.uploads["tenant-data"] = []objectstore.MultipartUploadRef{ + {Bucket: "tenant-data", Key: "big.img", UploadID: "u-1"}, + } + + require.NoError(t, newBucketReaper(fake).Delete(testCtx(t), "000000000042", + Resource{Kind: "s3-bucket", ID: "tenant-data"}, false)) + + assert.Contains(t, fake.deleted, "upload:tenant-data/u-1") + assert.Contains(t, fake.deleted, "bucket:tenant-data") +} + +// Trusting one page would leave every object past it behind, and the bucket +// delete would then fail for a reason that looks unrelated. +func TestBucketReaperFollowsObjectPagination(t *testing.T) { + fake := newFakeBuckets() + fake.pageSize = 2 + fake.objects["tenant-data"] = []string{"a", "b", "c", "d", "e"} + + require.NoError(t, newBucketReaper(fake).Delete(testCtx(t), "000000000042", + Resource{Kind: "s3-bucket", ID: "tenant-data"}, false)) + + assert.Empty(t, fake.objects["tenant-data"]) + assert.Contains(t, fake.deleted, "bucket:tenant-data") +} + +// Teardown re-runs after a crash. An object deleted by the previous attempt +// must not stop the second one before it starts. +func TestBucketReaperTreatsAMissingObjectAsDeleted(t *testing.T) { + fake := newFakeBuckets() + fake.objects["tenant-data"] = []string{"a.txt"} + fake.failWith["DeleteObject"] = &objectstore.NoSuchKeyError{Key: "a.txt"} + + err := newBucketReaper(fake).Delete(testCtx(t), "000000000042", + Resource{Kind: "s3-bucket", ID: "tenant-data"}, false) + + // The bucket is still not empty in the fake, so the delete is refused and + // the resource stays listed for the drain loop to retry. What matters is + // that the missing key did not fail the sweep. + require.Error(t, err) + assert.NotContains(t, err.Error(), "NoSuchKey") +} + +func TestBucketReaperTreatsAMissingBucketAsDeleted(t *testing.T) { + fake := newFakeBuckets() + fake.failWith["DeleteBucket"] = awserr.New("NoSuchBucket", "The specified bucket does not exist", nil) + + err := newBucketReaper(fake).Delete(testCtx(t), "000000000042", + Resource{Kind: "s3-bucket", ID: "gone"}, false) + + assert.NoError(t, err) +} + +// A bucket that will not empty has to stay listed. Reporting the delete as a +// success would let the account record go with the data still in place. +func TestBucketReaperReportsABucketItCannotEmpty(t *testing.T) { + fake := newFakeBuckets() + fake.objects["tenant-data"] = []string{"a.txt"} + fake.failWith["DeleteObject"] = errors.New("backend unavailable") + + err := newBucketReaper(fake).Delete(testCtx(t), "000000000042", + Resource{Kind: "s3-bucket", ID: "tenant-data"}, false) + + require.Error(t, err) + assert.NotContains(t, fake.deleted, "bucket:tenant-data") +} + +// Buckets are storage, not compute: they must not be reaped before the +// instances and images that read from them are gone. +func TestS3ReaperRunsInTheStorageStage(t *testing.T) { + reapers := S3Reapers(newFakeBuckets()) + + require.Len(t, reapers, 1) + assert.Equal(t, StageStorage, reapers[0].Stage()) + assert.Equal(t, "s3-bucket", reapers[0].Kind()) +} + +// The engine sorts reapers by stage and the sort is stable, so a bucket reaper +// registered after the EC2 storage reapers stays after them. +func TestSortReapersKeepsBucketsAfterVolumes(t *testing.T) { + reapers := append(EC2Reapers(nil, 1), S3Reapers(newFakeBuckets())...) + SortReapers(reapers) + + storage := make([]string, 0, len(reapers)) + for _, reaper := range reapers { + if reaper.Stage() == StageStorage { + storage = append(storage, reaper.Kind()) + } + } + + require.NotEmpty(t, storage) + assert.Greater(t, len(storage), 1, "the storage stage owns more than buckets") + assert.Equal(t, "s3-bucket", storage[len(storage)-1]) +} diff --git a/spinifex/accountteardown/types.go b/spinifex/accountteardown/types.go new file mode 100644 index 000000000..44d4afc42 --- /dev/null +++ b/spinifex/accountteardown/types.go @@ -0,0 +1,222 @@ +// Package accountteardown removes everything an account owns, in dependency +// order, and then the account itself. +// +// It is deliberately separate from the handlers it drives. Teardown is the one +// caller that must be able to delete a resource the customer-facing API would +// refuse — an attached volume, an instance whose node stopped answering — and +// keeping that ability in its own package means no ordinary request path can +// reach it by accident. +package accountteardown + +import ( + "context" + "errors" + "fmt" + "time" +) + +// Stage groups resources that must all be gone before the next group is +// touched. The ordering is the whole reason teardown converges: it is what +// stops a volume from being undeletable because an instance still holds it. +type Stage int + +const ( + // StageCompute releases everything that holds an attachment: instances, + // spot requests, container and database workloads. + StageCompute Stage = iota + 1 + + // StageAttachments verifies stage 1 actually landed. Detaches are + // asynchronous, so "the instance is gone" and "the volume is free" are + // separate facts and storage deletion depends on the second one. + StageAttachments + + // StageStorage removes snapshots, then volumes, then images and buckets. + StageStorage + + // StageNetwork unwinds the VPC in reverse creation order. + StageNetwork + + // StagePlatform removes account-scoped odds and ends with no ordering + // relationship to each other. + StagePlatform + + // StageIdentity removes credentials and IAM records. Access keys go first: + // it is the second quiesce, and it holds even if the status gate does not. + StageIdentity + + // StageAccount removes the account's own counters and reservations. The + // account record itself is deleted by the engine after this stage. + StageAccount +) + +var stageNames = map[Stage]string{ + StageCompute: "compute", + StageAttachments: "attachments", + StageStorage: "storage", + StageNetwork: "network", + StagePlatform: "platform", + StageIdentity: "identity", + StageAccount: "account", +} + +func (s Stage) String() string { + if name, ok := stageNames[s]; ok { + return name + } + return fmt.Sprintf("stage(%d)", int(s)) +} + +// Stages returns every stage in teardown order. +func Stages() []Stage { + return []Stage{ + StageCompute, StageAttachments, StageStorage, + StageNetwork, StagePlatform, StageIdentity, StageAccount, + } +} + +// Resource is one thing to delete. Detail carries whatever an operator needs +// to understand why it is here or why it will not go — an attachment, a +// dependency, the node holding it — and is never load-bearing. +type Resource struct { + Kind string `json:"kind"` + ID string `json:"id"` + Detail string `json:"detail,omitempty"` +} + +func (r Resource) String() string { + if r.Detail == "" { + return r.Kind + " " + r.ID + } + return r.Kind + " " + r.ID + " (" + r.Detail + ")" +} + +// Reaper enumerates and deletes one kind of resource for one account. +// +// List must be safe to call repeatedly: the engine re-lists after deleting to +// decide whether a stage has actually drained, because most deletes here are +// asynchronous. +// +// Delete must be idempotent. A resource that is already gone is a success, not +// a NotFound — teardown re-runs after a crash and would otherwise never finish. +type Reaper interface { + Kind() string + Stage() Stage + List(ctx context.Context, accountID string) ([]Resource, error) + Delete(ctx context.Context, accountID string, resource Resource, force bool) error +} + +// AccountStore is the slice of the IAM service teardown needs. Narrow on +// purpose: this package must not be able to create anything. +type AccountStore interface { + GetAccount(accountID string) (*Account, error) + SetAccountStatus(accountID, status string) (*Account, error) + DeleteAccount(accountID string) error +} + +// Account mirrors the IAM account record. Redeclared rather than imported so +// the engine's tests do not need an IAM service to run. +type Account struct { + AccountID string + AccountName string + Status string +} + +// Request is one teardown. +type Request struct { + AccountID string + + // AccountName must match the stored record. A mistyped account id then + // fails closed instead of emptying a real tenant. + AccountName string + + // Force escalates past state guards, hard-destroys on timeout, and treats + // an already-missing resource as deleted. It never reorders the stages and + // never widens scope beyond AccountID. + Force bool + + // DryRun reports what would be deleted and touches nothing. + DryRun bool +} + +// StageResult is what happened in one stage. +type StageResult struct { + Stage Stage `json:"stage"` + Deleted []Resource `json:"deleted,omitempty"` + Stuck []Stuck `json:"stuck,omitempty"` + Elapsed string `json:"elapsed"` +} + +// Stuck is a resource teardown could not remove, and why. It is the most +// valuable thing this package produces: each one is a bug in a delete path. +type Stuck struct { + Resource Resource `json:"resource"` + Reason string `json:"reason"` +} + +// Result is the outcome of a teardown or a dry run. +type Result struct { + AccountID string `json:"accountId"` + AccountName string `json:"accountName"` + DryRun bool `json:"dryRun"` + Forced bool `json:"forced"` + Stages []StageResult `json:"stages"` + + // AccountDeleted is false whenever anything was left stuck: the account + // stays TERMINATING so the residue stays attributable to it. + AccountDeleted bool `json:"accountDeleted"` +} + +// StuckCount totals resources left behind across every stage. +func (r *Result) StuckCount() int { + total := 0 + for _, stage := range r.Stages { + total += len(stage.Stuck) + } + return total +} + +// DeletedCount totals resources removed across every stage. +func (r *Result) DeletedCount() int { + total := 0 + for _, stage := range r.Stages { + total += len(stage.Deleted) + } + return total +} + +// ErrAccountNameMismatch reports a confirmation that did not match the stored +// account name. +var ErrAccountNameMismatch = errors.New("account name does not match the account id") + +// ErrProtectedAccount reports an attempt to delete the system or super-admin +// account. No credential grants this. +var ErrProtectedAccount = errors.New("account is protected and cannot be deleted") + +// ErrResourcesStuck reports a teardown that drained everything it could and +// still found resources it could not remove. +var ErrResourcesStuck = errors.New("teardown left resources behind") + +// protectedAccountIDs are the two accounts no teardown may ever remove. They +// mirror admin.SystemAccountID and admin.DefaultAccountID; a test asserts the +// two lists agree so a change to either is caught rather than diverging. +var protectedAccountIDs = map[string]string{ + "000000000000": "system", + "000000000001": "super admin", +} + +// Timeouts bounds how long teardown will wait for a stage to drain. +type Timeouts struct { + // StageDrain is the total budget for one stage, including re-listing after + // deletes. Exceeding it marks whatever is left as stuck rather than + // blocking the whole teardown on one resource. + StageDrain time.Duration + + // DrainPoll is the gap between re-listing a stage that has not emptied. + DrainPoll time.Duration +} + +// DefaultTimeouts are sized for instance termination, which is the long pole: +// a guest gets a graceful shutdown before the hypervisor takes the VM away. +func DefaultTimeouts() Timeouts { + return Timeouts{StageDrain: 5 * time.Minute, DrainPoll: 5 * time.Second} +} diff --git a/spinifex/admin/images_remove.go b/spinifex/admin/images_remove.go index d55d916e3..ea44b1f7c 100644 --- a/spinifex/admin/images_remove.go +++ b/spinifex/admin/images_remove.go @@ -147,7 +147,7 @@ func FindAMIDependents(store objectstore.ObjectStore, bucket, imageID string) (D if snapID == SnapPrefix(imageID) { continue } - cfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(store, bucket, snapID) + cfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(context.Background(), store, bucket, snapID) if err != nil { if objectstore.IsNoSuchKeyError(err) || errors.Is(err, handlers_ec2_snapshot.ErrCorruptSnapshotMetadata) { continue diff --git a/spinifex/awserrors/awserrors.go b/spinifex/awserrors/awserrors.go index 8ae391619..6f62e01cd 100644 --- a/spinifex/awserrors/awserrors.go +++ b/spinifex/awserrors/awserrors.go @@ -794,7 +794,7 @@ var ErrorLookup = map[string]ErrorMessage{ ErrorInsufficientVolumeCapacity: {HTTPCode: 503, Message: "There is not enough capacity to fulfill your EBS volume provision request. You can try to provision a different volume type, EBS volume in a different availability zone, or you can wait for additional capacity to become available."}, ErrorInterfaceInUseByTrafficMirrorSession: {HTTPCode: 409, Message: "The Traffic Mirror source that you are trying to create uses an interface that is already associated with a session. An interface can only be associated with a session, or with a target, but not both."}, ErrorInterfaceInUseByTrafficMirrorTarget: {HTTPCode: 409, Message: "The Traffic Mirror source that you are trying to create uses an interface that is already associated with a target. An interface can only be associated with a session, or with a target, but not both. If the interface is associated with a target, it cannot be associated with another target."}, - ErrorInternalError: {HTTPCode: 500, Message: "An internal error has occurred. Retry your request, but if the problem persists, contact us with details. "}, + ErrorInternalError: {HTTPCode: 500, Message: "An internal error has occurred. Retry your request, and if the problem persists, contact Mulga support with the details."}, ErrorInternalFailure: {HTTPCode: 404, Message: "The request processing has failed because of an unknown error, exception, or failure."}, ErrorInternetGatewayLimitExceeded: {HTTPCode: 400, Message: "You've reached the limit on the number of internet gateways that you can create. For more information, see Amazon VPC quotas."}, ErrorInvalidAMIAttributeItemValue: {HTTPCode: 400, Message: "The value of an item added to, or removed from, an image attribute is not valid. If you are specifying a userId, check that it is in the form of an AWS account ID, without hyphens."}, @@ -1107,7 +1107,7 @@ var ErrorLookup = map[string]ErrorMessage{ ErrorSecurityGroupsPerInstanceLimitExceeded: {HTTPCode: 400, Message: "You've reached the limit on the number of security groups that you can assign to an instance. For more information, see Amazon EC2 security groups."}, ErrorSecurityGroupsPerInterfaceLimitExceeded: {HTTPCode: 400, Message: "You've reached the limit on the number of security groups you can associate with the specified network interface. For more information, see Amazon VPC quotas."}, ErrorSerialConsoleSessionUnavailable: {HTTPCode: 403, Message: "The serial console access is not enabled for this account. Use EnableSerialConsoleAccess to enable access."}, - ErrorServerInternal: {HTTPCode: 500, Message: "An internal error has occurred. Retry your request, but if the problem persists, contact us with details by posting a message on AWS re:Post."}, + ErrorServerInternal: {HTTPCode: 500, Message: "An internal error has occurred. Retry your request, and if the problem persists, contact Mulga support with the details."}, ErrorServiceUnavailable: {HTTPCode: 503, Message: "The request has failed due to a temporary failure of the server."}, ErrorSignatureDoesNotMatch: {HTTPCode: 403, Message: "The request signature that Amazon has does not match the signature that you provided. Check your AWS credentials and signing method."}, ErrorSnapshotCopyUnsupportedInterRegion: {HTTPCode: 400, Message: "Inter-region snapshot copy is not supported for this AWS Region."}, diff --git a/spinifex/daemon/daemon.go b/spinifex/daemon/daemon.go index 79b3b5917..8161eacd7 100644 --- a/spinifex/daemon/daemon.go +++ b/spinifex/daemon/daemon.go @@ -851,6 +851,9 @@ func (d *Daemon) subscribeAll() error { {"ec2.DescribeVolumes", handleNATSRequest(d.volumeService.DescribeVolumes), "spinifex-workers"}, {"ec2.ModifyVolume", d.handleEC2ModifyVolume, "spinifex-workers"}, {"ec2.DeleteVolume", handleNATSRequest(d.volumeService.DeleteVolume), "spinifex-workers"}, + // Cluster-wide on purpose: the QMP detach is instance-scoped, so a + // volume held by an unreachable host has no other way out. + {"ec2.ForceDetachVolume", handleNATSRequest(d.volumeService.ForceDetachVolume), "spinifex-workers"}, {"ec2.DescribeVolumeStatus", handleNATSRequest(d.volumeService.DescribeVolumeStatus), "spinifex-workers"}, {"ec2.DescribeVolumesModifications", handleNATSRequest(d.volumeService.DescribeVolumesModifications), "spinifex-workers"}, {"ec2.CreateSnapshot", handleNATSRequest(d.snapshotService.CreateSnapshot), "spinifex-workers"}, diff --git a/spinifex/daemon/ecs_deps.go b/spinifex/daemon/ecs_deps.go index 30fa57890..f2bfd5e22 100644 --- a/spinifex/daemon/ecs_deps.go +++ b/spinifex/daemon/ecs_deps.go @@ -46,11 +46,21 @@ func (d *Daemon) buildECSServiceDeps() handlers_ecs.Deps { if d.clusterConfig != nil { clusterSize = len(d.clusterConfig.Nodes) } - if iamSvc, iamErr := handlers_iam.NewIAMServiceImpl(d.ctx, d.natsConn, masterKey, clusterSize); iamErr != nil { - slog.Warn("ECS: IAM service init failed; capacity provisioning disabled", "err", iamErr) - } else { - deps.IAM = iamSvc + // Retried, because the constructor runs a KV migration that needs JetStream + // to have responders. A daemon that started first got one attempt, failed + // it, and had capacity provisioning off for the life of the process with no + // symptom but a 500 from ProvisionCapacity. + iamSvc, iamErr := initServiceWithRetry("ECS IAM service", func() (*handlers_iam.IAMServiceImpl, error) { + return handlers_iam.NewIAMServiceImpl(d.ctx, d.natsConn, masterKey, clusterSize) + }) + if iamErr != nil { + // Off rather than fatal: ECS CRUD, EKS and EC2 do not need this, and a + // node that serves them is worth more than one that exits. + slog.Warn("ECS: IAM service unavailable; capacity provisioning disabled for this process", + "err", iamErr) + return deps } + deps.IAM = iamSvc return deps } diff --git a/spinifex/daemon/ecs_deps_test.go b/spinifex/daemon/ecs_deps_test.go new file mode 100644 index 000000000..d81972493 --- /dev/null +++ b/spinifex/daemon/ecs_deps_test.go @@ -0,0 +1,29 @@ +// Exercises buildECSServiceDeps, which is unexported wiring with no exported +// surface to drive it through. +// +//test:in-package +package daemon + +import ( + "path/filepath" + "testing" + + "github.com/mulgadc/spinifex/spinifex/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A worker with no master key still serves the rest of ECS. Capacity +// provisioning is the only thing the IAM service gates, so disabling it must +// not take the gateway URL, the image resolver or the launcher with it. +func TestBuildECSServiceDeps_NoMasterKeyDisablesOnlyCapacity(t *testing.T) { + d := &Daemon{ + config: &config.Config{}, + configPath: filepath.Join(t.TempDir(), "spinifex.toml"), + } + + deps := d.buildECSServiceDeps() + + assert.Nil(t, deps.IAM, "capacity provisioning is disabled without a master key") + require.NotNil(t, deps.RunInstances, "the launcher is independent of IAM") +} diff --git a/spinifex/ebsmetadata/store.go b/spinifex/ebsmetadata/store.go index 29341363e..162f864e7 100644 --- a/spinifex/ebsmetadata/store.go +++ b/spinifex/ebsmetadata/store.go @@ -7,6 +7,8 @@ import ( "fmt" "io" "log/slog" + "sync" + "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" @@ -113,12 +115,29 @@ const ( amiPrefix = "spinifex/ebsmetadata/v1/amis/" ) +// listFetchConcurrency bounds the object fetches a single listing may have in +// flight. A listing costs one round trip per document, so fetching them one at +// a time makes the call scale with the number of documents in the bucket rather +// than with the answer's size. +const listFetchConcurrency = 16 + +// listFetchTimeout bounds how long one document may hold up a listing. A +// document whose shards no longer reassemble does not fail quickly — the read +// runs its full retry budget first — so without a bound a single bad document +// adds its whole latency to every listing that walks the prefix. It sits well +// under the request timeouts callers work to, and comfortably above a healthy +// read of a metadata document. +var listFetchTimeout = 10 * time.Second + // listDocuments reads and decodes every document under prefix. // // skipCorrupt tolerates a document that cannot be fetched as well as one that // cannot be decoded: an object whose shards no longer join is exactly as // unusable as one whose bytes will not parse, and either kind read wholesale // as a failure turns a single bad document into a cluster-wide outage. +// +// Fetches run concurrently but results are assembled in key order, so a strict +// listing fails on the same document it would have failed on serially. func listDocuments[T any]( ctx context.Context, s *Store, @@ -133,28 +152,63 @@ func listDocuments[T any]( return nil, err } - documents := make([]T, 0, len(result.Contents)) - for _, object := range result.Contents { + // fetchErr and decodeErr are kept apart because they are tolerated on + // different terms: any fetch failure may be skipped, but only a decode + // failure that is ErrCorruptDocument may be. + type fetched struct { + key string + document T + fetchErr error + decodeErr error + empty bool + } + + results := make([]fetched, len(result.Contents)) + slots := make(chan struct{}, listFetchConcurrency) + var wg sync.WaitGroup + + for i, object := range result.Contents { if object.Key == nil { + results[i].empty = true continue } - data, err := s.get(ctx, *object.Key) - if err != nil { - if skipCorrupt { - slog.WarnContext(ctx, "skipping unreadable "+kind+" document", "key", *object.Key, "err", err) - continue + wg.Add(1) + slots <- struct{}{} + go func(idx int, key string) { + defer wg.Done() + defer func() { <-slots }() + + results[idx].key = key + fetchCtx, cancel := context.WithTimeout(ctx, listFetchTimeout) + defer cancel() + data, err := s.get(fetchCtx, key) + if err != nil { + results[idx].fetchErr = err + return } - return nil, err - } - document, err := unmarshal(data) - if err != nil { - if skipCorrupt && errors.Is(err, ErrCorruptDocument) { - slog.WarnContext(ctx, "skipping undecodable "+kind+" document", "key", *object.Key, "err", err) - continue + results[idx].document, results[idx].decodeErr = unmarshal(data) + }(i, *object.Key) + } + wg.Wait() + + documents := make([]T, 0, len(results)) + for _, r := range results { + switch { + case r.empty: + continue + case r.fetchErr != nil: + if !skipCorrupt { + return nil, r.fetchErr } - return nil, err + slog.WarnContext(ctx, "skipping unreadable "+kind+" document", "key", r.key, "err", r.fetchErr) + case r.decodeErr != nil: + if !skipCorrupt || !errors.Is(r.decodeErr, ErrCorruptDocument) { + return nil, r.decodeErr + } + slog.WarnContext(ctx, "skipping undecodable "+kind+" document", "key", r.key, "err", r.decodeErr) + default: + documents = append(documents, r.document) } - documents = append(documents, document) } return documents, nil } diff --git a/spinifex/ebsmetadata/store_test.go b/spinifex/ebsmetadata/store_test.go index 17fa5f2e4..4bf0f83cd 100644 --- a/spinifex/ebsmetadata/store_test.go +++ b/spinifex/ebsmetadata/store_test.go @@ -4,7 +4,10 @@ import ( "bytes" "context" "errors" + "fmt" + "sync/atomic" "testing" + "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" @@ -283,3 +286,163 @@ func TestListAMIs_SkipsCorruptButStrictDoesNot(t *testing.T) { _, err = store.ListAMIsStrict(ctx) require.ErrorIs(t, err, ErrCorruptDocument) } + +// TestListVolumes_FetchesDocumentsConcurrently is the reason listDocuments has +// a worker pool at all. A listing costs one object fetch per document, and +// fetching them one at a time makes DescribeVolumes scale with the number of +// volumes in the cluster rather than with the number the caller owns. +func TestListVolumes_FetchesDocumentsConcurrently(t *testing.T) { + objects := objectstore.NewMemoryObjectStore() + store := NewStore(objects, "control-plane") + ctx := context.Background() + + const documents = 32 + const delay = 20 * time.Millisecond + for i := range documents { + require.NoError(t, store.PutVolume(ctx, Volume{ + VolumeID: fmt.Sprintf("vol-%02d", i), TenantID: "acct-1", CapacityGiB: 1, + })) + } + + slow := &slowGetStore{ObjectStore: objects, delay: delay} + measured := NewStore(slow, "control-plane") + + start := time.Now() + volumes, err := measured.ListVolumes(ctx) + elapsed := time.Since(start) + + require.NoError(t, err) + require.Len(t, volumes, documents) + + serial := documents * delay + assert.Less(t, elapsed, serial/2, + "fetching %d documents took %s; serially it would be %s", documents, elapsed, serial) + assert.Equal(t, int64(documents), slow.gets.Load(), "every document must still be fetched") + assert.LessOrEqual(t, slow.peak(), int64(listFetchConcurrency), + "the pool bound must hold: an unbounded fan-out over a large bucket is its own problem") +} + +// TestListVolumes_ConcurrentFetchPreservesListingOrder pins the ordering the +// concurrent fetch has to reproduce: whatever order the listing came back in. +// Strict callers depend on it, because it decides which of several unusable +// documents is the one whose error is returned. +func TestListVolumes_ConcurrentFetchPreservesListingOrder(t *testing.T) { + objects := objectstore.NewMemoryObjectStore() + store := NewStore(objects, "control-plane") + ctx := context.Background() + + const documents = 24 + want := make([]string, 0, documents) + keys := make([]string, 0, documents) + for i := range documents { + id := fmt.Sprintf("vol-%02d", i) + require.NoError(t, store.PutVolume(ctx, Volume{VolumeID: id, TenantID: "acct-1", CapacityGiB: 1})) + key, err := VolumeKey(id) + require.NoError(t, err) + want = append(want, id) + keys = append(keys, key) + } + + fixed := NewStore(&fixedOrderStore{ObjectStore: objects, keys: keys}, "control-plane") + for range 5 { + volumes, err := fixed.ListVolumes(ctx) + require.NoError(t, err) + got := make([]string, 0, len(volumes)) + for _, v := range volumes { + got = append(got, v.VolumeID) + } + assert.Equal(t, want, got, "documents must follow the order the listing returned") + } +} + +// fixedOrderStore lists a known set of keys in a known order, because the +// in-memory store lists in map order and cannot pin an ordering guarantee. +type fixedOrderStore struct { + objectstore.ObjectStore + + keys []string +} + +func (s *fixedOrderStore) ListObjectsV2(_ context.Context, _ *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { + contents := make([]*s3.Object, 0, len(s.keys)) + for _, key := range s.keys { + contents = append(contents, &s3.Object{Key: aws.String(key)}) + } + return &s3.ListObjectsV2Output{Contents: contents}, nil +} + +// slowGetStore delays every fetch and records how many ran at once, so a test +// can tell a concurrent listing from a serial one and check the pool bound. +type slowGetStore struct { + objectstore.ObjectStore + + delay time.Duration + gets atomic.Int64 + inFlight atomic.Int64 + highest atomic.Int64 +} + +func (s *slowGetStore) GetObject(ctx context.Context, in *s3.GetObjectInput) (*s3.GetObjectOutput, error) { + s.gets.Add(1) + now := s.inFlight.Add(1) + for { + high := s.highest.Load() + if now <= high || s.highest.CompareAndSwap(high, now) { + break + } + } + defer s.inFlight.Add(-1) + + time.Sleep(s.delay) + return s.ObjectStore.GetObject(ctx, in) +} + +func (s *slowGetStore) peak() int64 { return s.highest.Load() } + +// hangingGetStore never answers for one key, standing in for a document whose +// read runs its full retry budget before failing. +type hangingGetStore struct { + objectstore.ObjectStore + + hangKey string +} + +func (s *hangingGetStore) GetObject(ctx context.Context, in *s3.GetObjectInput) (*s3.GetObjectOutput, error) { + if in.Key != nil && *in.Key == s.hangKey { + <-ctx.Done() + return nil, ctx.Err() + } + return s.ObjectStore.GetObject(ctx, in) +} + +// TestListVolumes_BoundsOneSlowDocument covers the shape that made a single bad +// volume document a cluster-wide problem: the read did not fail quickly, so its +// latency was added to every listing that walked the prefix. +func TestListVolumes_BoundsOneSlowDocument(t *testing.T) { + previous := listFetchTimeout + listFetchTimeout = 50 * time.Millisecond + t.Cleanup(func() { listFetchTimeout = previous }) + + objects := objectstore.NewMemoryObjectStore() + store := NewStore(objects, "control-plane") + ctx := context.Background() + + require.NoError(t, store.PutVolume(ctx, Volume{VolumeID: "vol-good", CapacityGiB: 1})) + require.NoError(t, store.PutVolume(ctx, Volume{VolumeID: "vol-slow", CapacityGiB: 1})) + key, err := VolumeKey("vol-slow") + require.NoError(t, err) + + hanging := NewStore(&hangingGetStore{ObjectStore: objects, hangKey: key}, "control-plane") + + start := time.Now() + volumes, err := hanging.ListVolumes(ctx) + elapsed := time.Since(start) + + require.NoError(t, err, "one slow document must not fail the whole listing") + require.Len(t, volumes, 1) + assert.Equal(t, "vol-good", volumes[0].VolumeID) + assert.Less(t, elapsed, time.Second, "the listing must not wait on the slow document indefinitely") + + _, err = hanging.ListVolumesStrict(ctx) + require.Error(t, err, "the strict listing must still report the document it could not read") +} diff --git a/spinifex/gateway/admin.go b/spinifex/gateway/admin.go index ebad1e2f5..06fcd4010 100644 --- a/spinifex/gateway/admin.go +++ b/spinifex/gateway/admin.go @@ -6,6 +6,7 @@ import ( "io" "log/slog" "net/http" + "sort" "strings" "github.com/go-chi/chi/v5" @@ -19,7 +20,22 @@ import ( // is rejected before any parsing, so adding a handler is a deliberate act // rather than a side effect of adding a function. var adminMethods = map[string]bool{ - "CreateAccount": true, + "CreateAccount": true, + "DeleteAccount": true, + "DescribeAccountDeletion": true, + "ListAccounts": true, +} + +// AdminMethodNames returns the callable /admin/ names in sorted order. +// Tooling that mints a credential grants these by name, so the list has to come +// from the router rather than from a copy that can drift out of step with it. +func AdminMethodNames() []string { + names := make([]string, 0, len(adminMethods)) + for name := range adminMethods { + names = append(names, name) + } + sort.Strings(names) + return names } // adminPathPrefix is the private admin surface's URL prefix. It is matched @@ -107,6 +123,12 @@ func (gw *GatewayConfig) Admin_Request(w http.ResponseWriter, r *http.Request) { switch method { case "CreateAccount": output, err = gw.adminCreateAccount(ctx, body) + case "DeleteAccount": + output, err = gw.adminDeleteAccount(ctx, body) + case "DescribeAccountDeletion": + output, err = gw.adminDescribeAccountDeletion(ctx, body) + case "ListAccounts": + output, err = gw.adminListAccounts(ctx, body) default: // Unreachable: adminMethods gates the switch. Fail closed anyway so // adding a name to the map without a case cannot silently return 200. diff --git a/spinifex/gateway/admin_deleteaccount.go b/spinifex/gateway/admin_deleteaccount.go new file mode 100644 index 000000000..c0cfedafe --- /dev/null +++ b/spinifex/gateway/admin_deleteaccount.go @@ -0,0 +1,571 @@ +package gateway + +import ( + "context" + "encoding/json" + "errors" + "log/slog" + "regexp" + "strings" + "sync" + "time" + + "github.com/google/uuid" + "github.com/mulgadc/spinifex/spinifex/accountteardown" + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/kvutil" + "github.com/nats-io/nats.go/jetstream" +) + +// KVBucketAccountDeletions holds one deletion job per account, keyed by account +// ID. Keying by account rather than by deletion ID is deliberate: an account is +// deleted once, and the record is what keeps "what happened to account X" +// answerable after the account itself is gone. +const KVBucketAccountDeletions = "spinifex-account-deletions" + +// accountDeletionTimeout is the outer bound on a whole teardown. It must stay +// well above the sum of the engine's per-stage drain budgets: the stages are +// what decide when to give up on a resource, and this cutting in first would +// abandon a teardown midway with no stuck report to show for it. +const accountDeletionTimeout = 2 * time.Hour + +// accountDeletionHeartbeat is how often a running job refreshes its record. A +// job whose heartbeat has stopped for accountDeletionStale is assumed dead — +// the gateway running it was restarted — and another gateway may take it over. +const ( + accountDeletionHeartbeat = 30 * time.Second + accountDeletionStale = 5 * time.Minute +) + +// accountDeletionInventoryTimeout bounds a dry run, which only lists. It is +// answered inline, so it must not hold an HTTP request for minutes. +const accountDeletionInventoryTimeout = 2 * time.Minute + +// Deletion job states. +const ( + DeletionStateRunning = "RUNNING" + DeletionStateCompleted = "COMPLETED" + DeletionStateFailed = "FAILED" +) + +var accountIDRE = regexp.MustCompile(`^[0-9]{12}$`) + +// DeleteAccountRequest is the /admin/DeleteAccount request body. +type DeleteAccountRequest struct { + AccountID string `json:"accountId"` + AccountName string `json:"accountName"` + ClientToken string `json:"clientToken"` + Force bool `json:"force,omitempty"` + DryRun bool `json:"dryRun,omitempty"` +} + +// DeleteAccountResponse acknowledges a started (or replayed) teardown. The +// teardown itself runs in the background: it takes minutes, and holding the +// request open for it would make every retry start a second one. +type DeleteAccountResponse struct { + DeletionID string `json:"deletionId"` + AccountID string `json:"accountId"` + State string `json:"state"` + DryRun bool `json:"dryRun"` + Inventory *accountteardown.Result `json:"inventory,omitempty"` +} + +// DescribeAccountDeletionRequest is the /admin/DescribeAccountDeletion body. +type DescribeAccountDeletionRequest struct { + AccountID string `json:"accountId"` +} + +// accountDeletionJob is the stored progress of one teardown. It is written +// before the teardown starts and updated after every stage, so a crash leaves +// an attributable record of how far it got. +type accountDeletionJob struct { + DeletionID string `json:"deletionId"` + AccountID string `json:"accountId"` + AccountName string `json:"accountName"` + ClientToken string `json:"clientToken"` + State string `json:"state"` + Force bool `json:"force"` + StartedAt string `json:"startedAt"` + // UpdatedAt is the running job's heartbeat. Its absence for long enough is + // what lets another gateway take over after a restart. + UpdatedAt string `json:"updatedAt"` + FinishedAt string `json:"finishedAt,omitempty"` + Stages []accountteardown.StageResult `json:"stages,omitempty"` + // Error is the failure reason, including the stuck-resource summary. The + // per-stage Stuck entries carry which resources and why. + Error string `json:"error,omitempty"` +} + +// adminDeleteAccount starts a teardown, or answers a dry run inline. +func (gw *GatewayConfig) adminDeleteAccount(ctx context.Context, body []byte) (any, error) { + var req DeleteAccountRequest + if err := json.Unmarshal(body, &req); err != nil { + slog.Debug("DeleteAccount: malformed JSON body", "err", err) + return nil, errors.New(awserrors.ErrorInvalidRequest) + } + if err := validateDeleteAccountRequest(&req); err != nil { + return nil, err + } + if gw.IAMService == nil { + slog.Error("DeleteAccount: IAM service not available") + return nil, errors.New(awserrors.ErrorInternalError) + } + + engine, err := accountteardown.NewClusterEngine(ctx, gw.NATSConn, gw.ExpectedNodes, gw.IAMService, gw.BucketStore) + if err != nil { + slog.Error("DeleteAccount: failed to build teardown engine", "err", err) + return nil, errors.New(awserrors.ErrorServiceUnavailable) + } + + if req.DryRun { + inventoryCtx, cancel := context.WithTimeout(ctx, accountDeletionInventoryTimeout) + defer cancel() + + inventory, err := engine.Inventory(inventoryCtx, req.AccountID) + if err != nil { + return nil, teardownError("DeleteAccount", req.AccountID, err) + } + return &DeleteAccountResponse{ + AccountID: req.AccountID, State: "DRY_RUN", DryRun: true, Inventory: inventory, + }, nil + } + + jobs, err := gw.accountDeletionStore(ctx) + if err != nil { + return nil, err + } + + // A retry carrying the stored token is answered from the record before + // anything else. Once the teardown has finished the account is gone, and an + // inventory would report the retry as a missing account. + replay, err := replayDeletionJob(ctx, jobs, &req) + if err != nil { + return nil, err + } + if replay != nil { + return deletionJobResponse(replay), nil + } + + // Confirm the account is deletable before claiming the job, so a mistyped + // id, a wrong name confirmation or a protected account fails the request + // rather than a background job nobody is watching. + if err := engine.Precheck(req.AccountID, req.AccountName); err != nil { + return nil, teardownError("DeleteAccount", req.AccountID, err) + } + + job, replay, err := claimDeletionJob(ctx, jobs, &req) + if err != nil { + return nil, err + } + if replay != nil { + return deletionJobResponse(replay), nil + } + + gw.runAccountDeletion(engine, jobs, job) + + return &DeleteAccountResponse{ + DeletionID: job.DeletionID, AccountID: job.AccountID, State: job.State, + }, nil +} + +// adminDescribeAccountDeletion returns the stored job for an account, including +// one that finished — the record outlives the account it describes. +func (gw *GatewayConfig) adminDescribeAccountDeletion(ctx context.Context, body []byte) (any, error) { + var req DescribeAccountDeletionRequest + if err := json.Unmarshal(body, &req); err != nil { + slog.Debug("DescribeAccountDeletion: malformed JSON body", "err", err) + return nil, errors.New(awserrors.ErrorInvalidRequest) + } + req.AccountID = strings.TrimSpace(req.AccountID) + if req.AccountID == "" { + return nil, errors.New(awserrors.ErrorMissingParameter) + } + if !accountIDRE.MatchString(req.AccountID) { + return nil, errors.New(awserrors.ErrorInvalidParameterValue) + } + + jobs, err := gw.accountDeletionStore(ctx) + if err != nil { + return nil, err + } + + job, err := readDeletionJob(ctx, jobs, req.AccountID) + if err != nil { + return nil, err + } + if job == nil { + return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) + } + return job, nil +} + +// AccountSummary is one row of /admin/ListAccounts. It carries no credential +// material: the caller can already create and delete accounts, but that is no +// reason for a listing to hand out keys. +type AccountSummary struct { + AccountID string `json:"accountId"` + AccountName string `json:"accountName"` + Status string `json:"status"` + CreatedAt string `json:"createdAt,omitempty"` +} + +// ListAccountsResponse is the /admin/ListAccounts success body. +type ListAccountsResponse struct { + Accounts []AccountSummary `json:"accounts"` + Count int `json:"count"` +} + +// adminListAccounts enumerates every account. It takes no parameters: the +// account cap bounds the listing, so paging would be complexity with no payer. +func (gw *GatewayConfig) adminListAccounts(_ context.Context, _ []byte) (any, error) { + if gw.IAMService == nil { + slog.Error("ListAccounts: IAM service not available") + return nil, errors.New(awserrors.ErrorInternalError) + } + + accounts, err := gw.IAMService.ListAccounts() + if err != nil { + slog.Error("ListAccounts: failed to list accounts", "err", err) + return nil, errors.New(awserrors.ErrorInternalError) + } + + response := &ListAccountsResponse{Accounts: make([]AccountSummary, 0, len(accounts))} + for _, account := range accounts { + if account == nil { + continue + } + response.Accounts = append(response.Accounts, AccountSummary{ + AccountID: account.AccountID, + AccountName: account.AccountName, + Status: account.Status, + CreatedAt: account.CreatedAt, + }) + } + response.Count = len(response.Accounts) + return response, nil +} + +// runAccountDeletion runs the teardown in the background on its own context. +// The request's context is cancelled the moment the response is written, and a +// teardown cancelled halfway leaves the account half-emptied. +func (gw *GatewayConfig) runAccountDeletion( + engine *accountteardown.Engine, + jobs jetstream.KeyValue, + job *accountDeletionJob, +) { + tracker := &deletionTracker{jobs: jobs, job: job} + engine.OnStage = tracker.stageFinished + + go func() { + ctx, cancel := context.WithTimeout(context.Background(), accountDeletionTimeout) + defer cancel() + + stop := tracker.startHeartbeat(ctx) + defer stop() + + result, err := engine.Teardown(ctx, accountteardown.Request{ + AccountID: job.AccountID, + AccountName: job.AccountName, + Force: job.Force, + }) + tracker.finished(ctx, result, err) + }() +} + +// deletionTracker keeps the stored job in step with a teardown as it runs. Its +// mutex covers the heartbeat and the stage callback, which write concurrently. +type deletionTracker struct { + jobs jetstream.KeyValue + + mu sync.Mutex + job *accountDeletionJob +} + +func (t *deletionTracker) stageFinished(ctx context.Context, stage accountteardown.StageResult) { + t.mu.Lock() + t.job.Stages = append(t.job.Stages, stage) + t.mu.Unlock() + t.persist(ctx) +} + +func (t *deletionTracker) finished(ctx context.Context, result *accountteardown.Result, err error) { + t.mu.Lock() + if result != nil { + t.job.Stages = result.Stages + } + if err != nil { + t.job.State = DeletionStateFailed + t.job.Error = err.Error() + } else { + t.job.State = DeletionStateCompleted + } + t.job.FinishedAt = time.Now().UTC().Format(time.RFC3339) + accountID, state := t.job.AccountID, t.job.State + t.mu.Unlock() + + t.persist(ctx) + if err != nil { + slog.Error("DeleteAccount: teardown failed", "accountID", accountID, "err", err) + return + } + slog.Info("DeleteAccount: teardown complete", "accountID", accountID, "state", state) +} + +// startHeartbeat refreshes UpdatedAt while the teardown runs. Without it a +// gateway restarted mid-teardown leaves a job that looks live forever, and the +// account can never be retried through the API. +func (t *deletionTracker) startHeartbeat(ctx context.Context) func() { + ticker := time.NewTicker(accountDeletionHeartbeat) + done := make(chan struct{}) + + go func() { + defer ticker.Stop() + for { + select { + case <-done: + return + case <-ctx.Done(): + return + case <-ticker.C: + t.persist(ctx) + } + } + }() + + return func() { close(done) } +} + +func (t *deletionTracker) persist(ctx context.Context) { + t.mu.Lock() + t.job.UpdatedAt = time.Now().UTC().Format(time.RFC3339) + record, err := json.Marshal(t.job) + accountID := t.job.AccountID + t.mu.Unlock() + + if err != nil { + slog.Error("DeleteAccount: failed to marshal job record", "accountID", accountID, "err", err) + return + } + if _, err := t.jobs.Put(ctx, accountID, record); err != nil { + // The teardown itself is unaffected; only its progress report is. Losing + // the run over a KV write would be the worse outcome. + slog.Error("DeleteAccount: failed to store job progress", "accountID", accountID, "err", err) + } +} + +// claimDeletionJob CAS-creates the job record, which is what stops two +// gateways tearing the same account down at once. It returns a non-nil replay +// when an existing job answers the request instead. +func claimDeletionJob( + ctx context.Context, + jobs jetstream.KeyValue, + req *DeleteAccountRequest, +) (*accountDeletionJob, *accountDeletionJob, error) { + now := time.Now().UTC() + job := &accountDeletionJob{ + DeletionID: uuid.NewString(), + AccountID: req.AccountID, + AccountName: req.AccountName, + ClientToken: req.ClientToken, + State: DeletionStateRunning, + Force: req.Force, + StartedAt: now.Format(time.RFC3339), + UpdatedAt: now.Format(time.RFC3339), + } + record, err := json.Marshal(job) + if err != nil { + slog.Error("DeleteAccount: failed to marshal job record", "err", err) + return nil, nil, errors.New(awserrors.ErrorInternalError) + } + + if _, err := jobs.Create(ctx, req.AccountID, record); err == nil { + return job, nil, nil + } else if !errors.Is(err, jetstream.ErrKeyExists) { + slog.Error("DeleteAccount: failed to claim deletion job", "accountID", req.AccountID, "err", err) + return nil, nil, errors.New(awserrors.ErrorInternalError) + } + + // Another gateway claimed it between the replay check and here, so the same + // token has to be recognised a second time. + if replay, err := replayDeletionJob(ctx, jobs, req); err != nil || replay != nil { + return nil, replay, err + } + + existing, err := readDeletionJob(ctx, jobs, req.AccountID) + if err != nil { + return nil, nil, err + } + if existing == nil { + // Deleted between Create and Get. Retryable rather than a fault. + return nil, nil, errors.New(awserrors.ErrorOperationInProgress) + } + + if existing.State == DeletionStateRunning && !deletionJobStale(existing, now) { + return nil, nil, errors.New(awserrors.ErrorOperationInProgress) + } + + // A failed job, or one whose gateway stopped heartbeating, is retried under + // a new deletion id. Teardown is idempotent: it re-lists what is left. + if _, err := jobs.Put(ctx, req.AccountID, record); err != nil { + slog.Error("DeleteAccount: failed to take over deletion job", "accountID", req.AccountID, "err", err) + return nil, nil, errors.New(awserrors.ErrorInternalError) + } + slog.Warn("DeleteAccount: retrying a job that did not finish", + "accountID", req.AccountID, "previousDeletionID", existing.DeletionID, "previousState", existing.State) + return job, nil, nil +} + +// replayDeletionJob answers a retry from the stored record. A token bound to a +// different account name is a client bug rather than a retry, so it is refused +// here too instead of starting a second teardown under the same token. +func replayDeletionJob( + ctx context.Context, + jobs jetstream.KeyValue, + req *DeleteAccountRequest, +) (*accountDeletionJob, error) { + existing, err := readDeletionJob(ctx, jobs, req.AccountID) + if err != nil || existing == nil { + return nil, err + } + if existing.ClientToken == req.ClientToken { + if existing.AccountName != req.AccountName { + return nil, errors.New(awserrors.ErrorIdempotentParameterMismatch) + } + return existing, nil + } + + // A finished teardown answers any token. The account is gone, so starting a + // second one would only fail on a missing account record — which is what a + // caller retrying under a fresh token would otherwise be told. + if existing.State == DeletionStateCompleted { + return existing, nil + } + return nil, nil +} + +// deletionJobResponse acknowledges an existing job. It carries no stage detail: +// DescribeAccountDeletion is where progress is read from, and duplicating it +// here would make the two disagree. +func deletionJobResponse(job *accountDeletionJob) *DeleteAccountResponse { + slog.Info("DeleteAccount: replayed existing job", + "accountID", job.AccountID, "deletionID", job.DeletionID, "state", job.State) + return &DeleteAccountResponse{ + DeletionID: job.DeletionID, AccountID: job.AccountID, State: job.State, + } +} + +// deletionJobStale reports whether a RUNNING job has stopped heartbeating for +// long enough that the gateway running it is presumed gone. +func deletionJobStale(job *accountDeletionJob, now time.Time) bool { + updated, err := time.Parse(time.RFC3339, job.UpdatedAt) + if err != nil { + // An unparseable heartbeat cannot prove the job is alive, and refusing + // forever on a malformed record is worse than allowing a retry. + return true + } + return now.Sub(updated) > accountDeletionStale +} + +func readDeletionJob(ctx context.Context, jobs jetstream.KeyValue, accountID string) (*accountDeletionJob, error) { + entry, err := jobs.Get(ctx, accountID) + if err != nil { + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, nil + } + slog.Error("DeleteAccount: failed to read deletion job", "accountID", accountID, "err", err) + return nil, errors.New(awserrors.ErrorInternalError) + } + + var job accountDeletionJob + if err := json.Unmarshal(entry.Value(), &job); err != nil { + slog.Error("DeleteAccount: failed to unmarshal deletion job", "accountID", accountID, "err", err) + return nil, errors.New(awserrors.ErrorInternalError) + } + return &job, nil +} + +// accountDeletionStore opens the job bucket. Opened per request, like the +// CreateAccount stores, so a cluster upgraded into this feature picks it up +// without a gateway restart. +func (gw *GatewayConfig) accountDeletionStore(ctx context.Context) (jetstream.KeyValue, error) { + js, err := jetstream.New(gw.NATSConn) + if err != nil { + slog.Error("DeleteAccount: failed to get JetStream context", "err", err) + return nil, errors.New(awserrors.ErrorServiceUnavailable) + } + + jobs, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketAccountDeletions, 5) + if err != nil { + slog.Error("DeleteAccount: failed to open deletion job bucket", "err", err) + return nil, errors.New(awserrors.ErrorServiceUnavailable) + } + return jobs, nil +} + +// teardownError maps an engine error onto the admin surface. +// +// A protected account returns AccessDenied rather than a description of why: +// no credential grants deleting it, so the answer is the same one an +// unauthorized caller gets. +func teardownError(method, accountID string, err error) error { + switch { + case errors.Is(err, accountteardown.ErrProtectedAccount): + slog.Warn(method+": refused a protected account", "accountID", accountID) + return errors.New(awserrors.ErrorAccessDenied) + case errors.Is(err, accountteardown.ErrAccountNameMismatch): + slog.Warn(method+": account name did not match", "accountID", accountID) + return errors.New(awserrors.ErrorIdempotentParameterMismatch) + case isAccountNotFound(err): + return errors.New(awserrors.ErrorIAMNoSuchEntity) + default: + slog.Error(method+": teardown could not start", "accountID", accountID, "err", err) + return errors.New(awserrors.ErrorInternalError) + } +} + +// isAccountNotFound recognises the IAM service's missing-account error, which +// is a plain error value rather than a typed one. Both spellings are matched +// because the account store and its KV layer word it differently. +func isAccountNotFound(err error) bool { + if err == nil { + return false + } + message := err.Error() + return strings.Contains(message, "account not found") || + strings.Contains(message, awserrors.ErrorIAMNoSuchEntity) +} + +// validateDeleteAccountRequest rejects malformed input. Messages name the +// offending field but never echo its value, which for accountName is a +// customer email address. +func validateDeleteAccountRequest(req *DeleteAccountRequest) error { + req.AccountID = strings.TrimSpace(req.AccountID) + req.AccountName = strings.TrimSpace(req.AccountName) + req.ClientToken = strings.TrimSpace(req.ClientToken) + + if req.AccountID == "" { + return errors.New(awserrors.ErrorMissingParameter) + } + if !accountIDRE.MatchString(req.AccountID) { + return errors.New(awserrors.ErrorInvalidParameterValue) + } + if req.DryRun { + // A dry run deletes nothing, so it needs neither the name confirmation + // nor a token to make it idempotent. + return nil + } + + // The name confirmation is the safety rail that makes a mistyped account id + // fail closed rather than empty a live tenant. + if req.AccountName == "" || req.ClientToken == "" { + return errors.New(awserrors.ErrorMissingParameter) + } + if len(req.AccountName) > accountNameMaxLen { + return errors.New(awserrors.ErrorInvalidParameterValue) + } + if len(req.ClientToken) < clientTokenMinLen || len(req.ClientToken) > clientTokenMaxLen || + !clientTokenRE.MatchString(req.ClientToken) { + return errors.New(awserrors.ErrorInvalidParameterValue) + } + return nil +} diff --git a/spinifex/gateway/admin_deleteaccount_job_test.go b/spinifex/gateway/admin_deleteaccount_job_test.go new file mode 100644 index 000000000..a5b70edd7 --- /dev/null +++ b/spinifex/gateway/admin_deleteaccount_job_test.go @@ -0,0 +1,490 @@ +package gateway + +//test:in-package — the deletion job record, its claim rules and the progress +// tracker are unexported, and they are what make a teardown survive a retry or +// a gateway restart. None of that is observable from the HTTP surface alone. + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go/service/s3" + "github.com/mulgadc/spinifex/spinifex/accountteardown" + "github.com/mulgadc/spinifex/spinifex/awserrors" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + "github.com/mulgadc/spinifex/spinifex/objectstore" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const deleteClientToken = "8b9c1d2e3f4a5b6c7d8e9f0a1b2c3d4e" + +// teardownSubjects are the service subjects the reapers drive. A subject with +// no responder makes the request time out and the stage fail, so this list has +// to keep step with the reapers NewClusterEngine wires. +var teardownSubjects = []string{"ec2.>", "ecs.>", "eks.>", "rds.>", "elbv2.>", "acm.>", "bedrock.>"} + +// newDeleteAccountGateway wires a gateway against a real IAM service and +// embedded JetStream, with a stub cluster answering the subjects the reapers +// drive. The tenant has no resources, so every stage drains empty and the test +// exercises the job record rather than the reapers. +func newDeleteAccountGateway(t *testing.T) *GatewayConfig { + t.Helper() + _, nc, _ := testutil.StartTestJetStream(t) + + masterKey, err := handlers_iam.GenerateMasterKey() + require.NoError(t, err) + svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey, 1) + require.NoError(t, err) + + for _, subject := range teardownSubjects { + sub, err := nc.Subscribe(subject, func(msg *nats.Msg) { + _ = msg.Respond([]byte(`{}`)) + }) + require.NoError(t, err) + t.Cleanup(func() { _ = sub.Unsubscribe() }) + } + require.NoError(t, nc.Flush()) + + return &GatewayConfig{ + DisableLogging: true, + IAMService: svc, + NATSConn: nc, + ExpectedNodes: 1, + BucketStore: emptyBucketStore{}, + } +} + +// emptyBucketStore stands in for predastore: the tenant owns no buckets, so +// the storage stage drains empty and these tests exercise the job record +// rather than the reapers. Teardown refuses without one, which is the point. +type emptyBucketStore struct{} + +func (emptyBucketStore) ListBucketsForOwner(context.Context, string) ([]string, error) { + return nil, nil +} + +func (emptyBucketStore) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { + return &s3.ListObjectsV2Output{}, nil +} + +func (emptyBucketStore) DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { + return &s3.DeleteObjectOutput{}, nil +} + +func (emptyBucketStore) ListMultipartUploads(context.Context, string) ([]objectstore.MultipartUploadRef, error) { + return nil, nil +} + +func (emptyBucketStore) AbortMultipartUpload(context.Context, objectstore.MultipartUploadRef) error { + return nil +} + +func (emptyBucketStore) DeleteBucket(context.Context, string) error { return nil } + +// newTenantAccount creates an account that is safe to delete. Account ids are +// sequential and the first belongs to the super admin, which is protected, so +// the tenant under test can never be the first one created. +func newTenantAccount(t *testing.T, gw *GatewayConfig, name string) *handlers_iam.Account { + t.Helper() + if accounts, err := gw.IAMService.ListAccounts(); err == nil && len(accounts) == 0 { + _, err := gw.IAMService.CreateAccount("super-admin@example.com") + require.NoError(t, err) + } + account, err := gw.IAMService.CreateAccount(name) + require.NoError(t, err) + return account +} + +func deleteAccountBody(t *testing.T, accountID, name, token string) []byte { + t.Helper() + body, err := json.Marshal(DeleteAccountRequest{ + AccountID: accountID, AccountName: name, ClientToken: token, + }) + require.NoError(t, err) + return body +} + +// storeDeletionJob seeds a job record directly, which is how a job left behind +// by another gateway is reproduced without racing a live teardown. +func storeDeletionJob(t *testing.T, gw *GatewayConfig, job *accountDeletionJob) jetstream.KeyValue { + t.Helper() + jobs, err := gw.accountDeletionStore(t.Context()) + require.NoError(t, err) + + record, err := json.Marshal(job) + require.NoError(t, err) + _, err = jobs.Put(t.Context(), job.AccountID, record) + require.NoError(t, err) + return jobs +} + +func describeDeletion(t *testing.T, gw *GatewayConfig, accountID string) *accountDeletionJob { + t.Helper() + body, err := json.Marshal(DescribeAccountDeletionRequest{AccountID: accountID}) + require.NoError(t, err) + + output, err := gw.adminDescribeAccountDeletion(t.Context(), body) + require.NoError(t, err) + job, ok := output.(*accountDeletionJob) + require.True(t, ok) + return job +} + +// A dry run is the operator's look before the leap. It must report what would +// go and leave the account untouched — including its status, since TERMINATING +// is a one-way door. +func TestAdminDeleteAccountDryRunChangesNothing(t *testing.T) { + gw := newDeleteAccountGateway(t) + account := newTenantAccount(t, gw, "tenant@example.com") + + body, err := json.Marshal(DeleteAccountRequest{AccountID: account.AccountID, DryRun: true}) + require.NoError(t, err) + + output, err := gw.adminDeleteAccount(t.Context(), body) + require.NoError(t, err) + + response, ok := output.(*DeleteAccountResponse) + require.True(t, ok) + assert.True(t, response.DryRun) + assert.Equal(t, "DRY_RUN", response.State) + assert.Empty(t, response.DeletionID, "a dry run must not claim a job") + require.NotNil(t, response.Inventory) + + stored, err := gw.IAMService.GetAccount(account.AccountID) + require.NoError(t, err) + assert.Equal(t, handlers_iam.AccountStatusActive, stored.Status) + + // No job record means a later real delete is not answered as a replay. + _, err = gw.adminDescribeAccountDeletion(t.Context(), + []byte(`{"accountId":"`+account.AccountID+`"}`)) + require.Error(t, err) + assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, err.Error()) +} + +// The request returns as soon as the job is claimed. A teardown takes minutes +// and holding the call open for it would make every timeout-and-retry start a +// second one. +func TestAdminDeleteAccountReturnsBeforeTheTeardownFinishes(t *testing.T) { + gw := newDeleteAccountGateway(t) + account := newTenantAccount(t, gw, "tenant@example.com") + + output, err := gw.adminDeleteAccount(t.Context(), + deleteAccountBody(t, account.AccountID, "tenant@example.com", deleteClientToken)) + require.NoError(t, err) + + response, ok := output.(*DeleteAccountResponse) + require.True(t, ok) + assert.NotEmpty(t, response.DeletionID) + assert.Equal(t, account.AccountID, response.AccountID) + assert.Equal(t, DeletionStateRunning, response.State) + + // Progress is polled, so the record has to reach a terminal state on its own. + require.Eventually(t, func() bool { + return describeDeletion(t, gw, account.AccountID).State == DeletionStateCompleted + }, 60*time.Second, 100*time.Millisecond) + + job := describeDeletion(t, gw, account.AccountID) + assert.Equal(t, response.DeletionID, job.DeletionID) + assert.NotEmpty(t, job.FinishedAt) + assert.Empty(t, job.Error) + assert.NotEmpty(t, job.Stages, "the stage report is the only view into a finished teardown") + + _, err = gw.IAMService.GetAccount(account.AccountID) + assert.Error(t, err, "the account record must be gone once the job completes") +} + +// The record outlives the account it describes, so a retry that arrives after +// the teardown finished is answered from it rather than as a missing account. +func TestAdminDeleteAccountReplaysAfterTheAccountIsGone(t *testing.T) { + gw := newDeleteAccountGateway(t) + account := newTenantAccount(t, gw, "tenant@example.com") + body := deleteAccountBody(t, account.AccountID, "tenant@example.com", deleteClientToken) + + first, err := gw.adminDeleteAccount(t.Context(), body) + require.NoError(t, err) + started := first.(*DeleteAccountResponse) + + require.Eventually(t, func() bool { + return describeDeletion(t, gw, account.AccountID).State == DeletionStateCompleted + }, 60*time.Second, 100*time.Millisecond) + + second, err := gw.adminDeleteAccount(t.Context(), body) + require.NoError(t, err) + + replayed, ok := second.(*DeleteAccountResponse) + require.True(t, ok) + assert.Equal(t, started.DeletionID, replayed.DeletionID) + assert.Equal(t, DeletionStateCompleted, replayed.State) + + // A caller that lost its token and generated a fresh one is told the same + // thing, rather than that the account it just deleted does not exist. + third, err := gw.adminDeleteAccount(t.Context(), + deleteAccountBody(t, account.AccountID, "tenant@example.com", strings.Repeat("f", 32))) + require.NoError(t, err) + assert.Equal(t, DeletionStateCompleted, third.(*DeleteAccountResponse).State) +} + +// Reusing a token for a different account name is a client bug. Treating it as +// a retry would confirm the stored name to a caller who guessed the wrong one. +func TestAdminDeleteAccountRejectsATokenBoundToAnotherName(t *testing.T) { + gw := newDeleteAccountGateway(t) + account := newTenantAccount(t, gw, "tenant@example.com") + storeDeletionJob(t, gw, &accountDeletionJob{ + DeletionID: "d-1", + AccountID: account.AccountID, + AccountName: "tenant@example.com", + ClientToken: deleteClientToken, + State: DeletionStateRunning, + UpdatedAt: time.Now().UTC().Format(time.RFC3339), + }) + + _, err := gw.adminDeleteAccount(t.Context(), + deleteAccountBody(t, account.AccountID, "someone-else@example.com", deleteClientToken)) + + require.Error(t, err) + assert.Equal(t, awserrors.ErrorIdempotentParameterMismatch, err.Error()) +} + +// Two teardowns on one account would race the reapers against each other, so a +// live job refuses a second caller rather than joining it. +func TestAdminDeleteAccountRefusesAConcurrentTeardown(t *testing.T) { + gw := newDeleteAccountGateway(t) + account := newTenantAccount(t, gw, "tenant@example.com") + storeDeletionJob(t, gw, &accountDeletionJob{ + DeletionID: "d-1", + AccountID: account.AccountID, + AccountName: "tenant@example.com", + ClientToken: deleteClientToken, + State: DeletionStateRunning, + UpdatedAt: time.Now().UTC().Format(time.RFC3339), + }) + + _, err := gw.adminDeleteAccount(t.Context(), + deleteAccountBody(t, account.AccountID, "tenant@example.com", strings.Repeat("b", 32))) + + require.Error(t, err) + assert.Equal(t, awserrors.ErrorOperationInProgress, err.Error()) +} + +// A gateway restarted mid-teardown leaves a job that looks live forever. +// Without takeover the account could never be retried through the API. +func TestAdminDeleteAccountTakesOverAStaleJob(t *testing.T) { + gw := newDeleteAccountGateway(t) + account := newTenantAccount(t, gw, "tenant@example.com") + storeDeletionJob(t, gw, &accountDeletionJob{ + DeletionID: "d-abandoned", + AccountID: account.AccountID, + AccountName: "tenant@example.com", + ClientToken: deleteClientToken, + State: DeletionStateRunning, + UpdatedAt: time.Now().UTC().Add(-2 * accountDeletionStale).Format(time.RFC3339), + }) + + output, err := gw.adminDeleteAccount(t.Context(), + deleteAccountBody(t, account.AccountID, "tenant@example.com", strings.Repeat("c", 32))) + require.NoError(t, err) + + response, ok := output.(*DeleteAccountResponse) + require.True(t, ok) + assert.NotEqual(t, "d-abandoned", response.DeletionID, "a takeover runs under its own deletion id") + assert.Equal(t, DeletionStateRunning, response.State) +} + +// A teardown that failed is retried rather than left wedged: the engine +// re-lists what is still there, so running it again is safe. +func TestAdminDeleteAccountRetriesAFailedJob(t *testing.T) { + gw := newDeleteAccountGateway(t) + account := newTenantAccount(t, gw, "tenant@example.com") + storeDeletionJob(t, gw, &accountDeletionJob{ + DeletionID: "d-failed", + AccountID: account.AccountID, + AccountName: "tenant@example.com", + ClientToken: deleteClientToken, + State: DeletionStateFailed, + Error: "volume vol-1 is stuck", + UpdatedAt: time.Now().UTC().Format(time.RFC3339), + }) + + output, err := gw.adminDeleteAccount(t.Context(), + deleteAccountBody(t, account.AccountID, "tenant@example.com", strings.Repeat("d", 32))) + require.NoError(t, err) + + response, ok := output.(*DeleteAccountResponse) + require.True(t, ok) + assert.NotEqual(t, "d-failed", response.DeletionID) + assert.Equal(t, DeletionStateRunning, response.State) +} + +// A completed job means the account is gone. Saying so beats the missing-account +// error a fresh token would otherwise get, which is what a harness retrying a +// lost response actually does. +func TestAdminDeleteAccountReportsACompletedJobToANewToken(t *testing.T) { + gw := newDeleteAccountGateway(t) + account := newTenantAccount(t, gw, "tenant@example.com") + storeDeletionJob(t, gw, &accountDeletionJob{ + DeletionID: "d-done", + AccountID: account.AccountID, + AccountName: "tenant@example.com", + ClientToken: deleteClientToken, + State: DeletionStateCompleted, + UpdatedAt: time.Now().UTC().Format(time.RFC3339), + FinishedAt: time.Now().UTC().Format(time.RFC3339), + }) + + output, err := gw.adminDeleteAccount(t.Context(), + deleteAccountBody(t, account.AccountID, "tenant@example.com", strings.Repeat("e", 32))) + require.NoError(t, err) + + response, ok := output.(*DeleteAccountResponse) + require.True(t, ok) + assert.Equal(t, "d-done", response.DeletionID) + assert.Equal(t, DeletionStateCompleted, response.State) +} + +// The protected accounts are refused the same way an unauthorized caller is: +// no credential grants deleting them, so the answer describes nothing. +func TestAdminDeleteAccountRefusesProtectedAccounts(t *testing.T) { + gw := newDeleteAccountGateway(t) + newTenantAccount(t, gw, "tenant@example.com") + + // The system account and the super admin, whichever names they carry. + for _, accountID := range []string{"000000000000", "000000000001"} { + t.Run(accountID, func(t *testing.T) { + _, err := gw.adminDeleteAccount(t.Context(), + deleteAccountBody(t, accountID, "super-admin@example.com", deleteClientToken)) + + require.Error(t, err) + assert.Equal(t, awserrors.ErrorAccessDenied, err.Error()) + }) + } +} + +// The name confirmation is what makes a mistyped account id fail closed rather +// than empty a live tenant, so it is checked before anything is deleted. +func TestAdminDeleteAccountRefusesAMismatchedName(t *testing.T) { + gw := newDeleteAccountGateway(t) + account := newTenantAccount(t, gw, "tenant@example.com") + + _, err := gw.adminDeleteAccount(t.Context(), + deleteAccountBody(t, account.AccountID, "wrong@example.com", deleteClientToken)) + + require.Error(t, err) + assert.Equal(t, awserrors.ErrorIdempotentParameterMismatch, err.Error()) + + stored, err := gw.IAMService.GetAccount(account.AccountID) + require.NoError(t, err) + assert.Equal(t, handlers_iam.AccountStatusActive, stored.Status) +} + +func TestAdminDeleteAccountRejectsAnUnknownAccount(t *testing.T) { + gw := newDeleteAccountGateway(t) + newTenantAccount(t, gw, "tenant@example.com") + + _, err := gw.adminDeleteAccount(t.Context(), + deleteAccountBody(t, "000000009999", "ghost@example.com", deleteClientToken)) + + require.Error(t, err) + assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, err.Error()) +} + +func TestAdminDeleteAccountRejectsAMalformedBody(t *testing.T) { + gw := newDeleteAccountGateway(t) + + _, err := gw.adminDeleteAccount(t.Context(), []byte(`{`)) + + require.Error(t, err) + assert.Equal(t, awserrors.ErrorInvalidRequest, err.Error()) +} + +// Without an IAM service there is no way to check what the account owns, and +// deleting on a guess is worse than refusing. +func TestAdminDeleteAccountRefusesWithoutAnIAMService(t *testing.T) { + gw := &GatewayConfig{DisableLogging: true} + + _, err := gw.adminDeleteAccount(context.Background(), + deleteAccountBody(t, "000000000042", "tenant@example.com", deleteClientToken)) + + require.Error(t, err) + assert.Equal(t, awserrors.ErrorInternalError, err.Error()) +} + +func TestAdminDescribeAccountDeletionRejectsAMalformedBody(t *testing.T) { + gw := newDeleteAccountGateway(t) + + _, err := gw.adminDescribeAccountDeletion(t.Context(), []byte(`{`)) + + require.Error(t, err) + assert.Equal(t, awserrors.ErrorInvalidRequest, err.Error()) +} + +// The stored stages are what an operator reads while a teardown runs, so the +// tracker has to write them as they finish rather than only at the end. +func TestDeletionTrackerRecordsEachStageAsItFinishes(t *testing.T) { + gw := newDeleteAccountGateway(t) + job := &accountDeletionJob{ + DeletionID: "d-1", + AccountID: "000000000042", + State: DeletionStateRunning, + } + jobs := storeDeletionJob(t, gw, job) + tracker := &deletionTracker{jobs: jobs, job: job} + + tracker.stageFinished(t.Context(), accountteardown.StageResult{ + Stage: accountteardown.StageCompute, + Deleted: []accountteardown.Resource{{Kind: "instance", ID: "i-1"}}, + }) + + stored := describeDeletion(t, gw, "000000000042") + require.Len(t, stored.Stages, 1) + assert.Equal(t, accountteardown.StageCompute, stored.Stages[0].Stage) + require.Len(t, stored.Stages[0].Deleted, 1) + assert.Equal(t, "i-1", stored.Stages[0].Deleted[0].ID) + assert.Equal(t, DeletionStateRunning, stored.State) + assert.NotEmpty(t, stored.UpdatedAt, "a running job must heartbeat or it reads as abandoned") +} + +// A teardown that gives up has to say so in the record. A job left RUNNING +// forever is indistinguishable from one still working. +func TestDeletionTrackerRecordsFailure(t *testing.T) { + gw := newDeleteAccountGateway(t) + job := &accountDeletionJob{ + DeletionID: "d-1", + AccountID: "000000000042", + State: DeletionStateRunning, + } + jobs := storeDeletionJob(t, gw, job) + tracker := &deletionTracker{jobs: jobs, job: job} + + tracker.finished(t.Context(), nil, assert.AnError) + + stored := describeDeletion(t, gw, "000000000042") + assert.Equal(t, DeletionStateFailed, stored.State) + assert.Contains(t, stored.Error, assert.AnError.Error()) + assert.NotEmpty(t, stored.FinishedAt) +} + +// The heartbeat is what lets another gateway tell an abandoned job from a live +// one, so stopping it must not leave the goroutine writing. +func TestDeletionTrackerHeartbeatStops(t *testing.T) { + gw := newDeleteAccountGateway(t) + job := &accountDeletionJob{ + DeletionID: "d-1", + AccountID: "000000000042", + State: DeletionStateRunning, + } + jobs := storeDeletionJob(t, gw, job) + tracker := &deletionTracker{jobs: jobs, job: job} + + stop := tracker.startHeartbeat(t.Context()) + stop() + + tracker.persist(t.Context()) + assert.NotEmpty(t, describeDeletion(t, gw, "000000000042").UpdatedAt) +} diff --git a/spinifex/gateway/admin_deleteaccount_test.go b/spinifex/gateway/admin_deleteaccount_test.go new file mode 100644 index 000000000..6acaba3ff --- /dev/null +++ b/spinifex/gateway/admin_deleteaccount_test.go @@ -0,0 +1,236 @@ +package gateway + +//test:in-package — the request validators, the job staleness rule and the +// engine-error mapping are unexported, and each is a safety rail that would go +// untested if it could only be reached through a live cluster. + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/accountteardown" + "github.com/mulgadc/spinifex/spinifex/awserrors" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// listAccountsIAMService answers the one call /admin/ListAccounts makes. +type listAccountsIAMService struct { + policyMockIAMService + + accounts []*handlers_iam.Account + err error +} + +func (m *listAccountsIAMService) ListAccounts() ([]*handlers_iam.Account, error) { + return m.accounts, m.err +} + +// A real delete needs the name confirmation and a client token; a dry run +// deletes nothing, so it needs neither. +func TestValidateDeleteAccountRequest(t *testing.T) { + token := strings.Repeat("a", clientTokenMinLen) + + tests := []struct { + name string + request DeleteAccountRequest + wantErr string + }{ + { + name: "complete request", + request: DeleteAccountRequest{AccountID: "000000000042", AccountName: "t@example.com", ClientToken: token}, + }, + { + name: "dry run needs only the account id", + request: DeleteAccountRequest{AccountID: "000000000042", DryRun: true}, + }, + { + name: "no account id", + request: DeleteAccountRequest{AccountName: "t@example.com", ClientToken: token}, + wantErr: awserrors.ErrorMissingParameter, + }, + { + name: "account id is not twelve digits", + request: DeleteAccountRequest{AccountID: "42", AccountName: "t@example.com", ClientToken: token}, + wantErr: awserrors.ErrorInvalidParameterValue, + }, + { + name: "account id is not numeric", + request: DeleteAccountRequest{AccountID: "00000000004x", AccountName: "t@example.com", ClientToken: token}, + wantErr: awserrors.ErrorInvalidParameterValue, + }, + { + name: "no name confirmation", + request: DeleteAccountRequest{AccountID: "000000000042", ClientToken: token}, + wantErr: awserrors.ErrorMissingParameter, + }, + { + name: "no client token", + request: DeleteAccountRequest{AccountID: "000000000042", AccountName: "t@example.com"}, + wantErr: awserrors.ErrorMissingParameter, + }, + { + name: "client token too short", + request: DeleteAccountRequest{AccountID: "000000000042", AccountName: "t@example.com", ClientToken: "short"}, + wantErr: awserrors.ErrorInvalidParameterValue, + }, + { + name: "account name too long", + request: DeleteAccountRequest{ + AccountID: "000000000042", + AccountName: strings.Repeat("x", accountNameMaxLen+1), + ClientToken: token, + }, + wantErr: awserrors.ErrorInvalidParameterValue, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := validateDeleteAccountRequest(&tc.request) + if tc.wantErr == "" { + assert.NoError(t, err) + return + } + require.Error(t, err) + assert.Equal(t, tc.wantErr, err.Error()) + }) + } +} + +// Whitespace around a confirmation name is a copy-paste artefact, not a +// mismatch — but it must be trimmed before the comparison, not after. +func TestValidateDeleteAccountRequestTrimsInput(t *testing.T) { + request := DeleteAccountRequest{ + AccountID: " 000000000042 ", + AccountName: " t@example.com ", + ClientToken: " " + strings.Repeat("a", clientTokenMinLen) + " ", + } + + require.NoError(t, validateDeleteAccountRequest(&request)) + + assert.Equal(t, "000000000042", request.AccountID) + assert.Equal(t, "t@example.com", request.AccountName) + assert.Equal(t, strings.Repeat("a", clientTokenMinLen), request.ClientToken) +} + +// A job is retryable only once its gateway has stopped heartbeating. Getting +// this wrong in either direction is bad: too eager runs two teardowns on one +// account, too slow wedges the account until an operator intervenes. +func TestDeletionJobStale(t *testing.T) { + now := time.Now().UTC() + + tests := []struct { + name string + updatedAt string + want bool + }{ + {"heartbeat just now", now.Format(time.RFC3339), false}, + {"heartbeat within the window", now.Add(-accountDeletionStale / 2).Format(time.RFC3339), false}, + {"heartbeat stopped", now.Add(-2 * accountDeletionStale).Format(time.RFC3339), true}, + {"no heartbeat recorded", "", true}, + {"unparseable heartbeat", "yesterday", true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + job := &accountDeletionJob{UpdatedAt: tc.updatedAt} + assert.Equal(t, tc.want, deletionJobStale(job, now)) + }) + } +} + +// A protected account must answer exactly as an unauthorized caller would: no +// credential grants deleting it, so the refusal describes nothing. +func TestTeardownErrorMapping(t *testing.T) { + tests := []struct { + name string + err error + want string + }{ + {"protected account", accountteardown.ErrProtectedAccount, awserrors.ErrorAccessDenied}, + { + "wrapped protected account", + fmt.Errorf("000000000001 is the super admin account: %w", accountteardown.ErrProtectedAccount), + awserrors.ErrorAccessDenied, + }, + {"name mismatch", accountteardown.ErrAccountNameMismatch, awserrors.ErrorIdempotentParameterMismatch}, + {"no such account", errors.New("account not found: 000000000042"), awserrors.ErrorIAMNoSuchEntity}, + {"anything else", errors.New("nats: timeout"), awserrors.ErrorInternalError}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := teardownError("DeleteAccount", "000000000042", tc.err) + require.Error(t, got) + assert.Equal(t, tc.want, got.Error()) + }) + } +} + +// The listing is an operator's index of tenants. It must carry the status — +// that is how a stuck TERMINATING account is noticed — and no key material. +func TestAdminListAccountsReportsStatus(t *testing.T) { + gw := &GatewayConfig{DisableLogging: true, IAMService: &listAccountsIAMService{ + accounts: []*handlers_iam.Account{ + {AccountID: "000000000042", AccountName: "t@example.com", Status: handlers_iam.AccountStatusActive}, + nil, + {AccountID: "000000000043", AccountName: "u@example.com", Status: handlers_iam.AccountStatusTerminating}, + }, + }} + + output, err := gw.adminListAccounts(context.Background(), nil) + require.NoError(t, err) + + response, ok := output.(*ListAccountsResponse) + require.True(t, ok) + assert.Equal(t, 2, response.Count) + assert.Equal(t, handlers_iam.AccountStatusActive, response.Accounts[0].Status) + assert.Equal(t, handlers_iam.AccountStatusTerminating, response.Accounts[1].Status) +} + +// Failing closed matters here: an empty listing reads as "no accounts", which +// is exactly what an operator would act on. +func TestAdminListAccountsFailsClosed(t *testing.T) { + gw := &GatewayConfig{DisableLogging: true, IAMService: &listAccountsIAMService{ + err: errors.New("kv unavailable"), + }} + + _, err := gw.adminListAccounts(context.Background(), nil) + + require.Error(t, err) + assert.Equal(t, awserrors.ErrorInternalError, err.Error()) +} + +// Each admin method is a separate grant. The signup Worker holds +// spinifex:CreateAccount and must not reach deletion with it. +func TestCreateAccountGrantDoesNotAllowDeleteAccount(t *testing.T) { + gw := &GatewayConfig{DisableLogging: true, IAMService: createAccountPolicy()} + + for _, method := range []string{"DeleteAccount", "DescribeAccountDeletion", "ListAccounts"} { + t.Run(method, func(t *testing.T) { + rec := authorizedAdminRequestForMethod(gw, method, `{}`) + + assert.Equal(t, awserrors.ErrorAccessDenied, decodeAdminError(t, rec).Error.Code) + }) + } +} + +// A missing account id must be refused before anything reads the cluster, so +// DescribeAccountDeletion cannot be used to probe with junk. +func TestDescribeAccountDeletionValidatesTheAccountID(t *testing.T) { + gw := &GatewayConfig{DisableLogging: true} + + _, err := gw.adminDescribeAccountDeletion(context.Background(), []byte(`{}`)) + require.Error(t, err) + assert.Equal(t, awserrors.ErrorMissingParameter, err.Error()) + + _, err = gw.adminDescribeAccountDeletion(context.Background(), []byte(`{"accountId":"nope"}`)) + require.Error(t, err) + assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error()) +} diff --git a/spinifex/gateway/admin_test.go b/spinifex/gateway/admin_test.go index ac6cc934a..9acb25241 100644 --- a/spinifex/gateway/admin_test.go +++ b/spinifex/gateway/admin_test.go @@ -9,6 +9,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "sort" "strings" "testing" @@ -28,7 +29,8 @@ func TestAdminPathMethod(t *testing.T) { wantOK bool }{ {"known method", "/admin/CreateAccount", "CreateAccount", true}, - {"unknown method", "/admin/DeleteAccount", "", false}, + {"deletion method", "/admin/DeleteAccount", "DeleteAccount", true}, + {"unknown method", "/admin/PurgeEverything", "", false}, {"nested path", "/admin/CreateAccount/extra", "", false}, {"bare prefix", "/admin/", "", false}, {"not admin", "/", "", false}, @@ -183,7 +185,7 @@ func decodeAdminError(t *testing.T, rec *httptest.ResponseRecorder) adminErrorBo func TestAdminRequestUnknownMethod(t *testing.T) { gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()} - rec := adminRequest(gw, "DeleteAccount", `{}`) + rec := adminRequest(gw, "PurgeEverything", `{}`) assert.Equal(t, http.StatusBadRequest, rec.Code) assert.Equal(t, awserrors.ErrorInvalidAction, decodeAdminError(t, rec).Error.Code) @@ -241,9 +243,15 @@ func TestAdminRequestRejectsNonPost(t *testing.T) { // authorizedAdminRequest builds a request that clears every authorization gate, // so the guards behind them can be tested on their own. func authorizedAdminRequest(gw *GatewayConfig, body string) *httptest.ResponseRecorder { - req := httptest.NewRequest(http.MethodPost, "/admin/CreateAccount", strings.NewReader(body)) + return authorizedAdminRequestForMethod(gw, "CreateAccount", body) +} + +// authorizedAdminRequestForMethod is the same for a named method, so a grant +// that covers one method can be shown not to cover another. +func authorizedAdminRequestForMethod(gw *GatewayConfig, method, body string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodPost, "/admin/"+method, strings.NewReader(body)) rctx := chi.NewRouteContext() - rctx.URLParams.Add("method", "CreateAccount") + rctx.URLParams.Add("method", method) ctx := context.WithValue(req.Context(), chi.RouteCtxKey, rctx) ctx = context.WithValue(ctx, ctxService, "spinifex") @@ -265,3 +273,17 @@ func TestAdminRequestReportsClusterUnavailable(t *testing.T) { assert.Equal(t, awserrors.ErrorServiceUnavailable, decodeAdminError(t, rec).Error.Code) } + +// The credential-minting CLI grants these by name. A method served but absent +// from the list would be uncallable by every key an operator issues. +func TestAdminMethodNamesCoverTheRoutedMethods(t *testing.T) { + names := AdminMethodNames() + + require.Len(t, names, len(adminMethods)) + assert.True(t, sort.StringsAreSorted(names), "names must be sorted for stable help text") + for _, name := range names { + method, ok := adminPathMethod(adminPathPrefix + name) + assert.True(t, ok, "%s is granted but not routed", name) + assert.Equal(t, name, method) + } +} diff --git a/spinifex/gateway/auth.go b/spinifex/gateway/auth.go index e6b3d092f..6479e3af7 100644 --- a/spinifex/gateway/auth.go +++ b/spinifex/gateway/auth.go @@ -29,6 +29,9 @@ func (gw *GatewayConfig) SigV4AuthMiddleware() func(http.Handler) http.Handler { if gw.RateLimiter == nil { gw.RateLimiter = NewAuthRateLimiter() } + if gw.accountStatus == nil { + gw.accountStatus = newAccountStatusCache() + } return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { clientIP := utils.ClientIP(r.RemoteAddr) @@ -147,6 +150,13 @@ func (gw *GatewayConfig) SigV4AuthMiddleware() func(http.Handler) http.Handler { return } + // Only after the signature holds: an unauthenticated prober must not + // be able to use this to tell a suspended account from a live one. + if errCode := gw.checkAccountActive(principal.accountID, clientIP); errCode != "" { + gw.writeSigV4Error(w, r, errCode) + return + } + // Parse rewound the body; re-read it for query-arg parsing, then rewind // again for the downstream handler. body, err := io.ReadAll(r.Body) diff --git a/spinifex/gateway/auth_accountstatus.go b/spinifex/gateway/auth_accountstatus.go new file mode 100644 index 000000000..b89709a03 --- /dev/null +++ b/spinifex/gateway/auth_accountstatus.go @@ -0,0 +1,97 @@ +package gateway + +import ( + "log/slog" + "sync" + "time" + + "github.com/mulgadc/spinifex/spinifex/awserrors" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" +) + +// accountActiveTTL bounds how long an account is trusted to still be ACTIVE +// without re-reading it. Only the ACTIVE answer is cached, so suspending or +// terminating an account takes effect within this window rather than being +// held open by a warm cache entry. +const accountActiveTTL = 5 * time.Second + +// accountStatusCache remembers which accounts were ACTIVE and when. Every +// authenticated request needs the answer and the account record changes +// perhaps twice in its lifetime, so an uncached read would double the +// auth path's KV traffic to learn nothing new. +type accountStatusCache struct { + mu sync.RWMutex + active map[string]time.Time +} + +func newAccountStatusCache() *accountStatusCache { + return &accountStatusCache{active: make(map[string]time.Time)} +} + +// activeUntil reports whether accountID was seen ACTIVE recently enough. +func (c *accountStatusCache) activeUntil(accountID string, now time.Time) bool { + if c == nil { + return false + } + c.mu.RLock() + defer c.mu.RUnlock() + expiry, ok := c.active[accountID] + return ok && now.Before(expiry) +} + +func (c *accountStatusCache) markActive(accountID string, now time.Time) { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + c.active[accountID] = now.Add(accountActiveTTL) +} + +// forget drops an entry so a status change takes effect immediately on the +// node that made it, rather than after the TTL. +func (c *accountStatusCache) forget(accountID string) { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + delete(c.active, accountID) +} + +// checkAccountActive rejects a request whose account is suspended or being +// torn down. It returns an empty string when the request may proceed. +// +// This is the only enforcement point for account status on the AWS surface: +// until it existed, suspending an account blocked ECR and nothing else. +func (gw *GatewayConfig) checkAccountActive(accountID, clientIP string) string { + if accountID == "" { + return "" + } + now := time.Now() + if gw.accountStatus.activeUntil(accountID, now) { + return "" + } + if gw.IAMService == nil { + slog.Error("Auth: IAM service not initialized for account status check") + return awserrors.ErrorInternalError + } + + account, err := gw.IAMService.GetAccount(accountID) + if err != nil { + // Fail closed. An account we cannot read is not evidence of an account + // in good standing, and this gate is what stops a terminating tenant + // from creating resources the teardown has already walked past. + slog.Error("Auth: failed to read account for status check", "accountID", accountID, "err", err) + return awserrors.ErrorInternalError + } + + if account.Status != handlers_iam.AccountStatusActive { + slog.Warn("Auth failure: account is not active", + "accountID", accountID, "status", account.Status, "sourceIP", clientIP) + return awserrors.ErrorAccessDenied + } + + gw.accountStatus.markActive(accountID, now) + return "" +} diff --git a/spinifex/gateway/auth_accountstatus_test.go b/spinifex/gateway/auth_accountstatus_test.go new file mode 100644 index 000000000..7f35ca240 --- /dev/null +++ b/spinifex/gateway/auth_accountstatus_test.go @@ -0,0 +1,127 @@ +package gateway + +//test:in-package — the status gate is an unexported step of the auth +// middleware, and its cache is deliberately not part of the package's API. + +import ( + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/go-chi/chi/v5" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" +) + +// statusGateRouter builds a signed-request harness whose only job is to report +// whether the auth middleware let the request through. +func statusGateRouter(t *testing.T, svc handlers_iam.IAMService) (chi.Router, *http.Request) { + t.Helper() + + gw := &GatewayConfig{DisableLogging: true, Region: testRegion, IAMService: svc} + r := chi.NewRouter() + r.Use(gw.SigV4AuthMiddleware()) + r.HandleFunc("/*", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) + + req := httptest.NewRequest(http.MethodGet, "/", nil) + req.Host = "localhost:9999" + signTestRequest(t, req, nil, testAccessKey, testSecretKey) + return r, req +} + +// statusGateService returns a mock holding one usable access key for account, +// with that account pinned to status. +func statusGateService(t *testing.T, accountID, status string) *mockIAMService { + t.Helper() + + encryptedSecret, err := handlers_iam.EncryptSecret(testSecretKey, testMasterKey) + if err != nil { + t.Fatalf("encrypt secret: %v", err) + } + return &mockIAMService{ + masterKey: testMasterKey, + accessKeys: map[string]*handlers_iam.AccessKey{ + testAccessKey: { + AccessKeyID: testAccessKey, + SecretAccessKey: encryptedSecret, + UserName: "alice", + AccountID: accountID, + Status: handlers_iam.AccessKeyStatusActive, + }, + }, + accounts: map[string]*handlers_iam.Account{ + accountID: {AccountID: accountID, Status: status}, + }, + } +} + +func TestAuthAllowsAnActiveAccount(t *testing.T) { + r, req := statusGateRouter(t, statusGateService(t, "123456789012", handlers_iam.AccountStatusActive)) + + if resp := doRequest(r, req); resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200 for an ACTIVE account, got %d", resp.StatusCode) + } +} + +// Until this gate existed, account status was enforced only in ECR's principal +// resolution, so suspending an account left EC2, S3 and STS wide open. +func TestAuthDeniesASuspendedAccount(t *testing.T) { + r, req := statusGateRouter(t, statusGateService(t, "123456789012", handlers_iam.AccountStatusSuspended)) + + if resp := doRequest(r, req); resp.StatusCode != http.StatusForbidden { + t.Fatalf("expected 403 for a SUSPENDED account, got %d", resp.StatusCode) + } +} + +// The quiesce that makes teardown converge: a terminating account must not be +// able to create resources the cascade has already walked past. +func TestAuthDeniesATerminatingAccount(t *testing.T) { + r, req := statusGateRouter(t, statusGateService(t, "123456789012", handlers_iam.AccountStatusTerminating)) + + if resp := doRequest(r, req); resp.StatusCode != http.StatusForbidden { + t.Fatalf("expected 403 for a TERMINATING account, got %d", resp.StatusCode) + } +} + +// unreadableAccountService answers key lookups but never account lookups. +type unreadableAccountService struct { + *mockIAMService +} + +func (s *unreadableAccountService) GetAccount(string) (*handlers_iam.Account, error) { + return nil, errors.New("account store unavailable") +} + +// An account we cannot read is not evidence of an account in good standing. +func TestAuthFailsClosedWhenTheAccountCannotBeRead(t *testing.T) { + svc := &unreadableAccountService{statusGateService(t, "123456789012", handlers_iam.AccountStatusActive)} + r, req := statusGateRouter(t, svc) + + if resp := doRequest(r, req); resp.StatusCode == http.StatusOK { + t.Fatal("expected an unreadable account to be denied, got 200") + } +} + +func TestAccountStatusCacheOnlyRemembersActive(t *testing.T) { + cache := newAccountStatusCache() + now := time.Now() + + if cache.activeUntil("123456789012", now) { + t.Fatal("an unseen account must not read as active") + } + + cache.markActive("123456789012", now) + if !cache.activeUntil("123456789012", now) { + t.Fatal("a just-marked account must read as active") + } + if cache.activeUntil("123456789012", now.Add(accountActiveTTL+time.Second)) { + t.Fatal("the entry must expire with the TTL") + } + + cache.markActive("123456789012", now) + cache.forget("123456789012") + if cache.activeUntil("123456789012", now) { + t.Fatal("forget must drop the entry so a status change takes effect at once") + } +} diff --git a/spinifex/gateway/auth_test.go b/spinifex/gateway/auth_test.go index b8782eba8..c1dd2a089 100644 --- a/spinifex/gateway/auth_test.go +++ b/spinifex/gateway/auth_test.go @@ -45,6 +45,11 @@ type mockIAMService struct { accessKeys map[string]*handlers_iam.AccessKey masterKey []byte + + // accounts overrides the status the auth gate reads. An empty map means + // every account is ACTIVE, which is what most tests here are asserting + // around rather than asserting about. + accounts map[string]*handlers_iam.Account } func (m *mockIAMService) LookupAccessKey(accessKeyID string) (*handlers_iam.AccessKey, error) { @@ -59,6 +64,13 @@ func (m *mockIAMService) DecryptSecret(ciphertext string) (string, error) { return handlers_iam.DecryptSecret(ciphertext, m.masterKey) } +func (m *mockIAMService) GetAccount(accountID string) (*handlers_iam.Account, error) { + if account, ok := m.accounts[accountID]; ok { + return account, nil + } + return &handlers_iam.Account{AccountID: accountID, Status: handlers_iam.AccountStatusActive}, nil +} + // testMasterKey is a fixed 32-byte key for deterministic tests. var testMasterKey []byte diff --git a/spinifex/gateway/discover_nodes_test.go b/spinifex/gateway/discover_nodes_test.go new file mode 100644 index 000000000..141735f10 --- /dev/null +++ b/spinifex/gateway/discover_nodes_test.go @@ -0,0 +1,112 @@ +package gateway_test + +import ( + "context" + "encoding/json" + "sync/atomic" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/gateway" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/mulgadc/spinifex/spinifex/types" + "github.com/nats-io/nats.go" +) + +// respondAsNodes answers the discovery subject as n distinct nodes and reports +// how many discovery rounds were served. +func respondAsNodes(t *testing.T, nc *nats.Conn, n int) *atomic.Int64 { + t.Helper() + + var rounds atomic.Int64 + for i := range n { + node := string(rune('a' + i)) + sub, err := nc.Subscribe("spinifex.nodes.discover", func(msg *nats.Msg) { + if node == "a" { + rounds.Add(1) + } + payload, err := json.Marshal(types.NodeDiscoverResponse{Node: node}) + if err != nil { + return + } + _ = msg.Respond(payload) + }) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + } + if err := nc.Flush(); err != nil { + t.Fatalf("flush: %v", err) + } + return &rounds +} + +// Discovery fronts eight EC2 handlers, so whatever it costs is added to every +// one of their calls. The gather counts whoever answers and therefore cannot +// know when to stop, so it always sits out its whole 500ms timeout — that cost +// has to be paid once in a while, not once per request. +func TestDiscoverActiveNodesKeepsItsCostOffTheRequestPath(t *testing.T) { + _, nc := testutil.StartTestNATS(t) + respondAsNodes(t, nc, 4) + + gw := &gateway.GatewayConfig{NATSConn: nc, ExpectedNodes: 4} + + if count := gw.DiscoverActiveNodes(context.Background()); count != 4 { + t.Fatalf("DiscoverActiveNodes = %d, want 4", count) + } + + start := time.Now() + for range 5 { + gw.DiscoverActiveNodes(context.Background()) + } + elapsed := time.Since(start) + + if elapsed > 100*time.Millisecond { + t.Errorf("five further discoveries took %v; each was paying the fan-out timeout again", elapsed) + } +} + +// Membership does not change per request. Re-deriving it on every call put a +// fan-out in front of every API call that needs it, which is most of them. +func TestDiscoverActiveNodesReusesARecentAnswer(t *testing.T) { + _, nc := testutil.StartTestNATS(t) + rounds := respondAsNodes(t, nc, 3) + + gw := &gateway.GatewayConfig{NATSConn: nc, ExpectedNodes: 3} + + if count := gw.DiscoverActiveNodes(context.Background()); count != 3 { + t.Fatalf("first discovery = %d, want 3", count) + } + first := rounds.Load() + + for range 5 { + if count := gw.DiscoverActiveNodes(context.Background()); count != 3 { + t.Fatalf("cached discovery = %d, want 3", count) + } + } + + if got := rounds.Load(); got != first { + t.Errorf("discovery ran %d more time(s) within the TTL, want the cached answer reused", got-first) + } +} + +// A fallback is not evidence of anything: caching one would make a momentary +// NATS problem outlive itself for the whole TTL. +func TestDiscoverActiveNodesDoesNotCacheAFallback(t *testing.T) { + _, nc := testutil.StartTestNATS(t) + + gw := &gateway.GatewayConfig{NATSConn: nc, ExpectedNodes: 2} + + if count := gw.DiscoverActiveNodes(context.Background()); count != 2 { + t.Fatalf("fallback = %d, want the configured node count", count) + } + + rounds := respondAsNodes(t, nc, 2) + if count := gw.DiscoverActiveNodes(context.Background()); count != 2 { + t.Fatalf("second discovery = %d, want 2", count) + } + if rounds.Load() == 0 { + t.Error("no discovery was published after a fallback, so the fallback had been cached") + } +} diff --git a/spinifex/gateway/ecrauth/ecrauth_test.go b/spinifex/gateway/ecrauth/ecrauth_test.go index 1adf69b6a..3504dcee5 100644 --- a/spinifex/gateway/ecrauth/ecrauth_test.go +++ b/spinifex/gateway/ecrauth/ecrauth_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/golang-jwt/jwt/v5" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -52,6 +53,69 @@ func TestLoadOrCreateSigningKey_CreatesThenReloads(t *testing.T) { assert.Len(t, verify2, 1) } +// One unreadable key in the shared bucket must not stop the gateway starting. +// It took every awsgw in the cluster down, permanently, over a key kept only to +// verify tokens that had long since expired. +func TestLoadOrCreateSigningKey_SkipsAnUnreadableKey(t *testing.T) { + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + + good, _, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + require.NoError(t, err) + + kv, err := js.KeyValue(t.Context(), SigningBucket) + require.NoError(t, err) + _, err = kv.Put(t.Context(), signingKeyName("stale-kid"), []byte("not decryptable with this master key")) + require.NoError(t, err) + + key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + require.NoError(t, err) + assert.Equal(t, good.Kid, key.Kid, "the readable key must still be active") + assert.Len(t, verify, 1, "an unreadable key contributes no verify key") +} + +// A key listed and then deleted before it is read is the ordinary race against +// another node's rotator pruning it, not a reason to refuse to start. +func TestLoadOrCreateSigningKey_SkipsAKeyDeletedWhileReading(t *testing.T) { + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + + good, _, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + require.NoError(t, err) + + kv, err := js.KeyValue(t.Context(), SigningBucket) + require.NoError(t, err) + require.NoError(t, kv.Delete(t.Context(), signingKeyName("never-existed"))) + + key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + require.NoError(t, err) + assert.Equal(t, good.Kid, key.Kid) + assert.Len(t, verify, 1) +} + +// Every key failing is the master key being wrong for the whole bucket. Minting +// a fresh one alongside would invalidate every issued token and hide the cause, +// so it fails closed instead. +func TestLoadOrCreateSigningKey_FailsWhenNoKeyDecrypts(t *testing.T) { + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + + _, err := openSigningBucket(t.Context(), js, testMasterKey, 1) + require.NoError(t, err) + kv, err := js.KeyValue(t.Context(), SigningBucket) + require.NoError(t, err) + _, err = kv.Put(t.Context(), signingKeyName("stale-kid"), []byte("not decryptable with this master key")) + require.NoError(t, err) + + _, _, err = LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + require.Error(t, err) + assert.Contains(t, err.Error(), "master key does not match") + + entries, err := kvutil.Keys(t.Context(), kv) + require.NoError(t, err) + assert.Len(t, entries, 1, "a failed load must not mint a key alongside the unreadable one") +} + func TestLoadOrCreateSigningKey_StoresEncrypted(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) diff --git a/spinifex/gateway/ecrauth/rotator.go b/spinifex/gateway/ecrauth/rotator.go index 1a557910c..c3b65b74d 100644 --- a/spinifex/gateway/ecrauth/rotator.go +++ b/spinifex/gateway/ecrauth/rotator.go @@ -119,7 +119,7 @@ func (r *Rotator) Run(ctx context.Context) { // rotateOnce runs one rotation/prune cycle and refreshes the issuer/verifier. // Errors are logged and skipped so a transient KV fault never crashes the loop. func (r *Rotator) rotateOnce(ctx context.Context, now time.Time) { - _, _, metas, err := reloadKeys(ctx, r.kv, r.masterKey) + _, _, metas, _, err := reloadKeys(ctx, r.kv, r.masterKey) if err != nil { slog.Warn("ECR signing-key rotation: load keys failed", "err", err) return @@ -147,7 +147,7 @@ func (r *Rotator) rotateOnce(ctx context.Context, now time.Time) { // refresh reloads the canonical key set and installs it on the issuer/verifier. func (r *Rotator) refresh(ctx context.Context) { - active, verify, _, err := reloadKeys(ctx, r.kv, r.masterKey) + active, verify, _, _, err := reloadKeys(ctx, r.kv, r.masterKey) if err != nil { slog.Warn("ECR signing-key rotation: refresh failed", "err", err) return diff --git a/spinifex/gateway/ecrauth/rotator_test.go b/spinifex/gateway/ecrauth/rotator_test.go index 83fa2f288..2a9907d80 100644 --- a/spinifex/gateway/ecrauth/rotator_test.go +++ b/spinifex/gateway/ecrauth/rotator_test.go @@ -109,7 +109,7 @@ func TestRotateOnce_MintsAndSwaps(t *testing.T) { _, err = verifier.Verify(newTok) require.NoError(t, err, "token from the rotated key must verify") - _, _, metas, err := reloadKeys(t.Context(), kv, testMasterKey) + _, _, metas, _, err := reloadKeys(t.Context(), kv, testMasterKey) require.NoError(t, err) assert.Len(t, metas, 2, "previous key retained alongside the new active key") } @@ -138,7 +138,7 @@ func TestRotateOnce_PrunesExpiredKey(t *testing.T) { } r.rotateOnce(t.Context(), time.Now().UTC()) - active, _, metas, err := reloadKeys(t.Context(), kv, testMasterKey) + active, _, metas, _, err := reloadKeys(t.Context(), kv, testMasterKey) require.NoError(t, err) require.Len(t, metas, 1, "the older key should be pruned") assert.Equal(t, second.Kid, active.Kid, "newest key remains active") @@ -191,7 +191,7 @@ func TestIssuerVerifier_ConcurrentSwap(t *testing.T) { require.NoError(t, err) k2, err := generateSigningKey(t.Context(), kv, testMasterKey) require.NoError(t, err) - _, v2, _, err := reloadKeys(t.Context(), kv, testMasterKey) + _, v2, _, _, err := reloadKeys(t.Context(), kv, testMasterKey) require.NoError(t, err) issuer := NewIssuer(k1, testAudience) diff --git a/spinifex/gateway/ecrauth/signingkey.go b/spinifex/gateway/ecrauth/signingkey.go index 6e89ccc29..e35a887b9 100644 --- a/spinifex/gateway/ecrauth/signingkey.go +++ b/spinifex/gateway/ecrauth/signingkey.go @@ -16,6 +16,7 @@ import ( "encoding/pem" "errors" "fmt" + "log/slog" "strings" "time" @@ -80,11 +81,18 @@ func LoadOrCreateSigningKey(ctx context.Context, js jetstream.JetStream, masterK if err != nil { return nil, nil, err } - active, verify, _, err := reloadKeys(ctx, kv, masterKey) + active, verify, _, undecodable, err := reloadKeys(ctx, kv, masterKey) if err != nil { return nil, nil, err } if active == nil { + // Keys are present and none of them opened. That is this node's master + // key being wrong for the whole bucket, not one stale entry, and minting + // a fresh key alongside would invalidate every issued token while hiding + // the cause. + if undecodable > 0 { + return nil, nil, fmt.Errorf("no signing key could be decrypted (%d stored): the IAM master key does not match this bucket", undecodable) + } active, err = generateSigningKey(ctx, kv, masterKey) if err != nil { return nil, nil, err @@ -107,12 +115,19 @@ func openSigningBucket(ctx context.Context, js jetstream.JetStream, masterKey [] } // reloadKeys reads every stored signing key, returning the verify set, each -// key's metadata, and the active (newest) decrypted key. active is nil for an -// empty bucket. Used by both startup and the rotation scheduler. -func reloadKeys(ctx context.Context, kv jetstream.KeyValue, masterKey []byte) (active *SigningKey, verify map[string]*ecdsa.PublicKey, metas []keyMeta, err error) { +// key's metadata, the active (newest) decrypted key, and how many stored keys +// could not be decrypted. active is nil for an empty bucket. Used by both +// startup and the rotation scheduler. +// +// A key that will not decrypt is counted and skipped rather than returned as an +// error. Old keys are kept only to verify tokens already issued under them, so +// one unreadable entry costs those tokens and nothing else — failing the whole +// load instead takes the gateway down on every node, permanently, over a key no +// live request needs. +func reloadKeys(ctx context.Context, kv jetstream.KeyValue, masterKey []byte) (active *SigningKey, verify map[string]*ecdsa.PublicKey, metas []keyMeta, undecodable int, err error) { names, err := kvutil.Keys(ctx, kv) if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { - return nil, nil, nil, fmt.Errorf("list signing keys: %w", err) + return nil, nil, nil, 0, fmt.Errorf("list signing keys: %w", err) } verify = make(map[string]*ecdsa.PublicKey) @@ -122,12 +137,22 @@ func reloadKeys(ctx context.Context, kv jetstream.KeyValue, masterKey []byte) (a continue } entry, err := kv.Get(ctx, name) + if errors.Is(err, jetstream.ErrKeyNotFound) { + // Listed but gone by the time it was read, which is the ordinary + // race against another node's rotator pruning it. Any other get + // error is the store failing and is still returned. + slog.WarnContext(ctx, "ECR auth bridge: signing key vanished between list and read", "key", name) + continue + } if err != nil { - return nil, nil, nil, fmt.Errorf("kv get %s: %w", name, err) + return nil, nil, nil, 0, fmt.Errorf("kv get %s: %w", name, err) } sk, err := decodeSigningKey(strings.TrimPrefix(name, signingKeyPrefix), entry.Value(), masterKey) if err != nil { - return nil, nil, nil, err + undecodable++ + slog.WarnContext(ctx, "ECR auth bridge: skipping unreadable signing key", + "key", name, "err", err) + continue } verify[sk.Kid] = &sk.priv.PublicKey m := keyMeta{kid: sk.Kid, created: entry.Created()} @@ -136,7 +161,7 @@ func reloadKeys(ctx context.Context, kv jetstream.KeyValue, masterKey []byte) (a active, activeMeta = sk, m } } - return active, verify, metas, nil + return active, verify, metas, undecodable, nil } // deleteSigningKey removes a rotated-out key from the bucket once its retention diff --git a/spinifex/gateway/gateway.go b/spinifex/gateway/gateway.go index 25c4fd4e3..815d91698 100644 --- a/spinifex/gateway/gateway.go +++ b/spinifex/gateway/gateway.go @@ -12,6 +12,7 @@ import ( "net/http" "net/url" "strings" + "sync" "time" "github.com/go-chi/chi/v5" @@ -22,6 +23,7 @@ import ( bbotel "github.com/mulgadc/bluebottle/pkg/otelsetup" "github.com/mulgadc/bluebottle/pkg/ratelimit" "github.com/mulgadc/bluebottle/pkg/sigv4" + "github.com/mulgadc/spinifex/spinifex/accountteardown" "github.com/mulgadc/spinifex/spinifex/awserrors" gateway_bedrock "github.com/mulgadc/spinifex/spinifex/gateway/bedrock" gateway_ecr "github.com/mulgadc/spinifex/spinifex/gateway/ecr" @@ -93,10 +95,16 @@ type GatewayConfig struct { // dialed directly for GetStorageStatus. Nil disables that verification, // so admin storage status reports every meta node unreachable rather than // dialing with no trust root. - RootCAs *x509.CertPool - ExpectedNodes int // Number of expected spinifex nodes for multi-node operations - Region string // Region this gateway is running in - InternalSuffix string // Internal DNS suffix for AWS-parity endpoints (e.g. spinifex.internal) + RootCAs *x509.CertPool + ExpectedNodes int // Number of expected spinifex nodes for multi-node operations + Region string // Region this gateway is running in + // The last discovered node count and when it was discovered, so the + // discovery fan-out runs once per activeNodesTTL rather than once per + // request that needs to know how many nodes to wait for. + activeNodesMu sync.RWMutex + activeNodesCount int + activeNodesAt time.Time + InternalSuffix string // Internal DNS suffix for AWS-parity endpoints (e.g. spinifex.internal) // RegistryPort is the gateway's advertised port, appended to the ECR // registry host so docker login/tag/push dial the right port. Empty or // "443" renders a port-less host (standard HTTPS parity). @@ -108,9 +116,17 @@ type GatewayConfig struct { RegistryHost string AZ string // Availability zone this gateway is running in IAMService handlers_iam.IAMService - STSService handlers_sts.STSService - RateLimiter *AuthRateLimiter // Per-IP auth failure rate limiter - Throttler *ratelimit.Throttler // Per-account+action API request throttler + // BucketStore reaps a tenant's S3 buckets during account teardown. It + // signs with the config service credential, which predastore already + // trusts to reach any bucket, so this grants enumeration, not access. + // Nil makes DeleteAccount refuse rather than tear down around the data. + BucketStore accountteardown.BucketStore + STSService handlers_sts.STSService + RateLimiter *AuthRateLimiter // Per-IP auth failure rate limiter + Throttler *ratelimit.Throttler // Per-account+action API request throttler + // accountStatus caches which accounts are ACTIVE, so enforcing account + // status does not add a KV read to every authenticated request. + accountStatus *accountStatusCache // Quota enforces per-account service quotas. Built unconditionally; a disabled // config yields a no-op Service whose Exempt always returns true. Nil only in // unit tests of unrelated routes, where no handler reaches the quota checks. @@ -782,16 +798,30 @@ func GenerateIAMErrorResponse(code, message, requestID string) (output []byte) { return output } +// How long a discovered node count is reused before it is gathered again. +// Membership does not change per request, and re-deriving it on every call put +// the discovery fan-out's whole timeout in front of every API call it fronts. +const activeNodesTTL = 5 * time.Second + // DiscoverActiveNodes discovers the number of active spinifex daemon nodes in the // cluster by publishing a discovery request and counting unique responses. It // carries the request context so the discovery fan-out joins the caller's trace. // Returns the number of active nodes (minimum 1 if fallback is needed). +// +// The result is cached for activeNodesTTL. The gather itself stays unbounded: +// it counts whoever answers, which may legitimately exceed the configured node +// count during a grow, and bounding it by that count would stop early and drop +// a live node's resources from every fan-out that follows. func (gw *GatewayConfig) DiscoverActiveNodes(ctx context.Context) int { if gw.NATSConn == nil { slog.WarnContext(ctx, "DiscoverActiveNodes: NATS connection not available, using ExpectedNodes fallback", "fallback", gw.ExpectedNodes) return gw.ExpectedNodes } + if count, ok := gw.cachedActiveNodes(); ok { + return count + } + frames, _, err := utils.Gather(ctx, gw.NATSConn, "spinifex.nodes.discover", []byte("{}"), utils.GatherOpts{Timeout: 500 * time.Millisecond}) if err != nil { @@ -815,10 +845,30 @@ func (gw *GatewayConfig) DiscoverActiveNodes(ctx context.Context) int { return gw.ExpectedNodes } + gw.rememberActiveNodes(activeNodes) slog.DebugContext(ctx, "DiscoverActiveNodes: Discovered active nodes", "count", activeNodes) return activeNodes } +// cachedActiveNodes returns a count discovered within the TTL. Only a real +// discovery is cached: a fallback is not evidence of anything and caching one +// would make a momentary NATS problem outlive itself. +func (gw *GatewayConfig) cachedActiveNodes() (int, bool) { + gw.activeNodesMu.RLock() + defer gw.activeNodesMu.RUnlock() + if gw.activeNodesCount == 0 || time.Since(gw.activeNodesAt) > activeNodesTTL { + return 0, false + } + return gw.activeNodesCount, true +} + +func (gw *GatewayConfig) rememberActiveNodes(count int) { + gw.activeNodesMu.Lock() + defer gw.activeNodesMu.Unlock() + gw.activeNodesCount = count + gw.activeNodesAt = time.Now() +} + // recordResolvedAction renames the current span to service.action, tags it // with aws.service/aws.action, and updates the request's metric action name. // Query-protocol services resolve their action during SigV4 auth, before diff --git a/spinifex/handlers/ec2/image/service_impl.go b/spinifex/handlers/ec2/image/service_impl.go index f6a873ce3..0ea0ca88d 100644 --- a/spinifex/handlers/ec2/image/service_impl.go +++ b/spinifex/handlers/ec2/image/service_impl.go @@ -549,7 +549,7 @@ func (s *ImageServiceImpl) GetAMISourceVolumeID(ctx context.Context, imageID str return "", errors.New(awserrors.ErrorInvalidAMIIDNotFound) } - snapCfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(s.store, s.bucketName, meta.SnapshotID) + snapCfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(ctx, s.store, s.bucketName, meta.SnapshotID) if err != nil { if objectstore.IsNoSuchKeyError(err) && meta.ImageOwnerAlias != "" && !utils.IsAccountID(meta.ImageOwnerAlias) { slog.WarnContext(ctx, "GetAMISourceVolumeID: system AMI has no snapshot metadata document, falling back to imageID as volume ID", @@ -736,7 +736,7 @@ func (s *ImageServiceImpl) CopyImage(ctx context.Context, input *ec2.CopyImageIn if srcMeta.SnapshotID == "" { return nil, errors.New(awserrors.ErrorInvalidAMIIDNotFound) } - srcSnap, err := handlers_ec2_snapshot.ReadSnapshotConfig(s.store, s.bucketName, srcMeta.SnapshotID) + srcSnap, err := handlers_ec2_snapshot.ReadSnapshotConfig(ctx, s.store, s.bucketName, srcMeta.SnapshotID) if err != nil { // Bundled system AMIs have no standalone snap-xxx/metadata.json; synthesize // a minimal snap view using VolumeID = sourceImageID so CopyImage succeeds. @@ -939,7 +939,7 @@ func (s *ImageServiceImpl) RegisterImage(ctx context.Context, input *ec2.Registe } snapshotID := *rootBDM.Ebs.SnapshotId - snapCfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(s.store, s.bucketName, snapshotID) + snapCfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(ctx, s.store, s.bucketName, snapshotID) if err != nil { // Corrupt snapshot is surfaced as NotFound, same as CopyImage. if objectstore.IsNoSuchKeyError(err) || errors.Is(err, handlers_ec2_snapshot.ErrCorruptSnapshotMetadata) { diff --git a/spinifex/handlers/ec2/image/service_impl_test.go b/spinifex/handlers/ec2/image/service_impl_test.go index c60466890..715fc5291 100644 --- a/spinifex/handlers/ec2/image/service_impl_test.go +++ b/spinifex/handlers/ec2/image/service_impl_test.go @@ -1677,7 +1677,7 @@ func TestCopyImage_NewSnapshotSharesSourceVolumeID(t *testing.T) { seedCopyableAMI(t, store, "ami-shareblocks", "shareblocks", testAccountID, "snap-orig", "vol-shared", 16) - srcSnap, err := handlers_ec2_snapshot.ReadSnapshotConfig(store, testBucket, "snap-orig") + srcSnap, err := handlers_ec2_snapshot.ReadSnapshotConfig(context.Background(), store, testBucket, "snap-orig") require.NoError(t, err) out, err := svc.CopyImage(context.Background(), validCopyImageServiceInput("ami-shareblocks", "shared-copy"), testAccountID) @@ -1687,7 +1687,7 @@ func TestCopyImage_NewSnapshotSharesSourceVolumeID(t *testing.T) { require.NoError(t, err) require.NotEqual(t, "snap-orig", newMeta.SnapshotID) - newSnap, err := handlers_ec2_snapshot.ReadSnapshotConfig(store, testBucket, newMeta.SnapshotID) + newSnap, err := handlers_ec2_snapshot.ReadSnapshotConfig(context.Background(), store, testBucket, newMeta.SnapshotID) require.NoError(t, err) // Compare against the source snapshot, not hard-coded literals — proves // the new snap truly inherits the source's VolumeID rather than happening @@ -1744,7 +1744,7 @@ func TestCopyImage_BundledSystemAMINoStandaloneSnap(t *testing.T) { // snap-ami-bundled01 viperblock reference. require.NotEqual(t, "snap-ami-bundled01", newMeta.SnapshotID, "copy must mint a new user-owned snap id, not borrow the source's") - newSnap, err := handlers_ec2_snapshot.ReadSnapshotConfig(store, testBucket, newMeta.SnapshotID) + newSnap, err := handlers_ec2_snapshot.ReadSnapshotConfig(context.Background(), store, testBucket, newMeta.SnapshotID) require.NoError(t, err) assert.Equal(t, "ami-bundled01", newSnap.VolumeID, "bundled fallback must point the new snap at the source AMI's prefix") diff --git a/spinifex/handlers/ec2/instance/service_impl.go b/spinifex/handlers/ec2/instance/service_impl.go index cf8428cc1..de32a33d9 100644 --- a/spinifex/handlers/ec2/instance/service_impl.go +++ b/spinifex/handlers/ec2/instance/service_impl.go @@ -497,6 +497,12 @@ func (s *InstanceServiceImpl) PrepareRunInstances(ctx context.Context, input *ec return nil, nil, nil, errors.New(awserrors.ErrorServerInternal) } if err := s.keyValidator.ValidateKeyPairExists(ctx, accountID, *input.KeyName); err != nil { + // Only a key the store answered for is absent. Reporting an unreadable + // store as a missing key sends the caller after a key that exists. + if err.Error() != awserrors.ErrorInvalidKeyPairNotFound { + slog.ErrorContext(ctx, "PrepareRunInstances: key pair unreadable", "keyName", *input.KeyName, "err", err) + return nil, nil, nil, errors.New(awserrors.ErrorServerInternal) + } slog.ErrorContext(ctx, "PrepareRunInstances: key pair not found", "keyName", *input.KeyName, "err", err) return nil, nil, nil, errors.New(awserrors.ErrorInvalidKeyPairNotFound) } diff --git a/spinifex/handlers/ec2/instance/service_impl_test.go b/spinifex/handlers/ec2/instance/service_impl_test.go index 417aeb814..cf8aa2f88 100644 --- a/spinifex/handlers/ec2/instance/service_impl_test.go +++ b/spinifex/handlers/ec2/instance/service_impl_test.go @@ -2536,7 +2536,7 @@ func TestPrepareRunInstances_KeyPairNotFound(t *testing.T) { amiLoader: &fakeAMILoader{byID: map[string]ebsmetadata.AMI{ "ami-1": {ImageOwnerAlias: "acc"}, }}, - keyValidator: &fakeKeyValidator{err: errors.New("no key")}, + keyValidator: &fakeKeyValidator{err: errors.New(awserrors.ErrorInvalidKeyPairNotFound)}, } _, _, _, err := svc.PrepareRunInstances(context.Background(), &ec2.RunInstancesInput{ InstanceType: aws.String("t3.micro"), @@ -2549,6 +2549,28 @@ func TestPrepareRunInstances_KeyPairNotFound(t *testing.T) { assert.Equal(t, awserrors.ErrorInvalidKeyPairNotFound, err.Error()) } +// A store that cannot be read says nothing about whether the key is there, and +// answering NotFound sends the caller after a key that exists. +func TestPrepareRunInstances_KeyPairUnreadableIsNotNotFound(t *testing.T) { + types, _ := defaultPrepareInstanceTypes() + svc := &InstanceServiceImpl{ + instanceTypes: types, + amiLoader: &fakeAMILoader{byID: map[string]ebsmetadata.AMI{ + "ami-1": {ImageOwnerAlias: "acc"}, + }}, + keyValidator: &fakeKeyValidator{err: errors.New("dial tcp: connection refused")}, + } + _, _, _, err := svc.PrepareRunInstances(context.Background(), &ec2.RunInstancesInput{ + InstanceType: aws.String("t3.micro"), + ImageId: aws.String("ami-1"), + KeyName: aws.String("real-key"), + MinCount: aws.Int64(1), + MaxCount: aws.Int64(1), + }, "acc", "") + require.Error(t, err) + assert.Equal(t, awserrors.ErrorServerInternal, err.Error()) +} + func TestPrepareRunInstances_InsufficientCapacity(t *testing.T) { types, it := defaultPrepareInstanceTypes() prov := &fakeResourceCapacityProvider{ diff --git a/spinifex/handlers/ec2/key/service_impl.go b/spinifex/handlers/ec2/key/service_impl.go index 139bf5dcd..e4ec64f1a 100644 --- a/spinifex/handlers/ec2/key/service_impl.go +++ b/spinifex/handlers/ec2/key/service_impl.go @@ -448,9 +448,25 @@ func (s *KeyServiceImpl) findKeyPairIdFromKeyName(ctx context.Context, accountID // ValidateKeyPairExists checks if a key pair with the given name exists. // Returns nil if the key pair exists, or an error with ErrorInvalidKeyPairNotFound if not. +// The stored material is addressed by name, so this reads it directly rather +// than listing the account's keys and scanning for a match. A listing is served +// from one replica's view and can lag a write that has already been +// acknowledged, which reports a key created moments earlier as absent. func (s *KeyServiceImpl) ValidateKeyPairExists(ctx context.Context, accountID, keyName string) error { - _, err := s.findKeyPairIdFromKeyName(ctx, accountID, keyName) - return err + keyPath := fmt.Sprintf("keys/%s/%s", accountID, keyName) + if _, err := s.store.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(s.bucketName), + Key: aws.String(keyPath), + }); err != nil { + if objectstore.IsNoSuchKeyError(err) { + return errors.New(awserrors.ErrorInvalidKeyPairNotFound) + } + // Not "absent": saying so would send the caller off to fix a key that is + // there, and a launch would keep failing until the read recovered. + slog.ErrorContext(ctx, "Failed to read key pair for validation", "keyName", keyName, "err", err) + return fmt.Errorf("validate key pair %s: %w", keyPath, err) + } + return nil } // GetPublicKeyMaterial returns the stored OpenSSH public key line, trimmed to a single diff --git a/spinifex/handlers/ec2/key/service_impl_test.go b/spinifex/handlers/ec2/key/service_impl_test.go index 6ae39064c..399b2a442 100644 --- a/spinifex/handlers/ec2/key/service_impl_test.go +++ b/spinifex/handlers/ec2/key/service_impl_test.go @@ -773,6 +773,18 @@ func TestValidateKeyPairExists_Found(t *testing.T) { assert.NoError(t, err) } +// The id-keyed metadata is what a listing has to find, and a listing is served +// from one replica's view. Validation must not depend on it: the material is +// addressed by name and a direct read of it answers the question on its own. +func TestValidateKeyPairExists_FoundWithoutTheMetadataListing(t *testing.T) { + svc, store := newTestKeyService() + + putStoredObject(t, store, "keys/"+testAccountID+"/listing-lagged", testED25519PubKey) + + err := svc.ValidateKeyPairExists(context.Background(), testAccountID, "listing-lagged") + assert.NoError(t, err) +} + func TestValidateKeyPairExists_NotFound(t *testing.T) { svc, _ := newTestKeyService() diff --git a/spinifex/handlers/ec2/snapshot/describe_fanout_test.go b/spinifex/handlers/ec2/snapshot/describe_fanout_test.go new file mode 100644 index 000000000..141469985 --- /dev/null +++ b/spinifex/handlers/ec2/snapshot/describe_fanout_test.go @@ -0,0 +1,104 @@ +package handlers_ec2_snapshot_test + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/config" + handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot" + "github.com/mulgadc/spinifex/spinifex/objectstore" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const fanoutAccountID = "111122223333" + +// slowMetadataStore delays every snapshot metadata read and records how many +// were in flight at once, standing in for an object store under load. +type slowMetadataStore struct { + objectstore.ObjectStore + + delay time.Duration + + mu sync.Mutex + inFlite int + peak int +} + +func (s *slowMetadataStore) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { + s.mu.Lock() + s.inFlite++ + if s.inFlite > s.peak { + s.peak = s.inFlite + } + s.mu.Unlock() + defer func() { + s.mu.Lock() + s.inFlite-- + s.mu.Unlock() + }() + + select { + case <-time.After(s.delay): + case <-ctx.Done(): + return nil, ctx.Err() + } + return s.ObjectStore.GetObject(ctx, input) +} + +func seedSnapshots(t *testing.T, store objectstore.ObjectStore, n int) *handlers_ec2_snapshot.SnapshotServiceImpl { + t.Helper() + cfg := &config.Config{Predastore: config.PredastoreConfig{Bucket: "test-bucket"}} + svc := handlers_ec2_snapshot.NewSnapshotServiceImplWithStore(cfg, store, nil) + for i := range n { + id := "snap-fanout" + string(rune('a'+i%26)) + string(rune('a'+i/26)) + require.NoError(t, handlers_ec2_snapshot.WriteSnapshotConfig(store, "test-bucket", id, + &handlers_ec2_snapshot.SnapshotConfig{ + SnapshotID: id, VolumeID: "vol-fanout", State: "completed", + OwnerID: fanoutAccountID, StartTime: time.Now(), + })) + } + return svc +} + +// TestDescribeSnapshotsReadsMetadataConcurrently pins the fix for a describe +// whose latency was the sum of every snapshot's metadata read: 24 snapshots at +// 100ms each took 2.4s serially, which is enough to miss the caller's timeout +// on an account that has done nothing wrong. +func TestDescribeSnapshotsReadsMetadataConcurrently(t *testing.T) { + store := &slowMetadataStore{ObjectStore: objectstore.NewMemoryObjectStore(), delay: 100 * time.Millisecond} + svc := seedSnapshots(t, store, 24) + + start := time.Now() + out, err := svc.DescribeSnapshots(context.Background(), &ec2.DescribeSnapshotsInput{}, fanoutAccountID) + elapsed := time.Since(start) + + require.NoError(t, err) + assert.Len(t, out.Snapshots, 24) + assert.Greater(t, store.peak, 1, "metadata reads must overlap, not run one after the other") + assert.Less(t, elapsed, time.Second, + "a describe must cost about the slowest read, not the sum of them all") +} + +// TestDescribeSnapshotsReportsAnExpiredDeadline covers the case that made this +// visible in production: when the caller has already given up, every metadata +// read fails, and an empty snapshot list would read as "this account has none". +func TestDescribeSnapshotsReportsAnExpiredDeadline(t *testing.T) { + store := &slowMetadataStore{ObjectStore: objectstore.NewMemoryObjectStore(), delay: time.Second} + svc := seedSnapshots(t, store, 4) + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + out, err := svc.DescribeSnapshots(ctx, &ec2.DescribeSnapshotsInput{}, fanoutAccountID) + require.Error(t, err) + assert.Nil(t, out) + assert.Equal(t, awserrors.ErrorServerInternal, err.Error(), + "a describe that ran out of time must not answer with a short list") + assert.ErrorIs(t, ctx.Err(), context.DeadlineExceeded) +} diff --git a/spinifex/handlers/ec2/snapshot/record_tags_test.go b/spinifex/handlers/ec2/snapshot/record_tags_test.go index 4a587c06e..e398ff25f 100644 --- a/spinifex/handlers/ec2/snapshot/record_tags_test.go +++ b/spinifex/handlers/ec2/snapshot/record_tags_test.go @@ -1,6 +1,7 @@ package handlers_ec2_snapshot import ( + "context" "testing" "github.com/aws/aws-sdk-go/aws" @@ -21,7 +22,7 @@ func TestSnapshotRecordTagsMirror(t *testing.T) { }, }, testAccountID)) - cfg, err := svc.getSnapshotConfig(snapID) + cfg, err := svc.getSnapshotConfig(context.Background(), snapID) require.NoError(t, err) assert.Equal(t, "yes", cfg.Tags["keep"]) assert.Equal(t, "v", cfg.Tags["drop"]) @@ -35,7 +36,7 @@ func TestSnapshotRecordTagsMirror(t *testing.T) { }, }, testAccountID)) - cfg, err = svc.getSnapshotConfig(snapID) + cfg, err = svc.getSnapshotConfig(context.Background(), snapID) require.NoError(t, err) assert.Equal(t, "yes", cfg.Tags["keep"]) _, ok := cfg.Tags["drop"] @@ -52,7 +53,7 @@ func TestSnapshotRecordTagsMirror_CrossTenantNoMutation(t *testing.T) { Tags: []*ec2.Tag{{Key: aws.String("evil"), Value: aws.String("1")}}, }, "999999999999")) - cfg, err := svc.getSnapshotConfig(snapID) + cfg, err := svc.getSnapshotConfig(context.Background(), snapID) require.NoError(t, err) assert.Equal(t, map[string]string{"orig": "1"}, cfg.Tags) } diff --git a/spinifex/handlers/ec2/snapshot/service_impl.go b/spinifex/handlers/ec2/snapshot/service_impl.go index e8ec7b5ef..febb97580 100644 --- a/spinifex/handlers/ec2/snapshot/service_impl.go +++ b/spinifex/handlers/ec2/snapshot/service_impl.go @@ -146,10 +146,12 @@ var ErrCorruptSnapshotMetadata = errors.New("corrupt snapshot metadata") // ReadSnapshotConfig reads {snapshotID}/metadata.json. Object-store errors are // returned unchanged; callers map NoSuchKey to their preferred AWS error. -// Decode failures wrap ErrCorruptSnapshotMetadata. -func ReadSnapshotConfig(store objectstore.ObjectStore, bucket, snapshotID string) (*SnapshotConfig, error) { +// Decode failures wrap ErrCorruptSnapshotMetadata. ctx carries the caller's +// deadline: a describe that has already given up must not keep the object +// store busy on its behalf. +func ReadSnapshotConfig(ctx context.Context, store objectstore.ObjectStore, bucket, snapshotID string) (*SnapshotConfig, error) { key := GetSnapshotKey(snapshotID) - result, err := store.GetObject(context.Background(), &s3.GetObjectInput{ + result, err := store.GetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), }) @@ -181,8 +183,8 @@ func WriteSnapshotConfig(store objectstore.ObjectStore, bucket, snapshotID strin } // getSnapshotConfig translates NoSuchKey to InvalidSnapshot.NotFound. -func (s *SnapshotServiceImpl) getSnapshotConfig(snapshotID string) (*SnapshotConfig, error) { - cfg, err := ReadSnapshotConfig(s.store, s.config.Predastore.Bucket, snapshotID) +func (s *SnapshotServiceImpl) getSnapshotConfig(ctx context.Context, snapshotID string) (*SnapshotConfig, error) { + cfg, err := ReadSnapshotConfig(ctx, s.store, s.config.Predastore.Bucket, snapshotID) if err != nil { if objectstore.IsNoSuchKeyError(err) { return nil, errors.New(awserrors.ErrorInvalidSnapshotNotFound) @@ -522,7 +524,7 @@ func (s *SnapshotServiceImpl) describeSnapshots(ctx context.Context, input *ec2. snapshotIDFilterValues = parsedFilters["snapshot-id"] } - var snapshots []*ec2.Snapshot + var wanted []string for _, prefix := range listResult.CommonPrefixes { if prefix.Prefix == nil { continue @@ -542,11 +544,27 @@ func (s *SnapshotServiceImpl) describeSnapshots(ctx context.Context, input *ec2. } } - cfg, err := s.getSnapshotConfig(snapshotID) - if err != nil { - slog.WarnContext(ctx, "DescribeSnapshots failed to get config", "snapshotId", snapshotID, "err", err) + wanted = append(wanted, snapshotID) + } + + configs, readErrs := s.readSnapshotConfigs(ctx, wanted) + + // Every read fails once the caller's deadline passes, and reporting that as + // an empty list would read as "this account has no snapshots". + if err := ctx.Err(); err != nil { + slog.WarnContext(ctx, "DescribeSnapshots gave up reading metadata", + "requested", len(wanted), "read", len(configs), "err", err) + return nil, errors.New(awserrors.ErrorServerInternal) + } + + var snapshots []*ec2.Snapshot + for _, snapshotID := range wanted { + cfg, ok := configs[snapshotID] + if !ok { + slog.WarnContext(ctx, "DescribeSnapshots failed to get config", + "snapshotId", snapshotID, "err", readErrs[snapshotID]) if strict { - return nil, fmt.Errorf("describe snapshot %s metadata: %w", snapshotID, err) + return nil, fmt.Errorf("describe snapshot %s metadata: %w", snapshotID, readErrs[snapshotID]) } continue } @@ -586,6 +604,46 @@ func (s *SnapshotServiceImpl) describeSnapshots(ctx context.Context, input *ec2. }, nil } +// describeSnapshotFanout bounds how many metadata reads a single describe has +// in flight. A describe used to read every snapshot's metadata.json one after +// the other, so its latency was the sum of them all and a single slow object +// made the whole listing miss its caller's deadline. The bound keeps a large +// account from turning one describe into a burst against the object store. +const describeSnapshotFanout = 16 + +// readSnapshotConfigs fetches metadata for each snapshot concurrently. It +// returns what it could read plus the error for each one it could not, so the +// caller decides whether an unreadable snapshot is fatal or simply skipped. +func (s *SnapshotServiceImpl) readSnapshotConfigs( + ctx context.Context, ids []string, +) (map[string]*SnapshotConfig, map[string]error) { + configs := make(map[string]*SnapshotConfig, len(ids)) + readErrs := make(map[string]error) + if len(ids) == 0 { + return configs, readErrs + } + + var mu sync.Mutex + sem := make(chan struct{}, describeSnapshotFanout) + var wg sync.WaitGroup + for _, id := range ids { + sem <- struct{}{} + wg.Go(func() { + defer func() { <-sem }() + cfg, err := s.getSnapshotConfig(ctx, id) + mu.Lock() + defer mu.Unlock() + if err != nil { + readErrs[id] = err + return + } + configs[id] = cfg + }) + } + wg.Wait() + return configs, readErrs +} + // snapshotMatchesFilters checks whether a SnapshotConfig satisfies all parsed filters. func snapshotMatchesFilters(cfg *SnapshotConfig, filters map[string][]string) bool { for name, values := range filters { @@ -645,7 +703,7 @@ func (s *SnapshotServiceImpl) DeleteSnapshot(ctx context.Context, input *ec2.Del slog.InfoContext(ctx, "DeleteSnapshot request", "snapshotId", snapshotID, "accountID", accountID) - cfg, err := s.getSnapshotConfig(snapshotID) + cfg, err := s.getSnapshotConfig(ctx, snapshotID) if err != nil { slog.ErrorContext(ctx, "DeleteSnapshot snapshot not found", "snapshotId", snapshotID, "err", err) return nil, err @@ -721,7 +779,7 @@ func (s *SnapshotServiceImpl) CopySnapshot(ctx context.Context, input *ec2.CopyS slog.InfoContext(ctx, "CopySnapshot request", "sourceSnapshotId", sourceSnapshotID, "accountID", accountID) - sourceCfg, err := s.getSnapshotConfig(sourceSnapshotID) + sourceCfg, err := s.getSnapshotConfig(ctx, sourceSnapshotID) if err != nil { slog.ErrorContext(ctx, "CopySnapshot source snapshot not found", "snapshotId", sourceSnapshotID, "err", err) return nil, err @@ -960,7 +1018,7 @@ func (s *SnapshotServiceImpl) mirrorSnapshotTags(resources []*string, accountID if res == nil || !strings.HasPrefix(*res, "snap-") { continue } - cfg, err := ReadSnapshotConfig(s.store, s.config.Predastore.Bucket, *res) + cfg, err := ReadSnapshotConfig(context.Background(), s.store, s.config.Predastore.Bucket, *res) if err != nil { if objectstore.IsNoSuchKeyError(err) { continue diff --git a/spinifex/handlers/ec2/snapshot/service_impl_test.go b/spinifex/handlers/ec2/snapshot/service_impl_test.go index 8cd89b423..86d8d151b 100644 --- a/spinifex/handlers/ec2/snapshot/service_impl_test.go +++ b/spinifex/handlers/ec2/snapshot/service_impl_test.go @@ -998,7 +998,7 @@ func TestCreateDeleteSnapshot_UsesInjectedProvider(t *testing.T) { snapshot, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-provider")}, testAccountID) require.NoError(t, err) require.NotNil(t, snapshot) - cfgStored, err := svc.getSnapshotConfig(aws.StringValue(snapshot.SnapshotId)) + cfgStored, err := svc.getSnapshotConfig(context.Background(), aws.StringValue(snapshot.SnapshotId)) require.NoError(t, err) assert.Equal(t, "memory://snapshot/"+aws.StringValue(snapshot.SnapshotId), cfgStored.ProviderHandle) _, err = svc.DeleteSnapshot(context.Background(), &ec2.DeleteSnapshotInput{SnapshotId: snapshot.SnapshotId}, testAccountID) @@ -1152,7 +1152,7 @@ func TestCopySnapshot_Provider_CreatesBackendSnapshot(t *testing.T) { require.True(t, ok, "the copy must exist in the provider, not only in the control plane") assert.Equal(t, "vol-copysrc", got.SourceVolumeID) - stored, err := svc.getSnapshotConfig(newID) + stored, err := svc.getSnapshotConfig(context.Background(), newID) require.NoError(t, err) assert.Equal(t, "vol-copysrc", stored.VolumeID) assert.Equal(t, "copied", stored.Description) @@ -1194,7 +1194,7 @@ func TestCopySnapshot_Provider_InheritsSourceDescription(t *testing.T) { out, err := svc.CopySnapshot(ctx, &ec2.CopySnapshotInput{SourceSnapshotId: src.SnapshotId}, testAccountID) require.NoError(t, err) - stored, err := svc.getSnapshotConfig(aws.StringValue(out.SnapshotId)) + stored, err := svc.getSnapshotConfig(context.Background(), aws.StringValue(out.SnapshotId)) require.NoError(t, err) assert.Equal(t, "original text", stored.Description) } diff --git a/spinifex/handlers/ec2/volume/force_detach_test.go b/spinifex/handlers/ec2/volume/force_detach_test.go new file mode 100644 index 000000000..e9491124f --- /dev/null +++ b/spinifex/handlers/ec2/volume/force_detach_test.go @@ -0,0 +1,115 @@ +package handlers_ec2_volume + +//test:in-package — seeding a volume's metadata document is how the attachment +// under test is created, and the metadata store is unexported. + +import ( + "context" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newForceDetachService(t *testing.T) *VolumeServiceImpl { + t.Helper() + svc := newTestVolumeService("ap-southeast-2a") + svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})) + return svc +} + +func seedAttachedVolume(t *testing.T, svc *VolumeServiceImpl, volumeID, accountID, instanceID string) { + t.Helper() + require.NoError(t, svc.metadata.PutVolume(context.Background(), ebsmetadata.Volume{ + VolumeID: volumeID, TenantID: accountID, CapacityGiB: 1, + State: "available", AvailabilityZone: "ap-southeast-2a", VolumeType: "gp3", + })) + require.NoError(t, svc.UpdateVolumeState(volumeID, "in-use", instanceID, "/dev/sdf")) +} + +// This is the escape from the deadlock teardown exists to break: a volume +// attached to an instance that will not terminate leaves both undeletable, so +// the attachment is cleared without the guest's cooperation. +func TestForceDetachVolumeClearsTheAttachment(t *testing.T) { + svc := newForceDetachService(t) + seedAttachedVolume(t, svc, "vol-forcedetach1", "acct-tenant", "i-stuck0000000000") + + attachment, err := svc.ForceDetachVolume(context.Background(), &ec2.DetachVolumeInput{ + VolumeId: aws.String("vol-forcedetach1"), Force: aws.Bool(true), + }, "acct-tenant") + require.NoError(t, err) + + // The previous instance is reported back: it is the only record of what + // the volume was torn away from. + assert.Equal(t, "i-stuck0000000000", aws.StringValue(attachment.InstanceId)) + assert.Equal(t, "detached", aws.StringValue(attachment.State)) + + meta, err := svc.GetVolumeMetadata("vol-forcedetach1") + require.NoError(t, err) + assert.Equal(t, "available", meta.State) + assert.Empty(t, meta.AttachedInstance) + assert.Empty(t, meta.DeviceName) +} + +// The force path is still account-scoped. Teardown may escalate past a state +// guard; it may never reach a volume belonging to another tenant. +func TestForceDetachVolumeRefusesAnotherAccountsVolume(t *testing.T) { + svc := newForceDetachService(t) + seedAttachedVolume(t, svc, "vol-forcedetach2", "acct-owner", "i-stuck0000000000") + + _, err := svc.ForceDetachVolume(context.Background(), &ec2.DetachVolumeInput{ + VolumeId: aws.String("vol-forcedetach2"), + }, "acct-intruder") + + require.Error(t, err) + // The same answer an absent volume gives, so this cannot be used to prove + // a volume id exists in someone else's account. + assert.Equal(t, awserrors.ErrorInvalidVolumeNotFound, err.Error()) + + meta, err := svc.GetVolumeMetadata("vol-forcedetach2") + require.NoError(t, err) + assert.Equal(t, "i-stuck0000000000", meta.AttachedInstance) +} + +func TestForceDetachVolumeRejectsBadInput(t *testing.T) { + svc := newForceDetachService(t) + + for _, input := range []*ec2.DetachVolumeInput{nil, {}, {VolumeId: aws.String("")}} { + _, err := svc.ForceDetachVolume(context.Background(), input, "acct-tenant") + require.Error(t, err) + assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error()) + } +} + +func TestForceDetachVolumeReportsAMissingVolume(t *testing.T) { + svc := newForceDetachService(t) + + _, err := svc.ForceDetachVolume(context.Background(), &ec2.DetachVolumeInput{ + VolumeId: aws.String("vol-doesnotexist"), + }, "acct-tenant") + + require.Error(t, err) + assert.Equal(t, awserrors.ErrorInvalidVolumeNotFound, err.Error()) +} + +// An unattached volume is already in the state the caller wants, so force +// detach is a no-op rather than an error — teardown re-runs after a crash. +func TestForceDetachVolumeOnAnUnattachedVolume(t *testing.T) { + svc := newForceDetachService(t) + require.NoError(t, svc.metadata.PutVolume(context.Background(), ebsmetadata.Volume{ + VolumeID: "vol-forcedetach3", TenantID: "acct-tenant", CapacityGiB: 1, + State: "available", AvailabilityZone: "ap-southeast-2a", VolumeType: "gp3", + })) + + attachment, err := svc.ForceDetachVolume(context.Background(), &ec2.DetachVolumeInput{ + VolumeId: aws.String("vol-forcedetach3"), + }, "acct-tenant") + require.NoError(t, err) + + assert.Empty(t, aws.StringValue(attachment.InstanceId)) +} diff --git a/spinifex/handlers/ec2/volume/service_impl.go b/spinifex/handlers/ec2/volume/service_impl.go index b9b4f1562..1e20ef401 100644 --- a/spinifex/handlers/ec2/volume/service_impl.go +++ b/spinifex/handlers/ec2/volume/service_impl.go @@ -1023,6 +1023,44 @@ func (s *VolumeServiceImpl) DetachVolumeOnTerminate(_ context.Context, volumeID, return s.UpdateVolumeState(volumeID, "available", "", "") } +// ForceDetachVolume clears a volume's attachment in the control plane without +// touching the guest, and is answered by any node rather than the one hosting +// the instance. +// +// The ordinary DetachVolume routes to ec2.cmd.{instanceID} for the QMP unplug, +// so a volume attached to an instance whose host stopped answering can never be +// detached and therefore never deleted. This exists for that deadlock and for +// nothing else: it leaves a live guest holding a device the control plane no +// longer believes in, which is only safe once that guest is being destroyed. +func (s *VolumeServiceImpl) ForceDetachVolume(ctx context.Context, input *ec2.DetachVolumeInput, accountID string) (*ec2.VolumeAttachment, error) { + if input == nil || input.VolumeId == nil || *input.VolumeId == "" { + return nil, errors.New(awserrors.ErrorInvalidParameterValue) + } + volumeID := *input.VolumeId + + meta, err := s.metadata.GetVolume(ctx, volumeID) + if err != nil { + return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound) + } + if meta.TenantID != accountID { + return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound) + } + + previous := meta.AttachedInstance + if err := s.UpdateVolumeState(volumeID, "available", "", ""); err != nil { + slog.ErrorContext(ctx, "ForceDetachVolume: failed to clear attachment", "volumeId", volumeID, "err", err) + return nil, errors.New(awserrors.ErrorServerInternal) + } + slog.WarnContext(ctx, "ForceDetachVolume: attachment cleared in the control plane only", + "volumeId", volumeID, "previousInstance", previous, "accountId", accountID) + + return &ec2.VolumeAttachment{ + VolumeId: aws.String(volumeID), + InstanceId: aws.String(previous), + State: aws.String("detached"), + }, nil +} + // DeleteVolume deletes an EBS volume: validates state, asks the EBS provider to // destroy the backing data, then removes the control-plane metadata document. func (s *VolumeServiceImpl) DeleteVolume(ctx context.Context, input *ec2.DeleteVolumeInput, accountID string) (*ec2.DeleteVolumeOutput, error) { diff --git a/spinifex/handlers/ecs/capacity.go b/spinifex/handlers/ecs/capacity.go index 994153073..651a3f06f 100644 --- a/spinifex/handlers/ecs/capacity.go +++ b/spinifex/handlers/ecs/capacity.go @@ -89,7 +89,7 @@ func (s *Service) ProvisionCapacity(ctx context.Context, input *ProvisionCapacit } if s.deps.IAM == nil { - return nil, errors.New("ecs: capacity provisioning requires the IAM service (master key not provisioned)") + return nil, errors.New("ecs: capacity provisioning requires the IAM service, which this worker could not start") } if s.deps.Images == nil { return nil, errors.New("ecs: capacity provisioning requires an image resolver") diff --git a/spinifex/handlers/ecs/instancerole.go b/spinifex/handlers/ecs/instancerole.go index 3ae09b609..4257801cb 100644 --- a/spinifex/handlers/ecs/instancerole.go +++ b/spinifex/handlers/ecs/instancerole.go @@ -2,6 +2,7 @@ package handlers_ecs import ( "fmt" + "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/iam" @@ -25,8 +26,16 @@ const ( // gateway enforces for assumed-role principals, plus ECR read so it can pull // task images via the instance role when a task carries no execution role. ecsInstanceRolePolicyDoc = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"ecs:*","Resource":"*"},{"Effect":"Allow","Action":["ecr:GetAuthorizationToken","ecr:BatchCheckLayerAvailability","ecr:GetDownloadUrlForLayer","ecr:BatchGetImage"],"Resource":"*"}]}` + + // ecsAttachRetries bounds the wait for a just-created policy to become + // visible to the attach, and ecsAttachRetryBase scales the linear backoff. + ecsAttachRetries = 5 + ecsAttachRetryBase = 100 * time.Millisecond ) +// ecsAttachRetrySleep is the test seam for the attach backoff. +var ecsAttachRetrySleep = time.Sleep + // ensureECSInstanceProfile find-or-creates the ecsInstanceRole, its ecs:* policy // attachment, and the matching instance profile, returning the profile ARN. // Every step is idempotent so concurrent ProvisionCapacity calls converge. @@ -88,14 +97,23 @@ func (s *Service) ensureECSRolePolicy(accountID string) error { return fmt.Errorf("create policy %q: %w", ecsInstanceRolePolicyName, err) } - _, err = s.deps.IAM.AttachRolePolicy(accountID, &iam.AttachRolePolicyInput{ - RoleName: aws.String(ecsInstanceRoleName), - PolicyArn: aws.String(policyARN), - }) - if err != nil && err.Error() != awserrors.ErrorIAMEntityAlreadyExists { - return fmt.Errorf("attach policy %q to role %q: %w", policyARN, ecsInstanceRoleName, err) + // The attach checks the policy exists, and the policy buckets are + // replicated, so a read that lands on a replica which has not caught up + // answers NoSuchEntity for the write a microsecond above. Retry that one + // code rather than fail a provision over a write known to have landed. + for attempt := 0; ; attempt++ { + _, err = s.deps.IAM.AttachRolePolicy(accountID, &iam.AttachRolePolicyInput{ + RoleName: aws.String(ecsInstanceRoleName), + PolicyArn: aws.String(policyARN), + }) + if err == nil || err.Error() == awserrors.ErrorIAMEntityAlreadyExists { + return nil + } + if err.Error() != awserrors.ErrorIAMNoSuchEntity || attempt == ecsAttachRetries { + return fmt.Errorf("attach policy %q to role %q: %w", policyARN, ecsInstanceRoleName, err) + } + ecsAttachRetrySleep(time.Duration(attempt+1) * ecsAttachRetryBase) } - return nil } // ensureECSInstanceProfileBinding guarantees the ecsInstanceRole instance diff --git a/spinifex/handlers/ecs/instancerole_retry_test.go b/spinifex/handlers/ecs/instancerole_retry_test.go new file mode 100644 index 000000000..6cc9fc3bb --- /dev/null +++ b/spinifex/handlers/ecs/instancerole_retry_test.go @@ -0,0 +1,90 @@ +// Drives ensureECSRolePolicy directly and swaps the unexported retry sleep, so +// it needs the package's own scope. +// +//test:in-package +package handlers_ecs + +import ( + "errors" + "testing" + "time" + + "github.com/aws/aws-sdk-go/service/iam" + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// flakyAttachIAM answers NoSuchEntity from AttachRolePolicy a set number of +// times before succeeding, standing in for a replica that has not yet caught up +// with the policy written immediately before. +type flakyAttachIAM struct { + stubIAM + + notFoundTimes int + attaches int +} + +func (f *flakyAttachIAM) AttachRolePolicy(_ string, _ *iam.AttachRolePolicyInput) (*iam.AttachRolePolicyOutput, error) { + f.attaches++ + if f.attaches <= f.notFoundTimes { + return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) + } + return &iam.AttachRolePolicyOutput{}, nil +} + +// alwaysDeniedIAM answers a code the retry must not swallow. +type alwaysDeniedIAM struct { + stubIAM + + attaches int +} + +func (a *alwaysDeniedIAM) AttachRolePolicy(_ string, _ *iam.AttachRolePolicyInput) (*iam.AttachRolePolicyOutput, error) { + a.attaches++ + return nil, errors.New(awserrors.ErrorAccessDenied) +} + +func noAttachSleep(t *testing.T) { + t.Helper() + original := ecsAttachRetrySleep + ecsAttachRetrySleep = func(time.Duration) {} + t.Cleanup(func() { ecsAttachRetrySleep = original }) +} + +// The policy is created and then attached. A NoSuchEntity from that attach is a +// stale read of a write that landed, so provisioning must converge rather than +// fail the whole call. +func TestEnsureECSRolePolicyRetriesAStaleNoSuchEntity(t *testing.T) { + noAttachSleep(t) + fake := &flakyAttachIAM{stubIAM: newStubIAM(), notFoundTimes: 3} + svc := &Service{deps: Deps{IAM: fake}} + + require.NoError(t, svc.ensureECSRolePolicy("000000000106")) + assert.Equal(t, 4, fake.attaches, "three stale reads then the one that sees the policy") +} + +// The retry is bounded: a policy that genuinely never appears still fails. +func TestEnsureECSRolePolicyGivesUpOnAPersistentNoSuchEntity(t *testing.T) { + noAttachSleep(t) + fake := &flakyAttachIAM{stubIAM: newStubIAM(), notFoundTimes: 1000} + svc := &Service{deps: Deps{IAM: fake}} + + err := svc.ensureECSRolePolicy("000000000106") + require.Error(t, err) + assert.ErrorContains(t, err, awserrors.ErrorIAMNoSuchEntity) + assert.Equal(t, ecsAttachRetries+1, fake.attaches, "bounded by ecsAttachRetries") +} + +// Only the stale-read code is retried. Anything else is a real answer and must +// come back on the first attempt. +func TestEnsureECSRolePolicyDoesNotRetryOtherErrors(t *testing.T) { + noAttachSleep(t) + fake := &alwaysDeniedIAM{stubIAM: newStubIAM()} + svc := &Service{deps: Deps{IAM: fake}} + + err := svc.ensureECSRolePolicy("000000000106") + require.Error(t, err) + assert.ErrorContains(t, err, awserrors.ErrorAccessDenied) + assert.Equal(t, 1, fake.attaches, "a denial is not a visibility race") +} diff --git a/spinifex/handlers/iam/account_lifecycle_test.go b/spinifex/handlers/iam/account_lifecycle_test.go new file mode 100644 index 000000000..06fd50e1e --- /dev/null +++ b/spinifex/handlers/iam/account_lifecycle_test.go @@ -0,0 +1,156 @@ +package handlers_iam_test + +import ( + "testing" + + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newLifecycleService(t *testing.T) handlers_iam.IAMService { + t.Helper() + _, nc, _ := testutil.StartTestJetStream(t) + + masterKey, err := handlers_iam.GenerateMasterKey() + require.NoError(t, err) + + svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey, 1) + require.NoError(t, err) + return svc +} + +func TestSetAccountStatusMovesBetweenStates(t *testing.T) { + svc := newLifecycleService(t) + account, err := svc.CreateAccount("tenant@example.com") + require.NoError(t, err) + + suspended, err := svc.SetAccountStatus(account.AccountID, handlers_iam.AccountStatusSuspended) + require.NoError(t, err) + assert.Equal(t, handlers_iam.AccountStatusSuspended, suspended.Status) + + // An operator hold is reversible, unlike a teardown. + restored, err := svc.SetAccountStatus(account.AccountID, handlers_iam.AccountStatusActive) + require.NoError(t, err) + assert.Equal(t, handlers_iam.AccountStatusActive, restored.Status) + + stored, err := svc.GetAccount(account.AccountID) + require.NoError(t, err) + assert.Equal(t, handlers_iam.AccountStatusActive, stored.Status) +} + +// There is no path back from TERMINATING. A half-torn-down account that +// resumes service is worse than one that stays down, and "undo" here would +// mean restoring deleted block storage. +func TestTerminatingIsAOneWayDoor(t *testing.T) { + svc := newLifecycleService(t) + account, err := svc.CreateAccount("tenant@example.com") + require.NoError(t, err) + + _, err = svc.SetAccountStatus(account.AccountID, handlers_iam.AccountStatusTerminating) + require.NoError(t, err) + + for _, status := range []string{handlers_iam.AccountStatusActive, handlers_iam.AccountStatusSuspended} { + _, err = svc.SetAccountStatus(account.AccountID, status) + require.Error(t, err) + } + + stored, err := svc.GetAccount(account.AccountID) + require.NoError(t, err) + assert.Equal(t, handlers_iam.AccountStatusTerminating, stored.Status) +} + +func TestSetAccountStatusRejectsAnUnknownStatus(t *testing.T) { + svc := newLifecycleService(t) + account, err := svc.CreateAccount("tenant@example.com") + require.NoError(t, err) + + _, err = svc.SetAccountStatus(account.AccountID, "PAUSED") + + assert.Error(t, err) +} + +// Deleting the record while the account is still ACTIVE would leave every +// resource it owns unattributable, so the status gate is the invariant. +func TestDeleteAccountRequiresTerminating(t *testing.T) { + svc := newLifecycleService(t) + // Account ids are sequential and the first is the super admin, which is + // protected — the tenant under test has to be the second. + _, err := svc.CreateAccount("super-admin@example.com") + require.NoError(t, err) + account, err := svc.CreateAccount("tenant@example.com") + require.NoError(t, err) + + require.Error(t, svc.DeleteAccount(account.AccountID)) + + _, err = svc.SetAccountStatus(account.AccountID, handlers_iam.AccountStatusTerminating) + require.NoError(t, err) + require.NoError(t, svc.DeleteAccount(account.AccountID)) + + _, err = svc.GetAccount(account.AccountID) + assert.Error(t, err) +} + +// No credential grants deleting these two, so the refusal lives in the library +// rather than only in the CLI that usually calls it. +func TestProtectedAccountsCanNeverBeDeleted(t *testing.T) { + svc := newLifecycleService(t) + + for accountID := range handlers_iam.UndeletableAccountIDs { + t.Run(accountID, func(t *testing.T) { + // Even a status that would otherwise permit deletion must not. + _, _ = svc.SetAccountStatus(accountID, handlers_iam.AccountStatusTerminating) + + err := svc.DeleteAccount(accountID) + + require.Error(t, err) + assert.ErrorIs(t, err, handlers_iam.ErrAccountUndeletable) + }) + } +} + +// A deleted account's name must be free again: a customer who leaves and comes +// back must not be told their own address is taken. +func TestReleaseDeletedFreesTheName(t *testing.T) { + index, _ := newAccountNameIndex(t) + ctx := t.Context() + + require.NoError(t, index.Reserve(ctx, "tenant@example.com", "token-1")) + require.NoError(t, index.Commit(ctx, "tenant@example.com", "000000000042", "token-1")) + + require.NoError(t, index.ReleaseDeleted(ctx, "tenant@example.com", "000000000042")) + + _, found, err := index.Lookup(ctx, "tenant@example.com") + require.NoError(t, err) + assert.False(t, found) + + // The same address can be claimed again, by a different account. + require.NoError(t, index.Reserve(ctx, "tenant@example.com", "token-2")) +} + +// Releasing a name that belongs to a live account would unindex it and let a +// second account take the same address. +func TestReleaseDeletedRefusesAnotherAccountsName(t *testing.T) { + index, _ := newAccountNameIndex(t) + ctx := t.Context() + + require.NoError(t, index.Reserve(ctx, "tenant@example.com", "token-1")) + require.NoError(t, index.Commit(ctx, "tenant@example.com", "000000000099", "token-1")) + + require.Error(t, index.ReleaseDeleted(ctx, "tenant@example.com", "000000000042")) + + accountID, found, err := index.Lookup(ctx, "tenant@example.com") + require.NoError(t, err) + require.True(t, found) + assert.Equal(t, "000000000099", accountID) +} + +// Teardown re-runs after a crash, so a name that is already released is a +// success rather than a failure that blocks the account record's deletion. +func TestReleaseDeletedIsIdempotent(t *testing.T) { + index, _ := newAccountNameIndex(t) + ctx := t.Context() + + assert.NoError(t, index.ReleaseDeleted(ctx, "never-reserved@example.com", "000000000042")) +} diff --git a/spinifex/handlers/iam/account_names.go b/spinifex/handlers/iam/account_names.go index 8b5f811d6..055367cf6 100644 --- a/spinifex/handlers/iam/account_names.go +++ b/spinifex/handlers/iam/account_names.go @@ -127,6 +127,29 @@ func (x *AccountNameIndex) Release(ctx context.Context, name, clientToken string return nil } +// ReleaseDeleted removes a committed entry once its account has been torn +// down, so the same email can sign up again. +// +// It is deliberately separate from Release, which must never unindex a live +// account: this one removes the entry only when it names exactly the account +// given, so a stale name cannot take another tenant's reservation with it. +func (x *AccountNameIndex) ReleaseDeleted(ctx context.Context, name, accountID string) error { + existing, err := x.get(ctx, name) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil + } + if err != nil { + return err + } + if existing.AccountID != accountID { + return fmt.Errorf("name %q is reserved by account %q, not %q", name, existing.AccountID, accountID) + } + if err := x.kv.Delete(ctx, nameKey(name)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + return fmt.Errorf("release account name: %w", err) + } + return nil +} + // Lookup returns the account ID indexed against name. found is false for both // an absent entry and an uncommitted reservation, since neither names an // account that exists. diff --git a/spinifex/handlers/iam/service.go b/spinifex/handlers/iam/service.go index cc506b5cb..ef2cdc5e9 100644 --- a/spinifex/handlers/iam/service.go +++ b/spinifex/handlers/iam/service.go @@ -136,6 +136,8 @@ type IAMService interface { CreateAccount(name string) (*Account, error) GetAccount(accountID string) (*Account, error) ListAccounts() ([]*Account, error) + SetAccountStatus(accountID, status string) (*Account, error) + DeleteAccount(accountID string) error // GetAccountSummary returns account-wide IAM usage counts plus AWS-parity // quota values as a SummaryMap. Read-only and account-scoped. diff --git a/spinifex/handlers/iam/service_impl.go b/spinifex/handlers/iam/service_impl.go index 4477650d4..537f12d76 100644 --- a/spinifex/handlers/iam/service_impl.go +++ b/spinifex/handlers/iam/service_impl.go @@ -1071,6 +1071,89 @@ func (s *IAMServiceImpl) ListAccounts() ([]*Account, error) { return accounts, nil } +// UndeletableAccountIDs are the two accounts no teardown may ever remove: the +// internal system account and the super admin. Held here rather than only in +// the caller so a new caller cannot bypass the rule by forgetting it. +var UndeletableAccountIDs = map[string]string{ + "000000000000": "system", + "000000000001": "super admin", +} + +// ErrAccountUndeletable reports a refusal to delete a protected account. It is +// deliberately distinct from a permissions error: no credential grants it. +var ErrAccountUndeletable = errors.New("account is protected and cannot be deleted") + +// SetAccountStatus writes a new status and returns the updated account. It +// refuses to move an account out of TERMINATING: teardown has already begun +// destroying data, so restoring service would hand back a hollowed-out account. +func (s *IAMServiceImpl) SetAccountStatus(accountID, status string) (*Account, error) { + switch status { + case AccountStatusActive, AccountStatusSuspended, AccountStatusTerminating: + default: + return nil, errors.New(awserrors.ErrorIAMInvalidInput) + } + + account, err := s.GetAccount(accountID) + if err != nil { + return nil, err + } + if account.Status == AccountStatusTerminating && status != AccountStatusTerminating { + return nil, fmt.Errorf("account %s is terminating: %w", accountID, ErrAccountUndeletable) + } + if account.Status == status { + return account, nil + } + + account.Status = status + data, err := json.Marshal(account) + if err != nil { + return nil, fmt.Errorf("marshal account: %w", err) + } + if _, err := s.accountsBucket.Put(context.Background(), accountID, data); err != nil { + return nil, fmt.Errorf("store account status: %w", err) + } + + slog.Info("Account status changed", "accountID", accountID, "status", status) + return account, nil +} + +// DeleteAccount removes the account record itself. It is the last step of a +// teardown and does nothing about the account's resources — a caller that has +// not emptied the account first strands every resource it owned, because the +// ownership record is what attributes them. +func (s *IAMServiceImpl) DeleteAccount(accountID string) error { + if reason, protected := UndeletableAccountIDs[accountID]; protected { + return fmt.Errorf("%s (%s): %w", accountID, reason, ErrAccountUndeletable) + } + + account, err := s.GetAccount(accountID) + if err != nil { + return err + } + if account.Status != AccountStatusTerminating { + return fmt.Errorf("account %s is %s, not %s: %w", + accountID, account.Status, AccountStatusTerminating, ErrAccountUndeletable) + } + + if err := s.accountsBucket.Delete(context.Background(), accountID); err != nil { + return fmt.Errorf("delete account: %w", err) + } + slog.Info("Account deleted", "accountID", accountID, "name", account.AccountName) + + if s.natsConn != nil { + evt, err := json.Marshal(struct { + AccountID string `json:"account_id"` + AccountName string `json:"account_name"` + }{AccountID: accountID, AccountName: account.AccountName}) + if err != nil { + slog.Error("Failed to marshal account deletion event", "accountID", accountID, "error", err) + } else if err := s.natsConn.Publish("iam.account.deleted", evt); err != nil { + slog.Error("Failed to publish account deletion event", "accountID", accountID, "error", err) + } + } + return nil +} + // --------------------------------------------------------------------------- // Policy CRUD // --------------------------------------------------------------------------- diff --git a/spinifex/handlers/iam/types.go b/spinifex/handlers/iam/types.go index 14e445fcb..a81ba5aae 100644 --- a/spinifex/handlers/iam/types.go +++ b/spinifex/handlers/iam/types.go @@ -13,6 +13,11 @@ const ( AccountStatusActive = "ACTIVE" AccountStatusSuspended = "SUSPENDED" + // AccountStatusTerminating marks an account whose teardown has started. + // There is no path back from it: a half-torn-down account that resumes + // service is worse than one that stays down. + AccountStatusTerminating = "TERMINATING" + PolicyEffectAllow = iampolicy.EffectAllow PolicyEffectDeny = iampolicy.EffectDeny ) @@ -163,6 +168,6 @@ type AdminBootstrapData struct { type Account struct { AccountID string `json:"account_id"` // 12-digit zero-padded, sequential AccountName string `json:"account_name"` // Friendly name - Status string `json:"status"` // "ACTIVE" or "SUSPENDED" + Status string `json:"status"` // ACTIVE, SUSPENDED or TERMINATING CreatedAt string `json:"created_at"` // RFC3339 } diff --git a/spinifex/loadgen/clients.go b/spinifex/loadgen/clients.go new file mode 100644 index 000000000..a74e22e4a --- /dev/null +++ b/spinifex/loadgen/clients.go @@ -0,0 +1,126 @@ +package loadgen + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net/http" + "os" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/aws/aws-sdk-go/service/sts" + "github.com/aws/aws-sdk-go/service/sts/stsiface" +) + +// Clients holds one tenant's SDK clients. They are built once and reused for +// the whole run: rebuilding a session per request would measure session +// construction, and holding a connection is what a real SDK client does. +type Clients struct { + EC2 ec2iface.EC2API + STS stsiface.STSAPI +} + +// Dial is what to point the clients at. +type Dial struct { + Endpoint string + Region string + // CABundle is needed only when the endpoint serves the cluster's own + // certificate. Through a publicly trusted proxy it is empty and system + // trust applies. + CABundle string + // MaxIdleConnsPerHost has to be at least the stage concurrency, or the + // transport closes and reopens connections under load and the run measures + // TLS handshakes. + MaxIdleConnsPerHost int + Timeout time.Duration +} + +// NewClients builds one tenant's clients from a shared-config profile. +func NewClients(dial Dial, profile string) (*Clients, error) { + if profile == "" { + return nil, errors.New("loadgen: profile is required") + } + + httpClient, err := httpClientFor(dial) + if err != nil { + return nil, err + } + + sess, err := session.NewSessionWithOptions(session.Options{ + Profile: profile, + SharedConfigState: session.SharedConfigEnable, + Config: aws.Config{ + Endpoint: aws.String(dial.Endpoint), + Region: aws.String(dial.Region), + S3ForcePathStyle: aws.Bool(true), + HTTPClient: httpClient, + MaxRetries: aws.Int(0), + }, + }) + if err != nil { + return nil, fmt.Errorf("loadgen: session for profile %s: %w", profile, err) + } + + return &Clients{EC2: ec2.New(sess), STS: sts.New(sess)}, nil +} + +// httpClientFor builds the transport every client shares. Retries are off in +// the session above on purpose: a retried request is a slow request, and +// hiding it inside one sample turns a failing cluster into a fast one. +func httpClientFor(dial Dial) (*http.Client, error) { + tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12} + if dial.CABundle != "" { + pem, err := os.ReadFile(dial.CABundle) + if err != nil { + return nil, fmt.Errorf("loadgen: read CA bundle: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("loadgen: no certificates in %s", dial.CABundle) + } + tlsConfig.RootCAs = pool + } + + idle := dial.MaxIdleConnsPerHost + if idle < 1 { + idle = 64 + } + timeout := dial.Timeout + if timeout <= 0 { + timeout = 30 * time.Second + } + + return &http.Client{ + Timeout: timeout, + Transport: &http.Transport{ + TLSClientConfig: tlsConfig, + MaxIdleConns: idle * 2, + MaxIdleConnsPerHost: idle, + IdleConnTimeout: 90 * time.Second, + }, + }, nil +} + +// ErrorCode classifies a failure for the error table. An AWS error keeps its +// service code so a 500 and a throttle never merge into one number; anything +// else is reported by its Go type, which is enough to tell a timeout from a +// connection refusal. +func ErrorCode(err error) string { + if err == nil { + return "" + } + var awsErr awserr.Error + if errors.As(err, &awsErr) { + return awsErr.Code() + } + if errors.Is(err, ErrShed) { + return shedCode + } + return fmt.Sprintf("%T", err) +} diff --git a/spinifex/loadgen/loadgen.go b/spinifex/loadgen/loadgen.go new file mode 100644 index 000000000..0a878bd16 --- /dev/null +++ b/spinifex/loadgen/loadgen.go @@ -0,0 +1,512 @@ +// Package loadgen drives a measured request load at the control plane and +// reports per-operation latency distributions. +// +// It exists because bash cannot measure this: a shell driving the AWS CLI pays +// over a second of interpreter start-up per call, which is larger than the +// latency being measured, so a percentile built from those samples is a +// percentile of the CLI. The engine here holds long-lived SDK clients and times +// only the request. +package loadgen + +import ( + "context" + "errors" + "fmt" + "math" + "sort" + "sync" + "time" +) + +// Mode is how a stage paces its requests, and the two answer different +// questions. Closed loop asks "with N callers in flight, how fast is it?" and +// slows down when the server does. Open loop asks "at R requests a second, does +// it keep up?" and does not, which is the only way to see a queue form. +type Mode string + +const ( + ModeClosed Mode = "closed" + ModeOpen Mode = "open" +) + +// Kind separates reads from writes because their SLOs differ by an order of +// magnitude: a Describe that takes two seconds is broken, a RunInstances that +// takes two seconds is normal. +type Kind string + +const ( + KindRead Kind = "read" + KindWrite Kind = "write" +) + +// ErrShed marks a request an open-loop stage could not issue because too many +// were already in flight. It is recorded rather than dropped: a stage that +// cannot sustain its rate has found the answer it was looking for. +var ErrShed = errors.New("shed: too many requests in flight") + +// shedCode is the error code shed requests are counted under. It is namespaced +// so it can never collide with a code the service returns. +const shedCode = "loadgen.Shed" + +// Op is one control-plane call under test. Call does the request and returns +// the service's error unchanged, so the code can be classified. +type Op struct { + Name string + Kind Kind + Call func(ctx context.Context, target *Target) error +} + +// Target is one tenant's wiring. A run spreads its workers across every target +// it is given, so the load looks like several customers rather than one. +type Target struct { + Account string + Profile string + Clients *Clients + // VPCID is resolved once before the run for operations that need a + // resource of the tenant's own to act on. Resolving it per request would + // put a second call inside every sample. + VPCID string + // VolumeID lets a run ask for one volume by id rather than for the whole + // listing. The difference between the two is the cost of the listing, + // which is otherwise indistinguishable from the cost of the request. + VolumeID string +} + +// Stage is one step of a ramp. Exactly one of Concurrency or RPS applies, +// depending on Mode. +type Stage struct { + Mode Mode `json:"mode"` + Concurrency int `json:"concurrency,omitempty"` + RPS float64 `json:"rps,omitempty"` + Duration time.Duration `json:"-"` + DurationSec float64 `json:"durationSeconds"` + // MaxInFlight caps how many open-loop requests may be outstanding at once. + // Zero means four seconds' worth of the stage's rate, which is enough slack + // for a slow cluster and not enough for a stalled one to exhaust memory. + MaxInFlight int `json:"maxInFlight,omitempty"` +} + +// SLO is the pass/fail line. A stage breaches when an operation's p99 exceeds +// the limit for its kind, or when any request fails for a reason other than a +// deliberately tripped quota. +type SLO struct { + ReadP99 time.Duration `json:"-"` + WriteP99 time.Duration `json:"-"` + // ExpectedErrorCodes are not failures. A quota rejection means the cluster + // is working; counting it as an error would fail a correct run. + ExpectedErrorCodes []string `json:"expectedErrorCodes,omitempty"` + ReadP99MS float64 `json:"readP99Ms"` + WriteP99MS float64 `json:"writeP99Ms"` +} + +// OpStats is one operation's result within one stage. +type OpStats struct { + Op string `json:"op"` + Kind Kind `json:"kind"` + Count int `json:"count"` + Errors int `json:"errors"` + ErrorCodes map[string]int `json:"errorCodes,omitempty"` + MeanMS float64 `json:"meanMs"` + P50MS float64 `json:"p50Ms"` + P90MS float64 `json:"p90Ms"` + P99MS float64 `json:"p99Ms"` + MaxMS float64 `json:"maxMs"` + // AchievedRPS is what the stage actually managed, which is the number that + // matters in open loop: asking for 40 and achieving 12 is the finding. + AchievedRPS float64 `json:"achievedRps"` +} + +// StageResult is one stage's outcome. +type StageResult struct { + Stage Stage `json:"stage"` + Ops map[string]OpStats `json:"ops"` + Started time.Time `json:"started"` + Ended time.Time `json:"ended"` + Breached []string `json:"breached,omitempty"` +} + +// Report is the whole run. It is written as JSON for the harness to fold into +// its run directory, and summarised for a human. +type Report struct { + Endpoint string `json:"endpoint"` + Region string `json:"region"` + Targets []string `json:"targets"` + SLO SLO `json:"slo"` + Started time.Time `json:"started"` + Ended time.Time `json:"ended"` + Stages []StageResult `json:"stages"` + // FirstBreach names the first stage of each mode whose p99 crossed the SLO. + // This is the capacity number the run exists to produce. + FirstBreach map[Mode]*Breach `json:"firstBreach,omitempty"` +} + +// Breach records where a mode stopped meeting its SLO. +type Breach struct { + StageIndex int `json:"stageIndex"` + Concurrency int `json:"concurrency,omitempty"` + RPS float64 `json:"rps,omitempty"` + Ops []string `json:"ops"` +} + +// sample is one timed request. +type sample struct { + op string + latency time.Duration + errCode string + failed bool +} + +// Run executes every stage in order and returns the report. Stages run back to +// back with no cool-down: a queue left by one stage is a real condition the +// next one should see, and hiding it would flatter the result. +func Run(ctx context.Context, targets []*Target, ops []Op, stages []Stage, slo SLO) (*Report, error) { + if len(targets) == 0 { + return nil, errors.New("loadgen: no targets") + } + if len(ops) == 0 { + return nil, errors.New("loadgen: no operations") + } + + report := &Report{ + SLO: slo, Started: time.Now().UTC(), FirstBreach: map[Mode]*Breach{}, + } + for _, target := range targets { + report.Targets = append(report.Targets, target.Account) + } + + for index, stage := range stages { + // An interrupted run reports the stages that finished rather than + // nothing: someone who stops a ramp still wants the rungs it climbed. + select { + case <-ctx.Done(): + report.Ended = time.Now().UTC() + return report, nil + default: + } + + result := runStage(ctx, targets, ops, stage) + result.Breached = breachedOps(result, slo) + report.Stages = append(report.Stages, result) + + if len(result.Breached) > 0 && report.FirstBreach[stage.Mode] == nil { + report.FirstBreach[stage.Mode] = &Breach{ + StageIndex: index, Concurrency: stage.Concurrency, + RPS: stage.RPS, Ops: result.Breached, + } + } + } + + report.Ended = time.Now().UTC() + return report, nil +} + +func runStage(ctx context.Context, targets []*Target, ops []Op, stage Stage) StageResult { + stage.DurationSec = stage.Duration.Seconds() + started := time.Now() + var samples []sample + switch stage.Mode { + case ModeOpen: + samples = runOpen(ctx, targets, ops, stage) + case ModeClosed: + fallthrough + default: + stage.Mode = ModeClosed + samples = runClosed(ctx, targets, ops, stage) + } + ended := time.Now() + + return StageResult{ + Stage: stage, + Ops: summarise(samples, ops, ended.Sub(started)), + Started: started.UTC(), + Ended: ended.UTC(), + } +} + +// runClosed holds a fixed number of callers in flight. Each worker walks the +// operation list so every operation gets the same share of the concurrency, +// rather than whichever one happens to be fastest getting the most turns. +func runClosed(ctx context.Context, targets []*Target, ops []Op, stage Stage) []sample { + concurrency := max(stage.Concurrency, 1) + stageCtx, cancel := context.WithTimeout(ctx, stage.Duration) + defer cancel() + + collected := make([][]sample, concurrency) + var wg sync.WaitGroup + for worker := range concurrency { + wg.Add(1) + go func(worker int) { + defer wg.Done() + target := targets[worker%len(targets)] + for turn := 0; stageCtx.Err() == nil; turn++ { + op := ops[(worker+turn)%len(ops)] + if result, ok := timeCall(stageCtx, target, op, time.Now()); ok { + collected[worker] = append(collected[worker], result) + } + } + }(worker) + } + wg.Wait() + + var samples []sample + for _, worker := range collected { + samples = append(samples, worker...) + } + return samples +} + +// runOpen issues requests on a fixed schedule regardless of how the server is +// coping, and times each one from when it was *due* rather than when it +// started. Timing from the start instead would hide the queue: a request held +// up for a second before it was issued would report as fast. +func runOpen(ctx context.Context, targets []*Target, ops []Op, stage Stage) []sample { + rps := stage.RPS + if rps <= 0 { + rps = 1 + } + interval := time.Duration(float64(time.Second) / rps) + stageCtx, cancel := context.WithTimeout(ctx, stage.Duration) + defer cancel() + + // In-flight requests are capped so a stalled cluster cannot make the + // generator allocate without bound. Requests over the cap are recorded as + // shed, which is the honest way to say the rate was not sustained. + maxInFlight := stage.MaxInFlight + if maxInFlight < 1 { + maxInFlight = int(math.Max(rps*maxInFlightSeconds, minInFlight)) + } + inFlight := make(chan struct{}, maxInFlight) + + var mu sync.Mutex + var samples []sample + var wg sync.WaitGroup + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for tick := 0; ; tick++ { + select { + case <-stageCtx.Done(): + wg.Wait() + return samples + case due := <-ticker.C: + target := targets[tick%len(targets)] + op := ops[tick%len(ops)] + + select { + case inFlight <- struct{}{}: + default: + mu.Lock() + samples = append(samples, sample{op: op.Name, errCode: shedCode, failed: true}) + mu.Unlock() + continue + } + + wg.Go(func() { + defer func() { <-inFlight }() + result, ok := timeCall(stageCtx, target, op, due) + if !ok { + return + } + mu.Lock() + samples = append(samples, result) + mu.Unlock() + }) + } + } +} + +const ( + // How many seconds of requests may be in flight at once in open loop. + maxInFlightSeconds = 4.0 + minInFlight = 8.0 +) + +// timeCall runs one operation and measures from due, which is the time the +// request was scheduled for. In closed loop that is the moment the call +// starts; in open loop it is the tick, so queueing counts against the result. +// +// It reports false for a request the stage's own deadline cut short. Those are +// not the cluster's failures — every worker has one in flight when time is up, +// so counting them would put a fixed error per worker into every stage and a +// truncated latency into every distribution. +func timeCall(ctx context.Context, target *Target, op Op, due time.Time) (sample, bool) { + err := op.Call(ctx, target) + if err != nil && ctx.Err() != nil && cancelledByStage(err) { + return sample{}, false + } + + result := sample{op: op.Name, latency: time.Since(due)} + if err != nil { + result.failed = true + result.errCode = ErrorCode(err) + } + return result, true +} + +// cancelledByStage reports whether an error is the shape a cancelled request +// takes. The caller checks the stage context separately, so a per-request +// timeout against a stalled cluster is still counted as the failure it is. +func cancelledByStage(err error) bool { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return true + } + return ErrorCode(err) == cancelledCode +} + +// The code aws-sdk-go gives a request cancelled through its context. +const cancelledCode = "RequestCanceled" + +func summarise(samples []sample, ops []Op, elapsed time.Duration) map[string]OpStats { + kinds := make(map[string]Kind, len(ops)) + for _, op := range ops { + kinds[op.Name] = op.Kind + } + + latencies := map[string][]float64{} + stats := map[string]OpStats{} + for _, s := range samples { + entry, ok := stats[s.op] + if !ok { + entry = OpStats{Op: s.op, Kind: kinds[s.op], ErrorCodes: map[string]int{}} + } + entry.Count++ + if s.failed { + entry.Errors++ + entry.ErrorCodes[s.errCode]++ + } + stats[s.op] = entry + // A shed request has no latency to report: it was never issued, and + // folding a zero into the distribution would improve the percentiles + // of a stage that failed to keep up. + if s.errCode != shedCode { + latencies[s.op] = append(latencies[s.op], float64(s.latency)/float64(time.Millisecond)) + } + } + + for name, entry := range stats { + values := latencies[name] + sort.Float64s(values) + entry.MeanMS = round2(mean(values)) + entry.P50MS = round2(percentile(values, 0.50)) + entry.P90MS = round2(percentile(values, 0.90)) + entry.P99MS = round2(percentile(values, 0.99)) + if len(values) > 0 { + entry.MaxMS = round2(values[len(values)-1]) + } + if elapsed > 0 { + entry.AchievedRPS = round2(float64(entry.Count) / elapsed.Seconds()) + } + if len(entry.ErrorCodes) == 0 { + entry.ErrorCodes = nil + } + stats[name] = entry + } + return stats +} + +// percentile is nearest-rank over sorted values: the smallest sample at or +// above the quantile. No interpolation, so every reported number is a request +// that actually happened. +func percentile(sorted []float64, quantile float64) float64 { + if len(sorted) == 0 { + return 0 + } + rank := min(max(int(math.Ceil(quantile*float64(len(sorted)))), 1), len(sorted)) + return sorted[rank-1] +} + +func mean(values []float64) float64 { + if len(values) == 0 { + return 0 + } + var total float64 + for _, v := range values { + total += v + } + return total / float64(len(values)) +} + +func round2(value float64) float64 { + return math.Round(value*100) / 100 +} + +// breachedOps names every operation in a stage that missed the SLO, either on +// p99 or by failing at all. An empty result means the stage passed. +func breachedOps(result StageResult, slo SLO) []string { + expected := make(map[string]bool, len(slo.ExpectedErrorCodes)) + for _, code := range slo.ExpectedErrorCodes { + expected[code] = true + } + + var breached []string + for name, stats := range result.Ops { + limit := slo.ReadP99 + if stats.Kind == KindWrite { + limit = slo.WriteP99 + } + if limit > 0 && stats.P99MS > float64(limit)/float64(time.Millisecond) { + breached = append(breached, name) + continue + } + for code, count := range stats.ErrorCodes { + if count > 0 && !expected[code] { + breached = append(breached, name) + break + } + } + } + sort.Strings(breached) + return breached +} + +// Summary renders the report the way an operator reads it: one line per +// operation per stage, then the capacity number. +func Summary(report *Report) string { + out := fmt.Sprintf("endpoint %s region %s tenants %d\n", + report.Endpoint, report.Region, len(report.Targets)) + for index, stage := range report.Stages { + out += fmt.Sprintf("\n[%d] %s ", index, stage.Stage.Mode) + if stage.Stage.Mode == ModeOpen { + out += fmt.Sprintf("%.0f rps", stage.Stage.RPS) + } else { + out += fmt.Sprintf("concurrency %d", stage.Stage.Concurrency) + } + out += fmt.Sprintf(" for %.0fs\n", stage.Stage.DurationSec) + + for _, name := range sortedOps(stage.Ops) { + s := stage.Ops[name] + out += fmt.Sprintf(" %-28s n=%-6d err=%-4d rps=%-7.2f p50=%-8.1f p90=%-8.1f p99=%-8.1f max=%.1f\n", + name, s.Count, s.Errors, s.AchievedRPS, s.P50MS, s.P90MS, s.P99MS, s.MaxMS) + } + if len(stage.Breached) > 0 { + out += fmt.Sprintf(" breached: %v\n", stage.Breached) + } + } + + out += "\n" + for _, mode := range []Mode{ModeClosed, ModeOpen} { + breach, ok := report.FirstBreach[mode] + if !ok || breach == nil { + out += fmt.Sprintf("%s: no breach at any stage run\n", mode) + continue + } + if mode == ModeOpen { + out += fmt.Sprintf("open: first breach at %.0f rps (stage %d) on %v\n", + breach.RPS, breach.StageIndex, breach.Ops) + continue + } + out += fmt.Sprintf("closed: first breach at concurrency %d (stage %d) on %v\n", + breach.Concurrency, breach.StageIndex, breach.Ops) + } + return out +} + +func sortedOps(ops map[string]OpStats) []string { + names := make([]string, 0, len(ops)) + for name := range ops { + names = append(names, name) + } + sort.Strings(names) + return names +} diff --git a/spinifex/loadgen/loadgen_test.go b/spinifex/loadgen/loadgen_test.go new file mode 100644 index 000000000..5210cd684 --- /dev/null +++ b/spinifex/loadgen/loadgen_test.go @@ -0,0 +1,358 @@ +package loadgen_test + +import ( + "context" + "errors" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/mulgadc/spinifex/spinifex/loadgen" +) + +// fixedOp answers after a fixed delay, so a stage's percentiles are known +// before it runs. +func fixedOp(name string, kind loadgen.Kind, latency time.Duration, err error) loadgen.Op { + return loadgen.Op{Name: name, Kind: kind, Call: func(ctx context.Context, _ *loadgen.Target) error { + select { + case <-time.After(latency): + case <-ctx.Done(): + } + return err + }} +} + +func targets(n int) []*loadgen.Target { + out := make([]*loadgen.Target, 0, n) + for i := range n { + out = append(out, &loadgen.Target{Account: string(rune('a' + i))}) + } + return out +} + +func TestClosedLoopHoldsTheRequestedConcurrency(t *testing.T) { + t.Parallel() + + var inFlight, peak int64 + op := loadgen.Op{Name: "Probe", Kind: loadgen.KindRead, + Call: func(ctx context.Context, _ *loadgen.Target) error { + current := atomic.AddInt64(&inFlight, 1) + defer atomic.AddInt64(&inFlight, -1) + for { + observed := atomic.LoadInt64(&peak) + if current <= observed || atomic.CompareAndSwapInt64(&peak, observed, current) { + break + } + } + select { + case <-time.After(5 * time.Millisecond): + case <-ctx.Done(): + } + return nil + }} + + stages := []loadgen.Stage{{Mode: loadgen.ModeClosed, Concurrency: 4, Duration: 150 * time.Millisecond}} + report, err := loadgen.Run(context.Background(), targets(2), []loadgen.Op{op}, stages, loadgen.SLO{}) + if err != nil { + t.Fatalf("Run: %v", err) + } + + if got := atomic.LoadInt64(&peak); got != 4 { + t.Errorf("peak concurrency = %d, want exactly the 4 requested", got) + } + if count := report.Stages[0].Ops["Probe"].Count; count < 4 { + t.Errorf("count = %d, want at least one request per worker", count) + } +} + +func TestOpenLoopIssuesRoughlyTheRequestedRate(t *testing.T) { + t.Parallel() + + stages := []loadgen.Stage{{Mode: loadgen.ModeOpen, RPS: 100, Duration: 300 * time.Millisecond}} + ops := []loadgen.Op{fixedOp("Probe", loadgen.KindRead, time.Millisecond, nil)} + + report, err := loadgen.Run(context.Background(), targets(1), ops, stages, loadgen.SLO{}) + if err != nil { + t.Fatalf("Run: %v", err) + } + + // 100/s for 300ms is 30 requests. Timer granularity makes the exact count + // unstable, so this asserts the order of magnitude, which is what a wrong + // pacing implementation would get wrong. + count := report.Stages[0].Ops["Probe"].Count + if count < 15 || count > 45 { + t.Errorf("count = %d, want roughly 30 for 100 rps over 300ms", count) + } +} + +// A stalled server must not let the generator allocate without bound: requests +// it cannot issue are recorded as shed, because a rate that was not sustained +// is the finding, not something to hide. +func TestOpenLoopShedsRatherThanQueueingWithoutBound(t *testing.T) { + t.Parallel() + + release := make(chan struct{}) + op := loadgen.Op{Name: "Stalled", Kind: loadgen.KindRead, + Call: func(ctx context.Context, _ *loadgen.Target) error { + select { + case <-release: + case <-ctx.Done(): + } + return nil + }} + + stages := []loadgen.Stage{{ + Mode: loadgen.ModeOpen, RPS: 200, Duration: 300 * time.Millisecond, MaxInFlight: 2, + }} + report, err := loadgen.Run(context.Background(), targets(1), []loadgen.Op{op}, stages, loadgen.SLO{}) + close(release) + if err != nil { + t.Fatalf("Run: %v", err) + } + + stats := report.Stages[0].Ops["Stalled"] + if stats.ErrorCodes["loadgen.Shed"] == 0 { + t.Errorf("no shed requests recorded against a server that never answered: %+v", stats.ErrorCodes) + } +} + +// A shed request was never issued, so folding a zero latency into the +// distribution would make the stage that failed to keep up look like the +// fastest one in the run. +func TestShedRequestsAreExcludedFromLatency(t *testing.T) { + t.Parallel() + + release := make(chan struct{}) + op := loadgen.Op{Name: "Slow", Kind: loadgen.KindRead, + Call: func(ctx context.Context, _ *loadgen.Target) error { + select { + case <-release: + case <-ctx.Done(): + } + return nil + }} + + stages := []loadgen.Stage{{Mode: loadgen.ModeOpen, RPS: 200, Duration: 250 * time.Millisecond}} + report, err := loadgen.Run(context.Background(), targets(1), []loadgen.Op{op}, stages, loadgen.SLO{}) + close(release) + if err != nil { + t.Fatalf("Run: %v", err) + } + + stats := report.Stages[0].Ops["Slow"] + if stats.ErrorCodes["loadgen.Shed"] == 0 { + t.Skip("no requests were shed on this machine; nothing to assert") + } + if stats.P50MS <= 0 { + t.Errorf("p50 = %v, want the issued requests' real latency, not a zero from a shed one", stats.P50MS) + } +} + +// Every worker has a request in flight when a stage's time is up. Counting +// those would put a fixed error per worker into every stage and a truncated +// latency into every distribution, which reads as a cluster fault and is ours. +func TestRequestsCutShortByTheStageDeadlineAreNotFailures(t *testing.T) { + t.Parallel() + + op := loadgen.Op{Name: "Blocked", Kind: loadgen.KindRead, + Call: func(ctx context.Context, _ *loadgen.Target) error { + <-ctx.Done() + return ctx.Err() + }} + + stages := []loadgen.Stage{{Mode: loadgen.ModeClosed, Concurrency: 3, Duration: 100 * time.Millisecond}} + report, err := loadgen.Run(context.Background(), targets(1), []loadgen.Op{op}, + stages, loadgen.SLO{ReadP99: time.Millisecond}) + if err != nil { + t.Fatalf("Run: %v", err) + } + + if stats, ok := report.Stages[0].Ops["Blocked"]; ok { + t.Errorf("cancelled requests were recorded: %+v", stats) + } + if len(report.Stages[0].Breached) != 0 { + t.Errorf("our own deadline breached the SLO: %v", report.Stages[0].Breached) + } +} + +func TestPercentilesComeFromRealSamples(t *testing.T) { + t.Parallel() + + stages := []loadgen.Stage{{Mode: loadgen.ModeClosed, Concurrency: 1, Duration: 200 * time.Millisecond}} + ops := []loadgen.Op{fixedOp("Fixed", loadgen.KindRead, 20*time.Millisecond, nil)} + + report, err := loadgen.Run(context.Background(), targets(1), ops, stages, loadgen.SLO{}) + if err != nil { + t.Fatalf("Run: %v", err) + } + + stats := report.Stages[0].Ops["Fixed"] + if stats.P50MS < 15 || stats.P50MS > 60 { + t.Errorf("p50 = %vms, want near the 20ms the operation actually takes", stats.P50MS) + } + if stats.P99MS < stats.P50MS { + t.Errorf("p99 %v is below p50 %v", stats.P99MS, stats.P50MS) + } + if stats.MaxMS < stats.P99MS { + t.Errorf("max %v is below p99 %v", stats.MaxMS, stats.P99MS) + } + if stats.AchievedRPS <= 0 { + t.Error("achieved rps was not reported") + } +} + +func TestFirstBreachNamesTheStageThatCrossedTheSLO(t *testing.T) { + t.Parallel() + + // Two stages: the second is slow enough to miss a 30ms read SLO. + fast := fixedOp("Op", loadgen.KindRead, 2*time.Millisecond, nil) + slow := fixedOp("Op", loadgen.KindRead, 60*time.Millisecond, nil) + + slo := loadgen.SLO{ReadP99: 30 * time.Millisecond} + first, err := loadgen.Run(context.Background(), targets(1), []loadgen.Op{fast}, + []loadgen.Stage{{Mode: loadgen.ModeClosed, Concurrency: 1, Duration: 100 * time.Millisecond}}, slo) + if err != nil { + t.Fatalf("Run: %v", err) + } + if len(first.Stages[0].Breached) != 0 { + t.Fatalf("fast stage reported a breach: %v", first.Stages[0].Breached) + } + + stages := []loadgen.Stage{ + {Mode: loadgen.ModeClosed, Concurrency: 1, Duration: 100 * time.Millisecond}, + {Mode: loadgen.ModeClosed, Concurrency: 2, Duration: 200 * time.Millisecond}, + } + report, err := loadgen.Run(context.Background(), targets(1), []loadgen.Op{slow}, stages, slo) + if err != nil { + t.Fatalf("Run: %v", err) + } + + breach := report.FirstBreach[loadgen.ModeClosed] + if breach == nil { + t.Fatal("no breach recorded for a stage well past the SLO") + } + if breach.StageIndex != 0 || breach.Concurrency != 1 { + t.Errorf("breach = stage %d concurrency %d, want the first stage that crossed", + breach.StageIndex, breach.Concurrency) + } +} + +// A quota rejection means the cluster is working. Counting it as a failure +// would fail a correct run, so declared codes are excluded from the verdict +// while still being reported. +func TestExpectedErrorCodesDoNotBreach(t *testing.T) { + t.Parallel() + + quota := awserr.New("RequestLimitExceeded", "quota", nil) + ops := []loadgen.Op{fixedOp("Op", loadgen.KindRead, time.Millisecond, quota)} + stages := []loadgen.Stage{{Mode: loadgen.ModeClosed, Concurrency: 1, Duration: 80 * time.Millisecond}} + + strict, err := loadgen.Run(context.Background(), targets(1), ops, stages, loadgen.SLO{}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if len(strict.Stages[0].Breached) == 0 { + t.Error("an undeclared error code did not breach") + } + + tolerant, err := loadgen.Run(context.Background(), targets(1), ops, stages, + loadgen.SLO{ExpectedErrorCodes: []string{"RequestLimitExceeded"}}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if len(tolerant.Stages[0].Breached) != 0 { + t.Errorf("a declared error code breached: %v", tolerant.Stages[0].Breached) + } + if got := tolerant.Stages[0].Ops["Op"].ErrorCodes["RequestLimitExceeded"]; got == 0 { + t.Error("a declared error code was not reported at all; it must still be counted") + } +} + +func TestErrorCodeKeepsTheServiceCode(t *testing.T) { + t.Parallel() + + if got := loadgen.ErrorCode(awserr.New("InternalError", "boom", nil)); got != "InternalError" { + t.Errorf("ErrorCode = %q, want the service code", got) + } + if got := loadgen.ErrorCode(nil); got != "" { + t.Errorf("ErrorCode(nil) = %q, want empty", got) + } + if got := loadgen.ErrorCode(errors.New("dial tcp: refused")); got == "" { + t.Error("a non-AWS error was classified as no error at all") + } +} + +func TestRunRejectsAnEmptyPlan(t *testing.T) { + t.Parallel() + + if _, err := loadgen.Run(context.Background(), nil, []loadgen.Op{fixedOp("Op", loadgen.KindRead, 0, nil)}, nil, loadgen.SLO{}); err == nil { + t.Error("no targets was accepted") + } + if _, err := loadgen.Run(context.Background(), targets(1), nil, nil, loadgen.SLO{}); err == nil { + t.Error("no operations was accepted") + } +} + +func TestResolveOpsRejectsAnUnknownOperation(t *testing.T) { + t.Parallel() + + if _, err := loadgen.ResolveOps([]string{"DescribeInstances"}); err != nil { + t.Fatalf("ResolveOps: %v", err) + } + _, err := loadgen.ResolveOps([]string{"DescribeInstances", "DeleteEverything"}) + if err == nil { + t.Fatal("an unknown operation was accepted, so the run would silently be shorter than asked for") + } + if !strings.Contains(err.Error(), "DeleteEverything") { + t.Errorf("error %q does not name the unknown operation", err) + } +} + +func TestSummaryReportsTheCapacityNumber(t *testing.T) { + t.Parallel() + + slo := loadgen.SLO{ReadP99: 5 * time.Millisecond} + stages := []loadgen.Stage{{Mode: loadgen.ModeClosed, Concurrency: 3, Duration: 80 * time.Millisecond}} + ops := []loadgen.Op{fixedOp("Op", loadgen.KindRead, 40*time.Millisecond, nil)} + + report, err := loadgen.Run(context.Background(), targets(1), ops, stages, slo) + if err != nil { + t.Fatalf("Run: %v", err) + } + report.Endpoint = "https://api.example.invalid" + + summary := loadgen.Summary(report) + for _, want := range []string{"https://api.example.invalid", "closed", "concurrency 3", "Op"} { + if !strings.Contains(summary, want) { + t.Errorf("summary does not mention %q:\n%s", want, summary) + } + } + if !strings.Contains(summary, "first breach at concurrency 3") { + t.Errorf("summary does not state where the SLO broke:\n%s", summary) + } +} + +// TestDescribeVolumesByIDRefusesToRunWithoutAVolume keeps the fast-path probe +// honest. Its whole purpose is to be compared against the full listing, so +// falling back to an unqualified read would silently measure the same thing +// twice and report the listing as costing nothing. +func TestDescribeVolumesByIDRefusesToRunWithoutAVolume(t *testing.T) { + t.Parallel() + + ops, err := loadgen.ResolveOps([]string{"DescribeVolumesByID"}) + if err != nil { + t.Fatalf("ResolveOps: %v", err) + } + if !loadgen.NeedsVolume(ops) { + t.Fatal("NeedsVolume must report true so the driver resolves one before the run") + } + if loadgen.NeedsVolume([]loadgen.Op{{Name: "DescribeVolumes"}}) { + t.Fatal("NeedsVolume must not claim the unqualified listing needs a volume id") + } + + if err := ops[0].Call(context.Background(), &loadgen.Target{Account: "000000000001"}); err == nil { + t.Fatal("an unresolved volume id must fail the call, not read the whole listing instead") + } +} diff --git a/spinifex/loadgen/ops.go b/spinifex/loadgen/ops.go new file mode 100644 index 000000000..1e34b5e18 --- /dev/null +++ b/spinifex/loadgen/ops.go @@ -0,0 +1,188 @@ +package loadgen + +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/sts" +) + +// The operations a run may drive. All but one are reads, deliberately: a load +// generator that creates guests would compete with the harness for the very +// capacity the harness sized the run against. CreateTags is the exception +// because a control plane whose reads are fast and whose writes are not is a +// broken control plane, and tagging costs no capacity. +var registry = map[string]Op{ + "DescribeInstances": { + Name: "DescribeInstances", Kind: KindRead, + Call: func(ctx context.Context, t *Target) error { + _, err := t.Clients.EC2.DescribeInstancesWithContext(ctx, &ec2.DescribeInstancesInput{}) + return err + }, + }, + "DescribeVpcs": { + Name: "DescribeVpcs", Kind: KindRead, + Call: func(ctx context.Context, t *Target) error { + _, err := t.Clients.EC2.DescribeVpcsWithContext(ctx, &ec2.DescribeVpcsInput{}) + return err + }, + }, + "DescribeSubnets": { + Name: "DescribeSubnets", Kind: KindRead, + Call: func(ctx context.Context, t *Target) error { + _, err := t.Clients.EC2.DescribeSubnetsWithContext(ctx, &ec2.DescribeSubnetsInput{}) + return err + }, + }, + "DescribeVolumes": { + Name: "DescribeVolumes", Kind: KindRead, + Call: func(ctx context.Context, t *Target) error { + _, err := t.Clients.EC2.DescribeVolumesWithContext(ctx, &ec2.DescribeVolumesInput{}) + return err + }, + }, + // The same read as DescribeVolumes but for one known id, which takes the + // handler's fast path and skips the bucket listing entirely. Run the two + // together and the gap between them is what the listing costs. + "DescribeVolumesByID": { + Name: "DescribeVolumesByID", Kind: KindRead, + Call: func(ctx context.Context, t *Target) error { + if t.VolumeID == "" { + return fmt.Errorf("DescribeVolumesByID: no volume resolved for %s", t.Account) + } + _, err := t.Clients.EC2.DescribeVolumesWithContext(ctx, &ec2.DescribeVolumesInput{ + VolumeIds: []*string{aws.String(t.VolumeID)}, + }) + return err + }, + }, + "DescribeSecurityGroups": { + Name: "DescribeSecurityGroups", Kind: KindRead, + Call: func(ctx context.Context, t *Target) error { + _, err := t.Clients.EC2.DescribeSecurityGroupsWithContext(ctx, &ec2.DescribeSecurityGroupsInput{}) + return err + }, + }, + // The console login path. A cluster whose EC2 API is healthy but whose STS + // is slow feels broken to every customer signing in, and nothing else in + // the run would notice. + "GetSessionToken": { + Name: "GetSessionToken", Kind: KindRead, + Call: func(ctx context.Context, t *Target) error { + _, err := t.Clients.STS.GetSessionTokenWithContext(ctx, &sts.GetSessionTokenInput{}) + return err + }, + }, + "CreateTags": { + Name: "CreateTags", Kind: KindWrite, + Call: func(ctx context.Context, t *Target) error { + if t.VPCID == "" { + return fmt.Errorf("CreateTags: no VPC resolved for %s", t.Account) + } + _, err := t.Clients.EC2.CreateTagsWithContext(ctx, &ec2.CreateTagsInput{ + Resources: []*string{aws.String(t.VPCID)}, + Tags: []*ec2.Tag{{ + Key: aws.String("spx:loadgen"), + Value: aws.String(t.Account), + }}, + }) + return err + }, + }, +} + +// DefaultOps is what a run drives when none are named: the three reads every +// console page and every terraform plan makes, so the number means something +// outside this harness. +var DefaultOps = []string{"DescribeInstances", "DescribeVpcs", "DescribeVolumes"} + +// OpNames lists every operation the registry knows, for the usage text. +func OpNames() []string { + names := make([]string, 0, len(registry)) + for name := range registry { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// ResolveOps turns names into operations, rejecting an unknown one rather than +// silently running a shorter list than was asked for. +func ResolveOps(names []string) ([]Op, error) { + ops := make([]Op, 0, len(names)) + for _, name := range names { + name = strings.TrimSpace(name) + if name == "" { + continue + } + op, ok := registry[name] + if !ok { + return nil, fmt.Errorf("loadgen: unknown operation %q (known: %s)", + name, strings.Join(OpNames(), ", ")) + } + ops = append(ops, op) + } + if len(ops) == 0 { + return nil, fmt.Errorf("loadgen: no operations selected") + } + return ops, nil +} + +// NeedsVPC reports whether any selected operation acts on a tenant resource, +// so the caller knows whether to pay for discovery before the run starts. +func NeedsVPC(ops []Op) bool { + for _, op := range ops { + if op.Name == "CreateTags" { + return true + } + } + return false +} + +// NeedsVolume reports whether any selected operation needs a volume id. +func NeedsVolume(ops []Op) bool { + for _, op := range ops { + if op.Name == "DescribeVolumesByID" { + return true + } + } + return false +} + +// ResolveVolume finds one volume for the target to ask about by id. A tenant +// with a running guest has a root volume; one with none cannot measure the +// fast path, and saying so is better than measuring an empty listing instead. +func ResolveVolume(ctx context.Context, target *Target) error { + out, err := target.Clients.EC2.DescribeVolumesWithContext(ctx, &ec2.DescribeVolumesInput{}) + if err != nil { + return fmt.Errorf("loadgen: resolve volume for %s: %w", target.Account, err) + } + for _, volume := range out.Volumes { + if volume.VolumeId != nil && *volume.VolumeId != "" { + target.VolumeID = *volume.VolumeId + return nil + } + } + return fmt.Errorf("loadgen: account %s has no volume to ask about", target.Account) +} + +// ResolveVPC finds a VPC for the target to act on. Every account has a default +// VPC created with it, so an empty answer means something is wrong with the +// tenant rather than that it has nothing yet. +func ResolveVPC(ctx context.Context, target *Target) error { + out, err := target.Clients.EC2.DescribeVpcsWithContext(ctx, &ec2.DescribeVpcsInput{}) + if err != nil { + return fmt.Errorf("loadgen: resolve VPC for %s: %w", target.Account, err) + } + for _, vpc := range out.Vpcs { + if vpc.VpcId != nil && *vpc.VpcId != "" { + target.VPCID = *vpc.VpcId + return nil + } + } + return fmt.Errorf("loadgen: account %s has no VPC to act on", target.Account) +} diff --git a/spinifex/network/host/mac_binding.go b/spinifex/network/host/mac_binding.go index 2a62d763c..cd07741e7 100644 --- a/spinifex/network/host/mac_binding.go +++ b/spinifex/network/host/mac_binding.go @@ -14,8 +14,10 @@ import ( // default-gateway MAC (the host shares the physical uplink), read from the kernel // neigh table and primed with one ping when absent. Idempotent: an existing // binding for the same lrpName+ip is replaced. Best-effort: a missing MAC logs -// and returns nil, leaving dynamic ARP as the fallback. -func SeedNexthopMAC(ctx context.Context, runner Runner, lrpName, nexthopIP string) error { +// and returns nil, leaving dynamic ARP as the fallback. nbAddr selects the NB DB +// to write to (empty uses the local default socket); a compute node runs no +// database, so a write without it would never reach the cluster. +func SeedNexthopMAC(ctx context.Context, runner Runner, nbAddr, lrpName, nexthopIP string) error { if lrpName == "" || nexthopIP == "" { return nil } @@ -40,9 +42,14 @@ func SeedNexthopMAC(ctx context.Context, runner Runner, lrpName, nexthopIP strin return nil } + var db []string + if nbAddr != "" { + db = []string{"--db=" + nbAddr, "--no-leader-only"} + } + // Idempotent: drop any stale binding (best-effort) before adding. - _, _ = runner.Run(ctx, "ovn-nbctl", "--if-exists", "static-mac-binding-del", lrpName, nexthopIP) - if _, err := runner.Run(ctx, "ovn-nbctl", "static-mac-binding-add", lrpName, nexthopIP, mac); err != nil { + _, _ = runner.Run(ctx, "ovn-nbctl", append(append([]string{}, db...), "--if-exists", "static-mac-binding-del", lrpName, nexthopIP)...) + if _, err := runner.Run(ctx, "ovn-nbctl", append(append([]string{}, db...), "static-mac-binding-add", lrpName, nexthopIP, mac)...); err != nil { return fmt.Errorf("ovn-nbctl static-mac-binding-add %s %s %s: %w", lrpName, nexthopIP, mac, err) } diff --git a/spinifex/network/host/mac_binding_test.go b/spinifex/network/host/mac_binding_test.go index e197118ba..2509aacf8 100644 --- a/spinifex/network/host/mac_binding_test.go +++ b/spinifex/network/host/mac_binding_test.go @@ -40,7 +40,7 @@ func TestSeedNexthopMAC_ResolvedOnFirstRead(t *testing.T) { "ip neigh show 192.168.1.1 dev br-wan": []byte("192.168.1.1 dev br-wan lladdr 04:f4:1c:fd:56:27 REACHABLE"), }, } - if err := SeedNexthopMAC(context.Background(), r, "gw-vpc-1", "192.168.1.1"); err != nil { + if err := SeedNexthopMAC(context.Background(), r, "", "gw-vpc-1", "192.168.1.1"); err != nil { t.Fatalf("SeedNexthopMAC: %v", err) } if calls := r.callArgs("ping"); len(calls) != 0 { @@ -64,6 +64,34 @@ func TestSeedNexthopMAC_ResolvedOnFirstRead(t *testing.T) { } } +// A compute node runs no local database, so the binding has to be addressed to +// the configured NB cluster or the write lands nowhere the fabric reads. +func TestSeedNexthopMAC_WritesToTheConfiguredNBAddress(t *testing.T) { + r := &scriptedRunner{ + responses: map[string][]byte{ + "ip route get 192.168.1.1": []byte("192.168.1.1 dev br-wan src 192.168.1.50"), + "ip neigh show 192.168.1.1 dev br-wan": []byte("192.168.1.1 dev br-wan lladdr 04:f4:1c:fd:56:27 REACHABLE"), + }, + } + nb := "tcp:10.2.0.2:6641,tcp:10.2.0.3:6641,tcp:10.2.0.4:6641" + if err := SeedNexthopMAC(context.Background(), r, nb, "gw-vpc-1", "192.168.1.1"); err != nil { + t.Fatalf("SeedNexthopMAC: %v", err) + } + + wantDel := "ovn-nbctl --db=" + nb + " --no-leader-only --if-exists static-mac-binding-del gw-vpc-1 192.168.1.1" + delCalls := r.callArgs("ovn-nbctl --db=") + if len(delCalls) != 2 { + t.Fatalf("expected both nbctl calls to carry --db, got %d: %v", len(delCalls), r.calls) + } + if got := strings.Join(delCalls[0], " "); got != wantDel { + t.Fatalf("del argv mismatch\n got: %s\nwant: %s", got, wantDel) + } + wantAdd := "ovn-nbctl --db=" + nb + " --no-leader-only static-mac-binding-add gw-vpc-1 192.168.1.1 04:f4:1c:fd:56:27" + if got := strings.Join(delCalls[1], " "); got != wantAdd { + t.Fatalf("add argv mismatch\n got: %s\nwant: %s", got, wantAdd) + } +} + func TestSeedNexthopMAC_ResolvedAfterPing(t *testing.T) { neighKey := "ip neigh show 192.168.1.1 dev br-wan" r := &scriptedRunner{ @@ -79,7 +107,7 @@ func TestSeedNexthopMAC_ResolvedAfterPing(t *testing.T) { firstOut: []byte(""), secondOut: []byte("192.168.1.1 dev br-wan lladdr 04:f4:1c:fd:56:27 REACHABLE"), } - if err := SeedNexthopMAC(context.Background(), wrapper, "gw-vpc-1", "192.168.1.1"); err != nil { + if err := SeedNexthopMAC(context.Background(), wrapper, "", "gw-vpc-1", "192.168.1.1"); err != nil { t.Fatalf("SeedNexthopMAC: %v", err) } if wrapper.neighCalls != 2 { @@ -132,7 +160,7 @@ func TestSeedNexthopMAC_UnresolvedEvenAfterPing(t *testing.T) { "ip neigh show 192.168.1.1 dev br-wan": []byte("192.168.1.1 dev br-wan INCOMPLETE"), }, } - if err := SeedNexthopMAC(context.Background(), r, "gw-vpc-1", "192.168.1.1"); err != nil { + if err := SeedNexthopMAC(context.Background(), r, "", "gw-vpc-1", "192.168.1.1"); err != nil { t.Fatalf("SeedNexthopMAC must return nil (best-effort) on unresolved MAC, got: %v", err) } if calls := r.callArgs("ping"); len(calls) != 1 { @@ -145,10 +173,10 @@ func TestSeedNexthopMAC_UnresolvedEvenAfterPing(t *testing.T) { func TestSeedNexthopMAC_EmptyArgsNoop(t *testing.T) { r := &scriptedRunner{} - if err := SeedNexthopMAC(context.Background(), r, "", "192.168.1.1"); err != nil { + if err := SeedNexthopMAC(context.Background(), r, "", "", "192.168.1.1"); err != nil { t.Fatalf("expected nil for empty lrpName, got %v", err) } - if err := SeedNexthopMAC(context.Background(), r, "gw-vpc-1", ""); err != nil { + if err := SeedNexthopMAC(context.Background(), r, "", "gw-vpc-1", ""); err != nil { t.Fatalf("expected nil for empty nexthopIP, got %v", err) } if len(r.calls) != 0 { @@ -162,7 +190,7 @@ func TestSeedNexthopMAC_RouteGetFailurePropagates(t *testing.T) { "ip route get 192.168.1.1": errors.New("network unreachable"), }, } - err := SeedNexthopMAC(context.Background(), r, "gw-vpc-1", "192.168.1.1") + err := SeedNexthopMAC(context.Background(), r, "", "gw-vpc-1", "192.168.1.1") if err == nil { t.Fatal("expected error when route resolution fails") } @@ -178,7 +206,7 @@ func TestSeedNexthopMAC_AddFailurePropagates(t *testing.T) { "ovn-nbctl static-mac-binding-add gw-vpc-1 192.168.1.1 04:f4:1c:fd:56:27": errors.New("exit 1"), }, } - err := SeedNexthopMAC(context.Background(), r, "gw-vpc-1", "192.168.1.1") + err := SeedNexthopMAC(context.Background(), r, "", "gw-vpc-1", "192.168.1.1") if err == nil { t.Fatal("expected error when static-mac-binding-add fails") } diff --git a/spinifex/objectstore/buckets.go b/spinifex/objectstore/buckets.go new file mode 100644 index 000000000..f4d8b4403 --- /dev/null +++ b/spinifex/objectstore/buckets.go @@ -0,0 +1,94 @@ +package objectstore + +import ( + "context" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/s3" +) + +// ownerAccountHeader asks predastore for another account's buckets. It is +// honoured only for the config service credential this store signs with, and +// the name has to match predastore's own constant — the header lives across a +// repository boundary and nothing links the two at compile time. +const ownerAccountHeader = "X-Predastore-Owner-Account-Id" + +// MultipartUploadRef identifies an upload well enough to abort it. An abort +// needs all three: the upload id alone does not say where to send it. +type MultipartUploadRef struct { + Bucket string + Key string + UploadID string +} + +// ListBucketsForOwner returns the buckets belonging to accountID rather than to +// the credential making the call. +// +// This is enumeration, not access: the service credential can already open any +// bucket it can name. Without it nothing can discover what an account owns, so +// nothing can delete it either. +func (s *S3ObjectStore) ListBucketsForOwner(ctx context.Context, accountID string) ([]string, error) { + out, err := s.client.ListBucketsWithContext(ctx, &s3.ListBucketsInput{}, withOwnerAccount(accountID)) + if err != nil { + return nil, err + } + + names := make([]string, 0, len(out.Buckets)) + for _, bucket := range out.Buckets { + if name := aws.StringValue(bucket.Name); name != "" { + names = append(names, name) + } + } + return names, nil +} + +// ListMultipartUploads returns the uploads started against bucket and not yet +// completed. They are invisible to an object listing and hold real storage, so +// a bucket can list as empty and still have parts in it. +func (s *S3ObjectStore) ListMultipartUploads(ctx context.Context, bucket string) ([]MultipartUploadRef, error) { + out, err := s.client.ListMultipartUploadsWithContext(ctx, &s3.ListMultipartUploadsInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + return nil, err + } + + refs := make([]MultipartUploadRef, 0, len(out.Uploads)) + for _, upload := range out.Uploads { + if upload == nil || aws.StringValue(upload.UploadId) == "" { + continue + } + refs = append(refs, MultipartUploadRef{ + Bucket: bucket, + Key: aws.StringValue(upload.Key), + UploadID: aws.StringValue(upload.UploadId), + }) + } + return refs, nil +} + +// AbortMultipartUpload discards an upload's parts. The object it would have +// written is untouched. +func (s *S3ObjectStore) AbortMultipartUpload(ctx context.Context, upload MultipartUploadRef) error { + _, err := s.client.AbortMultipartUploadWithContext(ctx, &s3.AbortMultipartUploadInput{ + Bucket: aws.String(upload.Bucket), + Key: aws.String(upload.Key), + UploadId: aws.String(upload.UploadID), + }) + return err +} + +// DeleteBucket removes an empty bucket. +func (s *S3ObjectStore) DeleteBucket(ctx context.Context, bucket string) error { + _, err := s.client.DeleteBucketWithContext(ctx, &s3.DeleteBucketInput{Bucket: aws.String(bucket)}) + return err +} + +// withOwnerAccount sets the owner header on the request before it is signed, so +// the header is covered by the signature rather than stripped as unsigned. +func withOwnerAccount(accountID string) request.Option { + return func(r *request.Request) { + r.HTTPRequest.Header.Set(ownerAccountHeader, accountID) + } +} diff --git a/spinifex/services/awsgw/awsgw.go b/spinifex/services/awsgw/awsgw.go index 43ea1e079..1c338abc5 100644 --- a/spinifex/services/awsgw/awsgw.go +++ b/spinifex/services/awsgw/awsgw.go @@ -441,6 +441,7 @@ func launchService(config *config.ClusterConfig) error { RegistryHost: registryHost, AZ: nodeConfig.AZ, IAMService: iamService, + BucketStore: objStore, STSService: stsService, Version: version, Commit: commit, diff --git a/spinifex/vm/teardown_reaper.go b/spinifex/vm/teardown_reaper.go index 94a255e1b..9f99c69a6 100644 --- a/spinifex/vm/teardown_reaper.go +++ b/spinifex/vm/teardown_reaper.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "log/slog" + "strings" + "sync" "time" ) @@ -24,14 +26,82 @@ const terminatedVisibilityWindow = 15 * time.Minute // this reaper finishes them rather than abandoning them. type TerminatedTeardownReaper struct { m *Manager + + mu sync.Mutex + backoff map[string]*depBackoff } +// depBackoff spaces out re-drives of a dependent that keeps failing. +// +// A dependent can fail for a reason no number of retries will fix — a volume +// whose metadata document cannot be reassembled, say. Re-driving it on every +// sweep costs a full object-store read each time, and one such record was +// enough to hold an object-store request slot open continuously and slow every +// other listing on the cluster behind it. The retry still happens, and the +// dependent still stays Failed; it just stops being free. +type depBackoff struct { + failures int + next time.Time +} + +// teardownRetryBase and teardownRetryMax bound the backoff. The first failure +// is retried on the next sweep, and the interval doubles from there. +const ( + teardownRetryBase = 30 * time.Second + teardownRetryMax = 30 * time.Minute +) + var _ Reaper = (*TerminatedTeardownReaper)(nil) // NewTerminatedTeardownReaper builds the reaper bound to this Manager's cleaner // and state store. func (m *Manager) NewTerminatedTeardownReaper() *TerminatedTeardownReaper { - return &TerminatedTeardownReaper{m: m} + return &TerminatedTeardownReaper{m: m, backoff: make(map[string]*depBackoff)} +} + +// dueForRetry reports whether a dependent may be re-driven on this sweep. It is +// in-memory and node-local: a restart simply retries everything once more. +func (r *TerminatedTeardownReaper) dueForRetry(instanceID, dep string) bool { + r.mu.Lock() + defer r.mu.Unlock() + b := r.backoff[instanceID+"/"+dep] + return b == nil || !time.Now().Before(b.next) +} + +// recordRetry advances or clears a dependent's backoff from its result. +func (r *TerminatedTeardownReaper) recordRetry(instanceID, dep string, state TeardownState) { + key := instanceID + "/" + dep + r.mu.Lock() + defer r.mu.Unlock() + if state == TeardownDone { + delete(r.backoff, key) + return + } + b := r.backoff[key] + if b == nil { + b = &depBackoff{} + r.backoff[key] = b + } + b.failures++ + wait := teardownRetryBase << min(b.failures-1, 16) + if wait > teardownRetryMax || wait <= 0 { + wait = teardownRetryMax + } + b.next = time.Now().Add(wait) + slog.Warn("vm/gc: teardown dependent failed, backing off", + "instanceId", instanceID, "dependent", dep, "failures", b.failures, "retry_in", wait) +} + +// forgetInstance drops an instance's backoff once its record is gone, so the +// map cannot grow without bound across the life of the process. +func (r *TerminatedTeardownReaper) forgetInstance(instanceID string) { + r.mu.Lock() + defer r.mu.Unlock() + for key := range r.backoff { + if strings.HasPrefix(key, instanceID+"/") { + delete(r.backoff, key) + } + } } func (r *TerminatedTeardownReaper) Class() string { return "instance-teardown" } @@ -108,30 +178,34 @@ func (r *TerminatedTeardownReaper) retryOutstanding(v *VM) map[string]TeardownSt } results := make(map[string]TeardownState) + retry := func(dep string, run func() error) { + if !outstanding(v, dep) || !r.dueForRetry(v.ID, dep) { + return + } + state := resultState(run()) + results[dep] = state + r.recordRetry(v.ID, dep, state) + } - if outstanding(v, TeardownVolumes) { - results[TeardownVolumes] = resultState(c.DeleteVolumes(v)) - } - if outstanding(v, TeardownGPU) { - results[TeardownGPU] = resultState(c.ReleaseGPU(v)) - } - if outstanding(v, TeardownPlacement) { - results[TeardownPlacement] = resultState(c.RemoveFromPlacementGroup(v)) - } + retry(TeardownVolumes, func() error { return c.DeleteVolumes(v) }) + retry(TeardownGPU, func() error { return c.ReleaseGPU(v) }) + retry(TeardownPlacement, func() error { return c.RemoveFromPlacementGroup(v) }) // NAT: re-publishes vpc.delete-nat + frees the IPAM slot. The cluster // reconciler heals the dataplane NAT rule. - if outstanding(v, TeardownNAT) { - results[TeardownNAT] = resultState(c.ReleasePublicIP(v)) - } + retry(TeardownNAT, func() error { return c.ReleasePublicIP(v) }) // ENI delete + OVN: deleting the ENI KV record turns its LSP into an orphan // the cluster reconcile prune reaps, so a successful ENI delete completes - // both eni and ovn. - if outstanding(v, TeardownENI) || outstanding(v, TeardownOVN) { + // both eni and ovn. Both share the ENI's backoff because one call decides + // them, and it is gated on whichever of the two is due. + if (outstanding(v, TeardownENI) || outstanding(v, TeardownOVN)) && + (r.dueForRetry(v.ID, TeardownENI) || r.dueForRetry(v.ID, TeardownOVN)) { eniErr := c.DetachAndDeleteENI(v) results[TeardownENI] = resultState(eniErr) results[TeardownOVN] = resultState(eniErr) + r.recordRetry(v.ID, TeardownENI, results[TeardownENI]) + r.recordRetry(v.ID, TeardownOVN, results[TeardownOVN]) } return results @@ -143,6 +217,7 @@ func (r *TerminatedTeardownReaper) purge(v *VM) bool { "instanceId", v.ID, "err", err) return false } + r.forgetInstance(v.ID) slog.Info("vm/gc: purged terminated record, teardown complete", "instanceId", v.ID) return true } diff --git a/spinifex/vm/teardown_reaper_test.go b/spinifex/vm/teardown_reaper_test.go new file mode 100644 index 000000000..d62361fd7 --- /dev/null +++ b/spinifex/vm/teardown_reaper_test.go @@ -0,0 +1,100 @@ +//test:in-package — drives the reaper's unexported per-dependent backoff state +//and reuses the in-package cleaner/state-store doubles. + +package vm + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestTeardownReaperBacksOffARepeatedlyFailingDependent covers the case that +// took the cluster down: a volume whose metadata document cannot be read fails +// every re-drive, and an ungated reaper spent a 20s object-store request on it +// on every sweep. The dependent must stay Failed (never dropped) but must not +// be retried again on the very next sweep. +func TestTeardownReaperBacksOffARepeatedlyFailingDependent(t *testing.T) { + m, cleaner, store, reaper, v := seedTerminated(t, map[string]string{ + TeardownVolumes: string(TeardownPending), + }) + _ = m + cleaner.deleteVolumesErr = errors.New("shard sizes do not match") + + _, err := reaper.Sweep(context.Background()) + require.NoError(t, err) + assert.Equal(t, []string{v.ID}, cleaner.deleteVolumes, "the first sweep must attempt the dependent") + assert.Equal(t, string(TeardownFailed), v.Teardown[TeardownVolumes], + "a failed retry must stay Failed, not be dropped") + + _, err = reaper.Sweep(context.Background()) + require.NoError(t, err) + assert.Equal(t, []string{v.ID}, cleaner.deleteVolumes, + "the next sweep must skip a dependent that is still backed off") + assert.Equal(t, string(TeardownFailed), v.Teardown[TeardownVolumes], + "a backed-off dependent stays Failed and its record stays in the store") + + remaining, err := store.ListTerminatedInstances() + require.NoError(t, err) + require.Len(t, remaining, 1, "a record with an outstanding dependent must not be purged") +} + +// TestTeardownReaperBackoffGrowsAndClearsOnSuccess checks the interval doubles +// per failure and that a dependent which eventually succeeds leaves no backoff +// state behind. +func TestTeardownReaperBackoffGrowsAndClearsOnSuccess(t *testing.T) { + _, cleaner, _, reaper, v := seedTerminated(t, map[string]string{ + TeardownVolumes: string(TeardownPending), + }) + cleaner.deleteVolumesErr = errors.New("shard sizes do not match") + + key := v.ID + "/" + TeardownVolumes + for i := 1; i <= 3; i++ { + expire(reaper, key) + _, err := reaper.Sweep(context.Background()) + require.NoError(t, err) + require.Len(t, cleaner.deleteVolumes, i) + + b := reaper.backoff[key] + require.NotNil(t, b) + assert.Equal(t, i, b.failures) + assert.WithinDuration(t, time.Now().Add(teardownRetryBase<<(i-1)), b.next, time.Minute) + } + + cleaner.deleteVolumesErr = nil + expire(reaper, key) + _, err := reaper.Sweep(context.Background()) + require.NoError(t, err) + assert.Len(t, cleaner.deleteVolumes, 4) + assert.Equal(t, string(TeardownDone), v.Teardown[TeardownVolumes]) + assert.Empty(t, reaper.backoff, "a dependent that succeeded must leave no backoff state") +} + +// TestTeardownReaperCapsTheBackoffInterval keeps the doubling from running away +// past the cap (or overflowing into a negative duration). +func TestTeardownReaperCapsTheBackoffInterval(t *testing.T) { + _, _, _, reaper, v := seedTerminated(t, nil) + + for range 80 { + reaper.recordRetry(v.ID, TeardownVolumes, TeardownFailed) + } + b := reaper.backoff[v.ID+"/"+TeardownVolumes] + require.NotNil(t, b) + assert.WithinDuration(t, time.Now().Add(teardownRetryMax), b.next, time.Minute) + assert.True(t, reaper.dueForRetry(v.ID, "some-other-dependent"), + "backoff is per dependent, not per instance") +} + +// expire brings a backoff entry's next-retry time forward so a test can take +// the following sweep without waiting for the real interval. +func expire(r *TerminatedTeardownReaper, key string) { + r.mu.Lock() + defer r.mu.Unlock() + if b := r.backoff[key]; b != nil { + b.next = time.Now().Add(-time.Second) + } +} diff --git a/spinifex/vpcd/flows_barrier_test.go b/spinifex/vpcd/flows_barrier_test.go new file mode 100644 index 000000000..4d2db0cc3 --- /dev/null +++ b/spinifex/vpcd/flows_barrier_test.go @@ -0,0 +1,34 @@ +// Exercises newFlowsBarrier, which is unexported wiring, by swapping the +// unexported waitForFlowsHV var it closes over. +// +//test:in-package +package vpcd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The barrier has to confirm against the same NB cluster vpcd writes through. +// A barrier with no address falls back to the local unix socket, which on a +// node outside the configured cluster is a database that never saw the write. +func TestNewFlowsBarrierUsesTheConfiguredNBAddress(t *testing.T) { + const nbAddr = "tcp:10.2.0.2:6641,tcp:10.2.0.3:6641,tcp:10.2.0.4:6641" + + original := waitForFlowsHV + t.Cleanup(func() { waitForFlowsHV = original }) + + var got string + var calls int + waitForFlowsHV = func(addr string) error { + got = addr + calls++ + return nil + } + + require.NoError(t, newFlowsBarrier(nbAddr)()) + assert.Equal(t, 1, calls, "the barrier runs the sync once per call") + assert.Equal(t, nbAddr, got, "the barrier confirms against the configured cluster") +} diff --git a/spinifex/vpcd/vpcd.go b/spinifex/vpcd/vpcd.go index d1e4cf0b4..b491b4973 100644 --- a/spinifex/vpcd/vpcd.go +++ b/spinifex/vpcd/vpcd.go @@ -43,11 +43,20 @@ const ( OvnExternalBridge = "br-ext" ) +// newFlowsBarrier binds the barrier to the same NB cluster the rest of vpcd +// writes through. Without the address `ovn-nbctl` uses the local unix socket, +// so a node that is not a member of the configured cluster confirms against a +// database that never saw the write — slowly, then not at all. +func newFlowsBarrier(nbAddr string) func() error { + return func() error { return waitForFlowsHV(nbAddr) } +} + // waitForFlowsHV runs `ovn-nbctl --wait=hv sync`, blocking until all chassis acknowledge the new NB sequence. // Bounded at 30 s; overruns log a Warn and return nil. Declared as a var so tests can stub it. -var waitForFlowsHV = func() error { +var waitForFlowsHV = func(nbAddr string) error { start := time.Now() cmd := sudoCommand("ovn-nbctl", + "--db="+nbAddr, "--no-leader-only", "--timeout=30", "--wait=hv", @@ -424,6 +433,8 @@ func launchService(cfg *Config) error { defer liveClient.Close() slog.Info("Connected to OVN NB DB", "endpoint", cfg.OVNNBAddr) + flowsBarrier := newFlowsBarrier(cfg.OVNNBAddr) + bridgeMode, wanBridge := resolveBridgeConfig(cfg.BridgeMode, cfg.ExternalInterface) slog.Info("External bridge mode", "mode", bridgeMode, "wan_bridge", wanBridge) if err := verifyBridgeMode(bridgeMode, cfg.ExternalInterface, wanBridge); err != nil { @@ -484,7 +495,7 @@ func launchService(cfg *Config) error { sgMgr := policy.NewSecurityGroupManager(liveClient) natOpts := []policy.Option{ - policy.WithFlowsBarrier(waitForFlowsHV), + policy.WithFlowsBarrier(flowsBarrier), policy.WithNeighFlusher(neighFlusher(wanBridge)), policy.WithNeighPrimer(neighPrimer(wanBridge)), } @@ -605,10 +616,10 @@ func launchService(cfg *Config) error { Allocator: gwAllocator, Chassis: chassisNames, NATMode: natMode, - FlowsBarrier: waitForFlowsHV, + FlowsBarrier: flowsBarrier, RoutedIngress: routedIngress, NexthopSeed: func(ctx context.Context, lrpName, nexthopIP string) error { - return host.SeedNexthopMAC(ctx, host.NewExecRunner(), lrpName, nexthopIP) + return host.SeedNexthopMAC(ctx, host.NewExecRunner(), cfg.OVNNBAddr, lrpName, nexthopIP) }, }) if err != nil { @@ -627,7 +638,7 @@ func launchService(cfg *Config) error { // starts, and no lease event will ever fire to correct them. reconcileGatewayLeases(ctx, dhcpMgr, igwMgr) } - eipMgr, err := external.NewEIPManager(natMgr, waitForFlowsHV) + eipMgr, err := external.NewEIPManager(natMgr, flowsBarrier) if err != nil { return fmt.Errorf("construct EIP manager: %w", err) }