diff --git a/cmd/pulse-control-plane/provider_msp.go b/cmd/pulse-control-plane/provider_msp.go index 3aab05813..7fdc6106c 100644 --- a/cmd/pulse-control-plane/provider_msp.go +++ b/cmd/pulse-control-plane/provider_msp.go @@ -16,6 +16,7 @@ func newProviderMSPCmd() *cobra.Command { cmd.AddCommand(newProviderMSPBackupCmd()) cmd.AddCommand(newProviderMSPPreflightCmd()) cmd.AddCommand(newProviderMSPProofCmd()) + cmd.AddCommand(newProviderMSPRecoverCmd()) cmd.AddCommand(newProviderMSPStatusCmd()) return cmd } diff --git a/cmd/pulse-control-plane/provider_msp_recover.go b/cmd/pulse-control-plane/provider_msp_recover.go new file mode 100644 index 000000000..a422751c0 --- /dev/null +++ b/cmd/pulse-control-plane/provider_msp_recover.go @@ -0,0 +1,94 @@ +package main + +import ( + "fmt" + "strings" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp" + "github.com/spf13/cobra" +) + +func newProviderMSPRecoverCmd() *cobra.Command { + var opts cloudcp.ProviderMSPRecoveryOptions + + cmd := &cobra.Command{ + Use: "recover", + Short: "Recover failed or degraded provider-hosted MSP client workspaces", + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := cloudcp.LoadConfig() + if err != nil { + return fmt.Errorf("load control plane config: %w", err) + } + report, err := cloudcp.RecoverProviderMSPWorkspaces(cmd.Context(), cfg, opts) + printProviderMSPRecoveryReport(report) + return err + }, + } + + cmd.Flags().StringArrayVar(&opts.TenantIDs, "tenant-id", nil, "Client workspace tenant ID to recover (repeatable)") + cmd.Flags().BoolVar(&opts.AllDegraded, "all-degraded", false, "Recover all failed, stuck provisioning, or unhealthy active client workspaces") + cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false, "Print the recovery plan without mutating tenant runtimes") + cmd.Flags().BoolVar(&opts.AllowEnvPlan, "allow-env-plan", false, "Allow CP_PROVIDER_MSP_PLAN_VERSION fallback instead of a signed provider MSP license file for local development") + cmd.Flags().StringVar(&opts.Image, "image", "", "Tenant runtime image to use during recovery (default: CP_PULSE_IMAGE)") + cmd.Flags().StringVar(&opts.RunID, "run-id", "", "Operator-visible recovery run identifier") + cmd.Flags().StringVar(&opts.SnapshotRoot, "snapshot-root", "", "Override tenant snapshot root (default: /backups/rollout)") + cmd.Flags().DurationVar(&opts.HealthTimeout, "health-timeout", 90*time.Second, "How long to wait for the recovered runtime to become healthy") + cmd.Flags().BoolVar(&opts.PrunePrevious, "prune-previous", false, "Remove the preserved pre-recovery container after success") + return cmd +} + +func printProviderMSPRecoveryReport(report *cloudcp.ProviderMSPRecoveryReport) { + if report == nil { + fmt.Println("provider_msp_recovery_ok=false") + return + } + fmt.Printf("provider_msp_recovery_ok=%t\n", report.OK) + fmt.Printf("dry_run=%t\n", report.DryRun) + fmt.Printf("plan_version=%s\n", report.PlanVersion) + fmt.Printf("plan_source=%s\n", report.PlanSource) + fmt.Printf("license_id=%s\n", report.LicenseID) + fmt.Printf("license_email=%s\n", report.LicenseEmail) + fmt.Printf("workspace_limit=%d\n", report.WorkspaceLimit) + fmt.Printf("recover_count=%d\n", report.RecoverCount) + fmt.Printf("recovered_count=%d\n", report.RecoveredCount) + fmt.Printf("skipped_count=%d\n", report.SkippedCount) + fmt.Printf("error_count=%d\n", report.ErrorCount) + for _, item := range report.Items { + fields := []string{ + "workspace=" + item.TenantID, + "display_name=" + quoteProviderMSPRecoveryField(item.DisplayName), + "state=" + item.State, + "action=" + item.Action, + "reason=" + quoteProviderMSPRecoveryField(item.Reason), + fmt.Sprintf("stuck_provisioning=%t", item.StuckProvisioning), + fmt.Sprintf("recovered=%t", item.Recovered), + } + if item.PreviousContainerID != "" { + fields = append(fields, "previous_container_id="+item.PreviousContainerID) + } + if item.ActiveContainerID != "" { + fields = append(fields, "active_container_id="+item.ActiveContainerID) + } + if item.ActiveImageRef != "" { + fields = append(fields, "active_image_ref="+item.ActiveImageRef) + } + if item.ActiveImageID != "" { + fields = append(fields, "active_image_id="+item.ActiveImageID) + } + if item.RestoredMissing { + fields = append(fields, "restored_missing=true") + } + if item.ReconciledOnly { + fields = append(fields, "reconciled_only=true") + } + if item.Error != "" { + fields = append(fields, "error="+quoteProviderMSPRecoveryField(item.Error)) + } + fmt.Println(strings.Join(fields, " ")) + } +} + +func quoteProviderMSPRecoveryField(value string) string { + return fmt.Sprintf("%q", strings.TrimSpace(value)) +} diff --git a/cmd/pulse-control-plane/provider_msp_recover_test.go b/cmd/pulse-control-plane/provider_msp_recover_test.go new file mode 100644 index 000000000..bc3506589 --- /dev/null +++ b/cmd/pulse-control-plane/provider_msp_recover_test.go @@ -0,0 +1,92 @@ +package main + +import ( + "bytes" + "io" + "os" + "strings" + "testing" + + "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp" +) + +func TestProviderMSPCommandExposesRecover(t *testing.T) { + cmd := newProviderMSPCmd() + for _, child := range cmd.Commands() { + if child.Name() == "recover" { + return + } + } + t.Fatal("provider-msp recover command is not registered") +} + +func TestPrintProviderMSPRecoveryReport(t *testing.T) { + var buf bytes.Buffer + stdout := captureStdoutForProviderMSPRecoverTest(t, &buf) + printProviderMSPRecoveryReport(&cloudcp.ProviderMSPRecoveryReport{ + OK: true, + DryRun: true, + PlanVersion: "msp_growth", + PlanSource: cloudcp.ProviderMSPPlanSourceLicenseFile, + LicenseID: "lic_test", + LicenseEmail: "provider@example.com", + WorkspaceLimit: 15, + RecoverCount: 1, + SkippedCount: 1, + Items: []cloudcp.ProviderMSPRecoveryItem{ + { + TenantID: "t-STUCK", + DisplayName: "Client A", + State: "provisioning", + Action: "recover", + Reason: "workspace is stuck in provisioning", + StuckProvisioning: true, + }, + { + TenantID: "t-HEALTHY", + DisplayName: "Client B", + State: "active", + Action: "skip", + Reason: "workspace is active and healthy", + }, + }, + }) + stdout() + + output := buf.String() + for _, want := range []string{ + "provider_msp_recovery_ok=true", + "dry_run=true", + "recover_count=1", + "skipped_count=1", + "workspace=t-STUCK", + "stuck_provisioning=true", + "reason=\"workspace is stuck in provisioning\"", + } { + if !strings.Contains(output, want) { + t.Fatalf("output missing %q:\n%s", want, output) + } + } +} + +func captureStdoutForProviderMSPRecoverTest(t *testing.T, buf *bytes.Buffer) func() { + t.Helper() + old := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("os.Pipe: %v", err) + } + os.Stdout = w + return func() { + if err := w.Close(); err != nil { + t.Fatalf("close stdout writer: %v", err) + } + if _, err := io.Copy(buf, r); err != nil { + t.Fatalf("copy stdout: %v", err) + } + if err := r.Close(); err != nil { + t.Fatalf("close stdout reader: %v", err) + } + os.Stdout = old + } +} diff --git a/deploy/provider-msp/.env.example b/deploy/provider-msp/.env.example index 6e5ba6c64..a3fb19a15 100644 --- a/deploy/provider-msp/.env.example +++ b/deploy/provider-msp/.env.example @@ -43,6 +43,12 @@ PULSE_EMAIL_REPLY_TO=support@example.com # Non-mutating operational status: # docker compose run --rm control-plane provider-msp status # +# Plan recovery for failed, stuck provisioning, or unhealthy client workspaces: +# docker compose run --rm control-plane provider-msp recover --all-degraded --dry-run +# +# Execute recovery after reviewing the dry-run: +# docker compose run --rm control-plane provider-msp recover --all-degraded +# # Recovery backup before upgrades or recovery drills: # docker compose run --rm control-plane provider-msp backup create # diff --git a/docs/release-control/v6/internal/subsystems/cloud-paid.md b/docs/release-control/v6/internal/subsystems/cloud-paid.md index e17bfffc3..66d31bc76 100644 --- a/docs/release-control/v6/internal/subsystems/cloud-paid.md +++ b/docs/release-control/v6/internal/subsystems/cloud-paid.md @@ -29,6 +29,10 @@ Provider-hosted MSP status uses the same tenant registry, health summary, and stuck-provisioning threshold as the control-plane cleanup loop, so operator readiness reports and automated failure handling cannot drift into separate definitions of a failed provider workspace. +Provider-hosted MSP recovery uses the same license-backed provider identity, +workspace registry, tenant data, and canonical tenant-runtime rollout path as +the rest of the control plane so failed, stuck, or unhealthy client workspaces +are repaired without introducing a second provisioning model. Provider-hosted MSP backup, verification, and restore are part of the cloud-paid operator contract because the recovery artifact must preserve the provider's license-backed plan identity, control-plane account/workspace registry, and @@ -117,6 +121,7 @@ tenant-local runtime state without depending on Stripe billing surfaces. 89. `internal/cloudcp/public_msp_signup_handlers.go` 90. `internal/cloudcp/provider_msp_bootstrap.go` 91. `internal/cloudcp/provider_msp_backup.go` +92. `internal/cloudcp/provider_msp_recovery.go` ## Shared Boundaries @@ -193,7 +198,12 @@ tenant-local runtime state without depending on Stripe billing surfaces. output, restore must recover that license as an explicit operator artifact, and the archive must stay Stripe-free so provider-hosted MSP recovery does not inherit Pulse-hosted SaaS billing assumptions. -11. `internal/cloudcp/tenant_runtime_rollout.go` shared with `deployment-installability`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary. +11. `internal/cloudcp/provider_msp_recovery.go` shared with `deployment-installability`: provider-hosted MSP failed-workspace recovery is both a cloud-paid license/account/runtime continuity boundary and a deployment-installability recovery artifact boundary. + Provider-hosted MSP recovery must require the signed provider MSP license + source by default, preserve the client workspace boundary, refuse to start + from empty tenant data, and mark a workspace active only after the canonical + tenant-runtime rollout path has produced a healthy runtime. +12. `internal/cloudcp/tenant_runtime_rollout.go` shared with `deployment-installability`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary. Hosted tenant runtime reconciliation must treat a registered tenant with preserved tenant data but no live Docker runtime as a recoverable managed state, not as a terminal skip. The control-plane-owned reconcile path must diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 75d219b89..da0594fb5 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -30,11 +30,13 @@ surfaces. 7. `cmd/pulse-control-plane/provider_msp_backup.go` 8. `cmd/pulse-control-plane/provider_msp_preflight.go` 9. `cmd/pulse-control-plane/provider_msp_proof.go` -10. `cmd/pulse-control-plane/provider_msp_status.go` -11. `internal/cloudcp/provider_msp_backup.go` -12. `internal/cloudcp/docker/manager.go` -13. `internal/cloudcp/docker/labels.go` -14. `internal/cloudcp/tenant_runtime_rollout.go` +10. `cmd/pulse-control-plane/provider_msp_recover.go` +11. `cmd/pulse-control-plane/provider_msp_status.go` +12. `internal/cloudcp/provider_msp_backup.go` +13. `internal/cloudcp/provider_msp_recovery.go` +14. `internal/cloudcp/docker/manager.go` +15. `internal/cloudcp/docker/labels.go` +16. `internal/cloudcp/tenant_runtime_rollout.go` 13. `.github/workflows/create-release.yml` 14. `.github/workflows/deploy-demo-server.yml` 15. `.github/workflows/helm-pages.yml` @@ -150,7 +152,13 @@ surfaces. snapshot, license artifact, and tenant runtime directories, and fail closed on restore when target provider MSP state already exists unless the operator explicitly uses the replace gate after stopping the control plane. -6. `internal/cloudcp/tenant_runtime_rollout.go` shared with `cloud-paid`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary. +6. `internal/cloudcp/provider_msp_recovery.go` shared with `cloud-paid`: provider-hosted MSP failed-workspace recovery is both a cloud-paid license/account/runtime continuity boundary and a deployment-installability recovery artifact boundary. + `pulse-control-plane provider-msp recover` must offer a dry-run plan and an + explicit execution path for failed, stuck provisioning, and unhealthy active + client workspaces; it must require the signed provider MSP license source by + default, refuse to recover from missing tenant data, and reuse the canonical + tenant-runtime rollout path before marking the workspace active again. +7. `internal/cloudcp/tenant_runtime_rollout.go` shared with `cloud-paid`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary. 7. `scripts/install.ps1` shared with `agent-lifecycle`: the Windows installer is both a deployment installability entry point and a canonical agent lifecycle runtime continuity boundary. It must expose a non-mutating preflight for the exact Windows agent architecture before Administrator-only install changes, accept token-file diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json index 43257345e..095ea2f07 100644 --- a/docs/release-control/v6/internal/subsystems/registry.json +++ b/docs/release-control/v6/internal/subsystems/registry.json @@ -642,6 +642,14 @@ "deployment-installability" ] }, + { + "path": "internal/cloudcp/provider_msp_recovery.go", + "rationale": "provider-hosted MSP failed-workspace recovery is both a cloud-paid license/account/runtime continuity boundary and a deployment-installability recovery artifact boundary", + "subsystems": [ + "cloud-paid", + "deployment-installability" + ] + }, { "path": "internal/cloudcp/tenant_runtime_rollout.go", "rationale": "hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary", @@ -2635,6 +2643,7 @@ "cmd/pulse-control-plane/provider_msp_backup.go", "cmd/pulse-control-plane/provider_msp_preflight.go", "cmd/pulse-control-plane/provider_msp_proof.go", + "cmd/pulse-control-plane/provider_msp_recover.go", "cmd/pulse-control-plane/provider_msp_status.go", "docker-compose.yml", "Dockerfile", @@ -2654,6 +2663,7 @@ "internal/cloudcp/docker/labels.go", "internal/cloudcp/docker/manager.go", "internal/cloudcp/provider_msp_backup.go", + "internal/cloudcp/provider_msp_recovery.go", "internal/cloudcp/tenant_runtime_rollout.go", "Makefile", "package-lock.json", @@ -2906,10 +2916,12 @@ "cmd/pulse-control-plane/provider_msp_backup.go", "cmd/pulse-control-plane/provider_msp_preflight.go", "cmd/pulse-control-plane/provider_msp_proof.go", + "cmd/pulse-control-plane/provider_msp_recover.go", "cmd/pulse-control-plane/provider_msp_status.go", "internal/cloudcp/docker/labels.go", "internal/cloudcp/docker/manager.go", "internal/cloudcp/provider_msp_backup.go", + "internal/cloudcp/provider_msp_recovery.go", "internal/cloudcp/tenant_runtime_rollout.go" ], "allow_same_subsystem_tests": false, @@ -2918,9 +2930,11 @@ "cmd/pulse-control-plane/provider_msp_backup_test.go", "cmd/pulse-control-plane/provider_msp_preflight_test.go", "cmd/pulse-control-plane/provider_msp_proof_test.go", + "cmd/pulse-control-plane/provider_msp_recover_test.go", "cmd/pulse-control-plane/provider_msp_status_test.go", "internal/cloudcp/docker/manager_test.go", "internal/cloudcp/provider_msp_backup_test.go", + "internal/cloudcp/provider_msp_recovery_test.go", "internal/cloudcp/tenant_runtime_rollout_test.go", "scripts/installtests/provider_msp_deploy_test.go" ] diff --git a/internal/cloudcp/provider_msp_recovery.go b/internal/cloudcp/provider_msp_recovery.go new file mode 100644 index 000000000..86c8dd2b1 --- /dev/null +++ b/internal/cloudcp/provider_msp_recovery.go @@ -0,0 +1,359 @@ +package cloudcp + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry" + runtimeconfig "github.com/rcourtman/pulse-go-rewrite/internal/config" + "github.com/rcourtman/pulse-go-rewrite/pkg/cloudauth" + pkglicensing "github.com/rcourtman/pulse-go-rewrite/pkg/licensing" +) + +const ( + providerMSPRecoveryActionRecover = "recover" + providerMSPRecoveryActionSkip = "skip" +) + +// ProviderMSPRecoveryOptions controls failed/degraded workspace recovery for a +// provider-hosted MSP control plane. +type ProviderMSPRecoveryOptions struct { + TenantIDs []string + AllDegraded bool + DryRun bool + AllowEnvPlan bool + Image string + RunID string + SnapshotRoot string + HealthTimeout time.Duration + PrunePrevious bool +} + +// ProviderMSPRecoveryItem is the operator-visible recovery decision for one +// client workspace. +type ProviderMSPRecoveryItem struct { + TenantID string + DisplayName string + State string + Action string + Reason string + Recovered bool + StuckProvisioning bool + PreviousContainerID string + ActiveContainerID string + ActiveImageRef string + ActiveImageID string + RestoredMissing bool + ReconciledOnly bool + Error string +} + +// ProviderMSPRecoveryReport describes the dry-run plan or executed recovery +// outcome. +type ProviderMSPRecoveryReport struct { + OK bool + DryRun bool + PlanVersion string + PlanSource string + LicenseID string + LicenseEmail string + WorkspaceLimit int + Items []ProviderMSPRecoveryItem + RecoverCount int + RecoveredCount int + SkippedCount int + ErrorCount int +} + +type providerMSPRecoveryDependencies struct { + OpenRegistry func(*CPConfig) (*registry.TenantRegistry, error) + RolloutTenantRuntime func(context.Context, *CPConfig, TenantRuntimeRolloutOptions) (*TenantRuntimeRolloutResult, error) + Now func() time.Time +} + +// RecoverProviderMSPWorkspaces plans or executes recovery for failed, stuck, or +// unhealthy provider MSP client workspaces. +func RecoverProviderMSPWorkspaces(ctx context.Context, cfg *CPConfig, opts ProviderMSPRecoveryOptions) (*ProviderMSPRecoveryReport, error) { + return recoverProviderMSPWorkspacesWithDependencies(ctx, cfg, opts, providerMSPRecoveryDependencies{}) +} + +func recoverProviderMSPWorkspacesWithDependencies(ctx context.Context, cfg *CPConfig, opts ProviderMSPRecoveryOptions, deps providerMSPRecoveryDependencies) (*ProviderMSPRecoveryReport, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if cfg == nil { + return nil, fmt.Errorf("control plane config is required") + } + opts = normalizeProviderMSPRecoveryOptions(opts) + if err := validateProviderMSPRecoveryOptions(opts); err != nil { + return nil, err + } + if err := validateProviderMSPRecoveryConfig(cfg, opts); err != nil { + return nil, err + } + deps = normalizeProviderMSPRecoveryDependencies(deps) + + workspaceLimit, _ := pkglicensing.WorkspaceLimitForPlan(cfg.ProviderMSPPlanVersion) + report := &ProviderMSPRecoveryReport{ + OK: true, + DryRun: opts.DryRun, + PlanVersion: strings.TrimSpace(cfg.ProviderMSPPlanVersion), + PlanSource: providerMSPPlanSourceOrDefault(cfg.ProviderMSPPlanSource), + LicenseID: strings.TrimSpace(cfg.ProviderMSPLicenseID), + LicenseEmail: strings.ToLower(strings.TrimSpace(cfg.ProviderMSPLicenseEmail)), + WorkspaceLimit: workspaceLimit, + } + + reg, err := deps.OpenRegistry(cfg) + if err != nil { + return nil, fmt.Errorf("open tenant registry: %w", err) + } + defer reg.Close() + + tenants, err := selectProviderMSPRecoveryTenants(reg, opts) + if err != nil { + return nil, err + } + now := deps.Now() + for _, tenant := range tenants { + item := planProviderMSPRecoveryItem(tenant, now) + if item.Action == providerMSPRecoveryActionRecover { + report.RecoverCount++ + } else { + report.SkippedCount++ + } + report.Items = append(report.Items, item) + } + if opts.DryRun { + return report, nil + } + + for idx := range report.Items { + item := &report.Items[idx] + if item.Action != providerMSPRecoveryActionRecover { + continue + } + if err := validateProviderMSPRecoveryTenantData(cfg, item.TenantID); err != nil { + item.Error = err.Error() + report.OK = false + report.ErrorCount++ + continue + } + + result, err := deps.RolloutTenantRuntime(ctx, cfg, TenantRuntimeRolloutOptions{ + TenantID: item.TenantID, + Image: opts.Image, + RunID: opts.RunID, + SnapshotRoot: opts.SnapshotRoot, + HealthTimeout: opts.HealthTimeout, + PrunePrevious: opts.PrunePrevious, + }) + if err != nil { + item.Error = err.Error() + report.OK = false + report.ErrorCount++ + continue + } + if result == nil || strings.TrimSpace(result.ActiveContainerID) == "" { + item.Error = "tenant runtime recovery did not return an active container" + report.OK = false + report.ErrorCount++ + continue + } + if err := markProviderMSPRecoveryTenantActive(reg, item.TenantID, deps.Now()); err != nil { + item.Error = err.Error() + report.OK = false + report.ErrorCount++ + continue + } + + item.Recovered = true + item.PreviousContainerID = result.PreviousContainerID + item.ActiveContainerID = result.ActiveContainerID + item.ActiveImageRef = result.ActiveImageRef + item.ActiveImageID = result.ActiveImageID + item.RestoredMissing = result.RestoredMissing + item.ReconciledOnly = result.ReconciledOnly + report.RecoveredCount++ + } + if report.ErrorCount > 0 { + return report, fmt.Errorf("provider MSP recovery failed for %d workspace(s)", report.ErrorCount) + } + return report, nil +} + +func normalizeProviderMSPRecoveryOptions(opts ProviderMSPRecoveryOptions) ProviderMSPRecoveryOptions { + opts.TenantIDs = dedupeProviderMSPRecoveryTenantIDs(opts.TenantIDs) + opts.Image = strings.TrimSpace(opts.Image) + opts.RunID = strings.TrimSpace(opts.RunID) + opts.SnapshotRoot = strings.TrimSpace(opts.SnapshotRoot) + return opts +} + +func validateProviderMSPRecoveryOptions(opts ProviderMSPRecoveryOptions) error { + if opts.AllDegraded && len(opts.TenantIDs) > 0 { + return fmt.Errorf("choose either --all-degraded or one or more --tenant-id values") + } + if !opts.AllDegraded && len(opts.TenantIDs) == 0 { + return fmt.Errorf("choose --all-degraded or at least one --tenant-id") + } + return nil +} + +func validateProviderMSPRecoveryConfig(cfg *CPConfig, opts ProviderMSPRecoveryOptions) error { + if !cfg.IsProviderHostedMSP() { + return fmt.Errorf("provider MSP recovery requires CP_CONTROL_PLANE_MODE=%s", ControlPlaneModeProviderHostedMSP) + } + if cfg.UsesStripeBilling() { + return fmt.Errorf("provider-hosted MSP recovery must be Stripe-free") + } + if _, known := pkglicensing.WorkspaceLimitForPlan(cfg.ProviderMSPPlanVersion); !known { + return fmt.Errorf("provider MSP plan %q has no known workspace limit", cfg.ProviderMSPPlanVersion) + } + if !opts.AllowEnvPlan && strings.TrimSpace(cfg.ProviderMSPPlanSource) != ProviderMSPPlanSourceLicenseFile { + return fmt.Errorf("provider MSP recovery requires %s plan source; rerun with --allow-env-plan only for local development", ProviderMSPPlanSourceLicenseFile) + } + return nil +} + +func normalizeProviderMSPRecoveryDependencies(deps providerMSPRecoveryDependencies) providerMSPRecoveryDependencies { + if deps.OpenRegistry == nil { + deps.OpenRegistry = func(cfg *CPConfig) (*registry.TenantRegistry, error) { + return registry.NewTenantRegistry(cfg.ControlPlaneDir()) + } + } + if deps.RolloutTenantRuntime == nil { + deps.RolloutTenantRuntime = RolloutTenantRuntime + } + if deps.Now == nil { + deps.Now = func() time.Time { return time.Now().UTC() } + } + return deps +} + +func selectProviderMSPRecoveryTenants(reg *registry.TenantRegistry, opts ProviderMSPRecoveryOptions) ([]*registry.Tenant, error) { + if opts.AllDegraded { + tenants, err := reg.List() + if err != nil { + return nil, fmt.Errorf("list tenants: %w", err) + } + return tenants, nil + } + + tenants := make([]*registry.Tenant, 0, len(opts.TenantIDs)) + for _, tenantID := range opts.TenantIDs { + tenant, err := reg.Get(tenantID) + if err != nil { + return nil, fmt.Errorf("load tenant %s: %w", tenantID, err) + } + if tenant == nil { + tenants = append(tenants, ®istry.Tenant{ID: tenantID}) + continue + } + tenants = append(tenants, tenant) + } + return tenants, nil +} + +func planProviderMSPRecoveryItem(tenant *registry.Tenant, now time.Time) ProviderMSPRecoveryItem { + if tenant == nil { + return ProviderMSPRecoveryItem{Action: providerMSPRecoveryActionSkip, Reason: "workspace is missing"} + } + item := ProviderMSPRecoveryItem{ + TenantID: strings.TrimSpace(tenant.ID), + DisplayName: strings.TrimSpace(tenant.DisplayName), + State: string(tenant.State), + Action: providerMSPRecoveryActionSkip, + } + if item.TenantID == "" { + item.Reason = "workspace id is missing" + return item + } + switch tenant.State { + case registry.TenantStateFailed: + item.Action = providerMSPRecoveryActionRecover + item.Reason = "workspace is failed" + case registry.TenantStateProvisioning: + if now.IsZero() { + now = time.Now().UTC() + } + if !tenant.CreatedAt.IsZero() && tenant.CreatedAt.UTC().After(now.UTC().Add(-provisioningTimeout)) { + item.Reason = "workspace is still within the provisioning timeout" + return item + } + item.Action = providerMSPRecoveryActionRecover + item.Reason = "workspace is stuck in provisioning" + item.StuckProvisioning = true + case registry.TenantStateActive: + if tenant.HealthCheckOK { + item.Reason = "workspace is active and healthy" + return item + } + item.Action = providerMSPRecoveryActionRecover + item.Reason = "workspace health check is failing" + default: + item.Reason = fmt.Sprintf("workspace state %q is not recoverable by provider MSP recovery", tenant.State) + } + return item +} + +func validateProviderMSPRecoveryTenantData(cfg *CPConfig, tenantID string) error { + tenantID = strings.TrimSpace(tenantID) + if tenantID == "" || strings.ContainsAny(tenantID, `/\`) { + return fmt.Errorf("unsafe tenant id %q", tenantID) + } + tenantDataDir := filepath.Join(cfg.TenantsDir(), tenantID) + if err := requireDirectory(tenantDataDir, "provider MSP tenant data dir"); err != nil { + return err + } + if err := requireRegularFile(filepath.Join(tenantDataDir, "secrets", "handoff.key"), "provider MSP tenant handoff key"); err != nil { + return err + } + if err := requireRegularFile(filepath.Join(tenantDataDir, cloudauth.HandoffKeyFile), "provider MSP tenant cloud handoff key"); err != nil { + return err + } + if !runtimeconfig.NewMultiTenantPersistence(tenantDataDir).OrgExists(tenantID) { + return fmt.Errorf("provider MSP tenant %s organization metadata is missing", tenantID) + } + return nil +} + +func markProviderMSPRecoveryTenantActive(reg *registry.TenantRegistry, tenantID string, now time.Time) error { + tenant, err := reg.Get(tenantID) + if err != nil { + return fmt.Errorf("reload recovered workspace %s: %w", tenantID, err) + } + if tenant == nil { + return fmt.Errorf("recovered workspace %s disappeared from registry", tenantID) + } + if now.IsZero() { + now = time.Now().UTC() + } + tenant.State = registry.TenantStateActive + tenant.HealthCheckOK = true + tenant.LastHealthCheck = &now + if err := reg.Update(tenant); err != nil { + return fmt.Errorf("mark recovered workspace %s active: %w", tenantID, err) + } + return nil +} + +func dedupeProviderMSPRecoveryTenantIDs(values []string) []string { + result := make([]string, 0, len(values)) + seen := map[string]struct{}{} + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + result = append(result, value) + } + return result +} diff --git a/internal/cloudcp/provider_msp_recovery_test.go b/internal/cloudcp/provider_msp_recovery_test.go new file mode 100644 index 000000000..d4ee957e6 --- /dev/null +++ b/internal/cloudcp/provider_msp_recovery_test.go @@ -0,0 +1,199 @@ +package cloudcp + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry" + runtimeconfig "github.com/rcourtman/pulse-go-rewrite/internal/config" + "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rcourtman/pulse-go-rewrite/pkg/cloudauth" +) + +func TestProviderMSPRecoveryDryRunPlansDegradedWorkspaces(t *testing.T) { + cfg := testProviderMSPBackupConfig(t) + now := time.Date(2026, 6, 2, 13, 0, 0, 0, time.UTC) + reg := seedProviderMSPRecoveryTenants(t, cfg, now) + reg.Close() + + report, err := recoverProviderMSPWorkspacesWithDependencies(context.Background(), cfg, ProviderMSPRecoveryOptions{ + AllDegraded: true, + DryRun: true, + }, providerMSPRecoveryDependencies{ + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatalf("RecoverProviderMSPWorkspaces dry-run: %v", err) + } + if !report.OK || !report.DryRun { + t.Fatalf("report status = ok %t dry-run %t", report.OK, report.DryRun) + } + if report.RecoverCount != 3 || report.SkippedCount != 2 { + t.Fatalf("counts = recover %d skipped %d", report.RecoverCount, report.SkippedCount) + } + items := providerMSPRecoveryItemsByTenant(report.Items) + for tenantID, wantReason := range map[string]string{ + "t-FAILED": "workspace is failed", + "t-STUCK": "workspace is stuck in provisioning", + "t-UNHEALTHY": "workspace health check is failing", + } { + item := items[tenantID] + if item.Action != providerMSPRecoveryActionRecover || item.Reason != wantReason { + t.Fatalf("%s item = %#v, want recover reason %q", tenantID, item, wantReason) + } + } + if items["t-RECENT"].Action != providerMSPRecoveryActionSkip || !strings.Contains(items["t-RECENT"].Reason, "provisioning timeout") { + t.Fatalf("recent provisioning item = %#v", items["t-RECENT"]) + } + if items["t-HEALTHY"].Action != providerMSPRecoveryActionSkip || !strings.Contains(items["t-HEALTHY"].Reason, "healthy") { + t.Fatalf("healthy item = %#v", items["t-HEALTHY"]) + } +} + +func TestProviderMSPRecoveryReactivatesRecoveredWorkspace(t *testing.T) { + cfg := testProviderMSPBackupConfig(t) + now := time.Date(2026, 6, 2, 13, 0, 0, 0, time.UTC) + reg := seedProviderMSPRecoveryTenants(t, cfg, now) + reg.Close() + writeProviderMSPRecoveryTenantData(t, cfg, "t-FAILED") + + var rolloutTenantID string + report, err := recoverProviderMSPWorkspacesWithDependencies(context.Background(), cfg, ProviderMSPRecoveryOptions{ + TenantIDs: []string{"t-FAILED"}, + RunID: "recovery-test", + }, providerMSPRecoveryDependencies{ + Now: func() time.Time { return now }, + RolloutTenantRuntime: func(_ context.Context, _ *CPConfig, opts TenantRuntimeRolloutOptions) (*TenantRuntimeRolloutResult, error) { + rolloutTenantID = opts.TenantID + return &TenantRuntimeRolloutResult{ + TenantID: opts.TenantID, + ActiveContainerID: "container-recovered", + ActiveImageRef: "pulse:test", + ActiveImageID: "sha256:recovered", + RestoredMissing: true, + }, nil + }, + }) + if err != nil { + t.Fatalf("RecoverProviderMSPWorkspaces: %v", err) + } + if !report.OK || report.RecoveredCount != 1 || report.ErrorCount != 0 { + t.Fatalf("report = %#v", report) + } + if rolloutTenantID != "t-FAILED" { + t.Fatalf("rollout tenant id = %q, want t-FAILED", rolloutTenantID) + } + item := report.Items[0] + if !item.Recovered || item.ActiveContainerID != "container-recovered" || !item.RestoredMissing { + t.Fatalf("recovery item = %#v", item) + } + + reloaded := getProviderMSPRecoveryTenant(t, cfg, "t-FAILED") + if reloaded.State != registry.TenantStateActive || !reloaded.HealthCheckOK || reloaded.LastHealthCheck == nil { + t.Fatalf("reloaded tenant = %#v", reloaded) + } +} + +func TestProviderMSPRecoveryRefusesMissingTenantData(t *testing.T) { + cfg := testProviderMSPBackupConfig(t) + now := time.Date(2026, 6, 2, 13, 0, 0, 0, time.UTC) + reg := seedProviderMSPRecoveryTenants(t, cfg, now) + reg.Close() + + rolloutCalled := false + report, err := recoverProviderMSPWorkspacesWithDependencies(context.Background(), cfg, ProviderMSPRecoveryOptions{ + TenantIDs: []string{"t-FAILED"}, + }, providerMSPRecoveryDependencies{ + Now: func() time.Time { return now }, + RolloutTenantRuntime: func(context.Context, *CPConfig, TenantRuntimeRolloutOptions) (*TenantRuntimeRolloutResult, error) { + rolloutCalled = true + return nil, nil + }, + }) + if err == nil { + t.Fatal("expected recovery to fail when tenant data is missing") + } + if rolloutCalled { + t.Fatal("rollout should not be called without tenant data") + } + if report == nil || report.OK || report.ErrorCount != 1 { + t.Fatalf("report = %#v", report) + } + if !strings.Contains(report.Items[0].Error, "provider MSP tenant data dir unavailable") { + t.Fatalf("item error = %q", report.Items[0].Error) + } +} + +func seedProviderMSPRecoveryTenants(t *testing.T, cfg *CPConfig, now time.Time) *registry.TenantRegistry { + t.Helper() + reg, err := registry.NewTenantRegistry(cfg.ControlPlaneDir()) + if err != nil { + t.Fatalf("NewTenantRegistry: %v", err) + } + tenants := []*registry.Tenant{ + {ID: "t-FAILED", DisplayName: "Failed", State: registry.TenantStateFailed, CreatedAt: now.Add(-time.Hour), HealthCheckOK: false}, + {ID: "t-STUCK", DisplayName: "Stuck", State: registry.TenantStateProvisioning, CreatedAt: now.Add(-time.Hour), HealthCheckOK: false}, + {ID: "t-RECENT", DisplayName: "Recent", State: registry.TenantStateProvisioning, CreatedAt: now.Add(-time.Minute), HealthCheckOK: false}, + {ID: "t-UNHEALTHY", DisplayName: "Unhealthy", State: registry.TenantStateActive, CreatedAt: now.Add(-time.Hour), HealthCheckOK: false}, + {ID: "t-HEALTHY", DisplayName: "Healthy", State: registry.TenantStateActive, CreatedAt: now.Add(-time.Hour), HealthCheckOK: true}, + } + for _, tenant := range tenants { + if err := reg.Create(tenant); err != nil { + t.Fatalf("Create(%s): %v", tenant.ID, err) + } + } + return reg +} + +func writeProviderMSPRecoveryTenantData(t *testing.T, cfg *CPConfig, tenantID string) { + t.Helper() + tenantDataDir := filepath.Join(cfg.TenantsDir(), tenantID) + secretsDir := filepath.Join(tenantDataDir, "secrets") + if err := os.MkdirAll(secretsDir, 0o700); err != nil { + t.Fatalf("create tenant secrets dir: %v", err) + } + if err := os.WriteFile(filepath.Join(secretsDir, "handoff.key"), []byte("handoff-key"), 0o600); err != nil { + t.Fatalf("write handoff key: %v", err) + } + if err := os.WriteFile(filepath.Join(tenantDataDir, cloudauth.HandoffKeyFile), []byte("cloud-handoff-key"), 0o600); err != nil { + t.Fatalf("write cloud handoff key: %v", err) + } + org := &models.Organization{ + ID: tenantID, + DisplayName: tenantID, + Status: models.OrgStatusActive, + CreatedAt: time.Now().UTC(), + } + if err := runtimeconfig.NewMultiTenantPersistence(tenantDataDir).SaveOrganization(org); err != nil { + t.Fatalf("save tenant organization: %v", err) + } +} + +func getProviderMSPRecoveryTenant(t *testing.T, cfg *CPConfig, tenantID string) *registry.Tenant { + t.Helper() + reg, err := registry.NewTenantRegistry(cfg.ControlPlaneDir()) + if err != nil { + t.Fatalf("NewTenantRegistry: %v", err) + } + defer reg.Close() + tenant, err := reg.Get(tenantID) + if err != nil { + t.Fatalf("Get(%s): %v", tenantID, err) + } + if tenant == nil { + t.Fatalf("tenant %s missing", tenantID) + } + return tenant +} + +func providerMSPRecoveryItemsByTenant(items []ProviderMSPRecoveryItem) map[string]ProviderMSPRecoveryItem { + result := make(map[string]ProviderMSPRecoveryItem, len(items)) + for _, item := range items { + result[item.TenantID] = item + } + return result +} diff --git a/scripts/installtests/provider_msp_deploy_test.go b/scripts/installtests/provider_msp_deploy_test.go index 204357218..e6b35b07b 100644 --- a/scripts/installtests/provider_msp_deploy_test.go +++ b/scripts/installtests/provider_msp_deploy_test.go @@ -49,6 +49,8 @@ func TestProviderMSPDeployEnvExampleMatchesBootstrapPath(t *testing.T) { "docker compose run --rm control-plane provider-msp bootstrap", "docker compose run --rm control-plane provider-msp preflight", "docker compose run --rm control-plane provider-msp status", + "docker compose run --rm control-plane provider-msp recover --all-degraded --dry-run", + "docker compose run --rm control-plane provider-msp recover --all-degraded", "docker compose run --rm control-plane provider-msp backup create", "docker compose run --rm control-plane provider-msp backup verify", "docker compose run --rm control-plane provider-msp backup restore", diff --git a/scripts/release_control/subsystem_lookup_test.py b/scripts/release_control/subsystem_lookup_test.py index 40570daca..9b25f4ff3 100644 --- a/scripts/release_control/subsystem_lookup_test.py +++ b/scripts/release_control/subsystem_lookup_test.py @@ -3408,9 +3408,11 @@ class SubsystemLookupTest(unittest.TestCase): "cmd/pulse-control-plane/provider_msp_backup_test.go", "cmd/pulse-control-plane/provider_msp_preflight_test.go", "cmd/pulse-control-plane/provider_msp_proof_test.go", + "cmd/pulse-control-plane/provider_msp_recover_test.go", "cmd/pulse-control-plane/provider_msp_status_test.go", "internal/cloudcp/docker/manager_test.go", "internal/cloudcp/provider_msp_backup_test.go", + "internal/cloudcp/provider_msp_recovery_test.go", "internal/cloudcp/tenant_runtime_rollout_test.go", "scripts/installtests/provider_msp_deploy_test.go", ],