Add provider MSP workspace recovery

2026-09-10 02:25:56 +00:00 · 2026-06-02 14:28:16 +01:00
parent a2e860dc8c
commit e5a7300aff
11 changed files with 794 additions and 7 deletions
@@ -16,6 +16,7 @@ func newProviderMSPCmd() *cobra.Command {
 	cmd.AddCommand(newProviderMSPBackupCmd())
 	cmd.AddCommand(newProviderMSPPreflightCmd())
 	cmd.AddCommand(newProviderMSPProofCmd())
+	cmd.AddCommand(newProviderMSPRecoverCmd())
 	cmd.AddCommand(newProviderMSPStatusCmd())
 	return cmd
 }
@@ -0,0 +1,94 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp"
+	"github.com/spf13/cobra"
+)
+
+func newProviderMSPRecoverCmd() *cobra.Command {
+	var opts cloudcp.ProviderMSPRecoveryOptions
+
+	cmd := &cobra.Command{
+		Use:   "recover",
+		Short: "Recover failed or degraded provider-hosted MSP client workspaces",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cfg, err := cloudcp.LoadConfig()
+			if err != nil {
+				return fmt.Errorf("load control plane config: %w", err)
+			}
+			report, err := cloudcp.RecoverProviderMSPWorkspaces(cmd.Context(), cfg, opts)
+			printProviderMSPRecoveryReport(report)
+			return err
+		},
+	}
+
+	cmd.Flags().StringArrayVar(&opts.TenantIDs, "tenant-id", nil, "Client workspace tenant ID to recover (repeatable)")
+	cmd.Flags().BoolVar(&opts.AllDegraded, "all-degraded", false, "Recover all failed, stuck provisioning, or unhealthy active client workspaces")
+	cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false, "Print the recovery plan without mutating tenant runtimes")
+	cmd.Flags().BoolVar(&opts.AllowEnvPlan, "allow-env-plan", false, "Allow CP_PROVIDER_MSP_PLAN_VERSION fallback instead of a signed provider MSP license file for local development")
+	cmd.Flags().StringVar(&opts.Image, "image", "", "Tenant runtime image to use during recovery (default: CP_PULSE_IMAGE)")
+	cmd.Flags().StringVar(&opts.RunID, "run-id", "", "Operator-visible recovery run identifier")
+	cmd.Flags().StringVar(&opts.SnapshotRoot, "snapshot-root", "", "Override tenant snapshot root (default: <CP_DATA_DIR>/backups/rollout)")
+	cmd.Flags().DurationVar(&opts.HealthTimeout, "health-timeout", 90*time.Second, "How long to wait for the recovered runtime to become healthy")
+	cmd.Flags().BoolVar(&opts.PrunePrevious, "prune-previous", false, "Remove the preserved pre-recovery container after success")
+	return cmd
+}
+
+func printProviderMSPRecoveryReport(report *cloudcp.ProviderMSPRecoveryReport) {
+	if report == nil {
+		fmt.Println("provider_msp_recovery_ok=false")
+		return
+	}
+	fmt.Printf("provider_msp_recovery_ok=%t\n", report.OK)
+	fmt.Printf("dry_run=%t\n", report.DryRun)
+	fmt.Printf("plan_version=%s\n", report.PlanVersion)
+	fmt.Printf("plan_source=%s\n", report.PlanSource)
+	fmt.Printf("license_id=%s\n", report.LicenseID)
+	fmt.Printf("license_email=%s\n", report.LicenseEmail)
+	fmt.Printf("workspace_limit=%d\n", report.WorkspaceLimit)
+	fmt.Printf("recover_count=%d\n", report.RecoverCount)
+	fmt.Printf("recovered_count=%d\n", report.RecoveredCount)
+	fmt.Printf("skipped_count=%d\n", report.SkippedCount)
+	fmt.Printf("error_count=%d\n", report.ErrorCount)
+	for _, item := range report.Items {
+		fields := []string{
+			"workspace=" + item.TenantID,
+			"display_name=" + quoteProviderMSPRecoveryField(item.DisplayName),
+			"state=" + item.State,
+			"action=" + item.Action,
+			"reason=" + quoteProviderMSPRecoveryField(item.Reason),
+			fmt.Sprintf("stuck_provisioning=%t", item.StuckProvisioning),
+			fmt.Sprintf("recovered=%t", item.Recovered),
+		}
+		if item.PreviousContainerID != "" {
+			fields = append(fields, "previous_container_id="+item.PreviousContainerID)
+		}
+		if item.ActiveContainerID != "" {
+			fields = append(fields, "active_container_id="+item.ActiveContainerID)
+		}
+		if item.ActiveImageRef != "" {
+			fields = append(fields, "active_image_ref="+item.ActiveImageRef)
+		}
+		if item.ActiveImageID != "" {
+			fields = append(fields, "active_image_id="+item.ActiveImageID)
+		}
+		if item.RestoredMissing {
+			fields = append(fields, "restored_missing=true")
+		}
+		if item.ReconciledOnly {
+			fields = append(fields, "reconciled_only=true")
+		}
+		if item.Error != "" {
+			fields = append(fields, "error="+quoteProviderMSPRecoveryField(item.Error))
+		}
+		fmt.Println(strings.Join(fields, " "))
+	}
+}
+
+func quoteProviderMSPRecoveryField(value string) string {
+	return fmt.Sprintf("%q", strings.TrimSpace(value))
+}
@@ -0,0 +1,92 @@
+package main
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp"
+)
+
+func TestProviderMSPCommandExposesRecover(t *testing.T) {
+	cmd := newProviderMSPCmd()
+	for _, child := range cmd.Commands() {
+		if child.Name() == "recover" {
+			return
+		}
+	}
+	t.Fatal("provider-msp recover command is not registered")
+}
+
+func TestPrintProviderMSPRecoveryReport(t *testing.T) {
+	var buf bytes.Buffer
+	stdout := captureStdoutForProviderMSPRecoverTest(t, &buf)
+	printProviderMSPRecoveryReport(&cloudcp.ProviderMSPRecoveryReport{
+		OK:             true,
+		DryRun:         true,
+		PlanVersion:    "msp_growth",
+		PlanSource:     cloudcp.ProviderMSPPlanSourceLicenseFile,
+		LicenseID:      "lic_test",
+		LicenseEmail:   "provider@example.com",
+		WorkspaceLimit: 15,
+		RecoverCount:   1,
+		SkippedCount:   1,
+		Items: []cloudcp.ProviderMSPRecoveryItem{
+			{
+				TenantID:          "t-STUCK",
+				DisplayName:       "Client A",
+				State:             "provisioning",
+				Action:            "recover",
+				Reason:            "workspace is stuck in provisioning",
+				StuckProvisioning: true,
+			},
+			{
+				TenantID:    "t-HEALTHY",
+				DisplayName: "Client B",
+				State:       "active",
+				Action:      "skip",
+				Reason:      "workspace is active and healthy",
+			},
+		},
+	})
+	stdout()
+
+	output := buf.String()
+	for _, want := range []string{
+		"provider_msp_recovery_ok=true",
+		"dry_run=true",
+		"recover_count=1",
+		"skipped_count=1",
+		"workspace=t-STUCK",
+		"stuck_provisioning=true",
+		"reason=\"workspace is stuck in provisioning\"",
+	} {
+		if !strings.Contains(output, want) {
+			t.Fatalf("output missing %q:\n%s", want, output)
+		}
+	}
+}
+
+func captureStdoutForProviderMSPRecoverTest(t *testing.T, buf *bytes.Buffer) func() {
+	t.Helper()
+	old := os.Stdout
+	r, w, err := os.Pipe()
+	if err != nil {
+		t.Fatalf("os.Pipe: %v", err)
+	}
+	os.Stdout = w
+	return func() {
+		if err := w.Close(); err != nil {
+			t.Fatalf("close stdout writer: %v", err)
+		}
+		if _, err := io.Copy(buf, r); err != nil {
+			t.Fatalf("copy stdout: %v", err)
+		}
+		if err := r.Close(); err != nil {
+			t.Fatalf("close stdout reader: %v", err)
+		}
+		os.Stdout = old
+	}
+}
@@ -43,6 +43,12 @@ PULSE_EMAIL_REPLY_TO=support@example.com
 # Non-mutating operational status:
 # docker compose run --rm control-plane provider-msp status
 #
+# Plan recovery for failed, stuck provisioning, or unhealthy client workspaces:
+# docker compose run --rm control-plane provider-msp recover --all-degraded --dry-run
+#
+# Execute recovery after reviewing the dry-run:
+# docker compose run --rm control-plane provider-msp recover --all-degraded
+#
 # Recovery backup before upgrades or recovery drills:
 # docker compose run --rm control-plane provider-msp backup create
 #
@@ -29,6 +29,10 @@ Provider-hosted MSP status uses the same tenant registry, health summary, and
 stuck-provisioning threshold as the control-plane cleanup loop, so operator
 readiness reports and automated failure handling cannot drift into separate
 definitions of a failed provider workspace.
+Provider-hosted MSP recovery uses the same license-backed provider identity,
+workspace registry, tenant data, and canonical tenant-runtime rollout path as
+the rest of the control plane so failed, stuck, or unhealthy client workspaces
+are repaired without introducing a second provisioning model.
 Provider-hosted MSP backup, verification, and restore are part of the cloud-paid
 operator contract because the recovery artifact must preserve the provider's
 license-backed plan identity, control-plane account/workspace registry, and
@@ -117,6 +121,7 @@ tenant-local runtime state without depending on Stripe billing surfaces.
 89. `internal/cloudcp/public_msp_signup_handlers.go`
 90. `internal/cloudcp/provider_msp_bootstrap.go`
 91. `internal/cloudcp/provider_msp_backup.go`
+92. `internal/cloudcp/provider_msp_recovery.go`

 ## Shared Boundaries

@@ -193,7 +198,12 @@ tenant-local runtime state without depending on Stripe billing surfaces.
    output, restore must recover that license as an explicit operator artifact,
    and the archive must stay Stripe-free so provider-hosted MSP recovery does
    not inherit Pulse-hosted SaaS billing assumptions.
-11. `internal/cloudcp/tenant_runtime_rollout.go` shared with `deployment-installability`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary.
+11. `internal/cloudcp/provider_msp_recovery.go` shared with `deployment-installability`: provider-hosted MSP failed-workspace recovery is both a cloud-paid license/account/runtime continuity boundary and a deployment-installability recovery artifact boundary.
+    Provider-hosted MSP recovery must require the signed provider MSP license
+    source by default, preserve the client workspace boundary, refuse to start
+    from empty tenant data, and mark a workspace active only after the canonical
+    tenant-runtime rollout path has produced a healthy runtime.
+12. `internal/cloudcp/tenant_runtime_rollout.go` shared with `deployment-installability`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary.
    Hosted tenant runtime reconciliation must treat a registered tenant with
    preserved tenant data but no live Docker runtime as a recoverable managed
    state, not as a terminal skip. The control-plane-owned reconcile path must
@@ -30,11 +30,13 @@ surfaces.
 7. `cmd/pulse-control-plane/provider_msp_backup.go`
 8. `cmd/pulse-control-plane/provider_msp_preflight.go`
 9. `cmd/pulse-control-plane/provider_msp_proof.go`
-10. `cmd/pulse-control-plane/provider_msp_status.go`
-11. `internal/cloudcp/provider_msp_backup.go`
-12. `internal/cloudcp/docker/manager.go`
-13. `internal/cloudcp/docker/labels.go`
-14. `internal/cloudcp/tenant_runtime_rollout.go`
+10. `cmd/pulse-control-plane/provider_msp_recover.go`
+11. `cmd/pulse-control-plane/provider_msp_status.go`
+12. `internal/cloudcp/provider_msp_backup.go`
+13. `internal/cloudcp/provider_msp_recovery.go`
+14. `internal/cloudcp/docker/manager.go`
+15. `internal/cloudcp/docker/labels.go`
+16. `internal/cloudcp/tenant_runtime_rollout.go`
 13. `.github/workflows/create-release.yml`
 14. `.github/workflows/deploy-demo-server.yml`
 15. `.github/workflows/helm-pages.yml`
@@ -150,7 +152,13 @@ surfaces.
   snapshot, license artifact, and tenant runtime directories, and fail closed
   on restore when target provider MSP state already exists unless the operator
   explicitly uses the replace gate after stopping the control plane.
-6. `internal/cloudcp/tenant_runtime_rollout.go` shared with `cloud-paid`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary.
+6. `internal/cloudcp/provider_msp_recovery.go` shared with `cloud-paid`: provider-hosted MSP failed-workspace recovery is both a cloud-paid license/account/runtime continuity boundary and a deployment-installability recovery artifact boundary.
+   `pulse-control-plane provider-msp recover` must offer a dry-run plan and an
+   explicit execution path for failed, stuck provisioning, and unhealthy active
+   client workspaces; it must require the signed provider MSP license source by
+   default, refuse to recover from missing tenant data, and reuse the canonical
+   tenant-runtime rollout path before marking the workspace active again.
+7. `internal/cloudcp/tenant_runtime_rollout.go` shared with `cloud-paid`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary.
 7. `scripts/install.ps1` shared with `agent-lifecycle`: the Windows installer is both a deployment installability entry point and a canonical agent lifecycle runtime continuity boundary.
   It must expose a non-mutating preflight for the exact Windows agent
   architecture before Administrator-only install changes, accept token-file
@@ -642,6 +642,14 @@
        "deployment-installability"
      ]
    },
+    {
+      "path": "internal/cloudcp/provider_msp_recovery.go",
+      "rationale": "provider-hosted MSP failed-workspace recovery is both a cloud-paid license/account/runtime continuity boundary and a deployment-installability recovery artifact boundary",
+      "subsystems": [
+        "cloud-paid",
+        "deployment-installability"
+      ]
+    },
    {
      "path": "internal/cloudcp/tenant_runtime_rollout.go",
      "rationale": "hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary",
@@ -2635,6 +2643,7 @@
        "cmd/pulse-control-plane/provider_msp_backup.go",
        "cmd/pulse-control-plane/provider_msp_preflight.go",
        "cmd/pulse-control-plane/provider_msp_proof.go",
+        "cmd/pulse-control-plane/provider_msp_recover.go",
        "cmd/pulse-control-plane/provider_msp_status.go",
        "docker-compose.yml",
        "Dockerfile",
@@ -2654,6 +2663,7 @@
        "internal/cloudcp/docker/labels.go",
        "internal/cloudcp/docker/manager.go",
        "internal/cloudcp/provider_msp_backup.go",
+        "internal/cloudcp/provider_msp_recovery.go",
        "internal/cloudcp/tenant_runtime_rollout.go",
        "Makefile",
        "package-lock.json",
@@ -2906,10 +2916,12 @@
              "cmd/pulse-control-plane/provider_msp_backup.go",
              "cmd/pulse-control-plane/provider_msp_preflight.go",
              "cmd/pulse-control-plane/provider_msp_proof.go",
+              "cmd/pulse-control-plane/provider_msp_recover.go",
              "cmd/pulse-control-plane/provider_msp_status.go",
              "internal/cloudcp/docker/labels.go",
              "internal/cloudcp/docker/manager.go",
              "internal/cloudcp/provider_msp_backup.go",
+              "internal/cloudcp/provider_msp_recovery.go",
              "internal/cloudcp/tenant_runtime_rollout.go"
            ],
            "allow_same_subsystem_tests": false,
@@ -2918,9 +2930,11 @@
              "cmd/pulse-control-plane/provider_msp_backup_test.go",
              "cmd/pulse-control-plane/provider_msp_preflight_test.go",
              "cmd/pulse-control-plane/provider_msp_proof_test.go",
+              "cmd/pulse-control-plane/provider_msp_recover_test.go",
              "cmd/pulse-control-plane/provider_msp_status_test.go",
              "internal/cloudcp/docker/manager_test.go",
              "internal/cloudcp/provider_msp_backup_test.go",
+              "internal/cloudcp/provider_msp_recovery_test.go",
              "internal/cloudcp/tenant_runtime_rollout_test.go",
              "scripts/installtests/provider_msp_deploy_test.go"
            ]
@@ -0,0 +1,359 @@
+package cloudcp
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry"
+	runtimeconfig "github.com/rcourtman/pulse-go-rewrite/internal/config"
+	"github.com/rcourtman/pulse-go-rewrite/pkg/cloudauth"
+	pkglicensing "github.com/rcourtman/pulse-go-rewrite/pkg/licensing"
+)
+
+const (
+	providerMSPRecoveryActionRecover = "recover"
+	providerMSPRecoveryActionSkip    = "skip"
+)
+
+// ProviderMSPRecoveryOptions controls failed/degraded workspace recovery for a
+// provider-hosted MSP control plane.
+type ProviderMSPRecoveryOptions struct {
+	TenantIDs     []string
+	AllDegraded   bool
+	DryRun        bool
+	AllowEnvPlan  bool
+	Image         string
+	RunID         string
+	SnapshotRoot  string
+	HealthTimeout time.Duration
+	PrunePrevious bool
+}
+
+// ProviderMSPRecoveryItem is the operator-visible recovery decision for one
+// client workspace.
+type ProviderMSPRecoveryItem struct {
+	TenantID            string
+	DisplayName         string
+	State               string
+	Action              string
+	Reason              string
+	Recovered           bool
+	StuckProvisioning   bool
+	PreviousContainerID string
+	ActiveContainerID   string
+	ActiveImageRef      string
+	ActiveImageID       string
+	RestoredMissing     bool
+	ReconciledOnly      bool
+	Error               string
+}
+
+// ProviderMSPRecoveryReport describes the dry-run plan or executed recovery
+// outcome.
+type ProviderMSPRecoveryReport struct {
+	OK             bool
+	DryRun         bool
+	PlanVersion    string
+	PlanSource     string
+	LicenseID      string
+	LicenseEmail   string
+	WorkspaceLimit int
+	Items          []ProviderMSPRecoveryItem
+	RecoverCount   int
+	RecoveredCount int
+	SkippedCount   int
+	ErrorCount     int
+}
+
+type providerMSPRecoveryDependencies struct {
+	OpenRegistry         func(*CPConfig) (*registry.TenantRegistry, error)
+	RolloutTenantRuntime func(context.Context, *CPConfig, TenantRuntimeRolloutOptions) (*TenantRuntimeRolloutResult, error)
+	Now                  func() time.Time
+}
+
+// RecoverProviderMSPWorkspaces plans or executes recovery for failed, stuck, or
+// unhealthy provider MSP client workspaces.
+func RecoverProviderMSPWorkspaces(ctx context.Context, cfg *CPConfig, opts ProviderMSPRecoveryOptions) (*ProviderMSPRecoveryReport, error) {
+	return recoverProviderMSPWorkspacesWithDependencies(ctx, cfg, opts, providerMSPRecoveryDependencies{})
+}
+
+func recoverProviderMSPWorkspacesWithDependencies(ctx context.Context, cfg *CPConfig, opts ProviderMSPRecoveryOptions, deps providerMSPRecoveryDependencies) (*ProviderMSPRecoveryReport, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	if cfg == nil {
+		return nil, fmt.Errorf("control plane config is required")
+	}
+	opts = normalizeProviderMSPRecoveryOptions(opts)
+	if err := validateProviderMSPRecoveryOptions(opts); err != nil {
+		return nil, err
+	}
+	if err := validateProviderMSPRecoveryConfig(cfg, opts); err != nil {
+		return nil, err
+	}
+	deps = normalizeProviderMSPRecoveryDependencies(deps)
+
+	workspaceLimit, _ := pkglicensing.WorkspaceLimitForPlan(cfg.ProviderMSPPlanVersion)
+	report := &ProviderMSPRecoveryReport{
+		OK:             true,
+		DryRun:         opts.DryRun,
+		PlanVersion:    strings.TrimSpace(cfg.ProviderMSPPlanVersion),
+		PlanSource:     providerMSPPlanSourceOrDefault(cfg.ProviderMSPPlanSource),
+		LicenseID:      strings.TrimSpace(cfg.ProviderMSPLicenseID),
+		LicenseEmail:   strings.ToLower(strings.TrimSpace(cfg.ProviderMSPLicenseEmail)),
+		WorkspaceLimit: workspaceLimit,
+	}
+
+	reg, err := deps.OpenRegistry(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("open tenant registry: %w", err)
+	}
+	defer reg.Close()
+
+	tenants, err := selectProviderMSPRecoveryTenants(reg, opts)
+	if err != nil {
+		return nil, err
+	}
+	now := deps.Now()
+	for _, tenant := range tenants {
+		item := planProviderMSPRecoveryItem(tenant, now)
+		if item.Action == providerMSPRecoveryActionRecover {
+			report.RecoverCount++
+		} else {
+			report.SkippedCount++
+		}
+		report.Items = append(report.Items, item)
+	}
+	if opts.DryRun {
+		return report, nil
+	}
+
+	for idx := range report.Items {
+		item := &report.Items[idx]
+		if item.Action != providerMSPRecoveryActionRecover {
+			continue
+		}
+		if err := validateProviderMSPRecoveryTenantData(cfg, item.TenantID); err != nil {
+			item.Error = err.Error()
+			report.OK = false
+			report.ErrorCount++
+			continue
+		}
+
+		result, err := deps.RolloutTenantRuntime(ctx, cfg, TenantRuntimeRolloutOptions{
+			TenantID:      item.TenantID,
+			Image:         opts.Image,
+			RunID:         opts.RunID,
+			SnapshotRoot:  opts.SnapshotRoot,
+			HealthTimeout: opts.HealthTimeout,
+			PrunePrevious: opts.PrunePrevious,
+		})
+		if err != nil {
+			item.Error = err.Error()
+			report.OK = false
+			report.ErrorCount++
+			continue
+		}
+		if result == nil || strings.TrimSpace(result.ActiveContainerID) == "" {
+			item.Error = "tenant runtime recovery did not return an active container"
+			report.OK = false
+			report.ErrorCount++
+			continue
+		}
+		if err := markProviderMSPRecoveryTenantActive(reg, item.TenantID, deps.Now()); err != nil {
+			item.Error = err.Error()
+			report.OK = false
+			report.ErrorCount++
+			continue
+		}
+
+		item.Recovered = true
+		item.PreviousContainerID = result.PreviousContainerID
+		item.ActiveContainerID = result.ActiveContainerID
+		item.ActiveImageRef = result.ActiveImageRef
+		item.ActiveImageID = result.ActiveImageID
+		item.RestoredMissing = result.RestoredMissing
+		item.ReconciledOnly = result.ReconciledOnly
+		report.RecoveredCount++
+	}
+	if report.ErrorCount > 0 {
+		return report, fmt.Errorf("provider MSP recovery failed for %d workspace(s)", report.ErrorCount)
+	}
+	return report, nil
+}
+
+func normalizeProviderMSPRecoveryOptions(opts ProviderMSPRecoveryOptions) ProviderMSPRecoveryOptions {
+	opts.TenantIDs = dedupeProviderMSPRecoveryTenantIDs(opts.TenantIDs)
+	opts.Image = strings.TrimSpace(opts.Image)
+	opts.RunID = strings.TrimSpace(opts.RunID)
+	opts.SnapshotRoot = strings.TrimSpace(opts.SnapshotRoot)
+	return opts
+}
+
+func validateProviderMSPRecoveryOptions(opts ProviderMSPRecoveryOptions) error {
+	if opts.AllDegraded && len(opts.TenantIDs) > 0 {
+		return fmt.Errorf("choose either --all-degraded or one or more --tenant-id values")
+	}
+	if !opts.AllDegraded && len(opts.TenantIDs) == 0 {
+		return fmt.Errorf("choose --all-degraded or at least one --tenant-id")
+	}
+	return nil
+}
+
+func validateProviderMSPRecoveryConfig(cfg *CPConfig, opts ProviderMSPRecoveryOptions) error {
+	if !cfg.IsProviderHostedMSP() {
+		return fmt.Errorf("provider MSP recovery requires CP_CONTROL_PLANE_MODE=%s", ControlPlaneModeProviderHostedMSP)
+	}
+	if cfg.UsesStripeBilling() {
+		return fmt.Errorf("provider-hosted MSP recovery must be Stripe-free")
+	}
+	if _, known := pkglicensing.WorkspaceLimitForPlan(cfg.ProviderMSPPlanVersion); !known {
+		return fmt.Errorf("provider MSP plan %q has no known workspace limit", cfg.ProviderMSPPlanVersion)
+	}
+	if !opts.AllowEnvPlan && strings.TrimSpace(cfg.ProviderMSPPlanSource) != ProviderMSPPlanSourceLicenseFile {
+		return fmt.Errorf("provider MSP recovery requires %s plan source; rerun with --allow-env-plan only for local development", ProviderMSPPlanSourceLicenseFile)
+	}
+	return nil
+}
+
+func normalizeProviderMSPRecoveryDependencies(deps providerMSPRecoveryDependencies) providerMSPRecoveryDependencies {
+	if deps.OpenRegistry == nil {
+		deps.OpenRegistry = func(cfg *CPConfig) (*registry.TenantRegistry, error) {
+			return registry.NewTenantRegistry(cfg.ControlPlaneDir())
+		}
+	}
+	if deps.RolloutTenantRuntime == nil {
+		deps.RolloutTenantRuntime = RolloutTenantRuntime
+	}
+	if deps.Now == nil {
+		deps.Now = func() time.Time { return time.Now().UTC() }
+	}
+	return deps
+}
+
+func selectProviderMSPRecoveryTenants(reg *registry.TenantRegistry, opts ProviderMSPRecoveryOptions) ([]*registry.Tenant, error) {
+	if opts.AllDegraded {
+		tenants, err := reg.List()
+		if err != nil {
+			return nil, fmt.Errorf("list tenants: %w", err)
+		}
+		return tenants, nil
+	}
+
+	tenants := make([]*registry.Tenant, 0, len(opts.TenantIDs))
+	for _, tenantID := range opts.TenantIDs {
+		tenant, err := reg.Get(tenantID)
+		if err != nil {
+			return nil, fmt.Errorf("load tenant %s: %w", tenantID, err)
+		}
+		if tenant == nil {
+			tenants = append(tenants, &registry.Tenant{ID: tenantID})
+			continue
+		}
+		tenants = append(tenants, tenant)
+	}
+	return tenants, nil
+}
+
+func planProviderMSPRecoveryItem(tenant *registry.Tenant, now time.Time) ProviderMSPRecoveryItem {
+	if tenant == nil {
+		return ProviderMSPRecoveryItem{Action: providerMSPRecoveryActionSkip, Reason: "workspace is missing"}
+	}
+	item := ProviderMSPRecoveryItem{
+		TenantID:    strings.TrimSpace(tenant.ID),
+		DisplayName: strings.TrimSpace(tenant.DisplayName),
+		State:       string(tenant.State),
+		Action:      providerMSPRecoveryActionSkip,
+	}
+	if item.TenantID == "" {
+		item.Reason = "workspace id is missing"
+		return item
+	}
+	switch tenant.State {
+	case registry.TenantStateFailed:
+		item.Action = providerMSPRecoveryActionRecover
+		item.Reason = "workspace is failed"
+	case registry.TenantStateProvisioning:
+		if now.IsZero() {
+			now = time.Now().UTC()
+		}
+		if !tenant.CreatedAt.IsZero() && tenant.CreatedAt.UTC().After(now.UTC().Add(-provisioningTimeout)) {
+			item.Reason = "workspace is still within the provisioning timeout"
+			return item
+		}
+		item.Action = providerMSPRecoveryActionRecover
+		item.Reason = "workspace is stuck in provisioning"
+		item.StuckProvisioning = true
+	case registry.TenantStateActive:
+		if tenant.HealthCheckOK {
+			item.Reason = "workspace is active and healthy"
+			return item
+		}
+		item.Action = providerMSPRecoveryActionRecover
+		item.Reason = "workspace health check is failing"
+	default:
+		item.Reason = fmt.Sprintf("workspace state %q is not recoverable by provider MSP recovery", tenant.State)
+	}
+	return item
+}
+
+func validateProviderMSPRecoveryTenantData(cfg *CPConfig, tenantID string) error {
+	tenantID = strings.TrimSpace(tenantID)
+	if tenantID == "" || strings.ContainsAny(tenantID, `/\`) {
+		return fmt.Errorf("unsafe tenant id %q", tenantID)
+	}
+	tenantDataDir := filepath.Join(cfg.TenantsDir(), tenantID)
+	if err := requireDirectory(tenantDataDir, "provider MSP tenant data dir"); err != nil {
+		return err
+	}
+	if err := requireRegularFile(filepath.Join(tenantDataDir, "secrets", "handoff.key"), "provider MSP tenant handoff key"); err != nil {
+		return err
+	}
+	if err := requireRegularFile(filepath.Join(tenantDataDir, cloudauth.HandoffKeyFile), "provider MSP tenant cloud handoff key"); err != nil {
+		return err
+	}
+	if !runtimeconfig.NewMultiTenantPersistence(tenantDataDir).OrgExists(tenantID) {
+		return fmt.Errorf("provider MSP tenant %s organization metadata is missing", tenantID)
+	}
+	return nil
+}
+
+func markProviderMSPRecoveryTenantActive(reg *registry.TenantRegistry, tenantID string, now time.Time) error {
+	tenant, err := reg.Get(tenantID)
+	if err != nil {
+		return fmt.Errorf("reload recovered workspace %s: %w", tenantID, err)
+	}
+	if tenant == nil {
+		return fmt.Errorf("recovered workspace %s disappeared from registry", tenantID)
+	}
+	if now.IsZero() {
+		now = time.Now().UTC()
+	}
+	tenant.State = registry.TenantStateActive
+	tenant.HealthCheckOK = true
+	tenant.LastHealthCheck = &now
+	if err := reg.Update(tenant); err != nil {
+		return fmt.Errorf("mark recovered workspace %s active: %w", tenantID, err)
+	}
+	return nil
+}
+
+func dedupeProviderMSPRecoveryTenantIDs(values []string) []string {
+	result := make([]string, 0, len(values))
+	seen := map[string]struct{}{}
+	for _, value := range values {
+		value = strings.TrimSpace(value)
+		if value == "" {
+			continue
+		}
+		if _, ok := seen[value]; ok {
+			continue
+		}
+		seen[value] = struct{}{}
+		result = append(result, value)
+	}
+	return result
+}
@@ -0,0 +1,199 @@
+package cloudcp
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry"
+	runtimeconfig "github.com/rcourtman/pulse-go-rewrite/internal/config"
+	"github.com/rcourtman/pulse-go-rewrite/internal/models"
+	"github.com/rcourtman/pulse-go-rewrite/pkg/cloudauth"
+)
+
+func TestProviderMSPRecoveryDryRunPlansDegradedWorkspaces(t *testing.T) {
+	cfg := testProviderMSPBackupConfig(t)
+	now := time.Date(2026, 6, 2, 13, 0, 0, 0, time.UTC)
+	reg := seedProviderMSPRecoveryTenants(t, cfg, now)
+	reg.Close()
+
+	report, err := recoverProviderMSPWorkspacesWithDependencies(context.Background(), cfg, ProviderMSPRecoveryOptions{
+		AllDegraded: true,
+		DryRun:      true,
+	}, providerMSPRecoveryDependencies{
+		Now: func() time.Time { return now },
+	})
+	if err != nil {
+		t.Fatalf("RecoverProviderMSPWorkspaces dry-run: %v", err)
+	}
+	if !report.OK || !report.DryRun {
+		t.Fatalf("report status = ok %t dry-run %t", report.OK, report.DryRun)
+	}
+	if report.RecoverCount != 3 || report.SkippedCount != 2 {
+		t.Fatalf("counts = recover %d skipped %d", report.RecoverCount, report.SkippedCount)
+	}
+	items := providerMSPRecoveryItemsByTenant(report.Items)
+	for tenantID, wantReason := range map[string]string{
+		"t-FAILED":    "workspace is failed",
+		"t-STUCK":     "workspace is stuck in provisioning",
+		"t-UNHEALTHY": "workspace health check is failing",
+	} {
+		item := items[tenantID]
+		if item.Action != providerMSPRecoveryActionRecover || item.Reason != wantReason {
+			t.Fatalf("%s item = %#v, want recover reason %q", tenantID, item, wantReason)
+		}
+	}
+	if items["t-RECENT"].Action != providerMSPRecoveryActionSkip || !strings.Contains(items["t-RECENT"].Reason, "provisioning timeout") {
+		t.Fatalf("recent provisioning item = %#v", items["t-RECENT"])
+	}
+	if items["t-HEALTHY"].Action != providerMSPRecoveryActionSkip || !strings.Contains(items["t-HEALTHY"].Reason, "healthy") {
+		t.Fatalf("healthy item = %#v", items["t-HEALTHY"])
+	}
+}
+
+func TestProviderMSPRecoveryReactivatesRecoveredWorkspace(t *testing.T) {
+	cfg := testProviderMSPBackupConfig(t)
+	now := time.Date(2026, 6, 2, 13, 0, 0, 0, time.UTC)
+	reg := seedProviderMSPRecoveryTenants(t, cfg, now)
+	reg.Close()
+	writeProviderMSPRecoveryTenantData(t, cfg, "t-FAILED")
+
+	var rolloutTenantID string
+	report, err := recoverProviderMSPWorkspacesWithDependencies(context.Background(), cfg, ProviderMSPRecoveryOptions{
+		TenantIDs: []string{"t-FAILED"},
+		RunID:     "recovery-test",
+	}, providerMSPRecoveryDependencies{
+		Now: func() time.Time { return now },
+		RolloutTenantRuntime: func(_ context.Context, _ *CPConfig, opts TenantRuntimeRolloutOptions) (*TenantRuntimeRolloutResult, error) {
+			rolloutTenantID = opts.TenantID
+			return &TenantRuntimeRolloutResult{
+				TenantID:          opts.TenantID,
+				ActiveContainerID: "container-recovered",
+				ActiveImageRef:    "pulse:test",
+				ActiveImageID:     "sha256:recovered",
+				RestoredMissing:   true,
+			}, nil
+		},
+	})
+	if err != nil {
+		t.Fatalf("RecoverProviderMSPWorkspaces: %v", err)
+	}
+	if !report.OK || report.RecoveredCount != 1 || report.ErrorCount != 0 {
+		t.Fatalf("report = %#v", report)
+	}
+	if rolloutTenantID != "t-FAILED" {
+		t.Fatalf("rollout tenant id = %q, want t-FAILED", rolloutTenantID)
+	}
+	item := report.Items[0]
+	if !item.Recovered || item.ActiveContainerID != "container-recovered" || !item.RestoredMissing {
+		t.Fatalf("recovery item = %#v", item)
+	}
+
+	reloaded := getProviderMSPRecoveryTenant(t, cfg, "t-FAILED")
+	if reloaded.State != registry.TenantStateActive || !reloaded.HealthCheckOK || reloaded.LastHealthCheck == nil {
+		t.Fatalf("reloaded tenant = %#v", reloaded)
+	}
+}
+
+func TestProviderMSPRecoveryRefusesMissingTenantData(t *testing.T) {
+	cfg := testProviderMSPBackupConfig(t)
+	now := time.Date(2026, 6, 2, 13, 0, 0, 0, time.UTC)
+	reg := seedProviderMSPRecoveryTenants(t, cfg, now)
+	reg.Close()
+
+	rolloutCalled := false
+	report, err := recoverProviderMSPWorkspacesWithDependencies(context.Background(), cfg, ProviderMSPRecoveryOptions{
+		TenantIDs: []string{"t-FAILED"},
+	}, providerMSPRecoveryDependencies{
+		Now: func() time.Time { return now },
+		RolloutTenantRuntime: func(context.Context, *CPConfig, TenantRuntimeRolloutOptions) (*TenantRuntimeRolloutResult, error) {
+			rolloutCalled = true
+			return nil, nil
+		},
+	})
+	if err == nil {
+		t.Fatal("expected recovery to fail when tenant data is missing")
+	}
+	if rolloutCalled {
+		t.Fatal("rollout should not be called without tenant data")
+	}
+	if report == nil || report.OK || report.ErrorCount != 1 {
+		t.Fatalf("report = %#v", report)
+	}
+	if !strings.Contains(report.Items[0].Error, "provider MSP tenant data dir unavailable") {
+		t.Fatalf("item error = %q", report.Items[0].Error)
+	}
+}
+
+func seedProviderMSPRecoveryTenants(t *testing.T, cfg *CPConfig, now time.Time) *registry.TenantRegistry {
+	t.Helper()
+	reg, err := registry.NewTenantRegistry(cfg.ControlPlaneDir())
+	if err != nil {
+		t.Fatalf("NewTenantRegistry: %v", err)
+	}
+	tenants := []*registry.Tenant{
+		{ID: "t-FAILED", DisplayName: "Failed", State: registry.TenantStateFailed, CreatedAt: now.Add(-time.Hour), HealthCheckOK: false},
+		{ID: "t-STUCK", DisplayName: "Stuck", State: registry.TenantStateProvisioning, CreatedAt: now.Add(-time.Hour), HealthCheckOK: false},
+		{ID: "t-RECENT", DisplayName: "Recent", State: registry.TenantStateProvisioning, CreatedAt: now.Add(-time.Minute), HealthCheckOK: false},
+		{ID: "t-UNHEALTHY", DisplayName: "Unhealthy", State: registry.TenantStateActive, CreatedAt: now.Add(-time.Hour), HealthCheckOK: false},
+		{ID: "t-HEALTHY", DisplayName: "Healthy", State: registry.TenantStateActive, CreatedAt: now.Add(-time.Hour), HealthCheckOK: true},
+	}
+	for _, tenant := range tenants {
+		if err := reg.Create(tenant); err != nil {
+			t.Fatalf("Create(%s): %v", tenant.ID, err)
+		}
+	}
+	return reg
+}
+
+func writeProviderMSPRecoveryTenantData(t *testing.T, cfg *CPConfig, tenantID string) {
+	t.Helper()
+	tenantDataDir := filepath.Join(cfg.TenantsDir(), tenantID)
+	secretsDir := filepath.Join(tenantDataDir, "secrets")
+	if err := os.MkdirAll(secretsDir, 0o700); err != nil {
+		t.Fatalf("create tenant secrets dir: %v", err)
+	}
+	if err := os.WriteFile(filepath.Join(secretsDir, "handoff.key"), []byte("handoff-key"), 0o600); err != nil {
+		t.Fatalf("write handoff key: %v", err)
+	}
+	if err := os.WriteFile(filepath.Join(tenantDataDir, cloudauth.HandoffKeyFile), []byte("cloud-handoff-key"), 0o600); err != nil {
+		t.Fatalf("write cloud handoff key: %v", err)
+	}
+	org := &models.Organization{
+		ID:          tenantID,
+		DisplayName: tenantID,
+		Status:      models.OrgStatusActive,
+		CreatedAt:   time.Now().UTC(),
+	}
+	if err := runtimeconfig.NewMultiTenantPersistence(tenantDataDir).SaveOrganization(org); err != nil {
+		t.Fatalf("save tenant organization: %v", err)
+	}
+}
+
+func getProviderMSPRecoveryTenant(t *testing.T, cfg *CPConfig, tenantID string) *registry.Tenant {
+	t.Helper()
+	reg, err := registry.NewTenantRegistry(cfg.ControlPlaneDir())
+	if err != nil {
+		t.Fatalf("NewTenantRegistry: %v", err)
+	}
+	defer reg.Close()
+	tenant, err := reg.Get(tenantID)
+	if err != nil {
+		t.Fatalf("Get(%s): %v", tenantID, err)
+	}
+	if tenant == nil {
+		t.Fatalf("tenant %s missing", tenantID)
+	}
+	return tenant
+}
+
+func providerMSPRecoveryItemsByTenant(items []ProviderMSPRecoveryItem) map[string]ProviderMSPRecoveryItem {
+	result := make(map[string]ProviderMSPRecoveryItem, len(items))
+	for _, item := range items {
+		result[item.TenantID] = item
+	}
+	return result
+}
@@ -49,6 +49,8 @@ func TestProviderMSPDeployEnvExampleMatchesBootstrapPath(t *testing.T) {
 		"docker compose run --rm control-plane provider-msp bootstrap",
 		"docker compose run --rm control-plane provider-msp preflight",
 		"docker compose run --rm control-plane provider-msp status",
+		"docker compose run --rm control-plane provider-msp recover --all-degraded --dry-run",
+		"docker compose run --rm control-plane provider-msp recover --all-degraded",
 		"docker compose run --rm control-plane provider-msp backup create",
 		"docker compose run --rm control-plane provider-msp backup verify",
 		"docker compose run --rm control-plane provider-msp backup restore",
@@ -3408,9 +3408,11 @@ class SubsystemLookupTest(unittest.TestCase):
                "cmd/pulse-control-plane/provider_msp_backup_test.go",
                "cmd/pulse-control-plane/provider_msp_preflight_test.go",
                "cmd/pulse-control-plane/provider_msp_proof_test.go",
+                "cmd/pulse-control-plane/provider_msp_recover_test.go",
                "cmd/pulse-control-plane/provider_msp_status_test.go",
                "internal/cloudcp/docker/manager_test.go",
                "internal/cloudcp/provider_msp_backup_test.go",
+                "internal/cloudcp/provider_msp_recovery_test.go",
                "internal/cloudcp/tenant_runtime_rollout_test.go",
                "scripts/installtests/provider_msp_deploy_test.go",
            ],