From 9dbaaa7efeb5068c8348af4cf50802b69405760b Mon Sep 17 00:00:00 2001 From: rcourtman Date: Fri, 24 Apr 2026 00:09:52 +0100 Subject: [PATCH] Recover missing hosted tenant runtimes --- cmd/pulse-control-plane/main.go | 3 + .../v6/internal/subsystems/cloud-paid.md | 6 ++ .../subsystems/deployment-installability.md | 5 ++ internal/cloudcp/tenant_runtime_rollout.go | 82 ++++++++++++++++++- .../cloudcp/tenant_runtime_rollout_test.go | 64 ++++++++++++++- 5 files changed, 155 insertions(+), 5 deletions(-) diff --git a/cmd/pulse-control-plane/main.go b/cmd/pulse-control-plane/main.go index 8a20616b2..cbd76a22e 100644 --- a/cmd/pulse-control-plane/main.go +++ b/cmd/pulse-control-plane/main.go @@ -172,6 +172,9 @@ func newTenantRuntimeReconcileCmd() *cobra.Command { result.ActiveImageID, result.ReconciledOnly, ) + if result.RestoredMissing { + fmt.Printf("restored_missing=%t\n", result.RestoredMissing) + } if result.PreviousContainerID != "" { fmt.Printf("previous_container_id=%s\n", result.PreviousContainerID) } diff --git a/docs/release-control/v6/internal/subsystems/cloud-paid.md b/docs/release-control/v6/internal/subsystems/cloud-paid.md index a90c03c35..3753b028b 100644 --- a/docs/release-control/v6/internal/subsystems/cloud-paid.md +++ b/docs/release-control/v6/internal/subsystems/cloud-paid.md @@ -141,6 +141,12 @@ cloud-specific enforcement rules. through the control-plane Docker manager so tenant runtime logging cannot fill the live Pulse Cloud host independently of tenant data quotas. 10. `internal/cloudcp/tenant_runtime_rollout.go` shared with `deployment-installability`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary. + Hosted tenant runtime reconciliation must treat a registered tenant with + preserved tenant data but no live Docker runtime as a recoverable managed + state, not as a terminal skip. The control-plane-owned reconcile path must + recreate the canonical tenant container, health-check it, and persist the + new runtime identity before hosted billing/auth surfaces are considered + coherent. The real `pulse-pro` license-server legacy checkout issuance, recurring renewals, manual issue, and legacy exchange flows are part of that same diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 0a37fb40d..300420212 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -192,6 +192,11 @@ server-side update execution surfaces. `cmd/pulse-control-plane/main.go`, `internal/cloudcp/docker/manager.go`, `internal/cloudcp/docker/labels.go`, and `internal/cloudcp/tenant_runtime_rollout.go` + The batch reconcile command must be restorative as well as corrective: + when a tenant registry row and tenant data remain but the canonical or + recorded Docker container is missing, dry-run must classify the tenant for + mutation and the live command must recreate the container, prove health, and + rewrite the registry runtime identity through the same control-plane path. 10. Add or change the canonical hosted staging smoke operator path through `scripts/run_hosted_staging_smoke.sh`, `tests/integration/scripts/bootstrap-hosted-mobile-onboarding.mjs`, `tests/integration/scripts/hosted-mobile-token-runtime.mjs`, `tests/integration/scripts/hosted-tenant-runtime.mjs`, and `tests/integration/scripts/relay-mobile-token-helper.go` ## Forbidden Paths diff --git a/internal/cloudcp/tenant_runtime_rollout.go b/internal/cloudcp/tenant_runtime_rollout.go index b21de3e23..2f2605cbf 100644 --- a/internal/cloudcp/tenant_runtime_rollout.go +++ b/internal/cloudcp/tenant_runtime_rollout.go @@ -43,6 +43,7 @@ type TenantRuntimeRolloutResult struct { ActiveImageID string BackupContainerName string ReconciledOnly bool + RestoredMissing bool } type TenantRuntimeContractReconcilePlanOptions struct { @@ -98,6 +99,7 @@ type tenantRuntimeRolloutService struct { registry tenantRuntimeRolloutRegistry docker tenantRuntimeRolloutDocker tenantsDir string + defaultImage string synchronizer tenantRuntimeRolloutSynchronizer now func() time.Time sleep func(time.Duration) @@ -105,6 +107,8 @@ type tenantRuntimeRolloutService struct { healthPoll time.Duration } +var errTenantRuntimeMissing = errors.New("tenant runtime container missing") + // RolloutTenantRuntime executes the canonical hosted tenant runtime rollout // path using the control plane's registry and Docker manager. func RolloutTenantRuntime(ctx context.Context, cfg *CPConfig, opts TenantRuntimeRolloutOptions) (*TenantRuntimeRolloutResult, error) { @@ -183,6 +187,7 @@ func newTenantRuntimeRolloutServiceFromConfig( registry: reg, docker: dockerMgr, tenantsDir: cfg.TenantsDir(), + defaultImage: strings.TrimSpace(image), synchronizer: filesystemTenantRuntimeSynchronizer{}, now: func() time.Time { return time.Now().UTC() }, sleep: time.Sleep, @@ -203,6 +208,9 @@ func (s *tenantRuntimeRolloutService) Rollout(ctx context.Context, opts TenantRu return nil, fmt.Errorf("tenant id is required") } image := strings.TrimSpace(opts.Image) + if image == "" { + image = strings.TrimSpace(s.defaultImage) + } if image == "" { return nil, fmt.Errorf("image is required") } @@ -227,6 +235,9 @@ func (s *tenantRuntimeRolloutService) Rollout(ctx context.Context, opts TenantRu live, err := s.resolveLiveContainer(ctx, tenant) if err != nil { + if errors.Is(err, errTenantRuntimeMissing) { + return s.recreateMissingRuntime(ctx, tenant, image, healthTimeout, healthPoll) + } return nil, err } if strings.TrimSpace(tenant.ContainerID) != "" && tenant.ContainerID != live.ID { @@ -369,6 +380,59 @@ func (s *tenantRuntimeRolloutService) Rollout(ctx context.Context, opts TenantRu }, nil } +func (s *tenantRuntimeRolloutService) recreateMissingRuntime( + ctx context.Context, + tenant *registry.Tenant, + image string, + healthTimeout time.Duration, + healthPoll time.Duration, +) (*TenantRuntimeRolloutResult, error) { + if tenant == nil { + return nil, fmt.Errorf("tenant is nil") + } + tenantID := strings.TrimSpace(tenant.ID) + if tenantID == "" { + return nil, fmt.Errorf("tenant id is required") + } + + tenantDataDir := filepath.Join(s.tenantsDir, tenantID) + newContainerID, err := s.docker.CreateAndStart(ctx, tenantID, tenantDataDir) + if err != nil { + return nil, fmt.Errorf("recreate missing tenant runtime for %s using image %s: %w", tenantID, image, err) + } + + healthy, err := s.waitForHealth(ctx, newContainerID, healthTimeout, healthPoll) + if err != nil || !healthy { + if err == nil { + err = fmt.Errorf("tenant runtime %s failed health checks", newContainerID) + } + if removeErr := s.docker.Remove(ctx, newContainerID); removeErr != nil { + return nil, fmt.Errorf("%w; cleanup failed for recreated container %s: %v", err, newContainerID, removeErr) + } + return nil, err + } + + newInfo, err := s.resolveLiveContainer(ctx, tenant) + if err != nil { + return nil, fmt.Errorf("inspect recreated tenant runtime %s: %w", tenantID, err) + } + if newInfo.Name != tenantRuntimeContainerName(tenantID) { + return nil, fmt.Errorf("recreated tenant runtime is not using canonical container name %s", tenantRuntimeContainerName(tenantID)) + } + if err := s.persistTenantRuntimeState(tenant, newInfo, true); err != nil { + return nil, err + } + + return &TenantRuntimeRolloutResult{ + TenantID: tenantID, + PreviousContainerID: strings.TrimSpace(tenant.ContainerID), + ActiveContainerID: newInfo.ID, + ActiveImageRef: newInfo.ImageRef, + ActiveImageID: newInfo.ImageID, + RestoredMissing: true, + }, nil +} + func tenantRuntimeMatchesContract( live *cpDocker.RuntimeContainerInfo, canonicalName string, @@ -404,8 +468,17 @@ func (s *tenantRuntimeRolloutService) PlanContractReconcile( } live, err := s.resolveLiveContainer(ctx, tenant) if err != nil { - item.Action = tenantRuntimeContractActionSkip - item.Reason = err.Error() + if errors.Is(err, errTenantRuntimeMissing) { + desiredRouting := s.docker.DesiredRuntimeRouting(tenant.ID) + item.ImageRef = strings.TrimSpace(s.defaultImage) + item.DesiredRouteHost = desiredRouting.Host + item.DesiredPublicURL = desiredRouting.PublicURL + item.Action = tenantRuntimeContractActionRollout + item.Reason = "tenant runtime container is missing; recreate from existing tenant data" + } else { + item.Action = tenantRuntimeContractActionSkip + item.Reason = err.Error() + } plan.Tenants = append(plan.Tenants, item) continue } @@ -506,10 +579,13 @@ func (s *tenantRuntimeRolloutService) resolveLiveContainer(ctx context.Context, containerID := strings.TrimSpace(tenant.ContainerID) if containerID == "" { - return nil, fmt.Errorf("tenant %s has no canonical runtime container and no registry container_id", tenant.ID) + return nil, fmt.Errorf("%w: tenant %s has no canonical runtime container and no registry container_id", errTenantRuntimeMissing, tenant.ID) } info, err = s.docker.Inspect(ctx, containerID) if err != nil { + if cpDocker.IsNotFound(err) { + return nil, fmt.Errorf("%w: inspect tenant container %s: %w", errTenantRuntimeMissing, containerID, err) + } return nil, fmt.Errorf("inspect tenant container %s: %w", containerID, err) } return info, nil diff --git a/internal/cloudcp/tenant_runtime_rollout_test.go b/internal/cloudcp/tenant_runtime_rollout_test.go index 082f99e22..a7ccfdf25 100644 --- a/internal/cloudcp/tenant_runtime_rollout_test.go +++ b/internal/cloudcp/tenant_runtime_rollout_test.go @@ -314,6 +314,59 @@ func TestTenantRuntimeRollout_RecreatesSameImageWhenRoutingContractDrifts(t *tes } } +func TestTenantRuntimeRollout_RecreatesMissingRuntimeFromTenantData(t *testing.T) { + tenant := ®istry.Tenant{ID: "t-MISSING01", ContainerID: "removed-container"} + reg := &fakeTenantRuntimeRolloutRegistry{tenant: tenant} + docker := newFakeTenantRuntimeRolloutDocker() + routing := docker.DesiredRuntimeRouting(tenant.ID) + docker.queueCreate(&cpDocker.RuntimeContainerInfo{ + ID: "recreated-container", + Name: tenantRuntimeContainerName(tenant.ID), + ImageRef: "pulse-runtime:stable", + ImageID: "sha256:stable", + Running: true, + RouteHost: routing.Host, + PublicURL: routing.PublicURL, + }, nil) + docker.health["recreated-container"] = []bool{true} + sync := &fakeTenantRuntimeRolloutSynchronizer{} + clock := newFakeTenantRuntimeRolloutClock() + + service := newTestTenantRuntimeRolloutService(reg, docker, sync, clock) + result, err := service.Rollout(context.Background(), TenantRuntimeRolloutOptions{ + TenantID: tenant.ID, + Image: "pulse-runtime:stable", + }) + if err != nil { + t.Fatalf("Rollout() error = %v", err) + } + + if !result.RestoredMissing { + t.Fatalf("RestoredMissing = false, want true") + } + if result.ReconciledOnly { + t.Fatalf("ReconciledOnly = true, want false") + } + if len(sync.snapshots) != 0 { + t.Fatalf("snapshot count = %d, want 0", len(sync.snapshots)) + } + if len(docker.createCalls) != 1 { + t.Fatalf("create call count = %d, want 1", len(docker.createCalls)) + } + if docker.createCalls[0].tenantDataDir != filepath.Join(tTempDirForRolloutService(), tenant.ID) { + t.Fatalf("created tenant data dir = %q", docker.createCalls[0].tenantDataDir) + } + if got := reg.updatedTenant.ContainerID; got != "recreated-container" { + t.Fatalf("updated tenant container id = %q, want recreated-container", got) + } + if got := reg.updatedTenant.CurrentImageDigest; got != "sha256:stable" { + t.Fatalf("updated tenant image digest = %q, want sha256:stable", got) + } + if !reg.updatedTenant.HealthCheckOK { + t.Fatalf("updated tenant health_check_ok = false, want true") + } +} + func TestTenantRuntimeContractReconcilePlan_AllTenantsClassifiesNoopRolloutAndSkip(t *testing.T) { tenantNoop := ®istry.Tenant{ID: "t-NOOP12345", ContainerID: "noop-live"} tenantDrift := ®istry.Tenant{ID: "t-Drift1234", ContainerID: "drift-live"} @@ -366,8 +419,14 @@ func TestTenantRuntimeContractReconcilePlan_AllTenantsClassifiesNoopRolloutAndSk if got[tenantDrift.ID].DesiredRouteHost != routingDrift.Host { t.Fatalf("drift tenant desired route host = %q, want %q", got[tenantDrift.ID].DesiredRouteHost, routingDrift.Host) } - if got[tenantMissing.ID].Action != tenantRuntimeContractActionSkip { - t.Fatalf("missing tenant action = %q, want %q", got[tenantMissing.ID].Action, tenantRuntimeContractActionSkip) + if got[tenantMissing.ID].Action != tenantRuntimeContractActionRollout { + t.Fatalf("missing tenant action = %q, want %q", got[tenantMissing.ID].Action, tenantRuntimeContractActionRollout) + } + if got[tenantMissing.ID].ImageRef != "pulse-runtime:stable" { + t.Fatalf("missing tenant image ref = %q, want pulse-runtime:stable", got[tenantMissing.ID].ImageRef) + } + if got[tenantMissing.ID].DesiredRouteHost == "" { + t.Fatalf("missing tenant desired route host is empty") } } @@ -416,6 +475,7 @@ func newTestTenantRuntimeRolloutService( registry: reg, docker: docker, tenantsDir: tTempDirForRolloutService(), + defaultImage: "pulse-runtime:stable", synchronizer: sync, now: clock.Now, sleep: clock.Sleep,