From 473a0ddee61673be5fb7aecd6fba9b2c80440d44 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Tue, 2 Jun 2026 16:45:03 +0100 Subject: [PATCH] Add provider MSP tenant rollout upgrade proof --- cmd/pulse-control-plane/main.go | 130 +++++++++++++- .../provider_msp_status.go | 79 ++++++++- .../provider_msp_status_test.go | 39 ++++- deploy/provider-msp/.env.example | 6 +- deploy/provider-msp/upgrade.sh | 20 ++- .../v6/internal/subsystems/cloud-paid.md | 7 + .../subsystems/deployment-installability.md | 6 +- internal/cloudcp/tenant_runtime_rollout.go | 162 ++++++++++++++++++ .../cloudcp/tenant_runtime_rollout_test.go | 62 +++++++ .../installtests/provider_msp_deploy_test.go | 5 +- 10 files changed, 492 insertions(+), 24 deletions(-) diff --git a/cmd/pulse-control-plane/main.go b/cmd/pulse-control-plane/main.go index 6642edc70..dbcfb0877 100644 --- a/cmd/pulse-control-plane/main.go +++ b/cmd/pulse-control-plane/main.go @@ -85,11 +85,13 @@ func newCloudAuditCmd() *cobra.Command { func newTenantRuntimeRolloutCmd() *cobra.Command { var tenantID string + var all bool var image string var runID string var snapshotRoot string var healthTimeout time.Duration var prunePrevious bool + var dryRun bool cmd := &cobra.Command{ Use: "rollout", @@ -99,6 +101,37 @@ func newTenantRuntimeRolloutCmd() *cobra.Command { if err != nil { return fmt.Errorf("load control plane config: %w", err) } + if all && strings.TrimSpace(tenantID) != "" { + return fmt.Errorf("choose either --all or --tenant-id") + } + if !all && strings.TrimSpace(tenantID) == "" { + return fmt.Errorf("--tenant-id is required unless --all is set") + } + if dryRun { + tenantIDs := []string{tenantID} + if all { + tenantIDs = nil + } + plan, err := cloudcp.PlanTenantRuntimeImageRollout(cmd.Context(), cfg, cloudcp.TenantRuntimeImageRolloutPlanOptions{ + TenantIDs: tenantIDs, + All: all, + Image: image, + }) + if err != nil { + return err + } + printTenantRuntimeImageRolloutPlan(plan) + return nil + } + if all { + return runTenantRuntimeRolloutAll(cmd.Context(), cfg, cloudcp.TenantRuntimeRolloutOptions{ + Image: image, + RunID: runID, + SnapshotRoot: snapshotRoot, + HealthTimeout: healthTimeout, + PrunePrevious: prunePrevious, + }) + } result, err := cloudcp.RolloutTenantRuntime(cmd.Context(), cfg, cloudcp.TenantRuntimeRolloutOptions{ TenantID: tenantID, Image: image, @@ -126,16 +159,67 @@ func newTenantRuntimeRolloutCmd() *cobra.Command { }, } cmd.Flags().StringVar(&tenantID, "tenant-id", "", "Hosted tenant ID to roll") + cmd.Flags().BoolVar(&all, "all", false, "Roll all active hosted tenants") cmd.Flags().StringVar(&image, "image", "", "Target Pulse runtime image reference") cmd.Flags().StringVar(&runID, "run-id", "", "Operator-visible rollout run identifier") cmd.Flags().StringVar(&snapshotRoot, "snapshot-root", "", "Override tenant snapshot root (default: /backups/rollout)") cmd.Flags().DurationVar(&healthTimeout, "health-timeout", 90*time.Second, "How long to wait for the target runtime to become healthy") cmd.Flags().BoolVar(&prunePrevious, "prune-previous", false, "Remove the preserved pre-rollout container after success") - _ = cmd.MarkFlagRequired("tenant-id") + cmd.Flags().BoolVar(&dryRun, "dry-run", false, "Print the target image rollout plan without mutating tenant runtimes") _ = cmd.MarkFlagRequired("image") return cmd } +func runTenantRuntimeRolloutAll(ctx context.Context, cfg *cloudcp.CPConfig, opts cloudcp.TenantRuntimeRolloutOptions) error { + reg, err := registry.NewTenantRegistry(cfg.ControlPlaneDir()) + if err != nil { + return fmt.Errorf("open tenant registry: %w", err) + } + defer reg.Close() + tenants, err := reg.ListByState(registry.TenantStateActive) + if err != nil { + return fmt.Errorf("list active tenants: %w", err) + } + + rolloutCount := 0 + failureCount := 0 + for _, tenant := range tenants { + if tenant == nil || strings.TrimSpace(tenant.ID) == "" { + continue + } + opts.TenantID = tenant.ID + result, err := cloudcp.RolloutTenantRuntime(ctx, cfg, opts) + if err != nil { + failureCount++ + fmt.Printf("tenant_id=%s\nstatus=error\nimage_ref=%s\nerror=%v\n\n", tenant.ID, opts.Image, err) + continue + } + rolloutCount++ + fmt.Printf("tenant_id=%s\nstatus=rolled\nactive_container_id=%s\nactive_image_ref=%s\nactive_image_id=%s\nreconciled_only=%t\n", + result.TenantID, + result.ActiveContainerID, + result.ActiveImageRef, + result.ActiveImageID, + result.ReconciledOnly, + ) + if result.RestoredMissing { + fmt.Printf("restored_missing=%t\n", result.RestoredMissing) + } + if result.PreviousContainerID != "" { + fmt.Printf("previous_container_id=%s\n", result.PreviousContainerID) + } + if result.BackupContainerName != "" { + fmt.Printf("backup_container_name=%s\n", result.BackupContainerName) + } + fmt.Println() + } + fmt.Printf("summary_rollout=%d\nsummary_error=%d\nsummary_total=%d\n", rolloutCount, failureCount, len(tenants)) + if failureCount > 0 { + return fmt.Errorf("%d tenant runtime rollouts failed", failureCount) + } + return nil +} + func newTenantRuntimeReconcileCmd() *cobra.Command { var tenantIDs []string var all bool @@ -273,6 +357,50 @@ func printTenantRuntimeReconcilePlan(plan *cloudcp.TenantRuntimeContractReconcil fmt.Printf("summary_rollout=%d\nsummary_noop=%d\nsummary_skip=%d\nsummary_total=%d\n", rolloutCount, noopCount, skipCount, len(plan.Tenants)) } +func printTenantRuntimeImageRolloutPlan(plan *cloudcp.TenantRuntimeImageRolloutPlan) { + if plan == nil { + fmt.Println("summary_total=0") + return + } + rolloutCount := 0 + noopCount := 0 + skipCount := 0 + for _, item := range plan.Tenants { + if item == nil { + continue + } + switch item.Action { + case "rollout": + rolloutCount++ + case "noop": + noopCount++ + default: + skipCount++ + } + fmt.Printf("tenant_id=%s\naction=%s\nreason=%s\n", item.TenantID, item.Action, item.Reason) + if item.State != "" { + fmt.Printf("tenant_state=%s\n", item.State) + } + if item.LiveContainerID != "" { + fmt.Printf("live_container_id=%s\n", item.LiveContainerID) + } + if item.LiveImageRef != "" { + fmt.Printf("live_image_ref=%s\n", item.LiveImageRef) + } + if item.TargetImageRef != "" { + fmt.Printf("target_image_ref=%s\n", item.TargetImageRef) + } + if item.LiveRouteHost != "" || item.DesiredRouteHost != "" { + fmt.Printf("live_route_host=%s\ndesired_route_host=%s\n", item.LiveRouteHost, item.DesiredRouteHost) + } + if item.LivePublicURL != "" || item.DesiredPublicURL != "" { + fmt.Printf("live_public_url=%s\ndesired_public_url=%s\n", item.LivePublicURL, item.DesiredPublicURL) + } + fmt.Println() + } + fmt.Printf("summary_rollout=%d\nsummary_noop=%d\nsummary_skip=%d\nsummary_total=%d\n", rolloutCount, noopCount, skipCount, len(plan.Tenants)) +} + func printCloudAuditReport(report *cloudcp.CloudAuditReport) { if report == nil { fmt.Println("audit_ok=false") diff --git a/cmd/pulse-control-plane/provider_msp_status.go b/cmd/pulse-control-plane/provider_msp_status.go index 75ed1ec9b..0dfe73f80 100644 --- a/cmd/pulse-control-plane/provider_msp_status.go +++ b/cmd/pulse-control-plane/provider_msp_status.go @@ -10,6 +10,7 @@ import ( "time" "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp" + cpDocker "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/docker" "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry" pkglicensing "github.com/rcourtman/pulse-go-rewrite/pkg/licensing" "github.com/spf13/cobra" @@ -64,10 +65,16 @@ type providerMSPBackupStatus struct { type providerMSPStatusDependencies struct { OpenRegistry func(*cloudcp.CPConfig) (*registry.TenantRegistry, error) RunPreflight func(context.Context, *cloudcp.CPConfig, providerMSPPreflightOptions) (*providerMSPPreflightReport, error) + NewDocker func(*cloudcp.CPConfig) (providerMSPStatusDocker, error) CheckBackup func(context.Context, *cloudcp.CPConfig) (*providerMSPBackupStatus, error) Now func() time.Time } +type providerMSPStatusDocker interface { + HealthCheck(context.Context, string) (bool, error) + Close() error +} + func newProviderMSPStatusCmd() *cobra.Command { opts := providerMSPStatusOptions{} cmd := &cobra.Command{ @@ -190,14 +197,17 @@ func runProviderMSPStatusWithDependencies(ctx context.Context, cfg *cloudcp.CPCo } } - healthy, unhealthy, err := reg.HealthSummary() + health, err := checkProviderMSPLiveRuntimeHealth(ctx, cfg, reg, deps) if err != nil { - addFailure("tenant health summary: %v", err) + addFailure("tenant runtime health summary: %v", err) } else { - report.HealthyTenants = healthy - report.UnhealthyTenants = unhealthy - if unhealthy > 0 { - addFailure("unhealthy active workspaces: %d", unhealthy) + report.HealthyTenants = health.Healthy + report.UnhealthyTenants = health.Unhealthy + for _, failure := range health.Failures { + addFailure("runtime health: %s", failure) + } + if health.Unhealthy > 0 { + addFailure("unhealthy active workspaces: %d", health.Unhealthy) } } @@ -243,6 +253,11 @@ func normalizeProviderMSPStatusDependencies(deps providerMSPStatusDependencies) if deps.RunPreflight == nil { deps.RunPreflight = runProviderMSPPreflight } + if deps.NewDocker == nil { + deps.NewDocker = func(cfg *cloudcp.CPConfig) (providerMSPStatusDocker, error) { + return cpDocker.NewManager(providerMSPDockerManagerConfig(cfg)) + } + } if deps.CheckBackup == nil { deps.CheckBackup = checkProviderMSPBackupStatus } @@ -252,6 +267,58 @@ func normalizeProviderMSPStatusDependencies(deps providerMSPStatusDependencies) return deps } +type providerMSPLiveRuntimeHealth struct { + Healthy int + Unhealthy int + Failures []string +} + +func checkProviderMSPLiveRuntimeHealth( + ctx context.Context, + cfg *cloudcp.CPConfig, + reg *registry.TenantRegistry, + deps providerMSPStatusDependencies, +) (*providerMSPLiveRuntimeHealth, error) { + if reg == nil { + return nil, fmt.Errorf("tenant registry is required") + } + dockerMgr, err := deps.NewDocker(cfg) + if err != nil { + return nil, fmt.Errorf("create Docker manager: %w", err) + } + defer dockerMgr.Close() + + tenants, err := reg.ListByState(registry.TenantStateActive) + if err != nil { + return nil, fmt.Errorf("list active tenants: %w", err) + } + + health := &providerMSPLiveRuntimeHealth{} + for _, tenant := range tenants { + if tenant == nil { + continue + } + containerID := strings.TrimSpace(tenant.ContainerID) + if containerID == "" { + health.Unhealthy++ + health.Failures = append(health.Failures, fmt.Sprintf("workspace %s has no runtime container id", tenant.ID)) + continue + } + healthy, checkErr := dockerMgr.HealthCheck(ctx, containerID) + if checkErr != nil { + health.Unhealthy++ + health.Failures = append(health.Failures, fmt.Sprintf("workspace %s container %s: %v", tenant.ID, containerID, checkErr)) + continue + } + if healthy { + health.Healthy++ + } else { + health.Unhealthy++ + } + } + return health, nil +} + func checkProviderMSPBackupStatus(ctx context.Context, cfg *cloudcp.CPConfig) (*providerMSPBackupStatus, error) { if cfg == nil { return nil, fmt.Errorf("control plane config is required") diff --git a/cmd/pulse-control-plane/provider_msp_status_test.go b/cmd/pulse-control-plane/provider_msp_status_test.go index 888a62a79..e6ea84fa7 100644 --- a/cmd/pulse-control-plane/provider_msp_status_test.go +++ b/cmd/pulse-control-plane/provider_msp_status_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "fmt" "strings" "testing" "time" @@ -26,9 +27,10 @@ func TestProviderMSPStatusReportsHealthyOperatorSurface(t *testing.T) { now := time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC) createProviderMSPStatusTenant(t, cfg, ®istry.Tenant{ ID: "t-HEALTHY", + ContainerID: "c-healthy", State: registry.TenantStateActive, CreatedAt: now.Add(-time.Hour), - HealthCheckOK: true, + HealthCheckOK: false, }) var gotPreflight providerMSPPreflightOptions @@ -37,6 +39,9 @@ func TestProviderMSPStatusReportsHealthyOperatorSurface(t *testing.T) { gotPreflight = opts return healthyProviderMSPStatusPreflightReport(), nil }, + NewDocker: healthyProviderMSPStatusDocker(map[string]bool{ + "c-healthy": true, + }), CheckBackup: func(context.Context, *cloudcp.CPConfig) (*providerMSPBackupStatus, error) { return healthyProviderMSPBackupStatus(now), nil }, @@ -73,9 +78,10 @@ func TestProviderMSPStatusFailsOnFailedUnhealthyAndStuckWorkspaces(t *testing.T) now := time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC) createProviderMSPStatusTenant(t, cfg, ®istry.Tenant{ ID: "t-UNHEALTHY", + ContainerID: "c-unhealthy", State: registry.TenantStateActive, CreatedAt: now.Add(-time.Hour), - HealthCheckOK: false, + HealthCheckOK: true, }) createProviderMSPStatusTenant(t, cfg, ®istry.Tenant{ ID: "t-FAILED", @@ -92,6 +98,9 @@ func TestProviderMSPStatusFailsOnFailedUnhealthyAndStuckWorkspaces(t *testing.T) RunPreflight: func(context.Context, *cloudcp.CPConfig, providerMSPPreflightOptions) (*providerMSPPreflightReport, error) { return healthyProviderMSPStatusPreflightReport(), nil }, + NewDocker: healthyProviderMSPStatusDocker(map[string]bool{ + "c-unhealthy": false, + }), Now: func() time.Time { return now }, }) if err == nil { @@ -118,6 +127,7 @@ func TestProviderMSPStatusBackupWarningBecomesFailureWhenRequired(t *testing.T) RunPreflight: func(context.Context, *cloudcp.CPConfig, providerMSPPreflightOptions) (*providerMSPPreflightReport, error) { return healthyProviderMSPStatusPreflightReport(), nil }, + NewDocker: healthyProviderMSPStatusDocker(nil), CheckBackup: func(context.Context, *cloudcp.CPConfig) (*providerMSPBackupStatus, error) { return &providerMSPBackupStatus{ Directory: "/data/backups/provider-msp", @@ -150,6 +160,31 @@ func TestProviderMSPStatusBackupWarningBecomesFailureWhenRequired(t *testing.T) } } +func healthyProviderMSPStatusDocker(health map[string]bool) func(*cloudcp.CPConfig) (providerMSPStatusDocker, error) { + return func(*cloudcp.CPConfig) (providerMSPStatusDocker, error) { + return &fakeProviderMSPStatusDocker{health: health}, nil + } +} + +type fakeProviderMSPStatusDocker struct { + health map[string]bool +} + +func (f *fakeProviderMSPStatusDocker) HealthCheck(_ context.Context, containerID string) (bool, error) { + if f.health == nil { + return false, fmt.Errorf("unexpected health check for %s", containerID) + } + healthy, ok := f.health[containerID] + if !ok { + return false, fmt.Errorf("unexpected health check for %s", containerID) + } + return healthy, nil +} + +func (f *fakeProviderMSPStatusDocker) Close() error { + return nil +} + func healthyProviderMSPStatusPreflightReport() *providerMSPPreflightReport { return &providerMSPPreflightReport{ OK: true, diff --git a/deploy/provider-msp/.env.example b/deploy/provider-msp/.env.example index ea6e70b77..08622980c 100644 --- a/deploy/provider-msp/.env.example +++ b/deploy/provider-msp/.env.example @@ -67,11 +67,11 @@ PULSE_EMAIL_REPLY_TO=support@example.com # Apply a provider control-plane upgrade after updating the image pins in this # file. The runner creates and verifies a fresh backup, dry-runs restore into a # separate target data dir, pulls provider images, starts Traefik/control-plane, -# and prints the tenant runtime reconcile plan: +# and prints the tenant runtime rollout plan for CP_PULSE_IMAGE: # ./upgrade.sh # -# Also reconcile all client runtimes to the configured CP_PULSE_IMAGE line after -# the provider services are updated: +# Also roll all client runtimes to the configured CP_PULSE_IMAGE line after the +# provider services are updated: # ./upgrade.sh --rollout-tenants # # Fresh-install provider proof. This bootstraps the owner account, checks install diff --git a/deploy/provider-msp/upgrade.sh b/deploy/provider-msp/upgrade.sh index 470222722..4b0cfb811 100755 --- a/deploy/provider-msp/upgrade.sh +++ b/deploy/provider-msp/upgrade.sh @@ -13,12 +13,12 @@ Runs the provider-hosted MSP pre-upgrade and upgrade flow: 3. creates and verifies a fresh provider MSP backup 4. dry-runs restore into a separate target data directory 5. pulls and starts the provider control-plane and Traefik services - 6. prints the tenant runtime reconcile plan - 7. optionally reconciles all tenant runtimes onto the configured runtime line + 6. prints the tenant runtime rollout plan for CP_PULSE_IMAGE + 7. optionally rolls all tenant runtimes onto CP_PULSE_IMAGE Options: --dry-run Print the non-mutating upgrade plan only - --rollout-tenants Execute tenant-runtime reconcile --all after provider services are updated + --rollout-tenants Execute tenant-runtime rollout --all after provider services are updated --prune-previous Remove preserved pre-rollout tenant containers after successful tenant rollout --skip-compose-pull Do not run docker compose pull for provider services --skip-runtime-image-pull Pass --skip-image-pull to provider-msp preflight @@ -144,6 +144,10 @@ fi provider_data_dir="$(env_value PULSE_PROVIDER_MSP_DATA_DIR .env)" provider_data_dir="${provider_data_dir:-/data}" +tenant_runtime_image="$(env_value CP_PULSE_IMAGE .env)" +if [[ -z "${tenant_runtime_image}" ]]; then + die "CP_PULSE_IMAGE is required" +fi if [[ -z "${restore_target}" ]]; then restore_target="${provider_data_dir%/}/upgrade-restore-drill" fi @@ -163,12 +167,13 @@ fi echo "provider_msp_upgrade_dry_run=$(truthy "${dry_run}" && echo true || echo false)" echo "provider_msp_upgrade_run_id=${run_id}" echo "provider_msp_upgrade_restore_target=${restore_target}" +echo "provider_msp_upgrade_tenant_runtime_image=${tenant_runtime_image}" run_control provider-msp status run_control "${preflight_args[@]}" if truthy "${dry_run}"; then - run_control tenant-runtime reconcile --all --dry-run + run_control tenant-runtime rollout --all --image "${tenant_runtime_image}" --dry-run echo "tenant_runtime_rollout_applied=false" echo "provider_msp_upgrade_plan_ok=true" exit 0 @@ -195,12 +200,13 @@ if ! truthy "${skip_compose_pull}"; then fi docker compose up -d traefik control-plane run_control provider-msp status --require-backup -run_control tenant-runtime reconcile --all --dry-run +run_control tenant-runtime rollout --all --image "${tenant_runtime_image}" --dry-run if truthy "${rollout_tenants}"; then reconcile_args=( - tenant-runtime reconcile + tenant-runtime rollout --all + --image "${tenant_runtime_image}" --run-id "${run_id}" --health-timeout "${health_timeout}" ) @@ -212,7 +218,7 @@ if truthy "${rollout_tenants}"; then echo "tenant_runtime_rollout_applied=true" else echo "tenant_runtime_rollout_applied=false" - echo "tenant_runtime_rollout_next_command=docker compose run --rm --no-deps control-plane tenant-runtime reconcile --all --run-id ${run_id} --health-timeout ${health_timeout}" + echo "tenant_runtime_rollout_next_command=docker compose run --rm --no-deps control-plane tenant-runtime rollout --all --image ${tenant_runtime_image} --run-id ${run_id} --health-timeout ${health_timeout}" fi docker compose ps diff --git a/docs/release-control/v6/internal/subsystems/cloud-paid.md b/docs/release-control/v6/internal/subsystems/cloud-paid.md index 66d31bc76..bb8071756 100644 --- a/docs/release-control/v6/internal/subsystems/cloud-paid.md +++ b/docs/release-control/v6/internal/subsystems/cloud-paid.md @@ -212,6 +212,13 @@ tenant-local runtime state without depending on Stripe billing surfaces. coherent. Tenant runtime rollout and missing-runtime restore must fail closed on the same storage admission guard before snapshotting or swapping containers. + `tenant-runtime rollout --all --image ` is the canonical hosted + fleet image upgrade path for active tenant runtimes. Its `--dry-run` mode + must print the target-image rollout plan before mutation, `--all` must + select active tenants only, and apply must still use the canonical + per-tenant snapshot, health-check, and rollback path. `tenant-runtime + reconcile --all` remains contract/routing drift repair for each tenant's + current image line, not an image-line upgrade path. The real `pulse-pro` license-server legacy checkout issuance, recurring renewals, manual issue, and legacy exchange flows are part of that same diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index da7188e96..fc7062cd3 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -156,9 +156,9 @@ surfaces. provider status and preflight, create and verify a fresh backup before apply, dry-run restore into a separate target data dir, require backup readiness before and after provider service replacement, update the packaged - Traefik/control-plane services, print the tenant runtime reconcile plan, and - only execute `tenant-runtime reconcile --all` when the operator explicitly - asks for tenant rollout. + Traefik/control-plane services, print the tenant runtime rollout plan for + `CP_PULSE_IMAGE`, and only execute `tenant-runtime rollout --all --image + ` when the operator explicitly asks for tenant rollout. `deploy/provider-msp/setup.sh` is the first-time provider host setup artifact. It must install the Docker/compose host prerequisites, create the provider data, backup, and Docker-network layout, copy the provider MSP diff --git a/internal/cloudcp/tenant_runtime_rollout.go b/internal/cloudcp/tenant_runtime_rollout.go index ab3957f1c..1e5d57379 100644 --- a/internal/cloudcp/tenant_runtime_rollout.go +++ b/internal/cloudcp/tenant_runtime_rollout.go @@ -67,6 +67,30 @@ type TenantRuntimeContractReconcilePlan struct { Tenants []*TenantRuntimeContractReconcilePlanItem } +type TenantRuntimeImageRolloutPlanOptions struct { + TenantIDs []string + All bool + Image string +} + +type TenantRuntimeImageRolloutPlanItem struct { + TenantID string + State string + LiveContainerID string + LiveImageRef string + TargetImageRef string + LiveRouteHost string + DesiredRouteHost string + LivePublicURL string + DesiredPublicURL string + Action string + Reason string +} + +type TenantRuntimeImageRolloutPlan struct { + Tenants []*TenantRuntimeImageRolloutPlanItem +} + const ( tenantRuntimeContractActionNoop = "noop" tenantRuntimeContractActionRollout = "rollout" @@ -147,6 +171,31 @@ func PlanTenantRuntimeContractReconcile( return service.PlanContractReconcile(ctx, opts) } +func PlanTenantRuntimeImageRollout( + ctx context.Context, + cfg *CPConfig, + opts TenantRuntimeImageRolloutPlanOptions, +) (*TenantRuntimeImageRolloutPlan, error) { + if cfg == nil { + return nil, fmt.Errorf("control plane config is required") + } + image := strings.TrimSpace(opts.Image) + if image == "" { + image = strings.TrimSpace(cfg.PulseImage) + } + if image == "" { + return nil, fmt.Errorf("missing tenant runtime image") + } + opts.Image = image + + service, cleanup, err := newTenantRuntimeRolloutServiceFromConfig(cfg, image) + if err != nil { + return nil, err + } + defer cleanup() + return service.PlanImageRollout(ctx, opts) +} + func newTenantRuntimeRolloutServiceFromConfig( cfg *CPConfig, image string, @@ -532,6 +581,77 @@ func (s *tenantRuntimeRolloutService) PlanContractReconcile( return plan, nil } +func (s *tenantRuntimeRolloutService) PlanImageRollout( + ctx context.Context, + opts TenantRuntimeImageRolloutPlanOptions, +) (*TenantRuntimeImageRolloutPlan, error) { + if s == nil { + return nil, fmt.Errorf("rollout service is nil") + } + targetImage := strings.TrimSpace(opts.Image) + if targetImage == "" { + targetImage = strings.TrimSpace(s.defaultImage) + } + if targetImage == "" { + return nil, fmt.Errorf("image is required") + } + tenants, err := s.selectImageRolloutTenants(opts) + if err != nil { + return nil, err + } + plan := &TenantRuntimeImageRolloutPlan{ + Tenants: make([]*TenantRuntimeImageRolloutPlanItem, 0, len(tenants)), + } + for _, tenant := range tenants { + item := &TenantRuntimeImageRolloutPlanItem{ + TenantID: strings.TrimSpace(tenant.ID), + State: string(tenant.State), + TargetImageRef: targetImage, + } + if tenant.State != registry.TenantStateActive { + item.Action = tenantRuntimeContractActionSkip + item.Reason = fmt.Sprintf("tenant state is %s, not active", tenant.State) + plan.Tenants = append(plan.Tenants, item) + continue + } + + desiredRouting := s.docker.DesiredRuntimeRouting(tenant.ID) + item.DesiredRouteHost = desiredRouting.Host + item.DesiredPublicURL = desiredRouting.PublicURL + + live, err := s.resolveLiveContainer(ctx, tenant) + if err != nil { + if errors.Is(err, errTenantRuntimeMissing) { + item.Action = tenantRuntimeContractActionRollout + item.Reason = "tenant runtime container is missing; recreate from existing tenant data" + } else { + item.Action = tenantRuntimeContractActionSkip + item.Reason = err.Error() + } + plan.Tenants = append(plan.Tenants, item) + continue + } + + item.LiveContainerID = strings.TrimSpace(live.ID) + item.LiveImageRef = strings.TrimSpace(live.ImageRef) + item.LiveRouteHost = strings.TrimSpace(live.RouteHost) + item.LivePublicURL = strings.TrimSpace(live.PublicURL) + + if tenantRuntimeMatchesContract(live, tenantRuntimeContainerName(tenant.ID), targetImage, desiredRouting) { + item.Action = tenantRuntimeContractActionNoop + item.Reason = "runtime already matches target image and canonical hosted contract" + } else if item.LiveImageRef != targetImage { + item.Action = tenantRuntimeContractActionRollout + item.Reason = "runtime image differs from target" + } else { + item.Action = tenantRuntimeContractActionRollout + item.Reason = "runtime contract drift detected" + } + plan.Tenants = append(plan.Tenants, item) + } + return plan, nil +} + func (s *tenantRuntimeRolloutService) selectContractReconcileTenants( opts TenantRuntimeContractReconcilePlanOptions, ) ([]*registry.Tenant, error) { @@ -569,6 +689,48 @@ func (s *tenantRuntimeRolloutService) selectContractReconcileTenants( return tenants, nil } +func (s *tenantRuntimeRolloutService) selectImageRolloutTenants( + opts TenantRuntimeImageRolloutPlanOptions, +) ([]*registry.Tenant, error) { + if s == nil { + return nil, fmt.Errorf("rollout service is nil") + } + if opts.All && len(dedupeNonEmptyStrings(opts.TenantIDs)) > 0 { + return nil, fmt.Errorf("choose either --all or one or more tenant ids") + } + if opts.All { + tenants, err := s.registry.List() + if err != nil { + return nil, fmt.Errorf("list tenants: %w", err) + } + active := make([]*registry.Tenant, 0, len(tenants)) + for _, tenant := range tenants { + if tenant != nil && tenant.State == registry.TenantStateActive { + active = append(active, tenant) + } + } + return active, nil + } + + tenantIDs := dedupeNonEmptyStrings(opts.TenantIDs) + if len(tenantIDs) == 0 { + return nil, fmt.Errorf("at least one tenant id or --all is required") + } + + tenants := make([]*registry.Tenant, 0, len(tenantIDs)) + for _, tenantID := range tenantIDs { + tenant, err := s.registry.Get(tenantID) + if err != nil { + return nil, fmt.Errorf("load tenant %s: %w", tenantID, err) + } + if tenant == nil { + return nil, fmt.Errorf("tenant %s not found", tenantID) + } + tenants = append(tenants, tenant) + } + return tenants, nil +} + func dedupeNonEmptyStrings(values []string) []string { result := make([]string, 0, len(values)) seen := make(map[string]struct{}, len(values)) diff --git a/internal/cloudcp/tenant_runtime_rollout_test.go b/internal/cloudcp/tenant_runtime_rollout_test.go index 2c5a5372d..acb5aeda8 100644 --- a/internal/cloudcp/tenant_runtime_rollout_test.go +++ b/internal/cloudcp/tenant_runtime_rollout_test.go @@ -543,6 +543,68 @@ func TestTenantRuntimeContractReconcilePlan_ExplicitTenantIDsDedupesAndPreserves } } +func TestTenantRuntimeImageRolloutPlan_AllActiveTenantsTargetsConfiguredImage(t *testing.T) { + tenantOldImage := ®istry.Tenant{ID: "t-OLDIMAGE1", State: registry.TenantStateActive, ContainerID: "old-live"} + tenantTargetImage := ®istry.Tenant{ID: "t-TARGET01", State: registry.TenantStateActive, ContainerID: "target-live"} + tenantMissing := ®istry.Tenant{ID: "t-MISSING02", State: registry.TenantStateActive, ContainerID: ""} + tenantSuspended := ®istry.Tenant{ID: "t-SUSPEND1", State: registry.TenantStateSuspended, ContainerID: "suspended-live"} + reg := &fakeTenantRuntimeRolloutRegistry{ + tenants: []*registry.Tenant{tenantOldImage, tenantTargetImage, tenantMissing, tenantSuspended}, + } + docker := newFakeTenantRuntimeRolloutDocker() + targetImage := "pulse-runtime:next" + for _, tc := range []struct { + tenant *registry.Tenant + image string + }{ + {tenantOldImage, "pulse-runtime:stable"}, + {tenantTargetImage, targetImage}, + {tenantSuspended, "pulse-runtime:stable"}, + } { + routing := docker.DesiredRuntimeRouting(tc.tenant.ID) + docker.addContainer(&cpDocker.RuntimeContainerInfo{ + ID: tc.tenant.ContainerID, + Name: tenantRuntimeContainerName(tc.tenant.ID), + ImageRef: tc.image, + ImageID: "sha256:test", + Running: true, + RouteHost: routing.Host, + PublicURL: routing.PublicURL, + }) + } + + service := newTestTenantRuntimeRolloutService(reg, docker, &fakeTenantRuntimeRolloutSynchronizer{}, newFakeTenantRuntimeRolloutClock()) + plan, err := service.PlanImageRollout(context.Background(), TenantRuntimeImageRolloutPlanOptions{ + All: true, + Image: targetImage, + }) + if err != nil { + t.Fatalf("PlanImageRollout() error = %v", err) + } + if len(plan.Tenants) != 3 { + t.Fatalf("plan tenant count = %d, want 3 active tenants", len(plan.Tenants)) + } + got := make(map[string]*TenantRuntimeImageRolloutPlanItem, len(plan.Tenants)) + for _, item := range plan.Tenants { + got[item.TenantID] = item + } + if got[tenantOldImage.ID].Action != tenantRuntimeContractActionRollout { + t.Fatalf("old image action = %q, want rollout", got[tenantOldImage.ID].Action) + } + if got[tenantOldImage.ID].LiveImageRef != "pulse-runtime:stable" || got[tenantOldImage.ID].TargetImageRef != targetImage { + t.Fatalf("old image refs = live %q target %q", got[tenantOldImage.ID].LiveImageRef, got[tenantOldImage.ID].TargetImageRef) + } + if got[tenantTargetImage.ID].Action != tenantRuntimeContractActionNoop { + t.Fatalf("target image action = %q, want noop", got[tenantTargetImage.ID].Action) + } + if got[tenantMissing.ID].Action != tenantRuntimeContractActionRollout { + t.Fatalf("missing tenant action = %q, want rollout", got[tenantMissing.ID].Action) + } + if _, ok := got[tenantSuspended.ID]; ok { + t.Fatalf("suspended tenant should not be selected by --all image rollout plan") + } +} + func newTestTenantRuntimeRolloutService( reg tenantRuntimeRolloutRegistry, docker tenantRuntimeRolloutDocker, diff --git a/scripts/installtests/provider_msp_deploy_test.go b/scripts/installtests/provider_msp_deploy_test.go index eac5a83e8..4ed6cf15e 100644 --- a/scripts/installtests/provider_msp_deploy_test.go +++ b/scripts/installtests/provider_msp_deploy_test.go @@ -160,9 +160,10 @@ func TestProviderMSPUpgradeRunnerMatchesComposeContract(t *testing.T) { "provider-msp backup verify", "provider-msp backup restore", "--target-data-dir", - "tenant-runtime reconcile --all --dry-run", - "tenant-runtime reconcile", + "tenant-runtime rollout --all --image", + "tenant-runtime rollout", "--all", + "--image", "--run-id", "--health-timeout", "--prune-previous",