Add provider MSP tenant rollout upgrade proof

This commit is contained in:
rcourtman
2026-06-02 16:45:03 +01:00
parent a0f2fcf1fe
commit 473a0ddee6
10 changed files with 492 additions and 24 deletions
+129 -1
View File
@@ -85,11 +85,13 @@ func newCloudAuditCmd() *cobra.Command {
func newTenantRuntimeRolloutCmd() *cobra.Command {
var tenantID string
var all bool
var image string
var runID string
var snapshotRoot string
var healthTimeout time.Duration
var prunePrevious bool
var dryRun bool
cmd := &cobra.Command{
Use: "rollout",
@@ -99,6 +101,37 @@ func newTenantRuntimeRolloutCmd() *cobra.Command {
if err != nil {
return fmt.Errorf("load control plane config: %w", err)
}
if all && strings.TrimSpace(tenantID) != "" {
return fmt.Errorf("choose either --all or --tenant-id")
}
if !all && strings.TrimSpace(tenantID) == "" {
return fmt.Errorf("--tenant-id is required unless --all is set")
}
if dryRun {
tenantIDs := []string{tenantID}
if all {
tenantIDs = nil
}
plan, err := cloudcp.PlanTenantRuntimeImageRollout(cmd.Context(), cfg, cloudcp.TenantRuntimeImageRolloutPlanOptions{
TenantIDs: tenantIDs,
All: all,
Image: image,
})
if err != nil {
return err
}
printTenantRuntimeImageRolloutPlan(plan)
return nil
}
if all {
return runTenantRuntimeRolloutAll(cmd.Context(), cfg, cloudcp.TenantRuntimeRolloutOptions{
Image: image,
RunID: runID,
SnapshotRoot: snapshotRoot,
HealthTimeout: healthTimeout,
PrunePrevious: prunePrevious,
})
}
result, err := cloudcp.RolloutTenantRuntime(cmd.Context(), cfg, cloudcp.TenantRuntimeRolloutOptions{
TenantID: tenantID,
Image: image,
@@ -126,16 +159,67 @@ func newTenantRuntimeRolloutCmd() *cobra.Command {
},
}
cmd.Flags().StringVar(&tenantID, "tenant-id", "", "Hosted tenant ID to roll")
cmd.Flags().BoolVar(&all, "all", false, "Roll all active hosted tenants")
cmd.Flags().StringVar(&image, "image", "", "Target Pulse runtime image reference")
cmd.Flags().StringVar(&runID, "run-id", "", "Operator-visible rollout run identifier")
cmd.Flags().StringVar(&snapshotRoot, "snapshot-root", "", "Override tenant snapshot root (default: <CP_DATA_DIR>/backups/rollout)")
cmd.Flags().DurationVar(&healthTimeout, "health-timeout", 90*time.Second, "How long to wait for the target runtime to become healthy")
cmd.Flags().BoolVar(&prunePrevious, "prune-previous", false, "Remove the preserved pre-rollout container after success")
_ = cmd.MarkFlagRequired("tenant-id")
cmd.Flags().BoolVar(&dryRun, "dry-run", false, "Print the target image rollout plan without mutating tenant runtimes")
_ = cmd.MarkFlagRequired("image")
return cmd
}
func runTenantRuntimeRolloutAll(ctx context.Context, cfg *cloudcp.CPConfig, opts cloudcp.TenantRuntimeRolloutOptions) error {
reg, err := registry.NewTenantRegistry(cfg.ControlPlaneDir())
if err != nil {
return fmt.Errorf("open tenant registry: %w", err)
}
defer reg.Close()
tenants, err := reg.ListByState(registry.TenantStateActive)
if err != nil {
return fmt.Errorf("list active tenants: %w", err)
}
rolloutCount := 0
failureCount := 0
for _, tenant := range tenants {
if tenant == nil || strings.TrimSpace(tenant.ID) == "" {
continue
}
opts.TenantID = tenant.ID
result, err := cloudcp.RolloutTenantRuntime(ctx, cfg, opts)
if err != nil {
failureCount++
fmt.Printf("tenant_id=%s\nstatus=error\nimage_ref=%s\nerror=%v\n\n", tenant.ID, opts.Image, err)
continue
}
rolloutCount++
fmt.Printf("tenant_id=%s\nstatus=rolled\nactive_container_id=%s\nactive_image_ref=%s\nactive_image_id=%s\nreconciled_only=%t\n",
result.TenantID,
result.ActiveContainerID,
result.ActiveImageRef,
result.ActiveImageID,
result.ReconciledOnly,
)
if result.RestoredMissing {
fmt.Printf("restored_missing=%t\n", result.RestoredMissing)
}
if result.PreviousContainerID != "" {
fmt.Printf("previous_container_id=%s\n", result.PreviousContainerID)
}
if result.BackupContainerName != "" {
fmt.Printf("backup_container_name=%s\n", result.BackupContainerName)
}
fmt.Println()
}
fmt.Printf("summary_rollout=%d\nsummary_error=%d\nsummary_total=%d\n", rolloutCount, failureCount, len(tenants))
if failureCount > 0 {
return fmt.Errorf("%d tenant runtime rollouts failed", failureCount)
}
return nil
}
func newTenantRuntimeReconcileCmd() *cobra.Command {
var tenantIDs []string
var all bool
@@ -273,6 +357,50 @@ func printTenantRuntimeReconcilePlan(plan *cloudcp.TenantRuntimeContractReconcil
fmt.Printf("summary_rollout=%d\nsummary_noop=%d\nsummary_skip=%d\nsummary_total=%d\n", rolloutCount, noopCount, skipCount, len(plan.Tenants))
}
func printTenantRuntimeImageRolloutPlan(plan *cloudcp.TenantRuntimeImageRolloutPlan) {
if plan == nil {
fmt.Println("summary_total=0")
return
}
rolloutCount := 0
noopCount := 0
skipCount := 0
for _, item := range plan.Tenants {
if item == nil {
continue
}
switch item.Action {
case "rollout":
rolloutCount++
case "noop":
noopCount++
default:
skipCount++
}
fmt.Printf("tenant_id=%s\naction=%s\nreason=%s\n", item.TenantID, item.Action, item.Reason)
if item.State != "" {
fmt.Printf("tenant_state=%s\n", item.State)
}
if item.LiveContainerID != "" {
fmt.Printf("live_container_id=%s\n", item.LiveContainerID)
}
if item.LiveImageRef != "" {
fmt.Printf("live_image_ref=%s\n", item.LiveImageRef)
}
if item.TargetImageRef != "" {
fmt.Printf("target_image_ref=%s\n", item.TargetImageRef)
}
if item.LiveRouteHost != "" || item.DesiredRouteHost != "" {
fmt.Printf("live_route_host=%s\ndesired_route_host=%s\n", item.LiveRouteHost, item.DesiredRouteHost)
}
if item.LivePublicURL != "" || item.DesiredPublicURL != "" {
fmt.Printf("live_public_url=%s\ndesired_public_url=%s\n", item.LivePublicURL, item.DesiredPublicURL)
}
fmt.Println()
}
fmt.Printf("summary_rollout=%d\nsummary_noop=%d\nsummary_skip=%d\nsummary_total=%d\n", rolloutCount, noopCount, skipCount, len(plan.Tenants))
}
func printCloudAuditReport(report *cloudcp.CloudAuditReport) {
if report == nil {
fmt.Println("audit_ok=false")
+73 -6
View File
@@ -10,6 +10,7 @@ import (
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp"
cpDocker "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/docker"
"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry"
pkglicensing "github.com/rcourtman/pulse-go-rewrite/pkg/licensing"
"github.com/spf13/cobra"
@@ -64,10 +65,16 @@ type providerMSPBackupStatus struct {
type providerMSPStatusDependencies struct {
OpenRegistry func(*cloudcp.CPConfig) (*registry.TenantRegistry, error)
RunPreflight func(context.Context, *cloudcp.CPConfig, providerMSPPreflightOptions) (*providerMSPPreflightReport, error)
NewDocker func(*cloudcp.CPConfig) (providerMSPStatusDocker, error)
CheckBackup func(context.Context, *cloudcp.CPConfig) (*providerMSPBackupStatus, error)
Now func() time.Time
}
type providerMSPStatusDocker interface {
HealthCheck(context.Context, string) (bool, error)
Close() error
}
func newProviderMSPStatusCmd() *cobra.Command {
opts := providerMSPStatusOptions{}
cmd := &cobra.Command{
@@ -190,14 +197,17 @@ func runProviderMSPStatusWithDependencies(ctx context.Context, cfg *cloudcp.CPCo
}
}
healthy, unhealthy, err := reg.HealthSummary()
health, err := checkProviderMSPLiveRuntimeHealth(ctx, cfg, reg, deps)
if err != nil {
addFailure("tenant health summary: %v", err)
addFailure("tenant runtime health summary: %v", err)
} else {
report.HealthyTenants = healthy
report.UnhealthyTenants = unhealthy
if unhealthy > 0 {
addFailure("unhealthy active workspaces: %d", unhealthy)
report.HealthyTenants = health.Healthy
report.UnhealthyTenants = health.Unhealthy
for _, failure := range health.Failures {
addFailure("runtime health: %s", failure)
}
if health.Unhealthy > 0 {
addFailure("unhealthy active workspaces: %d", health.Unhealthy)
}
}
@@ -243,6 +253,11 @@ func normalizeProviderMSPStatusDependencies(deps providerMSPStatusDependencies)
if deps.RunPreflight == nil {
deps.RunPreflight = runProviderMSPPreflight
}
if deps.NewDocker == nil {
deps.NewDocker = func(cfg *cloudcp.CPConfig) (providerMSPStatusDocker, error) {
return cpDocker.NewManager(providerMSPDockerManagerConfig(cfg))
}
}
if deps.CheckBackup == nil {
deps.CheckBackup = checkProviderMSPBackupStatus
}
@@ -252,6 +267,58 @@ func normalizeProviderMSPStatusDependencies(deps providerMSPStatusDependencies)
return deps
}
type providerMSPLiveRuntimeHealth struct {
Healthy int
Unhealthy int
Failures []string
}
func checkProviderMSPLiveRuntimeHealth(
ctx context.Context,
cfg *cloudcp.CPConfig,
reg *registry.TenantRegistry,
deps providerMSPStatusDependencies,
) (*providerMSPLiveRuntimeHealth, error) {
if reg == nil {
return nil, fmt.Errorf("tenant registry is required")
}
dockerMgr, err := deps.NewDocker(cfg)
if err != nil {
return nil, fmt.Errorf("create Docker manager: %w", err)
}
defer dockerMgr.Close()
tenants, err := reg.ListByState(registry.TenantStateActive)
if err != nil {
return nil, fmt.Errorf("list active tenants: %w", err)
}
health := &providerMSPLiveRuntimeHealth{}
for _, tenant := range tenants {
if tenant == nil {
continue
}
containerID := strings.TrimSpace(tenant.ContainerID)
if containerID == "" {
health.Unhealthy++
health.Failures = append(health.Failures, fmt.Sprintf("workspace %s has no runtime container id", tenant.ID))
continue
}
healthy, checkErr := dockerMgr.HealthCheck(ctx, containerID)
if checkErr != nil {
health.Unhealthy++
health.Failures = append(health.Failures, fmt.Sprintf("workspace %s container %s: %v", tenant.ID, containerID, checkErr))
continue
}
if healthy {
health.Healthy++
} else {
health.Unhealthy++
}
}
return health, nil
}
func checkProviderMSPBackupStatus(ctx context.Context, cfg *cloudcp.CPConfig) (*providerMSPBackupStatus, error) {
if cfg == nil {
return nil, fmt.Errorf("control plane config is required")
@@ -2,6 +2,7 @@ package main
import (
"context"
"fmt"
"strings"
"testing"
"time"
@@ -26,9 +27,10 @@ func TestProviderMSPStatusReportsHealthyOperatorSurface(t *testing.T) {
now := time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC)
createProviderMSPStatusTenant(t, cfg, &registry.Tenant{
ID: "t-HEALTHY",
ContainerID: "c-healthy",
State: registry.TenantStateActive,
CreatedAt: now.Add(-time.Hour),
HealthCheckOK: true,
HealthCheckOK: false,
})
var gotPreflight providerMSPPreflightOptions
@@ -37,6 +39,9 @@ func TestProviderMSPStatusReportsHealthyOperatorSurface(t *testing.T) {
gotPreflight = opts
return healthyProviderMSPStatusPreflightReport(), nil
},
NewDocker: healthyProviderMSPStatusDocker(map[string]bool{
"c-healthy": true,
}),
CheckBackup: func(context.Context, *cloudcp.CPConfig) (*providerMSPBackupStatus, error) {
return healthyProviderMSPBackupStatus(now), nil
},
@@ -73,9 +78,10 @@ func TestProviderMSPStatusFailsOnFailedUnhealthyAndStuckWorkspaces(t *testing.T)
now := time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC)
createProviderMSPStatusTenant(t, cfg, &registry.Tenant{
ID: "t-UNHEALTHY",
ContainerID: "c-unhealthy",
State: registry.TenantStateActive,
CreatedAt: now.Add(-time.Hour),
HealthCheckOK: false,
HealthCheckOK: true,
})
createProviderMSPStatusTenant(t, cfg, &registry.Tenant{
ID: "t-FAILED",
@@ -92,6 +98,9 @@ func TestProviderMSPStatusFailsOnFailedUnhealthyAndStuckWorkspaces(t *testing.T)
RunPreflight: func(context.Context, *cloudcp.CPConfig, providerMSPPreflightOptions) (*providerMSPPreflightReport, error) {
return healthyProviderMSPStatusPreflightReport(), nil
},
NewDocker: healthyProviderMSPStatusDocker(map[string]bool{
"c-unhealthy": false,
}),
Now: func() time.Time { return now },
})
if err == nil {
@@ -118,6 +127,7 @@ func TestProviderMSPStatusBackupWarningBecomesFailureWhenRequired(t *testing.T)
RunPreflight: func(context.Context, *cloudcp.CPConfig, providerMSPPreflightOptions) (*providerMSPPreflightReport, error) {
return healthyProviderMSPStatusPreflightReport(), nil
},
NewDocker: healthyProviderMSPStatusDocker(nil),
CheckBackup: func(context.Context, *cloudcp.CPConfig) (*providerMSPBackupStatus, error) {
return &providerMSPBackupStatus{
Directory: "/data/backups/provider-msp",
@@ -150,6 +160,31 @@ func TestProviderMSPStatusBackupWarningBecomesFailureWhenRequired(t *testing.T)
}
}
func healthyProviderMSPStatusDocker(health map[string]bool) func(*cloudcp.CPConfig) (providerMSPStatusDocker, error) {
return func(*cloudcp.CPConfig) (providerMSPStatusDocker, error) {
return &fakeProviderMSPStatusDocker{health: health}, nil
}
}
type fakeProviderMSPStatusDocker struct {
health map[string]bool
}
func (f *fakeProviderMSPStatusDocker) HealthCheck(_ context.Context, containerID string) (bool, error) {
if f.health == nil {
return false, fmt.Errorf("unexpected health check for %s", containerID)
}
healthy, ok := f.health[containerID]
if !ok {
return false, fmt.Errorf("unexpected health check for %s", containerID)
}
return healthy, nil
}
func (f *fakeProviderMSPStatusDocker) Close() error {
return nil
}
func healthyProviderMSPStatusPreflightReport() *providerMSPPreflightReport {
return &providerMSPPreflightReport{
OK: true,
+3 -3
View File
@@ -67,11 +67,11 @@ PULSE_EMAIL_REPLY_TO=support@example.com
# Apply a provider control-plane upgrade after updating the image pins in this
# file. The runner creates and verifies a fresh backup, dry-runs restore into a
# separate target data dir, pulls provider images, starts Traefik/control-plane,
# and prints the tenant runtime reconcile plan:
# and prints the tenant runtime rollout plan for CP_PULSE_IMAGE:
# ./upgrade.sh
#
# Also reconcile all client runtimes to the configured CP_PULSE_IMAGE line after
# the provider services are updated:
# Also roll all client runtimes to the configured CP_PULSE_IMAGE line after the
# provider services are updated:
# ./upgrade.sh --rollout-tenants
#
# Fresh-install provider proof. This bootstraps the owner account, checks install
+13 -7
View File
@@ -13,12 +13,12 @@ Runs the provider-hosted MSP pre-upgrade and upgrade flow:
3. creates and verifies a fresh provider MSP backup
4. dry-runs restore into a separate target data directory
5. pulls and starts the provider control-plane and Traefik services
6. prints the tenant runtime reconcile plan
7. optionally reconciles all tenant runtimes onto the configured runtime line
6. prints the tenant runtime rollout plan for CP_PULSE_IMAGE
7. optionally rolls all tenant runtimes onto CP_PULSE_IMAGE
Options:
--dry-run Print the non-mutating upgrade plan only
--rollout-tenants Execute tenant-runtime reconcile --all after provider services are updated
--rollout-tenants Execute tenant-runtime rollout --all after provider services are updated
--prune-previous Remove preserved pre-rollout tenant containers after successful tenant rollout
--skip-compose-pull Do not run docker compose pull for provider services
--skip-runtime-image-pull Pass --skip-image-pull to provider-msp preflight
@@ -144,6 +144,10 @@ fi
provider_data_dir="$(env_value PULSE_PROVIDER_MSP_DATA_DIR .env)"
provider_data_dir="${provider_data_dir:-/data}"
tenant_runtime_image="$(env_value CP_PULSE_IMAGE .env)"
if [[ -z "${tenant_runtime_image}" ]]; then
die "CP_PULSE_IMAGE is required"
fi
if [[ -z "${restore_target}" ]]; then
restore_target="${provider_data_dir%/}/upgrade-restore-drill"
fi
@@ -163,12 +167,13 @@ fi
echo "provider_msp_upgrade_dry_run=$(truthy "${dry_run}" && echo true || echo false)"
echo "provider_msp_upgrade_run_id=${run_id}"
echo "provider_msp_upgrade_restore_target=${restore_target}"
echo "provider_msp_upgrade_tenant_runtime_image=${tenant_runtime_image}"
run_control provider-msp status
run_control "${preflight_args[@]}"
if truthy "${dry_run}"; then
run_control tenant-runtime reconcile --all --dry-run
run_control tenant-runtime rollout --all --image "${tenant_runtime_image}" --dry-run
echo "tenant_runtime_rollout_applied=false"
echo "provider_msp_upgrade_plan_ok=true"
exit 0
@@ -195,12 +200,13 @@ if ! truthy "${skip_compose_pull}"; then
fi
docker compose up -d traefik control-plane
run_control provider-msp status --require-backup
run_control tenant-runtime reconcile --all --dry-run
run_control tenant-runtime rollout --all --image "${tenant_runtime_image}" --dry-run
if truthy "${rollout_tenants}"; then
reconcile_args=(
tenant-runtime reconcile
tenant-runtime rollout
--all
--image "${tenant_runtime_image}"
--run-id "${run_id}"
--health-timeout "${health_timeout}"
)
@@ -212,7 +218,7 @@ if truthy "${rollout_tenants}"; then
echo "tenant_runtime_rollout_applied=true"
else
echo "tenant_runtime_rollout_applied=false"
echo "tenant_runtime_rollout_next_command=docker compose run --rm --no-deps control-plane tenant-runtime reconcile --all --run-id ${run_id} --health-timeout ${health_timeout}"
echo "tenant_runtime_rollout_next_command=docker compose run --rm --no-deps control-plane tenant-runtime rollout --all --image ${tenant_runtime_image} --run-id ${run_id} --health-timeout ${health_timeout}"
fi
docker compose ps
@@ -212,6 +212,13 @@ tenant-local runtime state without depending on Stripe billing surfaces.
coherent.
Tenant runtime rollout and missing-runtime restore must fail closed on the
same storage admission guard before snapshotting or swapping containers.
`tenant-runtime rollout --all --image <target>` is the canonical hosted
fleet image upgrade path for active tenant runtimes. Its `--dry-run` mode
must print the target-image rollout plan before mutation, `--all` must
select active tenants only, and apply must still use the canonical
per-tenant snapshot, health-check, and rollback path. `tenant-runtime
reconcile --all` remains contract/routing drift repair for each tenant's
current image line, not an image-line upgrade path.
The real `pulse-pro` license-server legacy checkout issuance, recurring
renewals, manual issue, and legacy exchange flows are part of that same
@@ -156,9 +156,9 @@ surfaces.
provider status and preflight, create and verify a fresh backup before
apply, dry-run restore into a separate target data dir, require backup
readiness before and after provider service replacement, update the packaged
Traefik/control-plane services, print the tenant runtime reconcile plan, and
only execute `tenant-runtime reconcile --all` when the operator explicitly
asks for tenant rollout.
Traefik/control-plane services, print the tenant runtime rollout plan for
`CP_PULSE_IMAGE`, and only execute `tenant-runtime rollout --all --image
<CP_PULSE_IMAGE>` when the operator explicitly asks for tenant rollout.
`deploy/provider-msp/setup.sh` is the first-time provider host setup
artifact. It must install the Docker/compose host prerequisites, create the
provider data, backup, and Docker-network layout, copy the provider MSP
+162
View File
@@ -67,6 +67,30 @@ type TenantRuntimeContractReconcilePlan struct {
Tenants []*TenantRuntimeContractReconcilePlanItem
}
type TenantRuntimeImageRolloutPlanOptions struct {
TenantIDs []string
All bool
Image string
}
type TenantRuntimeImageRolloutPlanItem struct {
TenantID string
State string
LiveContainerID string
LiveImageRef string
TargetImageRef string
LiveRouteHost string
DesiredRouteHost string
LivePublicURL string
DesiredPublicURL string
Action string
Reason string
}
type TenantRuntimeImageRolloutPlan struct {
Tenants []*TenantRuntimeImageRolloutPlanItem
}
const (
tenantRuntimeContractActionNoop = "noop"
tenantRuntimeContractActionRollout = "rollout"
@@ -147,6 +171,31 @@ func PlanTenantRuntimeContractReconcile(
return service.PlanContractReconcile(ctx, opts)
}
func PlanTenantRuntimeImageRollout(
ctx context.Context,
cfg *CPConfig,
opts TenantRuntimeImageRolloutPlanOptions,
) (*TenantRuntimeImageRolloutPlan, error) {
if cfg == nil {
return nil, fmt.Errorf("control plane config is required")
}
image := strings.TrimSpace(opts.Image)
if image == "" {
image = strings.TrimSpace(cfg.PulseImage)
}
if image == "" {
return nil, fmt.Errorf("missing tenant runtime image")
}
opts.Image = image
service, cleanup, err := newTenantRuntimeRolloutServiceFromConfig(cfg, image)
if err != nil {
return nil, err
}
defer cleanup()
return service.PlanImageRollout(ctx, opts)
}
func newTenantRuntimeRolloutServiceFromConfig(
cfg *CPConfig,
image string,
@@ -532,6 +581,77 @@ func (s *tenantRuntimeRolloutService) PlanContractReconcile(
return plan, nil
}
func (s *tenantRuntimeRolloutService) PlanImageRollout(
ctx context.Context,
opts TenantRuntimeImageRolloutPlanOptions,
) (*TenantRuntimeImageRolloutPlan, error) {
if s == nil {
return nil, fmt.Errorf("rollout service is nil")
}
targetImage := strings.TrimSpace(opts.Image)
if targetImage == "" {
targetImage = strings.TrimSpace(s.defaultImage)
}
if targetImage == "" {
return nil, fmt.Errorf("image is required")
}
tenants, err := s.selectImageRolloutTenants(opts)
if err != nil {
return nil, err
}
plan := &TenantRuntimeImageRolloutPlan{
Tenants: make([]*TenantRuntimeImageRolloutPlanItem, 0, len(tenants)),
}
for _, tenant := range tenants {
item := &TenantRuntimeImageRolloutPlanItem{
TenantID: strings.TrimSpace(tenant.ID),
State: string(tenant.State),
TargetImageRef: targetImage,
}
if tenant.State != registry.TenantStateActive {
item.Action = tenantRuntimeContractActionSkip
item.Reason = fmt.Sprintf("tenant state is %s, not active", tenant.State)
plan.Tenants = append(plan.Tenants, item)
continue
}
desiredRouting := s.docker.DesiredRuntimeRouting(tenant.ID)
item.DesiredRouteHost = desiredRouting.Host
item.DesiredPublicURL = desiredRouting.PublicURL
live, err := s.resolveLiveContainer(ctx, tenant)
if err != nil {
if errors.Is(err, errTenantRuntimeMissing) {
item.Action = tenantRuntimeContractActionRollout
item.Reason = "tenant runtime container is missing; recreate from existing tenant data"
} else {
item.Action = tenantRuntimeContractActionSkip
item.Reason = err.Error()
}
plan.Tenants = append(plan.Tenants, item)
continue
}
item.LiveContainerID = strings.TrimSpace(live.ID)
item.LiveImageRef = strings.TrimSpace(live.ImageRef)
item.LiveRouteHost = strings.TrimSpace(live.RouteHost)
item.LivePublicURL = strings.TrimSpace(live.PublicURL)
if tenantRuntimeMatchesContract(live, tenantRuntimeContainerName(tenant.ID), targetImage, desiredRouting) {
item.Action = tenantRuntimeContractActionNoop
item.Reason = "runtime already matches target image and canonical hosted contract"
} else if item.LiveImageRef != targetImage {
item.Action = tenantRuntimeContractActionRollout
item.Reason = "runtime image differs from target"
} else {
item.Action = tenantRuntimeContractActionRollout
item.Reason = "runtime contract drift detected"
}
plan.Tenants = append(plan.Tenants, item)
}
return plan, nil
}
func (s *tenantRuntimeRolloutService) selectContractReconcileTenants(
opts TenantRuntimeContractReconcilePlanOptions,
) ([]*registry.Tenant, error) {
@@ -569,6 +689,48 @@ func (s *tenantRuntimeRolloutService) selectContractReconcileTenants(
return tenants, nil
}
func (s *tenantRuntimeRolloutService) selectImageRolloutTenants(
opts TenantRuntimeImageRolloutPlanOptions,
) ([]*registry.Tenant, error) {
if s == nil {
return nil, fmt.Errorf("rollout service is nil")
}
if opts.All && len(dedupeNonEmptyStrings(opts.TenantIDs)) > 0 {
return nil, fmt.Errorf("choose either --all or one or more tenant ids")
}
if opts.All {
tenants, err := s.registry.List()
if err != nil {
return nil, fmt.Errorf("list tenants: %w", err)
}
active := make([]*registry.Tenant, 0, len(tenants))
for _, tenant := range tenants {
if tenant != nil && tenant.State == registry.TenantStateActive {
active = append(active, tenant)
}
}
return active, nil
}
tenantIDs := dedupeNonEmptyStrings(opts.TenantIDs)
if len(tenantIDs) == 0 {
return nil, fmt.Errorf("at least one tenant id or --all is required")
}
tenants := make([]*registry.Tenant, 0, len(tenantIDs))
for _, tenantID := range tenantIDs {
tenant, err := s.registry.Get(tenantID)
if err != nil {
return nil, fmt.Errorf("load tenant %s: %w", tenantID, err)
}
if tenant == nil {
return nil, fmt.Errorf("tenant %s not found", tenantID)
}
tenants = append(tenants, tenant)
}
return tenants, nil
}
func dedupeNonEmptyStrings(values []string) []string {
result := make([]string, 0, len(values))
seen := make(map[string]struct{}, len(values))
@@ -543,6 +543,68 @@ func TestTenantRuntimeContractReconcilePlan_ExplicitTenantIDsDedupesAndPreserves
}
}
func TestTenantRuntimeImageRolloutPlan_AllActiveTenantsTargetsConfiguredImage(t *testing.T) {
tenantOldImage := &registry.Tenant{ID: "t-OLDIMAGE1", State: registry.TenantStateActive, ContainerID: "old-live"}
tenantTargetImage := &registry.Tenant{ID: "t-TARGET01", State: registry.TenantStateActive, ContainerID: "target-live"}
tenantMissing := &registry.Tenant{ID: "t-MISSING02", State: registry.TenantStateActive, ContainerID: ""}
tenantSuspended := &registry.Tenant{ID: "t-SUSPEND1", State: registry.TenantStateSuspended, ContainerID: "suspended-live"}
reg := &fakeTenantRuntimeRolloutRegistry{
tenants: []*registry.Tenant{tenantOldImage, tenantTargetImage, tenantMissing, tenantSuspended},
}
docker := newFakeTenantRuntimeRolloutDocker()
targetImage := "pulse-runtime:next"
for _, tc := range []struct {
tenant *registry.Tenant
image string
}{
{tenantOldImage, "pulse-runtime:stable"},
{tenantTargetImage, targetImage},
{tenantSuspended, "pulse-runtime:stable"},
} {
routing := docker.DesiredRuntimeRouting(tc.tenant.ID)
docker.addContainer(&cpDocker.RuntimeContainerInfo{
ID: tc.tenant.ContainerID,
Name: tenantRuntimeContainerName(tc.tenant.ID),
ImageRef: tc.image,
ImageID: "sha256:test",
Running: true,
RouteHost: routing.Host,
PublicURL: routing.PublicURL,
})
}
service := newTestTenantRuntimeRolloutService(reg, docker, &fakeTenantRuntimeRolloutSynchronizer{}, newFakeTenantRuntimeRolloutClock())
plan, err := service.PlanImageRollout(context.Background(), TenantRuntimeImageRolloutPlanOptions{
All: true,
Image: targetImage,
})
if err != nil {
t.Fatalf("PlanImageRollout() error = %v", err)
}
if len(plan.Tenants) != 3 {
t.Fatalf("plan tenant count = %d, want 3 active tenants", len(plan.Tenants))
}
got := make(map[string]*TenantRuntimeImageRolloutPlanItem, len(plan.Tenants))
for _, item := range plan.Tenants {
got[item.TenantID] = item
}
if got[tenantOldImage.ID].Action != tenantRuntimeContractActionRollout {
t.Fatalf("old image action = %q, want rollout", got[tenantOldImage.ID].Action)
}
if got[tenantOldImage.ID].LiveImageRef != "pulse-runtime:stable" || got[tenantOldImage.ID].TargetImageRef != targetImage {
t.Fatalf("old image refs = live %q target %q", got[tenantOldImage.ID].LiveImageRef, got[tenantOldImage.ID].TargetImageRef)
}
if got[tenantTargetImage.ID].Action != tenantRuntimeContractActionNoop {
t.Fatalf("target image action = %q, want noop", got[tenantTargetImage.ID].Action)
}
if got[tenantMissing.ID].Action != tenantRuntimeContractActionRollout {
t.Fatalf("missing tenant action = %q, want rollout", got[tenantMissing.ID].Action)
}
if _, ok := got[tenantSuspended.ID]; ok {
t.Fatalf("suspended tenant should not be selected by --all image rollout plan")
}
}
func newTestTenantRuntimeRolloutService(
reg tenantRuntimeRolloutRegistry,
docker tenantRuntimeRolloutDocker,
@@ -160,9 +160,10 @@ func TestProviderMSPUpgradeRunnerMatchesComposeContract(t *testing.T) {
"provider-msp backup verify",
"provider-msp backup restore",
"--target-data-dir",
"tenant-runtime reconcile --all --dry-run",
"tenant-runtime reconcile",
"tenant-runtime rollout --all --image",
"tenant-runtime rollout",
"--all",
"--image",
"--run-id",
"--health-timeout",
"--prune-previous",