mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 02:25:56 +00:00
Add provider MSP tenant rollout upgrade proof
This commit is contained in:
@@ -85,11 +85,13 @@ func newCloudAuditCmd() *cobra.Command {
|
||||
|
||||
func newTenantRuntimeRolloutCmd() *cobra.Command {
|
||||
var tenantID string
|
||||
var all bool
|
||||
var image string
|
||||
var runID string
|
||||
var snapshotRoot string
|
||||
var healthTimeout time.Duration
|
||||
var prunePrevious bool
|
||||
var dryRun bool
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "rollout",
|
||||
@@ -99,6 +101,37 @@ func newTenantRuntimeRolloutCmd() *cobra.Command {
|
||||
if err != nil {
|
||||
return fmt.Errorf("load control plane config: %w", err)
|
||||
}
|
||||
if all && strings.TrimSpace(tenantID) != "" {
|
||||
return fmt.Errorf("choose either --all or --tenant-id")
|
||||
}
|
||||
if !all && strings.TrimSpace(tenantID) == "" {
|
||||
return fmt.Errorf("--tenant-id is required unless --all is set")
|
||||
}
|
||||
if dryRun {
|
||||
tenantIDs := []string{tenantID}
|
||||
if all {
|
||||
tenantIDs = nil
|
||||
}
|
||||
plan, err := cloudcp.PlanTenantRuntimeImageRollout(cmd.Context(), cfg, cloudcp.TenantRuntimeImageRolloutPlanOptions{
|
||||
TenantIDs: tenantIDs,
|
||||
All: all,
|
||||
Image: image,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
printTenantRuntimeImageRolloutPlan(plan)
|
||||
return nil
|
||||
}
|
||||
if all {
|
||||
return runTenantRuntimeRolloutAll(cmd.Context(), cfg, cloudcp.TenantRuntimeRolloutOptions{
|
||||
Image: image,
|
||||
RunID: runID,
|
||||
SnapshotRoot: snapshotRoot,
|
||||
HealthTimeout: healthTimeout,
|
||||
PrunePrevious: prunePrevious,
|
||||
})
|
||||
}
|
||||
result, err := cloudcp.RolloutTenantRuntime(cmd.Context(), cfg, cloudcp.TenantRuntimeRolloutOptions{
|
||||
TenantID: tenantID,
|
||||
Image: image,
|
||||
@@ -126,16 +159,67 @@ func newTenantRuntimeRolloutCmd() *cobra.Command {
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVar(&tenantID, "tenant-id", "", "Hosted tenant ID to roll")
|
||||
cmd.Flags().BoolVar(&all, "all", false, "Roll all active hosted tenants")
|
||||
cmd.Flags().StringVar(&image, "image", "", "Target Pulse runtime image reference")
|
||||
cmd.Flags().StringVar(&runID, "run-id", "", "Operator-visible rollout run identifier")
|
||||
cmd.Flags().StringVar(&snapshotRoot, "snapshot-root", "", "Override tenant snapshot root (default: <CP_DATA_DIR>/backups/rollout)")
|
||||
cmd.Flags().DurationVar(&healthTimeout, "health-timeout", 90*time.Second, "How long to wait for the target runtime to become healthy")
|
||||
cmd.Flags().BoolVar(&prunePrevious, "prune-previous", false, "Remove the preserved pre-rollout container after success")
|
||||
_ = cmd.MarkFlagRequired("tenant-id")
|
||||
cmd.Flags().BoolVar(&dryRun, "dry-run", false, "Print the target image rollout plan without mutating tenant runtimes")
|
||||
_ = cmd.MarkFlagRequired("image")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func runTenantRuntimeRolloutAll(ctx context.Context, cfg *cloudcp.CPConfig, opts cloudcp.TenantRuntimeRolloutOptions) error {
|
||||
reg, err := registry.NewTenantRegistry(cfg.ControlPlaneDir())
|
||||
if err != nil {
|
||||
return fmt.Errorf("open tenant registry: %w", err)
|
||||
}
|
||||
defer reg.Close()
|
||||
tenants, err := reg.ListByState(registry.TenantStateActive)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list active tenants: %w", err)
|
||||
}
|
||||
|
||||
rolloutCount := 0
|
||||
failureCount := 0
|
||||
for _, tenant := range tenants {
|
||||
if tenant == nil || strings.TrimSpace(tenant.ID) == "" {
|
||||
continue
|
||||
}
|
||||
opts.TenantID = tenant.ID
|
||||
result, err := cloudcp.RolloutTenantRuntime(ctx, cfg, opts)
|
||||
if err != nil {
|
||||
failureCount++
|
||||
fmt.Printf("tenant_id=%s\nstatus=error\nimage_ref=%s\nerror=%v\n\n", tenant.ID, opts.Image, err)
|
||||
continue
|
||||
}
|
||||
rolloutCount++
|
||||
fmt.Printf("tenant_id=%s\nstatus=rolled\nactive_container_id=%s\nactive_image_ref=%s\nactive_image_id=%s\nreconciled_only=%t\n",
|
||||
result.TenantID,
|
||||
result.ActiveContainerID,
|
||||
result.ActiveImageRef,
|
||||
result.ActiveImageID,
|
||||
result.ReconciledOnly,
|
||||
)
|
||||
if result.RestoredMissing {
|
||||
fmt.Printf("restored_missing=%t\n", result.RestoredMissing)
|
||||
}
|
||||
if result.PreviousContainerID != "" {
|
||||
fmt.Printf("previous_container_id=%s\n", result.PreviousContainerID)
|
||||
}
|
||||
if result.BackupContainerName != "" {
|
||||
fmt.Printf("backup_container_name=%s\n", result.BackupContainerName)
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
fmt.Printf("summary_rollout=%d\nsummary_error=%d\nsummary_total=%d\n", rolloutCount, failureCount, len(tenants))
|
||||
if failureCount > 0 {
|
||||
return fmt.Errorf("%d tenant runtime rollouts failed", failureCount)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func newTenantRuntimeReconcileCmd() *cobra.Command {
|
||||
var tenantIDs []string
|
||||
var all bool
|
||||
@@ -273,6 +357,50 @@ func printTenantRuntimeReconcilePlan(plan *cloudcp.TenantRuntimeContractReconcil
|
||||
fmt.Printf("summary_rollout=%d\nsummary_noop=%d\nsummary_skip=%d\nsummary_total=%d\n", rolloutCount, noopCount, skipCount, len(plan.Tenants))
|
||||
}
|
||||
|
||||
func printTenantRuntimeImageRolloutPlan(plan *cloudcp.TenantRuntimeImageRolloutPlan) {
|
||||
if plan == nil {
|
||||
fmt.Println("summary_total=0")
|
||||
return
|
||||
}
|
||||
rolloutCount := 0
|
||||
noopCount := 0
|
||||
skipCount := 0
|
||||
for _, item := range plan.Tenants {
|
||||
if item == nil {
|
||||
continue
|
||||
}
|
||||
switch item.Action {
|
||||
case "rollout":
|
||||
rolloutCount++
|
||||
case "noop":
|
||||
noopCount++
|
||||
default:
|
||||
skipCount++
|
||||
}
|
||||
fmt.Printf("tenant_id=%s\naction=%s\nreason=%s\n", item.TenantID, item.Action, item.Reason)
|
||||
if item.State != "" {
|
||||
fmt.Printf("tenant_state=%s\n", item.State)
|
||||
}
|
||||
if item.LiveContainerID != "" {
|
||||
fmt.Printf("live_container_id=%s\n", item.LiveContainerID)
|
||||
}
|
||||
if item.LiveImageRef != "" {
|
||||
fmt.Printf("live_image_ref=%s\n", item.LiveImageRef)
|
||||
}
|
||||
if item.TargetImageRef != "" {
|
||||
fmt.Printf("target_image_ref=%s\n", item.TargetImageRef)
|
||||
}
|
||||
if item.LiveRouteHost != "" || item.DesiredRouteHost != "" {
|
||||
fmt.Printf("live_route_host=%s\ndesired_route_host=%s\n", item.LiveRouteHost, item.DesiredRouteHost)
|
||||
}
|
||||
if item.LivePublicURL != "" || item.DesiredPublicURL != "" {
|
||||
fmt.Printf("live_public_url=%s\ndesired_public_url=%s\n", item.LivePublicURL, item.DesiredPublicURL)
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
fmt.Printf("summary_rollout=%d\nsummary_noop=%d\nsummary_skip=%d\nsummary_total=%d\n", rolloutCount, noopCount, skipCount, len(plan.Tenants))
|
||||
}
|
||||
|
||||
func printCloudAuditReport(report *cloudcp.CloudAuditReport) {
|
||||
if report == nil {
|
||||
fmt.Println("audit_ok=false")
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp"
|
||||
cpDocker "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/docker"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry"
|
||||
pkglicensing "github.com/rcourtman/pulse-go-rewrite/pkg/licensing"
|
||||
"github.com/spf13/cobra"
|
||||
@@ -64,10 +65,16 @@ type providerMSPBackupStatus struct {
|
||||
type providerMSPStatusDependencies struct {
|
||||
OpenRegistry func(*cloudcp.CPConfig) (*registry.TenantRegistry, error)
|
||||
RunPreflight func(context.Context, *cloudcp.CPConfig, providerMSPPreflightOptions) (*providerMSPPreflightReport, error)
|
||||
NewDocker func(*cloudcp.CPConfig) (providerMSPStatusDocker, error)
|
||||
CheckBackup func(context.Context, *cloudcp.CPConfig) (*providerMSPBackupStatus, error)
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type providerMSPStatusDocker interface {
|
||||
HealthCheck(context.Context, string) (bool, error)
|
||||
Close() error
|
||||
}
|
||||
|
||||
func newProviderMSPStatusCmd() *cobra.Command {
|
||||
opts := providerMSPStatusOptions{}
|
||||
cmd := &cobra.Command{
|
||||
@@ -190,14 +197,17 @@ func runProviderMSPStatusWithDependencies(ctx context.Context, cfg *cloudcp.CPCo
|
||||
}
|
||||
}
|
||||
|
||||
healthy, unhealthy, err := reg.HealthSummary()
|
||||
health, err := checkProviderMSPLiveRuntimeHealth(ctx, cfg, reg, deps)
|
||||
if err != nil {
|
||||
addFailure("tenant health summary: %v", err)
|
||||
addFailure("tenant runtime health summary: %v", err)
|
||||
} else {
|
||||
report.HealthyTenants = healthy
|
||||
report.UnhealthyTenants = unhealthy
|
||||
if unhealthy > 0 {
|
||||
addFailure("unhealthy active workspaces: %d", unhealthy)
|
||||
report.HealthyTenants = health.Healthy
|
||||
report.UnhealthyTenants = health.Unhealthy
|
||||
for _, failure := range health.Failures {
|
||||
addFailure("runtime health: %s", failure)
|
||||
}
|
||||
if health.Unhealthy > 0 {
|
||||
addFailure("unhealthy active workspaces: %d", health.Unhealthy)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -243,6 +253,11 @@ func normalizeProviderMSPStatusDependencies(deps providerMSPStatusDependencies)
|
||||
if deps.RunPreflight == nil {
|
||||
deps.RunPreflight = runProviderMSPPreflight
|
||||
}
|
||||
if deps.NewDocker == nil {
|
||||
deps.NewDocker = func(cfg *cloudcp.CPConfig) (providerMSPStatusDocker, error) {
|
||||
return cpDocker.NewManager(providerMSPDockerManagerConfig(cfg))
|
||||
}
|
||||
}
|
||||
if deps.CheckBackup == nil {
|
||||
deps.CheckBackup = checkProviderMSPBackupStatus
|
||||
}
|
||||
@@ -252,6 +267,58 @@ func normalizeProviderMSPStatusDependencies(deps providerMSPStatusDependencies)
|
||||
return deps
|
||||
}
|
||||
|
||||
type providerMSPLiveRuntimeHealth struct {
|
||||
Healthy int
|
||||
Unhealthy int
|
||||
Failures []string
|
||||
}
|
||||
|
||||
func checkProviderMSPLiveRuntimeHealth(
|
||||
ctx context.Context,
|
||||
cfg *cloudcp.CPConfig,
|
||||
reg *registry.TenantRegistry,
|
||||
deps providerMSPStatusDependencies,
|
||||
) (*providerMSPLiveRuntimeHealth, error) {
|
||||
if reg == nil {
|
||||
return nil, fmt.Errorf("tenant registry is required")
|
||||
}
|
||||
dockerMgr, err := deps.NewDocker(cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create Docker manager: %w", err)
|
||||
}
|
||||
defer dockerMgr.Close()
|
||||
|
||||
tenants, err := reg.ListByState(registry.TenantStateActive)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list active tenants: %w", err)
|
||||
}
|
||||
|
||||
health := &providerMSPLiveRuntimeHealth{}
|
||||
for _, tenant := range tenants {
|
||||
if tenant == nil {
|
||||
continue
|
||||
}
|
||||
containerID := strings.TrimSpace(tenant.ContainerID)
|
||||
if containerID == "" {
|
||||
health.Unhealthy++
|
||||
health.Failures = append(health.Failures, fmt.Sprintf("workspace %s has no runtime container id", tenant.ID))
|
||||
continue
|
||||
}
|
||||
healthy, checkErr := dockerMgr.HealthCheck(ctx, containerID)
|
||||
if checkErr != nil {
|
||||
health.Unhealthy++
|
||||
health.Failures = append(health.Failures, fmt.Sprintf("workspace %s container %s: %v", tenant.ID, containerID, checkErr))
|
||||
continue
|
||||
}
|
||||
if healthy {
|
||||
health.Healthy++
|
||||
} else {
|
||||
health.Unhealthy++
|
||||
}
|
||||
}
|
||||
return health, nil
|
||||
}
|
||||
|
||||
func checkProviderMSPBackupStatus(ctx context.Context, cfg *cloudcp.CPConfig) (*providerMSPBackupStatus, error) {
|
||||
if cfg == nil {
|
||||
return nil, fmt.Errorf("control plane config is required")
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -26,9 +27,10 @@ func TestProviderMSPStatusReportsHealthyOperatorSurface(t *testing.T) {
|
||||
now := time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC)
|
||||
createProviderMSPStatusTenant(t, cfg, ®istry.Tenant{
|
||||
ID: "t-HEALTHY",
|
||||
ContainerID: "c-healthy",
|
||||
State: registry.TenantStateActive,
|
||||
CreatedAt: now.Add(-time.Hour),
|
||||
HealthCheckOK: true,
|
||||
HealthCheckOK: false,
|
||||
})
|
||||
|
||||
var gotPreflight providerMSPPreflightOptions
|
||||
@@ -37,6 +39,9 @@ func TestProviderMSPStatusReportsHealthyOperatorSurface(t *testing.T) {
|
||||
gotPreflight = opts
|
||||
return healthyProviderMSPStatusPreflightReport(), nil
|
||||
},
|
||||
NewDocker: healthyProviderMSPStatusDocker(map[string]bool{
|
||||
"c-healthy": true,
|
||||
}),
|
||||
CheckBackup: func(context.Context, *cloudcp.CPConfig) (*providerMSPBackupStatus, error) {
|
||||
return healthyProviderMSPBackupStatus(now), nil
|
||||
},
|
||||
@@ -73,9 +78,10 @@ func TestProviderMSPStatusFailsOnFailedUnhealthyAndStuckWorkspaces(t *testing.T)
|
||||
now := time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC)
|
||||
createProviderMSPStatusTenant(t, cfg, ®istry.Tenant{
|
||||
ID: "t-UNHEALTHY",
|
||||
ContainerID: "c-unhealthy",
|
||||
State: registry.TenantStateActive,
|
||||
CreatedAt: now.Add(-time.Hour),
|
||||
HealthCheckOK: false,
|
||||
HealthCheckOK: true,
|
||||
})
|
||||
createProviderMSPStatusTenant(t, cfg, ®istry.Tenant{
|
||||
ID: "t-FAILED",
|
||||
@@ -92,6 +98,9 @@ func TestProviderMSPStatusFailsOnFailedUnhealthyAndStuckWorkspaces(t *testing.T)
|
||||
RunPreflight: func(context.Context, *cloudcp.CPConfig, providerMSPPreflightOptions) (*providerMSPPreflightReport, error) {
|
||||
return healthyProviderMSPStatusPreflightReport(), nil
|
||||
},
|
||||
NewDocker: healthyProviderMSPStatusDocker(map[string]bool{
|
||||
"c-unhealthy": false,
|
||||
}),
|
||||
Now: func() time.Time { return now },
|
||||
})
|
||||
if err == nil {
|
||||
@@ -118,6 +127,7 @@ func TestProviderMSPStatusBackupWarningBecomesFailureWhenRequired(t *testing.T)
|
||||
RunPreflight: func(context.Context, *cloudcp.CPConfig, providerMSPPreflightOptions) (*providerMSPPreflightReport, error) {
|
||||
return healthyProviderMSPStatusPreflightReport(), nil
|
||||
},
|
||||
NewDocker: healthyProviderMSPStatusDocker(nil),
|
||||
CheckBackup: func(context.Context, *cloudcp.CPConfig) (*providerMSPBackupStatus, error) {
|
||||
return &providerMSPBackupStatus{
|
||||
Directory: "/data/backups/provider-msp",
|
||||
@@ -150,6 +160,31 @@ func TestProviderMSPStatusBackupWarningBecomesFailureWhenRequired(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func healthyProviderMSPStatusDocker(health map[string]bool) func(*cloudcp.CPConfig) (providerMSPStatusDocker, error) {
|
||||
return func(*cloudcp.CPConfig) (providerMSPStatusDocker, error) {
|
||||
return &fakeProviderMSPStatusDocker{health: health}, nil
|
||||
}
|
||||
}
|
||||
|
||||
type fakeProviderMSPStatusDocker struct {
|
||||
health map[string]bool
|
||||
}
|
||||
|
||||
func (f *fakeProviderMSPStatusDocker) HealthCheck(_ context.Context, containerID string) (bool, error) {
|
||||
if f.health == nil {
|
||||
return false, fmt.Errorf("unexpected health check for %s", containerID)
|
||||
}
|
||||
healthy, ok := f.health[containerID]
|
||||
if !ok {
|
||||
return false, fmt.Errorf("unexpected health check for %s", containerID)
|
||||
}
|
||||
return healthy, nil
|
||||
}
|
||||
|
||||
func (f *fakeProviderMSPStatusDocker) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func healthyProviderMSPStatusPreflightReport() *providerMSPPreflightReport {
|
||||
return &providerMSPPreflightReport{
|
||||
OK: true,
|
||||
|
||||
@@ -67,11 +67,11 @@ PULSE_EMAIL_REPLY_TO=support@example.com
|
||||
# Apply a provider control-plane upgrade after updating the image pins in this
|
||||
# file. The runner creates and verifies a fresh backup, dry-runs restore into a
|
||||
# separate target data dir, pulls provider images, starts Traefik/control-plane,
|
||||
# and prints the tenant runtime reconcile plan:
|
||||
# and prints the tenant runtime rollout plan for CP_PULSE_IMAGE:
|
||||
# ./upgrade.sh
|
||||
#
|
||||
# Also reconcile all client runtimes to the configured CP_PULSE_IMAGE line after
|
||||
# the provider services are updated:
|
||||
# Also roll all client runtimes to the configured CP_PULSE_IMAGE line after the
|
||||
# provider services are updated:
|
||||
# ./upgrade.sh --rollout-tenants
|
||||
#
|
||||
# Fresh-install provider proof. This bootstraps the owner account, checks install
|
||||
|
||||
@@ -13,12 +13,12 @@ Runs the provider-hosted MSP pre-upgrade and upgrade flow:
|
||||
3. creates and verifies a fresh provider MSP backup
|
||||
4. dry-runs restore into a separate target data directory
|
||||
5. pulls and starts the provider control-plane and Traefik services
|
||||
6. prints the tenant runtime reconcile plan
|
||||
7. optionally reconciles all tenant runtimes onto the configured runtime line
|
||||
6. prints the tenant runtime rollout plan for CP_PULSE_IMAGE
|
||||
7. optionally rolls all tenant runtimes onto CP_PULSE_IMAGE
|
||||
|
||||
Options:
|
||||
--dry-run Print the non-mutating upgrade plan only
|
||||
--rollout-tenants Execute tenant-runtime reconcile --all after provider services are updated
|
||||
--rollout-tenants Execute tenant-runtime rollout --all after provider services are updated
|
||||
--prune-previous Remove preserved pre-rollout tenant containers after successful tenant rollout
|
||||
--skip-compose-pull Do not run docker compose pull for provider services
|
||||
--skip-runtime-image-pull Pass --skip-image-pull to provider-msp preflight
|
||||
@@ -144,6 +144,10 @@ fi
|
||||
|
||||
provider_data_dir="$(env_value PULSE_PROVIDER_MSP_DATA_DIR .env)"
|
||||
provider_data_dir="${provider_data_dir:-/data}"
|
||||
tenant_runtime_image="$(env_value CP_PULSE_IMAGE .env)"
|
||||
if [[ -z "${tenant_runtime_image}" ]]; then
|
||||
die "CP_PULSE_IMAGE is required"
|
||||
fi
|
||||
if [[ -z "${restore_target}" ]]; then
|
||||
restore_target="${provider_data_dir%/}/upgrade-restore-drill"
|
||||
fi
|
||||
@@ -163,12 +167,13 @@ fi
|
||||
echo "provider_msp_upgrade_dry_run=$(truthy "${dry_run}" && echo true || echo false)"
|
||||
echo "provider_msp_upgrade_run_id=${run_id}"
|
||||
echo "provider_msp_upgrade_restore_target=${restore_target}"
|
||||
echo "provider_msp_upgrade_tenant_runtime_image=${tenant_runtime_image}"
|
||||
|
||||
run_control provider-msp status
|
||||
run_control "${preflight_args[@]}"
|
||||
|
||||
if truthy "${dry_run}"; then
|
||||
run_control tenant-runtime reconcile --all --dry-run
|
||||
run_control tenant-runtime rollout --all --image "${tenant_runtime_image}" --dry-run
|
||||
echo "tenant_runtime_rollout_applied=false"
|
||||
echo "provider_msp_upgrade_plan_ok=true"
|
||||
exit 0
|
||||
@@ -195,12 +200,13 @@ if ! truthy "${skip_compose_pull}"; then
|
||||
fi
|
||||
docker compose up -d traefik control-plane
|
||||
run_control provider-msp status --require-backup
|
||||
run_control tenant-runtime reconcile --all --dry-run
|
||||
run_control tenant-runtime rollout --all --image "${tenant_runtime_image}" --dry-run
|
||||
|
||||
if truthy "${rollout_tenants}"; then
|
||||
reconcile_args=(
|
||||
tenant-runtime reconcile
|
||||
tenant-runtime rollout
|
||||
--all
|
||||
--image "${tenant_runtime_image}"
|
||||
--run-id "${run_id}"
|
||||
--health-timeout "${health_timeout}"
|
||||
)
|
||||
@@ -212,7 +218,7 @@ if truthy "${rollout_tenants}"; then
|
||||
echo "tenant_runtime_rollout_applied=true"
|
||||
else
|
||||
echo "tenant_runtime_rollout_applied=false"
|
||||
echo "tenant_runtime_rollout_next_command=docker compose run --rm --no-deps control-plane tenant-runtime reconcile --all --run-id ${run_id} --health-timeout ${health_timeout}"
|
||||
echo "tenant_runtime_rollout_next_command=docker compose run --rm --no-deps control-plane tenant-runtime rollout --all --image ${tenant_runtime_image} --run-id ${run_id} --health-timeout ${health_timeout}"
|
||||
fi
|
||||
|
||||
docker compose ps
|
||||
|
||||
@@ -212,6 +212,13 @@ tenant-local runtime state without depending on Stripe billing surfaces.
|
||||
coherent.
|
||||
Tenant runtime rollout and missing-runtime restore must fail closed on the
|
||||
same storage admission guard before snapshotting or swapping containers.
|
||||
`tenant-runtime rollout --all --image <target>` is the canonical hosted
|
||||
fleet image upgrade path for active tenant runtimes. Its `--dry-run` mode
|
||||
must print the target-image rollout plan before mutation, `--all` must
|
||||
select active tenants only, and apply must still use the canonical
|
||||
per-tenant snapshot, health-check, and rollback path. `tenant-runtime
|
||||
reconcile --all` remains contract/routing drift repair for each tenant's
|
||||
current image line, not an image-line upgrade path.
|
||||
|
||||
The real `pulse-pro` license-server legacy checkout issuance, recurring
|
||||
renewals, manual issue, and legacy exchange flows are part of that same
|
||||
|
||||
@@ -156,9 +156,9 @@ surfaces.
|
||||
provider status and preflight, create and verify a fresh backup before
|
||||
apply, dry-run restore into a separate target data dir, require backup
|
||||
readiness before and after provider service replacement, update the packaged
|
||||
Traefik/control-plane services, print the tenant runtime reconcile plan, and
|
||||
only execute `tenant-runtime reconcile --all` when the operator explicitly
|
||||
asks for tenant rollout.
|
||||
Traefik/control-plane services, print the tenant runtime rollout plan for
|
||||
`CP_PULSE_IMAGE`, and only execute `tenant-runtime rollout --all --image
|
||||
<CP_PULSE_IMAGE>` when the operator explicitly asks for tenant rollout.
|
||||
`deploy/provider-msp/setup.sh` is the first-time provider host setup
|
||||
artifact. It must install the Docker/compose host prerequisites, create the
|
||||
provider data, backup, and Docker-network layout, copy the provider MSP
|
||||
|
||||
@@ -67,6 +67,30 @@ type TenantRuntimeContractReconcilePlan struct {
|
||||
Tenants []*TenantRuntimeContractReconcilePlanItem
|
||||
}
|
||||
|
||||
type TenantRuntimeImageRolloutPlanOptions struct {
|
||||
TenantIDs []string
|
||||
All bool
|
||||
Image string
|
||||
}
|
||||
|
||||
type TenantRuntimeImageRolloutPlanItem struct {
|
||||
TenantID string
|
||||
State string
|
||||
LiveContainerID string
|
||||
LiveImageRef string
|
||||
TargetImageRef string
|
||||
LiveRouteHost string
|
||||
DesiredRouteHost string
|
||||
LivePublicURL string
|
||||
DesiredPublicURL string
|
||||
Action string
|
||||
Reason string
|
||||
}
|
||||
|
||||
type TenantRuntimeImageRolloutPlan struct {
|
||||
Tenants []*TenantRuntimeImageRolloutPlanItem
|
||||
}
|
||||
|
||||
const (
|
||||
tenantRuntimeContractActionNoop = "noop"
|
||||
tenantRuntimeContractActionRollout = "rollout"
|
||||
@@ -147,6 +171,31 @@ func PlanTenantRuntimeContractReconcile(
|
||||
return service.PlanContractReconcile(ctx, opts)
|
||||
}
|
||||
|
||||
func PlanTenantRuntimeImageRollout(
|
||||
ctx context.Context,
|
||||
cfg *CPConfig,
|
||||
opts TenantRuntimeImageRolloutPlanOptions,
|
||||
) (*TenantRuntimeImageRolloutPlan, error) {
|
||||
if cfg == nil {
|
||||
return nil, fmt.Errorf("control plane config is required")
|
||||
}
|
||||
image := strings.TrimSpace(opts.Image)
|
||||
if image == "" {
|
||||
image = strings.TrimSpace(cfg.PulseImage)
|
||||
}
|
||||
if image == "" {
|
||||
return nil, fmt.Errorf("missing tenant runtime image")
|
||||
}
|
||||
opts.Image = image
|
||||
|
||||
service, cleanup, err := newTenantRuntimeRolloutServiceFromConfig(cfg, image)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer cleanup()
|
||||
return service.PlanImageRollout(ctx, opts)
|
||||
}
|
||||
|
||||
func newTenantRuntimeRolloutServiceFromConfig(
|
||||
cfg *CPConfig,
|
||||
image string,
|
||||
@@ -532,6 +581,77 @@ func (s *tenantRuntimeRolloutService) PlanContractReconcile(
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
func (s *tenantRuntimeRolloutService) PlanImageRollout(
|
||||
ctx context.Context,
|
||||
opts TenantRuntimeImageRolloutPlanOptions,
|
||||
) (*TenantRuntimeImageRolloutPlan, error) {
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("rollout service is nil")
|
||||
}
|
||||
targetImage := strings.TrimSpace(opts.Image)
|
||||
if targetImage == "" {
|
||||
targetImage = strings.TrimSpace(s.defaultImage)
|
||||
}
|
||||
if targetImage == "" {
|
||||
return nil, fmt.Errorf("image is required")
|
||||
}
|
||||
tenants, err := s.selectImageRolloutTenants(opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plan := &TenantRuntimeImageRolloutPlan{
|
||||
Tenants: make([]*TenantRuntimeImageRolloutPlanItem, 0, len(tenants)),
|
||||
}
|
||||
for _, tenant := range tenants {
|
||||
item := &TenantRuntimeImageRolloutPlanItem{
|
||||
TenantID: strings.TrimSpace(tenant.ID),
|
||||
State: string(tenant.State),
|
||||
TargetImageRef: targetImage,
|
||||
}
|
||||
if tenant.State != registry.TenantStateActive {
|
||||
item.Action = tenantRuntimeContractActionSkip
|
||||
item.Reason = fmt.Sprintf("tenant state is %s, not active", tenant.State)
|
||||
plan.Tenants = append(plan.Tenants, item)
|
||||
continue
|
||||
}
|
||||
|
||||
desiredRouting := s.docker.DesiredRuntimeRouting(tenant.ID)
|
||||
item.DesiredRouteHost = desiredRouting.Host
|
||||
item.DesiredPublicURL = desiredRouting.PublicURL
|
||||
|
||||
live, err := s.resolveLiveContainer(ctx, tenant)
|
||||
if err != nil {
|
||||
if errors.Is(err, errTenantRuntimeMissing) {
|
||||
item.Action = tenantRuntimeContractActionRollout
|
||||
item.Reason = "tenant runtime container is missing; recreate from existing tenant data"
|
||||
} else {
|
||||
item.Action = tenantRuntimeContractActionSkip
|
||||
item.Reason = err.Error()
|
||||
}
|
||||
plan.Tenants = append(plan.Tenants, item)
|
||||
continue
|
||||
}
|
||||
|
||||
item.LiveContainerID = strings.TrimSpace(live.ID)
|
||||
item.LiveImageRef = strings.TrimSpace(live.ImageRef)
|
||||
item.LiveRouteHost = strings.TrimSpace(live.RouteHost)
|
||||
item.LivePublicURL = strings.TrimSpace(live.PublicURL)
|
||||
|
||||
if tenantRuntimeMatchesContract(live, tenantRuntimeContainerName(tenant.ID), targetImage, desiredRouting) {
|
||||
item.Action = tenantRuntimeContractActionNoop
|
||||
item.Reason = "runtime already matches target image and canonical hosted contract"
|
||||
} else if item.LiveImageRef != targetImage {
|
||||
item.Action = tenantRuntimeContractActionRollout
|
||||
item.Reason = "runtime image differs from target"
|
||||
} else {
|
||||
item.Action = tenantRuntimeContractActionRollout
|
||||
item.Reason = "runtime contract drift detected"
|
||||
}
|
||||
plan.Tenants = append(plan.Tenants, item)
|
||||
}
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
func (s *tenantRuntimeRolloutService) selectContractReconcileTenants(
|
||||
opts TenantRuntimeContractReconcilePlanOptions,
|
||||
) ([]*registry.Tenant, error) {
|
||||
@@ -569,6 +689,48 @@ func (s *tenantRuntimeRolloutService) selectContractReconcileTenants(
|
||||
return tenants, nil
|
||||
}
|
||||
|
||||
func (s *tenantRuntimeRolloutService) selectImageRolloutTenants(
|
||||
opts TenantRuntimeImageRolloutPlanOptions,
|
||||
) ([]*registry.Tenant, error) {
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("rollout service is nil")
|
||||
}
|
||||
if opts.All && len(dedupeNonEmptyStrings(opts.TenantIDs)) > 0 {
|
||||
return nil, fmt.Errorf("choose either --all or one or more tenant ids")
|
||||
}
|
||||
if opts.All {
|
||||
tenants, err := s.registry.List()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list tenants: %w", err)
|
||||
}
|
||||
active := make([]*registry.Tenant, 0, len(tenants))
|
||||
for _, tenant := range tenants {
|
||||
if tenant != nil && tenant.State == registry.TenantStateActive {
|
||||
active = append(active, tenant)
|
||||
}
|
||||
}
|
||||
return active, nil
|
||||
}
|
||||
|
||||
tenantIDs := dedupeNonEmptyStrings(opts.TenantIDs)
|
||||
if len(tenantIDs) == 0 {
|
||||
return nil, fmt.Errorf("at least one tenant id or --all is required")
|
||||
}
|
||||
|
||||
tenants := make([]*registry.Tenant, 0, len(tenantIDs))
|
||||
for _, tenantID := range tenantIDs {
|
||||
tenant, err := s.registry.Get(tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load tenant %s: %w", tenantID, err)
|
||||
}
|
||||
if tenant == nil {
|
||||
return nil, fmt.Errorf("tenant %s not found", tenantID)
|
||||
}
|
||||
tenants = append(tenants, tenant)
|
||||
}
|
||||
return tenants, nil
|
||||
}
|
||||
|
||||
func dedupeNonEmptyStrings(values []string) []string {
|
||||
result := make([]string, 0, len(values))
|
||||
seen := make(map[string]struct{}, len(values))
|
||||
|
||||
@@ -543,6 +543,68 @@ func TestTenantRuntimeContractReconcilePlan_ExplicitTenantIDsDedupesAndPreserves
|
||||
}
|
||||
}
|
||||
|
||||
func TestTenantRuntimeImageRolloutPlan_AllActiveTenantsTargetsConfiguredImage(t *testing.T) {
|
||||
tenantOldImage := ®istry.Tenant{ID: "t-OLDIMAGE1", State: registry.TenantStateActive, ContainerID: "old-live"}
|
||||
tenantTargetImage := ®istry.Tenant{ID: "t-TARGET01", State: registry.TenantStateActive, ContainerID: "target-live"}
|
||||
tenantMissing := ®istry.Tenant{ID: "t-MISSING02", State: registry.TenantStateActive, ContainerID: ""}
|
||||
tenantSuspended := ®istry.Tenant{ID: "t-SUSPEND1", State: registry.TenantStateSuspended, ContainerID: "suspended-live"}
|
||||
reg := &fakeTenantRuntimeRolloutRegistry{
|
||||
tenants: []*registry.Tenant{tenantOldImage, tenantTargetImage, tenantMissing, tenantSuspended},
|
||||
}
|
||||
docker := newFakeTenantRuntimeRolloutDocker()
|
||||
targetImage := "pulse-runtime:next"
|
||||
for _, tc := range []struct {
|
||||
tenant *registry.Tenant
|
||||
image string
|
||||
}{
|
||||
{tenantOldImage, "pulse-runtime:stable"},
|
||||
{tenantTargetImage, targetImage},
|
||||
{tenantSuspended, "pulse-runtime:stable"},
|
||||
} {
|
||||
routing := docker.DesiredRuntimeRouting(tc.tenant.ID)
|
||||
docker.addContainer(&cpDocker.RuntimeContainerInfo{
|
||||
ID: tc.tenant.ContainerID,
|
||||
Name: tenantRuntimeContainerName(tc.tenant.ID),
|
||||
ImageRef: tc.image,
|
||||
ImageID: "sha256:test",
|
||||
Running: true,
|
||||
RouteHost: routing.Host,
|
||||
PublicURL: routing.PublicURL,
|
||||
})
|
||||
}
|
||||
|
||||
service := newTestTenantRuntimeRolloutService(reg, docker, &fakeTenantRuntimeRolloutSynchronizer{}, newFakeTenantRuntimeRolloutClock())
|
||||
plan, err := service.PlanImageRollout(context.Background(), TenantRuntimeImageRolloutPlanOptions{
|
||||
All: true,
|
||||
Image: targetImage,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("PlanImageRollout() error = %v", err)
|
||||
}
|
||||
if len(plan.Tenants) != 3 {
|
||||
t.Fatalf("plan tenant count = %d, want 3 active tenants", len(plan.Tenants))
|
||||
}
|
||||
got := make(map[string]*TenantRuntimeImageRolloutPlanItem, len(plan.Tenants))
|
||||
for _, item := range plan.Tenants {
|
||||
got[item.TenantID] = item
|
||||
}
|
||||
if got[tenantOldImage.ID].Action != tenantRuntimeContractActionRollout {
|
||||
t.Fatalf("old image action = %q, want rollout", got[tenantOldImage.ID].Action)
|
||||
}
|
||||
if got[tenantOldImage.ID].LiveImageRef != "pulse-runtime:stable" || got[tenantOldImage.ID].TargetImageRef != targetImage {
|
||||
t.Fatalf("old image refs = live %q target %q", got[tenantOldImage.ID].LiveImageRef, got[tenantOldImage.ID].TargetImageRef)
|
||||
}
|
||||
if got[tenantTargetImage.ID].Action != tenantRuntimeContractActionNoop {
|
||||
t.Fatalf("target image action = %q, want noop", got[tenantTargetImage.ID].Action)
|
||||
}
|
||||
if got[tenantMissing.ID].Action != tenantRuntimeContractActionRollout {
|
||||
t.Fatalf("missing tenant action = %q, want rollout", got[tenantMissing.ID].Action)
|
||||
}
|
||||
if _, ok := got[tenantSuspended.ID]; ok {
|
||||
t.Fatalf("suspended tenant should not be selected by --all image rollout plan")
|
||||
}
|
||||
}
|
||||
|
||||
func newTestTenantRuntimeRolloutService(
|
||||
reg tenantRuntimeRolloutRegistry,
|
||||
docker tenantRuntimeRolloutDocker,
|
||||
|
||||
@@ -160,9 +160,10 @@ func TestProviderMSPUpgradeRunnerMatchesComposeContract(t *testing.T) {
|
||||
"provider-msp backup verify",
|
||||
"provider-msp backup restore",
|
||||
"--target-data-dir",
|
||||
"tenant-runtime reconcile --all --dry-run",
|
||||
"tenant-runtime reconcile",
|
||||
"tenant-runtime rollout --all --image",
|
||||
"tenant-runtime rollout",
|
||||
"--all",
|
||||
"--image",
|
||||
"--run-id",
|
||||
"--health-timeout",
|
||||
"--prune-previous",
|
||||
|
||||
Reference in New Issue
Block a user