diff --git a/cmd/pulse-control-plane/provider_msp_install_proof.go b/cmd/pulse-control-plane/provider_msp_install_proof.go index 69a3395540..046c472491 100644 --- a/cmd/pulse-control-plane/provider_msp_install_proof.go +++ b/cmd/pulse-control-plane/provider_msp_install_proof.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "path/filepath" "strings" @@ -265,6 +266,12 @@ func runProviderMSPInstallProofWithDependencies(ctx context.Context, cfg *cloudc proofTenantIDs = providerMSPInstallProofTenantIDs(proof.Workspaces) } if err != nil { + // A proof that fails part-way returns no report, but it may already + // have created workspaces; remove those too. + var partial *providerMSPProofPartialError + if errors.As(err, &partial) && len(proofTenantIDs) == 0 { + proofTenantIDs = partial.CreatedTenantIDs + } addFailure("workspace proof: %v", err) return failAfterProof(err) } diff --git a/cmd/pulse-control-plane/provider_msp_install_proof_test.go b/cmd/pulse-control-plane/provider_msp_install_proof_test.go index 9e1fc07b75..a5a9e95025 100644 --- a/cmd/pulse-control-plane/provider_msp_install_proof_test.go +++ b/cmd/pulse-control-plane/provider_msp_install_proof_test.go @@ -407,3 +407,54 @@ func (f *fakeProviderMSPInstallProofRuntime) CleanupProviderMSPProofTenants(_ co func (f *fakeProviderMSPInstallProofRuntime) Close() { f.closed = true } + +// A proof that fails part-way returns no report, so the workspace list the +// install proof used to clean up from was empty and the workspaces it had +// already created stayed behind in the provider's client slots. +func TestProviderMSPInstallProofCleansUpWorkspacesFromAPartialProof(t *testing.T) { + cfg := testProviderMSPProofConfig(t) + fakeRuntime := &fakeProviderMSPInstallProofRuntime{ + proofErr: &providerMSPProofPartialError{ + err: errors.New("create proof workspace 2: workspace limit reached"), + CreatedTenantIDs: []string{"ws-proof-01"}, + }, + } + + report, err := runProviderMSPInstallProofWithDependencies(context.Background(), cfg, providerMSPInstallProofOptions{ + AccountName: "Example MSP", + OwnerEmail: "owner@example.com", + WorkspacePrefix: "Provider MSP Proof", + WorkspaceCount: 2, + InstallType: "pve", + TargetPath: "/settings/infrastructure?add=linux-host", + BackupOutput: "/tmp/provider-msp-install-proof.tar.gz", + RestoreTargetDataDir: "/tmp/provider-msp-restore-drill", + Cleanup: true, + }, providerMSPInstallProofDependencies{ + Bootstrap: func(context.Context, *cloudcp.CPConfig, cloudcp.ProviderMSPBootstrapOptions) (*cloudcp.ProviderMSPBootstrapResult, error) { + return healthyProviderMSPInstallProofBootstrap(), nil + }, + RunPreflight: func(context.Context, *cloudcp.CPConfig, providerMSPPreflightOptions) (*providerMSPPreflightReport, error) { + return healthyProviderMSPStatusPreflightReport(), nil + }, + RunStatus: func(context.Context, *cloudcp.CPConfig, providerMSPStatusOptions) (*providerMSPStatusReport, error) { + return &providerMSPStatusReport{OK: true}, nil + }, + NewProofRuntime: func(*cloudcp.CPConfig) (providerMSPInstallProofRuntime, error) { + return fakeRuntime, nil + }, + CreateBackup: func(context.Context, *cloudcp.CPConfig, string) (*cloudcp.ProviderMSPBackupCreateResult, error) { + t.Fatal("backup should not run after a failed proof") + return nil, nil + }, + }) + if err == nil || !strings.Contains(err.Error(), "workspace limit reached") { + t.Fatalf("install proof error = %v, want the proof failure", err) + } + if report == nil || report.OK || !report.CleanupOK { + t.Fatalf("report = %#v, want failed with cleanup done", report) + } + if !reflect.DeepEqual(fakeRuntime.cleanupTenantIDs, []string{"ws-proof-01"}) { + t.Fatalf("cleanup tenant ids = %#v, want the workspace the partial proof created", fakeRuntime.cleanupTenantIDs) + } +} diff --git a/cmd/pulse-control-plane/provider_msp_proof.go b/cmd/pulse-control-plane/provider_msp_proof.go index 8c4982645d..bb750dba33 100644 --- a/cmd/pulse-control-plane/provider_msp_proof.go +++ b/cmd/pulse-control-plane/provider_msp_proof.go @@ -260,7 +260,26 @@ func (rt *providerMSPProofRuntime) runProviderMSPProof(ctx context.Context, opts return nil, fmt.Errorf("bootstrap provider MSP account: %w", err) } + // The proof creates its workspaces on the provider's own account, so they + // count against the same client limit real clients do. Check for room + // before creating anything: on a two-client evaluation that already has a + // client, the second proof workspace would otherwise be refused after the + // first was created, leaving it behind in one of the evaluation's slots. + if bootstrap.WorkspaceLimit > 0 { + inUse, err := rt.registry.CountActiveByAccountID(bootstrap.AccountID) + if err != nil { + return nil, fmt.Errorf("count client workspaces: %w", err) + } + if free := bootstrap.WorkspaceLimit - inUse; free < opts.WorkspaceCount { + return nil, fmt.Errorf("the proof creates %d temporary client workspaces but this account has %d free (%d of %d in use); run it on a fresh platform, before adding clients", + opts.WorkspaceCount, max(free, 0), inUse, bootstrap.WorkspaceLimit) + } + } + createdTenants := make([]*registry.Tenant, 0, opts.WorkspaceCount) + partial := func(err error) error { + return &providerMSPProofPartialError{err: err, CreatedTenantIDs: providerMSPProofTenantIDs(createdTenants)} + } if opts.Cleanup { defer func() { _ = rt.cleanupProviderMSPProofTenants(context.Background(), createdTenants) @@ -292,13 +311,13 @@ func (rt *providerMSPProofRuntime) runProviderMSPProof(ctx context.Context, opts displayName := fmt.Sprintf("%s %02d", opts.WorkspacePrefix, idx+1) tenant, err := rt.createProviderMSPProofWorkspace(ctx, bootstrap.AccountID, displayName, bootstrap.OwnerEmail) if err != nil { - return nil, fmt.Errorf("create proof workspace %d: %w", idx+1, err) + return nil, partial(fmt.Errorf("create proof workspace %d: %w", idx+1, err)) } createdTenants = append(createdTenants, tenant) workspace, err := rt.proveProviderMSPWorkspace(ctx, tenant, bootstrap.OwnerUserID, opts.InstallType, opts.TargetPath) if err != nil { - return nil, fmt.Errorf("prove workspace %s: %w", tenant.ID, err) + return nil, partial(fmt.Errorf("prove workspace %s: %w", tenant.ID, err)) } if workspace.ContainerID == "" { report.RuntimeContainerVerified = false @@ -327,13 +346,35 @@ func (rt *providerMSPProofRuntime) runProviderMSPProof(ctx context.Context, opts boundaryOK, err := rt.verifyProviderMSPInstallTokenIsolation(report.Workspaces) if err != nil { - return nil, err + return nil, partial(err) } report.InstallTokenBoundaryOK = boundaryOK report.WorkspaceCount = len(report.Workspaces) return report, nil } +// providerMSPProofPartialError is a proof that failed after creating some of +// its workspaces. It carries their IDs so the caller can remove them: without +// that, a failure part-way through left proof workspaces occupying client +// slots, because no report (and so no workspace list) comes back on error. +type providerMSPProofPartialError struct { + err error + CreatedTenantIDs []string +} + +func (e *providerMSPProofPartialError) Error() string { return e.err.Error() } +func (e *providerMSPProofPartialError) Unwrap() error { return e.err } + +func providerMSPProofTenantIDs(tenants []*registry.Tenant) []string { + ids := make([]string, 0, len(tenants)) + for _, tenant := range tenants { + if tenant != nil && strings.TrimSpace(tenant.ID) != "" { + ids = append(ids, tenant.ID) + } + } + return ids +} + func normalizeProviderMSPProofOptions(opts providerMSPProofOptions) (providerMSPProofOptions, error) { opts.AccountName = strings.TrimSpace(opts.AccountName) opts.OwnerEmail = strings.ToLower(strings.TrimSpace(opts.OwnerEmail)) @@ -395,8 +436,9 @@ func (rt *providerMSPProofRuntime) createProviderMSPProofWorkspace(ctx context.C rt.registry, rt.provisioner, account.WorkspaceLimitPolicy{ - ProviderHostedMSP: true, - ProviderMSPPlanVersion: providerMSPProofPlanVersion(rt.cfg), + ProviderHostedMSP: true, + ProviderMSPPlanVersion: providerMSPProofPlanVersion(rt.cfg), + ProviderMSPLicenseLapsed: func() bool { return rt.cfg.ProviderMSPLicenseLapsed(time.Now()) }, }, ) mux := http.NewServeMux() diff --git a/cmd/pulse-control-plane/provider_msp_proof_test.go b/cmd/pulse-control-plane/provider_msp_proof_test.go index d81786bb45..e5192fd2e1 100644 --- a/cmd/pulse-control-plane/provider_msp_proof_test.go +++ b/cmd/pulse-control-plane/provider_msp_proof_test.go @@ -6,12 +6,14 @@ import ( "crypto/rand" "encoding/base64" "encoding/json" + "fmt" "os" "strings" "testing" "time" "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp" + "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry" pkglicensing "github.com/rcourtman/pulse-go-rewrite/pkg/licensing" ) @@ -430,3 +432,48 @@ func writeProviderMSPProofLicenseForTest(t *testing.T, licenseID, email, planVer } return path } + +// The proof's workspaces count against the provider's own client limit. When +// there is no room for all of them it must refuse before creating any: on a +// two-client evaluation with a client already added, it used to create the +// first proof workspace, get refused on the second, and leave the first +// occupying a slot. +func TestProviderMSPProofRefusesWithoutFreeSlotsAndCreatesNothing(t *testing.T) { + t.Setenv("DOCKER_HOST", "unix:///tmp/pulse-provider-msp-proof-missing-docker.sock") + t.Setenv("DOCKER_TLS_VERIFY", "") + t.Setenv("DOCKER_CERT_PATH", "") + + cfg := testProviderMSPProofConfig(t) + rt, err := newProviderMSPProofRuntimeFromConfig(cfg) + if err != nil { + t.Fatalf("newProviderMSPProofRuntimeFromConfig: %v", err) + } + defer rt.close() + + bootstrap, err := cloudcp.BootstrapProviderMSP(context.Background(), cfg, cloudcp.ProviderMSPBootstrapOptions{ + AccountName: "Acme Provider", + OwnerEmail: "owner@example.com", + }) + if err != nil { + t.Fatalf("BootstrapProviderMSP: %v", err) + } + if err := rt.registry.Create(®istry.Tenant{ID: "t-realclient", AccountID: bootstrap.AccountID, DisplayName: "Real Client", State: registry.TenantStateActive}); err != nil { + t.Fatalf("create existing client: %v", err) + } + + _, err = rt.runProviderMSPProof(context.Background(), providerMSPProofOptions{ + AccountName: "Acme Provider", + OwnerEmail: "owner@example.com", + WorkspacePrefix: "Provider MSP Proof", + WorkspaceCount: bootstrap.WorkspaceLimit, + InstallType: "pve", + TargetPath: "/settings/infrastructure?add=linux-host", + }) + want := fmt.Sprintf("has %d free (1 of %d in use)", bootstrap.WorkspaceLimit-1, bootstrap.WorkspaceLimit) + if err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("runProviderMSPProof error = %v, want refusal naming %q", err, want) + } + if count, err := rt.registry.CountActiveByAccountID(bootstrap.AccountID); err != nil || count != 1 { + t.Fatalf("active workspaces after refusal = %d, %v; want only the existing client", count, err) + } +} diff --git a/cmd/pulse-control-plane/provider_msp_status.go b/cmd/pulse-control-plane/provider_msp_status.go index 0dfe73f801..3dc8421b2b 100644 --- a/cmd/pulse-control-plane/provider_msp_status.go +++ b/cmd/pulse-control-plane/provider_msp_status.go @@ -23,15 +23,18 @@ type providerMSPStatusOptions struct { } type providerMSPStatusReport struct { - OK bool - Environment string - ControlMode string - BaseURL string - PlanVersion string - PlanSource string - LicenseID string - LicenseEmail string - WorkspaceLimit int + OK bool + Environment string + ControlMode string + BaseURL string + PlanVersion string + PlanSource string + LicenseID string + LicenseEmail string + WorkspaceLimit int + // LicenseLapsed is informational, not a failure: a lapsed install must + // still upgrade and keep its portal up so the provider can renew there. + LicenseLapsed bool RegistryReady bool TotalTenants int HealthyTenants int @@ -117,6 +120,7 @@ func runProviderMSPStatusWithDependencies(ctx context.Context, cfg *cloudcp.CPCo LicenseID: strings.TrimSpace(cfg.ProviderMSPLicenseID), LicenseEmail: strings.ToLower(strings.TrimSpace(cfg.ProviderMSPLicenseEmail)), WorkspaceLimit: workspaceLimit, + LicenseLapsed: cfg.ProviderMSPLicenseLapsed(deps.Now()), CountsByState: map[registry.TenantState]int{}, } addFailure := func(format string, args ...any) { @@ -399,6 +403,7 @@ func printProviderMSPStatusReport(report *providerMSPStatusReport) { fmt.Printf("license_id=%s\n", report.LicenseID) fmt.Printf("license_email=%s\n", report.LicenseEmail) fmt.Printf("workspace_limit=%d\n", report.WorkspaceLimit) + fmt.Printf("license_lapsed=%t\n", report.LicenseLapsed) fmt.Printf("registry_ready=%t\n", report.RegistryReady) fmt.Printf("total_tenants=%d\n", report.TotalTenants) fmt.Printf("healthy_tenants=%d\n", report.HealthyTenants) diff --git a/cmd/pulse-control-plane/provider_msp_status_test.go b/cmd/pulse-control-plane/provider_msp_status_test.go index e6ea84fa7e..ef01aada95 100644 --- a/cmd/pulse-control-plane/provider_msp_status_test.go +++ b/cmd/pulse-control-plane/provider_msp_status_test.go @@ -73,6 +73,30 @@ func TestProviderMSPStatusReportsHealthyOperatorSurface(t *testing.T) { } } +// A lapsed licence is reported, not failed: upgrade.sh gates on status, and a +// lapsed install must still upgrade and keep its portal up to renew. +func TestProviderMSPStatusReportsALapsedLicenceWithoutFailing(t *testing.T) { + cfg := testProviderMSPPreflightConfig(t, cloudcp.ProviderMSPPlanSourceLicenseFile) + now := time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC) + cfg.ProviderMSPLicenseExpiresAt = now.Add(-30 * 24 * time.Hour) + report, err := runProviderMSPStatusWithDependencies(context.Background(), cfg, providerMSPStatusOptions{}, providerMSPStatusDependencies{ + RunPreflight: func(context.Context, *cloudcp.CPConfig, providerMSPPreflightOptions) (*providerMSPPreflightReport, error) { + return healthyProviderMSPStatusPreflightReport(), nil + }, + NewDocker: healthyProviderMSPStatusDocker(map[string]bool{}), + CheckBackup: func(context.Context, *cloudcp.CPConfig) (*providerMSPBackupStatus, error) { + return healthyProviderMSPBackupStatus(now), nil + }, + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatalf("runProviderMSPStatusWithDependencies: %v", err) + } + if !report.OK || !report.LicenseLapsed { + t.Fatalf("report OK=%v LicenseLapsed=%v failures=%v, want OK and lapsed", report.OK, report.LicenseLapsed, report.Failures) + } +} + func TestProviderMSPStatusFailsOnFailedUnhealthyAndStuckWorkspaces(t *testing.T) { cfg := testProviderMSPPreflightConfig(t, cloudcp.ProviderMSPPlanSourceLicenseFile) now := time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC) diff --git a/deploy/provider-msp/.env.example b/deploy/provider-msp/.env.example index c5f271d119..59f1688bf8 100644 --- a/deploy/provider-msp/.env.example +++ b/deploy/provider-msp/.env.example @@ -113,10 +113,16 @@ PULSE_EMAIL_REPLY_TO=support@example.com # images: # ./upgrade.sh --dry-run # -# Apply a provider control-plane upgrade after updating the image pins in this -# file. The runner creates and verifies a fresh backup, dry-runs restore into a -# separate target data dir, pulls provider images, starts Traefik/control-plane, -# and prints the tenant runtime rollout plan for CP_PULSE_IMAGE: +# Move to a new release: download and verify its provider bundle as for a fresh +# install, then run upgrade.sh from the extracted bundle directory. It installs +# the bundle files here, keeps this file (and a pre-upgrade copy), re-pins +# CONTROL_PLANE_IMAGE and CP_PULSE_IMAGE to that release, creates and verifies a +# fresh backup, dry-runs restore into a separate target data dir, pulls provider +# images, starts Traefik/control-plane, and prints the tenant runtime rollout +# plan for CP_PULSE_IMAGE (--keep-image-pins keeps pins you set by hand): +# sudo -E bash ./upgrade.sh +# +# Run from this directory, upgrade.sh applies the image pins already here: # ./upgrade.sh # # Also roll all client runtimes to the configured CP_PULSE_IMAGE line after the diff --git a/deploy/provider-msp/upgrade.sh b/deploy/provider-msp/upgrade.sh index 2544a7e294..8ef8c0c272 100755 --- a/deploy/provider-msp/upgrade.sh +++ b/deploy/provider-msp/upgrade.sh @@ -7,6 +7,14 @@ usage() { Usage: ./upgrade.sh [options] +To move to a new release, download and verify its provider bundle exactly as +for a fresh install, then run ./upgrade.sh from the extracted bundle directory +(sudo -E bash ./upgrade.sh). It installs the bundle's files into the existing +install (PULSE_PROVIDER_MSP_INSTALL_DIR, default /opt/pulse-provider-msp), +keeps .env and a pre-upgrade copy of it, re-pins CONTROL_PLANE_IMAGE and +CP_PULSE_IMAGE to that release, and then runs the flow below. Run from the +install directory, it applies the image pins already in .env. + Runs the provider-hosted MSP pre-upgrade and upgrade flow: 1. validates .env and docker-compose.yml 2. checks provider status and install preflight @@ -26,6 +34,7 @@ Options: --restore-target DIR Restore dry-run target data dir (default: /upgrade-restore-drill) --run-id ID Operator-visible tenant reconcile run id --health-timeout DURATION Tenant rollout health timeout (default: 90s) + --keep-image-pins From a bundle: install its files but keep the image pins in .env -h, --help Show this help Environment equivalents: @@ -75,6 +84,98 @@ run_control() { docker compose run --rm --no-deps control-plane "$@" } +set_env_value() { + local key="$1" value="$2" env_path="$3" tmp + tmp="$(mktemp)" + if grep -q -E "^${key}=" "${env_path}"; then + awk -v key="${key}" -v value="${value}" 'BEGIN{done=0} $0 ~ "^" key "=" && done==0 { print key "=" value; done=1; next } { print }' "${env_path}" >"${tmp}" + else + cat "${env_path}" >"${tmp}" + printf '%s=%s\n' "${key}" "${value}" >>"${tmp}" + fi + cat "${tmp}" >"${env_path}" + rm -f "${tmp}" +} + +# resolve_image_digest turns a tag into an immutable digest ref, as setup.sh +# does, reading the registry without pulling the image. +resolve_image_digest() { + local ref="$1" manifest_json digest + manifest_json="$(docker buildx imagetools inspect "${ref}" --format '{{json .Manifest}}' 2>/dev/null || true)" + digest="$(printf '%s' "${manifest_json}" | jq -r 'if type == "object" then .digest // empty else empty end' 2>/dev/null || true)" + if [[ "${digest}" != sha256:* ]]; then + digest="$(docker buildx imagetools inspect "${ref}" 2>/dev/null | awk '$1 == "Digest:" {print $2; exit}' || true)" + fi + [[ "${digest}" == sha256:* ]] || return 1 + printf '%s@%s\n' "${ref%:*}" "${digest}" +} + +# The files setup.sh installs from a bundle. A bundle upgrade installs the same +# set, so an upgraded install matches a fresh one. +bundle_install_files=(docker-compose.yml traefik.yml traefik-dynamic.yml .env.example run-install-proof.sh upgrade.sh) + +# A bundle upgrade brings an existing install up to the release bundle this +# script was extracted with. Setup resolves the image pins in .env to digests +# once, so an upgrade used to re-pull only the release the provider first +# installed; moving on meant finding four digests by hand. Nothing on disk +# changes until the platform has been checked and backed up: those steps run +# the new release's control plane as a one-off through a shell override of the +# pins (compose prefers the shell over .env), which also carries fixes to the +# checks themselves. Only then are the bundle files installed and .env +# re-pinned, just before the new images start. +bundle_dir="" +bundle_version="" +bundle_pin_keys=() +bundle_pin_values=() + +plan_bundle_upgrade() { + local install_dir="$1" env_path key target current resolved + bundle_dir="$2" + bundle_version="$(tr -d '[:space:]' <"${bundle_dir}/VERSION")" + env_path="${install_dir}/.env" + [[ -f "${env_path}" ]] || die "no provider MSP install at ${install_dir} (missing .env); run ./setup.sh for a fresh install" + [[ -w "${install_dir}" && -w "${env_path}" ]] || die "cannot write ${install_dir}; run: sudo -E bash ./upgrade.sh" + echo "provider_msp_upgrade_bundle_version=${bundle_version}" + echo "provider_msp_upgrade_install_dir=${install_dir}" + if truthy "${keep_image_pins}"; then + echo "provider_msp_upgrade_image_pins=kept" + return 0 + fi + for key in CONTROL_PLANE_IMAGE CP_PULSE_IMAGE; do + target="$(env_value "${key}" "${bundle_dir}/.env.example")" + [[ -n "${target}" ]] || die "bundle .env.example has no ${key}; download the release bundle, not the source tree" + current="$(env_value "${key}" "${env_path}")" + if [[ "${target}" == *@sha256:* ]]; then + resolved="${target}" + elif ! resolved="$(resolve_image_digest "${target}")"; then + die "could not resolve a digest for ${target}; check this host can reach the registry" + fi + echo "provider_msp_upgrade_image_pin ${key} current=${current} target=${resolved}" + bundle_pin_keys+=("${key}") + bundle_pin_values+=("${resolved}") + export "${key}=${resolved}" + done +} + +apply_bundle_upgrade() { + local install_dir="$1" env_path stamp file i + env_path="${install_dir}/.env" + stamp="$(date -u +'%Y%m%dT%H%M%SZ')" + cp -p "${env_path}" "${env_path}.pre-upgrade-${stamp}" + echo "provider_msp_upgrade_env_backup=${env_path}.pre-upgrade-${stamp}" + for file in "${bundle_install_files[@]}"; do + [[ -f "${bundle_dir}/${file}" ]] || die "bundle is missing ${file}" + case "${file}" in + *.sh) install -m 0755 "${bundle_dir}/${file}" "${install_dir}/${file}" ;; + *) install -m 0644 "${bundle_dir}/${file}" "${install_dir}/${file}" ;; + esac + done + for ((i = 0; i < ${#bundle_pin_keys[@]}; i++)); do + set_env_value "${bundle_pin_keys[$i]}" "${bundle_pin_values[$i]}" "${env_path}" + done + echo "provider_msp_upgrade_bundle_installed=true" +} + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "${script_dir}" @@ -87,7 +188,7 @@ backup_output="${PROVIDER_MSP_UPGRADE_BACKUP_OUTPUT:-}" restore_target="${PROVIDER_MSP_UPGRADE_RESTORE_TARGET:-}" run_id="${PROVIDER_MSP_UPGRADE_RUN_ID:-provider-msp-upgrade-$(date -u +'%Y%m%dT%H%M%SZ')}" health_timeout="${PROVIDER_MSP_UPGRADE_HEALTH_TIMEOUT:-90s}" - +keep_image_pins="${PROVIDER_MSP_UPGRADE_KEEP_IMAGE_PINS:-0}" while (($# > 0)); do case "$1" in --dry-run) @@ -126,6 +227,10 @@ while (($# > 0)); do health_timeout="${2:-}" shift 2 ;; + --keep-image-pins) + keep_image_pins=1 + shift + ;; -h|--help) usage exit 0 @@ -138,13 +243,22 @@ while (($# > 0)); do esac done +# Run from an extracted release bundle (it carries VERSION and no .env), work +# on the existing install and move it to that release. +install_dir="${script_dir}" +if [[ -f "${script_dir}/VERSION" && ! -f "${script_dir}/.env" ]]; then + install_dir="${PULSE_PROVIDER_MSP_INSTALL_DIR:-/opt/pulse-provider-msp}" + plan_bundle_upgrade "${install_dir}" "${script_dir}" + cd "${install_dir}" +fi + if [[ ! -f .env ]]; then die "deploy/provider-msp/.env is required; copy .env.example to .env and fill in the provider values" fi provider_data_dir="$(env_value PULSE_PROVIDER_MSP_DATA_DIR .env)" provider_data_dir="${provider_data_dir:-/data}" -tenant_runtime_image="$(env_value CP_PULSE_IMAGE .env)" +tenant_runtime_image="${CP_PULSE_IMAGE:-$(env_value CP_PULSE_IMAGE .env)}" if [[ -z "${tenant_runtime_image}" ]]; then die "CP_PULSE_IMAGE is required" fi @@ -169,11 +283,21 @@ echo "provider_msp_upgrade_run_id=${run_id}" echo "provider_msp_upgrade_restore_target=${restore_target}" echo "provider_msp_upgrade_tenant_runtime_image=${tenant_runtime_image}" +# Status checks that the tenant runtime image is already on this host, and runs +# before anything else would pull it, so a new CP_PULSE_IMAGE failed the gate on +# every upgrade. Pulling changes nothing that is running. +if ! truthy "${skip_runtime_image_pull}"; then + docker pull "${tenant_runtime_image}" >/dev/null +fi + run_control provider-msp status run_control "${preflight_args[@]}" if truthy "${dry_run}"; then run_control tenant-runtime rollout --all --image "${tenant_runtime_image}" --dry-run + if [[ -n "${bundle_dir}" ]]; then + echo "provider_msp_upgrade_bundle_installed=false" + fi echo "tenant_runtime_rollout_applied=false" echo "provider_msp_upgrade_plan_ok=true" exit 0 @@ -195,6 +319,11 @@ run_control provider-msp backup verify "${archive_path}" run_control provider-msp backup restore "${archive_path}" --target-data-dir "${restore_target}" --dry-run run_control provider-msp status --require-backup +# The platform is checked and backed up; now install the bundle and re-pin. +if [[ -n "${bundle_dir}" ]]; then + apply_bundle_upgrade "${install_dir}" +fi + if ! truthy "${skip_compose_pull}"; then docker compose pull traefik docker-socket-proxy control-plane fi diff --git a/docs/AUDIT_LOGGING.md b/docs/AUDIT_LOGGING.md index 15ff2bdd32..b6d397f85f 100644 --- a/docs/AUDIT_LOGGING.md +++ b/docs/AUDIT_LOGGING.md @@ -31,7 +31,7 @@ Pulse automatically captures the following events: | `agent_profile_assigned` | Agent profile assignments | Profile `production` assigned to agent | | `agent_profile_unassigned` | Agent profile removals | Profile removed from agent | | `user_roles_updated` | RBAC role assignments changed | Updated roles for user jane: [operator] | -| `agent_config_fetch` | Agent configuration retrieval attempts | Agent config fetched successfully | +| `agent_config_fetch` | Every failed agent configuration fetch. A successful fetch is recorded on the agent's first delivery after Pulse starts, when its token or delivered configuration changes, and otherwise once a day; agents poll every minute, so repeat polls are not recorded | `agent_id=… token_id=… config=sha256:… reason=config_changed` | Each event includes: - **Timestamp** (UTC) diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 776c588425..c9090d8d1f 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -53,6 +53,28 @@ is exposed as unavailable, rather than replaced with a synthetic healthy state. Delivery-log diagnostic text is a notification-attempt projection, not an agent admission or liveness verdict. Masking embedded destination credentials preserves failure context without changing agent identities, credentials, session replacement or removal policy. Consumers must use the retained failureClass and outcome as delivery evidence only; a transport error does not establish that the monitored agent is offline. +### Accepted reports refresh the canonical store at most once per window — issue #2199 + +Every canonical store refresh is estate-wide, so refreshing on each accepted +report made total cost grow with agent count times estate size. Accepted host, +Docker and Kubernetes reports, including rejected-report liveness, therefore +refresh through `refreshUnifiedResourceStoreAfterAgentReport`, throttled to one +refresh per `agentReportRefreshInterval` (2 seconds, the same staleness the +read paths accept through `readPathRegistryFreshness`). The first report after +a quiet window is published before the handler returns. Reports inside the +window fold into one trailing refresh at the window's end, which reads the +latest state and broadcasts it, so a report reaches ReadState consumers within +the window rather than immediately. Agent removal and host-agent evaluation +still refresh immediately, so removed inventory retires at once. `Monitor.Stop` +cancels a pending trailing refresh and waits for a running one before closing +stores. Monitors built without `New` keep a zero window and refresh on every +report. `TestAgentReportRefreshFoldsReportsWithinWindow` and +`TestStopCancelsPendingAgentReportRefresh` pin the leading, trailing and +shutdown behaviour. The founder approved trading immediate visibility for this +bound on 24 September 2026. + +### Canonical Patrol and Assistant continuation, 2026-09-07 + ### Canonical Patrol and Assistant continuation, 2026-09-07 Patrol planning uses the canonical action broker's plan-only boundary. Policy @@ -582,6 +604,10 @@ acknowledgement stays a success rather than a rejection so a real agent does not read a demo server as an outage and retry-storm it. Tests that assert report admission or tenant isolation must therefore not enable mock mode, or the assertion passes without exercising the boundary. +Report admission, config fetch, and continuity lookups read the live host list +through `GetLiveHostsSnapshot`, which copies only hosts (see the monitoring +contract); it must not route through a full state snapshot, which copied every +guest on each agent report. Physical-disk evidence collected by a host agent must survive projection back into monitoring's models. Absent evidence has to carry its declared sentinel @@ -5212,6 +5238,11 @@ agent truth, effective enforcement, and bounded reason separately so lifecycle surfaces can show desired-disabled/applied-enabled and desired-enabled/applied-disabled as drift or attention, and no-report cases as pending or unknown rather than in-sync. + +Routine config polling is not a per-poll audit event: +`/api/agents/agent/{id}/config` audits every failed fetch, and a successful one +only on the agent's first delivery after start, a token or desired-config +change, or once a day (see the API contract). That same canonical /api/auto-register path must also complete the live post-registration contract after persistence: it must trigger discovery refresh and emit the canonical `node_auto_registered` WebSocket payload instead of diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index 005efb5512..8c14451813 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -6633,6 +6633,17 @@ The patrol-local `memory.ChangeDetector.GetChangesSummary` path now also delegates to the shared memory recent-change presentation helper, so any future fallback summary entry point inherits the same heading, resource prefixing, and change-type labels without re-implementing the markdown shape. +That detector is a reader of legacy history, not a detector. Its on-disk +history (`ai_changes.json`) was written by releases that still ran change +detection, which patrol dropped when it moved to agentic execution. The +detector loads that file once at construction and never modifies it, so it +holds no lock and has no background save to join before a data directory is +removed. `DetectChanges`, its snapshot diffing and `ResourceSnapshot` were +reachable only from tests and are retired; new change history belongs in the +unified-resource timeline, not in this fallback. Tests seed the fallback +through a legacy `ai_changes.json`, the same path production reads, and +`TestChangeDetector_LegacyHistoryIsReadOnly` proves reads leave the file and +directory untouched. Those unified-resource action and export audit records are now also exposed through the enterprise audit read surface so operators can inspect the execution trail without reaching into storage internals. diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index e79dbd405f..14b2c0078b 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -698,6 +698,20 @@ and slow-client eviction must use the same synchronized send/close ownership; recovering a send-on-closed panic is not a substitute for a race-free channel lifecycle. +Invalidation coalescing is a throttle, not a restart-on-signal debounce. The +first invalidation after a quiet period broadcasts after the 100 ms coalesce +window; later invalidations for the same audience (all clients, or one tenant) +replace the pending request without rescheduling it, and are broadcast no +sooner than `defaultStateBroadcastInterval` (2 seconds) after that audience's +previous broadcast. Every accepted agent report sends an invalidation and each +broadcast resolves and diffs the full frontend state (about 130 ms at 2,080 +synthetic resources), so the debounce let broadcast work scale with agent +count, and signals closer together than the window could postpone broadcasts +indefinitely (#2199). `TestStateBroadcastsSpacedByInterval`, +`TestTenantStateBroadcastsSpacedByInterval`, and +`TestStateBroadcastNotStarvedBySteadySignals` pin the spacing, latest-state, +and no-starvation behaviour. + Browser clients declare their inbound state-frame ceiling on the WebSocket upgrade URL before initial-state construction begins. If a complete snapshot or a later delta exceeds that ceiling, the hub sends a compact @@ -8694,6 +8708,16 @@ payload shape: `agentId`, `commandsEnabled`, `settings`, `issuedAt`, and signed command decision and the signed settings payload, restricted to the agent-applied settings key schema, rather than treating `desiredConfig` itself as a direct member of that signature payload. + +Its `agent_config_fetch` audit events record every failed fetch but a +successful fetch only when it is new audit information: the agent's first +delivery after Pulse starts, a different token or `desiredConfig` hash, or an +unchanged delivery last recorded 24 hours earlier. Agents poll every minute, +and auditing each poll made this event over 99.9% of audit rows. Recorded +successes carry `config=` and +`reason=first_since_start|token_changed|config_changed|daily` in their details. +`TestAgentConfigFetchAuditsNewDeliveriesAndEveryFailure` and +`TestAgentConfigFetchAuditTrackerRecordsOnlyNewDeliveries` pin the rule. Agent profile delete and unassign clients must now also route canonical `204` success handling through shared allowed-status helpers in `frontend-modern/src/api/responseUtils.ts` instead of open-coding local diff --git a/docs/release-control/v6/internal/subsystems/cloud-paid.md b/docs/release-control/v6/internal/subsystems/cloud-paid.md index 11207f1981..d83fda4213 100644 --- a/docs/release-control/v6/internal/subsystems/cloud-paid.md +++ b/docs/release-control/v6/internal/subsystems/cloud-paid.md @@ -3455,6 +3455,40 @@ Regression coverage: `TestEnsureTenantNetworkRejectsUnownedExistingNetwork` in `internal/cloudcp/docker/manager_test.go`. +### A provider control plane keeps its portal up on a lapsed licence + +A provider-hosted control plane used to refuse to start once its licence was +past expiry and the 7-day grace: `ValidateLicense` rejected it, so `LoadConfig` +failed and the portal went down at the next restart. That is exactly when the +provider needs the portal, because Plan is the only place to buy or renew; +the same happened to a paying provider whose subscription lapsed. Startup now +resolves an authentic, key-bound licence with +`pkglicensing.ValidateLicenseAllowingLapse`, preferring a current licence +(renewed first) and otherwise the lapsed one that expired later, and logs the +lapse. Nothing is unlocked by this: client runtimes still verify the provider +licence carried in each lease with `ValidateLicense` and drop MSP capabilities +once it lapses, workspace creation refuses with `provider_msp_license_lapsed`, +and the refresher still refuses to adopt a lapsed licence from the licence +server. The Plan panel reads `lapsed` from the plan state, says the evaluation +or plan ended and what that means for clients, and offers the plans, including +the same plan again after a paid plan ends; `provider-msp status` prints +`license_lapsed` without failing, so a lapsed install can still upgrade. The +portal also shows a control plane error's `message` in preference to its +machine `error` code, which had been surfacing codes such as +`provider_msp_license_lapsed` and `already_subscribed` as the whole message. +Verified on 2026-09-24 against the walkthrough lab: the control plane started +on an evaluation that lapsed 20 days earlier, the Plan panel offered Solo and +Starter, adding a client showed the lapse sentence, and with a live +subscription the refresher restored the paid licence on its own. Regression +coverage: `TestValidateLicenseAllowingLapseReturnsLapsedButStillAuthenticLicence` +in `pkg/licensing/service_lapsed_test.go`, +`TestLoadConfig_ProviderHostedMSPStartsOnALapsedLicence` in +`internal/cloudcp/config_test.go`, +`TestCreateWorkspace_ProviderHostedMSPRefusesNewClientsOnALapsedLicence` in +`internal/cloudcp/account/tenant_handlers_lapsed_test.go`, and +`TestProviderMSPLicenseRefreshRefusesALapsedLicence` in +`internal/cloudcp/provider_msp_license_refresh_test.go`. + ### Provider-hosted MSP platforms buy and renew their own licence A provider-hosted control plane now buys and renews its licence through the diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 419db400c3..efcb09cec7 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -2438,6 +2438,65 @@ artifact-selection behaviour. ## Current State +### Provider MSP upgrades move an install to a new release bundle + +`setup.sh` resolves the image pins in `.env` to digests once, so `upgrade.sh` +only ever re-pulled the release a provider first installed; moving to a new +release meant finding the new digests by hand, and none of that release's +fixes reached the install. Run from a newly extracted and verified release +bundle (it carries `VERSION` and no `.env`), `upgrade.sh` now works on the +existing install (`PULSE_PROVIDER_MSP_INSTALL_DIR`, default +`/opt/pulse-provider-msp`): it resolves the bundle's `CONTROL_PLANE_IMAGE` and +`CP_PULSE_IMAGE` tags (stamped by `scripts/build-release.sh`) to digests and +prints current and target pins. It runs the status gate, preflight and the +verified backup with the new release's control plane through a shell override, +which compose prefers over `.env`, so those checks carry the new release's +fixes while nothing on disk changes. Only then does it keep a +`.env.pre-upgrade-