Fix provider MSP evaluation setup flow

cmd/pulse-control-plane/provider_msp_proof.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
@@ -449,6 +450,7 @@ func (rt *providerMSPProofRuntime) proveProviderMSPWorkspace(ctx context.Context
 		OwnerUserID: ownerUserID,
 		BaseURL:     publicURL,
 	})
+	err = rt.reconcileProviderMSPProofRuntimeMutation(tenant, tenantDataDir, "install token generation", err)
 	if err != nil {
 		return providerMSPProofWorkspace{}, fmt.Errorf("generate hosted tenant install command: %w", err)
 	}
@@ -457,6 +459,7 @@ func (rt *providerMSPProofRuntime) proveProviderMSPWorkspace(ctx context.Context
 	}
 
 	tokenAuthVerified, err := markProviderMSPProofAgentTokenUsed(tenantDataDir, tenant.ID, install.Token)
+	err = rt.reconcileProviderMSPProofRuntimeMutation(tenant, tenantDataDir, "install token use", err)
 	if err != nil {
 		return providerMSPProofWorkspace{}, err
 	}
@@ -466,16 +469,19 @@ func (rt *providerMSPProofRuntime) proveProviderMSPWorkspace(ctx context.Context
 		facts.LastAgentSeenAt != nil
 
 	agentReport, err := rt.verifyProviderMSPProofAgentReportIngest(ctx, tenant, tenantDataDir, install.Token, install.TokenID)
+	err = rt.reconcileProviderMSPProofRuntimeMutation(tenant, tenantDataDir, "agent report ingest", err)
 	if err != nil {
 		return providerMSPProofWorkspace{}, err
 	}
 
 	rotation, err := rt.verifyProviderMSPProofInstallTokenRotation(ctx, tenant, tenantDataDir, install.Token, install.TokenID)
+	err = rt.reconcileProviderMSPProofRuntimeMutation(tenant, tenantDataDir, "install token rotation", err)
 	if err != nil {
 		return providerMSPProofWorkspace{}, err
 	}
 
 	exchangedTargetPath, err := rt.verifyProviderMSPProofHandoff(ctx, tenant, ownerUserID, targetPath)
+	err = rt.reconcileProviderMSPProofRuntimeMutation(tenant, tenantDataDir, "handoff exchange", err)
 	if err != nil {
 		return providerMSPProofWorkspace{}, err
 	}
@@ -486,6 +492,7 @@ func (rt *providerMSPProofRuntime) proveProviderMSPWorkspace(ctx context.Context
 	}
 
 	portalRollup, err := rt.verifyProviderMSPProofPortalRollup(ctx, tenant, tenantDataDir, agentReport)
+	err = rt.reconcileProviderMSPProofRuntimeMutation(tenant, tenantDataDir, "portal rollup", err)
 	if err != nil {
 		return providerMSPProofWorkspace{}, err
 	}
@@ -529,6 +536,23 @@ func (rt *providerMSPProofRuntime) proveProviderMSPWorkspace(ctx context.Context
 	}, nil
 }
 
+func (rt *providerMSPProofRuntime) reconcileProviderMSPProofRuntimeMutation(tenant *registry.Tenant, tenantDataDir, stage string, mutationErr error) error {
+	if rt == nil || rt.docker == nil || tenant == nil || strings.TrimSpace(tenant.ContainerID) == "" {
+		return mutationErr
+	}
+	reconcileErr := rt.docker.ReconcileTenantRuntimeMountSources(tenantDataDir)
+	if reconcileErr != nil {
+		reconcileErr = fmt.Errorf("reconcile tenant runtime ownership after %s: %w", stage, reconcileErr)
+	}
+	if mutationErr != nil && reconcileErr != nil {
+		return errors.Join(mutationErr, reconcileErr)
+	}
+	if mutationErr != nil {
+		return mutationErr
+	}
+	return reconcileErr
+}
+
 type providerMSPProofPortalRollup struct {
 	ReportScheduleCreated       bool
 	ReportScheduleID            string

deploy/provider-msp/setup.sh

@@ -302,8 +302,16 @@ default_image_ref() {
 # imagetools, which the Docker install above provides, and which reads the
 # registry without pulling the image.
 resolve_image_digest() {
-  local ref="$1" digest
-  digest="$(docker buildx imagetools inspect "${ref}" --format '{{.Manifest.Digest}}' 2>/dev/null || true)"
+  local ref="$1" manifest_json digest
+  manifest_json="$(docker buildx imagetools inspect "${ref}" --format '{{json .Manifest}}' 2>/dev/null || true)"
+  digest="$(printf '%s' "${manifest_json}" | jq -r 'if type == "object" then .digest // empty else empty end' 2>/dev/null || true)"
+  if [[ "${digest}" != sha256:* ]]; then
+    # Ubuntu's packaged Buildx and Docker's plugin have differed in which
+    # fields their Go template exposes. Keep a human-output fallback so an
+    # already-working Docker install is not rejected only because its Buildx
+    # formatter is older or distro-patched.
+    digest="$(docker buildx imagetools inspect "${ref}" 2>/dev/null | awk '$1 == "Digest:" {print $2; exit}' || true)"
+  fi
   [[ "${digest}" == sha256:* ]] || return 1
   printf '%s@%s\n' "${ref%:*}" "${digest}"
 }
@@ -715,6 +723,15 @@ The license must bind this key or the control plane will refuse to start."
 
 validate_compose_config() {
   log "validating compose config"
+  local license_file
+  license_file="$(env_value CP_PROVIDER_MSP_LICENSE_FILE "${PULSE_PROVIDER_MSP_INSTALL_DIR}/.env")"
+  if [[ "${1:-}" == "--allow-missing-evaluation-license" && -z "${license_file}" ]]; then
+    # Compose requires a non-empty secret source even for a config-only
+    # parse. Use a non-secret placeholder for this pre-issuance validation;
+    # the normal validation after ensure_eval_license uses the real file.
+    (cd "${PULSE_PROVIDER_MSP_INSTALL_DIR}" && CP_PROVIDER_MSP_LICENSE_FILE=/dev/null docker compose config --quiet)
+    return 0
+  fi
   (cd "${PULSE_PROVIDER_MSP_INSTALL_DIR}" && docker compose config --quiet)
 }
 
@@ -724,7 +741,13 @@ pull_provider_images() {
     return 0
   fi
   log "pulling provider MSP images"
-  (cd "${PULSE_PROVIDER_MSP_INSTALL_DIR}" && docker compose pull traefik docker-socket-proxy control-plane)
+  local env_path="${PULSE_PROVIDER_MSP_INSTALL_DIR}/.env"
+  local key image_ref
+  for key in TRAEFIK_IMAGE DOCKER_SOCKET_PROXY_IMAGE CONTROL_PLANE_IMAGE CP_PULSE_IMAGE; do
+    image_ref="$(env_value "${key}" "${env_path}")"
+    [[ "${image_ref}" == *@sha256:* ]] || die "${key} is not digest-pinned before image pull"
+    docker pull "${image_ref}"
+  done
 }
 
 run_install_proof_if_requested() {
@@ -820,12 +843,13 @@ main() {
   create_data_dirs
   ensure_docker_network
   block_container_metadata_service
-  validate_compose_config
+  validate_compose_config --allow-missing-evaluation-license
   pull_provider_images
   # Issue the evaluation only after the host is configured and the immutable
   # images are reachable. This makes an issued evaluation a useful activation
   # signal rather than a record created before setup can succeed.
   ensure_eval_license
+  validate_compose_config
   run_install_proof_if_requested
   print_summary
 }

docs/release-control/v6/internal/subsystems/cloud-paid.md

@@ -396,6 +396,10 @@ copy must not imply that the external agent independently sends alerts.
     including recursive ownership alignment to the configured tenant UID/GID and
     strict permissions on tenant key files, so `CapDrop: ["ALL"]` and
     read-only root filesystems remain compatible with first workspace startup.
+    Root-running control-plane proof paths that mutate files on behalf of an
+    already-running tenant must reuse that Docker-manager ownership boundary
+    before returning control to the rootless runtime; owner-only credential,
+    handoff, ingest, or report state must never remain readable only by root.
     Client-bound proof must cover the boundary itself: workspace limits must not
     be raceable past the licensed cap, handoff tokens must not be replayed or
     retargeted across workspaces, org-bound agent install/report tokens must not

docs/release-control/v6/internal/subsystems/deployment-installability.md

@@ -207,6 +207,11 @@ upgrade, update, release, or artifact-selection behavior.
    `license_file` must be the
    resolved provider MSP plan source unless the operator explicitly opts into
    the local-development `--allow-env-plan` escape hatch.
+   When the root-running provider proof mutates files on behalf of an already
+   running rootless tenant, it must reconcile the full tenant mount tree to the
+   Docker manager's configured runtime UID/GID before the next live-runtime
+   stage. Credential rotation, handoff, report ingest, and portal-rollup proof
+   must not leave owner-only tenant state readable only by the control plane.
    The same proof surface must also keep adversarial client-boundary probes in
    scope: workspace-limit check/create must be locked against concurrent cap
    bypass, handoff tokens must reject cross-workspace retargeting without being
@@ -256,6 +261,13 @@ upgrade, update, release, or artifact-selection behavior.
    the host Docker socket, the provider data directory must be mounted at the
    same absolute path inside the control-plane container that the host Docker
    daemon will later use for tenant runtime bind mounts.
+   Setup must perform its first compose parse with a non-secret placeholder
+   when no evaluation file exists yet, pull each resolved digest-pinned image
+   without requiring compose secret interpolation, issue the evaluation only
+   after those image pulls succeed, and then repeat normal compose validation
+   against the installed signed license. Image resolution must accept the
+   manifest JSON exposed by current Buildx and retain a human-output digest
+   fallback for supported distro-packaged Buildx variants.
    The setup artifact must also generate strong provider secrets when the
    template leaves `CP_ADMIN_KEY` or `CP_TRIAL_ACTIVATION_PRIVATE_KEY` blank,
    enforce minimum admin-key strength and a valid activation signing key before

internal/cloudcp/docker/manager.go

@@ -556,6 +556,21 @@ func (m *Manager) CreateAndStart(ctx context.Context, tenantID, tenantDataDir st
 	return resp.ID, nil
 }
 
+// ReconcileTenantRuntimeMountSources restores the configured rootless runtime
+// ownership after the provider control plane has deliberately mutated a live
+// tenant's files on its behalf. CreateAndStart performs the same preparation
+// before first boot; provider-hosted proof paths need this public boundary when
+// they hand files back to an already-running tenant.
+func (m *Manager) ReconcileTenantRuntimeMountSources(tenantDataDir string) error {
+	if m == nil {
+		return fmt.Errorf("docker manager is required")
+	}
+	if err := prepareTenantRuntimeMountSources(tenantDataDir, tenantRuntimeUIDFor(m.cfg), tenantRuntimeGIDFor(m.cfg)); err != nil {
+		return fmt.Errorf("reconcile tenant runtime mounts: %w", err)
+	}
+	return nil
+}
+
 func tenantRuntimeContainerConfig(tenantID string, cfg ManagerConfig, labels map[string]string, trustedProxyCIDRs []string) *container.Config {
 	return &container.Config{
 		Image:  cfg.Image,

internal/cloudcp/docker/manager_test.go

@@ -452,8 +452,9 @@ func TestPrepareTenantRuntimeMountSourcesAlignsOwnershipAndPermissions(t *testin
 	if err := os.WriteFile(nestedPath, []byte("state"), 0o644); err != nil {
 		t.Fatalf("write nested state: %v", err)
 	}
-	if err := prepareTenantRuntimeMountSources(tenantDataDir, uid, gid); err != nil {
-		t.Fatalf("prepareTenantRuntimeMountSources after nested state: %v", err)
+	mgr := &Manager{cfg: ManagerConfig{TenantRuntimeUID: uid, TenantRuntimeGID: gid}}
+	if err := mgr.ReconcileTenantRuntimeMountSources(tenantDataDir); err != nil {
+		t.Fatalf("ReconcileTenantRuntimeMountSources after nested state: %v", err)
 	}
 
 	for _, path := range []string{tenantDataDir, nestedDir, nestedPath} {

scripts/installtests/provider_msp_deploy_test.go

@@ -3,6 +3,7 @@ package installtests
 import (
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -156,7 +157,7 @@ func TestProviderMSPSetupScriptMatchesProviderContract(t *testing.T) {
 		"CP_ALLOW_DOCKERLESS_PROVISIONING must be false",
 		"CP_STORAGE_GUARDRAILS_ENABLED must be true",
 		"docker compose config --quiet",
-		"docker compose pull traefik docker-socket-proxy control-plane",
+		`docker pull "${image_ref}"`,
 		"PULSE_PROVIDER_MSP_ACCOUNT_NAME",
 		"PULSE_PROVIDER_MSP_OWNER_EMAIL",
 		"./run-install-proof.sh",
@@ -353,6 +354,12 @@ func TestProviderMSPSetupScriptSupportsUnlicensedEvaluation(t *testing.T) {
 		"default_image_ref",
 		"resolve_image_digest",
 		"buildx imagetools inspect",
+		`--format '{{json .Manifest}}'`,
+		`jq -r 'if type == "object" then .digest // empty else empty end'`,
+		`awk '$1 == "Digest:" {print $2; exit}'`,
+		"validate_compose_config --allow-missing-evaluation-license",
+		`CP_PROVIDER_MSP_LICENSE_FILE=/dev/null docker compose config --quiet`,
+		`docker pull "${image_ref}"`,
 		`if [[ "${current}" == *@sha256:*`,
 		`ref="${current}"`,
 		// Self-issue, and the three ways it must degrade instead of blocking.
@@ -369,6 +376,10 @@ func TestProviderMSPSetupScriptSupportsUnlicensedEvaluation(t *testing.T) {
 	if strings.LastIndex(script, "pull_provider_images\n") > strings.LastIndex(script, "ensure_eval_license\n") {
 		t.Fatal("evaluation must be issued only after pinned provider images are reachable")
 	}
+	setupSequence := "validate_compose_config --allow-missing-evaluation-license\n  pull_provider_images\n  # Issue the evaluation only after the host is configured and the immutable\n  # images are reachable. This makes an issued evaluation a useful activation\n  # signal rather than a record created before setup can succeed.\n  ensure_eval_license\n  validate_compose_config"
+	if !strings.Contains(script, setupSequence) {
+		t.Fatal("setup must validate compose, pull pinned images, issue the evaluation, then validate compose with the installed licence")
+	}
 
 	// The install must never abort because an evaluation licence could not be
 	// obtained. `|| true` inside the substitution does not achieve that: the
@@ -406,6 +417,39 @@ func TestProviderMSPSetupScriptSupportsUnlicensedEvaluation(t *testing.T) {
 	}
 }
 
+func TestProviderMSPResolveImageDigestAcceptsManifestJSON(t *testing.T) {
+	scriptBytes, err := os.ReadFile(repoFile("deploy", "provider-msp", "setup.sh"))
+	if err != nil {
+		t.Fatalf("read provider MSP setup: %v", err)
+	}
+	script := strings.Replace(string(scriptBytes), `main "$@"`, "", 1)
+	if script == string(scriptBytes) {
+		t.Fatal("provider MSP setup main invocation not found")
+	}
+
+	tempDir := t.TempDir()
+	digest := "sha256:" + strings.Repeat("a", 64)
+	fakeDocker := filepath.Join(tempDir, "docker")
+	if err := os.WriteFile(fakeDocker, []byte("#!/bin/sh\nprintf '%s\\n' '{\"digest\":\""+digest+"\"}'\n"), 0o755); err != nil {
+		t.Fatalf("write fake docker: %v", err)
+	}
+	runner := filepath.Join(tempDir, "resolve-image-digest.sh")
+	if err := os.WriteFile(runner, []byte(script+"\nresolve_image_digest example.invalid/provider:v1\n"), 0o755); err != nil {
+		t.Fatalf("write setup runner: %v", err)
+	}
+
+	cmd := exec.Command("bash", runner)
+	cmd.Env = append(os.Environ(), "PATH="+tempDir+":"+os.Getenv("PATH"))
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("resolve image digest: %v\n%s", err, output)
+	}
+	want := "example.invalid/provider@" + digest
+	if got := strings.TrimSpace(string(output)); got != want {
+		t.Fatalf("resolved image = %q, want %q", got, want)
+	}
+}
+
 func TestProviderMSPEvaluationDocsUsePublishedSignedBundle(t *testing.T) {
 	repoDocBytes, err := os.ReadFile(repoFile("docs", "MSP.md"))
 	if err != nil {