diff --git a/cmd/pulse-control-plane/provider_msp_proof.go b/cmd/pulse-control-plane/provider_msp_proof.go index e7e3671bd..166105930 100644 --- a/cmd/pulse-control-plane/provider_msp_proof.go +++ b/cmd/pulse-control-plane/provider_msp_proof.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "net/http" "net/http/httptest" @@ -449,6 +450,7 @@ func (rt *providerMSPProofRuntime) proveProviderMSPWorkspace(ctx context.Context OwnerUserID: ownerUserID, BaseURL: publicURL, }) + err = rt.reconcileProviderMSPProofRuntimeMutation(tenant, tenantDataDir, "install token generation", err) if err != nil { return providerMSPProofWorkspace{}, fmt.Errorf("generate hosted tenant install command: %w", err) } @@ -457,6 +459,7 @@ func (rt *providerMSPProofRuntime) proveProviderMSPWorkspace(ctx context.Context } tokenAuthVerified, err := markProviderMSPProofAgentTokenUsed(tenantDataDir, tenant.ID, install.Token) + err = rt.reconcileProviderMSPProofRuntimeMutation(tenant, tenantDataDir, "install token use", err) if err != nil { return providerMSPProofWorkspace{}, err } @@ -466,16 +469,19 @@ func (rt *providerMSPProofRuntime) proveProviderMSPWorkspace(ctx context.Context facts.LastAgentSeenAt != nil agentReport, err := rt.verifyProviderMSPProofAgentReportIngest(ctx, tenant, tenantDataDir, install.Token, install.TokenID) + err = rt.reconcileProviderMSPProofRuntimeMutation(tenant, tenantDataDir, "agent report ingest", err) if err != nil { return providerMSPProofWorkspace{}, err } rotation, err := rt.verifyProviderMSPProofInstallTokenRotation(ctx, tenant, tenantDataDir, install.Token, install.TokenID) + err = rt.reconcileProviderMSPProofRuntimeMutation(tenant, tenantDataDir, "install token rotation", err) if err != nil { return providerMSPProofWorkspace{}, err } exchangedTargetPath, err := rt.verifyProviderMSPProofHandoff(ctx, tenant, ownerUserID, targetPath) + err = rt.reconcileProviderMSPProofRuntimeMutation(tenant, tenantDataDir, "handoff exchange", err) if err != nil { return providerMSPProofWorkspace{}, err } @@ -486,6 +492,7 @@ func (rt *providerMSPProofRuntime) proveProviderMSPWorkspace(ctx context.Context } portalRollup, err := rt.verifyProviderMSPProofPortalRollup(ctx, tenant, tenantDataDir, agentReport) + err = rt.reconcileProviderMSPProofRuntimeMutation(tenant, tenantDataDir, "portal rollup", err) if err != nil { return providerMSPProofWorkspace{}, err } @@ -529,6 +536,23 @@ func (rt *providerMSPProofRuntime) proveProviderMSPWorkspace(ctx context.Context }, nil } +func (rt *providerMSPProofRuntime) reconcileProviderMSPProofRuntimeMutation(tenant *registry.Tenant, tenantDataDir, stage string, mutationErr error) error { + if rt == nil || rt.docker == nil || tenant == nil || strings.TrimSpace(tenant.ContainerID) == "" { + return mutationErr + } + reconcileErr := rt.docker.ReconcileTenantRuntimeMountSources(tenantDataDir) + if reconcileErr != nil { + reconcileErr = fmt.Errorf("reconcile tenant runtime ownership after %s: %w", stage, reconcileErr) + } + if mutationErr != nil && reconcileErr != nil { + return errors.Join(mutationErr, reconcileErr) + } + if mutationErr != nil { + return mutationErr + } + return reconcileErr +} + type providerMSPProofPortalRollup struct { ReportScheduleCreated bool ReportScheduleID string diff --git a/deploy/provider-msp/setup.sh b/deploy/provider-msp/setup.sh index 4c102e441..1796b1d92 100755 --- a/deploy/provider-msp/setup.sh +++ b/deploy/provider-msp/setup.sh @@ -302,8 +302,16 @@ default_image_ref() { # imagetools, which the Docker install above provides, and which reads the # registry without pulling the image. resolve_image_digest() { - local ref="$1" digest - digest="$(docker buildx imagetools inspect "${ref}" --format '{{.Manifest.Digest}}' 2>/dev/null || true)" + local ref="$1" manifest_json digest + manifest_json="$(docker buildx imagetools inspect "${ref}" --format '{{json .Manifest}}' 2>/dev/null || true)" + digest="$(printf '%s' "${manifest_json}" | jq -r 'if type == "object" then .digest // empty else empty end' 2>/dev/null || true)" + if [[ "${digest}" != sha256:* ]]; then + # Ubuntu's packaged Buildx and Docker's plugin have differed in which + # fields their Go template exposes. Keep a human-output fallback so an + # already-working Docker install is not rejected only because its Buildx + # formatter is older or distro-patched. + digest="$(docker buildx imagetools inspect "${ref}" 2>/dev/null | awk '$1 == "Digest:" {print $2; exit}' || true)" + fi [[ "${digest}" == sha256:* ]] || return 1 printf '%s@%s\n' "${ref%:*}" "${digest}" } @@ -715,6 +723,15 @@ The license must bind this key or the control plane will refuse to start." validate_compose_config() { log "validating compose config" + local license_file + license_file="$(env_value CP_PROVIDER_MSP_LICENSE_FILE "${PULSE_PROVIDER_MSP_INSTALL_DIR}/.env")" + if [[ "${1:-}" == "--allow-missing-evaluation-license" && -z "${license_file}" ]]; then + # Compose requires a non-empty secret source even for a config-only + # parse. Use a non-secret placeholder for this pre-issuance validation; + # the normal validation after ensure_eval_license uses the real file. + (cd "${PULSE_PROVIDER_MSP_INSTALL_DIR}" && CP_PROVIDER_MSP_LICENSE_FILE=/dev/null docker compose config --quiet) + return 0 + fi (cd "${PULSE_PROVIDER_MSP_INSTALL_DIR}" && docker compose config --quiet) } @@ -724,7 +741,13 @@ pull_provider_images() { return 0 fi log "pulling provider MSP images" - (cd "${PULSE_PROVIDER_MSP_INSTALL_DIR}" && docker compose pull traefik docker-socket-proxy control-plane) + local env_path="${PULSE_PROVIDER_MSP_INSTALL_DIR}/.env" + local key image_ref + for key in TRAEFIK_IMAGE DOCKER_SOCKET_PROXY_IMAGE CONTROL_PLANE_IMAGE CP_PULSE_IMAGE; do + image_ref="$(env_value "${key}" "${env_path}")" + [[ "${image_ref}" == *@sha256:* ]] || die "${key} is not digest-pinned before image pull" + docker pull "${image_ref}" + done } run_install_proof_if_requested() { @@ -820,12 +843,13 @@ main() { create_data_dirs ensure_docker_network block_container_metadata_service - validate_compose_config + validate_compose_config --allow-missing-evaluation-license pull_provider_images # Issue the evaluation only after the host is configured and the immutable # images are reachable. This makes an issued evaluation a useful activation # signal rather than a record created before setup can succeed. ensure_eval_license + validate_compose_config run_install_proof_if_requested print_summary } diff --git a/docs/release-control/v6/internal/subsystems/cloud-paid.md b/docs/release-control/v6/internal/subsystems/cloud-paid.md index 4f266a564..6a5b112d7 100644 --- a/docs/release-control/v6/internal/subsystems/cloud-paid.md +++ b/docs/release-control/v6/internal/subsystems/cloud-paid.md @@ -396,6 +396,10 @@ copy must not imply that the external agent independently sends alerts. including recursive ownership alignment to the configured tenant UID/GID and strict permissions on tenant key files, so `CapDrop: ["ALL"]` and read-only root filesystems remain compatible with first workspace startup. + Root-running control-plane proof paths that mutate files on behalf of an + already-running tenant must reuse that Docker-manager ownership boundary + before returning control to the rootless runtime; owner-only credential, + handoff, ingest, or report state must never remain readable only by root. Client-bound proof must cover the boundary itself: workspace limits must not be raceable past the licensed cap, handoff tokens must not be replayed or retargeted across workspaces, org-bound agent install/report tokens must not diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 8b3405ac7..efb53f995 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -207,6 +207,11 @@ upgrade, update, release, or artifact-selection behavior. `license_file` must be the resolved provider MSP plan source unless the operator explicitly opts into the local-development `--allow-env-plan` escape hatch. + When the root-running provider proof mutates files on behalf of an already + running rootless tenant, it must reconcile the full tenant mount tree to the + Docker manager's configured runtime UID/GID before the next live-runtime + stage. Credential rotation, handoff, report ingest, and portal-rollup proof + must not leave owner-only tenant state readable only by the control plane. The same proof surface must also keep adversarial client-boundary probes in scope: workspace-limit check/create must be locked against concurrent cap bypass, handoff tokens must reject cross-workspace retargeting without being @@ -256,6 +261,13 @@ upgrade, update, release, or artifact-selection behavior. the host Docker socket, the provider data directory must be mounted at the same absolute path inside the control-plane container that the host Docker daemon will later use for tenant runtime bind mounts. + Setup must perform its first compose parse with a non-secret placeholder + when no evaluation file exists yet, pull each resolved digest-pinned image + without requiring compose secret interpolation, issue the evaluation only + after those image pulls succeed, and then repeat normal compose validation + against the installed signed license. Image resolution must accept the + manifest JSON exposed by current Buildx and retain a human-output digest + fallback for supported distro-packaged Buildx variants. The setup artifact must also generate strong provider secrets when the template leaves `CP_ADMIN_KEY` or `CP_TRIAL_ACTIVATION_PRIVATE_KEY` blank, enforce minimum admin-key strength and a valid activation signing key before diff --git a/internal/cloudcp/docker/manager.go b/internal/cloudcp/docker/manager.go index 571094b4d..133a3b0f4 100644 --- a/internal/cloudcp/docker/manager.go +++ b/internal/cloudcp/docker/manager.go @@ -556,6 +556,21 @@ func (m *Manager) CreateAndStart(ctx context.Context, tenantID, tenantDataDir st return resp.ID, nil } +// ReconcileTenantRuntimeMountSources restores the configured rootless runtime +// ownership after the provider control plane has deliberately mutated a live +// tenant's files on its behalf. CreateAndStart performs the same preparation +// before first boot; provider-hosted proof paths need this public boundary when +// they hand files back to an already-running tenant. +func (m *Manager) ReconcileTenantRuntimeMountSources(tenantDataDir string) error { + if m == nil { + return fmt.Errorf("docker manager is required") + } + if err := prepareTenantRuntimeMountSources(tenantDataDir, tenantRuntimeUIDFor(m.cfg), tenantRuntimeGIDFor(m.cfg)); err != nil { + return fmt.Errorf("reconcile tenant runtime mounts: %w", err) + } + return nil +} + func tenantRuntimeContainerConfig(tenantID string, cfg ManagerConfig, labels map[string]string, trustedProxyCIDRs []string) *container.Config { return &container.Config{ Image: cfg.Image, diff --git a/internal/cloudcp/docker/manager_test.go b/internal/cloudcp/docker/manager_test.go index 550b589e3..e21439075 100644 --- a/internal/cloudcp/docker/manager_test.go +++ b/internal/cloudcp/docker/manager_test.go @@ -452,8 +452,9 @@ func TestPrepareTenantRuntimeMountSourcesAlignsOwnershipAndPermissions(t *testin if err := os.WriteFile(nestedPath, []byte("state"), 0o644); err != nil { t.Fatalf("write nested state: %v", err) } - if err := prepareTenantRuntimeMountSources(tenantDataDir, uid, gid); err != nil { - t.Fatalf("prepareTenantRuntimeMountSources after nested state: %v", err) + mgr := &Manager{cfg: ManagerConfig{TenantRuntimeUID: uid, TenantRuntimeGID: gid}} + if err := mgr.ReconcileTenantRuntimeMountSources(tenantDataDir); err != nil { + t.Fatalf("ReconcileTenantRuntimeMountSources after nested state: %v", err) } for _, path := range []string{tenantDataDir, nestedDir, nestedPath} { diff --git a/scripts/installtests/provider_msp_deploy_test.go b/scripts/installtests/provider_msp_deploy_test.go index 05222822f..24e082c82 100644 --- a/scripts/installtests/provider_msp_deploy_test.go +++ b/scripts/installtests/provider_msp_deploy_test.go @@ -3,6 +3,7 @@ package installtests import ( "os" "os/exec" + "path/filepath" "strings" "testing" @@ -156,7 +157,7 @@ func TestProviderMSPSetupScriptMatchesProviderContract(t *testing.T) { "CP_ALLOW_DOCKERLESS_PROVISIONING must be false", "CP_STORAGE_GUARDRAILS_ENABLED must be true", "docker compose config --quiet", - "docker compose pull traefik docker-socket-proxy control-plane", + `docker pull "${image_ref}"`, "PULSE_PROVIDER_MSP_ACCOUNT_NAME", "PULSE_PROVIDER_MSP_OWNER_EMAIL", "./run-install-proof.sh", @@ -353,6 +354,12 @@ func TestProviderMSPSetupScriptSupportsUnlicensedEvaluation(t *testing.T) { "default_image_ref", "resolve_image_digest", "buildx imagetools inspect", + `--format '{{json .Manifest}}'`, + `jq -r 'if type == "object" then .digest // empty else empty end'`, + `awk '$1 == "Digest:" {print $2; exit}'`, + "validate_compose_config --allow-missing-evaluation-license", + `CP_PROVIDER_MSP_LICENSE_FILE=/dev/null docker compose config --quiet`, + `docker pull "${image_ref}"`, `if [[ "${current}" == *@sha256:*`, `ref="${current}"`, // Self-issue, and the three ways it must degrade instead of blocking. @@ -369,6 +376,10 @@ func TestProviderMSPSetupScriptSupportsUnlicensedEvaluation(t *testing.T) { if strings.LastIndex(script, "pull_provider_images\n") > strings.LastIndex(script, "ensure_eval_license\n") { t.Fatal("evaluation must be issued only after pinned provider images are reachable") } + setupSequence := "validate_compose_config --allow-missing-evaluation-license\n pull_provider_images\n # Issue the evaluation only after the host is configured and the immutable\n # images are reachable. This makes an issued evaluation a useful activation\n # signal rather than a record created before setup can succeed.\n ensure_eval_license\n validate_compose_config" + if !strings.Contains(script, setupSequence) { + t.Fatal("setup must validate compose, pull pinned images, issue the evaluation, then validate compose with the installed licence") + } // The install must never abort because an evaluation licence could not be // obtained. `|| true` inside the substitution does not achieve that: the @@ -406,6 +417,39 @@ func TestProviderMSPSetupScriptSupportsUnlicensedEvaluation(t *testing.T) { } } +func TestProviderMSPResolveImageDigestAcceptsManifestJSON(t *testing.T) { + scriptBytes, err := os.ReadFile(repoFile("deploy", "provider-msp", "setup.sh")) + if err != nil { + t.Fatalf("read provider MSP setup: %v", err) + } + script := strings.Replace(string(scriptBytes), `main "$@"`, "", 1) + if script == string(scriptBytes) { + t.Fatal("provider MSP setup main invocation not found") + } + + tempDir := t.TempDir() + digest := "sha256:" + strings.Repeat("a", 64) + fakeDocker := filepath.Join(tempDir, "docker") + if err := os.WriteFile(fakeDocker, []byte("#!/bin/sh\nprintf '%s\\n' '{\"digest\":\""+digest+"\"}'\n"), 0o755); err != nil { + t.Fatalf("write fake docker: %v", err) + } + runner := filepath.Join(tempDir, "resolve-image-digest.sh") + if err := os.WriteFile(runner, []byte(script+"\nresolve_image_digest example.invalid/provider:v1\n"), 0o755); err != nil { + t.Fatalf("write setup runner: %v", err) + } + + cmd := exec.Command("bash", runner) + cmd.Env = append(os.Environ(), "PATH="+tempDir+":"+os.Getenv("PATH")) + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("resolve image digest: %v\n%s", err, output) + } + want := "example.invalid/provider@" + digest + if got := strings.TrimSpace(string(output)); got != want { + t.Fatalf("resolved image = %q, want %q", got, want) + } +} + func TestProviderMSPEvaluationDocsUsePublishedSignedBundle(t *testing.T) { repoDocBytes, err := os.ReadFile(repoFile("docs", "MSP.md")) if err != nil {