Fix provider MSP tenant rootless startup

This commit is contained in:
rcourtman
2026-06-02 21:31:17 +01:00
parent c7e50d5602
commit d8f5519eed
5 changed files with 420 additions and 8 deletions
@@ -226,6 +226,12 @@ Stripe-free and avoids a cloud-control-plane report data path across clients.
and bounded writable tmpfs mounts only for runtime scratch paths. These
defaults are part of the client isolation contract, not optional compose
decoration.
Tenant runtime containers must also start as the rootless Pulse runtime user
instead of entering the image's root-owned chown and `su-exec` branch. The
Docker manager owns host-side preparation of the tenant data directory,
including recursive ownership alignment to the configured tenant UID/GID and
strict permissions on tenant key files, so `CapDrop: ["ALL"]` and
read-only root filesystems remain compatible with first workspace startup.
Client-bound proof must cover the boundary itself: workspace limits must not
be raceable past the licensed cap, handoff tokens must not be replayed or
retargeted across workspaces, org-bound agent install/report tokens must not
@@ -144,6 +144,12 @@ TLS floor in the dynamic config.
`traefik.docker.network`. The packaged compose stack must label the Traefik
and control-plane support containers so the control plane can attach them to
each tenant bridge before starting the client runtime.
The client runtime must be started as the rootless Pulse UID/GID by the
Docker manager, with tenant data ownership prepared on the host before
container creation. Provider-hosted installability proof must therefore
exercise the actual `CreateAndStart` path with the real Pulse entrypoint
shape, not only raw Docker container creation, so capability drops and the
read-only root filesystem cannot break first tenant startup unnoticed.
`pulse-control-plane provider-msp proof` must exercise the first-client
onboarding path through workspace creation, client-bound install token
generation, tenant-local unified-agent report ingest, tenant-bound install
+60 -8
View File
@@ -31,6 +31,8 @@ type ManagerConfig struct {
TrialActivationPublicKey string
TrustedProxyCIDRs []string
TenantReportBrand TenantReportBrandConfig
TenantRuntimeUID int
TenantRuntimeGID int
MemoryLimit int64 // bytes
CPUShares int64
TenantLogMaxSize string
@@ -494,7 +496,7 @@ func (m *Manager) CreateAndStart(ctx context.Context, tenantID, tenantDataDir st
if _, _, err := m.ensureRuntimeImageAvailable(ctx, true); err != nil {
return "", fmt.Errorf("prepare tenant runtime image: %w", err)
}
if err := prepareTenantRuntimeMountSources(tenantDataDir, tenantRuntimeUID, tenantRuntimeGID); err != nil {
if err := prepareTenantRuntimeMountSources(tenantDataDir, tenantRuntimeUIDFor(m.cfg), tenantRuntimeGIDFor(m.cfg)); err != nil {
return "", fmt.Errorf("prepare tenant runtime mounts for %s: %w", tenantID, err)
}
@@ -509,11 +511,7 @@ func (m *Manager) CreateAndStart(ctx context.Context, tenantID, tenantDataDir st
containerName := "pulse-" + tenantID
resp, err := m.cli.ContainerCreate(ctx, client.ContainerCreateOptions{
Config: &container.Config{
Image: m.cfg.Image,
Labels: labels,
Env: tenantEnv(tenantID, m.cfg.BaseDomain, m.cfg.TrialActivationPublicKey, m.tenantTrustedProxyCIDRs(ctx, tenantNetworkName), m.cfg.TenantReportBrand),
},
Config: tenantRuntimeContainerConfig(tenantID, m.cfg, labels, m.tenantTrustedProxyCIDRs(ctx, tenantNetworkName)),
HostConfig: tenantRuntimeHostConfig(tenantDataDir, m.cfg),
NetworkingConfig: &network.NetworkingConfig{
EndpointsConfig: map[string]*network.EndpointSettings{
@@ -544,6 +542,37 @@ func (m *Manager) CreateAndStart(ctx context.Context, tenantID, tenantDataDir st
return resp.ID, nil
}
func tenantRuntimeContainerConfig(tenantID string, cfg ManagerConfig, labels map[string]string, trustedProxyCIDRs []string) *container.Config {
return &container.Config{
Image: cfg.Image,
User: tenantRuntimeUserFor(cfg),
Labels: labels,
Env: tenantEnvForRuntime(tenantID, cfg.BaseDomain, cfg.TrialActivationPublicKey, trustedProxyCIDRs, tenantRuntimeUIDFor(cfg), tenantRuntimeGIDFor(cfg), cfg.TenantReportBrand),
}
}
func tenantRuntimeUser() string {
return fmt.Sprintf("%d:%d", tenantRuntimeUID, tenantRuntimeGID)
}
func tenantRuntimeUserFor(cfg ManagerConfig) string {
return fmt.Sprintf("%d:%d", tenantRuntimeUIDFor(cfg), tenantRuntimeGIDFor(cfg))
}
func tenantRuntimeUIDFor(cfg ManagerConfig) int {
if cfg.TenantRuntimeUID > 0 {
return cfg.TenantRuntimeUID
}
return tenantRuntimeUID
}
func tenantRuntimeGIDFor(cfg ManagerConfig) int {
if cfg.TenantRuntimeGID > 0 {
return cfg.TenantRuntimeGID
}
return tenantRuntimeGID
}
func tenantRuntimeLogConfig(maxSize string, maxFile int) container.LogConfig {
maxSize = strings.TrimSpace(maxSize)
if maxSize == "" {
@@ -602,6 +631,10 @@ func tenantRuntimeOwnershipPaths() []string {
}
func tenantEnv(tenantID, baseDomain, trialActivationPublicKey string, trustedProxyCIDRs []string, reportBrand TenantReportBrandConfig) []string {
return tenantEnvForRuntime(tenantID, baseDomain, trialActivationPublicKey, trustedProxyCIDRs, tenantRuntimeUID, tenantRuntimeGID, reportBrand)
}
func tenantEnvForRuntime(tenantID, baseDomain, trialActivationPublicKey string, trustedProxyCIDRs []string, runtimeUID, runtimeGID int, reportBrand TenantReportBrandConfig) []string {
routing := CanonicalTenantRuntimeRouting(tenantID, baseDomain)
tenantID = strings.TrimSpace(tenantID)
@@ -610,8 +643,8 @@ func tenantEnv(tenantID, baseDomain, trialActivationPublicKey string, trustedPro
"PULSE_HOSTED_MODE=true",
"PULSE_TENANT_ID=" + tenantID,
"PULSE_MULTI_TENANT_ENABLED=true",
fmt.Sprintf("PUID=%d", tenantRuntimeUID),
fmt.Sprintf("PGID=%d", tenantRuntimeGID),
fmt.Sprintf("PUID=%d", runtimeUID),
fmt.Sprintf("PGID=%d", runtimeGID),
fmt.Sprintf("%s=%s", immutableOwnershipPathsEnv, strings.Join(tenantImmutableOwnershipPaths(), ":")),
}
if routing.PublicURL != "" {
@@ -751,8 +784,27 @@ func tenantMounts(tenantDataDir string) []mount.Mount {
}
func prepareTenantRuntimeMountSources(tenantDataDir string, uid, gid int) error {
if err := filepath.WalkDir(tenantDataDir, func(path string, entry os.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if err := os.Lchown(path, uid, gid); err != nil {
return fmt.Errorf("chown %s to %d:%d: %w", path, uid, gid, err)
}
return nil
}); err != nil {
return err
}
for _, relPath := range tenantRuntimeOwnershipPaths() {
path := filepath.Join(tenantDataDir, relPath)
info, err := os.Lstat(path)
if err != nil {
return fmt.Errorf("stat %s: %w", path, err)
}
if info.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("tenant runtime mount source %s must not be a symlink", path)
}
if err := os.Chmod(path, 0o600); err != nil {
return fmt.Errorf("chmod %s: %w", path, err)
}
@@ -1,11 +1,15 @@
package docker
import (
"archive/tar"
"bytes"
"context"
"crypto/rand"
"encoding/hex"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -15,6 +19,106 @@ import (
"github.com/moby/moby/client"
)
func TestIntegrationTenantRuntimeStartsRootlessWithHardenedHostConfig(t *testing.T) {
if os.Getenv("PULSE_RUN_DOCKER_INTEGRATION") != "1" {
t.Skip("set PULSE_RUN_DOCKER_INTEGRATION=1 to run live Docker tenant-runtime proof")
}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
suffix := dockerIntegrationSuffix(t)
imageTag := "pulse-it-rootless-" + suffix + ":latest"
providerNetwork := "pulse-it-provider-" + suffix
tenantNetworkPrefix := "pulse-it-tenant-" + suffix
tenantID := "t-rootless-" + suffix
scratchRoot, err := dockerIntegrationScratchRoot()
if err != nil {
t.Fatalf("resolve docker-visible scratch root: %v", err)
}
tenantDataDir, err := os.MkdirTemp(scratchRoot, "pulse-it-rootless-"+suffix+"-")
if err != nil {
t.Fatalf("create docker-visible tenant data dir: %v", err)
}
t.Cleanup(func() { _ = os.RemoveAll(tenantDataDir) })
if err := os.MkdirAll(filepath.Join(tenantDataDir, "secrets"), 0o755); err != nil {
t.Fatalf("mkdir tenant secrets: %v", err)
}
for _, path := range []string{
filepath.Join(tenantDataDir, "billing.json"),
filepath.Join(tenantDataDir, "secrets", "handoff.key"),
filepath.Join(tenantDataDir, ".cloud_handoff_key"),
} {
if err := os.WriteFile(path, []byte("secret"), 0o600); err != nil {
t.Fatalf("write %s: %v", path, err)
}
}
mgr, err := NewManager(ManagerConfig{
Image: imageTag,
Network: providerNetwork,
IsolateTenantNetworks: true,
TenantNetworkPrefix: tenantNetworkPrefix,
BaseDomain: "msp.example.test",
TenantRuntimeUID: os.Getuid(),
TenantRuntimeGID: os.Getgid(),
})
if err != nil {
t.Fatalf("NewManager: %v", err)
}
t.Cleanup(func() { _ = mgr.Close() })
dockerIntegrationBuildRootlessProofImage(t, ctx, mgr, imageTag)
t.Cleanup(func() {
_, _ = mgr.cli.ImageRemove(context.Background(), imageTag, client.ImageRemoveOptions{Force: true, PruneChildren: true})
})
if _, err := mgr.cli.NetworkCreate(ctx, providerNetwork, client.NetworkCreateOptions{
Driver: "bridge",
Labels: map[string]string{"pulse.integration": "tenant-rootless-proof"},
}); err != nil {
t.Fatalf("create provider network: %v", err)
}
t.Cleanup(func() {
_, _ = mgr.cli.NetworkRemove(context.Background(), providerNetwork, client.NetworkRemoveOptions{})
})
dockerIntegrationCreateSupportContainer(t, ctx, mgr, providerNetwork, providerSupportTraefikLabel)
dockerIntegrationCreateSupportContainer(t, ctx, mgr, providerNetwork, providerSupportControlPlaneLabel)
containerID, err := mgr.CreateAndStart(ctx, tenantID, tenantDataDir)
if err != nil {
t.Fatalf("CreateAndStart: %v", err)
}
t.Cleanup(func() { _ = mgr.Remove(context.Background(), containerID) })
inspect, err := mgr.cli.ContainerInspect(ctx, containerID, client.ContainerInspectOptions{})
if err != nil {
t.Fatalf("inspect tenant runtime: %v", err)
}
if inspect.Container.Config == nil {
t.Fatal("tenant runtime Config is nil")
}
if inspect.Container.Config.User != tenantRuntimeUserFor(mgr.cfg) {
t.Fatalf("tenant runtime Config.User = %q, want %q", inspect.Container.Config.User, tenantRuntimeUserFor(mgr.cfg))
}
if inspect.Container.HostConfig == nil {
t.Fatal("tenant runtime HostConfig is nil")
}
if !inspect.Container.HostConfig.ReadonlyRootfs {
t.Fatal("tenant runtime root filesystem is not read-only")
}
if got := inspect.Container.HostConfig.CapDrop; len(got) != 1 || got[0] != "ALL" {
t.Fatalf("tenant runtime CapDrop = %v, want [ALL]", got)
}
if len(inspect.Container.HostConfig.CapAdd) != 0 {
t.Fatalf("tenant runtime CapAdd = %v, want none", inspect.Container.HostConfig.CapAdd)
}
dockerIntegrationWaitForRootlessStartupProof(t, ctx, mgr, containerID, filepath.Join(tenantDataDir, "rootless-startup-proof"), tenantRuntimeUserFor(mgr.cfg))
}
func TestIntegrationTenantNetworksAreNotMutuallyReachable(t *testing.T) {
if os.Getenv("PULSE_RUN_DOCKER_INTEGRATION") != "1" {
t.Skip("set PULSE_RUN_DOCKER_INTEGRATION=1 to run live Docker tenant-network proof")
@@ -88,6 +192,177 @@ func dockerIntegrationSuffix(t *testing.T) string {
return strings.ToLower(hex.EncodeToString(raw[:]))
}
func dockerIntegrationScratchRoot() (string, error) {
if value := strings.TrimSpace(os.Getenv("PULSE_DOCKER_INTEGRATION_SCRATCH")); value != "" {
if err := os.MkdirAll(value, 0o700); err != nil {
return "", err
}
return value, nil
}
home, err := os.UserHomeDir()
if err != nil {
return "", err
}
root := filepath.Join(home, ".cache", "pulse-docker-integration")
if err := os.MkdirAll(root, 0o700); err != nil {
return "", err
}
return root, nil
}
func dockerIntegrationBuildRootlessProofImage(t *testing.T, ctx context.Context, mgr *Manager, imageTag string) {
t.Helper()
entrypointPath := filepath.Join(dockerIntegrationRepoRoot(t), "docker-entrypoint.sh")
entrypoint, err := os.ReadFile(entrypointPath)
if err != nil {
t.Fatalf("read docker-entrypoint.sh: %v", err)
}
dockerfile := []byte(`FROM alpine:3.20
RUN apk --no-cache add su-exec
RUN addgroup -g 1000 pulse && adduser -D -u 1000 -G pulse pulse
RUN mkdir -p /etc/pulse /data && chown -R pulse:pulse /etc/pulse /data
COPY docker-entrypoint.sh /docker-entrypoint.sh
RUN chmod +x /docker-entrypoint.sh
ENTRYPOINT ["/docker-entrypoint.sh"]
CMD ["sh", "-c", "printf '%s:%s\n' \"$(id -u)\" \"$(id -g)\" > /etc/pulse/rootless-startup-proof && sleep 300"]
`)
var contextTar bytes.Buffer
tw := tar.NewWriter(&contextTar)
dockerIntegrationAddTarFile(t, tw, "Dockerfile", 0o644, dockerfile)
dockerIntegrationAddTarFile(t, tw, "docker-entrypoint.sh", 0o755, entrypoint)
if err := tw.Close(); err != nil {
t.Fatalf("close image build context: %v", err)
}
build, err := mgr.cli.ImageBuild(ctx, &contextTar, client.ImageBuildOptions{
Tags: []string{imageTag},
Dockerfile: "Dockerfile",
Remove: true,
ForceRemove: true,
})
if err != nil {
t.Fatalf("build rootless proof image: %v", err)
}
defer build.Body.Close()
buildOutput, err := io.ReadAll(build.Body)
if err != nil {
t.Fatalf("read rootless proof image build output: %v", err)
}
if _, err := mgr.cli.ImageInspect(ctx, imageTag); err != nil {
t.Fatalf("inspect built rootless proof image: %v\n%s", err, string(buildOutput))
}
}
func dockerIntegrationRepoRoot(t *testing.T) string {
t.Helper()
dir, err := os.Getwd()
if err != nil {
t.Fatalf("getwd: %v", err)
}
for {
if _, err := os.Stat(filepath.Join(dir, "docker-entrypoint.sh")); err == nil {
return dir
}
parent := filepath.Dir(dir)
if parent == dir {
t.Fatalf("could not locate repo root from %s", dir)
}
dir = parent
}
}
func dockerIntegrationAddTarFile(t *testing.T, tw *tar.Writer, name string, mode int64, content []byte) {
t.Helper()
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: mode, Size: int64(len(content))}); err != nil {
t.Fatalf("write tar header %s: %v", name, err)
}
if _, err := tw.Write(content); err != nil {
t.Fatalf("write tar content %s: %v", name, err)
}
}
func dockerIntegrationCreateSupportContainer(t *testing.T, ctx context.Context, mgr *Manager, networkName, label string) string {
t.Helper()
key, value, ok := strings.Cut(label, "=")
if !ok || strings.TrimSpace(key) == "" || strings.TrimSpace(value) == "" {
t.Fatalf("invalid support label %q", label)
}
name := fmt.Sprintf("pulse-it-support-%s-%s", dockerIntegrationSuffix(t), strings.ReplaceAll(key, ".", "-"))
resp, err := mgr.cli.ContainerCreate(ctx, client.ContainerCreateOptions{
Config: &container.Config{
Image: mgr.cfg.Image,
Entrypoint: []string{"sleep"},
Cmd: []string{"300"},
Labels: map[string]string{
strings.TrimSpace(key): strings.TrimSpace(value),
"pulse.integration": "tenant-rootless-proof",
},
},
NetworkingConfig: &network.NetworkingConfig{
EndpointsConfig: map[string]*network.EndpointSettings{
networkName: {},
},
},
Name: name,
})
if err != nil {
t.Fatalf("create support container %s: %v", label, err)
}
t.Cleanup(func() {
_, _ = mgr.cli.ContainerRemove(context.Background(), resp.ID, client.ContainerRemoveOptions{Force: true})
})
if _, err := mgr.cli.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); err != nil {
t.Fatalf("start support container %s: %v", label, err)
}
return resp.ID
}
func dockerIntegrationWaitForRootlessStartupProof(t *testing.T, ctx context.Context, mgr *Manager, containerID, proofPath, wantUser string) {
t.Helper()
deadline := time.Now().Add(20 * time.Second)
for time.Now().Before(deadline) {
if content, err := os.ReadFile(proofPath); err == nil {
if got := strings.TrimSpace(string(content)); got == wantUser {
return
}
t.Fatalf("rootless startup proof = %q, want %q", strings.TrimSpace(string(content)), wantUser)
}
inspect, err := mgr.cli.ContainerInspect(ctx, containerID, client.ContainerInspectOptions{})
if err == nil && inspect.Container.State != nil && !inspect.Container.State.Running {
t.Fatalf("tenant runtime exited before rootless proof; state=%s exit=%d logs=%s",
inspect.Container.State.Status,
inspect.Container.State.ExitCode,
dockerIntegrationContainerLogs(t, ctx, mgr, containerID),
)
}
select {
case <-time.After(250 * time.Millisecond):
case <-ctx.Done():
t.Fatalf("wait for rootless startup proof: %v", ctx.Err())
}
}
t.Fatalf("tenant runtime did not write rootless proof at %s; logs=%s", proofPath, dockerIntegrationContainerLogs(t, ctx, mgr, containerID))
}
func dockerIntegrationContainerLogs(t *testing.T, ctx context.Context, mgr *Manager, containerID string) string {
t.Helper()
logs, err := mgr.cli.ContainerLogs(ctx, containerID, client.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Tail: "50"})
if err != nil {
return err.Error()
}
defer logs.Close()
content, err := io.ReadAll(logs)
if err != nil {
return err.Error()
}
return strings.TrimSpace(string(content))
}
func dockerIntegrationCreateContainer(t *testing.T, ctx context.Context, mgr *Manager, networkName string, cmd []string) string {
t.Helper()
name := fmt.Sprintf("pulse-it-%s-%s", dockerIntegrationSuffix(t), strings.ReplaceAll(networkName, "_", "-"))
+73
View File
@@ -253,6 +253,50 @@ func TestTenantRuntimeLogConfigBoundsJSONLogs(t *testing.T) {
}
}
func TestTenantRuntimeContainerConfigRunsRootless(t *testing.T) {
t.Parallel()
cfg := ManagerConfig{
Image: "pulse:test",
BaseDomain: "msp.example.com",
TrialActivationPublicKey: "pubkey-123",
}
labels := map[string]string{"pulse.managed": "true"}
got := tenantRuntimeContainerConfig("t-acme", cfg, labels, []string{"172.18.0.0/16"})
if got == nil {
t.Fatal("tenantRuntimeContainerConfig returned nil")
}
if got.User != "1000:1000" {
t.Fatalf("User = %q, want rootless tenant runtime user", got.User)
}
if got.Image != cfg.Image {
t.Fatalf("Image = %q, want %q", got.Image, cfg.Image)
}
if got.Labels["pulse.managed"] != "true" {
t.Fatalf("Labels not preserved: %v", got.Labels)
}
if envValue(got.Env, "PUID") != "1000" || envValue(got.Env, "PGID") != "1000" {
t.Fatalf("PUID/PGID env missing from rootless config: %v", got.Env)
}
if envValue(got.Env, "PULSE_TRUSTED_PROXY_CIDRS") != "172.18.0.0/16" {
t.Fatalf("trusted proxy env = %q", envValue(got.Env, "PULSE_TRUSTED_PROXY_CIDRS"))
}
custom := tenantRuntimeContainerConfig("t-acme", ManagerConfig{
Image: "pulse:test",
BaseDomain: "msp.example.com",
TenantRuntimeUID: 1234,
TenantRuntimeGID: 5678,
}, labels, nil)
if custom.User != "1234:5678" {
t.Fatalf("custom User = %q, want configured rootless runtime user", custom.User)
}
if envValue(custom.Env, "PUID") != "1234" || envValue(custom.Env, "PGID") != "5678" {
t.Fatalf("custom PUID/PGID env missing from rootless config: %v", custom.Env)
}
}
func TestTenantRuntimeHostConfigAppliesEscapeHardening(t *testing.T) {
t.Parallel()
@@ -338,6 +382,35 @@ func TestPrepareTenantRuntimeMountSourcesAlignsOwnershipAndPermissions(t *testin
t.Fatalf("prepareTenantRuntimeMountSources: %v", err)
}
nestedDir := filepath.Join(tenantDataDir, "state")
nestedPath := filepath.Join(nestedDir, "runtime.db")
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
t.Fatalf("mkdir nested state: %v", err)
}
if err := os.WriteFile(nestedPath, []byte("state"), 0o644); err != nil {
t.Fatalf("write nested state: %v", err)
}
if err := prepareTenantRuntimeMountSources(tenantDataDir, uid, gid); err != nil {
t.Fatalf("prepareTenantRuntimeMountSources after nested state: %v", err)
}
for _, path := range []string{tenantDataDir, nestedDir, nestedPath} {
info, err := os.Stat(path)
if err != nil {
t.Fatalf("stat %s: %v", path, err)
}
stat, ok := info.Sys().(*syscall.Stat_t)
if !ok {
t.Fatalf("%s stat type %T, want *syscall.Stat_t", path, info.Sys())
}
if int(stat.Uid) != uid {
t.Fatalf("%s uid = %d, want %d", path, stat.Uid, uid)
}
if int(stat.Gid) != gid {
t.Fatalf("%s gid = %d, want %d", path, stat.Gid, gid)
}
}
for _, path := range paths {
info, err := os.Stat(path)
if err != nil {