From d8f5519eed7b97539e10a601eac0af6a0377d2fc Mon Sep 17 00:00:00 2001 From: rcourtman Date: Tue, 2 Jun 2026 21:31:17 +0100 Subject: [PATCH] Fix provider MSP tenant rootless startup --- .../v6/internal/subsystems/cloud-paid.md | 6 + .../subsystems/deployment-installability.md | 6 + internal/cloudcp/docker/manager.go | 68 ++++- .../docker/manager_integration_test.go | 275 ++++++++++++++++++ internal/cloudcp/docker/manager_test.go | 73 +++++ 5 files changed, 420 insertions(+), 8 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/cloud-paid.md b/docs/release-control/v6/internal/subsystems/cloud-paid.md index e1fe56497..a6eff7a73 100644 --- a/docs/release-control/v6/internal/subsystems/cloud-paid.md +++ b/docs/release-control/v6/internal/subsystems/cloud-paid.md @@ -226,6 +226,12 @@ Stripe-free and avoids a cloud-control-plane report data path across clients. and bounded writable tmpfs mounts only for runtime scratch paths. These defaults are part of the client isolation contract, not optional compose decoration. + Tenant runtime containers must also start as the rootless Pulse runtime user + instead of entering the image's root-owned chown and `su-exec` branch. The + Docker manager owns host-side preparation of the tenant data directory, + including recursive ownership alignment to the configured tenant UID/GID and + strict permissions on tenant key files, so `CapDrop: ["ALL"]` and + read-only root filesystems remain compatible with first workspace startup. Client-bound proof must cover the boundary itself: workspace limits must not be raceable past the licensed cap, handoff tokens must not be replayed or retargeted across workspaces, org-bound agent install/report tokens must not diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index d9a5f096c..27e1d70f0 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -144,6 +144,12 @@ TLS floor in the dynamic config. `traefik.docker.network`. The packaged compose stack must label the Traefik and control-plane support containers so the control plane can attach them to each tenant bridge before starting the client runtime. + The client runtime must be started as the rootless Pulse UID/GID by the + Docker manager, with tenant data ownership prepared on the host before + container creation. Provider-hosted installability proof must therefore + exercise the actual `CreateAndStart` path with the real Pulse entrypoint + shape, not only raw Docker container creation, so capability drops and the + read-only root filesystem cannot break first tenant startup unnoticed. `pulse-control-plane provider-msp proof` must exercise the first-client onboarding path through workspace creation, client-bound install token generation, tenant-local unified-agent report ingest, tenant-bound install diff --git a/internal/cloudcp/docker/manager.go b/internal/cloudcp/docker/manager.go index 411d6e366..f8a082a07 100644 --- a/internal/cloudcp/docker/manager.go +++ b/internal/cloudcp/docker/manager.go @@ -31,6 +31,8 @@ type ManagerConfig struct { TrialActivationPublicKey string TrustedProxyCIDRs []string TenantReportBrand TenantReportBrandConfig + TenantRuntimeUID int + TenantRuntimeGID int MemoryLimit int64 // bytes CPUShares int64 TenantLogMaxSize string @@ -494,7 +496,7 @@ func (m *Manager) CreateAndStart(ctx context.Context, tenantID, tenantDataDir st if _, _, err := m.ensureRuntimeImageAvailable(ctx, true); err != nil { return "", fmt.Errorf("prepare tenant runtime image: %w", err) } - if err := prepareTenantRuntimeMountSources(tenantDataDir, tenantRuntimeUID, tenantRuntimeGID); err != nil { + if err := prepareTenantRuntimeMountSources(tenantDataDir, tenantRuntimeUIDFor(m.cfg), tenantRuntimeGIDFor(m.cfg)); err != nil { return "", fmt.Errorf("prepare tenant runtime mounts for %s: %w", tenantID, err) } @@ -509,11 +511,7 @@ func (m *Manager) CreateAndStart(ctx context.Context, tenantID, tenantDataDir st containerName := "pulse-" + tenantID resp, err := m.cli.ContainerCreate(ctx, client.ContainerCreateOptions{ - Config: &container.Config{ - Image: m.cfg.Image, - Labels: labels, - Env: tenantEnv(tenantID, m.cfg.BaseDomain, m.cfg.TrialActivationPublicKey, m.tenantTrustedProxyCIDRs(ctx, tenantNetworkName), m.cfg.TenantReportBrand), - }, + Config: tenantRuntimeContainerConfig(tenantID, m.cfg, labels, m.tenantTrustedProxyCIDRs(ctx, tenantNetworkName)), HostConfig: tenantRuntimeHostConfig(tenantDataDir, m.cfg), NetworkingConfig: &network.NetworkingConfig{ EndpointsConfig: map[string]*network.EndpointSettings{ @@ -544,6 +542,37 @@ func (m *Manager) CreateAndStart(ctx context.Context, tenantID, tenantDataDir st return resp.ID, nil } +func tenantRuntimeContainerConfig(tenantID string, cfg ManagerConfig, labels map[string]string, trustedProxyCIDRs []string) *container.Config { + return &container.Config{ + Image: cfg.Image, + User: tenantRuntimeUserFor(cfg), + Labels: labels, + Env: tenantEnvForRuntime(tenantID, cfg.BaseDomain, cfg.TrialActivationPublicKey, trustedProxyCIDRs, tenantRuntimeUIDFor(cfg), tenantRuntimeGIDFor(cfg), cfg.TenantReportBrand), + } +} + +func tenantRuntimeUser() string { + return fmt.Sprintf("%d:%d", tenantRuntimeUID, tenantRuntimeGID) +} + +func tenantRuntimeUserFor(cfg ManagerConfig) string { + return fmt.Sprintf("%d:%d", tenantRuntimeUIDFor(cfg), tenantRuntimeGIDFor(cfg)) +} + +func tenantRuntimeUIDFor(cfg ManagerConfig) int { + if cfg.TenantRuntimeUID > 0 { + return cfg.TenantRuntimeUID + } + return tenantRuntimeUID +} + +func tenantRuntimeGIDFor(cfg ManagerConfig) int { + if cfg.TenantRuntimeGID > 0 { + return cfg.TenantRuntimeGID + } + return tenantRuntimeGID +} + func tenantRuntimeLogConfig(maxSize string, maxFile int) container.LogConfig { maxSize = strings.TrimSpace(maxSize) if maxSize == "" { @@ -602,6 +631,10 @@ func tenantRuntimeOwnershipPaths() []string { } func tenantEnv(tenantID, baseDomain, trialActivationPublicKey string, trustedProxyCIDRs []string, reportBrand TenantReportBrandConfig) []string { + return tenantEnvForRuntime(tenantID, baseDomain, trialActivationPublicKey, trustedProxyCIDRs, tenantRuntimeUID, tenantRuntimeGID, reportBrand) +} + +func tenantEnvForRuntime(tenantID, baseDomain, trialActivationPublicKey string, trustedProxyCIDRs []string, runtimeUID, runtimeGID int, reportBrand TenantReportBrandConfig) []string { routing := CanonicalTenantRuntimeRouting(tenantID, baseDomain) tenantID = strings.TrimSpace(tenantID) @@ -610,8 +643,8 @@ func tenantEnv(tenantID, baseDomain, trialActivationPublicKey string, trustedPro "PULSE_HOSTED_MODE=true", "PULSE_TENANT_ID=" + tenantID, "PULSE_MULTI_TENANT_ENABLED=true", - fmt.Sprintf("PUID=%d", tenantRuntimeUID), - fmt.Sprintf("PGID=%d", tenantRuntimeGID), + fmt.Sprintf("PUID=%d", runtimeUID), + fmt.Sprintf("PGID=%d", runtimeGID), fmt.Sprintf("%s=%s", immutableOwnershipPathsEnv, strings.Join(tenantImmutableOwnershipPaths(), ":")), } if routing.PublicURL != "" { @@ -751,8 +784,27 @@ func tenantMounts(tenantDataDir string) []mount.Mount { } func prepareTenantRuntimeMountSources(tenantDataDir string, uid, gid int) error { + if err := filepath.WalkDir(tenantDataDir, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if err := os.Lchown(path, uid, gid); err != nil { + return fmt.Errorf("chown %s to %d:%d: %w", path, uid, gid, err) + } + return nil + }); err != nil { + return err + } + for _, relPath := range tenantRuntimeOwnershipPaths() { path := filepath.Join(tenantDataDir, relPath) + info, err := os.Lstat(path) + if err != nil { + return fmt.Errorf("stat %s: %w", path, err) + } + if info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("tenant runtime mount source %s must not be a symlink", path) + } if err := os.Chmod(path, 0o600); err != nil { return fmt.Errorf("chmod %s: %w", path, err) } diff --git a/internal/cloudcp/docker/manager_integration_test.go b/internal/cloudcp/docker/manager_integration_test.go index 47742bcc2..c00426b4c 100644 --- a/internal/cloudcp/docker/manager_integration_test.go +++ b/internal/cloudcp/docker/manager_integration_test.go @@ -1,11 +1,15 @@ package docker import ( + "archive/tar" + "bytes" "context" "crypto/rand" "encoding/hex" "fmt" + "io" "os" + "path/filepath" "strings" "testing" "time" @@ -15,6 +19,106 @@ import ( "github.com/moby/moby/client" ) +func TestIntegrationTenantRuntimeStartsRootlessWithHardenedHostConfig(t *testing.T) { + if os.Getenv("PULSE_RUN_DOCKER_INTEGRATION") != "1" { + t.Skip("set PULSE_RUN_DOCKER_INTEGRATION=1 to run live Docker tenant-runtime proof") + } + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + + suffix := dockerIntegrationSuffix(t) + imageTag := "pulse-it-rootless-" + suffix + ":latest" + providerNetwork := "pulse-it-provider-" + suffix + tenantNetworkPrefix := "pulse-it-tenant-" + suffix + tenantID := "t-rootless-" + suffix + scratchRoot, err := dockerIntegrationScratchRoot() + if err != nil { + t.Fatalf("resolve docker-visible scratch root: %v", err) + } + tenantDataDir, err := os.MkdirTemp(scratchRoot, "pulse-it-rootless-"+suffix+"-") + if err != nil { + t.Fatalf("create docker-visible tenant data dir: %v", err) + } + t.Cleanup(func() { _ = os.RemoveAll(tenantDataDir) }) + + if err := os.MkdirAll(filepath.Join(tenantDataDir, "secrets"), 0o755); err != nil { + t.Fatalf("mkdir tenant secrets: %v", err) + } + for _, path := range []string{ + filepath.Join(tenantDataDir, "billing.json"), + filepath.Join(tenantDataDir, "secrets", "handoff.key"), + filepath.Join(tenantDataDir, ".cloud_handoff_key"), + } { + if err := os.WriteFile(path, []byte("secret"), 0o600); err != nil { + t.Fatalf("write %s: %v", path, err) + } + } + + mgr, err := NewManager(ManagerConfig{ + Image: imageTag, + Network: providerNetwork, + IsolateTenantNetworks: true, + TenantNetworkPrefix: tenantNetworkPrefix, + BaseDomain: "msp.example.test", + TenantRuntimeUID: os.Getuid(), + TenantRuntimeGID: os.Getgid(), + }) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + t.Cleanup(func() { _ = mgr.Close() }) + + dockerIntegrationBuildRootlessProofImage(t, ctx, mgr, imageTag) + t.Cleanup(func() { + _, _ = mgr.cli.ImageRemove(context.Background(), imageTag, client.ImageRemoveOptions{Force: true, PruneChildren: true}) + }) + + if _, err := mgr.cli.NetworkCreate(ctx, providerNetwork, client.NetworkCreateOptions{ + Driver: "bridge", + Labels: map[string]string{"pulse.integration": "tenant-rootless-proof"}, + }); err != nil { + t.Fatalf("create provider network: %v", err) + } + t.Cleanup(func() { + _, _ = mgr.cli.NetworkRemove(context.Background(), providerNetwork, client.NetworkRemoveOptions{}) + }) + + dockerIntegrationCreateSupportContainer(t, ctx, mgr, providerNetwork, providerSupportTraefikLabel) + dockerIntegrationCreateSupportContainer(t, ctx, mgr, providerNetwork, providerSupportControlPlaneLabel) + + containerID, err := mgr.CreateAndStart(ctx, tenantID, tenantDataDir) + if err != nil { + t.Fatalf("CreateAndStart: %v", err) + } + t.Cleanup(func() { _ = mgr.Remove(context.Background(), containerID) }) + + inspect, err := mgr.cli.ContainerInspect(ctx, containerID, client.ContainerInspectOptions{}) + if err != nil { + t.Fatalf("inspect tenant runtime: %v", err) + } + if inspect.Container.Config == nil { + t.Fatal("tenant runtime Config is nil") + } + if inspect.Container.Config.User != tenantRuntimeUserFor(mgr.cfg) { + t.Fatalf("tenant runtime Config.User = %q, want %q", inspect.Container.Config.User, tenantRuntimeUserFor(mgr.cfg)) + } + if inspect.Container.HostConfig == nil { + t.Fatal("tenant runtime HostConfig is nil") + } + if !inspect.Container.HostConfig.ReadonlyRootfs { + t.Fatal("tenant runtime root filesystem is not read-only") + } + if got := inspect.Container.HostConfig.CapDrop; len(got) != 1 || got[0] != "ALL" { + t.Fatalf("tenant runtime CapDrop = %v, want [ALL]", got) + } + if len(inspect.Container.HostConfig.CapAdd) != 0 { + t.Fatalf("tenant runtime CapAdd = %v, want none", inspect.Container.HostConfig.CapAdd) + } + + dockerIntegrationWaitForRootlessStartupProof(t, ctx, mgr, containerID, filepath.Join(tenantDataDir, "rootless-startup-proof"), tenantRuntimeUserFor(mgr.cfg)) +} + func TestIntegrationTenantNetworksAreNotMutuallyReachable(t *testing.T) { if os.Getenv("PULSE_RUN_DOCKER_INTEGRATION") != "1" { t.Skip("set PULSE_RUN_DOCKER_INTEGRATION=1 to run live Docker tenant-network proof") @@ -88,6 +192,177 @@ func dockerIntegrationSuffix(t *testing.T) string { return strings.ToLower(hex.EncodeToString(raw[:])) } +func dockerIntegrationScratchRoot() (string, error) { + if value := strings.TrimSpace(os.Getenv("PULSE_DOCKER_INTEGRATION_SCRATCH")); value != "" { + if err := os.MkdirAll(value, 0o700); err != nil { + return "", err + } + return value, nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + root := filepath.Join(home, ".cache", "pulse-docker-integration") + if err := os.MkdirAll(root, 0o700); err != nil { + return "", err + } + return root, nil +} + +func dockerIntegrationBuildRootlessProofImage(t *testing.T, ctx context.Context, mgr *Manager, imageTag string) { + t.Helper() + + entrypointPath := filepath.Join(dockerIntegrationRepoRoot(t), "docker-entrypoint.sh") + entrypoint, err := os.ReadFile(entrypointPath) + if err != nil { + t.Fatalf("read docker-entrypoint.sh: %v", err) + } + + dockerfile := []byte(`FROM alpine:3.20 +RUN apk --no-cache add su-exec +RUN addgroup -g 1000 pulse && adduser -D -u 1000 -G pulse pulse +RUN mkdir -p /etc/pulse /data && chown -R pulse:pulse /etc/pulse /data +COPY docker-entrypoint.sh /docker-entrypoint.sh +RUN chmod +x /docker-entrypoint.sh +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["sh", "-c", "printf '%s:%s\n' \"$(id -u)\" \"$(id -g)\" > /etc/pulse/rootless-startup-proof && sleep 300"] +`) + + var contextTar bytes.Buffer + tw := tar.NewWriter(&contextTar) + dockerIntegrationAddTarFile(t, tw, "Dockerfile", 0o644, dockerfile) + dockerIntegrationAddTarFile(t, tw, "docker-entrypoint.sh", 0o755, entrypoint) + if err := tw.Close(); err != nil { + t.Fatalf("close image build context: %v", err) + } + + build, err := mgr.cli.ImageBuild(ctx, &contextTar, client.ImageBuildOptions{ + Tags: []string{imageTag}, + Dockerfile: "Dockerfile", + Remove: true, + ForceRemove: true, + }) + if err != nil { + t.Fatalf("build rootless proof image: %v", err) + } + defer build.Body.Close() + buildOutput, err := io.ReadAll(build.Body) + if err != nil { + t.Fatalf("read rootless proof image build output: %v", err) + } + if _, err := mgr.cli.ImageInspect(ctx, imageTag); err != nil { + t.Fatalf("inspect built rootless proof image: %v\n%s", err, string(buildOutput)) + } +} + +func dockerIntegrationRepoRoot(t *testing.T) string { + t.Helper() + dir, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + for { + if _, err := os.Stat(filepath.Join(dir, "docker-entrypoint.sh")); err == nil { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatalf("could not locate repo root from %s", dir) + } + dir = parent + } +} + +func dockerIntegrationAddTarFile(t *testing.T, tw *tar.Writer, name string, mode int64, content []byte) { + t.Helper() + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: mode, Size: int64(len(content))}); err != nil { + t.Fatalf("write tar header %s: %v", name, err) + } + if _, err := tw.Write(content); err != nil { + t.Fatalf("write tar content %s: %v", name, err) + } +} + +func dockerIntegrationCreateSupportContainer(t *testing.T, ctx context.Context, mgr *Manager, networkName, label string) string { + t.Helper() + key, value, ok := strings.Cut(label, "=") + if !ok || strings.TrimSpace(key) == "" || strings.TrimSpace(value) == "" { + t.Fatalf("invalid support label %q", label) + } + name := fmt.Sprintf("pulse-it-support-%s-%s", dockerIntegrationSuffix(t), strings.ReplaceAll(key, ".", "-")) + resp, err := mgr.cli.ContainerCreate(ctx, client.ContainerCreateOptions{ + Config: &container.Config{ + Image: mgr.cfg.Image, + Entrypoint: []string{"sleep"}, + Cmd: []string{"300"}, + Labels: map[string]string{ + strings.TrimSpace(key): strings.TrimSpace(value), + "pulse.integration": "tenant-rootless-proof", + }, + }, + NetworkingConfig: &network.NetworkingConfig{ + EndpointsConfig: map[string]*network.EndpointSettings{ + networkName: {}, + }, + }, + Name: name, + }) + if err != nil { + t.Fatalf("create support container %s: %v", label, err) + } + t.Cleanup(func() { + _, _ = mgr.cli.ContainerRemove(context.Background(), resp.ID, client.ContainerRemoveOptions{Force: true}) + }) + if _, err := mgr.cli.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); err != nil { + t.Fatalf("start support container %s: %v", label, err) + } + return resp.ID +} + +func dockerIntegrationWaitForRootlessStartupProof(t *testing.T, ctx context.Context, mgr *Manager, containerID, proofPath, wantUser string) { + t.Helper() + deadline := time.Now().Add(20 * time.Second) + for time.Now().Before(deadline) { + if content, err := os.ReadFile(proofPath); err == nil { + if got := strings.TrimSpace(string(content)); got == wantUser { + return + } + t.Fatalf("rootless startup proof = %q, want %q", strings.TrimSpace(string(content)), wantUser) + } + + inspect, err := mgr.cli.ContainerInspect(ctx, containerID, client.ContainerInspectOptions{}) + if err == nil && inspect.Container.State != nil && !inspect.Container.State.Running { + t.Fatalf("tenant runtime exited before rootless proof; state=%s exit=%d logs=%s", + inspect.Container.State.Status, + inspect.Container.State.ExitCode, + dockerIntegrationContainerLogs(t, ctx, mgr, containerID), + ) + } + + select { + case <-time.After(250 * time.Millisecond): + case <-ctx.Done(): + t.Fatalf("wait for rootless startup proof: %v", ctx.Err()) + } + } + t.Fatalf("tenant runtime did not write rootless proof at %s; logs=%s", proofPath, dockerIntegrationContainerLogs(t, ctx, mgr, containerID)) +} + +func dockerIntegrationContainerLogs(t *testing.T, ctx context.Context, mgr *Manager, containerID string) string { + t.Helper() + logs, err := mgr.cli.ContainerLogs(ctx, containerID, client.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Tail: "50"}) + if err != nil { + return err.Error() + } + defer logs.Close() + content, err := io.ReadAll(logs) + if err != nil { + return err.Error() + } + return strings.TrimSpace(string(content)) +} + func dockerIntegrationCreateContainer(t *testing.T, ctx context.Context, mgr *Manager, networkName string, cmd []string) string { t.Helper() name := fmt.Sprintf("pulse-it-%s-%s", dockerIntegrationSuffix(t), strings.ReplaceAll(networkName, "_", "-")) diff --git a/internal/cloudcp/docker/manager_test.go b/internal/cloudcp/docker/manager_test.go index ebfd1a689..edd736d28 100644 --- a/internal/cloudcp/docker/manager_test.go +++ b/internal/cloudcp/docker/manager_test.go @@ -253,6 +253,50 @@ func TestTenantRuntimeLogConfigBoundsJSONLogs(t *testing.T) { } } +func TestTenantRuntimeContainerConfigRunsRootless(t *testing.T) { + t.Parallel() + + cfg := ManagerConfig{ + Image: "pulse:test", + BaseDomain: "msp.example.com", + TrialActivationPublicKey: "pubkey-123", + } + labels := map[string]string{"pulse.managed": "true"} + + got := tenantRuntimeContainerConfig("t-acme", cfg, labels, []string{"172.18.0.0/16"}) + if got == nil { + t.Fatal("tenantRuntimeContainerConfig returned nil") + } + if got.User != "1000:1000" { + t.Fatalf("User = %q, want rootless tenant runtime user", got.User) + } + if got.Image != cfg.Image { + t.Fatalf("Image = %q, want %q", got.Image, cfg.Image) + } + if got.Labels["pulse.managed"] != "true" { + t.Fatalf("Labels not preserved: %v", got.Labels) + } + if envValue(got.Env, "PUID") != "1000" || envValue(got.Env, "PGID") != "1000" { + t.Fatalf("PUID/PGID env missing from rootless config: %v", got.Env) + } + if envValue(got.Env, "PULSE_TRUSTED_PROXY_CIDRS") != "172.18.0.0/16" { + t.Fatalf("trusted proxy env = %q", envValue(got.Env, "PULSE_TRUSTED_PROXY_CIDRS")) + } + + custom := tenantRuntimeContainerConfig("t-acme", ManagerConfig{ + Image: "pulse:test", + BaseDomain: "msp.example.com", + TenantRuntimeUID: 1234, + TenantRuntimeGID: 5678, + }, labels, nil) + if custom.User != "1234:5678" { + t.Fatalf("custom User = %q, want configured rootless runtime user", custom.User) + } + if envValue(custom.Env, "PUID") != "1234" || envValue(custom.Env, "PGID") != "5678" { + t.Fatalf("custom PUID/PGID env missing from rootless config: %v", custom.Env) + } +} + func TestTenantRuntimeHostConfigAppliesEscapeHardening(t *testing.T) { t.Parallel() @@ -338,6 +382,35 @@ func TestPrepareTenantRuntimeMountSourcesAlignsOwnershipAndPermissions(t *testin t.Fatalf("prepareTenantRuntimeMountSources: %v", err) } + nestedDir := filepath.Join(tenantDataDir, "state") + nestedPath := filepath.Join(nestedDir, "runtime.db") + if err := os.MkdirAll(nestedDir, 0o755); err != nil { + t.Fatalf("mkdir nested state: %v", err) + } + if err := os.WriteFile(nestedPath, []byte("state"), 0o644); err != nil { + t.Fatalf("write nested state: %v", err) + } + if err := prepareTenantRuntimeMountSources(tenantDataDir, uid, gid); err != nil { + t.Fatalf("prepareTenantRuntimeMountSources after nested state: %v", err) + } + + for _, path := range []string{tenantDataDir, nestedDir, nestedPath} { + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat %s: %v", path, err) + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + t.Fatalf("%s stat type %T, want *syscall.Stat_t", path, info.Sys()) + } + if int(stat.Uid) != uid { + t.Fatalf("%s uid = %d, want %d", path, stat.Uid, uid) + } + if int(stat.Gid) != gid { + t.Fatalf("%s gid = %d, want %d", path, stat.Gid, gid) + } + } + for _, path := range paths { info, err := os.Stat(path) if err != nil {