pulse/internal/cloudcp/docker/manager_test.go

package docker

import (
	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"
	"syscall"
	"testing"

	"github.com/moby/moby/api/types/container"
	mobynetwork "github.com/moby/moby/api/types/network"
)

func TestTenantImmutableOwnershipPaths(t *testing.T) {
	t.Parallel()

	got := tenantImmutableOwnershipPaths()
	want := []string{
		"/etc/pulse/secrets/handoff.key",
		"/etc/pulse/.cloud_handoff_key",
	}
	if len(got) != len(want) {
		t.Fatalf("len(paths) = %d, want %d", len(got), len(want))
	}
	for i := range want {
		if got[i] != want[i] {
			t.Fatalf("paths[%d] = %q, want %q", i, got[i], want[i])
		}
	}
}

func TestTenantMountsKeepImmutableFilesReadOnly(t *testing.T) {
	t.Parallel()

	tenantDataDir := filepath.Join("/tmp", "tenant-data")
	mounts := tenantMounts(tenantDataDir)

	if len(mounts) != 3 {
		t.Fatalf("len(mounts) = %d, want 3", len(mounts))
	}

	if mounts[0].Target != "/etc/pulse" {
		t.Fatalf("mounts[0].Target = %q, want %q", mounts[0].Target, "/etc/pulse")
	}
	if mounts[0].ReadOnly {
		t.Fatalf("mounts[0].ReadOnly = true, want false")
	}

	checkMount := func(index int, wantTarget, wantSource string) {
		t.Helper()
		if mounts[index].Target != wantTarget {
			t.Fatalf("mounts[%d].Target = %q, want %q", index, mounts[index].Target, wantTarget)
		}
		if mounts[index].Source != wantSource {
			t.Fatalf("mounts[%d].Source = %q, want %q", index, mounts[index].Source, wantSource)
		}
		if !mounts[index].ReadOnly {
			t.Fatalf("mounts[%d].ReadOnly = false, want true", index)
		}
	}

	for _, mounted := range mounts {
		if mounted.Target == "/etc/pulse/billing.json" {
			t.Fatalf("billing.json should be writable through the tenant data mount, got dedicated mount %+v", mounted)
		}
	}

	checkMount(1, "/etc/pulse/secrets/handoff.key", filepath.Join(tenantDataDir, "secrets", "handoff.key"))
	checkMount(2, "/etc/pulse/.cloud_handoff_key", filepath.Join(tenantDataDir, ".cloud_handoff_key"))
}

func TestTenantEnvIncludesImmutableOwnershipContract(t *testing.T) {
	t.Parallel()

	env := tenantEnv("t-example", "", "cloud.pulserelay.pro", "pubkey-123", "https://cp.example-msp.com", []string{"172.18.0.0/16", "127.0.0.1/32"}, TenantReportBrandConfig{})
	want := map[string]bool{
		"PULSE_DATA_DIR=/etc/pulse":       true,
		"PULSE_HOSTED_MODE=true":          true,
		"PULSE_TENANT_ID=t-example":       true,
		"PULSE_MULTI_TENANT_ENABLED=true": true,
		"PUID=1000":                       true,
		"PGID=1000":                       true,
		"PULSE_PUBLIC_URL=https://t-example.cloud.pulserelay.pro":                                    true,
		"PULSE_TRIAL_ACTIVATION_PUBLIC_KEY=pubkey-123":                                               true,
		"PULSE_PRO_TRIAL_SIGNUP_URL=https://cp.example-msp.com":                                      true,
		"PULSE_TRUSTED_PROXY_CIDRS=172.18.0.0/16,127.0.0.1/32":                                       true,
		immutableOwnershipPathsEnv + "=/etc/pulse/secrets/handoff.key:/etc/pulse/.cloud_handoff_key": true,
	}
	if len(env) != len(want) {
		t.Fatalf("len(env) = %d, want %d", len(env), len(want))
	}
	for _, item := range env {
		if !want[item] {
			t.Fatalf("unexpected env item %q", item)
		}
	}
}

func TestTenantEnvOmitsEntitlementRefreshURLWhenUnset(t *testing.T) {
	t.Parallel()

	env := tenantEnv("t-example", "", "cloud.pulserelay.pro", "", "", nil, TenantReportBrandConfig{})
	for _, item := range env {
		if strings.HasPrefix(item, "PULSE_PRO_TRIAL_SIGNUP_URL=") {
			t.Fatalf("PULSE_PRO_TRIAL_SIGNUP_URL should be omitted when no control plane base URL is configured, got %q", item)
		}
	}
}

func TestCanonicalTenantRuntimeRoutingLowercasesHostedAddressing(t *testing.T) {
	t.Parallel()

	got := CanonicalTenantRuntimeRouting("T-AbCd123", "Cloud.PulseRelay.Pro")
	if got.Host != "t-abcd123.cloud.pulserelay.pro" {
		t.Fatalf("Host = %q, want %q", got.Host, "t-abcd123.cloud.pulserelay.pro")
	}
	if got.PublicURL != "https://t-abcd123.cloud.pulserelay.pro" {
		t.Fatalf("PublicURL = %q, want %q", got.PublicURL, "https://t-abcd123.cloud.pulserelay.pro")
	}
}

func TestTraefikLabelsUseCanonicalLowercaseHost(t *testing.T) {
	t.Parallel()

	labels := TraefikLabels("T-AbCd123", "Cloud.PulseRelay.Pro", 7655)
	got := labels["traefik.http.routers.pulse-T-AbCd123.rule"]
	if got != "Host(`t-abcd123.cloud.pulserelay.pro`)" {
		t.Fatalf("router rule = %q, want %q", got, "Host(`t-abcd123.cloud.pulserelay.pro`)")
	}
}

func TestTraefikLabelsPinTenantRuntimeToIsolatedNetwork(t *testing.T) {
	t.Parallel()

	labels := TraefikLabels("t-acme", "msp.example.com", 7655, "pulse-provider-msp-tenant-t-acme")
	if got := labels["traefik.docker.network"]; got != "pulse-provider-msp-tenant-t-acme" {
		t.Fatalf("traefik.docker.network = %q, want isolated tenant network", got)
	}
}

func TestTenantNetworkNameIsDerivedPerTenant(t *testing.T) {
	t.Parallel()

	mgr := &Manager{cfg: ManagerConfig{
		Network:               "pulse-provider-msp",
		IsolateTenantNetworks: true,
		TenantNetworkPrefix:   "pulse-provider-msp-tenant",
	}}

	gotA := mgr.tenantNetworkName("T-Acme")
	gotB := mgr.tenantNetworkName("T-Beta")
	if gotA != "pulse-provider-msp-tenant-t-acme" {
		t.Fatalf("tenantNetworkName(T-Acme) = %q", gotA)
	}
	if gotB != "pulse-provider-msp-tenant-t-beta" {
		t.Fatalf("tenantNetworkName(T-Beta) = %q", gotB)
	}
	if gotA == gotB {
		t.Fatalf("tenant networks must be distinct, got %q", gotA)
	}
}

func TestHealthCheckPrefersProviderMSPIsolatedTenantNetwork(t *testing.T) {
	t.Parallel()

	mgr := &Manager{cfg: ManagerConfig{
		Network:               "pulse-provider-msp",
		IsolateTenantNetworks: true,
		TenantNetworkPrefix:   "pulse-provider-msp-tenant",
	}}
	inspect := container.InspectResponse{
		Config: &container.Config{
			Labels: map[string]string{"pulse.tenant.id": "t-acme"},
		},
		NetworkSettings: &container.NetworkSettings{
			Networks: map[string]*mobynetwork.EndpointSettings{
				"pulse-provider-msp":               {},
				"pulse-provider-msp-tenant-t-acme": {},
			},
		},
	}

	got := mgr.healthCheckNetworkCandidates(inspect)
	if len(got) < 2 {
		t.Fatalf("healthCheckNetworkCandidates = %v, want tenant and ingress networks", got)
	}
	if got[0] != "pulse-provider-msp-tenant-t-acme" {
		t.Fatalf("first health network = %q, want isolated tenant network; all=%v", got[0], got)
	}
}

func TestTenantEnvLowercasesPublicURLHost(t *testing.T) {
	t.Parallel()

	env := tenantEnv("T-AbCd123", "", "Cloud.PulseRelay.Pro", "", "", nil, TenantReportBrandConfig{})
	want := "PULSE_PUBLIC_URL=https://t-abcd123.cloud.pulserelay.pro"
	for _, item := range env {
		if item == want {
			return
		}
	}
	t.Fatalf("tenantEnv() missing %q in %v", want, env)
}

func TestTenantEnvOmitsPublicURLWithoutTenantContext(t *testing.T) {
	t.Parallel()

	env := tenantEnv("", "", "", "", "", nil, TenantReportBrandConfig{})
	sawTenantID := false
	for _, item := range env {
		if strings.HasPrefix(item, "PULSE_PUBLIC_URL=") {
			t.Fatalf("unexpected public URL env item %q", item)
		}
		if item == "PULSE_TENANT_ID=" {
			sawTenantID = true
		}
	}
	if !sawTenantID {
		t.Fatalf("expected explicit empty tenant id env item, got %v", env)
	}
}

func TestTenantEnvStampsWorkspaceDisplayName(t *testing.T) {
	t.Parallel()

	env := tenantEnv("t-acme", "  Acme Rentals  ", "cloud.pulserelay.pro", "", "", nil, TenantReportBrandConfig{})
	want := "PULSE_TENANT_NAME=Acme Rentals"
	for _, item := range env {
		if item == want {
			return
		}
	}
	t.Fatalf("tenantEnv() missing %q in %v", want, env)
}

func TestTenantEnvOmitsTenantNameWhenUnresolved(t *testing.T) {
	t.Parallel()

	env := tenantEnv("t-acme", "", "cloud.pulserelay.pro", "", "", nil, TenantReportBrandConfig{})
	for _, item := range env {
		if strings.HasPrefix(item, "PULSE_TENANT_NAME=") {
			t.Fatalf("unexpected tenant name env item %q; name must be omitted so payloads fall back to the tenant ID", item)
		}
	}
}

func TestTenantRuntimeContainerConfigResolvesTenantDisplayName(t *testing.T) {
	t.Parallel()

	cfg := ManagerConfig{
		Image:      "pulse:test",
		BaseDomain: "cloud.pulserelay.pro",
		TenantDisplayName: func(tenantID string) string {
			if tenantID != "t-acme" {
				t.Fatalf("resolver received tenant id %q, want t-acme", tenantID)
			}
			return "Acme Rentals"
		},
	}
	containerCfg := tenantRuntimeContainerConfig("t-acme", cfg, nil, nil)
	if got := envValue(containerCfg.Env, "PULSE_TENANT_NAME"); got != "Acme Rentals" {
		t.Fatalf("PULSE_TENANT_NAME = %q, want %q; env=%v", got, "Acme Rentals", containerCfg.Env)
	}

	withoutResolver := tenantRuntimeContainerConfig("t-acme", ManagerConfig{Image: "pulse:test", BaseDomain: "cloud.pulserelay.pro"}, nil, nil)
	for _, item := range withoutResolver.Env {
		if strings.HasPrefix(item, "PULSE_TENANT_NAME=") {
			t.Fatalf("unexpected tenant name env item %q without a resolver", item)
		}
	}
}

func TestTenantEnvIncludesProviderReportBrandDefaults(t *testing.T) {
	t.Parallel()

	env := tenantEnv("t-acme", "", "msp.example.com", "", "", nil, TenantReportBrandConfig{
		DisplayName: "Acme Managed IT",
		LogoBase64:  "iVBORw0KGgo=",
		LogoFormat:  "png",
	})
	want := map[string]bool{
		"PULSE_REPORT_PROVIDER_BRAND_DISPLAY_NAME=Acme Managed IT": true,
		"PULSE_REPORT_PROVIDER_BRAND_LOGO_BASE64=iVBORw0KGgo=":     true,
		"PULSE_REPORT_PROVIDER_BRAND_LOGO_FORMAT=png":              true,
	}
	for _, item := range env {
		delete(want, item)
	}
	if len(want) > 0 {
		t.Fatalf("tenantEnv() missing report brand env items: %v; env=%v", want, env)
	}
}

func TestTenantRuntimeLogConfigBoundsJSONLogs(t *testing.T) {
	t.Parallel()

	got := tenantRuntimeLogConfig("", 0)
	if got.Type != "json-file" {
		t.Fatalf("LogConfig.Type = %q, want json-file", got.Type)
	}
	if got.Config["max-size"] != defaultTenantLogMaxSize {
		t.Fatalf("max-size = %q, want %q", got.Config["max-size"], defaultTenantLogMaxSize)
	}
	if got.Config["max-file"] != "3" {
		t.Fatalf("max-file = %q, want 3", got.Config["max-file"])
	}

	custom := tenantRuntimeLogConfig("25m", 4)
	if custom.Config["max-size"] != "25m" {
		t.Fatalf("custom max-size = %q, want 25m", custom.Config["max-size"])
	}
	if custom.Config["max-file"] != "4" {
		t.Fatalf("custom max-file = %q, want 4", custom.Config["max-file"])
	}
}

func TestTenantRuntimeContainerConfigRunsRootless(t *testing.T) {
	t.Parallel()

	cfg := ManagerConfig{
		Image:                    "pulse:test",
		BaseDomain:               "msp.example.com",
		TrialActivationPublicKey: "pubkey-123",
	}
	labels := map[string]string{"pulse.managed": "true"}

	got := tenantRuntimeContainerConfig("t-acme", cfg, labels, []string{"172.18.0.0/16"})
	if got == nil {
		t.Fatal("tenantRuntimeContainerConfig returned nil")
	}
	if got.User != "1000:1000" {
		t.Fatalf("User = %q, want rootless tenant runtime user", got.User)
	}
	if got.Image != cfg.Image {
		t.Fatalf("Image = %q, want %q", got.Image, cfg.Image)
	}
	if got.Labels["pulse.managed"] != "true" {
		t.Fatalf("Labels not preserved: %v", got.Labels)
	}
	if envValue(got.Env, "PUID") != "1000" || envValue(got.Env, "PGID") != "1000" {
		t.Fatalf("PUID/PGID env missing from rootless config: %v", got.Env)
	}
	if envValue(got.Env, "PULSE_TRUSTED_PROXY_CIDRS") != "172.18.0.0/16" {
		t.Fatalf("trusted proxy env = %q", envValue(got.Env, "PULSE_TRUSTED_PROXY_CIDRS"))
	}

	custom := tenantRuntimeContainerConfig("t-acme", ManagerConfig{
		Image:            "pulse:test",
		BaseDomain:       "msp.example.com",
		TenantRuntimeUID: 1234,
		TenantRuntimeGID: 5678,
	}, labels, nil)
	if custom.User != "1234:5678" {
		t.Fatalf("custom User = %q, want configured rootless runtime user", custom.User)
	}
	if envValue(custom.Env, "PUID") != "1234" || envValue(custom.Env, "PGID") != "5678" {
		t.Fatalf("custom PUID/PGID env missing from rootless config: %v", custom.Env)
	}
}

func TestTenantRuntimeHostConfigAppliesEscapeHardening(t *testing.T) {
	t.Parallel()

	cfg := ManagerConfig{
		MemoryLimit:      512 * 1024 * 1024,
		CPUShares:        512,
		TenantLogMaxSize: "20m",
		TenantLogMaxFile: 2,
	}
	hostCfg := tenantRuntimeHostConfig("/data/tenants/t-acme", cfg)
	if hostCfg == nil {
		t.Fatal("tenantRuntimeHostConfig returned nil")
	}
	if !hostCfg.ReadonlyRootfs {
		t.Fatal("tenant runtime root filesystem must be read-only")
	}
	if got := hostCfg.SecurityOpt; len(got) != 1 || got[0] != "no-new-privileges:true" {
		t.Fatalf("SecurityOpt = %v, want no-new-privileges", got)
	}
	if got := hostCfg.CapDrop; len(got) != 1 || got[0] != "ALL" {
		t.Fatalf("CapDrop = %v, want [ALL]", got)
	}
	if _, ok := hostCfg.Tmpfs["/tmp"]; !ok {
		t.Fatalf("Tmpfs missing /tmp: %v", hostCfg.Tmpfs)
	}
	if _, ok := hostCfg.Tmpfs["/run"]; !ok {
		t.Fatalf("Tmpfs missing /run: %v", hostCfg.Tmpfs)
	}
	if hostCfg.LogConfig.Type != "json-file" || hostCfg.LogConfig.Config["max-size"] != "20m" || hostCfg.LogConfig.Config["max-file"] != "2" {
		t.Fatalf("LogConfig not preserved: %+v", hostCfg.LogConfig)
	}
	if hostCfg.Resources.Memory != cfg.MemoryLimit || hostCfg.Resources.CPUShares != cfg.CPUShares {
		t.Fatalf("Resources not preserved: %+v", hostCfg.Resources)
	}
}

func TestCanonicalTrustedProxyCIDR(t *testing.T) {
	t.Parallel()

	tests := []struct {
		input string
		want  string
	}{
		{input: "172.18.0.0/16", want: "172.18.0.0/16"},
		{input: "172.18.12.34/16", want: "172.18.0.0/16"},
		{input: "127.0.0.1", want: "127.0.0.1/32"},
		{input: "2001:db8::1", want: "2001:db8::1/128"},
		{input: "not-a-cidr", want: ""},
	}

	for _, tc := range tests {
		t.Run(tc.input, func(t *testing.T) {
			if got := canonicalTrustedProxyCIDR(tc.input); got != tc.want {
				t.Fatalf("canonicalTrustedProxyCIDR(%q) = %q, want %q", tc.input, got, tc.want)
			}
		})
	}
}

func TestPrepareTenantRuntimeMountSourcesAlignsOwnershipAndPermissions(t *testing.T) {
	t.Parallel()

	tenantDataDir := t.TempDir()
	secretsDir := filepath.Join(tenantDataDir, "secrets")
	if err := os.MkdirAll(secretsDir, 0o755); err != nil {
		t.Fatalf("mkdir secrets: %v", err)
	}

	paths := []string{
		filepath.Join(tenantDataDir, "billing.json"),
		filepath.Join(secretsDir, "handoff.key"),
		filepath.Join(tenantDataDir, ".cloud_handoff_key"),
	}
	for _, path := range paths {
		if err := os.WriteFile(path, []byte("secret"), 0o644); err != nil {
			t.Fatalf("write %s: %v", path, err)
		}
	}

	uid := os.Getuid()
	gid := os.Getgid()
	if err := prepareTenantRuntimeMountSources(tenantDataDir, uid, gid); err != nil {
		t.Fatalf("prepareTenantRuntimeMountSources: %v", err)
	}

	nestedDir := filepath.Join(tenantDataDir, "state")
	nestedPath := filepath.Join(nestedDir, "runtime.db")
	if err := os.MkdirAll(nestedDir, 0o755); err != nil {
		t.Fatalf("mkdir nested state: %v", err)
	}
	if err := os.WriteFile(nestedPath, []byte("state"), 0o644); err != nil {
		t.Fatalf("write nested state: %v", err)
	}
	mgr := &Manager{cfg: ManagerConfig{TenantRuntimeUID: uid, TenantRuntimeGID: gid}}
	if err := mgr.ReconcileTenantRuntimeMountSources(tenantDataDir); err != nil {
		t.Fatalf("ReconcileTenantRuntimeMountSources after nested state: %v", err)
	}

	for _, path := range []string{tenantDataDir, nestedDir, nestedPath} {
		info, err := os.Stat(path)
		if err != nil {
			t.Fatalf("stat %s: %v", path, err)
		}
		stat, ok := info.Sys().(*syscall.Stat_t)
		if !ok {
			t.Fatalf("%s stat type %T, want *syscall.Stat_t", path, info.Sys())
		}
		if int(stat.Uid) != uid {
			t.Fatalf("%s uid = %d, want %d", path, stat.Uid, uid)
		}
		if int(stat.Gid) != gid {
			t.Fatalf("%s gid = %d, want %d", path, stat.Gid, gid)
		}
	}

	for _, path := range paths {
		info, err := os.Stat(path)
		if err != nil {
			t.Fatalf("stat %s: %v", path, err)
		}
		if info.Mode().Perm() != 0o600 {
			t.Fatalf("%s perms = %o, want %o", path, info.Mode().Perm(), 0o600)
		}
		stat, ok := info.Sys().(*syscall.Stat_t)
		if !ok {
			t.Fatalf("%s stat type %T, want *syscall.Stat_t", path, info.Sys())
		}
		if int(stat.Uid) != uid {
			t.Fatalf("%s uid = %d, want %d", path, stat.Uid, uid)
		}
		if int(stat.Gid) != gid {
			t.Fatalf("%s gid = %d, want %d", path, stat.Gid, gid)
		}
	}
}

func TestCreateAndStartFailsBeforeImmutablePrepWhenDockerUnavailable(t *testing.T) {
	t.Setenv("DOCKER_HOST", "unix:///tmp/pulse-missing-docker.sock")
	t.Setenv("DOCKER_TLS_VERIFY", "")
	t.Setenv("DOCKER_CERT_PATH", "")

	mgr, err := NewManager(ManagerConfig{
		Image:      "pulse:test",
		Network:    "bridge",
		BaseDomain: "cloud.example.com",
	})
	if err != nil {
		t.Fatalf("NewManager: %v", err)
	}
	t.Cleanup(func() { _ = mgr.Close() })

	tenantDataDir := t.TempDir()
	secretsDir := filepath.Join(tenantDataDir, "secrets")
	if err := os.MkdirAll(secretsDir, 0o755); err != nil {
		t.Fatalf("mkdir secrets: %v", err)
	}
	for _, path := range []string{
		filepath.Join(tenantDataDir, "billing.json"),
		filepath.Join(secretsDir, "handoff.key"),
		filepath.Join(tenantDataDir, ".cloud_handoff_key"),
	} {
		if err := os.WriteFile(path, []byte("secret"), 0o600); err != nil {
			t.Fatalf("write %s: %v", path, err)
		}
	}

	_, err = mgr.CreateAndStart(context.Background(), "t-unavailable", tenantDataDir)
	if err == nil {
		t.Fatal("expected docker daemon error")
	}
	if !strings.Contains(err.Error(), "ping docker daemon") {
		t.Fatalf("CreateAndStart error = %v, want ping docker daemon failure", err)
	}
	if strings.Contains(err.Error(), "prepare tenant runtime mounts") {
		t.Fatalf("CreateAndStart error = %v, want daemon failure before mount preparation", err)
	}
}

func TestCheckRuntimePrerequisitesReportsMissingDockerDaemon(t *testing.T) {
	t.Setenv("DOCKER_HOST", "unix:///tmp/pulse-missing-docker-preflight.sock")
	t.Setenv("DOCKER_TLS_VERIFY", "")
	t.Setenv("DOCKER_CERT_PATH", "")

	mgr, err := NewManager(ManagerConfig{
		Image:      "pulse:test",
		Network:    "pulse-provider-msp",
		BaseDomain: "msp.example.com",
	})
	if err != nil {
		t.Fatalf("NewManager: %v", err)
	}
	t.Cleanup(func() { _ = mgr.Close() })

	report, err := mgr.CheckRuntimePrerequisites(context.Background(), RuntimePrerequisiteOptions{PullImage: false})
	if err != nil {
		t.Fatalf("CheckRuntimePrerequisites: %v", err)
	}
	if report == nil {
		t.Fatal("report is nil")
	}
	if report.OK {
		t.Fatalf("report.OK = true, want false")
	}
	if report.DockerReachable {
		t.Fatal("DockerReachable = true, want false")
	}
	if report.NetworkName != "pulse-provider-msp" {
		t.Fatalf("NetworkName = %q, want pulse-provider-msp", report.NetworkName)
	}
	if report.ImageRef != "pulse:test" {
		t.Fatalf("ImageRef = %q, want pulse:test", report.ImageRef)
	}
	if got := strings.Join(report.Failures, "; "); !strings.Contains(got, "ping docker daemon") {
		t.Fatalf("failures = %q, want docker daemon ping failure", got)
	}
}

// provider-msp status and upgrade.sh probe tenant health from a one-off
// container that is not attached to any isolated tenant network. The image's
// own Docker HEALTHCHECK result must decide health whenever it exists, so a
// healthy client does not read as unhealthy just because the caller cannot
// route to it; only a container with no HEALTHCHECK falls back to HTTP.
func TestDockerRecordedHealthDecidesWhenTheImageDeclaresOne(t *testing.T) {
	t.Parallel()

	for _, tc := range []struct {
		name         string
		health       *container.Health
		wantHealthy  bool
		wantRecorded bool
	}{
		{name: "healthy", health: &container.Health{Status: container.Healthy}, wantHealthy: true, wantRecorded: true},
		{name: "unhealthy", health: &container.Health{Status: container.Unhealthy}, wantHealthy: false, wantRecorded: true},
		{name: "starting", health: &container.Health{Status: container.Starting}, wantHealthy: false, wantRecorded: true},
		{name: "no healthcheck", health: &container.Health{Status: container.NoHealthcheck}, wantRecorded: false},
		{name: "health absent", health: nil, wantRecorded: false},
	} {
		inspect := container.InspectResponse{State: &container.State{Running: true, Health: tc.health}}
		healthy, recorded := dockerRecordedHealth(inspect)
		if healthy != tc.wantHealthy || recorded != tc.wantRecorded {
			t.Errorf("%s: dockerRecordedHealth = (%v, %v), want (%v, %v)", tc.name, healthy, recorded, tc.wantHealthy, tc.wantRecorded)
		}
	}
}

// A client created before per-client networks has no isolated network to
// rejoin. Reconciling it must be a quiet no-op, not a warning every minute.
func TestEnsureSupportContainersOnTenantNetworkSkipsMissingNetwork(t *testing.T) {
	var mutations []string
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Api-Version", "1.47")
		if strings.HasSuffix(r.URL.Path, "/_ping") {
			_, _ = w.Write([]byte("OK"))
			return
		}
		if r.Method != http.MethodGet {
			mutations = append(mutations, r.Method+" "+r.URL.Path)
		}
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusNotFound)
		_, _ = w.Write([]byte(`{"message":"network not found"}`))
	}))
	t.Cleanup(srv.Close)
	t.Setenv("DOCKER_HOST", "tcp://"+strings.TrimPrefix(srv.URL, "http://"))
	t.Setenv("DOCKER_TLS_VERIFY", "")
	t.Setenv("DOCKER_CERT_PATH", "")

	mgr, err := NewManager(ManagerConfig{Image: "pulse:test", Network: "pulse-provider-msp", IsolateTenantNetworks: true})
	if err != nil {
		t.Fatalf("NewManager: %v", err)
	}
	t.Cleanup(func() { _ = mgr.Close() })

	if err := mgr.EnsureSupportContainersOnTenantNetwork(context.Background(), "t-legacy"); err != nil {
		t.Fatalf("EnsureSupportContainersOnTenantNetwork = %v, want nil for a client without an isolated network", err)
	}
	if len(mutations) != 0 {
		t.Fatalf("reconcile changed the host for a missing network: %v", mutations)
	}
}

func TestEnsureSupportContainersOnTenantNetworkRejectsWrongOwner(t *testing.T) {
	var mutations []string
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Api-Version", "1.47")
		if strings.HasSuffix(r.URL.Path, "/_ping") {
			_, _ = w.Write([]byte("OK"))
			return
		}
		if r.Method != http.MethodGet {
			mutations = append(mutations, r.Method+" "+r.URL.Path)
		}
		w.Header().Set("Content-Type", "application/json")
		_, _ = w.Write([]byte(`{"Name":"pulse-provider-msp-tenant-t-acme","Id":"net-1","Labels":{"pulse.tenant.id":"t-other","pulse.provider-msp.network":"tenant"}}`))
	}))
	t.Cleanup(srv.Close)
	t.Setenv("DOCKER_HOST", "tcp://"+strings.TrimPrefix(srv.URL, "http://"))
	t.Setenv("DOCKER_TLS_VERIFY", "")
	t.Setenv("DOCKER_CERT_PATH", "")

	mgr, err := NewManager(ManagerConfig{Image: "pulse:test", Network: "pulse-provider-msp", IsolateTenantNetworks: true})
	if err != nil {
		t.Fatalf("NewManager: %v", err)
	}
	t.Cleanup(func() { _ = mgr.Close() })

	if err := mgr.EnsureSupportContainersOnTenantNetwork(context.Background(), "t-acme"); err == nil {
		t.Fatal("expected wrong-owner network to be rejected")
	}
	if len(mutations) != 0 {
		t.Fatalf("reconcile changed wrong-owner network: %v", mutations)
	}
}

func TestEnsureTenantNetworkRejectsUnownedExistingNetwork(t *testing.T) {
	var labels map[string]string
	var mutations []string
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Api-Version", "1.47")
		if strings.HasSuffix(r.URL.Path, "/_ping") {
			_, _ = w.Write([]byte("OK"))
			return
		}
		if r.Method != http.MethodGet {
			mutations = append(mutations, r.Method+" "+r.URL.Path)
		}
		w.Header().Set("Content-Type", "application/json")
		_ = json.NewEncoder(w).Encode(map[string]any{
			"Name":   "pulse-provider-msp-tenant-t-acme",
			"Id":     "net-1",
			"Labels": labels,
		})
	}))
	t.Cleanup(srv.Close)
	t.Setenv("DOCKER_HOST", "tcp://"+strings.TrimPrefix(srv.URL, "http://"))
	t.Setenv("DOCKER_TLS_VERIFY", "")
	t.Setenv("DOCKER_CERT_PATH", "")

	mgr, err := NewManager(ManagerConfig{Image: "pulse:test", Network: "pulse-provider-msp", IsolateTenantNetworks: true})
	if err != nil {
		t.Fatalf("NewManager: %v", err)
	}
	t.Cleanup(func() { _ = mgr.Close() })

	for _, tc := range []struct {
		name   string
		labels map[string]string
		wantOK bool
	}{
		{name: "unlabelled"},
		{name: "wrong tenant", labels: map[string]string{"pulse.tenant.id": "t-other", tenantRuntimeNetworkLabel: tenantRuntimeNetworkLabelValue}},
		{name: "missing runtime label", labels: map[string]string{"pulse.tenant.id": "t-acme"}},
		{name: "wrong runtime label", labels: map[string]string{"pulse.tenant.id": "t-acme", tenantRuntimeNetworkLabel: "ingress"}},
		{name: "owned tenant network", labels: map[string]string{"pulse.tenant.id": "t-acme", tenantRuntimeNetworkLabel: tenantRuntimeNetworkLabelValue}, wantOK: true},
	} {
		t.Run(tc.name, func(t *testing.T) {
			labels = tc.labels
			mutations = nil
			got, err := mgr.ensureTenantNetwork(context.Background(), "t-acme")
			if tc.wantOK {
				if err != nil || got != "pulse-provider-msp-tenant-t-acme" {
					t.Fatalf("ensureTenantNetwork = (%q, %v), want owned network", got, err)
				}
			} else if err == nil || got != "" {
				t.Fatalf("ensureTenantNetwork = (%q, %v), want fail closed", got, err)
			}
			if len(mutations) != 0 {
				t.Fatalf("ensureTenantNetwork mutated existing network: %v", mutations)
			}
		})
	}
	if got, err := mgr.ensureTenantNetwork(context.Background(), ""); err == nil || got != "" {
		t.Fatalf("ensureTenantNetwork(empty) = (%q, %v), want rejection", got, err)
	}
}

func TestTenantNetworkCleanupSelectsOnlyOwnedNetwork(t *testing.T) {
	ownedLabels := map[string]string{"pulse.tenant.id": "t-acme", tenantRuntimeNetworkLabel: tenantRuntimeNetworkLabelValue}
	var containerLabels, targetLabels map[string]string
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Api-Version", "1.47")
		w.Header().Set("Content-Type", "application/json")
		switch {
		case strings.HasSuffix(r.URL.Path, "/containers/client-a/json"):
			_ = json.NewEncoder(w).Encode(map[string]any{
				"Id": "client-a", "Config": map[string]any{"Labels": containerLabels},
				"NetworkSettings": map[string]any{"Networks": map[string]any{
					"provider-a-tenant-t-acme": map[string]any{},
					"provider-b-tenant-t-acme": map[string]any{},
				}},
			})
		case strings.HasSuffix(r.URL.Path, "/networks/provider-a-tenant-t-acme"):
			_ = json.NewEncoder(w).Encode(map[string]any{"Name": "provider-a-tenant-t-acme", "Labels": targetLabels})
		case strings.HasSuffix(r.URL.Path, "/networks/provider-b-tenant-t-acme"):
			_ = json.NewEncoder(w).Encode(map[string]any{"Name": "provider-b-tenant-t-acme", "Labels": ownedLabels})
		default:
			w.WriteHeader(http.StatusNotFound)
		}
	}))
	t.Cleanup(srv.Close)
	t.Setenv("DOCKER_HOST", "tcp://"+strings.TrimPrefix(srv.URL, "http://"))
	t.Setenv("DOCKER_TLS_VERIFY", "")
	t.Setenv("DOCKER_CERT_PATH", "")

	mgr, err := NewManager(ManagerConfig{Network: "provider-a", IsolateTenantNetworks: true})
	if err != nil {
		t.Fatalf("NewManager: %v", err)
	}
	t.Cleanup(func() { _ = mgr.Close() })
	for _, tc := range []struct {
		name            string
		containerLabels map[string]string
		targetLabels    map[string]string
		want            []string
	}{
		{name: "owned", containerLabels: map[string]string{"pulse.tenant.id": "t-acme", "pulse.managed": "true"}, targetLabels: ownedLabels, want: []string{"provider-a-tenant-t-acme"}},
		{name: "unlabelled network", containerLabels: map[string]string{"pulse.tenant.id": "t-acme", "pulse.managed": "true"}},
		{name: "wrong network tenant", containerLabels: map[string]string{"pulse.tenant.id": "t-acme", "pulse.managed": "true"}, targetLabels: map[string]string{"pulse.tenant.id": "t-other", tenantRuntimeNetworkLabel: tenantRuntimeNetworkLabelValue}},
		{name: "unmanaged container", containerLabels: map[string]string{"pulse.tenant.id": "t-acme"}, targetLabels: ownedLabels},
		{name: "no container tenant", containerLabels: map[string]string{"pulse.managed": "true"}, targetLabels: ownedLabels},
	} {
		t.Run(tc.name, func(t *testing.T) {
			containerLabels, targetLabels = tc.containerLabels, tc.targetLabels
			got := mgr.tenantNetworkNamesForContainer(context.Background(), "client-a")
			if len(got) != len(tc.want) || (len(got) > 0 && got[0] != tc.want[0]) {
				t.Fatalf("tenantNetworkNamesForContainer = %v, want %v", got, tc.want)
			}
		})
	}
}

func TestTenantNetworkCleanupDisconnectsOnlyAttachedLocalSupport(t *testing.T) {
	const target = "provider-a-tenant-t-acme"
	const localAttached = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
	const localReplacement = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
	const otherAttached = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
	var listFilters map[string]map[string]bool
	var disconnected []string
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Api-Version", "1.47")
		w.Header().Set("Content-Type", "application/json")
		switch {
		case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/networks/"+target):
			_ = json.NewEncoder(w).Encode(map[string]any{
				"Name": target, "Labels": map[string]string{"pulse.tenant.id": "t-acme", tenantRuntimeNetworkLabel: tenantRuntimeNetworkLabelValue},
				"Containers": map[string]any{localAttached: map[string]any{}, otherAttached: map[string]any{}},
			})
		case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/containers/json"):
			if err := json.Unmarshal([]byte(r.URL.Query().Get("filters")), &listFilters); err != nil {
				w.WriteHeader(http.StatusBadRequest)
				return
			}
			items := []map[string]string{{"Id": localAttached}, {"Id": localReplacement}}
			if !listFilters["network"]["provider-a"] {
				items = append(items, map[string]string{"Id": otherAttached})
			}
			_ = json.NewEncoder(w).Encode(items)
		case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/networks/"+target+"/disconnect"):
			var body struct{ Container string }
			if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
				w.WriteHeader(http.StatusBadRequest)
				return
			}
			disconnected = append(disconnected, body.Container)
			_, _ = w.Write([]byte(`{}`))
		default:
			w.WriteHeader(http.StatusNotFound)
		}
	}))
	t.Cleanup(srv.Close)
	t.Setenv("DOCKER_HOST", "tcp://"+strings.TrimPrefix(srv.URL, "http://"))
	t.Setenv("DOCKER_TLS_VERIFY", "")
	t.Setenv("DOCKER_CERT_PATH", "")

	mgr, err := NewManager(ManagerConfig{Network: "provider-a", IsolateTenantNetworks: true, SupportContainerLabels: []string{providerSupportTraefikLabel}})
	if err != nil {
		t.Fatalf("NewManager: %v", err)
	}
	t.Cleanup(func() { _ = mgr.Close() })
	if err := mgr.disconnectSupportContainersFromTenantNetwork(context.Background(), target); err != nil {
		t.Fatalf("disconnectSupportContainersFromTenantNetwork: %v", err)
	}
	if !listFilters["network"]["provider-a"] || !listFilters["label"][providerSupportTraefikLabel] {
		t.Fatalf("support selection did not identify the local provider: %v", listFilters)
	}
	if len(disconnected) != 1 || disconnected[0] != localAttached {
		t.Fatalf("disconnected = %v, want only attached local support %s", disconnected, localAttached)
	}
}