pulse/internal/monitoring/monitor_docker_test.go

package monitoring

import (
	"context"
	"errors"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/mock"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"
)

func newTestMonitor(t *testing.T) *Monitor {
	t.Helper()

	m := &Monitor{
		state:               models.NewState(),
		alertManager:        alerts.NewManager(),
		removedDockerHosts:  make(map[string]time.Time),
		rateTracker:         NewRateTracker(),
		metricsHistory:      NewMetricsHistory(1000, 24*time.Hour),
		dockerTokenBindings: make(map[string]string),
		guestMetadataStore:  config.NewGuestMetadataStore(t.TempDir(), nil),
		dockerMetadataStore: config.NewDockerMetadataStore(t.TempDir(), nil),
	}
	t.Cleanup(func() { m.alertManager.Stop() })
	return m
}

func TestApplyDockerReportGeneratesUniqueIDsForCollidingHosts(t *testing.T) {
	monitor := newTestMonitor(t)

	baseTimestamp := time.Now().UTC()
	baseReport := agentsdocker.Report{
		Agent: agentsdocker.AgentInfo{
			Version:         "1.0.0",
			IntervalSeconds: 30,
		},
		Host: agentsdocker.HostInfo{
			Hostname:         "docker-host",
			Name:             "Docker Host",
			MachineID:        "machine-duplicate",
			DockerVersion:    "26.0.0",
			TotalCPU:         4,
			TotalMemoryBytes: 8 << 30,
			UptimeSeconds:    120,
		},
		Containers: []agentsdocker.Container{
			{ID: "container-1", Name: "nginx"},
		},
		Timestamp: baseTimestamp,
	}

	token1 := &config.APITokenRecord{ID: "token-host-1", Name: "Host 1"}
	host1, err := monitor.ApplyDockerReport(baseReport, token1)
	if err != nil {
		t.Fatalf("ApplyDockerReport host1: %v", err)
	}
	if host1.ID == "" {
		t.Fatalf("expected host1 to have an identifier")
	}

	hosts := monitor.state.GetDockerHosts()
	if len(hosts) != 1 {
		t.Fatalf("expected 1 host after first report, got %d", len(hosts))
	}

	secondReport := baseReport
	secondReport.Host.Name = "Docker Host Clone"
	secondReport.Timestamp = baseTimestamp.Add(45 * time.Second)

	token2 := &config.APITokenRecord{ID: "token-host-2", Name: "Host 2"}
	host2, err := monitor.ApplyDockerReport(secondReport, token2)
	if err != nil {
		t.Fatalf("ApplyDockerReport host2: %v", err)
	}
	if host2.ID == "" {
		t.Fatalf("expected host2 to have an identifier")
	}
	if host2.ID == host1.ID {
		t.Fatalf("expected unique identifiers, but both hosts share %q", host2.ID)
	}

	hosts = monitor.state.GetDockerHosts()
	if len(hosts) != 2 {
		t.Fatalf("expected 2 hosts after second report, got %d", len(hosts))
	}

	secondReport.Timestamp = secondReport.Timestamp.Add(45 * time.Second)
	secondReport.Containers = append(secondReport.Containers, agentsdocker.Container{
		ID:   "container-2",
		Name: "redis",
	})

	updatedHost2, err := monitor.ApplyDockerReport(secondReport, token2)
	if err != nil {
		t.Fatalf("ApplyDockerReport host2 update: %v", err)
	}
	if updatedHost2.ID != host2.ID {
		t.Fatalf("expected host2 to retain identifier %q, got %q", host2.ID, updatedHost2.ID)
	}

	hosts = monitor.state.GetDockerHosts()
	var found models.DockerHost
	for _, h := range hosts {
		if h.ID == host2.ID {
			found = h
			break
		}
	}
	if found.ID == "" {
		t.Fatalf("failed to locate host2 in state after update")
	}
	if len(found.Containers) != 2 {
		t.Fatalf("expected host2 to have 2 containers after update, got %d", len(found.Containers))
	}
}

func TestApplyDockerReportUsesTokenToDisambiguateAgentIDCollisions(t *testing.T) {
	monitor := newTestMonitor(t)

	baseReport := agentsdocker.Report{
		Agent: agentsdocker.AgentInfo{
			ID:              "duplicate-agent",
			Version:         "1.0.0",
			IntervalSeconds: 30,
		},
		Host: agentsdocker.HostInfo{
			Hostname:         "docker-one",
			Name:             "Docker One",
			MachineID:        "machine-A",
			DockerVersion:    "26.0.0",
			TotalCPU:         4,
			TotalMemoryBytes: 16 << 30,
			UptimeSeconds:    120,
		},
		Containers: []agentsdocker.Container{
			{ID: "container-a", Name: "api"},
		},
		Timestamp: time.Now().UTC(),
	}

	tokenOne := &config.APITokenRecord{ID: "token-one", Name: "Token One"}
	hostOne, err := monitor.ApplyDockerReport(baseReport, tokenOne)
	if err != nil {
		t.Fatalf("ApplyDockerReport hostOne: %v", err)
	}
	if hostOne.ID == "" {
		t.Fatal("expected hostOne to receive an identifier")
	}

	secondReport := baseReport
	secondReport.Host.Hostname = "docker-two"
	secondReport.Host.Name = "Docker Two"
	secondReport.Host.MachineID = "machine-B"
	secondReport.Containers = []agentsdocker.Container{
		{ID: "container-b", Name: "db"},
	}
	secondReport.Timestamp = baseReport.Timestamp.Add(30 * time.Second)

	tokenTwo := &config.APITokenRecord{ID: "token-two", Name: "Token Two"}
	hostTwo, err := monitor.ApplyDockerReport(secondReport, tokenTwo)
	if err != nil {
		t.Fatalf("ApplyDockerReport hostTwo: %v", err)
	}

	if hostTwo.ID == "" {
		t.Fatal("expected hostTwo to receive an identifier")
	}
	if hostOne.ID == hostTwo.ID {
		t.Fatalf("expected different identifiers for hosts sharing an agent ID, got %q", hostOne.ID)
	}

	hosts := monitor.state.GetDockerHosts()
	if len(hosts) != 2 {
		t.Fatalf("expected 2 hosts after two reports, got %d", len(hosts))
	}

	updatedReport := baseReport
	updatedReport.Timestamp = baseReport.Timestamp.Add(60 * time.Second)
	updatedReport.Containers = append(updatedReport.Containers, agentsdocker.Container{
		ID:   "container-c",
		Name: "cache",
	})

	updatedHostOne, err := monitor.ApplyDockerReport(updatedReport, tokenOne)
	if err != nil {
		t.Fatalf("ApplyDockerReport hostOne update: %v", err)
	}
	if updatedHostOne.ID != hostOne.ID {
		t.Fatalf("expected hostOne to retain identifier %q, got %q", hostOne.ID, updatedHostOne.ID)
	}
}

func TestApplyDockerReportSkipsMetricsHistoryInMockMode(t *testing.T) {
	previous := mock.IsMockEnabled()
	mustSetMockEnabled(t, true)
	t.Cleanup(func() { mustSetMockEnabled(t, previous) })

	monitor := newTestMonitor(t)
	report := agentsdocker.Report{
		Agent: agentsdocker.AgentInfo{
			ID:              "docker-agent-1",
			Version:         "1.0.0",
			IntervalSeconds: 30,
		},
		Host: agentsdocker.HostInfo{
			Hostname:         "docker-demo",
			Name:             "Docker Demo",
			MachineID:        "docker-machine",
			DockerVersion:    "26.0.0",
			TotalCPU:         4,
			TotalMemoryBytes: 8 << 30,
			UptimeSeconds:    120,
			CPUUsagePercent:  37,
			Memory: agentsdocker.MemoryMetric{
				TotalBytes: 8 << 30,
				UsedBytes:  4 << 30,
				FreeBytes:  4 << 30,
				Usage:      50,
			},
			Disks: []agentsdocker.Disk{
				{
					Device:     "/dev/vda1",
					Mountpoint: "/",
					TotalBytes: 1000,
					UsedBytes:  500,
					FreeBytes:  500,
					Usage:      50,
				},
			},
		},
		Containers: []agentsdocker.Container{
			{
				ID:                  "docker-cont-1",
				Name:                "api",
				CPUPercent:          48,
				MemoryPercent:       62,
				RootFilesystemBytes: 1000,
				WritableLayerBytes:  240,
				NetworkRXBytes:      120,
				NetworkTXBytes:      80,
				BlockIO:             &agentsdocker.ContainerBlockIO{ReadBytes: 220, WriteBytes: 140},
			},
		},
		Timestamp: time.Now().UTC(),
	}

	host, err := monitor.ApplyDockerReport(report, nil)
	if err != nil {
		t.Fatalf("ApplyDockerReport: %v", err)
	}

	if got := monitor.metricsHistory.GetGuestMetrics("dockerHost:"+host.ID, "cpu", time.Hour); len(got) != 0 {
		t.Fatalf("expected mock mode to skip docker host metrics history, got %+v", got)
	}
	if got := monitor.metricsHistory.GetGuestMetrics("docker:docker-cont-1", "cpu", time.Hour); len(got) != 0 {
		t.Fatalf("expected mock mode to skip docker container metrics history, got %+v", got)
	}
}

func TestApplyDockerReportIncludesContainerDiskDetails(t *testing.T) {
	timestamp := time.Now().UTC()
	oomKilled := false
	report := agentsdocker.Report{
		Agent: agentsdocker.AgentInfo{
			ID:              "agent-1",
			Version:         "1.2.3",
			IntervalSeconds: 30,
		},
		Host: agentsdocker.HostInfo{
			Hostname: "disk-host",
		},
		Containers: []agentsdocker.Container{
			{
				ID:                  "ctr-1",
				Name:                "app",
				HealthcheckTargets:  []string{"dependency-service"},
				OOMKilled:           &oomKilled,
				WritableLayerBytes:  512 * 1024 * 1024,
				RootFilesystemBytes: 2 * 1024 * 1024 * 1024,
				BlockIO: &agentsdocker.ContainerBlockIO{
					ReadBytes:  123456,
					WriteBytes: 654321,
				},
				Mounts: []agentsdocker.ContainerMount{
					{
						Type:        "bind",
						Source:      "/srv/app/config",
						Destination: "/config",
						Mode:        "rw",
						RW:          true,
						Propagation: "rprivate",
						Name:        "",
						Driver:      "",
					},
				},
			},
		},
		Timestamp: timestamp,
	}

	monitor := newTestMonitor(t)
	host, err := monitor.ApplyDockerReport(report, nil)
	if err != nil {
		t.Fatalf("ApplyDockerReport returned error: %v", err)
	}

	if len(host.Containers) != 1 {
		t.Fatalf("expected 1 container, got %d", len(host.Containers))
	}

	container := host.Containers[0]
	if container.OOMKilled == nil || *container.OOMKilled {
		t.Fatalf("expected explicit non-OOM state, got %v", container.OOMKilled)
	}
	oomKilled = true
	if *container.OOMKilled {
		t.Fatal("expected monitoring state to own an independent OOM evidence value")
	}
	report.Containers[0].HealthcheckTargets[0] = "mutated"
	if container.HealthcheckTargets[0] != "dependency-service" {
		t.Fatal("expected monitoring state to own an independent health-check target slice")
	}
	if container.WritableLayerBytes != 512*1024*1024 {
		t.Fatalf("expected writable layer bytes to match, got %d", container.WritableLayerBytes)
	}
	if container.RootFilesystemBytes != 2*1024*1024*1024 {
		t.Fatalf("expected root filesystem bytes to match, got %d", container.RootFilesystemBytes)
	}

	if container.BlockIO == nil {
		t.Fatalf("expected block IO stats to be populated")
	}
	if container.BlockIO.ReadBytes != 123456 || container.BlockIO.WriteBytes != 654321 {
		t.Fatalf("unexpected block IO values: %+v", container.BlockIO)
	}
	if container.BlockIO.ReadRateBytesPerSecond != nil || container.BlockIO.WriteRateBytesPerSecond != nil {
		t.Fatalf("expected block IO rates to be unset on first sample: %+v", container.BlockIO)
	}

	if len(container.Mounts) != 1 {
		t.Fatalf("expected mounts to be preserved, got %d", len(container.Mounts))
	}
	mount := container.Mounts[0]
	if mount.Source != "/srv/app/config" || mount.Destination != "/config" || !mount.RW {
		t.Fatalf("unexpected mount payload: %+v", mount)
	}
}

func TestApplyDockerReportNormalizesContainerCPUCapacity(t *testing.T) {
	report := agentsdocker.Report{
		Agent: agentsdocker.AgentInfo{
			ID:              "agent-1",
			Version:         "1.2.3",
			IntervalSeconds: 30,
		},
		Host: agentsdocker.HostInfo{
			Hostname: "cpu-host",
			TotalCPU: 4,
		},
		Containers: []agentsdocker.Container{{
			ID:         "ctr-1",
			Name:       "transcoder",
			State:      "running",
			Status:     "running",
			CPUPercent: 240,
		}},
		Timestamp: time.Now().UTC(),
	}

	monitor := newTestMonitor(t)
	host, err := monitor.ApplyDockerReport(report, nil)
	if err != nil {
		t.Fatalf("ApplyDockerReport returned error: %v", err)
	}

	if len(host.Containers) != 1 {
		t.Fatalf("expected 1 container, got %d", len(host.Containers))
	}
	container := host.Containers[0]
	if container.CPUPercent != 240 {
		t.Fatalf("raw Docker CPU percent = %v, want 240", container.CPUPercent)
	}
	if container.CPUCapacityPercent != 60 {
		t.Fatalf("normalized Docker CPU capacity percent = %v, want 60", container.CPUCapacityPercent)
	}
}

func TestApplyDockerReportMigratesMetadataWhenContainerRuntimeIDChanges(t *testing.T) {
	monitor := newTestMonitor(t)

	baseTimestamp := time.Now().UTC()
	firstReport := agentsdocker.Report{
		Agent: agentsdocker.AgentInfo{
			ID:              "agent-1",
			Version:         "1.0.0",
			IntervalSeconds: 30,
		},
		Host: agentsdocker.HostInfo{
			Hostname:  "docker-host-1",
			MachineID: "machine-1",
		},
		Containers: []agentsdocker.Container{
			{ID: "container-old", Name: "app"},
		},
		Timestamp: baseTimestamp,
	}

	host, err := monitor.ApplyDockerReport(firstReport, nil)
	if err != nil {
		t.Fatalf("first ApplyDockerReport failed: %v", err)
	}
	if host.ID == "" {
		t.Fatal("expected docker host ID")
	}
	if err := monitor.dockerMetadataStore.Set(host.ID+":container:container-old", &config.DockerMetadata{
		CustomURL: "https://app.internal",
	}); err != nil {
		t.Fatalf("seed docker metadata: %v", err)
	}

	secondReport := firstReport
	secondReport.Timestamp = baseTimestamp.Add(30 * time.Second)
	secondReport.Containers = []agentsdocker.Container{
		{ID: "container-new", Name: "app"},
	}

	host, err = monitor.ApplyDockerReport(secondReport, nil)
	if err != nil {
		t.Fatalf("second ApplyDockerReport failed: %v", err)
	}

	meta := monitor.dockerMetadataStore.Get(host.ID + ":container:container-new")
	if meta == nil {
		t.Fatalf("expected migrated metadata for recreated container")
	}
	if meta.CustomURL != "https://app.internal" {
		t.Fatalf("expected migrated custom URL, got %#v", meta)
	}
}

func TestApplyDockerReportMigratesGuestMetadataToStableContainerName(t *testing.T) {
	monitor := newTestMonitor(t)

	baseTimestamp := time.Now().UTC()
	report := agentsdocker.Report{
		Agent: agentsdocker.AgentInfo{
			ID:              "agent-stable-guest",
			Version:         "1.0.0",
			IntervalSeconds: 30,
		},
		Host: agentsdocker.HostInfo{
			Hostname:  "docker-host-stable-guest",
			MachineID: "machine-stable-guest",
		},
		Containers: []agentsdocker.Container{
			{ID: "container-old", Name: "/app"},
		},
		Timestamp: baseTimestamp,
	}

	host, err := monitor.ApplyDockerReport(report, nil)
	if err != nil {
		t.Fatalf("first ApplyDockerReport failed: %v", err)
	}

	legacyKey := dockerAppContainerLegacyResourceID(host.ID, "container-old")
	if err := monitor.guestMetadataStore.Set(legacyKey, &config.GuestMetadata{
		CustomURL: "https://app.internal",
	}); err != nil {
		t.Fatalf("seed guest metadata: %v", err)
	}

	report.Timestamp = baseTimestamp.Add(30 * time.Second)
	if _, err := monitor.ApplyDockerReport(report, nil); err != nil {
		t.Fatalf("metadata seeding ApplyDockerReport failed: %v", err)
	}

	stableKey := dockerAppContainerMetadataKey(host.ID, "app")
	stableMeta := monitor.guestMetadataStore.Get(stableKey)
	if stableMeta == nil {
		t.Fatalf("expected stable guest metadata key %q", stableKey)
	}
	if stableMeta.CustomURL != "https://app.internal" {
		t.Fatalf("stable guest metadata URL = %q, want https://app.internal", stableMeta.CustomURL)
	}

	report.Timestamp = baseTimestamp.Add(60 * time.Second)
	report.Containers = nil
	if _, err := monitor.ApplyDockerReport(report, nil); err != nil {
		t.Fatalf("empty-container ApplyDockerReport failed: %v", err)
	}

	report.Timestamp = baseTimestamp.Add(90 * time.Second)
	report.Containers = []agentsdocker.Container{
		{ID: "container-new", Name: "app"},
	}
	host, err = monitor.ApplyDockerReport(report, nil)
	if err != nil {
		t.Fatalf("recreated-container ApplyDockerReport failed: %v", err)
	}

	resources := []unifiedresources.Resource{
		{
			ID:   dockerAppContainerLegacyResourceID(host.ID, "container-new"),
			Type: unifiedresources.ResourceTypeAppContainer,
			Name: "app",
			Docker: &unifiedresources.DockerData{
				HostSourceID: host.ID,
				ContainerID:  "container-new",
			},
		},
	}

	got := monitor.applyPersistedMetadataToUnifiedResources(resources)
	if got[0].CustomURL != "https://app.internal" {
		t.Fatalf("CustomURL after recreate gap = %q, want https://app.internal", got[0].CustomURL)
	}
}

func TestApplyDockerReportMovesStableMetadataOnRenameWithoutLeakingToReusedName(t *testing.T) {
	monitor := newTestMonitor(t)

	baseTimestamp := time.Now().UTC()
	report := agentsdocker.Report{
		Agent: agentsdocker.AgentInfo{
			ID:              "agent-rename",
			Version:         "1.0.0",
			IntervalSeconds: 30,
		},
		Host: agentsdocker.HostInfo{
			Hostname:  "docker-host-rename",
			MachineID: "machine-rename",
		},
		Containers: []agentsdocker.Container{
			{ID: "container-stable", Name: "app"},
		},
		Timestamp: baseTimestamp,
	}

	host, err := monitor.ApplyDockerReport(report, nil)
	if err != nil {
		t.Fatalf("first ApplyDockerReport failed: %v", err)
	}
	oldKey := dockerAppContainerMetadataKey(host.ID, "app")
	if err := monitor.guestMetadataStore.Set(oldKey, &config.GuestMetadata{
		CustomURL:     "https://app.internal",
		LastKnownName: "app",
		LastKnownType: "app-container",
	}); err != nil {
		t.Fatalf("seed stable guest metadata: %v", err)
	}
	oldDockerKey := dockerContainerNameMetadataKey(host.ID, "app")
	if err := monitor.dockerMetadataStore.Set(oldDockerKey, &config.DockerMetadata{
		CustomURL: "https://app-drawer.internal",
	}); err != nil {
		t.Fatalf("seed stable Docker metadata: %v", err)
	}

	report.Timestamp = baseTimestamp.Add(30 * time.Second)
	report.Containers = []agentsdocker.Container{
		{ID: "container-stable", Name: "renamed-app"},
	}
	if _, err := monitor.ApplyDockerReport(report, nil); err != nil {
		t.Fatalf("renamed-container ApplyDockerReport failed: %v", err)
	}

	newKey := dockerAppContainerMetadataKey(host.ID, "renamed-app")
	if meta := monitor.guestMetadataStore.Get(newKey); meta == nil || meta.CustomURL != "https://app.internal" {
		t.Fatalf("renamed stable metadata = %#v, want preserved URL", meta)
	}
	if meta := monitor.guestMetadataStore.Get(oldKey); meta != nil {
		t.Fatalf("old stable metadata key still exists after rename: %#v", meta)
	}
	newDockerKey := dockerContainerNameMetadataKey(host.ID, "renamed-app")
	if meta := monitor.dockerMetadataStore.Get(newDockerKey); meta == nil || meta.CustomURL != "https://app-drawer.internal" {
		t.Fatalf("renamed stable Docker metadata = %#v, want preserved URL", meta)
	}
	if meta := monitor.dockerMetadataStore.Get(oldDockerKey); meta != nil {
		t.Fatalf("old stable Docker metadata key still exists after rename: %#v", meta)
	}

	report.Timestamp = baseTimestamp.Add(60 * time.Second)
	report.Containers = []agentsdocker.Container{
		{ID: "container-stable", Name: "renamed-app"},
		{ID: "container-unrelated", Name: "app"},
	}
	if _, err := monitor.ApplyDockerReport(report, nil); err != nil {
		t.Fatalf("reused-name ApplyDockerReport failed: %v", err)
	}

	resources := []unifiedresources.Resource{
		{
			ID:   dockerAppContainerLegacyResourceID(host.ID, "container-stable"),
			Type: unifiedresources.ResourceTypeAppContainer,
			Name: "renamed-app",
			Docker: &unifiedresources.DockerData{
				HostSourceID: host.ID,
				ContainerID:  "container-stable",
			},
		},
		{
			ID:   dockerAppContainerLegacyResourceID(host.ID, "container-unrelated"),
			Type: unifiedresources.ResourceTypeAppContainer,
			Name: "app",
			Docker: &unifiedresources.DockerData{
				HostSourceID: host.ID,
				ContainerID:  "container-unrelated",
			},
		},
	}
	got := monitor.applyPersistedMetadataToUnifiedResources(resources)
	if got[0].CustomURL != "https://app.internal" {
		t.Fatalf("renamed container CustomURL = %q, want preserved URL", got[0].CustomURL)
	}
	if got[1].CustomURL != "" {
		t.Fatalf("unrelated container reusing old name inherited CustomURL %q", got[1].CustomURL)
	}
}

func TestDockerContainerStableMetadataMovesFailClosedOnAmbiguousNames(t *testing.T) {
	moves := dockerContainerStableMetadataMoves(
		"docker-host",
		[]models.DockerContainer{
			{ID: "container-a", Name: "/app"},
			{ID: "container-b", Name: "app"},
		},
		[]models.DockerContainer{
			{ID: "container-a", Name: "renamed-app"},
			{ID: "container-b", Name: "app"},
		},
		dockerAppContainerMetadataKey,
	)
	if len(moves) != 0 {
		t.Fatalf("ambiguous normalized source names produced moves: %#v", moves)
	}
}

func TestMigrateDockerContainerMetadataForRenamedContainersPreservesNameSwapOwnership(t *testing.T) {
	monitor := newTestMonitor(t)
	hostID := "docker-host-swap"
	if err := monitor.guestMetadataStore.Set(
		dockerAppContainerMetadataKey(hostID, "blue"),
		&config.GuestMetadata{CustomURL: "https://blue.internal"},
	); err != nil {
		t.Fatalf("seed blue metadata: %v", err)
	}
	if err := monitor.guestMetadataStore.Set(
		dockerAppContainerMetadataKey(hostID, "green"),
		&config.GuestMetadata{CustomURL: "https://green.internal"},
	); err != nil {
		t.Fatalf("seed green metadata: %v", err)
	}

	monitor.migrateDockerContainerMetadataForRenamedContainers(
		hostID,
		[]models.DockerContainer{
			{ID: "container-blue", Name: "blue"},
			{ID: "container-green", Name: "green"},
		},
		[]models.DockerContainer{
			{ID: "container-blue", Name: "green"},
			{ID: "container-green", Name: "blue"},
		},
	)

	if meta := monitor.guestMetadataStore.Get(dockerAppContainerMetadataKey(hostID, "green")); meta == nil || meta.CustomURL != "https://blue.internal" {
		t.Fatalf("container-blue metadata after swap = %#v", meta)
	}
	if meta := monitor.guestMetadataStore.Get(dockerAppContainerMetadataKey(hostID, "blue")); meta == nil || meta.CustomURL != "https://green.internal" {
		t.Fatalf("container-green metadata after swap = %#v", meta)
	}
}

func TestMigrateDockerContainerMetadataForRenamedContainersPreservesExistingDestination(t *testing.T) {
	monitor := newTestMonitor(t)
	hostID := "docker-host-destination"
	oldKey := dockerAppContainerMetadataKey(hostID, "old-name")
	newKey := dockerAppContainerMetadataKey(hostID, "reserved-name")
	if err := monitor.guestMetadataStore.Set(oldKey, &config.GuestMetadata{
		CustomURL: "https://old.internal",
	}); err != nil {
		t.Fatalf("seed old metadata: %v", err)
	}
	if err := monitor.guestMetadataStore.Set(newKey, &config.GuestMetadata{
		CustomURL: "https://reserved.internal",
	}); err != nil {
		t.Fatalf("seed destination metadata: %v", err)
	}

	monitor.migrateDockerContainerMetadataForRenamedContainers(
		hostID,
		[]models.DockerContainer{{ID: "container", Name: "old-name"}},
		[]models.DockerContainer{{ID: "container", Name: "reserved-name"}},
	)

	if meta := monitor.guestMetadataStore.Get(newKey); meta == nil || meta.CustomURL != "https://reserved.internal" {
		t.Fatalf("destination metadata = %#v", meta)
	}
	if meta := monitor.guestMetadataStore.Get(oldKey); meta != nil {
		t.Fatalf("obsolete source could leak onto later name reuse: %#v", meta)
	}
}

func TestApplyDockerReportFoldsRuntimeKeyDockerMetadataIntoStableGuestKey(t *testing.T) {
	monitor := newTestMonitor(t)

	baseTimestamp := time.Now().UTC()
	report := agentsdocker.Report{
		Agent: agentsdocker.AgentInfo{
			ID:              "agent-runtime-fold",
			Version:         "1.0.0",
			IntervalSeconds: 30,
		},
		Host: agentsdocker.HostInfo{
			Hostname:  "docker-host-runtime-fold",
			MachineID: "machine-runtime-fold",
		},
		Containers: []agentsdocker.Container{
			{ID: "container-wud", Name: "/wud"},
		},
		Timestamp: baseTimestamp,
	}

	host, err := monitor.ApplyDockerReport(report, nil)
	if err != nil {
		t.Fatalf("first ApplyDockerReport failed: %v", err)
	}

	// A URL saved through the resource drawer historically landed on the
	// runtime container key in the docker store.
	if err := monitor.dockerMetadataStore.Set(host.ID+":container:container-wud", &config.DockerMetadata{
		CustomURL: "https://wud.internal",
	}); err != nil {
		t.Fatalf("seed docker metadata: %v", err)
	}

	report.Timestamp = baseTimestamp.Add(30 * time.Second)
	if _, err := monitor.ApplyDockerReport(report, nil); err != nil {
		t.Fatalf("second ApplyDockerReport failed: %v", err)
	}

	stableKey := dockerAppContainerMetadataKey(host.ID, "wud")
	stableMeta := monitor.guestMetadataStore.Get(stableKey)
	if stableMeta == nil {
		t.Fatalf("expected runtime docker metadata folded into stable guest key %q", stableKey)
	}
	if stableMeta.CustomURL != "https://wud.internal" {
		t.Fatalf("stable guest metadata URL = %q, want https://wud.internal", stableMeta.CustomURL)
	}

	resources := []unifiedresources.Resource{
		{
			ID:   dockerAppContainerLegacyResourceID(host.ID, "container-wud"),
			Type: unifiedresources.ResourceTypeAppContainer,
			Name: "wud",
			Docker: &unifiedresources.DockerData{
				HostSourceID: host.ID,
				ContainerID:  "container-wud",
			},
		},
	}
	got := monitor.applyPersistedMetadataToUnifiedResources(resources)
	if got[0].CustomURL != "https://wud.internal" {
		t.Fatalf("projected CustomURL = %q, want https://wud.internal", got[0].CustomURL)
	}
}

func TestApplyDockerReportKeepsClearedStableGuestKeyOverRuntimeDockerMetadata(t *testing.T) {
	monitor := newTestMonitor(t)

	baseTimestamp := time.Now().UTC()
	report := agentsdocker.Report{
		Agent: agentsdocker.AgentInfo{
			ID:              "agent-cleared-stable",
			Version:         "1.0.0",
			IntervalSeconds: 30,
		},
		Host: agentsdocker.HostInfo{
			Hostname:  "docker-host-cleared-stable",
			MachineID: "machine-cleared-stable",
		},
		Containers: []agentsdocker.Container{
			{ID: "container-cleared", Name: "cleared"},
		},
		Timestamp: baseTimestamp,
	}

	host, err := monitor.ApplyDockerReport(report, nil)
	if err != nil {
		t.Fatalf("first ApplyDockerReport failed: %v", err)
	}

	// A user cleared the link: the stable guest key intentionally holds an
	// empty record. A stale runtime-key docker record must not overwrite or
	// outrank it.
	stableKey := dockerAppContainerMetadataKey(host.ID, "cleared")
	if err := monitor.guestMetadataStore.Set(stableKey, &config.GuestMetadata{
		CustomURL:     "",
		LastKnownName: "cleared",
		LastKnownType: "app-container",
	}); err != nil {
		t.Fatalf("seed cleared guest metadata: %v", err)
	}
	if err := monitor.dockerMetadataStore.Set(host.ID+":container:container-cleared", &config.DockerMetadata{
		CustomURL: "https://stale.internal",
	}); err != nil {
		t.Fatalf("seed docker metadata: %v", err)
	}

	report.Timestamp = baseTimestamp.Add(30 * time.Second)
	if _, err := monitor.ApplyDockerReport(report, nil); err != nil {
		t.Fatalf("second ApplyDockerReport failed: %v", err)
	}

	if meta := monitor.guestMetadataStore.Get(stableKey); meta == nil || meta.CustomURL != "" {
		t.Fatalf("cleared stable guest metadata was overwritten: %#v", meta)
	}

	resources := []unifiedresources.Resource{
		{
			ID:   dockerAppContainerLegacyResourceID(host.ID, "container-cleared"),
			Type: unifiedresources.ResourceTypeAppContainer,
			Name: "cleared",
			Docker: &unifiedresources.DockerData{
				HostSourceID: host.ID,
				ContainerID:  "container-cleared",
			},
		},
	}
	got := monitor.applyPersistedMetadataToUnifiedResources(resources)
	if got[0].CustomURL != "" {
		t.Fatalf("projected CustomURL = %q, want empty (cleared)", got[0].CustomURL)
	}
}

func TestApplyDockerReportSkipsMetadataMigrationForAmbiguousContainerNames(t *testing.T) {
	monitor := newTestMonitor(t)

	baseTimestamp := time.Now().UTC()
	firstReport := agentsdocker.Report{
		Agent: agentsdocker.AgentInfo{
			ID:              "agent-ambiguous",
			Version:         "1.0.0",
			IntervalSeconds: 30,
		},
		Host: agentsdocker.HostInfo{
			Hostname:  "docker-host-ambiguous",
			MachineID: "machine-ambiguous",
		},
		Containers: []agentsdocker.Container{
			{ID: "container-old-a", Name: "/app"},
			{ID: "container-old-b", Name: "app"},
		},
		Timestamp: baseTimestamp,
	}

	host, err := monitor.ApplyDockerReport(firstReport, nil)
	if err != nil {
		t.Fatalf("first ApplyDockerReport failed: %v", err)
	}
	if err := monitor.dockerMetadataStore.Set(host.ID+":container:container-old-a", &config.DockerMetadata{
		CustomURL: "https://ambiguous.internal",
	}); err != nil {
		t.Fatalf("seed docker metadata: %v", err)
	}

	secondReport := firstReport
	secondReport.Timestamp = baseTimestamp.Add(30 * time.Second)
	secondReport.Containers = []agentsdocker.Container{
		{ID: "container-new", Name: "app"},
	}

	host, err = monitor.ApplyDockerReport(secondReport, nil)
	if err != nil {
		t.Fatalf("second ApplyDockerReport failed: %v", err)
	}

	meta := monitor.dockerMetadataStore.Get(host.ID + ":container:container-new")
	if meta != nil {
		t.Fatalf("expected ambiguous metadata migration to be skipped, got %#v", meta)
	}
}

func TestApplyDockerMetadataToUnifiedResourcesAddsContainerCustomURL(t *testing.T) {
	monitor := newTestMonitor(t)
	if err := monitor.dockerMetadataStore.Set("docker-host-1:container:container-new", &config.DockerMetadata{
		CustomURL: " https://app.internal ",
	}); err != nil {
		t.Fatalf("seed docker metadata: %v", err)
	}

	resources := []unifiedresources.Resource{
		{
			ID:   "resource:app-container:app",
			Type: unifiedresources.ResourceTypeAppContainer,
			Docker: &unifiedresources.DockerData{
				HostSourceID: " docker-host-1 ",
				ContainerID:  " container-new ",
			},
		},
	}

	got := monitor.applyPersistedMetadataToUnifiedResources(resources)
	if got[0].CustomURL != "https://app.internal" {
		t.Fatalf("CustomURL = %q, want migrated Docker metadata URL", got[0].CustomURL)
	}
	if resources[0].CustomURL != "" {
		t.Fatalf("applyPersistedMetadataToUnifiedResources mutated input CustomURL to %q", resources[0].CustomURL)
	}
}

func TestApplyDockerMetadataToUnifiedResourcesPrefersCanonicalMetadataOverStaleResourceURL(t *testing.T) {
	monitor := newTestMonitor(t)
	if err := monitor.dockerMetadataStore.Set("docker-host-1:container:container-new", &config.DockerMetadata{
		CustomURL: "https://metadata.internal",
	}); err != nil {
		t.Fatalf("seed docker metadata: %v", err)
	}

	resources := []unifiedresources.Resource{
		{
			ID:        "resource:app-container:app",
			Type:      unifiedresources.ResourceTypeAppContainer,
			CustomURL: "https://resource.internal",
			Docker: &unifiedresources.DockerData{
				HostSourceID: "docker-host-1",
				ContainerID:  "container-new",
			},
		},
	}

	got := monitor.applyPersistedMetadataToUnifiedResources(resources)
	if got[0].CustomURL != "https://metadata.internal" {
		t.Fatalf("CustomURL = %q, want canonical metadata URL to win", got[0].CustomURL)
	}
}

func TestApplyDockerMetadataToUnifiedResourcesUsesStableDockerMetadata(t *testing.T) {
	monitor := newTestMonitor(t)
	if err := monitor.dockerMetadataStore.Set("docker-host-1:container-name:app", &config.DockerMetadata{
		CustomURL: "https://stable-docker.internal",
	}); err != nil {
		t.Fatalf("seed stable docker metadata: %v", err)
	}

	resources := []unifiedresources.Resource{
		{
			ID:   "resource:app-container:app",
			Type: unifiedresources.ResourceTypeAppContainer,
			Name: "app",
			Docker: &unifiedresources.DockerData{
				HostSourceID: "docker-host-1",
				ContainerID:  "container-new",
			},
		},
	}

	got := monitor.applyPersistedMetadataToUnifiedResources(resources)
	if got[0].CustomURL != "https://stable-docker.internal" {
		t.Fatalf("CustomURL = %q, want stable Docker metadata URL", got[0].CustomURL)
	}
}

func TestApplyDockerMetadataToUnifiedResourcesStableGuestMetadataBlocksLegacyFallback(t *testing.T) {
	monitor := newTestMonitor(t)
	if err := monitor.guestMetadataStore.Set(dockerAppContainerMetadataKey("docker-host-1", "app"), &config.GuestMetadata{
		CustomURL: "",
	}); err != nil {
		t.Fatalf("seed stable guest metadata: %v", err)
	}
	if err := monitor.dockerMetadataStore.Set("docker-host-1:container:container-new", &config.DockerMetadata{
		CustomURL: "https://legacy.internal",
	}); err != nil {
		t.Fatalf("seed legacy docker metadata: %v", err)
	}

	resources := []unifiedresources.Resource{
		{
			ID:   "resource:app-container:app",
			Type: unifiedresources.ResourceTypeAppContainer,
			Name: "app",
			Docker: &unifiedresources.DockerData{
				HostSourceID: "docker-host-1",
				ContainerID:  "container-new",
			},
		},
	}

	got := monitor.applyPersistedMetadataToUnifiedResources(resources)
	if got[0].CustomURL != "" {
		t.Fatalf("CustomURL = %q, want stable empty guest metadata to block legacy fallback", got[0].CustomURL)
	}
}

func TestApplyPersistedMetadataToUnifiedResourcesMigratesKubernetesURLToStableLogicalIdentity(t *testing.T) {
	monitor := newTestMonitor(t)
	legacyPodID := "k8s:cluster-a:pod:pod-uid-old"
	if err := monitor.guestMetadataStore.Set(legacyPodID, &config.GuestMetadata{
		CustomURL: "https://checkout.internal",
	}); err != nil {
		t.Fatalf("seed legacy pod metadata: %v", err)
	}

	oldPod := unifiedresources.Resource{
		ID:   "resource:pod:old",
		Type: unifiedresources.ResourceTypePod,
		Name: "checkout",
		Kubernetes: &unifiedresources.K8sData{
			ClusterID:    "cluster-a",
			Namespace:    "payments",
			ResourceKind: "Pod",
			PodUID:       "pod-uid-old",
		},
	}
	got := monitor.applyPersistedMetadataToUnifiedResources([]unifiedresources.Resource{oldPod})
	if got[0].CustomURL != "https://checkout.internal" {
		t.Fatalf("legacy pod CustomURL = %q, want migrated URL", got[0].CustomURL)
	}

	recreatedPod := oldPod
	recreatedPod.ID = "resource:pod:new"
	recreatedPod.Kubernetes = &unifiedresources.K8sData{
		ClusterID:    "cluster-a",
		Namespace:    "payments",
		ResourceKind: "Pod",
		PodUID:       "pod-uid-new",
	}
	got = monitor.applyPersistedMetadataToUnifiedResources([]unifiedresources.Resource{recreatedPod})
	if got[0].CustomURL != "https://checkout.internal" {
		t.Fatalf("recreated pod CustomURL = %q, want stable logical URL", got[0].CustomURL)
	}

	unrelated := []unifiedresources.Resource{
		{
			ID:   "resource:pod:other-cluster",
			Type: unifiedresources.ResourceTypePod,
			Name: "checkout",
			Kubernetes: &unifiedresources.K8sData{
				ClusterID:    "cluster-b",
				Namespace:    "payments",
				ResourceKind: "Pod",
				PodUID:       "pod-uid-other-cluster",
			},
		},
		{
			ID:   "resource:pod:other-namespace",
			Type: unifiedresources.ResourceTypePod,
			Name: "checkout",
			Kubernetes: &unifiedresources.K8sData{
				ClusterID:    "cluster-a",
				Namespace:    "staging",
				ResourceKind: "Pod",
				PodUID:       "pod-uid-other-namespace",
			},
		},
		{
			ID:   "resource:deployment:same-name",
			Type: unifiedresources.ResourceTypeK8sDeployment,
			Name: "checkout",
			Kubernetes: &unifiedresources.K8sData{
				ClusterID:    "cluster-a",
				Namespace:    "payments",
				ResourceKind: "Deployment",
				ResourceUID:  "deployment-uid",
			},
		},
		{
			ID:   "resource:pod:different-name",
			Type: unifiedresources.ResourceTypePod,
			Name: "checkout-canary",
			Kubernetes: &unifiedresources.K8sData{
				ClusterID:    "cluster-a",
				Namespace:    "payments",
				ResourceKind: "Pod",
				PodUID:       "pod-uid-different-name",
			},
		},
	}
	got = monitor.applyPersistedMetadataToUnifiedResources(unrelated)
	for i := range got {
		if got[i].CustomURL != "" {
			t.Fatalf("unrelated resource %d inherited CustomURL %q", i, got[i].CustomURL)
		}
	}
}

func TestApplyPersistedMetadataToUnifiedResourcesMigratesKubernetesControllersToStableLogicalIdentity(t *testing.T) {
	for _, testCase := range []struct {
		name         string
		resourceType unifiedresources.ResourceType
		kind         string
	}{
		{
			name:         "deployment",
			resourceType: unifiedresources.ResourceTypeK8sDeployment,
			kind:         "Deployment",
		},
		{
			name:         "service",
			resourceType: unifiedresources.ResourceTypeK8sService,
			kind:         "Service",
		},
	} {
		t.Run(testCase.name, func(t *testing.T) {
			monitor := newTestMonitor(t)
			oldResourceID := "resource:" + testCase.name + ":old"
			if err := monitor.guestMetadataStore.Set(oldResourceID, &config.GuestMetadata{
				CustomURL: "https://" + testCase.name + ".internal",
			}); err != nil {
				t.Fatalf("seed legacy metadata: %v", err)
			}

			resource := unifiedresources.Resource{
				ID:   oldResourceID,
				Type: testCase.resourceType,
				Name: "checkout",
				Kubernetes: &unifiedresources.K8sData{
					ClusterID:    "cluster-a",
					Namespace:    "payments",
					ResourceKind: testCase.kind,
					ResourceUID:  testCase.name + "-uid-old",
				},
			}
			got := monitor.applyPersistedMetadataToUnifiedResources([]unifiedresources.Resource{resource})
			if got[0].CustomURL != "https://"+testCase.name+".internal" {
				t.Fatalf("legacy %s CustomURL = %q", testCase.name, got[0].CustomURL)
			}

			resource.ID = "resource:" + testCase.name + ":new"
			resource.Kubernetes.ResourceUID = testCase.name + "-uid-new"
			got = monitor.applyPersistedMetadataToUnifiedResources([]unifiedresources.Resource{resource})
			if got[0].CustomURL != "https://"+testCase.name+".internal" {
				t.Fatalf("recreated %s CustomURL = %q", testCase.name, got[0].CustomURL)
			}
		})
	}
}

func TestApplyPersistedMetadataToUnifiedResourcesKubernetesStableClearBlocksLegacyFallback(t *testing.T) {
	monitor := newTestMonitor(t)
	resource := unifiedresources.Resource{
		ID:   "resource:pod:current",
		Type: unifiedresources.ResourceTypePod,
		Name: "checkout",
		Kubernetes: &unifiedresources.K8sData{
			ClusterID:    "cluster-a",
			Namespace:    "payments",
			ResourceKind: "Pod",
			PodUID:       "pod-uid-current",
		},
	}
	stableKey := "k8s-workload:cluster-a:pod:payments:checkout"
	if err := monitor.guestMetadataStore.Set(stableKey, &config.GuestMetadata{CustomURL: ""}); err != nil {
		t.Fatalf("seed cleared stable pod metadata: %v", err)
	}
	if err := monitor.guestMetadataStore.Set("k8s:cluster-a:pod:pod-uid-current", &config.GuestMetadata{
		CustomURL: "https://stale.internal",
	}); err != nil {
		t.Fatalf("seed legacy pod metadata: %v", err)
	}

	got := monitor.applyPersistedMetadataToUnifiedResources([]unifiedresources.Resource{resource})
	if got[0].CustomURL != "" {
		t.Fatalf("CustomURL = %q, want stable empty Kubernetes metadata to block legacy fallback", got[0].CustomURL)
	}
}

func TestApplyDockerReportComputesContainerNetworkAndDiskRates(t *testing.T) {
	monitor := newTestMonitor(t)
	baseTime := time.Now().UTC()

	baseReport := agentsdocker.Report{
		Agent: agentsdocker.AgentInfo{
			ID:              "agent-rates",
			Version:         "1.2.3",
			IntervalSeconds: 30,
		},
		Host: agentsdocker.HostInfo{
			Hostname: "rates-host",
		},
		Containers: []agentsdocker.Container{
			{
				ID:             "ctr-rates",
				Name:           "api",
				NetworkRXBytes: 10_000,
				NetworkTXBytes: 20_000,
				BlockIO: &agentsdocker.ContainerBlockIO{
					ReadBytes:  5_000,
					WriteBytes: 7_000,
				},
			},
		},
		Timestamp: baseTime,
	}

	first, err := monitor.ApplyDockerReport(baseReport, nil)
	if err != nil {
		t.Fatalf("first ApplyDockerReport returned error: %v", err)
	}
	if len(first.Containers) != 1 {
		t.Fatalf("expected 1 container on first report, got %d", len(first.Containers))
	}
	if first.Containers[0].NetInRate != 0 || first.Containers[0].NetOutRate != 0 {
		t.Fatalf("expected network rates unset on first sample, got in=%f out=%f", first.Containers[0].NetInRate, first.Containers[0].NetOutRate)
	}
	if first.Containers[0].BlockIO == nil {
		t.Fatal("expected block IO on first sample")
	}
	if first.Containers[0].BlockIO.ReadRateBytesPerSecond != nil || first.Containers[0].BlockIO.WriteRateBytesPerSecond != nil {
		t.Fatalf("expected disk rates unset on first sample, got %+v", first.Containers[0].BlockIO)
	}

	secondReport := baseReport
	secondReport.Timestamp = baseTime.Add(30 * time.Second)
	secondReport.Containers = []agentsdocker.Container{
		{
			ID:             "ctr-rates",
			Name:           "api",
			NetworkRXBytes: 16_000,
			NetworkTXBytes: 29_000,
			BlockIO: &agentsdocker.ContainerBlockIO{
				ReadBytes:  8_000,
				WriteBytes: 10_000,
			},
		},
	}

	second, err := monitor.ApplyDockerReport(secondReport, nil)
	if err != nil {
		t.Fatalf("second ApplyDockerReport returned error: %v", err)
	}
	if len(second.Containers) != 1 {
		t.Fatalf("expected 1 container on second report, got %d", len(second.Containers))
	}
	container := second.Containers[0]
	if container.NetInRate <= 0 {
		t.Fatalf("expected positive net in rate, got %f", container.NetInRate)
	}
	if container.NetOutRate <= 0 {
		t.Fatalf("expected positive net out rate, got %f", container.NetOutRate)
	}
	if container.BlockIO == nil {
		t.Fatal("expected block IO on second sample")
	}
	if container.BlockIO.ReadRateBytesPerSecond == nil || *container.BlockIO.ReadRateBytesPerSecond <= 0 {
		t.Fatalf("expected positive disk read rate, got %+v", container.BlockIO.ReadRateBytesPerSecond)
	}
	if container.BlockIO.WriteRateBytesPerSecond == nil || *container.BlockIO.WriteRateBytesPerSecond <= 0 {
		t.Fatalf("expected positive disk write rate, got %+v", container.BlockIO.WriteRateBytesPerSecond)
	}
}

func TestApplyDockerReportPodmanRuntimeMetadata(t *testing.T) {
	monitor := newTestMonitor(t)

	report := agentsdocker.Report{
		Agent: agentsdocker.AgentInfo{
			ID:              "agent-podman",
			Version:         "2.0.0",
			IntervalSeconds: 60,
		},
		Host: agentsdocker.HostInfo{
			Hostname:       "podman-host",
			Runtime:        "podman",
			RuntimeVersion: "4.9.3",
			DockerVersion:  "",
		},
		Timestamp: time.Now().UTC(),
	}

	host, err := monitor.ApplyDockerReport(report, nil)
	if err != nil {
		t.Fatalf("ApplyDockerReport returned error: %v", err)
	}

	if host.Runtime != "podman" {
		t.Fatalf("expected runtime podman, got %q", host.Runtime)
	}
	if host.RuntimeVersion != "4.9.3" {
		t.Fatalf("expected runtime version 4.9.3, got %q", host.RuntimeVersion)
	}
	if host.DockerVersion != "4.9.3" {
		t.Fatalf("expected docker version fallback to runtime version, got %q", host.DockerVersion)
	}
}

func TestApplyDockerReportDerivesDockerSecurityPosture(t *testing.T) {
	monitor := newTestMonitor(t)

	report := agentsdocker.Report{
		Agent: agentsdocker.AgentInfo{
			ID:              "agent-secure",
			Version:         "2.0.0",
			IntervalSeconds: 60,
		},
		Host: agentsdocker.HostInfo{
			Hostname: "secure-docker-host",
			Runtime:  "docker",
			Security: &agentsdocker.HostSecurityInfo{
				AuthorizationPlugins: []string{"opa", " audit "},
			},
		},
		Timestamp: time.Now().UTC(),
	}

	host, err := monitor.ApplyDockerReport(report, nil)
	if err != nil {
		t.Fatalf("ApplyDockerReport returned error: %v", err)
	}
	if host.Security == nil {
		t.Fatalf("expected host security posture to be populated")
	}
	if !host.Security.MutatingCommandsBlocked {
		t.Fatalf("expected mutating commands to be blocked")
	}
	if got := host.Security.AuthorizationPlugins; len(got) != 2 || got[0] != "opa" || got[1] != "audit" {
		t.Fatalf("expected normalized authorization plugins, got %#v", got)
	}
	if !strings.Contains(host.Security.MutatingCommandsBlockedReason, "GO-2026-4887") {
		t.Fatalf("expected advisory reason, got %q", host.Security.MutatingCommandsBlockedReason)
	}
}

func TestConvertDockerServices(t *testing.T) {
	t.Parallel()

	t.Run("nil input returns nil", func(t *testing.T) {
		t.Parallel()
		result := convertDockerServices(nil)
		if result != nil {
			t.Fatalf("expected nil, got %v", result)
		}
	})

	t.Run("empty slice returns nil", func(t *testing.T) {
		t.Parallel()
		result := convertDockerServices([]agentsdocker.Service{})
		if result != nil {
			t.Fatalf("expected nil, got %v", result)
		}
	})

	t.Run("basic fields are copied", func(t *testing.T) {
		t.Parallel()
		input := []agentsdocker.Service{{
			ID:             "svc-123",
			Name:           "web",
			Stack:          "mystack",
			Image:          "nginx:latest",
			Mode:           "replicated",
			DesiredTasks:   3,
			RunningTasks:   2,
			CompletedTasks: 1,
		}}

		result := convertDockerServices(input)
		if len(result) != 1 {
			t.Fatalf("expected 1 service, got %d", len(result))
		}

		svc := result[0]
		if svc.ID != "svc-123" {
			t.Errorf("ID = %q, want svc-123", svc.ID)
		}
		if svc.Name != "web" {
			t.Errorf("Name = %q, want web", svc.Name)
		}
		if svc.Stack != "mystack" {
			t.Errorf("Stack = %q, want mystack", svc.Stack)
		}
		if svc.Image != "nginx:latest" {
			t.Errorf("Image = %q, want nginx:latest", svc.Image)
		}
		if svc.Mode != "replicated" {
			t.Errorf("Mode = %q, want replicated", svc.Mode)
		}
		if svc.DesiredTasks != 3 {
			t.Errorf("DesiredTasks = %d, want 3", svc.DesiredTasks)
		}
		if svc.RunningTasks != 2 {
			t.Errorf("RunningTasks = %d, want 2", svc.RunningTasks)
		}
		if svc.CompletedTasks != 1 {
			t.Errorf("CompletedTasks = %d, want 1", svc.CompletedTasks)
		}
	})

	t.Run("labels are cloned when present", func(t *testing.T) {
		t.Parallel()
		input := []agentsdocker.Service{{
			ID:   "svc-1",
			Name: "web",
			Labels: map[string]string{
				"env":     "prod",
				"version": "1.0",
			},
		}}

		result := convertDockerServices(input)
		if result[0].Labels == nil {
			t.Fatal("expected labels to be present")
		}
		if result[0].Labels["env"] != "prod" {
			t.Errorf("Labels[env] = %q, want prod", result[0].Labels["env"])
		}
		if result[0].Labels["version"] != "1.0" {
			t.Errorf("Labels[version] = %q, want 1.0", result[0].Labels["version"])
		}

		// Verify it's a clone, not the same map
		input[0].Labels["env"] = "modified"
		if result[0].Labels["env"] == "modified" {
			t.Error("labels should be cloned, not shared")
		}
	})

	t.Run("empty labels are not copied", func(t *testing.T) {
		t.Parallel()
		input := []agentsdocker.Service{{
			ID:     "svc-1",
			Name:   "web",
			Labels: map[string]string{},
		}}

		result := convertDockerServices(input)
		if result[0].Labels != nil {
			t.Errorf("expected nil labels for empty map, got %v", result[0].Labels)
		}
	})

	t.Run("nil labels stay nil", func(t *testing.T) {
		t.Parallel()
		input := []agentsdocker.Service{{
			ID:     "svc-1",
			Name:   "web",
			Labels: nil,
		}}

		result := convertDockerServices(input)
		if result[0].Labels != nil {
			t.Errorf("expected nil labels, got %v", result[0].Labels)
		}
	})

	t.Run("endpoint ports are converted when present", func(t *testing.T) {
		t.Parallel()
		input := []agentsdocker.Service{{
			ID:   "svc-1",
			Name: "web",
			EndpointPorts: []agentsdocker.ServicePort{
				{
					Name:          "http",
					Protocol:      "tcp",
					TargetPort:    80,
					PublishedPort: 8080,
					PublishMode:   "ingress",
				},
				{
					Name:          "https",
					Protocol:      "tcp",
					TargetPort:    443,
					PublishedPort: 8443,
					PublishMode:   "host",
				},
			},
		}}

		result := convertDockerServices(input)
		if len(result[0].EndpointPorts) != 2 {
			t.Fatalf("expected 2 ports, got %d", len(result[0].EndpointPorts))
		}

		port1 := result[0].EndpointPorts[0]
		if port1.Name != "http" {
			t.Errorf("port[0].Name = %q, want http", port1.Name)
		}
		if port1.Protocol != "tcp" {
			t.Errorf("port[0].Protocol = %q, want tcp", port1.Protocol)
		}
		if port1.TargetPort != 80 {
			t.Errorf("port[0].TargetPort = %d, want 80", port1.TargetPort)
		}
		if port1.PublishedPort != 8080 {
			t.Errorf("port[0].PublishedPort = %d, want 8080", port1.PublishedPort)
		}
		if port1.PublishMode != "ingress" {
			t.Errorf("port[0].PublishMode = %q, want ingress", port1.PublishMode)
		}

		port2 := result[0].EndpointPorts[1]
		if port2.PublishMode != "host" {
			t.Errorf("port[1].PublishMode = %q, want host", port2.PublishMode)
		}
	})

	t.Run("empty endpoint ports are not copied", func(t *testing.T) {
		t.Parallel()
		input := []agentsdocker.Service{{
			ID:            "svc-1",
			Name:          "web",
			EndpointPorts: []agentsdocker.ServicePort{},
		}}

		result := convertDockerServices(input)
		if result[0].EndpointPorts != nil {
			t.Errorf("expected nil endpoint ports for empty slice, got %v", result[0].EndpointPorts)
		}
	})

	t.Run("nil endpoint ports stay nil", func(t *testing.T) {
		t.Parallel()
		input := []agentsdocker.Service{{
			ID:            "svc-1",
			Name:          "web",
			EndpointPorts: nil,
		}}

		result := convertDockerServices(input)
		if result[0].EndpointPorts != nil {
			t.Errorf("expected nil endpoint ports, got %v", result[0].EndpointPorts)
		}
	})

	t.Run("update status is converted when present", func(t *testing.T) {
		t.Parallel()
		completedAt := time.Date(2025, 1, 15, 10, 30, 0, 0, time.UTC)
		input := []agentsdocker.Service{{
			ID:   "svc-1",
			Name: "web",
			UpdateStatus: &agentsdocker.ServiceUpdate{
				State:       "completed",
				Message:     "update succeeded",
				CompletedAt: &completedAt,
			},
		}}

		result := convertDockerServices(input)
		if result[0].UpdateStatus == nil {
			t.Fatal("expected update status to be present")
		}
		if result[0].UpdateStatus.State != "completed" {
			t.Errorf("UpdateStatus.State = %q, want completed", result[0].UpdateStatus.State)
		}
		if result[0].UpdateStatus.Message != "update succeeded" {
			t.Errorf("UpdateStatus.Message = %q, want update succeeded", result[0].UpdateStatus.Message)
		}
		if result[0].UpdateStatus.CompletedAt == nil {
			t.Fatal("expected CompletedAt to be set")
		}
		if !result[0].UpdateStatus.CompletedAt.Equal(completedAt) {
			t.Errorf("UpdateStatus.CompletedAt = %v, want %v", result[0].UpdateStatus.CompletedAt, completedAt)
		}
	})

	t.Run("update status with nil CompletedAt", func(t *testing.T) {
		t.Parallel()
		input := []agentsdocker.Service{{
			ID:   "svc-1",
			Name: "web",
			UpdateStatus: &agentsdocker.ServiceUpdate{
				State:       "updating",
				Message:     "in progress",
				CompletedAt: nil,
			},
		}}

		result := convertDockerServices(input)
		if result[0].UpdateStatus == nil {
			t.Fatal("expected update status to be present")
		}
		if result[0].UpdateStatus.CompletedAt != nil {
			t.Errorf("expected nil CompletedAt, got %v", result[0].UpdateStatus.CompletedAt)
		}
	})

	t.Run("update status with zero CompletedAt", func(t *testing.T) {
		t.Parallel()
		zeroTime := time.Time{}
		input := []agentsdocker.Service{{
			ID:   "svc-1",
			Name: "web",
			UpdateStatus: &agentsdocker.ServiceUpdate{
				State:       "updating",
				CompletedAt: &zeroTime,
			},
		}}

		result := convertDockerServices(input)
		if result[0].UpdateStatus.CompletedAt != nil {
			t.Errorf("expected nil CompletedAt for zero time, got %v", result[0].UpdateStatus.CompletedAt)
		}
	})

	t.Run("nil update status stays nil", func(t *testing.T) {
		t.Parallel()
		input := []agentsdocker.Service{{
			ID:           "svc-1",
			Name:         "web",
			UpdateStatus: nil,
		}}

		result := convertDockerServices(input)
		if result[0].UpdateStatus != nil {
			t.Errorf("expected nil update status, got %v", result[0].UpdateStatus)
		}
	})

	t.Run("CreatedAt is copied when valid", func(t *testing.T) {
		t.Parallel()
		created := time.Date(2025, 1, 10, 8, 0, 0, 0, time.UTC)
		input := []agentsdocker.Service{{
			ID:        "svc-1",
			Name:      "web",
			CreatedAt: &created,
		}}

		result := convertDockerServices(input)
		if result[0].CreatedAt == nil {
			t.Fatal("expected CreatedAt to be set")
		}
		if !result[0].CreatedAt.Equal(created) {
			t.Errorf("CreatedAt = %v, want %v", result[0].CreatedAt, created)
		}
	})

	t.Run("nil CreatedAt stays nil", func(t *testing.T) {
		t.Parallel()
		input := []agentsdocker.Service{{
			ID:        "svc-1",
			Name:      "web",
			CreatedAt: nil,
		}}

		result := convertDockerServices(input)
		if result[0].CreatedAt != nil {
			t.Errorf("expected nil CreatedAt, got %v", result[0].CreatedAt)
		}
	})

	t.Run("zero CreatedAt is not copied", func(t *testing.T) {
		t.Parallel()
		zeroTime := time.Time{}
		input := []agentsdocker.Service{{
			ID:        "svc-1",
			Name:      "web",
			CreatedAt: &zeroTime,
		}}

		result := convertDockerServices(input)
		if result[0].CreatedAt != nil {
			t.Errorf("expected nil CreatedAt for zero time, got %v", result[0].CreatedAt)
		}
	})

	t.Run("UpdatedAt is copied when valid", func(t *testing.T) {
		t.Parallel()
		updated := time.Date(2025, 1, 12, 14, 30, 0, 0, time.UTC)
		input := []agentsdocker.Service{{
			ID:        "svc-1",
			Name:      "web",
			UpdatedAt: &updated,
		}}

		result := convertDockerServices(input)
		if result[0].UpdatedAt == nil {
			t.Fatal("expected UpdatedAt to be set")
		}
		if !result[0].UpdatedAt.Equal(updated) {
			t.Errorf("UpdatedAt = %v, want %v", result[0].UpdatedAt, updated)
		}
	})

	t.Run("nil UpdatedAt stays nil", func(t *testing.T) {
		t.Parallel()
		input := []agentsdocker.Service{{
			ID:        "svc-1",
			Name:      "web",
			UpdatedAt: nil,
		}}

		result := convertDockerServices(input)
		if result[0].UpdatedAt != nil {
			t.Errorf("expected nil UpdatedAt, got %v", result[0].UpdatedAt)
		}
	})

	t.Run("zero UpdatedAt is not copied", func(t *testing.T) {
		t.Parallel()
		zeroTime := time.Time{}
		input := []agentsdocker.Service{{
			ID:        "svc-1",
			Name:      "web",
			UpdatedAt: &zeroTime,
		}}

		result := convertDockerServices(input)
		if result[0].UpdatedAt != nil {
			t.Errorf("expected nil UpdatedAt for zero time, got %v", result[0].UpdatedAt)
		}
	})

	t.Run("multiple services are converted", func(t *testing.T) {
		t.Parallel()
		input := []agentsdocker.Service{
			{ID: "svc-1", Name: "web"},
			{ID: "svc-2", Name: "api"},
			{ID: "svc-3", Name: "worker"},
		}

		result := convertDockerServices(input)
		if len(result) != 3 {
			t.Fatalf("expected 3 services, got %d", len(result))
		}
		if result[0].ID != "svc-1" {
			t.Errorf("result[0].ID = %q, want svc-1", result[0].ID)
		}
		if result[1].ID != "svc-2" {
			t.Errorf("result[1].ID = %q, want svc-2", result[1].ID)
		}
		if result[2].ID != "svc-3" {
			t.Errorf("result[2].ID = %q, want svc-3", result[2].ID)
		}
	})

	t.Run("full service with all fields", func(t *testing.T) {
		t.Parallel()
		created := time.Date(2025, 1, 10, 8, 0, 0, 0, time.UTC)
		updated := time.Date(2025, 1, 15, 12, 0, 0, 0, time.UTC)
		completedAt := time.Date(2025, 1, 15, 11, 30, 0, 0, time.UTC)

		input := []agentsdocker.Service{{
			ID:             "svc-full",
			Name:           "fullservice",
			Stack:          "production",
			Image:          "myapp:v2.0",
			Mode:           "global",
			DesiredTasks:   5,
			RunningTasks:   5,
			CompletedTasks: 0,
			Labels: map[string]string{
				"com.docker.stack.namespace": "production",
			},
			EndpointPorts: []agentsdocker.ServicePort{
				{Name: "web", Protocol: "tcp", TargetPort: 8080, PublishedPort: 80, PublishMode: "ingress"},
			},
			UpdateStatus: &agentsdocker.ServiceUpdate{
				State:       "completed",
				Message:     "rollout complete",
				CompletedAt: &completedAt,
			},
			CreatedAt: &created,
			UpdatedAt: &updated,
		}}

		result := convertDockerServices(input)
		if len(result) != 1 {
			t.Fatalf("expected 1 service, got %d", len(result))
		}

		svc := result[0]
		if svc.ID != "svc-full" {
			t.Errorf("ID mismatch")
		}
		if svc.Mode != "global" {
			t.Errorf("Mode = %q, want global", svc.Mode)
		}
		if svc.Labels["com.docker.stack.namespace"] != "production" {
			t.Errorf("Labels mismatch")
		}
		if len(svc.EndpointPorts) != 1 || svc.EndpointPorts[0].PublishedPort != 80 {
			t.Errorf("EndpointPorts mismatch")
		}
		if svc.UpdateStatus == nil || svc.UpdateStatus.State != "completed" {
			t.Errorf("UpdateStatus mismatch")
		}
		if svc.CreatedAt == nil || !svc.CreatedAt.Equal(created) {
			t.Errorf("CreatedAt mismatch")
		}
		if svc.UpdatedAt == nil || !svc.UpdatedAt.Equal(updated) {
			t.Errorf("UpdatedAt mismatch")
		}
	})
}

func TestApplyDockerReport_MissingIdentifier(t *testing.T) {
	monitor := newTestMonitor(t)

	// Report with no agent ID and no hostname - should fail
	report := agentsdocker.Report{
		Host: agentsdocker.HostInfo{
			Hostname: "", // Empty hostname
		},
		Agent: agentsdocker.AgentInfo{
			ID: "", // Empty agent ID
		},
		Timestamp: time.Now(),
	}

	_, err := monitor.ApplyDockerReport(report, nil)
	if err == nil {
		t.Error("expected error for missing identifier")
	}
	if err != nil && !strings.Contains(err.Error(), "missing") {
		t.Errorf("expected 'missing' in error message, got: %v", err)
	}
}

func TestApplyDockerReport_RemovedHostRejection(t *testing.T) {
	monitor := newTestMonitor(t)

	// Mark host as removed
	hostID := "removed-docker-host"
	removedAt := time.Now().Add(-1 * time.Hour)
	monitor.mu.Lock()
	monitor.removedDockerHosts[hostID] = removedAt
	monitor.mu.Unlock()

	report := agentsdocker.Report{
		Host: agentsdocker.HostInfo{
			Hostname: hostID,
		},
		Agent: agentsdocker.AgentInfo{
			ID: hostID,
		},
		Timestamp: time.Now(),
	}

	_, err := monitor.ApplyDockerReport(report, nil)
	if err == nil {
		t.Error("expected error for removed host")
	}
	if err != nil && !strings.Contains(err.Error(), "monitoring stopped") {
		t.Errorf("expected 'monitoring stopped' in error message, got: %v", err)
	}
}

func TestApplyDockerReport_TokenBoundToDifferentAgent(t *testing.T) {
	monitor := newTestMonitor(t)

	tokenID := "shared-token"
	firstAgentID := "agent-first"
	secondAgentID := "agent-second"

	// Pre-bind token to first agent
	monitor.mu.Lock()
	monitor.dockerTokenBindings[tokenID] = firstAgentID
	monitor.mu.Unlock()

	// Report from second agent using same token
	report := agentsdocker.Report{
		Host: agentsdocker.HostInfo{
			Hostname: "second-host",
		},
		Agent: agentsdocker.AgentInfo{
			ID: secondAgentID,
		},
		Timestamp: time.Now(),
	}

	token := &config.APITokenRecord{ID: tokenID, Name: "TestToken"}

	_, err := monitor.ApplyDockerReport(report, token)
	if err == nil {
		t.Error("expected error for token bound to different agent")
	}
	if err != nil && !strings.Contains(err.Error(), "already in use by agent") {
		t.Errorf("expected 'already in use by agent' in error message, got: %v", err)
	}
}

func TestApplyDockerReport_MissingHostname(t *testing.T) {
	monitor := newTestMonitor(t)

	// Report with agent ID but no hostname
	report := agentsdocker.Report{
		Host: agentsdocker.HostInfo{
			Hostname: "", // Missing hostname
		},
		Agent: agentsdocker.AgentInfo{
			ID: "agent-with-id",
		},
		Timestamp: time.Now(),
	}

	_, err := monitor.ApplyDockerReport(report, nil)
	if err == nil {
		t.Error("expected error for missing hostname")
	}
	if err != nil && !strings.Contains(err.Error(), "missing hostname") {
		t.Errorf("expected 'missing hostname' in error message, got: %v", err)
	}
}

func TestApplyDockerReport_ReconnectWithoutMachineID(t *testing.T) {
	monitor := newTestMonitor(t)

	// Report with NO AgentID and NO MachineID, but valid Hostname.
	// This simulates a containerized agent without persistent ID or machine-id mount.
	report := agentsdocker.Report{
		Agent: agentsdocker.AgentInfo{
			ID:              "", // Empty AgentID
			Version:         "1.0.0",
			IntervalSeconds: 30,
		},
		Host: agentsdocker.HostInfo{
			Hostname:  "reconnect-host",
			MachineID: "", // Empty MachineID
		},
		Timestamp: time.Now().UTC(),
	}

	token := &config.APITokenRecord{ID: "token-reconnect", Name: "Test Token"}

	// First report - should succeed
	host1, err := monitor.ApplyDockerReport(report, token)
	if err != nil {
		t.Fatalf("First ApplyDockerReport failed: %v", err)
	}
	if host1.ID == "" {
		t.Fatal("First host has empty ID")
	}

	// Second report - identical to first
	// This must succeed by matching the existing host via Hostname + Token
	host2, err := monitor.ApplyDockerReport(report, token)
	if err != nil {
		t.Fatalf("Second ApplyDockerReport failed: %v", err)
	}

	// Verify that we matched the existing host instead of creating a new one
	if host1.ID != host2.ID {
		t.Errorf("Host IDs mismatch: %q vs %q - expected them to remain the same", host1.ID, host2.ID)
	}
}

func TestApplyDockerReport_SameHostnameDifferentTokens(t *testing.T) {
	monitor := newTestMonitor(t)

	// Two agents with the same hostname but different tokens should be treated as DIFFERENT hosts.
	// This tests that our Hostname+Token matching doesn't incorrectly merge unrelated agents.
	reportA := agentsdocker.Report{
		Agent: agentsdocker.AgentInfo{
			ID:              "", // Empty AgentID
			Version:         "1.0.0",
			IntervalSeconds: 30,
		},
		Host: agentsdocker.HostInfo{
			Hostname:  "shared-hostname",
			MachineID: "", // Empty MachineID
		},
		Timestamp: time.Now().UTC(),
	}

	tokenA := &config.APITokenRecord{ID: "token-A", Name: "Token A"}
	tokenB := &config.APITokenRecord{ID: "token-B", Name: "Token B"}

	// Agent A reports first
	hostA, err := monitor.ApplyDockerReport(reportA, tokenA)
	if err != nil {
		t.Fatalf("Agent A report failed: %v", err)
	}

	// Agent B reports with same hostname but different token
	hostB, err := monitor.ApplyDockerReport(reportA, tokenB)
	if err != nil {
		t.Fatalf("Agent B report failed: %v", err)
	}

	// They should have DIFFERENT IDs since they use different tokens
	if hostA.ID == hostB.ID {
		t.Errorf("Hosts should have different IDs but both have %q", hostA.ID)
	}

	// Verify both hosts exist in state
	hosts := monitor.state.GetDockerHosts()
	if len(hosts) != 2 {
		t.Errorf("Expected 2 hosts in state, got %d", len(hosts))
	}
}

// blockingDockerChecker simulates the field failure mode of the minipc
// incident (2026-08-20): a loaded Proxmox host where pct exec is arbitrarily
// slow, so probes are still executing when the next poll cycle starts.
type blockingDockerChecker struct {
	mu      sync.Mutex
	calls   []int
	started chan struct{} // closed when the first probe begins
	release chan struct{} // probes block here until the test closes it
	results map[int]bool
	errs    map[int]error
}

func (c *blockingDockerChecker) CheckDockerInContainer(ctx context.Context, node string, vmid int) (bool, error) {
	c.mu.Lock()
	c.calls = append(c.calls, vmid)
	first := len(c.calls) == 1
	c.mu.Unlock()
	if first {
		close(c.started)
	}
	select {
	case <-c.release:
	case <-ctx.Done():
		return false, ctx.Err()
	}
	if err, ok := c.errs[vmid]; ok {
		return false, err
	}
	return c.results[vmid], nil
}

func (c *blockingDockerChecker) callCount() int {
	c.mu.Lock()
	defer c.mu.Unlock()
	return len(c.calls)
}

// Bug 1 regression (minipc incident): while a probe for a guest is still
// executing, subsequent poll cycles must not re-dispatch the same probe. In
// the field the dispatcher re-issued the identical command every ~3s with
// fresh request IDs, so a slow host self-amplified until it ran out of
// resources.
func TestCheckContainersForDocker_InFlightProbeNotReissued(t *testing.T) {
	state := models.NewState()
	monitor := &Monitor{state: state}
	checker := &blockingDockerChecker{
		started: make(chan struct{}),
		release: make(chan struct{}),
		results: map[int]bool{101: true},
	}
	monitor.SetDockerChecker(checker)

	containers := func() []models.Container {
		return []models.Container{
			{ID: "ct-1", VMID: 101, Name: "slow-guest", Node: "node1", Status: "running"},
		}
	}

	firstDone := make(chan struct{})
	go func() {
		defer close(firstDone)
		monitor.CheckContainersForDocker(context.Background(), containers())
	}()

	select {
	case <-checker.started:
	case <-time.After(5 * time.Second):
		t.Fatalf("first probe never started")
	}

	// Second and third poll cycles arrive while the first probe is still
	// executing. Before the in-flight claim they would each dispatch the
	// same probe again.
	monitor.CheckContainersForDocker(context.Background(), containers())
	monitor.CheckContainersForDocker(context.Background(), containers())
	if got := checker.callCount(); got != 1 {
		t.Fatalf("expected 1 in-flight probe, got %d dispatches", got)
	}

	close(checker.release)
	select {
	case <-firstDone:
	case <-time.After(5 * time.Second):
		t.Fatalf("first poll cycle never finished")
	}

	// The probe completed, releasing the claim: the guest is probeable again
	// once the normal cadence asks for it.
	monitor.dockerProbeFailureMu.Lock()
	_, held := monitor.dockerProbesInFlight["ct-1"]
	monitor.dockerProbeFailureMu.Unlock()
	if held {
		t.Fatalf("expected in-flight claim to be released after probe completion")
	}
}

// Bug 3 regression (server side): a poll cycle whose enrichment context has
// already expired must not dispatch probes at all — in the field the server
// sent the command and then logged "Agent command canceled ... duration 0.05"
// while the agent kept executing it. An abandoned dispatch must also not
// count as a probe failure against the guest.
func TestCheckContainersForDocker_ExpiredContextDoesNotDispatch(t *testing.T) {
	state := models.NewState()
	checkedAt := time.Now().Add(-time.Hour)
	state.UpdateContainers([]models.Container{
		{ID: "ct-1", VMID: 101, Name: "guest", Node: "node1", Status: "running", HasDocker: true, DockerCheckedAt: checkedAt},
	})
	monitor := &Monitor{state: state}
	checker := &mockDockerChecker{results: map[int]bool{101: false}}
	monitor.SetDockerChecker(checker)

	ctx, cancel := context.WithCancel(context.Background())
	cancel()

	result := monitor.CheckContainersForDocker(ctx, []models.Container{
		{ID: "ct-1", VMID: 101, Name: "guest", Node: "node1", Status: "running"},
	})

	if got := checker.callCount(); got != 0 {
		t.Fatalf("expected no probe dispatch under expired context, got %d", got)
	}
	if !result[0].HasDocker || !result[0].DockerCheckedAt.Equal(checkedAt) {
		t.Fatalf("expected previous docker status preserved, got %+v", result[0])
	}
	monitor.dockerProbeFailureMu.Lock()
	failures := len(monitor.dockerProbeFailures)
	monitor.dockerProbeFailureMu.Unlock()
	if failures != 0 {
		t.Fatalf("expected no probe failures recorded for an undispatched cycle, got %d", failures)
	}
}

// A probe the server abandoned mid-flight (context canceled while waiting on
// the agent) may still be running on the agent. It must not be re-dispatched
// while the in-flight window holds, and it must not feed the failure backoff
// or the node circuit breaker — abandonment says nothing about the guest.
func TestCheckContainersForDocker_AbandonedProbeHoldsClaimWithoutFailure(t *testing.T) {
	state := models.NewState()
	monitor := &Monitor{state: state}
	checker := &mockDockerChecker{
		errs: map[int]error{101: context.Canceled},
	}
	monitor.SetDockerChecker(checker)

	containers := []models.Container{
		{ID: "ct-1", VMID: 101, Name: "guest", Node: "node1", Status: "running"},
	}

	monitor.CheckContainersForDocker(context.Background(), containers)
	if got := checker.callCount(); got != 1 {
		t.Fatalf("expected 1 dispatch, got %d", got)
	}

	monitor.dockerProbeFailureMu.Lock()
	failures := len(monitor.dockerProbeFailures)
	nodeFailures := len(monitor.dockerNodeProbeFailures)
	_, held := monitor.dockerProbesInFlight["ct-1"]
	monitor.dockerProbeFailureMu.Unlock()
	if failures != 0 || nodeFailures != 0 {
		t.Fatalf("abandoned probe must not record failures (guest=%d node=%d)", failures, nodeFailures)
	}
	if !held {
		t.Fatalf("abandoned probe must keep its in-flight claim")
	}

	// Next cycles: still claimed, no re-dispatch.
	monitor.CheckContainersForDocker(context.Background(), containers)
	monitor.CheckContainersForDocker(context.Background(), containers)
	if got := checker.callCount(); got != 1 {
		t.Fatalf("expected abandoned probe not to be re-issued, got %d dispatches", got)
	}

	// Once the in-flight window has passed the guest is probeable again.
	monitor.dockerProbeFailureMu.Lock()
	monitor.dockerProbesInFlight["ct-1"] = time.Now().Add(-2 * proxmoxGuestDockerProbeInFlightWindow)
	monitor.dockerProbeFailureMu.Unlock()
	monitor.CheckContainersForDocker(context.Background(), containers)
	if got := checker.callCount(); got != 2 {
		t.Fatalf("expected retry after in-flight window expiry, got %d dispatches", got)
	}
}

// Node circuit breaker: once a node accumulates enough consecutive command
// failures, no Docker probe is dispatched to it — even for a newly appearing
// guest that has no per-guest failure history. Other nodes are unaffected,
// and a success on the node closes the breaker.
func TestCheckContainersForDocker_NodeCircuitBreaker(t *testing.T) {
	state := models.NewState()
	monitor := &Monitor{state: state}
	checker := &mockDockerChecker{
		errs: map[int]error{
			101: errors.New("pct exec hung"),
			102: errors.New("pct exec hung"),
			103: errors.New("pct exec hung"),
		},
		results: map[int]bool{104: true, 201: true},
	}
	monitor.SetDockerChecker(checker)

	// Cycle 1: three guests on node1 fail, opening the breaker.
	monitor.CheckContainersForDocker(context.Background(), []models.Container{
		{ID: "ct-1", VMID: 101, Name: "a", Node: "node1", Status: "running"},
		{ID: "ct-2", VMID: 102, Name: "b", Node: "node1", Status: "running"},
		{ID: "ct-3", VMID: 103, Name: "c", Node: "node1", Status: "running"},
	})
	if got := checker.callCount(); got != 3 {
		t.Fatalf("expected 3 dispatches in first cycle, got %d", got)
	}
	if !monitor.dockerNodeProbeBreakerOpen("node1") {
		t.Fatalf("expected node1 breaker open after %d consecutive failures", proxmoxGuestDockerNodeFailureThreshold)
	}

	// Cycle 2: a brand-new guest on node1 must not be probed while the
	// breaker is open; a guest on node2 still is.
	monitor.CheckContainersForDocker(context.Background(), []models.Container{
		{ID: "ct-4", VMID: 104, Name: "new-on-bad-node", Node: "node1", Status: "running"},
		{ID: "ct-5", VMID: 201, Name: "on-good-node", Node: "node2", Status: "running"},
	})
	checker.mu.Lock()
	calls := append([]int(nil), checker.calls...)
	checker.mu.Unlock()
	if len(calls) != 4 || calls[3] != 201 {
		t.Fatalf("expected only node2 guest probed while breaker open, got dispatches %v", calls)
	}

	// Breaker backoff expires: node1 is retried, and the success closes the
	// breaker.
	monitor.dockerProbeFailureMu.Lock()
	monitor.dockerNodeProbeFailures["node1"].lastAt = time.Now().Add(-time.Hour)
	monitor.dockerProbeFailureMu.Unlock()
	monitor.CheckContainersForDocker(context.Background(), []models.Container{
		{ID: "ct-4", VMID: 104, Name: "new-on-bad-node", Node: "node1", Status: "running"},
	})
	if got := checker.callCount(); got != 5 {
		t.Fatalf("expected node1 retried after breaker backoff, got %d dispatches", got)
	}
	monitor.dockerProbeFailureMu.Lock()
	_, stillTracked := monitor.dockerNodeProbeFailures["node1"]
	monitor.dockerProbeFailureMu.Unlock()
	if stillTracked {
		t.Fatalf("expected breaker to close after a successful command on node1")
	}
}

// blockingInventoryCollector mirrors blockingDockerChecker for the inventory
// command path.
type blockingInventoryCollector struct {
	mu      sync.Mutex
	calls   int
	started chan struct{}
	release chan struct{}
}

func (c *blockingInventoryCollector) CollectDockerInventory(ctx context.Context, container models.Container) (agentsdocker.Report, bool, error) {
	c.mu.Lock()
	c.calls++
	first := c.calls == 1
	c.mu.Unlock()
	if first {
		close(c.started)
	}
	select {
	case <-c.release:
	case <-ctx.Done():
		return agentsdocker.Report{}, false, ctx.Err()
	}
	return agentsdocker.Report{}, false, nil
}

func (c *blockingInventoryCollector) callCount() int {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.calls
}

// The inventory command path has the same in-flight and expired-context rules
// as the socket probe: no dispatch under a dead context, and no re-dispatch
// while a previous inventory command for the guest is still executing.
func TestCollectProxmoxGuestDockerInventory_InFlightAndExpiredContext(t *testing.T) {
	state := models.NewState()
	monitor := &Monitor{state: state}
	collector := &blockingInventoryCollector{
		started: make(chan struct{}),
		release: make(chan struct{}),
	}
	monitor.SetDockerInventoryCollector(collector)

	containers := []models.Container{
		{ID: "ct-1", VMID: 101, Name: "guest", Node: "node1", Status: "running", HasDocker: true},
	}

	// Expired context: nothing dispatched.
	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	monitor.CollectProxmoxGuestDockerInventory(ctx, containers)
	if got := collector.callCount(); got != 0 {
		t.Fatalf("expected no inventory dispatch under expired context, got %d", got)
	}

	firstDone := make(chan struct{})
	go func() {
		defer close(firstDone)
		monitor.CollectProxmoxGuestDockerInventory(context.Background(), containers)
	}()
	select {
	case <-collector.started:
	case <-time.After(5 * time.Second):
		t.Fatalf("first inventory collection never started")
	}

	// While in flight, the next cycle must not re-dispatch.
	monitor.CollectProxmoxGuestDockerInventory(context.Background(), containers)
	if got := collector.callCount(); got != 1 {
		t.Fatalf("expected 1 in-flight inventory collection, got %d dispatches", got)
	}

	close(collector.release)
	select {
	case <-firstDone:
	case <-time.After(5 * time.Second):
		t.Fatalf("first inventory cycle never finished")
	}
}