pulse/internal/mock/generator_test.go

package mock

import (
	"fmt"
	"math"
	"sort"
	"strings"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)

func TestBuildFixtureStateIncludesDockerHosts(t *testing.T) {
	cfg := DefaultConfig
	cfg.DockerHostCount = 2
	cfg.DockerContainersPerHost = 5

	data := buildFixtureState(cfg)

	// Nested Docker-in-LXC hosts are appended on top of the configured
	// agent-reported hosts and deliberately carry no native engine
	// inventory, so the native-host expectations exclude them.
	nativeHosts := 0
	for _, host := range data.DockerHosts {
		if host.ID == "" {
			t.Fatalf("docker host missing id: %+v", host)
		}
		if strings.HasPrefix(host.ID, mockProxmoxLXCDockerHostSourcePrefix) {
			continue
		}
		nativeHosts++
		if len(host.Containers) == 0 {
			t.Fatalf("docker host %s has no containers", host.Hostname)
		}
		if len(host.Images) == 0 {
			t.Fatalf("docker host %s has no image inventory", host.Hostname)
		}
		if len(host.Volumes) == 0 {
			t.Fatalf("docker host %s has no volume inventory", host.Hostname)
		}
		if len(host.Networks) == 0 {
			t.Fatalf("docker host %s has no network inventory", host.Hostname)
		}
		if host.StorageUsage == nil || host.StorageUsage.Images.TotalCount == 0 || host.StorageUsage.Volumes.TotalCount == 0 {
			t.Fatalf("docker host %s has no engine storage usage inventory: %+v", host.Hostname, host.StorageUsage)
		}
	}
	if nativeHosts != cfg.DockerHostCount {
		t.Fatalf("expected %d native docker hosts, got %d", cfg.DockerHostCount, nativeHosts)
	}
}

func TestComputeGuestCountsHandlesZeroBaselines(t *testing.T) {
	cfg := MockConfig{
		VMsPerNode:  0,
		LXCsPerNode: 0,
	}

	roles := []string{"vm-heavy", "container-heavy", "light", "mixed"}
	for _, role := range roles {
		vmCount, lxcCount := computeGuestCounts(cfg, role)
		if vmCount < 0 || lxcCount < 0 {
			t.Fatalf("expected non-negative counts for role %q, got vm=%d lxc=%d", role, vmCount, lxcCount)
		}
	}
}

func TestBuildFixtureStateWithZeroGuestBaselinesDoesNotPanic(t *testing.T) {
	cfg := DefaultConfig
	cfg.NodeCount = 4
	cfg.VMsPerNode = 0
	cfg.LXCsPerNode = 0
	cfg.DockerHostCount = 0
	cfg.GenericHostCount = 0
	cfg.K8sClusterCount = 0

	for i := 0; i < 20; i++ {
		func() {
			defer func() {
				if r := recover(); r != nil {
					t.Fatalf("buildFixtureState panicked on iteration %d: %v", i, r)
				}
			}()
			_ = buildFixtureState(cfg)
		}()
	}
}

func TestBuildFixtureStateIncludesSwarmServices(t *testing.T) {
	cfg := DefaultConfig
	// Two hosts is the smallest mixed Docker/Podman estate generated by the
	// mock runtime and must still include a Docker manager inventory.
	cfg.DockerHostCount = 2
	cfg.DockerContainersPerHost = 6
	cfg.RandomMetrics = false

	data := buildFixtureState(cfg)

	found := false
	for _, host := range data.DockerHosts {
		if len(host.Services) == 0 {
			continue
		}
		if host.Swarm == nil {
			t.Fatalf("expected swarm metadata for host %s", host.ID)
		}
		if len(host.Tasks) == 0 {
			t.Fatalf("expected tasks for service host %s", host.ID)
		}
		if len(host.Secrets) == 0 {
			t.Fatalf("expected secrets for service host %s", host.ID)
		}
		if len(host.Configs) == 0 {
			t.Fatalf("expected configs for service host %s", host.ID)
		}
		if len(host.Nodes) == 0 {
			t.Fatalf("expected swarm nodes for service host %s", host.ID)
		}
		found = true
		break
	}

	if !found {
		t.Fatalf("expected at least one docker host with swarm services")
	}
}

func TestBuildFixtureStateIncludesHostAgents(t *testing.T) {
	cfg := DefaultConfig
	cfg.GenericHostCount = 5
	cfg.K8sClusterCount = 0
	cfg.RandomMetrics = false

	data := buildFixtureState(cfg)

	expectedMin := cfg.GenericHostCount + cfg.NodeCount
	if len(data.Hosts) < expectedMin {
		t.Fatalf("expected at least %d host agents, got %d", expectedMin, len(data.Hosts))
	}

	for _, host := range data.Hosts {
		if host.ID == "" {
			t.Fatalf("host agent missing id: %+v", host)
		}
		if host.Hostname == "" {
			t.Fatalf("host agent missing hostname: %+v", host)
		}
		if host.Status == "" {
			t.Fatalf("host agent missing status: %+v", host)
		}
		if host.Memory.Total > 0 {
			if host.Memory.Cache <= 0 {
				t.Fatalf("host agent %s should report reclaimable cache so the memory split is exercisable: %+v", host.ID, host.Memory)
			}
			if sum := host.Memory.Used + host.Memory.Cache + host.Memory.Free; sum > host.Memory.Total {
				t.Fatalf("host agent %s memory used+cache+free %d exceeds total %d", host.ID, sum, host.Memory.Total)
			}
		}
	}
}

func TestBuildFixtureStatePreservesStandaloneHostAgentsWhenLinkingNodes(t *testing.T) {
	cfg := DefaultConfig
	cfg.NodeCount = 3
	cfg.GenericHostCount = 4
	cfg.K8sClusterCount = 0
	cfg.RandomMetrics = false

	data := buildFixtureState(cfg)

	standalone := 0
	proxmoxLinked := 0
	for _, host := range data.Hosts {
		if host.LinkedNodeID != "" {
			proxmoxLinked++
		} else {
			standalone++
		}
	}

	if standalone != cfg.GenericHostCount {
		t.Fatalf("expected %d standalone agent hosts, got %d", cfg.GenericHostCount, standalone)
	}
	if proxmoxLinked != cfg.NodeCount {
		t.Fatalf("expected %d Proxmox-linked agent hosts, got %d", cfg.NodeCount, proxmoxLinked)
	}
}

func TestGenerateVMStoppedPreservesConfiguredCapacity(t *testing.T) {
	cfg := DefaultConfig
	cfg.StoppedPercent = 1

	vm := generateVM("node-01", "mock-cluster", 1001, cfg)

	if vm.Status != "stopped" {
		t.Fatalf("expected stopped VM, got %q", vm.Status)
	}
	if vm.Memory.Total <= 0 {
		t.Fatalf("expected stopped VM to keep configured memory total, got %d", vm.Memory.Total)
	}
	if vm.Memory.Used != 0 || vm.Memory.Usage != 0 {
		t.Fatalf("expected stopped VM memory usage to be zero, got used=%d usage=%f", vm.Memory.Used, vm.Memory.Usage)
	}
	if vm.Memory.Free != vm.Memory.Total {
		t.Fatalf("expected stopped VM free memory %d to equal total %d", vm.Memory.Free, vm.Memory.Total)
	}
}

func TestGenerateContainerStoppedPreservesConfiguredCapacity(t *testing.T) {
	cfg := DefaultConfig
	cfg.StoppedPercent = 1

	ct := generateContainer("node-01", "mock-cluster", 2001, cfg)

	if ct.Status != "stopped" {
		t.Fatalf("expected stopped container, got %q", ct.Status)
	}
	if ct.Memory.Total <= 0 {
		t.Fatalf("expected stopped container to keep configured memory total, got %d", ct.Memory.Total)
	}
	if ct.Memory.Used != 0 || ct.Memory.Usage != 0 {
		t.Fatalf("expected stopped container memory usage to be zero, got used=%d usage=%f", ct.Memory.Used, ct.Memory.Usage)
	}
	if ct.Memory.Free != ct.Memory.Total {
		t.Fatalf("expected stopped container free memory %d to equal total %d", ct.Memory.Free, ct.Memory.Total)
	}
}

func TestNormalizeMockBlendWeight_ComposesAcrossUpdateInterval(t *testing.T) {
	perMinuteWeight := 0.22
	perTickWeight := normalizeMockBlendWeight(perMinuteWeight, defaultMockUpdateInterval, time.Minute)
	compounded := 1 - math.Pow(1-perTickWeight, float64(time.Minute/defaultMockUpdateInterval))

	if perTickWeight >= perMinuteWeight {
		t.Fatalf("expected per-tick weight %.6f to be less than per-minute weight %.6f", perTickWeight, perMinuteWeight)
	}
	if math.Abs(compounded-perMinuteWeight) > 0.01 {
		t.Fatalf("expected compounded weight %.6f to stay close to %.6f", compounded, perMinuteWeight)
	}
}

func TestBuildFixtureStateLinksAllNodesToHostAgents(t *testing.T) {
	cfg := DefaultConfig
	cfg.NodeCount = 7
	cfg.GenericHostCount = 2
	cfg.RandomMetrics = false

	data := buildFixtureState(cfg)

	if len(data.Hosts) < cfg.NodeCount {
		t.Fatalf("expected enough hosts to link all nodes, got hosts=%d nodes=%d", len(data.Hosts), cfg.NodeCount)
	}

	hostsByID := make(map[string]models.Host, len(data.Hosts))
	for _, host := range data.Hosts {
		hostsByID[host.ID] = host
	}

	for _, node := range data.Nodes {
		if node.LinkedAgentID == "" {
			t.Fatalf("node %s is missing linked host agent id", node.ID)
		}
		host, ok := hostsByID[node.LinkedAgentID]
		if !ok {
			t.Fatalf("node %s linked host %s not found", node.ID, node.LinkedAgentID)
		}
		if host.LinkedNodeID != node.ID {
			t.Fatalf("host %s linkedNodeID=%q, want %q", host.ID, host.LinkedNodeID, node.ID)
		}
	}
}

// A node and its linked agent carry one CPU reading under the agent's real
// keys, taken from the series that seeds the agent's history, so the node
// drawer's headline continues the line drawn beneath it.
func TestFixtureNodeAndLinkedAgentShareOnePrimaryCPUTemperature(t *testing.T) {
	cfg := DefaultConfig
	cfg.NodeCount = 7
	cfg.GenericHostCount = 2
	cfg.RandomMetrics = true

	data := buildFixtureState(cfg)
	check := func(stage string, at time.Time, sampled bool) {
		t.Helper()
		hostsByID := make(map[string]models.Host, len(data.Hosts))
		for _, host := range data.Hosts {
			hostsByID[host.ID] = host
			if _, legacy := host.Sensors.TemperatureCelsius["cpu.package"]; legacy {
				t.Fatalf("%s: host %s carries a second package reading under cpu.package", stage, host.ID)
			}
		}
		compared := 0
		for _, node := range data.Nodes {
			nodeReading := models.PrimaryCPUTemperatureCelsius(node.Temperature)
			if nodeReading == nil || node.Status != "online" {
				continue
			}
			host, ok := hostsByID[node.LinkedAgentID]
			if !ok || host.Status == "offline" {
				continue
			}
			hostReading := models.HostSensorPrimaryCPUTemperatureCelsius(host.Sensors)
			if hostReading == nil || *hostReading != *nodeReading {
				t.Fatalf("%s: node %s reads %v but its agent %s reads %v", stage, node.ID, *nodeReading, host.ID, hostReading)
			}
			if want := SampleMetric("agent", host.ID, "temperature", at); sampled && *nodeReading != want {
				t.Fatalf("%s: node %s reads %v, want the agent series sample %v", stage, node.ID, *nodeReading, want)
			}
			compared++
		}
		if compared == 0 {
			t.Fatalf("%s: no online node with a CPU reading to compare", stage)
		}
	}
	check("build", time.Time{}, false)

	refreshAt := time.Now().Add(time.Minute)
	updateFixtureStateMetricsAt(&data, cfg, refreshAt)
	check("refresh", refreshAt, true)
}

func TestBuildFixtureStatePopulatesHostIORates(t *testing.T) {
	cfg := DefaultConfig
	cfg.NodeCount = 6
	cfg.GenericHostCount = 1
	cfg.RandomMetrics = true

	data := buildFixtureState(cfg)

	for _, host := range data.Hosts {
		if host.Status == "offline" {
			if host.NetInRate != 0 || host.NetOutRate != 0 || host.DiskReadRate != 0 || host.DiskWriteRate != 0 {
				t.Fatalf("offline host %s should have zero I/O rates", host.ID)
			}
			continue
		}
		if host.NetInRate <= 0 || host.NetOutRate <= 0 {
			t.Fatalf("host %s missing network rates: in=%f out=%f", host.ID, host.NetInRate, host.NetOutRate)
		}
		if host.DiskReadRate <= 0 || host.DiskWriteRate <= 0 {
			t.Fatalf("host %s missing disk rates: read=%f write=%f", host.ID, host.DiskReadRate, host.DiskWriteRate)
		}
	}
}

func TestBuildFixtureStatePopulatesDockerHostIORates(t *testing.T) {
	cfg := DefaultConfig
	cfg.DockerHostCount = 3
	cfg.RandomMetrics = true

	data := buildFixtureState(cfg)

	if len(data.DockerHosts) == 0 {
		t.Fatal("expected docker hosts in mock data")
	}

	for _, host := range data.DockerHosts {
		if strings.HasPrefix(host.ID, mockProxmoxLXCDockerHostSourcePrefix) {
			// Docker-in-LXC hosts mirror the production pct exec inventory,
			// which reports no host-level I/O rates or temperature.
			continue
		}
		if host.Status == "offline" {
			if host.NetInRate != 0 || host.NetOutRate != 0 || host.DiskReadRate != 0 || host.DiskWriteRate != 0 {
				t.Fatalf("offline docker host %s should have zero I/O rates", host.ID)
			}
			if host.Temperature != nil {
				t.Fatalf("offline docker host %s should not report temperature", host.ID)
			}
			continue
		}
		if host.NetInRate <= 0 || host.NetOutRate <= 0 {
			t.Fatalf("docker host %s missing network rates: in=%f out=%f", host.ID, host.NetInRate, host.NetOutRate)
		}
		if host.DiskReadRate <= 0 || host.DiskWriteRate <= 0 {
			t.Fatalf("docker host %s missing disk rates: read=%f write=%f", host.ID, host.DiskReadRate, host.DiskWriteRate)
		}
		if host.Temperature == nil {
			t.Fatalf("docker host %s missing temperature", host.ID)
		}
	}
}

func TestBuildFixtureStatePopulatesKubernetesUsageMetrics(t *testing.T) {
	cfg := DefaultConfig
	cfg.K8sClusterCount = 1
	cfg.K8sNodesPerCluster = 4
	cfg.K8sPodsPerCluster = 24
	cfg.RandomMetrics = false

	data := buildFixtureState(cfg)
	if len(data.KubernetesClusters) != 1 {
		t.Fatalf("expected exactly one kubernetes cluster, got %d", len(data.KubernetesClusters))
	}

	cluster := data.KubernetesClusters[0]
	readyNodeNames := make(map[string]struct{}, len(cluster.Nodes))
	for _, node := range cluster.Nodes {
		if node.Ready {
			readyNodeNames[strings.TrimSpace(node.Name)] = struct{}{}
		}
	}

	runningPodsWithUsage := 0
	for _, pod := range cluster.Pods {
		if !strings.EqualFold(pod.Phase, "running") {
			continue
		}
		if _, ok := readyNodeNames[strings.TrimSpace(pod.NodeName)]; !ok {
			continue
		}
		if pod.UsageCPUMilliCores <= 0 || pod.UsageMemoryBytes <= 0 {
			t.Fatalf("running pod %s missing cpu/memory usage: %+v", pod.Name, pod)
		}
		if pod.NetInRate <= 0 || pod.NetOutRate <= 0 {
			t.Fatalf("running pod %s missing network rates: in=%f out=%f", pod.Name, pod.NetInRate, pod.NetOutRate)
		}
		if pod.EphemeralStorageCapacityBytes <= 0 || pod.EphemeralStorageUsedBytes <= 0 {
			t.Fatalf("running pod %s missing ephemeral storage usage: %+v", pod.Name, pod)
		}
		if pod.DiskUsagePercent <= 0 {
			t.Fatalf("running pod %s missing disk usage percent: %+v", pod.Name, pod)
		}
		runningPodsWithUsage++
	}
	if runningPodsWithUsage == 0 {
		t.Fatal("expected at least one running kubernetes pod with usage metrics")
	}

	readyNodesWithUsage := 0
	for _, node := range cluster.Nodes {
		if !node.Ready {
			continue
		}
		if node.UsageCPUMilliCores <= 0 || node.UsageMemoryBytes <= 0 {
			t.Fatalf("ready node %s missing usage metrics: %+v", node.Name, node)
		}
		readyNodesWithUsage++
	}
	if readyNodesWithUsage == 0 {
		t.Fatal("expected at least one ready kubernetes node with usage metrics")
	}
}

func TestGenerateKubernetesNodesAlwaysRetainsSchedulableReadyNode(t *testing.T) {
	for count := 1; count <= 8; count++ {
		for iteration := 0; iteration < 100; iteration++ {
			nodes := generateKubernetesNodes("fixture-cluster", count)
			hasSchedulableReadyNode := false
			for _, node := range nodes {
				if node.Ready && !node.Unschedulable {
					hasSchedulableReadyNode = true
					break
				}
			}
			if !hasSchedulableReadyNode {
				t.Fatalf("node count %d iteration %d produced no schedulable ready node: %+v", count, iteration, nodes)
			}
		}
	}
}

func TestBuildFixtureStateIncludesKubernetesDeploymentAPIMetadata(t *testing.T) {
	cfg := DefaultConfig
	cfg.K8sClusterCount = 1
	cfg.K8sNodesPerCluster = 3
	cfg.K8sPodsPerCluster = 8
	cfg.K8sDeploymentsPerCluster = 6
	cfg.RandomMetrics = false

	data := buildFixtureState(cfg)
	if len(data.KubernetesClusters) != 1 {
		t.Fatalf("expected exactly one kubernetes cluster, got %d", len(data.KubernetesClusters))
	}

	cluster := data.KubernetesClusters[0]
	if len(cluster.Deployments) == 0 {
		t.Fatal("expected kubernetes deployment inventory")
	}
	for _, deployment := range cluster.Deployments {
		if deployment.UID == "" {
			t.Fatalf("deployment missing uid: %+v", deployment)
		}
		if deployment.CreatedAt.IsZero() {
			t.Fatalf("deployment missing creation metadata: %+v", deployment)
		}
		if deployment.ObservedGeneration == 0 {
			t.Fatalf("deployment missing observed generation: %+v", deployment)
		}
	}
}

func TestBuildFixtureStateCreatesHostEntriesForKubernetesNodes(t *testing.T) {
	cfg := DefaultConfig
	cfg.K8sClusterCount = 1
	cfg.K8sNodesPerCluster = 3
	cfg.K8sPodsPerCluster = 8
	cfg.NodeCount = 0
	cfg.RandomMetrics = false

	data := buildFixtureState(cfg)
	if len(data.KubernetesClusters) == 0 {
		t.Fatal("expected kubernetes clusters in mock data")
	}

	hostsByName := make(map[string]models.Host, len(data.Hosts))
	for _, host := range data.Hosts {
		hostsByName[strings.ToLower(strings.TrimSpace(host.Hostname))] = host
	}

	for _, node := range data.KubernetesClusters[0].Nodes {
		host, ok := hostsByName[strings.ToLower(strings.TrimSpace(node.Name))]
		if !ok {
			t.Fatalf("expected host entry for kubernetes node %s", node.Name)
		}
		if host.NetInRate <= 0 || host.NetOutRate <= 0 {
			t.Fatalf("kubernetes node host %s missing network rates: in=%f out=%f", host.Hostname, host.NetInRate, host.NetOutRate)
		}
		if host.DiskReadRate <= 0 || host.DiskWriteRate <= 0 {
			t.Fatalf("kubernetes node host %s missing disk rates: read=%f write=%f", host.Hostname, host.DiskReadRate, host.DiskWriteRate)
		}
		if len(host.Sensors.TemperatureCelsius) == 0 {
			t.Fatalf("kubernetes node host %s missing temperature sensors", host.Hostname)
		}
	}
}

// newMockKubernetesTestCluster builds ready, schedulable nodes with a fixed
// number of running pods each, so placement does not depend on the random
// readiness the fixture generator rolls.
func newMockKubernetesTestCluster(nodeCount, podsPerNode int) models.KubernetesCluster {
	cluster := models.KubernetesCluster{ID: "usage-test", Name: "usage-test", Status: "online"}
	for n := 0; n < nodeCount; n++ {
		nodeName := fmt.Sprintf("usage-test-%d", n)
		cluster.Nodes = append(cluster.Nodes, models.KubernetesNode{
			UID:                 nodeName + "-uid",
			Name:                nodeName,
			Ready:               true,
			CapacityCPU:         8,
			AllocCPU:            8,
			CapacityMemoryBytes: 32 << 30,
			AllocMemoryBytes:    32 << 30,
		})
		for p := 0; p < podsPerNode; p++ {
			cluster.Pods = append(cluster.Pods, models.KubernetesPod{
				UID:       fmt.Sprintf("%s-pod-%d-uid", nodeName, p),
				Name:      fmt.Sprintf("%s-pod-%d", nodeName, p),
				Namespace: "apps",
				NodeName:  nodeName,
				Phase:     "Running",
				OwnerKind: "Deployment",
			})
		}
	}
	return cluster
}

func TestMockKubernetesDenseNodesStayBelowAllocatableMemory(t *testing.T) {
	cluster := newMockKubernetesTestCluster(3, 13)
	data := models.StateSnapshot{KubernetesClusters: []models.KubernetesCluster{cluster}}
	for _, node := range cluster.Nodes {
		data.Hosts = append(data.Hosts, models.Host{ID: "host-" + node.Name, Hostname: node.Name, Status: "online"})
	}

	start := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
	for tick := 0; tick < 40; tick++ {
		initializeMockKubernetesClusterUsage(&data.KubernetesClusters[0], start.Add(time.Duration(tick)*15*time.Second), false)
		// The mock tick re-derives each linked host's memory from its node.
		syncMockKubernetesNodeHosts(&data)

		podMemory := make(map[string]float64, len(cluster.Nodes))
		for _, pod := range data.KubernetesClusters[0].Pods {
			podMemory[pod.NodeName] += pod.UsageMemoryPercent
		}
		for _, node := range data.KubernetesClusters[0].Nodes {
			if node.UsageMemoryBytes >= node.AllocMemoryBytes || node.UsageMemoryPercent >= 100 {
				t.Fatalf("tick %d: node %s with 13 running pods is pinned at allocatable memory (%.1f%%)", tick, node.Name, node.UsageMemoryPercent)
			}
			// Steady-state shares stay under the ceiling, which clips to it
			// exactly; a total at the ceiling means it bound without a transient.
			if podMemory[node.Name] >= mockKubernetesNodePodMemoryCeiling*100-0.01 {
				t.Fatalf("tick %d: node %s pods hold %.2f%%, the ceiling bound in steady state", tick, node.Name, podMemory[node.Name])
			}
		}
		for _, host := range data.Hosts {
			if host.Memory.Usage >= 100 {
				t.Fatalf("tick %d: linked host %s memory = %.1f%%, want the node's unsaturated reading", tick, host.Hostname, host.Memory.Usage)
			}
		}
	}
}

func TestMockKubernetesRescheduledPodsDoNotSaturateReceivingNode(t *testing.T) {
	cluster := newMockKubernetesTestCluster(3, 8)
	start := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
	for tick := 0; tick < 40; tick++ {
		initializeMockKubernetesClusterUsage(&cluster, start.Add(time.Duration(tick)*15*time.Second), false)
	}

	// Two nodes fail: their sixteen pods move to the survivor carrying
	// readings sized for nodes that each ran a third as many pods.
	cluster.Nodes[0].Ready = false
	cluster.Nodes[1].Ready = false
	initializeMockKubernetesClusterUsage(&cluster, start.Add(40*15*time.Second), false)

	survivor := cluster.Nodes[2]
	running := 0
	for _, pod := range cluster.Pods {
		if pod.NodeName == survivor.Name {
			running++
		}
	}
	if running != 24 {
		t.Fatalf("survivor runs %d pods after the failures, want all 24", running)
	}
	// Pods hold at most the ceiling and system overhead tops out at 26%.
	if limit := (mockKubernetesNodePodMemoryCeiling + 0.26) * 100; survivor.UsageMemoryPercent > limit+1e-9 {
		t.Fatalf("survivor memory = %.1f%% right after rescheduling, want at most %.1f%%", survivor.UsageMemoryPercent, limit)
	}
}

func TestMockKubernetesPodMemoryShareKeepsNodeWithinBudget(t *testing.T) {
	for pods := 0; pods <= 64; pods++ {
		share := mockKubernetesPodMemoryShare(pods)
		if share <= 0 || share > 1 {
			t.Fatalf("share(%d) = %f, want (0, 1]", pods, share)
		}
		if committed := float64(pods) * share; committed > mockKubernetesNodePodMemoryBudget+1e-9 {
			t.Fatalf("%d pods commit %.2f single-pod footprints, want at most %.2f", pods, committed, mockKubernetesNodePodMemoryBudget)
		}
	}
}

func TestMockKubernetesLoneHeavyPodStaysUnderNodeCeiling(t *testing.T) {
	// A five-minute cadence lets smoothing track each target closely, so the
	// pod reaches its real peaks.
	setMockUpdateInterval(5 * time.Minute)
	t.Cleanup(func() { setMockUpdateInterval(DefaultConfig.UpdateInterval) })

	// The heaviest footprint: the highest base and burst seeds on a
	// Guaranteed StatefulSet pod.
	uid, best := "", uint64(0)
	for i := 0; i < 2000; i++ {
		candidate := fmt.Sprintf("lone-pod-%d", i)
		weight := mockStableHash64(candidate, "mem-base")%20 + mockStableHash64(candidate, "mem-burst")%36
		if uid == "" || weight > best {
			uid, best = candidate, weight
		}
	}
	cluster := newMockKubernetesTestCluster(1, 1)
	cluster.Pods[0].UID = uid
	cluster.Pods[0].OwnerKind = "StatefulSet"
	cluster.Pods[0].QoSClass = "Guaranteed"

	start := time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)
	peak := 0.0
	for tick := 0; tick < 288; tick++ {
		initializeMockKubernetesClusterUsage(&cluster, start.Add(time.Duration(tick)*5*time.Minute), false)
		peak = math.Max(peak, cluster.Pods[0].UsageMemoryPercent)
	}
	if peak >= mockKubernetesNodePodMemoryCeiling*100-0.01 {
		t.Fatalf("lone pod %s peaked at %.2f%%: the ceiling clipped a steady-state reading", uid, peak)
	}
}

func TestGenerateNodesBoundsDirectConfiguration(t *testing.T) {
	cfg := DefaultConfig
	cfg.NodeCount = 1 << 30
	if got := len(generateNodes(cfg)); got != maxMockNodeCount {
		t.Fatalf("generateNodes() count = %d, want max %d", got, maxMockNodeCount)
	}

	cfg.NodeCount = -1
	if got := len(generateNodes(cfg)); got != minMockNodeCount {
		t.Fatalf("generateNodes() count = %d, want min %d", got, minMockNodeCount)
	}
}

func TestGenerateNodeIncludesProxmoxNetworkInventory(t *testing.T) {
	node := generateNode("pve1", false, DefaultConfig)
	if len(node.NetworkInterfaces) != 2 {
		t.Fatalf("network interface count = %d, want 2", len(node.NetworkInterfaces))
	}
	if node.NetworkInterfaces[0].Name != "eno1" || node.NetworkInterfaces[1].Name != "vmbr0" {
		t.Fatalf("unexpected network interfaces: %+v", node.NetworkInterfaces)
	}
	if len(node.NetworkInterfaces[1].Addresses) != 2 {
		t.Fatalf("bridge addresses = %v, want IPv4 and IPv6", node.NetworkInterfaces[1].Addresses)
	}
}

func TestBuildFixtureStateIncludesKubernetesStorageInventory(t *testing.T) {
	cfg := DefaultConfig
	cfg.K8sClusterCount = 1
	cfg.K8sNodesPerCluster = 3
	cfg.K8sPodsPerCluster = 8
	cfg.RandomMetrics = false

	data := buildFixtureState(cfg)
	if len(data.KubernetesClusters) != 1 {
		t.Fatalf("expected exactly one kubernetes cluster, got %d", len(data.KubernetesClusters))
	}

	cluster := data.KubernetesClusters[0]
	if len(cluster.StorageClasses) == 0 {
		t.Fatal("expected kubernetes storage class inventory")
	}
	if len(cluster.PersistentVolumes) == 0 {
		t.Fatal("expected kubernetes persistent volume inventory")
	}
	if len(cluster.PersistentVolumeClaims) == 0 {
		t.Fatal("expected kubernetes persistent volume claim inventory")
	}
}

func TestBuildFixtureStateIncludesKubernetesNetworkingInventory(t *testing.T) {
	cfg := DefaultConfig
	cfg.K8sClusterCount = 1
	cfg.K8sNodesPerCluster = 3
	cfg.K8sPodsPerCluster = 8
	cfg.RandomMetrics = false

	data := buildFixtureState(cfg)
	if len(data.KubernetesClusters) != 1 {
		t.Fatalf("expected exactly one kubernetes cluster, got %d", len(data.KubernetesClusters))
	}

	cluster := data.KubernetesClusters[0]
	if len(cluster.Services) == 0 {
		t.Fatal("expected kubernetes service inventory")
	}
	foundPublishedService := false
	for _, service := range cluster.Services {
		if service.ServiceType != "NodePort" || len(service.ExternalIPs) == 0 {
			continue
		}
		for _, port := range service.Ports {
			if port.NodePort > 0 {
				foundPublishedService = true
			}
		}
	}
	if !foundPublishedService {
		t.Fatal("expected kubernetes service inventory to include node-port/external service metadata")
	}
	if len(cluster.Ingresses) == 0 {
		t.Fatal("expected kubernetes ingress inventory")
	}
	if len(cluster.EndpointSlices) == 0 {
		t.Fatal("expected kubernetes endpoint slice inventory")
	}
}

func TestMockStateIncludesHostAgents(t *testing.T) {
	mustSetEnabled(t, true)
	t.Cleanup(func() {
		mustSetEnabled(t, false)
	})

	state := CurrentFixtureGraph().State
	if len(state.Hosts) == 0 {
		t.Fatalf("expected hosts in mock state, got %d", len(state.Hosts))
	}

	frontend := state.ToFrontend()
	if frontend.Resources == nil {
		t.Fatal("expected canonical frontend resources slice to be initialized")
	}
	if frontend.ConnectedInfrastructure == nil {
		t.Fatal("expected canonical connectedInfrastructure slice to be initialized")
	}
}

func TestUpdateMetricsMaintainsServiceHealth(t *testing.T) {
	cfg := DefaultConfig
	cfg.DockerHostCount = 3
	cfg.DockerContainersPerHost = 6

	data := buildFixtureState(cfg)
	updateFixtureStateMetrics(&data, cfg)

	for _, host := range data.DockerHosts {
		if len(host.Services) == 0 {
			continue
		}
		if host.Swarm == nil {
			t.Fatalf("expected swarm metadata for host %s after update", host.ID)
		}

		for _, svc := range host.Services {
			if svc.DesiredTasks < 0 {
				t.Fatalf("service %s has negative desired tasks", svc.Name)
			}
			if svc.RunningTasks < 0 {
				t.Fatalf("service %s has negative running tasks", svc.Name)
			}
			if svc.RunningTasks > svc.DesiredTasks && svc.DesiredTasks > 0 {
				t.Fatalf("service %s has running (%d) > desired (%d)", svc.Name, svc.RunningTasks, svc.DesiredTasks)
			}
		}
	}
}

func TestBuildFixtureStateIncludesPMGInstances(t *testing.T) {
	cfg := DefaultConfig

	data := buildFixtureState(cfg)

	if len(data.PMGInstances) == 0 {
		t.Fatalf("expected PMG instances in mock data")
	}

	for _, inst := range data.PMGInstances {
		if inst.Name == "" {
			t.Fatalf("PMG instance missing name: %+v", inst)
		}
		if inst.Status == "" {
			t.Fatalf("PMG instance missing status: %+v", inst)
		}
	}
}

func TestCloneStateCopiesPMGInstances(t *testing.T) {
	state := models.StateSnapshot{
		PMGInstances: []models.PMGInstance{
			{ID: "pmg-test", Name: "pmg-test", Status: "online"},
		},
	}

	cloned := cloneState(state)

	if len(cloned.PMGInstances) != 1 {
		t.Fatalf("expected cloned state to include PMG instances, got %d", len(cloned.PMGInstances))
	}

	cloned.PMGInstances[0].Name = "modified"
	if state.PMGInstances[0].Name == "modified" {
		t.Fatal("expected PMG instances to be deep-copied")
	}
}

func TestCloneStatePreservesIgnoredInfrastructureEntries(t *testing.T) {
	state := models.StateSnapshot{
		RemovedDockerHosts: []models.RemovedDockerHost{
			{ID: "docker-1", Hostname: "docker.local"},
		},
		RemovedHostAgents: []models.RemovedHostAgent{
			{ID: "agent-1", Hostname: "host.local"},
		},
		RemovedKubernetesClusters: []models.RemovedKubernetesCluster{
			{ID: "cluster-1", Name: "cluster.local"},
		},
	}

	cloned := cloneState(state)

	if len(cloned.RemovedDockerHosts) != 1 {
		t.Fatalf("expected cloned state to include removed docker hosts, got %d", len(cloned.RemovedDockerHosts))
	}
	if len(cloned.RemovedHostAgents) != 1 {
		t.Fatalf("expected cloned state to include removed host agents, got %d", len(cloned.RemovedHostAgents))
	}
	if len(cloned.RemovedKubernetesClusters) != 1 {
		t.Fatalf("expected cloned state to include removed kubernetes clusters, got %d", len(cloned.RemovedKubernetesClusters))
	}

	cloned.RemovedDockerHosts[0].Hostname = "mutated-docker.local"
	cloned.RemovedHostAgents[0].Hostname = "mutated-host.local"
	cloned.RemovedKubernetesClusters[0].Name = "mutated-cluster.local"

	if state.RemovedDockerHosts[0].Hostname == "mutated-docker.local" {
		t.Fatal("expected removed docker hosts to be copied")
	}
	if state.RemovedHostAgents[0].Hostname == "mutated-host.local" {
		t.Fatal("expected removed host agents to be copied")
	}
	if state.RemovedKubernetesClusters[0].Name == "mutated-cluster.local" {
		t.Fatal("expected removed kubernetes clusters to be copied")
	}
}

func TestBuildFixtureStateInitializesIgnoredInfrastructureSlices(t *testing.T) {
	data := buildFixtureState(DefaultConfig)

	if data.RemovedDockerHosts == nil {
		t.Fatal("expected buildFixtureState to initialize RemovedDockerHosts")
	}
	if data.RemovedHostAgents == nil {
		t.Fatal("expected buildFixtureState to initialize RemovedHostAgents")
	}
	if data.RemovedKubernetesClusters == nil {
		t.Fatal("expected buildFixtureState to initialize RemovedKubernetesClusters")
	}
}

// collectFixtureIdentities gathers every identity-bearing ID the fixture
// graph produces, keyed by category. Metric series resource IDs in the mock
// metrics store derive from these, so any per-boot churn here orphans stored
// history and breaks resource continuity across backend restarts.
func collectFixtureIdentities(graph FixtureGraph) map[string][]string {
	ids := map[string][]string{}
	add := func(category, id string) {
		ids[category] = append(ids[category], id)
	}

	state := graph.State
	for _, node := range state.Nodes {
		add("node", node.ID)
	}
	for _, vm := range state.VMs {
		add("vm", vm.ID)
	}
	for _, ct := range state.Containers {
		add("container", ct.ID)
	}
	for _, host := range state.Hosts {
		add("host", host.ID)
		add("host-hostname", host.Hostname)
		add("host-machine-id", host.ID+"="+host.MachineID)
	}
	for _, dockerHost := range state.DockerHosts {
		add("docker-host", dockerHost.ID)
		add("docker-host-machine-id", dockerHost.ID+"="+dockerHost.MachineID)
		add("docker-agent-id", dockerHost.ID+"="+dockerHost.AgentID)
		for _, c := range dockerHost.Containers {
			add("docker-container", c.ID)
		}
	}
	for _, cluster := range state.KubernetesClusters {
		add("k8s-cluster", cluster.ID)
		for _, node := range cluster.Nodes {
			add("k8s-node", node.UID)
		}
		for _, pod := range cluster.Pods {
			add("k8s-pod", pod.UID)
		}
		for _, deployment := range cluster.Deployments {
			add("k8s-deployment", deployment.UID)
		}
		for _, svc := range cluster.Services {
			add("k8s-service", svc.UID)
		}
	}
	for _, storage := range state.Storage {
		add("storage", storage.ID)
	}
	for _, ceph := range state.CephClusters {
		add("ceph-cluster", ceph.ID)
		add("ceph-fsid", ceph.FSID)
	}
	for _, disk := range state.PhysicalDisks {
		add("physical-disk", disk.ID)
	}

	resources, _ := graph.UnifiedResourceSnapshot()
	for _, resource := range resources {
		if resource.Type == unifiedresources.ResourceTypeAgent {
			add("canonical-agent", resource.ID)
		}
	}

	for category := range ids {
		sort.Strings(ids[category])
	}
	return ids
}

// TestFixtureIdentityStableAcrossBoots simulates two backend boots (the
// package-global rand source is in a different state for each build, exactly
// as two freshly seeded processes would be) and requires every identity set
// to match. Guards the generator's "stable IDs" contract end to end.
func TestFixtureIdentityStableAcrossBoots(t *testing.T) {
	now := time.Date(2026, time.June, 10, 12, 0, 0, 0, time.UTC)

	first := collectFixtureIdentities(buildFixtureGraph(DefaultConfig, now))
	second := collectFixtureIdentities(buildFixtureGraph(DefaultConfig, now))
	if len(first["canonical-agent"]) == 0 || len(second["canonical-agent"]) == 0 {
		t.Fatal("fixture graph must project canonical agents before comparing their identities")
	}

	categories := map[string]struct{}{}
	for category := range first {
		categories[category] = struct{}{}
	}
	for category := range second {
		categories[category] = struct{}{}
	}

	for category := range categories {
		diff := diffIdentitySets(first[category], second[category])
		if diff != "" {
			t.Errorf("%s identities churned across boots:\n%s", category, diff)
		}
	}
}

func diffIdentitySets(first, second []string) string {
	firstSet := make(map[string]struct{}, len(first))
	for _, id := range first {
		firstSet[id] = struct{}{}
	}
	secondSet := make(map[string]struct{}, len(second))
	for _, id := range second {
		secondSet[id] = struct{}{}
	}

	var lines []string
	for _, id := range first {
		if _, ok := secondSet[id]; !ok {
			lines = append(lines, fmt.Sprintf("  only in first boot:  %s", id))
		}
	}
	for _, id := range second {
		if _, ok := firstSet[id]; !ok {
			lines = append(lines, fmt.Sprintf("  only in second boot: %s", id))
		}
	}

	if len(lines) == 0 {
		return ""
	}
	sort.Strings(lines)
	out := ""
	for _, line := range lines {
		out += line + "\n"
	}
	return out
}

// Host-relative guest memory history is a byte series derived from fixture
// capacity, so the fixture registries must expose that capacity per guest
// without a caller cloning the whole graph on every lookup.
func TestMemoryTotalForResourceTracksFixtureCapacity(t *testing.T) {
	mustSetEnabled(t, true)
	t.Cleanup(func() {
		mustSetEnabled(t, false)
	})

	graph := CurrentFixtureGraph()

	checked := 0
	for _, vm := range graph.State.VMs {
		if vm.ID == "" || vm.Memory.Total <= 0 {
			continue
		}
		if got := MemoryTotalForResource("vm", vm.ID); got != float64(vm.Memory.Total) {
			t.Fatalf("MemoryTotalForResource(vm, %s) = %f, want %d", vm.ID, got, vm.Memory.Total)
		}
		checked++
		if checked >= 3 {
			break
		}
	}
	if checked == 0 {
		t.Fatal("fixture graph has no VM with a memory total")
	}

	for _, ct := range graph.State.Containers {
		if ct.ID == "" || ct.Memory.Total <= 0 {
			continue
		}
		if got := MemoryTotalForResource("container", ct.ID); got != float64(ct.Memory.Total) {
			t.Fatalf("MemoryTotalForResource(container, %s) = %f, want %d", ct.ID, got, ct.Memory.Total)
		}
		break
	}

	if got := MemoryTotalForResource("vm", "definitely-not-a-fixture-guest"); got != 0 {
		t.Fatalf("unknown guest capacity = %f, want 0", got)
	}
	if got := MemoryTotalForResource("vm", "   "); got != 0 {
		t.Fatalf("blank guest capacity = %f, want 0", got)
	}
}

func TestEnsureConfigOnlyMountFixtureSeedsUnknownUsageMount(t *testing.T) {
	data := buildFixtureState(DefaultConfig)

	var fixtureContainer *models.Container
	for i := range data.Containers {
		for _, disk := range data.Containers[i].Disks {
			if disk.Mountpoint == "/srv/archive" {
				fixtureContainer = &data.Containers[i]
			}
		}
	}
	if fixtureContainer == nil {
		t.Fatal("no container carries the config-only mount fixture (#1477)")
	}
	if fixtureContainer.Status != "running" {
		t.Fatalf("fixture container %s is %s, want running", fixtureContainer.Name, fixtureContainer.Status)
	}

	var rootfs, archive *models.Disk
	for i := range fixtureContainer.Disks {
		disk := &fixtureContainer.Disks[i]
		switch disk.Mountpoint {
		case "/":
			rootfs = disk
		case "/srv/archive":
			archive = disk
		}
	}
	if rootfs == nil || rootfs.Total <= 0 || rootfs.Usage < 0 {
		t.Fatalf("fixture container must keep a measured rootfs row, got %+v", rootfs)
	}
	if archive == nil {
		t.Fatal("config-only mount row missing")
	}
	if archive.Usage != -1 {
		t.Fatalf("config-only mount usage = %f, want the -1 unknown sentinel", archive.Usage)
	}
	if archive.Total != int64(20)*1024*1024*1024 {
		t.Fatalf("config-only mount total = %d, want the configured 20 GiB", archive.Total)
	}
	if archive.Type != "mp0" || archive.Device == "" {
		t.Fatalf("config-only mount must keep its mp key and device, got %+v", archive)
	}
}

func assertMockDockerInLXCFixture(t *testing.T, data *models.StateSnapshot) []models.DockerHost {
	t.Helper()

	var nested []models.DockerHost
	for _, host := range data.DockerHosts {
		if strings.HasPrefix(host.ID, mockProxmoxLXCDockerHostSourcePrefix) {
			nested = append(nested, host)
		}
	}
	if len(nested) != 2 {
		t.Fatalf("expected 2 nested Docker-in-LXC hosts, got %d", len(nested))
	}

	guestsBySourceID := make(map[string]models.Container, len(data.Containers))
	for _, ct := range data.Containers {
		key := fmt.Sprintf("%s%s:%s:%d", mockProxmoxLXCDockerHostSourcePrefix, ct.Instance, ct.Node, ct.VMID)
		guestsBySourceID[key] = ct
	}

	containerCounts := make(map[int]bool)
	for _, host := range nested {
		guest, ok := guestsBySourceID[host.ID]
		if !ok {
			t.Fatalf("nested docker host %s has no matching LXC guest", host.ID)
		}
		if guest.Status != "running" {
			t.Fatalf("nested docker host %s bound to non-running guest %s", host.ID, guest.ID)
		}
		if !guest.HasDocker || guest.DockerCheckedAt.IsZero() {
			t.Fatalf("parent LXC %s not marked as probed Docker host", guest.ID)
		}
		if host.AgentID != host.ID {
			t.Fatalf("nested docker host agent id %q must equal its source id %q", host.AgentID, host.ID)
		}
		if host.Hostname != guest.Name {
			t.Fatalf("nested docker host hostname %q should be the guest name %q", host.Hostname, guest.Name)
		}
		if host.Status != "online" {
			t.Fatalf("nested docker host %s bound to a running guest must be online, got %q", host.ID, host.Status)
		}
		if len(host.Containers) == 0 {
			t.Fatalf("nested docker host %s has no containers", host.ID)
		}
		containerCounts[len(host.Containers)] = true
		if len(host.Images) != 0 || len(host.Volumes) != 0 || len(host.Networks) != 0 || host.StorageUsage != nil {
			t.Fatalf("nested docker host %s must not fabricate native engine inventory (production pct exec inventory has none)", host.ID)
		}
		if host.Swarm != nil {
			t.Fatalf("nested docker host %s must not join the mock Swarm", host.ID)
		}
		if !data.ConnectionHealth[dockerConnectionPrefix+host.ID] {
			t.Fatalf("nested docker host %s missing connection health entry", host.ID)
		}
	}
	if !containerCounts[1] {
		t.Fatalf("expected one single-container nested host so the row cue renders its singular form, got counts %v", containerCounts)
	}
	return nested
}

func TestFixtureGraphIncludesDockerInLXCFixture(t *testing.T) {
	cfg := DefaultConfig
	now := time.Now()

	graph := buildFixtureGraph(cfg, now)
	first := assertMockDockerInLXCFixture(t, &graph.State)

	// The demo scenario layer reapplies on every metric tick; the fixture
	// must stay idempotent and the scenario's Docker fleet profiles must
	// keep their hands off the nested hosts.
	graph.UpdateMetrics(cfg, now.Add(time.Minute))
	second := assertMockDockerInLXCFixture(t, &graph.State)
	firstIDs := map[string]bool{}
	for _, host := range first {
		firstIDs[host.ID] = true
	}
	for _, host := range second {
		if !firstIDs[host.ID] {
			t.Fatalf("nested docker host identity changed across ticks: %s not in first pass", host.ID)
		}
	}

	// Disabling Docker in the mock estate disables the nested fixture too.
	cfgNoDocker := DefaultConfig
	cfgNoDocker.DockerHostCount = 0
	graphNoDocker := buildFixtureGraph(cfgNoDocker, now)
	for _, host := range graphNoDocker.State.DockerHosts {
		if strings.HasPrefix(host.ID, mockProxmoxLXCDockerHostSourcePrefix) {
			t.Fatalf("nested docker host %s generated despite DockerHostCount=0", host.ID)
		}
	}
	for _, ct := range graphNoDocker.State.Containers {
		if ct.HasDocker {
			t.Fatalf("LXC %s marked HasDocker despite DockerHostCount=0", ct.ID)
		}
	}
}

// assertMockBackupRunningFixture verifies the running-backup estate invariants
// and returns the set of flagged guest keys ("vm-<vmid>" / "ct-<vmid>").
func assertMockBackupRunningFixture(t *testing.T, state *models.StateSnapshot, now time.Time) map[string]bool {
	t.Helper()

	flagged := map[string]bool{}
	neverBackedUp := false
	backedUp := false
	for _, vm := range state.VMs {
		if !vm.BackupInProgress {
			continue
		}
		flagged[fmt.Sprintf("vm-%d", vm.VMID)] = true
		if vm.LastBackup.IsZero() {
			neverBackedUp = true
		} else {
			backedUp = true
		}
	}
	for _, ct := range state.Containers {
		if !ct.BackupInProgress {
			continue
		}
		flagged[fmt.Sprintf("ct-%d", ct.VMID)] = true
		if ct.LastBackup.IsZero() {
			neverBackedUp = true
		} else {
			backedUp = true
		}
	}
	if len(flagged) == 0 {
		t.Fatal("no guest flagged BackupInProgress")
	}
	if !neverBackedUp || !backedUp {
		t.Fatalf("running backups must spread across the age distribution; never=%v backed-up=%v", neverBackedUp, backedUp)
	}

	guests := len(state.VMs) + len(state.Containers)
	if len(flagged) > guests/10 {
		t.Fatalf("%d of %d guests flagged running; expected a small fraction", len(flagged), guests)
	}

	// The first-cycle showcase guests are pinned to specific protection
	// stories (failed run, no evidence at all); a running backup would mask
	// those states, so the fixture must never pick them.
	showcase := map[string]bool{}
	for _, profile := range demoVMProfiles {
		showcase[profile.Name] = true
	}
	for _, profile := range demoContainerProfiles {
		showcase[profile.Name] = true
	}
	for _, vm := range state.VMs {
		if vm.BackupInProgress && showcase[vm.Name] {
			t.Fatalf("showcase guest %s flagged BackupInProgress", vm.Name)
		}
	}
	for _, ct := range state.Containers {
		if ct.BackupInProgress && showcase[ct.Name] {
			t.Fatalf("showcase guest %s flagged BackupInProgress", ct.Name)
		}
	}

	inflight := map[string]bool{}
	for _, backup := range state.PBSBackups {
		if !strings.HasPrefix(backup.ID, mockBackupRunningIDPrefix) {
			if backup.InProgress {
				t.Fatalf("non-fixture PBS backup %s marked InProgress", backup.ID)
			}
			continue
		}
		key := fmt.Sprintf("%s-%s", backup.BackupType, backup.VMID)
		if inflight[key] {
			t.Fatalf("duplicate in-flight PBS backup for %s", key)
		}
		inflight[key] = true
		if !flagged[key] {
			t.Fatalf("in-flight PBS backup %s has no flagged guest", backup.ID)
		}
		if !backup.InProgress || backup.Size != 0 || backup.Verified {
			t.Fatalf("in-flight PBS backup %s must be InProgress with no size and no verification: %+v", backup.ID, backup)
		}
		wantBlob := "qemu-server.conf.blob"
		if backup.BackupType == "ct" {
			wantBlob = "pct.conf.blob"
		}
		if len(backup.Files) != 1 || backup.Files[0] != wantBlob {
			t.Fatalf("in-flight PBS backup %s files = %v, want only %s", backup.ID, backup.Files, wantBlob)
		}
		if age := now.Sub(backup.BackupTime); age <= 0 || age > time.Hour {
			t.Fatalf("in-flight PBS backup %s should have started minutes ago, started %s before now", backup.ID, age)
		}
	}
	if len(inflight) != len(flagged) {
		t.Fatalf("flagged guests = %d but in-flight PBS backups = %d", len(flagged), len(inflight))
	}

	runningTasks := map[string]bool{}
	for _, task := range state.PVEBackups.BackupTasks {
		if !strings.HasPrefix(task.ID, mockBackupRunningIDPrefix) {
			continue
		}
		kind := "vm"
		if task.Type == "lxc" {
			kind = "ct"
		}
		key := fmt.Sprintf("%s-%d", kind, task.VMID)
		if runningTasks[key] {
			t.Fatalf("duplicate running task for %s", key)
		}
		runningTasks[key] = true
		if !flagged[key] {
			t.Fatalf("running task %s has no flagged guest", task.ID)
		}
		if task.Status != "running" || !task.EndTime.IsZero() {
			t.Fatalf("fixture task %s must be running with no end time: %+v", task.ID, task)
		}
		if task.ObservedAt.Before(now.Add(-time.Minute)) {
			t.Fatalf("fixture task %s ObservedAt %s is stale relative to %s", task.ID, task.ObservedAt, now)
		}
	}
	if len(runningTasks) != len(flagged) {
		t.Fatalf("flagged guests = %d but running tasks = %d", len(flagged), len(runningTasks))
	}

	// The canonical Backups projection must mirror the fixture artifacts.
	mirrored := 0
	for _, backup := range state.Backups.PBS {
		if strings.HasPrefix(backup.ID, mockBackupRunningIDPrefix) {
			mirrored++
		}
	}
	if mirrored != len(inflight) {
		t.Fatalf("state.Backups.PBS mirrors %d in-flight backups, want %d", mirrored, len(inflight))
	}

	return flagged
}

func TestFixtureGraphIncludesBackupRunningFixture(t *testing.T) {
	cfg := DefaultConfig
	now := time.Now()

	graph := buildFixtureGraph(cfg, now)
	first := assertMockBackupRunningFixture(t, &graph.State, now)

	// The demo scenario layer reapplies on every metric tick; the fixture
	// must re-select the same guests and replace - not duplicate - its
	// in-flight artifacts.
	tick := now.Add(time.Minute)
	graph.UpdateMetrics(cfg, tick)
	second := assertMockBackupRunningFixture(t, &graph.State, tick)

	if len(first) != len(second) {
		t.Fatalf("running-guest selection changed size across ticks: %d then %d", len(first), len(second))
	}
	for key := range second {
		if !first[key] {
			t.Fatalf("running-guest selection changed across ticks: %s not in first pass", key)
		}
	}
}

func replicationFixtureEstate() ([]models.Node, []models.VM) {
	nodes := []models.Node{{Name: "pve1"}, {Name: "pve2"}, {Name: "pve3"}}
	vms := make([]models.VM, 0, 9)
	for i := 0; i < 9; i++ {
		// Every guest lives on pve1, so the old (i+1) modulo target picked the
		// source node for every third job.
		vms = append(vms, models.VM{VMID: 100 + i, Name: "guest", Node: "pve1", Instance: "cluster"})
	}
	return nodes, vms
}

func TestGenerateReplicationJobsNeverTargetsTheSourceNode(t *testing.T) {
	nodes, vms := replicationFixtureEstate()

	jobs := generateReplicationJobs(nodes, vms)
	if len(jobs) == 0 {
		t.Fatal("expected replication jobs for a three-node estate")
	}
	for _, job := range jobs {
		if job.TargetNode == job.SourceNode {
			t.Fatalf("job %s replicates %s to itself", job.JobID, job.SourceNode)
		}
	}
}

func TestGenerateReplicationJobsNeedsASecondNode(t *testing.T) {
	_, vms := replicationFixtureEstate()

	if jobs := generateReplicationJobs([]models.Node{{Name: "pve1"}}, vms); len(jobs) != 0 {
		t.Fatalf("expected no replication jobs without a target node, got %d", len(jobs))
	}
}

func TestRebaseMockReplicationJobsKeepsJobsOnSchedule(t *testing.T) {
	nodes, vms := replicationFixtureEstate()
	jobs := generateReplicationJobs(nodes, vms)
	jobs[0].FailCount = 2
	jobs[0].LastSyncStatus = "error"
	for i := 1; i < len(jobs); i++ {
		jobs[i].FailCount = 0
		jobs[i].LastSyncStatus = "ok"
	}

	start := time.Date(2026, 10, 2, 9, 7, 0, 0, time.UTC)
	// A long-running demo used to leave every job hours overdue; walk the
	// fixture clock well past one schedule window.
	for _, now := range []time.Time{start, start.Add(47 * time.Minute), start.Add(26 * time.Hour)} {
		rebaseMockReplicationJobs(jobs, now)
		for i, job := range jobs {
			if job.LastSyncTime == nil || job.NextSyncTime == nil {
				t.Fatalf("job %s lost its sync times", job.JobID)
			}
			if job.LastSyncUnix != job.LastSyncTime.Unix() || job.NextSyncUnix != job.NextSyncTime.Unix() {
				t.Fatalf("job %s unix companions drifted from its sync times", job.JobID)
			}
			if !job.NextSyncTime.After(now) || job.NextSyncTime.Sub(now) > mockReplicationInterval {
				t.Fatalf("job %s next sync %s is not within one interval after %s", job.JobID, job.NextSyncTime, now)
			}
			age := now.Sub(*job.LastSyncTime)
			if i == 0 {
				if age < 2*mockReplicationInterval {
					t.Fatalf("failing job last synced %s ago, want at least two missed intervals", age)
				}
				continue
			}
			if age < 0 || age >= mockReplicationInterval {
				t.Fatalf("healthy job %s last synced %s ago, want inside one interval", job.JobID, age)
			}
		}
	}
}

func TestRebaseMockReplicationJobsLeavesUnchangedJobsAlone(t *testing.T) {
	nodes, vms := replicationFixtureEstate()
	jobs := generateReplicationJobs(nodes, vms)
	start := time.Date(2026, 10, 2, 9, 0, 0, 0, time.UTC)
	rebaseMockReplicationJobs(jobs, start)
	before := append([]models.ReplicationJob(nil), jobs...)

	// One second later no job's phase can have passed for all of them, and a
	// job whose phase has not passed must keep its polled stamp and pointers so
	// the refresh does not manufacture a change for every job on every tick.
	rebaseMockReplicationJobs(jobs, start.Add(time.Second))
	unchanged := 0
	for i := range jobs {
		if jobs[i].LastSyncUnix != before[i].LastSyncUnix {
			continue
		}
		unchanged++
		if !jobs[i].LastPolled.Equal(before[i].LastPolled) || jobs[i].LastSyncTime != before[i].LastSyncTime {
			t.Fatalf("job %s was rewritten although its schedule phase had not passed", jobs[i].JobID)
		}
	}
	if unchanged < len(jobs)-1 {
		t.Fatalf("expected at most one job to roll over in one second, %d of %d stayed put", unchanged, len(jobs))
	}
}

func TestBuildFixtureStateMergesSharedPBSStorageLikeThePoller(t *testing.T) {
	cfg := DefaultConfig
	cfg.NodeCount = 5

	data := buildFixtureState(cfg)

	// Issue 1049: the poller folds a shared storage reported by every node
	// into one cluster-scoped row. The fixture must not fan it back out into
	// one row per node, each with its own usage figure.
	seen := map[string]models.Storage{}
	for _, storage := range data.Storage {
		if storage.Type != "pbs" {
			continue
		}
		key := storage.Instance + "/" + storage.Name
		if _, dup := seen[key]; dup {
			t.Fatalf("shared PBS storage %s appears more than once", key)
		}
		seen[key] = storage
		if storage.Node != "cluster" || storage.NodeCount < 2 || len(storage.Nodes) != storage.NodeCount {
			t.Fatalf("shared PBS storage %s is not cluster-scoped: node=%q nodes=%v count=%d", key, storage.Node, storage.Nodes, storage.NodeCount)
		}
		if !strings.HasSuffix(storage.ID, "-cluster-"+storage.Name) {
			t.Fatalf("shared PBS storage %s keeps a node-scoped id %q", key, storage.ID)
		}
	}
	if len(seen) == 0 {
		t.Fatal("expected shared PBS storage in the fixture")
	}
}

// Mock guests report where their memory reading came from, the way a live
// estate does, so the UI qualifies mock readings instead of treating every
// guest as unannotated ("Freshness unknown").
func TestGeneratedGuestMemoryProvenanceMatchesPowerStateAndOS(t *testing.T) {
	cfg := DefaultConfig
	cfg.StoppedPercent = 0.3
	now := time.Now()

	sources := map[string]int{}
	for i := 0; i < 300; i++ {
		vm := generateVM("node-01", "mock-cluster", 1000+i, cfg)
		obs := vm.Memory.Observation
		if vm.Status != "running" {
			if obs.State != "unavailable" || obs.Source != "powered-off" {
				t.Fatalf("stopped VM observation = %+v, want unavailable powered-off", obs)
			}
			continue
		}
		sources[obs.Source]++
		if obs.State != "current" || obs.ObservedAt.IsZero() || obs.ObservedAt.After(now.Add(time.Minute)) {
			t.Fatalf("running VM observation = %+v, want a current reading observed now", obs)
		}
		switch obs.Source {
		case "status-mem":
			// Proxmox's value carries no used | cache | free split.
			if vm.Memory.Cache != 0 {
				t.Fatalf("cache-inclusive VM reported a measured cache split: %+v", vm.Memory)
			}
		case "guest-agent-meminfo":
			// The QEMU guest agent reads Linux /proc/meminfo only.
			if strings.Contains(strings.ToLower(vm.OSName), "windows") {
				t.Fatalf("Windows VM %q claims Linux meminfo provenance", vm.OSName)
			}
		case "agent":
		default:
			t.Fatalf("running VM has unexpected memory source %q", obs.Source)
		}
	}
	if sources["status-mem"] == 0 || sources["guest-agent-meminfo"] == 0 || sources["agent"] == 0 {
		t.Fatalf("expected cache-inclusive, guest-agent and Pulse-agent VMs in the estate, got %v", sources)
	}

	sources = map[string]int{}
	for i := 0; i < 300; i++ {
		ct := generateContainer("node-01", "mock-cluster", 2000+i, cfg)
		obs := ct.Memory.Observation
		if ct.Status != "running" {
			if obs.State != "unavailable" || obs.Source != "powered-off" {
				t.Fatalf("stopped container observation = %+v, want unavailable powered-off", obs)
			}
			continue
		}
		sources[obs.Source]++
		if obs.State != "current" || obs.ObservedAt.IsZero() {
			t.Fatalf("running container observation = %+v, want a current reading", obs)
		}
		if obs.Source != "cluster-resources" && obs.Source != "agent" {
			t.Fatalf("running container has unexpected memory source %q", obs.Source)
		}
	}
	if sources["cluster-resources"] == 0 || sources["agent"] == 0 {
		t.Fatalf("expected cgroup and Pulse-agent containers in the estate, got %v", sources)
	}
}

func TestMockGuestMemoryObservationFollowsScenarioPowerState(t *testing.T) {
	cases := []struct {
		name    string
		before  models.MemoryObservation
		running bool
		want    models.MemoryObservation
	}{
		{
			name:    "forced stopped loses its reading",
			before:  models.MemoryObservation{State: "current", Source: "status-mem", ObservedAt: time.Now()},
			running: false,
			want:    models.MemoryObservation{State: "unavailable", Source: "powered-off"},
		},
		{
			name:    "forced running after generation gets a current source",
			before:  models.MemoryObservation{State: "unavailable", Source: "powered-off"},
			running: true,
			want:    models.MemoryObservation{State: "current", Source: "agent"},
		},
		{
			name:    "a running guest keeps its own source",
			before:  models.MemoryObservation{State: "current", Source: "status-mem", ObservedAt: time.Now()},
			running: true,
			want:    models.MemoryObservation{State: "current", Source: "status-mem"},
		},
		{
			name:    "a retained reading on a running guest is kept",
			before:  models.MemoryObservation{State: "last-known", Source: "status-mem", ObservedAt: time.Now().Add(-time.Hour)},
			running: true,
			want:    models.MemoryObservation{State: "last-known", Source: "status-mem"},
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			mem := models.Memory{Observation: tc.before}
			mockSettleGuestMemoryObservation(&mem, tc.running, "agent")
			got := mem.Observation
			if got.State != tc.want.State || got.Source != tc.want.Source {
				t.Fatalf("observation = %+v, want state %q source %q", got, tc.want.State, tc.want.Source)
			}
			if tc.want.State == "current" && got.ObservedAt.IsZero() {
				t.Fatalf("a current observation needs its observation time: %+v", got)
			}
		})
	}
}

func TestFixtureRefreshMovesCurrentGuestMemoryObservationTime(t *testing.T) {
	for _, randomMetrics := range []bool{true, false} {
		name := "random metrics"
		if !randomMetrics {
			name = "static metrics"
		}
		t.Run(name, func(t *testing.T) {
			cfg := DefaultConfig
			cfg.RandomMetrics = randomMetrics
			stale := time.Now().Add(-time.Hour)
			refreshAt := time.Now().Truncate(time.Second)

			data := &models.StateSnapshot{
				VMs: []models.VM{
					{ID: "vm-current", Node: "node-01", Status: "running", Memory: models.Memory{
						Total: 100, Observation: models.MemoryObservation{State: "current", Source: "status-mem", ObservedAt: stale},
					}},
					{ID: "vm-retained", Node: "node-01", Status: "running", Memory: models.Memory{
						Total: 100, Observation: models.MemoryObservation{State: "last-known", Source: "status-mem", ObservedAt: stale},
					}},
					{ID: "vm-stopped", Node: "node-01", Status: "stopped", Memory: models.Memory{
						Total: 100, Observation: models.MemoryObservation{State: "unavailable", Source: "powered-off"},
					}},
				},
				Containers: []models.Container{
					{ID: "ct-current", Node: "node-01", Status: "running", Memory: models.Memory{
						Total: 100, Observation: models.MemoryObservation{State: "current", Source: "cluster-resources", ObservedAt: stale},
					}},
				},
			}
			updateFixtureStateMetricsAt(data, cfg, refreshAt)

			if got := data.VMs[0].Memory.Observation; !got.ObservedAt.Equal(refreshAt) || got.Source != "status-mem" {
				t.Fatalf("current VM observation = %+v, want source kept and time %v", got, refreshAt)
			}
			if got := data.VMs[1].Memory.Observation; !got.ObservedAt.Equal(stale) || got.State != "last-known" {
				t.Fatalf("a retained observation keeps its own state and time, got %+v", got)
			}
			if got := data.VMs[2].Memory.Observation; !got.ObservedAt.IsZero() || got.Source != "powered-off" {
				t.Fatalf("a stopped guest has no observation time to move, got %+v", got)
			}
			if got := data.Containers[0].Memory.Observation; !got.ObservedAt.Equal(refreshAt) || got.Source != "cluster-resources" {
				t.Fatalf("current container observation = %+v, want source kept and time %v", got, refreshAt)
			}
		})
	}
}