pulse/internal/monitoring/memory_trust_characterization_test.go

package monitoring

import (
	"context"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/mock"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)

type vmMemoryTrustStubClient struct {
	*stubPVEClient
	vms                 []proxmox.VM
	containers          []proxmox.Container
	vmStatus            *proxmox.VMStatus
	vmAgentMemAvailable uint64
	vmRRDCalls          int
	vmAgentMemCalls     int
}

func TestCollectVMsWithNodesRetainsFailedNodeRuntimeState(t *testing.T) {
	monitor := newTestPVEMonitor("lab")
	defer monitor.alertManager.Stop()
	defer monitor.notificationMgr.Stop()
	monitor.state.UpdateVMsForInstance("lab", []models.VM{{
		ID:       "lab:node-b:201",
		VMID:     201,
		Name:     "database",
		Node:     "node-b",
		Instance: "lab",
		Status:   "running",
	}})

	client := &partialNodeGuestClient{
		stubPVEClient: &stubPVEClient{},
		failedNodes:   map[string]bool{"node-b": true},
	}
	vms := monitor.collectVMsWithNodes(
		context.Background(),
		"lab",
		"",
		false,
		client,
		[]proxmox.Node{{Node: "node-b", Status: "online"}},
		map[string]string{"node-b": "online"},
	)

	if len(vms) != 1 || vms[0].ID != "lab:node-b:201" || vms[0].Status != "running" {
		t.Fatalf("failed-node VM continuity = %+v, want retained source ID and running state", vms)
	}
}

func (s *vmMemoryTrustStubClient) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) {
	return s.vms, nil
}

func (s *vmMemoryTrustStubClient) GetContainers(ctx context.Context, node string) ([]proxmox.Container, error) {
	return s.containers, nil
}

func (s *vmMemoryTrustStubClient) GetVMStatus(ctx context.Context, node string, vmid int) (*proxmox.VMStatus, error) {
	return s.vmStatus, nil
}

// GetVMRRDData tracks lookups of the guest RRD endpoint. It is intentionally
// not part of PVEClientInterface anymore: guest rrddata carries no cache-aware
// memory columns (#1634), so the memory resolvers must never consult it.
func (s *vmMemoryTrustStubClient) GetVMRRDData(ctx context.Context, node string, vmid int, timeframe, cf string, ds []string) ([]proxmox.GuestRRDPoint, error) {
	s.vmRRDCalls++
	return nil, nil
}

func (s *vmMemoryTrustStubClient) GetVMMemAvailableFromAgent(ctx context.Context, node string, vmid int) (uint64, error) {
	s.vmAgentMemCalls++
	return s.vmAgentMemAvailable, nil
}

func TestPollPVENodeMemoryTrustCharacterization(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	const gib = uint64(1024 * 1024 * 1024)

	tests := []struct {
		name          string
		nodeStatus    *proxmox.NodeStatus
		rrdPoints     []proxmox.NodeRRDPoint
		wantSource    string
		wantFallback  string
		wantUsed      uint64
		wantRawSource string
		wantRRDUsed   uint64
		wantUnknown   bool
	}{
		{
			name: "missing MemAvailable derives from free+buffers+cached",
			nodeStatus: &proxmox.NodeStatus{
				Memory: &proxmox.MemoryStatus{
					Total:   32 * gib,
					Used:    26 * gib,
					Free:    2 * gib,
					Buffers: 3 * gib,
					Cached:  7 * gib,
				},
			},
			wantSource:    "derived-free-buffers-cached",
			wantFallback:  "",
			wantUsed:      20 * gib,
			wantRawSource: "node-status",
		},
		{
			name: "proxmox 8.4 field drift derives from total-minus-used gap",
			nodeStatus: &proxmox.NodeStatus{
				Memory: &proxmox.MemoryStatus{
					Total: 134794743808,
					Used:  107351023616,
					Free:  6471057408,
				},
			},
			wantSource:    "derived-total-minus-used",
			wantFallback:  "node-status-total-minus-used",
			wantUsed:      107351023616,
			wantRawSource: "node-status-total-minus-used",
		},
		{
			name: "RRD memused wins over MemFree and cache-inclusive status used",
			nodeStatus: &proxmox.NodeStatus{
				Memory: &proxmox.MemoryStatus{
					Total: 8 * gib,
					Used:  15 * gib / 2,
					Free:  gib / 2,
				},
			},
			rrdPoints:     []proxmox.NodeRRDPoint{{MemUsed: floatPtr(float64(6 * gib))}},
			wantSource:    "rrd-memused",
			wantFallback:  "rrd-memused",
			wantUsed:      6 * gib,
			wantRawSource: "rrd-memused",
			wantRRDUsed:   6 * gib,
		},
		{
			name: "explicit available wins over conflicting RRD",
			nodeStatus: &proxmox.NodeStatus{
				Memory: &proxmox.MemoryStatus{
					Total:     8 * gib,
					Used:      15 * gib / 2,
					Free:      gib / 2,
					Available: 2 * gib,
				},
			},
			rrdPoints:     []proxmox.NodeRRDPoint{{MemUsed: floatPtr(float64(7 * gib))}},
			wantSource:    "available-field",
			wantUsed:      6 * gib,
			wantRawSource: "node-status",
		},
		{
			name: "MemFree without cache evidence stays unknown",
			nodeStatus: &proxmox.NodeStatus{
				Memory: &proxmox.MemoryStatus{
					Total: 8 * gib,
					Used:  15 * gib / 2,
					Free:  gib / 2,
				},
			},
			wantSource:    "unavailable",
			wantFallback:  "cache-aware-memory-unavailable",
			wantRawSource: "cache-aware-unavailable",
			wantUnknown:   true,
		},
		{
			name: "conflicting cache components above total stay unknown",
			nodeStatus: &proxmox.NodeStatus{
				Memory: &proxmox.MemoryStatus{
					Total:   8 * gib,
					Used:    4 * gib,
					Free:    4 * gib,
					Buffers: 3 * gib,
					Cached:  3 * gib,
				},
			},
			wantSource:    "unavailable",
			wantFallback:  "cache-aware-memory-unavailable",
			wantRawSource: "cache-aware-unavailable",
			wantUnknown:   true,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			mon := newTestPVEMonitor("test")
			defer mon.alertManager.Stop()
			defer mon.notificationMgr.Stop()

			client := &stubPVEClient{nodeStatus: tt.nodeStatus, rrdPoints: tt.rrdPoints}
			node := proxmox.Node{
				Node:   "node1",
				Status: "online",
				MaxMem: tt.nodeStatus.Memory.Total,
				Mem:    tt.nodeStatus.Memory.Used,
				MaxCPU: 8,
			}

			modelNode, _, _, err := mon.pollPVENode(context.Background(), "test", &mon.config.PVEInstances[0], client, node, "healthy", nil, nil)
			if err != nil {
				t.Fatalf("pollPVENode() error = %v", err)
			}
			if got := uint64(modelNode.Memory.Used); got != tt.wantUsed {
				t.Fatalf("modelNode.Memory.Used = %d, want %d", got, tt.wantUsed)
			}
			if modelNode.Memory.UsageUnavailable != tt.wantUnknown {
				t.Fatalf("modelNode.Memory.UsageUnavailable = %t, want %t", modelNode.Memory.UsageUnavailable, tt.wantUnknown)
			}

			snap := mon.nodeSnapshots[makeNodeSnapshotKey("test", "node1")]
			if snap.MemorySource != tt.wantSource {
				t.Fatalf("snapshot.MemorySource = %q, want %q", snap.MemorySource, tt.wantSource)
			}
			if snap.FallbackReason != tt.wantFallback {
				t.Fatalf("snapshot.FallbackReason = %q, want %q", snap.FallbackReason, tt.wantFallback)
			}
			if snap.Raw.ProxmoxMemorySource != tt.wantRawSource {
				t.Fatalf("snapshot.Raw.ProxmoxMemorySource = %q, want %q", snap.Raw.ProxmoxMemorySource, tt.wantRawSource)
			}
			if snap.Raw.RRDUsed != tt.wantRRDUsed {
				t.Fatalf("snapshot.Raw.RRDUsed = %d, want %d", snap.Raw.RRDUsed, tt.wantRRDUsed)
			}
		})
	}
}

func TestPollPVENodePreservesPreviousSnapshotDuringTransientFallback(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	const gib = uint64(1024 * 1024 * 1024)

	mon := newTestPVEMonitor("test")
	defer mon.alertManager.Stop()
	defer mon.notificationMgr.Stop()

	client := &stubPVEClient{
		nodeStatus: &proxmox.NodeStatus{
			Memory: &proxmox.MemoryStatus{
				Total:     16 * gib,
				Used:      12 * gib,
				Free:      1 * gib,
				Available: 8 * gib,
			},
		},
	}
	node := proxmox.Node{
		Node:   "node1",
		Status: "online",
		MaxMem: 16 * gib,
		Mem:    15 * gib,
		MaxCPU: 8,
	}

	first, _, _, err := mon.pollPVENode(context.Background(), "test", &mon.config.PVEInstances[0], client, node, "healthy", nil, nil)
	if err != nil {
		t.Fatalf("first pollPVENode() error = %v", err)
	}
	if first.Memory.Used != int64(8*gib) {
		t.Fatalf("first.Memory.Used = %d, want %d", first.Memory.Used, 8*gib)
	}

	client.nodeStatus = nil
	second, _, _, err := mon.pollPVENode(
		context.Background(),
		"test",
		&mon.config.PVEInstances[0],
		client,
		node,
		"healthy",
		map[string]models.Memory{first.ID: first.Memory},
		nil,
	)
	if err != nil {
		t.Fatalf("second pollPVENode() error = %v", err)
	}
	if second.Memory.Used != first.Memory.Used {
		t.Fatalf("second.Memory.Used = %d, want preserved %d", second.Memory.Used, first.Memory.Used)
	}

	snap := mon.nodeSnapshots[makeNodeSnapshotKey("test", "node1")]
	if snap.MemorySource != "previous-snapshot" {
		t.Fatalf("snapshot.MemorySource = %q, want previous-snapshot", snap.MemorySource)
	}
	if snap.FallbackReason != "preserved-previous-snapshot" {
		t.Fatalf("snapshot.FallbackReason = %q, want preserved-previous-snapshot", snap.FallbackReason)
	}
	if snap.Memory.Used != first.Memory.Used {
		t.Fatalf("snapshot.Memory.Used = %d, want preserved %d", snap.Memory.Used, first.Memory.Used)
	}

	third, _, _, err := mon.pollPVENode(
		context.Background(),
		"test",
		&mon.config.PVEInstances[0],
		client,
		node,
		"healthy",
		map[string]models.Memory{second.ID: second.Memory},
		nil,
	)
	if err != nil {
		t.Fatalf("third pollPVENode() error = %v", err)
	}
	if !third.Memory.UsageUnavailable {
		t.Fatalf("third.Memory = %+v, want honest unavailable state after bounded carry-forward", third.Memory)
	}
}

func TestGuestDiskTrustCharacterizationCarriesForwardRecentSnapshot(t *testing.T) {
	now := time.Now()
	prev := &models.VM{
		Type:         "qemu",
		Status:       "running",
		LastSeen:     now.Add(-time.Minute),
		AgentVersion: "8.2.0",
		Disk: models.Disk{
			Total: 1000,
			Used:  400,
			Free:  600,
			Usage: 40,
		},
		Disks: []models.Disk{
			{Total: 1000, Used: 400, Free: 600, Usage: 40, Mountpoint: "/", Type: "ext4", Device: "/dev/vda"},
		},
	}

	total, used, free, usage, disks, reason := stabilizeGuestLowTrustDisk(
		prev,
		"running",
		1000,
		0,
		1000,
		-1,
		nil,
		"no-filesystems",
		false,
		now,
	)

	if total != 1000 || used != 400 || free != 600 || usage != 40 {
		t.Fatalf("unexpected carried-forward disk summary: total=%d used=%d free=%d usage=%.2f", total, used, free, usage)
	}
	if len(disks) != 1 || disks[0].Device != "/dev/vda" {
		t.Fatalf("expected previous individual disks to be preserved, got %#v", disks)
	}
	if reason != "prev-no-filesystems" {
		t.Fatalf("reason = %q, want prev-no-filesystems", reason)
	}
}

func TestHandleClusterVMResourceMemoryTrustCharacterization(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	const gib = uint64(1024 * 1024 * 1024)

	tests := []struct {
		name           string
		status         *proxmox.VMStatus
		agentAvailable uint64
		wantSource     string
		wantUsed       uint64
		wantGap        uint64
		wantUnknown    bool
	}{
		{
			name: "cache inflated Linux VM with bare meminfo total stays unknown",
			status: &proxmox.VMStatus{
				Status: "running",
				MaxMem: 8 * gib,
				Mem:    7 * gib,
				MemInfo: &proxmox.VMMemInfo{
					Total: 8 * gib,
				},
				Agent: proxmox.VMAgentField{Value: 1},
			},
			wantSource:  "unavailable",
			wantUnknown: true,
		},
		{
			name: "missing MemAvailable derives from free buffers cached",
			status: &proxmox.VMStatus{
				Status: "running",
				MaxMem: 8 * gib,
				Mem:    7 * gib,
				MemInfo: &proxmox.VMMemInfo{
					Total:   8 * gib,
					Free:    1 * gib,
					Buffers: gib / 2,
					Cached:  2 * gib,
				},
				Agent: proxmox.VMAgentField{Value: 1},
			},
			wantSource: "derived-free-buffers-cached",
			wantUsed:   uint64(8*gib) - (uint64(gib) + uint64(gib/2) + uint64(2*gib)),
		},
		{
			name: "missing Buffers and Cached derives from total minus used gap",
			status: &proxmox.VMStatus{
				Status: "running",
				MaxMem: 8 * gib,
				Mem:    7 * gib,
				MemInfo: &proxmox.VMMemInfo{
					Total: 8 * gib,
					Used:  5 * gib,
					Free:  gib / 2,
				},
				Agent: proxmox.VMAgentField{Value: 1},
			},
			wantSource: "derived-total-minus-used",
			wantUsed:   5 * gib,
			wantGap:    3 * gib,
		},
		{
			name: "partial Linux meminfo without cache aware fallback stays unknown",
			status: &proxmox.VMStatus{
				Status: "running",
				MaxMem: 8 * gib,
				Mem:    15 * gib / 2,
				MemInfo: &proxmox.VMMemInfo{
					Total: 8 * gib,
					Free:  gib / 2,
				},
				Agent: proxmox.VMAgentField{Value: 1},
			},
			wantSource:  "unavailable",
			wantUnknown: true,
		},
		{
			name: "MemAvailable conflicting with the selected capacity stays unknown",
			status: &proxmox.VMStatus{
				Status: "running",
				MaxMem: 8 * gib,
				Mem:    7 * gib,
				MemInfo: &proxmox.VMMemInfo{
					Total:     16 * gib,
					Available: 12 * gib,
				},
			},
			wantSource:  "unavailable",
			wantUnknown: true,
		},
		{
			name: "reclaimable components above the selected capacity stay unknown",
			status: &proxmox.VMStatus{
				Status: "running",
				MaxMem: 8 * gib,
				Mem:    7 * gib,
				MemInfo: &proxmox.VMMemInfo{
					Total:   8 * gib,
					Free:    2 * gib,
					Buffers: 2 * gib,
					Cached:  5 * gib,
				},
			},
			wantSource:  "unavailable",
			wantUnknown: true,
		},
		{
			name: "guest agent availability above the selected capacity stays unknown",
			status: &proxmox.VMStatus{
				Status: "running",
				MaxMem: 8 * gib,
				Mem:    7 * gib,
				MemInfo: &proxmox.VMMemInfo{
					Total: 8 * gib,
					Free:  gib / 2,
				},
				Agent: proxmox.VMAgentField{Value: 1},
			},
			agentAvailable: 9 * gib,
			wantSource:     "unavailable",
			wantUnknown:    true,
		},
		{
			name: "materially inconsistent status memory prefers freemem fallback",
			status: &proxmox.VMStatus{
				Status:  "running",
				MaxMem:  8 * gib,
				Mem:     7920 * 1024 * 1024,
				FreeMem: 5 * gib,
			},
			wantSource: "status-freemem",
			wantUsed:   3 * gib,
		},
		{
			name: "ballooned VM derives freemem fallback from balloon total",
			status: &proxmox.VMStatus{
				Status:  "running",
				MaxMem:  8 * gib,
				Mem:     4 * gib,
				Balloon: 4 * gib,
				FreeMem: 1 * gib,
			},
			wantSource: "status-freemem",
			wantUsed:   3 * gib,
		},
		{
			name: "saturated VM derives freemem fallback from ballooninfo",
			status: &proxmox.VMStatus{
				Status: "running",
				MaxMem: 8 * gib,
				Mem:    8 * gib,
				BalloonInfo: &proxmox.VMBalloonInfo{
					FreeMem:  5 * gib,
					TotalMem: 8 * gib,
				},
			},
			wantSource: "status-freemem",
			wantUsed:   3 * gib,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			mon := newTestPVEMonitor("test")
			defer mon.alertManager.Stop()
			defer mon.notificationMgr.Stop()

			client := &vmMemoryTrustStubClient{
				stubPVEClient:       &stubPVEClient{},
				vmStatus:            tt.status,
				vmAgentMemAvailable: tt.agentAvailable,
			}

			res := proxmox.ClusterResource{
				ID:     "qemu/101",
				Type:   "qemu",
				Node:   "node1",
				Name:   "vm-101",
				Status: "running",
				VMID:   101,
				MaxMem: tt.status.MaxMem,
				Mem:    tt.status.Mem,
				MaxCPU: 4,
			}

			vm, ok := mon.handleClusterVMResource(context.Background(), "test", res, makeGuestID("test", "node1", 101), client, nil, nil)
			if !ok {
				t.Fatal("handleClusterVMResource() returned ok=false")
			}
			if got := uint64(vm.Memory.Used); got != tt.wantUsed {
				t.Fatalf("vm.Memory.Used = %d, want %d", got, tt.wantUsed)
			}
			if vm.Memory.UsageUnavailable != tt.wantUnknown {
				t.Fatalf("vm.Memory.UsageUnavailable = %t, want %t", vm.Memory.UsageUnavailable, tt.wantUnknown)
			}

			snap := mon.guestSnapshots[makeGuestSnapshotKey("test", "qemu", "node1", 101)]
			if snap.MemorySource != tt.wantSource {
				t.Fatalf("snapshot.MemorySource = %q, want %q", snap.MemorySource, tt.wantSource)
			}
			if tt.wantSource == "derived-total-minus-used" && snap.FallbackReason != "derived-total-minus-used" {
				t.Fatalf("snapshot.FallbackReason = %q, want derived-total-minus-used", snap.FallbackReason)
			}
			if tt.wantGap > 0 && snap.Raw.MemInfoTotalMinusUsed != tt.wantGap {
				t.Fatalf("snapshot.Raw.MemInfoTotalMinusUsed = %d, want %d", snap.Raw.MemInfoTotalMinusUsed, tt.wantGap)
			}
		})
	}
}

func TestHandleClusterVMResourcePrefersGuestAgentMemAvailableForSaturatedIssue1319Status(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	const gib = uint64(1024 * 1024 * 1024)

	mon := newTestPVEMonitor("test")
	defer mon.alertManager.Stop()
	defer mon.notificationMgr.Stop()

	client := &vmMemoryTrustStubClient{
		stubPVEClient: &stubPVEClient{},
		vmStatus: &proxmox.VMStatus{
			Status: "running",
			MaxMem: 16 * gib,
			Mem:    16*gib + 512*1024*1024,
			Agent:  proxmox.VMAgentField{Value: 1},
		},
		vmAgentMemAvailable: 4 * gib,
	}
	res := proxmox.ClusterResource{
		ID:     "qemu/164",
		Type:   "qemu",
		Node:   "pve2",
		Name:   "linux-vm",
		Status: "running",
		VMID:   164,
		MaxMem: 16 * gib,
		Mem:    16*gib + 512*1024*1024,
		MaxCPU: 4,
	}

	vm, ok := mon.handleClusterVMResource(context.Background(), "cluster-a", res, makeGuestID("cluster-a", "pve2", 164), client, nil, nil)
	if !ok {
		t.Fatal("handleClusterVMResource() returned ok=false")
	}
	if got := uint64(vm.Memory.Used); got != 12*gib {
		t.Fatalf("vm.Memory.Used = %d, want %d", got, uint64(12*gib))
	}

	snap := mon.guestSnapshots[makeGuestSnapshotKey("cluster-a", "qemu", "pve2", 164)]
	if snap.MemorySource != "guest-agent-meminfo" {
		t.Fatalf("snapshot.MemorySource = %q, want guest-agent-meminfo", snap.MemorySource)
	}
	if snap.Raw.GuestAgentMemAvailable != 4*gib {
		t.Fatalf("snapshot.Raw.GuestAgentMemAvailable = %d, want %d", snap.Raw.GuestAgentMemAvailable, uint64(4*gib))
	}
	if client.vmAgentMemCalls != 1 {
		t.Fatalf("expected guest agent meminfo to be queried once, got %d calls", client.vmAgentMemCalls)
	}
	if client.vmRRDCalls != 0 {
		t.Fatalf("expected saturated guest-agent memory path to skip RRD, got %d RRD calls", client.vmRRDCalls)
	}
}

func TestPollVMsWithNodesPreservesProxmoxPool(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	mon := newTestPVEMonitor("test")
	defer mon.alertManager.Stop()
	defer mon.notificationMgr.Stop()

	client := &vmMemoryTrustStubClient{
		stubPVEClient: &stubPVEClient{},
		vms: []proxmox.VM{{
			VMID:   101,
			Name:   "vm-101",
			Node:   "node1",
			Pool:   "prod-vms",
			Status: "stopped",
			MaxMem: 8 * 1024,
			Mem:    2 * 1024,
			CPUs:   2,
		}},
	}

	nodes := []proxmox.Node{{Node: "node1", Status: "online"}}
	nodeEffectiveStatus := map[string]string{"node1": "online"}
	mon.pollVMsWithNodes(context.Background(), "test", "", false, client, nodes, nodeEffectiveStatus)

	vms := mon.state.GetSnapshot().VMs
	if len(vms) != 1 {
		t.Fatalf("expected 1 VM, got %d", len(vms))
	}
	if got := vms[0].Pool; got != "prod-vms" {
		t.Fatalf("vm pool = %q, want %q", got, "prod-vms")
	}
}

func TestPollVMsWithNodesRecordsQEMUTemplateBackupInventoryReadiness(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	mon := newTestPVEMonitor("test")
	defer mon.alertManager.Stop()
	defer mon.notificationMgr.Stop()

	client := &vmMemoryTrustStubClient{
		stubPVEClient: &stubPVEClient{},
		vms: []proxmox.VM{
			{
				VMID:     900,
				Name:     "tmpl-900",
				Node:     "node1",
				Status:   "stopped",
				Template: 1,
				MaxMem:   8 * 1024,
				CPUs:     2,
			},
			{
				VMID:   101,
				Name:   "vm-101",
				Node:   "node1",
				Status: "stopped",
				MaxMem: 8 * 1024,
				Mem:    2 * 1024,
				CPUs:   2,
			},
		},
	}

	nodes := []proxmox.Node{{Node: "node1", Status: "online"}}
	nodeEffectiveStatus := map[string]string{"node1": "online"}
	mon.pollVMsWithNodes(context.Background(), "test", "", false, client, nodes, nodeEffectiveStatus)

	vms := mon.state.GetSnapshot().VMs
	if len(vms) != 1 {
		t.Fatalf("expected only non-template VM in runtime state, got %d", len(vms))
	}
	if got := vms[0].VMID; got != 101 {
		t.Fatalf("runtime VMID = %d, want 101", got)
	}

	scope := mon.backupInventoryScopeForAlerts()
	if scope == nil {
		t.Fatal("expected backup inventory scope")
	}
	if !scope.PVEOrphanInventoryReady["test"]["qemu"] {
		t.Fatalf("expected qemu backup orphan inventory readiness for test instance")
	}
	templateSubject := pveBackupTemplateSubjectKey("test", "qemu", "node1", 900)
	if _, ok := scope.PVETemplateSubjects[templateSubject]; !ok {
		t.Fatalf("expected template subject %q in backup inventory scope", templateSubject)
	}
}

func TestPollVMsWithNodesMemoryTrustCharacterization(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	const gib = uint64(1024 * 1024 * 1024)

	tests := []struct {
		name       string
		status     *proxmox.VMStatus
		wantSource string
		wantUsed   uint64
	}{
		{
			name: "cache inflated Linux VM with bare meminfo total stays unknown",
			status: &proxmox.VMStatus{
				Status: "running",
				MaxMem: 8 * gib,
				Mem:    7 * gib,
				MemInfo: &proxmox.VMMemInfo{
					Total: 8 * gib,
				},
				Agent: proxmox.VMAgentField{Value: 1},
			},
			wantSource: "unavailable",
			wantUsed:   0,
		},
		{
			name: "missing Buffers and Cached derives from total minus used gap",
			status: &proxmox.VMStatus{
				Status: "running",
				MaxMem: 8 * gib,
				Mem:    7 * gib,
				MemInfo: &proxmox.VMMemInfo{
					Total: 8 * gib,
					Used:  5 * gib,
					Free:  gib / 2,
				},
				Agent: proxmox.VMAgentField{Value: 1},
			},
			wantSource: "derived-total-minus-used",
			wantUsed:   5 * gib,
		},
		{
			name: "materially inconsistent status memory prefers freemem fallback",
			status: &proxmox.VMStatus{
				Status:  "running",
				MaxMem:  8 * gib,
				Mem:     7920 * 1024 * 1024,
				FreeMem: 5 * gib,
			},
			wantSource: "status-freemem",
			wantUsed:   3 * gib,
		},
		{
			name: "ballooned VM derives freemem fallback from balloon total",
			status: &proxmox.VMStatus{
				Status:  "running",
				MaxMem:  8 * gib,
				Mem:     4 * gib,
				Balloon: 4 * gib,
				FreeMem: 1 * gib,
			},
			wantSource: "status-freemem",
			wantUsed:   3 * gib,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			mon := newTestPVEMonitor("test")
			defer mon.alertManager.Stop()
			defer mon.notificationMgr.Stop()

			client := &vmMemoryTrustStubClient{
				stubPVEClient: &stubPVEClient{},
				vms: []proxmox.VM{{
					VMID:   101,
					Name:   "vm-101",
					Node:   "node1",
					Status: "running",
					MaxMem: tt.status.MaxMem,
					Mem:    tt.status.Mem,
					CPUs:   2,
				}},
				vmStatus: tt.status,
			}

			nodes := []proxmox.Node{{Node: "node1", Status: "online"}}
			nodeEffectiveStatus := map[string]string{"node1": "online"}
			mon.pollVMsWithNodes(context.Background(), "test", "", false, client, nodes, nodeEffectiveStatus)

			key := makeGuestSnapshotKey("test", "qemu", "node1", 101)
			snap, ok := mon.guestSnapshots[key]
			if !ok {
				t.Fatalf("expected guest snapshot %q to be recorded", key)
			}
			if snap.MemorySource != tt.wantSource {
				t.Fatalf("snapshot.MemorySource = %q, want %q", snap.MemorySource, tt.wantSource)
			}
			if got := uint64(snap.Memory.Used); got != tt.wantUsed {
				t.Fatalf("snapshot.Memory.Used = %d, want %d", got, tt.wantUsed)
			}
			if tt.wantSource == "derived-total-minus-used" && snap.FallbackReason != "derived-total-minus-used" {
				t.Fatalf("snapshot.FallbackReason = %q, want derived-total-minus-used", snap.FallbackReason)
			}
		})
	}
}

func TestPollVMsWithNodes_SkipsNativeGuestMetricWritesInMockMode(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	previous := mock.IsMockEnabled()
	mustSetMockEnabled(t, true)
	t.Cleanup(func() { mustSetMockEnabled(t, previous) })

	mon := newTestPVEMonitor("test")
	defer mon.alertManager.Stop()
	defer mon.notificationMgr.Stop()

	client := &vmMemoryTrustStubClient{
		stubPVEClient: &stubPVEClient{},
		vms: []proxmox.VM{{
			VMID:   101,
			Name:   "vm-101",
			Node:   "node1",
			Status: "running",
			MaxMem: 8 * 1024,
			Mem:    3 * 1024,
			CPUs:   2,
			CPU:    0.42,
		}},
		vmStatus: &proxmox.VMStatus{
			Status: "running",
			MaxMem: 8 * 1024,
			Mem:    3 * 1024,
			Agent:  proxmox.VMAgentField{Value: 1},
		},
	}

	nodes := []proxmox.Node{{Node: "node1", Status: "online"}}
	nodeEffectiveStatus := map[string]string{"node1": "online"}
	mon.pollVMsWithNodes(context.Background(), "test", "", false, client, nodes, nodeEffectiveStatus)

	vms := mon.state.GetSnapshot().VMs
	if len(vms) != 1 {
		t.Fatalf("expected 1 VM, got %d", len(vms))
	}

	if got := mon.metricsHistory.GetGuestMetrics(vms[0].ID, "cpu", time.Hour); len(got) != 0 {
		t.Fatalf("expected mock mode to skip native VM metrics history writes, got %+v", got)
	}
	if got := mon.metricsHistory.GetGuestMetrics(vms[0].ID, "memory", time.Hour); len(got) != 0 {
		t.Fatalf("expected mock mode to skip native VM memory history writes, got %+v", got)
	}
}

func TestRecordGuestMetric_SkipsNativeWritesInMockMode(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	previous := mock.IsMockEnabled()
	mustSetMockEnabled(t, true)
	t.Cleanup(func() { mustSetMockEnabled(t, previous) })

	mon := newTestPVEMonitor("test")
	defer mon.alertManager.Stop()
	defer mon.notificationMgr.Stop()

	now := time.Now().UTC()
	mon.recordGuestMetric(
		"vm",
		"test:node1:101",
		42,
		48,
		1024,
		51,
		1024,
		512,
		256,
		128,
		now,
	)

	if got := mon.metricsHistory.GetGuestMetrics("test:node1:101", "cpu", time.Hour); len(got) != 0 {
		t.Fatalf("expected mock mode to skip helper cpu history writes, got %+v", got)
	}
	if got := mon.metricsHistory.GetGuestMetrics("test:node1:101", "memory", time.Hour); len(got) != 0 {
		t.Fatalf("expected mock mode to skip helper memory history writes, got %+v", got)
	}
}

func TestRecordGuestMetricPreservesHistoryWhenLiveMemoryIsUnavailable(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	mon := newTestPVEMonitor("test")
	defer mon.alertManager.Stop()
	defer mon.notificationMgr.Stop()

	now := time.Now().UTC()
	mon.recordGuestMetric("vm", "test:node1:101", 20, 76, 1024, 40, -1, -1, -1, -1, now.Add(-time.Minute))
	mon.recordGuestMetric("vm", "test:node1:101", 25, -1, -1, 42, -1, -1, -1, -1, now)

	points := mon.metricsHistory.GetGuestMetrics("test:node1:101", "memory", time.Hour)
	if len(points) != 1 {
		t.Fatalf("memory history points = %d, want 1 trusted point", len(points))
	}
	if points[0].Value != 76 {
		t.Fatalf("memory history value = %.2f, want 76", points[0].Value)
	}
	usedPoints := mon.metricsHistory.GetGuestMetrics("test:node1:101", "memoryused", time.Hour)
	if len(usedPoints) != 1 || usedPoints[0].Value != 1024 {
		t.Fatalf("memory-used history = %+v, want one trusted 1024-byte point", usedPoints)
	}
}

func TestHandleClusterContainerResourceMemoryTrustCharacterization(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	const gib = uint64(1024 * 1024 * 1024)

	tests := []struct {
		name        string
		res         proxmox.ClusterResource
		wantSource  string
		wantUsed    uint64
		wantStatus  string
		wantUnknown bool
	}{
		{
			name: "running LXC uses the cache-inclusive cluster resources listing (#1634)",
			res: proxmox.ClusterResource{
				ID:     "lxc/204",
				Type:   "lxc",
				Node:   "node1",
				Name:   "ct-204",
				Status: "running",
				VMID:   204,
				MaxMem: 8 * gib,
				Mem:    7 * gib,
				MaxCPU: 4,
			},
			wantSource: "cluster-resources",
			wantUsed:   7 * gib,
			wantStatus: "running",
		},
		{
			name: "running LXC without a listing memory value stays unknown",
			res: proxmox.ClusterResource{
				ID:     "lxc/206",
				Type:   "lxc",
				Node:   "node1",
				Name:   "ct-206",
				Status: "running",
				VMID:   206,
				MaxMem: 8 * gib,
				MaxCPU: 4,
			},
			wantSource:  "unavailable",
			wantStatus:  "running",
			wantUnknown: true,
		},
		{
			name: "stopped LXC reports powered off memory",
			res: proxmox.ClusterResource{
				ID:     "lxc/203",
				Type:   "lxc",
				Node:   "node1",
				Name:   "ct-203",
				Status: "stopped",
				VMID:   203,
				MaxMem: 8 * gib,
				Mem:    2 * gib,
				MaxCPU: 4,
			},
			wantSource: "powered-off",
			wantUsed:   0,
			wantStatus: "stopped",
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			mon := newTestPVEMonitor("test")
			defer mon.alertManager.Stop()
			defer mon.notificationMgr.Stop()

			client := &vmMemoryTrustStubClient{
				stubPVEClient: &stubPVEClient{},
			}

			container, ok := mon.handleClusterContainerResource(
				context.Background(),
				"test",
				tt.res,
				makeGuestID("test", "node1", tt.res.VMID),
				client,
				nil,
				nil,
			)
			if !ok {
				t.Fatal("handleClusterContainerResource() returned ok=false")
			}
			if container.Status != tt.wantStatus {
				t.Fatalf("container.Status = %q, want %q", container.Status, tt.wantStatus)
			}
			if got := uint64(container.Memory.Used); got != tt.wantUsed {
				t.Fatalf("container.Memory.Used = %d, want %d", got, tt.wantUsed)
			}
			if container.Memory.UsageUnavailable != tt.wantUnknown {
				t.Fatalf("container.Memory.UsageUnavailable = %t, want %t", container.Memory.UsageUnavailable, tt.wantUnknown)
			}

			key := makeGuestSnapshotKey("test", container.Type, "node1", tt.res.VMID)
			snap, ok := mon.guestSnapshots[key]
			if !ok {
				t.Fatalf("expected guest snapshot %q to be recorded", key)
			}
			if snap.MemorySource != tt.wantSource {
				t.Fatalf("snapshot.MemorySource = %q, want %q", snap.MemorySource, tt.wantSource)
			}
			if got := uint64(snap.Memory.Used); got != tt.wantUsed {
				t.Fatalf("snapshot.Memory.Used = %d, want %d", got, tt.wantUsed)
			}
		})
	}
}

func TestPollContainersWithNodesPreservesProxmoxPool(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	mon := newTestPVEMonitor("test")
	defer mon.alertManager.Stop()
	defer mon.notificationMgr.Stop()

	client := &vmMemoryTrustStubClient{
		stubPVEClient: &stubPVEClient{},
		containers: []proxmox.Container{{
			VMID:   201,
			Name:   "ct-201",
			Node:   "node1",
			Pool:   "ops-lxc",
			Status: "stopped",
			MaxMem: 8 * 1024,
			Mem:    2 * 1024,
			CPUs:   2,
		}},
	}

	nodes := []proxmox.Node{{Node: "node1", Status: "online"}}
	nodeEffectiveStatus := map[string]string{"node1": "online"}
	mon.pollContainersWithNodes(context.Background(), "test", "", false, client, nodes, nodeEffectiveStatus)

	containers := mon.state.GetSnapshot().Containers
	if len(containers) != 1 {
		t.Fatalf("expected 1 container, got %d", len(containers))
	}
	if got := containers[0].Pool; got != "ops-lxc" {
		t.Fatalf("container pool = %q, want %q", got, "ops-lxc")
	}
}

func TestPollContainersWithNodesMemoryTrustCharacterization(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	const gib = uint64(1024 * 1024 * 1024)

	tests := []struct {
		name       string
		container  proxmox.Container
		wantSource string
		wantUsed   uint64
	}{
		{
			name: "running LXC uses the cluster resources listing in node polling (#1634)",
			container: proxmox.Container{
				VMID:   301,
				Name:   "ct-301",
				Node:   "node1",
				Status: "running",
				MaxMem: 8 * gib,
				Mem:    7 * gib,
				CPUs:   2,
			},
			wantSource: "cluster-resources",
			wantUsed:   7 * gib,
		},
		{
			name: "stopped LXC reports powered off memory in node polling",
			container: proxmox.Container{
				VMID:   302,
				Name:   "ct-302",
				Node:   "node1",
				Status: "stopped",
				MaxMem: 8 * gib,
				Mem:    2 * gib,
				CPUs:   2,
			},
			wantSource: "powered-off",
			wantUsed:   0,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			mon := newTestPVEMonitor("test")
			defer mon.alertManager.Stop()
			defer mon.notificationMgr.Stop()

			client := &vmMemoryTrustStubClient{
				stubPVEClient: &stubPVEClient{},
				containers:    []proxmox.Container{tt.container},
			}

			nodes := []proxmox.Node{{Node: "node1", Status: "online"}}
			nodeEffectiveStatus := map[string]string{"node1": "online"}
			mon.pollContainersWithNodes(context.Background(), "test", "", false, client, nodes, nodeEffectiveStatus)

			key := makeGuestSnapshotKey("test", "lxc", "node1", int(tt.container.VMID))
			snap, ok := mon.guestSnapshots[key]
			if !ok {
				t.Fatalf("expected guest snapshot %q to be recorded", key)
			}
			if snap.MemorySource != tt.wantSource {
				t.Fatalf("snapshot.MemorySource = %q, want %q", snap.MemorySource, tt.wantSource)
			}
			if got := uint64(snap.Memory.Used); got != tt.wantUsed {
				t.Fatalf("snapshot.Memory.Used = %d, want %d", got, tt.wantUsed)
			}
		})
	}
}

// TestGracePeriodGuestContributesNoMemorySample extends the memory-trust
// boundary to carried-forward guests. A guest preserved while its node sits in
// the grace period has no fresh observation behind it, so recording one writes
// a memory reading Pulse did not take. Before the cycle guard this was hidden
// only by accident: the carried-forward projection leaves Memory without known
// usage, so historyMemoryUsage returned the -1 sentinel rather than a
// fabricated percentage. That is a coincidence of the projection, not a
// guarantee, and it would break the moment preservation started carrying
// memory forward.
func TestGracePeriodGuestContributesNoMemorySample(t *testing.T) {
	history := NewMetricsHistory(32, time.Hour)
	monitor := &Monitor{metricsHistory: history}

	cycleStart := time.Now().UTC()

	monitor.recordGuestMetrics(
		[]models.VM{{
			ID:       "vm-grace",
			Status:   "running",
			CPU:      0,
			CPUs:     4,
			LastSeen: cycleStart.Add(-2 * time.Minute),
			Memory:   models.Memory{Total: 8 << 30, Used: 4 << 30, Usage: 50},
		}},
		nil,
		cycleStart,
	)

	if got := history.GetGuestMetrics("vm-grace", "memory", time.Hour); len(got) != 0 {
		t.Fatalf("grace-period guest recorded %d memory sample(s): %+v", len(got), got)
	}
}