pulse/internal/monitoring/node_memory_sources_test.go

package monitoring

import (
	"context"
	"errors"
	"reflect"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)

type nodeNetworkFailureClient struct {
	PVEClientInterface
	err   error
	calls *int
}

func (c nodeNetworkFailureClient) GetNodeNetworkInterfaces(context.Context, string) ([]proxmox.NodeNetworkInterface, error) {
	if c.calls != nil {
		*c.calls++
	}
	return nil, c.err
}

func TestPVENodeNetworkFailurePreservesLastKnownInventory(t *testing.T) {
	previous := []models.Node{{
		ID: "lab-pve1", Instance: "lab", Name: "pve1",
		NetworkInterfaces: []models.HostNetworkInterface{{Name: "vmbr0", Addresses: []string{"192.0.2.10/24"}}},
	}}
	client := nodeNetworkFailureClient{err: errors.New("temporary network inventory failure")}

	got := collectPVENodeNetworkInterfaces(context.Background(), client, "lab", "lab-pve1", "pve1", previous)
	if !reflect.DeepEqual(got, previous[0].NetworkInterfaces) {
		t.Fatalf("network interfaces = %#v, want previous %#v", got, previous[0].NetworkInterfaces)
	}
	got[0].Addresses[0] = "mutated"
	if previous[0].NetworkInterfaces[0].Addresses[0] != "192.0.2.10/24" {
		t.Fatal("last-known interface fallback aliases previous state")
	}
}

func TestPVENodeNetworkRefreshIsBoundedWithinPollingWindow(t *testing.T) {
	calls := 0
	previous := []models.Node{{
		ID: "lab-pve1", Instance: "lab", Name: "pve1", LastSeen: time.Now(),
		NetworkInterfaces: []models.HostNetworkInterface{{Name: "vmbr0", Addresses: []string{"192.0.2.10/24"}}},
	}}
	client := nodeNetworkFailureClient{err: errors.New("must not be called"), calls: &calls}

	got := collectPVENodeNetworkInterfaces(context.Background(), client, "lab", "lab-pve1", "pve1", previous)
	if calls != 0 {
		t.Fatalf("network inventory calls = %d, want 0 within refresh window", calls)
	}
	if !reflect.DeepEqual(got, previous[0].NetworkInterfaces) {
		t.Fatalf("network interfaces = %#v, want previous %#v", got, previous[0].NetworkInterfaces)
	}
}

func TestResolveNodeMemoryCharacterization(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	const gib = uint64(1024 * 1024 * 1024)

	tests := []struct {
		name          string
		memory        *proxmox.MemoryStatus
		rrdPoints     []proxmox.NodeRRDPoint
		wantSource    string
		wantFallback  string
		wantUsed      uint64
		wantRawSource string
	}{
		{
			name: "missing MemAvailable derives from free buffers cached",
			memory: &proxmox.MemoryStatus{
				Total:   32 * gib,
				Used:    26 * gib,
				Free:    2 * gib,
				Buffers: 3 * gib,
				Cached:  7 * gib,
			},
			wantSource:    "derived-free-buffers-cached",
			wantFallback:  "",
			wantUsed:      20 * gib,
			wantRawSource: "node-status",
		},
		{
			name: "proxmox 8.4 field drift derives from total minus used gap",
			memory: &proxmox.MemoryStatus{
				Total: 134794743808,
				Used:  107351023616,
				Free:  6471057408,
			},
			wantSource:    "derived-total-minus-used",
			wantFallback:  "node-status-total-minus-used",
			wantUsed:      107351023616,
			wantRawSource: "node-status-total-minus-used",
		},
		{
			name: "missing cache fields falls back to rrd memused",
			memory: &proxmox.MemoryStatus{
				Total: 16 * gib,
				Used:  16 * gib,
				Free:  0,
			},
			rrdPoints: []proxmox.NodeRRDPoint{{
				MemTotal: floatPtr(float64(16 * gib)),
				MemUsed:  floatPtr(float64(6 * gib)),
			}},
			wantSource:    "rrd-memused",
			wantFallback:  "rrd-memused",
			wantUsed:      6 * gib,
			wantRawSource: "rrd-memused",
		},
		{
			name: "node status has only free so rrd memavailable is preferred over total-minus-free",
			memory: &proxmox.MemoryStatus{
				Total: 8 * gib,
				Used:  7 * gib,
				Free:  1 * gib,
			},
			rrdPoints: []proxmox.NodeRRDPoint{{
				MemTotal:     floatPtr(float64(8 * gib)),
				MemAvailable: floatPtr(float64(4 * gib)),
			}},
			wantSource:    "rrd-memavailable",
			wantFallback:  "rrd-memavailable",
			wantUsed:      4 * gib,
			wantRawSource: "rrd-memavailable",
		},
		{
			name: "explicit zero RRD memused is a valid idle sample",
			memory: &proxmox.MemoryStatus{
				Total: 8 * gib,
				Used:  7 * gib,
				Free:  1 * gib,
			},
			rrdPoints: []proxmox.NodeRRDPoint{{
				MemTotal: floatPtr(float64(8 * gib)),
				MemUsed:  floatPtr(0),
			}},
			wantSource:    "rrd-memused",
			wantFallback:  "rrd-memused",
			wantUsed:      0,
			wantRawSource: "rrd-memused",
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			mon := newTestPVEMonitor("test")
			defer mon.alertManager.Stop()
			defer mon.notificationMgr.Stop()

			client := &stubPVEClient{rrdPoints: tt.rrdPoints}

			memory, source, fallback, raw, ok := mon.resolveNodeMemory(
				context.Background(),
				client,
				"test",
				"node1",
				tt.memory,
				NodeMemoryRaw{},
			)
			if !ok {
				t.Fatal("resolveNodeMemory() = ok=false")
			}
			if got := uint64(memory.Used); got != tt.wantUsed {
				t.Fatalf("memory.Used = %d, want %d", got, tt.wantUsed)
			}
			if source != tt.wantSource {
				t.Fatalf("source = %q, want %q", source, tt.wantSource)
			}
			if fallback != tt.wantFallback {
				t.Fatalf("fallback = %q, want %q", fallback, tt.wantFallback)
			}
			if raw.ProxmoxMemorySource != tt.wantRawSource {
				t.Fatalf("raw.ProxmoxMemorySource = %q, want %q", raw.ProxmoxMemorySource, tt.wantRawSource)
			}
		})
	}
}

func TestPollPVENodePrefersLinkedHostDiskOverRootFS(t *testing.T) {
	mon := newTestPVEMonitor("test")
	defer mon.alertManager.Stop()
	defer mon.notificationMgr.Stop()
	mon.config.PVEInstances[0].IsCluster = true
	mon.config.PVEInstances[0].ClusterName = "production"
	mon.config.PVEInstances[0].ClusterEndpoints = []config.ClusterEndpoint{{
		NodeIdentity: "production-ebringa",
		NodeName:     "ebringa",
	}}
	mon.config.PVEInstances[0].ClusterNodeIdentities = []config.PVEClusterNodeIdentity{{
		ID:          "production-ebringa",
		NativeName:  "ebringa",
		DisplayName: "Render East",
	}}

	mon.state.UpsertHost(models.Host{
		ID:       "host-1",
		Hostname: "ebringa",
		Status:   "online",
		Disks: []models.Disk{
			{Mountpoint: "/", Type: "zfs", Total: 975, Used: 410, Free: 565, Usage: 42.0512820513},
		},
	})
	mon.state.UpdateNodesForInstance("test", []models.Node{
		{
			ID:            "production-ebringa",
			NodeIdentity:  "production-ebringa",
			Name:          "ebringa",
			DisplayName:   "Render East",
			Instance:      "test",
			LinkedAgentID: "host-1",
		},
	})

	client := &stubPVEClient{
		nodeStatus: &proxmox.NodeStatus{
			RootFS: &proxmox.RootFS{
				Total: 975,
				Used:  3,
				Free:  972,
			},
		},
	}

	modelNode, _, _, err := mon.pollPVENode(
		context.Background(),
		"test",
		&mon.config.PVEInstances[0],
		client,
		proxmox.Node{
			Node:    "ebringa",
			Status:  "online",
			MaxDisk: 975,
			Disk:    3,
		},
		"healthy",
		nil,
	)
	if err != nil {
		t.Fatalf("pollPVENode() error = %v", err)
	}

	if modelNode.Disk.Total != 975 || modelNode.Disk.Used != 410 || modelNode.Disk.Free != 565 {
		t.Fatalf("node disk = %+v, want linked host summary", modelNode.Disk)
	}
	if modelNode.Disk.Usage < 42 || modelNode.Disk.Usage > 42.1 {
		t.Fatalf("node disk usage = %.2f, want linked host usage around 42.05", modelNode.Disk.Usage)
	}
	if modelNode.NodeIdentity != "production-ebringa" ||
		modelNode.Name != "ebringa" ||
		modelNode.DisplayName != "Render East" {
		t.Fatalf("cluster presentation identity corrupted linked-agent lifecycle fields: %+v", modelNode)
	}
}

func TestResolveNodeDiskFallsBackToRootFSWithoutLinkedHost(t *testing.T) {
	mon := newTestPVEMonitor("test")
	defer mon.alertManager.Stop()
	defer mon.notificationMgr.Stop()

	disk, source := mon.resolveNodeDisk(
		"test",
		"test-node1",
		"node1",
		proxmox.Node{Node: "node1", Disk: 10, MaxDisk: 100},
		&proxmox.NodeStatus{
			RootFS: &proxmox.RootFS{
				Total: 200,
				Used:  50,
				Free:  150,
			},
		},
	)

	if source != "node-status-rootfs" {
		t.Fatalf("source = %q, want node-status-rootfs", source)
	}
	if disk.Total != 200 || disk.Used != 50 || disk.Free != 150 {
		t.Fatalf("disk = %+v, want rootfs values", disk)
	}
}

func TestPollPVEInstancePreservesNodesEndpointDiskWhenStorageFallbackExists(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	client := &stubPVEClient{
		nodes: []proxmox.Node{
			{
				Node:    "nuc",
				Status:  "online",
				CPU:     0.12,
				MaxCPU:  8,
				Mem:     4 * 1024 * 1024 * 1024,
				MaxMem:  8 * 1024 * 1024 * 1024,
				Disk:    92 * 1024 * 1024 * 1024,
				MaxDisk: 916 * 1024 * 1024 * 1024,
				Uptime:  3600,
			},
		},
		allStorage: []proxmox.Storage{{Storage: "local-zfs"}},
		storageByNode: map[string][]proxmox.Storage{
			"nuc": {
				{
					Storage:   "local-zfs",
					Type:      "zfspool",
					Content:   "images,rootdir",
					Shared:    0,
					Total:     944 * 1024 * 1024 * 1024,
					Used:      313 * 1024 * 1024 * 1024,
					Available: 631 * 1024 * 1024 * 1024,
				},
				{
					Storage:   "t7shield",
					Type:      "dir",
					Content:   "backup,iso,vztmpl",
					Shared:    0,
					Total:     916 * 1024 * 1024 * 1024,
					Used:      92 * 1024 * 1024 * 1024,
					Available: 824 * 1024 * 1024 * 1024,
				},
				{
					Storage:   "hetzner",
					Type:      "dir",
					Content:   "backup",
					Shared:    0,
					Total:     711 * 1024 * 1024 * 1024,
					Used:      109 * 1024 * 1024 * 1024,
					Available: 602 * 1024 * 1024 * 1024,
				},
			},
		},
	}

	mon := newTestPVEMonitor("test")
	defer mon.alertManager.Stop()
	defer mon.notificationMgr.Stop()
	mon.config.PVEInstances[0].MonitorStorage = true

	mon.pollPVEInstance(context.Background(), "test", client)

	snapshot := mon.state.GetSnapshot()
	if len(snapshot.Nodes) != 1 {
		t.Fatalf("expected one node in state, got %d", len(snapshot.Nodes))
	}

	node := snapshot.Nodes[0]
	wantTotal := int64(916 * 1024 * 1024 * 1024)
	wantUsed := int64(92 * 1024 * 1024 * 1024)
	wantFree := int64(824 * 1024 * 1024 * 1024)
	if node.Disk.Total != wantTotal || node.Disk.Used != wantUsed || node.Disk.Free != wantFree {
		t.Fatalf("node disk = %+v, want /nodes endpoint totals preserved", node.Disk)
	}
	if node.Disk.Usage < 10.0 || node.Disk.Usage > 10.1 {
		t.Fatalf("node disk usage = %.2f, want /nodes endpoint usage around 10.04", node.Disk.Usage)
	}
}

func TestPollPVEInstanceUsesStorageFallbackWhenNodeDiskUnavailable(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	client := &stubPVEClient{
		nodes: []proxmox.Node{
			{
				Node:    "nuc",
				Status:  "online",
				CPU:     0.12,
				MaxCPU:  8,
				Mem:     4 * 1024 * 1024 * 1024,
				MaxMem:  8 * 1024 * 1024 * 1024,
				Disk:    0,
				MaxDisk: 0,
				Uptime:  3600,
			},
		},
		nodeStatus: nil,
		allStorage: []proxmox.Storage{{Storage: "local-zfs"}},
		storageByNode: map[string][]proxmox.Storage{
			"nuc": {
				{
					Storage:   "local-zfs",
					Type:      "zfspool",
					Content:   "images,rootdir",
					Shared:    0,
					Total:     944 * 1024 * 1024 * 1024,
					Used:      313 * 1024 * 1024 * 1024,
					Available: 631 * 1024 * 1024 * 1024,
				},
			},
		},
	}

	mon := newTestPVEMonitor("test")
	defer mon.alertManager.Stop()
	defer mon.notificationMgr.Stop()
	mon.config.PVEInstances[0].MonitorStorage = true

	mon.pollPVEInstance(context.Background(), "test", client)

	snapshot := mon.state.GetSnapshot()
	if len(snapshot.Nodes) != 1 {
		t.Fatalf("expected one node in state, got %d", len(snapshot.Nodes))
	}

	node := snapshot.Nodes[0]
	wantTotal := int64(944 * 1024 * 1024 * 1024)
	wantUsed := int64(313 * 1024 * 1024 * 1024)
	wantFree := int64(631 * 1024 * 1024 * 1024)
	if node.Disk.Total != wantTotal || node.Disk.Used != wantUsed || node.Disk.Free != wantFree {
		t.Fatalf("node disk = %+v, want local-zfs fallback totals", node.Disk)
	}
	if node.Disk.Usage < 33.1 || node.Disk.Usage > 33.2 {
		t.Fatalf("node disk usage = %.2f, want local-zfs usage around 33.16", node.Disk.Usage)
	}
}