Restore LXC memory fallback to the cluster-resources listing

Real PVE guest RRD responses carry only the cache-inclusive mem/maxmem
columns; the cache-aware memused/memavailable columns exist only in node
RRD, so the LXC RRD branches can never match a live response. Removing
the cluster-resources fallback in bf67ba920 therefore left every running
LXC reporting unavailable memory, rendered as 0% (#1634).

Running containers now fall back to the listing value under the
low-trust cluster-resources source when cache-aware RRD evidence is
absent; unavailable remains reserved for running containers with no
listing evidence at all.

Fixes #1634

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

docs/release-control/v6/internal/subsystems/monitoring.md

@@ -1877,8 +1877,15 @@ being mistaken for an absent field.
 Node RRD fallback caches must key on `(instance, node)`, just as guest RRD and
 guest-agent caches key on `(instance, node, vmid)`, so identically named nodes
 in different Proxmox instances cannot exchange memory evidence.
-Running LXC cluster-resource memory is cache-inclusive and therefore cannot be
-used as live usage when both RRD `memavailable` and `memused` are absent.
+Running LXC memory acknowledges a Proxmox API reality: guest RRD responses
+carry only cache-inclusive `mem`/`maxmem` columns — the cache-aware
+`memavailable`/`memused` columns exist only in node RRD — so cache-aware LXC
+evidence is structurally absent from the guest RRD endpoint. When the
+cache-aware RRD branches do not match, a running container with a non-zero
+cluster-resource listing value must fall back to that cache-inclusive value
+under the low-trust `cluster-resources` source rather than reporting the
+guest unavailable; `unavailable` is reserved for running containers with no
+listing evidence at all (#1634).
 Unified Linux and Docker agent ingest likewise must not repair a missing used
 value from total minus free alone; it may use an explicit used/percentage or
 complete free-plus-cache evidence. In every collector, known capacity with no

internal/monitoring/memory_trust_characterization_test.go

@@ -1053,7 +1053,7 @@ func TestHandleClusterContainerResourceMemoryTrustCharacterization(t *testing.T)
 			wantStatus:   "running",
 		},
 		{
-			name: "running LXC without RRD memory stays unknown",
+			name: "running LXC without RRD memory falls back to cluster resources (#1634)",
 			res: proxmox.ClusterResource{
 				ID:     "lxc/204",
 				Type:   "lxc",
@@ -1065,6 +1065,22 @@ func TestHandleClusterContainerResourceMemoryTrustCharacterization(t *testing.T)
 				Mem:    7 * gib,
 				MaxCPU: 4,
 			},
+			wantSource: "cluster-resources",
+			wantUsed:   7 * gib,
+			wantStatus: "running",
+		},
+		{
+			name: "running LXC without RRD memory or listing value stays unknown",
+			res: proxmox.ClusterResource{
+				ID:     "lxc/206",
+				Type:   "lxc",
+				Node:   "node1",
+				Name:   "ct-206",
+				Status: "running",
+				VMID:   206,
+				MaxMem: 8 * gib,
+				MaxCPU: 4,
+			},
 			wantSource:  "unavailable",
 			wantStatus:  "running",
 			wantUnknown: true,

internal/monitoring/monitor_pve_guest_lxc.go

@@ -79,7 +79,17 @@ func (m *Monitor) calculateLXCMemory(
 				Str("instance", instanceName).
 				Str("container", res.Name).
 				Int("vmid", res.VMID).
-				Msg("RRD memory data unavailable for LXC; memory usage remains unavailable")
+				Msg("RRD memory data unavailable for LXC; falling back to cluster resources value")
+		}
+
+		// PVE guest RRD carries only mem/maxmem columns (cgroup values, cache
+		// included) — the cache-aware memused/memavailable columns exist only
+		// in node RRD, so the branches above cannot match a real response.
+		// Without this fallback every running container reports unavailable
+		// memory (issue #1634).
+		if memorySource == "unavailable" && res.Mem > 0 {
+			memUsed = res.Mem
+			memorySource = "cluster-resources"
 		}
 	}
 

internal/monitoring/monitor_pve_guest_lxc_test.go

@@ -175,3 +175,133 @@ func TestBuildContainerFromClusterResource_UsesContainerStatusCountersForRates(t
 		t.Fatalf("expected NetworkOut rate from container status counters, got %d", container.NetworkOut)
 	}
 }
+
+type stubPVEClientLXCRRD struct {
+	stubPVEClient
+
+	lxcRRDPoints []proxmox.GuestRRDPoint
+	lxcRRDErr    error
+}
+
+func (s *stubPVEClientLXCRRD) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe, cf string, ds []string) ([]proxmox.GuestRRDPoint, error) {
+	return s.lxcRRDPoints, s.lxcRRDErr
+}
+
+// Issue #1634: real PVE guest RRD responses carry only mem/maxmem, never the
+// cache-aware memused/memavailable columns, so running containers must fall
+// back to the cluster-resources listing value instead of reporting memory as
+// unavailable (rendered as 0%).
+func TestIssue1634LXCMemoryFallsBackToClusterResourcesOnRealRRDShape(t *testing.T) {
+	t.Parallel()
+
+	maxMem := float64(8 * 1024 * 1024 * 1024)
+	client := &stubPVEClientLXCRRD{
+		lxcRRDPoints: []proxmox.GuestRRDPoint{
+			{Time: 1785164640, MaxMem: &maxMem},
+		},
+	}
+
+	monitor := &Monitor{rateTracker: NewRateTracker()}
+	resource := proxmox.ClusterResource{
+		Type:   "lxc",
+		Node:   "pve-a",
+		Name:   "issue1634-ct",
+		Status: "running",
+		VMID:   108,
+		MaxMem: 8 * 1024 * 1024 * 1024,
+		Mem:    335716352,
+	}
+
+	container, _, memorySource, _, ok := monitor.buildContainerFromClusterResource(
+		context.Background(),
+		"cluster-a",
+		resource,
+		client,
+		map[int]bool{},
+	)
+	if !ok {
+		t.Fatal("expected container sample to be built")
+	}
+	if CanonicalMemorySource(memorySource) != "cluster-resources" {
+		t.Fatalf("memory source = %q, want cluster-resources fallback", memorySource)
+	}
+	if container.Memory.UsageUnavailable {
+		t.Fatal("running LXC memory marked unavailable despite listing value")
+	}
+	if container.Memory.Used != int64(resource.Mem) {
+		t.Fatalf("memory used = %d, want listing value %d", container.Memory.Used, resource.Mem)
+	}
+	if container.Memory.Usage <= 0 {
+		t.Fatalf("memory usage = %f, want > 0", container.Memory.Usage)
+	}
+	if !container.Memory.HasKnownUsage() {
+		t.Fatal("expected fallback memory to be usable for projections")
+	}
+}
+
+func TestIssue1634LXCMemoryFallsBackWhenRRDErrors(t *testing.T) {
+	t.Parallel()
+
+	client := &stubPVEClientLXCRRD{lxcRRDErr: context.DeadlineExceeded}
+
+	monitor := &Monitor{rateTracker: NewRateTracker()}
+	resource := proxmox.ClusterResource{
+		Type:   "lxc",
+		Node:   "pve-a",
+		Name:   "issue1634-rrd-err",
+		Status: "running",
+		VMID:   109,
+		MaxMem: 512 * 1024 * 1024,
+		Mem:    113344512,
+	}
+
+	container, _, memorySource, _, ok := monitor.buildContainerFromClusterResource(
+		context.Background(),
+		"cluster-a",
+		resource,
+		client,
+		map[int]bool{},
+	)
+	if !ok {
+		t.Fatal("expected container sample to be built")
+	}
+	if CanonicalMemorySource(memorySource) != "cluster-resources" {
+		t.Fatalf("memory source = %q, want cluster-resources fallback", memorySource)
+	}
+	if container.Memory.Used != int64(resource.Mem) || container.Memory.UsageUnavailable {
+		t.Fatalf("memory = %+v, want listing fallback", container.Memory)
+	}
+}
+
+func TestIssue1634LXCMemoryStaysUnavailableWithoutListingValue(t *testing.T) {
+	t.Parallel()
+
+	client := &stubPVEClientLXCRRD{}
+
+	monitor := &Monitor{rateTracker: NewRateTracker()}
+	resource := proxmox.ClusterResource{
+		Type:   "lxc",
+		Node:   "pve-a",
+		Name:   "issue1634-no-mem",
+		Status: "running",
+		VMID:   110,
+		MaxMem: 512 * 1024 * 1024,
+	}
+
+	container, _, memorySource, _, ok := monitor.buildContainerFromClusterResource(
+		context.Background(),
+		"cluster-a",
+		resource,
+		client,
+		map[int]bool{},
+	)
+	if !ok {
+		t.Fatal("expected container sample to be built")
+	}
+	if CanonicalMemorySource(memorySource) != "unavailable" {
+		t.Fatalf("memory source = %q, want unavailable when no evidence exists", memorySource)
+	}
+	if !container.Memory.UsageUnavailable {
+		t.Fatal("expected memory to stay marked unavailable without any usage evidence")
+	}
+}