From 8b56fc72e4ee79f3c219cf51f4c002e2e9343286 Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:24:33 +0100 Subject: [PATCH 1/5] fix(monitoring): prefer linked agent memory for Proxmox LXC Extend the #1962 linked-agent memory fallback to Proxmox LXC containers (#2148). A correlated online Pulse agent running inside a container now supplies the container's memory metric instead of the cache-inclusive cluster/resources fallback, which can badly under-report shared-memory workloads. The agent sample is only trusted when its total matches the container's configured limit, so an agent without lxcfs cannot leak host memory into the guest row. Threads the previous-guest agent map through the efficient and node-by-node container builders and reads agent-owned memory from merged ContainerView resources. Change-source: pulse-maintainer --- internal/monitoring/guest_memory_sources.go | 46 ++++++++ .../memory_trust_characterization_test.go | 1 + .../monitoring/monitor_additional_test.go | 39 ++++++ .../monitoring/monitor_polling_containers.go | 10 ++ internal/monitoring/monitor_previous_state.go | 25 ++-- .../monitoring/monitor_proxmox_pool_test.go | 1 + internal/monitoring/monitor_pve_guest_lxc.go | 11 ++ .../monitoring/monitor_pve_guest_lxc_test.go | 111 +++++++++++++++++- internal/monitoring/monitor_pve_guest_poll.go | 10 +- internal/unifiedresources/views.go | 21 +++- internal/unifiedresources/views_test.go | 31 +++++ 11 files changed, 289 insertions(+), 17 deletions(-) diff --git a/internal/monitoring/guest_memory_sources.go b/internal/monitoring/guest_memory_sources.go index 605c83d1a3..d26d7c48f0 100644 --- a/internal/monitoring/guest_memory_sources.go +++ b/internal/monitoring/guest_memory_sources.go @@ -382,3 +382,49 @@ func (m *Monitor) resolveGuestStatusMemory( return memTotal, memUsed, memorySource } + +// preferLinkedAgentLXCMemory replaces a low-trust Proxmox LXC memory reading +// with the linked Pulse agent's own sample when one is available. The agent runs +// inside the container, so its sample reflects the workload's real footprint, +// while cluster/resources is a cache-inclusive cgroup fallback that can badly +// under- or over-report shared-memory workloads (#2148, mirroring #1962). +// +// An agent inside a container without lxcfs sees the host's /proc/meminfo, so a +// sample is only trusted when its total matches the container's configured +// limit. Preferred provider sources are never overridden. +func preferLinkedAgentLXCMemory( + status string, + memTotal uint64, + memUsed uint64, + memorySource string, + guestRaw *VMMemoryRaw, + agentHost models.Host, + hasAgent bool, +) (uint64, uint64, string) { + if !hasAgent || status != "running" || memTotal == 0 { + return memTotal, memUsed, memorySource + } + switch CanonicalMemorySource(memorySource) { + case "cluster-resources", "unavailable": + default: + return memTotal, memUsed, memorySource + } + if !agentHost.Memory.HasKnownUsage() || agentHost.Memory.Total <= 0 || agentHost.Memory.Used < 0 { + return memTotal, memUsed, memorySource + } + agentTotal := uint64(agentHost.Memory.Total) + agentUsed := uint64(agentHost.Memory.Used) + if agentTotal != memTotal || agentUsed > memTotal { + return memTotal, memUsed, memorySource + } + if guestRaw != nil { + guestRaw.HostAgentTotal = agentTotal + guestRaw.HostAgentUsed = agentUsed + } + log.Debug(). + Uint64("total", memTotal). + Uint64("used", agentUsed). + Str("previousSource", memorySource). + Msg("LXC memory: using linked Pulse agent memory over cluster-resources fallback") + return memTotal, agentUsed, "agent" +} diff --git a/internal/monitoring/memory_trust_characterization_test.go b/internal/monitoring/memory_trust_characterization_test.go index f671d1678d..2d12f5091a 100644 --- a/internal/monitoring/memory_trust_characterization_test.go +++ b/internal/monitoring/memory_trust_characterization_test.go @@ -1004,6 +1004,7 @@ func TestHandleClusterContainerResourceMemoryTrustCharacterization(t *testing.T) makeGuestID("test", "node1", tt.res.VMID), client, nil, + nil, ) if !ok { t.Fatal("handleClusterContainerResource() returned ok=false") diff --git a/internal/monitoring/monitor_additional_test.go b/internal/monitoring/monitor_additional_test.go index bba390a312..15a9875f7b 100644 --- a/internal/monitoring/monitor_additional_test.go +++ b/internal/monitoring/monitor_additional_test.go @@ -1009,6 +1009,45 @@ func TestCorrelatedGuestMemoryNextPoll(t *testing.T) { } } +// Issue #2148: a correlated agent merged into a system-container resource must +// surface through the previous-guest context so the LXC builder prefers it over +// the cache-inclusive cluster/resources fallback. +func TestCorrelatedContainerMemoryNextPoll(t *testing.T) { + now := time.Now() + const gib = int64(1024 * 1024 * 1024) + guestID := makeGuestID("pve-a", "node1", 100) + store := unifiedresources.NewMemoryStore() + if err := store.AddLink(unifiedresources.ResourceLink{ResourceA: "ct-test", ResourceB: "agent-test", PrimaryID: "agent-test"}); err != nil { + t.Fatal(err) + } + registry := unifiedresources.NewRegistry(store) + total, used := int64(24)*gib, int64(11)*gib + resources := []unifiedresources.Resource{ + {ID: "ct-test", Type: unifiedresources.ResourceTypeSystemContainer, Name: "npu-ct", Status: unifiedresources.StatusOnline, LastSeen: now, + Sources: []unifiedresources.DataSource{unifiedresources.SourceProxmox}, + Proxmox: &unifiedresources.ProxmoxData{SourceID: guestID, Instance: "pve-a", NodeName: "node1", VMID: 100, RuntimeStatus: "running"}, + Metrics: &unifiedresources.ResourceMetrics{Memory: &unifiedresources.MetricValue{Total: &total, Used: &used, Percent: 45, Source: unifiedresources.SourceProxmox}}}, + {ID: "agent-test", Type: unifiedresources.ResourceTypeAgent, Name: "npu-ct-agent", Status: unifiedresources.StatusOnline, LastSeen: now, + Sources: []unifiedresources.DataSource{unifiedresources.SourceAgent}, + Agent: &unifiedresources.AgentData{AgentID: "agent-test", Memory: &unifiedresources.AgentMemoryMeta{Total: total, Used: used, Free: total - used}}}, + } + registry.IngestResources(resources) + if len(registry.Containers()) != 1 || len(registry.Hosts()) != 0 { + t.Fatalf("expected one merged container and no standalone hosts, got %d/%d", len(registry.Containers()), len(registry.Hosts())) + } + mon := &Monitor{state: models.NewState(), rateTracker: NewRateTracker(), config: &config.Config{}, resourceStore: unifiedresources.NewMonitorAdapter(registry)} + prev := mon.previousGuestContextForInstance("pve-a") + if _, ok := prev.hostAgentsByVMID[guestID]; !ok { + t.Fatalf("linked container agent missing from previous context: %+v", prev.hostAgentsByVMID) + } + container, _, source, _, ok := mon.buildContainerFromClusterResource(context.Background(), "pve-a", + proxmox.ClusterResource{Type: "lxc", Status: "running", Node: "node1", VMID: 100, Name: "npu-ct", MaxMem: uint64(total), Mem: 3459743744}, + &stubPVEClient{}, map[int]bool{}, prev.hostAgentsByVMID) + if !ok || source != "agent" || container.Memory.Used != used { + t.Fatalf("container memory not agent-backed: ok=%v source=%s mem=%+v", ok, source, container.Memory) + } +} + type correlatedMemoryClient struct{ stubPVEClient } func (*correlatedMemoryClient) GetVMStatus(context.Context, string, int) (*proxmox.VMStatus, error) { diff --git a/internal/monitoring/monitor_polling_containers.go b/internal/monitoring/monitor_polling_containers.go index c3148f7f37..f9313d81a0 100644 --- a/internal/monitoring/monitor_polling_containers.go +++ b/internal/monitoring/monitor_polling_containers.go @@ -154,6 +154,16 @@ func (m *Monitor) collectContainersWithNodes(ctx context.Context, instanceName s MaxMem: container.MaxMem, Mem: container.Mem, }) + agentHost, hasAgent := prevGuests.hostAgentsByVMID[guestID] + memTotal, memUsed, memorySource = preferLinkedAgentLXCMemory( + container.Status, + memTotal, + memUsed, + memorySource, + &guestRaw, + agentHost, + hasAgent, + ) memUsed, memorySource, _ = stabilizeGuestLowTrustMemory( m.previousGuestSnapshot(instanceName, "lxc", n.Node, int(container.VMID)), container.Status, diff --git a/internal/monitoring/monitor_previous_state.go b/internal/monitoring/monitor_previous_state.go index 86abbc1ac9..bfd30496c9 100644 --- a/internal/monitoring/monitor_previous_state.go +++ b/internal/monitoring/monitor_previous_state.go @@ -61,6 +61,9 @@ func (m *Monitor) previousGuestContextForInstance(instanceName string) previousG guestID := makeGuestID(container.Instance, container.Node, container.VMID) if guestID != "" { ctx.containersByID[guestID] = container + if memory, ok := ct.LinkedAgentMemory(); ok { + ctx.hostAgentsByVMID[guestID] = models.Host{LinkedVMID: guestID, Status: "online", Memory: memory} + } } if container.VMID > 0 && (strings.EqualFold(strings.TrimSpace(container.Type), "oci") || container.IsOCI) { ctx.containerOCIByVMID[container.VMID] = true @@ -72,10 +75,15 @@ func (m *Monitor) previousGuestContextForInstance(instanceName string) previousG continue } modelHost := previousHostFromView(host) - if modelHost.LinkedVMID == "" || modelHost.Status != "online" { + if modelHost.Status != "online" { continue } - ctx.hostAgentsByVMID[modelHost.LinkedVMID] = modelHost + if modelHost.LinkedVMID != "" { + ctx.hostAgentsByVMID[modelHost.LinkedVMID] = modelHost + } + if modelHost.LinkedContainerID != "" { + ctx.hostAgentsByVMID[modelHost.LinkedContainerID] = modelHost + } } return ctx @@ -172,12 +180,13 @@ func previousHostFromView(host *unifiedresources.HostView) models.Host { return models.Host{} } return models.Host{ - ID: host.ID(), - Hostname: host.Hostname(), - Status: string(host.Status()), - LinkedVMID: host.LinkedVMID(), - LastSeen: host.LastSeen(), - Disks: guestDisksFromReadStateView(host.Disks()), + ID: host.ID(), + Hostname: host.Hostname(), + Status: string(host.Status()), + LinkedVMID: host.LinkedVMID(), + LinkedContainerID: host.LinkedContainerID(), + LastSeen: host.LastSeen(), + Disks: guestDisksFromReadStateView(host.Disks()), Memory: models.Memory{ Used: host.MemoryUsed(), Total: host.MemoryTotal(), diff --git a/internal/monitoring/monitor_proxmox_pool_test.go b/internal/monitoring/monitor_proxmox_pool_test.go index 5e0ccca8ce..87eab29900 100644 --- a/internal/monitoring/monitor_proxmox_pool_test.go +++ b/internal/monitoring/monitor_proxmox_pool_test.go @@ -58,6 +58,7 @@ func TestBuildContainerFromClusterResource_PreservesProxmoxPool(t *testing.T) { }, nil, map[int]bool{}, + nil, ) if !ok { t.Fatal("expected container to be built") diff --git a/internal/monitoring/monitor_pve_guest_lxc.go b/internal/monitoring/monitor_pve_guest_lxc.go index 7cbb4c3799..6654435e9c 100644 --- a/internal/monitoring/monitor_pve_guest_lxc.go +++ b/internal/monitoring/monitor_pve_guest_lxc.go @@ -44,6 +44,7 @@ func (m *Monitor) buildContainerFromClusterResource( res proxmox.ClusterResource, client PVEClientInterface, prevContainerIsOCI map[int]bool, + vmIDToHostAgent map[string]models.Host, ) (models.Container, VMMemoryRaw, string, time.Time, bool) { // Skip templates if configured if res.Template == 1 { @@ -88,6 +89,16 @@ func (m *Monitor) buildContainerFromClusterResource( ) memTotal, memUsed, memorySource, guestRaw := m.calculateLXCMemory(res) + agentHost, hasAgent := vmIDToHostAgent[guestID] + memTotal, memUsed, memorySource = preferLinkedAgentLXCMemory( + res.Status, + memTotal, + memUsed, + memorySource, + &guestRaw, + agentHost, + hasAgent, + ) memUsed, memorySource, _ = stabilizeGuestLowTrustMemory( m.previousGuestSnapshot(instanceName, "lxc", res.Node, res.VMID), res.Status, diff --git a/internal/monitoring/monitor_pve_guest_lxc_test.go b/internal/monitoring/monitor_pve_guest_lxc_test.go index 4b1e82bb6f..93fd917bef 100644 --- a/internal/monitoring/monitor_pve_guest_lxc_test.go +++ b/internal/monitoring/monitor_pve_guest_lxc_test.go @@ -140,7 +140,7 @@ func TestIssue1613LXCStatusLagDoesNotEraseNewerListingDiskWrite(t *testing.T) { } if _, _, _, _, ok := monitor.buildContainerFromClusterResource( - context.Background(), "cluster-a", resource, client, map[int]bool{}, + context.Background(), "cluster-a", resource, client, map[int]bool{}, nil, ); !ok { t.Fatal("expected first LXC sample") } @@ -150,7 +150,7 @@ func TestIssue1613LXCStatusLagDoesNotEraseNewerListingDiskWrite(t *testing.T) { client.containerStatus.ObservedAt = resource.ObservedAt.Add(time.Second) container, _, _, _, ok := monitor.buildContainerFromClusterResource( - context.Background(), "cluster-a", resource, client, map[int]bool{}, + context.Background(), "cluster-a", resource, client, map[int]bool{}, nil, ) if !ok { t.Fatal("expected second LXC sample") @@ -199,6 +199,7 @@ func TestBuildContainerFromClusterResource_UsesContainerStatusCountersForRates(t resource, client, map[int]bool{}, + nil, ); !ok { t.Fatal("expected first container sample to be built") } @@ -219,6 +220,7 @@ func TestBuildContainerFromClusterResource_UsesContainerStatusCountersForRates(t resource, client, map[int]bool{}, + nil, ) if !ok { t.Fatal("expected second container sample to be built") @@ -282,6 +284,7 @@ func TestIssue1634LXCMemoryFallsBackToClusterResourcesOnRealRRDShape(t *testing. resource, client, map[int]bool{}, + nil, ) if !ok { t.Fatal("expected container sample to be built") @@ -327,6 +330,7 @@ func TestIssue1634LXCMemoryStaysUnavailableWithoutListingValue(t *testing.T) { resource, client, map[int]bool{}, + nil, ) if !ok { t.Fatal("expected container sample to be built") @@ -443,6 +447,7 @@ func TestBuildContainerFromClusterResource_AppliesAgentLXCFilesystems(t *testing resource, client, map[int]bool{}, + nil, ) if !ok { t.Fatal("expected container to be built") @@ -460,3 +465,105 @@ func TestBuildContainerFromClusterResource_AppliesAgentLXCFilesystems(t *testing t.Fatalf("expected /data mount with real usage from agent data, got %+v", container.Disks) } } + +// Issue #2148: a Proxmox LXC with a correlated, online Pulse agent must report +// the agent's own memory sample instead of the cache-inclusive +// cluster/resources fallback, which can badly under-report shared-memory +// workloads (mirrors the #1962 VM behaviour). +func TestIssue2148LXCPrefersLinkedAgentMemoryOverClusterResources(t *testing.T) { + t.Parallel() + + const gib = 1024 * 1024 * 1024 + + client := &stubPVEClientLXCRRD{} + monitor := &Monitor{rateTracker: NewRateTracker()} + resource := proxmox.ClusterResource{ + Type: "lxc", + Node: "pve-a", + Name: "npu-ct", + Status: "running", + VMID: 100, + MaxMem: 24 * gib, + Mem: 3459743744, + } + guestID := makeGuestID("cluster-a", "pve-a", 100) + agentHost := models.Host{ + LinkedVMID: guestID, + Status: "online", + Memory: models.Memory{ + Total: 24 * gib, + Used: 11025571200, + Free: 24*gib - 11025571200, + }, + } + + container, _, memorySource, _, ok := monitor.buildContainerFromClusterResource( + context.Background(), + "cluster-a", + resource, + client, + map[int]bool{}, + map[string]models.Host{guestID: agentHost}, + ) + if !ok { + t.Fatal("expected container sample to be built") + } + if memorySource != "agent" { + t.Fatalf("memory source = %q, want agent", memorySource) + } + if container.Memory.Used != agentHost.Memory.Used { + t.Fatalf("memory used = %d, want linked agent value %d", container.Memory.Used, agentHost.Memory.Used) + } + if container.Memory.UsageUnavailable || !container.Memory.HasKnownUsage() { + t.Fatalf("expected usable agent-backed memory, got %+v", container.Memory) + } +} + +// An agent inside a container without lxcfs sees the host's /proc/meminfo, so a +// sample whose total does not match the container's configured limit must not +// replace the provider reading. +func TestIssue2148LXCRejectsAgentMemoryWithMismatchedTotal(t *testing.T) { + t.Parallel() + + const gib = 1024 * 1024 * 1024 + + client := &stubPVEClientLXCRRD{} + monitor := &Monitor{rateTracker: NewRateTracker()} + resource := proxmox.ClusterResource{ + Type: "lxc", + Node: "pve-a", + Name: "host-memory-leak-ct", + Status: "running", + VMID: 101, + MaxMem: 24 * gib, + Mem: 3459743744, + } + guestID := makeGuestID("cluster-a", "pve-a", 101) + agentHost := models.Host{ + LinkedVMID: guestID, + Status: "online", + Memory: models.Memory{ + Total: 128 * gib, + Used: 40 * gib, + Free: 88 * gib, + }, + } + + container, _, memorySource, _, ok := monitor.buildContainerFromClusterResource( + context.Background(), + "cluster-a", + resource, + client, + map[int]bool{}, + map[string]models.Host{guestID: agentHost}, + ) + if !ok { + t.Fatal("expected container sample to be built") + } + if CanonicalMemorySource(memorySource) != "cluster-resources" { + t.Fatalf("memory source = %q, want cluster-resources fallback", memorySource) + } + if container.Memory.Used != int64(resource.Mem) { + t.Fatalf("memory used = %d, want provider value %d", container.Memory.Used, resource.Mem) + } +} diff --git a/internal/monitoring/monitor_pve_guest_poll.go b/internal/monitoring/monitor_pve_guest_poll.go index 879035d7cc..7c6f4e6bd0 100644 --- a/internal/monitoring/monitor_pve_guest_poll.go +++ b/internal/monitoring/monitor_pve_guest_poll.go @@ -147,7 +147,7 @@ func (m *Monitor) collectGuestsFromClusterResources( collectionWG.Add(1) go func() { defer collectionWG.Done() - allContainers = m.collectClusterContainerResources(ctx, instanceName, containerResources, client, prevContainerIsOCI, prevContainerByID) + allContainers = m.collectClusterContainerResources(ctx, instanceName, containerResources, client, prevContainerIsOCI, prevContainerByID, vmIDToHostAgent) }() } if len(vmResources) > 0 { @@ -253,6 +253,7 @@ func (m *Monitor) collectClusterContainerResources( client PVEClientInterface, prevContainerIsOCI map[int]bool, prevContainerByID map[string]models.Container, + vmIDToHostAgent map[string]models.Host, ) []models.Container { orderedContainers := make([]models.Container, len(resources)) orderedOK := make([]bool, len(resources)) @@ -263,13 +264,13 @@ func (m *Monitor) collectClusterContainerResources( var container models.Container var ok bool ran := m.runGuestAgentVMWork(ctx, func(workCtx context.Context) { - container, ok = m.handleClusterContainerResource(workCtx, instanceName, entry.resource, entry.guestID, client, prevContainerIsOCI) + container, ok = m.handleClusterContainerResource(workCtx, instanceName, entry.resource, entry.guestID, client, prevContainerIsOCI, vmIDToHostAgent) }) if !ran { // The enrichment budget is exhausted, but cluster/resources is still // authoritative inventory. A canceled context makes remote detail // calls fail immediately while the builder retains the base row. - container, ok = m.handleClusterContainerResource(ctx, instanceName, entry.resource, entry.guestID, client, prevContainerIsOCI) + container, ok = m.handleClusterContainerResource(ctx, instanceName, entry.resource, entry.guestID, client, prevContainerIsOCI, vmIDToHostAgent) } if ctx.Err() != nil { previous := prevContainerByID[entry.guestID] @@ -415,8 +416,9 @@ func (m *Monitor) handleClusterContainerResource( guestID string, client PVEClientInterface, prevContainerIsOCI map[int]bool, + vmIDToHostAgent map[string]models.Host, ) (models.Container, bool) { - container, guestRaw, memorySource, sampleTime, ok := m.buildContainerFromClusterResource(ctx, instanceName, res, client, prevContainerIsOCI) + container, guestRaw, memorySource, sampleTime, ok := m.buildContainerFromClusterResource(ctx, instanceName, res, client, prevContainerIsOCI, vmIDToHostAgent) if !ok { return models.Container{}, false } diff --git a/internal/unifiedresources/views.go b/internal/unifiedresources/views.go index ff0a0b79a0..d6440bdff2 100644 --- a/internal/unifiedresources/views.go +++ b/internal/unifiedresources/views.go @@ -78,13 +78,20 @@ func NewVMView(r *Resource) VMView { return VMView{r: r} } // LinkedAgentMemory returns the agent's own sample, not the platform-priority // merged metric. Freshness belongs to the agent source, not the VM row. func (v VMView) LinkedAgentMemory() (models.Memory, bool) { - if v.r == nil || v.r.Agent == nil || v.r.Agent.Stale || v.r.Agent.Memory == nil { + return linkedAgentMemoryFromResource(v.r) +} + +// linkedAgentMemoryFromResource reads the agent-owned memory sample attached to +// a merged guest resource. Correlation removes the standalone host row, so the +// guest view is the only place the agent sample survives (#1962, #2148). +func linkedAgentMemoryFromResource(r *Resource) (models.Memory, bool) { + if r == nil || r.Agent == nil || r.Agent.Stale || r.Agent.Memory == nil { return models.Memory{}, false } - if status, ok := v.r.SourceStatus[SourceAgent]; !ok || status.Status != "online" { + if status, ok := r.SourceStatus[SourceAgent]; !ok || status.Status != "online" { return models.Memory{}, false } - m := v.r.Agent.Memory + m := r.Agent.Memory memory := models.Memory{Total: m.Total, Used: m.Used, Free: m.Free, Cache: m.Cache, UsageUnavailable: m.UsageUnavailable} if m.Total <= 0 { return models.Memory{}, false @@ -397,6 +404,14 @@ type ContainerView struct{ r *Resource } func NewContainerView(r *Resource) ContainerView { return ContainerView{r: r} } +// LinkedAgentMemory returns the agent's own sample for a system container whose +// merged resource also carries an online Pulse agent source. Proxmox +// cluster/resources memory is a low-trust fallback that can badly misreport a +// container's real footprint, so the linked agent sample is preferred (#2148). +func (v ContainerView) LinkedAgentMemory() (models.Memory, bool) { + return linkedAgentMemoryFromResource(v.r) +} + func (v ContainerView) String() string { return fmt.Sprintf("ContainerView(%s, %q)", v.ID(), v.Name()) } func (v ContainerView) ID() string { diff --git a/internal/unifiedresources/views_test.go b/internal/unifiedresources/views_test.go index f01847ae18..2ec8712040 100644 --- a/internal/unifiedresources/views_test.go +++ b/internal/unifiedresources/views_test.go @@ -2267,3 +2267,34 @@ func TestVMViewLinkedAgentMemory(t *testing.T) { t.Fatal("nil view has memory") } } + +func TestContainerViewLinkedAgentMemory(t *testing.T) { + for _, tc := range []struct { + name string + status string + stale bool + total, used int64 + unavailable bool + want bool + }{ + {name: "live", status: "online", total: 24 << 30, used: 11 << 30, want: true}, + {name: "offline", status: "offline", total: 24 << 30, used: 11 << 30}, + {name: "stale agent", status: "online", stale: true, total: 24 << 30, used: 11 << 30}, + {name: "over capacity", status: "online", total: 24 << 30, used: 25 << 30}, + {name: "unavailable", status: "online", total: 24 << 30, used: 11 << 30, unavailable: true}, + } { + t.Run(tc.name, func(t *testing.T) { + r := &Resource{Status: StatusOnline, Agent: &AgentData{Stale: tc.stale, Memory: &AgentMemoryMeta{Total: tc.total, Used: tc.used, UsageUnavailable: tc.unavailable}}, SourceStatus: map[DataSource]SourceStatus{SourceAgent: {Status: tc.status}}, Metrics: &ResourceMetrics{Memory: &MetricValue{Percent: 100}}} + memory, ok := NewContainerView(r).LinkedAgentMemory() + if ok != tc.want { + t.Fatalf("valid=%v want=%v memory=%+v", ok, tc.want, memory) + } + if ok && memory.Used != tc.used { + t.Fatalf("not agent memory: %+v", memory) + } + }) + } + if _, ok := (ContainerView{}).LinkedAgentMemory(); ok { + t.Fatal("nil view has memory") + } +} From d5de3e26fcf8fe0330982426d84580a8b309b884 Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 04:28:39 +0100 Subject: [PATCH 2/5] fix(release-control): qualify release-line RCs against their own branch The secure-runtime RC qualification workflow (commit 79d87a4a2c) resolves the branch the control plane maps the version to and passes it as --main-ref, but secure_runtime_attestation_v6.verify_canonical_main_identity still required origin/main and required the candidate to be an ancestor of origin/main. Every release-train candidate therefore failed the pre-authentication step before any check ran; v6.4.5-rc.1 failed at 'Pre-authenticate exact qualification packet' with 'committed-main classification requires canonical origin/main' (run 35680955017), and all three historical runs failed. Accept exactly origin/main or the release train's origin/release/vX.Y, verify the ref against the canonical origin's live branch tip, and check candidate ancestry against that same ref. The release-candidate classification is unchanged. Update the deployment-installability contract and the v6 unit tests, including a mapped-release-branch acceptance case. Change-source: pulse-maintainer --- .../subsystems/deployment-installability.md | 7 +++- .../secure_runtime_attestation_v6.py | 27 +++++++----- .../secure_runtime_attestation_v6_test.py | 41 +++++++++++++++++-- 3 files changed, 61 insertions(+), 14 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index eaef072958..71c46fa75c 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -534,7 +534,12 @@ workflow execution ref and SHA equal the peeled tag commit. That workflow consum release assets and four release signatures, runs the canonical twenty-three-scenario schema-v7 lab, and applies the release-candidate attester against the exact GitHub release ID, tag, source commit, checksums, compiler provenance, assembly -provenance, and update-key fingerprint. The v7 host starts a rootful Docker +provenance, and update-key fingerprint. The attester anchors the candidate to +the branch the control plane maps the version to, either `origin/main` or the +release train's `origin/release/vX.Y`, and verifies that remote-tracking ref +against the canonical origin's live branch tip, so a release-train candidate is +qualified against the line that owns its version rather than being required to +appear on `main`. The v7 host starts a rootful Docker daemon inside the disposable systemd container without mounting the hosted runner socket or exposing a TCP listener. It imports an offline, source-bound fixture image and proves legacy inventory, summary-only typed-helper parity, diff --git a/scripts/release_control/secure_runtime_attestation_v6.py b/scripts/release_control/secure_runtime_attestation_v6.py index 3e063499fa..aae6b3f72a 100644 --- a/scripts/release_control/secure_runtime_attestation_v6.py +++ b/scripts/release_control/secure_runtime_attestation_v6.py @@ -32,6 +32,7 @@ ATTESTATION_TOOL_PATH = "scripts/release_control/secure_runtime_attestation_v6.p CANONICAL_REPOSITORY = "rcourtman/Pulse" CANONICAL_ORIGIN_URL = "https://github.com/rcourtman/Pulse.git" CANONICAL_MAIN_REF = "origin/main" +CANONICAL_BRANCH_REF_RE = re.compile(r"^origin/(main|release/v[0-9]+\.[0-9]+)$") ASSEMBLY_SIGNER_WORKFLOW = "github.com/rcourtman/Pulse/.github/workflows/build-release-candidate.yml" COMPILER_SIGNER_WORKFLOW = "github.com/rcourtman/Pulse/.github/workflows/compile-release-payload.yml" RELEASE_TAG_RE = re.compile(r"^v[0-9]+\.[0-9]+\.[0-9]+-rc\.[1-9][0-9]*$") @@ -167,17 +168,23 @@ def parse_remote_refs(raw: bytes) -> dict[str, str]: def verify_canonical_main_identity(checkout: Path, main_ref: str) -> str: - if main_ref != CANONICAL_MAIN_REF: - raise v5.AttestationError("committed-main classification requires canonical origin/main") + match = CANONICAL_BRANCH_REF_RE.fullmatch(main_ref) + if match is None: + raise v5.AttestationError( + "committed-main classification requires a canonical origin branch ref" + ) + branch = match.group(1) origin = v5.run_git(checkout, "remote", "get-url", "origin").stdout.decode().strip() if origin != CANONICAL_ORIGIN_URL: raise v5.AttestationError("origin remote URL is not the canonical Pulse repository URL") - local_main = v5.resolve_commit(checkout, CANONICAL_MAIN_REF, "canonical origin/main") - remote = v5.run_git(checkout, "ls-remote", "origin", "refs/heads/main") + local_branch = v5.resolve_commit(checkout, main_ref, f"canonical origin/{branch}") + remote = v5.run_git(checkout, "ls-remote", "origin", f"refs/heads/{branch}") remote_refs = parse_remote_refs(remote.stdout) - if remote_refs != {"refs/heads/main": local_main}: - raise v5.AttestationError("canonical origin/main does not match the remote main commit") - return local_main + if remote_refs != {f"refs/heads/{branch}": local_branch}: + raise v5.AttestationError( + f"canonical origin/{branch} does not match the remote branch commit" + ) + return local_branch def verify_release_candidate_tag_identity( @@ -609,7 +616,7 @@ def verify_and_snapshot_release_candidate_packet( raise v5.AttestationError("qualified commit must be the full canonical commit SHA") v5.require_detached_clean_checkout(checkout, resolved_commit) main_commit = verify_canonical_main_identity(checkout, main_ref) - v5.require_ancestor(checkout, resolved_commit, CANONICAL_MAIN_REF) + v5.require_ancestor(checkout, resolved_commit, main_ref) expected_assets = expected_release_asset_names(architecture) expected_versions = expected_release_artifact_versions(tag) @@ -952,7 +959,7 @@ def _create_attestation_with_snapshotted_artifacts( raise v5.AttestationError("qualified commit must be the full canonical commit SHA") v5.require_detached_clean_checkout(checkout, qualified_commit) main_commit = verify_canonical_main_identity(checkout, main_ref) - v5.require_ancestor(checkout, qualified_commit, CANONICAL_MAIN_REF) + v5.require_ancestor(checkout, qualified_commit, main_ref) with v6_contract(): receipt, receipt_bytes = v5.load_receipt(receipt_path) record_path = v5.canonical_repo_path(receipt_record_path, "receipt record path") @@ -1018,7 +1025,7 @@ def _create_attestation_with_snapshotted_artifacts( "qualified_ref_at_run": release_candidate_tag or qualified_commit, "main_ref_verified": main_ref, "main_ref_commit_at_attestation": main_commit, - "qualified_commit_reachable_from_main": True, + "qualified_commit_reachable_from_main": main_ref == CANONICAL_MAIN_REF, "build_checkout": "detached-worktree", "build_checkout_clean_except_lab_artifacts": True, "disposable_vm_guard_receipt_claim_validated": True, diff --git a/scripts/release_control/secure_runtime_attestation_v6_test.py b/scripts/release_control/secure_runtime_attestation_v6_test.py index 91db466794..0235605428 100644 --- a/scripts/release_control/secure_runtime_attestation_v6_test.py +++ b/scripts/release_control/secure_runtime_attestation_v6_test.py @@ -468,9 +468,9 @@ class SecureRuntimeAttestationV6Test(unittest.TestCase): verify_canonical_main_identity(self.root, CANONICAL_MAIN_REF), self.commit, ) - for caller_ref in ("HEAD", "main", "refs/heads/main", "scratch"): + for caller_ref in ("HEAD", "main", "refs/heads/main", "scratch", "origin/feature/x"): with self.subTest(caller_ref=caller_ref), self.assertRaisesRegex( - v5.AttestationError, "canonical origin/main" + v5.AttestationError, "canonical origin branch ref" ): verify_canonical_main_identity(self.root, caller_ref) @@ -482,10 +482,45 @@ class SecureRuntimeAttestationV6Test(unittest.TestCase): with ( mock.patch.object(v5, "run_git", side_effect=moved_remote), - self.assertRaisesRegex(v5.AttestationError, "does not match the remote main commit"), + self.assertRaisesRegex(v5.AttestationError, "does not match the remote branch commit"), ): verify_canonical_main_identity(self.root, CANONICAL_MAIN_REF) + def test_committed_main_identity_accepts_mapped_release_branch(self) -> None: + release_ref = "origin/release/v6.4" + + def canonical(_checkout, *args, **_kwargs): + command = tuple(args) + if command == ("remote", "get-url", "origin"): + output = CANONICAL_ORIGIN_URL + "\n" + elif command == ("rev-parse", "--verify", f"{release_ref}^{{commit}}"): + output = self.commit + "\n" + elif command == ("ls-remote", "origin", "refs/heads/release/v6.4"): + output = f"{self.commit}\trefs/heads/release/v6.4\n" + else: + raise AssertionError(f"unexpected git call {command}") + return subprocess.CompletedProcess(args, 0, output.encode(), b"") + + with mock.patch.object(v5, "run_git", side_effect=canonical): + self.assertEqual( + verify_canonical_main_identity(self.root, release_ref), + self.commit, + ) + + def moved_remote(_checkout, *args, **kwargs): + result = canonical(_checkout, *args, **kwargs) + if tuple(args) == ("ls-remote", "origin", "refs/heads/release/v6.4"): + return subprocess.CompletedProcess( + args, 0, f"{'c' * 40}\trefs/heads/release/v6.4\n".encode(), b"" + ) + return result + + with ( + mock.patch.object(v5, "run_git", side_effect=moved_remote), + self.assertRaisesRegex(v5.AttestationError, "does not match the remote branch commit"), + ): + verify_canonical_main_identity(self.root, release_ref) + def test_accepts_canonical_annotated_tag_only_as_release_packet_identity(self) -> None: with mock.patch.object(v5, "run_git", side_effect=lambda _checkout, *args, **_kwargs: self.git_result(*args)): identity = verify_release_candidate_tag_identity( From 50405af74397df5c217e8e83f17e33d73121da83 Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 08:01:27 +0100 Subject: [PATCH 3/5] fix(alerts): skip backup-age alerts for PBS host backups (#2136) A PBS host/config backup (backup-type host) is keyed by the node name, not a guest VMID, so CheckBackupsWithInventory raised a guest backup-age alert under a node-named subject (for example 'nas backup via proxmox-pbs') that never advanced while the node's guests were backed up, notifying repeatedly. Skip backup-age alerts for PBS subjects with no guest VMID: they are host/config backups, not guest workloads, and the Backups overview already keeps them out of guest coverage. PVE host backups already produce no subject and no alert, restoring consistency. Genuine orphaned PBS guest backups (numeric VMID) and PMG node backups still alert. The recovery point and rollup are retained, so inventory and artifact display are unchanged; only the spurious alert is cleared. Includes the alerts subsystem contract record and the regression tests. Change-source: pulse-maintainer --- .../v6/internal/subsystems/alerts.md | 12 +++ internal/alerts/alerts_test.go | 90 +++++++++++++++++++ internal/alerts/backup_snapshot.go | 10 +++ 3 files changed, 112 insertions(+) diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index 4f53856f97..8c9ef7805c 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -1620,6 +1620,18 @@ of a reporter installation or a migration that guesses operator intent. `TestMergeSnapshotOverrideInheritsZeroFields` in `internal/alerts/alerts_test.go` pin the merge, migration, and preservation behaviors. +Backup-age alerts are guest-workload scoped. A PBS recovery rollup whose subject +carries no guest VMID is a host/config backup (`backup-type host`, keyed by the +node name), not a guest workload, and must not raise a guest backup-age alert: +the Backups overview already excludes those entries from guest coverage, and a +stale host config backup otherwise notified forever under a node-named subject +while the node's guests were backed up. Evaluation skips PBS subjects with an +empty VMID, retaining their recovery point and rollup so inventory and artifact +display are unchanged. PVE host backups already produce no subject and no alert. +Genuinely orphaned PBS guest backups (numeric VMID) and PMG node backups still +alert. `TestCheckBackupsIgnoresPBSHostBackupSubject` and +`TestCheckBackupsStillAlertsOnOrphanedGuestPBSBackup` in +`internal/alerts/alerts_test.go` pin both sides of this boundary (#2136). Proxmox disk health alert evaluation now lives in `internal/alerts/disk_health.go`. That file owns Proxmox disk canonical identity, disk health assessment alerts, known-firmware health suppression, and diff --git a/internal/alerts/alerts_test.go b/internal/alerts/alerts_test.go index 0cca818bb0..48c8f041bc 100644 --- a/internal/alerts/alerts_test.go +++ b/internal/alerts/alerts_test.go @@ -2322,6 +2322,96 @@ func TestCheckBackupsHandlesPmgBackups(t *testing.T) { } } +// TestCheckBackupsIgnoresPBSHostBackupSubject covers #2136: a PBS host config +// backup (backup-type "host") is keyed by the node name, not a VMID, and must +// not raise a guest backup-age alert. The node name subject ("nas backup") +// otherwise looks like a stale guest and notifies forever even though the +// node's guests are backed up. +func TestCheckBackupsIgnoresPBSHostBackupSubject(t *testing.T) { + m := newTestManager(t) + m.ClearActiveAlerts() + + m.mu.Lock() + m.config.Enabled = true + m.config.BackupDefaults = BackupAlertConfig{ + Enabled: true, + WarningDays: 7, + CriticalDays: 14, + } + m.mu.Unlock() + + now := time.Now() + rollups := []recovery.ProtectionRollup{ + { + RollupID: "ext:pbs-host-nas", + SubjectRef: &recovery.ExternalRef{ + Type: "proxmox-guest", + Namespace: "pbs", + Name: "nas", + ID: "nas", + }, + LastSuccessAt: ptrTime(now.Add(-43.5 * 24 * time.Hour)), + LastOutcome: recovery.OutcomeSuccess, + Providers: []recovery.Provider{recovery.ProviderProxmoxPBS}, + }, + } + + m.CheckBackups(rollups, map[string]GuestLookup{}, map[string][]GuestLookup{}) + + m.mu.RLock() + defer m.mu.RUnlock() + for storageKey, alert := range m.activeAlerts { + if alert != nil && alert.Type == "backup-age" { + t.Fatalf("expected PBS host config backup to be excluded from guest backup-age alerts, found %s", effectiveAlertID(alert, storageKey)) + } + } +} + +// TestCheckBackupsStillAlertsOnOrphanedGuestPBSBackup is the companion guard: +// a genuinely orphaned PBS guest backup (numeric VMID, no live guest) must keep +// alerting, so the host-backup exclusion above does not silence real staleness. +func TestCheckBackupsStillAlertsOnOrphanedGuestPBSBackup(t *testing.T) { + m := newTestManager(t) + m.ClearActiveAlerts() + + m.mu.Lock() + m.config.Enabled = true + m.config.BackupDefaults = BackupAlertConfig{ + Enabled: true, + WarningDays: 7, + CriticalDays: 14, + } + m.mu.Unlock() + + now := time.Now() + rollups := []recovery.ProtectionRollup{ + { + RollupID: "ext:pbs-orphan-102", + SubjectRef: &recovery.ExternalRef{ + Type: "proxmox-vm", + Namespace: "pbs", + Name: "102", + ID: "102", + }, + LastSuccessAt: ptrTime(now.Add(-43.5 * 24 * time.Hour)), + LastOutcome: recovery.OutcomeSuccess, + Providers: []recovery.Provider{recovery.ProviderProxmoxPBS}, + }, + } + + m.CheckBackups(rollups, map[string]GuestLookup{}, map[string][]GuestLookup{}) + + m.mu.RLock() + defer m.mu.RUnlock() + if !testHasActiveAlert(t, m, buildCanonicalStateID("backup-subject:ext-pbs-orphan-102", "backup-subject:ext-pbs-orphan-102-backup-age")) { + var keys []string + for storageKey, active := range m.activeAlerts { + keys = append(keys, effectiveAlertID(active, storageKey)) + } + t.Fatalf("expected orphaned PBS guest backup alert, found keys: %v", keys) + } +} + func TestCheckBackupsSkipsOrphanedWhenDisabled(t *testing.T) { m := newTestManager(t) m.ClearActiveAlerts() diff --git a/internal/alerts/backup_snapshot.go b/internal/alerts/backup_snapshot.go index 3fcd264df7..c43c9f5714 100644 --- a/internal/alerts/backup_snapshot.go +++ b/internal/alerts/backup_snapshot.go @@ -859,6 +859,16 @@ func (m *Manager) CheckBackupsWithInventory( } ageDaysRounded := math.Round(ageDays*10) / 10 + // A PBS subject without a guest VMID is a host/config backup + // (PBS backup-type "host", keyed by the node name), not a guest + // workload. The Backups overview already keeps these out of guest + // coverage, so do not raise a guest backup-age alert for them: a + // stale host config backup otherwise notifies forever under a + // node-named subject while the node's guests are backed up (#2136). + if record.source == "PBS" && strings.TrimSpace(record.vmID) == "" { + continue + } + // Determine thresholds for this backup currentBackupCfg := backupCfg guestContext := guestSnapshotFromLookup(record.lookup, record.fallbackName) From 1026f5293b4c4e5a11651933e7ed36d398fd2f69 Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 08:46:29 +0100 Subject: [PATCH 4/5] docs(monitoring): record #2148 LXC linked-agent-memory contracts Complete the #2148 runtime change with the monitoring and unified-resources contract sections its canonical subsystem paths require, so the combined change satisfies the canonical completion guard without rewriting reviewed history. Change-source: pulse-maintainer --- .../v6/internal/subsystems/monitoring.md | 19 +++++++++++++++++++ .../internal/subsystems/unified-resources.md | 11 +++++++++++ 2 files changed, 30 insertions(+) diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 6f4255c924..4240bc62ab 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -17,6 +17,25 @@ ## Purpose +### Linked Pulse agent memory for Proxmox LXC — issue #2148 (22 September 2026) + +Both LXC memory paths (the efficient `cluster/resources` builder and the +per-node fallback) may substitute the linked Pulse agent's own memory sample for +a Proxmox fallback listing value. The substitution applies only when the current +memory source is a fallback (`cluster-resources` or `unavailable`), the guest is +`running`, the linked agent sample has known usage, and the agent total matches +the container's provisioned limit with used not exceeding it. A stopped +container, a non-fallback source, a missing linked agent, and a total mismatch +are left untouched; preferred or derived guest evidence is never replaced. The +provisioned platform total is retained, and the previous-guest agent map is +threaded through both builders. + +`TestIssue2148LXCPrefersLinkedAgentMemoryOverClusterResources` and +`TestIssue2148LXCRejectsAgentMemoryWithMismatchedTotal` pin the substitution and +the rejection boundary; `TestCorrelatedContainerMemoryNextPoll` pins the +next-poll merged-row path. This is synthetic runtime proof, not a live Proxmox +installation or browser receipt. + ### Preserve explicit agent filesystem selection at ingestion Both `ApplyHostReport` and `ApplyDockerReport` honour a disk's optional diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 4c0aefd1a6..a14354e1bc 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -37,6 +37,17 @@ policy remains authoritative; this accessor introduces no second timeout. `TestCorrelatedGuestMemoryNextPoll` verifies the manually merged row through the next-poll consumers. Existing host-list membership and metric priority are unchanged. +**System-container linked agent memory read — issue #2148 (22 September 2026)** + +`ContainerView.LinkedAgentMemory` exposes the same value-copy of the linked +agent's own memory sample for system containers, sharing the validity and +freshness rules with `VMView.LinkedAgentMemory`: missing/non-online agent source +status, explicit agent staleness, unavailable usage and invalid capacity or +counters are rejected, and trusted zero usage remains valid. The monitoring +layer consumes it only as a fallback replacement for Proxmox LXC memory, so +platform metric priority for a healthy agent is unchanged. +`TestContainerViewLinkedAgentMemory` pins those boundaries. + ### Bounded incident-history selection Canonical history queries filter exact alert identifiers and observation windows From e9ace60929c8c5df0f137d32159cce855cc29a65 Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 08:46:39 +0100 Subject: [PATCH 5/5] chore(governance): register the #2148 LXC memory completion pair Register 8b56fc72e4 with its contract-completion commit so canonical governance evaluates the reviewed Proxmox LXC linked-agent memory runtime change and its monitoring and unified-resources contract sections as one historical unit. Change-source: pulse-maintainer --- scripts/release_control/canonical_completion_history.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/release_control/canonical_completion_history.json b/scripts/release_control/canonical_completion_history.json index 7a898cda36..74092940cf 100644 --- a/scripts/release_control/canonical_completion_history.json +++ b/scripts/release_control/canonical_completion_history.json @@ -75,6 +75,11 @@ "incomplete_commit": "919009d7677318c071f888eb28c252d2d79d2728", "completion_commit": "9fa81dee8fba760b980d821d6e10560589c110f2", "reason": "Evaluate the PVE consolidation explicit-VerifySSL runtime change together with the follow-up commit that records its agent-lifecycle, api-contracts, security-privacy and dependent storage-recovery contract sections and the node API payload and persisted-field proofs, as one reviewed completion unit." + }, + { + "incomplete_commit": "8b56fc72e4ee79f3c219cf51f4c002e2e9343286", + "completion_commit": "1026f5293b4c4e5a11651933e7ed36d398fd2f69", + "reason": "Evaluate the Proxmox LXC linked-agent memory preference runtime change together with the follow-up commit that records its monitoring and unified-resources contract sections, as one reviewed completion unit." } ] }