fix(pbs): preserve metric alerts when node status is unavailable

A reachable PBS can deny or fail its node-status endpoint while datastore collection succeeds. The resulting zero values previously resolved active CPU and memory alerts without a healthy measurement. Carry internal availability evidence from polling and skip metric evaluation for those samples, preserving independent connectivity and policy suppression. Add failing-before unit coverage and an HTTP polling lifecycle regression for dispatch and recent recovery history.

Change-source: pulse-maintainer

docs/release-control/v6/internal/subsystems/agent-lifecycle.md

@@ -618,6 +618,15 @@ installer download and the agent's subsequent Pulse TLS connection.
 
 ## Shared Boundaries
 
+The shared `PBSInstance.NodeMetricsUnavailable` field belongs exclusively to
+provider polling and alert evaluation. It is retained by in-process state copies
+but excluded from JSON; it neither grants nor revokes host-agent identity,
+enrolment, removal or command authority. No agent lifecycle decision may infer
+host removal or re-enrolment from a PBS node-status failure. The shared-model
+boundary is verified in `internal/models/metrics_types_test.go`; this is not
+new host-agent removal or installed re-enrolment qualification.
+
+
 The shared security-status route may expose the validated caller currentUsername to authenticated clients, including organisation-scoped local sessions. This presentation identity is not an agent credential, enrollment grant or command authority; agent download, enrollment and token-scope checks remain independent of this field.
 
 `internal/models/models.go` and `internal/monitoring/monitor.go` also carry

docs/release-control/v6/internal/subsystems/alerts.md

@@ -371,6 +371,15 @@ default construction path still restores.
 
 ## Shared Boundaries
 
+PBS node-status availability is distinct from connectivity. While a connected
+PBS carries monitoring-owned `NodeMetricsUnavailable` evidence, CPU and memory
+evaluation must not treat zero-valued placeholders as recovery. Existing
+policy suppression and full-outage handling retain precedence. Valid low
+measurements resume normal recovery, including callbacks and recent history.
+Regression proof lives in `internal/alerts/telemetry_quality_test.go` and the
+HTTP-to-manager lifecycle in `internal/monitoring/monitor_pbs_coverage_test.go`.
+
+
 1. `frontend-modern/src/stores/websocket.ts` shared with `performance-and-scalability`: the connection-owned realtime store is both the canonical alert truth boundary and the fleet-scale resource reconciliation hot path.
    That shared store normalizes slimmed broadcast resources at ingestion —
    expanding `capabilitiesRef` through the state `capabilityCatalog` and

docs/release-control/v6/internal/subsystems/monitoring.md

@@ -669,6 +669,17 @@ cleanup so readers cannot retain orphaned runtime or alert projections.
 
 ## Shared Boundaries
 
+PBS polling owns the internal `PBSInstance.NodeMetricsUnavailable` discriminator:
+each poll starts unavailable and only a successful non-nil node-status result
+clears it. A denied or failed node-status endpoint does not invalidate successful
+connectivity or independently accessible datastore inventory. In-process state
+copies preserve this evidence; JSON deliberately does not carry it. The
+zero-value compatibility default is not persisted availability evidence.
+Proof: `internal/models/metrics_types_test.go` and
+`internal/monitoring/monitor_pbs_coverage_test.go`. The latter exercises real
+HTTP polling through normal alert-manager publication, not destination delivery.
+
+
 1. `internal/config/host_continuity.go` shared with `agent-lifecycle`: the durable host identity, report-order watermark, and removal tombstone journal is jointly owned by agent lifecycle admission and monitoring report continuity.
 2. `internal/kubernetesagent/agent.go` shared with `agent-lifecycle`: the Kubernetes native agent runtime is both a monitoring inventory source and an agent lifecycle Pulse control-plane transport client.
 3. `internal/mock/fixture_graph.go` shared with `performance-and-scalability`: the canonical mock fixture graph is both monitoring-owned runtime data and a protected large-estate demo transport hot path.

internal/alerts/pbs.go

@@ -125,6 +125,12 @@ func (m *Manager) CheckPBS(pbs models.PBSInstance) {
 		return
 	}
 
+	// Endpoint failure is not evidence of metric recovery. Connectivity above
+	// remains independent, as do explicit policy suppression and full outages.
+	if pbs.NodeMetricsUnavailable {
+		return
+	}
+
 	m.evaluateUnifiedMetrics(&UnifiedResourceInput{
 		ID:       pbs.ID,
 		Type:     "pbs",

internal/alerts/telemetry_quality_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/rcourtman/pulse-go-rewrite/internal/models"
 	"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
 )
 
@@ -192,3 +193,46 @@ func TestAlertQualityResolutionDurationBoundaries(t *testing.T) {
 }
 
 func alertQualityTimePtr(value time.Time) *time.Time { return &value }
+
+func TestPBSMissingMetricsDoNotResolve(t *testing.T) {
+	m := newUnifiedEvalParityManager(t)
+	m.UpdateConfig(AlertConfig{Enabled: true, PBSDefaults: ThresholdConfig{
+		CPU:    &HysteresisThreshold{Trigger: 80, Clear: 75},
+		Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
+	}})
+	disableTestTimeThresholds(m)
+	resolved := make(chan string, 8)
+	m.SetResolvedCallback(func(id string) { resolved <- id })
+	p := models.PBSInstance{ID: "pbs-missing", Name: "backup", Status: "online", CPU: 95, Memory: 95}
+	m.CheckPBS(p)
+	if len(m.GetActiveAlerts()) != 2 {
+		t.Fatal("expected two high utilisation alerts")
+	}
+	p.CPU, p.Memory = 0, 0
+	p.NodeMetricsUnavailable = true
+	for range 5 {
+		m.CheckPBS(p)
+	}
+	if len(m.GetActiveAlerts()) != 2 {
+		t.Fatal("missing metrics falsely resolved high utilisation")
+	}
+	select {
+	case id := <-resolved:
+		t.Fatalf("false recovery callback %s", id)
+	case <-time.After(50 * time.Millisecond):
+	}
+	p.NodeMetricsUnavailable = false
+	for range 5 {
+		m.CheckPBS(p)
+	}
+	if len(m.GetActiveAlerts()) != 0 {
+		t.Fatal("measured zero did not recover")
+	}
+	for range 2 {
+		select {
+		case <-resolved:
+		case <-time.After(time.Second):
+			t.Fatal("missing genuine recovery callback")
+		}
+	}
+}

internal/models/metrics_types_test.go

@@ -343,3 +343,40 @@ func TestPBSGuestConfirmationEvidenceStaysOutOfSerializedState(t *testing.T) {
 		t.Fatal("PBS guest confirmation evidence must not appear in snapshots")
 	}
 }
+
+// PBS poll evidence survives in-process copying but is not an agent wire input.
+func TestPBSNodeMetricAvailabilityModelBoundary(t *testing.T) {
+	state := NewState()
+	state.UpdatePBSInstance(PBSInstance{
+		ID: "pbs-availability", Name: "pbs", Status: "online",
+		NodeMetricsUnavailable: true,
+	})
+	snapshot := state.GetSnapshot()
+	if len(snapshot.PBSInstances) != 1 || !snapshot.PBSInstances[0].NodeMetricsUnavailable {
+		t.Fatal("snapshot lost unavailable-node-metrics evidence")
+	}
+	snapshot.PBSInstances[0].NodeMetricsUnavailable = false
+	if !state.GetSnapshot().PBSInstances[0].NodeMetricsUnavailable {
+		t.Fatal("snapshot mutation changed authoritative poll evidence")
+	}
+	missing := state.GetSnapshot().PBSInstances[0]
+	wire, err := json.Marshal(missing)
+	if err != nil {
+		t.Fatal(err)
+	}
+	missing.NodeMetricsUnavailable = false
+	availableWire, err := json.Marshal(missing)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(wire) != string(availableWire) {
+		t.Fatal("internal availability evidence changed the public wire contract")
+	}
+	var decoded PBSInstance
+	if err := json.Unmarshal([]byte(`{"id":"pbs","NodeMetricsUnavailable":true,"nodeMetricsUnavailable":true}`), &decoded); err != nil {
+		t.Fatal(err)
+	}
+	if decoded.NodeMetricsUnavailable {
+		t.Fatal("wire input asserted monitoring-owned availability evidence")
+	}
+}

internal/models/models.go

@@ -2699,6 +2699,10 @@ type PhysicalDisk struct {
 
 // PBSInstance represents a Proxmox Backup Server instance
 type PBSInstance struct {
+	// NodeMetricsUnavailable distinguishes a failed node-status poll from measured zero.
+	// Internal poll evidence; the zero value preserves legacy in-process callers.
+	NodeMetricsUnavailable bool `json:"-"`
+
 	ID                string                 `json:"id"`
 	Name              string                 `json:"name"`
 	Host              string                 `json:"host"`

internal/monitoring/monitor_pbs_coverage_test.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
 	"github.com/rcourtman/pulse-go-rewrite/internal/config"
 	"github.com/rcourtman/pulse-go-rewrite/internal/models"
 	"github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
@@ -500,3 +501,70 @@ func TestPBSAndPMGPollSkipDisabledInstances(t *testing.T) {
 		}
 	}
 }
+
+// Exercise real HTTP polling and one manager through missing metrics and recovery.
+func TestPBSMetricAvailabilityAlertLifecycle(t *testing.T) {
+	fixture := newPBSHealthTestServer(t)
+	instance := config.PBSInstance{Name: "pbs-lifecycle", Host: fixture.server.URL, MonitorDatastores: true}
+	monitor := newPBSHealthAuthorityMonitor([]config.PBSInstance{instance})
+	client := newPBSHealthTestClient(t, instance.Host)
+	manager := alerts.NewManagerWithDataDir(t.TempDir())
+	defer manager.Stop()
+	monitor.alertManager = manager
+	manager.UpdateConfig(alerts.AlertConfig{Enabled: true, ActivationState: alerts.ActivationActive,
+		TimeThresholds: map[string]int{"pbs": 0}, PBSDefaults: alerts.ThresholdConfig{
+			Memory: &alerts.HysteresisThreshold{Trigger: 40, Clear: 30},
+		}})
+	fired, resolved := make(chan string, 8), make(chan string, 8)
+	manager.SetAlertCallback(func(a *alerts.Alert) { fired <- a.ID })
+	manager.SetResolvedCallback(func(id string) { resolved <- id })
+	poll := func() models.PBSInstance {
+		monitor.pollPBSInstance(context.Background(), instance.Name, client)
+		return pbsInstanceByName(t, monitor.state.GetSnapshot(), instance.Name)
+	}
+	poll()
+	if len(manager.GetActiveAlerts()) != 1 {
+		t.Fatal("high memory failed to activate")
+	}
+	select {
+	case <-fired:
+	case <-time.After(time.Second):
+		t.Fatal("missing alert dispatch")
+	}
+	for _, mode := range []pbsHealthTestMode{pbsHealthTestNodeDenied, pbsHealthTestNodeGatewayFailure} {
+		fixture.setMode(mode)
+		missing := poll()
+		if missing.Status != "online" || !missing.NodeMetricsUnavailable {
+			t.Fatalf("bad missing projection: %+v", missing)
+		}
+		for range 5 {
+			poll()
+		}
+		if len(manager.GetActiveAlerts()) != 1 {
+			t.Fatal("endpoint failure resolved memory alert")
+		}
+		if len(manager.GetRecentlyResolved()) != 0 {
+			t.Fatal("false resolved history")
+		}
+		select {
+		case id := <-resolved:
+			t.Fatalf("false recovery dispatch %s", id)
+		case <-time.After(50 * time.Millisecond):
+		}
+	}
+	fixture.setMode(pbsHealthTestLowMemory)
+	for range 5 {
+		poll()
+	}
+	if len(manager.GetActiveAlerts()) != 0 {
+		t.Fatal("valid low memory failed to recover")
+	}
+	select {
+	case <-resolved:
+	case <-time.After(time.Second):
+		t.Fatal("missing recovery dispatch")
+	}
+	if len(manager.GetRecentlyResolved()) != 1 {
+		t.Fatal("missing resolved history")
+	}
+}

internal/monitoring/monitor_pbs_health_authority_test.go

@@ -25,6 +25,7 @@ const (
 	pbsHealthTestNodeDenied
 	pbsHealthTestNodeGatewayFailure
 	pbsHealthTestUnavailable
+	pbsHealthTestLowMemory
 )
 
 type pbsHealthTestServer struct {
@@ -75,8 +76,13 @@ func newPBSHealthTestServer(t *testing.T) *pbsHealthTestServer {
 			}
 			_ = json.NewEncoder(w).Encode(map[string]any{
 				"data": map[string]any{
-					"cpu":    0.15,
-					"memory": map[string]any{"used": 512, "total": 1024},
+					"cpu": 0.15,
+					"memory": map[string]any{"used": func() int {
+						if mode == pbsHealthTestLowMemory {
+							return 100
+						}
+						return 512
+					}(), "total": 1024},
 					"uptime": 120,
 				},
 			})
@@ -334,6 +340,9 @@ func TestPollPBSNodeMetricsFailureAndRecovery(t *testing.T) {
 			}
 			assertMetrics := func(got models.PBSInstance) {
 				t.Helper()
+				if got.NodeMetricsUnavailable {
+					t.Fatal("successful metrics marked unavailable")
+				}
 				if got.CPU != 0.15 || got.Memory != 50 || got.MemoryUsed != 512 || got.MemoryTotal != 1024 || got.Uptime != 120 {
 					t.Fatalf("successful node metrics not published: %+v", got)
 				}
@@ -347,6 +356,9 @@ func TestPollPBSNodeMetricsFailureAndRecovery(t *testing.T) {
 			if !status.LastSuccess.After(previous) || status.ConsecutiveFailures != 0 || status.LastErrorMessage != "" {
 				t.Fatalf("endpoint failure incorrectly affected connectivity ledger: %+v", status)
 			}
+			if !partial.NodeMetricsUnavailable {
+				t.Fatal("failed node endpoint not marked unavailable")
+			}
 			if partial.CPU != 0 || partial.Memory != 0 || partial.MemoryUsed != 0 || partial.MemoryTotal != 0 || partial.Uptime != 0 {
 				t.Fatalf("unavailable node metrics retained previous measurements: %+v", partial)
 			}

internal/monitoring/monitor_pbs_pmg.go

@@ -241,14 +241,15 @@ func (m *Monitor) pollPBSInstance(ctx context.Context, instanceName string, clie
 
 	// Initialize PBS instance with default values
 	pbsInst = models.PBSInstance{
-		ID:               PBSMonitorResourceID(instanceName),
-		Name:             instanceName,
-		Host:             instanceCfg.Host,
-		GuestURL:         instanceCfg.GuestURL,
-		Status:           "offline",
-		Version:          "unknown",
-		ConnectionHealth: "unhealthy",
-		LastSeen:         time.Now(),
+		NodeMetricsUnavailable: true,
+		ID:                     PBSMonitorResourceID(instanceName),
+		Name:                   instanceName,
+		Host:                   instanceCfg.Host,
+		GuestURL:               instanceCfg.GuestURL,
+		Status:                 "offline",
+		Version:                "unknown",
+		ConnectionHealth:       "unhealthy",
+		LastSeen:               time.Now(),
 	}
 	publishResult = true
 
@@ -336,6 +337,7 @@ func (m *Monitor) pollPBSInstance(ctx context.Context, instanceName string, clie
 			log.Debug().Err(err).Str("instance", instanceName).Msg("could not get PBS node status (may need Sys.Audit permission)")
 		}
 	} else if nodeStatus != nil {
+		pbsInst.NodeMetricsUnavailable = false
 		pbsInst.CPU = nodeStatus.CPU
 		if nodeStatus.Memory.Total > 0 {
 			pbsInst.Memory = float64(nodeStatus.Memory.Used) / float64(nodeStatus.Memory.Total) * 100