diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 87d6bc83c..deb287d4a 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -618,6 +618,15 @@ installer download and the agent's subsequent Pulse TLS connection. ## Shared Boundaries +The shared `PBSInstance.NodeMetricsUnavailable` field belongs exclusively to +provider polling and alert evaluation. It is retained by in-process state copies +but excluded from JSON; it neither grants nor revokes host-agent identity, +enrolment, removal or command authority. No agent lifecycle decision may infer +host removal or re-enrolment from a PBS node-status failure. The shared-model +boundary is verified in `internal/models/metrics_types_test.go`; this is not +new host-agent removal or installed re-enrolment qualification. + + The shared security-status route may expose the validated caller currentUsername to authenticated clients, including organisation-scoped local sessions. This presentation identity is not an agent credential, enrollment grant or command authority; agent download, enrollment and token-scope checks remain independent of this field. `internal/models/models.go` and `internal/monitoring/monitor.go` also carry diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index b2587c3e4..e09b14a01 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -371,6 +371,15 @@ default construction path still restores. ## Shared Boundaries +PBS node-status availability is distinct from connectivity. While a connected +PBS carries monitoring-owned `NodeMetricsUnavailable` evidence, CPU and memory +evaluation must not treat zero-valued placeholders as recovery. Existing +policy suppression and full-outage handling retain precedence. Valid low +measurements resume normal recovery, including callbacks and recent history. +Regression proof lives in `internal/alerts/telemetry_quality_test.go` and the +HTTP-to-manager lifecycle in `internal/monitoring/monitor_pbs_coverage_test.go`. + + 1. `frontend-modern/src/stores/websocket.ts` shared with `performance-and-scalability`: the connection-owned realtime store is both the canonical alert truth boundary and the fleet-scale resource reconciliation hot path. That shared store normalizes slimmed broadcast resources at ingestion — expanding `capabilitiesRef` through the state `capabilityCatalog` and diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 8777dc780..eb98c3fe3 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -669,6 +669,17 @@ cleanup so readers cannot retain orphaned runtime or alert projections. ## Shared Boundaries +PBS polling owns the internal `PBSInstance.NodeMetricsUnavailable` discriminator: +each poll starts unavailable and only a successful non-nil node-status result +clears it. A denied or failed node-status endpoint does not invalidate successful +connectivity or independently accessible datastore inventory. In-process state +copies preserve this evidence; JSON deliberately does not carry it. The +zero-value compatibility default is not persisted availability evidence. +Proof: `internal/models/metrics_types_test.go` and +`internal/monitoring/monitor_pbs_coverage_test.go`. The latter exercises real +HTTP polling through normal alert-manager publication, not destination delivery. + + 1. `internal/config/host_continuity.go` shared with `agent-lifecycle`: the durable host identity, report-order watermark, and removal tombstone journal is jointly owned by agent lifecycle admission and monitoring report continuity. 2. `internal/kubernetesagent/agent.go` shared with `agent-lifecycle`: the Kubernetes native agent runtime is both a monitoring inventory source and an agent lifecycle Pulse control-plane transport client. 3. `internal/mock/fixture_graph.go` shared with `performance-and-scalability`: the canonical mock fixture graph is both monitoring-owned runtime data and a protected large-estate demo transport hot path. diff --git a/internal/alerts/pbs.go b/internal/alerts/pbs.go index c07a0ca65..b7575126e 100644 --- a/internal/alerts/pbs.go +++ b/internal/alerts/pbs.go @@ -125,6 +125,12 @@ func (m *Manager) CheckPBS(pbs models.PBSInstance) { return } + // Endpoint failure is not evidence of metric recovery. Connectivity above + // remains independent, as do explicit policy suppression and full outages. + if pbs.NodeMetricsUnavailable { + return + } + m.evaluateUnifiedMetrics(&UnifiedResourceInput{ ID: pbs.ID, Type: "pbs", diff --git a/internal/alerts/telemetry_quality_test.go b/internal/alerts/telemetry_quality_test.go index e82d90e08..4a336b53d 100644 --- a/internal/alerts/telemetry_quality_test.go +++ b/internal/alerts/telemetry_quality_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust" ) @@ -192,3 +193,46 @@ func TestAlertQualityResolutionDurationBoundaries(t *testing.T) { } func alertQualityTimePtr(value time.Time) *time.Time { return &value } + +func TestPBSMissingMetricsDoNotResolve(t *testing.T) { + m := newUnifiedEvalParityManager(t) + m.UpdateConfig(AlertConfig{Enabled: true, PBSDefaults: ThresholdConfig{ + CPU: &HysteresisThreshold{Trigger: 80, Clear: 75}, + Memory: &HysteresisThreshold{Trigger: 85, Clear: 80}, + }}) + disableTestTimeThresholds(m) + resolved := make(chan string, 8) + m.SetResolvedCallback(func(id string) { resolved <- id }) + p := models.PBSInstance{ID: "pbs-missing", Name: "backup", Status: "online", CPU: 95, Memory: 95} + m.CheckPBS(p) + if len(m.GetActiveAlerts()) != 2 { + t.Fatal("expected two high utilisation alerts") + } + p.CPU, p.Memory = 0, 0 + p.NodeMetricsUnavailable = true + for range 5 { + m.CheckPBS(p) + } + if len(m.GetActiveAlerts()) != 2 { + t.Fatal("missing metrics falsely resolved high utilisation") + } + select { + case id := <-resolved: + t.Fatalf("false recovery callback %s", id) + case <-time.After(50 * time.Millisecond): + } + p.NodeMetricsUnavailable = false + for range 5 { + m.CheckPBS(p) + } + if len(m.GetActiveAlerts()) != 0 { + t.Fatal("measured zero did not recover") + } + for range 2 { + select { + case <-resolved: + case <-time.After(time.Second): + t.Fatal("missing genuine recovery callback") + } + } +} diff --git a/internal/models/metrics_types_test.go b/internal/models/metrics_types_test.go index 1b839bc69..2b53b9354 100644 --- a/internal/models/metrics_types_test.go +++ b/internal/models/metrics_types_test.go @@ -343,3 +343,40 @@ func TestPBSGuestConfirmationEvidenceStaysOutOfSerializedState(t *testing.T) { t.Fatal("PBS guest confirmation evidence must not appear in snapshots") } } + +// PBS poll evidence survives in-process copying but is not an agent wire input. +func TestPBSNodeMetricAvailabilityModelBoundary(t *testing.T) { + state := NewState() + state.UpdatePBSInstance(PBSInstance{ + ID: "pbs-availability", Name: "pbs", Status: "online", + NodeMetricsUnavailable: true, + }) + snapshot := state.GetSnapshot() + if len(snapshot.PBSInstances) != 1 || !snapshot.PBSInstances[0].NodeMetricsUnavailable { + t.Fatal("snapshot lost unavailable-node-metrics evidence") + } + snapshot.PBSInstances[0].NodeMetricsUnavailable = false + if !state.GetSnapshot().PBSInstances[0].NodeMetricsUnavailable { + t.Fatal("snapshot mutation changed authoritative poll evidence") + } + missing := state.GetSnapshot().PBSInstances[0] + wire, err := json.Marshal(missing) + if err != nil { + t.Fatal(err) + } + missing.NodeMetricsUnavailable = false + availableWire, err := json.Marshal(missing) + if err != nil { + t.Fatal(err) + } + if string(wire) != string(availableWire) { + t.Fatal("internal availability evidence changed the public wire contract") + } + var decoded PBSInstance + if err := json.Unmarshal([]byte(`{"id":"pbs","NodeMetricsUnavailable":true,"nodeMetricsUnavailable":true}`), &decoded); err != nil { + t.Fatal(err) + } + if decoded.NodeMetricsUnavailable { + t.Fatal("wire input asserted monitoring-owned availability evidence") + } +} diff --git a/internal/models/models.go b/internal/models/models.go index 318c8631a..f0e979774 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -2699,6 +2699,10 @@ type PhysicalDisk struct { // PBSInstance represents a Proxmox Backup Server instance type PBSInstance struct { + // NodeMetricsUnavailable distinguishes a failed node-status poll from measured zero. + // Internal poll evidence; the zero value preserves legacy in-process callers. + NodeMetricsUnavailable bool `json:"-"` + ID string `json:"id"` Name string `json:"name"` Host string `json:"host"` diff --git a/internal/monitoring/monitor_pbs_coverage_test.go b/internal/monitoring/monitor_pbs_coverage_test.go index d008a8209..038076ce4 100644 --- a/internal/monitoring/monitor_pbs_coverage_test.go +++ b/internal/monitoring/monitor_pbs_coverage_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "github.com/rcourtman/pulse-go-rewrite/internal/alerts" "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/pkg/pbs" @@ -500,3 +501,70 @@ func TestPBSAndPMGPollSkipDisabledInstances(t *testing.T) { } } } + +// Exercise real HTTP polling and one manager through missing metrics and recovery. +func TestPBSMetricAvailabilityAlertLifecycle(t *testing.T) { + fixture := newPBSHealthTestServer(t) + instance := config.PBSInstance{Name: "pbs-lifecycle", Host: fixture.server.URL, MonitorDatastores: true} + monitor := newPBSHealthAuthorityMonitor([]config.PBSInstance{instance}) + client := newPBSHealthTestClient(t, instance.Host) + manager := alerts.NewManagerWithDataDir(t.TempDir()) + defer manager.Stop() + monitor.alertManager = manager + manager.UpdateConfig(alerts.AlertConfig{Enabled: true, ActivationState: alerts.ActivationActive, + TimeThresholds: map[string]int{"pbs": 0}, PBSDefaults: alerts.ThresholdConfig{ + Memory: &alerts.HysteresisThreshold{Trigger: 40, Clear: 30}, + }}) + fired, resolved := make(chan string, 8), make(chan string, 8) + manager.SetAlertCallback(func(a *alerts.Alert) { fired <- a.ID }) + manager.SetResolvedCallback(func(id string) { resolved <- id }) + poll := func() models.PBSInstance { + monitor.pollPBSInstance(context.Background(), instance.Name, client) + return pbsInstanceByName(t, monitor.state.GetSnapshot(), instance.Name) + } + poll() + if len(manager.GetActiveAlerts()) != 1 { + t.Fatal("high memory failed to activate") + } + select { + case <-fired: + case <-time.After(time.Second): + t.Fatal("missing alert dispatch") + } + for _, mode := range []pbsHealthTestMode{pbsHealthTestNodeDenied, pbsHealthTestNodeGatewayFailure} { + fixture.setMode(mode) + missing := poll() + if missing.Status != "online" || !missing.NodeMetricsUnavailable { + t.Fatalf("bad missing projection: %+v", missing) + } + for range 5 { + poll() + } + if len(manager.GetActiveAlerts()) != 1 { + t.Fatal("endpoint failure resolved memory alert") + } + if len(manager.GetRecentlyResolved()) != 0 { + t.Fatal("false resolved history") + } + select { + case id := <-resolved: + t.Fatalf("false recovery dispatch %s", id) + case <-time.After(50 * time.Millisecond): + } + } + fixture.setMode(pbsHealthTestLowMemory) + for range 5 { + poll() + } + if len(manager.GetActiveAlerts()) != 0 { + t.Fatal("valid low memory failed to recover") + } + select { + case <-resolved: + case <-time.After(time.Second): + t.Fatal("missing recovery dispatch") + } + if len(manager.GetRecentlyResolved()) != 1 { + t.Fatal("missing resolved history") + } +} diff --git a/internal/monitoring/monitor_pbs_health_authority_test.go b/internal/monitoring/monitor_pbs_health_authority_test.go index 92af6b84b..dc603bafd 100644 --- a/internal/monitoring/monitor_pbs_health_authority_test.go +++ b/internal/monitoring/monitor_pbs_health_authority_test.go @@ -25,6 +25,7 @@ const ( pbsHealthTestNodeDenied pbsHealthTestNodeGatewayFailure pbsHealthTestUnavailable + pbsHealthTestLowMemory ) type pbsHealthTestServer struct { @@ -75,8 +76,13 @@ func newPBSHealthTestServer(t *testing.T) *pbsHealthTestServer { } _ = json.NewEncoder(w).Encode(map[string]any{ "data": map[string]any{ - "cpu": 0.15, - "memory": map[string]any{"used": 512, "total": 1024}, + "cpu": 0.15, + "memory": map[string]any{"used": func() int { + if mode == pbsHealthTestLowMemory { + return 100 + } + return 512 + }(), "total": 1024}, "uptime": 120, }, }) @@ -334,6 +340,9 @@ func TestPollPBSNodeMetricsFailureAndRecovery(t *testing.T) { } assertMetrics := func(got models.PBSInstance) { t.Helper() + if got.NodeMetricsUnavailable { + t.Fatal("successful metrics marked unavailable") + } if got.CPU != 0.15 || got.Memory != 50 || got.MemoryUsed != 512 || got.MemoryTotal != 1024 || got.Uptime != 120 { t.Fatalf("successful node metrics not published: %+v", got) } @@ -347,6 +356,9 @@ func TestPollPBSNodeMetricsFailureAndRecovery(t *testing.T) { if !status.LastSuccess.After(previous) || status.ConsecutiveFailures != 0 || status.LastErrorMessage != "" { t.Fatalf("endpoint failure incorrectly affected connectivity ledger: %+v", status) } + if !partial.NodeMetricsUnavailable { + t.Fatal("failed node endpoint not marked unavailable") + } if partial.CPU != 0 || partial.Memory != 0 || partial.MemoryUsed != 0 || partial.MemoryTotal != 0 || partial.Uptime != 0 { t.Fatalf("unavailable node metrics retained previous measurements: %+v", partial) } diff --git a/internal/monitoring/monitor_pbs_pmg.go b/internal/monitoring/monitor_pbs_pmg.go index 6021ed993..01f6e3ae7 100644 --- a/internal/monitoring/monitor_pbs_pmg.go +++ b/internal/monitoring/monitor_pbs_pmg.go @@ -241,14 +241,15 @@ func (m *Monitor) pollPBSInstance(ctx context.Context, instanceName string, clie // Initialize PBS instance with default values pbsInst = models.PBSInstance{ - ID: PBSMonitorResourceID(instanceName), - Name: instanceName, - Host: instanceCfg.Host, - GuestURL: instanceCfg.GuestURL, - Status: "offline", - Version: "unknown", - ConnectionHealth: "unhealthy", - LastSeen: time.Now(), + NodeMetricsUnavailable: true, + ID: PBSMonitorResourceID(instanceName), + Name: instanceName, + Host: instanceCfg.Host, + GuestURL: instanceCfg.GuestURL, + Status: "offline", + Version: "unknown", + ConnectionHealth: "unhealthy", + LastSeen: time.Now(), } publishResult = true @@ -336,6 +337,7 @@ func (m *Monitor) pollPBSInstance(ctx context.Context, instanceName string, clie log.Debug().Err(err).Str("instance", instanceName).Msg("could not get PBS node status (may need Sys.Audit permission)") } } else if nodeStatus != nil { + pbsInst.NodeMetricsUnavailable = false pbsInst.CPU = nodeStatus.CPU if nodeStatus.Memory.Total > 0 { pbsInst.Memory = float64(nodeStatus.Memory.Used) / float64(nodeStatus.Memory.Total) * 100