From 37d187490405c48cb59e3201d71b181e5f726c88 Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 20:23:20 +0100 Subject: [PATCH 1/2] docs(msp): verify published evaluation bundle guidance Replace obsolete v6.2.1 guidance using retained signed v6.4.1 delivery and payload evidence. Limit privacy claims to the evaluated licence request, synchronize the shipped guide, and adapt the registered deployment regression and contract without claiming installed onboarding acceptance. Change-source: pulse-maintainer --- docs/MSP.md | 19 ++++++++++++------- .../subsystems/deployment-installability.md | 17 +++++++++++++++++ frontend-modern/public/docs/MSP.md | 19 ++++++++++++------- .../installtests/provider_msp_deploy_test.go | 13 ++++++++++--- .../qualification/provider-delivery/README.md | 19 +++++++++++++++++++ 5 files changed, 70 insertions(+), 17 deletions(-) create mode 100644 tests/qualification/provider-delivery/README.md diff --git a/docs/MSP.md b/docs/MSP.md index 855a76365..cb746ec8a 100644 --- a/docs/MSP.md +++ b/docs/MSP.md @@ -271,7 +271,7 @@ before money changes hands. ### Evaluating without a licence Self-service evaluation is available from the signed provider bundle published -with Pulse v6.2.1. Use this exact release asset and its integrity sidecars; do +with Pulse v6.4.1. Use this exact release asset and its integrity sidecars; do **not** download the moving `main` branch archive or run its `setup.sh` as root. For a later release, first confirm its release page contains the versioned provider archive, checksum, and SSH signature before changing the @@ -281,7 +281,7 @@ Download the versioned asset, verify it with Pulse's pinned release key, and only then extract and run the guided setup: ```bash -export PULSE_VERSION=v6.2.1 +export PULSE_VERSION=v6.4.1 export PULSE_MSP_BUNDLE="pulse-provider-msp-${PULSE_VERSION}.tar.gz" export PULSE_RELEASE_BASE="https://github.com/rcourtman/Pulse/releases/download/${PULSE_VERSION}" @@ -301,8 +301,11 @@ cd "pulse-provider-msp-${PULSE_VERSION}" sudo -E bash ./setup.sh ``` -The v6.2.1 evaluation request is anonymous. If you want setup help, start from -the [Pulse MSP evaluation page](https://pulserelay.pro/msp.html#evaluate) first; +The licence request omits an email address unless you export +`PULSE_PROVIDER_MSP_EVAL_EMAIL` before running `setup.sh`, in which case that +address is included in the request. If you +want setup help, start from the +[Pulse MSP evaluation page](https://pulserelay.pro/msp.html#evaluate) first; that contact request remains separate from the licence activation. The host needs Ubuntu 24.04 or similar, a domain you can point at it, and @@ -313,9 +316,11 @@ Traefik dnsChallenge provider works by setting `ACME_DNS_PROVIDER` in `.env` and putting that provider's credential variables in `dns-credentials.env`. Leave `CP_PROVIDER_MSP_LICENSE_FILE` blank and `setup.sh` self-issues a -2-client evaluation licence. The v6.2.1 bundle sends only the public half of -the signing key generated on your host. The private key, client inventory, -credentials, and contact details never leave the machine. You can then onboard +2-client evaluation licence. The v6.4.1 bundle sends the public half of the +signing key generated on your host, a setup-stage marker, and a signup-source +label, plus the optional email address. This licence-request payload does not +include the private key, client inventory or credentials; this is not a claim +that setup makes no other network requests. You can then onboard two real clients and confirm the isolation boundary on your own infrastructure before buying. diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 02ffbb702..d99a4ef3e 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -5373,3 +5373,20 @@ trust suite verifies every consumer pin, absence of the affected auth assumption workflow trust controls and the retained native Windows command/lifecycle proof steps. Native Windows execution remains a hosted check, not a local Linux claim. This upgrade is independent of the grouped signing/Docker/Tailscale updates. + +### Published MSP evaluation bundle guidance + +The MSP guide and its shipped mirror select the signed v6.4.1 provider archive, +not a moving source archive. Verification of the pinned SSH identity/namespace +and checksum precedes extraction and privileged setup. The licence request +contains the generated public key, setup-stage and signup-source fields, plus +email only when explicitly supplied. Describe that request rather than asserting +anonymous setup or absence of all outbound credentials/network traffic. + +`TestProviderMSPEvaluationDocsUsePublishedSignedBundle` checks the synchronized +guides, exact version, verification commands and bounded payload disclosure. +Retained provider delivery verification establishes archive SHA256 +`a2f3f6df98385e7dbc47e5df9ca014046b8d81eba353420d8f22a668de621b88`, +a valid pinned SSH signature and the extracted request expression with absent +and synthetic email. It does not establish installed onboarding, server acceptance +or legal identity. See the qualification evidence below. diff --git a/frontend-modern/public/docs/MSP.md b/frontend-modern/public/docs/MSP.md index 855a76365..cb746ec8a 100644 --- a/frontend-modern/public/docs/MSP.md +++ b/frontend-modern/public/docs/MSP.md @@ -271,7 +271,7 @@ before money changes hands. ### Evaluating without a licence Self-service evaluation is available from the signed provider bundle published -with Pulse v6.2.1. Use this exact release asset and its integrity sidecars; do +with Pulse v6.4.1. Use this exact release asset and its integrity sidecars; do **not** download the moving `main` branch archive or run its `setup.sh` as root. For a later release, first confirm its release page contains the versioned provider archive, checksum, and SSH signature before changing the @@ -281,7 +281,7 @@ Download the versioned asset, verify it with Pulse's pinned release key, and only then extract and run the guided setup: ```bash -export PULSE_VERSION=v6.2.1 +export PULSE_VERSION=v6.4.1 export PULSE_MSP_BUNDLE="pulse-provider-msp-${PULSE_VERSION}.tar.gz" export PULSE_RELEASE_BASE="https://github.com/rcourtman/Pulse/releases/download/${PULSE_VERSION}" @@ -301,8 +301,11 @@ cd "pulse-provider-msp-${PULSE_VERSION}" sudo -E bash ./setup.sh ``` -The v6.2.1 evaluation request is anonymous. If you want setup help, start from -the [Pulse MSP evaluation page](https://pulserelay.pro/msp.html#evaluate) first; +The licence request omits an email address unless you export +`PULSE_PROVIDER_MSP_EVAL_EMAIL` before running `setup.sh`, in which case that +address is included in the request. If you +want setup help, start from the +[Pulse MSP evaluation page](https://pulserelay.pro/msp.html#evaluate) first; that contact request remains separate from the licence activation. The host needs Ubuntu 24.04 or similar, a domain you can point at it, and @@ -313,9 +316,11 @@ Traefik dnsChallenge provider works by setting `ACME_DNS_PROVIDER` in `.env` and putting that provider's credential variables in `dns-credentials.env`. Leave `CP_PROVIDER_MSP_LICENSE_FILE` blank and `setup.sh` self-issues a -2-client evaluation licence. The v6.2.1 bundle sends only the public half of -the signing key generated on your host. The private key, client inventory, -credentials, and contact details never leave the machine. You can then onboard +2-client evaluation licence. The v6.4.1 bundle sends the public half of the +signing key generated on your host, a setup-stage marker, and a signup-source +label, plus the optional email address. This licence-request payload does not +include the private key, client inventory or credentials; this is not a claim +that setup makes no other network requests. You can then onboard two real clients and confirm the isolation boundary on your own infrastructure before buying. diff --git a/scripts/installtests/provider_msp_deploy_test.go b/scripts/installtests/provider_msp_deploy_test.go index e53cace76..113218d23 100644 --- a/scripts/installtests/provider_msp_deploy_test.go +++ b/scripts/installtests/provider_msp_deploy_test.go @@ -498,20 +498,27 @@ func TestProviderMSPEvaluationDocsUsePublishedSignedBundle(t *testing.T) { doc := string(repoDocBytes) assertContainsAll(t, doc, "signed provider bundle published", - "with Pulse v6.2.1", + "with Pulse v6.4.1", "**not** download the moving `main` branch archive", - `export PULSE_VERSION=v6.2.1`, + `export PULSE_VERSION=v6.4.1`, `PULSE_MSP_BUNDLE="pulse-provider-msp-${PULSE_VERSION}.tar.gz"`, `releases/download/${PULSE_VERSION}`, "ssh-keygen -Y verify", `-s "${PULSE_MSP_BUNDLE}.sshsig" < "${PULSE_MSP_BUNDLE}"`, `sha256sum -c "${PULSE_MSP_BUNDLE}.sha256"`, - "The v6.2.1 evaluation request is anonymous", + "The licence request omits an email address unless you export", + "`PULSE_PROVIDER_MSP_EVAL_EMAIL` before running `setup.sh`", + "address is included in the request", + "a setup-stage marker, and a signup-source", + "this is not a claim", + "that setup makes no other network requests", "contact request remains separate from the licence activation", `sudo -E bash ./setup.sh`, ) assertNotContainsAny(t, doc, "Pulse/archive/refs/heads/main.tar.gz", + "evaluation request is anonymous", + "credentials never leave the", "cd Pulse-main/deploy/provider-msp", "sudo -E ./setup.sh", ) diff --git a/tests/qualification/provider-delivery/README.md b/tests/qualification/provider-delivery/README.md new file mode 100644 index 000000000..f117d2004 --- /dev/null +++ b/tests/qualification/provider-delivery/README.md @@ -0,0 +1,19 @@ +# Provider bundle delivery verification — 9 September 2026 + +The public v6.4.1 release delivered `pulse-provider-msp-v6.4.1.tar.gz` and its +`.sha256` and `.sshsig` sidecars. Retained verification: +- SHA256: a2f3f6df98385e7dbc47e5df9ca014046b8d81eba353420d8f22a668de621b88 +- Checksum verification passed. +- SSH signature verification passed using identity `pulse-installer`, + namespace `pulse-install`, fingerprint + `SHA256:WjzDnbyb4fF3hPGRE1ZLtYcXzLimGpJq6Ou4opquTV0`. +- Extracted setup.sh lines 23–24 default email empty and source + `provider_msp_setup`; lines 410–419 form the licence request. +- Offline execution of that exact jq expression passed with absent email and + synthetic .invalid email, checking the complete field allowlist. + +These are retained delivery and offline payload results, not a new installation +test. Setup, key generation, licence activation and customer lookup were not run. +The current documentation regression checks version selection, signature/checksum +commands, request-specific disclosure and byte-identical shipped documentation. +This does not qualify later provider archives or prove server-side acceptance. From ac636f98b1933c397bf6402fde3aa83c36267ac6 Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 20:00:21 +0100 Subject: [PATCH 2/2] fix(alerts): honour PBS datastore capacity policy Keep topology capacity bands as resource risk rather than duplicate parent and datastore alerts. The existing live PBS capacity evaluator owns thresholds, aliases and hysteresis; retain datastore state failures and reconcile old duplicates normally. Reproduce the 97.9% report with real registry projection and exercise threshold changes through synthetic PBS HTTP polling. Release adaptations require the live-poll evaluator, not this filter alone. Change-source: pulse-maintainer --- .../v6/internal/subsystems/alerts.md | 24 ++++ internal/alerts/unified_incidents.go | 13 ++ internal/alerts/unified_incidents_test.go | 127 +++++++++++++++--- .../monitoring/monitor_pbs_coverage_test.go | 25 +++- 4 files changed, 169 insertions(+), 20 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index 49dc1123c..fd7a30e86 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -2856,3 +2856,27 @@ This changes neither delivery scheduling nor the meaning of a successful test. The registered WebhookConfig regression covers alias test/save payload equality; the browser fixture exercises the real form at desktop and phone widths with synthetic callbacks, not a hosted Pushover destination or installed delivery. + +### PBS capacity has one alert policy owner + +PBS datastore capacity alerts are evaluated by `CheckStorageWithCapacityTrend` +from fresh PBS polls. Storage defaults, canonical datastore aliases, per-resource +overrides, hysteresis and predictive capacity policy govern that lifecycle. +The fixed 90/95% PBS topology assessment remains resource risk evidence; its +Pulse-generated `capacity_runway_low` incidents must not independently enter +active alerts on either the datastore or the parent backup server. Existing +copies retire through normal policy reconciliation, without deleting state or +claiming that capacity itself recovered. Other datastore state/error incidents +and native provider incidents retain their existing lifecycle. + +`TestPBSCapacityUsesStoragePolicyNotTopologyBands` in +`internal/alerts/unified_incidents_test.go` uses real registry projection to pin +both duplicate symptoms at 97.9%, canonical-alias 99% versus 90% policy, existing +alert retirement, threshold recovery, and preservation of datastore failure. +The posture and roll-up tests use datastore state failures, independent of +capacity. This relies on the live PBS poll evaluator; a release adaptation must +include that evaluator rather than remove the topology alerts in isolation. +`TestPBSPolledCapacityRequiresObservedRecovery` in +`internal/monitoring/monitor_pbs_coverage_test.go` additionally exercises the +97.9% policy transition through synthetic PBS HTTP polling, storage conversion +and unified alert synchronisation, including absence of duplicate parent alerts. diff --git a/internal/alerts/unified_incidents.go b/internal/alerts/unified_incidents.go index 4130235da..cb1239e78 100644 --- a/internal/alerts/unified_incidents.go +++ b/internal/alerts/unified_incidents.go @@ -94,6 +94,19 @@ func (m *Manager) SyncUnifiedResourceIncidents(resources []unifiedresources.Reso storageKey := canonicalTrackingKeyForSpec(spec, alert.ID) observedConditions[storageKey] = struct{}{} + // PBS capacity is evaluated by CheckStorageWithCapacityTrend using + // storage defaults, aliases, overrides and hysteresis. The topology's + // fixed risk bands are resource context, not a second alert policy. + // Keep the condition observed so pre-upgrade duplicates retire as a + // policy change without claiming that datastore health recovered. + if strings.EqualFold(strings.TrimSpace(incident.Provider), "pulse") && + incident.Code == "capacity_runway_low" && + (resource.Type == unifiedresources.ResourceTypePBS || + (resource.Type == unifiedresources.ResourceTypeStorage && resource.Storage != nil && + resource.Storage.Platform == "pbs" && resource.Storage.Type == "pbs-datastore")) { + continue + } + if alertType, ok := unifiedAlertResourceType(resource); ok { if disableAllKubernetes && isUnifiedKubernetesAlertType(alertType) { continue diff --git a/internal/alerts/unified_incidents_test.go b/internal/alerts/unified_incidents_test.go index 8bcac0c40..37976b396 100644 --- a/internal/alerts/unified_incidents_test.go +++ b/internal/alerts/unified_incidents_test.go @@ -4,6 +4,8 @@ import ( "testing" "time" + alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs" + "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust" "github.com/rcourtman/pulse-go-rewrite/internal/storagehealth" "github.com/rcourtman/pulse-go-rewrite/internal/truenas" @@ -576,7 +578,7 @@ func TestSyncUnifiedResourceIncidentsMarksPBSBackupPosture(t *testing.T) { Hostname: "pbs-main.local", DatastoreCount: 2, Datastores: []unifiedresources.PBSDatastoreMeta{ - {Name: "fast", Status: "online", Total: 100, Used: 96}, + {Name: "fast", Status: "ERROR", Total: 100, Used: 96}, {Name: "archive", Status: "online", Total: 100, Used: 40}, }, ProtectedWorkloadCount: 2, @@ -585,22 +587,22 @@ func TestSyncUnifiedResourceIncidentsMarksPBSBackupPosture(t *testing.T) { StorageRisk: &unifiedresources.StorageRisk{ Level: storagehealth.RiskCritical, Reasons: []unifiedresources.StorageRiskReason{ - {Code: "capacity_runway_low", Severity: storagehealth.RiskCritical, Summary: "PBS datastore fast is 96% full"}, + {Code: "pbs_datastore_state", Severity: storagehealth.RiskCritical, Summary: "PBS datastore fast is ERROR"}, }, }, }, Incidents: []unifiedresources.ResourceIncident{{ Provider: "pulse", - NativeID: "pbs-instance:pbs-main:capacity_runway_low", - Code: "capacity_runway_low", + NativeID: "pbs-instance:pbs-main:pbs_datastore_state", + Code: "pbs_datastore_state", Severity: storagehealth.RiskCritical, - Summary: "PBS datastore fast is 96% full", + Summary: "PBS datastore fast is ERROR", }}, } m.SyncUnifiedResourceIncidents([]unifiedresources.Resource{resource}) - alertID := "unified-incident-pbs-main-pulse-pbs-instance-pbs-main-capacity-runway-low-capacity-runway-low" + alertID := "unified-incident-pbs-main-pulse-pbs-instance-pbs-main-pbs-datastore-state-pbs-datastore-state" assertAlertPresent(t, m, alertID) m.mu.RLock() @@ -610,7 +612,7 @@ func TestSyncUnifiedResourceIncidentsMarksPBSBackupPosture(t *testing.T) { if alert.Type != "backup-posture-incident" { t.Fatalf("alert type = %q, want backup-posture-incident", alert.Type) } - wantMessage := "Backup server pbs-main has datastore capacity risk. Affects 1 backup datastore: fast" + wantMessage := "Backup server pbs-main has degraded datastore availability. Affects 1 backup datastore: fast" if alert.Message != wantMessage { t.Fatalf("message = %q, want %q", alert.Message, wantMessage) } @@ -734,17 +736,17 @@ func TestSyncUnifiedResourceIncidentsSuppressesPBSDatastoreChildWhenParentRollsU PBS: &unifiedresources.PBSData{ DatastoreCount: 1, Datastores: []unifiedresources.PBSDatastoreMeta{ - {Name: "fast", Status: "online", Total: 100, Used: 96}, + {Name: "fast", Status: "ERROR", Total: 100, Used: 96}, }, ProtectedWorkloadCount: 2, ProtectedWorkloadNames: []string{"media01", "app01"}, }, Incidents: []unifiedresources.ResourceIncident{{ Provider: "pulse", - NativeID: "pbs-instance:pbs-main:capacity_runway_low", - Code: "capacity_runway_low", + NativeID: "pbs-instance:pbs-main:pbs_datastore_state", + Code: "pbs_datastore_state", Severity: storagehealth.RiskCritical, - Summary: "PBS datastore fast is 96% full", + Summary: "PBS datastore fast is ERROR", }}, }, { @@ -761,10 +763,10 @@ func TestSyncUnifiedResourceIncidentsSuppressesPBSDatastoreChildWhenParentRollsU }, Incidents: []unifiedresources.ResourceIncident{{ Provider: "pulse", - NativeID: "pbs-instance:pbs-main:capacity_runway_low", - Code: "capacity_runway_low", + NativeID: "pbs-instance:pbs-main:pbs_datastore_state", + Code: "pbs_datastore_state", Severity: storagehealth.RiskCritical, - Summary: "PBS datastore fast is 96% full", + Summary: "PBS datastore fast is ERROR", }}, }, } @@ -1003,17 +1005,17 @@ func TestGetActiveAlertsPrioritizesBackupPostureExposure(t *testing.T) { PBS: &unifiedresources.PBSData{ DatastoreCount: 1, Datastores: []unifiedresources.PBSDatastoreMeta{ - {Name: "fast", Status: "online", Total: 100, Used: 96}, + {Name: "fast", Status: "ERROR", Total: 100, Used: 96}, }, ProtectedWorkloadCount: 2, ProtectedWorkloadNames: []string{"media01", "app01"}, }, Incidents: []unifiedresources.ResourceIncident{{ Provider: "pulse", - NativeID: "pbs-instance:pbs-main:capacity_runway_low", - Code: "capacity_runway_low", + NativeID: "pbs-instance:pbs-main:pbs_datastore_state", + Code: "pbs_datastore_state", Severity: storagehealth.RiskCritical, - Summary: "PBS datastore fast is 96% full", + Summary: "PBS datastore fast is ERROR", }}, }, { @@ -1332,3 +1334,92 @@ func TestTrueNASNativeCriticalTransition(t *testing.T) { }) } } + +// PBS capacity has one policy owner: CheckStorage. Topology risk remains +// visible, but must not create threshold-independent child/parent alerts. +func TestPBSCapacityUsesStoragePolicyNotTopologyBands(t *testing.T) { + m := newTestManager(t) + config := unifiedEvalBaseConfig() + config.StorageDefault = HysteresisThreshold{Trigger: 90, Clear: 85} + config.Overrides = map[string]ThresholdConfig{"pbs-main/fast": {Usage: &HysteresisThreshold{Trigger: 99, Clear: 98}}} + configureUnifiedEvalManager(t, m, config) + disableTestTimeThresholds(m) + instance := models.PBSInstance{ID: "pbs-main", Name: "main", Status: "online", LastSeen: time.Now(), Datastores: []models.PBSDatastore{{Name: "fast", Status: "online", Total: 1000, Used: 979, Free: 21, Usage: 97.9}}} + registry := unifiedresources.NewRegistry(unifiedresources.NewMemoryStore()) + registry.IngestSnapshot(models.StateSnapshot{PBSInstances: []models.PBSInstance{instance}}) + resources := registry.List() + capacityResources := 0 + for _, r := range resources { + for _, i := range r.Incidents { + if i.Code == "capacity_runway_low" { + capacityResources++ + break + } + } + } + if capacityResources != 2 { + t.Fatalf("want real parent and child capacity evidence, got %d", capacityResources) + } + storage := models.Storage{ID: "pbs-main-fast", AliasIDs: []string{"pbs-main/fast"}, Name: "fast", Instance: "pbs-main", Type: "pbs", Status: "online", Total: 1000, Used: 979, Free: 21, Usage: 97.9} + observe := func() { + for range 5 { + m.CheckStorage(storage) + m.SyncUnifiedResourceIncidents(resources) + } + } + observe() + if active := m.GetActiveAlerts(); len(active) != 0 { + t.Fatalf("99%% policy bypassed by topology incidents: %+v", active) + } + // Seed both pre-upgrade canonical alerts, with unchanged risk evidence. + // The next sync must retire them without deleting resource observations. + m.mu.Lock() + for _, spec := range alertspecs.BuildUnifiedResourceAlertSpecs(resources) { + if spec.Kind != alertspecs.AlertSpecKindProviderIncident { + continue + } + for _, resource := range resources { + if resource.ID != spec.ResourceID { + continue + } + incident, ok := incidentForProviderSpec(resource, spec) + if !ok || incident.Code != "capacity_runway_low" { + continue + } + alert := unifiedIncidentAlert(resource, incident, AlertLevelCritical, time.Now()) + applyCanonicalIdentity(alert, spec.ID, string(spec.Kind)) + m.setActiveAlertNoLock(canonicalTrackingKeyForSpec(spec, alert.ID), alert) + } + } + m.mu.Unlock() + if active := m.GetActiveAlerts(); len(active) != 2 { + t.Fatalf("want two pre-upgrade duplicates, got %d", len(active)) + } + observe() + if active := m.GetActiveAlerts(); len(active) != 0 { + t.Fatalf("pre-upgrade duplicates retained: %+v", active) + } + config.Overrides["pbs-main/fast"] = ThresholdConfig{Usage: &HysteresisThreshold{Trigger: 90, Clear: 85}} + m.UpdateConfig(config) + disableTestTimeThresholds(m) + observe() + active := m.GetActiveAlerts() + if len(active) != 1 || active[0].ResourceID != storage.ID || active[0].CanonicalKind != "metric-threshold" { + t.Fatalf("want one policy-owned capacity alert, got %+v", active) + } + config.Overrides["pbs-main/fast"] = ThresholdConfig{Usage: &HysteresisThreshold{Trigger: 99, Clear: 98}} + m.UpdateConfig(config) + disableTestTimeThresholds(m) + observe() + if active := m.GetActiveAlerts(); len(active) != 0 { + t.Fatalf("raising policy did not clear capacity: %+v", active) + } + // The same nearly-full datastore failing is still actionable. + instance.Datastores[0].Status = "ERROR" + registry.IngestSnapshot(models.StateSnapshot{PBSInstances: []models.PBSInstance{instance}}) + resources = registry.List() + observe() + if active := m.GetActiveAlerts(); len(active) == 0 { + t.Fatal("capacity policy hid datastore failure") + } +} diff --git a/internal/monitoring/monitor_pbs_coverage_test.go b/internal/monitoring/monitor_pbs_coverage_test.go index 34db989b4..a18a4ec6f 100644 --- a/internal/monitoring/monitor_pbs_coverage_test.go +++ b/internal/monitoring/monitor_pbs_coverage_test.go @@ -697,12 +697,33 @@ func TestPBSPolledCapacityRequiresObservedRecovery(t *testing.T) { t.Fatalf("incorrect recovery: %+v", resolved) } // Alternate PBS counter names must feed the same policy and identity. - // Stay below the separate 90% backup-posture incident threshold; the - // configured minimum delta of one permits this immediate recurrence. + // The configured minimum delta of one permits this immediate recurrence. response.Store(`{"data":{"total-space":1000,"used-space":860,"avail-space":140}}`) poll() active = manager.GetActiveAlerts() if len(active) != 1 || active[0].ID != original.ID || active[0].Value != 86 || !active[0].StartTime.After(original.StartTime) { t.Fatalf("incorrect recurrent incident: %+v", active) } + // The reported 97.9% crosses both topology bands. Neither the datastore + // nor parent posture may bypass the UI's 99% capacity policy. + response.Store(`{"data":{"total":1000,"used":979,"avail":21}}`) + highPolicy := basePolicy + highPolicy.Overrides = map[string]alerts.ThresholdConfig{"pbs-pbs-capacity/backups": { + Usage: &alerts.HysteresisThreshold{Trigger: 99, Clear: 98}, + }} + manager.UpdateConfig(highPolicy) + poll() + if active := manager.GetActiveAlerts(); len(active) != 0 { + t.Fatalf("97.9%% poll bypassed 99%% policy with topology incidents: %+v", active) + } + manager.UpdateConfig(basePolicy) + poll() + if active := manager.GetActiveAlerts(); len(active) != 1 || active[0].Type != "usage" { + t.Fatalf("high usage must have one policy-owned alert: %+v", active) + } + manager.UpdateConfig(highPolicy) + poll() + if active := manager.GetActiveAlerts(); len(active) != 0 { + t.Fatalf("raised policy did not clear high-usage alert: %+v", active) + } }