From 0df9747bd7ef97d768d5320e1952116ad4e675ea Mon Sep 17 00:00:00 2001 From: Pulse Test Date: Sat, 29 Aug 2026 13:59:13 +0100 Subject: [PATCH] Add privacy-safe alert quality telemetry --- docs/PRIVACY.md | 26 ++ .../v6/internal/subsystems/alerts.md | 19 + .../v6/internal/subsystems/api-contracts.md | 10 + .../v6/internal/subsystems/monitoring.md | 10 + .../v6/internal/subsystems/registry.json | 2 + .../internal/subsystems/security-privacy.md | 16 + frontend-modern/public/docs/PRIVACY.md | 26 ++ .../src/api/__tests__/settings.test.ts | 28 ++ frontend-modern/src/api/settings.ts | 28 ++ .../__tests__/useSystemSettingsState.test.ts | 28 ++ internal/alerts/telemetry_quality.go | 289 +++++++++++++ internal/alerts/telemetry_quality_test.go | 194 +++++++++ internal/monitoring/reload.go | 53 +-- internal/monitoring/reload_test.go | 6 +- internal/telemetry/telemetry.go | 106 ++++- internal/telemetry/telemetry_test.go | 117 ++++-- pkg/server/server.go | 28 ++ scripts/telemetry_adoption_report.py | 390 ++++++++++++++++-- .../tests/test_telemetry_adoption_report.py | 114 ++++- 19 files changed, 1364 insertions(+), 126 deletions(-) create mode 100644 internal/alerts/telemetry_quality.go create mode 100644 internal/alerts/telemetry_quality_test.go diff --git a/docs/PRIVACY.md b/docs/PRIVACY.md index 8be142c4a..af1859101 100644 --- a/docs/PRIVACY.md +++ b/docs/PRIVACY.md @@ -93,6 +93,21 @@ Every field is listed below with the reason it exists. Nothing else is included | Alerts fired 30d | `18` | Count unique locally retained alert occurrences in the current 30-day window without sending alert text, resource IDs, or timestamps | | Alerts acknowledged 30d | `7` | Count acknowledgements in the current 30-day window without sending actors, reasons, alert IDs, or timestamps | | Alerts resolved 30d | `12` | Count resolved alert records in the current 30-day window without sending resolution details, alert IDs, or resource IDs | +| Active alerts by severity | `1` info, `2` warning, `1` critical | Three aggregate counts that reconcile to active alerts | +| Active alert age buckets | `1` under 1h, `2` 1h-24h, `1` 1d-7d, `0` 7d+ | Four closed age buckets with no alert timestamps | +| Alerts fired by severity 30d | `2` info, `10` warning, `6` critical | Three aggregate occurrence counts that reconcile to alerts fired 30d | +| Alerts resolved by severity 30d | `1` info, `7` warning, `4` critical | Three aggregate occurrence counts that reconcile to alerts resolved 30d | +| Alert resolution duration buckets 30d | `3` under 15m, `2` 15m-1h, `5` 1h-24h, `2` 1d-7d, `0` 7d+ | Five closed time-to-resolution buckets with no occurrence timestamps | +| Repeat alert occurrences 30d | `4` | Count occurrences after the first local occurrence of the same canonical alert identity, without sending that identity | +| Snoozed alert occurrences 30d | `3` | Count occurrences with a durable suppression lifecycle transition | +| Alerts resolved while snoozed 30d | `2` | Count snoozed occurrences whose recovery transition arrived before an unsnooze transition | +| Alert manager tenants | `2` | Denominator for tenant-level alert configuration and persistence fields | +| Alert delivery active tenants | `2` | Count tenant alert managers with delivery enabled and activation state active | +| Alert flapping detection enabled tenants | `2` | Count tenant alert managers with flapping detection configured, not detected flapping episodes | +| Alert intent policy configured tenants | `1` | Count tenant alert managers with an explicitly revised default, resource-type, or resource intent policy | +| Alert event history authoritative tenants | `2` | Count tenant alert managers whose durable event history is authoritative | +| Alert active state authoritative tenants | `2` | Count tenant alert managers whose durable active-state projection is authoritative | +| Alert active state persistence degraded tenants | `0` | Count tenant alert managers carrying the fixed local degraded-state marker | | Notification attempts 7d | `14` | Count delivery attempts, including retry attempts, in the locally retained seven-day queue window without sending recipients, endpoints, titles, or message content | | Notification deliveries 7d | `11` | Count successfully delivered queue records in the local seven-day window without sending channel, recipient, endpoint, or content | | Notification failures 7d (terminal in schema v3) | `3` | Count terminal failed or dead-lettered delivery outcomes in the local seven-day window without sending retry-attempt failures, error text, endpoint, recipient, or message content | @@ -235,6 +250,16 @@ or usage counters to the new release. Listener, startup, runtime, API, UI, and asset failures collapse into fixed categories. Addresses, URLs, IP addresses, asset names, response bodies, and raw errors are neither sent nor stored. +Telemetry schema v14 adds the alert-quality aggregates listed above. The +sender folds identities and lifecycle transitions locally, then sends counts +only. Automatic versus operator resolution is deliberately absent because the +canonical alert lifecycle records recovery evidence but has no operator-resolve +action or resolution-actor provenance. Maintenance suppression effectiveness +is also absent because intent policies can suppress a candidate before an alert +occurrence exists, leaving no trustworthy fired-alert denominator. Detected +flapping episodes are not reported because their diagnostic event path may be +dropped under pressure. Configuration adoption is reported instead. + #### Server-side handling and retention - Telemetry pings are stored on the Pulse license server only for aggregate install/use analysis. @@ -243,6 +268,7 @@ asset names, response bodies, and raw errors are neither sent nor stored. - Aggregate reports preserve the closed deployment-method buckets, but treat them as best-effort current-runtime evidence. In particular, `container_other` and `binary_other` are unknown fallbacks for many upgraded installs, not precise original installation provenance. - Aggregate reports describe the known-age bucket as time since Pulse first created the schema-v2 lifecycle record. It is only a lower bound for an upgraded install and must not be presented as original installation age. - Aggregate release-health reports use the direct schema-v13 current and immediately previous observations. They do not infer new-release health from rolling update, feature, alert, or action counters. +- Aggregate alert-quality release comparisons require schema v14 on both cohorts, compare rolling counters only between consecutive same-version heartbeats, and stratify cohort exposure by known-age and heartbeat-count buckets. A first heartbeat is baseline-only, and cohorts with materially different exposure are not presented as like-for-like outcomes. - External-agent/MCP activity is stored only as a coarse adapter-origin flag plus capability-class counters: context, event stream, provisioning, operator state, findings, and action requests. - The receiver stores only fields in its versioned telemetry allowlist. A cross-repository parity check prevents client fields from being silently dropped and prevents the storage contract from growing beyond the disclosed payload. - Telemetry rows older than **90 days** are purged automatically. diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index f5447a8de..ec284e513 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -556,6 +556,25 @@ inspectability, or convert missing/stale evidence into health. ## Current State +### Alert-quality telemetry folds only canonical durable lifecycle truth + +`internal/alerts/telemetry_quality.go` folds active alerts and the canonical +30-day history projection into identity-free aggregate counts. Severity totals, +active-age buckets, resolution-duration buckets, repeated occurrences, and +snooze outcomes are computed locally. Canonical state and occurrence time may +be used transiently for folding, but neither leaves the process. The snapshot +also exposes tenant-level configuration and event/active-state authority counts +so downstream rates have denominators and persistence failures remain visible. + +The contract deliberately does not classify automatic versus operator +resolution because alert resolution currently records recovery evidence, not a +resolution actor or operator-resolve action. It does not claim maintenance +suppression effectiveness because an intent policy can suppress a candidate +before an alert occurrence exists. It also excludes detected flapping episode +counts because that diagnostic event path may drop under pressure. Flapping +configuration adoption remains measurable. `telemetry_quality_test.go` pins +occurrence folding, snooze sequencing, and exact bucket boundaries. + The alert webhook editor exposes the delivery contract's language-neutral `MessageKey`, event, resource type and node display name alongside the existing type, severity and metric fields. Presentation must advertise the backend-owned diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index 288545e3d..709a1665f 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -4209,6 +4209,16 @@ auto-register mutation boundary. ## Current State +### Telemetry preview is the canonical schema-v14 alert-quality payload + +The system settings telemetry preview continues to return the exact `Ping` +built by the outbound sender. Schema v14 adds aggregate alert lifecycle, +active-age, resolution-duration, repeat, snooze, configuration-adoption, and +persistence-health fields. `TelemetryPingPreview` declares the same fields, and +cross-repository schema parity covers the public Go payload, the frontend type, +and the private receiver allowlist. Older payload rows remain valid with zero +defaults for additive v14 columns. + Production package boundaries are active under `internal/api/alerting/` and `internal/api/configapi/`, backed by shared tenant-context, scope-enforcement, install-token, and agent-binding packages. Alert/notification and diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 5593de542..16ac2399e 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -1178,6 +1178,16 @@ cleanup so readers cannot retain orphaned runtime or alert projections. ## Current State +### Install snapshots aggregate alert quality across isolated tenant managers + +`ReloadableMonitor.AggregateInstallSnapshotCounts` loads each provisioned +organization through its own monitor, reads one privacy-bounded alert-quality +snapshot from that tenant's alert manager, and adds only aggregate integers to +the install snapshot. Organization IDs and alert identities are never copied +into telemetry state. Existing fired, acknowledged, and resolved totals now +come from the same canonical fold as the v14 quality fields, keeping legacy +consumers compatible while preventing two calculations from drifting. + ### Proxmox and agent disk observations share full hardware identity PVE may publish a RAID array volume's full NAA value as a bare serial while diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json index 0d145cabf..456b52fda 100644 --- a/docs/release-control/v6/internal/subsystems/registry.json +++ b/docs/release-control/v6/internal/subsystems/registry.json @@ -2585,6 +2585,7 @@ "internal/alerts/reducer_parity_refire_test.go", "internal/alerts/resolved_lock_discipline_test.go", "internal/alerts/synology_test.go", + "internal/alerts/telemetry_quality_test.go", "internal/alerts/threshold_resolution_shared_test.go", "internal/alerts/unified_incident_confirmation_test.go", "internal/alerts/update_alerts_test.go", @@ -6101,6 +6102,7 @@ "internal/monitoring/proxmox_large_cluster_poll_budget_test.go", "internal/monitoring/pve_protection_observation_test.go", "internal/monitoring/ratetracker_test.go", + "internal/monitoring/reload_test.go", "internal/monitoring/truenas_poller_test.go", "internal/unifiedresources/code_standards_test.go" ] diff --git a/docs/release-control/v6/internal/subsystems/security-privacy.md b/docs/release-control/v6/internal/subsystems/security-privacy.md index 839072fa1..6ba66d72c 100644 --- a/docs/release-control/v6/internal/subsystems/security-privacy.md +++ b/docs/release-control/v6/internal/subsystems/security-privacy.md @@ -740,6 +740,22 @@ tokens, and path-normalization variants. ## Current State +### Alert-quality telemetry remains aggregate and comparison-safe + +Telemetry schema v14 exports only counts and closed buckets. It exports no +resource or destination identity, hostname, alert or rule content, lifecycle +timestamp, actor, reason, error text, or clickstream. Public and frontend-served +privacy disclosures enumerate the fields and explain the deliberately excluded +resolution, maintenance-suppression, and detected-flapping classifications. + +The adoption report treats a version's first heartbeat as baseline-only and +uses only consecutive same-version changes for rolling counters. Optional +release comparisons require schema v14 in both cohorts, then match observations +by the latest known-install-age bucket and version heartbeat-count bucket. +Unmatched exposure is reported rather than pooled, and the specific 6.4.0 +versus 6.3.2 comparison is rejected as schema-incompatible instead of presenting +legacy alert volume as alert quality. + ### Docker update preflight remains read-only across the unified-agent bridge The `pulse-agent` late-bound Docker updater delegates the typed, read-only diff --git a/frontend-modern/public/docs/PRIVACY.md b/frontend-modern/public/docs/PRIVACY.md index 8be142c4a..af1859101 100644 --- a/frontend-modern/public/docs/PRIVACY.md +++ b/frontend-modern/public/docs/PRIVACY.md @@ -93,6 +93,21 @@ Every field is listed below with the reason it exists. Nothing else is included | Alerts fired 30d | `18` | Count unique locally retained alert occurrences in the current 30-day window without sending alert text, resource IDs, or timestamps | | Alerts acknowledged 30d | `7` | Count acknowledgements in the current 30-day window without sending actors, reasons, alert IDs, or timestamps | | Alerts resolved 30d | `12` | Count resolved alert records in the current 30-day window without sending resolution details, alert IDs, or resource IDs | +| Active alerts by severity | `1` info, `2` warning, `1` critical | Three aggregate counts that reconcile to active alerts | +| Active alert age buckets | `1` under 1h, `2` 1h-24h, `1` 1d-7d, `0` 7d+ | Four closed age buckets with no alert timestamps | +| Alerts fired by severity 30d | `2` info, `10` warning, `6` critical | Three aggregate occurrence counts that reconcile to alerts fired 30d | +| Alerts resolved by severity 30d | `1` info, `7` warning, `4` critical | Three aggregate occurrence counts that reconcile to alerts resolved 30d | +| Alert resolution duration buckets 30d | `3` under 15m, `2` 15m-1h, `5` 1h-24h, `2` 1d-7d, `0` 7d+ | Five closed time-to-resolution buckets with no occurrence timestamps | +| Repeat alert occurrences 30d | `4` | Count occurrences after the first local occurrence of the same canonical alert identity, without sending that identity | +| Snoozed alert occurrences 30d | `3` | Count occurrences with a durable suppression lifecycle transition | +| Alerts resolved while snoozed 30d | `2` | Count snoozed occurrences whose recovery transition arrived before an unsnooze transition | +| Alert manager tenants | `2` | Denominator for tenant-level alert configuration and persistence fields | +| Alert delivery active tenants | `2` | Count tenant alert managers with delivery enabled and activation state active | +| Alert flapping detection enabled tenants | `2` | Count tenant alert managers with flapping detection configured, not detected flapping episodes | +| Alert intent policy configured tenants | `1` | Count tenant alert managers with an explicitly revised default, resource-type, or resource intent policy | +| Alert event history authoritative tenants | `2` | Count tenant alert managers whose durable event history is authoritative | +| Alert active state authoritative tenants | `2` | Count tenant alert managers whose durable active-state projection is authoritative | +| Alert active state persistence degraded tenants | `0` | Count tenant alert managers carrying the fixed local degraded-state marker | | Notification attempts 7d | `14` | Count delivery attempts, including retry attempts, in the locally retained seven-day queue window without sending recipients, endpoints, titles, or message content | | Notification deliveries 7d | `11` | Count successfully delivered queue records in the local seven-day window without sending channel, recipient, endpoint, or content | | Notification failures 7d (terminal in schema v3) | `3` | Count terminal failed or dead-lettered delivery outcomes in the local seven-day window without sending retry-attempt failures, error text, endpoint, recipient, or message content | @@ -235,6 +250,16 @@ or usage counters to the new release. Listener, startup, runtime, API, UI, and asset failures collapse into fixed categories. Addresses, URLs, IP addresses, asset names, response bodies, and raw errors are neither sent nor stored. +Telemetry schema v14 adds the alert-quality aggregates listed above. The +sender folds identities and lifecycle transitions locally, then sends counts +only. Automatic versus operator resolution is deliberately absent because the +canonical alert lifecycle records recovery evidence but has no operator-resolve +action or resolution-actor provenance. Maintenance suppression effectiveness +is also absent because intent policies can suppress a candidate before an alert +occurrence exists, leaving no trustworthy fired-alert denominator. Detected +flapping episodes are not reported because their diagnostic event path may be +dropped under pressure. Configuration adoption is reported instead. + #### Server-side handling and retention - Telemetry pings are stored on the Pulse license server only for aggregate install/use analysis. @@ -243,6 +268,7 @@ asset names, response bodies, and raw errors are neither sent nor stored. - Aggregate reports preserve the closed deployment-method buckets, but treat them as best-effort current-runtime evidence. In particular, `container_other` and `binary_other` are unknown fallbacks for many upgraded installs, not precise original installation provenance. - Aggregate reports describe the known-age bucket as time since Pulse first created the schema-v2 lifecycle record. It is only a lower bound for an upgraded install and must not be presented as original installation age. - Aggregate release-health reports use the direct schema-v13 current and immediately previous observations. They do not infer new-release health from rolling update, feature, alert, or action counters. +- Aggregate alert-quality release comparisons require schema v14 on both cohorts, compare rolling counters only between consecutive same-version heartbeats, and stratify cohort exposure by known-age and heartbeat-count buckets. A first heartbeat is baseline-only, and cohorts with materially different exposure are not presented as like-for-like outcomes. - External-agent/MCP activity is stored only as a coarse adapter-origin flag plus capability-class counters: context, event stream, provisioning, operator state, findings, and action requests. - The receiver stores only fields in its versioned telemetry allowlist. A cross-repository parity check prevents client fields from being silently dropped and prevents the storage contract from growing beyond the disclosed payload. - Telemetry rows older than **90 days** are purged automatically. diff --git a/frontend-modern/src/api/__tests__/settings.test.ts b/frontend-modern/src/api/__tests__/settings.test.ts index 56bf711ad..56b601bc3 100644 --- a/frontend-modern/src/api/__tests__/settings.test.ts +++ b/frontend-modern/src/api/__tests__/settings.test.ts @@ -87,6 +87,34 @@ const mockTelemetryPreviewPayload = { alerts_fired_30d: 0, alerts_acknowledged_30d: 0, alerts_resolved_30d: 0, + active_alerts_info: 0, + active_alerts_warning: 0, + active_alerts_critical: 0, + active_alerts_age_under_1h: 0, + active_alerts_age_1h_24h: 0, + active_alerts_age_1d_7d: 0, + active_alerts_age_7d_plus: 0, + alerts_fired_info_30d: 0, + alerts_fired_warning_30d: 0, + alerts_fired_critical_30d: 0, + alerts_resolved_info_30d: 0, + alerts_resolved_warning_30d: 0, + alerts_resolved_critical_30d: 0, + alerts_resolution_under_15m_30d: 0, + alerts_resolution_15m_1h_30d: 0, + alerts_resolution_1h_24h_30d: 0, + alerts_resolution_1d_7d_30d: 0, + alerts_resolution_7d_plus_30d: 0, + alerts_repeat_occurrences_30d: 0, + alerts_snoozed_occurrences_30d: 0, + alerts_resolved_while_snoozed_30d: 0, + alert_manager_tenants: 0, + alert_delivery_active_tenants: 0, + alert_flapping_enabled_tenants: 0, + alert_intent_policy_configured_tenants: 0, + alert_event_history_authoritative_tenants: 0, + alert_active_state_authoritative_tenants: 0, + alert_active_state_persistence_degraded_tenants: 0, notification_attempts_7d: 0, notification_deliveries_7d: 0, notification_failures_7d: 0, diff --git a/frontend-modern/src/api/settings.ts b/frontend-modern/src/api/settings.ts index e345437a7..39b19ebf0 100644 --- a/frontend-modern/src/api/settings.ts +++ b/frontend-modern/src/api/settings.ts @@ -88,6 +88,34 @@ export interface TelemetryPingPreview { alerts_fired_30d: number; alerts_acknowledged_30d: number; alerts_resolved_30d: number; + active_alerts_info: number; + active_alerts_warning: number; + active_alerts_critical: number; + active_alerts_age_under_1h: number; + active_alerts_age_1h_24h: number; + active_alerts_age_1d_7d: number; + active_alerts_age_7d_plus: number; + alerts_fired_info_30d: number; + alerts_fired_warning_30d: number; + alerts_fired_critical_30d: number; + alerts_resolved_info_30d: number; + alerts_resolved_warning_30d: number; + alerts_resolved_critical_30d: number; + alerts_resolution_under_15m_30d: number; + alerts_resolution_15m_1h_30d: number; + alerts_resolution_1h_24h_30d: number; + alerts_resolution_1d_7d_30d: number; + alerts_resolution_7d_plus_30d: number; + alerts_repeat_occurrences_30d: number; + alerts_snoozed_occurrences_30d: number; + alerts_resolved_while_snoozed_30d: number; + alert_manager_tenants: number; + alert_delivery_active_tenants: number; + alert_flapping_enabled_tenants: number; + alert_intent_policy_configured_tenants: number; + alert_event_history_authoritative_tenants: number; + alert_active_state_authoritative_tenants: number; + alert_active_state_persistence_degraded_tenants: number; notification_attempts_7d: number; notification_deliveries_7d: number; notification_failures_7d: number; diff --git a/frontend-modern/src/components/Settings/__tests__/useSystemSettingsState.test.ts b/frontend-modern/src/components/Settings/__tests__/useSystemSettingsState.test.ts index 1fd5d4105..44869d371 100644 --- a/frontend-modern/src/components/Settings/__tests__/useSystemSettingsState.test.ts +++ b/frontend-modern/src/components/Settings/__tests__/useSystemSettingsState.test.ts @@ -93,6 +93,34 @@ const buildTelemetryPreviewPayload = ( alerts_fired_30d: 0, alerts_acknowledged_30d: 0, alerts_resolved_30d: 0, + active_alerts_info: 0, + active_alerts_warning: 0, + active_alerts_critical: 0, + active_alerts_age_under_1h: 0, + active_alerts_age_1h_24h: 0, + active_alerts_age_1d_7d: 0, + active_alerts_age_7d_plus: 0, + alerts_fired_info_30d: 0, + alerts_fired_warning_30d: 0, + alerts_fired_critical_30d: 0, + alerts_resolved_info_30d: 0, + alerts_resolved_warning_30d: 0, + alerts_resolved_critical_30d: 0, + alerts_resolution_under_15m_30d: 0, + alerts_resolution_15m_1h_30d: 0, + alerts_resolution_1h_24h_30d: 0, + alerts_resolution_1d_7d_30d: 0, + alerts_resolution_7d_plus_30d: 0, + alerts_repeat_occurrences_30d: 0, + alerts_snoozed_occurrences_30d: 0, + alerts_resolved_while_snoozed_30d: 0, + alert_manager_tenants: 0, + alert_delivery_active_tenants: 0, + alert_flapping_enabled_tenants: 0, + alert_intent_policy_configured_tenants: 0, + alert_event_history_authoritative_tenants: 0, + alert_active_state_authoritative_tenants: 0, + alert_active_state_persistence_degraded_tenants: 0, notification_attempts_7d: 0, notification_deliveries_7d: 0, notification_failures_7d: 0, diff --git a/internal/alerts/telemetry_quality.go b/internal/alerts/telemetry_quality.go new file mode 100644 index 000000000..9d9eac7bc --- /dev/null +++ b/internal/alerts/telemetry_quality.go @@ -0,0 +1,289 @@ +package alerts + +import ( + "fmt" + "sort" + "strings" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust" +) + +// AlertQualitySnapshot is an identity-free aggregate of the alert lifecycle. +// It is safe to add across tenant managers. Canonical alert identity is used +// only inside CalculateAlertQualitySnapshot to fold local history occurrences. +type AlertQualitySnapshot struct { + ActiveInfo int + ActiveWarning int + ActiveCritical int + ActiveAgeUnder1h int + ActiveAge1h24h int + ActiveAge1d7d int + ActiveAge7dPlus int + Fired30d int + FiredInfo30d int + FiredWarning30d int + FiredCritical30d int + Acknowledged30d int + Resolved30d int + ResolvedInfo30d int + ResolvedWarning30d int + ResolvedCritical30d int + ResolutionUnder15m30d int + Resolution15m1h30d int + Resolution1h24h30d int + Resolution1d7d30d int + Resolution7dPlus30d int + RepeatOccurrences30d int + SnoozedOccurrences30d int + ResolvedWhileSnoozed30d int + ManagerTenants int + DeliveryActiveTenants int + FlappingEnabledTenants int + IntentPolicyTenants int + EventHistoryHealthyTenants int + ActiveStateHealthyTenants int + ActiveStateDegradedTenants int +} + +// Add merges another tenant snapshot without retaining tenant identity. +func (snapshot *AlertQualitySnapshot) Add(other AlertQualitySnapshot) { + snapshot.ActiveInfo += other.ActiveInfo + snapshot.ActiveWarning += other.ActiveWarning + snapshot.ActiveCritical += other.ActiveCritical + snapshot.ActiveAgeUnder1h += other.ActiveAgeUnder1h + snapshot.ActiveAge1h24h += other.ActiveAge1h24h + snapshot.ActiveAge1d7d += other.ActiveAge1d7d + snapshot.ActiveAge7dPlus += other.ActiveAge7dPlus + snapshot.Fired30d += other.Fired30d + snapshot.FiredInfo30d += other.FiredInfo30d + snapshot.FiredWarning30d += other.FiredWarning30d + snapshot.FiredCritical30d += other.FiredCritical30d + snapshot.Acknowledged30d += other.Acknowledged30d + snapshot.Resolved30d += other.Resolved30d + snapshot.ResolvedInfo30d += other.ResolvedInfo30d + snapshot.ResolvedWarning30d += other.ResolvedWarning30d + snapshot.ResolvedCritical30d += other.ResolvedCritical30d + snapshot.ResolutionUnder15m30d += other.ResolutionUnder15m30d + snapshot.Resolution15m1h30d += other.Resolution15m1h30d + snapshot.Resolution1h24h30d += other.Resolution1h24h30d + snapshot.Resolution1d7d30d += other.Resolution1d7d30d + snapshot.Resolution7dPlus30d += other.Resolution7dPlus30d + snapshot.RepeatOccurrences30d += other.RepeatOccurrences30d + snapshot.SnoozedOccurrences30d += other.SnoozedOccurrences30d + snapshot.ResolvedWhileSnoozed30d += other.ResolvedWhileSnoozed30d + snapshot.ManagerTenants += other.ManagerTenants + snapshot.DeliveryActiveTenants += other.DeliveryActiveTenants + snapshot.FlappingEnabledTenants += other.FlappingEnabledTenants + snapshot.IntentPolicyTenants += other.IntentPolicyTenants + snapshot.EventHistoryHealthyTenants += other.EventHistoryHealthyTenants + snapshot.ActiveStateHealthyTenants += other.ActiveStateHealthyTenants + snapshot.ActiveStateDegradedTenants += other.ActiveStateDegradedTenants +} + +// AlertQualityTelemetrySnapshot returns the current manager's aggregate +// quality snapshot for a closed 30-day history window. +func (m *Manager) AlertQualityTelemetrySnapshot(now time.Time) AlertQualitySnapshot { + if m == nil { + return AlertQualitySnapshot{} + } + if now.IsZero() { + now = time.Now().UTC() + } else { + now = now.UTC() + } + cutoff := now.Add(-30 * 24 * time.Hour) + snapshot := CalculateAlertQualitySnapshot( + m.GetAlertHistorySince(cutoff, 0), + m.GetActiveAlerts(), + cutoff, + now, + ) + snapshot.ManagerTenants = 1 + + config := m.GetConfig() + if config.Enabled && config.ActivationState == ActivationActive { + snapshot.DeliveryActiveTenants = 1 + } + if config.FlappingEnabled { + snapshot.FlappingEnabledTenants = 1 + } + intent := m.GetIntentPolicies() + if intent.Revision > 0 || intent.UpdatedAt != nil || len(intent.Defaults) > 0 || len(intent.ResourceTypes) > 0 || len(intent.Resources) > 0 { + snapshot.IntentPolicyTenants = 1 + } + if m.eventLogStore() != nil && m.eventHistoryAuthoritative.Load() { + snapshot.EventHistoryHealthyTenants = 1 + } + if m.eventLogStore() != nil && m.activeStateAuthoritative.Load() { + snapshot.ActiveStateHealthyTenants = 1 + } + if m.activeStateDegraded() { + snapshot.ActiveStateDegradedTenants = 1 + } + return snapshot +} + +type alertQualityOccurrence struct { + alert Alert + acknowledged bool + resolved bool +} + +// CalculateAlertQualitySnapshot folds local lifecycle rows into aggregate +// counts. The input may contain repeated snapshots for one occurrence. +func CalculateAlertQualitySnapshot(history, active []Alert, cutoff, now time.Time) AlertQualitySnapshot { + if now.IsZero() { + now = time.Now().UTC() + } else { + now = now.UTC() + } + if cutoff.IsZero() { + cutoff = now.Add(-30 * 24 * time.Hour) + } else { + cutoff = cutoff.UTC() + } + + var snapshot AlertQualitySnapshot + for _, alert := range active { + addActiveAlertQuality(&snapshot, alert, now) + } + + occurrences := make(map[string]alertQualityOccurrence, len(history)) + for index, alert := range history { + identity := localAlertIdentity(alert) + occurredAt := alert.StartTime + if occurredAt.IsZero() && alert.OperationalRecord != nil { + occurredAt = alert.OperationalRecord.FirstObservedAt + } + key := fmt.Sprintf("%s:%d", identity, occurredAt.UnixNano()) + if identity == "" { + key = fmt.Sprintf("anonymous:%d", index) + } + current := occurrences[key] + current.alert = alert + if alert.AckTime != nil && !alert.AckTime.Before(cutoff) { + current.acknowledged = true + } + if alert.OperationalRecord != nil && alert.OperationalRecord.State == operationaltrust.OperationalResolved { + current.resolved = true + } + occurrences[key] = current + } + + identities := make(map[string]int, len(occurrences)) + for _, occurrence := range occurrences { + alert := occurrence.alert + snapshot.Fired30d++ + addSeverityCount(alert.Level, &snapshot.FiredInfo30d, &snapshot.FiredWarning30d, &snapshot.FiredCritical30d) + if occurrence.acknowledged { + snapshot.Acknowledged30d++ + } + identity := localAlertIdentity(alert) + if identity != "" { + identities[identity]++ + } + if snoozed, resolvedWhileSnoozed := snoozeOutcome(alert); snoozed { + snapshot.SnoozedOccurrences30d++ + if occurrence.resolved && resolvedWhileSnoozed { + snapshot.ResolvedWhileSnoozed30d++ + } + } + if !occurrence.resolved { + continue + } + snapshot.Resolved30d++ + addSeverityCount(alert.Level, &snapshot.ResolvedInfo30d, &snapshot.ResolvedWarning30d, &snapshot.ResolvedCritical30d) + addResolutionDuration(&snapshot, alert) + } + for _, count := range identities { + if count > 1 { + snapshot.RepeatOccurrences30d += count - 1 + } + } + return snapshot +} + +func localAlertIdentity(alert Alert) string { + if identity := strings.TrimSpace(alert.CanonicalState); identity != "" { + return identity + } + return strings.TrimSpace(alert.ID) +} + +func addSeverityCount(level AlertLevel, info, warning, critical *int) { + switch NormalizeAlertLevel(level) { + case AlertLevelInfo: + *info++ + case AlertLevelCritical: + *critical++ + default: + *warning++ + } +} + +func addActiveAlertQuality(snapshot *AlertQualitySnapshot, alert Alert, now time.Time) { + addSeverityCount(alert.Level, &snapshot.ActiveInfo, &snapshot.ActiveWarning, &snapshot.ActiveCritical) + startedAt := alert.StartTime + if startedAt.IsZero() && alert.OperationalRecord != nil { + startedAt = alert.OperationalRecord.FirstObservedAt + } + age := now.Sub(startedAt) + if startedAt.IsZero() || age < 0 { + age = 0 + } + switch { + case age < time.Hour: + snapshot.ActiveAgeUnder1h++ + case age < 24*time.Hour: + snapshot.ActiveAge1h24h++ + case age < 7*24*time.Hour: + snapshot.ActiveAge1d7d++ + default: + snapshot.ActiveAge7dPlus++ + } +} + +func addResolutionDuration(snapshot *AlertQualitySnapshot, alert Alert) { + if alert.OperationalRecord == nil || alert.OperationalRecord.ResolvedAt == nil { + return + } + startedAt := alert.StartTime + if startedAt.IsZero() { + startedAt = alert.OperationalRecord.FirstObservedAt + } + duration := alert.OperationalRecord.ResolvedAt.Sub(startedAt) + if startedAt.IsZero() || duration < 0 { + duration = 0 + } + switch { + case duration < 15*time.Minute: + snapshot.ResolutionUnder15m30d++ + case duration < time.Hour: + snapshot.Resolution15m1h30d++ + case duration < 24*time.Hour: + snapshot.Resolution1h24h30d++ + case duration < 7*24*time.Hour: + snapshot.Resolution1d7d30d++ + default: + snapshot.Resolution7dPlus30d++ + } +} + +func snoozeOutcome(alert Alert) (snoozed, resolvedWhileSnoozed bool) { + transitions := append([]operationaltrust.LifecycleTransition(nil), alert.Transitions...) + sort.SliceStable(transitions, func(i, j int) bool { return transitions[i].At.Before(transitions[j].At) }) + active := false + for _, transition := range transitions { + switch transition.Cause { + case operationaltrust.TransitionSuppression: + snoozed = true + active = true + case operationaltrust.TransitionSuppressionExpired: + active = false + case operationaltrust.TransitionRecoveryEvidence: + resolvedWhileSnoozed = active + } + } + return snoozed, resolvedWhileSnoozed +} diff --git a/internal/alerts/telemetry_quality_test.go b/internal/alerts/telemetry_quality_test.go new file mode 100644 index 000000000..e82d90e08 --- /dev/null +++ b/internal/alerts/telemetry_quality_test.go @@ -0,0 +1,194 @@ +package alerts + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust" +) + +func TestCalculateAlertQualitySnapshotBucketsAndOutcomes(t *testing.T) { + now := time.Date(2026, 8, 29, 12, 0, 0, 0, time.UTC) + resolvedAt := now.Add(-time.Hour) + resolved := Alert{ + ID: "private-alert-id", + CanonicalState: "private-resource|cpu", + Level: AlertLevelCritical, + StartTime: resolvedAt.Add(-24 * time.Hour), + AckTime: alertQualityTimePtr(resolvedAt.Add(-time.Hour)), + OperationalRecord: &operationaltrust.OperationalRecord{ + State: operationaltrust.OperationalResolved, + FirstObservedAt: resolvedAt.Add(-24 * time.Hour), + ResolvedAt: &resolvedAt, + }, + Transitions: []operationaltrust.LifecycleTransition{ + {At: resolvedAt.Add(-2 * time.Hour), Cause: operationaltrust.TransitionSuppression}, + {At: resolvedAt, Cause: operationaltrust.TransitionRecoveryEvidence}, + }, + } + repeat := resolved + repeat.StartTime = resolved.StartTime.Add(-48 * time.Hour) + repeatRecord := resolved.OperationalRecord.Clone() + repeat.OperationalRecord = &repeatRecord + repeat.OperationalRecord.FirstObservedAt = repeat.StartTime + repeatResolvedAt := repeat.StartTime.Add(10 * time.Minute) + repeat.OperationalRecord.ResolvedAt = &repeatResolvedAt + repeat.Transitions = nil + + snapshot := CalculateAlertQualitySnapshot( + []Alert{resolved, resolved, repeat}, + []Alert{ + {Level: AlertLevelInfo, StartTime: now.Add(-time.Hour + time.Nanosecond)}, + {Level: AlertLevelWarning, StartTime: now.Add(-time.Hour)}, + {Level: AlertLevelCritical, StartTime: now.Add(-24 * time.Hour)}, + {Level: AlertLevelCritical, StartTime: now.Add(-7 * 24 * time.Hour)}, + }, + now.Add(-30*24*time.Hour), + now, + ) + + if snapshot.Fired30d != 2 || snapshot.FiredCritical30d != 2 || snapshot.Resolved30d != 2 || snapshot.ResolvedCritical30d != 2 { + t.Fatalf("lifecycle counts = %+v", snapshot) + } + if snapshot.Acknowledged30d != 2 || snapshot.RepeatOccurrences30d != 1 { + t.Fatalf("ack/repeat counts = %+v", snapshot) + } + if snapshot.SnoozedOccurrences30d != 1 || snapshot.ResolvedWhileSnoozed30d != 1 { + t.Fatalf("snooze outcomes = %+v", snapshot) + } + if snapshot.ResolutionUnder15m30d != 1 || snapshot.Resolution1d7d30d != 1 { + t.Fatalf("resolution buckets = %+v", snapshot) + } + if snapshot.ActiveAgeUnder1h != 1 || snapshot.ActiveAge1h24h != 1 || snapshot.ActiveAge1d7d != 1 || snapshot.ActiveAge7dPlus != 1 { + t.Fatalf("active age boundaries = %+v", snapshot) + } +} + +func TestCalculateAlertQualitySnapshotUnsnoozePreventsEffectiveResolution(t *testing.T) { + now := time.Date(2026, 8, 29, 12, 0, 0, 0, time.UTC) + resolvedAt := now.Add(-time.Hour) + alert := Alert{ + ID: "alert-a", + Level: AlertLevelWarning, + StartTime: now.Add(-2 * time.Hour), + OperationalRecord: &operationaltrust.OperationalRecord{ + State: operationaltrust.OperationalResolved, + FirstObservedAt: now.Add(-2 * time.Hour), + ResolvedAt: &resolvedAt, + }, + Transitions: []operationaltrust.LifecycleTransition{ + {At: now.Add(-110 * time.Minute), Cause: operationaltrust.TransitionSuppression}, + {At: now.Add(-100 * time.Minute), Cause: operationaltrust.TransitionSuppressionExpired}, + {At: resolvedAt, Cause: operationaltrust.TransitionRecoveryEvidence}, + }, + } + snapshot := CalculateAlertQualitySnapshot([]Alert{alert}, nil, now.Add(-30*24*time.Hour), now) + if snapshot.SnoozedOccurrences30d != 1 || snapshot.ResolvedWhileSnoozed30d != 0 { + t.Fatalf("snooze outcome = %+v", snapshot) + } +} + +func TestAlertQualityAggregationDoesNotCreateCrossTenantRepeats(t *testing.T) { + now := time.Date(2026, 8, 29, 12, 0, 0, 0, time.UTC) + privateLocalIdentity := "resource-42|cpu" + tenantOccurrence := func(startedAt time.Time) Alert { + return Alert{ + CanonicalState: privateLocalIdentity, + Level: AlertLevelWarning, + StartTime: startedAt, + } + } + one := CalculateAlertQualitySnapshot( + []Alert{tenantOccurrence(now.Add(-time.Hour))}, nil, now.Add(-30*24*time.Hour), now, + ) + two := CalculateAlertQualitySnapshot( + []Alert{tenantOccurrence(now.Add(-2 * time.Hour))}, nil, now.Add(-30*24*time.Hour), now, + ) + one.Add(two) + if one.Fired30d != 2 || one.RepeatOccurrences30d != 0 { + t.Fatalf("cross-tenant aggregate = %+v", one) + } +} + +func TestAlertQualityTelemetrySnapshotReportsAdoptionAndPersistenceHealth(t *testing.T) { + dataDir := t.TempDir() + manager := NewManagerWithDataDir(dataDir, WithDurableAlertStore()) + defer manager.Stop() + config := manager.GetConfig() + config.Enabled = true + config.ActivationState = ActivationActive + config.FlappingEnabled = true + manager.UpdateConfig(config) + intent := manager.GetIntentPolicies() + grace := 30 + intent.Defaults[string(AlertIntentSignalDefault)] = AlertIntentRule{GraceSeconds: &grace} + if _, err := manager.UpdateIntentPolicies(intent); err != nil { + t.Fatalf("update intent policies: %v", err) + } + marker := filepath.Join(dataDir, "alerts", activeStateDegradedMarker) + if err := os.WriteFile(marker, []byte("degraded"), 0o600); err != nil { + t.Fatalf("write degraded marker: %v", err) + } + + snapshot := manager.AlertQualityTelemetrySnapshot(time.Now().UTC()) + if snapshot.ManagerTenants != 1 || snapshot.DeliveryActiveTenants != 1 || + snapshot.FlappingEnabledTenants != 1 || snapshot.IntentPolicyTenants != 1 || + snapshot.EventHistoryHealthyTenants != 1 || snapshot.ActiveStateHealthyTenants != 1 || + snapshot.ActiveStateDegradedTenants != 1 { + t.Fatalf("adoption/persistence snapshot = %+v", snapshot) + } +} + +func TestAlertQualityTelemetrySnapshotRespectsHistoryReset(t *testing.T) { + manager := NewManagerWithDataDir(t.TempDir()) + defer manager.Stop() + manager.historyManager.AddAlert(Alert{ + ID: "private-id", + Level: AlertLevelWarning, + StartTime: time.Now().Add(-time.Hour), + }) + if got := manager.AlertQualityTelemetrySnapshot(time.Now().UTC()).Fired30d; got != 1 { + t.Fatalf("fired before reset = %d, want 1", got) + } + if err := manager.ClearAlertHistory(); err != nil { + t.Fatalf("clear alert history: %v", err) + } + if got := manager.AlertQualityTelemetrySnapshot(time.Now().UTC()).Fired30d; got != 0 { + t.Fatalf("fired after reset = %d, want 0", got) + } +} + +func TestAlertQualityResolutionDurationBoundaries(t *testing.T) { + now := time.Date(2026, 8, 29, 12, 0, 0, 0, time.UTC) + durations := []time.Duration{ + 15*time.Minute - time.Nanosecond, + 15 * time.Minute, + time.Hour, + 24 * time.Hour, + 7 * 24 * time.Hour, + } + history := make([]Alert, 0, len(durations)) + for index, duration := range durations { + resolvedAt := now.Add(-time.Duration(index) * time.Minute) + history = append(history, Alert{ + ID: string(rune('a' + index)), + Level: AlertLevelWarning, + StartTime: resolvedAt.Add(-duration), + OperationalRecord: &operationaltrust.OperationalRecord{ + State: operationaltrust.OperationalResolved, + FirstObservedAt: resolvedAt.Add(-duration), + ResolvedAt: &resolvedAt, + }, + }) + } + snapshot := CalculateAlertQualitySnapshot(history, nil, now.Add(-30*24*time.Hour), now) + if snapshot.ResolutionUnder15m30d != 1 || snapshot.Resolution15m1h30d != 1 || + snapshot.Resolution1h24h30d != 1 || snapshot.Resolution1d7d30d != 1 || + snapshot.Resolution7dPlus30d != 1 { + t.Fatalf("resolution boundary buckets = %+v", snapshot) + } +} + +func alertQualityTimePtr(value time.Time) *time.Time { return &value } diff --git a/internal/monitoring/reload.go b/internal/monitoring/reload.go index 250c2602b..7d6c9120a 100644 --- a/internal/monitoring/reload.go +++ b/internal/monitoring/reload.go @@ -9,7 +9,6 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/alerts" "github.com/rcourtman/pulse-go-rewrite/internal/config" - "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" "github.com/rcourtman/pulse-go-rewrite/internal/websocket" "github.com/rs/zerolog/log" @@ -55,6 +54,7 @@ type InstallSnapshotCounts struct { NotificationFailuresConfiguration7d int NotificationFailuresRejected7d int NotificationFailuresUnknown7d int + AlertQuality alerts.AlertQualitySnapshot } // ReloadableMonitor wraps a Monitor with reload capability @@ -283,9 +283,12 @@ func accumulateInstallOutcomeCounts(counts *InstallSnapshotCounts, monitor *Moni if now.IsZero() { now = time.Now().UTC() } - alertCutoff := now.Add(-30 * 24 * time.Hour) if alertManager := monitor.GetAlertManager(); alertManager != nil { - accumulateAlertOutcomeCounts(counts, alertManager.GetAlertHistorySince(alertCutoff, 0), alertCutoff) + quality := alertManager.AlertQualityTelemetrySnapshot(now) + counts.AlertQuality.Add(quality) + counts.AlertsFired30d += quality.Fired30d + counts.AlertsAcknowledged30d += quality.Acknowledged30d + counts.AlertsResolved30d += quality.Resolved30d } if notificationManager := monitor.GetNotificationManager(); notificationManager != nil { stats, err := notificationManager.GetTelemetryStats(now.Add(-7 * 24 * time.Hour)) @@ -306,46 +309,18 @@ func accumulateInstallOutcomeCounts(counts *InstallSnapshotCounts, monitor *Moni } } +// accumulateAlertOutcomeCounts remains the focused history-folding seam used +// by monitoring tests. Production collection uses the manager snapshot so the +// same calculation also includes current age, adoption, and persistence health. func accumulateAlertOutcomeCounts(counts *InstallSnapshotCounts, history []alerts.Alert, cutoff time.Time) { if counts == nil { return } - type outcome struct { - acknowledged bool - resolved bool - } - outcomes := make(map[string]outcome, len(history)) - for index, alert := range history { - identity := strings.TrimSpace(alert.CanonicalState) - if identity == "" { - identity = strings.TrimSpace(alert.ID) - } - occurrence := alert.StartTime - if occurrence.IsZero() && alert.OperationalRecord != nil { - occurrence = alert.OperationalRecord.FirstObservedAt - } - key := fmt.Sprintf("%s:%d", identity, occurrence.UnixNano()) - if identity == "" { - key = fmt.Sprintf("anonymous:%d", index) - } - current := outcomes[key] - if alert.AckTime != nil && !alert.AckTime.Before(cutoff) { - current.acknowledged = true - } - if alert.OperationalRecord != nil && alert.OperationalRecord.State == operationaltrust.OperationalResolved { - current.resolved = true - } - outcomes[key] = current - } - for _, current := range outcomes { - counts.AlertsFired30d++ - if current.acknowledged { - counts.AlertsAcknowledged30d++ - } - if current.resolved { - counts.AlertsResolved30d++ - } - } + quality := alerts.CalculateAlertQualitySnapshot(history, nil, cutoff, cutoff.Add(30*24*time.Hour)) + counts.AlertQuality.Add(quality) + counts.AlertsFired30d += quality.Fired30d + counts.AlertsAcknowledged30d += quality.Acknowledged30d + counts.AlertsResolved30d += quality.Resolved30d } func accumulateInstallSnapshotCounts(counts *InstallSnapshotCounts, monitor *Monitor) { diff --git a/internal/monitoring/reload_test.go b/internal/monitoring/reload_test.go index fafc86c26..6219bb6ca 100644 --- a/internal/monitoring/reload_test.go +++ b/internal/monitoring/reload_test.go @@ -175,17 +175,21 @@ func TestAccumulateAlertOutcomeCountsUsesOnlyContentFreeLifecycleTotals(t *testi counts := InstallSnapshotCounts{} accumulateAlertOutcomeCounts(&counts, []alerts.Alert{ { + Level: alerts.AlertLevelCritical, AckTime: &recentAck, OperationalRecord: &operationaltrust.OperationalRecord{ State: operationaltrust.OperationalResolved, }, }, - {AckTime: &oldAck}, + {Level: alerts.AlertLevelWarning, AckTime: &oldAck}, }, now.Add(-30*24*time.Hour)) assert.Equal(t, 2, counts.AlertsFired30d) assert.Equal(t, 1, counts.AlertsAcknowledged30d) assert.Equal(t, 1, counts.AlertsResolved30d) + assert.Equal(t, 1, counts.AlertQuality.FiredCritical30d) + assert.Equal(t, 1, counts.AlertQuality.FiredWarning30d) + assert.Equal(t, 1, counts.AlertQuality.ResolvedCritical30d) } func TestAccumulateAlertOutcomeCountsDeduplicatesOccurrenceSnapshots(t *testing.T) { diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go index 0fee1aa39..16c5b9b90 100644 --- a/internal/telemetry/telemetry.go +++ b/internal/telemetry/telemetry.go @@ -39,6 +39,10 @@ // - Whether a paid license is active // - Whether any API tokens are configured // - Aggregate alert fired/acknowledged/resolved counts over 30 days +// - Aggregate alert lifecycle counts by severity, active-alert age and +// resolution-duration buckets, repeat and snooze outcomes over 30 days +// - Tenant counts for alert delivery, flapping and intent-policy adoption, +// plus event-history and active-state persistence health // - Aggregate notification attempt/delivery/failure counts over seven days // - Coarse update funnel counters and last failure category over the current install-ID rotation window // - Patrol, Assistant, and external-agent usage counters over the current install-ID rotation window: @@ -180,7 +184,10 @@ const ( // previous release observation. This separates a process that can emit // telemetry from one that is actually serving its API and frontend assets, // while retaining only fixed categories and release identity. - TelemetrySchemaVersion = 13 + // Schema v14 adds identity-free alert quality outcomes and their tenant + // denominators. It uses closed severity, age, and resolution-time buckets, + // and reports only aggregate lifecycle, adoption, and persistence health. + TelemetrySchemaVersion = 14 ) type installIDRecord struct { @@ -331,11 +338,42 @@ type Ping struct { // Core product outcomes. Alert history is retained locally for 30 days; // notification delivery rows are locally retention-bounded to seven days. - AlertsFired30d int `json:"alerts_fired_30d"` - AlertsAcknowledged30d int `json:"alerts_acknowledged_30d"` - AlertsResolved30d int `json:"alerts_resolved_30d"` - NotificationAttempts7d int `json:"notification_attempts_7d"` - NotificationDeliveries7d int `json:"notification_deliveries_7d"` + AlertsFired30d int `json:"alerts_fired_30d"` + AlertsAcknowledged30d int `json:"alerts_acknowledged_30d"` + AlertsResolved30d int `json:"alerts_resolved_30d"` + // Alert quality aggregates. Severity totals reconcile to the existing + // lifecycle totals, while age and resolution-duration buckets have fixed + // inclusive lower and exclusive upper boundaries. + ActiveAlertsInfo int `json:"active_alerts_info"` + ActiveAlertsWarning int `json:"active_alerts_warning"` + ActiveAlertsCritical int `json:"active_alerts_critical"` + ActiveAlertsAgeUnder1h int `json:"active_alerts_age_under_1h"` + ActiveAlertsAge1h24h int `json:"active_alerts_age_1h_24h"` + ActiveAlertsAge1d7d int `json:"active_alerts_age_1d_7d"` + ActiveAlertsAge7dPlus int `json:"active_alerts_age_7d_plus"` + AlertsFiredInfo30d int `json:"alerts_fired_info_30d"` + AlertsFiredWarning30d int `json:"alerts_fired_warning_30d"` + AlertsFiredCritical30d int `json:"alerts_fired_critical_30d"` + AlertsResolvedInfo30d int `json:"alerts_resolved_info_30d"` + AlertsResolvedWarning30d int `json:"alerts_resolved_warning_30d"` + AlertsResolvedCritical30d int `json:"alerts_resolved_critical_30d"` + AlertsResolutionUnder15m30d int `json:"alerts_resolution_under_15m_30d"` + AlertsResolution15m1h30d int `json:"alerts_resolution_15m_1h_30d"` + AlertsResolution1h24h30d int `json:"alerts_resolution_1h_24h_30d"` + AlertsResolution1d7d30d int `json:"alerts_resolution_1d_7d_30d"` + AlertsResolution7dPlus30d int `json:"alerts_resolution_7d_plus_30d"` + AlertsRepeatOccurrences30d int `json:"alerts_repeat_occurrences_30d"` + AlertsSnoozedOccurrences30d int `json:"alerts_snoozed_occurrences_30d"` + AlertsResolvedWhileSnoozed30d int `json:"alerts_resolved_while_snoozed_30d"` + AlertManagerTenants int `json:"alert_manager_tenants"` + AlertDeliveryActiveTenants int `json:"alert_delivery_active_tenants"` + AlertFlappingEnabledTenants int `json:"alert_flapping_enabled_tenants"` + AlertIntentPolicyConfiguredTenants int `json:"alert_intent_policy_configured_tenants"` + AlertEventHistoryAuthoritativeTenants int `json:"alert_event_history_authoritative_tenants"` + AlertActiveStateAuthoritativeTenants int `json:"alert_active_state_authoritative_tenants"` + AlertActiveStatePersistenceDegradedTenants int `json:"alert_active_state_persistence_degraded_tenants"` + NotificationAttempts7d int `json:"notification_attempts_7d"` + NotificationDeliveries7d int `json:"notification_deliveries_7d"` // NotificationFailures7d counts only terminal failed/dead-letter outcomes. // Retry-attempt failures remain represented in NotificationAttempts7d. NotificationFailures7d int `json:"notification_failures_7d"` @@ -491,6 +529,34 @@ type Snapshot struct { AlertsFired30d int AlertsAcknowledged30d int AlertsResolved30d int + ActiveAlertsInfo int + ActiveAlertsWarning int + ActiveAlertsCritical int + ActiveAlertsAgeUnder1h int + ActiveAlertsAge1h24h int + ActiveAlertsAge1d7d int + ActiveAlertsAge7dPlus int + AlertsFiredInfo30d int + AlertsFiredWarning30d int + AlertsFiredCritical30d int + AlertsResolvedInfo30d int + AlertsResolvedWarning30d int + AlertsResolvedCritical30d int + AlertsResolutionUnder15m30d int + AlertsResolution15m1h30d int + AlertsResolution1h24h30d int + AlertsResolution1d7d30d int + AlertsResolution7dPlus30d int + AlertsRepeatOccurrences30d int + AlertsSnoozedOccurrences30d int + AlertsResolvedWhileSnoozed30d int + AlertManagerTenants int + AlertDeliveryActiveTenants int + AlertFlappingEnabledTenants int + AlertIntentPolicyConfiguredTenants int + AlertEventHistoryAuthoritativeTenants int + AlertActiveStateAuthoritativeTenants int + AlertActiveStatePersistenceDegradedTenants int NotificationAttempts7d int NotificationDeliveries7d int NotificationFailures7d int @@ -1129,6 +1195,34 @@ func applySnapshot(base Ping, fn SnapshotFunc) Ping { ping.AlertsFired30d = s.AlertsFired30d ping.AlertsAcknowledged30d = s.AlertsAcknowledged30d ping.AlertsResolved30d = s.AlertsResolved30d + ping.ActiveAlertsInfo = s.ActiveAlertsInfo + ping.ActiveAlertsWarning = s.ActiveAlertsWarning + ping.ActiveAlertsCritical = s.ActiveAlertsCritical + ping.ActiveAlertsAgeUnder1h = s.ActiveAlertsAgeUnder1h + ping.ActiveAlertsAge1h24h = s.ActiveAlertsAge1h24h + ping.ActiveAlertsAge1d7d = s.ActiveAlertsAge1d7d + ping.ActiveAlertsAge7dPlus = s.ActiveAlertsAge7dPlus + ping.AlertsFiredInfo30d = s.AlertsFiredInfo30d + ping.AlertsFiredWarning30d = s.AlertsFiredWarning30d + ping.AlertsFiredCritical30d = s.AlertsFiredCritical30d + ping.AlertsResolvedInfo30d = s.AlertsResolvedInfo30d + ping.AlertsResolvedWarning30d = s.AlertsResolvedWarning30d + ping.AlertsResolvedCritical30d = s.AlertsResolvedCritical30d + ping.AlertsResolutionUnder15m30d = s.AlertsResolutionUnder15m30d + ping.AlertsResolution15m1h30d = s.AlertsResolution15m1h30d + ping.AlertsResolution1h24h30d = s.AlertsResolution1h24h30d + ping.AlertsResolution1d7d30d = s.AlertsResolution1d7d30d + ping.AlertsResolution7dPlus30d = s.AlertsResolution7dPlus30d + ping.AlertsRepeatOccurrences30d = s.AlertsRepeatOccurrences30d + ping.AlertsSnoozedOccurrences30d = s.AlertsSnoozedOccurrences30d + ping.AlertsResolvedWhileSnoozed30d = s.AlertsResolvedWhileSnoozed30d + ping.AlertManagerTenants = s.AlertManagerTenants + ping.AlertDeliveryActiveTenants = s.AlertDeliveryActiveTenants + ping.AlertFlappingEnabledTenants = s.AlertFlappingEnabledTenants + ping.AlertIntentPolicyConfiguredTenants = s.AlertIntentPolicyConfiguredTenants + ping.AlertEventHistoryAuthoritativeTenants = s.AlertEventHistoryAuthoritativeTenants + ping.AlertActiveStateAuthoritativeTenants = s.AlertActiveStateAuthoritativeTenants + ping.AlertActiveStatePersistenceDegradedTenants = s.AlertActiveStatePersistenceDegradedTenants ping.NotificationAttempts7d = s.NotificationAttempts7d ping.NotificationDeliveries7d = s.NotificationDeliveries7d // gitleaks:allow -- schema field name, not a credential ping.NotificationFailures7d = s.NotificationFailures7d diff --git a/internal/telemetry/telemetry_test.go b/internal/telemetry/telemetry_test.go index 2fd66c6b3..caa2acc52 100644 --- a/internal/telemetry/telemetry_test.go +++ b/internal/telemetry/telemetry_test.go @@ -335,40 +335,54 @@ func TestApplySnapshot(t *testing.T) { snap := func() Snapshot { return Snapshot{ - PVENodes: 3, - VMs: 10, - Containers: 5, - AgentHosts: 2, - DockerContainers: 12, - KubernetesPods: 18, - StoragePools: 4, - PhysicalDisks: 9, - TrueNASSystems: 1, - TrueNASApps: 3, - VMwareHosts: 2, - AvailabilityTargets: 6, - AIEnabled: true, - PatrolEnabled: true, - DiscoveryEnabled: true, - NotificationsEnabled: true, - AIActionsEnabled: true, - AlertAIEnabled: true, - ActiveAlerts: 2, - PaidLicense: true, - HasAPITokens: true, - RBACCustomRoles: 3, - RBACUserAssignments: 7, - AuditReads30d: 41, - ReportSchedules: 5, - ReportSchedulesEnabled: 4, - ReportSchedulesRun30d: 2, - AgentProfiles: 9, - UpdateAttempts30d: 4, - UpdateSuccesses30d: 2, - UpdateFailures30d: 1, - UpdateLastFailureCategory: "download", - PulseIntelligenceLoopConfigured: true, - PulseIntelligenceLoopActive30d: true, + PVENodes: 3, + VMs: 10, + Containers: 5, + AgentHosts: 2, + DockerContainers: 12, + KubernetesPods: 18, + StoragePools: 4, + PhysicalDisks: 9, + TrueNASSystems: 1, + TrueNASApps: 3, + VMwareHosts: 2, + AvailabilityTargets: 6, + AIEnabled: true, + PatrolEnabled: true, + DiscoveryEnabled: true, + NotificationsEnabled: true, + AIActionsEnabled: true, + AlertAIEnabled: true, + ActiveAlerts: 2, + PaidLicense: true, + HasAPITokens: true, + RBACCustomRoles: 3, + RBACUserAssignments: 7, + AuditReads30d: 41, + ReportSchedules: 5, + ReportSchedulesEnabled: 4, + ReportSchedulesRun30d: 2, + AgentProfiles: 9, + UpdateAttempts30d: 4, + UpdateSuccesses30d: 2, + UpdateFailures30d: 1, + UpdateLastFailureCategory: "download", + ActiveAlertsCritical: 2, + ActiveAlertsAge1h24h: 2, + AlertsFiredCritical30d: 5, + AlertsResolvedCritical30d: 3, + AlertsResolution1h24h30d: 3, + AlertsRepeatOccurrences30d: 2, + AlertsSnoozedOccurrences30d: 2, + AlertsResolvedWhileSnoozed30d: 1, + AlertManagerTenants: 2, + AlertDeliveryActiveTenants: 1, + AlertFlappingEnabledTenants: 2, + AlertIntentPolicyConfiguredTenants: 1, + AlertEventHistoryAuthoritativeTenants: 2, + AlertActiveStateAuthoritativeTenants: 2, + PulseIntelligenceLoopConfigured: true, + PulseIntelligenceLoopActive30d: true, PulseIntelligenceCompleteOperationsLoop30d: true, PulseIntelligenceApprovedExecutionLoop30d: true, PulseIntelligenceResolvedOperationsLoop30d: true, @@ -487,6 +501,15 @@ func TestApplySnapshot(t *testing.T) { ping.UpdateLastFailureCategory != "download" { t.Fatalf("update telemetry counters not applied: %#v", ping) } + if ping.ActiveAlertsCritical != 2 || ping.ActiveAlertsAge1h24h != 2 || + ping.AlertsFiredCritical30d != 5 || ping.AlertsResolvedCritical30d != 3 || + ping.AlertsResolution1h24h30d != 3 || ping.AlertsRepeatOccurrences30d != 2 || + ping.AlertsSnoozedOccurrences30d != 2 || ping.AlertsResolvedWhileSnoozed30d != 1 || + ping.AlertManagerTenants != 2 || ping.AlertDeliveryActiveTenants != 1 || + ping.AlertFlappingEnabledTenants != 2 || ping.AlertIntentPolicyConfiguredTenants != 1 || + ping.AlertEventHistoryAuthoritativeTenants != 2 || ping.AlertActiveStateAuthoritativeTenants != 2 { + t.Fatalf("alert quality snapshot not applied: %#v", ping) + } if !ping.PulseIntelligenceLoopConfigured || !ping.PulseIntelligenceLoopActive30d || !ping.PulseIntelligenceCompleteOperationsLoop30d || !ping.PulseIntelligenceApprovedExecutionLoop30d || @@ -868,6 +891,8 @@ func TestBuildPreview_UsesCurrentHeartbeatPayload(t *testing.T) { NotificationFailures7d: 3, NotificationFailuresAuthentication7d: 2, NotificationFailuresConnectivity7d: 1, + AlertsRepeatOccurrences30d: 4, + AlertManagerTenants: 2, } }, }) @@ -907,6 +932,9 @@ func TestBuildPreview_UsesCurrentHeartbeatPayload(t *testing.T) { preview.NotificationFailuresConnectivity7d != 1 { t.Fatalf("preview notification failure classes = %#v", preview) } + if preview.AlertsRepeatOccurrences30d != 4 || preview.AlertManagerTenants != 2 { + t.Fatalf("preview alert-quality fields = %#v", preview) + } if preview.InstallID == "" { t.Fatal("expected preview install ID") } @@ -932,6 +960,27 @@ func TestBuildPreview_UsesCurrentHeartbeatPayload(t *testing.T) { } } +func TestAlertQualityPayloadContainsOnlyAggregateFields(t *testing.T) { + payload, err := json.Marshal(Ping{ + AlertsRepeatOccurrences30d: 7, + AlertsResolvedWhileSnoozed30d: 5, + AlertIntentPolicyConfiguredTenants: 3, + AlertActiveStatePersistenceDegradedTenants: 1, + }) + if err != nil { + t.Fatalf("marshal ping: %v", err) + } + text := strings.ToLower(string(payload)) + for _, forbidden := range []string{ + "resource_id", "hostname", "alert_text", "alert_message", "rule_content", + "destination", "resolved_at", "start_time", "error_text", "click", + } { + if strings.Contains(text, forbidden) { + t.Fatalf("alert-quality payload contains forbidden key fragment %q: %s", forbidden, text) + } + } +} + func TestBuildPingAt_RotatesIdentifierDuringLongRunningSession(t *testing.T) { dir := t.TempDir() start := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) diff --git a/pkg/server/server.go b/pkg/server/server.go index 764f1208d..aec015c57 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -576,6 +576,34 @@ func Run(ctx context.Context, version string) (runErr error) { snap.AlertsFired30d = counts.AlertsFired30d snap.AlertsAcknowledged30d = counts.AlertsAcknowledged30d snap.AlertsResolved30d = counts.AlertsResolved30d + snap.ActiveAlertsInfo = counts.AlertQuality.ActiveInfo + snap.ActiveAlertsWarning = counts.AlertQuality.ActiveWarning + snap.ActiveAlertsCritical = counts.AlertQuality.ActiveCritical + snap.ActiveAlertsAgeUnder1h = counts.AlertQuality.ActiveAgeUnder1h + snap.ActiveAlertsAge1h24h = counts.AlertQuality.ActiveAge1h24h + snap.ActiveAlertsAge1d7d = counts.AlertQuality.ActiveAge1d7d + snap.ActiveAlertsAge7dPlus = counts.AlertQuality.ActiveAge7dPlus + snap.AlertsFiredInfo30d = counts.AlertQuality.FiredInfo30d + snap.AlertsFiredWarning30d = counts.AlertQuality.FiredWarning30d + snap.AlertsFiredCritical30d = counts.AlertQuality.FiredCritical30d + snap.AlertsResolvedInfo30d = counts.AlertQuality.ResolvedInfo30d + snap.AlertsResolvedWarning30d = counts.AlertQuality.ResolvedWarning30d + snap.AlertsResolvedCritical30d = counts.AlertQuality.ResolvedCritical30d + snap.AlertsResolutionUnder15m30d = counts.AlertQuality.ResolutionUnder15m30d + snap.AlertsResolution15m1h30d = counts.AlertQuality.Resolution15m1h30d + snap.AlertsResolution1h24h30d = counts.AlertQuality.Resolution1h24h30d + snap.AlertsResolution1d7d30d = counts.AlertQuality.Resolution1d7d30d + snap.AlertsResolution7dPlus30d = counts.AlertQuality.Resolution7dPlus30d + snap.AlertsRepeatOccurrences30d = counts.AlertQuality.RepeatOccurrences30d + snap.AlertsSnoozedOccurrences30d = counts.AlertQuality.SnoozedOccurrences30d + snap.AlertsResolvedWhileSnoozed30d = counts.AlertQuality.ResolvedWhileSnoozed30d + snap.AlertManagerTenants = counts.AlertQuality.ManagerTenants + snap.AlertDeliveryActiveTenants = counts.AlertQuality.DeliveryActiveTenants + snap.AlertFlappingEnabledTenants = counts.AlertQuality.FlappingEnabledTenants + snap.AlertIntentPolicyConfiguredTenants = counts.AlertQuality.IntentPolicyTenants + snap.AlertEventHistoryAuthoritativeTenants = counts.AlertQuality.EventHistoryHealthyTenants + snap.AlertActiveStateAuthoritativeTenants = counts.AlertQuality.ActiveStateHealthyTenants + snap.AlertActiveStatePersistenceDegradedTenants = counts.AlertQuality.ActiveStateDegradedTenants snap.NotificationAttempts7d = counts.NotificationAttempts7d snap.NotificationDeliveries7d = counts.NotificationDeliveries7d snap.NotificationFailures7d = counts.NotificationFailures7d diff --git a/scripts/telemetry_adoption_report.py b/scripts/telemetry_adoption_report.py index 6033e35f0..90be7fe16 100644 --- a/scripts/telemetry_adoption_report.py +++ b/scripts/telemetry_adoption_report.py @@ -9,7 +9,7 @@ published-release reporting. from __future__ import annotations import argparse -from collections import Counter +from collections import Counter, defaultdict from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone import gzip @@ -1024,6 +1024,37 @@ DEEP_SIGNAL_FIELDS = ( for field, label in PULSE_INTELLIGENCE_COUNT_FIELDS ), ) +ALERT_QUALITY_SCHEMA_VERSION = 14 +ALERT_QUALITY_COUNT_FIELDS = ( + ("active_alerts_info", "Active alerts: info"), + ("active_alerts_warning", "Active alerts: warning"), + ("active_alerts_critical", "Active alerts: critical"), + ("active_alerts_age_under_1h", "Active alerts under 1h"), + ("active_alerts_age_1h_24h", "Active alerts 1h-24h"), + ("active_alerts_age_1d_7d", "Active alerts 1d-7d"), + ("active_alerts_age_7d_plus", "Active alerts 7d+"), + ("alerts_fired_info_30d", "Alerts fired: info (30d)"), + ("alerts_fired_warning_30d", "Alerts fired: warning (30d)"), + ("alerts_fired_critical_30d", "Alerts fired: critical (30d)"), + ("alerts_resolved_info_30d", "Alerts resolved: info (30d)"), + ("alerts_resolved_warning_30d", "Alerts resolved: warning (30d)"), + ("alerts_resolved_critical_30d", "Alerts resolved: critical (30d)"), + ("alerts_resolution_under_15m_30d", "Resolution under 15m (30d)"), + ("alerts_resolution_15m_1h_30d", "Resolution 15m-1h (30d)"), + ("alerts_resolution_1h_24h_30d", "Resolution 1h-24h (30d)"), + ("alerts_resolution_1d_7d_30d", "Resolution 1d-7d (30d)"), + ("alerts_resolution_7d_plus_30d", "Resolution 7d+ (30d)"), + ("alerts_repeat_occurrences_30d", "Repeat alert occurrences (30d)"), + ("alerts_snoozed_occurrences_30d", "Snoozed alert occurrences (30d)"), + ("alerts_resolved_while_snoozed_30d", "Alerts resolved while snoozed (30d)"), + ("alert_manager_tenants", "Alert manager tenants"), + ("alert_delivery_active_tenants", "Alert delivery active tenants"), + ("alert_flapping_enabled_tenants", "Flapping detection enabled tenants"), + ("alert_intent_policy_configured_tenants", "Alert intent policy tenants"), + ("alert_event_history_authoritative_tenants", "Authoritative event-history tenants"), + ("alert_active_state_authoritative_tenants", "Authoritative active-state tenants"), + ("alert_active_state_persistence_degraded_tenants", "Degraded active-state tenants"), +) REPORT_ROW_COLUMNS = tuple( dict.fromkeys( ( @@ -1038,6 +1069,7 @@ REPORT_ROW_COLUMNS = tuple( "version_is_published_release", "platform", "notification_failures_7d", + *(key for key, _ in ALERT_QUALITY_COUNT_FIELDS), *(key for key, _ in ADOPTION_COUNT_FIELDS), *(key for key, _ in FEATURE_BOOL_FIELDS), *(key for key, _ in USER_BASE_CATEGORY_FIELDS), @@ -1064,6 +1096,7 @@ TARGET_RELEASE_ACTIVITY_COUNT_FIELDS = tuple( "alerts_fired_30d", "alerts_acknowledged_30d", "alerts_resolved_30d", + *(field for field, _ in ALERT_QUALITY_COUNT_FIELDS), "notification_attempts_7d", "notification_deliveries_7d", "notification_failures_7d", @@ -1088,6 +1121,7 @@ TARGET_RELEASE_ACTIVITY_LABELS = { **{key: label for key, label in ADOPTION_COUNT_FIELDS}, **{key: label for key, label in PULSE_INTELLIGENCE_COUNT_FIELDS}, "notification_failures_7d": "Notification failures (7d; schema-dependent semantics)", + **{key: label for key, label in ALERT_QUALITY_COUNT_FIELDS}, } GIT_DESCRIBE_RE = re.compile( r"^(?P\d+\.\d+\.\d+(?:-[0-9A-Za-z\.-]+)?)-(?P\d+)-g(?P[0-9a-fA-F]+)(?P-dirty)?$" @@ -1129,6 +1163,8 @@ class TargetReleaseInstallAnalysis: first_counts: tuple[int, ...] increase_totals: tuple[int, ...] decrease_totals: tuple[int, ...] + latest_schema_version: int + latest_known_install_age_bucket: str @dataclass @@ -1147,6 +1183,8 @@ class _TargetReleaseInstallAccumulator: adjacent_received_at: datetime | None = None adjacent_is_target: bool = False adjacent_counts: tuple[int, ...] = () + latest_schema_version: int = 0 + latest_known_install_age_bucket: str = "unknown" def observe(self, row: dict[str, Any], received_at: datetime, is_target: bool) -> None: counts = tuple( @@ -1160,6 +1198,13 @@ class _TargetReleaseInstallAccumulator: self.first_counts = counts if self.latest_received_at is None or received_at > self.latest_received_at: self.latest_received_at = received_at + self.latest_schema_version = parse_optional_nonnegative_int( + row.get("schema_version") + ) + self.latest_known_install_age_bucket = ( + str(row.get("known_install_age_bucket") or "unknown").strip() + or "unknown" + ) if ( is_target @@ -1194,6 +1239,8 @@ class _TargetReleaseInstallAccumulator: first_counts=self.first_counts, increase_totals=tuple(self.increase_totals), decrease_totals=tuple(self.decrease_totals), + latest_schema_version=self.latest_schema_version, + latest_known_install_age_bucket=self.latest_known_install_age_bucket, ) @@ -1657,6 +1704,7 @@ def fetch_rows_remote( db_path: str, since_days: int, target_version: str | None = None, + baseline_version: str | None = None, ) -> dict[str, Any]: # Let SQLite aggregate the history into sufficient per-install evidence. # Only the latest row and compact per-install facts cross the network, not @@ -1673,6 +1721,8 @@ column_names = sys.argv[3].split(",") intelligence_columns = sys.argv[4].split(",") target_version = sys.argv[5] target_activity_columns = sys.argv[6].split(",") if sys.argv[6] else [] +baseline_version = sys.argv[7] if len(sys.argv) > 7 else "" +analysis_versions = [value for value in (target_version, baseline_version) if value] if not column_names or any( not name or not name[0].isalpha() or not name.replace("_", "").isalnum() for name in column_names @@ -1734,7 +1784,7 @@ analysis_sql = ( "GROUP BY install_id" ) target_analysis_sql = None -if target_version: +if analysis_versions: target_match = "(ordered_version = ? OR ordered_version = ?)" previous_target_match = "(previous_version = ? OR previous_version = ?)" pair_match = previous_target_match + " AND received_at <> previous_received_at" @@ -1744,6 +1794,8 @@ if target_version: "SUM(CASE WHEN " + pair_match + " THEN 1 ELSE 0 END)", "MIN(received_at)", "MAX(received_at)", + "MAX(CASE WHEN last_rank = 1 THEN COALESCE(schema_version, 0) END)", + "MAX(CASE WHEN last_rank = 1 THEN COALESCE(known_install_age_bucket, 'unknown') END)", ] for name in target_activity_columns: target_selects.append( @@ -1762,6 +1814,8 @@ if target_version: "received_at", "rowid AS source_rowid", "TRIM(version) AS ordered_version", + ("schema_version" if "schema_version" in available_columns else "0 AS schema_version"), + ("known_install_age_bucket" if "known_install_age_bucket" in available_columns else "'unknown' AS known_install_age_bucket"), *( name if name in available_columns else "0 AS " + name for name in target_activity_columns @@ -1779,6 +1833,8 @@ if target_version: "received_at", "source_rowid", "ordered_version", + "schema_version", + "known_install_age_bucket", *target_activity_columns, "previous_received_at", "previous_version", @@ -1797,6 +1853,9 @@ if target_version: "ROW_NUMBER() OVER (" "PARTITION BY install_id ORDER BY received_at ASC, source_rowid ASC" ") AS first_rank " + ", ROW_NUMBER() OVER (" + "PARTITION BY install_id ORDER BY received_at DESC, source_rowid DESC" + ") AS last_rank " "FROM ordered WHERE " + target_match + ") SELECT " + ", ".join(target_selects) + " " "FROM target_rows GROUP BY install_id" @@ -1832,31 +1891,32 @@ try: signal_fields, free_signal_fields, ]}) if target_analysis_sql is not None: - for row in conn.execute( - target_analysis_sql, - ( - cutoff, - target_version, - "v" + target_version, - *( - value - for _ in range(1 + (len(target_activity_columns) * 2)) - for value in (target_version, "v" + target_version) + for analysis_version in analysis_versions: + for row in conn.execute( + target_analysis_sql, + ( + cutoff, + analysis_version, + "v" + analysis_version, + *( + value + for _ in range(1 + (len(target_activity_columns) * 2)) + for value in (analysis_version, "v" + analysis_version) + ), ), - ), - ): - first_counts = [] - increase_totals = [] - decrease_totals = [] - for signal_index in range(len(target_activity_columns)): - value_index = 5 + (signal_index * 3) - first_counts.append(row[value_index]) - increase_totals.append(row[value_index + 1]) - decrease_totals.append(row[value_index + 2]) - emit({"t": [ - row[0], row[1], row[2], row[3], row[4], first_counts, - increase_totals, decrease_totals, - ]}) + ): + first_counts = [] + increase_totals = [] + decrease_totals = [] + for signal_index in range(len(target_activity_columns)): + value_index = 7 + (signal_index * 3) + first_counts.append(row[value_index]) + increase_totals.append(row[value_index + 1]) + decrease_totals.append(row[value_index + 2]) + emit({"t": [ + analysis_version, row[0], row[1], row[2], row[3], row[4], + row[5], row[6], first_counts, increase_totals, decrease_totals, + ]}) for row in conn.execute(rows_sql, (cutoff,)): emit({"r": list(row)}) finally: @@ -1875,6 +1935,7 @@ finally: ",".join(REPORT_HISTORY_SIGNAL_COLUMNS), normalize_release_tag(target_version or ""), ",".join(TARGET_RELEASE_ACTIVITY_COUNT_FIELDS), + normalize_release_tag(baseline_version or ""), ], input=remote_script.encode("utf-8"), capture_output=True, @@ -1918,18 +1979,27 @@ finally: ) elif "t" in record: values = record["t"] - if len(values) != 8: + if len(values) not in {8, 11}: raise RuntimeError(f"invalid target release analysis from remote telemetry fetch on {ssh_host}") + if len(values) == 8: + values = [ + normalize_release_tag(target_version or ""), + values[0], values[1], values[2], values[3], values[4], + 0, "unknown", values[5], values[6], values[7], + ] target_release_facts.append( { - "install_id": values[0], - "heartbeat_count": values[1], - "same_version_pair_count": values[2], - "first_received_at": values[3], - "latest_received_at": values[4], - "first_counts": values[5], - "increase_totals": values[6], - "decrease_totals": values[7], + "version": values[0], + "install_id": values[1], + "heartbeat_count": values[2], + "same_version_pair_count": values[3], + "first_received_at": values[4], + "latest_received_at": values[5], + "latest_schema_version": values[6], + "latest_known_install_age_bucket": values[7], + "first_counts": values[8], + "increase_totals": values[9], + "decrease_totals": values[10], } ) else: @@ -2740,10 +2810,15 @@ def analyze_target_release_rows( def analyze_target_release_facts( facts: Iterable[dict[str, Any]], + target_version: str | None = None, ) -> dict[str, TargetReleaseInstallAnalysis]: analyses: dict[str, TargetReleaseInstallAnalysis] = {} expected_count_length = len(TARGET_RELEASE_ACTIVITY_COUNT_FIELDS) + normalized_target = normalize_release_tag(target_version or "") for fact in facts: + fact_version = normalize_release_tag(str(fact.get("version") or "")) + if normalized_target and fact_version and fact_version != normalized_target: + continue first_counts = tuple( parse_optional_nonnegative_int(value) for value in fact.get("first_counts") or () @@ -2772,6 +2847,13 @@ def analyze_target_release_facts( first_counts=first_counts, increase_totals=increase_totals, decrease_totals=decrease_totals, + latest_schema_version=parse_optional_nonnegative_int( + fact.get("latest_schema_version") + ), + latest_known_install_age_bucket=( + str(fact.get("latest_known_install_age_bucket") or "unknown").strip() + or "unknown" + ), ) return analyses @@ -2789,6 +2871,176 @@ def summarize_target_release_followup( for install_id, analysis in analyses_by_install.items() if install_id in latest_by_install } + return _summarize_target_release_followup_analysis( + latest_by_install, + published_versions, + normalized_target, + target_identity, + analyses, + ) + + +def _heartbeat_exposure_bucket(heartbeat_count: int) -> str: + if heartbeat_count <= 1: + return "1" + if heartbeat_count == 2: + return "2" + if heartbeat_count <= 6: + return "3_6" + return "7_plus" + + +def _alert_quality_release_cohort( + version: str, + analyses: dict[str, TargetReleaseInstallAnalysis], +) -> dict[str, Any]: + age_buckets: Counter[str] = Counter() + heartbeat_buckets: Counter[str] = Counter() + schema_versions: Counter[str] = Counter() + schema_capable = 0 + comparable = 0 + for analysis in analyses.values(): + age_buckets[analysis.latest_known_install_age_bucket] += 1 + heartbeat_buckets[_heartbeat_exposure_bucket(analysis.heartbeat_count)] += 1 + schema_versions[str(analysis.latest_schema_version)] += 1 + if analysis.latest_schema_version >= ALERT_QUALITY_SCHEMA_VERSION: + schema_capable += 1 + if analysis.same_version_pair_count > 0: + comparable += 1 + return { + "version": normalize_release_tag(version), + "installs": len(analyses), + "heartbeat_rows": sum(item.heartbeat_count for item in analyses.values()), + "first_heartbeat_baseline_only_installs": sum( + 1 for item in analyses.values() if item.same_version_pair_count == 0 + ), + "schema_capable_installs": schema_capable, + "same_version_comparable_installs": comparable, + "known_age_buckets": counter_entries(age_buckets, "bucket"), + "heartbeat_count_buckets": counter_entries(heartbeat_buckets, "bucket"), + "schema_versions": counter_entries(schema_versions, "schema_version"), + } + + +def summarize_alert_quality_release_comparison( + target_version: str, + baseline_version: str, + target_analyses: dict[str, TargetReleaseInstallAnalysis], + baseline_analyses: dict[str, TargetReleaseInstallAnalysis], +) -> dict[str, Any]: + target_cohort = _alert_quality_release_cohort(target_version, target_analyses) + baseline_cohort = _alert_quality_release_cohort(baseline_version, baseline_analyses) + result: dict[str, Any] = { + "target": target_cohort, + "baseline": baseline_cohort, + "matched_exposure_strata": [], + "interpretation": { + "first_heartbeat": ( + "The first heartbeat for a version is baseline-only. Alert-quality changes require " + "a later consecutive heartbeat from the same version and install." + ), + "exposure": ( + "Comparisons are separated by the latest known-install-age bucket and version heartbeat-count " + "bucket. Unmatched strata are not pooled into a release outcome." + ), + "rolling_window": ( + "Positive and negative rolling-window changes remain separate. Ratios below use positive " + "same-version changes only and are descriptive, not causal release attribution." + ), + }, + } + if target_cohort["schema_capable_installs"] == 0 or baseline_cohort["schema_capable_installs"] == 0: + result["status"] = "schema_not_comparable" + result["reason"] = ( + f"Both releases need telemetry schema {ALERT_QUALITY_SCHEMA_VERSION} or newer. " + f"{normalize_release_tag(target_version)} and {normalize_release_tag(baseline_version)} " + "cannot be compared with alert-quality outcomes from legacy volume counters." + ) + return result + + quality_index = { + field: TARGET_RELEASE_ACTIVITY_COUNT_FIELDS.index(field) + for field in ( + "alerts_fired_30d", + "alerts_resolved_30d", + "alerts_repeat_occurrences_30d", + "alerts_snoozed_occurrences_30d", + "alerts_resolved_while_snoozed_30d", + "alerts_resolution_under_15m_30d", + "alerts_resolution_15m_1h_30d", + "alerts_resolution_1h_24h_30d", + "alerts_resolution_1d_7d_30d", + "alerts_resolution_7d_plus_30d", + ) + } + + def stratify( + analyses: dict[str, TargetReleaseInstallAnalysis], + ) -> dict[tuple[str, str], list[TargetReleaseInstallAnalysis]]: + strata: dict[tuple[str, str], list[TargetReleaseInstallAnalysis]] = defaultdict(list) + for analysis in analyses.values(): + if ( + analysis.latest_schema_version < ALERT_QUALITY_SCHEMA_VERSION + or analysis.same_version_pair_count == 0 + ): + continue + strata[( + analysis.latest_known_install_age_bucket, + _heartbeat_exposure_bucket(analysis.heartbeat_count), + )].append(analysis) + return strata + + def summarize_stratum(items: list[TargetReleaseInstallAnalysis]) -> dict[str, Any]: + totals = { + field: sum(item.increase_totals[index] for item in items) + for field, index in quality_index.items() + } + fired = totals["alerts_fired_30d"] + resolved = totals["alerts_resolved_30d"] + snoozed = totals["alerts_snoozed_occurrences_30d"] + return { + "installs": len(items), + "same_version_pairs": sum(item.same_version_pair_count for item in items), + "positive_changes": totals, + "resolution_per_fired": (resolved / fired) if fired else None, + "repeat_per_fired": ( + totals["alerts_repeat_occurrences_30d"] / fired if fired else None + ), + "resolved_while_snoozed_per_snoozed": ( + totals["alerts_resolved_while_snoozed_30d"] / snoozed + if snoozed + else None + ), + } + + target_strata = stratify(target_analyses) + baseline_strata = stratify(baseline_analyses) + for age_bucket, heartbeat_bucket in sorted(set(target_strata) & set(baseline_strata)): + key = (age_bucket, heartbeat_bucket) + result["matched_exposure_strata"].append({ + "known_install_age_bucket": age_bucket, + "heartbeat_count_bucket": heartbeat_bucket, + "target": summarize_stratum(target_strata[key]), + "baseline": summarize_stratum(baseline_strata[key]), + }) + if not result["matched_exposure_strata"]: + result["status"] = "no_matched_exposure" + result["reason"] = ( + "Schema-capable rows exist, but no known-age and heartbeat-count stratum has " + "same-version follow-up in both releases." + ) + else: + result["status"] = "descriptive_matched_strata" + return result + + +def _summarize_target_release_followup_analysis( + latest_by_install: dict[str, dict[str, Any]], + published_versions: set[str], + normalized_target: str, + target_identity: ClassifiedVersion, + analyses: dict[str, TargetReleaseInstallAnalysis], +) -> dict[str, Any]: signals = { field: { "field": field, @@ -3052,6 +3304,7 @@ def summarize_rows( rows: Iterable[dict[str, Any]], published_versions: set[str], target_version: str | None = None, + baseline_version: str | None = None, include_mock_fleet: bool = False, *, now: datetime | None = None, @@ -3087,17 +3340,33 @@ def summarize_rows( for install_id, analysis in pulse_intelligence_analysis.items() if install_id in latest_by_install } + release_fact_list = ( + list(target_release_analysis_facts) + if target_release_analysis_facts is not None + else None + ) target_release_analysis: dict[str, TargetReleaseInstallAnalysis] = {} if target_version: target_release_analysis = ( - analyze_target_release_facts(target_release_analysis_facts) - if target_release_analysis_facts is not None + analyze_target_release_facts(release_fact_list, target_version) + if release_fact_list is not None else analyze_target_release_rows( row_list, published_versions, target_version, ) ) + baseline_release_analysis: dict[str, TargetReleaseInstallAnalysis] = {} + if baseline_version: + baseline_release_analysis = ( + analyze_target_release_facts(release_fact_list, baseline_version) + if release_fact_list is not None + else analyze_target_release_rows( + row_list, + published_versions, + baseline_version, + ) + ) latest_install_windows = summarize_latest_install_windows( latest_by_install, published_versions, @@ -3166,6 +3435,14 @@ def summarize_rows( if target_version else None, "target_release_followup": target_release_followup, + "alert_quality_release_comparison": summarize_alert_quality_release_comparison( + target_version, + baseline_version, + target_release_analysis, + baseline_release_analysis, + ) + if target_version and baseline_version + else None, "active_latest": { "active_24h": latest_install_windows["24h"]["active_installs"], "active_72h": summary_72h["active_installs"], @@ -3559,6 +3836,36 @@ def format_text(summary: dict[str, Any], repo: str, since_days: int) -> str: else: lines.append(" - none") + quality_comparison = summary.get("alert_quality_release_comparison") + if quality_comparison: + target = quality_comparison["target"] + baseline = quality_comparison["baseline"] + lines.extend([ + "", + f"Alert-quality release comparison ({target['version']} vs {baseline['version']}):", + f"- status: {quality_comparison['status']}", + f"- target exposure: {target['installs']} install(s), {target['heartbeat_rows']} heartbeat(s), " + f"{target['same_version_comparable_installs']} schema-capable same-version follow-up install(s)", + f"- baseline exposure: {baseline['installs']} install(s), {baseline['heartbeat_rows']} heartbeat(s), " + f"{baseline['same_version_comparable_installs']} schema-capable same-version follow-up install(s)", + "- first heartbeat for each version and install is baseline-only", + "- comparisons use only matched known-age and heartbeat-count exposure strata", + ]) + if quality_comparison.get("reason"): + lines.append(f"- reason: {quality_comparison['reason']}") + strata = quality_comparison.get("matched_exposure_strata", []) + lines.append("- matched exposure strata:") + if strata: + for stratum in strata: + lines.append( + " - age " + stratum["known_install_age_bucket"] + + ", heartbeats " + stratum["heartbeat_count_bucket"] + + f": target {stratum['target']['installs']} install(s), " + f"baseline {stratum['baseline']['installs']} install(s)" + ) + else: + lines.append(" - none") + target_coverage = summary.get("target_release_coverage_7d") if target_coverage: lines.extend( @@ -3633,6 +3940,13 @@ def parse_args(argv: list[str]) -> argparse.Namespace: "the latest published stable release, falling back to the latest RC" ), ) + parser.add_argument( + "--baseline-version", + help=( + "optional earlier release for schema-gated alert-quality comparison; " + "cohorts are separated by known-age and heartbeat-count exposure" + ), + ) parser.add_argument( "--include-mock-fleet", action="store_true", @@ -3664,6 +3978,7 @@ def main(argv: list[str] | None = None) -> int: args.db_path, args.since_days, target_version=target_version, + baseline_version=args.baseline_version, ) if args.ssh_host else fetch_rows_local(args.db_path, args.since_days) @@ -3673,6 +3988,7 @@ def main(argv: list[str] | None = None) -> int: source["rows"], published_versions, target_version=target_version, + baseline_version=args.baseline_version, include_mock_fleet=args.include_mock_fleet, source_window_days=args.since_days, pulse_intelligence_analysis_facts=source.get( diff --git a/scripts/tests/test_telemetry_adoption_report.py b/scripts/tests/test_telemetry_adoption_report.py index db962d7c5..c0a6d0efe 100644 --- a/scripts/tests/test_telemetry_adoption_report.py +++ b/scripts/tests/test_telemetry_adoption_report.py @@ -55,11 +55,14 @@ class TelemetryAdoptionReportTest(unittest.TestCase): decrease_totals = list(first_counts) target_release_facts = [ { + "version": "6.3.0-rc.3", "install_id": "a", "heartbeat_count": 2, "same_version_pair_count": 1, "first_received_at": "2026-07-16 23:00:00", "latest_received_at": "2026-07-17 00:00:00", + "latest_schema_version": 14, + "latest_known_install_age_bucket": "31_90d", "first_counts": first_counts, "increase_totals": increase_totals, "decrease_totals": decrease_totals, @@ -93,11 +96,14 @@ class TelemetryAdoptionReportTest(unittest.TestCase): json.dumps( { "t": [ + fact["version"], fact["install_id"], fact["heartbeat_count"], fact["same_version_pair_count"], fact["first_received_at"], fact["latest_received_at"], + fact["latest_schema_version"], + fact["latest_known_install_age_bucket"], fact["first_counts"], fact["increase_totals"], fact["decrease_totals"], @@ -149,18 +155,19 @@ class TelemetryAdoptionReportTest(unittest.TestCase): self.assertNotIn("SELECT *", remote_script) self.assertIn("received_at >= datetime('now', ?)", remote_script) self.assertEqual( - run_mock.call_args.args[0][-4], + run_mock.call_args.args[0][-5], ",".join(report.REPORT_ROW_COLUMNS), ) self.assertEqual( - run_mock.call_args.args[0][-3], + run_mock.call_args.args[0][-4], ",".join(report.REPORT_HISTORY_SIGNAL_COLUMNS), ) - self.assertEqual(run_mock.call_args.args[0][-2], "6.3.0-rc.3") + self.assertEqual(run_mock.call_args.args[0][-3], "6.3.0-rc.3") self.assertEqual( - run_mock.call_args.args[0][-1], + run_mock.call_args.args[0][-2], ",".join(report.TARGET_RELEASE_ACTIVITY_COUNT_FIELDS), ) + self.assertEqual(run_mock.call_args.args[0][-1], "") self.assertIn("ROW_NUMBER() OVER (", remote_script) self.assertIn("GROUP BY install_id", remote_script) self.assertIn("MIN(CASE WHEN paid_license = 0", remote_script) @@ -300,11 +307,12 @@ class TelemetryAdoptionReportTest(unittest.TestCase): ] target_record = next(record["t"] for record in records if "t" in record) alerts_index = report.TARGET_RELEASE_ACTIVITY_COUNT_FIELDS.index("alerts_fired_30d") - self.assertEqual(target_record[1], 2) - self.assertEqual(target_record[2], 1) - self.assertEqual(target_record[5][alerts_index], 4) - self.assertEqual(target_record[6][alerts_index], 2) - self.assertEqual(target_record[7][alerts_index], 0) + self.assertEqual(target_record[0], "6.3.0-rc.3") + self.assertEqual(target_record[2], 2) + self.assertEqual(target_record[3], 1) + self.assertEqual(target_record[8][alerts_index], 4) + self.assertEqual(target_record[9][alerts_index], 2) + self.assertEqual(target_record[10][alerts_index], 0) def test_compact_intelligence_facts_match_full_history_analysis(self) -> None: numeric_columns = { @@ -684,6 +692,94 @@ class TelemetryAdoptionReportTest(unittest.TestCase): self.assertIn("rollback transitions:", rendered) self.assertIn("6.2.1: 1 install(s)", rendered) + def test_alert_quality_comparison_rejects_640_vs_632_legacy_schemas(self) -> None: + now = datetime(2026, 8, 29, 12, tzinfo=timezone.utc) + rows = [ + { + "install_id": "target-install", + "version": "6.4.0", + "schema_version": 13, + "known_install_age_bucket": "31_90d", + "received_at": now.replace(hour=8).strftime("%Y-%m-%d %H:%M:%S"), + "alerts_fired_30d": 100, + }, + { + "install_id": "target-install", + "version": "6.4.0", + "schema_version": 13, + "known_install_age_bucket": "31_90d", + "received_at": now.replace(hour=10).strftime("%Y-%m-%d %H:%M:%S"), + "alerts_fired_30d": 102, + }, + { + "install_id": "baseline-install", + "version": "6.3.2", + "schema_version": 12, + "known_install_age_bucket": "91_365d", + "received_at": now.replace(hour=7).strftime("%Y-%m-%d %H:%M:%S"), + "alerts_fired_30d": 40, + }, + { + "install_id": "baseline-install", + "version": "6.3.2", + "schema_version": 12, + "known_install_age_bucket": "91_365d", + "received_at": now.replace(hour=11).strftime("%Y-%m-%d %H:%M:%S"), + "alerts_fired_30d": 80, + }, + ] + full_summary = report.summarize_rows( + {"latest_ping": rows[-1]["received_at"], "total_rows": 4, "total_distinct_installs": 2}, + rows, + {"6.4.0", "6.3.2"}, + target_version="6.4.0", + baseline_version="6.3.2", + now=now, + ) + summary = full_summary["alert_quality_release_comparison"] + self.assertEqual(summary["status"], "schema_not_comparable") + self.assertEqual(summary["matched_exposure_strata"], []) + self.assertIn("schema 14", summary["reason"]) + rendered = report.format_text(full_summary, "rcourtman/Pulse", 7) + self.assertIn("Alert-quality release comparison (6.4.0 vs 6.3.2):", rendered) + self.assertIn("status: schema_not_comparable", rendered) + + def test_alert_quality_comparison_uses_only_matched_followup_exposure(self) -> None: + now = datetime(2026, 8, 29, 12, tzinfo=timezone.utc) + + def rows_for(version: str, install: str, fired: tuple[int, int], resolved: tuple[int, int]): + return [ + { + "install_id": install, + "version": version, + "schema_version": 14, + "known_install_age_bucket": "31_90d", + "received_at": now.replace(hour=hour).strftime("%Y-%m-%d %H:%M:%S"), + "alerts_fired_30d": fired[index], + "alerts_resolved_30d": resolved[index], + "alerts_repeat_occurrences_30d": index, + } + for index, hour in enumerate((8, 10)) + ] + + rows = rows_for("6.4.1", "target", (100, 104), (70, 73)) + rows_for( + "6.4.0", "baseline", (30, 35), (20, 22) + ) + summary = report.summarize_rows( + {"latest_ping": rows[-1]["received_at"], "total_rows": 4, "total_distinct_installs": 2}, + rows, + {"6.4.1", "6.4.0"}, + target_version="6.4.1", + baseline_version="6.4.0", + now=now, + )["alert_quality_release_comparison"] + self.assertEqual(summary["status"], "descriptive_matched_strata") + self.assertEqual(len(summary["matched_exposure_strata"]), 1) + stratum = summary["matched_exposure_strata"][0] + self.assertEqual(stratum["heartbeat_count_bucket"], "2") + self.assertEqual(stratum["target"]["positive_changes"]["alerts_fired_30d"], 4) + self.assertEqual(stratum["baseline"]["positive_changes"]["alerts_fired_30d"], 5) + self.assertEqual(stratum["target"]["resolution_per_fired"], 0.75) def test_compact_target_release_facts_match_local_one_pass_analysis(self) -> None: rows = [ {