diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index 0f3a5adcb..33643aa42 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -2390,3 +2390,28 @@ against a managed local backend. The browser phase must exercise active-alert snooze, delivery diagnosis, restart persistence, unsnooze, incident evidence, history migration, virtualization, and clear tombstones without replacing the backend APIs with route mocks. + +### Noisy gauges use one trust-stable lifecycle + +Memory, temperature, and disk-temperature warnings use one continuous +stability window on both lifecycle edges: an unchanged factory configuration +requires five minutes above the trigger to open and five minutes at or below +the recovery threshold to close. A return to the hysteresis band or a renewed +breach resets the recovery run. Critical metric evidence may bypass only this +factory or legacy activation delay; an explicit alert-intent policy remains +authoritative. A metric-specific delay, including zero, remains the simple +operator override and becomes the matching recovery window for these gauges. + +Recovery timing uses the reducer's monotonic runtime evidence when supplied, +so wall-clock changes cannot prematurely close an incident. Missing or unknown +observations do not constitute healthy evidence and therefore cannot advance a +recovery run. Restarts conservatively restart an in-progress recovery window +rather than manufacturing elapsed health. + +For guest disk capacity, valid per-filesystem evidence owns the alert identity +and suppresses the less actionable guest-wide aggregate. This guarantees one +incident for one full filesystem while retaining the aggregate only when no +usable filesystem evidence exists. OS-managed transient mounts, including +macOS Gatekeeper App Translocation volumes, are removed at the shared +filesystem collection boundary before they can enter capacity inventory or +alert evaluation. diff --git a/internal/alerts/canonical_metric.go b/internal/alerts/canonical_metric.go index d4befbc79..e7b15c494 100644 --- a/internal/alerts/canonical_metric.go +++ b/internal/alerts/canonical_metric.go @@ -170,6 +170,7 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec var intent *reducer.DiscreteIntent delaySeconds := 0 effectiveIntent := m.resolveEffectiveIntentPolicyNoLock(spec.ResourceID, resourceType, MetricAlertIntentSignal(metricType)) + stability := metricStabilityPolicy{} if effectiveIntent.Explicit { decision := m.evaluateIntentNoLock(spec.ResourceID, resourceType, MetricAlertIntentSignal(metricType), trackingKey, observedAt, triggered, BackupIntentContext{}) if decision.StateChanged { @@ -184,6 +185,9 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec } else if triggered { delaySeconds = m.getTimeThreshold(spec.ResourceID, resourceType, metricType) } + if !effectiveIntent.Explicit { + stability = metricStabilityFor(metricType, effectiveIntent.GraceSeconds) + } evidence := alertspecs.AlertEvidence{ ObservedAt: observedAt, @@ -210,12 +214,14 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec RuntimeTickValid: true, ObservedAt: observedAt, }, reducer.MetricRule{ - Trigger: spec.MetricThreshold.Trigger, - Clear: clearThreshold, - Critical: spec.MetricThreshold.Critical, - CriticalDisabled: spec.MetricThreshold.Critical == nil, - DelaySeconds: delaySeconds, - Intent: intent, + Trigger: spec.MetricThreshold.Trigger, + Clear: clearThreshold, + Critical: spec.MetricThreshold.Critical, + CriticalDisabled: spec.MetricThreshold.Critical == nil, + DelaySeconds: delaySeconds, + CriticalBypassesDelay: stability.CriticalBypassesDelay, + RecoveryDelaySeconds: stability.RecoveryDelaySeconds, + Intent: intent, }) primary := reducer.EventType("") if len(events) > 0 { @@ -243,6 +249,10 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec "clearThreshold": metricClearThreshold(spec.MetricThreshold, threshold), "monitorOnly": monitorOnly, } + if stability.RecoveryDelaySeconds > 0 { + alertMetadata["stabilityWindowSeconds"] = stability.RecoveryDelaySeconds + alertMetadata["criticalBypassesStability"] = stability.CriticalBypassesDelay + } if unit != "" { alertMetadata["unit"] = unit } diff --git a/internal/alerts/guest.go b/internal/alerts/guest.go index 257a5acb5..5eb502e55 100644 --- a/internal/alerts/guest.go +++ b/internal/alerts/guest.go @@ -261,40 +261,22 @@ func (m *Manager) CheckGuest(guest any, instanceName string) { diskReadMetric, diskWriteMetric, networkInMetric, networkOutMetric := guestIORateMetrics(snapshot) diskMetric := &UnifiedResourceMetric{Percent: diskUsage} if len(disks) > 0 { - var aggregateTotal, aggregateUsed int64 - hasDedicatedDiskOverride := false seenAggregateDiskKeys := make(map[string]struct{}) for idx, disk := range disks { if disk.Total <= 0 || disk.Usage < 0 { continue } - label, keySource, sanitizedKey := guestDiskIdentity(disk, idx) + _, _, sanitizedKey := guestDiskIdentity(disk, idx) if _, exists := seenAggregateDiskKeys[sanitizedKey]; exists { continue } seenAggregateDiskKeys[sanitizedKey] = struct{}{} - - m.mu.RLock() - diskOverride, hasDiskOverride := lookupGuestDiskOverride(m.config.Overrides, guest, guestID, keySource) - m.mu.RUnlock() - if hasDiskOverride && (diskOverride.Disabled || diskOverride.Disk != nil) { - hasDedicatedDiskOverride = true - log.Debug(). - Str("guest", name). - Str("diskLabel", label). - Msg("Excluding guest filesystem with a dedicated override from aggregate disk alert") - continue - } - aggregateTotal += disk.Total - aggregateUsed += disk.Used } - if hasDedicatedDiskOverride { - if aggregateTotal > 0 { - diskMetric.Percent = float64(aggregateUsed) / float64(aggregateTotal) * 100 - } else { - diskMetric = nil - m.clearAlert(canonicalMetricStateID(guestID, "disk")) - } + if len(seenAggregateDiskKeys) > 0 { + // Filesystem evidence is more actionable than the guest-wide rollup. + // Evaluating both creates two incidents for the same full filesystem. + diskMetric = nil + m.clearAlert(canonicalMetricStateID(guestID, "disk")) } } m.evaluateUnifiedMetrics(&UnifiedResourceInput{ diff --git a/internal/alerts/metric_runtime.go b/internal/alerts/metric_runtime.go index 498f50a3f..8e82ae1a5 100644 --- a/internal/alerts/metric_runtime.go +++ b/internal/alerts/metric_runtime.go @@ -11,6 +11,44 @@ import ( "github.com/rs/zerolog/log" ) +type metricStabilityPolicy struct { + CriticalBypassesDelay bool + RecoveryDelaySeconds int +} + +const ( + defaultLegacyMetricDelaySeconds = 5 + defaultNoisyGaugeStabilitySeconds = 5 * 60 +) + +func defaultMetricStabilityDelay(metricType string) int { + switch strings.ToLower(strings.TrimSpace(metricType)) { + case "memory", "temperature", "disktemperature", "disk_temperature": + return defaultNoisyGaugeStabilitySeconds + default: + return 0 + } +} + +// metricStabilityFor keeps noisy gauges trustworthy without adding a second +// operator-facing policy surface. The configured alert delay is the one +// stability window on both lifecycle edges; critical evidence bypasses only +// the legacy/default activation delay, never an explicit intent policy. +func metricStabilityFor(metricType string, delaySeconds int) metricStabilityPolicy { + if delaySeconds <= 0 { + return metricStabilityPolicy{} + } + switch strings.ToLower(strings.TrimSpace(metricType)) { + case "memory", "temperature", "disktemperature", "disk_temperature": + return metricStabilityPolicy{ + CriticalBypassesDelay: true, + RecoveryDelaySeconds: delaySeconds, + } + default: + return metricStabilityPolicy{} + } +} + func isMetricThresholdAlertType(metricType string) bool { switch metricType { case "cpu", "memory", "disk", "diskRead", "diskWrite", "networkIn", "networkOut", "temperature", "usage": @@ -66,23 +104,58 @@ func (m *Manager) getLegacyTimeThresholdWithSource(resourceType, metricType stri if delay, ok := m.getMetricTimeThreshold(resourceType, metricType); ok { return delay, "legacy.metricTimeThresholds." + strings.ToLower(strings.TrimSpace(resourceType)) + "." + strings.ToLower(strings.TrimSpace(metricType)), true } - - base, hasTypeSpecific := m.getBaseTimeThreshold(resourceType) - - if !hasTypeSpecific { - if delay, ok := m.getGlobalMetricTimeThreshold(metricType); ok { - return delay, "legacy.metricTimeThresholds.all." + strings.ToLower(strings.TrimSpace(metricType)), true - } + if delay, ok := m.getGlobalExactMetricTimeThreshold(metricType); ok { + return delay, "legacy.metricTimeThresholds.all." + strings.ToLower(strings.TrimSpace(metricType)), true } + base, hasTypeSpecific := m.getBaseTimeThreshold(resourceType) if hasTypeSpecific { + // The legacy factory delay was five seconds for every metric. Upgrade + // only that unchanged default for noisy gauges; a deliberate type-level + // value remains authoritative, as does any metric-specific override above. + if base == defaultLegacyMetricDelaySeconds { + if delay := defaultMetricStabilityDelay(metricType); delay > 0 { + return delay, "factory.metricStability." + strings.ToLower(strings.TrimSpace(metricType)), true + } + } return base, "legacy.timeThresholds." + strings.ToLower(strings.TrimSpace(resourceType)), true } + if delay, ok := m.getGlobalMetricTimeThreshold(metricType); ok { + return delay, "legacy.metricTimeThresholds.all." + strings.ToLower(strings.TrimSpace(metricType)), true + } if base != 0 { + if base == defaultLegacyMetricDelaySeconds { + if delay := defaultMetricStabilityDelay(metricType); delay > 0 { + return delay, "factory.metricStability." + strings.ToLower(strings.TrimSpace(metricType)), true + } + } return base, "legacy.timeThresholds.all", true } return 0, "", false } +// getGlobalExactMetricTimeThreshold returns only an explicit all. +// override. Broad all.default/all.* fallbacks retain their legacy position +// after resource-type delays and therefore cannot accidentally erase those +// more specific policies. +func (m *Manager) getGlobalExactMetricTimeThreshold(metricType string) (int, bool) { + if len(m.config.MetricTimeThresholds) == 0 { + return 0, false + } + + perType, ok := m.config.MetricTimeThresholds["all"] + if !ok || len(perType) == 0 { + return 0, false + } + + metricKey := strings.ToLower(strings.TrimSpace(metricType)) + if metricKey == "" { + return 0, false + } + + delay, ok := perType[metricKey] + return delay, ok +} + // getMetricTimeThreshold returns a metric-specific delay if configured at the resource-type level. func (m *Manager) getMetricTimeThreshold(resourceType, metricType string) (int, bool) { if len(m.config.MetricTimeThresholds) == 0 { @@ -278,6 +351,7 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource effectiveIntent := m.resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType)) var intent *reducer.DiscreteIntent delaySeconds := 0 + stability := metricStabilityPolicy{} if effectiveIntent.Explicit { decision := m.evaluateIntentNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType), trackingKey, time.Now(), conditionActive, BackupIntentContext{}) if decision.StateChanged { @@ -292,6 +366,9 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource } else if conditionActive { delaySeconds = m.getTimeThreshold(resourceID, resourceType, metricType) } + if !effectiveIntent.Explicit { + stability = metricStabilityFor(metricType, effectiveIntent.GraceSeconds) + } events := m.core.ApplyMetric(reducer.MetricSignal{ ResourceID: resourceID, @@ -302,10 +379,12 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource RuntimeTickValid: true, ObservedAt: m.policyNow(), }, reducer.MetricRule{ - Trigger: threshold.Trigger, - Clear: threshold.Clear, - DelaySeconds: delaySeconds, - Intent: intent, + Trigger: threshold.Trigger, + Clear: threshold.Clear, + DelaySeconds: delaySeconds, + CriticalBypassesDelay: stability.CriticalBypassesDelay, + RecoveryDelaySeconds: stability.RecoveryDelaySeconds, + Intent: intent, }) primary := reducer.EventType("") if len(events) > 0 { diff --git a/internal/alerts/reducer/reducer.go b/internal/alerts/reducer/reducer.go index b05a85878..35cb37059 100644 --- a/internal/alerts/reducer/reducer.go +++ b/internal/alerts/reducer/reducer.go @@ -65,6 +65,15 @@ type MetricRule struct { // an explicit Intent gate is supplied — the manager's explicit intent // policies replace the legacy time-threshold delay. DelaySeconds int + // CriticalBypassesDelay lets urgent evidence activate immediately while + // warning-level evidence still has to survive the stability window. It is + // intentionally ignored when an explicit Intent gate is present: explicit + // operator policy remains authoritative. + CriticalBypassesDelay bool + // RecoveryDelaySeconds is the continuous time at or below Clear required + // before a firing incident resolves. A value <= 0 preserves immediate + // hysteresis recovery. + RecoveryDelaySeconds int // Intent is the resolved intent-policy context for this observation // (metric. signals); nil means no gate. Intent *DiscreteIntent @@ -108,9 +117,15 @@ type Incident struct { // toward DiscreteRule.RecoveryConfirmations; reset by any matching // observation. Unused by the metric family. RecoveryCount int - Acknowledged bool - AckUser string - AckAt time.Time + // RecoverySince and its monotonic companions track a continuous healthy + // run for metric rules with a recovery stability window. + RecoverySince time.Time + RecoveryElapsed time.Duration + RecoveryTicksSupplied bool + LastRecoveryRuntimeTick time.Duration + Acknowledged bool + AckUser string + AckAt time.Time // Backup-run bookkeeping for the intent gate's backup-offline deferral // sub-policy (confirmation family only). BackupActive bool @@ -390,6 +405,7 @@ func (s *State) ApplyMetric(signal MetricSignal, rule MetricRule) []Event { // recomputes level on every tick, so severity can demote as well as // escalate while firing. if incident != nil && incident.State == StateFiring { + resetMetricRecoveryRun(incident) previous := incident.Severity incident.Severity = severityFor(signal.Value, rule, signal.Metric) incident.LastValue = signal.Value @@ -444,7 +460,9 @@ func (s *State) ApplyMetric(signal MetricSignal, rule MetricRule) []Event { return pendingResult() } } else if rule.DelaySeconds > 0 { - if signal.ObservedAt.Sub(incident.PendingSince) < time.Duration(rule.DelaySeconds)*time.Second { + critical := severityFor(signal.Value, rule, signal.Metric) == SeverityCritical + if !(critical && rule.CriticalBypassesDelay) && + signal.ObservedAt.Sub(incident.PendingSince) < time.Duration(rule.DelaySeconds)*time.Second { return pendingResult() } } @@ -473,6 +491,31 @@ func (s *State) ApplyMetric(signal MetricSignal, rule MetricRule) []Event { clear = rule.Trigger } if signal.Value <= clear { + if rule.RecoveryDelaySeconds > 0 { + if incident.RecoverySince.IsZero() { + incident.RecoverySince = signal.ObservedAt + incident.RecoveryTicksSupplied = signal.RuntimeTickValid + incident.LastRecoveryRuntimeTick = signal.RuntimeTick + incident.RecoveryElapsed = 0 + incident.LastObservedAt = signal.ObservedAt + return nil + } + if signal.RuntimeTickValid { + if incident.RecoveryTicksSupplied && signal.RuntimeTick >= incident.LastRecoveryRuntimeTick { + incident.RecoveryElapsed += signal.RuntimeTick - incident.LastRecoveryRuntimeTick + } + incident.RecoveryTicksSupplied = true + incident.LastRecoveryRuntimeTick = signal.RuntimeTick + } + elapsed := signal.ObservedAt.Sub(incident.RecoverySince) + if incident.RecoveryTicksSupplied { + elapsed = incident.RecoveryElapsed + } + incident.LastObservedAt = signal.ObservedAt + if elapsed < time.Duration(rule.RecoveryDelaySeconds)*time.Second { + return nil + } + } severity := incident.Severity delete(s.incidents, key) s.markAckInactive(key, signal.ObservedAt) @@ -482,9 +525,20 @@ func (s *State) ApplyMetric(signal MetricSignal, rule MetricRule) []Event { // Hysteresis hold: between clear and trigger the incident stays firing. // The manager does not refresh value or last-seen in this band; the // reducer mirrors that so parity diffs stay exact. + resetMetricRecoveryRun(incident) return nil } +func resetMetricRecoveryRun(incident *Incident) { + if incident == nil { + return + } + incident.RecoverySince = time.Time{} + incident.RecoveryElapsed = 0 + incident.RecoveryTicksSupplied = false + incident.LastRecoveryRuntimeTick = 0 +} + // SeedFiringIncident installs a firing incident directly, bypassing the // transition functions. Shadow mode uses it to align the reducer with // pre-existing manager state (persisted-alert restore, divergence resync); diff --git a/internal/alerts/reducer/reducer_test.go b/internal/alerts/reducer/reducer_test.go index 33b699221..dc6f25659 100644 --- a/internal/alerts/reducer/reducer_test.go +++ b/internal/alerts/reducer/reducer_test.go @@ -132,6 +132,52 @@ func TestDelayFiresWithPendingStartAndDipResets(t *testing.T) { } } +func TestCriticalCanBypassWarningStabilityDelay(t *testing.T) { + state := NewState() + rule := MetricRule{Trigger: 80, Clear: 75, DelaySeconds: 300, CriticalBypassesDelay: true} + + events := state.ApplyMetric(signalAt(95, 0), rule) + if len(events) != 1 || events[0].Type != EventFired || events[0].Severity != SeverityCritical { + t.Fatalf("events = %+v, want immediate critical fire", events) + } +} + +func TestCriticalDoesNotBypassExplicitIntent(t *testing.T) { + state := NewState() + rule := MetricRule{ + Trigger: 80, Clear: 75, CriticalBypassesDelay: true, + Intent: &DiscreteIntent{Explicit: true, GraceSeconds: 300}, + } + + events := state.ApplyMetric(signalAt(95, 0), rule) + if len(events) != 1 || events[0].Type != EventPending { + t.Fatalf("events = %+v, want explicit intent to keep critical pending", events) + } +} + +func TestMetricRecoveryRequiresContinuousHealthyWindow(t *testing.T) { + state := NewState() + rule := MetricRule{Trigger: 80, Clear: 75, RecoveryDelaySeconds: 120} + state.ApplyMetric(signalAt(90, 0), rule) + + if events := state.ApplyMetric(signalAt(70, time.Minute), rule); len(events) != 0 { + t.Fatalf("first healthy observation emitted %+v", events) + } + if events := state.ApplyMetric(signalAt(78, 90*time.Second), rule); len(events) != 0 { + t.Fatalf("hysteresis-band interruption emitted %+v", events) + } + if events := state.ApplyMetric(signalAt(70, 2*time.Minute), rule); len(events) != 0 { + t.Fatalf("restarted healthy run emitted %+v", events) + } + if events := state.ApplyMetric(signalAt(70, 3*time.Minute+59*time.Second), rule); len(events) != 0 { + t.Fatalf("recovery fired before continuous window elapsed: %+v", events) + } + events := state.ApplyMetric(signalAt(70, 4*time.Minute), rule) + if len(events) != 1 || events[0].Type != EventResolved { + t.Fatalf("events = %+v, want recovery after continuous healthy window", events) + } +} + func TestClearFallsBackToTriggerWhenUnset(t *testing.T) { state := NewState() rule := MetricRule{Trigger: 80, Clear: 0} diff --git a/internal/alerts/threshold_resolution_shared_test.go b/internal/alerts/threshold_resolution_shared_test.go index c4c960865..65e5ee266 100644 --- a/internal/alerts/threshold_resolution_shared_test.go +++ b/internal/alerts/threshold_resolution_shared_test.go @@ -938,6 +938,32 @@ func TestGuestFilesystemThresholdOverrideOwnsMountEvaluation(t *testing.T) { } } +func TestGuestFilesystemEvidenceSuppressesDuplicateAggregateAlert(t *testing.T) { + m := newTestManager(t) + guestID := BuildGuestKey("pve1", "node1", 109) + m.mu.Lock() + m.config.TimeThresholds = map[string]int{} + m.config.GuestDefaults = ThresholdConfig{Disk: &HysteresisThreshold{Trigger: 80, Clear: 75}} + m.mu.Unlock() + + m.CheckGuest(models.VM{ + ID: guestID, VMID: 109, Name: "pulse-dev", Node: "node1", Instance: "pve1", Status: "running", + Disk: models.Disk{Usage: 96.5}, + Disks: []models.Disk{{Mountpoint: "/", Device: "/dev/vda1", Usage: 96.9, Total: 100, Used: 97, Free: 3}}, + }, "pve1") + + aggregateID := canonicalMetricStateID(guestID, "disk") + filesystemID := canonicalMetricStateID(guestID+"-disk-dev-vda1", "disk") + m.mu.RLock() + defer m.mu.RUnlock() + if _, exists := testLookupActiveAlert(t, m, aggregateID); exists { + t.Fatalf("aggregate alert %q duplicated filesystem evidence", aggregateID) + } + if _, exists := testLookupActiveAlert(t, m, filesystemID); !exists { + t.Fatalf("actionable filesystem alert %q was not created", filesystemID) + } +} + // Regression: checkMetric stores canonical-identity alerts under the // canonical state key, so hysteresis resolution must not remove only the // unregistered legacy "-" ID. diff --git a/internal/alerts/time_threshold_test.go b/internal/alerts/time_threshold_test.go index 2396e7861..83a182f30 100644 --- a/internal/alerts/time_threshold_test.go +++ b/internal/alerts/time_threshold_test.go @@ -85,6 +85,59 @@ func TestGetTimeThresholdMetricOverrides(t *testing.T) { } } +func TestGetTimeThresholdUsesTrustStableNoisyGaugeDefaults(t *testing.T) { + manager := NewManagerWithDataDir(t.TempDir(), WithoutPersistedAlertRestore()) + + t.Cleanup(manager.Stop) + + if got := manager.getTimeThreshold("vm-resource", "VM", "memory"); got != defaultNoisyGaugeStabilitySeconds { + t.Fatalf("default memory stability = %d, want %d", got, defaultNoisyGaugeStabilitySeconds) + } + if got := manager.getTimeThreshold("node-1", "Node", "temperature"); got != defaultNoisyGaugeStabilitySeconds { + t.Fatalf("default temperature stability = %d, want %d", got, defaultNoisyGaugeStabilitySeconds) + } + if got := manager.getTimeThreshold("vm-resource", "VM", "cpu"); got != defaultLegacyMetricDelaySeconds { + t.Fatalf("default CPU delay = %d, want %d", got, defaultLegacyMetricDelaySeconds) + } + + manager.mu.Lock() + manager.config.MetricTimeThresholds = map[string]map[string]int{ + "all": {"memory": 0}, + } + manager.mu.Unlock() + + if got := manager.getTimeThreshold("vm-resource", "VM", "memory"); got != 0 { + t.Fatalf("explicit global memory override = %d, want 0", got) + } +} + +func TestCheckMetricNoisyWarningWaitsButCriticalFiresImmediately(t *testing.T) { + manager := NewManagerWithDataDir(t.TempDir(), WithoutPersistedAlertRestore()) + t.Cleanup(manager.Stop) + + threshold := &HysteresisThreshold{Trigger: 85, Clear: 80} + canonicalState := buildCanonicalStateID("vm-resource", "metric-threshold:memory") + + manager.checkMetric("vm-resource", "database", "node-1", "pve-1", "VM", "memory", 90, threshold, nil) + manager.mu.RLock() + _, warningActive := manager.getActiveAlertNoLock(canonicalState) + manager.mu.RUnlock() + if warningActive { + t.Fatal("brief warning-level memory evidence opened an alert") + } + + manager.checkMetric("vm-resource", "database", "node-1", "pve-1", "VM", "memory", 96, threshold, nil) + manager.mu.RLock() + alert, criticalActive := manager.getActiveAlertNoLock(canonicalState) + manager.mu.RUnlock() + if !criticalActive { + t.Fatal("critical memory evidence did not bypass the factory stability delay") + } + if alert.Level != AlertLevelCritical { + t.Fatalf("critical memory alert level = %q, want %q", alert.Level, AlertLevelCritical) + } +} + func TestCheckMetricUsesPendingStartTime(t *testing.T) { manager := NewManager() diff --git a/internal/alerts/unified_eval_test.go b/internal/alerts/unified_eval_test.go index 9584df70d..de010d214 100644 --- a/internal/alerts/unified_eval_test.go +++ b/internal/alerts/unified_eval_test.go @@ -1559,3 +1559,27 @@ func TestUnifiedStorageForecastKeepsCanonicalMetricSpecAcrossPlatforms(t *testin t.Fatalf("CanonicalSpecID = %q, want canonical usage spec", alert.CanonicalSpecID) } } + +func TestUnifiedMetricNoisyWarningWaitsButCriticalFiresImmediately(t *testing.T) { + m := newTestManager(t) + input := &UnifiedResourceInput{ + ID: "vm-noisy-memory", + Type: "vm", + Name: "database", + Memory: &UnifiedResourceMetric{Percent: 90}, + } + alertID := canonicalMetricStateID(input.ID, "memory") + + m.CheckUnifiedResource(input) + assertAlertMissing(t, m, alertID) + + input.Memory.Percent = 96 + m.CheckUnifiedResource(input) + alert := testRequireActiveAlert(t, m, alertID) + if alert.Level != AlertLevelCritical { + t.Fatalf("critical memory alert level = %q, want %q", alert.Level, AlertLevelCritical) + } + if got := alert.Metadata["stabilityWindowSeconds"]; got != defaultNoisyGaugeStabilitySeconds { + t.Fatalf("stabilityWindowSeconds = %v, want %d", got, defaultNoisyGaugeStabilitySeconds) + } +} diff --git a/pkg/fsfilters/filters.go b/pkg/fsfilters/filters.go index ed58b7b39..41f145236 100644 --- a/pkg/fsfilters/filters.go +++ b/pkg/fsfilters/filters.go @@ -129,6 +129,13 @@ var containerPathPrefixes = []string{ "/mnt/.ix-apps/docker/", // TrueNAS SCALE Docker overlay mounts } +// transientMountPatterns identify OS-managed mounts whose lifetime follows an +// application launch rather than operator-managed storage. They must not +// participate in capacity inventory or alerting. +var transientMountPatterns = []string{ + "/AppTranslocation/", // macOS Gatekeeper translocated application image +} + // ShouldSkipFilesystem determines if a filesystem should be excluded from disk // usage aggregation. It checks for read-only filesystems, virtual/pseudo filesystems, // network mounts, and special system mountpoints. Returns skip=true if the filesystem @@ -168,6 +175,13 @@ func ShouldSkipFilesystem(fsType, mountpoint string, totalBytes, usedBytes uint6 reasons = append(reasons, "special-mountpoint") } + for _, pattern := range transientMountPatterns { + if strings.Contains(mountpoint, pattern) { + reasons = append(reasons, "transient-mountpoint") + break + } + } + // Windows System Reserved partition if mountpoint == "System Reserved" || strings.Contains(mountpoint, "System Reserved") { reasons = append(reasons, "special-mountpoint") diff --git a/pkg/fsfilters/filters_test.go b/pkg/fsfilters/filters_test.go index 4b6bf0e92..985c74ea0 100644 --- a/pkg/fsfilters/filters_test.go +++ b/pkg/fsfilters/filters_test.go @@ -204,6 +204,7 @@ func TestShouldSkipFilesystem(t *testing.T) { {"macos data companion volume", "apfs", "/System/Volumes/Data", 245107195904, 213573853184, true}, {"macos preboot companion volume", "apfs", "/System/Volumes/Preboot", 245107195904, 213573853184, true}, {"macos simulator runtime volume", "apfs", "/Library/Developer/CoreSimulator/Volumes/iOS_23E254a", 18058575872, 17593331712, true}, + {"macos app translocation volume", "apfs", "/private/var/folders/xy/random/T/AppTranslocation/2A9C/d/Pulse.app", 245107195904, 221809991680, true}, // Regular filesystems that should NOT be skipped {"ext4 root", "ext4", "/", 100 * 1024 * 1024 * 1024, 50 * 1024 * 1024 * 1024, false},