fix(alerts): stabilize noisy lifecycle signals

This commit is contained in:
Pulse Test
2026-08-29 17:32:17 +01:00
parent a506cdde8b
commit d671f05408
11 changed files with 359 additions and 45 deletions
@@ -2390,3 +2390,28 @@ against a managed local backend. The browser phase must exercise active-alert
snooze, delivery diagnosis, restart persistence, unsnooze, incident evidence,
history migration, virtualization, and clear tombstones without replacing the
backend APIs with route mocks.
### Noisy gauges use one trust-stable lifecycle
Memory, temperature, and disk-temperature warnings use one continuous
stability window on both lifecycle edges: an unchanged factory configuration
requires five minutes above the trigger to open and five minutes at or below
the recovery threshold to close. A return to the hysteresis band or a renewed
breach resets the recovery run. Critical metric evidence may bypass only this
factory or legacy activation delay; an explicit alert-intent policy remains
authoritative. A metric-specific delay, including zero, remains the simple
operator override and becomes the matching recovery window for these gauges.
Recovery timing uses the reducer's monotonic runtime evidence when supplied,
so wall-clock changes cannot prematurely close an incident. Missing or unknown
observations do not constitute healthy evidence and therefore cannot advance a
recovery run. Restarts conservatively restart an in-progress recovery window
rather than manufacturing elapsed health.
For guest disk capacity, valid per-filesystem evidence owns the alert identity
and suppresses the less actionable guest-wide aggregate. This guarantees one
incident for one full filesystem while retaining the aggregate only when no
usable filesystem evidence exists. OS-managed transient mounts, including
macOS Gatekeeper App Translocation volumes, are removed at the shared
filesystem collection boundary before they can enter capacity inventory or
alert evaluation.
+16 -6
View File
@@ -170,6 +170,7 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
var intent *reducer.DiscreteIntent
delaySeconds := 0
effectiveIntent := m.resolveEffectiveIntentPolicyNoLock(spec.ResourceID, resourceType, MetricAlertIntentSignal(metricType))
stability := metricStabilityPolicy{}
if effectiveIntent.Explicit {
decision := m.evaluateIntentNoLock(spec.ResourceID, resourceType, MetricAlertIntentSignal(metricType), trackingKey, observedAt, triggered, BackupIntentContext{})
if decision.StateChanged {
@@ -184,6 +185,9 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
} else if triggered {
delaySeconds = m.getTimeThreshold(spec.ResourceID, resourceType, metricType)
}
if !effectiveIntent.Explicit {
stability = metricStabilityFor(metricType, effectiveIntent.GraceSeconds)
}
evidence := alertspecs.AlertEvidence{
ObservedAt: observedAt,
@@ -210,12 +214,14 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
RuntimeTickValid: true,
ObservedAt: observedAt,
}, reducer.MetricRule{
Trigger: spec.MetricThreshold.Trigger,
Clear: clearThreshold,
Critical: spec.MetricThreshold.Critical,
CriticalDisabled: spec.MetricThreshold.Critical == nil,
DelaySeconds: delaySeconds,
Intent: intent,
Trigger: spec.MetricThreshold.Trigger,
Clear: clearThreshold,
Critical: spec.MetricThreshold.Critical,
CriticalDisabled: spec.MetricThreshold.Critical == nil,
DelaySeconds: delaySeconds,
CriticalBypassesDelay: stability.CriticalBypassesDelay,
RecoveryDelaySeconds: stability.RecoveryDelaySeconds,
Intent: intent,
})
primary := reducer.EventType("")
if len(events) > 0 {
@@ -243,6 +249,10 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
"clearThreshold": metricClearThreshold(spec.MetricThreshold, threshold),
"monitorOnly": monitorOnly,
}
if stability.RecoveryDelaySeconds > 0 {
alertMetadata["stabilityWindowSeconds"] = stability.RecoveryDelaySeconds
alertMetadata["criticalBypassesStability"] = stability.CriticalBypassesDelay
}
if unit != "" {
alertMetadata["unit"] = unit
}
+6 -24
View File
@@ -261,40 +261,22 @@ func (m *Manager) CheckGuest(guest any, instanceName string) {
diskReadMetric, diskWriteMetric, networkInMetric, networkOutMetric := guestIORateMetrics(snapshot)
diskMetric := &UnifiedResourceMetric{Percent: diskUsage}
if len(disks) > 0 {
var aggregateTotal, aggregateUsed int64
hasDedicatedDiskOverride := false
seenAggregateDiskKeys := make(map[string]struct{})
for idx, disk := range disks {
if disk.Total <= 0 || disk.Usage < 0 {
continue
}
label, keySource, sanitizedKey := guestDiskIdentity(disk, idx)
_, _, sanitizedKey := guestDiskIdentity(disk, idx)
if _, exists := seenAggregateDiskKeys[sanitizedKey]; exists {
continue
}
seenAggregateDiskKeys[sanitizedKey] = struct{}{}
m.mu.RLock()
diskOverride, hasDiskOverride := lookupGuestDiskOverride(m.config.Overrides, guest, guestID, keySource)
m.mu.RUnlock()
if hasDiskOverride && (diskOverride.Disabled || diskOverride.Disk != nil) {
hasDedicatedDiskOverride = true
log.Debug().
Str("guest", name).
Str("diskLabel", label).
Msg("Excluding guest filesystem with a dedicated override from aggregate disk alert")
continue
}
aggregateTotal += disk.Total
aggregateUsed += disk.Used
}
if hasDedicatedDiskOverride {
if aggregateTotal > 0 {
diskMetric.Percent = float64(aggregateUsed) / float64(aggregateTotal) * 100
} else {
diskMetric = nil
m.clearAlert(canonicalMetricStateID(guestID, "disk"))
}
if len(seenAggregateDiskKeys) > 0 {
// Filesystem evidence is more actionable than the guest-wide rollup.
// Evaluating both creates two incidents for the same full filesystem.
diskMetric = nil
m.clearAlert(canonicalMetricStateID(guestID, "disk"))
}
}
m.evaluateUnifiedMetrics(&UnifiedResourceInput{
+90 -11
View File
@@ -11,6 +11,44 @@ import (
"github.com/rs/zerolog/log"
)
type metricStabilityPolicy struct {
CriticalBypassesDelay bool
RecoveryDelaySeconds int
}
const (
defaultLegacyMetricDelaySeconds = 5
defaultNoisyGaugeStabilitySeconds = 5 * 60
)
func defaultMetricStabilityDelay(metricType string) int {
switch strings.ToLower(strings.TrimSpace(metricType)) {
case "memory", "temperature", "disktemperature", "disk_temperature":
return defaultNoisyGaugeStabilitySeconds
default:
return 0
}
}
// metricStabilityFor keeps noisy gauges trustworthy without adding a second
// operator-facing policy surface. The configured alert delay is the one
// stability window on both lifecycle edges; critical evidence bypasses only
// the legacy/default activation delay, never an explicit intent policy.
func metricStabilityFor(metricType string, delaySeconds int) metricStabilityPolicy {
if delaySeconds <= 0 {
return metricStabilityPolicy{}
}
switch strings.ToLower(strings.TrimSpace(metricType)) {
case "memory", "temperature", "disktemperature", "disk_temperature":
return metricStabilityPolicy{
CriticalBypassesDelay: true,
RecoveryDelaySeconds: delaySeconds,
}
default:
return metricStabilityPolicy{}
}
}
func isMetricThresholdAlertType(metricType string) bool {
switch metricType {
case "cpu", "memory", "disk", "diskRead", "diskWrite", "networkIn", "networkOut", "temperature", "usage":
@@ -66,23 +104,58 @@ func (m *Manager) getLegacyTimeThresholdWithSource(resourceType, metricType stri
if delay, ok := m.getMetricTimeThreshold(resourceType, metricType); ok {
return delay, "legacy.metricTimeThresholds." + strings.ToLower(strings.TrimSpace(resourceType)) + "." + strings.ToLower(strings.TrimSpace(metricType)), true
}
base, hasTypeSpecific := m.getBaseTimeThreshold(resourceType)
if !hasTypeSpecific {
if delay, ok := m.getGlobalMetricTimeThreshold(metricType); ok {
return delay, "legacy.metricTimeThresholds.all." + strings.ToLower(strings.TrimSpace(metricType)), true
}
if delay, ok := m.getGlobalExactMetricTimeThreshold(metricType); ok {
return delay, "legacy.metricTimeThresholds.all." + strings.ToLower(strings.TrimSpace(metricType)), true
}
base, hasTypeSpecific := m.getBaseTimeThreshold(resourceType)
if hasTypeSpecific {
// The legacy factory delay was five seconds for every metric. Upgrade
// only that unchanged default for noisy gauges; a deliberate type-level
// value remains authoritative, as does any metric-specific override above.
if base == defaultLegacyMetricDelaySeconds {
if delay := defaultMetricStabilityDelay(metricType); delay > 0 {
return delay, "factory.metricStability." + strings.ToLower(strings.TrimSpace(metricType)), true
}
}
return base, "legacy.timeThresholds." + strings.ToLower(strings.TrimSpace(resourceType)), true
}
if delay, ok := m.getGlobalMetricTimeThreshold(metricType); ok {
return delay, "legacy.metricTimeThresholds.all." + strings.ToLower(strings.TrimSpace(metricType)), true
}
if base != 0 {
if base == defaultLegacyMetricDelaySeconds {
if delay := defaultMetricStabilityDelay(metricType); delay > 0 {
return delay, "factory.metricStability." + strings.ToLower(strings.TrimSpace(metricType)), true
}
}
return base, "legacy.timeThresholds.all", true
}
return 0, "", false
}
// getGlobalExactMetricTimeThreshold returns only an explicit all.<metric>
// override. Broad all.default/all.* fallbacks retain their legacy position
// after resource-type delays and therefore cannot accidentally erase those
// more specific policies.
func (m *Manager) getGlobalExactMetricTimeThreshold(metricType string) (int, bool) {
if len(m.config.MetricTimeThresholds) == 0 {
return 0, false
}
perType, ok := m.config.MetricTimeThresholds["all"]
if !ok || len(perType) == 0 {
return 0, false
}
metricKey := strings.ToLower(strings.TrimSpace(metricType))
if metricKey == "" {
return 0, false
}
delay, ok := perType[metricKey]
return delay, ok
}
// getMetricTimeThreshold returns a metric-specific delay if configured at the resource-type level.
func (m *Manager) getMetricTimeThreshold(resourceType, metricType string) (int, bool) {
if len(m.config.MetricTimeThresholds) == 0 {
@@ -278,6 +351,7 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
effectiveIntent := m.resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType))
var intent *reducer.DiscreteIntent
delaySeconds := 0
stability := metricStabilityPolicy{}
if effectiveIntent.Explicit {
decision := m.evaluateIntentNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType), trackingKey, time.Now(), conditionActive, BackupIntentContext{})
if decision.StateChanged {
@@ -292,6 +366,9 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
} else if conditionActive {
delaySeconds = m.getTimeThreshold(resourceID, resourceType, metricType)
}
if !effectiveIntent.Explicit {
stability = metricStabilityFor(metricType, effectiveIntent.GraceSeconds)
}
events := m.core.ApplyMetric(reducer.MetricSignal{
ResourceID: resourceID,
@@ -302,10 +379,12 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
RuntimeTickValid: true,
ObservedAt: m.policyNow(),
}, reducer.MetricRule{
Trigger: threshold.Trigger,
Clear: threshold.Clear,
DelaySeconds: delaySeconds,
Intent: intent,
Trigger: threshold.Trigger,
Clear: threshold.Clear,
DelaySeconds: delaySeconds,
CriticalBypassesDelay: stability.CriticalBypassesDelay,
RecoveryDelaySeconds: stability.RecoveryDelaySeconds,
Intent: intent,
})
primary := reducer.EventType("")
if len(events) > 0 {
+58 -4
View File
@@ -65,6 +65,15 @@ type MetricRule struct {
// an explicit Intent gate is supplied — the manager's explicit intent
// policies replace the legacy time-threshold delay.
DelaySeconds int
// CriticalBypassesDelay lets urgent evidence activate immediately while
// warning-level evidence still has to survive the stability window. It is
// intentionally ignored when an explicit Intent gate is present: explicit
// operator policy remains authoritative.
CriticalBypassesDelay bool
// RecoveryDelaySeconds is the continuous time at or below Clear required
// before a firing incident resolves. A value <= 0 preserves immediate
// hysteresis recovery.
RecoveryDelaySeconds int
// Intent is the resolved intent-policy context for this observation
// (metric.<name> signals); nil means no gate.
Intent *DiscreteIntent
@@ -108,9 +117,15 @@ type Incident struct {
// toward DiscreteRule.RecoveryConfirmations; reset by any matching
// observation. Unused by the metric family.
RecoveryCount int
Acknowledged bool
AckUser string
AckAt time.Time
// RecoverySince and its monotonic companions track a continuous healthy
// run for metric rules with a recovery stability window.
RecoverySince time.Time
RecoveryElapsed time.Duration
RecoveryTicksSupplied bool
LastRecoveryRuntimeTick time.Duration
Acknowledged bool
AckUser string
AckAt time.Time
// Backup-run bookkeeping for the intent gate's backup-offline deferral
// sub-policy (confirmation family only).
BackupActive bool
@@ -390,6 +405,7 @@ func (s *State) ApplyMetric(signal MetricSignal, rule MetricRule) []Event {
// recomputes level on every tick, so severity can demote as well as
// escalate while firing.
if incident != nil && incident.State == StateFiring {
resetMetricRecoveryRun(incident)
previous := incident.Severity
incident.Severity = severityFor(signal.Value, rule, signal.Metric)
incident.LastValue = signal.Value
@@ -444,7 +460,9 @@ func (s *State) ApplyMetric(signal MetricSignal, rule MetricRule) []Event {
return pendingResult()
}
} else if rule.DelaySeconds > 0 {
if signal.ObservedAt.Sub(incident.PendingSince) < time.Duration(rule.DelaySeconds)*time.Second {
critical := severityFor(signal.Value, rule, signal.Metric) == SeverityCritical
if !(critical && rule.CriticalBypassesDelay) &&
signal.ObservedAt.Sub(incident.PendingSince) < time.Duration(rule.DelaySeconds)*time.Second {
return pendingResult()
}
}
@@ -473,6 +491,31 @@ func (s *State) ApplyMetric(signal MetricSignal, rule MetricRule) []Event {
clear = rule.Trigger
}
if signal.Value <= clear {
if rule.RecoveryDelaySeconds > 0 {
if incident.RecoverySince.IsZero() {
incident.RecoverySince = signal.ObservedAt
incident.RecoveryTicksSupplied = signal.RuntimeTickValid
incident.LastRecoveryRuntimeTick = signal.RuntimeTick
incident.RecoveryElapsed = 0
incident.LastObservedAt = signal.ObservedAt
return nil
}
if signal.RuntimeTickValid {
if incident.RecoveryTicksSupplied && signal.RuntimeTick >= incident.LastRecoveryRuntimeTick {
incident.RecoveryElapsed += signal.RuntimeTick - incident.LastRecoveryRuntimeTick
}
incident.RecoveryTicksSupplied = true
incident.LastRecoveryRuntimeTick = signal.RuntimeTick
}
elapsed := signal.ObservedAt.Sub(incident.RecoverySince)
if incident.RecoveryTicksSupplied {
elapsed = incident.RecoveryElapsed
}
incident.LastObservedAt = signal.ObservedAt
if elapsed < time.Duration(rule.RecoveryDelaySeconds)*time.Second {
return nil
}
}
severity := incident.Severity
delete(s.incidents, key)
s.markAckInactive(key, signal.ObservedAt)
@@ -482,9 +525,20 @@ func (s *State) ApplyMetric(signal MetricSignal, rule MetricRule) []Event {
// Hysteresis hold: between clear and trigger the incident stays firing.
// The manager does not refresh value or last-seen in this band; the
// reducer mirrors that so parity diffs stay exact.
resetMetricRecoveryRun(incident)
return nil
}
func resetMetricRecoveryRun(incident *Incident) {
if incident == nil {
return
}
incident.RecoverySince = time.Time{}
incident.RecoveryElapsed = 0
incident.RecoveryTicksSupplied = false
incident.LastRecoveryRuntimeTick = 0
}
// SeedFiringIncident installs a firing incident directly, bypassing the
// transition functions. Shadow mode uses it to align the reducer with
// pre-existing manager state (persisted-alert restore, divergence resync);
+46
View File
@@ -132,6 +132,52 @@ func TestDelayFiresWithPendingStartAndDipResets(t *testing.T) {
}
}
func TestCriticalCanBypassWarningStabilityDelay(t *testing.T) {
state := NewState()
rule := MetricRule{Trigger: 80, Clear: 75, DelaySeconds: 300, CriticalBypassesDelay: true}
events := state.ApplyMetric(signalAt(95, 0), rule)
if len(events) != 1 || events[0].Type != EventFired || events[0].Severity != SeverityCritical {
t.Fatalf("events = %+v, want immediate critical fire", events)
}
}
func TestCriticalDoesNotBypassExplicitIntent(t *testing.T) {
state := NewState()
rule := MetricRule{
Trigger: 80, Clear: 75, CriticalBypassesDelay: true,
Intent: &DiscreteIntent{Explicit: true, GraceSeconds: 300},
}
events := state.ApplyMetric(signalAt(95, 0), rule)
if len(events) != 1 || events[0].Type != EventPending {
t.Fatalf("events = %+v, want explicit intent to keep critical pending", events)
}
}
func TestMetricRecoveryRequiresContinuousHealthyWindow(t *testing.T) {
state := NewState()
rule := MetricRule{Trigger: 80, Clear: 75, RecoveryDelaySeconds: 120}
state.ApplyMetric(signalAt(90, 0), rule)
if events := state.ApplyMetric(signalAt(70, time.Minute), rule); len(events) != 0 {
t.Fatalf("first healthy observation emitted %+v", events)
}
if events := state.ApplyMetric(signalAt(78, 90*time.Second), rule); len(events) != 0 {
t.Fatalf("hysteresis-band interruption emitted %+v", events)
}
if events := state.ApplyMetric(signalAt(70, 2*time.Minute), rule); len(events) != 0 {
t.Fatalf("restarted healthy run emitted %+v", events)
}
if events := state.ApplyMetric(signalAt(70, 3*time.Minute+59*time.Second), rule); len(events) != 0 {
t.Fatalf("recovery fired before continuous window elapsed: %+v", events)
}
events := state.ApplyMetric(signalAt(70, 4*time.Minute), rule)
if len(events) != 1 || events[0].Type != EventResolved {
t.Fatalf("events = %+v, want recovery after continuous healthy window", events)
}
}
func TestClearFallsBackToTriggerWhenUnset(t *testing.T) {
state := NewState()
rule := MetricRule{Trigger: 80, Clear: 0}
@@ -938,6 +938,32 @@ func TestGuestFilesystemThresholdOverrideOwnsMountEvaluation(t *testing.T) {
}
}
func TestGuestFilesystemEvidenceSuppressesDuplicateAggregateAlert(t *testing.T) {
m := newTestManager(t)
guestID := BuildGuestKey("pve1", "node1", 109)
m.mu.Lock()
m.config.TimeThresholds = map[string]int{}
m.config.GuestDefaults = ThresholdConfig{Disk: &HysteresisThreshold{Trigger: 80, Clear: 75}}
m.mu.Unlock()
m.CheckGuest(models.VM{
ID: guestID, VMID: 109, Name: "pulse-dev", Node: "node1", Instance: "pve1", Status: "running",
Disk: models.Disk{Usage: 96.5},
Disks: []models.Disk{{Mountpoint: "/", Device: "/dev/vda1", Usage: 96.9, Total: 100, Used: 97, Free: 3}},
}, "pve1")
aggregateID := canonicalMetricStateID(guestID, "disk")
filesystemID := canonicalMetricStateID(guestID+"-disk-dev-vda1", "disk")
m.mu.RLock()
defer m.mu.RUnlock()
if _, exists := testLookupActiveAlert(t, m, aggregateID); exists {
t.Fatalf("aggregate alert %q duplicated filesystem evidence", aggregateID)
}
if _, exists := testLookupActiveAlert(t, m, filesystemID); !exists {
t.Fatalf("actionable filesystem alert %q was not created", filesystemID)
}
}
// Regression: checkMetric stores canonical-identity alerts under the
// canonical state key, so hysteresis resolution must not remove only the
// unregistered legacy "<resourceID>-<metric>" ID.
+53
View File
@@ -85,6 +85,59 @@ func TestGetTimeThresholdMetricOverrides(t *testing.T) {
}
}
func TestGetTimeThresholdUsesTrustStableNoisyGaugeDefaults(t *testing.T) {
manager := NewManagerWithDataDir(t.TempDir(), WithoutPersistedAlertRestore())
t.Cleanup(manager.Stop)
if got := manager.getTimeThreshold("vm-resource", "VM", "memory"); got != defaultNoisyGaugeStabilitySeconds {
t.Fatalf("default memory stability = %d, want %d", got, defaultNoisyGaugeStabilitySeconds)
}
if got := manager.getTimeThreshold("node-1", "Node", "temperature"); got != defaultNoisyGaugeStabilitySeconds {
t.Fatalf("default temperature stability = %d, want %d", got, defaultNoisyGaugeStabilitySeconds)
}
if got := manager.getTimeThreshold("vm-resource", "VM", "cpu"); got != defaultLegacyMetricDelaySeconds {
t.Fatalf("default CPU delay = %d, want %d", got, defaultLegacyMetricDelaySeconds)
}
manager.mu.Lock()
manager.config.MetricTimeThresholds = map[string]map[string]int{
"all": {"memory": 0},
}
manager.mu.Unlock()
if got := manager.getTimeThreshold("vm-resource", "VM", "memory"); got != 0 {
t.Fatalf("explicit global memory override = %d, want 0", got)
}
}
func TestCheckMetricNoisyWarningWaitsButCriticalFiresImmediately(t *testing.T) {
manager := NewManagerWithDataDir(t.TempDir(), WithoutPersistedAlertRestore())
t.Cleanup(manager.Stop)
threshold := &HysteresisThreshold{Trigger: 85, Clear: 80}
canonicalState := buildCanonicalStateID("vm-resource", "metric-threshold:memory")
manager.checkMetric("vm-resource", "database", "node-1", "pve-1", "VM", "memory", 90, threshold, nil)
manager.mu.RLock()
_, warningActive := manager.getActiveAlertNoLock(canonicalState)
manager.mu.RUnlock()
if warningActive {
t.Fatal("brief warning-level memory evidence opened an alert")
}
manager.checkMetric("vm-resource", "database", "node-1", "pve-1", "VM", "memory", 96, threshold, nil)
manager.mu.RLock()
alert, criticalActive := manager.getActiveAlertNoLock(canonicalState)
manager.mu.RUnlock()
if !criticalActive {
t.Fatal("critical memory evidence did not bypass the factory stability delay")
}
if alert.Level != AlertLevelCritical {
t.Fatalf("critical memory alert level = %q, want %q", alert.Level, AlertLevelCritical)
}
}
func TestCheckMetricUsesPendingStartTime(t *testing.T) {
manager := NewManager()
+24
View File
@@ -1559,3 +1559,27 @@ func TestUnifiedStorageForecastKeepsCanonicalMetricSpecAcrossPlatforms(t *testin
t.Fatalf("CanonicalSpecID = %q, want canonical usage spec", alert.CanonicalSpecID)
}
}
func TestUnifiedMetricNoisyWarningWaitsButCriticalFiresImmediately(t *testing.T) {
m := newTestManager(t)
input := &UnifiedResourceInput{
ID: "vm-noisy-memory",
Type: "vm",
Name: "database",
Memory: &UnifiedResourceMetric{Percent: 90},
}
alertID := canonicalMetricStateID(input.ID, "memory")
m.CheckUnifiedResource(input)
assertAlertMissing(t, m, alertID)
input.Memory.Percent = 96
m.CheckUnifiedResource(input)
alert := testRequireActiveAlert(t, m, alertID)
if alert.Level != AlertLevelCritical {
t.Fatalf("critical memory alert level = %q, want %q", alert.Level, AlertLevelCritical)
}
if got := alert.Metadata["stabilityWindowSeconds"]; got != defaultNoisyGaugeStabilitySeconds {
t.Fatalf("stabilityWindowSeconds = %v, want %d", got, defaultNoisyGaugeStabilitySeconds)
}
}
+14
View File
@@ -129,6 +129,13 @@ var containerPathPrefixes = []string{
"/mnt/.ix-apps/docker/", // TrueNAS SCALE Docker overlay mounts
}
// transientMountPatterns identify OS-managed mounts whose lifetime follows an
// application launch rather than operator-managed storage. They must not
// participate in capacity inventory or alerting.
var transientMountPatterns = []string{
"/AppTranslocation/", // macOS Gatekeeper translocated application image
}
// ShouldSkipFilesystem determines if a filesystem should be excluded from disk
// usage aggregation. It checks for read-only filesystems, virtual/pseudo filesystems,
// network mounts, and special system mountpoints. Returns skip=true if the filesystem
@@ -168,6 +175,13 @@ func ShouldSkipFilesystem(fsType, mountpoint string, totalBytes, usedBytes uint6
reasons = append(reasons, "special-mountpoint")
}
for _, pattern := range transientMountPatterns {
if strings.Contains(mountpoint, pattern) {
reasons = append(reasons, "transient-mountpoint")
break
}
}
// Windows System Reserved partition
if mountpoint == "System Reserved" || strings.Contains(mountpoint, "System Reserved") {
reasons = append(reasons, "special-mountpoint")
+1
View File
@@ -204,6 +204,7 @@ func TestShouldSkipFilesystem(t *testing.T) {
{"macos data companion volume", "apfs", "/System/Volumes/Data", 245107195904, 213573853184, true},
{"macos preboot companion volume", "apfs", "/System/Volumes/Preboot", 245107195904, 213573853184, true},
{"macos simulator runtime volume", "apfs", "/Library/Developer/CoreSimulator/Volumes/iOS_23E254a", 18058575872, 17593331712, true},
{"macos app translocation volume", "apfs", "/private/var/folders/xy/random/T/AppTranslocation/2A9C/d/Pulse.app", 245107195904, 221809991680, true},
// Regular filesystems that should NOT be skipped
{"ext4 root", "ext4", "/", 100 * 1024 * 1024 * 1024, 50 * 1024 * 1024 * 1024, false},