mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 02:25:56 +00:00
fix(alerts): stabilize noisy lifecycle signals
This commit is contained in:
@@ -2390,3 +2390,28 @@ against a managed local backend. The browser phase must exercise active-alert
|
||||
snooze, delivery diagnosis, restart persistence, unsnooze, incident evidence,
|
||||
history migration, virtualization, and clear tombstones without replacing the
|
||||
backend APIs with route mocks.
|
||||
|
||||
### Noisy gauges use one trust-stable lifecycle
|
||||
|
||||
Memory, temperature, and disk-temperature warnings use one continuous
|
||||
stability window on both lifecycle edges: an unchanged factory configuration
|
||||
requires five minutes above the trigger to open and five minutes at or below
|
||||
the recovery threshold to close. A return to the hysteresis band or a renewed
|
||||
breach resets the recovery run. Critical metric evidence may bypass only this
|
||||
factory or legacy activation delay; an explicit alert-intent policy remains
|
||||
authoritative. A metric-specific delay, including zero, remains the simple
|
||||
operator override and becomes the matching recovery window for these gauges.
|
||||
|
||||
Recovery timing uses the reducer's monotonic runtime evidence when supplied,
|
||||
so wall-clock changes cannot prematurely close an incident. Missing or unknown
|
||||
observations do not constitute healthy evidence and therefore cannot advance a
|
||||
recovery run. Restarts conservatively restart an in-progress recovery window
|
||||
rather than manufacturing elapsed health.
|
||||
|
||||
For guest disk capacity, valid per-filesystem evidence owns the alert identity
|
||||
and suppresses the less actionable guest-wide aggregate. This guarantees one
|
||||
incident for one full filesystem while retaining the aggregate only when no
|
||||
usable filesystem evidence exists. OS-managed transient mounts, including
|
||||
macOS Gatekeeper App Translocation volumes, are removed at the shared
|
||||
filesystem collection boundary before they can enter capacity inventory or
|
||||
alert evaluation.
|
||||
|
||||
@@ -170,6 +170,7 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
|
||||
var intent *reducer.DiscreteIntent
|
||||
delaySeconds := 0
|
||||
effectiveIntent := m.resolveEffectiveIntentPolicyNoLock(spec.ResourceID, resourceType, MetricAlertIntentSignal(metricType))
|
||||
stability := metricStabilityPolicy{}
|
||||
if effectiveIntent.Explicit {
|
||||
decision := m.evaluateIntentNoLock(spec.ResourceID, resourceType, MetricAlertIntentSignal(metricType), trackingKey, observedAt, triggered, BackupIntentContext{})
|
||||
if decision.StateChanged {
|
||||
@@ -184,6 +185,9 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
|
||||
} else if triggered {
|
||||
delaySeconds = m.getTimeThreshold(spec.ResourceID, resourceType, metricType)
|
||||
}
|
||||
if !effectiveIntent.Explicit {
|
||||
stability = metricStabilityFor(metricType, effectiveIntent.GraceSeconds)
|
||||
}
|
||||
|
||||
evidence := alertspecs.AlertEvidence{
|
||||
ObservedAt: observedAt,
|
||||
@@ -210,12 +214,14 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
|
||||
RuntimeTickValid: true,
|
||||
ObservedAt: observedAt,
|
||||
}, reducer.MetricRule{
|
||||
Trigger: spec.MetricThreshold.Trigger,
|
||||
Clear: clearThreshold,
|
||||
Critical: spec.MetricThreshold.Critical,
|
||||
CriticalDisabled: spec.MetricThreshold.Critical == nil,
|
||||
DelaySeconds: delaySeconds,
|
||||
Intent: intent,
|
||||
Trigger: spec.MetricThreshold.Trigger,
|
||||
Clear: clearThreshold,
|
||||
Critical: spec.MetricThreshold.Critical,
|
||||
CriticalDisabled: spec.MetricThreshold.Critical == nil,
|
||||
DelaySeconds: delaySeconds,
|
||||
CriticalBypassesDelay: stability.CriticalBypassesDelay,
|
||||
RecoveryDelaySeconds: stability.RecoveryDelaySeconds,
|
||||
Intent: intent,
|
||||
})
|
||||
primary := reducer.EventType("")
|
||||
if len(events) > 0 {
|
||||
@@ -243,6 +249,10 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
|
||||
"clearThreshold": metricClearThreshold(spec.MetricThreshold, threshold),
|
||||
"monitorOnly": monitorOnly,
|
||||
}
|
||||
if stability.RecoveryDelaySeconds > 0 {
|
||||
alertMetadata["stabilityWindowSeconds"] = stability.RecoveryDelaySeconds
|
||||
alertMetadata["criticalBypassesStability"] = stability.CriticalBypassesDelay
|
||||
}
|
||||
if unit != "" {
|
||||
alertMetadata["unit"] = unit
|
||||
}
|
||||
|
||||
@@ -261,40 +261,22 @@ func (m *Manager) CheckGuest(guest any, instanceName string) {
|
||||
diskReadMetric, diskWriteMetric, networkInMetric, networkOutMetric := guestIORateMetrics(snapshot)
|
||||
diskMetric := &UnifiedResourceMetric{Percent: diskUsage}
|
||||
if len(disks) > 0 {
|
||||
var aggregateTotal, aggregateUsed int64
|
||||
hasDedicatedDiskOverride := false
|
||||
seenAggregateDiskKeys := make(map[string]struct{})
|
||||
for idx, disk := range disks {
|
||||
if disk.Total <= 0 || disk.Usage < 0 {
|
||||
continue
|
||||
}
|
||||
label, keySource, sanitizedKey := guestDiskIdentity(disk, idx)
|
||||
_, _, sanitizedKey := guestDiskIdentity(disk, idx)
|
||||
if _, exists := seenAggregateDiskKeys[sanitizedKey]; exists {
|
||||
continue
|
||||
}
|
||||
seenAggregateDiskKeys[sanitizedKey] = struct{}{}
|
||||
|
||||
m.mu.RLock()
|
||||
diskOverride, hasDiskOverride := lookupGuestDiskOverride(m.config.Overrides, guest, guestID, keySource)
|
||||
m.mu.RUnlock()
|
||||
if hasDiskOverride && (diskOverride.Disabled || diskOverride.Disk != nil) {
|
||||
hasDedicatedDiskOverride = true
|
||||
log.Debug().
|
||||
Str("guest", name).
|
||||
Str("diskLabel", label).
|
||||
Msg("Excluding guest filesystem with a dedicated override from aggregate disk alert")
|
||||
continue
|
||||
}
|
||||
aggregateTotal += disk.Total
|
||||
aggregateUsed += disk.Used
|
||||
}
|
||||
if hasDedicatedDiskOverride {
|
||||
if aggregateTotal > 0 {
|
||||
diskMetric.Percent = float64(aggregateUsed) / float64(aggregateTotal) * 100
|
||||
} else {
|
||||
diskMetric = nil
|
||||
m.clearAlert(canonicalMetricStateID(guestID, "disk"))
|
||||
}
|
||||
if len(seenAggregateDiskKeys) > 0 {
|
||||
// Filesystem evidence is more actionable than the guest-wide rollup.
|
||||
// Evaluating both creates two incidents for the same full filesystem.
|
||||
diskMetric = nil
|
||||
m.clearAlert(canonicalMetricStateID(guestID, "disk"))
|
||||
}
|
||||
}
|
||||
m.evaluateUnifiedMetrics(&UnifiedResourceInput{
|
||||
|
||||
@@ -11,6 +11,44 @@ import (
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
type metricStabilityPolicy struct {
|
||||
CriticalBypassesDelay bool
|
||||
RecoveryDelaySeconds int
|
||||
}
|
||||
|
||||
const (
|
||||
defaultLegacyMetricDelaySeconds = 5
|
||||
defaultNoisyGaugeStabilitySeconds = 5 * 60
|
||||
)
|
||||
|
||||
func defaultMetricStabilityDelay(metricType string) int {
|
||||
switch strings.ToLower(strings.TrimSpace(metricType)) {
|
||||
case "memory", "temperature", "disktemperature", "disk_temperature":
|
||||
return defaultNoisyGaugeStabilitySeconds
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// metricStabilityFor keeps noisy gauges trustworthy without adding a second
|
||||
// operator-facing policy surface. The configured alert delay is the one
|
||||
// stability window on both lifecycle edges; critical evidence bypasses only
|
||||
// the legacy/default activation delay, never an explicit intent policy.
|
||||
func metricStabilityFor(metricType string, delaySeconds int) metricStabilityPolicy {
|
||||
if delaySeconds <= 0 {
|
||||
return metricStabilityPolicy{}
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(metricType)) {
|
||||
case "memory", "temperature", "disktemperature", "disk_temperature":
|
||||
return metricStabilityPolicy{
|
||||
CriticalBypassesDelay: true,
|
||||
RecoveryDelaySeconds: delaySeconds,
|
||||
}
|
||||
default:
|
||||
return metricStabilityPolicy{}
|
||||
}
|
||||
}
|
||||
|
||||
func isMetricThresholdAlertType(metricType string) bool {
|
||||
switch metricType {
|
||||
case "cpu", "memory", "disk", "diskRead", "diskWrite", "networkIn", "networkOut", "temperature", "usage":
|
||||
@@ -66,23 +104,58 @@ func (m *Manager) getLegacyTimeThresholdWithSource(resourceType, metricType stri
|
||||
if delay, ok := m.getMetricTimeThreshold(resourceType, metricType); ok {
|
||||
return delay, "legacy.metricTimeThresholds." + strings.ToLower(strings.TrimSpace(resourceType)) + "." + strings.ToLower(strings.TrimSpace(metricType)), true
|
||||
}
|
||||
|
||||
base, hasTypeSpecific := m.getBaseTimeThreshold(resourceType)
|
||||
|
||||
if !hasTypeSpecific {
|
||||
if delay, ok := m.getGlobalMetricTimeThreshold(metricType); ok {
|
||||
return delay, "legacy.metricTimeThresholds.all." + strings.ToLower(strings.TrimSpace(metricType)), true
|
||||
}
|
||||
if delay, ok := m.getGlobalExactMetricTimeThreshold(metricType); ok {
|
||||
return delay, "legacy.metricTimeThresholds.all." + strings.ToLower(strings.TrimSpace(metricType)), true
|
||||
}
|
||||
base, hasTypeSpecific := m.getBaseTimeThreshold(resourceType)
|
||||
if hasTypeSpecific {
|
||||
// The legacy factory delay was five seconds for every metric. Upgrade
|
||||
// only that unchanged default for noisy gauges; a deliberate type-level
|
||||
// value remains authoritative, as does any metric-specific override above.
|
||||
if base == defaultLegacyMetricDelaySeconds {
|
||||
if delay := defaultMetricStabilityDelay(metricType); delay > 0 {
|
||||
return delay, "factory.metricStability." + strings.ToLower(strings.TrimSpace(metricType)), true
|
||||
}
|
||||
}
|
||||
return base, "legacy.timeThresholds." + strings.ToLower(strings.TrimSpace(resourceType)), true
|
||||
}
|
||||
if delay, ok := m.getGlobalMetricTimeThreshold(metricType); ok {
|
||||
return delay, "legacy.metricTimeThresholds.all." + strings.ToLower(strings.TrimSpace(metricType)), true
|
||||
}
|
||||
if base != 0 {
|
||||
if base == defaultLegacyMetricDelaySeconds {
|
||||
if delay := defaultMetricStabilityDelay(metricType); delay > 0 {
|
||||
return delay, "factory.metricStability." + strings.ToLower(strings.TrimSpace(metricType)), true
|
||||
}
|
||||
}
|
||||
return base, "legacy.timeThresholds.all", true
|
||||
}
|
||||
return 0, "", false
|
||||
}
|
||||
|
||||
// getGlobalExactMetricTimeThreshold returns only an explicit all.<metric>
|
||||
// override. Broad all.default/all.* fallbacks retain their legacy position
|
||||
// after resource-type delays and therefore cannot accidentally erase those
|
||||
// more specific policies.
|
||||
func (m *Manager) getGlobalExactMetricTimeThreshold(metricType string) (int, bool) {
|
||||
if len(m.config.MetricTimeThresholds) == 0 {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
perType, ok := m.config.MetricTimeThresholds["all"]
|
||||
if !ok || len(perType) == 0 {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
metricKey := strings.ToLower(strings.TrimSpace(metricType))
|
||||
if metricKey == "" {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
delay, ok := perType[metricKey]
|
||||
return delay, ok
|
||||
}
|
||||
|
||||
// getMetricTimeThreshold returns a metric-specific delay if configured at the resource-type level.
|
||||
func (m *Manager) getMetricTimeThreshold(resourceType, metricType string) (int, bool) {
|
||||
if len(m.config.MetricTimeThresholds) == 0 {
|
||||
@@ -278,6 +351,7 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
|
||||
effectiveIntent := m.resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType))
|
||||
var intent *reducer.DiscreteIntent
|
||||
delaySeconds := 0
|
||||
stability := metricStabilityPolicy{}
|
||||
if effectiveIntent.Explicit {
|
||||
decision := m.evaluateIntentNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType), trackingKey, time.Now(), conditionActive, BackupIntentContext{})
|
||||
if decision.StateChanged {
|
||||
@@ -292,6 +366,9 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
|
||||
} else if conditionActive {
|
||||
delaySeconds = m.getTimeThreshold(resourceID, resourceType, metricType)
|
||||
}
|
||||
if !effectiveIntent.Explicit {
|
||||
stability = metricStabilityFor(metricType, effectiveIntent.GraceSeconds)
|
||||
}
|
||||
|
||||
events := m.core.ApplyMetric(reducer.MetricSignal{
|
||||
ResourceID: resourceID,
|
||||
@@ -302,10 +379,12 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
|
||||
RuntimeTickValid: true,
|
||||
ObservedAt: m.policyNow(),
|
||||
}, reducer.MetricRule{
|
||||
Trigger: threshold.Trigger,
|
||||
Clear: threshold.Clear,
|
||||
DelaySeconds: delaySeconds,
|
||||
Intent: intent,
|
||||
Trigger: threshold.Trigger,
|
||||
Clear: threshold.Clear,
|
||||
DelaySeconds: delaySeconds,
|
||||
CriticalBypassesDelay: stability.CriticalBypassesDelay,
|
||||
RecoveryDelaySeconds: stability.RecoveryDelaySeconds,
|
||||
Intent: intent,
|
||||
})
|
||||
primary := reducer.EventType("")
|
||||
if len(events) > 0 {
|
||||
|
||||
@@ -65,6 +65,15 @@ type MetricRule struct {
|
||||
// an explicit Intent gate is supplied — the manager's explicit intent
|
||||
// policies replace the legacy time-threshold delay.
|
||||
DelaySeconds int
|
||||
// CriticalBypassesDelay lets urgent evidence activate immediately while
|
||||
// warning-level evidence still has to survive the stability window. It is
|
||||
// intentionally ignored when an explicit Intent gate is present: explicit
|
||||
// operator policy remains authoritative.
|
||||
CriticalBypassesDelay bool
|
||||
// RecoveryDelaySeconds is the continuous time at or below Clear required
|
||||
// before a firing incident resolves. A value <= 0 preserves immediate
|
||||
// hysteresis recovery.
|
||||
RecoveryDelaySeconds int
|
||||
// Intent is the resolved intent-policy context for this observation
|
||||
// (metric.<name> signals); nil means no gate.
|
||||
Intent *DiscreteIntent
|
||||
@@ -108,9 +117,15 @@ type Incident struct {
|
||||
// toward DiscreteRule.RecoveryConfirmations; reset by any matching
|
||||
// observation. Unused by the metric family.
|
||||
RecoveryCount int
|
||||
Acknowledged bool
|
||||
AckUser string
|
||||
AckAt time.Time
|
||||
// RecoverySince and its monotonic companions track a continuous healthy
|
||||
// run for metric rules with a recovery stability window.
|
||||
RecoverySince time.Time
|
||||
RecoveryElapsed time.Duration
|
||||
RecoveryTicksSupplied bool
|
||||
LastRecoveryRuntimeTick time.Duration
|
||||
Acknowledged bool
|
||||
AckUser string
|
||||
AckAt time.Time
|
||||
// Backup-run bookkeeping for the intent gate's backup-offline deferral
|
||||
// sub-policy (confirmation family only).
|
||||
BackupActive bool
|
||||
@@ -390,6 +405,7 @@ func (s *State) ApplyMetric(signal MetricSignal, rule MetricRule) []Event {
|
||||
// recomputes level on every tick, so severity can demote as well as
|
||||
// escalate while firing.
|
||||
if incident != nil && incident.State == StateFiring {
|
||||
resetMetricRecoveryRun(incident)
|
||||
previous := incident.Severity
|
||||
incident.Severity = severityFor(signal.Value, rule, signal.Metric)
|
||||
incident.LastValue = signal.Value
|
||||
@@ -444,7 +460,9 @@ func (s *State) ApplyMetric(signal MetricSignal, rule MetricRule) []Event {
|
||||
return pendingResult()
|
||||
}
|
||||
} else if rule.DelaySeconds > 0 {
|
||||
if signal.ObservedAt.Sub(incident.PendingSince) < time.Duration(rule.DelaySeconds)*time.Second {
|
||||
critical := severityFor(signal.Value, rule, signal.Metric) == SeverityCritical
|
||||
if !(critical && rule.CriticalBypassesDelay) &&
|
||||
signal.ObservedAt.Sub(incident.PendingSince) < time.Duration(rule.DelaySeconds)*time.Second {
|
||||
return pendingResult()
|
||||
}
|
||||
}
|
||||
@@ -473,6 +491,31 @@ func (s *State) ApplyMetric(signal MetricSignal, rule MetricRule) []Event {
|
||||
clear = rule.Trigger
|
||||
}
|
||||
if signal.Value <= clear {
|
||||
if rule.RecoveryDelaySeconds > 0 {
|
||||
if incident.RecoverySince.IsZero() {
|
||||
incident.RecoverySince = signal.ObservedAt
|
||||
incident.RecoveryTicksSupplied = signal.RuntimeTickValid
|
||||
incident.LastRecoveryRuntimeTick = signal.RuntimeTick
|
||||
incident.RecoveryElapsed = 0
|
||||
incident.LastObservedAt = signal.ObservedAt
|
||||
return nil
|
||||
}
|
||||
if signal.RuntimeTickValid {
|
||||
if incident.RecoveryTicksSupplied && signal.RuntimeTick >= incident.LastRecoveryRuntimeTick {
|
||||
incident.RecoveryElapsed += signal.RuntimeTick - incident.LastRecoveryRuntimeTick
|
||||
}
|
||||
incident.RecoveryTicksSupplied = true
|
||||
incident.LastRecoveryRuntimeTick = signal.RuntimeTick
|
||||
}
|
||||
elapsed := signal.ObservedAt.Sub(incident.RecoverySince)
|
||||
if incident.RecoveryTicksSupplied {
|
||||
elapsed = incident.RecoveryElapsed
|
||||
}
|
||||
incident.LastObservedAt = signal.ObservedAt
|
||||
if elapsed < time.Duration(rule.RecoveryDelaySeconds)*time.Second {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
severity := incident.Severity
|
||||
delete(s.incidents, key)
|
||||
s.markAckInactive(key, signal.ObservedAt)
|
||||
@@ -482,9 +525,20 @@ func (s *State) ApplyMetric(signal MetricSignal, rule MetricRule) []Event {
|
||||
// Hysteresis hold: between clear and trigger the incident stays firing.
|
||||
// The manager does not refresh value or last-seen in this band; the
|
||||
// reducer mirrors that so parity diffs stay exact.
|
||||
resetMetricRecoveryRun(incident)
|
||||
return nil
|
||||
}
|
||||
|
||||
func resetMetricRecoveryRun(incident *Incident) {
|
||||
if incident == nil {
|
||||
return
|
||||
}
|
||||
incident.RecoverySince = time.Time{}
|
||||
incident.RecoveryElapsed = 0
|
||||
incident.RecoveryTicksSupplied = false
|
||||
incident.LastRecoveryRuntimeTick = 0
|
||||
}
|
||||
|
||||
// SeedFiringIncident installs a firing incident directly, bypassing the
|
||||
// transition functions. Shadow mode uses it to align the reducer with
|
||||
// pre-existing manager state (persisted-alert restore, divergence resync);
|
||||
|
||||
@@ -132,6 +132,52 @@ func TestDelayFiresWithPendingStartAndDipResets(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCriticalCanBypassWarningStabilityDelay(t *testing.T) {
|
||||
state := NewState()
|
||||
rule := MetricRule{Trigger: 80, Clear: 75, DelaySeconds: 300, CriticalBypassesDelay: true}
|
||||
|
||||
events := state.ApplyMetric(signalAt(95, 0), rule)
|
||||
if len(events) != 1 || events[0].Type != EventFired || events[0].Severity != SeverityCritical {
|
||||
t.Fatalf("events = %+v, want immediate critical fire", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCriticalDoesNotBypassExplicitIntent(t *testing.T) {
|
||||
state := NewState()
|
||||
rule := MetricRule{
|
||||
Trigger: 80, Clear: 75, CriticalBypassesDelay: true,
|
||||
Intent: &DiscreteIntent{Explicit: true, GraceSeconds: 300},
|
||||
}
|
||||
|
||||
events := state.ApplyMetric(signalAt(95, 0), rule)
|
||||
if len(events) != 1 || events[0].Type != EventPending {
|
||||
t.Fatalf("events = %+v, want explicit intent to keep critical pending", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricRecoveryRequiresContinuousHealthyWindow(t *testing.T) {
|
||||
state := NewState()
|
||||
rule := MetricRule{Trigger: 80, Clear: 75, RecoveryDelaySeconds: 120}
|
||||
state.ApplyMetric(signalAt(90, 0), rule)
|
||||
|
||||
if events := state.ApplyMetric(signalAt(70, time.Minute), rule); len(events) != 0 {
|
||||
t.Fatalf("first healthy observation emitted %+v", events)
|
||||
}
|
||||
if events := state.ApplyMetric(signalAt(78, 90*time.Second), rule); len(events) != 0 {
|
||||
t.Fatalf("hysteresis-band interruption emitted %+v", events)
|
||||
}
|
||||
if events := state.ApplyMetric(signalAt(70, 2*time.Minute), rule); len(events) != 0 {
|
||||
t.Fatalf("restarted healthy run emitted %+v", events)
|
||||
}
|
||||
if events := state.ApplyMetric(signalAt(70, 3*time.Minute+59*time.Second), rule); len(events) != 0 {
|
||||
t.Fatalf("recovery fired before continuous window elapsed: %+v", events)
|
||||
}
|
||||
events := state.ApplyMetric(signalAt(70, 4*time.Minute), rule)
|
||||
if len(events) != 1 || events[0].Type != EventResolved {
|
||||
t.Fatalf("events = %+v, want recovery after continuous healthy window", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClearFallsBackToTriggerWhenUnset(t *testing.T) {
|
||||
state := NewState()
|
||||
rule := MetricRule{Trigger: 80, Clear: 0}
|
||||
|
||||
@@ -938,6 +938,32 @@ func TestGuestFilesystemThresholdOverrideOwnsMountEvaluation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGuestFilesystemEvidenceSuppressesDuplicateAggregateAlert(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
guestID := BuildGuestKey("pve1", "node1", 109)
|
||||
m.mu.Lock()
|
||||
m.config.TimeThresholds = map[string]int{}
|
||||
m.config.GuestDefaults = ThresholdConfig{Disk: &HysteresisThreshold{Trigger: 80, Clear: 75}}
|
||||
m.mu.Unlock()
|
||||
|
||||
m.CheckGuest(models.VM{
|
||||
ID: guestID, VMID: 109, Name: "pulse-dev", Node: "node1", Instance: "pve1", Status: "running",
|
||||
Disk: models.Disk{Usage: 96.5},
|
||||
Disks: []models.Disk{{Mountpoint: "/", Device: "/dev/vda1", Usage: 96.9, Total: 100, Used: 97, Free: 3}},
|
||||
}, "pve1")
|
||||
|
||||
aggregateID := canonicalMetricStateID(guestID, "disk")
|
||||
filesystemID := canonicalMetricStateID(guestID+"-disk-dev-vda1", "disk")
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
if _, exists := testLookupActiveAlert(t, m, aggregateID); exists {
|
||||
t.Fatalf("aggregate alert %q duplicated filesystem evidence", aggregateID)
|
||||
}
|
||||
if _, exists := testLookupActiveAlert(t, m, filesystemID); !exists {
|
||||
t.Fatalf("actionable filesystem alert %q was not created", filesystemID)
|
||||
}
|
||||
}
|
||||
|
||||
// Regression: checkMetric stores canonical-identity alerts under the
|
||||
// canonical state key, so hysteresis resolution must not remove only the
|
||||
// unregistered legacy "<resourceID>-<metric>" ID.
|
||||
|
||||
@@ -85,6 +85,59 @@ func TestGetTimeThresholdMetricOverrides(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetTimeThresholdUsesTrustStableNoisyGaugeDefaults(t *testing.T) {
|
||||
manager := NewManagerWithDataDir(t.TempDir(), WithoutPersistedAlertRestore())
|
||||
|
||||
t.Cleanup(manager.Stop)
|
||||
|
||||
if got := manager.getTimeThreshold("vm-resource", "VM", "memory"); got != defaultNoisyGaugeStabilitySeconds {
|
||||
t.Fatalf("default memory stability = %d, want %d", got, defaultNoisyGaugeStabilitySeconds)
|
||||
}
|
||||
if got := manager.getTimeThreshold("node-1", "Node", "temperature"); got != defaultNoisyGaugeStabilitySeconds {
|
||||
t.Fatalf("default temperature stability = %d, want %d", got, defaultNoisyGaugeStabilitySeconds)
|
||||
}
|
||||
if got := manager.getTimeThreshold("vm-resource", "VM", "cpu"); got != defaultLegacyMetricDelaySeconds {
|
||||
t.Fatalf("default CPU delay = %d, want %d", got, defaultLegacyMetricDelaySeconds)
|
||||
}
|
||||
|
||||
manager.mu.Lock()
|
||||
manager.config.MetricTimeThresholds = map[string]map[string]int{
|
||||
"all": {"memory": 0},
|
||||
}
|
||||
manager.mu.Unlock()
|
||||
|
||||
if got := manager.getTimeThreshold("vm-resource", "VM", "memory"); got != 0 {
|
||||
t.Fatalf("explicit global memory override = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckMetricNoisyWarningWaitsButCriticalFiresImmediately(t *testing.T) {
|
||||
manager := NewManagerWithDataDir(t.TempDir(), WithoutPersistedAlertRestore())
|
||||
t.Cleanup(manager.Stop)
|
||||
|
||||
threshold := &HysteresisThreshold{Trigger: 85, Clear: 80}
|
||||
canonicalState := buildCanonicalStateID("vm-resource", "metric-threshold:memory")
|
||||
|
||||
manager.checkMetric("vm-resource", "database", "node-1", "pve-1", "VM", "memory", 90, threshold, nil)
|
||||
manager.mu.RLock()
|
||||
_, warningActive := manager.getActiveAlertNoLock(canonicalState)
|
||||
manager.mu.RUnlock()
|
||||
if warningActive {
|
||||
t.Fatal("brief warning-level memory evidence opened an alert")
|
||||
}
|
||||
|
||||
manager.checkMetric("vm-resource", "database", "node-1", "pve-1", "VM", "memory", 96, threshold, nil)
|
||||
manager.mu.RLock()
|
||||
alert, criticalActive := manager.getActiveAlertNoLock(canonicalState)
|
||||
manager.mu.RUnlock()
|
||||
if !criticalActive {
|
||||
t.Fatal("critical memory evidence did not bypass the factory stability delay")
|
||||
}
|
||||
if alert.Level != AlertLevelCritical {
|
||||
t.Fatalf("critical memory alert level = %q, want %q", alert.Level, AlertLevelCritical)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckMetricUsesPendingStartTime(t *testing.T) {
|
||||
manager := NewManager()
|
||||
|
||||
|
||||
@@ -1559,3 +1559,27 @@ func TestUnifiedStorageForecastKeepsCanonicalMetricSpecAcrossPlatforms(t *testin
|
||||
t.Fatalf("CanonicalSpecID = %q, want canonical usage spec", alert.CanonicalSpecID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnifiedMetricNoisyWarningWaitsButCriticalFiresImmediately(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
input := &UnifiedResourceInput{
|
||||
ID: "vm-noisy-memory",
|
||||
Type: "vm",
|
||||
Name: "database",
|
||||
Memory: &UnifiedResourceMetric{Percent: 90},
|
||||
}
|
||||
alertID := canonicalMetricStateID(input.ID, "memory")
|
||||
|
||||
m.CheckUnifiedResource(input)
|
||||
assertAlertMissing(t, m, alertID)
|
||||
|
||||
input.Memory.Percent = 96
|
||||
m.CheckUnifiedResource(input)
|
||||
alert := testRequireActiveAlert(t, m, alertID)
|
||||
if alert.Level != AlertLevelCritical {
|
||||
t.Fatalf("critical memory alert level = %q, want %q", alert.Level, AlertLevelCritical)
|
||||
}
|
||||
if got := alert.Metadata["stabilityWindowSeconds"]; got != defaultNoisyGaugeStabilitySeconds {
|
||||
t.Fatalf("stabilityWindowSeconds = %v, want %d", got, defaultNoisyGaugeStabilitySeconds)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -129,6 +129,13 @@ var containerPathPrefixes = []string{
|
||||
"/mnt/.ix-apps/docker/", // TrueNAS SCALE Docker overlay mounts
|
||||
}
|
||||
|
||||
// transientMountPatterns identify OS-managed mounts whose lifetime follows an
|
||||
// application launch rather than operator-managed storage. They must not
|
||||
// participate in capacity inventory or alerting.
|
||||
var transientMountPatterns = []string{
|
||||
"/AppTranslocation/", // macOS Gatekeeper translocated application image
|
||||
}
|
||||
|
||||
// ShouldSkipFilesystem determines if a filesystem should be excluded from disk
|
||||
// usage aggregation. It checks for read-only filesystems, virtual/pseudo filesystems,
|
||||
// network mounts, and special system mountpoints. Returns skip=true if the filesystem
|
||||
@@ -168,6 +175,13 @@ func ShouldSkipFilesystem(fsType, mountpoint string, totalBytes, usedBytes uint6
|
||||
reasons = append(reasons, "special-mountpoint")
|
||||
}
|
||||
|
||||
for _, pattern := range transientMountPatterns {
|
||||
if strings.Contains(mountpoint, pattern) {
|
||||
reasons = append(reasons, "transient-mountpoint")
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Windows System Reserved partition
|
||||
if mountpoint == "System Reserved" || strings.Contains(mountpoint, "System Reserved") {
|
||||
reasons = append(reasons, "special-mountpoint")
|
||||
|
||||
@@ -204,6 +204,7 @@ func TestShouldSkipFilesystem(t *testing.T) {
|
||||
{"macos data companion volume", "apfs", "/System/Volumes/Data", 245107195904, 213573853184, true},
|
||||
{"macos preboot companion volume", "apfs", "/System/Volumes/Preboot", 245107195904, 213573853184, true},
|
||||
{"macos simulator runtime volume", "apfs", "/Library/Developer/CoreSimulator/Volumes/iOS_23E254a", 18058575872, 17593331712, true},
|
||||
{"macos app translocation volume", "apfs", "/private/var/folders/xy/random/T/AppTranslocation/2A9C/d/Pulse.app", 245107195904, 221809991680, true},
|
||||
|
||||
// Regular filesystems that should NOT be skipped
|
||||
{"ext4 root", "ext4", "/", 100 * 1024 * 1024 * 1024, 50 * 1024 * 1024 * 1024, false},
|
||||
|
||||
Reference in New Issue
Block a user