Add predictive storage capacity alerts

This commit is contained in:
rcourtman
2026-08-27 20:29:17 +01:00
parent 1ccd973050
commit b0c1eca4cd
20 changed files with 1002 additions and 35 deletions
@@ -193,6 +193,22 @@ lifecycle when the agent heartbeat is unhealthy, and resolves only when a
fresh result from the currently assigned agent arrives. Notification delivery,
acknowledgement, history, and recovery reuse the normal alert pipeline.
Storage capacity forecasting is an alert-grade evidence boundary, not an AI
prose feature. It requires at least 24 hours of valid history, hourly median
normalization, a fresh terminal sample, agreement between full-window and
recent positive slopes, and confidence of at least 0.80 before it can fire.
Raw poll count alone must never manufacture confidence. Forecast risk opens
only when projected exhaustion is within seven days (critical within one day)
and recovers only after the projection moves beyond fourteen days or trusted
evidence proves growth stopped. Missing, shallow, stale, or low-confidence
history is unknown rather than recovery for an already-active forecast.
Predictive and static usage policy share `metric-threshold:usage` as one
canonical occurrence: a forecast that later crosses the percentage threshold
keeps its start time, acknowledgement, history, and timeline instead of
emitting a forecast recovery plus a second capacity alert. The same contract
applies to Proxmox, Ceph, TrueNAS pools/datasets, and vSphere datastores through
their existing platform thresholds and disable policy.
Active-alert restore is opt-out at construction. `NewManagerWithDataDir` accepts
`ManagerOption` values, and `WithoutPersistedAlertRestore` starts the manager
with an empty active-alert set instead of reading `active-alerts.json`. Mock
@@ -252,6 +268,7 @@ default construction path still restores.
45. `internal/alerts/docker.go`
46. `internal/alerts/pbs.go`
47. `internal/alerts/storage.go`
47a. `internal/alerts/capacity_forecast.go`
48. `internal/alerts/node.go`
49. `internal/alerts/host.go`
50. `internal/alerts/backup_snapshot.go`
@@ -420,6 +420,18 @@ lifecycle, and treats a fresh result from that same assignment as recovery.
Assignment trackers are removed with their targets and reset when agent identity
changes.
Storage capacity forecasting consumes monitoring-owned percentage history
through `internal/monitoring/storage_capacity_forecast.go`. The bridge combines
the durable SQLite series with the in-memory tail and current observation, then
caches the alert-owned trend result for a bounded interval. Durable history is
required in the read path so restart cannot erase a previously earned
confidence floor. Canonical storage resources must resolve their metrics target
before trend evaluation, keeping TrueNAS and vSphere forecast identity aligned
with the series written by `syncUnifiedStorageMetrics`; Proxmox and Ceph retain
their source-native storage history IDs. Monitoring supplies evidence only and
must not choose forecast horizons, severity, hysteresis, notification routing,
or lifecycle identity.
Monitoring ingest keeps mock mode hermetic. The unified read path already
substitutes the mock snapshot wholesale, so anything that runs after that
substitution has to be suppressed explicitly rather than assumed hidden. Server
@@ -510,6 +522,7 @@ cleanup so readers cannot retain orphaned runtime or alert projections.
50. `internal/models/ceph_cluster_identity.go`
51. `internal/truenas/types.go`
52. `internal/monitoring/monitor_alert_sync.go`
52a. `internal/monitoring/storage_capacity_forecast.go`
53. `internal/monitoring/platform_poller_shared.go`
54. `internal/monitoring/monitor_backups.go`
55. `internal/monitoring/resource_stale_thresholds.go`
+24
View File
@@ -20878,3 +20878,27 @@ func TestCleanupRemovesOnlyStaleSMARTCounterSnapshots(t *testing.T) {
t.Fatal("current SMART counter snapshot was removed")
}
}
func TestCheckStorageForecastUsesCanonicalUsageAlertIdentity(t *testing.T) {
m := newTestManager(t)
m.mu.Lock()
m.config.TimeThresholds = map[string]int{}
m.config.StorageDefault = HysteresisThreshold{Trigger: 80, Clear: 70}
m.mu.Unlock()
storage := models.Storage{ID: "forecast-proof", Name: "archive", Status: "active", Usage: 72}
trend := CapacityTrendObservation{
Ready: true, Reason: "increasing", ObservedAt: time.Now(), DailyChange: 5,
Confidence: 0.99, SampleCount: 300, BucketCount: 48, CoverageSpan: 48 * time.Hour,
}
m.CheckStorageWithCapacityTrend(storage, trend)
m.CheckStorageWithCapacityTrend(storage, trend)
alert := testRequireActiveAlert(t, m, canonicalMetricStateID(storage.ID, "usage"))
if alert.CanonicalSpecID != canonicalMetricSpecID(storage.ID, "usage") {
t.Fatalf("CanonicalSpecID = %q, want canonical usage spec", alert.CanonicalSpecID)
}
if got := alert.Metadata[capacityAlertOriginKey]; got != capacityAlertOriginForecast {
t.Fatalf("capacity origin = %v, want forecast", got)
}
}
+31 -19
View File
@@ -13,23 +13,27 @@ import (
)
type canonicalLifecycleAlertParams struct {
Spec alertspecs.ResourceAlertSpec
Evidence alertspecs.AlertEvidence
IntentSignal string
PolicyDisabledNoLock func() bool
AlertID string
AlertType string
ResourceID string
ResourceName string
Node string
Instance string
Message string
Metadata map[string]interface{}
AddToRecent bool
AddToHistory bool
RateLimit bool
DispatchAsync bool
IntentBackup BackupIntentContext
Spec alertspecs.ResourceAlertSpec
Evidence alertspecs.AlertEvidence
IntentSignal string
PolicyDisabledNoLock func() bool
AlertID string
AlertType string
ResourceID string
ResourceName string
Node string
Instance string
Message string
Value float64
Threshold float64
Metadata map[string]interface{}
AddToRecent bool
AddToHistory bool
RateLimit bool
DispatchAsync bool
NotifyOnSeverityChange bool
AddToHistoryOnSeverityChange bool
IntentBackup BackupIntentContext
}
type canonicalStatefulAlertParams struct {
@@ -460,8 +464,8 @@ func (m *Manager) evaluateCanonicalLifecycleAlert(params canonicalLifecycleAlert
Node: params.Node,
Instance: params.Instance,
Message: params.Message,
Value: 0,
Threshold: 0,
Value: params.Value,
Threshold: params.Threshold,
StartTime: incident.StartedAt,
LastSeen: params.Evidence.ObservedAt,
Metadata: cloneMetadata(params.Metadata),
@@ -501,6 +505,14 @@ func (m *Manager) evaluateCanonicalLifecycleAlert(params canonicalLifecycleAlert
if existing != nil {
if primary == reducer.EventSeverityChanged {
result.Transition = transition(alertspecs.EvaluationTransitionSeverityChanged, alertspecs.AlertStateFiring, alertspecs.AlertStateFiring)
if params.AddToHistoryOnSeverityChange {
m.historyManager.AddAlertTransition(*alert)
}
if params.NotifyOnSeverityChange {
if !params.RateLimit || m.checkRateLimit(trackingKey) {
m.dispatchAlert(alert, params.DispatchAsync)
}
}
}
return result, true
}
+5
View File
@@ -327,6 +327,11 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
if existingAlert.Metadata == nil {
existingAlert.Metadata = map[string]interface{}{}
}
if opts != nil {
for _, key := range opts.RemoveMetadata {
delete(existingAlert.Metadata, key)
}
}
for k, v := range alertMetadata {
existingAlert.Metadata[k] = v
}
+424
View File
@@ -0,0 +1,424 @@
package alerts
import (
"fmt"
"math"
"sort"
"strconv"
"time"
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rs/zerolog/log"
)
const (
capacityForecastLookback = 7 * 24 * time.Hour
capacityForecastBucketWidth = time.Hour
capacityForecastMinimumSpan = 24 * time.Hour
capacityForecastFreshness = 2 * time.Hour
capacityForecastMinimumBuckets = 12
capacityForecastMinimumRate = 0.1
capacityForecastMinConfidence = 0.80
capacityForecastWarningHorizon = 7 * 24 * time.Hour
capacityForecastCriticalWindow = 24 * time.Hour
capacityForecastRecoveryWindow = 14 * 24 * time.Hour
)
// CapacityMetricPoint is one percentage-utilization observation used to
// estimate when a capacity-backed resource will fill. Callers may provide raw
// samples at any cadence; the estimator normalizes them into hourly medians so
// a fast poller cannot manufacture confidence by repeating nearly identical
// observations.
type CapacityMetricPoint struct {
Timestamp time.Time
Value float64
}
// CapacityTrendObservation is detector evidence, not an alert decision.
// Ready means the evidence coverage is sufficient to say whether a trend is
// actionable. A non-ready observation must not be interpreted as recovery.
type CapacityTrendObservation struct {
Ready bool
Reason string
ObservedAt time.Time
DailyChange float64
Confidence float64
SampleCount int
BucketCount int
CoverageSpan time.Duration
}
type capacityBucketPoint struct {
timestamp time.Time
value float64
}
// EstimateCapacityTrend produces a conservative, time-aware capacity trend.
// Alert policy (warning/critical horizons, threshold coexistence, and
// hysteresis) remains in Manager; this function only establishes trustworthy
// trend evidence.
func EstimateCapacityTrend(points []CapacityMetricPoint, now time.Time) CapacityTrendObservation {
if now.IsZero() {
now = time.Now()
}
result := CapacityTrendObservation{Reason: "insufficient-history"}
if len(points) == 0 {
return result
}
cutoff := now.Add(-capacityForecastLookback)
buckets := make(map[int64][]float64)
validSamples := 0
latest := time.Time{}
for _, point := range points {
if point.Timestamp.IsZero() || point.Timestamp.Before(cutoff) || point.Timestamp.After(now.Add(5*time.Minute)) {
continue
}
if math.IsNaN(point.Value) || math.IsInf(point.Value, 0) || point.Value < 0 || point.Value > 100 {
continue
}
bucket := point.Timestamp.Unix() / int64(capacityForecastBucketWidth/time.Second)
buckets[bucket] = append(buckets[bucket], point.Value)
validSamples++
if point.Timestamp.After(latest) {
latest = point.Timestamp
}
}
result.SampleCount = validSamples
if validSamples == 0 || latest.IsZero() {
result.Reason = "no-valid-samples"
return result
}
result.ObservedAt = latest
if now.Sub(latest) > capacityForecastFreshness {
result.Reason = "stale-history"
return result
}
normalized := make([]capacityBucketPoint, 0, len(buckets))
for bucket, values := range buckets {
sort.Float64s(values)
median := values[len(values)/2]
if len(values)%2 == 0 {
median = (values[len(values)/2-1] + values[len(values)/2]) / 2
}
normalized = append(normalized, capacityBucketPoint{
timestamp: time.Unix(bucket*int64(capacityForecastBucketWidth/time.Second), 0),
value: median,
})
}
sort.Slice(normalized, func(i, j int) bool {
return normalized[i].timestamp.Before(normalized[j].timestamp)
})
result.BucketCount = len(normalized)
if len(normalized) < capacityForecastMinimumBuckets {
result.Reason = "insufficient-hourly-coverage"
return result
}
result.CoverageSpan = normalized[len(normalized)-1].timestamp.Sub(normalized[0].timestamp)
if result.CoverageSpan < capacityForecastMinimumSpan {
result.Reason = "history-window-too-short"
return result
}
overallSlope, overallR2 := capacityLinearRegression(normalized)
recentStart := len(normalized) / 2
if len(normalized)-recentStart < 8 {
recentStart = len(normalized) - 8
}
recentSlope, _ := capacityLinearRegression(normalized[recentStart:])
overallDaily := overallSlope * 24
recentDaily := recentSlope * 24
result.DailyChange = overallDaily
// A historic rise that has flattened or reversed is not an impending
// exhaustion signal. Requiring both windows to rise also rejects one-off
// capacity jumps caused by a resize or a telemetry discontinuity.
if overallDaily <= capacityForecastMinimumRate || recentDaily <= capacityForecastMinimumRate {
result.Ready = true
result.Reason = "not-increasing"
return result
}
spanFactor := math.Min(1, result.CoverageSpan.Hours()/48)
agreement := math.Min(overallDaily, recentDaily) / math.Max(overallDaily, recentDaily)
result.Confidence = clampCapacityConfidence(overallR2 * spanFactor * agreement)
result.Ready = true
if result.Confidence < capacityForecastMinConfidence {
result.Reason = "low-confidence"
return result
}
result.Reason = "increasing"
return result
}
func capacityLinearRegression(points []capacityBucketPoint) (slopePerHour, rSquared float64) {
if len(points) < 2 {
return 0, 0
}
start := points[0].timestamp
n := float64(len(points))
var sumX, sumY, sumXY, sumX2 float64
for _, point := range points {
x := point.timestamp.Sub(start).Hours()
sumX += x
sumY += point.value
sumXY += x * point.value
sumX2 += x * x
}
denominator := n*sumX2 - sumX*sumX
if denominator == 0 {
return 0, 0
}
slopePerHour = (n*sumXY - sumX*sumY) / denominator
intercept := (sumY - slopePerHour*sumX) / n
meanY := sumY / n
var residual, total float64
for _, point := range points {
x := point.timestamp.Sub(start).Hours()
predicted := intercept + slopePerHour*x
residual += math.Pow(point.value-predicted, 2)
total += math.Pow(point.value-meanY, 2)
}
if total == 0 {
return slopePerHour, 0
}
return slopePerHour, clampCapacityConfidence(1 - residual/total)
}
func clampCapacityConfidence(value float64) float64 {
if value < 0 {
return 0
}
if value > 1 {
return 1
}
return value
}
const (
capacityAlertOriginKey = "capacityAlertOrigin"
capacityAlertOriginThreshold = "threshold"
capacityAlertOriginForecast = "forecast"
)
var capacityForecastMetadataKeys = []string{
"forecastConfidence",
"forecastDailyChangePct",
"forecastDaysToFull",
"forecastObservedAt",
"forecastSampleCount",
"forecastBucketCount",
"forecastCoverageSeconds",
}
func (m *Manager) evaluateStorageCapacity(storage models.Storage, thresholds ThresholdConfig, trend CapacityTrendObservation) {
input := &UnifiedResourceInput{
ID: storage.ID,
Type: "storage",
Name: storage.Name,
Node: storage.Node,
Instance: storage.Instance,
Disk: &UnifiedResourceMetric{Percent: storage.Usage},
}
m.evaluateUnifiedCapacity(input, thresholds, trend, func() bool {
if !m.config.Enabled {
return true
}
allDisabled, _ := m.alertPolicyTypeSwitchesNoLock("storage")
return allDisabled || m.resolveStorageThresholdsNoLock(storage).Disabled
})
}
func (m *Manager) evaluateUnifiedCapacity(input *UnifiedResourceInput, thresholds ThresholdConfig, trend CapacityTrendObservation, policyDisabledNoLock func() bool) {
if input == nil || input.Disk == nil {
return
}
threshold := thresholds.Usage
if threshold == nil || threshold.Trigger <= 0 {
m.evaluateUnifiedMetrics(input, thresholds, nil)
return
}
current := input.DiskValue()
active, origin := m.activeStorageCapacityOrigin(input.ID)
staticTriggered := current >= threshold.Trigger
staticLatched := active && origin == capacityAlertOriginThreshold && threshold.Clear > 0 && current >= threshold.Clear
if staticTriggered || staticLatched {
m.evaluateUnifiedMetrics(input, thresholds, &metricOptions{
Metadata: map[string]interface{}{
capacityAlertOriginKey: capacityAlertOriginThreshold,
},
RemoveMetadata: capacityForecastMetadataKeys,
})
return
}
// Insufficient or low-confidence history is unknown, not recovery. Keep an
// already-firing forecast occurrence untouched until trustworthy evidence
// says the risk has receded; normal stale-alert cleanup remains the final
// bound if telemetry never becomes usable again.
recoveryFloor := threshold.Clear
if recoveryFloor <= 0 {
recoveryFloor = threshold.Trigger
}
if active && origin == capacityAlertOriginForecast && current >= recoveryFloor && (!trend.Ready || trend.Reason == "low-confidence") {
return
}
eta, hasETA := capacityTimeToFull(current, trend.DailyChange)
forecastTrusted := trend.Ready && trend.Confidence >= capacityForecastMinConfidence && hasETA
forecastTriggered := forecastTrusted && eta <= capacityForecastWarningHorizon
forecastLatched := active && origin == capacityAlertOriginForecast && forecastTrusted && eta <= capacityForecastRecoveryWindow
if forecastTriggered || forecastLatched {
m.evaluateCapacityForecast(input, thresholds, trend, eta, policyDisabledNoLock)
return
}
// No predictive risk remains. Run the ordinary metric evaluator so a
// forecast recovery and every existing static hysteresis rule retain the
// same canonical state, history, and notification behavior.
m.evaluateUnifiedMetrics(input, thresholds, &metricOptions{
Metadata: map[string]interface{}{
capacityAlertOriginKey: capacityAlertOriginThreshold,
},
RemoveMetadata: capacityForecastMetadataKeys,
})
}
func (m *Manager) activeStorageCapacityOrigin(resourceID string) (bool, string) {
trackingKey := canonicalMetricStateID(resourceID, "usage")
m.mu.RLock()
defer m.mu.RUnlock()
alert, exists := m.getActiveAlertNoLock(trackingKey)
if !exists || alert == nil {
return false, ""
}
if alert.Metadata == nil {
return true, capacityAlertOriginThreshold
}
origin, _ := alert.Metadata[capacityAlertOriginKey].(string)
if origin == "" {
origin = capacityAlertOriginThreshold
}
return true, origin
}
func capacityTimeToFull(current, dailyChange float64) (time.Duration, bool) {
if current < 0 || current >= 100 || dailyChange <= capacityForecastMinimumRate || math.IsNaN(dailyChange) || math.IsInf(dailyChange, 0) {
return 0, false
}
days := (100 - current) / dailyChange
if days <= 0 || math.IsNaN(days) || math.IsInf(days, 0) {
return 0, false
}
return time.Duration(days * float64(24*time.Hour)), true
}
func (m *Manager) evaluateCapacityForecast(input *UnifiedResourceInput, thresholds ThresholdConfig, trend CapacityTrendObservation, eta time.Duration, policyDisabledNoLock func() bool) {
resourceID := input.ID
specID := canonicalMetricSpecID(resourceID, "usage")
resourceType, ok := unifiedMetricResourceType(input.Type)
if !ok {
return
}
spec, err := buildCanonicalSeverityThresholdSpec(
specID,
resourceID,
input.Name,
resourceType,
"capacity-risk",
1,
2,
false,
)
if err != nil {
log.Warn().Err(err).Str("resourceID", input.ID).Msg("Skipping invalid storage capacity forecast spec")
return
}
spec.ConfirmationsRequired = 2
if err := spec.Validate(); err != nil {
log.Warn().Err(err).Str("resourceID", input.ID).Msg("Skipping invalid confirmed storage capacity forecast spec")
return
}
riskScore := 1.0
if eta <= capacityForecastCriticalWindow {
riskScore = 2
}
observedAt := m.policyNow()
daysToFull := eta.Hours() / 24
message := fmt.Sprintf(
"%s projected to fill in %s (%.1f%% used, +%.2f%%/day)",
unifiedAlertType(input.Type),
formatCapacityETA(eta),
input.DiskValue(),
trend.DailyChange,
)
attributes := map[string]string{
"capacity_alert_origin": capacityAlertOriginForecast,
"confidence": strconv.FormatFloat(trend.Confidence, 'f', 3, 64),
"current_usage_percent": strconv.FormatFloat(input.DiskValue(), 'f', 2, 64),
"daily_change_percent": strconv.FormatFloat(trend.DailyChange, 'f', 3, 64),
"forecast_days_to_full": strconv.FormatFloat(daysToFull, 'f', 2, 64),
"history_bucket_count": strconv.Itoa(trend.BucketCount),
"history_coverage_seconds": strconv.FormatInt(int64(trend.CoverageSpan/time.Second), 10),
}
metadata := map[string]interface{}{
"resourceType": input.Type,
"clearThreshold": thresholds.Usage.Clear,
capacityAlertOriginKey: capacityAlertOriginForecast,
"forecastConfidence": trend.Confidence,
"forecastDailyChangePct": trend.DailyChange,
"forecastDaysToFull": daysToFull,
"forecastObservedAt": trend.ObservedAt,
"forecastSampleCount": trend.SampleCount,
"forecastBucketCount": trend.BucketCount,
"forecastCoverageSeconds": int64(trend.CoverageSpan / time.Second),
}
_, _ = m.evaluateCanonicalLifecycleAlert(canonicalLifecycleAlertParams{
Spec: spec,
Evidence: alertspecs.AlertEvidence{
ObservedAt: observedAt,
Summary: message,
Attributes: attributes,
SeverityThreshold: &alertspecs.SeverityThresholdEvidence{
Metric: "capacity-risk",
Direction: alertspecs.ThresholdDirectionAbove,
Observed: riskScore,
},
},
IntentSignal: MetricAlertIntentSignal("usage"),
PolicyDisabledNoLock: policyDisabledNoLock,
AlertID: canonicalMetricStateID(resourceID, "usage"),
AlertType: "usage",
ResourceID: resourceID,
ResourceName: input.Name,
Node: input.Node,
Instance: input.Instance,
Message: message,
Value: input.DiskValue(),
Threshold: 100,
Metadata: metadata,
AddToRecent: true,
AddToHistory: true,
RateLimit: true,
NotifyOnSeverityChange: true,
AddToHistoryOnSeverityChange: true,
})
}
func formatCapacityETA(eta time.Duration) string {
if eta < 2*time.Hour {
return "about 1 hour"
}
if eta < 24*time.Hour {
return fmt.Sprintf("about %.0f hours", math.Ceil(eta.Hours()))
}
days := int(math.Ceil(eta.Hours() / 24))
if days == 1 {
return "about 1 day"
}
return fmt.Sprintf("about %d days", days)
}
+227
View File
@@ -0,0 +1,227 @@
package alerts
import (
"math"
"strings"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
)
func TestEstimateCapacityTrendRequiresCoverageAndRecentAgreement(t *testing.T) {
now := time.Date(2026, time.August, 27, 12, 0, 0, 0, time.UTC)
t.Run("trusts a clean multi-day fill trend", func(t *testing.T) {
points := make([]CapacityMetricPoint, 0, 73)
for hour := 72; hour >= 0; hour-- {
age := time.Duration(hour) * time.Hour
points = append(points, CapacityMetricPoint{
Timestamp: now.Add(-age),
Value: 55 + float64(72-hour)*0.08,
})
}
trend := EstimateCapacityTrend(points, now)
if !trend.Ready || trend.Reason != "increasing" {
t.Fatalf("trend = %+v, want trusted increasing evidence", trend)
}
if math.Abs(trend.DailyChange-1.92) > 0.02 {
t.Fatalf("DailyChange = %.3f, want about 1.92", trend.DailyChange)
}
if trend.Confidence < capacityForecastMinConfidence {
t.Fatalf("Confidence = %.3f, want >= %.2f", trend.Confidence, capacityForecastMinConfidence)
}
})
t.Run("rejects dense samples without a full-day span", func(t *testing.T) {
points := make([]CapacityMetricPoint, 0, 300)
for i := 0; i < 300; i++ {
points = append(points, CapacityMetricPoint{
Timestamp: now.Add(-time.Duration(300-i) * time.Minute),
Value: 50 + float64(i)*0.02,
})
}
trend := EstimateCapacityTrend(points, now)
if trend.Ready || trend.Reason != "insufficient-hourly-coverage" {
t.Fatalf("trend = %+v, want insufficient hourly coverage", trend)
}
})
t.Run("does not extrapolate a historic jump after growth stops", func(t *testing.T) {
points := make([]CapacityMetricPoint, 0, 73)
for hour := 72; hour >= 0; hour-- {
elapsed := 72 - hour
value := 50.0
if elapsed >= 24 {
value = 70
}
points = append(points, CapacityMetricPoint{Timestamp: now.Add(-time.Duration(hour) * time.Hour), Value: value})
}
trend := EstimateCapacityTrend(points, now)
if !trend.Ready || trend.Reason != "not-increasing" {
t.Fatalf("trend = %+v, want a ready non-increasing decision", trend)
}
})
}
func TestStorageForecastSharesLifecycleWithStaticUsageAlert(t *testing.T) {
m := newTestManager(t)
m.ClearActiveAlerts()
m.mu.Lock()
m.config.TimeThresholds = map[string]int{}
m.config.SuppressionWindow = 0
m.config.MinimumDelta = 0
m.config.ActivationState = ActivationActive
m.config.StorageDefault = HysteresisThreshold{Trigger: 80, Clear: 70}
m.mu.Unlock()
deliveries := 0
resolved := 0
m.SetAlertCallback(func(*Alert) { deliveries++ })
m.SetResolvedCallback(func(string) { resolved++ })
storage := models.Storage{
ID: "storage-forecast-1",
Name: "archive",
Node: "pve1",
Instance: "lab",
Status: "active",
Usage: 72,
}
trend := CapacityTrendObservation{
Ready: true,
Reason: "increasing",
ObservedAt: time.Now(),
DailyChange: 5,
Confidence: 0.98,
SampleCount: 400,
BucketCount: 72,
CoverageSpan: 72 * time.Hour,
}
// Forecast alerts require two independent evaluation cycles.
m.CheckStorageWithCapacityTrend(storage, trend)
if testHasActiveAlert(t, m, canonicalMetricStateID(storage.ID, "usage")) {
t.Fatal("forecast activated before its confirmation floor")
}
m.CheckStorageWithCapacityTrend(storage, trend)
alert := testRequireActiveAlert(t, m, canonicalMetricStateID(storage.ID, "usage"))
if alert.Type != "usage" || alert.Level != AlertLevelWarning {
t.Fatalf("forecast alert = type %q level %q, want usage warning", alert.Type, alert.Level)
}
if got := alert.Metadata[capacityAlertOriginKey]; got != capacityAlertOriginForecast {
t.Fatalf("capacity origin = %v, want forecast", got)
}
if !strings.Contains(alert.Message, "projected to fill") {
t.Fatalf("message = %q, want predictive explanation", alert.Message)
}
if deliveries != 1 {
t.Fatalf("deliveries = %d, want one forecast activation", deliveries)
}
start := alert.StartTime
if err := m.AcknowledgeAlert(alert.ID, "operator"); err != nil {
t.Fatalf("acknowledge forecast: %v", err)
}
// A real threshold breach upgrades the same occurrence. It must not emit a
// forecast recovery or a second unacknowledged activation.
storage.Usage = 91
m.CheckStorageWithCapacityTrend(storage, trend)
upgraded := testRequireActiveAlert(t, m, canonicalMetricStateID(storage.ID, "usage"))
if upgraded.StartTime != start {
t.Fatalf("occurrence start changed from %s to %s", start, upgraded.StartTime)
}
if !upgraded.Acknowledged || upgraded.AckUser != "operator" {
t.Fatalf("acknowledgement was not preserved: %+v", upgraded)
}
if upgraded.Level != AlertLevelCritical {
t.Fatalf("upgraded level = %q, want critical", upgraded.Level)
}
if got := upgraded.Metadata[capacityAlertOriginKey]; got != capacityAlertOriginThreshold {
t.Fatalf("capacity origin = %v, want threshold", got)
}
if _, exists := upgraded.Metadata["forecastDaysToFull"]; exists {
t.Fatal("stale forecast metadata survived the static threshold transition")
}
if deliveries != 1 {
t.Fatalf("deliveries = %d, want acknowledged transition not to page again", deliveries)
}
if resolved != 0 {
t.Fatalf("resolved callbacks = %d, forecast-to-threshold transition must stay one incident", resolved)
}
if got := len(m.GetActiveAlerts()); got != 1 {
t.Fatalf("active alerts = %d, want one canonical capacity incident", got)
}
}
func TestStorageForecastTreatsUncertainEvidenceAsUnknown(t *testing.T) {
m := newTestManager(t)
m.ClearActiveAlerts()
m.mu.Lock()
m.config.TimeThresholds = map[string]int{}
m.config.StorageDefault = HysteresisThreshold{Trigger: 80, Clear: 70}
m.mu.Unlock()
storage := models.Storage{ID: "storage-forecast-unknown", Name: "media", Status: "active", Usage: 70}
trusted := CapacityTrendObservation{
Ready: true, Reason: "increasing", ObservedAt: time.Now(), DailyChange: 6,
Confidence: 0.99, SampleCount: 300, BucketCount: 48, CoverageSpan: 48 * time.Hour,
}
m.CheckStorageWithCapacityTrend(storage, trusted)
m.CheckStorageWithCapacityTrend(storage, trusted)
before := testRequireActiveAlert(t, m, canonicalMetricStateID(storage.ID, "usage"))
uncertain := trusted
uncertain.Confidence = 0.3
uncertain.Reason = "low-confidence"
m.CheckStorageWithCapacityTrend(storage, uncertain)
after := testRequireActiveAlert(t, m, canonicalMetricStateID(storage.ID, "usage"))
if after.StartTime != before.StartTime {
t.Fatal("uncertain evidence replaced the active forecast occurrence")
}
recovered := trusted
recovered.DailyChange = 0
recovered.Confidence = 0
recovered.Reason = "not-increasing"
m.CheckStorageWithCapacityTrend(storage, recovered)
if testHasActiveAlert(t, m, canonicalMetricStateID(storage.ID, "usage")) {
t.Fatal("positive non-increasing evidence did not recover the forecast")
}
}
func TestUnifiedStorageForecastUsesPlatformPolicyAndCanonicalIdentity(t *testing.T) {
m := newTestManager(t)
m.ClearActiveAlerts()
usage := &HysteresisThreshold{Trigger: 85, Clear: 75}
m.mu.Lock()
m.config.TimeThresholds = map[string]int{}
m.config.TrueNASDefaults.Usage = usage
m.mu.Unlock()
input := &UnifiedResourceInput{
ID: "truenas:atlas/pool:tank",
Type: "truenas-pool",
Name: "tank",
Node: "atlas",
Instance: "TrueNAS",
Disk: &UnifiedResourceMetric{Percent: 70},
}
trend := CapacityTrendObservation{
Ready: true, Reason: "increasing", ObservedAt: time.Now(), DailyChange: 7,
Confidence: 0.99, SampleCount: 300, BucketCount: 48, CoverageSpan: 48 * time.Hour,
}
m.CheckUnifiedResourceWithCapacityTrend(input, trend)
m.CheckUnifiedResourceWithCapacityTrend(input, trend)
alert := testRequireActiveAlert(t, m, canonicalMetricStateID(input.ID, "usage"))
if alert.ResourceID != input.ID || alert.Type != "usage" {
t.Fatalf("alert identity = resource %q type %q", alert.ResourceID, alert.Type)
}
if got := alert.Metadata["resourceType"]; got != "truenas-pool" {
t.Fatalf("resourceType metadata = %v, want truenas-pool", got)
}
if got := alert.Metadata[capacityAlertOriginKey]; got != capacityAlertOriginForecast {
t.Fatalf("capacity origin = %v, want forecast", got)
}
}
+8 -2
View File
@@ -166,8 +166,9 @@ func (m *Manager) getGlobalMetricTimeThreshold(metricType string) (int, bool) {
// checkMetric checks a single metric against its threshold with hysteresis.
type metricOptions struct {
Metadata map[string]interface{}
Message string
Metadata map[string]interface{}
RemoveMetadata []string
Message string
// MonitorOnly suppresses external notifications while still tracking the alert.
MonitorOnly bool
}
@@ -333,6 +334,11 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
if existingAlert.Metadata == nil {
existingAlert.Metadata = map[string]interface{}{}
}
if opts != nil {
for _, key := range opts.RemoveMetadata {
delete(existingAlert.Metadata, key)
}
}
existingAlert.Metadata["resourceType"] = resourceType
existingAlert.Metadata["clearThreshold"] = threshold.Clear
existingAlert.Metadata["monitorOnly"] = monitorOnly
+7 -8
View File
@@ -15,6 +15,12 @@ import (
// CheckStorage checks storage against thresholds
func (m *Manager) CheckStorage(storage models.Storage) {
m.CheckStorageWithCapacityTrend(storage, CapacityTrendObservation{})
}
// CheckStorageWithCapacityTrend evaluates static storage policy and optional
// predictive evidence as one canonical capacity lifecycle.
func (m *Manager) CheckStorageWithCapacityTrend(storage models.Storage, trend CapacityTrendObservation) {
m.mu.RLock()
if !m.config.Enabled {
m.mu.RUnlock()
@@ -97,14 +103,7 @@ func (m *Manager) CheckStorage(storage models.Storage) {
// Check usage if storage is online - checkMetric will skip if threshold is nil or <= 0
if storage.Status != "offline" && storage.Status != "unavailable" && storage.Usage > 0 {
m.evaluateUnifiedMetrics(&UnifiedResourceInput{
ID: storage.ID,
Type: "storage",
Name: storage.Name,
Node: storage.Node,
Instance: storage.Instance,
Disk: &UnifiedResourceMetric{Percent: storage.Usage},
}, thresholds, nil)
m.evaluateStorageCapacity(storage, thresholds, trend)
}
// Check ZFS pool status if this is ZFS storage
+21 -1
View File
@@ -462,6 +462,12 @@ func unifiedMetricResourceType(typeKey string) (unifiedresources.ResourceType, b
}
func (m *Manager) CheckUnifiedResourceMetrics(resources []unifiedresources.Resource) {
m.CheckUnifiedResourceMetricsWithCapacityTrends(resources, nil)
}
// CheckUnifiedResourceMetricsWithCapacityTrends evaluates canonical resources
// with optional alert-grade capacity evidence keyed by canonical resource ID.
func (m *Manager) CheckUnifiedResourceMetricsWithCapacityTrends(resources []unifiedresources.Resource, capacityTrends map[string]CapacityTrendObservation) {
if m == nil {
return
}
@@ -470,7 +476,7 @@ func (m *Manager) CheckUnifiedResourceMetrics(resources []unifiedresources.Resou
if !ok {
continue
}
m.CheckUnifiedResource(input)
m.CheckUnifiedResourceWithCapacityTrend(input, capacityTrends[input.ID])
}
}
@@ -730,6 +736,13 @@ func (i *UnifiedResourceInput) TemperatureValue() float64 {
// available metric. Discrete event alerts (offline, RAID, backup age, etc.)
// are NOT evaluated here — they remain in the typed Check* methods.
func (m *Manager) CheckUnifiedResource(input *UnifiedResourceInput) {
m.CheckUnifiedResourceWithCapacityTrend(input, CapacityTrendObservation{})
}
// CheckUnifiedResourceWithCapacityTrend keeps predictive and static capacity
// evaluation on the same canonical lifecycle for every admitted storage
// platform.
func (m *Manager) CheckUnifiedResourceWithCapacityTrend(input *UnifiedResourceInput, trend CapacityTrendObservation) {
if input == nil {
return
}
@@ -757,5 +770,12 @@ func (m *Manager) CheckUnifiedResource(input *UnifiedResourceInput) {
Str("resourceType", unifiedAlertType(input.Type)).
Msg("Evaluating unified resource metrics")
if unifiedStorageUsageResourceType(input.Type) && input.Disk != nil {
m.evaluateUnifiedCapacity(input, thresholds, trend, func() bool {
return !m.config.Enabled || m.unifiedPlatformAlertsDisabledNoLock(input.Type) || m.resolveResourceThresholds(input.Type, input.ID).Disabled
})
return
}
m.evaluateUnifiedMetrics(input, thresholds, nil)
}
+22
View File
@@ -1537,3 +1537,25 @@ func TestUnifiedResourceInputExcludesDockerAppContainers(t *testing.T) {
t.Fatalf("truenas app containers must stay unified-eval owned, got %+v ok=%v", input, ok)
}
}
func TestUnifiedStorageForecastKeepsCanonicalMetricSpecAcrossPlatforms(t *testing.T) {
m := newTestManager(t)
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
input := &UnifiedResourceInput{
ID: "vmware:lab/datastore:archive",
Type: "vmware-datastore",
Name: "archive",
Disk: &UnifiedResourceMetric{Percent: 70},
}
trend := CapacityTrendObservation{
Ready: true, Reason: "increasing", ObservedAt: time.Now(), DailyChange: 7,
Confidence: 0.99, SampleCount: 300, BucketCount: 48, CoverageSpan: 48 * time.Hour,
}
m.CheckUnifiedResourceWithCapacityTrend(input, trend)
m.CheckUnifiedResourceWithCapacityTrend(input, trend)
alert := testRequireActiveAlert(t, m, canonicalMetricStateID(input.ID, "usage"))
if alert.CanonicalSpecID != canonicalMetricSpecID(input.ID, "usage") {
t.Fatalf("CanonicalSpecID = %q, want canonical usage spec", alert.CanonicalSpecID)
}
}
@@ -494,7 +494,7 @@ func TestUnifiedResourceAlertSyncEvaluatesMetricsBeforeIncidents(t *testing.T) {
}
source := string(data)
metricIndex := strings.Index(source, "CheckUnifiedResourceMetrics(resources)")
metricIndex := strings.Index(source, "CheckUnifiedResourceMetricsWithCapacityTrends(resources")
incidentIndex := strings.Index(source, "SyncUnifiedResourceIncidents(resources)")
if metricIndex < 0 {
t.Fatalf("monitor alert sync must run unified resource metric evaluation")
+1 -1
View File
@@ -93,7 +93,7 @@ func (m *Monitor) checkCephPoolStorage(cluster models.CephCluster) {
}
}
if m.alertManager != nil {
m.alertManager.CheckStorage(storage)
m.alertManager.CheckStorageWithCapacityTrend(storage, m.storageCapacityTrend(storage, timestamp))
}
}
}
+3
View File
@@ -53,6 +53,9 @@ type MetricsHistory struct {
diskMetrics map[string]*DiskMetrics // key: disk metrics resource ID
maxDataPoints int
retentionTime time.Duration
capacityForecastMu sync.Mutex
capacityForecastCache map[string]storageCapacityForecastCacheEntry
}
// NewMetricsHistory creates a new metrics history tracker
@@ -3,6 +3,8 @@ package monitoring
import (
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
)
func TestNewMetricsHistory(t *testing.T) {
@@ -1395,3 +1397,27 @@ func TestCleanupMetricsReturnsNilForExpiredData(t *testing.T) {
t.Errorf("cleanupMetrics should return nil for fully expired data, got slice with len=%d cap=%d", len(result), cap(result))
}
}
func TestMetricsHistorySuppliesAlertGradeStorageCapacityTrend(t *testing.T) {
now := time.Now().UTC().Truncate(time.Hour)
history := NewMetricsHistory(1000, 7*24*time.Hour)
for hour := 72; hour >= 0; hour-- {
history.AddStorageMetric(
"forecast-history-proof",
"usage",
60+float64(72-hour)*0.08,
now.Add(-time.Duration(hour)*time.Hour),
)
}
monitor := &Monitor{metricsHistory: history}
trend := monitor.storageCapacityTrend(models.Storage{
ID: "forecast-history-proof", Name: "archive", Status: "active", Usage: 65.76,
}, now)
if !trend.Ready || trend.Reason != "increasing" {
t.Fatalf("trend = %+v, want trusted increasing evidence", trend)
}
if trend.Confidence < 0.8 {
t.Fatalf("confidence = %.3f, want alert-grade evidence", trend.Confidence)
}
}
+2 -1
View File
@@ -2,6 +2,7 @@ package monitoring
import (
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/logging"
@@ -132,7 +133,7 @@ func (m *Monitor) syncUnifiedResourceAlertsToState(resources []unifiedresources.
m.migrateAvailabilityLinksToCanonicalIDs(resources)
m.alertManager.CheckUnifiedResourceMetrics(resources)
m.alertManager.CheckUnifiedResourceMetricsWithCapacityTrends(resources, m.unifiedStorageCapacityTrends(resources, time.Now()))
m.alertManager.SyncUnifiedResourceIncidents(resources)
m.syncAlertsToState()
}
+1 -1
View File
@@ -549,7 +549,7 @@ func (m *Monitor) checkMockAlerts() {
Str("name", storage.Name).
Float64("usage", storage.Usage).
Msg("Checking storage for alerts")
m.alertManager.CheckStorage(storage)
m.alertManager.CheckStorageWithCapacityTrend(storage, m.storageCapacityTrend(storage, time.Now()))
}
// Check alerts for PBS instances
@@ -710,7 +710,7 @@ func (m *Monitor) pollStorageWithNodes(ctx context.Context, instanceName string,
}
if m.alertManager != nil {
m.alertManager.CheckStorage(storage)
m.alertManager.CheckStorageWithCapacityTrend(storage, m.storageCapacityTrend(storage, time.Now()))
}
}
@@ -0,0 +1,115 @@
package monitoring
import (
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rs/zerolog/log"
)
const (
storageCapacityForecastLookback = 7 * 24 * time.Hour
storageCapacityForecastRefresh = 15 * time.Minute
)
type storageCapacityForecastCacheEntry struct {
trend alerts.CapacityTrendObservation
expiresAt time.Time
}
// storageCapacityTrend bridges monitoring-owned history into alert-owned
// predictive policy. SQLite history is included so a process restart does not
// erase confidence; the in-memory tail and current observation cover buffered
// writes and installations where durable metrics are unavailable.
func (m *Monitor) storageCapacityTrend(storage models.Storage, now time.Time) alerts.CapacityTrendObservation {
if m == nil || storage.ID == "" || storage.Usage <= 0 {
return alerts.CapacityTrendObservation{Reason: "invalid-current-usage"}
}
return m.storageCapacityTrendFor(storage.ID, storage.ID, storage.Usage, now)
}
func (m *Monitor) storageCapacityTrendFor(cacheID, historyID string, currentUsage float64, now time.Time) alerts.CapacityTrendObservation {
if m == nil || cacheID == "" || historyID == "" || currentUsage <= 0 {
return alerts.CapacityTrendObservation{Reason: "invalid-current-usage"}
}
if now.IsZero() {
now = time.Now()
}
if m.metricsHistory != nil {
m.metricsHistory.capacityForecastMu.Lock()
if cached, ok := m.metricsHistory.capacityForecastCache[cacheID]; ok && now.Before(cached.expiresAt) {
m.metricsHistory.capacityForecastMu.Unlock()
return cached.trend
}
m.metricsHistory.capacityForecastMu.Unlock()
}
points := make([]alerts.CapacityMetricPoint, 0, 256)
if m.metricsStore != nil {
stored, err := m.metricsStore.Query(
"storage",
historyID,
"usage",
now.Add(-storageCapacityForecastLookback),
now,
int64(time.Hour/time.Second),
)
if err != nil {
log.Debug().Err(err).Str("storage", cacheID).Msg("Persistent capacity history unavailable; using in-memory tail")
} else {
for _, point := range stored {
points = append(points, alerts.CapacityMetricPoint{Timestamp: point.Timestamp, Value: point.Value})
}
}
}
if m.metricsHistory != nil {
for _, point := range m.metricsHistory.GetAllStorageMetrics(historyID, storageCapacityForecastLookback)["usage"] {
points = append(points, alerts.CapacityMetricPoint{Timestamp: point.Timestamp, Value: point.Value})
}
}
points = append(points, alerts.CapacityMetricPoint{Timestamp: now, Value: currentUsage})
trend := alerts.EstimateCapacityTrend(points, now)
if m.metricsHistory != nil {
m.metricsHistory.capacityForecastMu.Lock()
if m.metricsHistory.capacityForecastCache == nil {
m.metricsHistory.capacityForecastCache = make(map[string]storageCapacityForecastCacheEntry)
}
m.metricsHistory.capacityForecastCache[cacheID] = storageCapacityForecastCacheEntry{
trend: trend,
expiresAt: now.Add(storageCapacityForecastRefresh),
}
m.metricsHistory.capacityForecastMu.Unlock()
}
return trend
}
func (m *Monitor) unifiedStorageCapacityTrends(resources []unifiedresources.Resource, now time.Time) map[string]alerts.CapacityTrendObservation {
trends := make(map[string]alerts.CapacityTrendObservation)
var resolver MetricsTargetResourceStore
if candidate, ok := m.resourceStore.(MetricsTargetResourceStore); ok {
resolver = candidate
}
for _, resource := range resources {
input, ok := alerts.UnifiedResourceInputFromResource(resource)
if !ok || input.Disk == nil {
continue
}
switch input.Type {
case "truenas-pool", "truenas-dataset", "vmware-datastore":
default:
continue
}
historyID := input.ID
if resolver != nil {
if target := resolver.MetricsTargetForResource(resource.ID); target != nil && target.ResourceType == "storage" && target.ResourceID != "" {
historyID = target.ResourceID
}
}
trends[input.ID] = m.storageCapacityTrendFor(input.ID, historyID, input.DiskValue(), now)
}
return trends
}
@@ -0,0 +1,53 @@
package monitoring
import (
"path/filepath"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
)
func TestStorageCapacityTrendUsesDurableHistoryAfterRestart(t *testing.T) {
now := time.Now().UTC().Truncate(time.Hour)
config := metrics.DefaultConfig(t.TempDir())
config.DBPath = filepath.Join(t.TempDir(), "capacity-metrics.db")
store, err := metrics.NewStore(config)
if err != nil {
t.Fatalf("new metrics store: %v", err)
}
t.Cleanup(func() { _ = store.Close() })
writes := make([]metrics.WriteMetric, 0, 72)
for hour := 72; hour >= 1; hour-- {
writes = append(writes, metrics.WriteMetric{
ResourceType: "storage",
ResourceID: "durable-storage",
MetricType: "usage",
Value: 60 + float64(72-hour)*0.08,
Timestamp: now.Add(-time.Duration(hour) * time.Hour),
Tier: metrics.TierHourly,
})
}
store.WriteBatchSync(writes)
// Deliberately omit MetricsHistory: this models the process immediately
// after restart, when the in-memory ring is empty but SQLite remains.
monitor := &Monitor{metricsStore: store}
trend := monitor.storageCapacityTrend(models.Storage{
ID: "durable-storage",
Name: "archive",
Status: "active",
Usage: 65.76,
}, now)
if !trend.Ready || trend.Reason != "increasing" {
t.Fatalf("trend = %+v, want trusted increasing evidence from durable history", trend)
}
if trend.CoverageSpan < 48*time.Hour {
t.Fatalf("coverage = %s, want at least 48h", trend.CoverageSpan)
}
if trend.Confidence < 0.8 {
t.Fatalf("confidence = %.3f, want alert-grade evidence", trend.Confidence)
}
}