pulse/internal/alerts/capacity_forecast.go

package alerts

import (
	"fmt"
	"math"
	"sort"
	"strconv"
	"time"

	alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rs/zerolog/log"
)

const (
	capacityForecastLookback       = 7 * 24 * time.Hour
	capacityForecastBucketWidth    = time.Hour
	capacityForecastMinimumSpan    = 24 * time.Hour
	capacityForecastFreshness      = 2 * time.Hour
	capacityForecastMinimumBuckets = 12
	capacityForecastMinimumRate    = 0.1
	capacityForecastMinConfidence  = 0.80
	capacityForecastWarningHorizon = 7 * 24 * time.Hour
	capacityForecastCriticalWindow = 24 * time.Hour
	capacityForecastRecoveryWindow = 14 * 24 * time.Hour
)

// CapacityMetricPoint is one percentage-utilization observation used to
// estimate when a capacity-backed resource will fill. Callers may provide raw
// samples at any cadence; the estimator normalizes them into hourly medians so
// a fast poller cannot manufacture confidence by repeating nearly identical
// observations.
type CapacityMetricPoint struct {
	Timestamp time.Time
	Value     float64
}

// CapacityTrendObservation is detector evidence, not an alert decision.
// Ready means the evidence coverage is sufficient to say whether a trend is
// actionable. A non-ready observation must not be interpreted as recovery.
type CapacityTrendObservation struct {
	Ready        bool
	Reason       string
	ObservedAt   time.Time
	DailyChange  float64
	Confidence   float64
	SampleCount  int
	BucketCount  int
	CoverageSpan time.Duration
}

type capacityBucketPoint struct {
	timestamp time.Time
	value     float64
}

// EstimateCapacityTrend produces a conservative, time-aware capacity trend.
// Alert policy (warning/critical horizons, threshold coexistence, and
// hysteresis) remains in Manager; this function only establishes trustworthy
// trend evidence.
func EstimateCapacityTrend(points []CapacityMetricPoint, now time.Time) CapacityTrendObservation {
	if now.IsZero() {
		now = time.Now()
	}
	result := CapacityTrendObservation{Reason: "insufficient-history"}
	if len(points) == 0 {
		return result
	}

	cutoff := now.Add(-capacityForecastLookback)
	buckets := make(map[int64][]float64)
	validSamples := 0
	latest := time.Time{}
	for _, point := range points {
		if point.Timestamp.IsZero() || point.Timestamp.Before(cutoff) || point.Timestamp.After(now.Add(5*time.Minute)) {
			continue
		}
		if math.IsNaN(point.Value) || math.IsInf(point.Value, 0) || point.Value < 0 || point.Value > 100 {
			continue
		}
		bucket := point.Timestamp.Unix() / int64(capacityForecastBucketWidth/time.Second)
		buckets[bucket] = append(buckets[bucket], point.Value)
		validSamples++
		if point.Timestamp.After(latest) {
			latest = point.Timestamp
		}
	}
	result.SampleCount = validSamples
	if validSamples == 0 || latest.IsZero() {
		result.Reason = "no-valid-samples"
		return result
	}
	result.ObservedAt = latest
	if now.Sub(latest) > capacityForecastFreshness {
		result.Reason = "stale-history"
		return result
	}

	normalized := make([]capacityBucketPoint, 0, len(buckets))
	for bucket, values := range buckets {
		sort.Float64s(values)
		median := values[len(values)/2]
		if len(values)%2 == 0 {
			median = (values[len(values)/2-1] + values[len(values)/2]) / 2
		}
		normalized = append(normalized, capacityBucketPoint{
			timestamp: time.Unix(bucket*int64(capacityForecastBucketWidth/time.Second), 0),
			value:     median,
		})
	}
	sort.Slice(normalized, func(i, j int) bool {
		return normalized[i].timestamp.Before(normalized[j].timestamp)
	})
	result.BucketCount = len(normalized)
	if len(normalized) < capacityForecastMinimumBuckets {
		result.Reason = "insufficient-hourly-coverage"
		return result
	}
	result.CoverageSpan = normalized[len(normalized)-1].timestamp.Sub(normalized[0].timestamp)
	if result.CoverageSpan < capacityForecastMinimumSpan {
		result.Reason = "history-window-too-short"
		return result
	}

	overallSlope, overallR2 := capacityLinearRegression(normalized)
	recentStart := len(normalized) / 2
	if len(normalized)-recentStart < 8 {
		recentStart = len(normalized) - 8
	}
	recentSlope, _ := capacityLinearRegression(normalized[recentStart:])
	overallDaily := overallSlope * 24
	recentDaily := recentSlope * 24
	result.DailyChange = overallDaily

	// A historic rise that has flattened or reversed is not an impending
	// exhaustion signal. Requiring both windows to rise also rejects one-off
	// capacity jumps caused by a resize or a telemetry discontinuity.
	if overallDaily <= capacityForecastMinimumRate || recentDaily <= capacityForecastMinimumRate {
		result.Ready = true
		result.Reason = "not-increasing"
		return result
	}

	spanFactor := math.Min(1, result.CoverageSpan.Hours()/48)
	agreement := math.Min(overallDaily, recentDaily) / math.Max(overallDaily, recentDaily)
	result.Confidence = clampCapacityConfidence(overallR2 * spanFactor * agreement)
	result.Ready = true
	if result.Confidence < capacityForecastMinConfidence {
		result.Reason = "low-confidence"
		return result
	}
	result.Reason = "increasing"
	return result
}

func capacityLinearRegression(points []capacityBucketPoint) (slopePerHour, rSquared float64) {
	if len(points) < 2 {
		return 0, 0
	}
	start := points[0].timestamp
	n := float64(len(points))
	var sumX, sumY, sumXY, sumX2 float64
	for _, point := range points {
		x := point.timestamp.Sub(start).Hours()
		sumX += x
		sumY += point.value
		sumXY += x * point.value
		sumX2 += x * x
	}
	denominator := n*sumX2 - sumX*sumX
	if denominator == 0 {
		return 0, 0
	}
	slopePerHour = (n*sumXY - sumX*sumY) / denominator
	intercept := (sumY - slopePerHour*sumX) / n
	meanY := sumY / n
	var residual, total float64
	for _, point := range points {
		x := point.timestamp.Sub(start).Hours()
		predicted := intercept + slopePerHour*x
		residual += math.Pow(point.value-predicted, 2)
		total += math.Pow(point.value-meanY, 2)
	}
	if total == 0 {
		return slopePerHour, 0
	}
	return slopePerHour, clampCapacityConfidence(1 - residual/total)
}

func clampCapacityConfidence(value float64) float64 {
	if value < 0 {
		return 0
	}
	if value > 1 {
		return 1
	}
	return value
}

const (
	capacityAlertOriginKey       = "capacityAlertOrigin"
	capacityAlertOriginThreshold = "threshold"
	capacityAlertOriginForecast  = "forecast"
)

var capacityForecastMetadataKeys = []string{
	"forecastConfidence",
	"forecastDailyChangePct",
	"forecastDaysToFull",
	"forecastObservedAt",
	"forecastSampleCount",
	"forecastBucketCount",
	"forecastCoverageSeconds",
}

func (m *Manager) evaluateStorageCapacity(storage models.Storage, thresholds ThresholdConfig, trend CapacityTrendObservation) {
	input := &UnifiedResourceInput{
		ID:             storage.ID,
		StorageAliases: storage.AliasIDs,
		Type:           "storage",
		Name:           storage.Name,
		Node:           storage.Node,
		Instance:       storage.Instance,
		Disk:           &UnifiedResourceMetric{Percent: storage.Usage},
	}
	m.evaluateUnifiedCapacity(input, thresholds, trend, func() bool {
		if !m.config.Enabled {
			return true
		}
		allDisabled, _ := m.alertPolicyTypeSwitchesNoLock("storage")
		return allDisabled || m.resolveStorageThresholdsNoLock(storage).Disabled
	})
}

func (m *Manager) evaluateUnifiedCapacity(input *UnifiedResourceInput, thresholds ThresholdConfig, trend CapacityTrendObservation, policyDisabledNoLock func() bool) {
	if input == nil || input.Disk == nil {
		return
	}
	threshold := thresholds.Usage
	if threshold == nil || threshold.Trigger <= 0 {
		m.evaluateUnifiedMetrics(input, thresholds, nil)
		return
	}

	current := input.DiskValue()
	active, origin := m.activeStorageCapacityOrigin(input.ID)
	staticTriggered := current >= threshold.Trigger
	staticLatched := active && origin == capacityAlertOriginThreshold && threshold.Clear > 0 && current >= threshold.Clear
	if staticTriggered || staticLatched {
		m.evaluateUnifiedMetrics(input, thresholds, &metricOptions{
			Metadata: map[string]interface{}{
				capacityAlertOriginKey: capacityAlertOriginThreshold,
			},
			RemoveMetadata: capacityForecastMetadataKeys,
		})
		return
	}

	// Insufficient or low-confidence history is unknown, not recovery. Keep an
	// already-firing forecast occurrence untouched until trustworthy evidence
	// says the risk has receded; normal stale-alert cleanup remains the final
	// bound if telemetry never becomes usable again.
	recoveryFloor := threshold.Clear
	if recoveryFloor <= 0 {
		recoveryFloor = threshold.Trigger
	}
	if active && origin == capacityAlertOriginForecast && current >= recoveryFloor && (!trend.Ready || trend.Reason == "low-confidence") {
		return
	}

	eta, hasETA := capacityTimeToFull(current, trend.DailyChange)
	forecastTrusted := trend.Ready && trend.Confidence >= capacityForecastMinConfidence && hasETA
	forecastTriggered := forecastTrusted && eta <= capacityForecastWarningHorizon
	forecastLatched := active && origin == capacityAlertOriginForecast && forecastTrusted && eta <= capacityForecastRecoveryWindow
	if forecastTriggered || forecastLatched {
		m.evaluateCapacityForecast(input, thresholds, trend, eta, policyDisabledNoLock)
		return
	}

	// No predictive risk remains. Run the ordinary metric evaluator so a
	// forecast recovery and every existing static hysteresis rule retain the
	// same canonical state, history, and notification behavior.
	m.evaluateUnifiedMetrics(input, thresholds, &metricOptions{
		Metadata: map[string]interface{}{
			capacityAlertOriginKey: capacityAlertOriginThreshold,
		},
		RemoveMetadata: capacityForecastMetadataKeys,
	})
}

func (m *Manager) activeStorageCapacityOrigin(resourceID string) (bool, string) {
	trackingKey := canonicalMetricStateID(resourceID, "usage")
	m.mu.RLock()
	defer m.mu.RUnlock()
	alert, exists := m.getActiveAlertNoLock(trackingKey)
	if !exists || alert == nil {
		return false, ""
	}
	if alert.Metadata == nil {
		return true, capacityAlertOriginThreshold
	}
	origin, _ := alert.Metadata[capacityAlertOriginKey].(string)
	if origin == "" {
		origin = capacityAlertOriginThreshold
	}
	return true, origin
}

func capacityTimeToFull(current, dailyChange float64) (time.Duration, bool) {
	if current < 0 || current >= 100 || dailyChange <= capacityForecastMinimumRate || math.IsNaN(dailyChange) || math.IsInf(dailyChange, 0) {
		return 0, false
	}
	days := (100 - current) / dailyChange
	if days <= 0 || math.IsNaN(days) || math.IsInf(days, 0) {
		return 0, false
	}
	return time.Duration(days * float64(24*time.Hour)), true
}

func (m *Manager) evaluateCapacityForecast(input *UnifiedResourceInput, thresholds ThresholdConfig, trend CapacityTrendObservation, eta time.Duration, policyDisabledNoLock func() bool) {
	resourceID := input.ID
	specID := canonicalMetricSpecID(resourceID, "usage")
	resourceType, ok := unifiedMetricResourceType(input.Type)
	if !ok {
		return
	}
	spec, err := buildCanonicalSeverityThresholdSpec(
		specID,
		resourceID,
		input.Name,
		resourceType,
		"capacity-risk",
		1,
		2,
		false,
	)
	if err != nil {
		log.Warn().Err(err).Str("resourceID", input.ID).Msg("Skipping invalid storage capacity forecast spec")
		return
	}
	spec.ConfirmationsRequired = 2
	if err := spec.Validate(); err != nil {
		log.Warn().Err(err).Str("resourceID", input.ID).Msg("Skipping invalid confirmed storage capacity forecast spec")
		return
	}

	riskScore := 1.0
	if eta <= capacityForecastCriticalWindow {
		riskScore = 2
	}
	observedAt := m.policyNow()
	daysToFull := eta.Hours() / 24
	message := fmt.Sprintf(
		"%s projected to fill in %s (%.1f%% used, +%.2f%%/day)",
		unifiedAlertType(input.Type),
		formatCapacityETA(eta),
		input.DiskValue(),
		trend.DailyChange,
	)
	attributes := map[string]string{
		"capacity_alert_origin":    capacityAlertOriginForecast,
		"confidence":               strconv.FormatFloat(trend.Confidence, 'f', 3, 64),
		"current_usage_percent":    strconv.FormatFloat(input.DiskValue(), 'f', 2, 64),
		"daily_change_percent":     strconv.FormatFloat(trend.DailyChange, 'f', 3, 64),
		"forecast_days_to_full":    strconv.FormatFloat(daysToFull, 'f', 2, 64),
		"history_bucket_count":     strconv.Itoa(trend.BucketCount),
		"history_coverage_seconds": strconv.FormatInt(int64(trend.CoverageSpan/time.Second), 10),
	}
	metadata := map[string]interface{}{
		"resourceType":            input.Type,
		"clearThreshold":          thresholds.Usage.Clear,
		capacityAlertOriginKey:    capacityAlertOriginForecast,
		"forecastConfidence":      trend.Confidence,
		"forecastDailyChangePct":  trend.DailyChange,
		"forecastDaysToFull":      daysToFull,
		"forecastObservedAt":      trend.ObservedAt,
		"forecastSampleCount":     trend.SampleCount,
		"forecastBucketCount":     trend.BucketCount,
		"forecastCoverageSeconds": int64(trend.CoverageSpan / time.Second),
	}
	if len(input.StorageAliases) > 0 {
		metadata[storagePolicyAliasesKey] = append([]string(nil), input.StorageAliases...)
	}
	if platformType := unifiedAlertPlatformType(input.Type); platformType != "" {
		metadata[alertPlatformTypeKey] = platformType
	}
	_, _ = m.evaluateCanonicalLifecycleAlert(canonicalLifecycleAlertParams{
		Spec: spec,
		Evidence: alertspecs.AlertEvidence{
			ObservedAt: observedAt,
			Summary:    message,
			Attributes: attributes,
			SeverityThreshold: &alertspecs.SeverityThresholdEvidence{
				Metric:    "capacity-risk",
				Direction: alertspecs.ThresholdDirectionAbove,
				Observed:  riskScore,
			},
		},
		IntentSignal:                 MetricAlertIntentSignal("usage"),
		PolicyDisabledNoLock:         policyDisabledNoLock,
		AlertID:                      canonicalMetricStateID(resourceID, "usage"),
		AlertType:                    "usage",
		ResourceID:                   resourceID,
		ResourceName:                 input.Name,
		Node:                         input.Node,
		Instance:                     input.Instance,
		Message:                      message,
		Value:                        input.DiskValue(),
		Threshold:                    100,
		Metadata:                     metadata,
		AddToRecent:                  true,
		AddToHistory:                 true,
		RateLimit:                    true,
		NotifyOnSeverityChange:       true,
		AddToHistoryOnSeverityChange: true,
	})
}

func formatCapacityETA(eta time.Duration) string {
	if eta < 2*time.Hour {
		return "about 1 hour"
	}
	if eta < 24*time.Hour {
		return fmt.Sprintf("about %.0f hours", math.Ceil(eta.Hours()))
	}
	days := int(math.Ceil(eta.Hours() / 24))
	if days == 1 {
		return "about 1 day"
	}
	return fmt.Sprintf("about %d days", days)
}