mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-19 18:32:29 +00:00
rcourtman
a0e8896893
Move PBS connectivity and metric evaluation into internal/alerts/pbs.go, and move storage connectivity, usage, and ZFS health evaluation into internal/alerts/storage.go. Keep the Manager API unchanged while recording PBS and storage as resource-checker owners in the alerts subsystem contract, with focused characterization tests for PBS offline normalization and ZFS device labels. Proof: go test ./internal/alerts/...
295 lines
11 KiB
Go
295 lines
11 KiB
Go
package alerts
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
// CheckStorage checks storage against thresholds
|
|
func (m *Manager) CheckStorage(storage models.Storage) {
|
|
m.mu.RLock()
|
|
if !m.config.Enabled {
|
|
m.mu.RUnlock()
|
|
return
|
|
}
|
|
if m.config.DisableAllStorage {
|
|
m.mu.RUnlock()
|
|
// Clear any existing storage alerts when all storage alerts are disabled
|
|
m.mu.Lock()
|
|
usageAlertID := canonicalMetricStateID(storage.ID, "usage")
|
|
if m.clearActiveAlertIfPresentNoLock(usageAlertID) {
|
|
log.Info().
|
|
Str("alertID", usageAlertID).
|
|
Str("storage", storage.Name).
|
|
Msg("Cleared usage alert - all storage alerts disabled")
|
|
}
|
|
offlineAlertID := canonicalConnectivityStateID(storage.ID)
|
|
if m.clearActiveAlertIfPresentNoLock(offlineAlertID) {
|
|
log.Info().
|
|
Str("alertID", offlineAlertID).
|
|
Str("storage", storage.Name).
|
|
Msg("Cleared offline alert - all storage alerts disabled")
|
|
}
|
|
m.mu.Unlock()
|
|
return
|
|
}
|
|
|
|
thresholds := m.resolveResourceThresholds("storage", storage.ID)
|
|
m.mu.RUnlock()
|
|
|
|
if thresholds.Disabled {
|
|
m.mu.Lock()
|
|
delete(m.offlineConfirmations, storage.ID)
|
|
m.mu.Unlock()
|
|
m.clearAlert(canonicalMetricStateID(storage.ID, "usage"))
|
|
m.clearAlert(canonicalConnectivityStateID(storage.ID))
|
|
return
|
|
}
|
|
|
|
// Check if storage is truly offline/unavailable (not just inactive from other nodes)
|
|
// Note: In a cluster, local storage from other nodes shows as inactive which is normal
|
|
if thresholds.DisableConnectivity {
|
|
m.mu.Lock()
|
|
delete(m.offlineConfirmations, storage.ID)
|
|
m.mu.Unlock()
|
|
m.clearAlert(canonicalConnectivityStateID(storage.ID))
|
|
} else if storage.Status == "offline" || storage.Status == "unavailable" {
|
|
m.checkStorageOffline(storage)
|
|
} else {
|
|
// Clear any existing offline alert if storage is back online
|
|
m.clearStorageOfflineAlert(storage)
|
|
}
|
|
|
|
// Check usage if storage has valid data (even if not currently active on this node)
|
|
// In clusters, storage may show as inactive on nodes where it's not currently mounted
|
|
// but we still want to alert on high usage
|
|
log.Debug().
|
|
Str("storage", storage.Name).
|
|
Str("id", storage.ID).
|
|
Float64("usage", storage.Usage).
|
|
Str("status", storage.Status).
|
|
Float64("trigger", thresholds.Usage.Trigger).
|
|
Float64("clear", thresholds.Usage.Clear).
|
|
Msg("Checking storage thresholds")
|
|
|
|
// Check usage if storage is online - checkMetric will skip if threshold is nil or <= 0
|
|
if storage.Status != "offline" && storage.Status != "unavailable" && storage.Usage > 0 {
|
|
m.evaluateUnifiedMetrics(&UnifiedResourceInput{
|
|
ID: storage.ID,
|
|
Type: "storage",
|
|
Name: storage.Name,
|
|
Node: storage.Node,
|
|
Instance: storage.Instance,
|
|
Disk: &UnifiedResourceMetric{Percent: storage.Usage},
|
|
}, thresholds, nil)
|
|
}
|
|
|
|
// Check ZFS pool status if this is ZFS storage
|
|
if storage.ZFSPool != nil {
|
|
m.checkZFSPoolHealth(storage)
|
|
}
|
|
}
|
|
|
|
// checkZFSPoolHealth checks ZFS pool for errors and degraded state
|
|
func (m *Manager) checkZFSPoolHealth(storage models.Storage) {
|
|
pool := storage.ZFSPool
|
|
if pool == nil {
|
|
return
|
|
}
|
|
|
|
poolResourceName := fmt.Sprintf("%s (%s)", storage.Name, pool.Name)
|
|
poolAssessment := storagehealth.AssessZFSPool(*pool)
|
|
|
|
// Check pool state (DEGRADED, FAULTED, etc.)
|
|
stateAlertID := fmt.Sprintf("zfs-pool-state-%s", storage.ID)
|
|
stateMetadata := map[string]interface{}{
|
|
"pool_name": pool.Name,
|
|
"pool_state": pool.State,
|
|
}
|
|
stateReasons := filterStorageHealthReasonsByCodes(poolAssessment.Reasons, zfsPoolAssessmentCodes)
|
|
stateResult, _ := m.syncCanonicalHealthAssessmentAlert(canonicalHealthAssessmentAlertParams{
|
|
SpecID: storage.ID + "/zfs-pool:" + sanitizeHostComponent(pool.Name) + "-state",
|
|
Signal: "zfs-pool",
|
|
Codes: zfsPoolAssessmentCodes,
|
|
Reasons: stateReasons,
|
|
AlertID: stateAlertID,
|
|
AlertType: "zfs-pool-state",
|
|
SpecResourceID: storage.ID + "/zfs-pool:" + sanitizeHostComponent(pool.Name),
|
|
ResourceID: storage.ID,
|
|
ResourceName: poolResourceName,
|
|
ResourceType: unifiedresources.ResourceTypeStorage,
|
|
Node: storage.Node,
|
|
Instance: storage.Instance,
|
|
Metadata: stateMetadata,
|
|
})
|
|
if stateResult.Transition != nil && stateResult.Transition.Kind == alertspecs.EvaluationTransitionActivated {
|
|
log.Warn().
|
|
Str("pool", pool.Name).
|
|
Str("state", pool.State).
|
|
Str("node", storage.Node).
|
|
Msg("ZFS pool is not healthy")
|
|
}
|
|
|
|
// Check for read/write/checksum errors
|
|
totalErrors := pool.ReadErrors + pool.WriteErrors + pool.ChecksumErrors
|
|
errorsAlertID := fmt.Sprintf("zfs-pool-errors-%s", storage.ID)
|
|
errorsSpecResourceID := storage.ID + "/zfs-pool:" + sanitizeHostComponent(pool.Name)
|
|
errorsSpecID := errorsSpecResourceID + "-errors"
|
|
if totalErrors > 0 {
|
|
existingValue, exists := m.activeAlertValue(buildCanonicalStateID(errorsSpecResourceID, errorsSpecID))
|
|
if !exists || float64(totalErrors) > existingValue {
|
|
errorMetadata := map[string]interface{}{
|
|
"pool_name": pool.Name,
|
|
"read_errors": pool.ReadErrors,
|
|
"write_errors": pool.WriteErrors,
|
|
"checksum_errors": pool.ChecksumErrors,
|
|
}
|
|
errorReasons := filterStorageHealthReasonsByCodes(poolAssessment.Reasons, zfsPoolErrorAssessmentCodes)
|
|
result, _ := m.syncCanonicalHealthAssessmentAlert(canonicalHealthAssessmentAlertParams{
|
|
SpecID: errorsSpecID,
|
|
Signal: "zfs-pool-errors",
|
|
Codes: zfsPoolErrorAssessmentCodes,
|
|
Reasons: errorReasons,
|
|
AlertID: errorsAlertID,
|
|
AlertType: "zfs-pool-errors",
|
|
SpecResourceID: errorsSpecResourceID,
|
|
ResourceID: storage.ID,
|
|
ResourceName: poolResourceName,
|
|
ResourceType: unifiedresources.ResourceTypeStorage,
|
|
Node: storage.Node,
|
|
Instance: storage.Instance,
|
|
Metadata: errorMetadata,
|
|
MessageBuilder: func(result alertspecs.EvaluationResult) (string, float64, float64) {
|
|
return fmt.Sprintf(
|
|
"ZFS pool '%s' has errors: %d read, %d write, %d checksum",
|
|
pool.Name, pool.ReadErrors, pool.WriteErrors, pool.ChecksumErrors,
|
|
), float64(totalErrors), 0
|
|
},
|
|
})
|
|
if result.Transition != nil && result.Transition.Kind == alertspecs.EvaluationTransitionActivated {
|
|
log.Error().
|
|
Str("pool", pool.Name).
|
|
Int64("read_errors", pool.ReadErrors).
|
|
Int64("write_errors", pool.WriteErrors).
|
|
Int64("checksum_errors", pool.ChecksumErrors).
|
|
Str("node", storage.Node).
|
|
Msg("ZFS pool has I/O errors")
|
|
}
|
|
}
|
|
} else {
|
|
m.clearAlert(buildCanonicalStateID(errorsSpecResourceID, errorsSpecID))
|
|
}
|
|
|
|
// Check individual devices for errors
|
|
for _, device := range pool.Devices {
|
|
alertID := fmt.Sprintf("zfs-device-%s-%s", storage.ID, device.Name)
|
|
deviceAssessment := zfsDeviceAssessment(device)
|
|
metadata := map[string]interface{}{
|
|
"pool_name": pool.Name,
|
|
"device_name": device.Name,
|
|
"device_state": device.State,
|
|
"read_errors": device.ReadErrors,
|
|
"write_errors": device.WriteErrors,
|
|
"checksum_errors": device.ChecksumErrors,
|
|
}
|
|
result, _ := m.syncCanonicalHealthAssessmentAlert(canonicalHealthAssessmentAlertParams{
|
|
SpecID: storage.ID + "/zfs-pool:" + sanitizeHostComponent(pool.Name) + "/device:" + sanitizeHostComponent(device.Name) + "-health",
|
|
Signal: "zfs-device",
|
|
Codes: zfsDeviceAssessmentCodes,
|
|
Reasons: deviceAssessment.Reasons,
|
|
AlertID: alertID,
|
|
AlertType: "zfs-device",
|
|
SpecResourceID: storage.ID + "/zfs-pool:" + sanitizeHostComponent(pool.Name) + "/device:" + sanitizeHostComponent(device.Name),
|
|
ResourceID: storage.ID,
|
|
ResourceName: formatZFSDeviceResourceName(storage.Name, pool.Name, device.Name),
|
|
ResourceType: unifiedresources.ResourceTypeStorage,
|
|
Node: storage.Node,
|
|
Instance: storage.Instance,
|
|
Metadata: metadata,
|
|
MessageBuilder: func(result alertspecs.EvaluationResult) (string, float64, float64) {
|
|
return strings.Join(storageHealthReasonSummaries(deviceAssessment.Reasons), "; "), float64(device.ReadErrors + device.WriteErrors + device.ChecksumErrors), 0
|
|
},
|
|
})
|
|
if result.Transition != nil && result.Transition.Kind == alertspecs.EvaluationTransitionActivated {
|
|
log.Warn().
|
|
Str("pool", pool.Name).
|
|
Str("device", device.Name).
|
|
Str("state", device.State).
|
|
Int64("errors", device.ReadErrors+device.WriteErrors+device.ChecksumErrors).
|
|
Str("node", storage.Node).
|
|
Msg("ZFS device has issues")
|
|
}
|
|
}
|
|
}
|
|
|
|
func formatZFSDeviceResourceName(storageName, poolName, deviceName string) string {
|
|
storageName = strings.TrimSpace(storageName)
|
|
if storageName == "" {
|
|
storageName = "storage"
|
|
}
|
|
|
|
parts := make([]string, 0, 2)
|
|
if poolName = strings.TrimSpace(poolName); poolName != "" {
|
|
parts = append(parts, poolName)
|
|
}
|
|
if deviceName = strings.TrimSpace(deviceName); deviceName != "" {
|
|
parts = append(parts, deviceName)
|
|
}
|
|
if len(parts) == 0 {
|
|
return storageName
|
|
}
|
|
return fmt.Sprintf("%s (%s)", storageName, strings.Join(parts, ", "))
|
|
}
|
|
|
|
// checkStorageOffline creates an alert for offline/unavailable storage
|
|
func (m *Manager) checkStorageOffline(storage models.Storage) {
|
|
alertID := fmt.Sprintf("storage-offline-%s", storage.ID)
|
|
|
|
m.mu.Lock()
|
|
delete(m.offlineRecoveryConfirmations, canonicalConnectivityStateID(storage.ID))
|
|
m.mu.Unlock()
|
|
|
|
thresholds := m.resolveResourceThresholds("storage", storage.ID)
|
|
spec, err := buildCanonicalConnectivitySpec(storage.ID, storage.Name, unifiedresources.ResourceTypeStorage, AlertLevelWarning, 2, thresholds.Disabled || thresholds.DisableConnectivity)
|
|
if err != nil {
|
|
log.Warn().
|
|
Err(err).
|
|
Str("storage", storage.Name).
|
|
Str("storageID", storage.ID).
|
|
Msg("Skipping invalid canonical storage connectivity spec")
|
|
return
|
|
}
|
|
|
|
_, _ = m.evaluateCanonicalLifecycleAlert(canonicalLifecycleAlertParams{
|
|
Spec: spec,
|
|
Evidence: alertspecs.AlertEvidence{ObservedAt: time.Now(), Connectivity: &alertspecs.ConnectivityEvidence{Signal: "status", Connected: false}},
|
|
Tracking: m.offlineConfirmations,
|
|
TrackingKey: storage.ID,
|
|
AlertID: alertID,
|
|
AlertType: "offline",
|
|
ResourceID: storage.ID,
|
|
ResourceName: storage.Name,
|
|
Node: storage.Node,
|
|
Instance: storage.Instance,
|
|
Message: fmt.Sprintf("Storage %s on node %s is unavailable", storage.Name, storage.Node),
|
|
Metadata: map[string]interface{}{
|
|
"resourceType": "storage",
|
|
"status": storage.Status,
|
|
},
|
|
RateLimit: true,
|
|
DispatchAsync: true,
|
|
})
|
|
}
|
|
|
|
// clearStorageOfflineAlert removes offline alert when storage comes back online
|
|
func (m *Manager) clearStorageOfflineAlert(storage models.Storage) {
|
|
m.clearResourceOfflineAlert(storage.ID, storage.Name, storage.Node, "Storage", offlineRecoveryConfirmationsStorage)
|
|
}
|