pulse/internal/alerts/threshold_resolution_shared_test.go

package alerts

import (
	"encoding/json"
	"math"
	"os"
	"path/filepath"
	"reflect"
	"testing"
	"time"

	alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)

func boolPtr(v bool) *bool {
	return &v
}

func TestApplyThresholdOverrideIncludesLifecycleFields(t *testing.T) {
	m := newTestManager(t)

	base := ThresholdConfig{
		PoweredOffSeverity: AlertLevelWarning,
		Backup: &BackupAlertConfig{
			Enabled:       true,
			WarningDays:   3,
			CriticalDays:  7,
			FreshHours:    24,
			StaleHours:    72,
			AlertOrphaned: boolPtr(true),
			IgnoreVMIDs:   []string{"100"},
		},
		Snapshot: &SnapshotAlertConfig{
			Enabled:      true,
			WarningDays:  7,
			CriticalDays: 14,
		},
	}

	override := ThresholdConfig{
		PoweredOffSeverity: AlertLevelCritical,
		Backup: &BackupAlertConfig{
			Enabled:       false,
			WarningDays:   10,
			CriticalDays:  20,
			FreshHours:    12,
			StaleHours:    36,
			AlertOrphaned: boolPtr(false),
			IgnoreVMIDs:   []string{"200", "201"},
		},
		Snapshot: &SnapshotAlertConfig{
			Enabled:         false,
			WarningDays:     30,
			CriticalDays:    60,
			WarningSizeGiB:  10,
			CriticalSizeGiB: 20,
		},
	}

	got := m.applyThresholdOverride(base, override)

	if got.PoweredOffSeverity != AlertLevelCritical {
		t.Fatalf("PoweredOffSeverity = %q, want %q", got.PoweredOffSeverity, AlertLevelCritical)
	}
	if got.Backup == nil || got.Backup.Enabled {
		t.Fatalf("Backup override not applied: %+v", got.Backup)
	}
	if got.Backup.AlertOrphaned == nil || *got.Backup.AlertOrphaned {
		t.Fatalf("Backup AlertOrphaned override not applied: %+v", got.Backup)
	}
	if len(got.Backup.IgnoreVMIDs) != 2 || got.Backup.IgnoreVMIDs[0] != "200" {
		t.Fatalf("Backup IgnoreVMIDs override not applied: %+v", got.Backup.IgnoreVMIDs)
	}
	if got.Snapshot == nil || got.Snapshot.Enabled {
		t.Fatalf("Snapshot override not applied: %+v", got.Snapshot)
	}
}

func TestCheckGuestStoppedUsesResolvedThresholdsForPoweredOff(t *testing.T) {
	m := newTestManager(t)
	guestID := BuildGuestKey("pve1", "node1", 100)

	m.mu.Lock()
	m.config.Enabled = true
	m.config.CustomRules = []CustomAlertRule{
		{
			Name:     "disable-powered-off",
			Enabled:  true,
			Priority: 10,
			FilterConditions: FilterStack{
				LogicalOperator: "AND",
			},
			Thresholds: ThresholdConfig{
				DisableConnectivity: true,
			},
		},
	}
	m.mu.Unlock()

	vm := models.VM{
		ID:       guestID,
		VMID:     100,
		Name:     "app01",
		Node:     "node1",
		Instance: "pve1",
		Status:   "stopped",
	}

	m.CheckGuest(vm, "pve1")
	m.CheckGuest(vm, "pve1")

	m.mu.RLock()
	_, alertExists := m.activeAlerts["guest-powered-off-"+guestID]
	confirmationExists := testCoreHasIncident(m, guestID, canonicalPoweredStateSpecID(guestID))
	m.mu.RUnlock()

	if alertExists {
		t.Fatalf("expected no powered-off alert for %q when resolved thresholds disable connectivity", guestID)
	}
	if confirmationExists {
		t.Fatalf("expected no powered-off confirmations for %q when resolved thresholds disable connectivity", guestID)
	}
}

func TestCheckUnifiedResourceUsesCanonicalGuestOverrideKey(t *testing.T) {
	m := newTestManager(t)
	resourceID := BuildGuestKey("pve1", "node1", 100)

	m.mu.Lock()
	m.config.Enabled = true
	m.config.TimeThresholds = map[string]int{}
	m.config.GuestDefaults = ThresholdConfig{
		CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
	}
	m.config.Overrides = map[string]ThresholdConfig{
		resourceID: {
			CPU: &HysteresisThreshold{Trigger: 60, Clear: 55},
		},
	}
	m.mu.Unlock()

	m.CheckUnifiedResource(&UnifiedResourceInput{
		ID:       resourceID,
		Type:     "vm",
		Name:     "app01",
		Node:     "node1",
		Instance: "pve1",
		CPU:      &UnifiedResourceMetric{Percent: 65},
	})

	exists := testHasActiveAlert(t, m, canonicalMetricStateID(resourceID, "cpu"))

	if !exists {
		t.Fatalf("expected canonical resource ID %q to be used for override lookup and alert IDs", resourceID)
	}
}

func TestCheckUnifiedResourceUsesStableClusteredGuestOverrideKey(t *testing.T) {
	m := newTestManager(t)
	resourceID := BuildGuestKey("pve1", "node2", 101)

	m.mu.Lock()
	m.config.Enabled = true
	m.config.TimeThresholds = map[string]int{}
	m.config.GuestDefaults = ThresholdConfig{
		CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
	}
	m.config.Overrides = map[string]ThresholdConfig{
		stableGuestOverrideKey("pve1", 101): {
			CPU: &HysteresisThreshold{Trigger: 60, Clear: 55},
		},
	}
	m.mu.Unlock()

	m.CheckUnifiedResource(&UnifiedResourceInput{
		ID:       resourceID,
		Type:     "vm",
		Name:     "app02",
		Node:     "node2",
		Instance: "pve1",
		CPU:      &UnifiedResourceMetric{Percent: 65},
	})

	exists := testHasActiveAlert(t, m, canonicalMetricStateID(resourceID, "cpu"))
	if !exists {
		t.Fatalf("expected stable clustered guest override to resolve for canonical resource ID %q", resourceID)
	}
}

func TestCheckNodeDisabledOverrideClearsExistingAlerts(t *testing.T) {
	m := newTestManager(t)
	node := models.Node{
		ID:       "node/pve-1",
		Name:     "pve-1",
		Instance: "pve1",
		Status:   "offline",
	}

	m.mu.Lock()
	m.config.Enabled = true
	m.config.Overrides = map[string]ThresholdConfig{
		node.ID: {Disabled: true},
	}
	cpuState, cpuAlert := testNewCanonicalAlert(node.ID, canonicalMetricSpecID(node.ID, "cpu"), string(alertspecs.AlertSpecKindMetricThreshold), "cpu")
	offlineState, offlineAlert := testNewCanonicalAlert(node.ID, canonicalConnectivitySpecID(node.ID), string(alertspecs.AlertSpecKindConnectivity), "offline")
	m.setActiveAlertNoLock(cpuState, cpuAlert)
	m.setActiveAlertNoLock(offlineState, offlineAlert)
	m.mu.Unlock()

	m.CheckNode(node)

	m.mu.RLock()
	_, cpuExists := m.activeAlerts[cpuState]
	_, offlineExists := m.activeAlerts[offlineState]
	countExists := testCoreHasIncident(m, node.ID, canonicalConnectivitySpecID(node.ID))
	m.mu.RUnlock()

	if cpuExists {
		t.Fatalf("expected CPU alert to be cleared for disabled node override")
	}
	if offlineExists {
		t.Fatalf("expected offline alert to be cleared for disabled node override")
	}
	if countExists {
		t.Fatalf("expected node offline tracking to be cleared for disabled node override")
	}
}

func testActiveAlertIDsOfType(m *Manager, alertType string) []string {
	var ids []string
	for _, alert := range m.GetActiveAlerts() {
		if alert.Type == alertType {
			ids = append(ids, alert.ID)
		}
	}
	return ids
}

func testNodeWithHostAgent() (models.Node, models.Host) {
	node := models.Node{
		ID:       "homelab-delly2",
		Name:     "delly2",
		Instance: "homelab",
		Status:   "online",
		CPU:      0.10,
		Memory:   models.Memory{Total: 100, Used: 40, Free: 60, Usage: 40},
		Disk:     models.Disk{Usage: 30},
	}
	host := models.Host{
		ID:           "agent-delly2",
		Hostname:     "delly2",
		LinkedNodeID: node.ID,
		CPUUsage:     10,
		Memory:       models.Memory{Total: 100, Used: 40, Free: 60, Usage: 40},
	}
	return node, host
}

// A host agent on a Proxmox node takes over CPU, memory and disk alerts, but it
// has no CPU temperature metric, so the node must keep raising temperature.
func TestCheckNodeKeepsTemperatureAlertWhenHostAgentMonitorsNode(t *testing.T) {
	m := newTestManager(t)
	m.mu.Lock()
	m.config.Enabled = true
	m.config.TimeThresholds = map[string]int{}
	m.config.NodeDefaults.Temperature = &HysteresisThreshold{Trigger: 80, Clear: 75}
	m.mu.Unlock()

	node, host := testNodeWithHostAgent()
	m.CheckHost(host)
	if !m.hasHostAgentForNode(node.ID) {
		t.Fatalf("expected CheckHost to link %q to node %q for deduplication", host.ID, node.ID)
	}

	node.Temperature = &models.Temperature{Available: true, CPUPackage: 90}
	m.CheckNode(node)
	m.CheckHost(host)

	tempAlertID := canonicalMetricStateID(node.ID, "temperature")
	if got := testActiveAlertIDsOfType(m, "temperature"); len(got) != 1 || got[0] != tempAlertID {
		t.Fatalf("expected exactly one temperature alert %q, got %v", tempAlertID, got)
	}

	node.Temperature = &models.Temperature{Available: true, CPUPackage: 60}
	m.CheckNode(node)
	if got := testActiveAlertIDsOfType(m, "temperature"); len(got) != 0 {
		t.Fatalf("expected node temperature alert to recover while the agent is registered, got %v", got)
	}
}

// A node alert that was open when the host agent registered must be handed
// over, not frozen until the agent goes offline.
func TestCheckNodeReleasesOpenMetricAlertWhenHostAgentRegisters(t *testing.T) {
	m := newTestManager(t)
	m.mu.Lock()
	m.config.Enabled = true
	m.config.TimeThresholds = map[string]int{}
	m.config.NodeDefaults.Memory = &HysteresisThreshold{Trigger: 85, Clear: 80}
	m.config.AgentDefaults.Memory = &HysteresisThreshold{Trigger: 85, Clear: 80}
	m.mu.Unlock()

	node, host := testNodeWithHostAgent()
	node.Memory = models.Memory{Total: 100, Used: 95, Free: 5, Usage: 95}
	host.Memory = node.Memory

	m.CheckNode(node)
	nodeAlertID := canonicalMetricStateID(node.ID, "memory")
	if !testHasActiveAlert(t, m, nodeAlertID) {
		t.Fatalf("expected node memory alert %q before the agent registers", nodeAlertID)
	}

	m.CheckHost(host)
	m.CheckNode(node)

	if testHasActiveAlert(t, m, nodeAlertID) {
		t.Fatalf("expected node memory alert %q to be released once the host agent owns memory", nodeAlertID)
	}
	m.mu.RLock()
	nodeIncident := testCoreHasIncident(m, node.ID, canonicalMetricSpecID(node.ID, "memory"))
	m.mu.RUnlock()
	if nodeIncident {
		t.Fatalf("expected no node memory incident left in the reducer core after handover")
	}
	agentAlertID := canonicalMetricStateID(hostResourceID(host.ID), "memory")
	if got := testActiveAlertIDsOfType(m, "memory"); len(got) != 1 || got[0] != agentAlertID {
		t.Fatalf("expected exactly one memory alert %q, got %v", agentAlertID, got)
	}
}

// Missing temperature telemetry is not a reading: it must not resolve an open
// node temperature alert, while a disabled threshold still clears it.
func TestCheckNodeMissingTemperatureDoesNotResolveOpenAlert(t *testing.T) {
	m := newTestManager(t)
	m.mu.Lock()
	m.config.Enabled = true
	m.config.TimeThresholds = map[string]int{}
	m.config.NodeDefaults.Temperature = &HysteresisThreshold{Trigger: 80, Clear: 75}
	m.mu.Unlock()

	node, _ := testNodeWithHostAgent()
	node.Temperature = &models.Temperature{Available: true, CPUPackage: 90}
	m.CheckNode(node)
	tempAlertID := canonicalMetricStateID(node.ID, "temperature")
	if !testHasActiveAlert(t, m, tempAlertID) {
		t.Fatalf("expected node temperature alert %q", tempAlertID)
	}

	for _, missing := range []*models.Temperature{nil, {Available: false}, {Available: true}} {
		node.Temperature = missing
		m.CheckNode(node)
		if !testHasActiveAlert(t, m, tempAlertID) {
			t.Fatalf("expected temperature alert to stay open without a reading (temperature %+v)", missing)
		}
	}

	m.mu.Lock()
	m.config.NodeDefaults.Temperature = &HysteresisThreshold{Trigger: 0, Clear: 0}
	m.mu.Unlock()
	m.CheckNode(node)
	if testHasActiveAlert(t, m, tempAlertID) {
		t.Fatalf("expected disabled temperature threshold to clear %q without a reading", tempAlertID)
	}
}

// Deduplication only hands the agent the usage metrics it evaluates. With agent
// alerts switched off, or one agent threshold off, the node keeps those alerts
// so the machine is never left unmonitored.
func TestCheckNodeKeepsUsageMetricsTheAgentDoesNotEvaluate(t *testing.T) {
	setup := func(t *testing.T) (*Manager, models.Node, models.Host) {
		m := newTestManager(t)
		m.mu.Lock()
		m.config.Enabled = true
		m.config.TimeThresholds = map[string]int{}
		m.config.NodeDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 75}
		m.config.NodeDefaults.Memory = &HysteresisThreshold{Trigger: 85, Clear: 80}
		m.config.AgentDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 75}
		m.config.AgentDefaults.Memory = &HysteresisThreshold{Trigger: 85, Clear: 80}
		m.mu.Unlock()
		node, host := testNodeWithHostAgent()
		node.CPU = 0.95
		node.Memory = models.Memory{Total: 100, Used: 95, Free: 5, Usage: 95}
		host.CPUUsage = 95
		host.Memory = node.Memory
		return m, node, host
	}
	nodeCPU := func(node models.Node) string { return canonicalMetricStateID(node.ID, "cpu") }
	nodeMemory := func(node models.Node) string { return canonicalMetricStateID(node.ID, "memory") }

	t.Run("agent_alerts_disabled", func(t *testing.T) {
		m, node, host := setup(t)
		m.CheckHost(host)
		m.CheckNode(node)
		if testHasActiveAlert(t, m, nodeCPU(node)) {
			t.Fatalf("expected the agent to own CPU while its alerts are enabled")
		}

		m.mu.Lock()
		m.config.DisableAllAgents = true
		m.mu.Unlock()
		m.CheckHost(host)
		m.CheckNode(node)
		if !testHasActiveAlert(t, m, nodeCPU(node)) || !testHasActiveAlert(t, m, nodeMemory(node)) {
			t.Fatalf("expected the node to alert on CPU and memory while agent alerts are disabled")
		}
	})

	t.Run("agent_cpu_threshold_off", func(t *testing.T) {
		m, node, host := setup(t)
		m.mu.Lock()
		m.config.AgentDefaults.CPU = &HysteresisThreshold{Trigger: 0, Clear: 0}
		m.mu.Unlock()
		m.CheckHost(host)
		m.CheckNode(node)
		if !testHasActiveAlert(t, m, nodeCPU(node)) {
			t.Fatalf("expected the node to keep CPU when the agent CPU threshold is off")
		}
		if testHasActiveAlert(t, m, nodeMemory(node)) {
			t.Fatalf("expected the agent to keep owning memory")
		}
		if got := testActiveAlertIDsOfType(m, "memory"); len(got) != 1 {
			t.Fatalf("expected exactly one memory alert for the machine, got %v", got)
		}
	})
}

// Ownership follows what the linked agents are configured to evaluate, not one
// report's data: a missing agent memory reading must not hand memory back to
// the node, a second linked agent's metrics count, and the node's disk metric
// belongs to the agent only when the agent evaluates that same filesystem.
func TestCheckNodeUsageOwnershipFollowsWhatAgentsEvaluate(t *testing.T) {
	setup := func(t *testing.T) (*Manager, models.Node, models.Host) {
		m := newTestManager(t)
		m.mu.Lock()
		m.config.Enabled = true
		m.config.TimeThresholds = map[string]int{}
		m.config.NodeDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 75}
		m.config.NodeDefaults.Memory = &HysteresisThreshold{Trigger: 85, Clear: 80}
		m.config.NodeDefaults.Disk = &HysteresisThreshold{Trigger: 90, Clear: 85}
		m.config.AgentDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 75}
		m.config.AgentDefaults.Memory = &HysteresisThreshold{Trigger: 85, Clear: 80}
		m.config.AgentDefaults.Disk = &HysteresisThreshold{Trigger: 90, Clear: 85}
		m.mu.Unlock()
		node, host := testNodeWithHostAgent()
		return m, node, host
	}

	t.Run("missing_agent_memory_reading", func(t *testing.T) {
		m, node, host := setup(t)
		node.Memory = models.Memory{Total: 100, Used: 95, Free: 5, Usage: 95}
		host.Memory = node.Memory
		m.CheckHost(host)
		m.CheckNode(node)

		host.Memory = models.Memory{Total: 100, UsageUnavailable: true}
		m.CheckHost(host)
		m.CheckNode(node)
		agentAlertID := canonicalMetricStateID(hostResourceID(host.ID), "memory")
		if got := testActiveAlertIDsOfType(m, "memory"); len(got) != 1 || got[0] != agentAlertID {
			t.Fatalf("expected only the agent memory alert while its reading is missing, got %v", got)
		}
	})

	t.Run("second_linked_agent", func(t *testing.T) {
		m, node, host := setup(t)
		node.CPU = 0.95
		m.mu.Lock()
		m.config.Overrides = map[string]ThresholdConfig{
			"agent-a": {CPU: &HysteresisThreshold{Trigger: 0, Clear: 0}},
		}
		m.mu.Unlock()
		agentA := host
		agentA.ID = "agent-a"
		agentB := host
		agentB.ID = "agent-b"
		agentB.CPUUsage = 95
		m.CheckHost(agentA)
		m.CheckHost(agentB)
		m.CheckNode(node)
		agentBAlertID := canonicalMetricStateID(hostResourceID(agentB.ID), "cpu")
		if got := testActiveAlertIDsOfType(m, "cpu"); len(got) != 1 || got[0] != agentBAlertID {
			t.Fatalf("expected only agent-b's CPU alert when it covers CPU for the node, got %v", got)
		}
	})

	t.Run("summary_disk", func(t *testing.T) {
		m, node, host := setup(t)
		root := models.Disk{Mountpoint: "/", Device: "/dev/sda1", Total: 100, Used: 95, Free: 5, Usage: 95}
		data := models.Disk{Mountpoint: "/data", Device: "/dev/sdb1", Total: 100, Used: 10, Free: 90, Usage: 10}
		host.Disks = []models.Disk{root, data}
		node.Disk = models.Disk{Total: 100, Used: 95, Free: 5, Usage: 95}
		rootResourceID, _ := hostDiskResourceID(host, root)
		nodeDiskAlertID := canonicalMetricStateID(node.ID, "disk")

		m.mu.Lock()
		m.config.Overrides = map[string]ThresholdConfig{rootResourceID: {Disabled: true}}
		m.mu.Unlock()
		m.CheckHost(host)
		m.CheckNode(node)
		if !testHasActiveAlert(t, m, nodeDiskAlertID) {
			t.Fatalf("expected the node to keep its root disk alert while the agent skips that filesystem")
		}

		m.mu.Lock()
		m.config.Overrides = nil
		m.mu.Unlock()
		m.CheckHost(host)
		m.CheckNode(node)
		if testHasActiveAlert(t, m, nodeDiskAlertID) {
			t.Fatalf("expected the node to release its disk alert once the agent evaluates the same filesystem")
		}
		if !testHasActiveAlert(t, m, canonicalMetricStateID(rootResourceID, "disk")) {
			t.Fatalf("expected the agent's root filesystem alert")
		}
	})
}

// A missing reading keeps the incident but must not let a sustained-for or
// recovery delay complete across the gap in evidence.
func TestCheckNodeMissingTemperatureInterruptsTimingRuns(t *testing.T) {
	m := newTestManager(t)
	m.mu.Lock()
	m.config.Enabled = true
	m.config.TimeThresholds = map[string]int{}
	m.config.MetricTimeThresholds = map[string]map[string]int{"node": {"temperature": 300}}
	m.config.NodeDefaults.Temperature = &HysteresisThreshold{Trigger: 80, Clear: 75}
	m.mu.Unlock()

	node, _ := testNodeWithHostAgent()
	tempAlertID := canonicalMetricStateID(node.ID, "temperature")
	specID := canonicalMetricSpecID(node.ID, "temperature")
	hot := &models.Temperature{Available: true, CPUPackage: 85}

	node.Temperature = hot
	m.CheckNode(node)
	m.mu.Lock()
	pending := testCoreIsPending(m, node.ID, specID)
	m.core.ShiftPending(-10 * time.Minute)
	m.mu.Unlock()
	if !pending {
		t.Fatalf("expected a pending temperature run before the reading went missing")
	}

	node.Temperature = nil
	m.CheckNode(node)
	node.Temperature = hot
	m.CheckNode(node)
	if testHasActiveAlert(t, m, tempAlertID) {
		t.Fatalf("expected the sustained-for delay to restart after a missing reading, not fire on the first sample back")
	}

	m.mu.Lock()
	m.core.ShiftPending(-10 * time.Minute)
	m.mu.Unlock()
	m.CheckNode(node)
	if !testHasActiveAlert(t, m, tempAlertID) {
		t.Fatalf("expected the temperature alert to fire after a sustained run")
	}

	node.Temperature = &models.Temperature{Available: true, CPUPackage: 60}
	m.CheckNode(node)
	m.mu.RLock()
	incident, _ := m.core.Incident(node.ID, specID)
	m.mu.RUnlock()
	if incident.RecoverySince.IsZero() {
		t.Fatalf("expected a recovery run to start below the clear threshold")
	}

	node.Temperature = nil
	m.CheckNode(node)
	m.mu.RLock()
	incident, _ = m.core.Incident(node.ID, specID)
	m.mu.RUnlock()
	if !incident.RecoverySince.IsZero() {
		t.Fatalf("expected a missing reading to restart the recovery run")
	}
	if !testHasActiveAlert(t, m, tempAlertID) {
		t.Fatalf("expected the temperature alert to stay open without a reading")
	}
}

// Node alerts keep the PVE instance name in Instance. A config save must judge
// them against node thresholds, not fall through to guest thresholds, which
// have no temperature threshold and used to resolve a live temperature alert.
func TestConfigSaveKeepsNodeTemperatureAlertOverTrigger(t *testing.T) {
	m := newTestManager(t)
	m.mu.Lock()
	m.config.Enabled = true
	m.config.TimeThresholds = map[string]int{}
	m.config.NodeDefaults.Temperature = &HysteresisThreshold{Trigger: 80, Clear: 75}
	m.mu.Unlock()

	node, _ := testNodeWithHostAgent()
	node.Temperature = &models.Temperature{Available: true, CPUPackage: 90}
	m.CheckNode(node)
	tempAlertID := canonicalMetricStateID(node.ID, "temperature")
	if !testHasActiveAlert(t, m, tempAlertID) {
		t.Fatalf("expected node temperature alert %q", tempAlertID)
	}

	m.UpdateConfig(m.GetConfig())
	if !testHasActiveAlert(t, m, tempAlertID) {
		t.Fatalf("expected a config save to keep a node temperature alert still over its trigger")
	}

	config := m.GetConfig()
	config.Overrides = map[string]ThresholdConfig{
		node.ID: {Temperature: &HysteresisThreshold{Trigger: 95, Clear: 92}},
	}
	m.UpdateConfig(config)
	if testHasActiveAlert(t, m, tempAlertID) {
		t.Fatalf("expected a node override raising the trigger above the reading to resolve the alert")
	}
}

func TestReevaluateActiveAlertsUsesSharedAgentOverrideResolution(t *testing.T) {
	m := newTestManager(t)

	m.mu.Lock()
	m.config.Enabled = true
	m.config.AgentDefaults = ThresholdConfig{
		CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
	}
	m.config.Overrides = map[string]ThresholdConfig{
		"host1": {
			Disabled: true,
		},
	}
	m.activeAlerts["agent:host1-cpu"] = &Alert{
		ID:        "agent:host1-cpu",
		Type:      "cpu",
		Value:     95,
		Threshold: 80,
		Metadata: map[string]interface{}{
			"resourceType": "Agent",
		},
	}
	m.reevaluateActiveAlertsLocked()
	m.mu.Unlock()

	m.mu.RLock()
	_, exists := m.activeAlerts["agent:host1-cpu"]
	m.mu.RUnlock()

	if exists {
		t.Fatalf("expected reevaluation to resolve agent alert using raw host override key")
	}
}

func TestReevaluateActiveAlertsUsesSharedStorageOverrideResolution(t *testing.T) {
	m := newTestManager(t)

	m.mu.Lock()
	m.config.Enabled = true
	m.config.StorageDefault = HysteresisThreshold{Trigger: 85, Clear: 80}
	m.config.Overrides = map[string]ThresholdConfig{
		"storage1": {
			Disabled: true,
		},
	}
	m.activeAlerts["storage1-usage"] = &Alert{
		ID:        "storage1-usage",
		Type:      "usage",
		Value:     95,
		Threshold: 85,
		Instance:  "Storage",
	}
	m.reevaluateActiveAlertsLocked()
	m.mu.Unlock()

	m.mu.RLock()
	_, exists := m.activeAlerts["storage1-usage"]
	m.mu.RUnlock()

	if exists {
		t.Fatalf("expected reevaluation to resolve storage alert when shared override disables storage alerting")
	}
}

func TestReevaluateActiveAlertsUsesStorageMetadataForCephPoolAlias(t *testing.T) {
	m := newTestManager(t)
	resourceID := "pve5-ceph-pool-data_replication"

	m.mu.Lock()
	m.config.Enabled = true
	m.config.StorageDefault = HysteresisThreshold{Trigger: 95, Clear: 90}
	m.config.Overrides = map[string]ThresholdConfig{
		"agent:pve5-ceph-pool-data_replication": {
			Usage: &HysteresisThreshold{Trigger: 50, Clear: 45},
		},
	}
	state, alert := testNewCanonicalAlert(resourceID, canonicalMetricSpecID(resourceID, "usage"), string(alertspecs.AlertSpecKindMetricThreshold), "usage")
	alert.Value = 60
	alert.Threshold = 50
	alert.ResourceName = "data_replication"
	alert.Node = "pve5"
	alert.Instance = "pve5"
	alert.Metadata = map[string]interface{}{
		"resourceType": "Storage",
	}
	m.setActiveAlertNoLock(state, alert)
	m.reevaluateActiveAlertsLocked()
	m.mu.Unlock()

	m.mu.RLock()
	_, exists := m.activeAlerts[state]
	m.mu.RUnlock()

	if !exists {
		t.Fatalf("expected Ceph pool storage alert to remain active under agent-prefixed alias override")
	}
}

func TestStorageThresholdResolutionUsesAliasIDs(t *testing.T) {
	m := newTestManager(t)

	m.mu.Lock()
	m.config.StorageDefault = HysteresisThreshold{Trigger: 95, Clear: 90}
	m.config.Overrides = map[string]ThresholdConfig{
		"agent:pve5-ceph-pool-data_replication": {
			Usage: &HysteresisThreshold{Trigger: 50, Clear: 45},
		},
	}
	thresholds := m.resolveStorageThresholdsNoLock(models.Storage{
		ID:       "pve5-ceph-pool-data_replication",
		AliasIDs: []string{"agent:pve5-ceph-pool-data_replication"},
	})
	m.mu.Unlock()

	if thresholds.Usage == nil {
		t.Fatalf("expected usage threshold to be resolved")
	}
	if thresholds.Usage.Trigger != 50 || thresholds.Usage.Clear != 45 {
		t.Fatalf("usage threshold = %#v, want alias override trigger 50 clear 45", thresholds.Usage)
	}
}

func TestCheckStorageOfflineUsesSharedThresholdResolution(t *testing.T) {
	m := newTestManager(t)
	storage := models.Storage{
		ID:     "storage1",
		Name:   "tank",
		Status: "offline",
	}

	m.mu.Lock()
	m.config.Overrides = map[string]ThresholdConfig{
		storage.ID: {
			DisableConnectivity: true,
		},
	}
	m.activeAlerts["storage-offline-"+storage.ID] = &Alert{ID: "storage-offline-" + storage.ID}
	m.mu.Unlock()

	m.checkStorageOffline(storage)

	m.mu.RLock()
	_, alertExists := m.activeAlerts["storage-offline-"+storage.ID]
	confirmExists := testCoreHasIncident(m, storage.ID, canonicalConnectivitySpecID(storage.ID))
	m.mu.RUnlock()

	if alertExists {
		t.Fatalf("expected storage offline alert to clear when shared thresholds disable connectivity")
	}
	if confirmExists {
		t.Fatalf("expected storage offline confirmations to clear when shared thresholds disable connectivity")
	}
}

func TestCheckSnapshotsUsesGuestContextForCustomRules(t *testing.T) {
	m := newTestManager(t)
	m.ClearActiveAlerts()

	cfg := AlertConfig{
		Enabled: true,
		SnapshotDefaults: SnapshotAlertConfig{
			Enabled:      true,
			WarningDays:  7,
			CriticalDays: 14,
		},
		CustomRules: []CustomAlertRule{
			{
				Name:     "db-snapshots",
				Enabled:  true,
				Priority: 10,
				FilterConditions: FilterStack{
					LogicalOperator: "AND",
					Filters: []FilterCondition{
						{Type: "text", Field: "name", Value: "db"},
					},
				},
				Thresholds: ThresholdConfig{
					Snapshot: &SnapshotAlertConfig{
						Enabled:      true,
						WarningDays:  15,
						CriticalDays: 20,
					},
				},
			},
		},
	}
	m.UpdateConfig(cfg)
	m.mu.Lock()
	m.config.TimeThresholds = map[string]int{}
	m.mu.Unlock()

	now := time.Now()
	snapshots := []models.GuestSnapshot{
		{
			ID:       "inst-node-100-weekly",
			Name:     "weekly",
			Node:     "node",
			Instance: "inst",
			Type:     "qemu",
			VMID:     100,
			Time:     now.Add(-10 * 24 * time.Hour),
		},
		{
			ID:       "inst-node-101-weekly",
			Name:     "weekly",
			Node:     "node",
			Instance: "inst",
			Type:     "qemu",
			VMID:     101,
			Time:     now.Add(-10 * 24 * time.Hour),
		},
	}
	guestLookups := map[string]GuestLookup{
		BuildGuestKey("inst", "node", 100): {Name: "db-server"},
		BuildGuestKey("inst", "node", 101): {Name: "web-server"},
	}

	m.CheckSnapshotsForInstance("inst", snapshots, guestLookups)

	m.mu.RLock()
	_, dbExists := testLookupActiveAlert(t, m, "snapshot-age-inst-node-100-weekly")
	_, webExists := testLookupActiveAlert(t, m, "snapshot-age-inst-node-101-weekly")
	m.mu.RUnlock()

	if dbExists {
		t.Fatalf("expected db snapshot alert to be suppressed by custom rule thresholds")
	}
	if !webExists {
		t.Fatalf("expected non-matching snapshot alert to use default thresholds")
	}
}

func TestCheckBackupsUsesGuestContextForCustomRules(t *testing.T) {
	m := newTestManager(t)
	m.ClearActiveAlerts()

	cfg := AlertConfig{
		Enabled: true,
		BackupDefaults: BackupAlertConfig{
			Enabled:      true,
			WarningDays:  7,
			CriticalDays: 14,
		},
		CustomRules: []CustomAlertRule{
			{
				Name:     "db-backups",
				Enabled:  true,
				Priority: 10,
				FilterConditions: FilterStack{
					LogicalOperator: "AND",
					Filters: []FilterCondition{
						{Type: "text", Field: "name", Value: "db"},
					},
				},
				Thresholds: ThresholdConfig{
					Backup: &BackupAlertConfig{
						Enabled:      true,
						WarningDays:  15,
						CriticalDays: 20,
					},
				},
			},
		},
	}
	m.UpdateConfig(cfg)

	now := time.Now()
	rollups := []recovery.ProtectionRollup{
		{
			RollupID: "db-rollup",
			SubjectRef: &recovery.ExternalRef{
				Type:      "proxmox-vm",
				Namespace: "inst",
				Name:      "db-server",
				ID:        BuildGuestKey("inst", "node", 100),
				Class:     "node",
			},
			LastSuccessAt: ptrTime(now.Add(-10 * 24 * time.Hour)),
			LastOutcome:   recovery.OutcomeSuccess,
			Providers:     []recovery.Provider{recovery.ProviderProxmoxPVE},
		},
		{
			RollupID: "web-rollup",
			SubjectRef: &recovery.ExternalRef{
				Type:      "proxmox-vm",
				Namespace: "inst",
				Name:      "web-server",
				ID:        BuildGuestKey("inst", "node", 101),
				Class:     "node",
			},
			LastSuccessAt: ptrTime(now.Add(-10 * 24 * time.Hour)),
			LastOutcome:   recovery.OutcomeSuccess,
			Providers:     []recovery.Provider{recovery.ProviderProxmoxPVE},
		},
	}

	guest100 := GuestLookup{
		ResourceID: BuildGuestKey("inst", "node", 100),
		Name:       "db-server",
		Instance:   "inst",
		Node:       "node",
		Type:       "qemu",
		VMID:       100,
	}
	guest101 := GuestLookup{
		ResourceID: BuildGuestKey("inst", "node", 101),
		Name:       "web-server",
		Instance:   "inst",
		Node:       "node",
		Type:       "qemu",
		VMID:       101,
	}
	guestsByKey := map[string]GuestLookup{
		guest100.ResourceID: guest100,
		guest101.ResourceID: guest101,
	}
	guestsByVMID := map[string][]GuestLookup{
		"100": {guest100},
		"101": {guest101},
	}

	m.CheckBackups(rollups, guestsByKey, guestsByVMID)

	m.mu.RLock()
	_, dbExists := testLookupActiveAlert(t, m, "backup-age-"+sanitizeAlertKey(guest100.ResourceID))
	_, webExists := testLookupActiveAlert(t, m, "backup-age-"+sanitizeAlertKey(guest101.ResourceID))
	m.mu.RUnlock()

	if dbExists {
		t.Fatalf("expected db backup alert to be suppressed by custom rule thresholds")
	}
	if !webExists {
		t.Fatalf("expected non-matching backup alert to use default thresholds")
	}
}

func TestReevaluateActiveAlertsUsesGuestContextForMetricCustomRules(t *testing.T) {
	m := newTestManager(t)
	resourceID := BuildGuestKey("pve1", "node1", 100)

	m.mu.Lock()
	m.config.Enabled = true
	m.config.GuestDefaults = ThresholdConfig{
		CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
	}
	m.config.CustomRules = []CustomAlertRule{
		{
			Name:     "db-metrics",
			Enabled:  true,
			Priority: 10,
			FilterConditions: FilterStack{
				LogicalOperator: "AND",
				Filters: []FilterCondition{
					{Type: "text", Field: "name", Value: "db"},
				},
			},
			Thresholds: ThresholdConfig{
				CPU: &HysteresisThreshold{Trigger: 95, Clear: 90},
			},
		},
	}
	state, alert := testNewCanonicalAlert(resourceID, canonicalMetricSpecID(resourceID, "cpu"), string(alertspecs.AlertSpecKindMetricThreshold), "cpu")
	alert.Value = 90
	alert.Threshold = 80
	alert.ResourceName = "db-server"
	alert.Node = "node1"
	alert.Instance = "pve1"
	alert.Metadata = map[string]interface{}{
		"resourceType": "vm",
	}
	m.setActiveAlertNoLock(state, alert)
	m.reevaluateActiveAlertsLocked()
	m.mu.Unlock()

	m.mu.RLock()
	_, exists := m.activeAlerts[state]
	m.mu.RUnlock()

	if exists {
		t.Fatalf("expected guest metric alert to resolve when custom rule raises the trigger")
	}
}

func TestReevaluateActiveAlertsUsesGuestContextForBackupCustomRules(t *testing.T) {
	m := newTestManager(t)
	resourceID := BuildGuestKey("pve1", "node1", 100)

	m.mu.Lock()
	m.config.Enabled = true
	m.config.BackupDefaults = BackupAlertConfig{
		Enabled:      true,
		WarningDays:  7,
		CriticalDays: 14,
	}
	m.config.CustomRules = []CustomAlertRule{
		{
			Name:     "db-backup-reeval",
			Enabled:  true,
			Priority: 10,
			FilterConditions: FilterStack{
				LogicalOperator: "AND",
				Filters: []FilterCondition{
					{Type: "text", Field: "name", Value: "db"},
				},
			},
			Thresholds: ThresholdConfig{
				Backup: &BackupAlertConfig{
					Enabled:      true,
					WarningDays:  15,
					CriticalDays: 20,
				},
			},
		},
	}
	state, alert := testNewCanonicalAlert(resourceID, resourceID+"-backup-age", string(alertspecs.AlertSpecKindPostureThreshold), "backup-age")
	alert.Value = 10
	alert.Threshold = 7
	alert.ResourceName = "db-server backup"
	alert.Node = "node1"
	alert.Instance = "pve1"
	alert.Metadata = map[string]interface{}{
		"ageDays":       10.0,
		"guestName":     "db-server",
		"guestType":     "qemu",
		"guestInstance": "pve1",
		"guestNode":     "node1",
		"guestVmid":     100,
		"orphaned":      false,
	}
	m.setActiveAlertNoLock(state, alert)
	m.reevaluateActiveAlertsLocked()
	m.mu.Unlock()

	m.mu.RLock()
	_, exists := m.activeAlerts[state]
	m.mu.RUnlock()

	if exists {
		t.Fatalf("expected guest backup alert to resolve when custom rule raises backup thresholds")
	}
}

func TestReevaluateActiveAlertsUsesGuestContextForPoweredOffCustomRules(t *testing.T) {
	m := newTestManager(t)
	resourceID := BuildGuestKey("pve1", "node1", 100)

	m.mu.Lock()
	m.config.Enabled = true
	m.config.CustomRules = []CustomAlertRule{
		{
			Name:     "db-no-powered-off",
			Enabled:  true,
			Priority: 10,
			FilterConditions: FilterStack{
				LogicalOperator: "AND",
				Filters: []FilterCondition{
					{Type: "text", Field: "name", Value: "db"},
				},
			},
			Thresholds: ThresholdConfig{
				DisableConnectivity: true,
			},
		},
	}
	state, alert := testNewCanonicalAlert(resourceID, canonicalPoweredStateSpecID(resourceID), string(alertspecs.AlertSpecKindPoweredState), "powered-off")
	alert.ResourceName = "db-server"
	alert.Node = "node1"
	alert.Instance = "pve1"
	alert.Metadata = map[string]interface{}{
		"resourceType": "vm",
	}
	m.setActiveAlertNoLock(state, alert)
	m.reevaluateActiveAlertsLocked()
	m.mu.Unlock()

	m.mu.RLock()
	_, exists := m.activeAlerts[state]
	m.mu.RUnlock()

	if exists {
		t.Fatalf("expected guest powered-off alert to resolve when custom rule disables connectivity")
	}
}

func TestReevaluateActiveAlertsUsesPBSResourceTypeMetadata(t *testing.T) {
	m := newTestManager(t)

	m.mu.Lock()
	m.config.Enabled = true
	m.config.PBSDefaults = ThresholdConfig{
		CPU:    &HysteresisThreshold{Trigger: 99, Clear: 94},
		Memory: &HysteresisThreshold{Trigger: 0, Clear: 0},
	}
	state, alert := testNewCanonicalAlert("pbs-1", canonicalMetricSpecID("pbs-1", "memory"), string(alertspecs.AlertSpecKindMetricThreshold), "memory")
	alert.Value = 90.8
	alert.Threshold = 85
	alert.ResourceName = "pbs"
	alert.Node = "pbs.local"
	alert.Instance = "pbs"
	alert.Metadata = map[string]interface{}{
		"resourceType": "PBS",
	}
	m.setActiveAlertNoLock(state, alert)
	m.reevaluateActiveAlertsLocked()
	m.mu.Unlock()

	m.mu.RLock()
	_, exists := m.activeAlerts[state]
	m.mu.RUnlock()

	if exists {
		t.Fatalf("expected PBS metric alert to resolve when PBS memory threshold is disabled")
	}
}

func TestReevaluateActiveAlertsUsesKubernetesResourceTypeMetadata(t *testing.T) {
	m := newTestManager(t)

	resourceID := "k8s:prod/ns:default/pod:api-7d9f"
	m.mu.Lock()
	m.config.Enabled = true
	m.config.KubernetesDefaults = ThresholdConfig{
		CPU: &HysteresisThreshold{Trigger: 0, Clear: 0},
	}
	state, alert := testNewCanonicalAlert(resourceID, canonicalMetricSpecID(resourceID, "cpu"), string(alertspecs.AlertSpecKindMetricThreshold), "cpu")
	alert.Value = 88
	alert.Threshold = 80
	alert.ResourceName = "api-7d9f"
	alert.Node = "worker-1"
	alert.Instance = "prod"
	alert.Metadata = map[string]interface{}{
		"resourceType": "Kubernetes Pod",
	}
	m.setActiveAlertNoLock(state, alert)
	m.reevaluateActiveAlertsLocked()
	m.mu.Unlock()

	m.mu.RLock()
	_, exists := m.activeAlerts[state]
	m.mu.RUnlock()

	if exists {
		t.Fatalf("expected Kubernetes metric alert to resolve when Kubernetes CPU threshold is disabled")
	}
}

func TestReevaluateActiveAlertsUsesTrueNASResourceTypeMetadata(t *testing.T) {
	m := newTestManager(t)

	resourceID := "storage:truenas-main/pool:tank"
	m.mu.Lock()
	m.config.Enabled = true
	m.config.StorageDefault = HysteresisThreshold{Trigger: 95, Clear: 90}
	m.config.TrueNASDefaults = ThresholdConfig{
		Usage: &HysteresisThreshold{Trigger: 0, Clear: 0},
	}
	state, alert := testNewCanonicalAlert(resourceID, canonicalMetricSpecID(resourceID, "usage"), string(alertspecs.AlertSpecKindMetricThreshold), "usage")
	alert.Value = 90
	alert.Threshold = 85
	alert.ResourceName = "tank"
	alert.Node = "truenas-main"
	alert.Instance = "TrueNAS"
	alert.Metadata = map[string]interface{}{
		"resourceType": "TrueNAS Pool",
	}
	m.setActiveAlertNoLock(state, alert)
	m.reevaluateActiveAlertsLocked()
	m.mu.Unlock()

	m.mu.RLock()
	_, exists := m.activeAlerts[state]
	m.mu.RUnlock()

	if exists {
		t.Fatalf("expected TrueNAS metric alert to resolve when TrueNAS usage threshold is disabled")
	}
}

func TestReevaluateActiveAlertsUsesVMwareResourceTypeMetadata(t *testing.T) {
	m := newTestManager(t)

	resourceID := "vmware:vc-1:datastore:datastore-301"
	m.mu.Lock()
	m.config.Enabled = true
	m.config.StorageDefault = HysteresisThreshold{Trigger: 95, Clear: 90}
	m.config.VMwareDefaults = ThresholdConfig{
		Usage: &HysteresisThreshold{Trigger: 0, Clear: 0},
	}
	state, alert := testNewCanonicalAlert(resourceID, canonicalMetricSpecID(resourceID, "usage"), string(alertspecs.AlertSpecKindMetricThreshold), "usage")
	alert.Value = 90
	alert.Threshold = 85
	alert.ResourceName = "nvme-primary"
	alert.Node = "Lab Datacenter"
	alert.Instance = "Lab vCenter"
	alert.Metadata = map[string]interface{}{
		"resourceType": "vSphere Datastore",
	}
	m.setActiveAlertNoLock(state, alert)
	m.reevaluateActiveAlertsLocked()
	m.mu.Unlock()

	m.mu.RLock()
	_, exists := m.activeAlerts[state]
	m.mu.RUnlock()

	if exists {
		t.Fatalf("expected vSphere metric alert to resolve when vSphere usage threshold is disabled")
	}
}

func TestGuestFilesystemDisableOverrideExcludesMountAndAggregate(t *testing.T) {
	m := newTestManager(t)
	guestID := BuildGuestKey("pve1", "node1", 101)

	m.mu.Lock()
	m.config.TimeThresholds = map[string]int{}
	m.config.GuestDefaults = ThresholdConfig{
		Disk: &HysteresisThreshold{Trigger: 80, Clear: 75},
	}
	m.config.Overrides = map[string]ThresholdConfig{
		"guest-disk:pve1:node1:101/disk:boot-dev-vda1": {Disabled: true},
	}
	m.mu.Unlock()

	m.CheckGuest(models.VM{
		ID:       guestID,
		VMID:     101,
		Name:     "flatcar",
		Node:     "node1",
		Instance: "pve1",
		Status:   "running",
		Disk:     models.Disk{Usage: 97},
		Disks: []models.Disk{
			{Mountpoint: "/boot", Device: "/dev/vda1", Usage: 97, Total: 100, Used: 97, Free: 3},
			{Mountpoint: "/", Device: "/dev/vda2", Usage: 50, Total: 100, Used: 50, Free: 50},
		},
	}, "pve1")

	bootAlertID := canonicalMetricStateID(guestID+"-disk-boot-dev-vda1", "disk")
	rootAlertID := canonicalMetricStateID(guestID+"-disk-dev-vda2", "disk")
	aggregateAlertID := canonicalMetricStateID(guestID, "disk")

	m.mu.RLock()
	defer m.mu.RUnlock()
	if _, exists := testLookupActiveAlert(t, m, bootAlertID); exists {
		t.Fatalf("disabled /boot filesystem created alert %q", bootAlertID)
	}
	if _, exists := testLookupActiveAlert(t, m, rootAlertID); exists {
		t.Fatalf("healthy root filesystem created alert %q", rootAlertID)
	}
	if _, exists := testLookupActiveAlert(t, m, aggregateAlertID); exists {
		t.Fatalf("disabled /boot filesystem leaked into aggregate alert %q", aggregateAlertID)
	}
}

func TestGuestFilesystemThresholdOverrideOwnsMountEvaluation(t *testing.T) {
	m := newTestManager(t)
	guestID := BuildGuestKey("pve1", "node1", 101)

	m.mu.Lock()
	m.config.TimeThresholds = map[string]int{}
	m.config.GuestDefaults = ThresholdConfig{
		Disk: &HysteresisThreshold{Trigger: 80, Clear: 75},
	}
	m.config.Overrides = map[string]ThresholdConfig{
		"guest-disk:guest:pve1:101/disk:boot-dev-vda1": {
			Disk: &HysteresisThreshold{Trigger: 99, Clear: 95},
		},
	}
	aggregateAlertID := canonicalMetricStateID(guestID, "disk")
	m.activeAlerts[aggregateAlertID] = &Alert{
		ID:         aggregateAlertID,
		ResourceID: guestID,
		Type:       "disk",
		Value:      97,
	}
	m.mu.Unlock()

	vm := models.VM{
		ID:       guestID,
		VMID:     101,
		Name:     "flatcar",
		Node:     "node1",
		Instance: "pve1",
		Status:   "running",
		Disks: []models.Disk{
			{Mountpoint: "/boot", Device: "/dev/vda1", Usage: 97, Total: 100, Used: 97, Free: 3},
		},
	}
	m.CheckGuest(vm, "pve1")

	bootAlertID := canonicalMetricStateID(guestID+"-disk-boot-dev-vda1", "disk")
	m.mu.RLock()
	_, existsAt97 := testLookupActiveAlert(t, m, bootAlertID)
	_, aggregateExists := testLookupActiveAlert(t, m, aggregateAlertID)
	m.mu.RUnlock()
	if existsAt97 {
		t.Fatalf("/boot alert fired below its 99%% override")
	}
	if aggregateExists {
		t.Fatalf("dedicated filesystem override left stale aggregate alert %q", aggregateAlertID)
	}

	vm.Disks[0].Usage = 100
	vm.Disks[0].Used = 100
	vm.Disks[0].Free = 0
	m.CheckGuest(vm, "pve1")

	m.mu.RLock()
	defer m.mu.RUnlock()
	if _, exists := testLookupActiveAlert(t, m, bootAlertID); !exists {
		t.Fatalf("/boot alert did not fire above its 99%% override")
	}
}

func TestGuestFilesystemEvidenceSuppressesDuplicateAggregateAlert(t *testing.T) {
	m := newTestManager(t)
	guestID := BuildGuestKey("pve1", "node1", 109)
	m.mu.Lock()
	m.config.TimeThresholds = map[string]int{}
	m.config.GuestDefaults = ThresholdConfig{Disk: &HysteresisThreshold{Trigger: 80, Clear: 75}}
	m.mu.Unlock()

	m.CheckGuest(models.VM{
		ID: guestID, VMID: 109, Name: "pulse-dev", Node: "node1", Instance: "pve1", Status: "running",
		Disk:  models.Disk{Usage: 96.5},
		Disks: []models.Disk{{Mountpoint: "/", Device: "/dev/vda1", Usage: 96.9, Total: 100, Used: 97, Free: 3}},
	}, "pve1")

	aggregateID := canonicalMetricStateID(guestID, "disk")
	filesystemID := canonicalMetricStateID(guestID+"-disk-dev-vda1", "disk")
	m.mu.RLock()
	defer m.mu.RUnlock()
	if _, exists := testLookupActiveAlert(t, m, aggregateID); exists {
		t.Fatalf("aggregate alert %q duplicated filesystem evidence", aggregateID)
	}
	if _, exists := testLookupActiveAlert(t, m, filesystemID); !exists {
		t.Fatalf("actionable filesystem alert %q was not created", filesystemID)
	}
}

// A config save re-judges every active alert. A filesystem alert must be judged
// by the threshold its evaluator used for that filesystem, not the host or guest
// default: saving any unrelated setting used to resolve an alert a per-disk
// threshold of 70 raised at 80 under a default of 90, notifying recovery, and
// the next poll raised it again as a new alert.
func TestConfigSaveJudgesFilesystemAlertsByTheirOwnThreshold(t *testing.T) {
	perDisk := func() *HysteresisThreshold { return &HysteresisThreshold{Trigger: 70, Clear: 65} }
	abovePerDisk := func() *HysteresisThreshold { return &HysteresisThreshold{Trigger: 85, Clear: 80} }

	agentHost := func(device string) models.Host {
		return models.Host{
			ID: "host-fs", Hostname: "host-fs", Platform: "linux", Status: "online",
			IntervalSeconds: 30, LastSeen: time.Now(),
			Disks: []models.Disk{{Mountpoint: "/data", Device: device, Usage: 80, Total: 100, Used: 80, Free: 20}},
		}
	}
	guestID := BuildGuestKey("pve1", "node1", 101)
	guestVM := models.VM{
		ID: guestID, VMID: 101, Name: "db", Node: "node1", Instance: "pve1", Status: "running",
		Disks: []models.Disk{{Mountpoint: "/data", Device: "/dev/vdb1", Usage: 80, Total: 100, Used: 80, Free: 20}},
	}
	guestAlertID := canonicalMetricStateID(guestID+"-disk-data-dev-vdb1", "disk")

	type diskCase struct {
		name      string
		alertID   string
		configure func(cfg *AlertConfig)
		evaluate  func(m *Manager)
		// raise lifts the threshold that judges this filesystem above its reading.
		raise func(cfg *AlertConfig)
	}

	agentDiskID, _ := hostDiskResourceID(agentHost("/dev/sdb1"), agentHost("/dev/sdb1").Disks[0])
	guestDiskOverrideID := guestDiskOverrideKey(guestVM, guestID, "/data-/dev/vdb1")
	guestOverrideID := guestOverridePrimaryKey(guestVM, guestID)

	cases := []diskCase{
		{
			name:    "agent filesystem override",
			alertID: canonicalMetricStateID(agentDiskID, "disk"),
			configure: func(cfg *AlertConfig) {
				cfg.Overrides = map[string]ThresholdConfig{agentDiskID: {Disk: perDisk()}}
			},
			evaluate: func(m *Manager) { m.CheckHost(agentHost("/dev/sdb1")) },
			raise: func(cfg *AlertConfig) {
				cfg.Overrides = map[string]ThresholdConfig{agentDiskID: {Disk: abovePerDisk()}}
			},
		},
		{
			name:    "agent disk fill by hardware type",
			alertID: canonicalMetricStateID(agentDiskID, "disk"),
			configure: func(cfg *AlertConfig) {
				cfg.DiskFillByType["nvme"] = *perDisk()
			},
			evaluate: func(m *Manager) { m.CheckHost(agentHost("/dev/nvme0n1p1")) },
			raise: func(cfg *AlertConfig) {
				cfg.DiskFillByType["nvme"] = *abovePerDisk()
			},
		},
		{
			name:    "guest filesystem override",
			alertID: guestAlertID,
			configure: func(cfg *AlertConfig) {
				cfg.Overrides = map[string]ThresholdConfig{guestDiskOverrideID: {Disk: perDisk()}}
			},
			evaluate: func(m *Manager) { m.CheckGuest(guestVM, "pve1") },
			raise: func(cfg *AlertConfig) {
				cfg.Overrides = map[string]ThresholdConfig{guestDiskOverrideID: {Disk: abovePerDisk()}}
			},
		},
		{
			name:    "guest override on a filesystem alert",
			alertID: guestAlertID,
			configure: func(cfg *AlertConfig) {
				cfg.Overrides = map[string]ThresholdConfig{guestOverrideID: {Disk: perDisk()}}
			},
			evaluate: func(m *Manager) { m.CheckGuest(guestVM, "pve1") },
			raise: func(cfg *AlertConfig) {
				cfg.Overrides = map[string]ThresholdConfig{guestOverrideID: {Disk: abovePerDisk()}}
			},
		},
	}

	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			m := newTestManager(t)
			cfg := m.GetConfig()
			cfg.Enabled = true
			// UpdateConfig fills missing delays with 5s; zero fires at once.
			cfg.TimeThresholds = map[string]int{"agent": 0, "guest": 0}
			cfg.AgentDefaults.Disk = &HysteresisThreshold{Trigger: 90, Clear: 85}
			cfg.GuestDefaults.Disk = &HysteresisThreshold{Trigger: 90, Clear: 85}
			tc.configure(&cfg)
			m.UpdateConfig(cfg)

			tc.evaluate(m)
			startedAt := testRequireActiveAlert(t, m, tc.alertID).StartTime

			m.UpdateConfig(m.GetConfig())
			if !testHasActiveAlert(t, m, tc.alertID) {
				t.Fatalf("an unchanged config save resolved %q, which its own threshold of 70 still holds at 80", tc.alertID)
			}
			tc.evaluate(m)
			if got := testRequireActiveAlert(t, m, tc.alertID).StartTime; !got.Equal(startedAt) {
				t.Fatalf("alert %q was raised again after the save: started %v, want %v", tc.alertID, got, startedAt)
			}

			cfg = m.GetConfig()
			tc.raise(&cfg)
			m.UpdateConfig(cfg)
			if testHasActiveAlert(t, m, tc.alertID) {
				t.Fatalf("a save lifting %q's own trigger above its reading kept the alert", tc.alertID)
			}
		})
	}
}

// Regression: checkMetric stores canonical-identity alerts under the
// canonical state key, so hysteresis resolution must not remove only the
// unregistered legacy "<resourceID>-<metric>" ID.
func TestCheckMetricResolveRemovesCanonicallyKeyedAlert(t *testing.T) {
	manager := NewManagerWithDataDir(t.TempDir(), WithoutPersistedAlertRestore())
	t.Cleanup(manager.Stop)

	manager.mu.Lock()
	manager.config.Enabled = true
	manager.config.TimeThresholds = map[string]int{}
	manager.mu.Unlock()

	threshold := &HysteresisThreshold{Trigger: 80, Clear: 75}
	resourceID := "guest-123/disk:root"

	manager.checkMetric(resourceID, "test-vm", "node1", "qemu/123", "VM", "disk", 85, threshold, nil)

	manager.mu.Lock()
	activeAfterFire := len(manager.activeAlerts)
	manager.mu.Unlock()
	if activeAfterFire != 1 {
		t.Fatalf("active alerts after fire = %d, want 1", activeAfterFire)
	}

	manager.checkMetric(resourceID, "test-vm", "node1", "qemu/123", "VM", "disk", 70, threshold, nil)

	manager.mu.Lock()
	activeAfterResolve := len(manager.activeAlerts)
	manager.mu.Unlock()
	if activeAfterResolve != 0 {
		t.Fatalf("active alerts after resolve = %d, want 0 (stale alert left behind)", activeAfterResolve)
	}

	if resolved := manager.GetResolvedAlert(buildCanonicalStateID(resourceID, "metric-threshold:disk")); resolved == nil {
		t.Fatal("expected a recently-resolved entry for the cleared alert")
	}
}

// DiskTemperatureThreshold is the policy Patrol and the Physical Disks Health
// verdict judge disk heat by, so it must resolve every disk type exactly as
// CheckHost does: the per-type trigger the user set, else the agent default,
// and nothing at all once the agent default is switched off.
func TestDiskTemperatureThresholdMatchesCheckHostPolicy(t *testing.T) {
	m := configureDiskTempTypeHostManager(t)

	for diskType, want := range map[string]HysteresisThreshold{
		"nvme":  {Trigger: 70, Clear: 65},
		" NVMe": {Trigger: 70, Clear: 65},
		"sas":   {Trigger: 65, Clear: 60},
		"sata":  {Trigger: 55, Clear: 50},
		"hdd":   {Trigger: 55, Clear: 50},
		"":      {Trigger: 55, Clear: 50},
	} {
		if got := m.DiskTemperatureThreshold(diskType); got == nil || *got != want {
			t.Errorf("DiskTemperatureThreshold(%q) = %+v, want %+v", diskType, got, want)
		}
		if got := DefaultDiskTemperatureThreshold(diskType); got == nil || *got != want {
			t.Errorf("DefaultDiskTemperatureThreshold(%q) = %+v, want the factory %+v", diskType, got, want)
		}
	}

	// A user who raised the NVMe trigger to 75 gets no alert at 72C, and the
	// policy says the same disk is not hot.
	m.mu.Lock()
	m.config.DiskTempByType["nvme"] = HysteresisThreshold{Trigger: 75, Clear: 70}
	m.mu.Unlock()
	if got := m.DiskTemperatureThreshold("nvme"); got == nil || got.Trigger != 75 || got.Clear != 70 {
		t.Fatalf("raised nvme threshold = %+v, want 75/70", got)
	}
	host := hostWithSMARTDiskTemp("host-temp-raised-nvme", "nvme", 72)
	m.CheckHost(host)
	if _, exists := testLookupActiveAlert(t, m, hostDiskTempAlertID(host)); exists {
		t.Fatalf("expected no alert for nvme at 72C under a raised 75C trigger, active: %v", alertKeys(m))
	}

	// The returned threshold is a copy.
	m.DiskTemperatureThreshold("nvme").Trigger = 1
	if got := m.DiskTemperatureThreshold("nvme"); got.Trigger != 75 {
		t.Fatalf("caller mutation reached the config: %+v", got)
	}

	m.mu.Lock()
	m.config.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 0, Clear: 0}
	m.mu.Unlock()
	for _, diskType := range []string{"nvme", "sata", "hdd"} {
		if got := m.DiskTemperatureThreshold(diskType); got == nil || got.Trigger > 0 {
			t.Errorf("DiskTemperatureThreshold(%q) with disk temperature alerting off = %+v, want a disabled threshold", diskType, got)
		}
	}

	var nilManager *Manager
	if got := nilManager.DiskTemperatureThreshold("nvme"); got == nil || got.Trigger != 70 {
		t.Fatalf("nil manager threshold = %+v, want the factory nvme 70", got)
	}
}

// An agent owns a usage metric only with usable evidence: it evaluated the
// metric this report, or it still holds an alert or run for it. Configuration
// alone is not coverage.
func TestCheckNodeKeepsMetricsTheAgentCannotEvaluate(t *testing.T) {
	setup := func(t *testing.T) (*Manager, models.Node, models.Host) {
		m := newTestManager(t)
		m.mu.Lock()
		m.config.Enabled = true
		m.config.TimeThresholds = map[string]int{}
		m.config.NodeDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 75}
		m.config.NodeDefaults.Memory = &HysteresisThreshold{Trigger: 85, Clear: 80}
		m.config.AgentDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 75}
		m.config.AgentDefaults.Memory = &HysteresisThreshold{Trigger: 85, Clear: 80}
		m.mu.Unlock()
		node, host := testNodeWithHostAgent()
		node.CPU = 0.95
		node.Memory = models.Memory{Total: 100, Used: 95, Free: 5, Usage: 95}
		return m, node, host
	}

	t.Run("memory_unknown_from_the_start", func(t *testing.T) {
		m, node, host := setup(t)
		host.Memory = models.Memory{Total: 100, UsageUnavailable: true}
		m.CheckHost(host)
		m.CheckNode(node)
		if !testHasActiveAlert(t, m, canonicalMetricStateID(node.ID, "memory")) {
			t.Fatalf("expected the node to alert on memory while the agent has no memory reading")
		}
	})

	t.Run("non_finite_cpu", func(t *testing.T) {
		m, node, host := setup(t)
		host.CPUUsage = math.NaN()
		m.CheckHost(host)
		m.CheckNode(node)
		if !testHasActiveAlert(t, m, canonicalMetricStateID(node.ID, "cpu")) {
			t.Fatalf("expected the node to alert on CPU while the agent reports no usable CPU value")
		}
	})
}

// Turning an agent threshold off releases its pending run even when the agent
// reports no reading for that metric, so the old run cannot keep ownership and
// leave the node silent.
func TestDisabledAgentThresholdReleasesPendingRunWithoutReading(t *testing.T) {
	m := newTestManager(t)
	m.mu.Lock()
	m.config.Enabled = true
	m.config.TimeThresholds = map[string]int{}
	m.config.MetricTimeThresholds = map[string]map[string]int{"agent": {"memory": 300}}
	m.config.NodeDefaults.Memory = &HysteresisThreshold{Trigger: 85, Clear: 80}
	m.config.AgentDefaults.Memory = &HysteresisThreshold{Trigger: 85, Clear: 80}
	m.mu.Unlock()
	node, host := testNodeWithHostAgent()
	// 90% stays below the 95% critical level, which would bypass the delay.
	node.Memory = models.Memory{Total: 100, Used: 90, Free: 10, Usage: 90}
	host.Memory = node.Memory
	agentResourceID := hostResourceID(host.ID)
	agentMemorySpec := canonicalMetricSpecID(agentResourceID, "memory")

	m.CheckHost(host)
	m.mu.RLock()
	pending := testCoreIsPending(m, agentResourceID, agentMemorySpec)
	m.mu.RUnlock()
	if !pending {
		t.Fatalf("expected a pending agent memory run")
	}

	m.mu.Lock()
	m.config.AgentDefaults.Memory = &HysteresisThreshold{Trigger: 0, Clear: 0}
	m.mu.Unlock()
	host.Memory = models.Memory{Total: 100, UsageUnavailable: true}
	m.CheckHost(host)
	m.CheckNode(node)

	m.mu.RLock()
	agentIncident := testCoreHasIncident(m, agentResourceID, agentMemorySpec)
	nodeIncident := testCoreHasIncident(m, node.ID, canonicalMetricSpecID(node.ID, "memory"))
	m.mu.RUnlock()
	if agentIncident {
		t.Fatalf("expected the disabled agent memory threshold to release its pending run")
	}
	if !nodeIncident {
		t.Fatalf("expected the node to evaluate memory once the agent no longer covers it")
	}
}

// Ownership follows the config at once and link updates apply in report order:
// a config change revokes what it no longer supports without waiting for the
// agent's next report, an older report can neither restore revoked ownership
// nor remove a newer report's link, and a report started before the agent went
// offline is stale.
func TestHostAgentNodeLinkFollowsConfigAndReportOrder(t *testing.T) {
	m := newTestManager(t)
	m.mu.Lock()
	m.config.Enabled = true
	m.config.TimeThresholds = map[string]int{}
	m.config.NodeDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 75}
	m.config.AgentDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 75}
	m.mu.Unlock()
	node, host := testNodeWithHostAgent()
	node.CPU = 0.95
	host.CPUUsage = 50

	m.CheckHost(host)
	m.CheckNode(node)
	nodeCPUSpec := canonicalMetricSpecID(node.ID, "cpu")
	m.mu.RLock()
	evaluatedBefore := testCoreHasIncident(m, node.ID, nodeCPUSpec)
	m.mu.RUnlock()
	if evaluatedBefore {
		t.Fatalf("expected the agent to own CPU before the config change")
	}

	// A report already in flight when agent alerts are disabled must not
	// restore ownership once it lands.
	inFlight := m.beginHostAgentReport(host.ID)

	// No agent report follows the config change; the node must evaluate CPU on
	// its very next check. (UpdateConfig restores the default activation delay,
	// so the run is pending rather than firing.)
	config := m.GetConfig()
	config.DisableAllAgents = true
	m.UpdateConfig(config)
	m.CheckNode(node)
	m.mu.RLock()
	evaluatedAfter := testCoreHasIncident(m, node.ID, nodeCPUSpec)
	m.mu.RUnlock()
	if !evaluatedAfter {
		t.Fatalf("expected the node to evaluate CPU right after agent alerts were disabled")
	}

	covering := hostAgentNodeLink{agentID: host.ID, nodeID: node.ID, cpu: true}
	m.applyHostAgentNodeLink(covering, inFlight)
	if m.hasHostAgentForNode(node.ID) {
		t.Fatalf("expected a report evaluated before agent alerts were disabled not to restore ownership")
	}

	config.DisableAllAgents = false
	m.UpdateConfig(config)
	older := m.beginHostAgentReport(host.ID)
	newer := m.beginHostAgentReport(host.ID)
	m.applyHostAgentNodeLink(covering, newer)
	m.applyHostAgentNodeLink(hostAgentNodeLink{agentID: host.ID, nodeID: node.ID}, older)
	if !m.hasHostAgentForNode(node.ID) {
		t.Fatalf("expected an older report not to remove the newer report's link")
	}

	olderCovering := m.beginHostAgentReport(host.ID)
	newerRemoving := m.beginHostAgentReport(host.ID)
	m.applyHostAgentNodeLink(hostAgentNodeLink{agentID: host.ID, nodeID: node.ID}, newerRemoving)
	m.applyHostAgentNodeLink(covering, olderCovering)
	if m.hasHostAgentForNode(node.ID) {
		t.Fatalf("expected an older report not to restore ownership a newer report removed")
	}

	beforeOffline := m.beginHostAgentReport(host.ID)
	m.HandleHostOffline(host)
	m.applyHostAgentNodeLink(covering, beforeOffline)
	if m.hasHostAgentForNode(node.ID) {
		t.Fatalf("expected a report started before the agent went offline not to re-register")
	}
}

// An agent that reports unlinked hands its former node the usage metrics back
// on the node's next check, even while the agent's own alert is still firing.
func TestCheckNodeTakesBackMetricsWhenAgentUnlinks(t *testing.T) {
	m := newTestManager(t)
	m.mu.Lock()
	m.config.Enabled = true
	m.config.TimeThresholds = map[string]int{}
	m.config.NodeDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 75}
	m.config.AgentDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 75}
	m.mu.Unlock()
	node, host := testNodeWithHostAgent()
	node.CPU = 0.95
	host.CPUUsage = 95

	m.CheckHost(host)
	m.CheckNode(node)
	nodeCPU := canonicalMetricStateID(node.ID, "cpu")
	if testHasActiveAlert(t, m, nodeCPU) {
		t.Fatalf("expected the linked agent to own CPU")
	}

	host.LinkedNodeID = ""
	m.CheckHost(host)
	m.CheckNode(node)
	if !testHasActiveAlert(t, m, nodeCPU) {
		t.Fatalf("expected the node to alert on CPU once its agent reports unlinked")
	}
}

// An explicit disk override, including one inherited from the linked node,
// beats the per-type DiskFillByType default; otherwise the node released its
// disk alert to an agent applying a looser per-type threshold.
func TestHostDiskOverrideBeatsDiskFillByType(t *testing.T) {
	for _, overrideOn := range []string{"linked node", "agent"} {
		t.Run(overrideOn, func(t *testing.T) {
			m := newTestManager(t)
			node, host := testNodeWithHostAgent()
			overrideID := node.ID
			if overrideOn == "agent" {
				overrideID = host.ID
			}
			m.mu.Lock()
			m.config.Enabled = true
			m.config.TimeThresholds = map[string]int{}
			m.config.AgentDefaults.Disk = &HysteresisThreshold{Trigger: 90, Clear: 85}
			m.config.DiskFillByType = map[string]HysteresisThreshold{"nvme": {Trigger: 92, Clear: 88}}
			m.config.Overrides = map[string]ThresholdConfig{
				overrideID: {Disk: &HysteresisThreshold{Trigger: 80, Clear: 75}},
			}
			m.mu.Unlock()
			root := models.Disk{Mountpoint: "/", Device: "/dev/nvme0n1p2", Total: 100, Used: 85, Free: 15, Usage: 85}
			host.Disks = []models.Disk{root}

			m.CheckHost(host)
			rootResourceID, _ := hostDiskResourceID(host, root)
			if !testHasActiveAlert(t, m, canonicalMetricStateID(rootResourceID, "disk")) {
				t.Fatalf("expected the 80%% %s disk override to raise the agent's root disk alert at 85%%", overrideOn)
			}

			// Config-save reevaluation must preserve the same explicit override
			// when the upstream per-filesystem resolver is composed with agent
			// ownership. The looser NVMe default must not clear this occurrence.
			alertID := canonicalMetricStateID(rootResourceID, "disk")
			before, _ := testLookupActiveAlert(t, m, alertID)
			m.UpdateConfig(m.GetConfig())
			after, exists := testLookupActiveAlert(t, m, alertID)
			if !exists || after.StartTime != before.StartTime {
				t.Fatalf("config save lost the %s disk override occurrence", overrideOn)
			}
			m.CheckHost(host)
			if !testHasActiveAlert(t, m, alertID) {
				t.Fatalf("next host report lost the %s disk override alert", overrideOn)
			}
		})
	}
}

// Releasing a metric drops its explicit intent grace run too, so a later
// resume does not activate at once on time counted while nothing evaluated.
func TestReleaseCanonicalMetricAlertClearsIntentPending(t *testing.T) {
	m := newTestManager(t)
	spec, err := buildCanonicalMetricSpec("homelab-delly2", "delly2", "node", "cpu", nil)
	if err != nil {
		t.Fatalf("spec: %v", err)
	}
	trackingKey := canonicalTrackingKeyForSpec(spec, spec.ID)
	m.mu.Lock()
	m.intentPending[trackingKey] = IntentPendingState{
		TrackingKey:    trackingKey,
		ResourceID:     spec.ResourceID,
		Signal:         MetricAlertIntentSignal("cpu"),
		FirstMatchedAt: time.Now().Add(-10 * time.Minute),
		LastObservedAt: time.Now().Add(-9 * time.Minute),
	}
	m.mu.Unlock()

	m.releaseCanonicalMetricAlert(spec, "delly2", "delly2", "homelab", "node", 0, nil)

	m.mu.RLock()
	_, stillPending := m.intentPending[trackingKey]
	m.mu.RUnlock()
	if stillPending {
		t.Fatalf("expected releasing the metric to clear its intent pending state")
	}
}

// Missing storage observations break confirmation runs, not the occurrence.
// Exercise the public evaluator and both live/shadow reducers, not seeded state.
func TestStorageConnectivityObservationGaps(t *testing.T) {
	for _, gap := range []string{"", "unknown", " UNKNOWN "} {
		t.Run("activation/"+gap, func(t *testing.T) {
			m := newShadowFeedManager(t)
			s := models.Storage{ID: "storage-a", Name: "backups", Status: "offline"}
			other := s
			other.ID = "storage-ab"
			id := canonicalConnectivityStateID(s.ID)
			m.CheckStorage(s)
			m.CheckStorage(other)
			s.Status = gap
			m.CheckStorage(s)
			s.Status = "unavailable"
			m.CheckStorage(s)
			if testHasActiveAlert(t, m, id) {
				t.Fatal("non-consecutive outage observations fired storage alert")
			}
			m.CheckStorage(other)
			testRequireActiveAlert(t, m, canonicalConnectivityStateID(other.ID))
			m.CheckStorage(s)
			testRequireActiveAlert(t, m, id)
			if m.ShadowDivergences() != 0 {
				t.Fatal("storage interruption diverged from shadow")
			}
		})
		t.Run("recovery/"+gap, func(t *testing.T) {
			m := newShadowFeedManager(t)
			s := models.Storage{ID: "storage-a", Name: "backups", Status: "offline"}
			id := canonicalConnectivityStateID(s.ID)
			for range 2 {
				m.CheckStorage(s)
			}
			if err := m.AcknowledgeAlert(id, "operator"); err != nil {
				t.Fatal(err)
			}
			s.Status = "available"
			m.CheckStorage(s)
			before := testRequireActiveAlert(t, m, id).Clone()
			s.Status = gap
			m.CheckStorage(s)
			if !reflect.DeepEqual(before, testRequireActiveAlert(t, m, id).Clone()) || m.GetResolvedAlert(id) != nil {
				t.Fatal("missing connectivity changed acknowledged occurrence")
			}
			s.Status = "available"
			m.CheckStorage(s)
			if !testHasActiveAlert(t, m, id) {
				t.Fatal("non-consecutive healthy observations resolved storage outage")
			}
			m.CheckStorage(s)
			if testHasActiveAlert(t, m, id) {
				t.Fatal("fresh consecutive healthy observations failed to recover")
			}
			resolved := m.GetResolvedAlert(id)
			if resolved == nil || !resolved.StartTime.Equal(before.StartTime) {
				t.Fatal("confirmed recovery lost original occurrence")
			}
			if m.ShadowDivergences() != 0 {
				t.Fatal("storage recovery diverged from shadow")
			}
		})
	}
}

func TestStorageConnectivityGapRestartsIntentGrace(t *testing.T) {
	m := newShadowFeedManager(t)
	var tick time.Duration
	m.intentClock = func() time.Duration { return tick }
	doc := NewAlertIntentPolicyDocument()
	doc.Defaults[string(AlertIntentSignalOffline)] = AlertIntentRule{GraceSeconds: intPointer(60)}
	if err := m.LoadIntentPolicies(doc); err != nil {
		t.Fatal(err)
	}
	s := models.Storage{ID: "storage-a", Name: "backups", Status: "offline"}
	id := canonicalConnectivityStateID(s.ID)
	m.CheckStorage(s)
	tick = 40 * time.Second
	m.CheckStorage(s)
	s.Status = "unknown"
	m.CheckStorage(s)
	m.mu.RLock()
	_, pending := m.intentPending[id]
	_, ticking := m.intentRuntimeTicks[id]
	m.mu.RUnlock()
	if pending || ticking {
		t.Error("missing status retained intent grace bookkeeping")
	}
	if err := m.SaveActiveAlerts(); err != nil {
		t.Fatal(err)
	}
	data, err := os.ReadFile(filepath.Join(m.getAlertsDir(), intentPendingFileName))
	if err != nil {
		t.Fatal(err)
	}
	var saved []IntentPendingState
	if err := json.Unmarshal(data, &saved); err != nil || len(saved) != 0 {
		t.Fatalf("interrupted intent remained checkpointed: %s, %v", data, err)
	}
	s.Status = "offline"
	for _, next := range []time.Duration{70, 90, 129} {
		tick = next * time.Second
		m.CheckStorage(s)
		if testHasActiveAlert(t, m, id) {
			t.Fatal("intent grace accrued across missing storage status")
		}
	}
	tick = 130 * time.Second
	m.CheckStorage(s)
	testRequireActiveAlert(t, m, id)
	if m.ShadowDivergences() != 0 {
		t.Fatal("intent interruption diverged from shadow")
	}
}

// Usage has delayed activation but immediate measured recovery. Do not invent
// a recovery stability window to fit the memory/temperature timing fixtures.
func TestStorageCapacityGapHoldsOccurrence(t *testing.T) {
	for _, gap := range []string{"missing", "unconfirmed-zero", "negative", "offline", "unavailable"} {
		t.Run(gap, func(t *testing.T) {
			m, elapsed := continuityManager(t, false)
			id, metric, observe := continuityObserver(t, m, "storage", gap)
			alertID := canonicalMetricStateID(id, metric)
			observe(95, false)
			elapsed.Store(int64(time.Minute))
			observe(95, false)
			if err := m.AcknowledgeAlert(alertID, "operator"); err != nil {
				t.Fatal(err)
			}
			before := testRequireActiveAlert(t, m, alertID).Clone()
			elapsed.Store(int64(2 * time.Minute))
			observe(10, true)
			if !reflect.DeepEqual(before, testRequireActiveAlert(t, m, alertID).Clone()) || m.GetResolvedAlert(alertID) != nil {
				t.Fatal("missing capacity changed the acknowledged occurrence")
			}
			// A real, independently observed empty capacity can clear even
			// while connectivity is unknown; neither channel stands in for the other.
			empty := models.Storage{ID: id, Name: "backups", Status: "unknown", Total: 1000, Free: 1000}
			m.CheckStorage(empty)
			if testHasActiveAlert(t, m, alertID) {
				t.Fatal("confirmed empty capacity did not recover immediately")
			}
			resolved := m.GetResolvedAlert(alertID)
			if resolved == nil || !resolved.StartTime.Equal(before.StartTime) || resolved.Value != 0 {
				t.Fatal("measured recovery lost the original occurrence or clearing value")
			}
			if m.ShadowDivergences() != 0 {
				t.Fatal("capacity interruption diverged from shadow")
			}
		})
	}
}

// HostDiskTemperatureThreshold is the disk heat policy for a disk a host agent
// reports, so it must agree with CheckHost on every host-level override: an
// explicit Disk Temp override on the host (or the canonical resource its ID
// resolves to), or inherited from its linked node or guest, replaces the
// per-type entry; the first override in the chain decides even when it sets no
// Disk Temp; and a host whose alerts are off judges no heat.
func TestHostDiskTemperatureThresholdMatchesCheckHostOverrides(t *testing.T) {
	const hostID = "host-temp-policy"
	const nodeID = "pve1-node1"
	const vmID = "pve1-node1-105"
	const canonicalID = "agent-canonical-temp-policy"
	perType := &HysteresisThreshold{Trigger: 70, Clear: 65}

	for _, tc := range []struct {
		name      string
		overrides map[string]ThresholdConfig
		linkNode  bool
		linkVM    bool
		canonical bool
		want      *HysteresisThreshold
		off       bool
	}{
		{name: "no override", want: perType},
		{
			name:      "host disk temp override",
			overrides: map[string]ThresholdConfig{hostID: {DiskTemperature: &HysteresisThreshold{Trigger: 80, Clear: 75}}},
			want:      &HysteresisThreshold{Trigger: 80, Clear: 75},
		},
		{
			name:      "host override without disk temp",
			overrides: map[string]ThresholdConfig{hostID: {Memory: &HysteresisThreshold{Trigger: 90, Clear: 85}}},
			want:      perType,
		},
		{
			name:      "linked node disk temp override",
			overrides: map[string]ThresholdConfig{nodeID: {DiskTemperature: &HysteresisThreshold{Trigger: 78, Clear: 72}}},
			linkNode:  true,
			want:      &HysteresisThreshold{Trigger: 78, Clear: 72},
		},
		{
			name: "host override shadows linked node",
			overrides: map[string]ThresholdConfig{
				hostID: {Memory: &HysteresisThreshold{Trigger: 90, Clear: 85}},
				nodeID: {DiskTemperature: &HysteresisThreshold{Trigger: 78, Clear: 72}},
			},
			linkNode: true,
			want:     perType,
		},
		{
			// The agent ID resolves to the machine's canonical resource ID.
			name:      "canonical machine disk temp override",
			overrides: map[string]ThresholdConfig{canonicalID: {DiskTemperature: &HysteresisThreshold{Trigger: 82, Clear: 77}}},
			canonical: true,
			want:      &HysteresisThreshold{Trigger: 82, Clear: 77},
		},
		{
			name:      "linked guest disk temp override",
			overrides: map[string]ThresholdConfig{vmID: {DiskTemperature: &HysteresisThreshold{Trigger: 76, Clear: 71}}},
			linkVM:    true,
			want:      &HysteresisThreshold{Trigger: 76, Clear: 71},
		},
		{
			name:      "linked guest alerts disabled",
			overrides: map[string]ThresholdConfig{vmID: {Disabled: true}},
			linkVM:    true,
		},
		{
			name:      "host disk temp switched off",
			overrides: map[string]ThresholdConfig{hostID: {DiskTemperature: &HysteresisThreshold{}}},
			off:       true,
		},
		{
			name:      "host alerts disabled",
			overrides: map[string]ThresholdConfig{hostID: {Disabled: true}},
		},
		{
			name:      "linked node alerts disabled",
			overrides: map[string]ThresholdConfig{nodeID: {Disabled: true}},
			linkNode:  true,
		},
	} {
		t.Run(tc.name, func(t *testing.T) {
			m := configureDiskTempTypeHostManager(t)
			m.mu.Lock()
			for id, override := range tc.overrides {
				m.config.Overrides[id] = override
			}
			m.mu.Unlock()
			if tc.canonical {
				m.SetResourceIntentIdentityResolver(func(resourceID string) (string, bool) {
					return canonicalID, resourceID == hostID
				})
			}

			owner := DiskTemperatureHost{ID: hostID}
			if tc.linkNode {
				owner.LinkedNodeID = nodeID
			}
			if tc.linkVM {
				owner.LinkedVMID = vmID
			}
			got := m.HostDiskTemperatureThreshold(owner, "nvme")
			switch {
			case tc.off:
				if got == nil || got.Trigger > 0 {
					t.Fatalf("HostDiskTemperatureThreshold = %+v, want a switched-off threshold", got)
				}
			case (got == nil) != (tc.want == nil) || (got != nil && *got != *tc.want):
				t.Fatalf("HostDiskTemperatureThreshold = %+v, want %+v", got, tc.want)
			}

			check := func(temperature int) bool {
				host := hostWithSMARTDiskTemp(hostID, "nvme", temperature)
				host.LinkedNodeID = owner.LinkedNodeID
				host.LinkedVMID = owner.LinkedVMID
				m.CheckHost(host)
				_, exists := testLookupActiveAlert(t, m, hostDiskTempAlertID(host))
				return exists
			}
			if got == nil || got.Trigger <= 0 {
				if check(99) {
					t.Fatalf("CheckHost alerted at 99C on a host the policy judges no heat for, active: %v", alertKeys(m))
				}
				return
			}
			trigger := int(got.Trigger)
			if check(trigger - 1) {
				t.Fatalf("CheckHost alerted at %dC, under the policy trigger %v", trigger-1, got.Trigger)
			}
			if !check(trigger) {
				t.Fatalf("CheckHost stayed quiet at the policy trigger %dC, active: %v", trigger, alertKeys(m))
			}
		})
	}

	m := configureDiskTempTypeHostManager(t)
	if got, want := m.HostDiskTemperatureThreshold(DiskTemperatureHost{}, "sas"), m.DiskTemperatureThreshold("sas"); got == nil || want == nil || *got != *want {
		t.Fatalf("hostless policy = %+v, want DiskTemperatureThreshold %+v", got, want)
	}
	var nilManager *Manager
	if got := nilManager.HostDiskTemperatureThreshold(DiskTemperatureHost{ID: hostID}, "nvme"); got == nil || got.Trigger != 70 {
		t.Fatalf("nil manager host policy = %+v, want the factory nvme 70", got)
	}
}

// TrueNASDiskTemperatureThreshold is the disk heat policy for a TrueNAS disk,
// so it must agree with the disk's temperature alert on every tier: the
// disk's own override, then the TrueNAS-wide value, then the per-type policy.
// A host override under the TrueNAS system's synthetic agent ID is not one of
// them. Disable all TrueNAS silences the alert without changing the policy,
// as the agent switches do for agent disks.
func TestTrueNASDiskTemperatureThresholdMatchesTrueNASDiskAlerts(t *testing.T) {
	disk := trueNASTemperatureDisk("nvme0n1", "nvme", 0)

	const canonicalDiskID = "physical-disk:canonical-nvme0n1"
	for _, tc := range []struct {
		name     string
		mutate   func(*AlertConfig)
		setup    func(*Manager)
		want     *HysteresisThreshold
		off      bool
		silenced bool
	}{
		{name: "per-type policy", want: &HysteresisThreshold{Trigger: 70, Clear: 65}},
		{
			name: "raised per-type trigger",
			mutate: func(cfg *AlertConfig) {
				cfg.DiskTempByType = map[string]HysteresisThreshold{"nvme": {Trigger: 75, Clear: 70}}
			},
			want: &HysteresisThreshold{Trigger: 75, Clear: 70},
		},
		{
			name: "TrueNAS-wide value",
			mutate: func(cfg *AlertConfig) {
				cfg.TrueNASDiskDefaults.Temperature = &HysteresisThreshold{Trigger: 62, Clear: 57}
			},
			want: &HysteresisThreshold{Trigger: 62, Clear: 57},
		},
		{
			name: "per-disk override beats the TrueNAS-wide value",
			mutate: func(cfg *AlertConfig) {
				cfg.TrueNASDiskDefaults.Temperature = &HysteresisThreshold{Trigger: 62, Clear: 57}
				cfg.Overrides = map[string]ThresholdConfig{disk.ID: {Temperature: &HysteresisThreshold{Trigger: 75, Clear: 70}}}
			},
			want: &HysteresisThreshold{Trigger: 75, Clear: 70},
		},
		{
			name: "per-disk override without a clear value",
			mutate: func(cfg *AlertConfig) {
				cfg.Overrides = map[string]ThresholdConfig{disk.ID: {Temperature: &HysteresisThreshold{Trigger: 75}}}
			},
			want: &HysteresisThreshold{Trigger: 75, Clear: 70},
		},
		{
			name: "per-disk override under the canonical identity",
			mutate: func(cfg *AlertConfig) {
				cfg.Overrides = map[string]ThresholdConfig{canonicalDiskID: {Temperature: &HysteresisThreshold{Trigger: 78, Clear: 73}}}
			},
			setup: func(m *Manager) {
				m.SetResourceIntentIdentityResolver(func(resourceID string) (string, bool) {
					return canonicalDiskID, resourceID == disk.ID
				})
			},
			want: &HysteresisThreshold{Trigger: 78, Clear: 73},
		},
		{
			name: "host override under the TrueNAS system ID does not reach its disks",
			mutate: func(cfg *AlertConfig) {
				cfg.Overrides = map[string]ThresholdConfig{"truenas-main": {DiskTemperature: &HysteresisThreshold{Trigger: 50, Clear: 45}}}
			},
			want: &HysteresisThreshold{Trigger: 70, Clear: 65},
		},
		{
			name: "per-disk override switches the disk off",
			mutate: func(cfg *AlertConfig) {
				cfg.Overrides = map[string]ThresholdConfig{disk.ID: {Temperature: &HysteresisThreshold{}}}
			},
			off: true,
		},
		{
			name: "per-disk override disables the disk's alerts",
			mutate: func(cfg *AlertConfig) {
				cfg.Overrides = map[string]ThresholdConfig{disk.ID: {Disabled: true}}
			},
		},
		{
			name: "TrueNAS-wide off",
			mutate: func(cfg *AlertConfig) {
				cfg.TrueNASDiskDefaults.Temperature = &HysteresisThreshold{}
			},
			off: true,
		},
		{
			name: "TrueNAS Disks defaults disabled",
			mutate: func(cfg *AlertConfig) {
				cfg.TrueNASDiskDefaults.Disabled = true
			},
		},
		{
			name: "agent Disk Temp default off",
			mutate: func(cfg *AlertConfig) {
				cfg.AgentDefaults.DiskTemperature = &HysteresisThreshold{}
			},
			off: true,
		},
		{
			name: "Disable all TrueNAS keeps the policy",
			mutate: func(cfg *AlertConfig) {
				cfg.DisableAllTrueNAS = true
			},
			want:     &HysteresisThreshold{Trigger: 70, Clear: 65},
			silenced: true,
		},
	} {
		t.Run(tc.name, func(t *testing.T) {
			m := newTestManager(t)
			cfg := unifiedEvalBaseConfig()
			cfg.TrueNASDiskDefaults = ThresholdConfig{}
			if tc.mutate != nil {
				tc.mutate(&cfg)
			}
			configureUnifiedEvalManager(t, m, cfg)
			if tc.setup != nil {
				tc.setup(m)
			}

			got := m.TrueNASDiskTemperatureThreshold(" "+disk.ID+" ", " NVMe")
			switch {
			case tc.off:
				if got == nil || got.Trigger > 0 {
					t.Fatalf("TrueNASDiskTemperatureThreshold = %+v, want a switched-off threshold", got)
				}
			case (got == nil) != (tc.want == nil) || (got != nil && *got != *tc.want):
				t.Fatalf("TrueNASDiskTemperatureThreshold = %+v, want %+v", got, tc.want)
			}

			var alert *Alert
			fires := func(temperature int) bool {
				reading := disk
				meta := *disk.PhysicalDisk
				meta.Temperature = temperature
				reading.PhysicalDisk = &meta
				checkTrueNASTemperatureDisk(t, m, reading)
				var firing bool
				alert, firing = trueNASDiskTemperatureAlert(t, m, reading)
				return firing
			}
			if tc.silenced {
				if fires(99) {
					t.Fatalf("alert fired at 99C with TrueNAS alerts disabled, active: %v", alertKeys(m))
				}
				return
			}
			if got == nil || got.Trigger <= 0 {
				if fires(99) {
					t.Fatalf("alert fired at 99C on a disk the policy judges no heat for, active: %v", alertKeys(m))
				}
				return
			}
			trigger := int(got.Trigger)
			if fires(trigger - 1) {
				t.Fatalf("alert fired at %dC, under the policy trigger %v", trigger-1, got.Trigger)
			}
			if !fires(trigger) {
				t.Fatalf("alert stayed quiet at the policy trigger %dC, active: %v", trigger, alertKeys(m))
			}
			// The alert recovers at the clear value Patrol reads.
			if alert.Threshold != got.Trigger || alert.Metadata["clearThreshold"] != got.Clear {
				t.Fatalf("alert trigger/clear = %v/%v, want the policy %v/%v", alert.Threshold, alert.Metadata["clearThreshold"], got.Trigger, got.Clear)
			}
		})
	}

	// A disk whose type TrueNAS never reported follows the agent Disk Temp
	// default, as its alert does.
	t.Run("untyped disk", func(t *testing.T) {
		m := newTestManager(t)
		cfg := unifiedEvalBaseConfig()
		cfg.TrueNASDiskDefaults = ThresholdConfig{}
		cfg.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 58, Clear: 53}
		configureUnifiedEvalManager(t, m, cfg)
		untypedDisk := trueNASTemperatureDisk("sdx", "", 58)
		if got := m.TrueNASDiskTemperatureThreshold(untypedDisk.ID, ""); got == nil || got.Trigger != 58 || got.Clear != 53 {
			t.Fatalf("untyped disk = %+v, want the agent default 58/53", got)
		}
		checkTrueNASTemperatureDisk(t, m, untypedDisk)
		if _, firing := trueNASDiskTemperatureAlert(t, m, untypedDisk); !firing {
			t.Fatalf("untyped disk at the agent default 58C raised no alert, active: %v", alertKeys(m))
		}
	})

	m := configureDiskTempTypeHostManager(t)
	m.TrueNASDiskTemperatureThreshold(disk.ID, "nvme").Trigger = 1
	if got := m.TrueNASDiskTemperatureThreshold(disk.ID, "nvme"); got == nil || got.Trigger != 70 {
		t.Fatalf("caller mutation reached the config: %+v", got)
	}
	var nilManager *Manager
	if got := nilManager.TrueNASDiskTemperatureThreshold(disk.ID, "sata"); got == nil || got.Trigger != 55 {
		t.Fatalf("nil manager TrueNAS disk policy = %+v, want the factory sata 55", got)
	}
}

// IsTrueNASDiskResource names the disks the unified evaluator judges as
// TrueNAS disks, the ones TrueNASDiskTemperatureThreshold resolves.
func TestIsTrueNASDiskResourceMatchesTheUnifiedEvaluator(t *testing.T) {
	agentDisk := unifiedresources.Resource{
		ID: "physical-disk:host-1/nvme0n1", Type: unifiedresources.ResourceTypePhysicalDisk,
		Sources:      []unifiedresources.DataSource{unifiedresources.SourceAgent},
		PhysicalDisk: &unifiedresources.PhysicalDiskMeta{DiskType: "nvme"},
	}
	trueNASFacetDisk := agentDisk
	trueNASFacetDisk.TrueNAS = &unifiedresources.TrueNASData{}
	mergedDisk := agentDisk
	mergedDisk.Sources = []unifiedresources.DataSource{unifiedresources.SourceAgent, unifiedresources.SourceTrueNAS}
	noMetaDisk := trueNASTemperatureDisk("sdy", "sata", 40)
	noMetaDisk.PhysicalDisk = nil
	trueNASSystem := unifiedresources.Resource{
		ID: "agent:truenas-main", Type: unifiedresources.ResourceTypeAgent,
		Sources: []unifiedresources.DataSource{unifiedresources.SourceTrueNAS},
	}
	for name, tc := range map[string]struct {
		resource unifiedresources.Resource
		want     bool
	}{
		"TrueNAS-sourced disk": {resource: trueNASTemperatureDisk("sda", "sata", 40), want: true},
		"TrueNAS facet disk":   {resource: trueNASFacetDisk, want: true},
		"agent and TrueNAS":    {resource: mergedDisk, want: true},
		"no disk metadata":     {resource: noMetaDisk, want: true},
		"agent disk":           {resource: agentDisk},
		"TrueNAS system":       {resource: trueNASSystem},
	} {
		if got := IsTrueNASDiskResource(tc.resource); got != tc.want {
			t.Errorf("%s: IsTrueNASDiskResource = %v, want %v", name, got, tc.want)
		}
		input, ok := UnifiedResourceInputFromResource(tc.resource)
		if evaluated := ok && input.Type == "truenas-disk"; evaluated != tc.want {
			t.Errorf("%s: unified evaluator judges it as a TrueNAS disk = %v, want %v", name, evaluated, tc.want)
		}
	}
}