pulse/internal/monitoring/reload_test.go

package monitoring

import (
	"context"
	"errors"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestReloadableMonitor_Lifecycle_Coverage(t *testing.T) {
	// Create minimal config
	cfg := &config.Config{
		DataPath: t.TempDir(),
	}
	persistence := config.NewMultiTenantPersistence(cfg.DataPath)

	// Create ReloadableMonitor
	rm, err := NewReloadableMonitor(cfg, persistence, nil)
	require.NoError(t, err)
	require.NotNil(t, rm)

	// Test GetConfig
	assert.Equal(t, cfg, rm.GetConfig())

	// Test Start
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	rm.Start(ctx)

	// Test GetMultiTenantMonitor
	mtm := rm.GetMultiTenantMonitor()
	require.NotNil(t, mtm)

	// Test GetMonitor (default legacy shim)
	// Should initialize default monitor on demand
	m := rm.GetMonitor()
	require.NotNil(t, m)

	// Test ReadSnapshot (default)
	state := rm.ReadSnapshot("default")
	require.NotNil(t, state)

	// Test ReadSnapshot (non-existent) - should not auto-provision an unprovisioned tenant
	stateMissing := rm.ReadSnapshot("missing-org")
	assert.Nil(t, stateMissing)

	// Test ReadSnapshot with invalid OrgID (should fail persistence check)
	// Assuming "../" or similar might be rejected by GetPersistence or underlying path logic
	// If GetMonitor is robust, checking error branch might require mocking persistence failure.
	// For now, attempting path traversal char.
	// If Pulse cleans it, it might pass. Checking code: persistence joins path.
	// Let's try an error injection if possible, or skip if too complex.
	// Actually, persistence.GetPersistence returns error if newPersistence fails? No, usually succeeds unless mkdir fails.
	// We'll skip complex mocking just for this line, accepting high coverage.

	// Start reload in background
	errChan := make(chan error)
	go func() {
		errChan <- rm.Reload()
	}()

	// Wait for reload (it sleeps for 1s in doReload)
	select {
	case err := <-errChan:
		require.NoError(t, err)
	case <-time.After(3 * time.Second):
		t.Fatal("Reload timed out")
	}

	// Verify internal state after reload (ctx should be new)
	assert.NotNil(t, rm.ctx)

	// Test Stop
	rm.Stop()
}

func TestReloadPreservesLiveAgentCredentialInventory(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())
	runtimeConfig, err := config.LoadWithoutLoggingInit()
	require.NoError(t, err)
	rm, err := NewReloadableMonitor(runtimeConfig, config.NewMultiTenantPersistence(runtimeConfig.DataPath), nil)
	require.NoError(t, err)
	ctx, cancel := context.WithCancel(context.Background())
	t.Cleanup(cancel)
	rm.Start(ctx)
	t.Cleanup(rm.Stop)

	// The server and authentication handlers retain runtimeConfig across reloads.
	// Token creation and revocation replace its slice after the monitor restarts.
	for generation := 0; generation < 2; generation++ {
		require.NoError(t, rm.Reload())
		monitor := rm.GetMonitor()
		require.NotNil(t, monitor)
		now := time.Now().UTC()
		config.Mu.Lock()
		runtimeConfig.APITokens = []config.APITokenRecord{{
			ID: "reload-agent-token", CreatedAt: now, LastUsedAt: &now,
			Scopes: []string{config.ScopeAgentExec},
		}}
		config.Mu.Unlock()
		monitor.state.UpsertHost(models.Host{
			ID: "reload-agent", Hostname: "reload-host", Status: "online",
			LastSeen: now, AgentVersion: "6.4.1", TokenID: "reload-agent-token",
			TokenLastUsedAt: &now, CommandsEnabled: true,
		})
		agent := requireAgentDiagnostic(t, monitor.GetAgentFleetDiagnostics("6.4.1", now), "agent-reload-agent")
		require.False(t, diagnosticHasAnyReason(agent.Reasons,
			AgentFleetReasonCredentialMissing, AgentFleetReasonCredentialUnlisted, AgentFleetReasonExecScopeMissing),
			"live credential is absent from diagnostics after reload: %+v", agent.Reasons)
		require.Same(t, runtimeConfig, rm.GetConfig())
		require.Same(t, runtimeConfig, monitor.GetConfig())

		// Revocation must also reach diagnostics. Recent usage still produces the
		// existing contradiction warning, rather than hiding a missing credential.
		config.Mu.Lock()
		runtimeConfig.APITokens = nil
		config.Mu.Unlock()
		agent = requireAgentDiagnostic(t, monitor.GetAgentFleetDiagnostics("6.4.1", now), "agent-reload-agent")
		requireReasonCode(t, agent, AgentFleetReasonCredentialUnlisted)
	}
}

func TestReloadableMonitorAggregateInstallSnapshotCountsIncludesProvisionedTenants(t *testing.T) {
	baseDir := t.TempDir()
	cfg := &config.Config{DataPath: baseDir}
	persistence := config.NewMultiTenantPersistence(baseDir)

	for _, orgID := range []string{"default", "org-a", "org-b"} {
		_, err := persistence.GetPersistence(orgID)
		require.NoError(t, err)
	}

	rm, err := NewReloadableMonitor(cfg, persistence, nil)
	require.NoError(t, err)

	mtm := rm.GetMultiTenantMonitor()
	require.NotNil(t, mtm)
	mtm.monitors["default"] = testTelemetryMonitor(
		[]models.Node{{ID: "node-default", Name: "node-default", Instance: "pve-default"}},
		[]models.VM{{ID: "vm-default", VMID: 101, Name: "vm-default", Instance: "pve-default"}},
		[]models.Container{{ID: "ct-default", VMID: 201, Name: "ct-default", Instance: "pve-default"}},
		[]models.PBSInstance{{Name: "pbs-default", Host: "pbs-default.local"}},
		[]models.PMGInstance{{Name: "pmg-default", Host: "pmg-default.local"}},
		[]models.DockerHost{{ID: "docker-default", Hostname: "docker-default"}},
		[]models.KubernetesCluster{{ID: "k8s-default", Name: "k8s-default"}},
		1,
	)
	mtm.monitors["org-a"] = testTelemetryMonitor(
		[]models.Node{{ID: "node-a", Name: "node-a", Instance: "pve-a"}},
		[]models.VM{
			{ID: "vm-a1", VMID: 102, Name: "vm-a1", Instance: "pve-a"},
			{ID: "vm-a2", VMID: 103, Name: "vm-a2", Instance: "pve-a"},
		},
		nil,
		[]models.PBSInstance{{Name: "pbs-a", Host: "pbs-a.local"}},
		nil,
		nil,
		nil,
		2,
	)
	mtm.monitors["org-b"] = testTelemetryMonitor(
		nil,
		nil,
		[]models.Container{{ID: "ct-b1", VMID: 202, Name: "ct-b1", Instance: "pve-b"}},
		nil,
		[]models.PMGInstance{{Name: "pmg-b", Host: "pmg-b.local"}},
		[]models.DockerHost{{ID: "docker-b", Hostname: "docker-b"}},
		[]models.KubernetesCluster{{ID: "k8s-b", Name: "k8s-b"}},
		3,
	)

	counts := rm.AggregateInstallSnapshotCounts()

	assert.Equal(t, 2, counts.PVENodes)
	assert.Equal(t, 2, counts.PBSInstances)
	assert.Equal(t, 2, counts.PMGInstances)
	assert.Equal(t, 3, counts.VMs)
	assert.Equal(t, 2, counts.Containers)
	assert.Equal(t, 2, counts.DockerHosts)
	assert.Equal(t, 2, counts.KubernetesClusters)
	assert.Equal(t, 6, counts.ActiveAlerts)
}

func TestAccumulateInstallSnapshotUnifiedResourceCountsUsesCoarseV6AdoptionSignals(t *testing.T) {
	counts := InstallSnapshotCounts{}
	accumulateInstallSnapshotUnifiedResourceCounts(&counts, []unifiedresources.Resource{
		{Type: unifiedresources.ResourceTypeAgent, TrueNAS: &unifiedresources.TrueNASData{}},
		{Type: unifiedresources.ResourceTypeVM, TrueNAS: &unifiedresources.TrueNASData{}},
		{Type: unifiedresources.ResourceTypeAppContainer, TrueNAS: &unifiedresources.TrueNASData{}},
		{Type: unifiedresources.ResourceTypeAgent, VMware: &unifiedresources.VMwareData{}},
		{Type: unifiedresources.ResourceTypeVM, VMware: &unifiedresources.VMwareData{}},
		{Type: unifiedresources.ResourceTypeStorage, VMware: &unifiedresources.VMwareData{}},
		{Type: unifiedresources.ResourceTypeCeph},
		{Type: unifiedresources.ResourceTypeNetworkShare},
		{Type: unifiedresources.ResourceTypeNetworkEndpoint, Availability: &unifiedresources.AvailabilityData{Enabled: true}},
		{Type: unifiedresources.ResourceTypeNetworkEndpoint, Availability: &unifiedresources.AvailabilityData{Enabled: false}},
	})

	assert.Equal(t, 1, counts.TrueNASSystems)
	assert.Equal(t, 1, counts.TrueNASVMs)
	assert.Equal(t, 1, counts.TrueNASApps)
	assert.Equal(t, 1, counts.VMwareHosts)
	assert.Equal(t, 1, counts.VMwareVMs)
	assert.Equal(t, 1, counts.VMwareDatastores)
	assert.Equal(t, 1, counts.CephClusters)
	assert.Equal(t, 1, counts.NetworkShares)
	assert.Equal(t, 1, counts.AvailabilityTargets)
}

func TestAccumulateAlertOutcomeCountsUsesOnlyContentFreeLifecycleTotals(t *testing.T) {
	now := time.Now().UTC()
	recentAck := now.Add(-time.Hour)
	oldAck := now.Add(-40 * 24 * time.Hour)
	counts := InstallSnapshotCounts{}
	accumulateAlertOutcomeCounts(&counts, []alerts.Alert{
		{
			Level:   alerts.AlertLevelCritical,
			AckTime: &recentAck,
			OperationalRecord: &operationaltrust.OperationalRecord{
				State: operationaltrust.OperationalResolved,
			},
		},
		{Level: alerts.AlertLevelWarning, AckTime: &oldAck},
	}, now.Add(-30*24*time.Hour))

	assert.Equal(t, 2, counts.AlertsFired30d)
	assert.Equal(t, 1, counts.AlertsAcknowledged30d)
	assert.Equal(t, 1, counts.AlertsResolved30d)
	assert.Equal(t, 1, counts.AlertQuality.FiredCritical30d)
	assert.Equal(t, 1, counts.AlertQuality.FiredWarning30d)
	assert.Equal(t, 1, counts.AlertQuality.ResolvedCritical30d)
}

func TestAccumulateAlertOutcomeCountsDeduplicatesOccurrenceSnapshots(t *testing.T) {
	now := time.Now().UTC()
	startedAt := now.Add(-2 * time.Hour)
	acknowledgedAt := now.Add(-time.Hour)
	open := alerts.Alert{
		ID:             "resource::cpu",
		CanonicalState: "resource::cpu",
		StartTime:      startedAt,
	}
	acknowledged := *open.Clone()
	acknowledged.AckTime = &acknowledgedAt
	resolved := *acknowledged.Clone()
	resolved.OperationalRecord = &operationaltrust.OperationalRecord{
		State: operationaltrust.OperationalResolved,
	}

	counts := InstallSnapshotCounts{}
	accumulateAlertOutcomeCounts(
		&counts,
		[]alerts.Alert{open, acknowledged, resolved},
		now.Add(-30*24*time.Hour),
	)

	assert.Equal(t, 1, counts.AlertsFired30d)
	assert.Equal(t, 1, counts.AlertsAcknowledged30d)
	assert.Equal(t, 1, counts.AlertsResolved30d)
}

func testTelemetryMonitor(
	nodes []models.Node,
	vms []models.VM,
	containers []models.Container,
	pbsInstances []models.PBSInstance,
	pmgInstances []models.PMGInstance,
	dockerHosts []models.DockerHost,
	k8sClusters []models.KubernetesCluster,
	activeAlerts int,
) *Monitor {
	state := models.NewState()
	state.UpdateNodes(nodes)
	state.UpdateVMs(vms)
	state.UpdateContainers(containers)
	state.UpdatePBSInstances(pbsInstances)
	state.UpdatePMGInstances(pmgInstances)
	for _, host := range dockerHosts {
		state.UpsertDockerHost(host)
	}
	for _, cluster := range k8sClusters {
		state.UpsertKubernetesCluster(cluster)
	}

	alerts := make([]models.Alert, 0, activeAlerts)
	for i := 0; i < activeAlerts; i++ {
		alerts = append(alerts, models.Alert{ID: time.Now().Add(time.Duration(i) * time.Second).Format(time.RFC3339Nano)})
	}
	state.UpdateActiveAlerts(alerts)

	return &Monitor{state: state}
}

// These fixtures exercise only the reload handshake; no monitor or storage is started.
func TestReloadCancellation(t *testing.T) {
	for _, phase := range []string{"before-request", "blocked-request", "awaiting-result"} {
		t.Run(phase, func(t *testing.T) {
			ctx, cancel := context.WithCancel(context.Background())
			defer cancel()
			rm := &ReloadableMonitor{parentCtx: ctx, reloadChan: make(chan chan error)}
			if phase == "before-request" {
				cancel()
			}
			result := make(chan error, 1)
			go func() { result <- rm.Reload() }()
			var reply chan error
			if phase == "awaiting-result" {
				select {
				case reply = <-rm.reloadChan:
				case <-time.After(time.Second):
					t.Fatal("request not submitted")
				}
			}
			cancel()
			select {
			case err := <-result:
				if !errors.Is(err, context.Canceled) {
					t.Fatalf("got %v, want context.Canceled", err)
				}
			case <-time.After(time.Second):
				// Release the old implementation's blocked caller before failing.
				if reply == nil {
					reply = <-rm.reloadChan
				}
				reply <- nil
				<-result
				t.Fatal("Reload remained blocked after cancellation")
			}
			if reply != nil {
				// Completion after caller cancellation must not strand the watcher.
				select {
				case reply <- nil:
				default:
					t.Fatal("late completion would block watcher")
				}
			}
		})
	}
}

func TestReloadHandshakeResult(t *testing.T) {
	for _, expected := range []error{nil, errors.New("reload failed")} {
		rm := &ReloadableMonitor{parentCtx: context.Background(), reloadChan: make(chan chan error)}
		go func() { reply := <-rm.reloadChan; reply <- expected }()
		if err := rm.Reload(); err != expected {
			t.Fatalf("got %v, want %v", err, expected)
		}
	}
}

func TestReloadBeforeStart(t *testing.T) {
	rm := &ReloadableMonitor{reloadChan: make(chan chan error, 1)}
	if err := rm.Reload(); err == nil {
		t.Fatal("Reload before Start must return an error")
	}
	if len(rm.reloadChan) != 0 {
		t.Fatal("unstarted reload was queued")
	}
}