package monitoring import ( "context" "errors" "strings" "testing" "time" "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" "github.com/rcourtman/pulse-go-rewrite/internal/vmware" ) func TestVMwarePollerPollsConfiguredConnections(t *testing.T) { previous := vmware.IsFeatureEnabled() vmware.SetFeatureEnabled(true) t.Cleanup(func() { vmware.SetFeatureEnabled(previous) }) mtp, persistence := newTestTenantPersistence(t) connection := config.NewVMwareVCenterInstance() connection.ID = "vc-1" connection.Name = "Lab VC" connection.Host = "vc.lab.local" connection.Username = "administrator@vsphere.local" connection.Password = "super-secret" if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{connection}); err != nil { t.Fatalf("SaveVMwareConfig() error = %v", err) } poller := NewVMwarePoller(mtp, 20*time.Millisecond) poller.newProvider = func(instance config.VMwareVCenterInstance) (vmwarePollerProvider, error) { return vmware.NewProvider(vmware.InventorySnapshot{ ConnectionID: instance.ID, ConnectionName: instance.Name, VCenterHost: instance.Host, CollectedAt: time.Now().UTC(), Hosts: []vmware.InventoryHost{{ Host: "host-101", Name: "esxi-01.lab.local", ConnectionState: "CONNECTED", PowerState: "POWERED_ON", HostUUID: "uuid-host-1", RecentTasks: []vmware.InventoryTask{{ Task: "task-11", Name: "Reconnect host", State: "running", StartedAt: time.Now().UTC().Add(-time.Minute), }}, }}, VMs: []vmware.InventoryVM{{ VM: "vm-201", Name: "app-01", PowerState: "POWERED_ON", CPUCount: 2, MemorySizeMiB: 4096, RecentEvents: []vmware.InventoryEvent{{ Event: "event-201", Type: "VmMessageEvent", Message: "Snapshot completed successfully", User: "vpxuser", CreatedAt: time.Now().UTC().Add(-2 * time.Minute), }}, }}, Datastores: []vmware.InventoryDatastore{{ Datastore: "datastore-11", Name: "nvme-primary", Type: "VMFS", FreeSpace: 50, Capacity: 100, }}, }), nil } poller.Start(context.Background()) t.Cleanup(poller.Stop) waitForCondition(t, 2*time.Second, func() bool { return len(poller.GetCurrentRecordsForOrg("default")) == 3 }, "expected VMware poller to ingest projected records") records := poller.GetCurrentRecordsForOrg("default") if len(records) != 3 { t.Fatalf("expected 3 VMware records, got %d", len(records)) } gotSourceIDs := make([]string, 0, len(records)) for _, record := range records { gotSourceIDs = append(gotSourceIDs, record.SourceID) } wantSourceIDs := []string{ "vc-1:host:host-101", "vc-1:vm:vm-201", "vc-1:datastore:datastore-11", } for i, want := range wantSourceIDs { if gotSourceIDs[i] != want { t.Fatalf("record %d sourceID = %q, want %q", i, gotSourceIDs[i], want) } } ownedSources := poller.SnapshotOwnedSourcesForOrg("default") if len(ownedSources) != 1 || ownedSources[0] != unifiedresources.SourceVMware { t.Fatalf("owned sources = %#v, want [%q]", ownedSources, unifiedresources.SourceVMware) } changes := poller.GetCurrentChangesForOrg("default") if len(changes) != 2 { t.Fatalf("expected 2 VMware activity changes, got %d", len(changes)) } if changes[0].Kind != unifiedresources.ChangeActivity { t.Fatalf("latest VMware change kind = %q, want %q", changes[0].Kind, unifiedresources.ChangeActivity) } if changes[0].SourceAdapter != unifiedresources.AdapterVMware { t.Fatalf("latest VMware change source adapter = %q, want %q", changes[0].SourceAdapter, unifiedresources.AdapterVMware) } if supplemental := poller.SupplementalChanges(nil, "default"); len(supplemental) != 2 { t.Fatalf("expected SupplementalChanges to mirror cached VMware changes, got %d", len(supplemental)) } waitForCondition(t, 2*time.Second, func() bool { summary, ok := poller.ConnectionSummaries("default", []config.VMwareVCenterInstance{connection})[connection.ID] return ok && summary.Poll != nil && summary.Poll.LastSuccessAt != nil && summary.Observed != nil }, "expected VMware poller to publish connection summary") summary := poller.ConnectionSummaries("default", []config.VMwareVCenterInstance{connection})[connection.ID] if summary.Poll == nil || summary.Poll.IntervalSeconds != 60 { t.Fatalf("expected VMware poll interval 60, got %+v", summary.Poll) } if summary.Observed == nil || summary.Observed.Hosts != 1 || summary.Observed.VMs != 1 || summary.Observed.Datastores != 1 { t.Fatalf("unexpected observed summary: %+v", summary.Observed) } } func TestVMwarePollerSupplementalInventoryReadyAtUsesPersistedActiveConnections(t *testing.T) { mtp, persistence := newTestTenantPersistence(t) connection := config.NewVMwareVCenterInstance() connection.ID = "vc-ready" connection.Host = "vc-ready.lab.local" connection.Username = "administrator@vsphere.local" connection.Password = "secret" if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{connection}); err != nil { t.Fatalf("SaveVMwareConfig() error = %v", err) } poller := NewVMwarePoller(mtp, time.Second) if readyAt, settled := poller.SupplementalInventoryReadyAt(nil, "default"); settled || !readyAt.IsZero() { t.Fatalf("SupplementalInventoryReadyAt() before any attempt = (%v, %t), want (zero, false)", readyAt, settled) } attemptedAt := time.Now().UTC() poller.RecordConnectionTestSuccess("default", connection.ID, &vmware.InventorySummary{}, attemptedAt) readyAt, settled := poller.SupplementalInventoryReadyAt(nil, "default") if !settled { t.Fatal("expected readiness to settle after the first recorded attempt") } if !readyAt.Equal(attemptedAt) { t.Fatalf("SupplementalInventoryReadyAt() = %v, want %v", readyAt, attemptedAt) } } func TestVMwarePollerFeatureFlagGate(t *testing.T) { previous := vmware.IsFeatureEnabled() vmware.SetFeatureEnabled(false) t.Cleanup(func() { vmware.SetFeatureEnabled(previous) }) mtp, persistence := newTestTenantPersistence(t) connection := config.NewVMwareVCenterInstance() connection.ID = "vc-flag-off" connection.Host = "vc.disabled.local" connection.Username = "user" connection.Password = "secret" if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{connection}); err != nil { t.Fatalf("SaveVMwareConfig() error = %v", err) } poller := NewVMwarePoller(mtp, 20*time.Millisecond) initialStopped := poller.stopped poller.Start(context.Background()) if poller.cancel != nil { t.Fatal("expected Start() to be a no-op with feature flag disabled") } if poller.stopped != initialStopped { t.Fatal("expected stopped channel to remain unchanged when Start() is gated") } if records := poller.GetCurrentRecordsForOrg("default"); len(records) != 0 { t.Fatalf("expected no VMware records when feature flag is disabled, got %d", len(records)) } } func TestVMwarePollerConnectionSummariesCaptureFailuresWithoutClearingObservedSummary(t *testing.T) { poller := NewVMwarePoller(nil, time.Minute) connection := config.NewVMwareVCenterInstance() connection.ID = "vc-1" connection.Host = "vc.lab.local" connection.Username = "administrator@vsphere.local" connection.Password = "secret" successAt := time.Date(2026, 3, 31, 9, 0, 0, 0, time.UTC) failureAt := successAt.Add(5 * time.Minute) poller.RecordConnectionTestSuccess("default", connection.ID, &vmware.InventorySummary{ Hosts: 2, VMs: 14, Datastores: 3, Networks: 5, VIRelease: "8.0.3", }, successAt) poller.RecordConnectionTestFailure("default", connection.ID, &vmware.ConnectionError{ Category: "permission", Message: "VMware permissions are insufficient for vmware performance metrics", }, failureAt) summary := poller.ConnectionSummaries("default", []config.VMwareVCenterInstance{connection})[connection.ID] if summary.Poll == nil || summary.Poll.LastError == nil { t.Fatalf("expected poll failure summary, got %+v", summary.Poll) } if summary.Poll.ConsecutiveFailures != 1 || summary.Poll.LastError.Category != "permission" { t.Fatalf("unexpected poll failure details: %+v", summary.Poll) } if summary.Observed == nil || summary.Observed.VMs != 14 || summary.Observed.Networks != 5 || summary.Observed.VIRelease != "8.0.3" { t.Fatalf("expected observed summary to be preserved after failure, got %+v", summary.Observed) } } func TestVMwarePollerSyncRecordsProviderConstructionFailure(t *testing.T) { mtp, persistence := newTestTenantPersistence(t) connection := config.NewVMwareVCenterInstance() connection.ID = "vc-build-fail" connection.Host = "vc.build-fail.local" connection.Username = "administrator@vsphere.local" connection.Password = "secret" if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{connection}); err != nil { t.Fatalf("SaveVMwareConfig() error = %v", err) } poller := NewVMwarePoller(mtp, time.Minute) poller.newProvider = func(config.VMwareVCenterInstance) (vmwarePollerProvider, error) { return nil, &vmware.ConnectionError{ Category: "auth", Message: "VMware credentials were rejected", } } poller.syncConnections() poller.syncConnections() if providers := poller.providersByOrg["default"]; len(providers) != 0 { t.Fatalf("expected provider construction failure to leave no providers, got %d", len(providers)) } summary := poller.ConnectionSummaries("default", []config.VMwareVCenterInstance{connection})[connection.ID] if summary.Poll == nil || summary.Poll.LastAttemptAt == nil || summary.Poll.LastError == nil { t.Fatalf("expected provider construction failure in poll summary, got %+v", summary.Poll) } if summary.Poll.ConsecutiveFailures != 2 { t.Fatalf("consecutive failures = %d, want 2", summary.Poll.ConsecutiveFailures) } if summary.Poll.LastError.Category != "auth" || !strings.Contains(summary.Poll.LastError.Message, "credentials were rejected") { t.Fatalf("unexpected provider construction error summary: %+v", summary.Poll.LastError) } } type failingVMwarePollerProvider struct { err error } func (p *failingVMwarePollerProvider) Refresh(context.Context) error { return p.err } func (p *failingVMwarePollerProvider) Records() []unifiedresources.IngestRecord { return nil } func (p *failingVMwarePollerProvider) ActivityChanges() []unifiedresources.ResourceChange { return nil } func (p *failingVMwarePollerProvider) Snapshot() *vmware.InventorySnapshot { return nil } func (p *failingVMwarePollerProvider) Close() {} func TestVMwarePollerLiveFailureKeepsCachedRecords(t *testing.T) { previous := vmware.IsFeatureEnabled() vmware.SetFeatureEnabled(true) t.Cleanup(func() { vmware.SetFeatureEnabled(previous) }) mtp, persistence := newTestTenantPersistence(t) connection := config.NewVMwareVCenterInstance() connection.ID = "vc-live-fail" connection.Host = "vc.fail.local" connection.Username = "administrator@vsphere.local" connection.Password = "secret" if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{connection}); err != nil { t.Fatalf("SaveVMwareConfig() error = %v", err) } providerCalls := 0 poller := NewVMwarePoller(mtp, 20*time.Millisecond) poller.newProvider = func(instance config.VMwareVCenterInstance) (vmwarePollerProvider, error) { providerCalls++ if providerCalls == 1 { return vmware.NewProvider(vmware.InventorySnapshot{ ConnectionID: instance.ID, CollectedAt: time.Now().UTC(), VIRelease: "8.0.3", Hosts: []vmware.InventoryHost{{Host: "host-1", Name: "esxi-01"}}, }), nil } return &failingVMwarePollerProvider{ err: errors.New("refresh vmware inventory: VMware permissions are insufficient for vmware performance metrics"), }, nil } poller.syncConnections() poller.pollAll(context.Background()) initialRecords := poller.GetCurrentRecordsForOrg("default") if len(initialRecords) != 1 { t.Fatalf("expected initial cached VMware records, got %d", len(initialRecords)) } poller.providersByOrg["default"][connection.ID] = &failingVMwarePollerProvider{ err: &vmware.ConnectionError{Category: "permission", Message: "VMware permissions are insufficient for vmware performance metrics"}, } poller.pollAll(context.Background()) recordsAfterFailure := poller.GetCurrentRecordsForOrg("default") if len(recordsAfterFailure) != 1 { t.Fatalf("expected cached VMware records to survive live failure, got %d", len(recordsAfterFailure)) } summary := poller.ConnectionSummaries("default", []config.VMwareVCenterInstance{connection})[connection.ID] if summary.Poll == nil || summary.Poll.LastError == nil || summary.Poll.LastError.Category != "permission" { t.Fatalf("expected permission poll error after live failure, got %+v", summary.Poll) } } func TestVMwarePollerPublishesDegradedObservedSummaryOnPartialSuccess(t *testing.T) { poller := NewVMwarePoller(nil, time.Minute) connection := config.NewVMwareVCenterInstance() connection.ID = "vc-partial" connection.Host = "vc.partial.local" connection.Username = "administrator@vsphere.local" connection.Password = "secret" collectedAt := time.Date(2026, 3, 31, 10, 0, 0, 0, time.UTC) poller.newProvider = func(instance config.VMwareVCenterInstance) (vmwarePollerProvider, error) { return vmware.NewProvider(vmware.InventorySnapshot{ ConnectionID: instance.ID, CollectedAt: collectedAt, VIRelease: "8.0.3", Hosts: []vmware.InventoryHost{{Host: "host-101", Name: "esxi-01"}}, EnrichmentIssues: []vmware.InventoryEnrichmentIssue{ {Stage: "signals", EntityType: "host", EntityID: "host-101", Category: "permission", Message: "VMware permissions are insufficient for HostSystem overall status"}, {Stage: "signals", EntityType: "vm", EntityID: "vm-201", Category: "permission", Message: "VMware permissions are insufficient for HostSystem overall status"}, {Stage: "topology", EntityType: "vm", EntityID: "vm-201", Category: "unavailable", Message: "VMware vm guest identity is temporarily unavailable"}, }, }), nil } poller.providersByOrg["default"] = map[string]vmwarePollerProvider{} provider, err := poller.newProvider(connection) if err != nil { t.Fatalf("newProvider() error = %v", err) } poller.providersByOrg["default"][connection.ID] = provider poller.configsByOrg["default"] = map[string]config.VMwareVCenterInstance{connection.ID: connection} poller.pollAll(context.Background()) summary := poller.ConnectionSummaries("default", []config.VMwareVCenterInstance{connection})[connection.ID] if summary.Poll == nil || summary.Poll.LastSuccessAt == nil || summary.Poll.LastError != nil { t.Fatalf("expected successful degraded poll summary, got %+v", summary.Poll) } if summary.Observed == nil || !summary.Observed.Degraded || summary.Observed.IssueCount != 3 { t.Fatalf("expected degraded observed summary, got %+v", summary.Observed) } if len(summary.Observed.Issues) != 2 { t.Fatalf("expected aggregated observed issues, got %+v", summary.Observed.Issues) } if summary.Observed.Issues[0].Occurrences != 2 || summary.Observed.Issues[0].Category != "permission" { t.Fatalf("expected aggregated permission issue first, got %+v", summary.Observed.Issues[0]) } } type sequencedVMwarePollerProvider struct { snapshots []*vmware.InventorySnapshot index int current *vmware.InventorySnapshot } func (p *sequencedVMwarePollerProvider) Refresh(context.Context) error { if len(p.snapshots) == 0 { p.current = nil return nil } if p.index >= len(p.snapshots) { p.current = p.snapshots[len(p.snapshots)-1] return nil } p.current = p.snapshots[p.index] p.index++ return nil } func (p *sequencedVMwarePollerProvider) Records() []unifiedresources.IngestRecord { return nil } func (p *sequencedVMwarePollerProvider) ActivityChanges() []unifiedresources.ResourceChange { return nil } func (p *sequencedVMwarePollerProvider) Snapshot() *vmware.InventorySnapshot { return p.current } func (p *sequencedVMwarePollerProvider) Close() {} func TestVMwarePollerLogsDegradedTransitionsOnlyWhenStateChanges(t *testing.T) { logs := captureVMwarePollerLogs(t) poller := NewVMwarePoller(nil, time.Minute) connection := config.NewVMwareVCenterInstance() connection.ID = "vc-log" connection.Host = "vc.log.local" connection.Username = "administrator@vsphere.local" connection.Password = "secret" issueA := vmware.InventoryEnrichmentIssue{ Stage: "signals", Category: "permission", Message: "VMware permissions are insufficient for HostSystem overall status", } issueB := vmware.InventoryEnrichmentIssue{ Stage: "topology", Category: "unavailable", Message: "VMware vm guest identity is temporarily unavailable", } provider := &sequencedVMwarePollerProvider{ snapshots: []*vmware.InventorySnapshot{ { ConnectionID: connection.ID, CollectedAt: time.Date(2026, 3, 31, 11, 0, 0, 0, time.UTC), Hosts: []vmware.InventoryHost{{Host: "host-101", Name: "esxi-01"}}, EnrichmentIssues: []vmware.InventoryEnrichmentIssue{issueA, issueA}, }, { ConnectionID: connection.ID, CollectedAt: time.Date(2026, 3, 31, 11, 1, 0, 0, time.UTC), Hosts: []vmware.InventoryHost{{Host: "host-101", Name: "esxi-01"}}, EnrichmentIssues: []vmware.InventoryEnrichmentIssue{issueA, issueA}, }, { ConnectionID: connection.ID, CollectedAt: time.Date(2026, 3, 31, 11, 2, 0, 0, time.UTC), Hosts: []vmware.InventoryHost{{Host: "host-101", Name: "esxi-01"}}, EnrichmentIssues: []vmware.InventoryEnrichmentIssue{issueB}, }, { ConnectionID: connection.ID, CollectedAt: time.Date(2026, 3, 31, 11, 3, 0, 0, time.UTC), Hosts: []vmware.InventoryHost{{Host: "host-101", Name: "esxi-01"}}, }, }, } entry := vmwarePollerProviderEntry{ orgID: "default", connectionID: connection.ID, provider: provider, } for i := 0; i < 4; i++ { poller.pollConnection(context.Background(), entry) } output := logs.String() if got := strings.Count(output, `"action":"refresh_partial"`); got != 1 { t.Fatalf("expected one refresh_partial warning, got %d logs: %s", got, output) } if got := strings.Count(output, `"action":"refresh_partial_changed"`); got != 1 { t.Fatalf("expected one refresh_partial_changed warning, got %d logs: %s", got, output) } if got := strings.Count(output, `"action":"refresh_recovered"`); got != 1 { t.Fatalf("expected one refresh_recovered info log, got %d logs: %s", got, output) } summary := poller.ConnectionSummaries("default", []config.VMwareVCenterInstance{connection})[connection.ID] if summary.Observed == nil || summary.Observed.Degraded { t.Fatalf("expected recovered observed summary after final healthy poll, got %+v", summary.Observed) } } func captureVMwarePollerLogs(t *testing.T) *monitoringLogCapture { t.Helper() return newMonitoringLogCapture(t) }