mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-21 19:25:57 +00:00
1254 lines
44 KiB
Go
1254 lines
44 KiB
Go
package api
|
|
|
|
import (
|
|
"reflect"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/platformsupport"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/remoteconfig"
|
|
)
|
|
|
|
func ptrTime(t time.Time) *time.Time { return &t }
|
|
|
|
func ptrBool(v bool) *bool { return &v }
|
|
|
|
func healthEntry(lastSuccess *time.Time, errMessage, errCategory string, breakerState string) monitoring.InstanceHealth {
|
|
ps := monitoring.InstancePollStatus{LastSuccess: lastSuccess}
|
|
if errMessage != "" {
|
|
ps.LastError = &monitoring.ErrorDetail{
|
|
At: time.Now(),
|
|
Message: errMessage,
|
|
Category: errCategory,
|
|
}
|
|
}
|
|
return monitoring.InstanceHealth{
|
|
PollStatus: ps,
|
|
Breaker: monitoring.InstanceBreaker{State: breakerState},
|
|
}
|
|
}
|
|
|
|
func desiredAgentConfig(t *testing.T, commandsEnabled *bool, settings map[string]interface{}) connectionAgentDesiredConfig {
|
|
t.Helper()
|
|
metadata, err := remoteconfig.BuildDesiredConfigMetadata(commandsEnabled, settings)
|
|
if err != nil {
|
|
t.Fatalf("BuildDesiredConfigMetadata: %v", err)
|
|
}
|
|
fingerprint := ConnectionFleetConfigFingerprint{
|
|
Version: metadata.Version,
|
|
Hash: metadata.Hash,
|
|
}
|
|
return connectionAgentDesiredConfig{
|
|
Fingerprint: &fingerprint,
|
|
CommandsEnabled: cloneBoolPtr(commandsEnabled),
|
|
}
|
|
}
|
|
|
|
func TestDeriveConnectionState_Paused(t *testing.T) {
|
|
state, reason, _, _ := deriveConnectionState(false, monitoring.InstanceHealth{}, time.Now(), 0)
|
|
if state != ConnectionStatePaused {
|
|
t.Fatalf("got %q, want %q", state, ConnectionStatePaused)
|
|
}
|
|
if reason == "" {
|
|
t.Fatal("expected non-empty reason for paused state")
|
|
}
|
|
}
|
|
|
|
func TestDeriveConnectionState_Pending(t *testing.T) {
|
|
state, _, lastSeen, lastError := deriveConnectionState(true, monitoring.InstanceHealth{}, time.Now(), 0)
|
|
if state != ConnectionStatePending {
|
|
t.Fatalf("got %q, want %q", state, ConnectionStatePending)
|
|
}
|
|
if lastSeen != nil || lastError != nil {
|
|
t.Fatal("expected nil lastSeen and lastError in pending state")
|
|
}
|
|
}
|
|
|
|
func TestDeriveConnectionState_Unauthorized(t *testing.T) {
|
|
now := time.Now()
|
|
h := healthEntry(ptrTime(now.Add(-30*time.Second)), "401 Unauthorized: token invalid", "auth", "closed")
|
|
state, reason, _, err := deriveConnectionState(true, h, now, 0)
|
|
if state != ConnectionStateUnauthorized {
|
|
t.Fatalf("got %q, want %q", state, ConnectionStateUnauthorized)
|
|
}
|
|
if reason == "" {
|
|
t.Fatal("expected reason to include error message")
|
|
}
|
|
if err == nil || err.Message != "401 Unauthorized: token invalid" {
|
|
t.Fatalf("unexpected lastError: %+v", err)
|
|
}
|
|
}
|
|
|
|
func TestDeriveConnectionState_Unreachable(t *testing.T) {
|
|
now := time.Now()
|
|
h := healthEntry(ptrTime(now.Add(-30*time.Second)), "connection refused", "network", "open")
|
|
state, _, _, _ := deriveConnectionState(true, h, now, 0)
|
|
if state != ConnectionStateUnreachable {
|
|
t.Fatalf("got %q, want %q", state, ConnectionStateUnreachable)
|
|
}
|
|
}
|
|
|
|
func TestDeriveConnectionState_Stale(t *testing.T) {
|
|
now := time.Now()
|
|
stale := now.Add(-5 * time.Minute)
|
|
h := healthEntry(ptrTime(stale), "", "", "closed")
|
|
state, reason, _, _ := deriveConnectionState(true, h, now, 0)
|
|
if state != ConnectionStateStale {
|
|
t.Fatalf("got %q, want %q", state, ConnectionStateStale)
|
|
}
|
|
if reason == "" {
|
|
t.Fatal("expected reason to describe staleness")
|
|
}
|
|
}
|
|
|
|
func TestDeriveConnectionState_Active(t *testing.T) {
|
|
now := time.Now()
|
|
h := healthEntry(ptrTime(now.Add(-10*time.Second)), "", "", "closed")
|
|
state, _, _, _ := deriveConnectionState(true, h, now, 0)
|
|
if state != ConnectionStateActive {
|
|
t.Fatalf("got %q, want %q", state, ConnectionStateActive)
|
|
}
|
|
}
|
|
|
|
func TestBuildConnections_SortsByTypeThenName(t *testing.T) {
|
|
now := time.Now()
|
|
in := aggregatorInputs{
|
|
pveInstances: []config.PVEInstance{
|
|
{Name: "beta", Host: "https://b.lan:8006", MonitorVMs: true},
|
|
{Name: "alpha", Host: "https://a.lan:8006", MonitorVMs: true},
|
|
},
|
|
pbsInstances: []config.PBSInstance{
|
|
{Name: "backups", Host: "https://bkp.lan:8007"},
|
|
},
|
|
now: now,
|
|
}
|
|
got := buildConnections(in)
|
|
if len(got) != 3 {
|
|
t.Fatalf("expected 3 connections, got %d", len(got))
|
|
}
|
|
if got[0].Type != ConnectionTypePBS {
|
|
t.Fatalf("expected PBS first, got %q", got[0].Type)
|
|
}
|
|
if got[1].Name != "alpha" || got[2].Name != "beta" {
|
|
t.Fatalf("PVE order wrong: %q, %q", got[1].Name, got[2].Name)
|
|
}
|
|
}
|
|
|
|
func TestBuildConnections_PVEPausedRespectsDisabled(t *testing.T) {
|
|
in := aggregatorInputs{
|
|
pveInstances: []config.PVEInstance{{Name: "pve1", Host: "https://pve1.lan:8006", Disabled: true}},
|
|
now: time.Now(),
|
|
}
|
|
got := buildConnections(in)
|
|
if len(got) != 1 {
|
|
t.Fatalf("expected 1 connection, got %d", len(got))
|
|
}
|
|
if got[0].State != ConnectionStatePaused {
|
|
t.Fatalf("expected paused, got %q", got[0].State)
|
|
}
|
|
if got[0].Enabled {
|
|
t.Fatal("expected Enabled=false for Disabled PVE instance")
|
|
}
|
|
}
|
|
|
|
func TestBuildConnections_AgentStateFromLastSeen(t *testing.T) {
|
|
now := time.Now()
|
|
in := aggregatorInputs{
|
|
hosts: []models.Host{
|
|
{ID: "fresh", Hostname: "h1", LastSeen: now.Add(-10 * time.Second), IntervalSeconds: 30},
|
|
{ID: "stale", Hostname: "h2", LastSeen: now.Add(-5*time.Minute - time.Second), IntervalSeconds: 30},
|
|
{ID: "never", Hostname: "h3"},
|
|
},
|
|
now: now,
|
|
}
|
|
got := buildConnections(in)
|
|
byID := map[string]Connection{}
|
|
for _, c := range got {
|
|
byID[c.ID] = c
|
|
}
|
|
if byID["agent:fresh"].State != ConnectionStateActive {
|
|
t.Fatalf("fresh agent: got %q, want active", byID["agent:fresh"].State)
|
|
}
|
|
if byID["agent:stale"].State != ConnectionStateStale {
|
|
t.Fatalf("stale agent: got %q, want stale", byID["agent:stale"].State)
|
|
}
|
|
if byID["agent:never"].State != ConnectionStatePending {
|
|
t.Fatalf("never-reported agent: got %q, want pending", byID["agent:never"].State)
|
|
}
|
|
for _, c := range got {
|
|
if c.Capabilities.SupportsPause || c.Capabilities.SupportsScope {
|
|
t.Fatalf("agents must not advertise pause/scope capabilities: %+v", c)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestBuildConnections_AvailabilityTargetStateFromProbeStatus(t *testing.T) {
|
|
checkedAt := time.Date(2026, 5, 6, 10, 0, 0, 0, time.UTC)
|
|
in := aggregatorInputs{
|
|
availabilityTargets: []config.AvailabilityTarget{
|
|
{
|
|
ID: "sensor-1",
|
|
Name: "Energy monitor",
|
|
Address: "192.0.2.10",
|
|
Protocol: config.AvailabilityProbeICMP,
|
|
Enabled: true,
|
|
FailureThreshold: 2,
|
|
},
|
|
},
|
|
availabilityStatuses: map[string]monitoring.AvailabilityProbeStatus{
|
|
"sensor-1": {
|
|
TargetID: "sensor-1",
|
|
Available: false,
|
|
LastChecked: checkedAt,
|
|
ConsecutiveFailures: 1,
|
|
LastError: "timeout",
|
|
},
|
|
},
|
|
now: checkedAt.Add(5 * time.Second),
|
|
}
|
|
|
|
got := buildConnections(in)
|
|
if len(got) != 1 {
|
|
t.Fatalf("expected 1 connection, got %d", len(got))
|
|
}
|
|
connection := got[0]
|
|
if connection.ID != "availability:sensor-1" {
|
|
t.Fatalf("ID = %q, want availability:sensor-1", connection.ID)
|
|
}
|
|
if connection.Type != ConnectionTypeAvailability {
|
|
t.Fatalf("Type = %q, want availability", connection.Type)
|
|
}
|
|
if connection.State != ConnectionStateUnreachable {
|
|
t.Fatalf("State = %q, want unreachable", connection.State)
|
|
}
|
|
if connection.LastError == nil || connection.LastError.Category != "availability" {
|
|
t.Fatalf("LastError = %+v, want availability category", connection.LastError)
|
|
}
|
|
if !reflect.DeepEqual(connection.Surfaces, []string{"availability"}) {
|
|
t.Fatalf("Surfaces = %+v, want [availability]", connection.Surfaces)
|
|
}
|
|
if !connection.Capabilities.SupportsPause || connection.Capabilities.SupportsScope || !connection.Capabilities.SupportsTest {
|
|
t.Fatalf("Capabilities = %+v, want pause/test without scope", connection.Capabilities)
|
|
}
|
|
if !reflect.DeepEqual(connection.HostAliases, []string{"192.0.2.10"}) {
|
|
t.Fatalf("HostAliases = %+v, want endpoint address", connection.HostAliases)
|
|
}
|
|
}
|
|
|
|
func TestDeriveAvailabilityConnectionStateUsesCanonicalRemoteProbeFreshness(t *testing.T) {
|
|
receivedAt := time.Date(2026, 7, 29, 10, 0, 0, 0, time.UTC)
|
|
agentCheckedAt := receivedAt.Add(-24 * time.Hour)
|
|
target := config.NormalizeAvailabilityTarget(config.AvailabilityTarget{
|
|
ID: "remote-sensor",
|
|
Name: "Remote sensor",
|
|
Address: "sensor.example",
|
|
Protocol: config.AvailabilityProbeICMP,
|
|
Enabled: true,
|
|
PollIntervalSecs: 60,
|
|
ProbeAgentID: "probe-agent",
|
|
})
|
|
status := monitoring.AvailabilityProbeStatus{
|
|
TargetID: target.ID,
|
|
Available: true,
|
|
LastChecked: agentCheckedAt,
|
|
LastSuccess: agentCheckedAt,
|
|
ProbeAgentID: "probe-agent",
|
|
ProbeReportReceivedAt: receivedAt,
|
|
}
|
|
|
|
state, reason, lastSeen, lastError := deriveAvailabilityConnectionState(
|
|
target,
|
|
status,
|
|
receivedAt.Add(4*time.Minute),
|
|
)
|
|
if state != ConnectionStateActive || reason != "" || lastError != nil {
|
|
t.Fatalf("connection = state %q reason %q error %+v, want active", state, reason, lastError)
|
|
}
|
|
if lastSeen == nil || !lastSeen.Equal(receivedAt) {
|
|
t.Fatalf("last seen = %v, want server receipt %v", lastSeen, receivedAt)
|
|
}
|
|
}
|
|
|
|
func TestBuildConnections_AgentHostAliasesIncludeReportedIdentityHints(t *testing.T) {
|
|
now := time.Now()
|
|
in := aggregatorInputs{
|
|
hosts: []models.Host{
|
|
{
|
|
ID: "pi",
|
|
Hostname: "pi",
|
|
Platform: "slackware",
|
|
OSName: "Unraid",
|
|
OSVersion: "7.1.0",
|
|
KernelVersion: "6.12.0",
|
|
Architecture: "x86_64",
|
|
ReportIP: "192.168.0.2",
|
|
LastSeen: now,
|
|
AgentVersion: "6.0.0",
|
|
CommandsEnabled: true,
|
|
NetworkInterfaces: []models.HostNetworkInterface{
|
|
{Name: "eth0", Addresses: []string{"192.168.0.2/24", "fe80::1/64"}},
|
|
},
|
|
},
|
|
},
|
|
expectedAgentVersion: "6.0.1",
|
|
now: now,
|
|
}
|
|
|
|
got := buildConnections(in)
|
|
if len(got) != 1 {
|
|
t.Fatalf("expected 1 connection, got %d", len(got))
|
|
}
|
|
if !reflect.DeepEqual(got[0].HostAliases, []string{"pi", "192.168.0.2", "fe80::1"}) {
|
|
t.Fatalf("agent host aliases = %+v, want %+v", got[0].HostAliases, []string{"pi", "192.168.0.2", "fe80::1"})
|
|
}
|
|
if got[0].AgentIdentity == nil {
|
|
t.Fatal("expected agent identity metadata")
|
|
}
|
|
if !reflect.DeepEqual(
|
|
got[0].AgentIdentity,
|
|
&ConnectionAgentIdentity{
|
|
Hostname: "pi",
|
|
Platform: "linux",
|
|
HostProfile: "unraid",
|
|
OSName: "Unraid",
|
|
OSVersion: "7.1.0",
|
|
KernelVersion: "6.12.0",
|
|
Architecture: "x86_64",
|
|
ReportIP: "192.168.0.2",
|
|
CommandsEnabled: true,
|
|
},
|
|
) {
|
|
t.Fatalf("agent identity = %+v", got[0].AgentIdentity)
|
|
}
|
|
}
|
|
|
|
func TestBuildConnections_ProjectsLegacyLinuxDistroToRuntimePlatform(t *testing.T) {
|
|
now := time.Now()
|
|
got := buildConnections(aggregatorInputs{
|
|
hosts: []models.Host{{
|
|
ID: "mageia-host",
|
|
Hostname: "mageia-host",
|
|
Platform: "mageia",
|
|
OSName: "Mageia",
|
|
Architecture: "x86_64",
|
|
LastSeen: now,
|
|
AgentVersion: "6.1.0-rc.4",
|
|
CommandsEnabled: true,
|
|
}},
|
|
expectedAgentVersion: "6.1.1",
|
|
now: now,
|
|
})
|
|
|
|
if len(got) != 1 || got[0].AgentIdentity == nil {
|
|
t.Fatalf("expected one connection with agent identity, got %+v", got)
|
|
}
|
|
if got[0].AgentIdentity.Platform != platformsupport.RuntimePlatformLinux {
|
|
t.Fatalf("agent platform = %q, want linux", got[0].AgentIdentity.Platform)
|
|
}
|
|
if got[0].AgentIdentity.OSName != "Mageia" {
|
|
t.Fatalf("agent OS name = %q, want Mageia", got[0].AgentIdentity.OSName)
|
|
}
|
|
}
|
|
|
|
func TestBuildConnections_AgentHostProfileFromUnraidStorageFacts(t *testing.T) {
|
|
now := time.Now()
|
|
in := aggregatorInputs{
|
|
hosts: []models.Host{
|
|
{
|
|
ID: "tower",
|
|
Hostname: "tower",
|
|
Platform: "linux",
|
|
Unraid: &models.HostUnraidStorage{ArrayStarted: true},
|
|
LastSeen: now,
|
|
AgentVersion: "6.12.10",
|
|
CommandsEnabled: false,
|
|
},
|
|
},
|
|
now: now,
|
|
}
|
|
|
|
got := buildConnections(in)
|
|
if len(got) != 1 {
|
|
t.Fatalf("expected 1 connection, got %d", len(got))
|
|
}
|
|
identity := got[0].AgentIdentity
|
|
if identity == nil {
|
|
t.Fatal("expected agent identity metadata")
|
|
}
|
|
if identity.HostProfile != "unraid" {
|
|
t.Fatalf("host profile = %q, want %q", identity.HostProfile, "unraid")
|
|
}
|
|
if identity.Platform != "linux" {
|
|
t.Fatalf("platform = %q, want %q", identity.Platform, "linux")
|
|
}
|
|
if identity.OSName != "" {
|
|
t.Fatalf("os name = %q, want empty when source identity is absent", identity.OSName)
|
|
}
|
|
}
|
|
|
|
func TestBuildConnections_AgentVersionUpdateAvailability(t *testing.T) {
|
|
now := time.Now()
|
|
in := aggregatorInputs{
|
|
hosts: []models.Host{
|
|
{ID: "current", Hostname: "current", LastSeen: now, AgentVersion: "6.0.3", AgentUpdate: &models.AgentUpdateStatus{State: "disabled"}},
|
|
{ID: "outdated", Hostname: "outdated", LastSeen: now, AgentVersion: "6.0.0", AgentUpdate: &models.AgentUpdateStatus{State: "disabled"}},
|
|
{
|
|
ID: "failed",
|
|
Hostname: "failed",
|
|
LastSeen: now,
|
|
AgentVersion: "6.0.3",
|
|
AgentUpdate: &models.AgentUpdateStatus{State: "error", LastError: "signature verification failed"},
|
|
AgentModules: []models.AgentModuleStatus{{
|
|
Name: "docker", Enabled: true, State: "retrying", LastError: "socket unavailable", UpdatedAt: now,
|
|
}},
|
|
},
|
|
{ID: "unknown", Hostname: "unknown", LastSeen: now},
|
|
},
|
|
expectedAgentVersion: "6.0.3",
|
|
now: now,
|
|
}
|
|
|
|
got := buildConnections(in)
|
|
byID := map[string]Connection{}
|
|
for _, connection := range got {
|
|
byID[connection.ID] = connection
|
|
}
|
|
|
|
if byID["agent:current"].AgentVersion != "6.0.3" {
|
|
t.Fatalf("current agent version = %q, want %q", byID["agent:current"].AgentVersion, "6.0.3")
|
|
}
|
|
if byID["agent:current"].ExpectedAgentVersion != "6.0.3" {
|
|
t.Fatalf(
|
|
"current expected agent version = %q, want %q",
|
|
byID["agent:current"].ExpectedAgentVersion,
|
|
"6.0.3",
|
|
)
|
|
}
|
|
if byID["agent:current"].AgentUpdateAvailable {
|
|
t.Fatal("current agent should not report an update available")
|
|
}
|
|
if byID["agent:current"].Fleet.UpdateStatus != fleetStateCurrent {
|
|
t.Fatalf("current agent update status = %q, want current", byID["agent:current"].Fleet.UpdateStatus)
|
|
}
|
|
|
|
if byID["agent:outdated"].AgentVersion != "6.0.0" {
|
|
t.Fatalf(
|
|
"outdated agent version = %q, want %q",
|
|
byID["agent:outdated"].AgentVersion,
|
|
"6.0.0",
|
|
)
|
|
}
|
|
if !byID["agent:outdated"].AgentUpdateAvailable {
|
|
t.Fatal("outdated agent should report an update available")
|
|
}
|
|
if byID["agent:outdated"].Fleet.UpdateStatus != fleetStateDisabled {
|
|
t.Fatalf("outdated disabled agent update status = %q, want disabled", byID["agent:outdated"].Fleet.UpdateStatus)
|
|
}
|
|
|
|
if byID["agent:failed"].Fleet.UpdateStatus != fleetStateFailed || byID["agent:failed"].AgentUpdate == nil || byID["agent:failed"].AgentUpdate.LastError != "signature verification failed" {
|
|
t.Fatalf("failed agent update status = %+v", byID["agent:failed"])
|
|
}
|
|
if byID["agent:failed"].Fleet.AdapterHealth != fleetStateDegraded || len(byID["agent:failed"].AgentModules) != 1 {
|
|
t.Fatalf("failed agent module status = %+v", byID["agent:failed"])
|
|
}
|
|
|
|
if byID["agent:unknown"].AgentVersion != "" {
|
|
t.Fatalf("unknown agent version = %q, want empty", byID["agent:unknown"].AgentVersion)
|
|
}
|
|
if byID["agent:unknown"].AgentUpdateAvailable {
|
|
t.Fatal("agent without a reported version should not raise an update flag")
|
|
}
|
|
}
|
|
|
|
func TestBuildConnections_AgentFleetGovernance(t *testing.T) {
|
|
now := time.Now()
|
|
currentDesiredConfig := desiredAgentConfig(t, ptrBool(true), nil)
|
|
currentDesired := *currentDesiredConfig.Fingerprint
|
|
in := aggregatorInputs{
|
|
hosts: []models.Host{
|
|
{
|
|
ID: "current",
|
|
Hostname: "current",
|
|
LastSeen: now,
|
|
AgentVersion: "6.0.3",
|
|
CommandsEnabled: true,
|
|
AppliedConfig: &models.AgentConfigFingerprint{
|
|
Version: currentDesired.Version,
|
|
Hash: currentDesired.Hash,
|
|
},
|
|
},
|
|
{
|
|
ID: "outdated",
|
|
Hostname: "outdated",
|
|
LastSeen: now,
|
|
AgentVersion: "6.0.0",
|
|
},
|
|
{
|
|
ID: "pending",
|
|
Hostname: "pending",
|
|
},
|
|
},
|
|
agentDesiredConfigs: map[string]connectionAgentDesiredConfig{
|
|
"current": currentDesiredConfig,
|
|
},
|
|
expectedAgentVersion: "6.0.3",
|
|
now: now,
|
|
}
|
|
|
|
got := buildConnections(in)
|
|
byID := map[string]Connection{}
|
|
for _, connection := range got {
|
|
byID[connection.ID] = connection
|
|
}
|
|
|
|
current := byID["agent:current"].Fleet
|
|
if current.EnrollmentState != fleetStateEnrolled ||
|
|
current.LivenessState != fleetStateActive ||
|
|
current.VersionDrift != fleetStateCurrent ||
|
|
current.AdapterHealth != fleetStateHealthy ||
|
|
current.ConfigRollout != fleetStateReported ||
|
|
current.CredentialStatus != fleetStateVerified ||
|
|
current.UpdateStatus != fleetStateCurrent ||
|
|
current.RemoteControl != fleetStateEnabled {
|
|
t.Fatalf("current agent fleet governance = %+v", current)
|
|
}
|
|
if current.ConfigDrift == nil ||
|
|
current.ConfigDrift.Status != fleetConfigDriftCurrent ||
|
|
current.ConfigDrift.Desired == nil ||
|
|
current.ConfigDrift.Applied == nil ||
|
|
current.ConfigDrift.Desired.Version != connectionAgentConfigFingerprintVersion ||
|
|
current.ConfigDrift.Desired.Hash != currentDesired.Hash ||
|
|
current.ConfigDrift.Applied.Hash != currentDesired.Hash {
|
|
t.Fatalf("current agent config drift = %+v", current.ConfigDrift)
|
|
}
|
|
if current.Rollout == nil || current.Rollout.Status != fleetStateCurrent || current.Rollout.Stage != fleetRolloutStageApplied {
|
|
t.Fatalf("current agent rollout = %+v", current.Rollout)
|
|
}
|
|
if current.CommandPolicy == nil ||
|
|
current.CommandPolicy.Status != fleetStateEnabled ||
|
|
current.CommandPolicy.Desired != fleetStateEnabled ||
|
|
current.CommandPolicy.Applied != fleetStateEnabled ||
|
|
current.CommandPolicy.Enforcement != fleetCommandPolicyInSync {
|
|
t.Fatalf("current agent command policy = %+v", current.CommandPolicy)
|
|
}
|
|
|
|
outdated := byID["agent:outdated"].Fleet
|
|
if outdated.VersionDrift != fleetStateBehind ||
|
|
outdated.UpdateStatus != fleetStateUpdateAvailable ||
|
|
outdated.RemoteControl != fleetStateDisabled {
|
|
t.Fatalf("outdated agent fleet governance = %+v", outdated)
|
|
}
|
|
if outdated.CommandPolicy == nil ||
|
|
outdated.CommandPolicy.Status != fleetStateDisabled ||
|
|
outdated.CommandPolicy.Desired != fleetStateUnknown ||
|
|
outdated.CommandPolicy.Applied != fleetStateDisabled ||
|
|
outdated.CommandPolicy.Enforcement != fleetCommandPolicyNotApplicable {
|
|
t.Fatalf("outdated agent command policy = %+v", outdated.CommandPolicy)
|
|
}
|
|
|
|
pending := byID["agent:pending"].Fleet
|
|
if pending.EnrollmentState != fleetStatePending ||
|
|
pending.LivenessState != string(ConnectionStatePending) ||
|
|
pending.AdapterHealth != fleetStateDegraded ||
|
|
pending.ConfigRollout != fleetStateUnknown ||
|
|
pending.CredentialStatus != fleetStateUnknown {
|
|
t.Fatalf("pending agent fleet governance = %+v", pending)
|
|
}
|
|
if pending.ConfigDrift == nil || pending.ConfigDrift.Status != fleetStateUnknown {
|
|
t.Fatalf("pending agent config drift = %+v", pending.ConfigDrift)
|
|
}
|
|
if pending.Rollout == nil || pending.Rollout.Status != fleetStatePending {
|
|
t.Fatalf("pending agent rollout = %+v", pending.Rollout)
|
|
}
|
|
if pending.CommandPolicy == nil ||
|
|
pending.CommandPolicy.Desired != fleetStateUnknown ||
|
|
pending.CommandPolicy.Applied != fleetStateUnknown ||
|
|
pending.CommandPolicy.Enforcement != fleetStatePending {
|
|
t.Fatalf("pending agent command policy = %+v", pending.CommandPolicy)
|
|
}
|
|
}
|
|
|
|
func TestBuildConnections_AgentWithoutManagedDesiredConfigDoesNotReportRolloutPending(t *testing.T) {
|
|
now := time.Now()
|
|
in := aggregatorInputs{
|
|
hosts: []models.Host{
|
|
{
|
|
ID: "default-agent",
|
|
Hostname: "default-agent",
|
|
LastSeen: now,
|
|
CommandsEnabled: false,
|
|
},
|
|
},
|
|
agentDesiredConfigs: map[string]connectionAgentDesiredConfig{
|
|
"default-agent": {},
|
|
},
|
|
now: now,
|
|
}
|
|
|
|
got := buildConnections(in)
|
|
if len(got) != 1 {
|
|
t.Fatalf("expected 1 connection, got %d", len(got))
|
|
}
|
|
|
|
fleet := got[0].Fleet
|
|
if fleet.ConfigDrift == nil ||
|
|
fleet.ConfigDrift.Status != fleetConfigDriftNotApplicable ||
|
|
fleet.ConfigDrift.Desired != nil ||
|
|
fleet.ConfigDrift.Applied != nil {
|
|
t.Fatalf("default agent config drift = %+v", fleet.ConfigDrift)
|
|
}
|
|
if fleet.Rollout == nil ||
|
|
fleet.Rollout.Status != fleetStateCurrent ||
|
|
fleet.Rollout.Stage != fleetRolloutStageApplied {
|
|
t.Fatalf("default agent rollout = %+v", fleet.Rollout)
|
|
}
|
|
}
|
|
|
|
func TestBuildConnections_AgentCommandPolicyDesiredAppliedConvergence(t *testing.T) {
|
|
now := time.Now()
|
|
desiredDisabled := desiredAgentConfig(t, ptrBool(false), nil)
|
|
desiredEnabled := desiredAgentConfig(t, ptrBool(true), nil)
|
|
in := aggregatorInputs{
|
|
hosts: []models.Host{
|
|
{
|
|
ID: "desired-disabled-applied-enabled",
|
|
Hostname: "desired-disabled-applied-enabled",
|
|
LastSeen: now,
|
|
CommandsEnabled: true,
|
|
},
|
|
{
|
|
ID: "desired-enabled-applied-disabled",
|
|
Hostname: "desired-enabled-applied-disabled",
|
|
LastSeen: now,
|
|
CommandsEnabled: false,
|
|
},
|
|
{
|
|
ID: "desired-disabled-unreported",
|
|
Hostname: "desired-disabled-unreported",
|
|
},
|
|
},
|
|
agentDesiredConfigs: map[string]connectionAgentDesiredConfig{
|
|
"desired-disabled-applied-enabled": desiredDisabled,
|
|
"desired-enabled-applied-disabled": desiredEnabled,
|
|
"desired-disabled-unreported": desiredDisabled,
|
|
},
|
|
now: now,
|
|
}
|
|
|
|
got := buildConnections(in)
|
|
byID := map[string]Connection{}
|
|
for _, connection := range got {
|
|
byID[connection.ID] = connection
|
|
}
|
|
|
|
disabledDesired := byID["agent:desired-disabled-applied-enabled"].Fleet
|
|
if disabledDesired.RemoteControl != fleetStateEnabled {
|
|
t.Fatalf("remote control for desired-disabled/applied-enabled = %q, want applied enabled", disabledDesired.RemoteControl)
|
|
}
|
|
if disabledDesired.CommandPolicy == nil ||
|
|
disabledDesired.CommandPolicy.Status != fleetStateEnabled ||
|
|
disabledDesired.CommandPolicy.Desired != fleetStateDisabled ||
|
|
disabledDesired.CommandPolicy.Applied != fleetStateEnabled ||
|
|
disabledDesired.CommandPolicy.Enforcement != fleetCommandPolicyDrifted {
|
|
t.Fatalf("desired-disabled/applied-enabled command policy = %+v", disabledDesired.CommandPolicy)
|
|
}
|
|
|
|
enabledDesired := byID["agent:desired-enabled-applied-disabled"].Fleet
|
|
if enabledDesired.RemoteControl != fleetStateDisabled {
|
|
t.Fatalf("remote control for desired-enabled/applied-disabled = %q, want applied disabled", enabledDesired.RemoteControl)
|
|
}
|
|
if enabledDesired.CommandPolicy == nil ||
|
|
enabledDesired.CommandPolicy.Status != fleetStateDisabled ||
|
|
enabledDesired.CommandPolicy.Desired != fleetStateEnabled ||
|
|
enabledDesired.CommandPolicy.Applied != fleetStateDisabled ||
|
|
enabledDesired.CommandPolicy.Enforcement != fleetCommandPolicyDrifted {
|
|
t.Fatalf("desired-enabled/applied-disabled command policy = %+v", enabledDesired.CommandPolicy)
|
|
}
|
|
|
|
unreported := byID["agent:desired-disabled-unreported"].Fleet
|
|
if unreported.RemoteControl != fleetStateUnknown {
|
|
t.Fatalf("remote control for unreported agent = %q, want unknown", unreported.RemoteControl)
|
|
}
|
|
if unreported.CommandPolicy == nil ||
|
|
unreported.CommandPolicy.Status != fleetStateUnknown ||
|
|
unreported.CommandPolicy.Desired != fleetStateDisabled ||
|
|
unreported.CommandPolicy.Applied != fleetStateUnknown ||
|
|
unreported.CommandPolicy.Enforcement != fleetStatePending {
|
|
t.Fatalf("unreported command policy = %+v", unreported.CommandPolicy)
|
|
}
|
|
}
|
|
|
|
func TestBuildConnections_AgentConfigDriftUsesCanonicalDesiredMetadataWithoutSelfComparing(t *testing.T) {
|
|
now := time.Now()
|
|
desiredConfig := desiredAgentConfig(t, ptrBool(true), map[string]interface{}{
|
|
"interval": "10s",
|
|
})
|
|
desired := *desiredConfig.Fingerprint
|
|
in := aggregatorInputs{
|
|
hosts: []models.Host{
|
|
{
|
|
ID: "agent-1",
|
|
Hostname: "agent-1",
|
|
LastSeen: now,
|
|
CommandsEnabled: false,
|
|
DiskExclude: []string{"/dev/loop*"},
|
|
},
|
|
},
|
|
agentDesiredConfigs: map[string]connectionAgentDesiredConfig{
|
|
"agent-1": desiredConfig,
|
|
},
|
|
now: now,
|
|
}
|
|
|
|
got := buildConnections(in)
|
|
if len(got) != 1 {
|
|
t.Fatalf("expected 1 connection, got %d", len(got))
|
|
}
|
|
drift := got[0].Fleet.ConfigDrift
|
|
if drift == nil {
|
|
t.Fatal("expected agent config drift metadata")
|
|
}
|
|
if drift.Status != fleetStatePending {
|
|
t.Fatalf("config drift status = %q, want pending", drift.Status)
|
|
}
|
|
if drift.Desired == nil || *drift.Desired != desired {
|
|
t.Fatalf("desired config drift fingerprint = %+v, want %+v", drift.Desired, desired)
|
|
}
|
|
if drift.Applied != nil {
|
|
t.Fatalf("applied config fingerprint should be absent until agent reports a comparable fingerprint, got %+v", drift.Applied)
|
|
}
|
|
|
|
selfCompared := connectionConfigFingerprint(connectionAgentConfigFingerprintVersion, map[string]any{
|
|
"commandsEnabled": false,
|
|
"diskExclude": []string{"/dev/loop*"},
|
|
})
|
|
if selfCompared == nil {
|
|
t.Fatal("expected local self-comparison fingerprint to be derivable")
|
|
}
|
|
if drift.Desired.Hash == selfCompared.Hash {
|
|
t.Fatalf("desired config hash reused report-field fingerprint %q", drift.Desired.Hash)
|
|
}
|
|
if got[0].Fleet.Rollout == nil || got[0].Fleet.Rollout.Status == fleetStateCurrent {
|
|
t.Fatalf("rollout should not claim current without an applied config comparison, got %+v", got[0].Fleet.Rollout)
|
|
}
|
|
}
|
|
|
|
func TestConnectionAgentDesiredConfigFingerprintsSkipsEmptyDefaultConfig(t *testing.T) {
|
|
monitor, err := monitoring.New(&config.Config{DataPath: t.TempDir()})
|
|
if err != nil {
|
|
t.Fatalf("monitoring.New: %v", err)
|
|
}
|
|
t.Cleanup(func() { monitor.Stop() })
|
|
|
|
hostID := "default-agent"
|
|
got := connectionAgentDesiredConfigFingerprints(monitor, []models.Host{{ID: hostID}}, nil)
|
|
desired, ok := got[hostID]
|
|
if !ok {
|
|
t.Fatalf("expected resolved desired config entry for %q, got %+v", hostID, got)
|
|
}
|
|
if desired.Fingerprint != nil {
|
|
t.Fatalf("empty default config should not create rollout fingerprint, got %+v", desired.Fingerprint)
|
|
}
|
|
if desired.CommandsEnabled != nil {
|
|
t.Fatalf("empty default config should not set command override, got %+v", desired.CommandsEnabled)
|
|
}
|
|
|
|
commandsEnabled := true
|
|
if err := monitor.UpdateHostAgentConfig(hostID, &commandsEnabled); err != nil {
|
|
t.Fatalf("UpdateHostAgentConfig: %v", err)
|
|
}
|
|
got = connectionAgentDesiredConfigFingerprints(monitor, []models.Host{{ID: hostID}}, nil)
|
|
desired = got[hostID]
|
|
if desired.Fingerprint == nil {
|
|
t.Fatalf("managed command override should create desired config fingerprint")
|
|
}
|
|
if desired.CommandsEnabled == nil || !*desired.CommandsEnabled {
|
|
t.Fatalf("managed command override = %+v, want true", desired.CommandsEnabled)
|
|
}
|
|
}
|
|
|
|
func TestConnectionAgentDesiredConfigFingerprintsUsesTokenEffectiveCommandPolicy(t *testing.T) {
|
|
monitor, err := monitoring.New(&config.Config{DataPath: t.TempDir()})
|
|
if err != nil {
|
|
t.Fatalf("monitoring.New: %v", err)
|
|
}
|
|
t.Cleanup(func() { monitor.Stop() })
|
|
|
|
hostID := "agent-token-gated"
|
|
rawDesiredCommands := true
|
|
if err := monitor.UpdateHostAgentConfig(hostID, &rawDesiredCommands); err != nil {
|
|
t.Fatalf("UpdateHostAgentConfig: %v", err)
|
|
}
|
|
|
|
now := time.Now()
|
|
host := models.Host{
|
|
ID: hostID,
|
|
Hostname: "agent-token-gated.local",
|
|
LastSeen: now,
|
|
CommandsEnabled: false,
|
|
TokenID: "runtime-without-exec",
|
|
}
|
|
tokens := []config.APITokenRecord{
|
|
{
|
|
ID: "runtime-without-exec",
|
|
Scopes: []string{config.ScopeAgentReport, config.ScopeAgentConfigRead},
|
|
},
|
|
}
|
|
|
|
got := connectionAgentDesiredConfigFingerprints(monitor, []models.Host{host}, tokens)
|
|
desired, ok := got[hostID]
|
|
if !ok {
|
|
t.Fatalf("expected resolved desired config entry for %q, got %+v", hostID, got)
|
|
}
|
|
if desired.CommandsEnabled == nil || *desired.CommandsEnabled {
|
|
t.Fatalf("effective desired CommandsEnabled = %+v, want false", desired.CommandsEnabled)
|
|
}
|
|
if desired.Fingerprint == nil {
|
|
t.Fatal("effective command override should still create a desired config fingerprint")
|
|
}
|
|
|
|
effectiveDisabled := false
|
|
effectiveMetadata, err := remoteconfig.BuildDesiredConfigMetadata(&effectiveDisabled, nil)
|
|
if err != nil {
|
|
t.Fatalf("BuildDesiredConfigMetadata disabled: %v", err)
|
|
}
|
|
rawMetadata, err := remoteconfig.BuildDesiredConfigMetadata(&rawDesiredCommands, nil)
|
|
if err != nil {
|
|
t.Fatalf("BuildDesiredConfigMetadata raw: %v", err)
|
|
}
|
|
if desired.Fingerprint.Hash != effectiveMetadata.Hash {
|
|
t.Fatalf("desired fingerprint hash = %q, want effective disabled hash %q", desired.Fingerprint.Hash, effectiveMetadata.Hash)
|
|
}
|
|
if desired.Fingerprint.Hash == rawMetadata.Hash {
|
|
t.Fatalf("desired fingerprint reused raw unsanitized command policy hash %q", desired.Fingerprint.Hash)
|
|
}
|
|
|
|
connections := buildConnections(aggregatorInputs{
|
|
hosts: []models.Host{host},
|
|
agentDesiredConfigs: got,
|
|
now: now,
|
|
})
|
|
if len(connections) != 1 {
|
|
t.Fatalf("expected one connection, got %d", len(connections))
|
|
}
|
|
policy := connections[0].Fleet.CommandPolicy
|
|
if policy == nil ||
|
|
policy.Status != fleetStateDisabled ||
|
|
policy.Desired != fleetStateDisabled ||
|
|
policy.Applied != fleetStateDisabled ||
|
|
policy.Enforcement != fleetCommandPolicyInSync {
|
|
t.Fatalf("command policy = %+v, want effective disabled in sync", policy)
|
|
}
|
|
}
|
|
|
|
func TestConnectionFleetAgentConfigDriftComparesAppliedFingerprintsWhenAvailable(t *testing.T) {
|
|
now := time.Now()
|
|
conn := Connection{
|
|
Type: ConnectionTypeAgent,
|
|
State: ConnectionStateActive,
|
|
Enabled: true,
|
|
LastSeen: &now,
|
|
}
|
|
desired := &ConnectionFleetConfigFingerprint{Version: connectionAgentConfigFingerprintVersion, Hash: "sha256:desired"}
|
|
applied := &ConnectionFleetConfigFingerprint{Version: connectionAgentConfigFingerprintVersion, Hash: "sha256:applied"}
|
|
|
|
drifted := connectionFleetAgentConfigDriftForFingerprints(conn, desired, applied)
|
|
if drifted.Status != fleetConfigDriftDrifted ||
|
|
drifted.Desired != desired ||
|
|
drifted.Applied != applied ||
|
|
drifted.LastObservedAt == nil {
|
|
t.Fatalf("drifted config comparison = %+v", drifted)
|
|
}
|
|
|
|
matchingApplied := &ConnectionFleetConfigFingerprint{Version: desired.Version, Hash: desired.Hash}
|
|
current := connectionFleetAgentConfigDriftForFingerprints(conn, desired, matchingApplied)
|
|
if current.Status != fleetConfigDriftCurrent ||
|
|
current.Desired != desired ||
|
|
current.Applied != matchingApplied ||
|
|
current.LastObservedAt == nil {
|
|
t.Fatalf("current config comparison = %+v", current)
|
|
}
|
|
}
|
|
|
|
func TestBuildConnections_PlatformFleetGovernance(t *testing.T) {
|
|
now := time.Now()
|
|
lastSuccess := now.Add(-30 * time.Second)
|
|
in := aggregatorInputs{
|
|
pveInstances: []config.PVEInstance{
|
|
{Name: "healthy", Host: "https://healthy.lan:8006"},
|
|
{Name: "bad-token", Host: "https://bad-token.lan:8006"},
|
|
{Name: "paused", Host: "https://paused.lan:8006", Disabled: true},
|
|
},
|
|
instanceHealth: map[string]monitoring.InstanceHealth{
|
|
"pve::healthy": healthEntry(&lastSuccess, "", "", "closed"),
|
|
"pve::bad-token": healthEntry(&lastSuccess, "403 forbidden", "auth", "closed"),
|
|
},
|
|
now: now,
|
|
}
|
|
|
|
got := buildConnections(in)
|
|
byID := map[string]Connection{}
|
|
for _, connection := range got {
|
|
byID[connection.ID] = connection
|
|
}
|
|
|
|
healthy := byID["pve:healthy"].Fleet
|
|
if healthy.EnrollmentState != fleetStateConfigured ||
|
|
healthy.LivenessState != fleetStateActive ||
|
|
healthy.VersionDrift != fleetStateNotApplicable ||
|
|
healthy.AdapterHealth != fleetStateHealthy ||
|
|
healthy.ConfigRollout != fleetStateConfigured ||
|
|
healthy.CredentialStatus != fleetStateVerified ||
|
|
healthy.UpdateStatus != fleetStateNotApplicable ||
|
|
healthy.RemoteControl != fleetStateNotApplicable {
|
|
t.Fatalf("healthy platform fleet governance = %+v", healthy)
|
|
}
|
|
if healthy.ConfigDrift == nil ||
|
|
healthy.ConfigDrift.Status != fleetConfigDriftCurrent ||
|
|
healthy.ConfigDrift.Desired == nil ||
|
|
healthy.ConfigDrift.Applied == nil ||
|
|
healthy.ConfigDrift.Desired.Hash != healthy.ConfigDrift.Applied.Hash {
|
|
t.Fatalf("healthy platform config drift = %+v", healthy.ConfigDrift)
|
|
}
|
|
if healthy.Rollout == nil || healthy.Rollout.Status != fleetStateCurrent {
|
|
t.Fatalf("healthy platform rollout = %+v", healthy.Rollout)
|
|
}
|
|
if healthy.CredentialHealth == nil || healthy.CredentialHealth.Status != fleetStateVerified {
|
|
t.Fatalf("healthy platform credential health = %+v", healthy.CredentialHealth)
|
|
}
|
|
|
|
badToken := byID["pve:bad-token"].Fleet
|
|
if badToken.AdapterHealth != fleetStateBlocked ||
|
|
badToken.CredentialStatus != fleetStateInvalid ||
|
|
badToken.LivenessState != string(ConnectionStateUnauthorized) {
|
|
t.Fatalf("bad token fleet governance = %+v", badToken)
|
|
}
|
|
if badToken.CredentialHealth == nil ||
|
|
badToken.CredentialHealth.Status != fleetStateInvalid ||
|
|
badToken.CredentialHealth.LastFailedAt == nil {
|
|
t.Fatalf("bad token credential health = %+v", badToken.CredentialHealth)
|
|
}
|
|
if badToken.Rollout == nil || badToken.Rollout.Status != fleetRolloutBlocked {
|
|
t.Fatalf("bad token rollout = %+v", badToken.Rollout)
|
|
}
|
|
|
|
paused := byID["pve:paused"].Fleet
|
|
if paused.EnrollmentState != fleetStatePaused ||
|
|
paused.AdapterHealth != fleetStatePaused ||
|
|
paused.ConfigRollout != fleetStatePaused ||
|
|
paused.CredentialStatus != fleetStatePaused {
|
|
t.Fatalf("paused platform fleet governance = %+v", paused)
|
|
}
|
|
if paused.ConfigDrift == nil || paused.ConfigDrift.Status != fleetStatePaused {
|
|
t.Fatalf("paused config drift = %+v", paused.ConfigDrift)
|
|
}
|
|
if paused.Rollout == nil || paused.Rollout.Status != fleetStatePaused {
|
|
t.Fatalf("paused rollout = %+v", paused.Rollout)
|
|
}
|
|
}
|
|
|
|
func TestBuildConnections_VMwareAndTrueNASEnabledFlag(t *testing.T) {
|
|
in := aggregatorInputs{
|
|
vmwareInstances: []config.VMwareVCenterInstance{{
|
|
ID: "vc1", Name: "vc", Host: "vc.lan", Enabled: false,
|
|
MonitorVMs: true, MonitorHosts: true, MonitorDatastores: false,
|
|
}},
|
|
truenasInstances: []config.TrueNASInstance{{
|
|
ID: "tn1", Name: "tn", Host: "tn.lan", Enabled: true, UseHTTPS: true,
|
|
MonitorDatasets: true, MonitorPools: false, MonitorReplication: true,
|
|
}},
|
|
now: time.Now(),
|
|
}
|
|
got := buildConnections(in)
|
|
var vmw, tn Connection
|
|
for _, c := range got {
|
|
switch c.Type {
|
|
case ConnectionTypeVMware:
|
|
vmw = c
|
|
case ConnectionTypeTrueNAS:
|
|
tn = c
|
|
}
|
|
}
|
|
if vmw.State != ConnectionStatePaused || vmw.Enabled {
|
|
t.Fatalf("vmware with Enabled=false should be paused, got state=%q enabled=%v", vmw.State, vmw.Enabled)
|
|
}
|
|
if !vmw.Capabilities.SupportsScope {
|
|
t.Fatal("vmware connections must advertise scope capability")
|
|
}
|
|
if vmw.Scope["datastores"] || !vmw.Scope["vms"] || !vmw.Scope["hosts"] {
|
|
t.Fatalf("vmware scope map should reflect Monitor* fields, got %+v", vmw.Scope)
|
|
}
|
|
if tn.State != ConnectionStatePending {
|
|
t.Fatalf("truenas with no health yet should be pending, got %q", tn.State)
|
|
}
|
|
if !tn.Enabled {
|
|
t.Fatal("truenas with Enabled=true should surface enabled=true")
|
|
}
|
|
if !tn.Capabilities.SupportsScope {
|
|
t.Fatal("truenas connections must advertise scope capability")
|
|
}
|
|
if tn.Scope["pools"] || !tn.Scope["datasets"] || !tn.Scope["replication"] {
|
|
t.Fatalf("truenas scope map should reflect Monitor* fields, got %+v", tn.Scope)
|
|
}
|
|
}
|
|
|
|
func TestBuildConnections_PlatformPollerSummariesDriveRuntimeState(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
vmwareSuccess := now.Add(-40 * time.Second)
|
|
trueNASSuccess := now.Add(-30 * time.Second)
|
|
in := aggregatorInputs{
|
|
vmwareInstances: []config.VMwareVCenterInstance{{
|
|
ID: "vc1", Name: "vc", Host: "vc.lan", Enabled: true,
|
|
MonitorVMs: true, MonitorHosts: true, MonitorDatastores: true,
|
|
}},
|
|
vmwareSummaries: map[string]monitoring.VMwareConnectionSummary{
|
|
"vc1": {
|
|
Poll: &monitoring.VMwareConnectionPollStatus{
|
|
LastSuccessAt: &vmwareSuccess,
|
|
},
|
|
},
|
|
},
|
|
truenasInstances: []config.TrueNASInstance{{
|
|
ID: "tn1", Name: "tn", Host: "tn.lan", Enabled: true, UseHTTPS: true,
|
|
APIKey: "secret", MonitorDatasets: true, MonitorPools: true, MonitorReplication: true,
|
|
}},
|
|
truenasSummaries: map[string]monitoring.TrueNASConnectionSummary{
|
|
"tn1": {
|
|
Poll: &monitoring.TrueNASConnectionPollStatus{
|
|
LastSuccessAt: &trueNASSuccess,
|
|
},
|
|
},
|
|
},
|
|
now: now,
|
|
}
|
|
|
|
got := buildConnections(in)
|
|
byID := map[string]Connection{}
|
|
for _, connection := range got {
|
|
byID[connection.ID] = connection
|
|
}
|
|
|
|
vmw := byID["vmware:vc1"]
|
|
if vmw.State != ConnectionStateActive {
|
|
t.Fatalf("vmware state = %q, want active", vmw.State)
|
|
}
|
|
if vmw.LastSeen == nil || !vmw.LastSeen.Equal(vmwareSuccess) {
|
|
t.Fatalf("vmware lastSeen = %+v, want %s", vmw.LastSeen, vmwareSuccess)
|
|
}
|
|
if vmw.Fleet.CredentialStatus != fleetStateVerified || vmw.Fleet.CredentialHealth == nil || vmw.Fleet.CredentialHealth.Status != fleetStateVerified {
|
|
t.Fatalf("vmware credential governance = %+v", vmw.Fleet)
|
|
}
|
|
|
|
tn := byID["truenas:tn1"]
|
|
if tn.State != ConnectionStateActive {
|
|
t.Fatalf("truenas state = %q, want active", tn.State)
|
|
}
|
|
if tn.LastSeen == nil || !tn.LastSeen.Equal(trueNASSuccess) {
|
|
t.Fatalf("truenas lastSeen = %+v, want %s", tn.LastSeen, trueNASSuccess)
|
|
}
|
|
if tn.Fleet.CredentialStatus != fleetStateVerified || tn.Fleet.CredentialHealth == nil || tn.Fleet.CredentialHealth.Status != fleetStateVerified {
|
|
t.Fatalf("truenas credential governance = %+v", tn.Fleet)
|
|
}
|
|
}
|
|
|
|
func TestDeriveConnectionState_FirstPollFailureIsUnreachable(t *testing.T) {
|
|
now := time.Now()
|
|
h := healthEntry(nil, "connection refused", "network", "closed")
|
|
state, reason, lastSeen, lastError := deriveConnectionState(true, h, now, 0)
|
|
if state != ConnectionStateUnreachable {
|
|
t.Fatalf("state = %q, want unreachable", state)
|
|
}
|
|
if reason != "connection refused" {
|
|
t.Fatalf("reason = %q, want connection refused", reason)
|
|
}
|
|
if lastSeen != nil {
|
|
t.Fatalf("lastSeen = %+v, want nil", lastSeen)
|
|
}
|
|
if lastError == nil || lastError.Message != "connection refused" {
|
|
t.Fatalf("lastError = %+v, want connection refused", lastError)
|
|
}
|
|
}
|
|
|
|
func TestDeriveConnectionState_CurrentFailureOverridesRecentSuccess(t *testing.T) {
|
|
now := time.Now()
|
|
lastSuccess := now.Add(-5 * time.Second)
|
|
|
|
for _, test := range []struct {
|
|
name string
|
|
message string
|
|
wantState ConnectionState
|
|
}{
|
|
{
|
|
name: "authentication rejected",
|
|
message: "401 Unauthorized: token invalid",
|
|
wantState: ConnectionStateUnauthorized,
|
|
},
|
|
{
|
|
name: "network timeout",
|
|
message: "context deadline exceeded while awaiting response headers",
|
|
wantState: ConnectionStateUnreachable,
|
|
},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
h := healthEntry(&lastSuccess, test.message, "transient", "closed")
|
|
state, reason, gotLastSeen, lastError := deriveConnectionState(true, h, now, 0)
|
|
if state != test.wantState {
|
|
t.Fatalf("state = %q, want %q", state, test.wantState)
|
|
}
|
|
if reason != test.message || lastError == nil || lastError.Message != test.message {
|
|
t.Fatalf("failure detail = reason %q error %+v, want %q", reason, lastError, test.message)
|
|
}
|
|
if gotLastSeen == nil || !gotLastSeen.Equal(lastSuccess) {
|
|
t.Fatalf("lastSeen = %+v, want preserved success %s", gotLastSeen, lastSuccess)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestBuildConnectionsKeepsDistinctPBSSystemsWithSameHostname(t *testing.T) {
|
|
now := time.Now()
|
|
lastSuccess := now.Add(-10 * time.Second)
|
|
got := buildConnections(aggregatorInputs{
|
|
pbsInstances: []config.PBSInstance{
|
|
{Name: "pbs-east", Host: "https://backup.local:8007"},
|
|
{Name: "pbs-west", Host: "https://backup.local:8007"},
|
|
},
|
|
instanceHealth: map[string]monitoring.InstanceHealth{
|
|
"pbs::pbs-east": healthEntry(&lastSuccess, "", "", "closed"),
|
|
"pbs::pbs-west": healthEntry(&lastSuccess, "dial tcp: i/o timeout", "transient", "closed"),
|
|
},
|
|
now: now,
|
|
})
|
|
if len(got) != 2 {
|
|
t.Fatalf("connections = %d, want two distinct PBS rows", len(got))
|
|
}
|
|
byID := make(map[string]Connection, len(got))
|
|
for _, connection := range got {
|
|
byID[connection.ID] = connection
|
|
}
|
|
if byID["pbs:pbs-east"].State != ConnectionStateActive {
|
|
t.Fatalf("pbs-east = %+v, want active", byID["pbs:pbs-east"])
|
|
}
|
|
if byID["pbs:pbs-west"].State != ConnectionStateUnreachable {
|
|
t.Fatalf("pbs-west = %+v, want unreachable", byID["pbs:pbs-west"])
|
|
}
|
|
|
|
alertSnapshots := snapshotConnectionsForAlerts(got)
|
|
if len(alertSnapshots) != 2 {
|
|
t.Fatalf("alert snapshots = %+v, want both PBS systems", alertSnapshots)
|
|
}
|
|
alertsByID := make(map[string]alerts.ConnectionSnapshot, len(alertSnapshots))
|
|
for _, snapshot := range alertSnapshots {
|
|
alertsByID[snapshot.ID] = snapshot
|
|
}
|
|
if alertsByID["pbs:pbs-east"].State != alerts.ConnectionStateActive {
|
|
t.Fatalf("pbs-east alert state = %+v, want active", alertsByID["pbs:pbs-east"])
|
|
}
|
|
if west := alertsByID["pbs:pbs-west"]; west.State != alerts.ConnectionStateUnreachable || west.LastError == nil {
|
|
t.Fatalf("pbs-west alert state = %+v, want unreachable with current error", west)
|
|
}
|
|
}
|
|
|
|
func TestBuildConnections_UsesHealthLookup(t *testing.T) {
|
|
now := time.Now()
|
|
ls := now.Add(-15 * time.Second)
|
|
in := aggregatorInputs{
|
|
pveInstances: []config.PVEInstance{{Name: "pve1", Host: "https://pve1.lan:8006"}},
|
|
instanceHealth: map[string]monitoring.InstanceHealth{
|
|
"pve::pve1": healthEntry(&ls, "", "", "closed"),
|
|
},
|
|
now: now,
|
|
}
|
|
got := buildConnections(in)
|
|
if got[0].State != ConnectionStateActive {
|
|
t.Fatalf("expected active, got %q", got[0].State)
|
|
}
|
|
if got[0].LastSeen == nil || !got[0].LastSeen.Equal(ls) {
|
|
t.Fatalf("lastSeen not propagated: %+v", got[0].LastSeen)
|
|
}
|
|
}
|
|
|
|
func TestConnectionStaleThresholdFor(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
interval time.Duration
|
|
want time.Duration
|
|
}{
|
|
{name: "unknown cadence keeps floor", interval: 0, want: connectionStaleThreshold},
|
|
{name: "negative cadence keeps floor", interval: -time.Minute, want: connectionStaleThreshold},
|
|
{name: "fast cadence keeps floor", interval: 10 * time.Second, want: connectionStaleThreshold},
|
|
{name: "floor boundary cadence keeps floor", interval: 40 * time.Second, want: connectionStaleThreshold},
|
|
{name: "slow cadence scales to 3x", interval: 5 * time.Minute, want: 15 * time.Minute},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
if got := connectionStaleThresholdFor(test.interval); got != test.want {
|
|
t.Fatalf("connectionStaleThresholdFor(%s) = %s, want %s", test.interval, got, test.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Regression for https://github.com/rcourtman/Pulse/issues/1620: with a
|
|
// 5-minute polling cadence the previous fixed 2-minute cutoff marked every
|
|
// connection permanently stale, latching the connection-degraded alert.
|
|
func TestDeriveConnectionState_SlowCadenceIsNotPermanentlyStale(t *testing.T) {
|
|
now := time.Now()
|
|
interval := 5 * time.Minute
|
|
|
|
// A last success exactly one interval old is the steady state when the
|
|
// connection check runs concurrently with the poll on the same tick; it
|
|
// must derive active, not stale.
|
|
h := healthEntry(ptrTime(now.Add(-interval)), "", "", "closed")
|
|
state, _, _, _ := deriveConnectionState(true, h, now, interval)
|
|
if state != ConnectionStateActive {
|
|
t.Fatalf("state at 1x interval = %q, want %q", state, ConnectionStateActive)
|
|
}
|
|
|
|
// Beyond 3x the interval the connection is genuinely stale.
|
|
h = healthEntry(ptrTime(now.Add(-3*interval-time.Second)), "", "", "closed")
|
|
state, _, _, _ = deriveConnectionState(true, h, now, interval)
|
|
if state != ConnectionStateStale {
|
|
t.Fatalf("state beyond 3x interval = %q, want %q", state, ConnectionStateStale)
|
|
}
|
|
}
|
|
|
|
func TestBuildConnections_ScalesStaleThresholdWithPollingIntervals(t *testing.T) {
|
|
now := time.Now()
|
|
lastSuccess := now.Add(-5 * time.Minute)
|
|
in := aggregatorInputs{
|
|
pveInstances: []config.PVEInstance{{Name: "pve1", Host: "https://pve1.lan:8006"}},
|
|
pbsInstances: []config.PBSInstance{{Name: "pbs1", Host: "https://pbs1.lan:8007"}},
|
|
truenasInstances: []config.TrueNASInstance{
|
|
{ID: "tn1", Name: "tn1", Host: "tn1.lan", Enabled: true, PollIntervalSecs: 300},
|
|
},
|
|
instanceHealth: map[string]monitoring.InstanceHealth{
|
|
"pve::pve1": healthEntry(&lastSuccess, "", "", "closed"),
|
|
"pbs::pbs1": healthEntry(&lastSuccess, "", "", "closed"),
|
|
"truenas::tn1": healthEntry(&lastSuccess, "", "", "closed"),
|
|
},
|
|
pvePollingInterval: 5 * time.Minute,
|
|
now: now,
|
|
}
|
|
got := buildConnections(in)
|
|
byID := make(map[string]Connection, len(got))
|
|
for _, conn := range got {
|
|
byID[conn.ID] = conn
|
|
}
|
|
|
|
if state := byID["pve:pve1"].State; state != ConnectionStateActive {
|
|
t.Fatalf("pve state = %q, want active (threshold scaled by pvePollingInterval)", state)
|
|
}
|
|
if state := byID["truenas:tn1"].State; state != ConnectionStateActive {
|
|
t.Fatalf("truenas state = %q, want active (threshold scaled by PollIntervalSecs)", state)
|
|
}
|
|
// PBS was left at the zero-value interval, so the 2-minute floor still
|
|
// applies and the same 5-minute-old success reads as stale.
|
|
if state := byID["pbs:pbs1"].State; state != ConnectionStateStale {
|
|
t.Fatalf("pbs state = %q, want stale under the floor threshold", state)
|
|
}
|
|
}
|