package monitoring import ( "context" "errors" "fmt" "net" "os" "path/filepath" "sync/atomic" "testing" "time" "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/pkg/tlsutil" ) // Regression tests for discussion #1638: the cluster-endpoint discovery-policy // check ran raw DNS lookups per node per poll cycle, and the SSH collectors // re-executed ssh against hosts that kept failing. The policy check must now go // through the shared cached resolver — which keeps DNS at roughly one query per // host per cache refresh while still enforcing the blocklist against resolved // addresses — and the SSH backoffs must only escalate for work that actually // ran. func issue1638CountingLookup(t *testing.T, calls *int, ips map[string][]net.IP) { t.Helper() oldLookup := lookupIPFunc lookupIPFunc = func(host string) ([]net.IP, error) { *calls++ if resolved, ok := ips[host]; ok { return resolved, nil } return nil, errors.New("no such host") } t.Cleanup(func() { lookupIPFunc = oldLookup }) } // issue1638FakeClock pins the SSH backoff clock so compounding, decay, and // expiry are observable without sleeping. type issue1638FakeClock struct { now time.Time } func (c *issue1638FakeClock) advance(d time.Duration) { c.now = c.now.Add(d) } func issue1638UseFakeClock(t *testing.T) *issue1638FakeClock { t.Helper() clock := &issue1638FakeClock{now: time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)} old := sshBackoffNow sshBackoffNow = func() time.Time { return clock.now } t.Cleanup(func() { sshBackoffNow = old }) return clock } // TestIssue1638DefaultPolicyBlocksHostnameResolvingToLinkLocal pins the // blocklist enforcement that the first fix traded away. The default policy is // just the injected 169.254.0.0/16 blocklist, and it has to apply to whatever a // hostname endpoint resolves to, not only to literal link-local IPs — otherwise // a name pointed at the metadata range walks straight through. func TestIssue1638DefaultPolicyBlocksHostnameResolvingToLinkLocal(t *testing.T) { calls := 0 issue1638CountingLookup(t, &calls, map[string][]net.IP{ "imds.example.com": {net.ParseIP("169.254.169.254")}, "node-a.local": {net.ParseIP("192.168.1.5")}, }) // NormalizeDiscoveryConfig injects the default link-local blocklist, so // this is what every install without an explicit policy runs with. discoveryCfg := config.NormalizeDiscoveryConfig(config.DiscoveryConfig{}) blocked := config.ClusterEndpoint{NodeName: "node-imds", Host: "imds.example.com"} if got := clusterEndpointRuntimeURL(blocked, true, false, discoveryCfg); got != "" { t.Fatalf("hostname resolving into 169.254.0.0/16 was allowed: %q", got) } allowed := config.ClusterEndpoint{NodeName: "node-a", Host: "node-a.local"} if got := clusterEndpointRuntimeURL(allowed, true, false, discoveryCfg); got != "https://node-a.local:8006" { t.Fatalf("ordinary hostname endpoint URL = %q, want %q", got, "https://node-a.local:8006") } } func TestIssue1638DefaultPolicyStillBlocksLiteralLinkLocalWithoutDNS(t *testing.T) { calls := 0 issue1638CountingLookup(t, &calls, nil) discoveryCfg := config.NormalizeDiscoveryConfig(config.DiscoveryConfig{}) endpoint := config.ClusterEndpoint{NodeName: "node-b", IP: "169.254.10.20"} if got := clusterEndpointRuntimeURL(endpoint, false, false, discoveryCfg); got != "" { t.Fatalf("literal link-local endpoint allowed through default blocklist: %q", got) } if calls != 0 { t.Fatalf("literal IP evaluation hit the resolver %d times, want 0", calls) } } func TestIssue1638CustomPolicyEnforcedPerPoll(t *testing.T) { calls := 0 issue1638CountingLookup(t, &calls, map[string][]net.IP{ "allowed.local": {net.ParseIP("10.0.0.10")}, "blocked.local": {net.ParseIP("192.168.1.10")}, }) discoveryCfg := config.NormalizeDiscoveryConfig(config.DiscoveryConfig{ SubnetAllowlist: []string{"10.0.0.0/8"}, }) allowed := config.ClusterEndpoint{NodeName: "node-a", Host: "allowed.local"} blocked := config.ClusterEndpoint{NodeName: "node-b", Host: "blocked.local"} for poll := 0; poll < 20; poll++ { if got := clusterEndpointRuntimeURL(allowed, true, false, discoveryCfg); got != "https://allowed.local:8006" { t.Fatalf("poll %d: allowed endpoint URL = %q", poll, got) } if got := clusterEndpointRuntimeURL(blocked, true, false, discoveryCfg); got != "" { t.Fatalf("poll %d: blocked endpoint unexpectedly allowed: %q", poll, got) } } } // issue1638CountingResolver counts the DNS queries the shared cached resolver // actually issues, as opposed to the lookups the policy check asks for. type issue1638CountingResolver struct { queries int32 answers map[string][]string } func (r *issue1638CountingResolver) LookupHost(_ context.Context, host string) ([]string, error) { atomic.AddInt32(&r.queries, 1) if answer, ok := r.answers[host]; ok { return answer, nil } return nil, &net.DNSError{Err: "no such host", Name: host, IsNotFound: true} } func (r *issue1638CountingResolver) LookupAddr(_ context.Context, addr string) ([]string, error) { atomic.AddInt32(&r.queries, 1) return nil, &net.DNSError{Err: "no such host", Name: addr, IsNotFound: true} } // TestIssue1638PolicyLookupsStayCachedAcrossPolls is the #1638 win itself: // with the decision cache gone, repeat poll cycles must still not generate DNS // traffic, because the policy check resolves through the same process-global // cached resolver that pkg/tlsutil dials with. func TestIssue1638PolicyLookupsStayCachedAcrossPolls(t *testing.T) { // Names unique to this test so the process-global cache cannot be warmed // or poisoned by another test in the package. const ( allowedHost = "issue1638-allowed.invalid" blockedHost = "issue1638-blocked.invalid" ) counting := &issue1638CountingResolver{answers: map[string][]string{ allowedHost: {"10.0.0.10"}, blockedHost: {"169.254.169.254"}, }} resolver := tlsutil.GetDNSResolver() previous := resolver.Resolver resolver.Resolver = counting t.Cleanup(func() { resolver.Resolver = previous }) discoveryCfg := config.NormalizeDiscoveryConfig(config.DiscoveryConfig{}) allowed := config.ClusterEndpoint{NodeName: "node-a", Host: allowedHost} blocked := config.ClusterEndpoint{NodeName: "node-b", Host: blockedHost} for poll := 0; poll < 20; poll++ { if got := clusterEndpointRuntimeURL(allowed, true, false, discoveryCfg); got != "https://"+allowedHost+":8006" { t.Fatalf("poll %d: allowed endpoint URL = %q", poll, got) } if got := clusterEndpointRuntimeURL(blocked, true, false, discoveryCfg); got != "" { t.Fatalf("poll %d: link-local endpoint unexpectedly allowed: %q", poll, got) } } if queries := atomic.LoadInt32(&counting.queries); queries != 2 { t.Fatalf("20 poll cycles issued %d DNS queries, want exactly 2 (one per endpoint host)", queries) } } func TestIssue1638KeyscanFailureIsNotRetriedEveryCycle(t *testing.T) { clock := issue1638UseFakeClock(t) scans := 0 manager, err := NewKnownHostsManager( filepath.Join(t.TempDir(), "known_hosts"), WithKeyscanFunc(func(ctx context.Context, host string, port int, timeout time.Duration) ([]byte, error) { scans++ return nil, errors.New("connection refused") }), ) if err != nil { t.Fatalf("NewKnownHostsManager: %v", err) } for cycle := 0; cycle < 5; cycle++ { if err := manager.Ensure(context.Background(), "unreachable.local"); err == nil { t.Fatalf("cycle %d: expected error from failing keyscan", cycle) } } if scans != 1 { t.Fatalf("failing host was keyscanned %d times within backoff window, want 1", scans) } // Suppressed calls must be distinguishable from a real refusal so callers // don't escalate their own backoff for work that never ran. suppressed := manager.Ensure(context.Background(), "unreachable.local") if !errors.Is(suppressed, ErrKeyscanSuppressed) { t.Fatalf("suppressed Ensure error = %v, want it to wrap ErrKeyscanSuppressed", suppressed) } // Resetting the manager (as applying settings does) retries immediately // rather than waiting out the window. manager.ResetFailures() if err := manager.Ensure(context.Background(), "unreachable.local"); err == nil { t.Fatal("expected error from failing keyscan after reset") } if scans != 2 { t.Fatalf("reset did not retry the keyscan, scans = %d, want 2", scans) } // Backoff compounds only while failures keep arriving inside the window. clock.advance(keyscanFailureInitialBackoff + time.Second) if err := manager.Ensure(context.Background(), "unreachable.local"); err == nil { t.Fatal("expected error once the first backoff window expired") } if scans != 3 { t.Fatalf("expired backoff did not retry, scans = %d, want 3", scans) } clock.advance(keyscanFailureInitialBackoff + time.Second) if err := manager.Ensure(context.Background(), "unreachable.local"); !errors.Is(err, ErrKeyscanSuppressed) { t.Fatalf("second failure should have doubled the window, got %v", err) } if scans != 3 { t.Fatalf("doubled backoff window still re-scanned, scans = %d, want 3", scans) } } func TestIssue1638KeyscanTimeoutDoesNotCompound(t *testing.T) { clock := issue1638UseFakeClock(t) scans := 0 manager, err := NewKnownHostsManager( filepath.Join(t.TempDir(), "known_hosts"), WithKeyscanFunc(func(ctx context.Context, host string, port int, timeout time.Duration) ([]byte, error) { scans++ return nil, fmt.Errorf("ssh-keyscan gave up: %w", context.DeadlineExceeded) }), ) if err != nil { t.Fatalf("NewKnownHostsManager: %v", err) } // Three consecutive timeouts, each one window apart. A compounding backoff // would have suppressed the third; a timeout is not evidence about the // host, so the window stays at the floor. for cycle := 0; cycle < 3; cycle++ { if err := manager.Ensure(context.Background(), "slow.local"); err == nil { t.Fatalf("cycle %d: expected timeout error", cycle) } clock.advance(keyscanFailureInitialBackoff + time.Second) } if scans != 3 { t.Fatalf("timeouts compounded the keyscan backoff, scans = %d, want 3", scans) } } type issue1638CountingRunner struct { runs int err error } func (r *issue1638CountingRunner) Run(ctx context.Context, name string, args ...string) ([]byte, error) { r.runs++ if r.err != nil { return nil, r.err } return nil, errors.New("connection refused") } func issue1638TemperatureCollector(t *testing.T) (*TemperatureCollector, *issue1638CountingRunner) { t.Helper() tc, runner, _ := issue1638TemperatureCollectorWithKey(t) return tc, runner } func issue1638TemperatureCollectorWithKey(t *testing.T) (*TemperatureCollector, *issue1638CountingRunner, string) { t.Helper() keyPath := filepath.Join(t.TempDir(), "id_ed25519_test") if err := os.WriteFile(keyPath, []byte("dummy"), 0o600); err != nil { t.Fatalf("write key: %v", err) } tc := NewTemperatureCollectorWithPort("root", keyPath, 22) tc.hostKeys = nil runner := &issue1638CountingRunner{} tc.runner = runner return tc, runner, keyPath } // TestIssue1638RepairedSSHKeyClearsBackoff pins the escape hatch from the // backoff: replacing the SSH key is the usual fix for whatever opened the // window, so the next cycle must retry instead of waiting the window out. func TestIssue1638RepairedSSHKeyClearsBackoff(t *testing.T) { issue1638UseFakeClock(t) tc, runner, keyPath := issue1638TemperatureCollectorWithKey(t) if _, err := tc.CollectTemperature(context.Background(), "node1.local", "node1"); err != nil { t.Fatalf("CollectTemperature: %v", err) } if runner.runs != 2 { t.Fatalf("first cycle ran %d ssh commands, want 2", runner.runs) } if _, err := tc.CollectTemperature(context.Background(), "node1.local", "node1"); err != nil { t.Fatalf("CollectTemperature inside backoff: %v", err) } if runner.runs != 2 { t.Fatalf("backoff window still ran ssh, total %d commands, want 2", runner.runs) } // Operator installs a working key. if err := os.WriteFile(keyPath, []byte("repaired"), 0o600); err != nil { t.Fatalf("rewrite key: %v", err) } if err := os.Chtimes(keyPath, time.Now().Add(time.Minute), time.Now().Add(time.Minute)); err != nil { t.Fatalf("touch key: %v", err) } if _, err := tc.CollectTemperature(context.Background(), "node1.local", "node1"); err != nil { t.Fatalf("CollectTemperature after key repair: %v", err) } if runner.runs != 4 { t.Fatalf("repaired key did not clear the backoff, total %d commands, want 4", runner.runs) } } func TestIssue1638TemperatureSSHFailureBacksOff(t *testing.T) { clock := issue1638UseFakeClock(t) tc, runner := issue1638TemperatureCollector(t) // First cycle attempts both the sensors and the RPi fallback command. if _, err := tc.CollectTemperature(context.Background(), "node1.local", "node1"); err != nil { t.Fatalf("CollectTemperature: %v", err) } if runner.runs != 2 { t.Fatalf("first cycle ran %d ssh commands, want 2", runner.runs) } // Subsequent cycles inside the backoff window must not exec ssh again. for cycle := 0; cycle < 5; cycle++ { if _, err := tc.CollectTemperature(context.Background(), "node1.local", "node1"); err != nil { t.Fatalf("cycle %d: CollectTemperature: %v", cycle, err) } } if runner.runs != 2 { t.Fatalf("backoff window still ran ssh, total %d commands, want 2", runner.runs) } // Once the first window expires the host is retried, and that second hard // failure doubles the window: a poll one initial window later is suppressed. clock.advance(temperatureSSHFailureInitialBackoff + time.Second) if _, err := tc.CollectTemperature(context.Background(), "node1.local", "node1"); err != nil { t.Fatalf("CollectTemperature after expiry: %v", err) } if runner.runs != 4 { t.Fatalf("expired backoff did not retry, total %d commands, want 4", runner.runs) } clock.advance(temperatureSSHFailureInitialBackoff + time.Second) if _, err := tc.CollectTemperature(context.Background(), "node1.local", "node1"); err != nil { t.Fatalf("CollectTemperature inside doubled window: %v", err) } if runner.runs != 4 { t.Fatalf("backoff did not compound, total %d commands, want 4", runner.runs) } // Applying settings clears the backoff so a repaired key is picked up on // the next cycle instead of waiting the window out. tc.ResetSSHFailures() if _, err := tc.CollectTemperature(context.Background(), "node1.local", "node1"); err != nil { t.Fatalf("CollectTemperature after reset: %v", err) } if runner.runs != 6 { t.Fatalf("reset did not retry ssh, total %d commands, want 6", runner.runs) } } // TestIssue1638TemperatureBackoffDecaysAfterQuietPeriod pins that a host whose // backoff expired long ago starts over at the floor rather than resuming the // compounding it had reached. func TestIssue1638TemperatureBackoffDecaysAfterQuietPeriod(t *testing.T) { clock := issue1638UseFakeClock(t) tc, runner := issue1638TemperatureCollector(t) if _, err := tc.CollectTemperature(context.Background(), "node1.local", "node1"); err != nil { t.Fatalf("CollectTemperature: %v", err) } if runner.runs != 2 { t.Fatalf("first cycle ran %d ssh commands, want 2", runner.runs) } // Sit out the window and one further window on top of it, then fail again. clock.advance(2*temperatureSSHFailureInitialBackoff + time.Second) if _, err := tc.CollectTemperature(context.Background(), "node1.local", "node1"); err != nil { t.Fatalf("CollectTemperature after quiet period: %v", err) } if runner.runs != 4 { t.Fatalf("quiet-period retry did not run, total %d commands, want 4", runner.runs) } // The decayed backoff is back at the floor, so one initial window is enough // to retry rather than the doubled window a compounding-only policy keeps. clock.advance(temperatureSSHFailureInitialBackoff + time.Second) if _, err := tc.CollectTemperature(context.Background(), "node1.local", "node1"); err != nil { t.Fatalf("CollectTemperature after decayed window: %v", err) } if runner.runs != 6 { t.Fatalf("backoff did not decay to the floor, total %d commands, want 6", runner.runs) } } // TestIssue1638TemperatureDoesNotEscalateWhenNoSSHRan covers the case where the // knownhosts manager suppresses the call inside its own backoff. Nothing was // executed, so the temperature layer must not record a failure of its own — // otherwise two independent backoffs compound against a single event. func TestIssue1638TemperatureDoesNotEscalateWhenNoSSHRan(t *testing.T) { issue1638UseFakeClock(t) tc, runner := issue1638TemperatureCollector(t) scans := 0 manager, err := NewKnownHostsManager( filepath.Join(t.TempDir(), "known_hosts"), WithKeyscanFunc(func(ctx context.Context, host string, port int, timeout time.Duration) ([]byte, error) { scans++ return nil, errors.New("connection refused") }), ) if err != nil { t.Fatalf("NewKnownHostsManager: %v", err) } tc.hostKeys = manager // Put the knownhosts manager into its backoff window first. if err := manager.Ensure(context.Background(), "node1.local"); err == nil { t.Fatal("expected keyscan failure") } if scans != 1 { t.Fatalf("keyscan ran %d times, want 1", scans) } if _, err := tc.CollectTemperature(context.Background(), "node1.local", "node1"); err != nil { t.Fatalf("CollectTemperature: %v", err) } if runner.runs != 0 { t.Fatalf("ssh was executed %d times despite a suppressed host key scan, want 0", runner.runs) } if scans != 1 { t.Fatalf("ssh-keyscan re-ran inside its backoff, scans = %d, want 1", scans) } tc.sshFailureMu.Lock() recorded := len(tc.sshFailures) tc.sshFailureMu.Unlock() if recorded != 0 { t.Fatalf("temperature layer recorded %d failures for an attempt that never ran, want 0", recorded) } } // TestIssue1638TemperatureDeadlineDoesNotCompound pins the lenient treatment of // our own collection deadline: it is not evidence that the host is broken, so // the window holds at the floor and the wasted second probe is skipped. func TestIssue1638TemperatureDeadlineDoesNotCompound(t *testing.T) { clock := issue1638UseFakeClock(t) tc, runner := issue1638TemperatureCollector(t) runner.err = errors.New("signal: killed") expired := func() (context.Context, context.CancelFunc) { return context.WithDeadline(context.Background(), time.Now().Add(-time.Second)) } for cycle := 0; cycle < 3; cycle++ { ctx, cancel := expired() if _, err := tc.CollectTemperature(ctx, "node1.local", "node1"); err != nil { cancel() t.Fatalf("cycle %d: CollectTemperature: %v", cycle, err) } cancel() clock.advance(temperatureSSHFailureInitialBackoff + time.Second) } // One probe per cycle (the RPi fallback is pointless once the deadline has // passed) and no compounding, so all three cycles ran. if runner.runs != 3 { t.Fatalf("deadline handling ran %d ssh commands, want 3 (one per cycle, no fallback, no compounding)", runner.runs) } } // TestIssue1638MonitorResetSSHFailureBackoffDelegates pins the settings-save // escape hatch: Monitor.ResetSSHFailureBackoff must clear the temperature // collector's per-host SSH backoff (the settings API calls it on every live // tenant monitor after a save), and must be safe on a nil monitor or a monitor // without a temperature collector. func TestIssue1638MonitorResetSSHFailureBackoffDelegates(t *testing.T) { issue1638UseFakeClock(t) tc, runner := issue1638TemperatureCollector(t) // Open a backoff window with a failing first cycle (sensors + fallback). if _, err := tc.CollectTemperature(context.Background(), "node1.local", "node1"); err != nil { t.Fatalf("CollectTemperature: %v", err) } if runner.runs != 2 { t.Fatalf("first cycle ran %d ssh commands, want 2", runner.runs) } if _, err := tc.CollectTemperature(context.Background(), "node1.local", "node1"); err != nil { t.Fatalf("CollectTemperature inside window: %v", err) } if runner.runs != 2 { t.Fatalf("backoff window still ran ssh, total %d commands, want 2", runner.runs) } m := &Monitor{tempCollector: tc} m.ResetSSHFailureBackoff() if _, err := tc.CollectTemperature(context.Background(), "node1.local", "node1"); err != nil { t.Fatalf("CollectTemperature after reset: %v", err) } if runner.runs != 4 { t.Fatalf("reset did not retry ssh, total %d commands, want 4", runner.runs) } // Nil receiver and nil collector must both be no-ops, not panics: the // settings API calls this on whatever monitor the tenant lookup returns. var nilMonitor *Monitor nilMonitor.ResetSSHFailureBackoff() (&Monitor{}).ResetSSHFailureBackoff() }