From c70431caaf768f9cb86b92f48ff3376e5f5fd3c0 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Thu, 16 Jul 2026 09:26:53 +0100 Subject: [PATCH] Honor configured availability poll interval in the scheduler An availability target's configured poll interval only seeded the adaptive scheduler: BuildPlan derived every instance's cadence from the global adaptive bounds, and a failing probe raised the staleness score and error penalty, collapsing the probe interval toward the global 5-second minimum. With interval 120s and failure threshold 4 the alert was promised after ~8 minutes of downtime but fired within the first minute because the four consecutive failures accumulated at the collapsed cadence (#1582). Availability checks promise pollInterval x failureThreshold as the detection window, so the cadence is a user contract, not a scheduling hint. Add a FixedIntervalPollProvider extension that pins an instance to its configured interval, implement it for availability targets, and bypass adaptive selection wherever the next run is computed (plan building, rescheduling, and the non-adaptive fallback paths). --- internal/monitoring/availability_poller.go | 8 +++++ internal/monitoring/monitor.go | 18 ++++++++-- .../monitoring/monitor_polling_schedule.go | 9 ++++- internal/monitoring/poll_providers.go | 7 ++++ internal/monitoring/scheduler.go | 28 ++++++++++----- internal/monitoring/scheduler_test.go | 36 +++++++++++++++++++ 6 files changed, 93 insertions(+), 13 deletions(-) diff --git a/internal/monitoring/availability_poller.go b/internal/monitoring/availability_poller.go index 6189c2d68..8dfc931cf 100644 --- a/internal/monitoring/availability_poller.go +++ b/internal/monitoring/availability_poller.go @@ -76,6 +76,14 @@ func (availabilityPollProvider) BaseInterval(m *Monitor) time.Duration { return clampInterval(minInterval, 10*time.Second, time.Hour) } +func (availabilityPollProvider) FixedInstanceInterval(m *Monitor, instanceName string) time.Duration { + target, ok := m.availabilityTargetByID(instanceName) + if !ok || !target.Enabled { + return 0 + } + return clampInterval(time.Duration(target.EffectivePollIntervalSecs())*time.Second, 10*time.Second, time.Hour) +} + func (availabilityPollProvider) BuildPollTask(m *Monitor, instanceName string) (PollTask, error) { target, ok := m.availabilityTargetByID(instanceName) if !ok || !target.Enabled { diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go index 2dbe348be..c6814c5fb 100644 --- a/internal/monitoring/monitor.go +++ b/internal/monitoring/monitor.go @@ -1740,6 +1740,13 @@ func (m *Monitor) effectivePVEPollingInterval() time.Duration { return effectivePVEPollingIntervalForConfig(m.config) } +func (m *Monitor) fixedIntervalForInstance(instanceType InstanceType, instanceName string) time.Duration { + if provider, ok := m.getPollProvider(instanceType).(FixedIntervalPollProvider); ok && provider != nil { + return provider.FixedInstanceInterval(m, instanceName) + } + return 0 +} + func (m *Monitor) baseIntervalForInstanceType(instanceType InstanceType) time.Duration { if provider := m.getPollProvider(instanceType); provider != nil { if interval := provider.BaseInterval(m); interval > 0 { @@ -2174,11 +2181,15 @@ func (m *Monitor) rescheduleTask(task ScheduledTask) { return } + fixedInterval := m.fixedIntervalForInstance(task.InstanceType, task.InstanceName) + if m.scheduler == nil { - baseInterval := m.baseIntervalForInstanceType(task.InstanceType) - nextInterval := task.Interval + nextInterval := fixedInterval if nextInterval <= 0 { - nextInterval = baseInterval + nextInterval = task.Interval + } + if nextInterval <= 0 { + nextInterval = m.baseIntervalForInstanceType(task.InstanceType) } if nextInterval <= 0 { nextInterval = DefaultSchedulerConfig().BaseInterval @@ -2195,6 +2206,7 @@ func (m *Monitor) rescheduleTask(task ScheduledTask) { Type: task.InstanceType, LastInterval: task.Interval, LastScheduled: task.NextRun, + FixedInterval: fixedInterval, } if m.stalenessTracker != nil { if snap, ok := m.stalenessTracker.snapshot(task.InstanceType, task.InstanceName); ok { diff --git a/internal/monitoring/monitor_polling_schedule.go b/internal/monitoring/monitor_polling_schedule.go index dbc8dc3bf..6d35820a7 100644 --- a/internal/monitoring/monitor_polling_schedule.go +++ b/internal/monitoring/monitor_polling_schedule.go @@ -35,11 +35,15 @@ func (m *Monitor) describeInstancesForScheduler() []InstanceDescriptor { sort.Strings(names) providerType := provider.Type() + fixedProvider, _ := provider.(FixedIntervalPollProvider) for _, name := range names { desc := InstanceDescriptor{ Name: name, Type: providerType, } + if fixedProvider != nil { + desc.FixedInterval = fixedProvider.FixedInstanceInterval(m, name) + } if m.scheduler != nil { if last, ok := m.scheduler.LastScheduled(providerType, name); ok { desc.LastScheduled = last.NextRun @@ -74,7 +78,10 @@ func (m *Monitor) buildScheduledTasks(now time.Time) []ScheduledTask { if m.scheduler == nil { tasks := make([]ScheduledTask, 0, len(descriptors)) for _, desc := range descriptors { - interval := m.baseIntervalForInstanceType(desc.Type) + interval := desc.FixedInterval + if interval <= 0 { + interval = m.baseIntervalForInstanceType(desc.Type) + } if interval <= 0 { interval = DefaultSchedulerConfig().BaseInterval } diff --git a/internal/monitoring/poll_providers.go b/internal/monitoring/poll_providers.go index 94ce1600f..97eb74e9d 100644 --- a/internal/monitoring/poll_providers.go +++ b/internal/monitoring/poll_providers.go @@ -48,6 +48,13 @@ type ConnectionHealthKeyPollProvider interface { ConnectionHealthKey(m *Monitor, instanceName string) string } +// FixedIntervalPollProvider is an optional PollProvider extension for providers +// whose instances carry a user-configured poll cadence that must be honored +// exactly, bypassing adaptive interval selection. +type FixedIntervalPollProvider interface { + FixedInstanceInterval(m *Monitor, instanceName string) time.Duration +} + // SupplementalRecordsPollProvider is an optional PollProvider extension for // providers that can emit source-native unified ingest records. type SupplementalRecordsPollProvider interface { diff --git a/internal/monitoring/scheduler.go b/internal/monitoring/scheduler.go index d21833f26..e8ecd2fd9 100644 --- a/internal/monitoring/scheduler.go +++ b/internal/monitoring/scheduler.go @@ -64,6 +64,11 @@ type InstanceDescriptor struct { LastFailure time.Time LastScheduled time.Time LastInterval time.Duration + // FixedInterval pins the instance to a user-configured cadence. When set, + // the adaptive selector is bypassed entirely: availability checks promise + // pollInterval x failureThreshold as the detection window, so the + // scheduler must not probe faster on failure or slower when idle. + FixedInterval time.Duration ErrorCount int Metadata TaskMetadata } @@ -181,15 +186,20 @@ func (s *AdaptiveScheduler) BuildPlan(now time.Time, inventory []InstanceDescrip InstanceType: inst.Type, } - nextInterval := s.interval.SelectInterval(req) - if nextInterval <= 0 { - nextInterval = s.cfg.BaseInterval - } - if nextInterval < s.cfg.MinInterval { - nextInterval = s.cfg.MinInterval - } - if nextInterval > s.cfg.MaxInterval { - nextInterval = s.cfg.MaxInterval + var nextInterval time.Duration + if inst.FixedInterval > 0 { + nextInterval = inst.FixedInterval + } else { + nextInterval = s.interval.SelectInterval(req) + if nextInterval <= 0 { + nextInterval = s.cfg.BaseInterval + } + if nextInterval < s.cfg.MinInterval { + nextInterval = s.cfg.MinInterval + } + if nextInterval > s.cfg.MaxInterval { + nextInterval = s.cfg.MaxInterval + } } nextRun := now diff --git a/internal/monitoring/scheduler_test.go b/internal/monitoring/scheduler_test.go index a019298bf..8aee3417b 100644 --- a/internal/monitoring/scheduler_test.go +++ b/internal/monitoring/scheduler_test.go @@ -1862,6 +1862,42 @@ func TestBuildPlan_SingleInstance(t *testing.T) { } } +// TestBuildPlan_FixedInterval pins the user-configured cadence contract: +// a descriptor with FixedInterval bypasses the adaptive selector and the +// scheduler's min/max clamps entirely (#1582). +func TestBuildPlan_FixedInterval(t *testing.T) { + t.Parallel() + + cfg := SchedulerConfig{ + BaseInterval: 10 * time.Second, + MinInterval: 5 * time.Second, + MaxInterval: 60 * time.Second, + } + + // Selector would pick 5s (failing endpoint pushed to the floor); staleness + // is high. The fixed interval must win anyway, even above MaxInterval. + staleness := mockStalenessSource{scores: map[string]float64{"availability:check1": 1.0}} + scheduler := NewAdaptiveScheduler(cfg, staleness, mockIntervalSelector{interval: 5 * time.Second}, nil) + + now := time.Now() + inventory := []InstanceDescriptor{ + { + Name: "check1", + Type: InstanceTypeAvailability, + FixedInterval: 120 * time.Second, + }, + } + + tasks := scheduler.BuildPlan(now, inventory, 0) + + if len(tasks) != 1 { + t.Fatalf("expected 1 task, got %d", len(tasks)) + } + if tasks[0].Interval != 120*time.Second { + t.Errorf("expected fixed interval 120s to bypass the selector, got %v", tasks[0].Interval) + } +} + // TestBuildPlan_MultipleInstances tests scheduling multiple instances and ordering func TestBuildPlan_MultipleInstances(t *testing.T) { t.Parallel()