Honor configured availability poll interval in the scheduler

An availability target's configured poll interval only seeded the
adaptive scheduler: BuildPlan derived every instance's cadence from the
global adaptive bounds, and a failing probe raised the staleness score
and error penalty, collapsing the probe interval toward the global
5-second minimum. With interval 120s and failure threshold 4 the alert
was promised after ~8 minutes of downtime but fired within the first
minute because the four consecutive failures accumulated at the
collapsed cadence (#1582).

Availability checks promise pollInterval x failureThreshold as the
detection window, so the cadence is a user contract, not a scheduling
hint. Add a FixedIntervalPollProvider extension that pins an instance
to its configured interval, implement it for availability targets, and
bypass adaptive selection wherever the next run is computed (plan
building, rescheduling, and the non-adaptive fallback paths).
This commit is contained in:
rcourtman
2026-07-16 09:26:53 +01:00
parent 4421945e9b
commit c70431caaf
6 changed files with 93 additions and 13 deletions
@@ -76,6 +76,14 @@ func (availabilityPollProvider) BaseInterval(m *Monitor) time.Duration {
return clampInterval(minInterval, 10*time.Second, time.Hour)
}
func (availabilityPollProvider) FixedInstanceInterval(m *Monitor, instanceName string) time.Duration {
target, ok := m.availabilityTargetByID(instanceName)
if !ok || !target.Enabled {
return 0
}
return clampInterval(time.Duration(target.EffectivePollIntervalSecs())*time.Second, 10*time.Second, time.Hour)
}
func (availabilityPollProvider) BuildPollTask(m *Monitor, instanceName string) (PollTask, error) {
target, ok := m.availabilityTargetByID(instanceName)
if !ok || !target.Enabled {
+15 -3
View File
@@ -1740,6 +1740,13 @@ func (m *Monitor) effectivePVEPollingInterval() time.Duration {
return effectivePVEPollingIntervalForConfig(m.config)
}
func (m *Monitor) fixedIntervalForInstance(instanceType InstanceType, instanceName string) time.Duration {
if provider, ok := m.getPollProvider(instanceType).(FixedIntervalPollProvider); ok && provider != nil {
return provider.FixedInstanceInterval(m, instanceName)
}
return 0
}
func (m *Monitor) baseIntervalForInstanceType(instanceType InstanceType) time.Duration {
if provider := m.getPollProvider(instanceType); provider != nil {
if interval := provider.BaseInterval(m); interval > 0 {
@@ -2174,11 +2181,15 @@ func (m *Monitor) rescheduleTask(task ScheduledTask) {
return
}
fixedInterval := m.fixedIntervalForInstance(task.InstanceType, task.InstanceName)
if m.scheduler == nil {
baseInterval := m.baseIntervalForInstanceType(task.InstanceType)
nextInterval := task.Interval
nextInterval := fixedInterval
if nextInterval <= 0 {
nextInterval = baseInterval
nextInterval = task.Interval
}
if nextInterval <= 0 {
nextInterval = m.baseIntervalForInstanceType(task.InstanceType)
}
if nextInterval <= 0 {
nextInterval = DefaultSchedulerConfig().BaseInterval
@@ -2195,6 +2206,7 @@ func (m *Monitor) rescheduleTask(task ScheduledTask) {
Type: task.InstanceType,
LastInterval: task.Interval,
LastScheduled: task.NextRun,
FixedInterval: fixedInterval,
}
if m.stalenessTracker != nil {
if snap, ok := m.stalenessTracker.snapshot(task.InstanceType, task.InstanceName); ok {
@@ -35,11 +35,15 @@ func (m *Monitor) describeInstancesForScheduler() []InstanceDescriptor {
sort.Strings(names)
providerType := provider.Type()
fixedProvider, _ := provider.(FixedIntervalPollProvider)
for _, name := range names {
desc := InstanceDescriptor{
Name: name,
Type: providerType,
}
if fixedProvider != nil {
desc.FixedInterval = fixedProvider.FixedInstanceInterval(m, name)
}
if m.scheduler != nil {
if last, ok := m.scheduler.LastScheduled(providerType, name); ok {
desc.LastScheduled = last.NextRun
@@ -74,7 +78,10 @@ func (m *Monitor) buildScheduledTasks(now time.Time) []ScheduledTask {
if m.scheduler == nil {
tasks := make([]ScheduledTask, 0, len(descriptors))
for _, desc := range descriptors {
interval := m.baseIntervalForInstanceType(desc.Type)
interval := desc.FixedInterval
if interval <= 0 {
interval = m.baseIntervalForInstanceType(desc.Type)
}
if interval <= 0 {
interval = DefaultSchedulerConfig().BaseInterval
}
+7
View File
@@ -48,6 +48,13 @@ type ConnectionHealthKeyPollProvider interface {
ConnectionHealthKey(m *Monitor, instanceName string) string
}
// FixedIntervalPollProvider is an optional PollProvider extension for providers
// whose instances carry a user-configured poll cadence that must be honored
// exactly, bypassing adaptive interval selection.
type FixedIntervalPollProvider interface {
FixedInstanceInterval(m *Monitor, instanceName string) time.Duration
}
// SupplementalRecordsPollProvider is an optional PollProvider extension for
// providers that can emit source-native unified ingest records.
type SupplementalRecordsPollProvider interface {
+19 -9
View File
@@ -64,6 +64,11 @@ type InstanceDescriptor struct {
LastFailure time.Time
LastScheduled time.Time
LastInterval time.Duration
// FixedInterval pins the instance to a user-configured cadence. When set,
// the adaptive selector is bypassed entirely: availability checks promise
// pollInterval x failureThreshold as the detection window, so the
// scheduler must not probe faster on failure or slower when idle.
FixedInterval time.Duration
ErrorCount int
Metadata TaskMetadata
}
@@ -181,15 +186,20 @@ func (s *AdaptiveScheduler) BuildPlan(now time.Time, inventory []InstanceDescrip
InstanceType: inst.Type,
}
nextInterval := s.interval.SelectInterval(req)
if nextInterval <= 0 {
nextInterval = s.cfg.BaseInterval
}
if nextInterval < s.cfg.MinInterval {
nextInterval = s.cfg.MinInterval
}
if nextInterval > s.cfg.MaxInterval {
nextInterval = s.cfg.MaxInterval
var nextInterval time.Duration
if inst.FixedInterval > 0 {
nextInterval = inst.FixedInterval
} else {
nextInterval = s.interval.SelectInterval(req)
if nextInterval <= 0 {
nextInterval = s.cfg.BaseInterval
}
if nextInterval < s.cfg.MinInterval {
nextInterval = s.cfg.MinInterval
}
if nextInterval > s.cfg.MaxInterval {
nextInterval = s.cfg.MaxInterval
}
}
nextRun := now
+36
View File
@@ -1862,6 +1862,42 @@ func TestBuildPlan_SingleInstance(t *testing.T) {
}
}
// TestBuildPlan_FixedInterval pins the user-configured cadence contract:
// a descriptor with FixedInterval bypasses the adaptive selector and the
// scheduler's min/max clamps entirely (#1582).
func TestBuildPlan_FixedInterval(t *testing.T) {
t.Parallel()
cfg := SchedulerConfig{
BaseInterval: 10 * time.Second,
MinInterval: 5 * time.Second,
MaxInterval: 60 * time.Second,
}
// Selector would pick 5s (failing endpoint pushed to the floor); staleness
// is high. The fixed interval must win anyway, even above MaxInterval.
staleness := mockStalenessSource{scores: map[string]float64{"availability:check1": 1.0}}
scheduler := NewAdaptiveScheduler(cfg, staleness, mockIntervalSelector{interval: 5 * time.Second}, nil)
now := time.Now()
inventory := []InstanceDescriptor{
{
Name: "check1",
Type: InstanceTypeAvailability,
FixedInterval: 120 * time.Second,
},
}
tasks := scheduler.BuildPlan(now, inventory, 0)
if len(tasks) != 1 {
t.Fatalf("expected 1 task, got %d", len(tasks))
}
if tasks[0].Interval != 120*time.Second {
t.Errorf("expected fixed interval 120s to bypass the selector, got %v", tasks[0].Interval)
}
}
// TestBuildPlan_MultipleInstances tests scheduling multiple instances and ordering
func TestBuildPlan_MultipleInstances(t *testing.T) {
t.Parallel()