mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 02:25:56 +00:00
Honor configured availability poll interval in the scheduler
An availability target's configured poll interval only seeded the adaptive scheduler: BuildPlan derived every instance's cadence from the global adaptive bounds, and a failing probe raised the staleness score and error penalty, collapsing the probe interval toward the global 5-second minimum. With interval 120s and failure threshold 4 the alert was promised after ~8 minutes of downtime but fired within the first minute because the four consecutive failures accumulated at the collapsed cadence (#1582). Availability checks promise pollInterval x failureThreshold as the detection window, so the cadence is a user contract, not a scheduling hint. Add a FixedIntervalPollProvider extension that pins an instance to its configured interval, implement it for availability targets, and bypass adaptive selection wherever the next run is computed (plan building, rescheduling, and the non-adaptive fallback paths).
This commit is contained in:
@@ -76,6 +76,14 @@ func (availabilityPollProvider) BaseInterval(m *Monitor) time.Duration {
|
||||
return clampInterval(minInterval, 10*time.Second, time.Hour)
|
||||
}
|
||||
|
||||
func (availabilityPollProvider) FixedInstanceInterval(m *Monitor, instanceName string) time.Duration {
|
||||
target, ok := m.availabilityTargetByID(instanceName)
|
||||
if !ok || !target.Enabled {
|
||||
return 0
|
||||
}
|
||||
return clampInterval(time.Duration(target.EffectivePollIntervalSecs())*time.Second, 10*time.Second, time.Hour)
|
||||
}
|
||||
|
||||
func (availabilityPollProvider) BuildPollTask(m *Monitor, instanceName string) (PollTask, error) {
|
||||
target, ok := m.availabilityTargetByID(instanceName)
|
||||
if !ok || !target.Enabled {
|
||||
|
||||
@@ -1740,6 +1740,13 @@ func (m *Monitor) effectivePVEPollingInterval() time.Duration {
|
||||
return effectivePVEPollingIntervalForConfig(m.config)
|
||||
}
|
||||
|
||||
func (m *Monitor) fixedIntervalForInstance(instanceType InstanceType, instanceName string) time.Duration {
|
||||
if provider, ok := m.getPollProvider(instanceType).(FixedIntervalPollProvider); ok && provider != nil {
|
||||
return provider.FixedInstanceInterval(m, instanceName)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (m *Monitor) baseIntervalForInstanceType(instanceType InstanceType) time.Duration {
|
||||
if provider := m.getPollProvider(instanceType); provider != nil {
|
||||
if interval := provider.BaseInterval(m); interval > 0 {
|
||||
@@ -2174,11 +2181,15 @@ func (m *Monitor) rescheduleTask(task ScheduledTask) {
|
||||
return
|
||||
}
|
||||
|
||||
fixedInterval := m.fixedIntervalForInstance(task.InstanceType, task.InstanceName)
|
||||
|
||||
if m.scheduler == nil {
|
||||
baseInterval := m.baseIntervalForInstanceType(task.InstanceType)
|
||||
nextInterval := task.Interval
|
||||
nextInterval := fixedInterval
|
||||
if nextInterval <= 0 {
|
||||
nextInterval = baseInterval
|
||||
nextInterval = task.Interval
|
||||
}
|
||||
if nextInterval <= 0 {
|
||||
nextInterval = m.baseIntervalForInstanceType(task.InstanceType)
|
||||
}
|
||||
if nextInterval <= 0 {
|
||||
nextInterval = DefaultSchedulerConfig().BaseInterval
|
||||
@@ -2195,6 +2206,7 @@ func (m *Monitor) rescheduleTask(task ScheduledTask) {
|
||||
Type: task.InstanceType,
|
||||
LastInterval: task.Interval,
|
||||
LastScheduled: task.NextRun,
|
||||
FixedInterval: fixedInterval,
|
||||
}
|
||||
if m.stalenessTracker != nil {
|
||||
if snap, ok := m.stalenessTracker.snapshot(task.InstanceType, task.InstanceName); ok {
|
||||
|
||||
@@ -35,11 +35,15 @@ func (m *Monitor) describeInstancesForScheduler() []InstanceDescriptor {
|
||||
sort.Strings(names)
|
||||
|
||||
providerType := provider.Type()
|
||||
fixedProvider, _ := provider.(FixedIntervalPollProvider)
|
||||
for _, name := range names {
|
||||
desc := InstanceDescriptor{
|
||||
Name: name,
|
||||
Type: providerType,
|
||||
}
|
||||
if fixedProvider != nil {
|
||||
desc.FixedInterval = fixedProvider.FixedInstanceInterval(m, name)
|
||||
}
|
||||
if m.scheduler != nil {
|
||||
if last, ok := m.scheduler.LastScheduled(providerType, name); ok {
|
||||
desc.LastScheduled = last.NextRun
|
||||
@@ -74,7 +78,10 @@ func (m *Monitor) buildScheduledTasks(now time.Time) []ScheduledTask {
|
||||
if m.scheduler == nil {
|
||||
tasks := make([]ScheduledTask, 0, len(descriptors))
|
||||
for _, desc := range descriptors {
|
||||
interval := m.baseIntervalForInstanceType(desc.Type)
|
||||
interval := desc.FixedInterval
|
||||
if interval <= 0 {
|
||||
interval = m.baseIntervalForInstanceType(desc.Type)
|
||||
}
|
||||
if interval <= 0 {
|
||||
interval = DefaultSchedulerConfig().BaseInterval
|
||||
}
|
||||
|
||||
@@ -48,6 +48,13 @@ type ConnectionHealthKeyPollProvider interface {
|
||||
ConnectionHealthKey(m *Monitor, instanceName string) string
|
||||
}
|
||||
|
||||
// FixedIntervalPollProvider is an optional PollProvider extension for providers
|
||||
// whose instances carry a user-configured poll cadence that must be honored
|
||||
// exactly, bypassing adaptive interval selection.
|
||||
type FixedIntervalPollProvider interface {
|
||||
FixedInstanceInterval(m *Monitor, instanceName string) time.Duration
|
||||
}
|
||||
|
||||
// SupplementalRecordsPollProvider is an optional PollProvider extension for
|
||||
// providers that can emit source-native unified ingest records.
|
||||
type SupplementalRecordsPollProvider interface {
|
||||
|
||||
@@ -64,6 +64,11 @@ type InstanceDescriptor struct {
|
||||
LastFailure time.Time
|
||||
LastScheduled time.Time
|
||||
LastInterval time.Duration
|
||||
// FixedInterval pins the instance to a user-configured cadence. When set,
|
||||
// the adaptive selector is bypassed entirely: availability checks promise
|
||||
// pollInterval x failureThreshold as the detection window, so the
|
||||
// scheduler must not probe faster on failure or slower when idle.
|
||||
FixedInterval time.Duration
|
||||
ErrorCount int
|
||||
Metadata TaskMetadata
|
||||
}
|
||||
@@ -181,15 +186,20 @@ func (s *AdaptiveScheduler) BuildPlan(now time.Time, inventory []InstanceDescrip
|
||||
InstanceType: inst.Type,
|
||||
}
|
||||
|
||||
nextInterval := s.interval.SelectInterval(req)
|
||||
if nextInterval <= 0 {
|
||||
nextInterval = s.cfg.BaseInterval
|
||||
}
|
||||
if nextInterval < s.cfg.MinInterval {
|
||||
nextInterval = s.cfg.MinInterval
|
||||
}
|
||||
if nextInterval > s.cfg.MaxInterval {
|
||||
nextInterval = s.cfg.MaxInterval
|
||||
var nextInterval time.Duration
|
||||
if inst.FixedInterval > 0 {
|
||||
nextInterval = inst.FixedInterval
|
||||
} else {
|
||||
nextInterval = s.interval.SelectInterval(req)
|
||||
if nextInterval <= 0 {
|
||||
nextInterval = s.cfg.BaseInterval
|
||||
}
|
||||
if nextInterval < s.cfg.MinInterval {
|
||||
nextInterval = s.cfg.MinInterval
|
||||
}
|
||||
if nextInterval > s.cfg.MaxInterval {
|
||||
nextInterval = s.cfg.MaxInterval
|
||||
}
|
||||
}
|
||||
|
||||
nextRun := now
|
||||
|
||||
@@ -1862,6 +1862,42 @@ func TestBuildPlan_SingleInstance(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildPlan_FixedInterval pins the user-configured cadence contract:
|
||||
// a descriptor with FixedInterval bypasses the adaptive selector and the
|
||||
// scheduler's min/max clamps entirely (#1582).
|
||||
func TestBuildPlan_FixedInterval(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg := SchedulerConfig{
|
||||
BaseInterval: 10 * time.Second,
|
||||
MinInterval: 5 * time.Second,
|
||||
MaxInterval: 60 * time.Second,
|
||||
}
|
||||
|
||||
// Selector would pick 5s (failing endpoint pushed to the floor); staleness
|
||||
// is high. The fixed interval must win anyway, even above MaxInterval.
|
||||
staleness := mockStalenessSource{scores: map[string]float64{"availability:check1": 1.0}}
|
||||
scheduler := NewAdaptiveScheduler(cfg, staleness, mockIntervalSelector{interval: 5 * time.Second}, nil)
|
||||
|
||||
now := time.Now()
|
||||
inventory := []InstanceDescriptor{
|
||||
{
|
||||
Name: "check1",
|
||||
Type: InstanceTypeAvailability,
|
||||
FixedInterval: 120 * time.Second,
|
||||
},
|
||||
}
|
||||
|
||||
tasks := scheduler.BuildPlan(now, inventory, 0)
|
||||
|
||||
if len(tasks) != 1 {
|
||||
t.Fatalf("expected 1 task, got %d", len(tasks))
|
||||
}
|
||||
if tasks[0].Interval != 120*time.Second {
|
||||
t.Errorf("expected fixed interval 120s to bypass the selector, got %v", tasks[0].Interval)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildPlan_MultipleInstances tests scheduling multiple instances and ordering
|
||||
func TestBuildPlan_MultipleInstances(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
Reference in New Issue
Block a user