Files
pulse/internal/monitoring/monitor_runtime_settings.go
T
rcourtman 912d7ceb51 Judge connection freshness by the monitor's live poll cadence
Non-default organisations run their monitors against a detached copy of
the server config (#1619). A system-settings save reaches those monitors
only as runtime PBS and PMG polling overrides, so the copy that the
connections list reads through the org's config handlers kept the old
interval until that monitor was rebuilt. The active-to-stale cutoff is
3 x the polling interval, so raising PBS from 1m to 5m left a 3m cutoff
and Settings > Infrastructure badges read "stale" for part of every
cycle, while lowering it delayed the badge by the same factor.

Add Monitor.BasePollInterval, an override-aware reader for the clamped
PVE, PBS and PMG base cadence that never takes m.mu, and have the
connections input builder use it whenever a monitor is present. The
fixed-cadence branch of resourcePollIntervals reads the same accessor, so
resource freshness and connection freshness share one derivation.

The system settings GET is not affected: both production routes serve
SystemSettingsHandler, which reads the base config and persisted
settings; the configapi handler that reads the tenant copy is only a
fallback when that handler is absent.
2026-10-10 07:37:03 +01:00

205 lines
6.8 KiB
Go

package monitoring
import "time"
// Runtime-tunable polling cadence settings.
//
// Each tenant monitor polls against a detached DeepCopy of the base config,
// so mutating the base config after a system-settings save never reaches a
// running monitor (#1619). These setters push the saved values into live
// monitors directly, shadowing the config fields. The overrides live behind
// runtimePollingMu because polling goroutines read them every cycle while
// the settings API writes them.
// runtimePollingOverrides holds the polling-cadence values the settings API
// has pushed into this live monitor, shadowing the corresponding fields of
// the monitor's detached config copy. nil means "use the config value".
type runtimePollingOverrides struct {
backupEnabled *bool
backupInterval *time.Duration
pbsInterval *time.Duration
pmgInterval *time.Duration
}
// backupPollingEnabledSetting returns the effective EnableBackupPolling value.
func (m *Monitor) backupPollingEnabledSetting() bool {
if m == nil || m.config == nil {
return false
}
m.runtimePollingMu.RLock()
override := m.runtimePollingOverride.backupEnabled
m.runtimePollingMu.RUnlock()
if override != nil {
return *override
}
return m.config.EnableBackupPolling
}
// backupPollingIntervalSetting returns the effective BackupPollingInterval.
// Zero means cycle-based scheduling (BackupPollingCycles).
func (m *Monitor) backupPollingIntervalSetting() time.Duration {
if m == nil || m.config == nil {
return 0
}
m.runtimePollingMu.RLock()
override := m.runtimePollingOverride.backupInterval
m.runtimePollingMu.RUnlock()
if override != nil {
return *override
}
return m.config.BackupPollingInterval
}
// pbsPollingIntervalSetting returns the effective PBSPollingInterval before
// clamping.
func (m *Monitor) pbsPollingIntervalSetting() time.Duration {
if m == nil || m.config == nil {
return 0
}
m.runtimePollingMu.RLock()
override := m.runtimePollingOverride.pbsInterval
m.runtimePollingMu.RUnlock()
if override != nil {
return *override
}
return m.config.PBSPollingInterval
}
// pmgPollingIntervalSetting returns the effective PMGPollingInterval before
// clamping.
func (m *Monitor) pmgPollingIntervalSetting() time.Duration {
if m == nil || m.config == nil {
return 0
}
m.runtimePollingMu.RLock()
override := m.runtimePollingOverride.pmgInterval
m.runtimePollingMu.RUnlock()
if override != nil {
return *override
}
return m.config.PMGPollingInterval
}
// BasePollInterval returns the clamped base cadence this monitor polls PVE,
// PBS or PMG at, honouring the runtime polling overrides. For the built-in
// providers these are the values the fixed-cadence scheduler reads
// (baseIntervalForInstanceType); a replacement provider or a task already
// queued at its previous interval can differ until the next planning pass.
// This accessor skips the poll-provider lookup, so it never takes m.mu.
// Non-default tenant monitors poll against a detached config copy (#1619), so
// a saved PBS or PMG interval reaches them only as an override. Code outside
// the monitor that judges poll freshness must read the cadence here, not from
// GetConfig. An adaptive scheduler plans its own intervals independently of
// this base; see PlannedPollInterval. Zero means the cadence is unknown:
// another instance type, or a monitor without config.
func (m *Monitor) BasePollInterval(instanceType InstanceType) time.Duration {
if m == nil || m.config == nil {
return 0
}
switch instanceType {
case InstanceTypePVE:
return m.effectivePVEPollingInterval()
case InstanceTypePBS:
return clampInterval(m.pbsPollingIntervalSetting(), 10*time.Second, time.Hour)
case InstanceTypePMG:
return clampInterval(m.pmgPollingIntervalSetting(), 10*time.Second, time.Hour)
default:
return 0
}
}
// SetBackupPollingEnabled toggles backup polling on the live monitor.
// Re-enabling clears the per-instance last-poll timestamps so the next
// polling cycle runs an immediate catch-up poll, matching the behavior of
// a full monitor reload.
func (m *Monitor) SetBackupPollingEnabled(enabled bool) {
if m == nil {
return
}
m.runtimePollingMu.Lock()
wasEnabled := m.config != nil && m.config.EnableBackupPolling
if m.runtimePollingOverride.backupEnabled != nil {
wasEnabled = *m.runtimePollingOverride.backupEnabled
}
m.runtimePollingOverride.backupEnabled = &enabled
m.runtimePollingMu.Unlock()
if enabled && !wasEnabled {
m.resetBackupPollTimestamps()
}
}
// SetBackupPollingInterval updates the backup polling cadence on the live
// monitor. Lowering the interval (or switching from cycle-based scheduling
// to an interval) clears the per-instance last-poll timestamps so the next
// polling cycle runs an immediate catch-up poll rather than waiting out the
// remainder of the old interval.
func (m *Monitor) SetBackupPollingInterval(interval time.Duration) {
if m == nil {
return
}
if interval < 0 {
interval = 0
}
m.runtimePollingMu.Lock()
previous := time.Duration(0)
if m.config != nil {
previous = m.config.BackupPollingInterval
}
if m.runtimePollingOverride.backupInterval != nil {
previous = *m.runtimePollingOverride.backupInterval
}
m.runtimePollingOverride.backupInterval = &interval
m.runtimePollingMu.Unlock()
if interval > 0 && (previous <= 0 || interval < previous) {
m.resetBackupPollTimestamps()
}
}
// SetPBSPollingInterval updates the PBS polling cadence on the live monitor.
// The scheduler re-reads the base interval every cycle, so no further action
// is needed.
func (m *Monitor) SetPBSPollingInterval(interval time.Duration) {
if m == nil || interval <= 0 {
return
}
m.runtimePollingMu.Lock()
m.runtimePollingOverride.pbsInterval = &interval
m.runtimePollingMu.Unlock()
}
// SetPMGPollingInterval updates the PMG polling cadence on the live monitor.
// The scheduler re-reads the base interval every cycle, so no further action
// is needed.
func (m *Monitor) SetPMGPollingInterval(interval time.Duration) {
if m == nil || interval <= 0 {
return
}
m.runtimePollingMu.Lock()
m.runtimePollingOverride.pmgInterval = &interval
m.runtimePollingMu.Unlock()
}
// ResetSSHFailureBackoff clears the temperature collector's per-host SSH
// backoff and the knownhosts keyscan backoff. A system-settings save is an
// operator touchpoint that often follows repairing SSH access, and the
// on-disk key-change check only notices key file replacement, so the save
// itself also drops the backoff windows rather than making the operator wait
// one out (#1638).
func (m *Monitor) ResetSSHFailureBackoff() {
if m == nil {
return
}
m.tempCollector.ResetSSHFailures()
}
// resetBackupPollTimestamps clears the per-instance backup poll timestamps
// so the next cycle polls immediately.
func (m *Monitor) resetBackupPollTimestamps() {
m.mu.Lock()
m.lastPVEBackupPoll = make(map[string]time.Time)
m.lastPBSBackupPoll = make(map[string]time.Time)
m.mu.Unlock()
}