mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-25 21:20:00 +00:00
912d7ceb51
Non-default organisations run their monitors against a detached copy of the server config (#1619). A system-settings save reaches those monitors only as runtime PBS and PMG polling overrides, so the copy that the connections list reads through the org's config handlers kept the old interval until that monitor was rebuilt. The active-to-stale cutoff is 3 x the polling interval, so raising PBS from 1m to 5m left a 3m cutoff and Settings > Infrastructure badges read "stale" for part of every cycle, while lowering it delayed the badge by the same factor. Add Monitor.BasePollInterval, an override-aware reader for the clamped PVE, PBS and PMG base cadence that never takes m.mu, and have the connections input builder use it whenever a monitor is present. The fixed-cadence branch of resourcePollIntervals reads the same accessor, so resource freshness and connection freshness share one derivation. The system settings GET is not affected: both production routes serve SystemSettingsHandler, which reads the base config and persisted settings; the configapi handler that reads the tenant copy is only a fallback when that handler is absent.
205 lines
6.8 KiB
Go
205 lines
6.8 KiB
Go
package monitoring
|
|
|
|
import "time"
|
|
|
|
// Runtime-tunable polling cadence settings.
|
|
//
|
|
// Each tenant monitor polls against a detached DeepCopy of the base config,
|
|
// so mutating the base config after a system-settings save never reaches a
|
|
// running monitor (#1619). These setters push the saved values into live
|
|
// monitors directly, shadowing the config fields. The overrides live behind
|
|
// runtimePollingMu because polling goroutines read them every cycle while
|
|
// the settings API writes them.
|
|
|
|
// runtimePollingOverrides holds the polling-cadence values the settings API
|
|
// has pushed into this live monitor, shadowing the corresponding fields of
|
|
// the monitor's detached config copy. nil means "use the config value".
|
|
type runtimePollingOverrides struct {
|
|
backupEnabled *bool
|
|
backupInterval *time.Duration
|
|
pbsInterval *time.Duration
|
|
pmgInterval *time.Duration
|
|
}
|
|
|
|
// backupPollingEnabledSetting returns the effective EnableBackupPolling value.
|
|
func (m *Monitor) backupPollingEnabledSetting() bool {
|
|
if m == nil || m.config == nil {
|
|
return false
|
|
}
|
|
m.runtimePollingMu.RLock()
|
|
override := m.runtimePollingOverride.backupEnabled
|
|
m.runtimePollingMu.RUnlock()
|
|
if override != nil {
|
|
return *override
|
|
}
|
|
return m.config.EnableBackupPolling
|
|
}
|
|
|
|
// backupPollingIntervalSetting returns the effective BackupPollingInterval.
|
|
// Zero means cycle-based scheduling (BackupPollingCycles).
|
|
func (m *Monitor) backupPollingIntervalSetting() time.Duration {
|
|
if m == nil || m.config == nil {
|
|
return 0
|
|
}
|
|
m.runtimePollingMu.RLock()
|
|
override := m.runtimePollingOverride.backupInterval
|
|
m.runtimePollingMu.RUnlock()
|
|
if override != nil {
|
|
return *override
|
|
}
|
|
return m.config.BackupPollingInterval
|
|
}
|
|
|
|
// pbsPollingIntervalSetting returns the effective PBSPollingInterval before
|
|
// clamping.
|
|
func (m *Monitor) pbsPollingIntervalSetting() time.Duration {
|
|
if m == nil || m.config == nil {
|
|
return 0
|
|
}
|
|
m.runtimePollingMu.RLock()
|
|
override := m.runtimePollingOverride.pbsInterval
|
|
m.runtimePollingMu.RUnlock()
|
|
if override != nil {
|
|
return *override
|
|
}
|
|
return m.config.PBSPollingInterval
|
|
}
|
|
|
|
// pmgPollingIntervalSetting returns the effective PMGPollingInterval before
|
|
// clamping.
|
|
func (m *Monitor) pmgPollingIntervalSetting() time.Duration {
|
|
if m == nil || m.config == nil {
|
|
return 0
|
|
}
|
|
m.runtimePollingMu.RLock()
|
|
override := m.runtimePollingOverride.pmgInterval
|
|
m.runtimePollingMu.RUnlock()
|
|
if override != nil {
|
|
return *override
|
|
}
|
|
return m.config.PMGPollingInterval
|
|
}
|
|
|
|
// BasePollInterval returns the clamped base cadence this monitor polls PVE,
|
|
// PBS or PMG at, honouring the runtime polling overrides. For the built-in
|
|
// providers these are the values the fixed-cadence scheduler reads
|
|
// (baseIntervalForInstanceType); a replacement provider or a task already
|
|
// queued at its previous interval can differ until the next planning pass.
|
|
// This accessor skips the poll-provider lookup, so it never takes m.mu.
|
|
// Non-default tenant monitors poll against a detached config copy (#1619), so
|
|
// a saved PBS or PMG interval reaches them only as an override. Code outside
|
|
// the monitor that judges poll freshness must read the cadence here, not from
|
|
// GetConfig. An adaptive scheduler plans its own intervals independently of
|
|
// this base; see PlannedPollInterval. Zero means the cadence is unknown:
|
|
// another instance type, or a monitor without config.
|
|
func (m *Monitor) BasePollInterval(instanceType InstanceType) time.Duration {
|
|
if m == nil || m.config == nil {
|
|
return 0
|
|
}
|
|
switch instanceType {
|
|
case InstanceTypePVE:
|
|
return m.effectivePVEPollingInterval()
|
|
case InstanceTypePBS:
|
|
return clampInterval(m.pbsPollingIntervalSetting(), 10*time.Second, time.Hour)
|
|
case InstanceTypePMG:
|
|
return clampInterval(m.pmgPollingIntervalSetting(), 10*time.Second, time.Hour)
|
|
default:
|
|
return 0
|
|
}
|
|
}
|
|
|
|
// SetBackupPollingEnabled toggles backup polling on the live monitor.
|
|
// Re-enabling clears the per-instance last-poll timestamps so the next
|
|
// polling cycle runs an immediate catch-up poll, matching the behavior of
|
|
// a full monitor reload.
|
|
func (m *Monitor) SetBackupPollingEnabled(enabled bool) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
m.runtimePollingMu.Lock()
|
|
wasEnabled := m.config != nil && m.config.EnableBackupPolling
|
|
if m.runtimePollingOverride.backupEnabled != nil {
|
|
wasEnabled = *m.runtimePollingOverride.backupEnabled
|
|
}
|
|
m.runtimePollingOverride.backupEnabled = &enabled
|
|
m.runtimePollingMu.Unlock()
|
|
|
|
if enabled && !wasEnabled {
|
|
m.resetBackupPollTimestamps()
|
|
}
|
|
}
|
|
|
|
// SetBackupPollingInterval updates the backup polling cadence on the live
|
|
// monitor. Lowering the interval (or switching from cycle-based scheduling
|
|
// to an interval) clears the per-instance last-poll timestamps so the next
|
|
// polling cycle runs an immediate catch-up poll rather than waiting out the
|
|
// remainder of the old interval.
|
|
func (m *Monitor) SetBackupPollingInterval(interval time.Duration) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
if interval < 0 {
|
|
interval = 0
|
|
}
|
|
m.runtimePollingMu.Lock()
|
|
previous := time.Duration(0)
|
|
if m.config != nil {
|
|
previous = m.config.BackupPollingInterval
|
|
}
|
|
if m.runtimePollingOverride.backupInterval != nil {
|
|
previous = *m.runtimePollingOverride.backupInterval
|
|
}
|
|
m.runtimePollingOverride.backupInterval = &interval
|
|
m.runtimePollingMu.Unlock()
|
|
|
|
if interval > 0 && (previous <= 0 || interval < previous) {
|
|
m.resetBackupPollTimestamps()
|
|
}
|
|
}
|
|
|
|
// SetPBSPollingInterval updates the PBS polling cadence on the live monitor.
|
|
// The scheduler re-reads the base interval every cycle, so no further action
|
|
// is needed.
|
|
func (m *Monitor) SetPBSPollingInterval(interval time.Duration) {
|
|
if m == nil || interval <= 0 {
|
|
return
|
|
}
|
|
m.runtimePollingMu.Lock()
|
|
m.runtimePollingOverride.pbsInterval = &interval
|
|
m.runtimePollingMu.Unlock()
|
|
}
|
|
|
|
// SetPMGPollingInterval updates the PMG polling cadence on the live monitor.
|
|
// The scheduler re-reads the base interval every cycle, so no further action
|
|
// is needed.
|
|
func (m *Monitor) SetPMGPollingInterval(interval time.Duration) {
|
|
if m == nil || interval <= 0 {
|
|
return
|
|
}
|
|
m.runtimePollingMu.Lock()
|
|
m.runtimePollingOverride.pmgInterval = &interval
|
|
m.runtimePollingMu.Unlock()
|
|
}
|
|
|
|
// ResetSSHFailureBackoff clears the temperature collector's per-host SSH
|
|
// backoff and the knownhosts keyscan backoff. A system-settings save is an
|
|
// operator touchpoint that often follows repairing SSH access, and the
|
|
// on-disk key-change check only notices key file replacement, so the save
|
|
// itself also drops the backoff windows rather than making the operator wait
|
|
// one out (#1638).
|
|
func (m *Monitor) ResetSSHFailureBackoff() {
|
|
if m == nil {
|
|
return
|
|
}
|
|
m.tempCollector.ResetSSHFailures()
|
|
}
|
|
|
|
// resetBackupPollTimestamps clears the per-instance backup poll timestamps
|
|
// so the next cycle polls immediately.
|
|
func (m *Monitor) resetBackupPollTimestamps() {
|
|
m.mu.Lock()
|
|
m.lastPVEBackupPoll = make(map[string]time.Time)
|
|
m.lastPBSBackupPoll = make(map[string]time.Time)
|
|
m.mu.Unlock()
|
|
}
|