mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-27 05:57:27 +00:00
06b7f5a0b9
SetMockMode(false) switched the mock package off and then cleared active alerts, but every path that evaluates mode-dependent data checked the mode once, up front. A mock alert pass (started from the ticker with go, so passes overlap), a frontend or ingest refresh evaluating a GetState snapshot, a backup-age check over fixture recovery rollups, or a connection check over the mock ledger could already hold fixture data when the switch happened. Its evaluations then landed after the clear, in live mode, where nothing evaluates or resolves fixture resources, so the alerts stayed until the 24-hour stale sweep. A pass-level lock around checkMockAlerts was tried and reverted: it left the other paths open and made the mock-mode API wait behind a whole pass of synchronous event-log writes. The monitor now owns a mock-mode epoch fence. The mock pass, backup-age and snapshot checks, connection checks, Docker pruning and the resource-store refresh take a scope before reading anything mode-dependent and send each alert-manager call through it. SetMockMode, now serialized, flips the mode, ends the epoch, waits only for calls already admitted, and then clears. Unified metrics are evaluated one resource per call, so the wait covers single calls, not passes. Registry rebuilds are tracked rather than waited on. A refresh evaluates the registry it read back only if no rebuild from an ended epoch overlapped it, and the readers that open or remove alerts from the inventory use current state until a clean rebuild in the new epoch replaces the previous mode's registry. The mock supplemental fixture sources now serve nothing outside mock mode: the router swaps them for the real pollers only after the switch, refreshing between sources, so a single switch to live mode used to raise fixture vSphere incidents. The new tests park a real evaluation at a fixed point (a held lock or a blocking supplier), switch modes and release it. Against the unfenced code the mock pass left about 40 fixture alerts, the frontend refresh 5 vSphere incidents and the backup check 137 backup-age alerts in live mode, and a mock-mode connection observation counted toward a live connection's confirmation. The drain, republish and pruning tests fail against a fence without its wait, overlap check or registry tracking. Switching mock mode with several tenant monitors running remains uncovered and is noted in the monitoring contract.
121 lines
3.8 KiB
Go
121 lines
3.8 KiB
Go
package monitoring
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestMockModeFenceRefusesCallsFromAnEndedEpoch(t *testing.T) {
|
|
var fence mockModeFence
|
|
stale := fence.begin()
|
|
fence.advance()
|
|
|
|
if stale.current() {
|
|
t.Fatal("a scope from an ended epoch still reads as current")
|
|
}
|
|
if stale.run(func() { t.Fatal("a call from an ended epoch ran") }) {
|
|
t.Fatal("run reported a refused call as admitted")
|
|
}
|
|
ran := false
|
|
if !fence.begin().run(func() { ran = true }) || !ran {
|
|
t.Fatal("a scope taken after the epoch ended was refused")
|
|
}
|
|
}
|
|
|
|
func TestMockModeFenceAdvanceWaitsForAdmittedCalls(t *testing.T) {
|
|
var fence mockModeFence
|
|
scope := fence.begin()
|
|
entered, release := make(chan struct{}), make(chan struct{})
|
|
nestedAdmitted := make(chan bool, 1)
|
|
callDone := make(chan struct{})
|
|
go func() {
|
|
defer close(callDone)
|
|
scope.run(func() {
|
|
close(entered)
|
|
<-release
|
|
// A call made from inside an admitted one, after the epoch
|
|
// ended, is refused rather than deadlocking the waiting advance.
|
|
nestedAdmitted <- scope.run(func() {})
|
|
})
|
|
}()
|
|
waitForSignal(t, entered, 10*time.Second, "admitted call never started")
|
|
|
|
advanced := make(chan struct{})
|
|
go func() {
|
|
defer close(advanced)
|
|
fence.advance()
|
|
}()
|
|
waitForCondition(t, 10*time.Second, func() bool { return !scope.current() }, "advance never ended the epoch")
|
|
select {
|
|
case <-advanced:
|
|
t.Fatal("advance returned while a call admitted under the ended epoch was still running")
|
|
default:
|
|
}
|
|
if scope.run(func() { t.Error("a call started after the epoch ended ran") }) {
|
|
t.Fatal("run admitted a call after the epoch ended")
|
|
}
|
|
|
|
close(release)
|
|
waitForSignal(t, callDone, 10*time.Second, "admitted call did not finish")
|
|
waitForSignal(t, advanced, 10*time.Second, "advance did not return once the admitted call finished")
|
|
if <-nestedAdmitted {
|
|
t.Fatal("a nested call made after the epoch ended was admitted")
|
|
}
|
|
}
|
|
|
|
func TestMockModeFenceFlagsReadsOverlappedByAnEndedEpochPublish(t *testing.T) {
|
|
var fence mockModeFence
|
|
if !fence.registryCurrent() {
|
|
t.Fatal("a monitor that never switched modes distrusts its registry")
|
|
}
|
|
stale := fence.begin()
|
|
entered, release := make(chan struct{}), make(chan struct{})
|
|
staleDone := make(chan struct{})
|
|
go func() {
|
|
defer close(staleDone)
|
|
stale.publish(func() bool {
|
|
close(entered)
|
|
<-release
|
|
return true
|
|
})
|
|
}()
|
|
waitForSignal(t, entered, 10*time.Second, "publish never started")
|
|
|
|
// advance does not wait for publishes.
|
|
advanced := make(chan struct{})
|
|
go func() {
|
|
defer close(advanced)
|
|
fence.advance()
|
|
}()
|
|
waitForSignal(t, advanced, 10*time.Second, "advance waited for a publish")
|
|
if fence.registryCurrent() {
|
|
t.Fatal("the registry counted as current right after a switch")
|
|
}
|
|
if _, ok := stale.publish(func() bool { t.Error("a publish from an ended epoch ran"); return true }); ok {
|
|
t.Fatal("publish admitted an ended epoch")
|
|
}
|
|
|
|
live := fence.begin()
|
|
mark, ok := live.publish(func() bool { return true })
|
|
if !ok {
|
|
t.Fatal("publish refused the current epoch")
|
|
}
|
|
if live.undisturbedSince(mark) || fence.registryCurrent() {
|
|
t.Fatal("a publish overlapped by a running ended-epoch publish counted as undisturbed")
|
|
}
|
|
close(release)
|
|
waitForSignal(t, staleDone, 10*time.Second, "ended-epoch publish did not finish")
|
|
if live.undisturbedSince(mark) || fence.registryCurrent() {
|
|
t.Fatal("a publish overlapped by a finished ended-epoch publish counted as undisturbed")
|
|
}
|
|
|
|
// A refresh that skipped its rebuild replaced nothing.
|
|
if _, ok := live.publish(func() bool { return false }); !ok || fence.registryCurrent() {
|
|
t.Fatal("a publish that replaced nothing made the registry current")
|
|
}
|
|
next, ok := live.publish(func() bool { return true })
|
|
if !ok || !live.undisturbedSince(next) || !fence.registryCurrent() {
|
|
t.Fatal("a rebuild after the ended-epoch publish finished did not make the registry current")
|
|
}
|
|
}
|