From dbc20a9f665bd2614a897dd8f2d989724ff2a604 Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Wed, 22 Jul 2026 01:33:26 +0200 Subject: [PATCH] test(deletion-maintenance): de-flake watchdog tests via event gates The three watchdog tests raced a fixed fanout-body sleep against sub-second warn/severe thresholds (e.g. 150ms body vs 500ms severe). On a loaded CI runner the whole RunCoreAsync body overran the severe threshold before the finally-block cancelled the watchdog, so the severe OpsEvent fired and "LongRunning past warning" flaked (3100/3101). Replace the wall-clock race with event gates: the fanout body now blocks on WaitForEventAsync() instead of sleeping, so the target watchdog is guaranteed to have fired before the body returns. The opposite threshold is pushed to minutes (warn-only tests) or the body waits on the severe event (both-fire test) so warn is necessarily first. OpsEvents recording is now lock-guarded since watchdog tasks record concurrently. No production change. Watchdog class ran 5x green (16/16); full suite 3110/3110. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...SessionDeletionMaintenanceFunctionTests.cs | 59 +++++++++++++++---- 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionMaintenanceFunctionTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionMaintenanceFunctionTests.cs index 6794658c..c3c2f51d 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionMaintenanceFunctionTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionMaintenanceFunctionTests.cs @@ -78,15 +78,16 @@ public class SessionDeletionMaintenanceFunctionTests [Fact] public async Task RunCore_emits_LongRunning_when_body_runs_past_warning_threshold() { - // Set watchdog thresholds to sub-second; make the fanout sleep long enough to trip warning - // but not severe. - var harness = new Harness(watchdogWarn: TimeSpan.FromMilliseconds(50), watchdogSevere: TimeSpan.FromMilliseconds(500)); + // Deterministic (no wall-clock race): the fanout body blocks until the WARN watchdog has + // actually recorded its OpsEvent, then returns. Severe sits minutes out, so it can never + // fire in the moment between warn firing and the body completing (which cancels it). + var harness = new Harness(watchdogWarn: TimeSpan.FromMilliseconds(20), watchdogSevere: TimeSpan.FromMinutes(10)); harness.SetKillSwitch(false); harness.Fanout.Setup(f => f.RunAsync(It.IsAny(), It.IsAny(), It.IsAny())) .Returns(async (SessionRetentionFanoutService.FanoutResult r, DateTime _, CancellationToken ct) => { r.TenantsProcessed = 1; - await Task.Delay(150, ct); + await harness.WaitForEventAsync("SessionDeletionMaintenanceLongRunning"); }); await harness.Sut.RunCoreAsync("Test", CancellationToken.None); @@ -102,14 +103,17 @@ public class SessionDeletionMaintenanceFunctionTests // assigned only AFTER the fanout returned, so every LongRunning warning showed zeros. // The fanout now mutates the caller-supplied result incrementally; the watchdog snapshot // must reflect that progress mid-run. - var harness = new Harness(watchdogWarn: TimeSpan.FromMilliseconds(50), watchdogSevere: TimeSpan.FromSeconds(5)); + // Deterministic: counters are set at fanout entry, then the body blocks until the WARN + // watchdog fires and snapshots them. Severe sits minutes out and never fires — no + // body-sleep-vs-threshold race. + var harness = new Harness(watchdogWarn: TimeSpan.FromMilliseconds(50), watchdogSevere: TimeSpan.FromMinutes(10)); harness.SetKillSwitch(false); harness.Fanout.Setup(f => f.RunAsync(It.IsAny(), It.IsAny(), It.IsAny())) .Returns(async (SessionRetentionFanoutService.FanoutResult r, DateTime _, CancellationToken ct) => { r.TenantsProcessed = 3; r.SessionsEnqueued = 7; - await Task.Delay(200, ct); // long enough for the 50ms watchdog to fire mid-run + await harness.WaitForEventAsync("SessionDeletionMaintenanceLongRunning"); }); await harness.Sut.RunCoreAsync("Test", CancellationToken.None); @@ -122,17 +126,20 @@ public class SessionDeletionMaintenanceFunctionTests [Fact] public async Task RunCore_emits_LongRunningSevere_when_body_runs_past_severe_threshold() { - var harness = new Harness(watchdogWarn: TimeSpan.FromMilliseconds(50), watchdogSevere: TimeSpan.FromMilliseconds(150)); + // Deterministic: the body blocks until the SEVERE watchdog records its OpsEvent. Warn has a + // smaller threshold, so it is guaranteed to have fired first — neither depends on a body + // sleep outlasting a fixed wall-clock threshold. + var harness = new Harness(watchdogWarn: TimeSpan.FromMilliseconds(20), watchdogSevere: TimeSpan.FromMilliseconds(60)); harness.SetKillSwitch(false); harness.Fanout.Setup(f => f.RunAsync(It.IsAny(), It.IsAny(), It.IsAny())) .Returns(async (SessionRetentionFanoutService.FanoutResult _, DateTime _, CancellationToken ct) => { - await Task.Delay(300, ct); + await harness.WaitForEventAsync("SessionDeletionMaintenanceLongRunningSevere"); }); await harness.Sut.RunCoreAsync("Test", CancellationToken.None); - // Both watchdogs fire: warning at 50ms, severe at 150ms. + // Both watchdogs fire: warning first (smaller threshold), then severe (which the body waited for). Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceLongRunning"); Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceLongRunningSevere"); } @@ -345,6 +352,12 @@ public class SessionDeletionMaintenanceFunctionTests private readonly Dictionary> _sessionsByState = new(StringComparer.Ordinal); private readonly List _ttlBlobs = new(); + // Watchdog OpsEvents are recorded from concurrent watchdog tasks; _opsSync serializes the + // OpsEvents append + waiter signalling, and _eventWaiters lets a test's fanout body block + // until a specific watchdog OpsEvent has actually fired (see WaitForEventAsync). + private readonly object _opsSync = new(); + private readonly Dictionary _eventWaiters = new(StringComparer.Ordinal); + public Harness(TimeSpan? watchdogWarn = null, TimeSpan? watchdogSevere = null) { StorageMock = new Mock(Mock.Of(), NullLogger.Instance); @@ -376,7 +389,12 @@ public class SessionDeletionMaintenanceFunctionTests opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny())) .Returns(e => { - OpsEvents.Add(new CapturedOpsEvent(e.EventType, e.Severity, e.Message, e.Details ?? string.Empty)); + lock (_opsSync) + { + OpsEvents.Add(new CapturedOpsEvent(e.EventType, e.Severity, e.Message, e.Details ?? string.Empty)); + if (_eventWaiters.TryGetValue(e.EventType, out var waiter)) + waiter.TrySetResult(); + } return Task.CompletedTask; }); var alertDispatch = new OpsAlertDispatchService( @@ -426,6 +444,27 @@ public class SessionDeletionMaintenanceFunctionTests public void SetKillSwitch(bool active) => AdminConfig.Setup(a => a.IsSessionDeletionKillSwitchActiveAsync()).ReturnsAsync(active); + /// + /// Deterministic watchdog gate: returns a task that completes once an OpsEvent whose type is + /// has been recorded. A test's fanout body awaits this instead of + /// sleeping a fixed span, so the watchdog is guaranteed to have fired before the body returns — + /// removing the wall-clock race (body sleep vs sub-second thresholds) that flaked under CI load. + /// + public Task WaitForEventAsync(string eventName) + { + lock (_opsSync) + { + if (OpsEvents.Any(e => e.EventName == eventName)) + return Task.CompletedTask; + if (!_eventWaiters.TryGetValue(eventName, out var waiter)) + { + waiter = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _eventWaiters[eventName] = waiter; + } + return waiter.Task; + } + } + public void SeedTtlBlobs(params (string TenantId, string SessionId, string ManifestId, DateTime LastModified)[] entries) { foreach (var e in entries)