Files
AutopilotMonitor/src/Backend/AutopilotMonitor.Functions.Tests/HealthCheckServicePoisonQueueTests.cs
T
Oliver Kieselbach c8ce7cdb37 backend: remove the V2 index tables and their dual-write pipeline (D-196)
The five cross-session index tables (SessionsByTerminal, SessionsByStage,
DeadEndsByReason, ClassifierVerdictsByIdLevel, SignalsByKind) were write-only
since M5.d: no endpoint, MCP tool or web page ever read them, while the
dual-write cost a serial queue send per signal/transition on the ingest path,
a 5-13 min full scan of Signals + DecisionTransitions every two hours and five
property-filter scans per session deletion (audit 2026-09-05 F05).

Removed: the tables from TableNames.All (schema hash changes, next cold start
runs a full pass without them), the telemetry-index-reconcile queue name,
producer, envelope, handler, queue function and 2h timer, the cross-partition
Timestamp scans on the signal/transition repositories, the EnableIndexDualWrite
flag (model, config mapping, admin UI, generated wire types), cascade steps
11-15 (later steps renumbered 11-18), the deletion wrappers and the offboarding
wipe entries. DiscriminatorPkProp stays as a step class so manifests written
before this change still resume.

Order of operations: deploy first, then delete the queue pair and the five
tables in the portal (B-88h). Deleting the tables before the deploy 500s every
delete preview/enqueue and poisons offboarding.

Submodules: .claude/commands (import_session.py), internal (D-196, docs, plan).

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-05 04:27:48 +02:00

215 lines
8.4 KiB
C#

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Monitoring;
using AutopilotMonitor.Shared;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging.Abstractions;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Service-level tests for the poison-queue HealthCheck card. Drives
/// <see cref="HealthCheckService.CheckPoisonQueuesAsync"/> with an in-memory probe
/// and asserts the health card's status/message/details shape. The pure
/// boundary classifier and seen-index extractor live in
/// <see cref="MaintenanceServicePoisonQueueTests"/> next to the canonical definitions.
/// </summary>
public class HealthCheckServicePoisonQueueTests
{
[Fact]
public async Task Check_AllPoisonQueuesEmpty_ReportsHealthy()
{
// Existing-but-drained poison queues: enumerated, probed, all zero.
var probe = new FakePoisonQueueProbe();
probe.Counts[Constants.QueueNames.AnalyzeOnEnrollmentEnd + "-poison"] = 0;
probe.Counts[Constants.QueueNames.VulnerabilityCorrelate + "-poison"] = 0;
probe.Counts[Constants.QueueNames.TenantAutoApprove + "-poison"] = 0;
var svc = BuildService(probe);
var check = await svc.CheckPoisonQueuesAsync();
Assert.Equal("Poison Queues", check.Name);
Assert.Equal("healthy", check.Status);
Assert.Equal("All poison queues empty", check.Message);
Assert.Equal(3, check.Details!.Count);
Assert.All(check.Details.Values, v => Assert.Equal("0 messages", v));
}
[Fact]
public async Task Check_NoPoisonQueuesExist_ReportsHealthy()
{
// Steady state on a healthy system: poison queues are created lazily on
// first poison-move, so an empty enumeration means nothing ever failed.
var svc = BuildService(new FakePoisonQueueProbe());
var check = await svc.CheckPoisonQueuesAsync();
Assert.Equal("healthy", check.Status);
Assert.Equal("All poison queues empty", check.Message);
Assert.Empty(check.Details!);
}
[Fact]
public async Task Check_EnumerationFails_ReportsWarningWithError()
{
// A failed listing must never read as an all-clear.
var probe = new FakePoisonQueueProbe
{
ListException = new InvalidOperationException("auth denied")
};
var svc = BuildService(probe);
var check = await svc.CheckPoisonQueuesAsync();
Assert.Equal("warning", check.Status);
Assert.Contains("enumeration failed", check.Message);
Assert.Contains("auth denied", (string)check.Details!["error"]);
}
[Fact]
public async Task Check_SingleQueueWithBacklog_ReportsWarning()
{
var probe = new FakePoisonQueueProbe();
probe.Counts[Constants.QueueNames.AnalyzeOnEnrollmentEnd + "-poison"] = 3;
probe.Counts[Constants.QueueNames.VulnerabilityCorrelate + "-poison"] = 0;
probe.Counts[Constants.QueueNames.TenantAutoApprove + "-poison"] = 0;
var svc = BuildService(probe);
var check = await svc.CheckPoisonQueuesAsync();
Assert.Equal("warning", check.Status);
Assert.Equal("Operator review required — failed messages parked after 5 retries", check.Message);
Assert.Equal("3 messages",
check.Details![Constants.QueueNames.AnalyzeOnEnrollmentEnd + "-poison"]);
Assert.Equal("0 messages",
check.Details[Constants.QueueNames.VulnerabilityCorrelate + "-poison"]);
}
[Fact]
public async Task Check_QueueAboveCritical_ReportsUnhealthy()
{
var probe = new FakePoisonQueueProbe();
probe.Counts[Constants.QueueNames.AnalyzeOnEnrollmentEnd + "-poison"] = 15;
probe.Counts[Constants.QueueNames.VulnerabilityCorrelate + "-poison"] = 0;
probe.Counts[Constants.QueueNames.TenantAutoApprove + "-poison"] = 2;
var svc = BuildService(probe);
var check = await svc.CheckPoisonQueuesAsync();
Assert.Equal("unhealthy", check.Status);
Assert.Contains("Poison backlog accumulating", check.Message);
Assert.Equal("15 messages",
check.Details![Constants.QueueNames.AnalyzeOnEnrollmentEnd + "-poison"]);
}
[Fact]
public async Task Check_ProbeThrows_DetailsRecordErrorAndForceWarning()
{
var probe = new FakePoisonQueueProbe
{
ExceptionFor =
{
[Constants.QueueNames.VulnerabilityCorrelate + "-poison"] =
new InvalidOperationException("storage 500")
}
};
probe.Counts[Constants.QueueNames.AnalyzeOnEnrollmentEnd + "-poison"] = 0;
probe.Counts[Constants.QueueNames.TenantAutoApprove + "-poison"] = 0;
var svc = BuildService(probe);
var check = await svc.CheckPoisonQueuesAsync();
Assert.Equal("warning", check.Status);
var errorEntry = (string)check.Details![Constants.QueueNames.VulnerabilityCorrelate + "-poison"];
Assert.StartsWith("error:", errorEntry);
Assert.Contains("storage 500", errorEntry);
}
[Fact]
public async Task Check_CustomThresholdsFromConfig_AreRespected()
{
var probe = new FakePoisonQueueProbe();
probe.Counts[Constants.QueueNames.AnalyzeOnEnrollmentEnd + "-poison"] = 4;
probe.Counts[Constants.QueueNames.VulnerabilityCorrelate + "-poison"] = 0;
probe.Counts[Constants.QueueNames.TenantAutoApprove + "-poison"] = 0;
// warn=5 → 4 messages now classifies as healthy (would be warning under defaults)
var svc = BuildService(probe, new Dictionary<string, string?>
{
["PoisonQueueWarningThreshold"] = "5",
["PoisonQueueCriticalThreshold"] = "50",
});
var check = await svc.CheckPoisonQueuesAsync();
Assert.Equal("healthy", check.Status);
}
[Fact]
public async Task Check_OneMessage_FormatsAsSingularInDetails()
{
var probe = new FakePoisonQueueProbe();
probe.Counts[Constants.QueueNames.AnalyzeOnEnrollmentEnd + "-poison"] = 1;
probe.Counts[Constants.QueueNames.VulnerabilityCorrelate + "-poison"] = 0;
probe.Counts[Constants.QueueNames.TenantAutoApprove + "-poison"] = 0;
var svc = BuildService(probe);
var check = await svc.CheckPoisonQueuesAsync();
Assert.Equal("1 message",
check.Details![Constants.QueueNames.AnalyzeOnEnrollmentEnd + "-poison"]);
}
/// <summary>
/// Builds a HealthCheckService wired only with the dependencies that
/// <see cref="HealthCheckService.CheckPoisonQueuesAsync"/> touches.
/// The other constructor parameters are passed as <c>null!</c> — they
/// belong to sibling checks that this test class never invokes.
/// </summary>
private static HealthCheckService BuildService(
FakePoisonQueueProbe probe,
IDictionary<string, string?>? extraConfig = null)
{
var config = new ConfigurationBuilder()
.AddInMemoryCollection(extraConfig ?? new Dictionary<string, string?>())
.Build();
return new HealthCheckService(
NullLogger<HealthCheckService>.Instance,
adminConfigService: null!,
httpClientFactory: null!,
metricsReader: null!,
poisonQueueProbe: probe,
configuration: config);
}
private sealed class FakePoisonQueueProbe : IPoisonQueueProbe
{
public Dictionary<string, long> Counts { get; } = new();
public Dictionary<string, Exception> ExceptionFor { get; } = new();
public Exception? ListException { get; set; }
public Task<IReadOnlyList<string>> ListPoisonQueuesAsync(CancellationToken ct)
{
if (ListException is not null)
throw ListException;
IReadOnlyList<string> names = Counts.Keys
.Concat(ExceptionFor.Keys)
.Distinct()
.OrderBy(n => n, StringComparer.Ordinal)
.ToList();
return Task.FromResult(names);
}
public Task<long> GetApproximateMessageCountAsync(string queueName, CancellationToken ct)
{
if (ExceptionFor.TryGetValue(queueName, out var ex))
throw ex;
if (Counts.TryGetValue(queueName, out var count))
return Task.FromResult(count);
return Task.FromResult(0L);
}
}
}