diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/VerdictCalibrationTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/VerdictCalibrationTests.cs new file mode 100644 index 00000000..6e52af24 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/VerdictCalibrationTests.cs @@ -0,0 +1,301 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using AutopilotMonitor.Functions.Functions.Metrics; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Shared.Models; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Verdict calibration aggregate (docs/backend/verdict-calibration.md): the pure bucketing core, +/// the persistence round-trip and the matrix arithmetic of the read endpoint. +/// +public class VerdictCalibrationTests +{ + private const string TenantA = "11111111-1111-1111-1111-111111111111"; + private const string TenantB = "22222222-2222-2222-2222-222222222222"; + private static readonly DateTime Now = new(2026, 8, 23, 12, 0, 0, DateTimeKind.Utc); + + private static SessionSummary Session(string tenant, string id, SessionStatus status, DateTime started, + string? path = null, string? priorPath = null, string? priorStatus = null, string? adminMarked = null, + DateTime? completed = null, string serial = "SN-1", string? failureReason = null, string? reconcileReason = null) => new() + { + TenantId = tenant, SessionId = id, SerialNumber = serial, Status = status, StartedAt = started, + CompletedAt = completed, VerdictPath = path, PriorVerdictPath = priorPath, PriorStatus = priorStatus, + AdminMarkedAction = adminMarked, FailureReason = failureReason ?? string.Empty, + ReconcileReason = reconcileReason ?? string.Empty, FailureSource = string.Empty, + }; + + private static DeviceHistory History(string tenant, string serial, params (string Id, DateTime Started, DateTime? Completed, string Status)[] refs) => new() + { + TenantId = tenant, SerialKey = serial.ToLowerInvariant(), SerialNumber = serial, + Chain = refs.Select(r => new DeviceSessionRef { SessionId = r.Id, StartedAt = r.Started, CompletedAt = r.Completed, Status = r.Status }).ToList(), + }; + + private static VerdictCalibrationBucket Bucket(VerdictCalibrationDailyAggregate row, string path, string status) + => Assert.Single(row.Buckets, b => b.VerdictPath == path && b.Status == status); + + // ---- BuildVerdictCalibrationAggregates ---- + + [Fact] + public void Buckets_by_tenant_date_path_and_status_with_global_mirror() + { + var d1 = new DateTime(2026, 8, 10, 9, 0, 0, DateTimeKind.Utc); + var sessions = new List + { + Session(TenantA, "a1", SessionStatus.Succeeded, d1, path: "agent:complete", completed: d1.AddHours(1)), + Session(TenantA, "a2", SessionStatus.Succeeded, d1.AddHours(2), path: "agent:complete", completed: d1.AddHours(3)), + Session(TenantA, "a3", SessionStatus.Incomplete, d1, path: "sweep:r6", completed: d1.AddHours(6)), + Session(TenantA, "a4", SessionStatus.Stalled, d1.AddDays(1), path: "sweep:stalled"), + Session(TenantB, "b1", SessionStatus.Failed, d1, path: "agent:failed", completed: d1.AddHours(1)), + }; + var rows = MaintenanceService.BuildVerdictCalibrationAggregates(sessions, new Dictionary>(), Now); + + var a10 = Assert.Single(rows, r => r.TenantId == TenantA && r.Date == "2026-08-10"); + Assert.Equal(3, a10.SessionCount); + Assert.Equal(3, a10.TerminalSessionCount); + Assert.Equal(2, Bucket(a10, "agent:complete", "Succeeded").Count); + Assert.Equal(1, Bucket(a10, "sweep:r6", "Incomplete").Count); + + var a11 = Assert.Single(rows, r => r.TenantId == TenantA && r.Date == "2026-08-11"); + Assert.Equal(1, a11.SessionCount); + Assert.Equal(0, a11.TerminalSessionCount); // Stalled is not terminal + Assert.Equal(1, Bucket(a11, "sweep:stalled", "Stalled").Count); + + var g10 = Assert.Single(rows, r => r.TenantId == "global" && r.Date == "2026-08-10"); + Assert.Equal(4, g10.SessionCount); + Assert.Equal(1, Bucket(g10, "agent:failed", "Failed").Count); + Assert.Equal(VerdictCalibrationDailyAggregate.CurrentVersion, g10.Version); + Assert.Equal(Now, g10.ComputedAt); + + // Deterministic ordering: rows by tenant then date, buckets by path then status. + Assert.Equal(rows.Select(r => (r.TenantId, r.Date)).OrderBy(k => k.TenantId, StringComparer.Ordinal).ThenBy(k => k.Date, StringComparer.Ordinal), rows.Select(r => (r.TenantId, r.Date))); + Assert.Equal(a10.Buckets.Select(b => b.VerdictPath).OrderBy(p => p, StringComparer.Ordinal), a10.Buckets.Select(b => b.VerdictPath)); + } + + [Fact] + public void Unstamped_rows_are_derived_and_counted_as_derived() + { + var d = new DateTime(2026, 8, 10, 9, 0, 0, DateTimeKind.Utc); + var sessions = new List + { + Session(TenantA, "a1", SessionStatus.Succeeded, d, path: null, completed: d.AddHours(1)), // legacy agent completion + Session(TenantA, "a2", SessionStatus.Incomplete, d, path: null, completed: d.AddHours(5), + failureReason: "No Device Setup completion or explicit failure signal observed before timeout"), + Session(TenantA, "a3", SessionStatus.Succeeded, d, path: "agent:complete", completed: d.AddHours(1)), + }; + var row = MaintenanceService.BuildVerdictCalibrationAggregates(sessions, new Dictionary>(), Now) + .Single(r => r.TenantId == TenantA); + + var complete = Bucket(row, "agent:complete", "Succeeded"); + Assert.Equal(2, complete.Count); + Assert.Equal(1, complete.DerivedCount); + var r6 = Bucket(row, "legacy:r6", "Incomplete"); + Assert.Equal(1, r6.Count); + Assert.Equal(1, r6.DerivedCount); + } + + [Fact] + public void Overrides_are_attributed_to_the_prior_path() + { + var d = new DateTime(2026, 8, 10, 9, 0, 0, DateTimeKind.Utc); + var sessions = new List + { + // sweep said Incomplete, agent later completed → late completion against sweep:r5_incomplete + Session(TenantA, "a1", SessionStatus.Succeeded, d, path: "agent:complete", priorPath: "sweep:r5_incomplete", priorStatus: "Incomplete", completed: d.AddHours(1)), + // admin flipped an agent success to Failed + Session(TenantA, "a2", SessionStatus.Failed, d, path: "manual:failed", priorPath: "agent:complete", priorStatus: "Succeeded", adminMarked: "Failed", completed: d.AddHours(1)), + // retro reclassification Failed → Incomplete + Session(TenantA, "a3", SessionStatus.Incomplete, d, path: "retro:r6", priorPath: "agent:failed", priorStatus: "Failed", completed: d.AddHours(1)), + // admin mark to Succeeded counts as admin, not late completion + Session(TenantA, "a4", SessionStatus.Succeeded, d, path: "manual:succeeded", priorPath: "sweep:r6", priorStatus: "Incomplete", adminMarked: "Succeeded", completed: d.AddHours(1)), + }; + var row = MaintenanceService.BuildVerdictCalibrationAggregates(sessions, new Dictionary>(), Now) + .Single(r => r.TenantId == TenantA); + + var priorSweep = Bucket(row, "sweep:r5_incomplete", "Incomplete"); + Assert.Equal(0, priorSweep.Count); // nobody is on it any more + Assert.Equal(1, priorSweep.OverriddenByLateCompletion); + + var priorAgent = Bucket(row, "agent:complete", "Succeeded"); + Assert.Equal(1, priorAgent.Count); // a1 is on it now + Assert.Equal(1, priorAgent.OverriddenByAdmin); // a2 used to be + + Assert.Equal(1, Bucket(row, "agent:failed", "Failed").OverriddenOther); + Assert.Equal(1, Bucket(row, "sweep:r6", "Incomplete").OverriddenByAdmin); + Assert.Equal(0, Bucket(row, "manual:succeeded", "Succeeded").OverriddenByAdmin); + } + + [Fact] + public void ReEnrollment_proxy_counts_only_eligible_terminal_sessions() + { + var old = Now.AddDays(-20); + var young = Now.AddDays(-2); + var sessions = new List + { + // old, re-enrolled 3 days later → eligible + re-enrolled + Session(TenantA, "a1", SessionStatus.Incomplete, old, path: "sweep:r6", completed: old.AddHours(5), serial: "SN-A"), + // old, next attempt 10 days later → eligible, not re-enrolled + Session(TenantA, "a2", SessionStatus.Failed, old, path: "agent:failed", completed: old.AddHours(1), serial: "SN-B"), + // old, no next attempt → eligible, not re-enrolled + Session(TenantA, "a3", SessionStatus.Succeeded, old, path: "agent:complete", completed: old.AddHours(1), serial: "SN-C"), + // too young to judge → not eligible even though the chain shows a follow-up + Session(TenantA, "a4", SessionStatus.Failed, young, path: "agent:failed", completed: young.AddHours(1), serial: "SN-D"), + // non-terminal → never eligible + Session(TenantA, "a5", SessionStatus.AwaitingUser, old, path: "sweep:r5_awaiting", serial: "SN-E"), + }; + var histories = new Dictionary> + { + [TenantA] = new() + { + History(TenantA, "SN-A", ("a1", old, old.AddHours(5), "Incomplete"), ("x1", old.AddDays(3), old.AddDays(3).AddHours(1), "Succeeded")), + History(TenantA, "SN-B", ("a2", old, old.AddHours(1), "Failed"), ("x2", old.AddDays(10), null, "Succeeded")), + History(TenantA, "SN-C", ("a3", old, old.AddHours(1), "Succeeded")), + History(TenantA, "SN-D", ("a4", young, young.AddHours(1), "Failed"), ("x4", young.AddHours(2), null, "Succeeded")), + }, + }; + var rows = MaintenanceService.BuildVerdictCalibrationAggregates(sessions, histories, Now); + + var oldRow = rows.Single(r => r.TenantId == TenantA && r.Date == old.ToString("yyyy-MM-dd")); + var r6 = Bucket(oldRow, "sweep:r6", "Incomplete"); + Assert.Equal((1, 1), (r6.Eligible7d, r6.ReEnrolled7d)); + var failed = Bucket(oldRow, "agent:failed", "Failed"); + Assert.Equal((1, 0), (failed.Eligible7d, failed.ReEnrolled7d)); + var complete = Bucket(oldRow, "agent:complete", "Succeeded"); + Assert.Equal((1, 0), (complete.Eligible7d, complete.ReEnrolled7d)); + var awaiting = Bucket(oldRow, "sweep:r5_awaiting", "AwaitingUser"); + Assert.Equal((0, 0), (awaiting.Eligible7d, awaiting.ReEnrolled7d)); + + var youngRow = rows.Single(r => r.TenantId == TenantA && r.Date == young.ToString("yyyy-MM-dd")); + var youngFailed = Bucket(youngRow, "agent:failed", "Failed"); + Assert.Equal((0, 0), (youngFailed.Eligible7d, youngFailed.ReEnrolled7d)); + } + + [Fact] + public void Sessions_inside_a_deletion_cascade_are_skipped() + { + var d = new DateTime(2026, 8, 10, 9, 0, 0, DateTimeKind.Utc); + var deleting = Session(TenantA, "a1", SessionStatus.Succeeded, d, path: "agent:complete"); + deleting.DeletionState = "Pending"; + var rows = MaintenanceService.BuildVerdictCalibrationAggregates(new[] { deleting }, new Dictionary>(), Now); + Assert.Empty(rows); + } + + // ---- Persistence round-trip ---- + + [Fact] + public void Aggregate_round_trips_through_the_table_entity() + { + var aggregate = new VerdictCalibrationDailyAggregate + { + TenantId = TenantA, Date = "2026-08-10", Version = 1, SessionCount = 5, TerminalSessionCount = 4, ComputedAt = Now, + Buckets = + { + new VerdictCalibrationBucket { VerdictPath = "agent:complete", Status = "Succeeded", Count = 3, DerivedCount = 1, Eligible7d = 3, ReEnrolled7d = 1, OverriddenByAdmin = 1, OverriddenByLateCompletion = 0, OverriddenOther = 2 }, + new VerdictCalibrationBucket { VerdictPath = "sweep:r6", Status = "Incomplete", Count = 1 }, + }, + }; + var entity = TableStorageService.BuildVerdictCalibrationAggregateEntity(aggregate); + Assert.Equal(TenantA, entity.PartitionKey); + Assert.Equal("2026-08-10", entity.RowKey); + + var back = TableStorageService.MapToVerdictCalibrationAggregate(entity); + Assert.Equal(JsonSerializer.Serialize(aggregate), JsonSerializer.Serialize(back)); + } + + // ---- Response builder ---- + + private static VerdictCalibrationDailyAggregate Day(string tenant, DateTime date, int sessions, params (string Path, string Status, int Count)[] buckets) => new() + { + TenantId = tenant, Date = date.ToString("yyyy-MM-dd"), SessionCount = sessions, TerminalSessionCount = sessions, Version = 1, ComputedAt = Now, + Buckets = buckets.Select(b => new VerdictCalibrationBucket { VerdictPath = b.Path, Status = b.Status, Count = b.Count, Eligible7d = b.Count, ReEnrolled7d = b.Count / 5 }).ToList(), + }; + + // Same naming policy as the worker host (Program.cs: camelCase) so the assertions pin the wire shape. + private static readonly JsonSerializerOptions Wire = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; + + private static JsonElement BuildJson(IReadOnlyList daily, DateTime today, int days) + => JsonSerializer.SerializeToElement(VerdictCalibrationResponse.Build(daily, "global", today, days), Wire); + + [Fact] + public void Response_sums_the_window_and_computes_share_rate_and_trend() + { + var today = new DateTime(2026, 8, 23, 0, 0, 0, DateTimeKind.Utc); + var daily = new List(); + // 35 days: baseline days carry 10 sessions (1 sweep:r6), the last 7 days carry 10 sessions (3 sweep:r6) → share 10% → 30%, lift 3.0 + for (var i = 34; i >= 0; i--) + { + var date = today.AddDays(-i); + var r6 = i < 7 ? 3 : 1; + daily.Add(Day("global", date, 10, ("agent:complete", "Succeeded", 10 - r6), ("sweep:r6", "Incomplete", r6))); + } + var json = BuildJson(daily, today, days: 30); + + Assert.True(json.GetProperty("success").GetBoolean()); + Assert.Equal(30, json.GetProperty("windowDays").GetInt32()); + Assert.Equal(300, json.GetProperty("totals").GetProperty("sessions").GetInt32()); + Assert.Equal(30, json.GetProperty("totals").GetProperty("days").GetInt32()); + + var paths = json.GetProperty("paths").EnumerateArray().ToList(); + var r6Row = paths.Single(p => p.GetProperty("verdictPath").GetString() == "sweep:r6"); + Assert.Equal(7 * 3 + 23 * 1, r6Row.GetProperty("count").GetInt32()); + Assert.Equal(Math.Round(100.0 * 44 / 300, 1), r6Row.GetProperty("sharePct").GetDouble()); + Assert.Equal(30.0, r6Row.GetProperty("window7").GetProperty("sharePct").GetDouble()); + Assert.Equal(10.0, r6Row.GetProperty("baseline28").GetProperty("sharePct").GetDouble()); + Assert.Equal(3.0, r6Row.GetProperty("lift").GetDouble()); + // Eligible 44 ≥ 20 → rate stated; ReEnrolled7d = count/5 per day → 7*0 + 23*0 = 0 → 0% + Assert.Equal(0.0, r6Row.GetProperty("reEnrollRatePct").GetDouble()); + + // Sorted by count desc → agent:complete first. + Assert.Equal("agent:complete", paths[0].GetProperty("verdictPath").GetString()); + } + + [Fact] + public void Response_withholds_rate_below_twenty_eligible_and_lift_without_baseline() + { + var today = new DateTime(2026, 8, 23, 0, 0, 0, DateTimeKind.Utc); + var daily = new List + { + Day("global", today, 5, ("maxlife:r3", "Incomplete", 5)), + }; + var json = BuildJson(daily, today, days: 7); + var row = json.GetProperty("paths").EnumerateArray().Single(); + Assert.Equal(JsonValueKind.Null, row.GetProperty("reEnrollRatePct").ValueKind); + Assert.Equal(JsonValueKind.Null, row.GetProperty("lift").ValueKind); // no baseline rows → never invented + Assert.Equal(100.0, row.GetProperty("sharePct").GetDouble()); + } + + [Fact] + public void Response_keeps_a_zero_count_bucket_that_carries_overrides() + { + var today = new DateTime(2026, 8, 23, 0, 0, 0, DateTimeKind.Utc); + var day = Day("global", today, 1, ("agent:complete", "Succeeded", 1)); + day.Buckets.Add(new VerdictCalibrationBucket { VerdictPath = "sweep:r5_incomplete", Status = "Incomplete", Count = 0, OverriddenByLateCompletion = 1 }); + var json = BuildJson(new[] { day }, today, days: 7); + var rows = json.GetProperty("paths").EnumerateArray().ToList(); + var prior = rows.Single(p => p.GetProperty("verdictPath").GetString() == "sweep:r5_incomplete"); + Assert.Equal(0, prior.GetProperty("count").GetInt32()); + Assert.Equal(1, prior.GetProperty("overriddenByLateCompletion").GetInt32()); + } + + [Fact] + public void Response_is_empty_but_well_formed_before_the_first_sweep() + { + var json = BuildJson(Array.Empty(), new DateTime(2026, 8, 23), 30); + Assert.True(json.GetProperty("success").GetBoolean()); + Assert.Empty(json.GetProperty("paths").EnumerateArray()); + Assert.Equal(0, json.GetProperty("totals").GetProperty("sessions").GetInt32()); + Assert.Equal(JsonValueKind.Null, json.GetProperty("computedAt").ValueKind); + } + + [Theory] + [InlineData(null, 30)] + [InlineData("abc", 30)] + [InlineData("0", 1)] + [InlineData("500", 180)] + [InlineData("14", 14)] + public void ClampDays_bounds_the_window(string? raw, int expected) + => Assert.Equal(expected, VerdictCalibrationResponse.ClampDays(raw)); +} diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMetricsRepository.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMetricsRepository.cs index fb7ba235..dc0a5d9b 100644 --- a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMetricsRepository.cs +++ b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMetricsRepository.cs @@ -159,5 +159,17 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage public Task DeleteDeviceJourneyAggregatesOlderThanAsync(DateTime cutoffDate) => _storage.DeleteDeviceJourneyAggregatesOlderThanAsync(cutoffDate); + + public Task SaveVerdictCalibrationAggregateAsync(VerdictCalibrationDailyAggregate aggregate) + => _storage.SaveVerdictCalibrationAggregateAsync(aggregate); + + public Task> GetVerdictCalibrationAggregatesAsync(string tenantId, DateTime startDate, DateTime endDate) + => _storage.GetVerdictCalibrationAggregatesAsync(tenantId, startDate, endDate); + + public Task DeleteVerdictCalibrationAggregateAsync(string tenantId, string dateKey) + => _storage.DeleteVerdictCalibrationAggregateAsync(tenantId, dateKey); + + public Task DeleteVerdictCalibrationAggregatesOlderThanAsync(DateTime cutoffDate) + => _storage.DeleteVerdictCalibrationAggregatesOlderThanAsync(cutoffDate); } } diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetVerdictCalibrationFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetVerdictCalibrationFunction.cs new file mode 100644 index 00000000..e5d50a6b --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetVerdictCalibrationFunction.cs @@ -0,0 +1,220 @@ +using System.Globalization; +using System.Net; +using AutopilotMonitor.Functions.Helpers; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Functions.Metrics +{ + /// + /// Operator-only verdict calibration matrix (docs/backend/verdict-calibration.md): per verdict + /// path, how many sessions it produced in the window, its share, how often it was overridden + /// (admin / late completion / other), the 7-day re-enrollment proxy, and a 7d-vs-28d trend. + /// Platform scope ("global" partition) or one tenant via ?tenantId=. Serves the admin + /// metrics page and the get_verdict_calibration MCP tool. Never customer-facing — + /// these are platform-internal classifier diagnostics, not product features. + /// + public class GetVerdictCalibrationFunction + { + private readonly ILogger _logger; + private readonly IMetricsRepository _metricsRepo; + + public GetVerdictCalibrationFunction(ILogger logger, IMetricsRepository metricsRepo) + { + _logger = logger; + _metricsRepo = metricsRepo; + } + + [Function("GetVerdictCalibration")] + public async Task Run( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "global/metrics/verdict-calibration")] HttpRequestData req) + { + try + { + // Authentication + GlobalReadOrAdmin authorization enforced by PolicyEnforcementMiddleware + var query = System.Web.HttpUtility.ParseQueryString(req.Url.Query); + var tenantIdFilter = query["tenantId"]; + var days = VerdictCalibrationResponse.ClampDays(query["days"]); + var partition = "global"; + if (!string.IsNullOrWhiteSpace(tenantIdFilter)) + { + if (!Guid.TryParse(tenantIdFilter, out var parsed)) + { + var bad = req.CreateResponse(HttpStatusCode.BadRequest); + await bad.WriteAsJsonAsync(new { success = false, message = "tenantId must be a GUID" }); + return bad; + } + partition = parsed.ToString("D"); + } + + var today = DateTime.UtcNow.Date; + // One read covers both the requested window and the trend horizon (7d window + 28d baseline). + var readDays = Math.Max(days, VerdictCalibrationResponse.TrendHorizonDays); + var daily = await _metricsRepo.GetVerdictCalibrationAggregatesAsync( + partition, VerdictCalibrationResponse.InclusiveWindowStart(today, readDays), today); + + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteAsJsonAsync(VerdictCalibrationResponse.Build(daily, partition, today, days)); + return response; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error fetching verdict calibration"); + var errorResponse = req.CreateResponse(HttpStatusCode.InternalServerError); + await errorResponse.WriteAsJsonAsync(new { success = false, message = "Internal server error" }); + return errorResponse; + } + } + } + + /// Pure response builder — internal static so the matrix arithmetic is pinned by tests. + internal static class VerdictCalibrationResponse + { + internal const int DefaultWindowDays = 30; + internal const int MaxWindowDays = 180; // aggregate retention + internal const int TrendWindowDays = 7; + internal const int TrendBaselineDays = 28; + internal const int TrendHorizonDays = TrendWindowDays + TrendBaselineDays; + + internal static int ClampDays(string? raw) + { + if (!int.TryParse(raw, out var days)) return DefaultWindowDays; + return Math.Clamp(days, 1, MaxWindowDays); + } + + /// "Last N days" = exactly N calendar day keys including today (both range ends inclusive). + internal static DateTime InclusiveWindowStart(DateTime today, int days) => today.AddDays(-(days - 1)); + + internal sealed class PathRow + { + public string VerdictPath { get; set; } = string.Empty; + public string Status { get; set; } = string.Empty; + public int Count { get; set; } + public double SharePct { get; set; } + public int DerivedCount { get; set; } + public int Eligible7d { get; set; } + public int ReEnrolled7d { get; set; } + /// Null below the minimum eligible count — never a rate on a handful of sessions. + public double? ReEnrollRatePct { get; set; } + public int OverriddenByAdmin { get; set; } + public int OverriddenByLateCompletion { get; set; } + public int OverriddenOther { get; set; } + public TrendWindow Window7 { get; set; } = new(); + public TrendWindow Baseline28 { get; set; } = new(); + /// Window share ÷ baseline share; null when the baseline share is 0 (a new path has no finite lift — never invented). + public double? Lift { get; set; } + } + + internal sealed class TrendWindow + { + public int Count { get; set; } + public int Sessions { get; set; } + public double SharePct { get; set; } + } + + /// Minimum eligible sessions before a re-enrollment rate is stated (truthfulness: no rate on n<20). + internal const int MinEligibleForRate = 20; + + internal static object Build(IReadOnlyList daily, string partition, DateTime today, int days) + { + var windowStart = Key(InclusiveWindowStart(today, days)); + var trendWindowStart = Key(InclusiveWindowStart(today, TrendWindowDays)); + var baselineStart = Key(InclusiveWindowStart(today, TrendHorizonDays)); + + var rows = new Dictionary<(string Path, string Status), PathRow>(); + int sessions = 0, terminal = 0, derived = 0; + int trendSessions = 0, baselineSessions = 0; + DateTime? computedAt = null; + var versions = new HashSet(); + + PathRow Row(VerdictCalibrationBucket b) + { + var key = (b.VerdictPath, b.Status); + if (!rows.TryGetValue(key, out var row)) + { + row = new PathRow { VerdictPath = b.VerdictPath, Status = b.Status }; + rows[key] = row; + } + return row; + } + + foreach (var day in daily) + { + var inWindow = string.CompareOrdinal(day.Date, windowStart) >= 0; + var inTrendWindow = string.CompareOrdinal(day.Date, trendWindowStart) >= 0; + var inBaseline = !inTrendWindow && string.CompareOrdinal(day.Date, baselineStart) >= 0; + + if (inWindow) + { + sessions += day.SessionCount; + terminal += day.TerminalSessionCount; + versions.Add(day.Version); + if (computedAt == null || day.ComputedAt > computedAt) computedAt = day.ComputedAt; + } + if (inTrendWindow) trendSessions += day.SessionCount; + if (inBaseline) baselineSessions += day.SessionCount; + + foreach (var b in day.Buckets) + { + // Overrides are attributed to the prior path and can exist on a bucket with Count=0. + var row = Row(b); + if (inWindow) + { + row.Count += b.Count; + row.DerivedCount += b.DerivedCount; + row.Eligible7d += b.Eligible7d; + row.ReEnrolled7d += b.ReEnrolled7d; + row.OverriddenByAdmin += b.OverriddenByAdmin; + row.OverriddenByLateCompletion += b.OverriddenByLateCompletion; + row.OverriddenOther += b.OverriddenOther; + derived += b.DerivedCount; + } + if (inTrendWindow) row.Window7.Count += b.Count; + if (inBaseline) row.Baseline28.Count += b.Count; + } + } + + var list = rows.Values + .Where(r => r.Count > 0 || r.OverriddenByAdmin + r.OverriddenByLateCompletion + r.OverriddenOther > 0) + .OrderByDescending(r => r.Count) + .ThenBy(r => r.VerdictPath, StringComparer.Ordinal) + .ThenBy(r => r.Status, StringComparer.Ordinal) + .ToList(); + + foreach (var r in list) + { + r.SharePct = Pct(r.Count, sessions); + r.ReEnrollRatePct = r.Eligible7d >= MinEligibleForRate ? Pct(r.ReEnrolled7d, r.Eligible7d) : null; + r.Window7.Sessions = trendSessions; + r.Window7.SharePct = Pct(r.Window7.Count, trendSessions); + r.Baseline28.Sessions = baselineSessions; + r.Baseline28.SharePct = Pct(r.Baseline28.Count, baselineSessions); + r.Lift = r.Baseline28.SharePct > 0 && trendSessions > 0 + ? Math.Round(r.Window7.SharePct / r.Baseline28.SharePct, 2) + : null; + } + + return new + { + success = true, + tenantId = partition, + windowDays = days, + windowStart, + windowEnd = Key(today), + computedAt, + versions = versions.OrderBy(v => v).ToArray(), + totals = new { sessions, terminal, derived, days = daily.Count(d => string.CompareOrdinal(d.Date, windowStart) >= 0) }, + trend = new { windowDays = TrendWindowDays, baselineDays = TrendBaselineDays, windowSessions = trendSessions, baselineSessions }, + paths = list, + alerts = Array.Empty(), + }; + } + + private static string Key(DateTime d) => d.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture); + + private static double Pct(int part, int whole) => whole <= 0 ? 0 : Math.Round(100.0 * part / whole, 1); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs b/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs index bc67d109..e7c568d7 100644 --- a/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs +++ b/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs @@ -464,6 +464,7 @@ public static class EndpointAccessPolicyCatalog new("GET", "global/metrics/fleet-health", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam), new("GET", "global/metrics/time-attribution", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam), new("GET", "global/metrics/device-journeys", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam), + new("GET", "global/metrics/verdict-calibration", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam), new("GET", "global/apps/list", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam), new("GET", "global/apps/{appName}/analytics", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam), new("GET", "global/apps/{appName}/sessions", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam), diff --git a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.Aggregation.cs b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.Aggregation.cs index 73782bc9..12af6b19 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.Aggregation.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.Aggregation.cs @@ -1102,6 +1102,19 @@ namespace AutopilotMonitor.Functions.Services { _logger.LogWarning(ex, "Failed to cleanup old device-journey aggregates"); } + + // Verdict-calibration daily aggregates: same 180d window (regenerable; the radar + // needs 35d, the matrix serves up to 180d). + try + { + var deleted = await _metricsRepo.DeleteVerdictCalibrationAggregatesOlderThanAsync(now.AddDays(-usageMetricsRetentionDays)); + if (deleted > 0) + _logger.LogInformation("Verdict-calibration aggregate cleanup: deleted {Count} rows older than {Days} days", deleted, usageMetricsRetentionDays); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to cleanup old verdict-calibration aggregates"); + } } /// /// Detects and cleans up orphaned events — events stored in the Events table diff --git a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.VerdictCalibration.cs b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.VerdictCalibration.cs new file mode 100644 index 00000000..e82c4413 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.VerdictCalibration.cs @@ -0,0 +1,259 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.Linq; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Helpers; +using AutopilotMonitor.Shared.Models; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Services +{ + /// + /// Partial: verdict-calibration daily aggregates (docs/backend/verdict-calibration.md) — the + /// operator thermometer for the rule classifier: which verdict path produced how many + /// sessions per day, how many of those were later overridden (admin / late completion / + /// other), and how many devices re-enrolled within 7 days. + /// + public partial class MaintenanceService + { + /// App setting kill switch: "true" skips the sweep. Fail-open — the sweep only writes its own regenerable table. + internal const string VerdictCalibrationKillSwitchSetting = "VerdictCalibrationSweepDisabled"; + + /// Rolling window: sessions started in the last 30 days (same as the F1/F2 sweeps). + private const int VerdictCalibrationSweepDays = 30; + + /// Re-enrollment proxy horizon: another terminal session of the same device within this many days. + internal const int VerdictCalibrationReEnrollDays = 7; + + /// + /// Recomputes the verdict-calibration rows for the rolling window — idempotent per + /// (tenant × StartedAt date) plus "global" mirror rows. Reads the window's sessions (one + /// cross-partition scan, same as the F1/F2 sweeps) and each window tenant's device + /// histories (one partition scan per tenant) for the re-enrollment proxy. Stale date rows + /// a run did not regenerate are deleted so window sums never serve ghost counts. + /// + private async Task SweepVerdictCalibrationAsync() + { + try + { + if (string.Equals(_configuration[VerdictCalibrationKillSwitchSetting], "true", StringComparison.OrdinalIgnoreCase)) + { + _logger.LogInformation("Verdict-calibration sweep skipped ({Setting}=true)", VerdictCalibrationKillSwitchSetting); + return; + } + + var sw = Stopwatch.StartNew(); + var now = DateTime.UtcNow; + var today = now.Date; + var windowStart = today.AddDays(-VerdictCalibrationSweepDays); + var windowEnd = today.AddDays(1); + + var sessions = await _maintenanceRepo.GetSessionsByDateRangeAsync(windowStart, windowEnd); + + var historiesByTenant = new Dictionary>(StringComparer.OrdinalIgnoreCase); + foreach (var tenantId in sessions.Select(s => s.TenantId).Distinct(StringComparer.OrdinalIgnoreCase)) + { + historiesByTenant[tenantId] = await _metricsRepo.GetDeviceHistoriesByTenantAsync(tenantId); + } + + var aggregates = BuildVerdictCalibrationAggregates(sessions, historiesByTenant, now); + var saved = 0; + foreach (var aggregate in aggregates) + { + if (await _metricsRepo.SaveVerdictCalibrationAggregateAsync(aggregate)) + saved++; + } + + var generatedKeys = new HashSet<(string TenantId, string Date)>(aggregates.Select(a => (a.TenantId, a.Date))); + var removedStale = 0; + foreach (var partition in aggregates.Select(a => a.TenantId).Distinct().ToList()) + { + var existingRows = await _metricsRepo.GetVerdictCalibrationAggregatesAsync(partition, windowStart, windowEnd.AddDays(-1)); + foreach (var row in existingRows) + { + if (generatedKeys.Contains((partition, row.Date))) continue; + await _metricsRepo.DeleteVerdictCalibrationAggregateAsync(partition, row.Date); + removedStale++; + } + } + + sw.Stop(); + _logger.LogInformation( + "Verdict-calibration sweep: {Sessions} window sessions across {Tenants} tenants, {Rows} rows written, {Removed} stale rows removed in {Ms}ms", + sessions.Count, historiesByTenant.Count, saved, removedStale, sw.ElapsedMilliseconds); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Verdict-calibration sweep failed (non-fatal)"); + } + } + + /// + /// Pure aggregation core behind — internal + /// static so the contract is pinned by unit tests. Per session: the current + /// (VerdictPath, Status) bucket gets Count (+ DerivedCount when the path was + /// derived read-side), terminal sessions old enough get Eligible7d and, if the + /// device's history chain shows another terminal session starting within 7 days of this + /// one's end, ReEnrolled7d; a session carrying a prior verdict additionally + /// increments the PRIOR bucket's Overridden* counter. Sessions inside a deletion + /// cascade are skipped. Rows are emitted per (tenant, StartedAt date) and mirrored into + /// the "global" partition; buckets are sorted for deterministic output. + /// + internal static List BuildVerdictCalibrationAggregates( + IReadOnlyList sessions, + IReadOnlyDictionary> historiesByTenant, + DateTime nowUtc) + { + var nextStartBySession = BuildNextStartLookup(historiesByTenant); + var eligibleCutoff = nowUtc.AddDays(-VerdictCalibrationReEnrollDays); + + var rows = new Dictionary<(string TenantId, string Date), RowAccumulator>(); + + RowAccumulator Row(string tenantId, string date) + { + var key = (tenantId, date); + if (!rows.TryGetValue(key, out var acc)) + { + acc = new RowAccumulator(); + rows[key] = acc; + } + return acc; + } + + foreach (var session in sessions) + { + if (session == null) continue; + if (!string.IsNullOrEmpty(session.DeletionState) && session.DeletionState != "None") + continue; + + var date = session.StartedAt.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture); + var (path, derived) = VerdictPathDerivation.Derive(session); + var statusName = session.Status.ToString(); + var isTerminal = DeviceJourneyCalculator.IsTerminal(session.Status); + + var eligible = false; + var reEnrolled = false; + if (isTerminal) + { + var end = session.CompletedAt ?? session.LastEventAt ?? session.StartedAt; + eligible = end <= eligibleCutoff; + if (eligible + && nextStartBySession.TryGetValue((session.TenantId, session.SessionId), out var next) + && next.NextStart.HasValue + && (next.NextStart.Value - next.End) < TimeSpan.FromDays(VerdictCalibrationReEnrollDays)) + { + reEnrolled = true; + } + } + + var overridden = ClassifyOverride(session); + + foreach (var partition in new[] { session.TenantId, "global" }) + { + var acc = Row(partition, date); + acc.SessionCount++; + if (isTerminal) acc.TerminalSessionCount++; + + var bucket = acc.Bucket(path, statusName); + bucket.Count++; + if (derived) bucket.DerivedCount++; + if (eligible) bucket.Eligible7d++; + if (reEnrolled) bucket.ReEnrolled7d++; + + if (overridden.HasValue) + { + var prior = acc.Bucket(session.PriorVerdictPath!, session.PriorStatus!); + switch (overridden.Value) + { + case OverrideKind.Admin: prior.OverriddenByAdmin++; break; + case OverrideKind.LateCompletion: prior.OverriddenByLateCompletion++; break; + default: prior.OverriddenOther++; break; + } + } + } + } + + return rows + .OrderBy(r => r.Key.TenantId, StringComparer.Ordinal) + .ThenBy(r => r.Key.Date, StringComparer.Ordinal) + .Select(r => new VerdictCalibrationDailyAggregate + { + TenantId = r.Key.TenantId, + Date = r.Key.Date, + Version = VerdictCalibrationDailyAggregate.CurrentVersion, + SessionCount = r.Value.SessionCount, + TerminalSessionCount = r.Value.TerminalSessionCount, + Buckets = r.Value.Buckets.Values + .OrderBy(b => b.VerdictPath, StringComparer.Ordinal) + .ThenBy(b => b.Status, StringComparer.Ordinal) + .ToList(), + ComputedAt = nowUtc, + }) + .ToList(); + } + + private enum OverrideKind { Admin, LateCompletion, Other } + + /// + /// A session whose current status overrode a prior verdict names the writer that did it: + /// an admin mark (AdminMarkedAction set), a late agent completion (now Succeeded without + /// admin involvement), or anything else (retro-reclassification, grace expiry, supersede). + /// + private static OverrideKind? ClassifyOverride(SessionSummary session) + { + if (string.IsNullOrEmpty(session.PriorVerdictPath) || string.IsNullOrEmpty(session.PriorStatus)) + return null; + if (!string.IsNullOrEmpty(session.AdminMarkedAction)) + return OverrideKind.Admin; + if (session.Status == SessionStatus.Succeeded) + return OverrideKind.LateCompletion; + return OverrideKind.Other; + } + + /// + /// (tenant, sessionId) → this session's chain end and the next chain entry's StartedAt — + /// the re-enrollment proxy source. Chains are ascending by StartedAt and capped at 20, so + /// a window session older than the cap simply has no entry (counts as not re-enrolled + /// only when eligible — the cap is a known, small bias toward under-counting). + /// + internal static Dictionary<(string TenantId, string SessionId), (DateTime End, DateTime? NextStart)> BuildNextStartLookup( + IReadOnlyDictionary> historiesByTenant) + { + var lookup = new Dictionary<(string, string), (DateTime, DateTime?)>(); + foreach (var pair in historiesByTenant) + { + foreach (var history in pair.Value) + { + var chain = history.Chain; + for (var i = 0; i < chain.Count; i++) + { + var end = chain[i].CompletedAt ?? chain[i].StartedAt; + DateTime? nextStart = i + 1 < chain.Count ? chain[i + 1].StartedAt : null; + lookup[(pair.Key, chain[i].SessionId)] = (end, nextStart); + } + } + } + return lookup; + } + + private sealed class RowAccumulator + { + public int SessionCount; + public int TerminalSessionCount; + public readonly Dictionary<(string Path, string Status), VerdictCalibrationBucket> Buckets = new(); + + public VerdictCalibrationBucket Bucket(string path, string status) + { + var key = (path, status); + if (!Buckets.TryGetValue(key, out var bucket)) + { + bucket = new VerdictCalibrationBucket { VerdictPath = path, Status = status }; + Buckets[key] = bucket; + } + return bucket; + } + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs index 3a640a29..583c742d 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs @@ -158,6 +158,10 @@ namespace AutopilotMonitor.Functions.Services // F2 PR4: device-history chain heal (incl. deleted-session ref cleanup) + daily // FTR aggregates over the same rolling window. Fail-soft internally. await SweepDeviceJourneysAsync(); + // Verdict calibration: per-verdict-path daily buckets over the same rolling + // window, AFTER the device-journey sweep so the re-enrollment proxy reads + // freshly merged chains. Fail-soft internally. + await SweepVerdictCalibrationAsync(); // Plan §5 PR6 / §16 R14: session retention fanout extracted out of the 2h timer // into the dedicated 12h SessionDeletionMaintenanceFunction so cascade-lifecycle // work has independent cadence + kill-switch + OpsEvent watchdogs. The non-session @@ -232,6 +236,7 @@ namespace AutopilotMonitor.Functions.Services // (rolling 30d windows, cheap once converged). await SweepTimeAttributionAsync(); await SweepDeviceJourneysAsync(); + await SweepVerdictCalibrationAsync(); if (!aggregateOnly) { diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/TenantOffboardingHandler.cs b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/TenantOffboardingHandler.cs index 782bcc81..c0dfd44d 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/TenantOffboardingHandler.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/TenantOffboardingHandler.cs @@ -83,6 +83,8 @@ namespace AutopilotMonitor.Functions.Services.Offboarding // whole-row deletion besides chain-emptying. Constants.TableNames.DeviceHistories, Constants.TableNames.DeviceJourneyAggregates, + // Verdict calibration daily buckets (PK=TenantId) — regenerable from Sessions, wiped with the tenant. + Constants.TableNames.VerdictCalibrationAggregates, // Pre-write config snapshots (PK=tenantId). Wiped in the bulk phase; the // TenantConfiguration row itself is deleted LAST via SafeWipe (a delete, not a // save), so the wipe cannot re-create a snapshot afterwards. The admin-config diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.VerdictCalibration.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.VerdictCalibration.cs new file mode 100644 index 00000000..b8aaa992 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.VerdictCalibration.cs @@ -0,0 +1,125 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Threading.Tasks; +using Azure.Data.Tables; +using AutopilotMonitor.Functions.Security; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.Models; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Services +{ + /// + /// Partial: verdict-calibration daily aggregate persistence (docs/backend/verdict-calibration.md). + /// VerdictCalibrationAggregates: PK=TenantId ("global" for the cross-tenant row), + /// RK="{yyyy-MM-dd}" — one row per (tenant, StartedAt date) holding the per-verdict-path + /// buckets as a JSON column. Recomputed whole (Replace) by the maintenance sweep, 180d + /// retention, regenerable from Sessions + DeviceHistories. + /// + public partial class TableStorageService + { + public async Task SaveVerdictCalibrationAggregateAsync(VerdictCalibrationDailyAggregate aggregate) + { + try + { + var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.VerdictCalibrationAggregates); + await tableClient.UpsertEntityAsync(BuildVerdictCalibrationAggregateEntity(aggregate), TableUpdateMode.Replace); + return true; + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to save verdict-calibration aggregate {Date} for tenant {TenantId}", + aggregate.Date, aggregate.TenantId); + return false; + } + } + + public async Task> GetVerdictCalibrationAggregatesAsync( + string tenantId, DateTime startDate, DateTime endDate) + { + if (tenantId != "global") + SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); + try + { + var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.VerdictCalibrationAggregates); + var filter = $"PartitionKey eq '{tenantId}' and RowKey ge '{startDate:yyyy-MM-dd}' and RowKey le '{endDate:yyyy-MM-dd}'"; + var results = new List(); + await foreach (var entity in tableClient.QueryAsync(filter: filter)) + { + results.Add(MapToVerdictCalibrationAggregate(entity)); + } + return results; + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to query verdict-calibration aggregates for tenant {TenantId}", tenantId); + return new List(); + } + } + + // internal static: pinned by round-trip tests together with MapToVerdictCalibrationAggregate. + internal static TableEntity BuildVerdictCalibrationAggregateEntity(VerdictCalibrationDailyAggregate aggregate) + { + return new TableEntity(aggregate.TenantId, aggregate.Date) + { + ["Date"] = aggregate.Date, + ["Version"] = aggregate.Version, + ["SessionCount"] = aggregate.SessionCount, + ["TerminalSessionCount"] = aggregate.TerminalSessionCount, + ["BucketsJson"] = JsonSerializer.Serialize(aggregate.Buckets), + ["ComputedAt"] = EnsureUtc(aggregate.ComputedAt), + }; + } + + internal static VerdictCalibrationDailyAggregate MapToVerdictCalibrationAggregate(TableEntity entity) + { + return new VerdictCalibrationDailyAggregate + { + TenantId = entity.PartitionKey, + Date = entity.GetString("Date") ?? entity.RowKey, + Version = entity.GetInt32("Version") ?? 0, + SessionCount = entity.GetInt32("SessionCount") ?? 0, + TerminalSessionCount = entity.GetInt32("TerminalSessionCount") ?? 0, + Buckets = DeserializeJsonColumn(entity.GetString("BucketsJson")), + ComputedAt = entity.GetDateTimeOffset("ComputedAt")?.UtcDateTime ?? DateTime.MinValue, + }; + } + + public async Task DeleteVerdictCalibrationAggregateAsync(string tenantId, string dateKey) + { + try + { + var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.VerdictCalibrationAggregates); + await tableClient.DeleteEntityAsync(tenantId, dateKey); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to delete stale verdict-calibration aggregate {Date} for tenant {TenantId}", + dateKey, tenantId); + } + } + + public async Task DeleteVerdictCalibrationAggregatesOlderThanAsync(DateTime cutoffDate) + { + try + { + var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.VerdictCalibrationAggregates); + var filter = $"RowKey lt '{cutoffDate:yyyy-MM-dd}'"; + var deleted = 0; + await foreach (var entity in tableClient.QueryAsync( + filter: filter, select: new[] { "PartitionKey", "RowKey" })) + { + await tableClient.DeleteEntityAsync(entity.PartitionKey, entity.RowKey); + deleted++; + } + return deleted; + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to delete old verdict-calibration aggregates"); + return 0; + } + } + } +} diff --git a/src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-catalog-order.test.ts b/src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-catalog-order.test.ts index c3abefad..06d18ce0 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-catalog-order.test.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-catalog-order.test.ts @@ -136,6 +136,9 @@ describe('role catalog snapshot — privilege-leak guard', () => { // routing to the GlobalReadOrAdmin variant — same placement as get_app_install_metrics. 'get_time_attribution', 'get_usage_metrics', + // Platform scope (GA + Global Reader): GlobalReadOrAdmin endpoint, operator-only + // classifier diagnostics. + 'get_verdict_calibration', 'get_vulnerability_summary', 'list_blocked_devices', 'list_session_annotations', @@ -197,6 +200,7 @@ describe('role catalog snapshot — privilege-leak guard', () => { 'get_platform_metrics', 'get_tenant_config', 'get_tenant_config_schema', + 'get_verdict_calibration', 'list_blocked_devices', 'list_session_annotations', 'list_session_reports', diff --git a/src/McpServer/autopilot-monitor-mcp/src/tools/admin.ts b/src/McpServer/autopilot-monitor-mcp/src/tools/admin.ts index 22fc70e4..40e7e175 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/tools/admin.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/tools/admin.ts @@ -838,6 +838,45 @@ export function registerAdminTools(server: McpServer, ga: boolean, strictGa: boo }) ); + // Tool: get_verdict_calibration — platform scope (GA + Global Reader). Operator diagnostics for + // the session-verdict classifier (docs/backend/verdict-calibration.md) — never a customer surface. + if (ga) server.registerTool( + 'get_verdict_calibration', + { + title: 'Verdict Calibration (classifier thermometer)', + description: + 'Answer "which code path produced our session verdicts, and does any of them look wrong?". Per verdict ' + + 'path (VerdictPaths vocabulary — agent:complete, agent:failed, sweep:r5_incomplete, maxlife:r6, ' + + 'late:r4, retro:r6, rule:, manual:failed, register:superseded, legacy:* for pre-instrumentation ' + + 'rows derived read-side): session count and share in the window, derivedCount (rows attributed by ' + + 'derivation — weaker evidence), the 7-day re-enrollment proxy (eligible7d = terminal sessions old ' + + 'enough to judge, reEnrolled7d = the same device registered another terminal session within 7 days; ' + + 'reEnrollRatePct is null below 20 eligible), the correction stream attributed to the PRIOR path ' + + '(overriddenByAdmin / overriddenByLateCompletion / overriddenOther — a path that is frequently ' + + 'overridden was wrong), and a 7d-vs-28d share trend (window7, baseline28, lift; lift null without ' + + 'baseline). Read it as calibration: a Succeeded path whose re-enrollment rate sits at Failed level ' + + 'reconciles too boldly; an Incomplete path whose rate matches the Succeeded background is too ' + + 'cautious; a rising sweep:*/maxlife:* share is an agent-liveness signal, not a classifier one. ' + + 'Omit tenantId for the cross-tenant aggregate, or pass tenantId to scope to one tenant. Rows come ' + + 'from the 2-hourly maintenance sweep (computedAt); empty before its first run.', + inputSchema: { + tenantId: z.string().optional().describe('Optional — scope to one tenant. Omit for the cross-tenant aggregate.'), + days: z.coerce.number().int().min(1).max(180).optional() + .describe('Window in days (default 30, max 180 — aggregate retention). The 7d/28d trend is always today-anchored.'), + }, + annotations: READ_ONLY, + }, + async (args) => withToolTelemetry('get_verdict_calibration', args, async () => { + try { + const { tenantId, days } = args; + const data = await apiFetch(`/api/global/metrics/verdict-calibration${buildQuery({ tenantId, days })}`); + return toolResultText(data, MAX_RESULT_SIZE_CHARS.adminStream); + } catch (error: unknown) { + return toolError('get_verdict_calibration', args, error); + } + }) + ); + // Tool 18: list_session_reports — Global Admin only; not registered for normal users. if (ga) server.registerTool( 'list_session_reports', diff --git a/src/Shared/AutopilotMonitor.Shared/Constants.cs b/src/Shared/AutopilotMonitor.Shared/Constants.cs index 027e8e68..973e3775 100644 --- a/src/Shared/AutopilotMonitor.Shared/Constants.cs +++ b/src/Shared/AutopilotMonitor.Shared/Constants.cs @@ -988,6 +988,11 @@ namespace AutopilotMonitor.Shared // like UsageMetrics. public const string DeviceHistories = "DeviceHistories"; public const string DeviceJourneyAggregates = "DeviceJourneyAggregates"; + // VerdictCalibrationAggregates: PK=TenantId (or "global"), RK="{yyyy-MM-dd}" — daily + // per-verdict-path buckets (count, derived, re-enrolled-within-7d, overridden-by-admin/ + // late-completion) the operator calibration matrix + drift radar read + // (docs/backend/verdict-calibration.md). Additive across days; 180d retention; regenerable. + public const string VerdictCalibrationAggregates = "VerdictCalibrationAggregates"; // Configuration tables public const string TenantConfiguration = "TenantConfiguration"; @@ -1198,6 +1203,7 @@ namespace AutopilotMonitor.Shared TimeAttributionAggregates, DeviceHistories, DeviceJourneyAggregates, + VerdictCalibrationAggregates, TenantConfiguration, AdminConfiguration, GlobalAdmins, diff --git a/src/Shared/AutopilotMonitor.Shared/DataAccess/IMetricsRepository.cs b/src/Shared/AutopilotMonitor.Shared/DataAccess/IMetricsRepository.cs index 8d2c7e61..c58173b3 100644 --- a/src/Shared/AutopilotMonitor.Shared/DataAccess/IMetricsRepository.cs +++ b/src/Shared/AutopilotMonitor.Shared/DataAccess/IMetricsRepository.cs @@ -128,6 +128,14 @@ namespace AutopilotMonitor.Shared.DataAccess /// Deletes one daily FTR row — the sweep's stale-bucket reconcile. Task DeleteDeviceJourneyAggregateAsync(string tenantId, string dateKey); Task DeleteDeviceJourneyAggregatesOlderThanAsync(DateTime cutoffDate); + + // --- Verdict calibration (docs/backend/verdict-calibration.md) --- + Task SaveVerdictCalibrationAggregateAsync(VerdictCalibrationDailyAggregate aggregate); + /// Daily verdict-path rows of one tenant partition ("global" allowed), inclusive date range; counts are additive across days. + Task> GetVerdictCalibrationAggregatesAsync(string tenantId, DateTime startDate, DateTime endDate); + /// Deletes one daily row — the sweep's stale-bucket reconcile. + Task DeleteVerdictCalibrationAggregateAsync(string tenantId, string dateKey); + Task DeleteVerdictCalibrationAggregatesOlderThanAsync(DateTime cutoffDate); } /// diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Metrics/VerdictCalibrationDailyAggregate.cs b/src/Shared/AutopilotMonitor.Shared/Models/Metrics/VerdictCalibrationDailyAggregate.cs new file mode 100644 index 00000000..377fea5b --- /dev/null +++ b/src/Shared/AutopilotMonitor.Shared/Models/Metrics/VerdictCalibrationDailyAggregate.cs @@ -0,0 +1,78 @@ +using System; +using System.Collections.Generic; + +namespace AutopilotMonitor.Shared.Models +{ + /// + /// One (verdict path × status) bucket of a daily verdict-calibration row. Count is the + /// sessions whose CURRENT status was produced by this path; the Overridden* counters + /// are sessions that once carried this path and were then overridden (their + /// PriorVerdictPath names this bucket) — the correction stream. Re-enrollment is the + /// delayed proxy signal: of the sessions old enough to judge (), how + /// many saw the same device register another terminal session within 7 days. + /// + public class VerdictCalibrationBucket + { + /// value, or the read-side derivation for pre-instrumentation rows. + public string VerdictPath { get; set; } = string.Empty; + + /// name the path produced. + public string Status { get; set; } = string.Empty; + + /// Sessions currently on this path/status. + public int Count { get; set; } + + /// Of : rows without a stamped VerdictPath, attributed by VerdictPathDerivation (weaker evidence). + public int DerivedCount { get; set; } + + /// Of : sessions whose terminal moment lies ≥ 7 days before the compute time — the re-enrollment denominator. + public int Eligible7d { get; set; } + + /// Of : the device registered another terminal session within 7 days of this one's end. + public int ReEnrolled7d { get; set; } + + /// Sessions that carried this path and were then flipped by an administrator (MarkSessionSucceeded/Failed). + public int OverriddenByAdmin { get; set; } + + /// Sessions that carried this path and were then upgraded to Succeeded by a late agent completion report. + public int OverriddenByLateCompletion { get; set; } + + /// Sessions that carried this path and were overridden by any other writer (retro-reclassification, grace expiry, supersede). + public int OverriddenOther { get; set; } + } + + /// + /// Daily verdict-calibration rollup for one (tenant, date) — the operator's thermometer for the + /// rule classifier (docs/backend/verdict-calibration.md). A session buckets on its StartedAt + /// date (same convention as every other daily aggregate; the rolling sweep re-buckets + /// late-terminating sessions idempotently). Every status is counted — non-terminal rows are + /// part of the picture (a growing Stalled share is a signal) — and + /// carries the share denominator for terminal paths. + /// Counts are additive across days; a window is the sum of daily rows. "global" mirror rows + /// sum all tenants. Regenerable from Sessions + DeviceHistories at any time. + /// + public class VerdictCalibrationDailyAggregate + { + /// Bucketing/derivation algorithm version — a definition change never silently mixes semantics. + public const int CurrentVersion = 1; + + /// Calendar day (UTC, "yyyy-MM-dd") the sessions STARTED. + public string Date { get; set; } = string.Empty; + + /// Tenant GUID, or "global" for the cross-tenant row. + public string TenantId { get; set; } = string.Empty; + + public int Version { get; set; } = CurrentVersion; + + /// All sessions started on this date (any status). + public int SessionCount { get; set; } + + /// Sessions whose status is Succeeded / Failed / Incomplete — the share denominator for terminal paths. + public int TerminalSessionCount { get; set; } + + /// Ordered by VerdictPath, then Status (ordinal) for deterministic output. + public List Buckets { get; set; } = new List(); + + public DateTime ComputedAt { get; set; } + } +} diff --git a/src/Web/autopilot-monitor-web/app/admin/metrics/[section]/SectionClient.tsx b/src/Web/autopilot-monitor-web/app/admin/metrics/[section]/SectionClient.tsx index 9685335c..6c3b0227 100644 --- a/src/Web/autopilot-monitor-web/app/admin/metrics/[section]/SectionClient.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/metrics/[section]/SectionClient.tsx @@ -5,11 +5,13 @@ import { type MetricsSectionId } from "../metricsNavSections"; import { SectionAgentMetrics } from "../sections/SectionAgentMetrics"; import { SectionPlatformUsage } from "../sections/SectionPlatformUsage"; import { SectionMcpUsage } from "../sections/SectionMcpUsage"; +import { SectionVerdictCalibration } from "../sections/SectionVerdictCalibration"; const SECTION_COMPONENTS: Record = { "platform-metrics": SectionAgentMetrics, "usage": SectionPlatformUsage, "mcp-usage": SectionMcpUsage, + "verdict-calibration": SectionVerdictCalibration, }; export function SectionClient({ section }: { section: string }) { diff --git a/src/Web/autopilot-monitor-web/app/admin/metrics/metricsNavSections.ts b/src/Web/autopilot-monitor-web/app/admin/metrics/metricsNavSections.ts index d6387997..0e7c2989 100644 --- a/src/Web/autopilot-monitor-web/app/admin/metrics/metricsNavSections.ts +++ b/src/Web/autopilot-monitor-web/app/admin/metrics/metricsNavSections.ts @@ -2,6 +2,7 @@ export const METRICS_NAV_SECTIONS = [ { id: "platform-metrics", label: "Platform Metrics", description: "Agent performance, delivery latency, crash rates, and platform health" }, { id: "usage", label: "Platform Usage", description: "Platform usage statistics across all tenants" }, { id: "mcp-usage", label: "MCP Usage", description: "MCP API usage metrics across all users" }, + { id: "verdict-calibration", label: "Verdict Calibration", description: "Which code path produced each session verdict, overrides and re-enrollment — classifier diagnostics" }, ] as const; export type MetricsSectionId = (typeof METRICS_NAV_SECTIONS)[number]["id"]; diff --git a/src/Web/autopilot-monitor-web/app/admin/metrics/sections/SectionVerdictCalibration.tsx b/src/Web/autopilot-monitor-web/app/admin/metrics/sections/SectionVerdictCalibration.tsx new file mode 100644 index 00000000..d72b663d --- /dev/null +++ b/src/Web/autopilot-monitor-web/app/admin/metrics/sections/SectionVerdictCalibration.tsx @@ -0,0 +1,225 @@ +"use client"; + +import { useCallback, useEffect, useState } from "react"; +import { useAuth } from "../../../../contexts/AuthContext"; +import { authenticatedFetch, TokenExpiredError } from "@/lib/authenticatedFetch"; +import { api } from "@/lib/api"; +import { useAggregatedAdminScope } from "@/hooks"; +import { TenantScopeSelector } from "@/components/TenantScopeSelector"; +import { SessionStatusBadge } from "@/components/SessionStatusBadge"; +import { SegmentedControl, TIME_RANGE_OPTIONS } from "@/components/SegmentedControl"; +import { + VERDICT_PATH_ORIGIN_LABELS, + groupPathsByOrigin, + trendGlyph, + type VerdictCalibrationPath, + type VerdictCalibrationResponse, +} from "./verdictCalibrationLogic"; + +type DateRange = "7d" | "30d" | "90d"; + +const RANGE_DAYS: Record = { "7d": 7, "30d": 30, "90d": 90 }; + +export function SectionVerdictCalibration() { + const { getAccessToken } = useAuth(); + const scope = useAggregatedAdminScope({ defaultAggregated: true }); + const { selectedTenantId, scopeInitialized } = scope; + const [data, setData] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [dateRange, setDateRange] = useState("30d"); + + const fetchData = useCallback( + async (range: DateRange, tenantId: string) => { + setLoading(true); + setError(null); + try { + const res = await authenticatedFetch( + api.metrics.globalVerdictCalibration(RANGE_DAYS[range], tenantId || undefined), + getAccessToken + ); + if (res.status === 404) { + // Deploy skew: backend without the route yet — reads as "no rows", never as an error. + setData(null); + return; + } + if (!res.ok) throw new Error(`Verdict calibration: ${res.status}`); + setData((await res.json()) as VerdictCalibrationResponse); + } catch (err) { + if (err instanceof TokenExpiredError) { + setError("Session expired. Please refresh the page."); + } else { + setError(err instanceof Error ? err.message : "Failed to load verdict calibration"); + } + } finally { + setLoading(false); + } + }, + [getAccessToken] + ); + + useEffect(() => { + if (!scopeInitialized) return; + const run = async () => { + await fetchData(dateRange, selectedTenantId); + }; + void run(); + }, [fetchData, dateRange, selectedTenantId, scopeInitialized]); + + const groups = data ? groupPathsByOrigin(data.paths) : []; + + return ( +
+
+
+

Verdict Calibration

+

+ Which code path produced each session verdict, how often it was overridden, and whether the device came back. + Platform-internal classifier diagnostics — correlations only, never causal claims. +

+
+
+ + setDateRange(v as DateRange)} /> + +
+
+ + {error && ( +
{error}
+ )} + + {data && ( +
+ + + + +
+ )} + +
+ {loading && !data ? ( +

Loading…

+ ) : !data || data.paths.length === 0 ? ( +

+ No calibration rows yet. The maintenance sweep writes them every two hours; the first rows appear after its + next pass. +

+ ) : ( +
+ + + + + + + + + + + + + + {groups.map((group) => ( + + ))} + +
Verdict pathStatusSessionsShare + 7d trend + + Re-enrolled ≤7d + + Overridden (admin / late / other) +
+
+ )} + {data?.computedAt && ( +

+ Rows computed {new Date(data.computedAt).toLocaleString()} · algorithm version{" "} + {data.versions.join(", ") || "—"} +

+ )} +
+
+ ); +} + +function Stat({ label, value, hint }: { label: string; value: string; hint: string }) { + return ( +
+

{label}

+

{value}

+

{hint}

+
+ ); +} + +function GroupRows({ origin, rows }: { origin: string; rows: VerdictCalibrationPath[] }) { + return ( + <> + + + {VERDICT_PATH_ORIGIN_LABELS[origin] ?? origin} + + + {rows.map((row) => ( + + ))} + + ); +} + +function PathRow({ row }: { row: VerdictCalibrationPath }) { + const trend = trendGlyph(row); + const overridden = row.overriddenByAdmin + row.overriddenByLateCompletion + row.overriddenOther; + return ( + + + {row.verdictPath} + {row.derivedCount > 0 && ( + + derived {row.derivedCount} + + )} + + + + + {row.count.toLocaleString()} + {row.sharePct.toFixed(1)}% + + {trend.text} + + + {row.reEnrollRatePct == null ? — (n={row.eligible7d}) : `${row.reEnrollRatePct.toFixed(1)}%`} + + + {overridden === 0 ? ( + — + ) : ( + `${row.overriddenByAdmin} / ${row.overriddenByLateCompletion} / ${row.overriddenOther}` + )} + + + ); +} diff --git a/src/Web/autopilot-monitor-web/app/admin/metrics/sections/__tests__/verdictCalibrationLogic.test.ts b/src/Web/autopilot-monitor-web/app/admin/metrics/sections/__tests__/verdictCalibrationLogic.test.ts new file mode 100644 index 00000000..11425788 --- /dev/null +++ b/src/Web/autopilot-monitor-web/app/admin/metrics/sections/__tests__/verdictCalibrationLogic.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from "vitest"; +import { + VERDICT_PATH_ORIGIN_LABELS, + groupPathsByOrigin, + pathOrigin, + trendGlyph, + type VerdictCalibrationPath, +} from "../verdictCalibrationLogic"; + +function row(verdictPath: string, overrides: Partial = {}): VerdictCalibrationPath { + return { + verdictPath, + status: "Succeeded", + count: 10, + sharePct: 10, + derivedCount: 0, + eligible7d: 10, + reEnrolled7d: 1, + reEnrollRatePct: null, + overriddenByAdmin: 0, + overriddenByLateCompletion: 0, + overriddenOther: 0, + window7: { count: 5, sessions: 50, sharePct: 10 }, + baseline28: { count: 20, sessions: 200, sharePct: 10 }, + lift: 1, + ...overrides, + }; +} + +describe("pathOrigin", () => { + it("splits on the first colon and tolerates rule ids with colons-free detail", () => { + expect(pathOrigin("sweep:r5_incomplete")).toBe("sweep"); + expect(pathOrigin("rule:ANALYZE-ESP-001")).toBe("rule"); + expect(pathOrigin("legacy:unknown")).toBe("legacy"); + expect(pathOrigin("weird")).toBe("weird"); + }); +}); + +describe("groupPathsByOrigin", () => { + it("orders groups by the documented origin order, unknown origins last, rows untouched", () => { + const groups = groupPathsByOrigin([ + row("zzz:x"), + row("manual:failed"), + row("sweep:r6"), + row("sweep:r5_incomplete"), + row("agent:complete"), + ]); + expect(groups.map((g) => g.origin)).toEqual(["agent", "sweep", "manual", "zzz"]); + expect(groups[1].rows.map((r) => r.verdictPath)).toEqual(["sweep:r6", "sweep:r5_incomplete"]); + }); + + it("has a label for every origin in the order list", () => { + for (const origin of ["agent", "ingest", "sweep", "maxlife", "late", "retro", "register", "rule", "manual", "legacy"]) { + expect(VERDICT_PATH_ORIGIN_LABELS[origin]).toBeTruthy(); + } + }); +}); + +describe("trendGlyph", () => { + it("flags a doubled share with an up arrow and a halved share with a down arrow", () => { + expect(trendGlyph(row("sweep:r6", { lift: 2.4 })).text).toBe("↑ 2.4×"); + expect(trendGlyph(row("sweep:r6", { lift: 0.4 })).text).toBe("↓ 0.4×"); + expect(trendGlyph(row("sweep:r6", { lift: 1.3 })).text).toBe("≈ 1.3×"); + }); + + it("withholds the arrow below five window hits — one session can double a tiny share", () => { + const g = trendGlyph(row("sweep:r6", { lift: 3, window7: { count: 3, sessions: 50, sharePct: 6 } })); + expect(g.text).toBe("≈ 3.0×"); + expect(g.className).toContain("text-gray-400"); + }); + + it("never invents a lift without a baseline", () => { + expect(trendGlyph(row("sweep:r6", { lift: null, baseline28: { count: 0, sessions: 200, sharePct: 0 } })).text).toBe("new"); + expect(trendGlyph(row("sweep:r6", { lift: null, window7: { count: 0, sessions: 0, sharePct: 0 }, baseline28: { count: 0, sessions: 0, sharePct: 0 } })).text).toBe("—"); + }); +}); diff --git a/src/Web/autopilot-monitor-web/app/admin/metrics/sections/verdictCalibrationLogic.ts b/src/Web/autopilot-monitor-web/app/admin/metrics/sections/verdictCalibrationLogic.ts new file mode 100644 index 00000000..75c37813 --- /dev/null +++ b/src/Web/autopilot-monitor-web/app/admin/metrics/sections/verdictCalibrationLogic.ts @@ -0,0 +1,109 @@ +/** + * Pure helpers for the Verdict Calibration matrix (docs/backend/verdict-calibration.md). + * Wire shape mirrors GetVerdictCalibrationFunction / VerdictCalibrationResponse (camelCase). + */ + +export interface TrendWindow { + count: number; + sessions: number; + sharePct: number; +} + +export interface VerdictCalibrationPath { + verdictPath: string; + status: string; + count: number; + sharePct: number; + derivedCount: number; + eligible7d: number; + reEnrolled7d: number; + /** null below 20 eligible sessions — never a rate on a handful */ + reEnrollRatePct: number | null; + overriddenByAdmin: number; + overriddenByLateCompletion: number; + overriddenOther: number; + window7: TrendWindow; + baseline28: TrendWindow; + /** window share ÷ baseline share; null without a baseline */ + lift: number | null; +} + +export interface VerdictCalibrationResponse { + success: boolean; + tenantId: string; + windowDays: number; + windowStart: string; + windowEnd: string; + computedAt: string | null; + versions: number[]; + totals: { sessions: number; terminal: number; derived: number; days: number }; + trend: { windowDays: number; baselineDays: number; windowSessions: number; baselineSessions: number }; + paths: VerdictCalibrationPath[]; + alerts: unknown[]; +} + +/** The `origin` half of a verdict path (`sweep` for `sweep:r6`); the whole string without a colon. */ +export function pathOrigin(path: string): string { + const i = path.indexOf(":"); + return i < 0 ? path : path.slice(0, i); +} + +/** Display order + labels for the origin groups; unknown origins sort last, alphabetically. */ +export const VERDICT_PATH_ORIGIN_LABELS: Record = { + agent: "Agent-declared", + ingest: "Ingest mapping", + sweep: "Maintenance sweep (silence classifier)", + maxlife: "Agent max-lifetime shutdown (silence classifier)", + late: "Late telemetry reconcile (silence classifier)", + retro: "Retro reclassification", + register: "Session registration", + rule: "Analyze rule (MarkSessionAsFailed)", + manual: "Administrator", + legacy: "Pre-instrumentation (derived, origin unknown)", +}; + +const ORIGIN_ORDER = Object.keys(VERDICT_PATH_ORIGIN_LABELS); + +export interface OriginGroup { + origin: string; + rows: VerdictCalibrationPath[]; +} + +/** Groups rows by origin in display order; rows inside a group keep the server order (count desc). */ +export function groupPathsByOrigin(paths: VerdictCalibrationPath[]): OriginGroup[] { + const byOrigin = new Map(); + for (const p of paths) { + const origin = pathOrigin(p.verdictPath); + const list = byOrigin.get(origin); + if (list) list.push(p); + else byOrigin.set(origin, [p]); + } + return Array.from(byOrigin.entries()) + .sort(([a], [b]) => { + const ia = ORIGIN_ORDER.indexOf(a); + const ib = ORIGIN_ORDER.indexOf(b); + if (ia !== -1 || ib !== -1) return (ia === -1 ? ORIGIN_ORDER.length : ia) - (ib === -1 ? ORIGIN_ORDER.length : ib); + return a.localeCompare(b); + }) + .map(([origin, rows]) => ({ origin, rows })); +} + +/** + * 7d-vs-28d trend cell. Lift ≥ 2 (share at least doubled) and ≤ 0.5 (halved) get the arrow + + * color; anything in between is a neutral "≈". No baseline → "new" (a new signal has no finite + * lift — never invented). Below 5 window hits the arrow is withheld: one session can double a + * tiny share. + */ +export function trendGlyph(row: VerdictCalibrationPath): { text: string; className: string } { + if (row.lift == null) { + return row.window7.count > 0 && row.baseline28.sessions > 0 + ? { text: "new", className: "text-gray-500" } + : { text: "—", className: "text-gray-400" }; + } + const lift = row.lift; + const label = `${lift.toFixed(1)}×`; + if (row.window7.count < 5) return { text: `≈ ${label}`, className: "text-gray-400" }; + if (lift >= 2) return { text: `↑ ${label}`, className: "text-red-600 font-medium" }; + if (lift <= 0.5) return { text: `↓ ${label}`, className: "text-green-700" }; + return { text: `≈ ${label}`, className: "text-gray-500" }; +} diff --git a/src/Web/autopilot-monitor-web/lib/api.ts b/src/Web/autopilot-monitor-web/lib/api.ts index 58acedd7..db63dff0 100644 --- a/src/Web/autopilot-monitor-web/lib/api.ts +++ b/src/Web/autopilot-monitor-web/lib/api.ts @@ -292,6 +292,10 @@ export const api = { `${API_BASE_URL}/api/metrics/device-journeys${qs({ days: String(days) })}`, globalDeviceJourneys: (days: number, tenantId?: string) => `${API_BASE_URL}/api/global/metrics/device-journeys${qs({ days: String(days), tenantId })}`, + // Verdict calibration matrix (operator-only; GlobalReadOrAdmin). Omit tenantId for the + // cross-tenant aggregate. Daily rows are additive, so the window follows days honestly. + globalVerdictCalibration: (days: number, tenantId?: string) => + `${API_BASE_URL}/api/global/metrics/verdict-calibration${qs({ days: String(days), tenantId })}`, geographic: (tenantId: string, days: number, groupBy: string) => `${API_BASE_URL}/api/metrics/geographic${qs({ tenantId, days: String(days), groupBy })}`, globalGeographic: (days: number, groupBy: string, tenantId?: string) => diff --git a/src/Web/autopilot-monitor-web/lib/globalNavConfig.tsx b/src/Web/autopilot-monitor-web/lib/globalNavConfig.tsx index 11326623..f3c43c6a 100644 --- a/src/Web/autopilot-monitor-web/lib/globalNavConfig.tsx +++ b/src/Web/autopilot-monitor-web/lib/globalNavConfig.tsx @@ -261,6 +261,7 @@ export const EXPANDABLE_NAV_GROUPS: ExpandableNavGroup[] = [ { id: "ga-usage", label: "Platform Usage", href: route("/admin/metrics/usage") }, { id: "ga-active-users", label: "Active Users", href: "/admin/presence" }, { id: "ga-mcp-usage", label: "MCP Usage", href: route("/admin/metrics/mcp-usage") }, + { id: "ga-verdict-calibration", label: "Verdict Calibration", href: route("/admin/metrics/verdict-calibration") }, ], }, {