mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 12:42:38 +00:00
feat: verdict calibration aggregate, GA endpoint, MCP tool and operator matrix
Daily (tenant x StartedAt date) buckets per verdict path: count, derived attribution, 7-day re-enrollment proxy (eligible/re-enrolled), and the correction stream attributed to the PRIOR path (admin / late completion / other). Written by the 2h maintenance sweep after the device-journey pass, 180d retention, regenerable. Read via GET global/metrics/verdict-calibration (GlobalReadOrAdmin, 7d-vs-28d share trend, rates withheld below n=20), the get_verdict_calibration MCP tool, and the admin metrics section. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,301 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using AutopilotMonitor.Functions.Functions.Metrics;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Verdict calibration aggregate (docs/backend/verdict-calibration.md): the pure bucketing core,
|
||||
/// the persistence round-trip and the matrix arithmetic of the read endpoint.
|
||||
/// </summary>
|
||||
public class VerdictCalibrationTests
|
||||
{
|
||||
private const string TenantA = "11111111-1111-1111-1111-111111111111";
|
||||
private const string TenantB = "22222222-2222-2222-2222-222222222222";
|
||||
private static readonly DateTime Now = new(2026, 8, 23, 12, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private static SessionSummary Session(string tenant, string id, SessionStatus status, DateTime started,
|
||||
string? path = null, string? priorPath = null, string? priorStatus = null, string? adminMarked = null,
|
||||
DateTime? completed = null, string serial = "SN-1", string? failureReason = null, string? reconcileReason = null) => new()
|
||||
{
|
||||
TenantId = tenant, SessionId = id, SerialNumber = serial, Status = status, StartedAt = started,
|
||||
CompletedAt = completed, VerdictPath = path, PriorVerdictPath = priorPath, PriorStatus = priorStatus,
|
||||
AdminMarkedAction = adminMarked, FailureReason = failureReason ?? string.Empty,
|
||||
ReconcileReason = reconcileReason ?? string.Empty, FailureSource = string.Empty,
|
||||
};
|
||||
|
||||
private static DeviceHistory History(string tenant, string serial, params (string Id, DateTime Started, DateTime? Completed, string Status)[] refs) => new()
|
||||
{
|
||||
TenantId = tenant, SerialKey = serial.ToLowerInvariant(), SerialNumber = serial,
|
||||
Chain = refs.Select(r => new DeviceSessionRef { SessionId = r.Id, StartedAt = r.Started, CompletedAt = r.Completed, Status = r.Status }).ToList(),
|
||||
};
|
||||
|
||||
private static VerdictCalibrationBucket Bucket(VerdictCalibrationDailyAggregate row, string path, string status)
|
||||
=> Assert.Single(row.Buckets, b => b.VerdictPath == path && b.Status == status);
|
||||
|
||||
// ---- BuildVerdictCalibrationAggregates ----
|
||||
|
||||
[Fact]
|
||||
public void Buckets_by_tenant_date_path_and_status_with_global_mirror()
|
||||
{
|
||||
var d1 = new DateTime(2026, 8, 10, 9, 0, 0, DateTimeKind.Utc);
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
Session(TenantA, "a1", SessionStatus.Succeeded, d1, path: "agent:complete", completed: d1.AddHours(1)),
|
||||
Session(TenantA, "a2", SessionStatus.Succeeded, d1.AddHours(2), path: "agent:complete", completed: d1.AddHours(3)),
|
||||
Session(TenantA, "a3", SessionStatus.Incomplete, d1, path: "sweep:r6", completed: d1.AddHours(6)),
|
||||
Session(TenantA, "a4", SessionStatus.Stalled, d1.AddDays(1), path: "sweep:stalled"),
|
||||
Session(TenantB, "b1", SessionStatus.Failed, d1, path: "agent:failed", completed: d1.AddHours(1)),
|
||||
};
|
||||
var rows = MaintenanceService.BuildVerdictCalibrationAggregates(sessions, new Dictionary<string, List<DeviceHistory>>(), Now);
|
||||
|
||||
var a10 = Assert.Single(rows, r => r.TenantId == TenantA && r.Date == "2026-08-10");
|
||||
Assert.Equal(3, a10.SessionCount);
|
||||
Assert.Equal(3, a10.TerminalSessionCount);
|
||||
Assert.Equal(2, Bucket(a10, "agent:complete", "Succeeded").Count);
|
||||
Assert.Equal(1, Bucket(a10, "sweep:r6", "Incomplete").Count);
|
||||
|
||||
var a11 = Assert.Single(rows, r => r.TenantId == TenantA && r.Date == "2026-08-11");
|
||||
Assert.Equal(1, a11.SessionCount);
|
||||
Assert.Equal(0, a11.TerminalSessionCount); // Stalled is not terminal
|
||||
Assert.Equal(1, Bucket(a11, "sweep:stalled", "Stalled").Count);
|
||||
|
||||
var g10 = Assert.Single(rows, r => r.TenantId == "global" && r.Date == "2026-08-10");
|
||||
Assert.Equal(4, g10.SessionCount);
|
||||
Assert.Equal(1, Bucket(g10, "agent:failed", "Failed").Count);
|
||||
Assert.Equal(VerdictCalibrationDailyAggregate.CurrentVersion, g10.Version);
|
||||
Assert.Equal(Now, g10.ComputedAt);
|
||||
|
||||
// Deterministic ordering: rows by tenant then date, buckets by path then status.
|
||||
Assert.Equal(rows.Select(r => (r.TenantId, r.Date)).OrderBy(k => k.TenantId, StringComparer.Ordinal).ThenBy(k => k.Date, StringComparer.Ordinal), rows.Select(r => (r.TenantId, r.Date)));
|
||||
Assert.Equal(a10.Buckets.Select(b => b.VerdictPath).OrderBy(p => p, StringComparer.Ordinal), a10.Buckets.Select(b => b.VerdictPath));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Unstamped_rows_are_derived_and_counted_as_derived()
|
||||
{
|
||||
var d = new DateTime(2026, 8, 10, 9, 0, 0, DateTimeKind.Utc);
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
Session(TenantA, "a1", SessionStatus.Succeeded, d, path: null, completed: d.AddHours(1)), // legacy agent completion
|
||||
Session(TenantA, "a2", SessionStatus.Incomplete, d, path: null, completed: d.AddHours(5),
|
||||
failureReason: "No Device Setup completion or explicit failure signal observed before timeout"),
|
||||
Session(TenantA, "a3", SessionStatus.Succeeded, d, path: "agent:complete", completed: d.AddHours(1)),
|
||||
};
|
||||
var row = MaintenanceService.BuildVerdictCalibrationAggregates(sessions, new Dictionary<string, List<DeviceHistory>>(), Now)
|
||||
.Single(r => r.TenantId == TenantA);
|
||||
|
||||
var complete = Bucket(row, "agent:complete", "Succeeded");
|
||||
Assert.Equal(2, complete.Count);
|
||||
Assert.Equal(1, complete.DerivedCount);
|
||||
var r6 = Bucket(row, "legacy:r6", "Incomplete");
|
||||
Assert.Equal(1, r6.Count);
|
||||
Assert.Equal(1, r6.DerivedCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Overrides_are_attributed_to_the_prior_path()
|
||||
{
|
||||
var d = new DateTime(2026, 8, 10, 9, 0, 0, DateTimeKind.Utc);
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
// sweep said Incomplete, agent later completed → late completion against sweep:r5_incomplete
|
||||
Session(TenantA, "a1", SessionStatus.Succeeded, d, path: "agent:complete", priorPath: "sweep:r5_incomplete", priorStatus: "Incomplete", completed: d.AddHours(1)),
|
||||
// admin flipped an agent success to Failed
|
||||
Session(TenantA, "a2", SessionStatus.Failed, d, path: "manual:failed", priorPath: "agent:complete", priorStatus: "Succeeded", adminMarked: "Failed", completed: d.AddHours(1)),
|
||||
// retro reclassification Failed → Incomplete
|
||||
Session(TenantA, "a3", SessionStatus.Incomplete, d, path: "retro:r6", priorPath: "agent:failed", priorStatus: "Failed", completed: d.AddHours(1)),
|
||||
// admin mark to Succeeded counts as admin, not late completion
|
||||
Session(TenantA, "a4", SessionStatus.Succeeded, d, path: "manual:succeeded", priorPath: "sweep:r6", priorStatus: "Incomplete", adminMarked: "Succeeded", completed: d.AddHours(1)),
|
||||
};
|
||||
var row = MaintenanceService.BuildVerdictCalibrationAggregates(sessions, new Dictionary<string, List<DeviceHistory>>(), Now)
|
||||
.Single(r => r.TenantId == TenantA);
|
||||
|
||||
var priorSweep = Bucket(row, "sweep:r5_incomplete", "Incomplete");
|
||||
Assert.Equal(0, priorSweep.Count); // nobody is on it any more
|
||||
Assert.Equal(1, priorSweep.OverriddenByLateCompletion);
|
||||
|
||||
var priorAgent = Bucket(row, "agent:complete", "Succeeded");
|
||||
Assert.Equal(1, priorAgent.Count); // a1 is on it now
|
||||
Assert.Equal(1, priorAgent.OverriddenByAdmin); // a2 used to be
|
||||
|
||||
Assert.Equal(1, Bucket(row, "agent:failed", "Failed").OverriddenOther);
|
||||
Assert.Equal(1, Bucket(row, "sweep:r6", "Incomplete").OverriddenByAdmin);
|
||||
Assert.Equal(0, Bucket(row, "manual:succeeded", "Succeeded").OverriddenByAdmin);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReEnrollment_proxy_counts_only_eligible_terminal_sessions()
|
||||
{
|
||||
var old = Now.AddDays(-20);
|
||||
var young = Now.AddDays(-2);
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
// old, re-enrolled 3 days later → eligible + re-enrolled
|
||||
Session(TenantA, "a1", SessionStatus.Incomplete, old, path: "sweep:r6", completed: old.AddHours(5), serial: "SN-A"),
|
||||
// old, next attempt 10 days later → eligible, not re-enrolled
|
||||
Session(TenantA, "a2", SessionStatus.Failed, old, path: "agent:failed", completed: old.AddHours(1), serial: "SN-B"),
|
||||
// old, no next attempt → eligible, not re-enrolled
|
||||
Session(TenantA, "a3", SessionStatus.Succeeded, old, path: "agent:complete", completed: old.AddHours(1), serial: "SN-C"),
|
||||
// too young to judge → not eligible even though the chain shows a follow-up
|
||||
Session(TenantA, "a4", SessionStatus.Failed, young, path: "agent:failed", completed: young.AddHours(1), serial: "SN-D"),
|
||||
// non-terminal → never eligible
|
||||
Session(TenantA, "a5", SessionStatus.AwaitingUser, old, path: "sweep:r5_awaiting", serial: "SN-E"),
|
||||
};
|
||||
var histories = new Dictionary<string, List<DeviceHistory>>
|
||||
{
|
||||
[TenantA] = new()
|
||||
{
|
||||
History(TenantA, "SN-A", ("a1", old, old.AddHours(5), "Incomplete"), ("x1", old.AddDays(3), old.AddDays(3).AddHours(1), "Succeeded")),
|
||||
History(TenantA, "SN-B", ("a2", old, old.AddHours(1), "Failed"), ("x2", old.AddDays(10), null, "Succeeded")),
|
||||
History(TenantA, "SN-C", ("a3", old, old.AddHours(1), "Succeeded")),
|
||||
History(TenantA, "SN-D", ("a4", young, young.AddHours(1), "Failed"), ("x4", young.AddHours(2), null, "Succeeded")),
|
||||
},
|
||||
};
|
||||
var rows = MaintenanceService.BuildVerdictCalibrationAggregates(sessions, histories, Now);
|
||||
|
||||
var oldRow = rows.Single(r => r.TenantId == TenantA && r.Date == old.ToString("yyyy-MM-dd"));
|
||||
var r6 = Bucket(oldRow, "sweep:r6", "Incomplete");
|
||||
Assert.Equal((1, 1), (r6.Eligible7d, r6.ReEnrolled7d));
|
||||
var failed = Bucket(oldRow, "agent:failed", "Failed");
|
||||
Assert.Equal((1, 0), (failed.Eligible7d, failed.ReEnrolled7d));
|
||||
var complete = Bucket(oldRow, "agent:complete", "Succeeded");
|
||||
Assert.Equal((1, 0), (complete.Eligible7d, complete.ReEnrolled7d));
|
||||
var awaiting = Bucket(oldRow, "sweep:r5_awaiting", "AwaitingUser");
|
||||
Assert.Equal((0, 0), (awaiting.Eligible7d, awaiting.ReEnrolled7d));
|
||||
|
||||
var youngRow = rows.Single(r => r.TenantId == TenantA && r.Date == young.ToString("yyyy-MM-dd"));
|
||||
var youngFailed = Bucket(youngRow, "agent:failed", "Failed");
|
||||
Assert.Equal((0, 0), (youngFailed.Eligible7d, youngFailed.ReEnrolled7d));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Sessions_inside_a_deletion_cascade_are_skipped()
|
||||
{
|
||||
var d = new DateTime(2026, 8, 10, 9, 0, 0, DateTimeKind.Utc);
|
||||
var deleting = Session(TenantA, "a1", SessionStatus.Succeeded, d, path: "agent:complete");
|
||||
deleting.DeletionState = "Pending";
|
||||
var rows = MaintenanceService.BuildVerdictCalibrationAggregates(new[] { deleting }, new Dictionary<string, List<DeviceHistory>>(), Now);
|
||||
Assert.Empty(rows);
|
||||
}
|
||||
|
||||
// ---- Persistence round-trip ----
|
||||
|
||||
[Fact]
|
||||
public void Aggregate_round_trips_through_the_table_entity()
|
||||
{
|
||||
var aggregate = new VerdictCalibrationDailyAggregate
|
||||
{
|
||||
TenantId = TenantA, Date = "2026-08-10", Version = 1, SessionCount = 5, TerminalSessionCount = 4, ComputedAt = Now,
|
||||
Buckets =
|
||||
{
|
||||
new VerdictCalibrationBucket { VerdictPath = "agent:complete", Status = "Succeeded", Count = 3, DerivedCount = 1, Eligible7d = 3, ReEnrolled7d = 1, OverriddenByAdmin = 1, OverriddenByLateCompletion = 0, OverriddenOther = 2 },
|
||||
new VerdictCalibrationBucket { VerdictPath = "sweep:r6", Status = "Incomplete", Count = 1 },
|
||||
},
|
||||
};
|
||||
var entity = TableStorageService.BuildVerdictCalibrationAggregateEntity(aggregate);
|
||||
Assert.Equal(TenantA, entity.PartitionKey);
|
||||
Assert.Equal("2026-08-10", entity.RowKey);
|
||||
|
||||
var back = TableStorageService.MapToVerdictCalibrationAggregate(entity);
|
||||
Assert.Equal(JsonSerializer.Serialize(aggregate), JsonSerializer.Serialize(back));
|
||||
}
|
||||
|
||||
// ---- Response builder ----
|
||||
|
||||
private static VerdictCalibrationDailyAggregate Day(string tenant, DateTime date, int sessions, params (string Path, string Status, int Count)[] buckets) => new()
|
||||
{
|
||||
TenantId = tenant, Date = date.ToString("yyyy-MM-dd"), SessionCount = sessions, TerminalSessionCount = sessions, Version = 1, ComputedAt = Now,
|
||||
Buckets = buckets.Select(b => new VerdictCalibrationBucket { VerdictPath = b.Path, Status = b.Status, Count = b.Count, Eligible7d = b.Count, ReEnrolled7d = b.Count / 5 }).ToList(),
|
||||
};
|
||||
|
||||
// Same naming policy as the worker host (Program.cs: camelCase) so the assertions pin the wire shape.
|
||||
private static readonly JsonSerializerOptions Wire = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase };
|
||||
|
||||
private static JsonElement BuildJson(IReadOnlyList<VerdictCalibrationDailyAggregate> daily, DateTime today, int days)
|
||||
=> JsonSerializer.SerializeToElement(VerdictCalibrationResponse.Build(daily, "global", today, days), Wire);
|
||||
|
||||
[Fact]
|
||||
public void Response_sums_the_window_and_computes_share_rate_and_trend()
|
||||
{
|
||||
var today = new DateTime(2026, 8, 23, 0, 0, 0, DateTimeKind.Utc);
|
||||
var daily = new List<VerdictCalibrationDailyAggregate>();
|
||||
// 35 days: baseline days carry 10 sessions (1 sweep:r6), the last 7 days carry 10 sessions (3 sweep:r6) → share 10% → 30%, lift 3.0
|
||||
for (var i = 34; i >= 0; i--)
|
||||
{
|
||||
var date = today.AddDays(-i);
|
||||
var r6 = i < 7 ? 3 : 1;
|
||||
daily.Add(Day("global", date, 10, ("agent:complete", "Succeeded", 10 - r6), ("sweep:r6", "Incomplete", r6)));
|
||||
}
|
||||
var json = BuildJson(daily, today, days: 30);
|
||||
|
||||
Assert.True(json.GetProperty("success").GetBoolean());
|
||||
Assert.Equal(30, json.GetProperty("windowDays").GetInt32());
|
||||
Assert.Equal(300, json.GetProperty("totals").GetProperty("sessions").GetInt32());
|
||||
Assert.Equal(30, json.GetProperty("totals").GetProperty("days").GetInt32());
|
||||
|
||||
var paths = json.GetProperty("paths").EnumerateArray().ToList();
|
||||
var r6Row = paths.Single(p => p.GetProperty("verdictPath").GetString() == "sweep:r6");
|
||||
Assert.Equal(7 * 3 + 23 * 1, r6Row.GetProperty("count").GetInt32());
|
||||
Assert.Equal(Math.Round(100.0 * 44 / 300, 1), r6Row.GetProperty("sharePct").GetDouble());
|
||||
Assert.Equal(30.0, r6Row.GetProperty("window7").GetProperty("sharePct").GetDouble());
|
||||
Assert.Equal(10.0, r6Row.GetProperty("baseline28").GetProperty("sharePct").GetDouble());
|
||||
Assert.Equal(3.0, r6Row.GetProperty("lift").GetDouble());
|
||||
// Eligible 44 ≥ 20 → rate stated; ReEnrolled7d = count/5 per day → 7*0 + 23*0 = 0 → 0%
|
||||
Assert.Equal(0.0, r6Row.GetProperty("reEnrollRatePct").GetDouble());
|
||||
|
||||
// Sorted by count desc → agent:complete first.
|
||||
Assert.Equal("agent:complete", paths[0].GetProperty("verdictPath").GetString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Response_withholds_rate_below_twenty_eligible_and_lift_without_baseline()
|
||||
{
|
||||
var today = new DateTime(2026, 8, 23, 0, 0, 0, DateTimeKind.Utc);
|
||||
var daily = new List<VerdictCalibrationDailyAggregate>
|
||||
{
|
||||
Day("global", today, 5, ("maxlife:r3", "Incomplete", 5)),
|
||||
};
|
||||
var json = BuildJson(daily, today, days: 7);
|
||||
var row = json.GetProperty("paths").EnumerateArray().Single();
|
||||
Assert.Equal(JsonValueKind.Null, row.GetProperty("reEnrollRatePct").ValueKind);
|
||||
Assert.Equal(JsonValueKind.Null, row.GetProperty("lift").ValueKind); // no baseline rows → never invented
|
||||
Assert.Equal(100.0, row.GetProperty("sharePct").GetDouble());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Response_keeps_a_zero_count_bucket_that_carries_overrides()
|
||||
{
|
||||
var today = new DateTime(2026, 8, 23, 0, 0, 0, DateTimeKind.Utc);
|
||||
var day = Day("global", today, 1, ("agent:complete", "Succeeded", 1));
|
||||
day.Buckets.Add(new VerdictCalibrationBucket { VerdictPath = "sweep:r5_incomplete", Status = "Incomplete", Count = 0, OverriddenByLateCompletion = 1 });
|
||||
var json = BuildJson(new[] { day }, today, days: 7);
|
||||
var rows = json.GetProperty("paths").EnumerateArray().ToList();
|
||||
var prior = rows.Single(p => p.GetProperty("verdictPath").GetString() == "sweep:r5_incomplete");
|
||||
Assert.Equal(0, prior.GetProperty("count").GetInt32());
|
||||
Assert.Equal(1, prior.GetProperty("overriddenByLateCompletion").GetInt32());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Response_is_empty_but_well_formed_before_the_first_sweep()
|
||||
{
|
||||
var json = BuildJson(Array.Empty<VerdictCalibrationDailyAggregate>(), new DateTime(2026, 8, 23), 30);
|
||||
Assert.True(json.GetProperty("success").GetBoolean());
|
||||
Assert.Empty(json.GetProperty("paths").EnumerateArray());
|
||||
Assert.Equal(0, json.GetProperty("totals").GetProperty("sessions").GetInt32());
|
||||
Assert.Equal(JsonValueKind.Null, json.GetProperty("computedAt").ValueKind);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null, 30)]
|
||||
[InlineData("abc", 30)]
|
||||
[InlineData("0", 1)]
|
||||
[InlineData("500", 180)]
|
||||
[InlineData("14", 14)]
|
||||
public void ClampDays_bounds_the_window(string? raw, int expected)
|
||||
=> Assert.Equal(expected, VerdictCalibrationResponse.ClampDays(raw));
|
||||
}
|
||||
+12
@@ -159,5 +159,17 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
|
||||
public Task<int> DeleteDeviceJourneyAggregatesOlderThanAsync(DateTime cutoffDate)
|
||||
=> _storage.DeleteDeviceJourneyAggregatesOlderThanAsync(cutoffDate);
|
||||
|
||||
public Task<bool> SaveVerdictCalibrationAggregateAsync(VerdictCalibrationDailyAggregate aggregate)
|
||||
=> _storage.SaveVerdictCalibrationAggregateAsync(aggregate);
|
||||
|
||||
public Task<List<VerdictCalibrationDailyAggregate>> GetVerdictCalibrationAggregatesAsync(string tenantId, DateTime startDate, DateTime endDate)
|
||||
=> _storage.GetVerdictCalibrationAggregatesAsync(tenantId, startDate, endDate);
|
||||
|
||||
public Task DeleteVerdictCalibrationAggregateAsync(string tenantId, string dateKey)
|
||||
=> _storage.DeleteVerdictCalibrationAggregateAsync(tenantId, dateKey);
|
||||
|
||||
public Task<int> DeleteVerdictCalibrationAggregatesOlderThanAsync(DateTime cutoffDate)
|
||||
=> _storage.DeleteVerdictCalibrationAggregatesOlderThanAsync(cutoffDate);
|
||||
}
|
||||
}
|
||||
|
||||
+220
@@ -0,0 +1,220 @@
|
||||
using System.Globalization;
|
||||
using System.Net;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Functions.Metrics
|
||||
{
|
||||
/// <summary>
|
||||
/// Operator-only verdict calibration matrix (docs/backend/verdict-calibration.md): per verdict
|
||||
/// path, how many sessions it produced in the window, its share, how often it was overridden
|
||||
/// (admin / late completion / other), the 7-day re-enrollment proxy, and a 7d-vs-28d trend.
|
||||
/// Platform scope ("global" partition) or one tenant via ?tenantId=. Serves the admin
|
||||
/// metrics page and the <c>get_verdict_calibration</c> MCP tool. Never customer-facing —
|
||||
/// these are platform-internal classifier diagnostics, not product features.
|
||||
/// </summary>
|
||||
public class GetVerdictCalibrationFunction
|
||||
{
|
||||
private readonly ILogger<GetVerdictCalibrationFunction> _logger;
|
||||
private readonly IMetricsRepository _metricsRepo;
|
||||
|
||||
public GetVerdictCalibrationFunction(ILogger<GetVerdictCalibrationFunction> logger, IMetricsRepository metricsRepo)
|
||||
{
|
||||
_logger = logger;
|
||||
_metricsRepo = metricsRepo;
|
||||
}
|
||||
|
||||
[Function("GetVerdictCalibration")]
|
||||
public async Task<HttpResponseData> Run(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "global/metrics/verdict-calibration")] HttpRequestData req)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Authentication + GlobalReadOrAdmin authorization enforced by PolicyEnforcementMiddleware
|
||||
var query = System.Web.HttpUtility.ParseQueryString(req.Url.Query);
|
||||
var tenantIdFilter = query["tenantId"];
|
||||
var days = VerdictCalibrationResponse.ClampDays(query["days"]);
|
||||
var partition = "global";
|
||||
if (!string.IsNullOrWhiteSpace(tenantIdFilter))
|
||||
{
|
||||
if (!Guid.TryParse(tenantIdFilter, out var parsed))
|
||||
{
|
||||
var bad = req.CreateResponse(HttpStatusCode.BadRequest);
|
||||
await bad.WriteAsJsonAsync(new { success = false, message = "tenantId must be a GUID" });
|
||||
return bad;
|
||||
}
|
||||
partition = parsed.ToString("D");
|
||||
}
|
||||
|
||||
var today = DateTime.UtcNow.Date;
|
||||
// One read covers both the requested window and the trend horizon (7d window + 28d baseline).
|
||||
var readDays = Math.Max(days, VerdictCalibrationResponse.TrendHorizonDays);
|
||||
var daily = await _metricsRepo.GetVerdictCalibrationAggregatesAsync(
|
||||
partition, VerdictCalibrationResponse.InclusiveWindowStart(today, readDays), today);
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
await response.WriteAsJsonAsync(VerdictCalibrationResponse.Build(daily, partition, today, days));
|
||||
return response;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Error fetching verdict calibration");
|
||||
var errorResponse = req.CreateResponse(HttpStatusCode.InternalServerError);
|
||||
await errorResponse.WriteAsJsonAsync(new { success = false, message = "Internal server error" });
|
||||
return errorResponse;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Pure response builder — internal static so the matrix arithmetic is pinned by tests.</summary>
|
||||
internal static class VerdictCalibrationResponse
|
||||
{
|
||||
internal const int DefaultWindowDays = 30;
|
||||
internal const int MaxWindowDays = 180; // aggregate retention
|
||||
internal const int TrendWindowDays = 7;
|
||||
internal const int TrendBaselineDays = 28;
|
||||
internal const int TrendHorizonDays = TrendWindowDays + TrendBaselineDays;
|
||||
|
||||
internal static int ClampDays(string? raw)
|
||||
{
|
||||
if (!int.TryParse(raw, out var days)) return DefaultWindowDays;
|
||||
return Math.Clamp(days, 1, MaxWindowDays);
|
||||
}
|
||||
|
||||
/// <summary>"Last N days" = exactly N calendar day keys including today (both range ends inclusive).</summary>
|
||||
internal static DateTime InclusiveWindowStart(DateTime today, int days) => today.AddDays(-(days - 1));
|
||||
|
||||
internal sealed class PathRow
|
||||
{
|
||||
public string VerdictPath { get; set; } = string.Empty;
|
||||
public string Status { get; set; } = string.Empty;
|
||||
public int Count { get; set; }
|
||||
public double SharePct { get; set; }
|
||||
public int DerivedCount { get; set; }
|
||||
public int Eligible7d { get; set; }
|
||||
public int ReEnrolled7d { get; set; }
|
||||
/// <summary>Null below the minimum eligible count — never a rate on a handful of sessions.</summary>
|
||||
public double? ReEnrollRatePct { get; set; }
|
||||
public int OverriddenByAdmin { get; set; }
|
||||
public int OverriddenByLateCompletion { get; set; }
|
||||
public int OverriddenOther { get; set; }
|
||||
public TrendWindow Window7 { get; set; } = new();
|
||||
public TrendWindow Baseline28 { get; set; } = new();
|
||||
/// <summary>Window share ÷ baseline share; null when the baseline share is 0 (a new path has no finite lift — never invented).</summary>
|
||||
public double? Lift { get; set; }
|
||||
}
|
||||
|
||||
internal sealed class TrendWindow
|
||||
{
|
||||
public int Count { get; set; }
|
||||
public int Sessions { get; set; }
|
||||
public double SharePct { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Minimum eligible sessions before a re-enrollment rate is stated (truthfulness: no rate on n<20).</summary>
|
||||
internal const int MinEligibleForRate = 20;
|
||||
|
||||
internal static object Build(IReadOnlyList<VerdictCalibrationDailyAggregate> daily, string partition, DateTime today, int days)
|
||||
{
|
||||
var windowStart = Key(InclusiveWindowStart(today, days));
|
||||
var trendWindowStart = Key(InclusiveWindowStart(today, TrendWindowDays));
|
||||
var baselineStart = Key(InclusiveWindowStart(today, TrendHorizonDays));
|
||||
|
||||
var rows = new Dictionary<(string Path, string Status), PathRow>();
|
||||
int sessions = 0, terminal = 0, derived = 0;
|
||||
int trendSessions = 0, baselineSessions = 0;
|
||||
DateTime? computedAt = null;
|
||||
var versions = new HashSet<int>();
|
||||
|
||||
PathRow Row(VerdictCalibrationBucket b)
|
||||
{
|
||||
var key = (b.VerdictPath, b.Status);
|
||||
if (!rows.TryGetValue(key, out var row))
|
||||
{
|
||||
row = new PathRow { VerdictPath = b.VerdictPath, Status = b.Status };
|
||||
rows[key] = row;
|
||||
}
|
||||
return row;
|
||||
}
|
||||
|
||||
foreach (var day in daily)
|
||||
{
|
||||
var inWindow = string.CompareOrdinal(day.Date, windowStart) >= 0;
|
||||
var inTrendWindow = string.CompareOrdinal(day.Date, trendWindowStart) >= 0;
|
||||
var inBaseline = !inTrendWindow && string.CompareOrdinal(day.Date, baselineStart) >= 0;
|
||||
|
||||
if (inWindow)
|
||||
{
|
||||
sessions += day.SessionCount;
|
||||
terminal += day.TerminalSessionCount;
|
||||
versions.Add(day.Version);
|
||||
if (computedAt == null || day.ComputedAt > computedAt) computedAt = day.ComputedAt;
|
||||
}
|
||||
if (inTrendWindow) trendSessions += day.SessionCount;
|
||||
if (inBaseline) baselineSessions += day.SessionCount;
|
||||
|
||||
foreach (var b in day.Buckets)
|
||||
{
|
||||
// Overrides are attributed to the prior path and can exist on a bucket with Count=0.
|
||||
var row = Row(b);
|
||||
if (inWindow)
|
||||
{
|
||||
row.Count += b.Count;
|
||||
row.DerivedCount += b.DerivedCount;
|
||||
row.Eligible7d += b.Eligible7d;
|
||||
row.ReEnrolled7d += b.ReEnrolled7d;
|
||||
row.OverriddenByAdmin += b.OverriddenByAdmin;
|
||||
row.OverriddenByLateCompletion += b.OverriddenByLateCompletion;
|
||||
row.OverriddenOther += b.OverriddenOther;
|
||||
derived += b.DerivedCount;
|
||||
}
|
||||
if (inTrendWindow) row.Window7.Count += b.Count;
|
||||
if (inBaseline) row.Baseline28.Count += b.Count;
|
||||
}
|
||||
}
|
||||
|
||||
var list = rows.Values
|
||||
.Where(r => r.Count > 0 || r.OverriddenByAdmin + r.OverriddenByLateCompletion + r.OverriddenOther > 0)
|
||||
.OrderByDescending(r => r.Count)
|
||||
.ThenBy(r => r.VerdictPath, StringComparer.Ordinal)
|
||||
.ThenBy(r => r.Status, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
|
||||
foreach (var r in list)
|
||||
{
|
||||
r.SharePct = Pct(r.Count, sessions);
|
||||
r.ReEnrollRatePct = r.Eligible7d >= MinEligibleForRate ? Pct(r.ReEnrolled7d, r.Eligible7d) : null;
|
||||
r.Window7.Sessions = trendSessions;
|
||||
r.Window7.SharePct = Pct(r.Window7.Count, trendSessions);
|
||||
r.Baseline28.Sessions = baselineSessions;
|
||||
r.Baseline28.SharePct = Pct(r.Baseline28.Count, baselineSessions);
|
||||
r.Lift = r.Baseline28.SharePct > 0 && trendSessions > 0
|
||||
? Math.Round(r.Window7.SharePct / r.Baseline28.SharePct, 2)
|
||||
: null;
|
||||
}
|
||||
|
||||
return new
|
||||
{
|
||||
success = true,
|
||||
tenantId = partition,
|
||||
windowDays = days,
|
||||
windowStart,
|
||||
windowEnd = Key(today),
|
||||
computedAt,
|
||||
versions = versions.OrderBy(v => v).ToArray(),
|
||||
totals = new { sessions, terminal, derived, days = daily.Count(d => string.CompareOrdinal(d.Date, windowStart) >= 0) },
|
||||
trend = new { windowDays = TrendWindowDays, baselineDays = TrendBaselineDays, windowSessions = trendSessions, baselineSessions },
|
||||
paths = list,
|
||||
alerts = Array.Empty<object>(),
|
||||
};
|
||||
}
|
||||
|
||||
private static string Key(DateTime d) => d.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture);
|
||||
|
||||
private static double Pct(int part, int whole) => whole <= 0 ? 0 : Math.Round(100.0 * part / whole, 1);
|
||||
}
|
||||
}
|
||||
@@ -464,6 +464,7 @@ public static class EndpointAccessPolicyCatalog
|
||||
new("GET", "global/metrics/fleet-health", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam),
|
||||
new("GET", "global/metrics/time-attribution", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam),
|
||||
new("GET", "global/metrics/device-journeys", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam),
|
||||
new("GET", "global/metrics/verdict-calibration", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam),
|
||||
new("GET", "global/apps/list", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam),
|
||||
new("GET", "global/apps/{appName}/analytics", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam),
|
||||
new("GET", "global/apps/{appName}/sessions", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam),
|
||||
|
||||
@@ -1102,6 +1102,19 @@ namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to cleanup old device-journey aggregates");
|
||||
}
|
||||
|
||||
// Verdict-calibration daily aggregates: same 180d window (regenerable; the radar
|
||||
// needs 35d, the matrix serves up to 180d).
|
||||
try
|
||||
{
|
||||
var deleted = await _metricsRepo.DeleteVerdictCalibrationAggregatesOlderThanAsync(now.AddDays(-usageMetricsRetentionDays));
|
||||
if (deleted > 0)
|
||||
_logger.LogInformation("Verdict-calibration aggregate cleanup: deleted {Count} rows older than {Days} days", deleted, usageMetricsRetentionDays);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to cleanup old verdict-calibration aggregates");
|
||||
}
|
||||
}
|
||||
/// <summary>
|
||||
/// Detects and cleans up orphaned events — events stored in the Events table
|
||||
|
||||
+259
@@ -0,0 +1,259 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
/// <summary>
|
||||
/// Partial: verdict-calibration daily aggregates (docs/backend/verdict-calibration.md) — the
|
||||
/// operator thermometer for the rule classifier: which verdict path produced how many
|
||||
/// sessions per day, how many of those were later overridden (admin / late completion /
|
||||
/// other), and how many devices re-enrolled within 7 days.
|
||||
/// </summary>
|
||||
public partial class MaintenanceService
|
||||
{
|
||||
/// <summary>App setting kill switch: "true" skips the sweep. Fail-open — the sweep only writes its own regenerable table.</summary>
|
||||
internal const string VerdictCalibrationKillSwitchSetting = "VerdictCalibrationSweepDisabled";
|
||||
|
||||
/// <summary>Rolling window: sessions started in the last 30 days (same as the F1/F2 sweeps).</summary>
|
||||
private const int VerdictCalibrationSweepDays = 30;
|
||||
|
||||
/// <summary>Re-enrollment proxy horizon: another terminal session of the same device within this many days.</summary>
|
||||
internal const int VerdictCalibrationReEnrollDays = 7;
|
||||
|
||||
/// <summary>
|
||||
/// Recomputes the verdict-calibration rows for the rolling window — idempotent per
|
||||
/// (tenant × StartedAt date) plus "global" mirror rows. Reads the window's sessions (one
|
||||
/// cross-partition scan, same as the F1/F2 sweeps) and each window tenant's device
|
||||
/// histories (one partition scan per tenant) for the re-enrollment proxy. Stale date rows
|
||||
/// a run did not regenerate are deleted so window sums never serve ghost counts.
|
||||
/// </summary>
|
||||
private async Task SweepVerdictCalibrationAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (string.Equals(_configuration[VerdictCalibrationKillSwitchSetting], "true", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
_logger.LogInformation("Verdict-calibration sweep skipped ({Setting}=true)", VerdictCalibrationKillSwitchSetting);
|
||||
return;
|
||||
}
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
var now = DateTime.UtcNow;
|
||||
var today = now.Date;
|
||||
var windowStart = today.AddDays(-VerdictCalibrationSweepDays);
|
||||
var windowEnd = today.AddDays(1);
|
||||
|
||||
var sessions = await _maintenanceRepo.GetSessionsByDateRangeAsync(windowStart, windowEnd);
|
||||
|
||||
var historiesByTenant = new Dictionary<string, List<DeviceHistory>>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var tenantId in sessions.Select(s => s.TenantId).Distinct(StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
historiesByTenant[tenantId] = await _metricsRepo.GetDeviceHistoriesByTenantAsync(tenantId);
|
||||
}
|
||||
|
||||
var aggregates = BuildVerdictCalibrationAggregates(sessions, historiesByTenant, now);
|
||||
var saved = 0;
|
||||
foreach (var aggregate in aggregates)
|
||||
{
|
||||
if (await _metricsRepo.SaveVerdictCalibrationAggregateAsync(aggregate))
|
||||
saved++;
|
||||
}
|
||||
|
||||
var generatedKeys = new HashSet<(string TenantId, string Date)>(aggregates.Select(a => (a.TenantId, a.Date)));
|
||||
var removedStale = 0;
|
||||
foreach (var partition in aggregates.Select(a => a.TenantId).Distinct().ToList())
|
||||
{
|
||||
var existingRows = await _metricsRepo.GetVerdictCalibrationAggregatesAsync(partition, windowStart, windowEnd.AddDays(-1));
|
||||
foreach (var row in existingRows)
|
||||
{
|
||||
if (generatedKeys.Contains((partition, row.Date))) continue;
|
||||
await _metricsRepo.DeleteVerdictCalibrationAggregateAsync(partition, row.Date);
|
||||
removedStale++;
|
||||
}
|
||||
}
|
||||
|
||||
sw.Stop();
|
||||
_logger.LogInformation(
|
||||
"Verdict-calibration sweep: {Sessions} window sessions across {Tenants} tenants, {Rows} rows written, {Removed} stale rows removed in {Ms}ms",
|
||||
sessions.Count, historiesByTenant.Count, saved, removedStale, sw.ElapsedMilliseconds);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Verdict-calibration sweep failed (non-fatal)");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pure aggregation core behind <see cref="SweepVerdictCalibrationAsync"/> — internal
|
||||
/// static so the contract is pinned by unit tests. Per session: the current
|
||||
/// (VerdictPath, Status) bucket gets <c>Count</c> (+ <c>DerivedCount</c> when the path was
|
||||
/// derived read-side), terminal sessions old enough get <c>Eligible7d</c> and, if the
|
||||
/// device's history chain shows another terminal session starting within 7 days of this
|
||||
/// one's end, <c>ReEnrolled7d</c>; a session carrying a prior verdict additionally
|
||||
/// increments the PRIOR bucket's <c>Overridden*</c> counter. Sessions inside a deletion
|
||||
/// cascade are skipped. Rows are emitted per (tenant, StartedAt date) and mirrored into
|
||||
/// the "global" partition; buckets are sorted for deterministic output.
|
||||
/// </summary>
|
||||
internal static List<VerdictCalibrationDailyAggregate> BuildVerdictCalibrationAggregates(
|
||||
IReadOnlyList<SessionSummary> sessions,
|
||||
IReadOnlyDictionary<string, List<DeviceHistory>> historiesByTenant,
|
||||
DateTime nowUtc)
|
||||
{
|
||||
var nextStartBySession = BuildNextStartLookup(historiesByTenant);
|
||||
var eligibleCutoff = nowUtc.AddDays(-VerdictCalibrationReEnrollDays);
|
||||
|
||||
var rows = new Dictionary<(string TenantId, string Date), RowAccumulator>();
|
||||
|
||||
RowAccumulator Row(string tenantId, string date)
|
||||
{
|
||||
var key = (tenantId, date);
|
||||
if (!rows.TryGetValue(key, out var acc))
|
||||
{
|
||||
acc = new RowAccumulator();
|
||||
rows[key] = acc;
|
||||
}
|
||||
return acc;
|
||||
}
|
||||
|
||||
foreach (var session in sessions)
|
||||
{
|
||||
if (session == null) continue;
|
||||
if (!string.IsNullOrEmpty(session.DeletionState) && session.DeletionState != "None")
|
||||
continue;
|
||||
|
||||
var date = session.StartedAt.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture);
|
||||
var (path, derived) = VerdictPathDerivation.Derive(session);
|
||||
var statusName = session.Status.ToString();
|
||||
var isTerminal = DeviceJourneyCalculator.IsTerminal(session.Status);
|
||||
|
||||
var eligible = false;
|
||||
var reEnrolled = false;
|
||||
if (isTerminal)
|
||||
{
|
||||
var end = session.CompletedAt ?? session.LastEventAt ?? session.StartedAt;
|
||||
eligible = end <= eligibleCutoff;
|
||||
if (eligible
|
||||
&& nextStartBySession.TryGetValue((session.TenantId, session.SessionId), out var next)
|
||||
&& next.NextStart.HasValue
|
||||
&& (next.NextStart.Value - next.End) < TimeSpan.FromDays(VerdictCalibrationReEnrollDays))
|
||||
{
|
||||
reEnrolled = true;
|
||||
}
|
||||
}
|
||||
|
||||
var overridden = ClassifyOverride(session);
|
||||
|
||||
foreach (var partition in new[] { session.TenantId, "global" })
|
||||
{
|
||||
var acc = Row(partition, date);
|
||||
acc.SessionCount++;
|
||||
if (isTerminal) acc.TerminalSessionCount++;
|
||||
|
||||
var bucket = acc.Bucket(path, statusName);
|
||||
bucket.Count++;
|
||||
if (derived) bucket.DerivedCount++;
|
||||
if (eligible) bucket.Eligible7d++;
|
||||
if (reEnrolled) bucket.ReEnrolled7d++;
|
||||
|
||||
if (overridden.HasValue)
|
||||
{
|
||||
var prior = acc.Bucket(session.PriorVerdictPath!, session.PriorStatus!);
|
||||
switch (overridden.Value)
|
||||
{
|
||||
case OverrideKind.Admin: prior.OverriddenByAdmin++; break;
|
||||
case OverrideKind.LateCompletion: prior.OverriddenByLateCompletion++; break;
|
||||
default: prior.OverriddenOther++; break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return rows
|
||||
.OrderBy(r => r.Key.TenantId, StringComparer.Ordinal)
|
||||
.ThenBy(r => r.Key.Date, StringComparer.Ordinal)
|
||||
.Select(r => new VerdictCalibrationDailyAggregate
|
||||
{
|
||||
TenantId = r.Key.TenantId,
|
||||
Date = r.Key.Date,
|
||||
Version = VerdictCalibrationDailyAggregate.CurrentVersion,
|
||||
SessionCount = r.Value.SessionCount,
|
||||
TerminalSessionCount = r.Value.TerminalSessionCount,
|
||||
Buckets = r.Value.Buckets.Values
|
||||
.OrderBy(b => b.VerdictPath, StringComparer.Ordinal)
|
||||
.ThenBy(b => b.Status, StringComparer.Ordinal)
|
||||
.ToList(),
|
||||
ComputedAt = nowUtc,
|
||||
})
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private enum OverrideKind { Admin, LateCompletion, Other }
|
||||
|
||||
/// <summary>
|
||||
/// A session whose current status overrode a prior verdict names the writer that did it:
|
||||
/// an admin mark (AdminMarkedAction set), a late agent completion (now Succeeded without
|
||||
/// admin involvement), or anything else (retro-reclassification, grace expiry, supersede).
|
||||
/// </summary>
|
||||
private static OverrideKind? ClassifyOverride(SessionSummary session)
|
||||
{
|
||||
if (string.IsNullOrEmpty(session.PriorVerdictPath) || string.IsNullOrEmpty(session.PriorStatus))
|
||||
return null;
|
||||
if (!string.IsNullOrEmpty(session.AdminMarkedAction))
|
||||
return OverrideKind.Admin;
|
||||
if (session.Status == SessionStatus.Succeeded)
|
||||
return OverrideKind.LateCompletion;
|
||||
return OverrideKind.Other;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// (tenant, sessionId) → this session's chain end and the next chain entry's StartedAt —
|
||||
/// the re-enrollment proxy source. Chains are ascending by StartedAt and capped at 20, so
|
||||
/// a window session older than the cap simply has no entry (counts as not re-enrolled
|
||||
/// only when eligible — the cap is a known, small bias toward under-counting).
|
||||
/// </summary>
|
||||
internal static Dictionary<(string TenantId, string SessionId), (DateTime End, DateTime? NextStart)> BuildNextStartLookup(
|
||||
IReadOnlyDictionary<string, List<DeviceHistory>> historiesByTenant)
|
||||
{
|
||||
var lookup = new Dictionary<(string, string), (DateTime, DateTime?)>();
|
||||
foreach (var pair in historiesByTenant)
|
||||
{
|
||||
foreach (var history in pair.Value)
|
||||
{
|
||||
var chain = history.Chain;
|
||||
for (var i = 0; i < chain.Count; i++)
|
||||
{
|
||||
var end = chain[i].CompletedAt ?? chain[i].StartedAt;
|
||||
DateTime? nextStart = i + 1 < chain.Count ? chain[i + 1].StartedAt : null;
|
||||
lookup[(pair.Key, chain[i].SessionId)] = (end, nextStart);
|
||||
}
|
||||
}
|
||||
}
|
||||
return lookup;
|
||||
}
|
||||
|
||||
private sealed class RowAccumulator
|
||||
{
|
||||
public int SessionCount;
|
||||
public int TerminalSessionCount;
|
||||
public readonly Dictionary<(string Path, string Status), VerdictCalibrationBucket> Buckets = new();
|
||||
|
||||
public VerdictCalibrationBucket Bucket(string path, string status)
|
||||
{
|
||||
var key = (path, status);
|
||||
if (!Buckets.TryGetValue(key, out var bucket))
|
||||
{
|
||||
bucket = new VerdictCalibrationBucket { VerdictPath = path, Status = status };
|
||||
Buckets[key] = bucket;
|
||||
}
|
||||
return bucket;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -158,6 +158,10 @@ namespace AutopilotMonitor.Functions.Services
|
||||
// F2 PR4: device-history chain heal (incl. deleted-session ref cleanup) + daily
|
||||
// FTR aggregates over the same rolling window. Fail-soft internally.
|
||||
await SweepDeviceJourneysAsync();
|
||||
// Verdict calibration: per-verdict-path daily buckets over the same rolling
|
||||
// window, AFTER the device-journey sweep so the re-enrollment proxy reads
|
||||
// freshly merged chains. Fail-soft internally.
|
||||
await SweepVerdictCalibrationAsync();
|
||||
// Plan §5 PR6 / §16 R14: session retention fanout extracted out of the 2h timer
|
||||
// into the dedicated 12h SessionDeletionMaintenanceFunction so cascade-lifecycle
|
||||
// work has independent cadence + kill-switch + OpsEvent watchdogs. The non-session
|
||||
@@ -232,6 +236,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
// (rolling 30d windows, cheap once converged).
|
||||
await SweepTimeAttributionAsync();
|
||||
await SweepDeviceJourneysAsync();
|
||||
await SweepVerdictCalibrationAsync();
|
||||
|
||||
if (!aggregateOnly)
|
||||
{
|
||||
|
||||
+2
@@ -83,6 +83,8 @@ namespace AutopilotMonitor.Functions.Services.Offboarding
|
||||
// whole-row deletion besides chain-emptying.
|
||||
Constants.TableNames.DeviceHistories,
|
||||
Constants.TableNames.DeviceJourneyAggregates,
|
||||
// Verdict calibration daily buckets (PK=TenantId) — regenerable from Sessions, wiped with the tenant.
|
||||
Constants.TableNames.VerdictCalibrationAggregates,
|
||||
// Pre-write config snapshots (PK=tenantId). Wiped in the bulk phase; the
|
||||
// TenantConfiguration row itself is deleted LAST via SafeWipe (a delete, not a
|
||||
// save), so the wipe cannot re-create a snapshot afterwards. The admin-config
|
||||
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json;
|
||||
using System.Threading.Tasks;
|
||||
using Azure.Data.Tables;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
/// <summary>
|
||||
/// Partial: verdict-calibration daily aggregate persistence (docs/backend/verdict-calibration.md).
|
||||
/// VerdictCalibrationAggregates: PK=TenantId ("global" for the cross-tenant row),
|
||||
/// RK="{yyyy-MM-dd}" — one row per (tenant, StartedAt date) holding the per-verdict-path
|
||||
/// buckets as a JSON column. Recomputed whole (Replace) by the maintenance sweep, 180d
|
||||
/// retention, regenerable from Sessions + DeviceHistories.
|
||||
/// </summary>
|
||||
public partial class TableStorageService
|
||||
{
|
||||
public async Task<bool> SaveVerdictCalibrationAggregateAsync(VerdictCalibrationDailyAggregate aggregate)
|
||||
{
|
||||
try
|
||||
{
|
||||
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.VerdictCalibrationAggregates);
|
||||
await tableClient.UpsertEntityAsync(BuildVerdictCalibrationAggregateEntity(aggregate), TableUpdateMode.Replace);
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to save verdict-calibration aggregate {Date} for tenant {TenantId}",
|
||||
aggregate.Date, aggregate.TenantId);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<List<VerdictCalibrationDailyAggregate>> GetVerdictCalibrationAggregatesAsync(
|
||||
string tenantId, DateTime startDate, DateTime endDate)
|
||||
{
|
||||
if (tenantId != "global")
|
||||
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
|
||||
try
|
||||
{
|
||||
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.VerdictCalibrationAggregates);
|
||||
var filter = $"PartitionKey eq '{tenantId}' and RowKey ge '{startDate:yyyy-MM-dd}' and RowKey le '{endDate:yyyy-MM-dd}'";
|
||||
var results = new List<VerdictCalibrationDailyAggregate>();
|
||||
await foreach (var entity in tableClient.QueryAsync<TableEntity>(filter: filter))
|
||||
{
|
||||
results.Add(MapToVerdictCalibrationAggregate(entity));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to query verdict-calibration aggregates for tenant {TenantId}", tenantId);
|
||||
return new List<VerdictCalibrationDailyAggregate>();
|
||||
}
|
||||
}
|
||||
|
||||
// internal static: pinned by round-trip tests together with MapToVerdictCalibrationAggregate.
|
||||
internal static TableEntity BuildVerdictCalibrationAggregateEntity(VerdictCalibrationDailyAggregate aggregate)
|
||||
{
|
||||
return new TableEntity(aggregate.TenantId, aggregate.Date)
|
||||
{
|
||||
["Date"] = aggregate.Date,
|
||||
["Version"] = aggregate.Version,
|
||||
["SessionCount"] = aggregate.SessionCount,
|
||||
["TerminalSessionCount"] = aggregate.TerminalSessionCount,
|
||||
["BucketsJson"] = JsonSerializer.Serialize(aggregate.Buckets),
|
||||
["ComputedAt"] = EnsureUtc(aggregate.ComputedAt),
|
||||
};
|
||||
}
|
||||
|
||||
internal static VerdictCalibrationDailyAggregate MapToVerdictCalibrationAggregate(TableEntity entity)
|
||||
{
|
||||
return new VerdictCalibrationDailyAggregate
|
||||
{
|
||||
TenantId = entity.PartitionKey,
|
||||
Date = entity.GetString("Date") ?? entity.RowKey,
|
||||
Version = entity.GetInt32("Version") ?? 0,
|
||||
SessionCount = entity.GetInt32("SessionCount") ?? 0,
|
||||
TerminalSessionCount = entity.GetInt32("TerminalSessionCount") ?? 0,
|
||||
Buckets = DeserializeJsonColumn<VerdictCalibrationBucket>(entity.GetString("BucketsJson")),
|
||||
ComputedAt = entity.GetDateTimeOffset("ComputedAt")?.UtcDateTime ?? DateTime.MinValue,
|
||||
};
|
||||
}
|
||||
|
||||
public async Task DeleteVerdictCalibrationAggregateAsync(string tenantId, string dateKey)
|
||||
{
|
||||
try
|
||||
{
|
||||
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.VerdictCalibrationAggregates);
|
||||
await tableClient.DeleteEntityAsync(tenantId, dateKey);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to delete stale verdict-calibration aggregate {Date} for tenant {TenantId}",
|
||||
dateKey, tenantId);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<int> DeleteVerdictCalibrationAggregatesOlderThanAsync(DateTime cutoffDate)
|
||||
{
|
||||
try
|
||||
{
|
||||
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.VerdictCalibrationAggregates);
|
||||
var filter = $"RowKey lt '{cutoffDate:yyyy-MM-dd}'";
|
||||
var deleted = 0;
|
||||
await foreach (var entity in tableClient.QueryAsync<TableEntity>(
|
||||
filter: filter, select: new[] { "PartitionKey", "RowKey" }))
|
||||
{
|
||||
await tableClient.DeleteEntityAsync(entity.PartitionKey, entity.RowKey);
|
||||
deleted++;
|
||||
}
|
||||
return deleted;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to delete old verdict-calibration aggregates");
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -136,6 +136,9 @@ describe('role catalog snapshot — privilege-leak guard', () => {
|
||||
// routing to the GlobalReadOrAdmin variant — same placement as get_app_install_metrics.
|
||||
'get_time_attribution',
|
||||
'get_usage_metrics',
|
||||
// Platform scope (GA + Global Reader): GlobalReadOrAdmin endpoint, operator-only
|
||||
// classifier diagnostics.
|
||||
'get_verdict_calibration',
|
||||
'get_vulnerability_summary',
|
||||
'list_blocked_devices',
|
||||
'list_session_annotations',
|
||||
@@ -197,6 +200,7 @@ describe('role catalog snapshot — privilege-leak guard', () => {
|
||||
'get_platform_metrics',
|
||||
'get_tenant_config',
|
||||
'get_tenant_config_schema',
|
||||
'get_verdict_calibration',
|
||||
'list_blocked_devices',
|
||||
'list_session_annotations',
|
||||
'list_session_reports',
|
||||
|
||||
@@ -838,6 +838,45 @@ export function registerAdminTools(server: McpServer, ga: boolean, strictGa: boo
|
||||
})
|
||||
);
|
||||
|
||||
// Tool: get_verdict_calibration — platform scope (GA + Global Reader). Operator diagnostics for
|
||||
// the session-verdict classifier (docs/backend/verdict-calibration.md) — never a customer surface.
|
||||
if (ga) server.registerTool(
|
||||
'get_verdict_calibration',
|
||||
{
|
||||
title: 'Verdict Calibration (classifier thermometer)',
|
||||
description:
|
||||
'Answer "which code path produced our session verdicts, and does any of them look wrong?". Per verdict ' +
|
||||
'path (VerdictPaths vocabulary — agent:complete, agent:failed, sweep:r5_incomplete, maxlife:r6, ' +
|
||||
'late:r4, retro:r6, rule:<id>, manual:failed, register:superseded, legacy:* for pre-instrumentation ' +
|
||||
'rows derived read-side): session count and share in the window, derivedCount (rows attributed by ' +
|
||||
'derivation — weaker evidence), the 7-day re-enrollment proxy (eligible7d = terminal sessions old ' +
|
||||
'enough to judge, reEnrolled7d = the same device registered another terminal session within 7 days; ' +
|
||||
'reEnrollRatePct is null below 20 eligible), the correction stream attributed to the PRIOR path ' +
|
||||
'(overriddenByAdmin / overriddenByLateCompletion / overriddenOther — a path that is frequently ' +
|
||||
'overridden was wrong), and a 7d-vs-28d share trend (window7, baseline28, lift; lift null without ' +
|
||||
'baseline). Read it as calibration: a Succeeded path whose re-enrollment rate sits at Failed level ' +
|
||||
'reconciles too boldly; an Incomplete path whose rate matches the Succeeded background is too ' +
|
||||
'cautious; a rising sweep:*/maxlife:* share is an agent-liveness signal, not a classifier one. ' +
|
||||
'Omit tenantId for the cross-tenant aggregate, or pass tenantId to scope to one tenant. Rows come ' +
|
||||
'from the 2-hourly maintenance sweep (computedAt); empty before its first run.',
|
||||
inputSchema: {
|
||||
tenantId: z.string().optional().describe('Optional — scope to one tenant. Omit for the cross-tenant aggregate.'),
|
||||
days: z.coerce.number().int().min(1).max(180).optional()
|
||||
.describe('Window in days (default 30, max 180 — aggregate retention). The 7d/28d trend is always today-anchored.'),
|
||||
},
|
||||
annotations: READ_ONLY,
|
||||
},
|
||||
async (args) => withToolTelemetry('get_verdict_calibration', args, async () => {
|
||||
try {
|
||||
const { tenantId, days } = args;
|
||||
const data = await apiFetch(`/api/global/metrics/verdict-calibration${buildQuery({ tenantId, days })}`);
|
||||
return toolResultText(data, MAX_RESULT_SIZE_CHARS.adminStream);
|
||||
} catch (error: unknown) {
|
||||
return toolError('get_verdict_calibration', args, error);
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
// Tool 18: list_session_reports — Global Admin only; not registered for normal users.
|
||||
if (ga) server.registerTool(
|
||||
'list_session_reports',
|
||||
|
||||
@@ -988,6 +988,11 @@ namespace AutopilotMonitor.Shared
|
||||
// like UsageMetrics.
|
||||
public const string DeviceHistories = "DeviceHistories";
|
||||
public const string DeviceJourneyAggregates = "DeviceJourneyAggregates";
|
||||
// VerdictCalibrationAggregates: PK=TenantId (or "global"), RK="{yyyy-MM-dd}" — daily
|
||||
// per-verdict-path buckets (count, derived, re-enrolled-within-7d, overridden-by-admin/
|
||||
// late-completion) the operator calibration matrix + drift radar read
|
||||
// (docs/backend/verdict-calibration.md). Additive across days; 180d retention; regenerable.
|
||||
public const string VerdictCalibrationAggregates = "VerdictCalibrationAggregates";
|
||||
|
||||
// Configuration tables
|
||||
public const string TenantConfiguration = "TenantConfiguration";
|
||||
@@ -1198,6 +1203,7 @@ namespace AutopilotMonitor.Shared
|
||||
TimeAttributionAggregates,
|
||||
DeviceHistories,
|
||||
DeviceJourneyAggregates,
|
||||
VerdictCalibrationAggregates,
|
||||
TenantConfiguration,
|
||||
AdminConfiguration,
|
||||
GlobalAdmins,
|
||||
|
||||
@@ -128,6 +128,14 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
/// <summary>Deletes one daily FTR row — the sweep's stale-bucket reconcile.</summary>
|
||||
Task DeleteDeviceJourneyAggregateAsync(string tenantId, string dateKey);
|
||||
Task<int> DeleteDeviceJourneyAggregatesOlderThanAsync(DateTime cutoffDate);
|
||||
|
||||
// --- Verdict calibration (docs/backend/verdict-calibration.md) ---
|
||||
Task<bool> SaveVerdictCalibrationAggregateAsync(VerdictCalibrationDailyAggregate aggregate);
|
||||
/// <summary>Daily verdict-path rows of one tenant partition ("global" allowed), inclusive date range; counts are additive across days.</summary>
|
||||
Task<List<VerdictCalibrationDailyAggregate>> GetVerdictCalibrationAggregatesAsync(string tenantId, DateTime startDate, DateTime endDate);
|
||||
/// <summary>Deletes one daily row — the sweep's stale-bucket reconcile.</summary>
|
||||
Task DeleteVerdictCalibrationAggregateAsync(string tenantId, string dateKey);
|
||||
Task<int> DeleteVerdictCalibrationAggregatesOlderThanAsync(DateTime cutoffDate);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace AutopilotMonitor.Shared.Models
|
||||
{
|
||||
/// <summary>
|
||||
/// One (verdict path × status) bucket of a daily verdict-calibration row. <c>Count</c> is the
|
||||
/// sessions whose CURRENT status was produced by this path; the <c>Overridden*</c> counters
|
||||
/// are sessions that once carried this path and were then overridden (their
|
||||
/// <c>PriorVerdictPath</c> names this bucket) — the correction stream. Re-enrollment is the
|
||||
/// delayed proxy signal: of the sessions old enough to judge (<see cref="Eligible7d"/>), how
|
||||
/// many saw the same device register another terminal session within 7 days.
|
||||
/// </summary>
|
||||
public class VerdictCalibrationBucket
|
||||
{
|
||||
/// <summary><see cref="VerdictPaths"/> value, or the read-side derivation for pre-instrumentation rows.</summary>
|
||||
public string VerdictPath { get; set; } = string.Empty;
|
||||
|
||||
/// <summary><see cref="SessionStatus"/> name the path produced.</summary>
|
||||
public string Status { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Sessions currently on this path/status.</summary>
|
||||
public int Count { get; set; }
|
||||
|
||||
/// <summary>Of <see cref="Count"/>: rows without a stamped VerdictPath, attributed by <c>VerdictPathDerivation</c> (weaker evidence).</summary>
|
||||
public int DerivedCount { get; set; }
|
||||
|
||||
/// <summary>Of <see cref="Count"/>: sessions whose terminal moment lies ≥ 7 days before the compute time — the re-enrollment denominator.</summary>
|
||||
public int Eligible7d { get; set; }
|
||||
|
||||
/// <summary>Of <see cref="Eligible7d"/>: the device registered another terminal session within 7 days of this one's end.</summary>
|
||||
public int ReEnrolled7d { get; set; }
|
||||
|
||||
/// <summary>Sessions that carried this path and were then flipped by an administrator (MarkSessionSucceeded/Failed).</summary>
|
||||
public int OverriddenByAdmin { get; set; }
|
||||
|
||||
/// <summary>Sessions that carried this path and were then upgraded to Succeeded by a late agent completion report.</summary>
|
||||
public int OverriddenByLateCompletion { get; set; }
|
||||
|
||||
/// <summary>Sessions that carried this path and were overridden by any other writer (retro-reclassification, grace expiry, supersede).</summary>
|
||||
public int OverriddenOther { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Daily verdict-calibration rollup for one (tenant, date) — the operator's thermometer for the
|
||||
/// rule classifier (docs/backend/verdict-calibration.md). A session buckets on its StartedAt
|
||||
/// date (same convention as every other daily aggregate; the rolling sweep re-buckets
|
||||
/// late-terminating sessions idempotently). Every status is counted — non-terminal rows are
|
||||
/// part of the picture (a growing Stalled share is a signal) — and
|
||||
/// <see cref="TerminalSessionCount"/> carries the share denominator for terminal paths.
|
||||
/// Counts are additive across days; a window is the sum of daily rows. "global" mirror rows
|
||||
/// sum all tenants. Regenerable from Sessions + DeviceHistories at any time.
|
||||
/// </summary>
|
||||
public class VerdictCalibrationDailyAggregate
|
||||
{
|
||||
/// <summary>Bucketing/derivation algorithm version — a definition change never silently mixes semantics.</summary>
|
||||
public const int CurrentVersion = 1;
|
||||
|
||||
/// <summary>Calendar day (UTC, "yyyy-MM-dd") the sessions STARTED.</summary>
|
||||
public string Date { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Tenant GUID, or "global" for the cross-tenant row.</summary>
|
||||
public string TenantId { get; set; } = string.Empty;
|
||||
|
||||
public int Version { get; set; } = CurrentVersion;
|
||||
|
||||
/// <summary>All sessions started on this date (any status).</summary>
|
||||
public int SessionCount { get; set; }
|
||||
|
||||
/// <summary>Sessions whose status is Succeeded / Failed / Incomplete — the share denominator for terminal paths.</summary>
|
||||
public int TerminalSessionCount { get; set; }
|
||||
|
||||
/// <summary>Ordered by VerdictPath, then Status (ordinal) for deterministic output.</summary>
|
||||
public List<VerdictCalibrationBucket> Buckets { get; set; } = new List<VerdictCalibrationBucket>();
|
||||
|
||||
public DateTime ComputedAt { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -5,11 +5,13 @@ import { type MetricsSectionId } from "../metricsNavSections";
|
||||
import { SectionAgentMetrics } from "../sections/SectionAgentMetrics";
|
||||
import { SectionPlatformUsage } from "../sections/SectionPlatformUsage";
|
||||
import { SectionMcpUsage } from "../sections/SectionMcpUsage";
|
||||
import { SectionVerdictCalibration } from "../sections/SectionVerdictCalibration";
|
||||
|
||||
const SECTION_COMPONENTS: Record<MetricsSectionId, React.ComponentType> = {
|
||||
"platform-metrics": SectionAgentMetrics,
|
||||
"usage": SectionPlatformUsage,
|
||||
"mcp-usage": SectionMcpUsage,
|
||||
"verdict-calibration": SectionVerdictCalibration,
|
||||
};
|
||||
|
||||
export function SectionClient({ section }: { section: string }) {
|
||||
|
||||
@@ -2,6 +2,7 @@ export const METRICS_NAV_SECTIONS = [
|
||||
{ id: "platform-metrics", label: "Platform Metrics", description: "Agent performance, delivery latency, crash rates, and platform health" },
|
||||
{ id: "usage", label: "Platform Usage", description: "Platform usage statistics across all tenants" },
|
||||
{ id: "mcp-usage", label: "MCP Usage", description: "MCP API usage metrics across all users" },
|
||||
{ id: "verdict-calibration", label: "Verdict Calibration", description: "Which code path produced each session verdict, overrides and re-enrollment — classifier diagnostics" },
|
||||
] as const;
|
||||
|
||||
export type MetricsSectionId = (typeof METRICS_NAV_SECTIONS)[number]["id"];
|
||||
|
||||
+225
@@ -0,0 +1,225 @@
|
||||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import { useAuth } from "../../../../contexts/AuthContext";
|
||||
import { authenticatedFetch, TokenExpiredError } from "@/lib/authenticatedFetch";
|
||||
import { api } from "@/lib/api";
|
||||
import { useAggregatedAdminScope } from "@/hooks";
|
||||
import { TenantScopeSelector } from "@/components/TenantScopeSelector";
|
||||
import { SessionStatusBadge } from "@/components/SessionStatusBadge";
|
||||
import { SegmentedControl, TIME_RANGE_OPTIONS } from "@/components/SegmentedControl";
|
||||
import {
|
||||
VERDICT_PATH_ORIGIN_LABELS,
|
||||
groupPathsByOrigin,
|
||||
trendGlyph,
|
||||
type VerdictCalibrationPath,
|
||||
type VerdictCalibrationResponse,
|
||||
} from "./verdictCalibrationLogic";
|
||||
|
||||
type DateRange = "7d" | "30d" | "90d";
|
||||
|
||||
const RANGE_DAYS: Record<DateRange, number> = { "7d": 7, "30d": 30, "90d": 90 };
|
||||
|
||||
export function SectionVerdictCalibration() {
|
||||
const { getAccessToken } = useAuth();
|
||||
const scope = useAggregatedAdminScope({ defaultAggregated: true });
|
||||
const { selectedTenantId, scopeInitialized } = scope;
|
||||
const [data, setData] = useState<VerdictCalibrationResponse | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [dateRange, setDateRange] = useState<DateRange>("30d");
|
||||
|
||||
const fetchData = useCallback(
|
||||
async (range: DateRange, tenantId: string) => {
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
try {
|
||||
const res = await authenticatedFetch(
|
||||
api.metrics.globalVerdictCalibration(RANGE_DAYS[range], tenantId || undefined),
|
||||
getAccessToken
|
||||
);
|
||||
if (res.status === 404) {
|
||||
// Deploy skew: backend without the route yet — reads as "no rows", never as an error.
|
||||
setData(null);
|
||||
return;
|
||||
}
|
||||
if (!res.ok) throw new Error(`Verdict calibration: ${res.status}`);
|
||||
setData((await res.json()) as VerdictCalibrationResponse);
|
||||
} catch (err) {
|
||||
if (err instanceof TokenExpiredError) {
|
||||
setError("Session expired. Please refresh the page.");
|
||||
} else {
|
||||
setError(err instanceof Error ? err.message : "Failed to load verdict calibration");
|
||||
}
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
},
|
||||
[getAccessToken]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (!scopeInitialized) return;
|
||||
const run = async () => {
|
||||
await fetchData(dateRange, selectedTenantId);
|
||||
};
|
||||
void run();
|
||||
}, [fetchData, dateRange, selectedTenantId, scopeInitialized]);
|
||||
|
||||
const groups = data ? groupPathsByOrigin(data.paths) : [];
|
||||
|
||||
return (
|
||||
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8 space-y-6">
|
||||
<div className="flex flex-col gap-3 sm:flex-row sm:items-center sm:justify-between">
|
||||
<div>
|
||||
<h2 className="text-lg font-semibold text-gray-900">Verdict Calibration</h2>
|
||||
<p className="text-sm text-gray-500">
|
||||
Which code path produced each session verdict, how often it was overridden, and whether the device came back.
|
||||
Platform-internal classifier diagnostics — correlations only, never causal claims.
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex flex-wrap items-center gap-3">
|
||||
<TenantScopeSelector scope={scope} allowAggregated />
|
||||
<SegmentedControl options={TIME_RANGE_OPTIONS} value={dateRange} onChange={(v) => setDateRange(v as DateRange)} />
|
||||
<button
|
||||
onClick={() => fetchData(dateRange, selectedTenantId)}
|
||||
disabled={loading}
|
||||
className="px-3 py-1.5 text-sm bg-white border border-gray-300 rounded-md hover:bg-gray-50 disabled:opacity-50"
|
||||
>
|
||||
Refresh
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="bg-red-50 border border-red-200 text-red-700 text-sm rounded-md px-4 py-3">{error}</div>
|
||||
)}
|
||||
|
||||
{data && (
|
||||
<div className="grid grid-cols-2 md:grid-cols-4 gap-4">
|
||||
<Stat label="Sessions in window" value={data.totals.sessions.toLocaleString()} hint={`${data.totals.days} day rows · ${data.windowStart} – ${data.windowEnd}`} />
|
||||
<Stat label="Terminal" value={data.totals.terminal.toLocaleString()} hint="Succeeded / Failed / Incomplete" />
|
||||
<Stat label="Derived attribution" value={data.totals.derived.toLocaleString()} hint="rows without a stamped path (pre-instrumentation)" />
|
||||
<Stat
|
||||
label="Trend basis"
|
||||
value={`${data.trend.windowSessions.toLocaleString()} / ${data.trend.baselineSessions.toLocaleString()}`}
|
||||
hint={`last ${data.trend.windowDays}d vs prior ${data.trend.baselineDays}d sessions`}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="bg-white shadow rounded-lg p-6">
|
||||
{loading && !data ? (
|
||||
<p className="text-sm text-gray-500">Loading…</p>
|
||||
) : !data || data.paths.length === 0 ? (
|
||||
<p className="text-sm text-gray-500">
|
||||
No calibration rows yet. The maintenance sweep writes them every two hours; the first rows appear after its
|
||||
next pass.
|
||||
</p>
|
||||
) : (
|
||||
<div className="overflow-x-auto">
|
||||
<table className="min-w-full divide-y divide-gray-200 text-sm">
|
||||
<thead>
|
||||
<tr className="text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
<th className="px-3 py-2">Verdict path</th>
|
||||
<th className="px-3 py-2">Status</th>
|
||||
<th className="px-3 py-2 text-right">Sessions</th>
|
||||
<th className="px-3 py-2 text-right">Share</th>
|
||||
<th className="px-3 py-2 text-right" title="Share in the last 7 days vs the prior 28 days (lift = window share ÷ baseline share)">
|
||||
7d trend
|
||||
</th>
|
||||
<th className="px-3 py-2 text-right" title="Of terminal sessions at least 7 days old: the same device registered another terminal session within 7 days. Blank below 20 eligible sessions.">
|
||||
Re-enrolled ≤7d
|
||||
</th>
|
||||
<th className="px-3 py-2 text-right" title="Sessions that carried this path and were then overridden: by an administrator / by a late agent completion / by another writer">
|
||||
Overridden (admin / late / other)
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-100">
|
||||
{groups.map((group) => (
|
||||
<GroupRows key={group.origin} origin={group.origin} rows={group.rows} />
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
)}
|
||||
{data?.computedAt && (
|
||||
<p className="mt-3 text-xs text-gray-400">
|
||||
Rows computed {new Date(data.computedAt).toLocaleString()} · algorithm version{" "}
|
||||
{data.versions.join(", ") || "—"}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function Stat({ label, value, hint }: { label: string; value: string; hint: string }) {
|
||||
return (
|
||||
<div className="bg-white shadow rounded-lg p-4">
|
||||
<p className="text-xs font-medium text-gray-500 uppercase tracking-wider">{label}</p>
|
||||
<p className="mt-1 text-2xl font-semibold text-gray-900">{value}</p>
|
||||
<p className="text-xs text-gray-400">{hint}</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function GroupRows({ origin, rows }: { origin: string; rows: VerdictCalibrationPath[] }) {
|
||||
return (
|
||||
<>
|
||||
<tr className="bg-gray-50">
|
||||
<td colSpan={7} className="px-3 py-1.5 text-xs font-semibold text-gray-600">
|
||||
{VERDICT_PATH_ORIGIN_LABELS[origin] ?? origin}
|
||||
</td>
|
||||
</tr>
|
||||
{rows.map((row) => (
|
||||
<PathRow key={`${row.verdictPath}|${row.status}`} row={row} />
|
||||
))}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function PathRow({ row }: { row: VerdictCalibrationPath }) {
|
||||
const trend = trendGlyph(row);
|
||||
const overridden = row.overriddenByAdmin + row.overriddenByLateCompletion + row.overriddenOther;
|
||||
return (
|
||||
<tr>
|
||||
<td className="px-3 py-2 font-mono text-xs text-gray-800 whitespace-nowrap">
|
||||
{row.verdictPath}
|
||||
{row.derivedCount > 0 && (
|
||||
<span
|
||||
className="ml-2 inline-flex items-center px-2 py-0.5 rounded text-xs font-medium bg-gray-100 text-gray-600 border border-gray-200"
|
||||
title={`${row.derivedCount} of ${row.count} attributed read-side (no stamped path)`}
|
||||
>
|
||||
derived {row.derivedCount}
|
||||
</span>
|
||||
)}
|
||||
</td>
|
||||
<td className="px-3 py-2">
|
||||
<SessionStatusBadge status={row.status} />
|
||||
</td>
|
||||
<td className="px-3 py-2 text-right tabular-nums">{row.count.toLocaleString()}</td>
|
||||
<td className="px-3 py-2 text-right tabular-nums">{row.sharePct.toFixed(1)}%</td>
|
||||
<td
|
||||
className={`px-3 py-2 text-right tabular-nums ${trend.className}`}
|
||||
title={`last 7d: ${row.window7.count} of ${row.window7.sessions} (${row.window7.sharePct.toFixed(1)}%) · prior 28d: ${row.baseline28.count} of ${row.baseline28.sessions} (${row.baseline28.sharePct.toFixed(1)}%)`}
|
||||
>
|
||||
{trend.text}
|
||||
</td>
|
||||
<td
|
||||
className="px-3 py-2 text-right tabular-nums"
|
||||
title={`${row.reEnrolled7d} of ${row.eligible7d} eligible`}
|
||||
>
|
||||
{row.reEnrollRatePct == null ? <span className="text-gray-400">— (n={row.eligible7d})</span> : `${row.reEnrollRatePct.toFixed(1)}%`}
|
||||
</td>
|
||||
<td className="px-3 py-2 text-right tabular-nums" title={overridden === 0 ? undefined : `${overridden} overridden in total`}>
|
||||
{overridden === 0 ? (
|
||||
<span className="text-gray-400">—</span>
|
||||
) : (
|
||||
`${row.overriddenByAdmin} / ${row.overriddenByLateCompletion} / ${row.overriddenOther}`
|
||||
)}
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
}
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
VERDICT_PATH_ORIGIN_LABELS,
|
||||
groupPathsByOrigin,
|
||||
pathOrigin,
|
||||
trendGlyph,
|
||||
type VerdictCalibrationPath,
|
||||
} from "../verdictCalibrationLogic";
|
||||
|
||||
function row(verdictPath: string, overrides: Partial<VerdictCalibrationPath> = {}): VerdictCalibrationPath {
|
||||
return {
|
||||
verdictPath,
|
||||
status: "Succeeded",
|
||||
count: 10,
|
||||
sharePct: 10,
|
||||
derivedCount: 0,
|
||||
eligible7d: 10,
|
||||
reEnrolled7d: 1,
|
||||
reEnrollRatePct: null,
|
||||
overriddenByAdmin: 0,
|
||||
overriddenByLateCompletion: 0,
|
||||
overriddenOther: 0,
|
||||
window7: { count: 5, sessions: 50, sharePct: 10 },
|
||||
baseline28: { count: 20, sessions: 200, sharePct: 10 },
|
||||
lift: 1,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("pathOrigin", () => {
|
||||
it("splits on the first colon and tolerates rule ids with colons-free detail", () => {
|
||||
expect(pathOrigin("sweep:r5_incomplete")).toBe("sweep");
|
||||
expect(pathOrigin("rule:ANALYZE-ESP-001")).toBe("rule");
|
||||
expect(pathOrigin("legacy:unknown")).toBe("legacy");
|
||||
expect(pathOrigin("weird")).toBe("weird");
|
||||
});
|
||||
});
|
||||
|
||||
describe("groupPathsByOrigin", () => {
|
||||
it("orders groups by the documented origin order, unknown origins last, rows untouched", () => {
|
||||
const groups = groupPathsByOrigin([
|
||||
row("zzz:x"),
|
||||
row("manual:failed"),
|
||||
row("sweep:r6"),
|
||||
row("sweep:r5_incomplete"),
|
||||
row("agent:complete"),
|
||||
]);
|
||||
expect(groups.map((g) => g.origin)).toEqual(["agent", "sweep", "manual", "zzz"]);
|
||||
expect(groups[1].rows.map((r) => r.verdictPath)).toEqual(["sweep:r6", "sweep:r5_incomplete"]);
|
||||
});
|
||||
|
||||
it("has a label for every origin in the order list", () => {
|
||||
for (const origin of ["agent", "ingest", "sweep", "maxlife", "late", "retro", "register", "rule", "manual", "legacy"]) {
|
||||
expect(VERDICT_PATH_ORIGIN_LABELS[origin]).toBeTruthy();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("trendGlyph", () => {
|
||||
it("flags a doubled share with an up arrow and a halved share with a down arrow", () => {
|
||||
expect(trendGlyph(row("sweep:r6", { lift: 2.4 })).text).toBe("↑ 2.4×");
|
||||
expect(trendGlyph(row("sweep:r6", { lift: 0.4 })).text).toBe("↓ 0.4×");
|
||||
expect(trendGlyph(row("sweep:r6", { lift: 1.3 })).text).toBe("≈ 1.3×");
|
||||
});
|
||||
|
||||
it("withholds the arrow below five window hits — one session can double a tiny share", () => {
|
||||
const g = trendGlyph(row("sweep:r6", { lift: 3, window7: { count: 3, sessions: 50, sharePct: 6 } }));
|
||||
expect(g.text).toBe("≈ 3.0×");
|
||||
expect(g.className).toContain("text-gray-400");
|
||||
});
|
||||
|
||||
it("never invents a lift without a baseline", () => {
|
||||
expect(trendGlyph(row("sweep:r6", { lift: null, baseline28: { count: 0, sessions: 200, sharePct: 0 } })).text).toBe("new");
|
||||
expect(trendGlyph(row("sweep:r6", { lift: null, window7: { count: 0, sessions: 0, sharePct: 0 }, baseline28: { count: 0, sessions: 0, sharePct: 0 } })).text).toBe("—");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,109 @@
|
||||
/**
|
||||
* Pure helpers for the Verdict Calibration matrix (docs/backend/verdict-calibration.md).
|
||||
* Wire shape mirrors GetVerdictCalibrationFunction / VerdictCalibrationResponse (camelCase).
|
||||
*/
|
||||
|
||||
export interface TrendWindow {
|
||||
count: number;
|
||||
sessions: number;
|
||||
sharePct: number;
|
||||
}
|
||||
|
||||
export interface VerdictCalibrationPath {
|
||||
verdictPath: string;
|
||||
status: string;
|
||||
count: number;
|
||||
sharePct: number;
|
||||
derivedCount: number;
|
||||
eligible7d: number;
|
||||
reEnrolled7d: number;
|
||||
/** null below 20 eligible sessions — never a rate on a handful */
|
||||
reEnrollRatePct: number | null;
|
||||
overriddenByAdmin: number;
|
||||
overriddenByLateCompletion: number;
|
||||
overriddenOther: number;
|
||||
window7: TrendWindow;
|
||||
baseline28: TrendWindow;
|
||||
/** window share ÷ baseline share; null without a baseline */
|
||||
lift: number | null;
|
||||
}
|
||||
|
||||
export interface VerdictCalibrationResponse {
|
||||
success: boolean;
|
||||
tenantId: string;
|
||||
windowDays: number;
|
||||
windowStart: string;
|
||||
windowEnd: string;
|
||||
computedAt: string | null;
|
||||
versions: number[];
|
||||
totals: { sessions: number; terminal: number; derived: number; days: number };
|
||||
trend: { windowDays: number; baselineDays: number; windowSessions: number; baselineSessions: number };
|
||||
paths: VerdictCalibrationPath[];
|
||||
alerts: unknown[];
|
||||
}
|
||||
|
||||
/** The `origin` half of a verdict path (`sweep` for `sweep:r6`); the whole string without a colon. */
|
||||
export function pathOrigin(path: string): string {
|
||||
const i = path.indexOf(":");
|
||||
return i < 0 ? path : path.slice(0, i);
|
||||
}
|
||||
|
||||
/** Display order + labels for the origin groups; unknown origins sort last, alphabetically. */
|
||||
export const VERDICT_PATH_ORIGIN_LABELS: Record<string, string> = {
|
||||
agent: "Agent-declared",
|
||||
ingest: "Ingest mapping",
|
||||
sweep: "Maintenance sweep (silence classifier)",
|
||||
maxlife: "Agent max-lifetime shutdown (silence classifier)",
|
||||
late: "Late telemetry reconcile (silence classifier)",
|
||||
retro: "Retro reclassification",
|
||||
register: "Session registration",
|
||||
rule: "Analyze rule (MarkSessionAsFailed)",
|
||||
manual: "Administrator",
|
||||
legacy: "Pre-instrumentation (derived, origin unknown)",
|
||||
};
|
||||
|
||||
const ORIGIN_ORDER = Object.keys(VERDICT_PATH_ORIGIN_LABELS);
|
||||
|
||||
export interface OriginGroup {
|
||||
origin: string;
|
||||
rows: VerdictCalibrationPath[];
|
||||
}
|
||||
|
||||
/** Groups rows by origin in display order; rows inside a group keep the server order (count desc). */
|
||||
export function groupPathsByOrigin(paths: VerdictCalibrationPath[]): OriginGroup[] {
|
||||
const byOrigin = new Map<string, VerdictCalibrationPath[]>();
|
||||
for (const p of paths) {
|
||||
const origin = pathOrigin(p.verdictPath);
|
||||
const list = byOrigin.get(origin);
|
||||
if (list) list.push(p);
|
||||
else byOrigin.set(origin, [p]);
|
||||
}
|
||||
return Array.from(byOrigin.entries())
|
||||
.sort(([a], [b]) => {
|
||||
const ia = ORIGIN_ORDER.indexOf(a);
|
||||
const ib = ORIGIN_ORDER.indexOf(b);
|
||||
if (ia !== -1 || ib !== -1) return (ia === -1 ? ORIGIN_ORDER.length : ia) - (ib === -1 ? ORIGIN_ORDER.length : ib);
|
||||
return a.localeCompare(b);
|
||||
})
|
||||
.map(([origin, rows]) => ({ origin, rows }));
|
||||
}
|
||||
|
||||
/**
|
||||
* 7d-vs-28d trend cell. Lift ≥ 2 (share at least doubled) and ≤ 0.5 (halved) get the arrow +
|
||||
* color; anything in between is a neutral "≈". No baseline → "new" (a new signal has no finite
|
||||
* lift — never invented). Below 5 window hits the arrow is withheld: one session can double a
|
||||
* tiny share.
|
||||
*/
|
||||
export function trendGlyph(row: VerdictCalibrationPath): { text: string; className: string } {
|
||||
if (row.lift == null) {
|
||||
return row.window7.count > 0 && row.baseline28.sessions > 0
|
||||
? { text: "new", className: "text-gray-500" }
|
||||
: { text: "—", className: "text-gray-400" };
|
||||
}
|
||||
const lift = row.lift;
|
||||
const label = `${lift.toFixed(1)}×`;
|
||||
if (row.window7.count < 5) return { text: `≈ ${label}`, className: "text-gray-400" };
|
||||
if (lift >= 2) return { text: `↑ ${label}`, className: "text-red-600 font-medium" };
|
||||
if (lift <= 0.5) return { text: `↓ ${label}`, className: "text-green-700" };
|
||||
return { text: `≈ ${label}`, className: "text-gray-500" };
|
||||
}
|
||||
@@ -292,6 +292,10 @@ export const api = {
|
||||
`${API_BASE_URL}/api/metrics/device-journeys${qs({ days: String(days) })}`,
|
||||
globalDeviceJourneys: (days: number, tenantId?: string) =>
|
||||
`${API_BASE_URL}/api/global/metrics/device-journeys${qs({ days: String(days), tenantId })}`,
|
||||
// Verdict calibration matrix (operator-only; GlobalReadOrAdmin). Omit tenantId for the
|
||||
// cross-tenant aggregate. Daily rows are additive, so the window follows days honestly.
|
||||
globalVerdictCalibration: (days: number, tenantId?: string) =>
|
||||
`${API_BASE_URL}/api/global/metrics/verdict-calibration${qs({ days: String(days), tenantId })}`,
|
||||
geographic: (tenantId: string, days: number, groupBy: string) =>
|
||||
`${API_BASE_URL}/api/metrics/geographic${qs({ tenantId, days: String(days), groupBy })}`,
|
||||
globalGeographic: (days: number, groupBy: string, tenantId?: string) =>
|
||||
|
||||
@@ -261,6 +261,7 @@ export const EXPANDABLE_NAV_GROUPS: ExpandableNavGroup[] = [
|
||||
{ id: "ga-usage", label: "Platform Usage", href: route("/admin/metrics/usage") },
|
||||
{ id: "ga-active-users", label: "Active Users", href: "/admin/presence" },
|
||||
{ id: "ga-mcp-usage", label: "MCP Usage", href: route("/admin/metrics/mcp-usage") },
|
||||
{ id: "ga-verdict-calibration", label: "Verdict Calibration", href: route("/admin/metrics/verdict-calibration") },
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user