Retention: derived statistics and the audit trail follow the tenant's retention (D-337)

First-time-right, time-attribution and rule-statistics rows lived on fixed lifetimes (180,
180 and 90 days), a storage bound reused from the never-pruned tables, never weighed against
tenant retention: a Community tenant could read first-time-right rows for days 91-180 through
the API and MCP, a Pro tenant keeping 365 days lost them after 180 and its rule statistics
after 90.

The retention fanout now prunes per tenant, before its session scan: these rows at the
tenant's effective retention (downgrade grace included), the audit trail at
max(180, retention) up to 365. The maintenance cleanups become outer bounds and platform
lifetimes in DataLifetimes: no tenant row beyond 365 days, global first-time-right rows 365,
time attribution 180, global and legacy rule statistics 90, the platform audit partition and
legacy audit rows 180. Device histories already follow their sessions and stay as they are.

The device-journeys window goes to 365 days (route cap and MCP get_device_history). The
SessionDeletionMaintenanceCompleted ops event carries tenantRowsPruned.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Oliver Kieselbach
2026-10-09 18:56:12 +02:00
parent bd064b1164
commit b2f6b7bd7f
20 changed files with 383 additions and 78 deletions
@@ -108,7 +108,8 @@ public class OffboardingSessionEnumeratorTests
public Task<List<AuditLogEntry>> GetAllAuditLogsAsync(DateTime? dateFrom = null, DateTime? dateTo = null, AuditLogQueryFilters? filters = null) => throw new NotImplementedException();
public Task<RawPage<AuditLogEntry>> GetAuditLogsPageAsync(string tenantId, DateTime? dateFrom, DateTime? dateTo, int pageSize, string? continuation, bool excludeDeletions = false, AuditLogQueryFilters? filters = null) => throw new NotImplementedException();
public Task<RawPage<AuditLogEntry>> GetAllAuditLogsPageAsync(DateTime? dateFrom, DateTime? dateTo, int pageSize, string? continuation, bool excludeDeletions = false, AuditLogQueryFilters? filters = null) => throw new NotImplementedException();
public Task<int> DeleteAuditLogsOlderThanAsync(DateTime cutoffUtc) => throw new NotImplementedException();
public Task<int> DeleteAuditLogsOlderThanAsync(DateTime cutoffUtc, DateTime legacyCutoffUtc) => throw new NotImplementedException();
public Task<int> DeleteTenantAuditLogsOlderThanAsync(string tenantId, DateTime cutoffUtc) => throw new NotImplementedException();
public Task<List<SessionSummary>> GetSessionsOlderThanAsync(string tenantId, DateTime cutoffDate, int maxResults = int.MaxValue, bool excludeInFlightDeletions = false) => throw new NotImplementedException();
public Task<List<SessionSummary>> GetSessionsByDateRangeAsync(DateTime startDate, DateTime endDate, string? tenantId = null, CancellationToken cancellationToken = default) => throw new NotImplementedException();
public Task<List<SessionSummary>> GetUsageWindowSessionsAsync(DateTime startDate, DateTime endDate, string? tenantId = null) => throw new NotImplementedException();
@@ -196,7 +196,7 @@ public class RuleStatsRepositoryTests
// ---------- cleanup ----------
[Fact]
public async Task Cleanup_deletes_per_scope_and_only_bare_date_rows_from_the_legacy_query()
public async Task Cleanup_bounds_tenants_at_the_ceiling_global_and_legacy_at_the_platform_cutoff()
{
var h = new Harness
{
@@ -207,20 +207,42 @@ public class RuleStatsRepositoryTests
new TableEntity("2020a3f1-0000-0000-0000-000000000000_2026-05-01", "ANALYZE-A"), // GUID inside the date range → keep
}
: f.StartsWith($"PartitionKey ge '{Tenant}_'")
? new[] { new TableEntity($"{Tenant}_2026-05-30", "ANALYZE-B") }
? new[] { new TableEntity($"{Tenant}_2025-09-30", "ANALYZE-B") }
: Array.Empty<TableEntity>()
};
var deleted = await h.Sut.DeleteRuleStatsOlderThanAsync(new DateTime(2026, 6, 7, 0, 0, 0, DateTimeKind.Utc), new[] { Tenant });
var deleted = await h.Sut.DeleteRuleStatsOlderThanAsync(
tenantCutoffDate: new DateTime(2025, 10, 9, 0, 0, 0, DateTimeKind.Utc),
platformCutoffDate: new DateTime(2026, 6, 7, 0, 0, 0, DateTimeKind.Utc),
new[] { Tenant, "global" });
Assert.Contains($"PartitionKey ge '{Tenant}_' and PartitionKey lt '{Tenant}_2026-06-07'", h.Filters);
Assert.Contains($"PartitionKey ge '{Tenant}_' and PartitionKey lt '{Tenant}_2025-10-09'", h.Filters);
Assert.Contains("PartitionKey ge 'global_' and PartitionKey lt 'global_2026-06-07'", h.Filters);
Assert.Contains("PartitionKey ge '2000-01-01' and PartitionKey lt '2026-06-07'", h.Filters);
Assert.Single(h.Filters, f => f.StartsWith("PartitionKey ge 'global_'")); // a "global" tenant id is no second pass
Assert.Equal(2, deleted);
Assert.Contains(("2026-05-01", $"{Tenant}_ANALYZE-A"), h.Deleted);
Assert.Contains(($"{Tenant}_2026-05-30", "ANALYZE-B"), h.Deleted);
Assert.Contains(($"{Tenant}_2025-09-30", "ANALYZE-B"), h.Deleted);
Assert.DoesNotContain(h.Deleted, d => d.Pk.StartsWith("2020a3f1"));
}
[Fact]
public async Task Tenant_cleanup_deletes_only_that_tenants_partitions_before_its_cutoff()
{
var h = new Harness
{
RowsFor = f => f.StartsWith($"PartitionKey ge '{Tenant}_'")
? new[] { new TableEntity($"{Tenant}_2026-05-30", "ANALYZE-B") }
: Array.Empty<TableEntity>()
};
var deleted = await h.Sut.DeleteTenantRuleStatsOlderThanAsync(Tenant, new DateTime(2026, 6, 7, 0, 0, 0, DateTimeKind.Utc));
Assert.Equal(new[] { $"PartitionKey ge '{Tenant}_' and PartitionKey lt '{Tenant}_2026-06-07'" }, h.Filters);
Assert.Equal(1, deleted);
Assert.Equal(new[] { ($"{Tenant}_2026-05-30", "ANALYZE-B") }, h.Deleted);
}
// ---------- platform stats (D-198) ----------
[Fact]
@@ -413,6 +413,7 @@ public class SessionDeletionMaintenanceFunctionTests
Fanout = new Mock<SessionRetentionFanoutService>(
maintRepo.Object,
Mock.Of<IMetricsRepository>(),
new Mock<TenantConfigurationService>(Mock.Of<IConfigRepository>(), NullLogger<TenantConfigurationService>.Instance, memCache).Object,
Mock.Of<ISessionDeletionEnqueuer>(),
AdminConfig.Object,
@@ -421,11 +421,88 @@ public class SessionRetentionFanoutServiceTests
"t1", now.AddDays(-90), It.IsAny<int>(), It.IsAny<bool>()), Times.Once);
}
// ============================ Rows outside the cascade follow retention ====
[Fact]
public async Task TenantRows_PrunedAtTheRetentionCutoff_AuditAtItsFloor_EvenWithNoSessionToDelete()
{
var now = new DateTime(2026, 10, 9, 12, 0, 0, DateTimeKind.Utc);
var harness = new Harness(utcNow: () => now);
harness.WithTenant("t1", retentionDays: 90, sessions: Array.Empty<SessionSummary>());
harness.MetricsRepo.Setup(m => m.DeleteTenantDeviceJourneyAggregatesOlderThanAsync("t1", now.AddDays(-90))).ReturnsAsync(2);
harness.MetricsRepo.Setup(m => m.DeleteTenantTimeAttributionAggregatesOlderThanAsync("t1", now.AddDays(-90))).ReturnsAsync(3);
harness.MetricsRepo.Setup(m => m.DeleteTenantRuleStatsOlderThanAsync("t1", now.AddDays(-90))).ReturnsAsync(4);
harness.MaintenanceRepo.Setup(m => m.DeleteTenantAuditLogsOlderThanAsync("t1", now.AddDays(-180))).ReturnsAsync(5);
var result = await harness.RunAsync();
Assert.Equal(14, result.TenantRowsPruned);
harness.MetricsRepo.Verify(m => m.DeleteTenantDeviceJourneyAggregatesOlderThanAsync("t1", now.AddDays(-90)), Times.Once);
harness.MetricsRepo.Verify(m => m.DeleteTenantTimeAttributionAggregatesOlderThanAsync("t1", now.AddDays(-90)), Times.Once);
harness.MetricsRepo.Verify(m => m.DeleteTenantRuleStatsOlderThanAsync("t1", now.AddDays(-90)), Times.Once);
harness.MaintenanceRepo.Verify(m => m.DeleteTenantAuditLogsOlderThanAsync("t1", now.AddDays(-180)), Times.Once);
}
[Fact]
public async Task TenantRows_ProWithLongerRetention_KeepRowsAndAuditTrailThatLong()
{
var now = new DateTime(2026, 10, 9, 12, 0, 0, DateTimeKind.Utc);
var harness = new Harness(utcNow: () => now);
harness.WithTenant("t1", retentionDays: 365, sessions: Array.Empty<SessionSummary>(), planTier: "pro");
await harness.RunAsync();
harness.MetricsRepo.Verify(m => m.DeleteTenantDeviceJourneyAggregatesOlderThanAsync("t1", now.AddDays(-365)), Times.Once);
harness.MetricsRepo.Verify(m => m.DeleteTenantRuleStatsOlderThanAsync("t1", now.AddDays(-365)), Times.Once);
harness.MaintenanceRepo.Verify(m => m.DeleteTenantAuditLogsOlderThanAsync("t1", now.AddDays(-365)), Times.Once);
}
[Fact]
public async Task TenantRows_ShortRetention_AuditTrailKeepsTheFloor()
{
var now = new DateTime(2026, 10, 9, 12, 0, 0, DateTimeKind.Utc);
var harness = new Harness(utcNow: () => now);
harness.WithTenant("t1", retentionDays: 30, sessions: Array.Empty<SessionSummary>());
await harness.RunAsync();
harness.MetricsRepo.Verify(m => m.DeleteTenantRuleStatsOlderThanAsync("t1", now.AddDays(-30)), Times.Once);
harness.MaintenanceRepo.Verify(m => m.DeleteTenantAuditLogsOlderThanAsync("t1", now.AddDays(-180)), Times.Once);
}
[Fact]
public async Task TenantRows_RetentionZero_NoTenantPrune()
{
var harness = new Harness();
harness.WithTenant("t1", retentionDays: 0, sessions: Array.Empty<SessionSummary>());
var result = await harness.RunAsync();
Assert.Equal(0, result.TenantRowsPruned);
harness.MetricsRepo.VerifyNoOtherCalls();
harness.MaintenanceRepo.Verify(m => m.DeleteTenantAuditLogsOlderThanAsync(It.IsAny<string>(), It.IsAny<DateTime>()), Times.Never);
}
[Fact]
public async Task TenantRows_FailingPrune_DoesNotBlockTheSessionFanout()
{
var harness = new Harness();
harness.WithTenant("t1", retentionDays: 90, sessions: new[] { Old("s1", ageDays: 120) });
harness.MetricsRepo.Setup(m => m.DeleteTenantDeviceJourneyAggregatesOlderThanAsync(It.IsAny<string>(), It.IsAny<DateTime>()))
.ThrowsAsync(new InvalidOperationException("storage down"));
var result = await harness.RunAsync();
Assert.Equal(0, result.TenantRowsPruned);
Assert.Equal(1, result.SessionsEnqueued);
}
// ============================================================ Harness ====
private sealed class Harness
{
public Mock<IMaintenanceRepository> MaintenanceRepo { get; }
public Mock<IMetricsRepository> MetricsRepo { get; } = new();
public Mock<TenantConfigurationService> TenantConfig { get; }
public Mock<ISessionDeletionEnqueuer> Enqueuer { get; }
public Mock<AdminConfigurationService> AdminConfig { get; }
@@ -465,7 +542,7 @@ public class SessionRetentionFanoutServiceTests
// Use internal ctor to inject a no-op throttle so the rate-limit test runs in real-time
// (50ms × 100 = 5s otherwise) and an optional scripted clock for the budget tests.
Sut = new SessionRetentionFanoutService(
MaintenanceRepo.Object, TenantConfig.Object, Enqueuer.Object,
MaintenanceRepo.Object, MetricsRepo.Object, TenantConfig.Object, Enqueuer.Object,
AdminConfig.Object,
NullLogger<SessionRetentionFanoutService>.Instance,
throttle: throttle ?? ((_, _) => Task.CompletedTask),
@@ -48,8 +48,11 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
bool excludeDeletions = false, AuditLogQueryFilters? filters = null)
=> _storage.GetAllAuditLogsPageAsync(dateFrom, dateTo, pageSize, continuation, excludeDeletions, filters);
public Task<int> DeleteAuditLogsOlderThanAsync(DateTime cutoffUtc)
=> _storage.DeleteAuditLogsOlderThanAsync(cutoffUtc);
public Task<int> DeleteAuditLogsOlderThanAsync(DateTime cutoffUtc, DateTime legacyCutoffUtc)
=> _storage.DeleteAuditLogsOlderThanAsync(cutoffUtc, legacyCutoffUtc);
public Task<int> DeleteTenantAuditLogsOlderThanAsync(string tenantId, DateTime cutoffUtc)
=> _storage.DeleteTenantAuditLogsOlderThanAsync(tenantId, cutoffUtc);
public Task<List<SessionSummary>> GetSessionsOlderThanAsync(string tenantId, DateTime cutoffDate, int maxResults = int.MaxValue, bool excludeInFlightDeletions = false)
=> _storage.GetSessionsOlderThanAsync(tenantId, cutoffDate, maxResults, excludeInFlightDeletions);
@@ -137,8 +137,11 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
string? endDate = null, string? ruleType = null, int maxResultsPerTenant = 10000)
=> _storage.GetRuleStatsForTenantsAsync(tenantIds, startDate, endDate, ruleType, maxResultsPerTenant);
public Task<int> DeleteRuleStatsOlderThanAsync(DateTime cutoffDate, IReadOnlyCollection<string> tenantIds)
=> _storage.DeleteRuleStatsOlderThanAsync(cutoffDate, tenantIds);
public Task<int> DeleteRuleStatsOlderThanAsync(DateTime tenantCutoffDate, DateTime platformCutoffDate, IReadOnlyCollection<string> tenantIds)
=> _storage.DeleteRuleStatsOlderThanAsync(tenantCutoffDate, platformCutoffDate, tenantIds);
public Task<int> DeleteTenantRuleStatsOlderThanAsync(string tenantId, DateTime cutoffDate)
=> _storage.DeleteTenantRuleStatsOlderThanAsync(tenantId, cutoffDate);
public Task<SessionTimeBreakdown?> ComputeAndStoreSessionTimeBreakdownAsync(string tenantId, string sessionId)
=> _storage.ComputeAndStoreSessionTimeBreakdownAsync(tenantId, sessionId);
@@ -167,6 +170,9 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
public Task<int> DeleteTimeAttributionAggregatesOlderThanAsync(DateTime cutoffDate)
=> _storage.DeleteTimeAttributionAggregatesOlderThanAsync(cutoffDate);
public Task<int> DeleteTenantTimeAttributionAggregatesOlderThanAsync(string tenantId, DateTime cutoffDate)
=> _storage.DeleteTenantTimeAttributionAggregatesOlderThanAsync(tenantId, cutoffDate);
public Task<DeviceHistory?> GetDeviceHistoryAsync(string tenantId, string serialKey)
=> _storage.GetDeviceHistoryAsync(tenantId, serialKey);
@@ -191,6 +197,9 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
public Task<int> DeleteDeviceJourneyAggregatesOlderThanAsync(DateTime cutoffDate)
=> _storage.DeleteDeviceJourneyAggregatesOlderThanAsync(cutoffDate);
public Task<int> DeleteTenantDeviceJourneyAggregatesOlderThanAsync(string tenantId, DateTime cutoffDate)
=> _storage.DeleteTenantDeviceJourneyAggregatesOlderThanAsync(tenantId, cutoffDate);
public Task<bool> SaveVerdictCalibrationAggregateAsync(VerdictCalibrationDailyAggregate aggregate)
=> _storage.SaveVerdictCalibrationAggregateAsync(aggregate);
@@ -240,7 +240,8 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance
strandedQueuedDetected: strandedQueuedDetected,
durationMs: (int)sw.ElapsedMilliseconds,
abortedByKillSwitch: fanoutResult.AbortedByKillSwitch,
abortedByBudget: fanoutResult.AbortedByBudget);
abortedByBudget: fanoutResult.AbortedByBudget,
tenantRowsPruned: fanoutResult.TenantRowsPruned);
_logger.LogInformation(
"SessionDeletionMaintenance: completed in {Ms}ms — killSwitch={KillSwitch} tenants={Tenants} enqueued={Enqueued} skipped={Skipped} blobs={Blobs} preparing={Preparing} stranded={Stranded} tombstones={Tombstones} abortedByBudget={Budget}",
@@ -193,7 +193,8 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
internal static class DeviceJourneyMetricsResponseBuilder
{
internal const int DefaultWindowDays = 30;
internal const int MaxWindowDays = 180; // aggregate retention — older rows no longer exist
// Tenant rows follow the tenant's retention; none is older than the ceiling, the "global" rows live as long.
internal const int MaxWindowDays = DataLifetimes.TenantRowCeilingDays;
internal const int MaxRepeatDevices = 10;
/// <summary>
@@ -80,7 +80,7 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
internal static class VerdictCalibrationResponseBuilder
{
internal const int DefaultWindowDays = 30;
internal const int MaxWindowDays = 180; // aggregate retention
internal const int MaxWindowDays = DataLifetimes.VerdictCalibrationDays; // older rows no longer exist
internal const int TrendWindowDays = 7;
internal const int TrendBaselineDays = 28;
internal const int TrendHorizonDays = TrendWindowDays + TrendBaselineDays;
@@ -0,0 +1,47 @@
using System;
namespace AutopilotMonitor.Functions.Helpers
{
/// <summary>
/// Lifetimes of the rows derived from sessions and of the audit trail. Tenant rows follow the
/// tenant's effective retention and are pruned per tenant by <c>SessionRetentionFanoutService</c>;
/// the fixed values here bound that from outside: the audit floor, the ceiling no tenant row
/// outlives (retention 0, a tenant the fanout did not reach), and the lifetimes of platform rows
/// that belong to no tenant. Route caps over these tables refer to the same constants.
/// </summary>
public static class DataLifetimes
{
/// <summary>Ceiling for every tenant row: the Pro retention cap, the longest window any route reads.</summary>
public const int TenantRowCeilingDays = 365;
/// <summary>
/// A tenant's audit trail is kept at least this long, whatever its retention; the platform partition
/// (<c>Constants.AuditGlobalTenantId</c>) and legacy rows without an event time keep exactly this.
/// </summary>
public const int AuditLogFloorDays = 180;
/// <summary>Platform ("global") first-time-right rows: as far back as the Global Admin route reads.</summary>
public const int GlobalDeviceJourneyDays = TenantRowCeilingDays;
/// <summary>Platform ("global") rule stats and the legacy layout until its cutover.</summary>
public const int GlobalRuleStatsDays = 90;
/// <summary>Time-attribution rows: the route reads only the newest 30-day window.</summary>
public const int TimeAttributionDays = 180;
/// <summary>Verdict-calibration rows (Global Admin only).</summary>
public const int VerdictCalibrationDays = 180;
/// <summary>Daily usage-metrics snapshots.</summary>
public const int UsageMetricsDays = 180;
/// <summary>
/// A tenant's audit trail: its effective retention, never below the floor nor above the ceiling;
/// 0 (never delete) keeps it to the ceiling.
/// </summary>
public static int TenantAuditDays(int effectiveRetentionDays)
=> effectiveRetentionDays <= 0
? TenantRowCeilingDays
: Math.Clamp(effectiveRetentionDays, AuditLogFloorDays, TenantRowCeilingDays);
}
}
@@ -2,6 +2,7 @@ using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Helpers;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using AutopilotMonitor.Shared.Models.Deletion;
@@ -22,6 +23,12 @@ namespace AutopilotMonitor.Functions.Services.Deletion
/// Bounds the cost of a maintenance fan-out when a tenant has months of backlog behind a
/// freshly-shortened retention setting.
/// </para>
/// <para>
/// <b>Rows outside the session cascade</b> follow the same retention: the tenant's
/// first-time-right and time-attribution aggregates and rule statistics, and its audit trail
/// with its own floor (<see cref="DataLifetimes"/>), are pruned per tenant before the session
/// scan. The cross-tenant cleanups in the maintenance service only bound them from outside.
/// </para>
/// </summary>
public class SessionRetentionFanoutService
{
@@ -29,6 +36,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion
public static readonly TimeSpan EnqueueThrottleDelay = TimeSpan.FromMilliseconds(50);
private readonly IMaintenanceRepository _maintenanceRepo;
private readonly IMetricsRepository _metricsRepo;
private readonly TenantConfigurationService _tenantConfig;
private readonly ISessionDeletionEnqueuer _enqueuer;
private readonly AdminConfigurationService _adminConfig;
@@ -38,11 +46,12 @@ namespace AutopilotMonitor.Functions.Services.Deletion
public SessionRetentionFanoutService(
IMaintenanceRepository maintenanceRepo,
IMetricsRepository metricsRepo,
TenantConfigurationService tenantConfig,
ISessionDeletionEnqueuer enqueuer,
AdminConfigurationService adminConfig,
ILogger<SessionRetentionFanoutService> logger)
: this(maintenanceRepo, tenantConfig, enqueuer, adminConfig, logger, throttle: Task.Delay, utcNow: () => DateTime.UtcNow)
: this(maintenanceRepo, metricsRepo, tenantConfig, enqueuer, adminConfig, logger, throttle: Task.Delay, utcNow: () => DateTime.UtcNow)
{
}
@@ -53,6 +62,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion
/// </summary>
internal SessionRetentionFanoutService(
IMaintenanceRepository maintenanceRepo,
IMetricsRepository metricsRepo,
TenantConfigurationService tenantConfig,
ISessionDeletionEnqueuer enqueuer,
AdminConfigurationService adminConfig,
@@ -61,6 +71,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion
Func<DateTime>? utcNow = null)
{
_maintenanceRepo = maintenanceRepo;
_metricsRepo = metricsRepo;
_tenantConfig = tenantConfig;
_enqueuer = enqueuer;
_adminConfig = adminConfig;
@@ -83,6 +94,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion
public int RateLimitedTenants { get; set; } // tenants that hit MaxEnqueuesPerTenantPerRun
public bool AbortedByKillSwitch { get; set; } // kill-switch flipped mid-run
public bool AbortedByBudget { get; set; } // run-budget deadline crossed mid-run
public int TenantRowsPruned { get; set; } // aggregate/rule-stats/audit rows beyond tenant retention
}
/// <summary>
@@ -183,6 +195,9 @@ namespace AutopilotMonitor.Functions.Services.Deletion
var cutoffUtc = nowUtc.AddDays(-retentionDays);
// Before the session scan: its early return would skip a tenant with no session left to delete.
result.TenantRowsPruned += await PruneTenantRowsAsync(tenantId, retentionDays, nowUtc).ConfigureAwait(false);
// Server-bounded read: fetch one more than the per-run dispatch cap. The loop below
// only ever advances MaxEnqueuesPerTenantPerRun sessions, so loading the whole backlog
// every run is wasted I/O that grows with the backlog. The "+1" is a cheap probe that
@@ -315,6 +330,37 @@ namespace AutopilotMonitor.Functions.Services.Deletion
tenantId, cutoffUtc, Math.Min(oldSessions.Count, MaxEnqueuesPerTenantPerRun), moreRemaining, enqueued, skipped);
}
/// <summary>
/// Deletes the tenant's rows that follow its retention outside the session cascade:
/// first-time-right and time-attribution aggregates and rule statistics at the retention
/// cutoff, the audit trail at its own window (<see cref="DataLifetimes.TenantAuditDays"/>).
/// The storage calls are fail-soft; a failure never blocks the session fanout, the next run
/// retries.
/// </summary>
private async Task<int> PruneTenantRowsAsync(string tenantId, int retentionDays, DateTime nowUtc)
{
var cutoffUtc = nowUtc.AddDays(-retentionDays);
var auditDays = DataLifetimes.TenantAuditDays(retentionDays);
try
{
var deleted =
await _metricsRepo.DeleteTenantDeviceJourneyAggregatesOlderThanAsync(tenantId, cutoffUtc).ConfigureAwait(false)
+ await _metricsRepo.DeleteTenantTimeAttributionAggregatesOlderThanAsync(tenantId, cutoffUtc).ConfigureAwait(false)
+ await _metricsRepo.DeleteTenantRuleStatsOlderThanAsync(tenantId, cutoffUtc).ConfigureAwait(false)
+ await _maintenanceRepo.DeleteTenantAuditLogsOlderThanAsync(tenantId, nowUtc.AddDays(-auditDays)).ConfigureAwait(false);
if (deleted > 0)
_logger.LogInformation(
"Tenant {TenantId}: pruned {Count} aggregate/rule-stats/audit rows beyond retention ({Days} days, audit {AuditDays} days)",
tenantId, deleted, retentionDays, auditDays);
return deleted;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Tenant {TenantId}: pruning aggregate/rule-stats/audit rows failed — the session fanout continues", tenantId);
return 0;
}
}
/// <summary>
/// Enqueue a cascade for one session. The producer handles CAS-Preparing, manifest build,
/// blob upload, and queue send; we just translate the outcome into a counter bump (or
@@ -590,15 +590,19 @@ namespace AutopilotMonitor.Functions.Services
_logger.LogWarning(ex, "Failed to cleanup old tenant MCP usage counters");
}
// Rule stats retention: delete entries older than 90 days
// Rule stats: tenant partitions follow the tenant's retention (pruned per tenant by the
// retention fanout); this pass is their ceiling, plus the platform rows and the legacy
// layout at their own lifetime.
try
{
var ruleStatsCutoff = DateTime.UtcNow.AddDays(-90);
var ruleStatsTenantIds = await _maintenanceRepo.GetAllTenantIdsAsync();
var deletedRuleStats = await _metricsRepo.DeleteRuleStatsOlderThanAsync(ruleStatsCutoff, ruleStatsTenantIds);
var deletedRuleStats = await _metricsRepo.DeleteRuleStatsOlderThanAsync(
DateTime.UtcNow.AddDays(-DataLifetimes.TenantRowCeilingDays),
DateTime.UtcNow.AddDays(-DataLifetimes.GlobalRuleStatsDays),
ruleStatsTenantIds);
if (deletedRuleStats > 0)
_logger.LogInformation("Rule stats cleanup: deleted {Count} entries older than 90 days", deletedRuleStats);
_logger.LogInformation("Rule stats cleanup: deleted {Count} entries beyond the tenant ceiling or platform lifetime", deletedRuleStats);
}
catch (Exception ex)
{
@@ -1043,9 +1047,11 @@ namespace AutopilotMonitor.Functions.Services
/// <summary>
/// Retention cleanup for append-only tables that previously had no purge mechanism and
/// therefore grew unbounded: GlobalNotifications + TenantNotifications (hybrid: dismissed 30d /
/// unread 180d), HardwareRejectionNotificationTracker (30d), AuditLogs (180d), UsageMetrics
/// (180d), BackupJobs (365d). Each table is handled in its own try/catch so one failure never
/// blocks the others. Retention windows are fixed product decisions (no AdminConfig knob),
/// unread 180d), HardwareRejectionNotificationTracker (30d), UsageMetrics (180d), BackupJobs
/// (365d), plus the outer bounds of the rows that follow tenant retention (see
/// <see cref="DataLifetimes"/>): AuditLogs, time-attribution, device-journey and
/// verdict-calibration aggregates. Each table is handled in its own try/catch so one failure
/// never blocks the others. Retention windows are fixed product decisions (no AdminConfig knob),
/// mirroring the DistressReports pattern. PlatformStats is intentionally excluded — it is a
/// single upserted row, not append-only.
/// </summary>
@@ -1057,8 +1063,6 @@ namespace AutopilotMonitor.Functions.Services
const int notificationDismissedRetentionDays = 30;
const int notificationUnreadRetentionDays = 180;
const int hardwareRejectionRetentionDays = 30;
const int auditLogRetentionDays = 180;
const int usageMetricsRetentionDays = 180;
const int backupJobRetentionDays = 365;
var now = DateTime.UtcNow;
@@ -1125,11 +1129,21 @@ namespace AutopilotMonitor.Functions.Services
_logger.LogWarning(ex, "Failed to cleanup old delegation invitation rows");
}
// Audit trail: each tenant's rows follow its retention with a 180-day floor, pruned per
// tenant by the retention fanout; this pass is the ceiling for tenant rows, while the
// platform partition (no tenant, no retention) and the legacy rows without a recoverable
// event time keep the floor.
try
{
var deleted = await _maintenanceRepo.DeleteAuditLogsOlderThanAsync(now.AddDays(-auditLogRetentionDays));
var floorCutoff = now.AddDays(-DataLifetimes.AuditLogFloorDays);
var deleted =
await _maintenanceRepo.DeleteAuditLogsOlderThanAsync(
now.AddDays(-DataLifetimes.TenantRowCeilingDays), legacyCutoffUtc: floorCutoff)
+ await _maintenanceRepo.DeleteTenantAuditLogsOlderThanAsync(
AutopilotMonitor.Shared.Constants.AuditGlobalTenantId, floorCutoff);
if (deleted > 0)
_logger.LogInformation("Audit log cleanup: deleted {Count} entries older than {Days} days", deleted, auditLogRetentionDays);
_logger.LogInformation("Audit log cleanup: deleted {Count} entries beyond the tenant ceiling or the platform/legacy floor",
deleted);
}
catch (Exception ex)
{
@@ -1138,7 +1152,7 @@ namespace AutopilotMonitor.Functions.Services
try
{
var cutoffDate = now.AddDays(-usageMetricsRetentionDays).ToString("yyyy-MM-dd");
var cutoffDate = now.AddDays(-DataLifetimes.UsageMetricsDays).ToString("yyyy-MM-dd");
var deleted = await _metricsRepo.DeleteUsageMetricsSnapshotsOlderThanAsync(cutoffDate);
if (deleted > 0)
_logger.LogInformation("Usage metrics cleanup: deleted {Count} snapshots older than {Cutoff}", deleted, cutoffDate);
@@ -1159,41 +1173,44 @@ namespace AutopilotMonitor.Functions.Services
_logger.LogWarning(ex, "Failed to cleanup old backup job records");
}
// Time-attribution daily aggregates: same 180d window as the usage-metrics snapshots
// they sit beside. (SessionTimeBreakdowns needs no age sweep — breakdown rows are
// deleted with their session via the deletion-manifest cascade / offboarding wipe.)
// Time-attribution daily aggregates: tenant rows follow the tenant's retention (pruned per
// tenant by the retention fanout); this pass bounds every partition, the platform rows
// included, at the route's needs. (SessionTimeBreakdowns needs no age sweep — breakdown
// rows are deleted with their session via the deletion-manifest cascade / offboarding wipe.)
try
{
var deleted = await _metricsRepo.DeleteTimeAttributionAggregatesOlderThanAsync(now.AddDays(-usageMetricsRetentionDays));
var deleted = await _metricsRepo.DeleteTimeAttributionAggregatesOlderThanAsync(now.AddDays(-DataLifetimes.TimeAttributionDays));
if (deleted > 0)
_logger.LogInformation("Time-attribution aggregate cleanup: deleted {Count} rows older than {Days} days", deleted, usageMetricsRetentionDays);
_logger.LogInformation("Time-attribution aggregate cleanup: deleted {Count} rows older than {Days} days", deleted, DataLifetimes.TimeAttributionDays);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to cleanup old time-attribution aggregates");
}
// Device-journey (FTR) daily aggregates: same 180d window. (DeviceHistories needs no
// age sweep — chain refs of deleted sessions are pruned tombstone-driven by
// SweepDeviceJourneysAsync and the rows die with tenant offboarding.)
// Device-journey (FTR) daily aggregates: tenant rows follow the tenant's retention (pruned
// per tenant by the retention fanout); this pass is the ceiling for tenant rows and the
// lifetime of the platform rows. (DeviceHistories needs no age sweep — chain refs of
// deleted sessions are pruned tombstone-driven by SweepDeviceJourneysAsync and the rows
// die with tenant offboarding.)
try
{
var deleted = await _metricsRepo.DeleteDeviceJourneyAggregatesOlderThanAsync(now.AddDays(-usageMetricsRetentionDays));
var deleted = await _metricsRepo.DeleteDeviceJourneyAggregatesOlderThanAsync(now.AddDays(-DataLifetimes.GlobalDeviceJourneyDays));
if (deleted > 0)
_logger.LogInformation("Device-journey aggregate cleanup: deleted {Count} rows older than {Days} days", deleted, usageMetricsRetentionDays);
_logger.LogInformation("Device-journey aggregate cleanup: deleted {Count} rows older than {Days} days", deleted, DataLifetimes.GlobalDeviceJourneyDays);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to cleanup old device-journey aggregates");
}
// Verdict-calibration daily aggregates: same 180d window (regenerable; the radar
// needs 35d, the matrix serves up to 180d).
// Verdict-calibration daily aggregates (regenerable; the radar needs 35d, the matrix
// serves up to the lifetime).
try
{
var deleted = await _metricsRepo.DeleteVerdictCalibrationAggregatesOlderThanAsync(now.AddDays(-usageMetricsRetentionDays));
var deleted = await _metricsRepo.DeleteVerdictCalibrationAggregatesOlderThanAsync(now.AddDays(-DataLifetimes.VerdictCalibrationDays));
if (deleted > 0)
_logger.LogInformation("Verdict-calibration aggregate cleanup: deleted {Count} rows older than {Days} days", deleted, usageMetricsRetentionDays);
_logger.LogInformation("Verdict-calibration aggregate cleanup: deleted {Count} rows older than {Days} days", deleted, DataLifetimes.VerdictCalibrationDays);
}
catch (Exception ex)
{
@@ -287,13 +287,15 @@ namespace AutopilotMonitor.Functions.Services
public Task RecordSessionDeletionMaintenanceCompletedAsync(
bool killSwitchActive, int tenantsProcessed, int sessionsEnqueued,
int sessionsSkipped, int rateLimitedTenants, int blobsTtlGced, int preparingRowsCleared,
int strandedQueuedDetected, int durationMs, bool abortedByKillSwitch, bool abortedByBudget)
int strandedQueuedDetected, int durationMs, bool abortedByKillSwitch, bool abortedByBudget,
int tenantRowsPruned)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionDeletionMaintenanceCompleted, OpsEventSeverity.Info,
$"SessionDeletionMaintenance completed in {durationMs}ms — tenants={tenantsProcessed} enqueued={sessionsEnqueued} skipped={sessionsSkipped} blobsTtlGced={blobsTtlGced} preparingCleared={preparingRowsCleared} stranded={strandedQueuedDetected} killSwitch={killSwitchActive} abortedByBudget={abortedByBudget}",
$"SessionDeletionMaintenance completed in {durationMs}ms — tenants={tenantsProcessed} enqueued={sessionsEnqueued} skipped={sessionsSkipped} tenantRowsPruned={tenantRowsPruned} blobsTtlGced={blobsTtlGced} preparingCleared={preparingRowsCleared} stranded={strandedQueuedDetected} killSwitch={killSwitchActive} abortedByBudget={abortedByBudget}",
null, "System.Maintenance", new {
killSwitchActive, tenantsProcessed, sessionsEnqueued,
sessionsSkipped, rateLimitedTenants, blobsTtlGced, preparingRowsCleared,
strandedQueuedDetected, durationMs, abortedByKillSwitch, abortedByBudget,
tenantRowsPruned,
});
/// <summary>
@@ -306,17 +306,24 @@ namespace AutopilotMonitor.Functions.Services
}
/// <summary>
/// Retention: deletes FTR aggregate rows older than the cutoff (RowKey IS the date, so a
/// string compare works across partitions). Mirrors the UsageMetrics 180d policy.
/// DeviceHistories needs no age sweep — refs are pruned tombstone-driven and the rows
/// die with tenant offboarding.
/// Retention ceiling: deletes FTR aggregate rows older than the cutoff in every partition
/// (RowKey IS the date, so a string compare works across partitions) — the lifetime of the
/// "global" rows and the outer bound for tenant rows. DeviceHistories needs no age sweep —
/// refs are pruned tombstone-driven and the rows die with tenant offboarding.
/// </summary>
public async Task<int> DeleteDeviceJourneyAggregatesOlderThanAsync(DateTime cutoffDate)
public Task<int> DeleteDeviceJourneyAggregatesOlderThanAsync(DateTime cutoffDate)
=> DeleteDeviceJourneyAggregatesAsync($"RowKey lt '{cutoffDate:yyyy-MM-dd}'");
/// <summary>Tenant retention: deletes one tenant's FTR rows older than the cutoff.</summary>
public Task<int> DeleteTenantDeviceJourneyAggregatesOlderThanAsync(string tenantId, DateTime cutoffDate)
=> DeleteDeviceJourneyAggregatesAsync(
$"PartitionKey eq '{ODataSanitizer.EscapeValue(tenantId)}' and RowKey lt '{cutoffDate:yyyy-MM-dd}'");
private async Task<int> DeleteDeviceJourneyAggregatesAsync(string filter)
{
try
{
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.DeviceJourneyAggregates);
var filter = $"RowKey lt '{cutoffDate:yyyy-MM-dd}'";
var deleted = 0;
await foreach (var entity in tableClient.QueryAsync<TableEntity>(
filter: filter, select: new[] { "PartitionKey", "RowKey" }))
@@ -73,17 +73,20 @@ namespace AutopilotMonitor.Functions.Services
}
/// <summary>
/// Retention cleanup: deletes audit log entries whose business time is older than the
/// cutoff. The AuditLogs table is append-only (one row per admin action) and is otherwise
/// only wiped on tenant offboarding, so without this it grows unbounded.
/// Retention cleanup across tenants: deletes audit log entries whose business time is older
/// than the cutoff — the ceiling; each tenant's own window is applied by
/// <see cref="DeleteTenantAuditLogsOlderThanAsync"/>. The AuditLogs table is append-only (one
/// row per admin action) and is otherwise only wiped on tenant offboarding, so without this
/// it grows unbounded.
/// Two passes: (1) time-encoded '!'-rows via an index-backed RowKey range — the system
/// Timestamp cannot be used because storage migrations reset it (post-migration it would
/// delete nothing for a retention period, then the whole pre-migration corpus at once);
/// (2) legacy bare-GUID rows (true event time unrecoverable) via their frozen system
/// Timestamp — they all age out ~180d after the 2026-07-18 migration.
/// Timestamp at <paramref name="legacyCutoffUtc"/> — they all age out ~180d after the
/// 2026-07-18 migration.
/// TODO: Remove pass 2 after 2027-03-01 (no bare-GUID rows can remain by then).
/// </summary>
public async Task<int> DeleteAuditLogsOlderThanAsync(DateTime cutoffUtc)
public async Task<int> DeleteAuditLogsOlderThanAsync(DateTime cutoffUtc, DateTime legacyCutoffUtc)
{
try
{
@@ -92,10 +95,11 @@ namespace AutopilotMonitor.Functions.Services
deleted += await DeleteAuditLogsByFilterAsync(tableClient, BusinessTimestamp.AuditRetentionClause(cutoffUtc));
// 'RowKey ge '0'' cannot match any '!'-row (0x21 < 0x30), only legacy GUIDs.
deleted += await DeleteAuditLogsByFilterAsync(tableClient,
$"RowKey ge '0' and Timestamp lt datetime'{cutoffUtc:yyyy-MM-ddTHH:mm:ss}Z'");
$"RowKey ge '0' and Timestamp lt datetime'{legacyCutoffUtc:yyyy-MM-ddTHH:mm:ss}Z'");
if (deleted > 0)
_logger.LogInformation("Deleted {Count} audit log entries older than {Cutoff:yyyy-MM-dd}", deleted, cutoffUtc);
_logger.LogInformation("Deleted {Count} audit log entries (older than {Cutoff:yyyy-MM-dd}, legacy older than {LegacyCutoff:yyyy-MM-dd})",
deleted, cutoffUtc, legacyCutoffUtc);
return deleted;
}
@@ -106,6 +110,25 @@ namespace AutopilotMonitor.Functions.Services
}
}
/// <summary>
/// Tenant retention: deletes one tenant's time-encoded audit entries older than the cutoff.
/// Legacy rows without an event time are left to the cross-tenant pass.
/// </summary>
public async Task<int> DeleteTenantAuditLogsOlderThanAsync(string tenantId, DateTime cutoffUtc)
{
try
{
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.AuditLogs);
return await DeleteAuditLogsByFilterAsync(tableClient,
$"PartitionKey eq '{ODataSanitizer.EscapeValue(tenantId)}' and {BusinessTimestamp.AuditRetentionClause(cutoffUtc)}");
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to delete old audit log entries for tenant {TenantId}", tenantId);
return 0;
}
}
private async Task<int> DeleteAuditLogsByFilterAsync(TableClient tableClient, string filter)
{
var query = tableClient.QueryAsync<TableEntity>(filter: filter, select: new[] { "PartitionKey", "RowKey" });
@@ -2003,29 +2003,38 @@ namespace AutopilotMonitor.Functions.Services
}
/// <summary>
/// Retention cleanup: per scope (every tenant plus "global") a PartitionKey range delete;
/// while legacy rows can still exist, additionally the legacy date-keyed rows — checked
/// row by row, because a bare date range would also match tenant GUIDs that sort inside
/// it. Per-row failures are logged and skipped so one 404 does not abort the sweep.
/// Retention cleanup: per scope a PartitionKey range delete — every tenant at
/// <paramref name="tenantCutoffDate"/> (the ceiling; each tenant's own retention is applied by
/// <see cref="DeleteTenantRuleStatsOlderThanAsync"/>), "global" at <paramref name="platformCutoffDate"/>;
/// while legacy rows can still exist, additionally the legacy date-keyed rows at the platform
/// cutoff — checked row by row, because a bare date range would also match tenant GUIDs that
/// sort inside it. Per-row failures are logged and skipped so one 404 does not abort the sweep.
/// </summary>
public async Task<int> DeleteRuleStatsOlderThanAsync(DateTime cutoffDate, IReadOnlyCollection<string> tenantIds)
public async Task<int> DeleteRuleStatsOlderThanAsync(
DateTime tenantCutoffDate, DateTime platformCutoffDate, IReadOnlyCollection<string> tenantIds)
{
var cutoffStr = cutoffDate.ToString("yyyy-MM-dd");
var tenantCutoffStr = tenantCutoffDate.ToString("yyyy-MM-dd");
var platformCutoffStr = platformCutoffDate.ToString("yyyy-MM-dd");
var deleted = 0;
try
{
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.RuleStats);
var scopes = (tenantIds ?? Array.Empty<string>()).Append(RuleStatsKeys.GlobalScope).Distinct(StringComparer.OrdinalIgnoreCase);
var tenantScopes = (tenantIds ?? Array.Empty<string>())
.Where(t => !string.Equals(t, RuleStatsKeys.GlobalScope, StringComparison.OrdinalIgnoreCase))
.Distinct(StringComparer.OrdinalIgnoreCase);
foreach (var scope in scopes)
deleted += await DeleteRuleStatsRowsAsync(tableClient, BuildRuleStatsCleanupFilter(scope, cutoffStr), requireLegacyKey: false).ConfigureAwait(false);
foreach (var scope in tenantScopes)
deleted += await DeleteRuleStatsRowsAsync(tableClient, BuildRuleStatsCleanupFilter(scope, tenantCutoffStr), requireLegacyKey: false).ConfigureAwait(false);
deleted += await DeleteRuleStatsRowsAsync(tableClient, BuildRuleStatsCleanupFilter(RuleStatsKeys.GlobalScope, platformCutoffStr), requireLegacyKey: false).ConfigureAwait(false);
if (RuleStatsKeys.LegacyLayoutActive(DateTime.UtcNow))
deleted += await DeleteRuleStatsRowsAsync(tableClient, $"PartitionKey ge '2000-01-01' and PartitionKey lt '{cutoffStr}'", requireLegacyKey: true).ConfigureAwait(false);
deleted += await DeleteRuleStatsRowsAsync(tableClient, $"PartitionKey ge '2000-01-01' and PartitionKey lt '{platformCutoffStr}'", requireLegacyKey: true).ConfigureAwait(false);
if (deleted > 0)
_logger.LogInformation("Deleted {Count} rule stats entries older than {Cutoff}", deleted, cutoffStr);
_logger.LogInformation("Deleted {Count} rule stats entries (tenant ceiling {TenantCutoff}, platform {PlatformCutoff})",
deleted, tenantCutoffStr, platformCutoffStr);
}
catch (Exception ex)
{
@@ -2035,6 +2044,22 @@ namespace AutopilotMonitor.Functions.Services
return deleted;
}
/// <summary>Tenant retention: deletes one tenant's partitions older than the cutoff.</summary>
public async Task<int> DeleteTenantRuleStatsOlderThanAsync(string tenantId, DateTime cutoffDate)
{
try
{
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.RuleStats);
return await DeleteRuleStatsRowsAsync(
tableClient, BuildRuleStatsCleanupFilter(tenantId, cutoffDate.ToString("yyyy-MM-dd")), requireLegacyKey: false).ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to delete old rule stats entries for tenant {TenantId}", tenantId);
return 0;
}
}
private async Task<int> DeleteRuleStatsRowsAsync(TableClient tableClient, string filter, bool requireLegacyKey)
{
var deleted = 0;
@@ -303,15 +303,23 @@ namespace AutopilotMonitor.Functions.Services
}
/// <summary>
/// Retention: deletes aggregate rows older than the cutoff (RowKey starts with the date,
/// so a string compare works across partitions). Mirrors the UsageMetrics 180d policy.
/// Retention bound: deletes daily aggregate rows older than the cutoff in every partition
/// (RowKey starts with the date, so a string compare works across partitions; the
/// "rolling30" rows sort above every date and stay).
/// </summary>
public async Task<int> DeleteTimeAttributionAggregatesOlderThanAsync(DateTime cutoffDate)
public Task<int> DeleteTimeAttributionAggregatesOlderThanAsync(DateTime cutoffDate)
=> DeleteTimeAttributionAggregatesAsync($"RowKey lt '{cutoffDate:yyyy-MM-dd}'");
/// <summary>Tenant retention: deletes one tenant's daily rows older than the cutoff; its rolling rows stay.</summary>
public Task<int> DeleteTenantTimeAttributionAggregatesOlderThanAsync(string tenantId, DateTime cutoffDate)
=> DeleteTimeAttributionAggregatesAsync(
$"PartitionKey eq '{ODataSanitizer.EscapeValue(tenantId)}' and RowKey lt '{cutoffDate:yyyy-MM-dd}'");
private async Task<int> DeleteTimeAttributionAggregatesAsync(string filter)
{
try
{
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.TimeAttributionAggregates);
var filter = $"RowKey lt '{cutoffDate:yyyy-MM-dd}'";
var deleted = 0;
await foreach (var entity in tableClient.QueryAsync<TableEntity>(
filter: filter, select: new[] { "PartitionKey", "RowKey" }))
@@ -2122,8 +2122,8 @@ export function registerAdminTools(server: McpServer, ga: boolean, strictGa: boo
inputSchema: {
serialNumber: z.string().min(1).max(128).optional()
.describe('Device serial number (trimmed, case-insensitive). Omit for the fleet FTR rollup.'),
days: z.coerce.number().int().min(1).max(180).optional()
.describe(daysDescription(30, 180, 'Fleet mode only.')),
days: z.coerce.number().int().min(1).max(365).optional()
.describe(daysDescription(30, 365, 'Fleet mode only.')),
tenantId: z.string().optional().describe(tenantIdDescription(ga, delegated,
'Device mode: the device\'s tenant. Fleet mode: scope to one tenant; omit for cross-tenant.',
'Ignored — data is scoped to your tenant.')),
@@ -52,10 +52,15 @@ namespace AutopilotMonitor.Shared.DataAccess
bool excludeDeletions = false, AuditLogQueryFilters? filters = null);
/// <summary>
/// Retention cleanup: deletes audit log entries older than <paramref name="cutoffUtc"/>
/// across all tenants. Returns the number of rows deleted.
/// Retention cleanup across all tenants: time-encoded entries older than <paramref name="cutoffUtc"/>
/// (the ceiling; each tenant's own window is applied by <see cref="DeleteTenantAuditLogsOlderThanAsync"/>)
/// and legacy entries without a recoverable event time older than <paramref name="legacyCutoffUtc"/>.
/// Returns the number of rows deleted.
/// </summary>
Task<int> DeleteAuditLogsOlderThanAsync(DateTime cutoffUtc);
Task<int> DeleteAuditLogsOlderThanAsync(DateTime cutoffUtc, DateTime legacyCutoffUtc);
/// <summary>One tenant's time-encoded audit entries older than the cutoff; returns the number of rows deleted.</summary>
Task<int> DeleteTenantAuditLogsOlderThanAsync(string tenantId, DateTime cutoffUtc);
// --- Data Retention Queries ---
/// <summary>
@@ -146,8 +146,14 @@ namespace AutopilotMonitor.Shared.DataAccess
/// <summary>Many tenants, one range read each with bounded concurrency; never includes "global".</summary>
Task<List<RuleStatsEntry>> GetRuleStatsForTenantsAsync(IReadOnlyCollection<string> tenantIds, string? startDate = null,
string? endDate = null, string? ruleType = null, int maxResultsPerTenant = 10000);
/// <summary>Retention cleanup per scope (every tenant plus "global"); returns the number of rows deleted.</summary>
Task<int> DeleteRuleStatsOlderThanAsync(DateTime cutoffDate, IReadOnlyCollection<string> tenantIds);
/// <summary>
/// Retention cleanup across scopes: tenant partitions older than <paramref name="tenantCutoffDate"/> (their
/// ceiling; each tenant's own retention is applied by <see cref="DeleteTenantRuleStatsOlderThanAsync"/>), "global"
/// and the legacy layout older than <paramref name="platformCutoffDate"/>. Returns the number of rows deleted.
/// </summary>
Task<int> DeleteRuleStatsOlderThanAsync(DateTime tenantCutoffDate, DateTime platformCutoffDate, IReadOnlyCollection<string> tenantIds);
/// <summary>One tenant's partitions older than the cutoff (tenant retention); returns the number of rows deleted.</summary>
Task<int> DeleteTenantRuleStatsOlderThanAsync(string tenantId, DateTime cutoffDate);
// --- F1 Time Attribution (insights spec §F1, PR2) ---
/// <summary>Computes + stores the session's time breakdown; null when not computable (non-terminal, no duration, events aged out) — fail-soft.</summary>
@@ -165,6 +171,8 @@ namespace AutopilotMonitor.Shared.DataAccess
/// <summary>Deletes one aggregate row (daily date key or "rolling30") — the sweep's stale-bucket reconcile.</summary>
Task DeleteTimeAttributionAggregateAsync(string tenantId, string dateKey, string enrollmentClass);
Task<int> DeleteTimeAttributionAggregatesOlderThanAsync(DateTime cutoffDate);
/// <summary>One tenant's daily rows older than the cutoff (tenant retention); its rolling rows stay.</summary>
Task<int> DeleteTenantTimeAttributionAggregatesOlderThanAsync(string tenantId, DateTime cutoffDate);
// --- F2 Device History / First-Time-Right (insights spec §F2, PR4) ---
/// <summary>Point-reads one device's history row by NORMALIZED serial; null when absent.</summary>
@@ -180,6 +188,8 @@ namespace AutopilotMonitor.Shared.DataAccess
/// <summary>Deletes one daily FTR row — the sweep's stale-bucket reconcile.</summary>
Task DeleteDeviceJourneyAggregateAsync(string tenantId, string dateKey);
Task<int> DeleteDeviceJourneyAggregatesOlderThanAsync(DateTime cutoffDate);
/// <summary>One tenant's daily FTR rows older than the cutoff (tenant retention).</summary>
Task<int> DeleteTenantDeviceJourneyAggregatesOlderThanAsync(string tenantId, DateTime cutoffDate);
// --- Verdict calibration (internal/docs/backend/verdict-calibration.md) ---
Task<bool> SaveVerdictCalibrationAggregateAsync(VerdictCalibrationDailyAggregate aggregate);