From 5f4519d635d1fc3c8b5712e7ddcfb355155c4923 Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Sat, 7 Feb 2026 20:54:48 +0100 Subject: [PATCH] DailyMaintananceFunction restructuring --- .../Functions/DailyMaintenanceFunction.cs | 503 +----------------- .../Functions/TriggerMaintenanceFunction.cs | 8 +- .../AutopilotMonitor.Functions/Program.cs | 3 +- .../Services/MaintenanceService.cs | 482 +++++++++++++++++ 4 files changed, 494 insertions(+), 502 deletions(-) create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/DailyMaintenanceFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/DailyMaintenanceFunction.cs index a58549b2..f3a68f75 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/DailyMaintenanceFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/DailyMaintenanceFunction.cs @@ -1,38 +1,24 @@ -using System; -using System.Diagnostics; -using System.Linq; using System.Threading.Tasks; using AutopilotMonitor.Functions.Services; -using AutopilotMonitor.Shared.Models; using Microsoft.Azure.Functions.Worker; using Microsoft.Extensions.Logging; -using Newtonsoft.Json; namespace AutopilotMonitor.Functions.Functions { /// - /// Daily maintenance job that: - /// 1. Marks stalled sessions (InProgress for too long) as timed out - /// 2. Aggregates yesterday's metrics into historical snapshots - /// 3. Deletes old sessions and events based on tenant retention policies - /// Runs daily at 2 AM UTC + /// Timer-triggered Azure Function that runs daily maintenance. + /// Delegates all logic to MaintenanceService. /// public class DailyMaintenanceFunction { - private readonly TableStorageService _storageService; - private readonly TenantConfigurationService _tenantConfigService; - private readonly UsageMetricsService _usageMetricsService; + private readonly MaintenanceService _maintenanceService; private readonly ILogger _logger; public DailyMaintenanceFunction( - TableStorageService storageService, - TenantConfigurationService tenantConfigService, - UsageMetricsService usageMetricsService, + MaintenanceService maintenanceService, ILogger logger) { - _storageService = storageService; - _tenantConfigService = tenantConfigService; - _usageMetricsService = usageMetricsService; + _maintenanceService = maintenanceService; _logger = logger; } @@ -43,483 +29,8 @@ namespace AutopilotMonitor.Functions.Functions [Function("DailyMaintenance")] public async Task Run([TimerTrigger("0 0 2 * * *")] object timer) { - _logger.LogInformation($"Daily maintenance started at {DateTime.UtcNow}"); - var maintenanceStart = Stopwatch.StartNew(); - - try - { - // Step 1: Mark stalled sessions as timed out - await MarkStalledSessionsAsTimedOutAsync(); - - // Step 2: Aggregate yesterday's metrics into historical snapshots - await AggregateYesterdayMetricsAsync(); - - // Step 3: Clean up old data based on retention policies - await CleanupOldDataAsync(); - - // Step 4: Recompute platform-wide stats for landing page - await RecomputePlatformStatsAsync(); - - maintenanceStart.Stop(); - _logger.LogInformation($"Daily maintenance completed in {maintenanceStart.ElapsedMilliseconds}ms"); - } - catch (Exception ex) - { - _logger.LogError(ex, "Daily maintenance failed"); - } + _logger.LogInformation("DailyMaintenance timer trigger fired"); + await _maintenanceService.RunAllAsync(); } - - /// - /// Manually triggered maintenance with flexible date selection - /// - public async Task RunManualAsync(DateTime? targetDate = null, bool aggregateOnly = false, string triggeredBy = "Unknown") - { - _logger.LogInformation($"Manual maintenance triggered by {triggeredBy} at {DateTime.UtcNow}"); - var maintenanceStart = Stopwatch.StartNew(); - var result = new MaintenanceResult { TriggeredBy = triggeredBy, TriggeredAt = DateTime.UtcNow }; - - try - { - if (!aggregateOnly) - { - // Step 1: Mark stalled sessions as timed out - await MarkStalledSessionsAsTimedOutAsync(); - result.StalledSessionsChecked = true; - } - - // Step 2: Aggregate metrics for specified date (or yesterday if not specified) - var dateToAggregate = targetDate ?? DateTime.UtcNow.AddDays(-1).Date; - await AggregateMetricsForDateAsync(dateToAggregate); - result.AggregatedDate = dateToAggregate.ToString("yyyy-MM-dd"); - result.MetricsAggregated = true; - - if (!aggregateOnly) - { - // Step 3: Clean up old data based on retention policies - await CleanupOldDataAsync(); - result.DataCleanupExecuted = true; - } - - // Step 4: Recompute platform-wide stats - await RecomputePlatformStatsAsync(); - result.PlatformStatsRecomputed = true; - - maintenanceStart.Stop(); - result.DurationMs = (int)maintenanceStart.ElapsedMilliseconds; - result.Success = true; - - _logger.LogInformation($"Manual maintenance completed in {maintenanceStart.ElapsedMilliseconds}ms"); - return result; - } - catch (Exception ex) - { - _logger.LogError(ex, "Manual maintenance failed"); - result.Success = false; - result.Error = ex.Message; - maintenanceStart.Stop(); - result.DurationMs = (int)maintenanceStart.ElapsedMilliseconds; - return result; - } - } - - /// - /// Marks sessions that have been in "InProgress" status for too long as "Failed - Timed Out" - /// This prevents stalled sessions from running indefinitely and skewing statistics - /// - private async Task MarkStalledSessionsAsTimedOutAsync() - { - _logger.LogInformation("Checking for stalled sessions..."); - var stalledStart = Stopwatch.StartNew(); - - try - { - // Get all unique tenant IDs - var tenantIds = await _storageService.GetAllTenantIdsAsync(); - - int totalSessionsTimedOut = 0; - - foreach (var tenantId in tenantIds) - { - try - { - // Get tenant configuration to determine session timeout - var config = await _tenantConfigService.GetConfigurationAsync(tenantId); - var timeoutHours = config?.SessionTimeoutHours ?? 5; // Default 5 hours if not configured - - var cutoffTime = DateTime.UtcNow.AddHours(-timeoutHours); - - // Get all sessions for this tenant - var sessions = await _storageService.GetSessionsAsync(tenantId, maxResults: 1000000); - - // Find sessions that are still in progress but started before cutoff time - var stalledSessions = sessions - .Where(s => s.Status == SessionStatus.InProgress && s.StartedAt < cutoffTime) - .ToList(); - - if (stalledSessions.Count == 0) - { - _logger.LogInformation($"Tenant {tenantId}: No stalled sessions found (timeout: {timeoutHours}h)"); - continue; - } - - int sessionCount = 0; - - foreach (var session in stalledSessions) - { - // Mark session as failed with timeout reason - await _storageService.UpdateSessionStatusAsync( - session.TenantId, - session.SessionId, - SessionStatus.Failed, - failureReason: $"Session timed out after {timeoutHours} hours (started at {session.StartedAt:yyyy-MM-dd HH:mm:ss} UTC)" - ); - sessionCount++; - } - - totalSessionsTimedOut += sessionCount; - - // Log audit entry - await _storageService.LogAuditEntryAsync( - tenantId, - "SessionTimeout", - "Session", - $"{sessionCount} sessions", - "System.DailyMaintenance", - new System.Collections.Generic.Dictionary - { - { "SessionsTimedOut", sessionCount.ToString() }, - { "TimeoutHours", timeoutHours.ToString() }, - { "CutoffTime", cutoffTime.ToString("yyyy-MM-dd HH:mm:ss") } - }); - - _logger.LogInformation($"Tenant {tenantId}: Marked {sessionCount} stalled sessions as timed out (timeout: {timeoutHours}h)"); - } - catch (Exception ex) - { - _logger.LogError(ex, $"Failed to check stalled sessions for tenant {tenantId}"); - } - } - - stalledStart.Stop(); - _logger.LogInformation($"Stalled session check completed: {totalSessionsTimedOut} sessions timed out in {stalledStart.ElapsedMilliseconds}ms"); - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to check for stalled sessions"); - } - } - - /// - /// Aggregates yesterday's metrics and saves them as historical snapshots - /// - private async Task AggregateYesterdayMetricsAsync() - { - var yesterday = DateTime.UtcNow.AddDays(-1).Date; - await AggregateMetricsForDateAsync(yesterday); - } - - /// - /// Aggregates metrics for a specific date and saves them as historical snapshots - /// - private async Task AggregateMetricsForDateAsync(DateTime targetDate) - { - _logger.LogInformation($"Aggregating metrics for {targetDate:yyyy-MM-dd}..."); - var aggregateStart = Stopwatch.StartNew(); - - try - { - var targetDateStr = targetDate.ToString("yyyy-MM-dd"); - - // Get all sessions from target date - var allSessions = await _storageService.GetAllSessionsAsync(maxResults: 1000000); - var targetDateSessions = allSessions.Where(s => s.StartedAt.Date == targetDate).ToList(); - - if (targetDateSessions.Count == 0) - { - _logger.LogInformation($"No sessions found for {targetDateStr}"); - return; - } - - // Aggregate global metrics (cross-tenant) - var globalMetrics = await ComputeUsageMetricsSnapshotAsync(targetDateStr, "global", targetDateSessions); - await _storageService.SaveUsageMetricsSnapshotAsync(globalMetrics); - - // Aggregate per-tenant metrics - var tenantGroups = targetDateSessions.GroupBy(s => s.TenantId); - foreach (var tenantGroup in tenantGroups) - { - var tenantMetrics = await ComputeUsageMetricsSnapshotAsync(targetDateStr, tenantGroup.Key, tenantGroup.ToList()); - await _storageService.SaveUsageMetricsSnapshotAsync(tenantMetrics); - } - - aggregateStart.Stop(); - _logger.LogInformation($"Aggregated metrics for {targetDateSessions.Count} sessions from {targetDateStr} in {aggregateStart.ElapsedMilliseconds}ms"); - } - catch (Exception ex) - { - _logger.LogError(ex, $"Failed to aggregate metrics for {targetDate:yyyy-MM-dd}"); - throw; // Re-throw for manual trigger error handling - } - } - - /// - /// Computes historical metrics for a specific date and tenant - /// - private async Task ComputeUsageMetricsSnapshotAsync(string date, string tenantId, List sessions) - { - var computeStart = Stopwatch.StartNew(); - - // Session statistics - var completed = sessions.Where(s => s.Status == SessionStatus.Succeeded || s.Status == SessionStatus.Failed).ToList(); - var succeeded = sessions.Count(s => s.Status == SessionStatus.Succeeded); - var successRate = completed.Count > 0 ? Math.Round((succeeded / (double)completed.Count) * 100, 1) : 0; - - // Performance statistics - var completedWithDuration = sessions.Where(s => s.DurationSeconds.HasValue && s.DurationSeconds.Value > 0).ToList(); - double avgDuration = 0, medianDuration = 0, p95Duration = 0, p99Duration = 0; - - if (completedWithDuration.Any()) - { - var durations = completedWithDuration.Select(s => s.DurationSeconds!.Value / 60.0).OrderBy(d => d).ToList(); - avgDuration = Math.Round(durations.Average(), 1); - medianDuration = CalculatePercentile(durations, 50); - p95Duration = CalculatePercentile(durations, 95); - p99Duration = CalculatePercentile(durations, 99); - } - - // Hardware statistics (top 5 only) - var manufacturers = sessions - .GroupBy(s => s.Manufacturer) - .Select(g => new { Name = g.Key, Count = g.Count(), Percentage = Math.Round((g.Count() / (double)sessions.Count) * 100, 1) }) - .OrderByDescending(x => x.Count) - .Take(5) - .ToList(); - - var models = sessions - .GroupBy(s => s.Model) - .Select(g => new { Name = g.Key, Count = g.Count(), Percentage = Math.Round((g.Count() / (double)sessions.Count) * 100, 1) }) - .OrderByDescending(x => x.Count) - .Take(5) - .ToList(); - - // User activity for this date - var targetDate = DateTime.ParseExact(date, "yyyy-MM-dd", null); - var (uniqueUsers, loginCount) = await _storageService.GetUserActivityForDateAsync( - tenantId == "global" ? null : tenantId, targetDate); - - computeStart.Stop(); - - return new UsageMetricsSnapshot - { - Date = date, - TenantId = tenantId, - ComputedAt = DateTime.UtcNow, - ComputeDurationMs = (int)computeStart.ElapsedMilliseconds, - SessionsTotal = sessions.Count, - SessionsSucceeded = succeeded, - SessionsFailed = sessions.Count(s => s.Status == SessionStatus.Failed), - SessionsInProgress = sessions.Count(s => s.Status == SessionStatus.InProgress), - SessionsSuccessRate = successRate, - AvgDurationMinutes = avgDuration, - MedianDurationMinutes = medianDuration, - P95DurationMinutes = p95Duration, - P99DurationMinutes = p99Duration, - UniqueTenants = tenantId == "global" ? sessions.Select(s => s.TenantId).Distinct().Count() : 0, - UniqueUsers = uniqueUsers, - LoginCount = loginCount, - TopManufacturers = JsonConvert.SerializeObject(manufacturers), - TopModels = JsonConvert.SerializeObject(models) - }; - } - - /// - /// Cleans up old sessions and events based on tenant retention policies - /// - private async Task CleanupOldDataAsync() - { - _logger.LogInformation("Starting data retention cleanup..."); - var cleanupStart = Stopwatch.StartNew(); - - try - { - // Get all unique tenant IDs - var tenantIds = await _storageService.GetAllTenantIdsAsync(); - - int totalSessionsDeleted = 0; - int totalEventsDeleted = 0; - - foreach (var tenantId in tenantIds) - { - try - { - // Get tenant configuration to determine retention period - var config = await _tenantConfigService.GetConfigurationAsync(tenantId); - var retentionDays = config?.DataRetentionDays ?? 90; // Default 90 days if not configured - - var cutoffDate = DateTime.UtcNow.AddDays(-retentionDays); - - // Find sessions older than retention period - var oldSessions = await _storageService.GetSessionsOlderThanAsync(tenantId, cutoffDate); - - if (oldSessions.Count == 0) - { - _logger.LogInformation($"Tenant {tenantId}: No sessions older than {retentionDays} days"); - continue; - } - - // Delete each session and all associated data - int sessionCount = 0; - int eventCount = 0; - int ruleResultCount = 0; - int appSummaryCount = 0; - - foreach (var session in oldSessions) - { - // Delete all events for this session - var deletedEvents = await _storageService.DeleteSessionEventsAsync(session.TenantId, session.SessionId); - eventCount += deletedEvents; - - // Delete all rule results for this session - var deletedRuleResults = await _storageService.DeleteSessionRuleResultsAsync(session.TenantId, session.SessionId); - ruleResultCount += deletedRuleResults; - - // Delete all app install summaries for this session - var deletedAppSummaries = await _storageService.DeleteSessionAppInstallSummariesAsync(session.TenantId, session.SessionId); - appSummaryCount += deletedAppSummaries; - - // Delete the session itself - await _storageService.DeleteSessionAsync(session.TenantId, session.SessionId); - sessionCount++; - } - - totalSessionsDeleted += sessionCount; - totalEventsDeleted += eventCount; - - // Log audit entry - await _storageService.LogAuditEntryAsync( - tenantId, - "DataRetentionCleanup", - "Session", - $"{sessionCount} sessions", - "System.DailyMaintenance", - new System.Collections.Generic.Dictionary - { - { "SessionsDeleted", sessionCount.ToString() }, - { "EventsDeleted", eventCount.ToString() }, - { "RuleResultsDeleted", ruleResultCount.ToString() }, - { "AppSummariesDeleted", appSummaryCount.ToString() }, - { "RetentionDays", retentionDays.ToString() }, - { "CutoffDate", cutoffDate.ToString("yyyy-MM-dd") } - }); - - _logger.LogInformation($"Tenant {tenantId}: Deleted {sessionCount} sessions, {eventCount} events, {ruleResultCount} rule results, {appSummaryCount} app summaries (retention: {retentionDays} days)"); - } - catch (Exception ex) - { - _logger.LogError(ex, $"Failed to cleanup data for tenant {tenantId}"); - } - } - - cleanupStart.Stop(); - _logger.LogInformation($"Data retention cleanup completed: {totalSessionsDeleted} sessions and {totalEventsDeleted} events deleted in {cleanupStart.ElapsedMilliseconds}ms"); - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to cleanup old data"); - } - } - - /// - /// Recomputes platform-wide stats from all tables. - /// Used on the public landing page (no auth required). - /// - private async Task RecomputePlatformStatsAsync() - { - _logger.LogInformation("Recomputing platform stats..."); - var sw = Stopwatch.StartNew(); - - try - { - var tenantIds = await _storageService.GetAllTenantIdsAsync(); - long totalEnrollments = 0; - long successfulEnrollments = 0; - long totalEvents = 0; - long totalUsers = 0; - var uniqueModels = new HashSet(StringComparer.OrdinalIgnoreCase); - - foreach (var tid in tenantIds) - { - var sessions = await _storageService.GetSessionsAsync(tid, maxResults: 10000); - totalEnrollments += sessions.Count; - successfulEnrollments += sessions.Count(s => s.Status == SessionStatus.Succeeded); - - foreach (var s in sessions) - { - var modelKey = $"{s.Manufacturer} {s.Model}".Trim(); - if (!string.IsNullOrEmpty(modelKey)) - uniqueModels.Add(modelKey); - totalEvents += s.EventCount; - } - - var userMetrics = await _storageService.GetUserActivityMetricsAsync(tid); - totalUsers += userMetrics.TotalUniqueUsers; - } - - // Preserve IssuesDetected from existing stats (incremented during ingestion) - var existingStats = await _storageService.GetPlatformStatsAsync(); - - var stats = new PlatformStats - { - TotalEnrollments = totalEnrollments, - TotalUsers = totalUsers, - TotalTenants = tenantIds.Count, - UniqueDeviceModels = uniqueModels.Count, - TotalEventsProcessed = totalEvents, - SuccessfulEnrollments = successfulEnrollments, - IssuesDetected = existingStats?.IssuesDetected ?? 0, - LastFullCompute = DateTime.UtcNow, - LastUpdated = DateTime.UtcNow - }; - - await _storageService.SavePlatformStatsAsync(stats); - - sw.Stop(); - _logger.LogInformation($"Platform stats recomputed in {sw.ElapsedMilliseconds}ms: " + - $"{totalEnrollments} enrollments, {totalUsers} users, {tenantIds.Count} tenants, {uniqueModels.Count} models"); - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to recompute platform stats"); - } - } - - /// - /// Calculates percentile value from sorted list - /// - private double CalculatePercentile(List sortedValues, int percentile) - { - if (sortedValues.Count == 0) return 0; - - var index = (int)Math.Ceiling((percentile / 100.0) * sortedValues.Count) - 1; - index = Math.Max(0, Math.Min(index, sortedValues.Count - 1)); - return Math.Round(sortedValues[index], 1); - } - } - - /// - /// Result of a maintenance operation - /// - public class MaintenanceResult - { - public bool Success { get; set; } - public string? Error { get; set; } - public string TriggeredBy { get; set; } = string.Empty; - public DateTime TriggeredAt { get; set; } - public int DurationMs { get; set; } - public bool StalledSessionsChecked { get; set; } - public bool MetricsAggregated { get; set; } - public string? AggregatedDate { get; set; } - public bool DataCleanupExecuted { get; set; } - public bool PlatformStatsRecomputed { get; set; } } } diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/TriggerMaintenanceFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/TriggerMaintenanceFunction.cs index 88364fd0..8c3bb779 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/TriggerMaintenanceFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/TriggerMaintenanceFunction.cs @@ -16,16 +16,16 @@ namespace AutopilotMonitor.Functions.Functions public class TriggerMaintenanceFunction { private readonly ILogger _logger; - private readonly DailyMaintenanceFunction _maintenanceFunction; + private readonly MaintenanceService _maintenanceService; private readonly GalacticAdminService _galacticAdminService; public TriggerMaintenanceFunction( ILogger logger, - DailyMaintenanceFunction maintenanceFunction, + MaintenanceService maintenanceService, GalacticAdminService galacticAdminService) { _logger = logger; - _maintenanceFunction = maintenanceFunction; + _maintenanceService = maintenanceService; _galacticAdminService = galacticAdminService; } @@ -99,7 +99,7 @@ namespace AutopilotMonitor.Functions.Functions bool aggregateOnly = aggregateOnlyParam?.ToLower() == "true"; // Execute maintenance tasks - var result = await _maintenanceFunction.RunManualAsync(targetDate, aggregateOnly, userEmail); + var result = await _maintenanceService.RunManualAsync(targetDate, aggregateOnly, userEmail); var response = req.CreateResponse(HttpStatusCode.OK); await response.WriteAsJsonAsync(new diff --git a/src/Backend/AutopilotMonitor.Functions/Program.cs b/src/Backend/AutopilotMonitor.Functions/Program.cs index 2198c85d..59a56e46 100644 --- a/src/Backend/AutopilotMonitor.Functions/Program.cs +++ b/src/Backend/AutopilotMonitor.Functions/Program.cs @@ -1,6 +1,5 @@ using System.Text.Json; using System.Text.Json.Serialization; -using AutopilotMonitor.Functions.Functions; using AutopilotMonitor.Functions.Middleware; using AutopilotMonitor.Functions.Services; using Microsoft.AspNetCore.Authentication.JwtBearer; @@ -101,6 +100,6 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); -builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Build().Run(); diff --git a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs new file mode 100644 index 00000000..350560ec --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs @@ -0,0 +1,482 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using AutopilotMonitor.Shared.Models; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +namespace AutopilotMonitor.Functions.Services +{ + /// + /// Result of a maintenance operation + /// + public class MaintenanceResult + { + public bool Success { get; set; } + public string? Error { get; set; } + public string TriggeredBy { get; set; } = string.Empty; + public DateTime TriggeredAt { get; set; } + public int DurationMs { get; set; } + public bool StalledSessionsChecked { get; set; } + public bool MetricsAggregated { get; set; } + public string? AggregatedDate { get; set; } + public bool DataCleanupExecuted { get; set; } + public bool PlatformStatsRecomputed { get; set; } + } + + /// + /// Dedicated service for maintenance tasks: + /// 1. Marks stalled sessions (InProgress for too long) as timed out + /// 2. Aggregates metrics into historical snapshots + /// 3. Deletes old sessions and events based on tenant retention policies + /// 4. Recomputes platform-wide stats for the landing page + /// + public class MaintenanceService + { + private readonly TableStorageService _storageService; + private readonly TenantConfigurationService _tenantConfigService; + private readonly UsageMetricsService _usageMetricsService; + private readonly ILogger _logger; + + public MaintenanceService( + TableStorageService storageService, + TenantConfigurationService tenantConfigService, + UsageMetricsService usageMetricsService, + ILogger logger) + { + _storageService = storageService; + _tenantConfigService = tenantConfigService; + _usageMetricsService = usageMetricsService; + _logger = logger; + } + + /// + /// Runs all maintenance tasks (used by the daily timer trigger) + /// + public async Task RunAllAsync() + { + _logger.LogInformation($"Daily maintenance started at {DateTime.UtcNow}"); + var maintenanceStart = Stopwatch.StartNew(); + + try + { + await MarkStalledSessionsAsTimedOutAsync(); + await AggregateYesterdayMetricsAsync(); + await CleanupOldDataAsync(); + await RecomputePlatformStatsAsync(); + + maintenanceStart.Stop(); + _logger.LogInformation($"Daily maintenance completed in {maintenanceStart.ElapsedMilliseconds}ms"); + } + catch (Exception ex) + { + _logger.LogError(ex, "Daily maintenance failed"); + } + } + + /// + /// Manually triggered maintenance with flexible date selection + /// + public async Task RunManualAsync(DateTime? targetDate = null, bool aggregateOnly = false, string triggeredBy = "Unknown") + { + _logger.LogInformation($"Manual maintenance triggered by {triggeredBy} at {DateTime.UtcNow}"); + var maintenanceStart = Stopwatch.StartNew(); + var result = new MaintenanceResult { TriggeredBy = triggeredBy, TriggeredAt = DateTime.UtcNow }; + + try + { + if (!aggregateOnly) + { + await MarkStalledSessionsAsTimedOutAsync(); + result.StalledSessionsChecked = true; + } + + var dateToAggregate = targetDate ?? DateTime.UtcNow.AddDays(-1).Date; + await AggregateMetricsForDateAsync(dateToAggregate); + result.AggregatedDate = dateToAggregate.ToString("yyyy-MM-dd"); + result.MetricsAggregated = true; + + if (!aggregateOnly) + { + await CleanupOldDataAsync(); + result.DataCleanupExecuted = true; + } + + await RecomputePlatformStatsAsync(); + result.PlatformStatsRecomputed = true; + + maintenanceStart.Stop(); + result.DurationMs = (int)maintenanceStart.ElapsedMilliseconds; + result.Success = true; + + _logger.LogInformation($"Manual maintenance completed in {maintenanceStart.ElapsedMilliseconds}ms"); + return result; + } + catch (Exception ex) + { + _logger.LogError(ex, "Manual maintenance failed"); + result.Success = false; + result.Error = ex.Message; + maintenanceStart.Stop(); + result.DurationMs = (int)maintenanceStart.ElapsedMilliseconds; + return result; + } + } + + /// + /// Marks sessions that have been in "InProgress" status for too long as "Failed - Timed Out" + /// + private async Task MarkStalledSessionsAsTimedOutAsync() + { + _logger.LogInformation("Checking for stalled sessions..."); + var stalledStart = Stopwatch.StartNew(); + + try + { + var tenantIds = await _storageService.GetAllTenantIdsAsync(); + int totalSessionsTimedOut = 0; + + foreach (var tenantId in tenantIds) + { + try + { + var config = await _tenantConfigService.GetConfigurationAsync(tenantId); + var timeoutHours = config?.SessionTimeoutHours ?? 5; + var cutoffTime = DateTime.UtcNow.AddHours(-timeoutHours); + + var sessions = await _storageService.GetSessionsAsync(tenantId, maxResults: 1000000); + var stalledSessions = sessions + .Where(s => s.Status == SessionStatus.InProgress && s.StartedAt < cutoffTime) + .ToList(); + + if (stalledSessions.Count == 0) + { + _logger.LogInformation($"Tenant {tenantId}: No stalled sessions found (timeout: {timeoutHours}h)"); + continue; + } + + int sessionCount = 0; + + foreach (var session in stalledSessions) + { + await _storageService.UpdateSessionStatusAsync( + session.TenantId, + session.SessionId, + SessionStatus.Failed, + failureReason: $"Session timed out after {timeoutHours} hours (started at {session.StartedAt:yyyy-MM-dd HH:mm:ss} UTC)" + ); + sessionCount++; + } + + totalSessionsTimedOut += sessionCount; + + await _storageService.LogAuditEntryAsync( + tenantId, + "SessionTimeout", + "Session", + $"{sessionCount} sessions", + "System.DailyMaintenance", + new Dictionary + { + { "SessionsTimedOut", sessionCount.ToString() }, + { "TimeoutHours", timeoutHours.ToString() }, + { "CutoffTime", cutoffTime.ToString("yyyy-MM-dd HH:mm:ss") } + }); + + _logger.LogInformation($"Tenant {tenantId}: Marked {sessionCount} stalled sessions as timed out (timeout: {timeoutHours}h)"); + } + catch (Exception ex) + { + _logger.LogError(ex, $"Failed to check stalled sessions for tenant {tenantId}"); + } + } + + stalledStart.Stop(); + _logger.LogInformation($"Stalled session check completed: {totalSessionsTimedOut} sessions timed out in {stalledStart.ElapsedMilliseconds}ms"); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to check for stalled sessions"); + } + } + + /// + /// Aggregates yesterday's metrics and saves them as historical snapshots + /// + private async Task AggregateYesterdayMetricsAsync() + { + var yesterday = DateTime.UtcNow.AddDays(-1).Date; + await AggregateMetricsForDateAsync(yesterday); + } + + /// + /// Aggregates metrics for a specific date and saves them as historical snapshots + /// + private async Task AggregateMetricsForDateAsync(DateTime targetDate) + { + _logger.LogInformation($"Aggregating metrics for {targetDate:yyyy-MM-dd}..."); + var aggregateStart = Stopwatch.StartNew(); + + try + { + var targetDateStr = targetDate.ToString("yyyy-MM-dd"); + + var allSessions = await _storageService.GetAllSessionsAsync(maxResults: 1000000); + var targetDateSessions = allSessions.Where(s => s.StartedAt.Date == targetDate).ToList(); + + if (targetDateSessions.Count == 0) + { + _logger.LogInformation($"No sessions found for {targetDateStr}"); + return; + } + + var globalMetrics = await ComputeUsageMetricsSnapshotAsync(targetDateStr, "global", targetDateSessions); + await _storageService.SaveUsageMetricsSnapshotAsync(globalMetrics); + + var tenantGroups = targetDateSessions.GroupBy(s => s.TenantId); + foreach (var tenantGroup in tenantGroups) + { + var tenantMetrics = await ComputeUsageMetricsSnapshotAsync(targetDateStr, tenantGroup.Key, tenantGroup.ToList()); + await _storageService.SaveUsageMetricsSnapshotAsync(tenantMetrics); + } + + aggregateStart.Stop(); + _logger.LogInformation($"Aggregated metrics for {targetDateSessions.Count} sessions from {targetDateStr} in {aggregateStart.ElapsedMilliseconds}ms"); + } + catch (Exception ex) + { + _logger.LogError(ex, $"Failed to aggregate metrics for {targetDate:yyyy-MM-dd}"); + throw; + } + } + + /// + /// Computes historical metrics for a specific date and tenant + /// + private async Task ComputeUsageMetricsSnapshotAsync(string date, string tenantId, List sessions) + { + var computeStart = Stopwatch.StartNew(); + + var completed = sessions.Where(s => s.Status == SessionStatus.Succeeded || s.Status == SessionStatus.Failed).ToList(); + var succeeded = sessions.Count(s => s.Status == SessionStatus.Succeeded); + var successRate = completed.Count > 0 ? Math.Round((succeeded / (double)completed.Count) * 100, 1) : 0; + + var completedWithDuration = sessions.Where(s => s.DurationSeconds.HasValue && s.DurationSeconds.Value > 0).ToList(); + double avgDuration = 0, medianDuration = 0, p95Duration = 0, p99Duration = 0; + + if (completedWithDuration.Any()) + { + var durations = completedWithDuration.Select(s => s.DurationSeconds!.Value / 60.0).OrderBy(d => d).ToList(); + avgDuration = Math.Round(durations.Average(), 1); + medianDuration = CalculatePercentile(durations, 50); + p95Duration = CalculatePercentile(durations, 95); + p99Duration = CalculatePercentile(durations, 99); + } + + var manufacturers = sessions + .GroupBy(s => s.Manufacturer) + .Select(g => new { Name = g.Key, Count = g.Count(), Percentage = Math.Round((g.Count() / (double)sessions.Count) * 100, 1) }) + .OrderByDescending(x => x.Count) + .Take(5) + .ToList(); + + var models = sessions + .GroupBy(s => s.Model) + .Select(g => new { Name = g.Key, Count = g.Count(), Percentage = Math.Round((g.Count() / (double)sessions.Count) * 100, 1) }) + .OrderByDescending(x => x.Count) + .Take(5) + .ToList(); + + var targetDate = DateTime.ParseExact(date, "yyyy-MM-dd", null); + var (uniqueUsers, loginCount) = await _storageService.GetUserActivityForDateAsync( + tenantId == "global" ? null : tenantId, targetDate); + + computeStart.Stop(); + + return new UsageMetricsSnapshot + { + Date = date, + TenantId = tenantId, + ComputedAt = DateTime.UtcNow, + ComputeDurationMs = (int)computeStart.ElapsedMilliseconds, + SessionsTotal = sessions.Count, + SessionsSucceeded = succeeded, + SessionsFailed = sessions.Count(s => s.Status == SessionStatus.Failed), + SessionsInProgress = sessions.Count(s => s.Status == SessionStatus.InProgress), + SessionsSuccessRate = successRate, + AvgDurationMinutes = avgDuration, + MedianDurationMinutes = medianDuration, + P95DurationMinutes = p95Duration, + P99DurationMinutes = p99Duration, + UniqueTenants = tenantId == "global" ? sessions.Select(s => s.TenantId).Distinct().Count() : 0, + UniqueUsers = uniqueUsers, + LoginCount = loginCount, + TopManufacturers = JsonConvert.SerializeObject(manufacturers), + TopModels = JsonConvert.SerializeObject(models) + }; + } + + /// + /// Cleans up old sessions and events based on tenant retention policies + /// + private async Task CleanupOldDataAsync() + { + _logger.LogInformation("Starting data retention cleanup..."); + var cleanupStart = Stopwatch.StartNew(); + + try + { + var tenantIds = await _storageService.GetAllTenantIdsAsync(); + int totalSessionsDeleted = 0; + int totalEventsDeleted = 0; + + foreach (var tenantId in tenantIds) + { + try + { + var config = await _tenantConfigService.GetConfigurationAsync(tenantId); + var retentionDays = config?.DataRetentionDays ?? 90; + var cutoffDate = DateTime.UtcNow.AddDays(-retentionDays); + + var oldSessions = await _storageService.GetSessionsOlderThanAsync(tenantId, cutoffDate); + + if (oldSessions.Count == 0) + { + _logger.LogInformation($"Tenant {tenantId}: No sessions older than {retentionDays} days"); + continue; + } + + int sessionCount = 0; + int eventCount = 0; + int ruleResultCount = 0; + int appSummaryCount = 0; + + foreach (var session in oldSessions) + { + var deletedEvents = await _storageService.DeleteSessionEventsAsync(session.TenantId, session.SessionId); + eventCount += deletedEvents; + + var deletedRuleResults = await _storageService.DeleteSessionRuleResultsAsync(session.TenantId, session.SessionId); + ruleResultCount += deletedRuleResults; + + var deletedAppSummaries = await _storageService.DeleteSessionAppInstallSummariesAsync(session.TenantId, session.SessionId); + appSummaryCount += deletedAppSummaries; + + await _storageService.DeleteSessionAsync(session.TenantId, session.SessionId); + sessionCount++; + } + + totalSessionsDeleted += sessionCount; + totalEventsDeleted += eventCount; + + await _storageService.LogAuditEntryAsync( + tenantId, + "DataRetentionCleanup", + "Session", + $"{sessionCount} sessions", + "System.DailyMaintenance", + new Dictionary + { + { "SessionsDeleted", sessionCount.ToString() }, + { "EventsDeleted", eventCount.ToString() }, + { "RuleResultsDeleted", ruleResultCount.ToString() }, + { "AppSummariesDeleted", appSummaryCount.ToString() }, + { "RetentionDays", retentionDays.ToString() }, + { "CutoffDate", cutoffDate.ToString("yyyy-MM-dd") } + }); + + _logger.LogInformation($"Tenant {tenantId}: Deleted {sessionCount} sessions, {eventCount} events, {ruleResultCount} rule results, {appSummaryCount} app summaries (retention: {retentionDays} days)"); + } + catch (Exception ex) + { + _logger.LogError(ex, $"Failed to cleanup data for tenant {tenantId}"); + } + } + + cleanupStart.Stop(); + _logger.LogInformation($"Data retention cleanup completed: {totalSessionsDeleted} sessions and {totalEventsDeleted} events deleted in {cleanupStart.ElapsedMilliseconds}ms"); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to cleanup old data"); + } + } + + /// + /// Recomputes platform-wide stats from all tables. + /// Used on the public landing page (no auth required). + /// + private async Task RecomputePlatformStatsAsync() + { + _logger.LogInformation("Recomputing platform stats..."); + var sw = Stopwatch.StartNew(); + + try + { + var tenantIds = await _storageService.GetAllTenantIdsAsync(); + long totalEnrollments = 0; + long successfulEnrollments = 0; + long totalEvents = 0; + long totalUsers = 0; + var uniqueModels = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var tid in tenantIds) + { + var sessions = await _storageService.GetSessionsAsync(tid, maxResults: 10000); + totalEnrollments += sessions.Count; + successfulEnrollments += sessions.Count(s => s.Status == SessionStatus.Succeeded); + + foreach (var s in sessions) + { + var modelKey = $"{s.Manufacturer} {s.Model}".Trim(); + if (!string.IsNullOrEmpty(modelKey)) + uniqueModels.Add(modelKey); + totalEvents += s.EventCount; + } + + var userMetrics = await _storageService.GetUserActivityMetricsAsync(tid); + totalUsers += userMetrics.TotalUniqueUsers; + } + + var existingStats = await _storageService.GetPlatformStatsAsync(); + + var stats = new PlatformStats + { + TotalEnrollments = totalEnrollments, + TotalUsers = totalUsers, + TotalTenants = tenantIds.Count, + UniqueDeviceModels = uniqueModels.Count, + TotalEventsProcessed = totalEvents, + SuccessfulEnrollments = successfulEnrollments, + IssuesDetected = existingStats?.IssuesDetected ?? 0, + LastFullCompute = DateTime.UtcNow, + LastUpdated = DateTime.UtcNow + }; + + await _storageService.SavePlatformStatsAsync(stats); + + sw.Stop(); + _logger.LogInformation($"Platform stats recomputed in {sw.ElapsedMilliseconds}ms: " + + $"{totalEnrollments} enrollments, {totalUsers} users, {tenantIds.Count} tenants, {uniqueModels.Count} models"); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to recompute platform stats"); + } + } + + /// + /// Calculates percentile value from sorted list + /// + private double CalculatePercentile(List sortedValues, int percentile) + { + if (sortedValues.Count == 0) return 0; + + var index = (int)Math.Ceiling((percentile / 100.0) * sortedValues.Count) - 1; + index = Math.Max(0, Math.Min(index, sortedValues.Count - 1)); + return Math.Round(sortedValues[index], 1); + } + } +}