AutopilotMonitor/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;

namespace AutopilotMonitor.Functions.Services
{
    /// <summary>
    /// Result of a maintenance operation
    /// </summary>
    public class MaintenanceResult
    {
        public bool Success { get; set; }
        public string? Error { get; set; }
        public string TriggeredBy { get; set; } = string.Empty;
        public DateTime TriggeredAt { get; set; }
        public int DurationMs { get; set; }
        public bool StalledSessionsChecked { get; set; }
        public bool MetricsAggregated { get; set; }
        public string? AggregatedDate { get; set; }
        public bool DataCleanupExecuted { get; set; }
        public bool PlatformStatsRecomputed { get; set; }
        public int DevicesBlockedForExcessiveData { get; set; }
    }

    /// <summary>
    /// Dedicated service for maintenance tasks:
    /// 1. Marks stalled sessions (InProgress for too long) as timed out
    /// 2. Aggregates metrics into historical snapshots
    /// 3. Deletes old sessions and events based on tenant retention policies
    /// 4. Recomputes platform-wide stats for the landing page
    /// </summary>
    public partial class MaintenanceService
    {
        private readonly IMaintenanceRepository _maintenanceRepo;
        private readonly ISessionRepository _sessionRepo;
        private readonly IMetricsRepository _metricsRepo;
        private readonly TenantConfigurationService _tenantConfigService;
        private readonly UsageMetricsService _usageMetricsService;
        private readonly AdminConfigurationService _adminConfigurationService;
        private readonly BlockedDeviceService _blockedDeviceService;
        private readonly TenantAdminsService _tenantAdminsService;
        private readonly IUserUsageRepository _userUsageRepo;
        private readonly IDistressReportRepository _distressReportRepo;
        private readonly IOpsEventRepository _opsEventRepo;
        private readonly OpsEventService _opsEventService;
        private readonly IHttpClientFactory _httpClientFactory;
        private readonly ILogger<MaintenanceService> _logger;

        private const string PlatformStatsAliasFileName = "platform-stats.json";
        private const string PlatformStatsAliasCacheControl = "public, max-age=300, stale-while-revalidate=86400";
        private const string PlatformStatsVersionedCacheControl = "public, max-age=31536000, immutable";

        public MaintenanceService(
            IMaintenanceRepository maintenanceRepo,
            ISessionRepository sessionRepo,
            IMetricsRepository metricsRepo,
            TenantConfigurationService tenantConfigService,
            UsageMetricsService usageMetricsService,
            AdminConfigurationService adminConfigurationService,
            BlockedDeviceService blockedDeviceService,
            TenantAdminsService tenantAdminsService,
            IUserUsageRepository userUsageRepo,
            IDistressReportRepository distressReportRepo,
            IOpsEventRepository opsEventRepo,
            OpsEventService opsEventService,
            IHttpClientFactory httpClientFactory,
            ILogger<MaintenanceService> logger)
        {
            _maintenanceRepo = maintenanceRepo;
            _sessionRepo = sessionRepo;
            _metricsRepo = metricsRepo;
            _tenantConfigService = tenantConfigService;
            _usageMetricsService = usageMetricsService;
            _adminConfigurationService = adminConfigurationService;
            _blockedDeviceService = blockedDeviceService;
            _tenantAdminsService = tenantAdminsService;
            _userUsageRepo = userUsageRepo;
            _distressReportRepo = distressReportRepo;
            _opsEventRepo = opsEventRepo;
            _opsEventService = opsEventService;
            _httpClientFactory = httpClientFactory;
            _logger = logger;
        }

        /// <summary>
        /// Runs all maintenance tasks (used by the daily timer trigger)
        /// </summary>
        public async Task RunAllAsync()
        {
            _logger.LogInformation($"Daily maintenance started at {DateTime.UtcNow}");
            var maintenanceStart = Stopwatch.StartNew();

            try
            {
                await MarkStalledSessionsAsTimedOutAsync();
                await DetectExcessiveEventSessionsAsync();
                await BlockExcessiveDataSendersAsync();
                await AggregateMetricsWithCatchUpAsync();
                await CleanupOldDataAsync();
                await CleanupOldDistressReportsAsync();
                await CleanupOldOpsEventsAsync();
                await CheckAgentBlobStorageAsync();
                await RecomputePlatformStatsAsync();

                // Backfill and repair tasks run only via manual trigger (RunManualAsync)
                // to keep the timer-triggered path lightweight. See RunManualAsync for:
                // - BackfillSessionIndexAsync (safety net for missing index entries)
                // - CleanupGhostSessionIndexEntriesAsync (safety net for ghost entries)

                maintenanceStart.Stop();
                _logger.LogInformation($"Daily maintenance completed in {maintenanceStart.ElapsedMilliseconds}ms");
                await _opsEventService.RecordMaintenanceCompletedAsync((int)maintenanceStart.ElapsedMilliseconds, "Timer");
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Daily maintenance failed");
                await _opsEventService.RecordMaintenanceFailedAsync(ex.Message, "Timer");
            }
        }

        /// <summary>
        /// Manually triggered maintenance with flexible date selection
        /// </summary>
        public async Task<MaintenanceResult> RunManualAsync(DateTime? targetDate = null, bool aggregateOnly = false, string triggeredBy = "Unknown")
        {
            _logger.LogInformation($"Manual maintenance triggered by {triggeredBy} at {DateTime.UtcNow}");
            var maintenanceStart = Stopwatch.StartNew();
            var result = new MaintenanceResult { TriggeredBy = triggeredBy, TriggeredAt = DateTime.UtcNow };

            try
            {
                if (!aggregateOnly)
                {
                    await MarkStalledSessionsAsTimedOutAsync();
                    result.StalledSessionsChecked = true;
                    await DetectExcessiveEventSessionsAsync();
                }

                var dateToAggregate = targetDate ?? DateTime.UtcNow.AddDays(-1).Date;
                await AggregateMetricsForDateAsync(dateToAggregate);
                result.AggregatedDate = dateToAggregate.ToString("yyyy-MM-dd");
                result.MetricsAggregated = true;

                if (!aggregateOnly)
                {
                    await CleanupOldDataAsync();
                    result.DataCleanupExecuted = true;

                    // --- Backfill & repair tasks (manual-only, not in timer path) ---

                    // Safety net: backfill any sessions missing from SessionsIndex
                    await _maintenanceRepo.BackfillSessionIndexAsync();

                    // Safety net: remove ghost SessionsIndex entries caused by the
                    // StoreSessionAsync Replace-mode IndexRowKey bug (now fixed).
                    await _maintenanceRepo.CleanupGhostSessionIndexEntriesAsync();

                }

                await RecomputePlatformStatsAsync();
                result.PlatformStatsRecomputed = true;

                maintenanceStart.Stop();
                result.DurationMs = (int)maintenanceStart.ElapsedMilliseconds;
                result.Success = true;

                _logger.LogInformation($"Manual maintenance completed in {maintenanceStart.ElapsedMilliseconds}ms");
                await _opsEventService.RecordMaintenanceCompletedAsync(result.DurationMs, triggeredBy);
                return result;
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Manual maintenance failed");
                result.Success = false;
                result.Error = ex.Message;
                maintenanceStart.Stop();
                result.DurationMs = (int)maintenanceStart.ElapsedMilliseconds;
                await _opsEventService.RecordMaintenanceFailedAsync(ex.Message, triggeredBy);
                return result;
            }
        }

        /// <summary>
        /// Two-stage sweep for stuck sessions:
        /// 1. **Agent-silent Stalled marker** (2h fixed): Sessions still InProgress but with no events
        ///    for more than 2h are marked as Stalled (non-terminal). Picks up agents that cannot emit
        ///    session_stalled themselves (bluescreen, network loss, power off). Excludes WhiteGlove.
        /// 2. **Session timeout Failed** (5h default): Sessions that exceed the full SessionTimeoutHours
        ///    window (InProgress or Stalled) graduate to terminal Failed state.
        /// Both stages run in the same 2h maintenance pass so no new timers are introduced
        /// (preserving Container App scale-to-zero).
        /// </summary>
        private async Task MarkStalledSessionsAsTimedOutAsync()
        {
            _logger.LogInformation("Checking for stalled sessions...");
            var stalledStart = Stopwatch.StartNew();

            try
            {
                var tenantIds = await _maintenanceRepo.GetAllTenantIdsAsync();
                int totalSessionsTimedOut = 0;
                int totalSessionsMarkedStalled = 0;

                foreach (var tenantId in tenantIds)
                {
                    try
                    {
                        var config = await _tenantConfigService.GetConfigurationAsync(tenantId);
                        var timeoutHours = config?.SessionTimeoutHours ?? 5;
                        const int agentSilenceHours = 2; // fixed policy: 2h silence → Stalled intermediate
                        var now = DateTime.UtcNow;
                        var cutoffTime = now.AddHours(-timeoutHours);
                        var silenceCutoff = now.AddHours(-agentSilenceHours);

                        // -------- Stage 1: Agent-silent Stalled marker --------
                        // InProgress sessions with no events in > 2h but not yet older than 5h
                        // (otherwise Stage 2 picks them up directly as Failed).
                        var silentSessions = await _maintenanceRepo.GetAgentSilentSessionsAsync(
                            tenantId, silenceCutoff, hardCutoff: cutoffTime);

                        int silentMarked = 0;
                        foreach (var silent in silentSessions)
                        {
                            var lastEventAt = silent.LastEventAt ?? silent.StartedAt;
                            var silentMinutes = (int)(now - lastEventAt).TotalMinutes;
                            await _sessionRepo.UpdateSessionStatusAsync(
                                silent.TenantId,
                                silent.SessionId,
                                SessionStatus.Stalled,
                                stalledAt: now,
                                failureReason: $"Agent silent for {silentMinutes}min (detected by maintenance sweep)");
                            silentMarked++;
                        }

                        if (silentMarked > 0)
                        {
                            totalSessionsMarkedStalled += silentMarked;
                            _logger.LogInformation($"Tenant {tenantId}: Marked {silentMarked} agent-silent sessions as Stalled (silence threshold: {agentSilenceHours}h)");
                            await _maintenanceRepo.LogAuditEntryAsync(
                                tenantId,
                                "SessionStalled",
                                "Session",
                                $"{silentMarked} sessions",
                                "System.Maintenance",
                                new Dictionary<string, string>
                                {
                                    { "SessionsMarkedStalled", silentMarked.ToString() },
                                    { "AgentSilenceHours", agentSilenceHours.ToString() },
                                    { "SilenceCutoff", silenceCutoff.ToString("yyyy-MM-ddTHH:mm:ss") }
                                });
                        }

                        // -------- Stage 2: Terminal timeout (5h default) --------
                        var stalledSessions = await _maintenanceRepo.GetStalledSessionsAsync(tenantId, cutoffTime);

                        if (stalledSessions.Count == 0)
                        {
                            _logger.LogInformation($"Tenant {tenantId}: No stalled sessions found (timeout: {timeoutHours}h)");
                            continue;
                        }

                        int sessionCount = 0;

                        foreach (var session in stalledSessions)
                        {
                            await _sessionRepo.UpdateSessionStatusAsync(
                                session.TenantId,
                                session.SessionId,
                                SessionStatus.Failed,
                                failureReason: $"Session timed out after {timeoutHours} hours (started at {session.StartedAt:yyyy-MM-dd HH:mm:ss} UTC)"
                            );
                            sessionCount++;
                        }

                        totalSessionsTimedOut += sessionCount;

                        await _maintenanceRepo.LogAuditEntryAsync(
                            tenantId,
                            "SessionTimeout",
                            "Session",
                            $"{sessionCount} sessions",
                            "System.Maintenance",
                            new Dictionary<string, string>
                            {
                                { "SessionsTimedOut", sessionCount.ToString() },
                                { "TimeoutHours", timeoutHours.ToString() },
                                { "CutoffTime", cutoffTime.ToString("yyyy-MM-dd HH:mm:ss") }
                            });

                        _logger.LogInformation($"Tenant {tenantId}: Marked {sessionCount} stalled sessions as timed out (timeout: {timeoutHours}h)");
                        await _opsEventService.RecordSessionTimeoutsAsync(tenantId, sessionCount, timeoutHours);
                    }
                    catch (Exception ex)
                    {
                        _logger.LogError(ex, $"Failed to check stalled sessions for tenant {tenantId}");
                    }
                }

                stalledStart.Stop();
                _logger.LogInformation($"Stalled session check completed: {totalSessionsMarkedStalled} marked Stalled, {totalSessionsTimedOut} timed out in {stalledStart.ElapsedMilliseconds}ms");
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Failed to check for stalled sessions");
            }
        }

        /// <summary>
        /// Runaway-session threshold: sessions with more events than this are very likely produced
        /// by an agent loop bug (largest real sessions observed to date are &lt;~500 events).
        /// </summary>
        private const int ExcessiveEventCountThreshold = 2000;

        /// <summary>
        /// Scans every tenant for sessions whose EventCount exceeds <see cref="ExcessiveEventCountThreshold"/>.
        /// Emits one ExcessiveSessionEvents ops event per session (dispatched to Telegram/Teams via OpsAlertDispatchService)
        /// and marks the session so subsequent maintenance runs do not re-alert.
        /// </summary>
        private async Task DetectExcessiveEventSessionsAsync()
        {
            try
            {
                var tenantIds = await _maintenanceRepo.GetAllTenantIdsAsync();
                int totalAlerted = 0;

                foreach (var tenantId in tenantIds)
                {
                    try
                    {
                        var runaways = await _sessionRepo.GetSessionsWithEventCountAboveAsync(tenantId, ExcessiveEventCountThreshold);
                        foreach (var session in runaways)
                        {
                            if (session.ExcessiveEventsAlerted) continue;

                            await _opsEventService.RecordExcessiveSessionEventsAsync(
                                tenantId, session.SessionId, session.EventCount, ExcessiveEventCountThreshold);
                            await _sessionRepo.MarkExcessiveEventsAlertedAsync(tenantId, session.SessionId);
                            totalAlerted++;
                        }
                    }
                    catch (Exception ex)
                    {
                        _logger.LogError(ex, "Failed excessive-event scan for tenant {TenantId}", tenantId);
                    }
                }

                if (totalAlerted > 0)
                    _logger.LogWarning("Excessive-event scan: {Count} new runaway session(s) alerted (threshold {Threshold})", totalAlerted, ExcessiveEventCountThreshold);
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Failed to scan for excessive-event sessions");
            }
        }

        /// <summary>
        /// Detects devices that are still actively sending data beyond the configured maximum session
        /// window and blocks them automatically. Status-independent: uses LastEventAt (written on every
        /// event batch) combined with StartedAt to identify sessions spanning too long a data window.
        /// Controlled by AdminConfiguration.MaxSessionWindowHours (0 = disabled).
        /// </summary>
        private async Task<int> BlockExcessiveDataSendersAsync()
        {
            _logger.LogInformation("Checking for excessive data senders...");
            var sw = Stopwatch.StartNew();

            try
            {
                var adminConfig = await _adminConfigurationService.GetConfigurationAsync();
                if (adminConfig.MaxSessionWindowHours == 0)
                {
                    _logger.LogInformation("Excessive data sender detection disabled (MaxSessionWindowHours = 0)");
                    return 0;
                }

                var windowCutoff = DateTime.UtcNow.AddHours(-adminConfig.MaxSessionWindowHours);
                var blockDurationHours = adminConfig.MaintenanceBlockDurationHours;
                var tenantIds = await _maintenanceRepo.GetAllTenantIdsAsync();
                int totalBlocked = 0;

                foreach (var tenantId in tenantIds)
                {
                    try
                    {
                        var sessions = await _maintenanceRepo.GetExcessiveDataSendersAsync(tenantId, windowCutoff, adminConfig.MaxSessionWindowHours);

                        if (sessions.Count == 0)
                        {
                            _logger.LogInformation($"Tenant {tenantId}: No excessive data sender sessions found (window: {adminConfig.MaxSessionWindowHours}h)");
                            continue;
                        }

                        int blockedCount = 0;

                        foreach (var session in sessions)
                        {
                            if (string.IsNullOrEmpty(session.SerialNumber))
                                continue;

                            var (isBlocked, _, _, _) = await _blockedDeviceService.IsBlockedAsync(tenantId, session.SerialNumber);
                            if (isBlocked)
                                continue;

                            var reason = $"Excessive data window: session active for >{adminConfig.MaxSessionWindowHours}h " +
                                         $"(started {session.StartedAt:yyyy-MM-dd HH:mm:ss} UTC, " +
                                         $"last event {session.LastEventAt:yyyy-MM-dd HH:mm:ss} UTC)";

                            await _blockedDeviceService.BlockDeviceAsync(
                                tenantId,
                                session.SerialNumber,
                                durationHours: blockDurationHours,
                                blockedByEmail: "System.Maintenance",
                                reason: reason,
                                blockedSessionId: session.SessionId);

                            blockedCount++;
                        }

                        if (blockedCount > 0)
                        {
                            await _maintenanceRepo.LogAuditEntryAsync(
                                tenantId,
                                "ExcessiveDataBlock",
                                "Device",
                                $"{blockedCount} devices",
                                "System.Maintenance",
                                new Dictionary<string, string>
                                {
                                    { "DevicesBlocked", blockedCount.ToString() },
                                    { "WindowHours", adminConfig.MaxSessionWindowHours.ToString() },
                                    { "BlockDurationHours", blockDurationHours.ToString() }
                                });

                            _logger.LogInformation($"Tenant {tenantId}: Blocked {blockedCount} excessive data sender device(s) (window: {adminConfig.MaxSessionWindowHours}h, block: {blockDurationHours}h)");
                            await _opsEventService.RecordExcessiveDataBlockedAsync(tenantId, blockedCount, adminConfig.MaxSessionWindowHours);
                        }

                        totalBlocked += blockedCount;
                    }
                    catch (Exception ex)
                    {
                        _logger.LogError(ex, $"Failed to check excessive data senders for tenant {tenantId}");
                    }
                }

                sw.Stop();
                _logger.LogInformation($"Excessive data sender check completed: {totalBlocked} devices blocked in {sw.ElapsedMilliseconds}ms");
                return totalBlocked;
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Failed to check for excessive data senders");
                return 0;
            }
        }


        /// <summary>
        /// Aggregates metrics for any missed days in the last 7 days, plus yesterday.
        /// Checks the UsageMetrics table for existing snapshots to avoid re-aggregation.
        /// </summary>
    }
}