AutopilotMonitor/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services.Monitoring;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;

namespace AutopilotMonitor.Functions.Services
{
    /// <summary>
    /// Result of a maintenance operation
    /// </summary>
    public class MaintenanceResult
    {
        public bool Success { get; set; }
        public string? Error { get; set; }
        public string TriggeredBy { get; set; } = string.Empty;
        public DateTime TriggeredAt { get; set; }
        public int DurationMs { get; set; }
        public bool StalledSessionsChecked { get; set; }
        public bool MetricsAggregated { get; set; }
        public string? AggregatedDate { get; set; }
        public bool DataCleanupExecuted { get; set; }
        public bool PlatformStatsRecomputed { get; set; }
        public int DevicesBlockedForExcessiveData { get; set; }
    }

    /// <summary>
    /// Dedicated service for maintenance tasks:
    /// 1. Marks stalled sessions (InProgress for too long) as timed out
    /// 2. Aggregates metrics into historical snapshots
    /// 3. Deletes old sessions and events based on tenant retention policies
    /// 4. Recomputes platform-wide stats for the landing page
    /// </summary>
    public partial class MaintenanceService
    {
        private readonly IMaintenanceRepository _maintenanceRepo;
        private readonly ISessionRepository _sessionRepo;
        private readonly IMetricsRepository _metricsRepo;
        private readonly TenantConfigurationService _tenantConfigService;
        private readonly UsageMetricsService _usageMetricsService;
        private readonly AdminConfigurationService _adminConfigurationService;
        private readonly BlockedDeviceService _blockedDeviceService;
        private readonly TenantAdminsService _tenantAdminsService;
        private readonly IUserUsageRepository _userUsageRepo;
        private readonly IDistressReportRepository _distressReportRepo;
        private readonly IOpsEventRepository _opsEventRepo;
        private readonly OpsEventService _opsEventService;
        private readonly IAzureMonitorMetricsReader _metricsReader;
        private readonly IConfiguration _configuration;
        private readonly IHttpClientFactory _httpClientFactory;
        private readonly ILogger<MaintenanceService> _logger;

        private const string PlatformStatsAliasFileName = "platform-stats.json";
        private const string PlatformStatsAliasCacheControl = "public, max-age=300, stale-while-revalidate=86400";
        private const string PlatformStatsVersionedCacheControl = "public, max-age=31536000, immutable";

        public MaintenanceService(
            IMaintenanceRepository maintenanceRepo,
            ISessionRepository sessionRepo,
            IMetricsRepository metricsRepo,
            TenantConfigurationService tenantConfigService,
            UsageMetricsService usageMetricsService,
            AdminConfigurationService adminConfigurationService,
            BlockedDeviceService blockedDeviceService,
            TenantAdminsService tenantAdminsService,
            IUserUsageRepository userUsageRepo,
            IDistressReportRepository distressReportRepo,
            IOpsEventRepository opsEventRepo,
            OpsEventService opsEventService,
            IAzureMonitorMetricsReader metricsReader,
            IConfiguration configuration,
            IHttpClientFactory httpClientFactory,
            ILogger<MaintenanceService> logger)
        {
            _maintenanceRepo = maintenanceRepo;
            _sessionRepo = sessionRepo;
            _metricsRepo = metricsRepo;
            _tenantConfigService = tenantConfigService;
            _usageMetricsService = usageMetricsService;
            _adminConfigurationService = adminConfigurationService;
            _blockedDeviceService = blockedDeviceService;
            _tenantAdminsService = tenantAdminsService;
            _userUsageRepo = userUsageRepo;
            _distressReportRepo = distressReportRepo;
            _opsEventRepo = opsEventRepo;
            _opsEventService = opsEventService;
            _metricsReader = metricsReader;
            _configuration = configuration;
            _httpClientFactory = httpClientFactory;
            _logger = logger;
        }

        /// <summary>
        /// Runs all maintenance tasks (used by the daily timer trigger)
        /// </summary>
        public async Task RunAllAsync()
        {
            _logger.LogInformation($"Daily maintenance started at {DateTime.UtcNow}");
            var maintenanceStart = Stopwatch.StartNew();

            try
            {
                await MarkStalledSessionsAsTimedOutAsync();
                await DetectExcessiveEventSessionsAsync();
                await BlockExcessiveDataSendersAsync();
                await AggregateMetricsWithCatchUpAsync();
                await CleanupOldDataAsync();
                await CleanupOldDistressReportsAsync();
                await CleanupOldOpsEventsAsync();
                await CleanupOrphanedEventsAsync();
                await CheckAgentBlobStorageAsync();
                await CheckEmbeddedCertExpiryAsync();
                await RecomputePlatformStatsAsync();

                // Backfill and repair tasks run only via manual trigger (RunManualAsync)
                // to keep the timer-triggered path lightweight. See RunManualAsync for:
                // - BackfillSessionIndexAsync (safety net for missing index entries)
                // - CleanupGhostSessionIndexEntriesAsync (safety net for ghost entries)

                maintenanceStart.Stop();
                _logger.LogInformation($"Daily maintenance completed in {maintenanceStart.ElapsedMilliseconds}ms");
                await _opsEventService.RecordMaintenanceCompletedAsync((int)maintenanceStart.ElapsedMilliseconds, "Timer");
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Daily maintenance failed");
                await _opsEventService.RecordMaintenanceFailedAsync(ex.Message, "Timer");
            }
        }

        /// <summary>
        /// Manually triggered maintenance with flexible date selection
        /// </summary>
        public async Task<MaintenanceResult> RunManualAsync(DateTime? targetDate = null, bool aggregateOnly = false, string triggeredBy = "Unknown")
        {
            _logger.LogInformation($"Manual maintenance triggered by {triggeredBy} at {DateTime.UtcNow}");
            var maintenanceStart = Stopwatch.StartNew();
            var result = new MaintenanceResult { TriggeredBy = triggeredBy, TriggeredAt = DateTime.UtcNow };

            try
            {
                if (!aggregateOnly)
                {
                    await MarkStalledSessionsAsTimedOutAsync();
                    result.StalledSessionsChecked = true;
                    await DetectExcessiveEventSessionsAsync();
                }

                var dateToAggregate = targetDate ?? DateTime.UtcNow.AddDays(-1).Date;
                await AggregateMetricsForDateAsync(dateToAggregate);
                result.AggregatedDate = dateToAggregate.ToString("yyyy-MM-dd");
                result.MetricsAggregated = true;

                if (!aggregateOnly)
                {
                    await CleanupOldDataAsync();
                    result.DataCleanupExecuted = true;

                    // --- Backfill & repair tasks (manual-only, not in timer path) ---

                    // Safety net: backfill any sessions missing from SessionsIndex
                    await _maintenanceRepo.BackfillSessionIndexAsync();

                    // Safety net: remove ghost SessionsIndex entries caused by the
                    // StoreSessionAsync Replace-mode IndexRowKey bug (now fixed).
                    await _maintenanceRepo.CleanupGhostSessionIndexEntriesAsync();

                    // Mirror the timer path: check embedded Intune cert bundle for
                    // expiring members so manual triggers also exercise the watcher.
                    await CheckEmbeddedCertExpiryAsync();

                    // Same parity for SignalR quota - the dedicated 1h timer is the
                    // primary cadence, but operators triggering maintenance manually
                    // expect every health/quota check to run.
                    await CheckSignalRQuotaAsync();
                }

                await RecomputePlatformStatsAsync();
                result.PlatformStatsRecomputed = true;

                maintenanceStart.Stop();
                result.DurationMs = (int)maintenanceStart.ElapsedMilliseconds;
                result.Success = true;

                _logger.LogInformation($"Manual maintenance completed in {maintenanceStart.ElapsedMilliseconds}ms");
                await _opsEventService.RecordMaintenanceCompletedAsync(result.DurationMs, triggeredBy);
                return result;
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Manual maintenance failed");
                result.Success = false;
                result.Error = ex.Message;
                maintenanceStart.Stop();
                result.DurationMs = (int)maintenanceStart.ElapsedMilliseconds;
                await _opsEventService.RecordMaintenanceFailedAsync(ex.Message, triggeredBy);
                return result;
            }
        }

        /// <summary>
        /// Two-stage sweep for stuck sessions:
        /// 1. **Agent-silent Stalled marker** (2h fixed): Sessions still InProgress but with no events
        ///    for more than 2h are marked as Stalled (non-terminal). Picks up agents that cannot emit
        ///    session_stalled themselves (bluescreen, network loss, power off). Excludes WhiteGlove.
        /// 2. **Session timeout Failed** (5h default): Sessions that exceed the full SessionTimeoutHours
        ///    window (InProgress or Stalled) graduate to terminal Failed state.
        /// Both stages run in the same 2h maintenance pass so no new timers are introduced
        /// (preserving Container App scale-to-zero).
        /// </summary>
        private async Task MarkStalledSessionsAsTimedOutAsync()
        {
            _logger.LogInformation("Checking for stalled sessions...");
            var stalledStart = Stopwatch.StartNew();

            try
            {
                var tenantIds = await _maintenanceRepo.GetAllTenantIdsAsync();
                int totalSessionsTimedOut = 0;
                int totalSessionsMarkedStalled = 0;

                foreach (var tenantId in tenantIds)
                {
                    try
                    {
                        var config = await _tenantConfigService.GetConfigurationAsync(tenantId);
                        var timeoutHours = config?.SessionTimeoutHours ?? 5;
                        const int agentSilenceHours = 2; // fixed policy: 2h silence → Stalled intermediate
                        var now = DateTime.UtcNow;
                        var cutoffTime = now.AddHours(-timeoutHours);
                        var silenceCutoff = now.AddHours(-agentSilenceHours);

                        // -------- Stage 1: Agent-silent Stalled marker --------
                        // InProgress sessions with no events in > 2h but not yet older than 5h
                        // (otherwise Stage 2 picks them up directly as Failed).
                        var silentSessions = await _maintenanceRepo.GetAgentSilentSessionsAsync(
                            tenantId, silenceCutoff, hardCutoff: cutoffTime);

                        int silentMarked = 0;
                        foreach (var silent in silentSessions)
                        {
                            var lastEventAt = silent.LastEventAt ?? silent.StartedAt;
                            var silentMinutes = (int)(now - lastEventAt).TotalMinutes;
                            await _sessionRepo.UpdateSessionStatusAsync(
                                silent.TenantId,
                                silent.SessionId,
                                SessionStatus.Stalled,
                                stalledAt: now,
                                failureReason: $"Agent silent for {silentMinutes}min (detected by maintenance sweep)");
                            silentMarked++;
                        }

                        if (silentMarked > 0)
                        {
                            totalSessionsMarkedStalled += silentMarked;
                            _logger.LogInformation($"Tenant {tenantId}: Marked {silentMarked} agent-silent sessions as Stalled (silence threshold: {agentSilenceHours}h)");
                            await _maintenanceRepo.LogAuditEntryAsync(
                                tenantId,
                                "SessionStalled",
                                "Session",
                                $"{silentMarked} sessions",
                                "System.Maintenance",
                                new Dictionary<string, string>
                                {
                                    { "SessionsMarkedStalled", silentMarked.ToString() },
                                    { "AgentSilenceHours", agentSilenceHours.ToString() },
                                    { "SilenceCutoff", silenceCutoff.ToString("yyyy-MM-ddTHH:mm:ss") }
                                });
                        }

                        // -------- Stage 2: Terminal timeout (5h default) --------
                        var stalledSessions = await _maintenanceRepo.GetStalledSessionsAsync(tenantId, cutoffTime);

                        if (stalledSessions.Count == 0)
                        {
                            _logger.LogInformation($"Tenant {tenantId}: No stalled sessions found (timeout: {timeoutHours}h)");
                            continue;
                        }

                        int sessionCount = 0;

                        foreach (var session in stalledSessions)
                        {
                            // Hybrid User-Driven completion-gap fix (2026-05-01): before
                            // graduating the session to terminal Failed, build a compact
                            // snapshot of "what we last knew" so operators don't have to
                            // scroll through hundreds of events to reconstruct where the
                            // session was stuck. Best-effort — a snapshot read failure
                            // must never block the timeout transition itself.
                            string? snapshotJson = null;
                            try
                            {
                                var sessionEvents = await _sessionRepo.GetSessionEventsAsync(
                                    session.TenantId, session.SessionId, maxResults: 1000);
                                snapshotJson = FailureSnapshotBuilder.Build(sessionEvents, now);
                            }
                            catch (Exception snapEx)
                            {
                                _logger.LogWarning(snapEx,
                                    $"Failed to build failure snapshot for session {session.SessionId}; proceeding with timeout transition without snapshot");
                            }

                            await _sessionRepo.UpdateSessionStatusAsync(
                                session.TenantId,
                                session.SessionId,
                                SessionStatus.Failed,
                                failureReason: $"Session timed out after {timeoutHours} hours (started at {session.StartedAt:yyyy-MM-dd HH:mm:ss} UTC)",
                                failureSnapshotJson: snapshotJson
                            );
                            sessionCount++;
                        }

                        totalSessionsTimedOut += sessionCount;

                        await _maintenanceRepo.LogAuditEntryAsync(
                            tenantId,
                            "SessionTimeout",
                            "Session",
                            $"{sessionCount} sessions",
                            "System.Maintenance",
                            new Dictionary<string, string>
                            {
                                { "SessionsTimedOut", sessionCount.ToString() },
                                { "TimeoutHours", timeoutHours.ToString() },
                                { "CutoffTime", cutoffTime.ToString("yyyy-MM-dd HH:mm:ss") }
                            });

                        _logger.LogInformation($"Tenant {tenantId}: Marked {sessionCount} stalled sessions as timed out (timeout: {timeoutHours}h)");
                        await _opsEventService.RecordSessionTimeoutsAsync(tenantId, sessionCount, timeoutHours);
                    }
                    catch (Exception ex)
                    {
                        _logger.LogError(ex, $"Failed to check stalled sessions for tenant {tenantId}");
                    }
                }

                stalledStart.Stop();
                _logger.LogInformation($"Stalled session check completed: {totalSessionsMarkedStalled} marked Stalled, {totalSessionsTimedOut} timed out in {stalledStart.ElapsedMilliseconds}ms");
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Failed to check for stalled sessions");
            }
        }

        /// <summary>
        /// Scans every tenant for sessions whose EventCount exceeds the configured
        /// <c>ExcessiveEventCountThreshold</c> (AdminConfiguration, default 2000).
        /// Emits one ExcessiveSessionEvents ops event per session (dispatched to Telegram/Teams
        /// via OpsAlertDispatchService) and marks the session so subsequent runs do not re-alert.
        /// A threshold of 0 disables the scan entirely.
        /// </summary>
        private async Task DetectExcessiveEventSessionsAsync()
        {
            try
            {
                var adminConfig = await _adminConfigurationService.GetConfigurationAsync();
                var threshold = adminConfig?.ExcessiveEventCountThreshold ?? 2000;
                if (threshold <= 0)
                {
                    _logger.LogDebug("Excessive-event scan disabled (threshold = 0)");
                    return;
                }

                var tenantIds = await _maintenanceRepo.GetAllTenantIdsAsync();
                int totalAlerted = 0;

                foreach (var tenantId in tenantIds)
                {
                    try
                    {
                        var runaways = await _sessionRepo.GetSessionsWithEventCountAboveAsync(tenantId, threshold);
                        foreach (var session in runaways)
                        {
                            if (session.ExcessiveEventsAlerted) continue;

                            await _opsEventService.RecordExcessiveSessionEventsAsync(
                                tenantId, session.SessionId, session.EventCount, threshold);
                            await _sessionRepo.MarkExcessiveEventsAlertedAsync(tenantId, session.SessionId);
                            totalAlerted++;
                        }
                    }
                    catch (Exception ex)
                    {
                        _logger.LogError(ex, "Failed excessive-event scan for tenant {TenantId}", tenantId);
                    }
                }

                if (totalAlerted > 0)
                    _logger.LogWarning("Excessive-event scan: {Count} new runaway session(s) alerted (threshold {Threshold})", totalAlerted, threshold);
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Failed to scan for excessive-event sessions");
            }
        }

        /// <summary>
        /// Detects devices that are still actively sending data beyond the configured maximum session
        /// window and blocks them automatically. Status-independent: uses LastEventAt (written on every
        /// event batch) combined with StartedAt to identify sessions spanning too long a data window.
        /// Controlled by AdminConfiguration.MaxSessionWindowHours (0 = disabled).
        /// </summary>
        private async Task<int> BlockExcessiveDataSendersAsync()
        {
            _logger.LogInformation("Checking for excessive data senders...");
            var sw = Stopwatch.StartNew();

            try
            {
                var adminConfig = await _adminConfigurationService.GetConfigurationAsync();
                if (adminConfig.MaxSessionWindowHours == 0)
                {
                    _logger.LogInformation("Excessive data sender detection disabled (MaxSessionWindowHours = 0)");
                    return 0;
                }

                var windowCutoff = DateTime.UtcNow.AddHours(-adminConfig.MaxSessionWindowHours);
                var blockDurationHours = adminConfig.MaintenanceBlockDurationHours;
                var tenantIds = await _maintenanceRepo.GetAllTenantIdsAsync();
                int totalBlocked = 0;

                foreach (var tenantId in tenantIds)
                {
                    try
                    {
                        var sessions = await _maintenanceRepo.GetExcessiveDataSendersAsync(tenantId, windowCutoff, adminConfig.MaxSessionWindowHours);

                        if (sessions.Count == 0)
                        {
                            _logger.LogInformation($"Tenant {tenantId}: No excessive data sender sessions found (window: {adminConfig.MaxSessionWindowHours}h)");
                            continue;
                        }

                        int blockedCount = 0;

                        foreach (var session in sessions)
                        {
                            if (string.IsNullOrEmpty(session.SerialNumber))
                                continue;

                            var (isBlocked, _, _, _) = await _blockedDeviceService.IsBlockedAsync(tenantId, session.SerialNumber);
                            if (isBlocked)
                                continue;

                            var reason = $"Excessive data window: session active for >{adminConfig.MaxSessionWindowHours}h " +
                                         $"(started {session.StartedAt:yyyy-MM-dd HH:mm:ss} UTC, " +
                                         $"last event {session.LastEventAt:yyyy-MM-dd HH:mm:ss} UTC)";

                            await _blockedDeviceService.BlockDeviceAsync(
                                tenantId,
                                session.SerialNumber,
                                durationHours: blockDurationHours,
                                blockedByEmail: "System.Maintenance",
                                reason: reason,
                                blockedSessionId: session.SessionId);

                            blockedCount++;
                        }

                        if (blockedCount > 0)
                        {
                            await _maintenanceRepo.LogAuditEntryAsync(
                                tenantId,
                                "ExcessiveDataBlock",
                                "Device",
                                $"{blockedCount} devices",
                                "System.Maintenance",
                                new Dictionary<string, string>
                                {
                                    { "DevicesBlocked", blockedCount.ToString() },
                                    { "WindowHours", adminConfig.MaxSessionWindowHours.ToString() },
                                    { "BlockDurationHours", blockDurationHours.ToString() }
                                });

                            _logger.LogInformation($"Tenant {tenantId}: Blocked {blockedCount} excessive data sender device(s) (window: {adminConfig.MaxSessionWindowHours}h, block: {blockDurationHours}h)");
                            await _opsEventService.RecordExcessiveDataBlockedAsync(tenantId, blockedCount, adminConfig.MaxSessionWindowHours);
                        }

                        totalBlocked += blockedCount;
                    }
                    catch (Exception ex)
                    {
                        _logger.LogError(ex, $"Failed to check excessive data senders for tenant {tenantId}");
                    }
                }

                sw.Stop();
                _logger.LogInformation($"Excessive data sender check completed: {totalBlocked} devices blocked in {sw.ElapsedMilliseconds}ms");
                return totalBlocked;
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Failed to check for excessive data senders");
                return 0;
            }
        }


        /// <summary>
        /// Aggregates metrics for any missed days in the last 7 days, plus yesterday.
        /// Checks the UsageMetrics table for existing snapshots to avoid re-aggregation.
        /// </summary>
    }
}