added agent emergency reporter

src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.cs

@@ -62,6 +62,10 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
         private int _consecutiveAuthFailures = 0;
         private DateTime? _firstAuthFailureTime = null;
 
+        // Non-auth upload failure tracking for the emergency channel
+        private int _consecutiveUploadFailures = 0;
+        private EmergencyReporter _emergencyReporter;
+
         public MonitoringService(AgentConfiguration configuration, AgentLogger logger, string agentVersion)
         {
             _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
@@ -75,6 +79,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
 
             _spool = new EventSpool(_configuration.SpoolDirectory);
             _apiClient = new BackendApiClient(_configuration.ApiBaseUrl, _configuration, _logger);
+            _emergencyReporter = new EmergencyReporter(_apiClient, _configuration.SessionId, _configuration.TenantId, agentVersion, _logger);
             _cleanupService = new CleanupService(_configuration, _logger);
             _diagnosticsService = new DiagnosticsPackageService(_configuration, _logger, _apiClient);
 
@@ -832,9 +837,10 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
                     _spool.RemoveEvents(events);
                     _logger.Info($"Uploaded {response.EventsProcessed} events");
 
-                    // Reset auth failure counter on success
+                    // Reset failure counters on success
                     _consecutiveAuthFailures = 0;
                     _firstAuthFailureTime = null;
+                    _consecutiveUploadFailures = 0;
                 }
                 else
                 {
@@ -848,7 +854,19 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
             }
             catch (Exception ex)
             {
+                _consecutiveUploadFailures++;
                 _logger.Error("Error uploading events", ex);
+
+                // After N consecutive non-auth failures, signal the emergency channel once.
+                // The EmergencyReporter deduplicates further calls for the same error category.
+                if (_consecutiveUploadFailures == EmergencyReporter.ConsecutiveFailureThreshold)
+                {
+                    _ = _emergencyReporter.TrySendAsync(
+                        AgentErrorType.IngestFailed,
+                        ex.Message,
+                        httpStatusCode: null,
+                        sequenceNumber: _eventSequence);
+                }
             }
         }
 

src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Network/BackendApiClient.cs

@@ -6,6 +6,7 @@ using System.Linq;
 using System.Net.Http;
 using System.Security.Cryptography.X509Certificates;
 using System.Text;
+using System.Threading;
 using System.Threading.Tasks;
 using AutopilotMonitor.Agent.Core.Configuration;
 using AutopilotMonitor.Agent.Core.Security;
@@ -208,6 +209,39 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Network
             return response;
         }
 
+        /// <summary>
+        /// Sends a critical error report to the emergency channel endpoint.
+        /// Fire-and-forget: swallows all exceptions so a failure here never cascades.
+        /// Uses a 5-second per-request timeout so it cannot block the upload loop.
+        /// </summary>
+        public async Task ReportAgentErrorAsync(AgentErrorReport report)
+        {
+            try
+            {
+                var url = $"{_baseUrl}{Constants.ApiEndpoints.ReportAgentError}";
+                var json = JsonConvert.SerializeObject(report);
+
+                var httpRequest = new HttpRequestMessage(HttpMethod.Post, url)
+                {
+                    Content = new StringContent(json, Encoding.UTF8, "application/json")
+                };
+
+                // Security validation on the backend requires tenant ID before parsing the body
+                httpRequest.Headers.Add("X-Tenant-Id", report.TenantId);
+
+                // Hardware headers are required for full security validation
+                AddSecurityHeaders(httpRequest);
+
+                using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
+                await _httpClient.SendAsync(httpRequest, cts.Token);
+                // Response status is deliberately ignored — endpoint always returns 200
+            }
+            catch
+            {
+                // Swallow all exceptions — the emergency channel must never cascade failures
+            }
+        }
+
         private async Task<TResponse> PostAsync<TRequest, TResponse>(string url, TRequest data)
         {
             var json = JsonConvert.SerializeObject(data);

src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Network/EmergencyReporter.cs

@@ -0,0 +1,111 @@
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using AutopilotMonitor.Agent.Core.Logging;
+using AutopilotMonitor.Shared.Models;
+
+namespace AutopilotMonitor.Agent.Core.Monitoring.Network
+{
+    /// <summary>
+    /// Sends critical agent-side errors to the backend emergency channel endpoint.
+    ///
+    /// Anti-flood guarantees (per session):
+    ///   - Only fires after <see cref="ConsecutiveFailureThreshold"/> consecutive upload failures
+    ///   - Same error key (ErrorType + HTTP status code) is only ever sent once per session
+    ///   - At most <see cref="MaxReportsPerSession"/> emergency reports total per session
+    ///   - At most one report every <see cref="MinIntervalMinutes"/> minutes regardless of error type
+    ///
+    /// A failure in this reporter is silently swallowed — it must never cascade into the main loop.
+    /// </summary>
+    internal class EmergencyReporter
+    {
+        /// <summary>
+        /// Number of consecutive upload failures required before the first emergency report is sent.
+        /// </summary>
+        public const int ConsecutiveFailureThreshold = 3;
+
+        private const int MaxReportsPerSession = 5;
+        private const int MinIntervalMinutes = 10;
+
+        private readonly BackendApiClient _apiClient;
+        private readonly string _sessionId;
+        private readonly string _tenantId;
+        private readonly string _agentVersion;
+        private readonly AgentLogger _logger;
+
+        private readonly HashSet<string> _reportedKeys = new HashSet<string>();
+        private DateTime? _lastReportTime;
+        private int _reportCount;
+
+        public EmergencyReporter(
+            BackendApiClient apiClient,
+            string sessionId,
+            string tenantId,
+            string agentVersion,
+            AgentLogger logger)
+        {
+            _apiClient = apiClient;
+            _sessionId = sessionId;
+            _tenantId = tenantId;
+            _agentVersion = agentVersion;
+            _logger = logger;
+        }
+
+        /// <summary>
+        /// Attempts to send an emergency report. Returns immediately without awaiting
+        /// (intended to be called as fire-and-forget via <c>_ = reporter.TrySendAsync(...)</c>).
+        ///
+        /// All anti-flood checks are applied synchronously before the async HTTP call,
+        /// so repeated calls are cheap when suppressed.
+        /// </summary>
+        public async Task TrySendAsync(
+            AgentErrorType errorType,
+            string message,
+            int? httpStatusCode = null,
+            long? sequenceNumber = null)
+        {
+            // Deduplicate by error type + status code: same failure category sent only once per session
+            var key = $"{errorType}:{httpStatusCode}";
+
+            if (_reportCount >= MaxReportsPerSession)
+            {
+                _logger?.Debug($"[EmergencyChannel] Suppressed ({_reportCount}/{MaxReportsPerSession} reports used): {key}");
+                return;
+            }
+
+            if (_reportedKeys.Contains(key))
+            {
+                _logger?.Debug($"[EmergencyChannel] Suppressed (already reported): {key}");
+                return;
+            }
+
+            if (_lastReportTime != null &&
+                (DateTime.UtcNow - _lastReportTime.Value).TotalMinutes < MinIntervalMinutes)
+            {
+                _logger?.Debug($"[EmergencyChannel] Suppressed (cooldown active, last report at {_lastReportTime.Value:HH:mm:ss}): {key}");
+                return;
+            }
+
+            _reportedKeys.Add(key);
+            _reportCount++;
+            _lastReportTime = DateTime.UtcNow;
+
+            var statusText = httpStatusCode.HasValue ? $" HTTP {httpStatusCode}" : string.Empty;
+            _logger?.Warning($"[EmergencyChannel] Sending report {_reportCount}/{MaxReportsPerSession}: {errorType}{statusText}");
+
+            var report = new AgentErrorReport
+            {
+                SessionId = _sessionId,
+                TenantId = _tenantId,
+                ErrorType = errorType,
+                Message = message,
+                HttpStatusCode = httpStatusCode,
+                SequenceNumber = sequenceNumber,
+                AgentVersion = _agentVersion,
+                Timestamp = DateTime.UtcNow
+            };
+
+            await _apiClient.ReportAgentErrorAsync(report);
+        }
+    }
+}

src/Agent/AutopilotMonitor.Agent/buildcounter.txt

@@ -1 +1 @@
-154
+158

src/Backend/AutopilotMonitor.Functions/Functions/ReportAgentErrorFunction.cs

@@ -0,0 +1,135 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Text.Json;
+using AutopilotMonitor.Functions.Security;
+using AutopilotMonitor.Functions.Services;
+using AutopilotMonitor.Shared.Models;
+using Microsoft.ApplicationInsights;
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.Azure.Functions.Worker.Http;
+using Microsoft.Extensions.Logging;
+
+namespace AutopilotMonitor.Functions.Functions
+{
+    /// <summary>
+    /// Emergency channel endpoint: receives critical agent-side error reports when the
+    /// normal ingest path is unavailable or returning errors.
+    ///
+    /// Security: full ValidateSecurityAsync (cert + rate limit + hardware whitelist + Autopilot).
+    /// Storage: logs to Application Insights as a structured custom event "AgentEmergencyError".
+    ///          No Table Storage — App Insights is sufficient for incident investigation via KQL.
+    ///
+    /// Response: always 200 OK so the agent never retries based on backend-side issues here.
+    /// </summary>
+    public class ReportAgentErrorFunction
+    {
+        private readonly ILogger<ReportAgentErrorFunction> _logger;
+        private readonly TenantConfigurationService _configService;
+        private readonly RateLimitService _rateLimitService;
+        private readonly AutopilotDeviceValidator _autopilotDeviceValidator;
+        private readonly TelemetryClient _telemetryClient;
+
+        public ReportAgentErrorFunction(
+            ILogger<ReportAgentErrorFunction> logger,
+            TenantConfigurationService configService,
+            RateLimitService rateLimitService,
+            AutopilotDeviceValidator autopilotDeviceValidator,
+            TelemetryClient telemetryClient)
+        {
+            _logger = logger;
+            _configService = configService;
+            _rateLimitService = rateLimitService;
+            _autopilotDeviceValidator = autopilotDeviceValidator;
+            _telemetryClient = telemetryClient;
+        }
+
+        [Function("ReportAgentError")]
+        public async Task<HttpResponseData> Run(
+            [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "agent/error")] HttpRequestData req)
+        {
+            try
+            {
+                // Security checks FIRST — same pattern as /api/agent/ingest.
+                // TenantId from X-Tenant-Id header allows validation before parsing the body.
+                var tenantId = req.Headers.Contains("X-Tenant-Id")
+                    ? req.Headers.GetValues("X-Tenant-Id").FirstOrDefault()
+                    : null;
+
+                if (string.IsNullOrEmpty(tenantId))
+                {
+                    return req.CreateResponse(HttpStatusCode.BadRequest);
+                }
+
+                // Full security validation: cert, rate limit, hardware whitelist, Autopilot (if enabled).
+                // Autopilot positive cache TTL is 30 min — devices that have recently called /ingest
+                // will get a cache hit here at essentially zero cost.
+                var (_, errorResponse) = await req.ValidateSecurityAsync(
+                    tenantId,
+                    _configService,
+                    _rateLimitService,
+                    _autopilotDeviceValidator,
+                    _logger
+                );
+
+                if (errorResponse != null)
+                {
+                    return errorResponse;
+                }
+
+                // Parse the report body
+                AgentErrorReport? report = null;
+                try
+                {
+                    report = await JsonSerializer.DeserializeAsync<AgentErrorReport>(
+                        req.Body,
+                        new JsonSerializerOptions { PropertyNameCaseInsensitive = true });
+                }
+                catch
+                {
+                    // Body is malformed — still return 200 so the agent does not retry
+                    _logger.LogWarning("ReportAgentError: Could not parse request body from tenant {TenantId}", tenantId);
+                    return req.CreateResponse(HttpStatusCode.OK);
+                }
+
+                if (report == null)
+                {
+                    return req.CreateResponse(HttpStatusCode.OK);
+                }
+
+                // Emit a structured log entry (captured by App Insights as a trace)
+                _logger.LogCritical(
+                    "AgentEmergencyError [{ErrorType}] tenant={TenantId} session={SessionId} http={HttpStatusCode} seq={SequenceNumber} ver={AgentVersion}: {Message}",
+                    report.ErrorType,
+                    tenantId,
+                    report.SessionId,
+                    report.HttpStatusCode,
+                    report.SequenceNumber,
+                    report.AgentVersion,
+                    report.Message);
+
+                // Also emit as a custom event for easy KQL queries in App Insights:
+                //   customEvents | where name == "AgentEmergencyError" | order by timestamp desc
+                _telemetryClient.TrackEvent("AgentEmergencyError", new Dictionary<string, string>
+                {
+                    ["TenantId"]       = tenantId,
+                    ["SessionId"]      = report.SessionId ?? string.Empty,
+                    ["ErrorType"]      = report.ErrorType.ToString(),
+                    ["HttpStatusCode"] = report.HttpStatusCode?.ToString() ?? string.Empty,
+                    ["SequenceNumber"] = report.SequenceNumber?.ToString() ?? string.Empty,
+                    ["AgentVersion"]   = report.AgentVersion ?? string.Empty,
+                    ["Message"]        = report.Message ?? string.Empty,
+                    ["AgentTimestamp"] = report.Timestamp.ToString("O"),
+                });
+
+                return req.CreateResponse(HttpStatusCode.OK);
+            }
+            catch (Exception ex)
+            {
+                // Return 200 even on unexpected errors — the agent must not retry the emergency channel
+                _logger.LogError(ex, "ReportAgentError: Unexpected error handling emergency report");
+                return req.CreateResponse(HttpStatusCode.OK);
+            }
+        }
+    }
+}

src/Shared/AutopilotMonitor.Shared/Constants.cs

@@ -73,6 +73,7 @@ namespace AutopilotMonitor.Shared
             public const string GatherRules              = "/api/rules/gather";
             public const string AnalyzeRules             = "/api/rules/analyze";
             public const string GetDiagnosticsUploadUrl  = "/api/agent/upload-url";
+            public const string ReportAgentError         = "/api/agent/error";
             public const string BlockDevice              = "/api/devices/block";
             public const string GetBlockedDevices        = "/api/devices/blocked";
             public const string ImeLogPatterns           = "/api/rules/ime-log-patterns";

src/Shared/AutopilotMonitor.Shared/Models/AgentErrorReport.cs

@@ -0,0 +1,78 @@
+using System;
+
+namespace AutopilotMonitor.Shared.Models
+{
+    /// <summary>
+    /// Classification of agent-side critical errors reported via the emergency channel.
+    /// </summary>
+    public enum AgentErrorType
+    {
+        /// <summary>
+        /// POST /api/agent/ingest returned a non-auth error (5xx / network failure).
+        /// Events remain in the spool but are not reaching the backend.
+        /// </summary>
+        IngestFailed = 0,
+
+        /// <summary>
+        /// GET /api/agent/config failed — agent continues with defaults.
+        /// </summary>
+        ConfigFetchFailed = 1,
+
+        /// <summary>
+        /// POST /api/agent/register-session failed — session may not be tracked correctly.
+        /// </summary>
+        RegisterSessionFailed = 2,
+    }
+
+    /// <summary>
+    /// Lightweight payload sent by the agent to the emergency channel endpoint
+    /// when a critical backend communication failure occurs.
+    /// Never stored on disk; fire-and-forget from the agent.
+    /// </summary>
+    public class AgentErrorReport
+    {
+        /// <summary>
+        /// The agent's current session ID (from SessionPersistence).
+        /// Allows correlation with session data already in the backend.
+        /// </summary>
+        public string SessionId { get; set; }
+
+        /// <summary>
+        /// Tenant ID, from the device's MDM enrollment registry key.
+        /// </summary>
+        public string TenantId { get; set; }
+
+        /// <summary>
+        /// Classification of the error.
+        /// </summary>
+        public AgentErrorType ErrorType { get; set; }
+
+        /// <summary>
+        /// Error message from the exception or HTTP response.
+        /// Kept short — no stack traces, no sensitive data.
+        /// </summary>
+        public string Message { get; set; }
+
+        /// <summary>
+        /// HTTP status code returned by the backend, if applicable.
+        /// Null for network-level failures (timeout, DNS, TLS).
+        /// </summary>
+        public int? HttpStatusCode { get; set; }
+
+        /// <summary>
+        /// The agent's current spool sequence number at the time of failure.
+        /// Useful for estimating how many events are at risk of being lost.
+        /// </summary>
+        public long? SequenceNumber { get; set; }
+
+        /// <summary>
+        /// Agent version string for diagnosing version-specific issues.
+        /// </summary>
+        public string AgentVersion { get; set; }
+
+        /// <summary>
+        /// UTC timestamp of when the error occurred on the agent.
+        /// </summary>
+        public DateTime Timestamp { get; set; } = DateTime.UtcNow;
+    }
+}