diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.cs b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.cs index 32036c6c..42ecc3ef 100644 --- a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.cs +++ b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.cs @@ -62,6 +62,10 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core private int _consecutiveAuthFailures = 0; private DateTime? _firstAuthFailureTime = null; + // Non-auth upload failure tracking for the emergency channel + private int _consecutiveUploadFailures = 0; + private EmergencyReporter _emergencyReporter; + public MonitoringService(AgentConfiguration configuration, AgentLogger logger, string agentVersion) { _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); @@ -75,6 +79,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core _spool = new EventSpool(_configuration.SpoolDirectory); _apiClient = new BackendApiClient(_configuration.ApiBaseUrl, _configuration, _logger); + _emergencyReporter = new EmergencyReporter(_apiClient, _configuration.SessionId, _configuration.TenantId, agentVersion, _logger); _cleanupService = new CleanupService(_configuration, _logger); _diagnosticsService = new DiagnosticsPackageService(_configuration, _logger, _apiClient); @@ -832,9 +837,10 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core _spool.RemoveEvents(events); _logger.Info($"Uploaded {response.EventsProcessed} events"); - // Reset auth failure counter on success + // Reset failure counters on success _consecutiveAuthFailures = 0; _firstAuthFailureTime = null; + _consecutiveUploadFailures = 0; } else { @@ -848,7 +854,19 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core } catch (Exception ex) { + _consecutiveUploadFailures++; _logger.Error("Error uploading events", ex); + + // After N consecutive non-auth failures, signal the emergency channel once. + // The EmergencyReporter deduplicates further calls for the same error category. + if (_consecutiveUploadFailures == EmergencyReporter.ConsecutiveFailureThreshold) + { + _ = _emergencyReporter.TrySendAsync( + AgentErrorType.IngestFailed, + ex.Message, + httpStatusCode: null, + sequenceNumber: _eventSequence); + } } } diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Network/BackendApiClient.cs b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Network/BackendApiClient.cs index b4da03ea..c50b0ad5 100644 --- a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Network/BackendApiClient.cs +++ b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Network/BackendApiClient.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Net.Http; using System.Security.Cryptography.X509Certificates; using System.Text; +using System.Threading; using System.Threading.Tasks; using AutopilotMonitor.Agent.Core.Configuration; using AutopilotMonitor.Agent.Core.Security; @@ -208,6 +209,39 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Network return response; } + /// + /// Sends a critical error report to the emergency channel endpoint. + /// Fire-and-forget: swallows all exceptions so a failure here never cascades. + /// Uses a 5-second per-request timeout so it cannot block the upload loop. + /// + public async Task ReportAgentErrorAsync(AgentErrorReport report) + { + try + { + var url = $"{_baseUrl}{Constants.ApiEndpoints.ReportAgentError}"; + var json = JsonConvert.SerializeObject(report); + + var httpRequest = new HttpRequestMessage(HttpMethod.Post, url) + { + Content = new StringContent(json, Encoding.UTF8, "application/json") + }; + + // Security validation on the backend requires tenant ID before parsing the body + httpRequest.Headers.Add("X-Tenant-Id", report.TenantId); + + // Hardware headers are required for full security validation + AddSecurityHeaders(httpRequest); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await _httpClient.SendAsync(httpRequest, cts.Token); + // Response status is deliberately ignored — endpoint always returns 200 + } + catch + { + // Swallow all exceptions — the emergency channel must never cascade failures + } + } + private async Task PostAsync(string url, TRequest data) { var json = JsonConvert.SerializeObject(data); diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Network/EmergencyReporter.cs b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Network/EmergencyReporter.cs new file mode 100644 index 00000000..79b34849 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Network/EmergencyReporter.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using AutopilotMonitor.Agent.Core.Logging; +using AutopilotMonitor.Shared.Models; + +namespace AutopilotMonitor.Agent.Core.Monitoring.Network +{ + /// + /// Sends critical agent-side errors to the backend emergency channel endpoint. + /// + /// Anti-flood guarantees (per session): + /// - Only fires after consecutive upload failures + /// - Same error key (ErrorType + HTTP status code) is only ever sent once per session + /// - At most emergency reports total per session + /// - At most one report every minutes regardless of error type + /// + /// A failure in this reporter is silently swallowed — it must never cascade into the main loop. + /// + internal class EmergencyReporter + { + /// + /// Number of consecutive upload failures required before the first emergency report is sent. + /// + public const int ConsecutiveFailureThreshold = 3; + + private const int MaxReportsPerSession = 5; + private const int MinIntervalMinutes = 10; + + private readonly BackendApiClient _apiClient; + private readonly string _sessionId; + private readonly string _tenantId; + private readonly string _agentVersion; + private readonly AgentLogger _logger; + + private readonly HashSet _reportedKeys = new HashSet(); + private DateTime? _lastReportTime; + private int _reportCount; + + public EmergencyReporter( + BackendApiClient apiClient, + string sessionId, + string tenantId, + string agentVersion, + AgentLogger logger) + { + _apiClient = apiClient; + _sessionId = sessionId; + _tenantId = tenantId; + _agentVersion = agentVersion; + _logger = logger; + } + + /// + /// Attempts to send an emergency report. Returns immediately without awaiting + /// (intended to be called as fire-and-forget via _ = reporter.TrySendAsync(...)). + /// + /// All anti-flood checks are applied synchronously before the async HTTP call, + /// so repeated calls are cheap when suppressed. + /// + public async Task TrySendAsync( + AgentErrorType errorType, + string message, + int? httpStatusCode = null, + long? sequenceNumber = null) + { + // Deduplicate by error type + status code: same failure category sent only once per session + var key = $"{errorType}:{httpStatusCode}"; + + if (_reportCount >= MaxReportsPerSession) + { + _logger?.Debug($"[EmergencyChannel] Suppressed ({_reportCount}/{MaxReportsPerSession} reports used): {key}"); + return; + } + + if (_reportedKeys.Contains(key)) + { + _logger?.Debug($"[EmergencyChannel] Suppressed (already reported): {key}"); + return; + } + + if (_lastReportTime != null && + (DateTime.UtcNow - _lastReportTime.Value).TotalMinutes < MinIntervalMinutes) + { + _logger?.Debug($"[EmergencyChannel] Suppressed (cooldown active, last report at {_lastReportTime.Value:HH:mm:ss}): {key}"); + return; + } + + _reportedKeys.Add(key); + _reportCount++; + _lastReportTime = DateTime.UtcNow; + + var statusText = httpStatusCode.HasValue ? $" HTTP {httpStatusCode}" : string.Empty; + _logger?.Warning($"[EmergencyChannel] Sending report {_reportCount}/{MaxReportsPerSession}: {errorType}{statusText}"); + + var report = new AgentErrorReport + { + SessionId = _sessionId, + TenantId = _tenantId, + ErrorType = errorType, + Message = message, + HttpStatusCode = httpStatusCode, + SequenceNumber = sequenceNumber, + AgentVersion = _agentVersion, + Timestamp = DateTime.UtcNow + }; + + await _apiClient.ReportAgentErrorAsync(report); + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent/buildcounter.txt b/src/Agent/AutopilotMonitor.Agent/buildcounter.txt index a2ecc456..4c5c8078 100644 --- a/src/Agent/AutopilotMonitor.Agent/buildcounter.txt +++ b/src/Agent/AutopilotMonitor.Agent/buildcounter.txt @@ -1 +1 @@ -154 +158 diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/ReportAgentErrorFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/ReportAgentErrorFunction.cs new file mode 100644 index 00000000..ef68ce57 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Functions/ReportAgentErrorFunction.cs @@ -0,0 +1,135 @@ +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Text.Json; +using AutopilotMonitor.Functions.Security; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Shared.Models; +using Microsoft.ApplicationInsights; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Functions +{ + /// + /// Emergency channel endpoint: receives critical agent-side error reports when the + /// normal ingest path is unavailable or returning errors. + /// + /// Security: full ValidateSecurityAsync (cert + rate limit + hardware whitelist + Autopilot). + /// Storage: logs to Application Insights as a structured custom event "AgentEmergencyError". + /// No Table Storage — App Insights is sufficient for incident investigation via KQL. + /// + /// Response: always 200 OK so the agent never retries based on backend-side issues here. + /// + public class ReportAgentErrorFunction + { + private readonly ILogger _logger; + private readonly TenantConfigurationService _configService; + private readonly RateLimitService _rateLimitService; + private readonly AutopilotDeviceValidator _autopilotDeviceValidator; + private readonly TelemetryClient _telemetryClient; + + public ReportAgentErrorFunction( + ILogger logger, + TenantConfigurationService configService, + RateLimitService rateLimitService, + AutopilotDeviceValidator autopilotDeviceValidator, + TelemetryClient telemetryClient) + { + _logger = logger; + _configService = configService; + _rateLimitService = rateLimitService; + _autopilotDeviceValidator = autopilotDeviceValidator; + _telemetryClient = telemetryClient; + } + + [Function("ReportAgentError")] + public async Task Run( + [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "agent/error")] HttpRequestData req) + { + try + { + // Security checks FIRST — same pattern as /api/agent/ingest. + // TenantId from X-Tenant-Id header allows validation before parsing the body. + var tenantId = req.Headers.Contains("X-Tenant-Id") + ? req.Headers.GetValues("X-Tenant-Id").FirstOrDefault() + : null; + + if (string.IsNullOrEmpty(tenantId)) + { + return req.CreateResponse(HttpStatusCode.BadRequest); + } + + // Full security validation: cert, rate limit, hardware whitelist, Autopilot (if enabled). + // Autopilot positive cache TTL is 30 min — devices that have recently called /ingest + // will get a cache hit here at essentially zero cost. + var (_, errorResponse) = await req.ValidateSecurityAsync( + tenantId, + _configService, + _rateLimitService, + _autopilotDeviceValidator, + _logger + ); + + if (errorResponse != null) + { + return errorResponse; + } + + // Parse the report body + AgentErrorReport? report = null; + try + { + report = await JsonSerializer.DeserializeAsync( + req.Body, + new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); + } + catch + { + // Body is malformed — still return 200 so the agent does not retry + _logger.LogWarning("ReportAgentError: Could not parse request body from tenant {TenantId}", tenantId); + return req.CreateResponse(HttpStatusCode.OK); + } + + if (report == null) + { + return req.CreateResponse(HttpStatusCode.OK); + } + + // Emit a structured log entry (captured by App Insights as a trace) + _logger.LogCritical( + "AgentEmergencyError [{ErrorType}] tenant={TenantId} session={SessionId} http={HttpStatusCode} seq={SequenceNumber} ver={AgentVersion}: {Message}", + report.ErrorType, + tenantId, + report.SessionId, + report.HttpStatusCode, + report.SequenceNumber, + report.AgentVersion, + report.Message); + + // Also emit as a custom event for easy KQL queries in App Insights: + // customEvents | where name == "AgentEmergencyError" | order by timestamp desc + _telemetryClient.TrackEvent("AgentEmergencyError", new Dictionary + { + ["TenantId"] = tenantId, + ["SessionId"] = report.SessionId ?? string.Empty, + ["ErrorType"] = report.ErrorType.ToString(), + ["HttpStatusCode"] = report.HttpStatusCode?.ToString() ?? string.Empty, + ["SequenceNumber"] = report.SequenceNumber?.ToString() ?? string.Empty, + ["AgentVersion"] = report.AgentVersion ?? string.Empty, + ["Message"] = report.Message ?? string.Empty, + ["AgentTimestamp"] = report.Timestamp.ToString("O"), + }); + + return req.CreateResponse(HttpStatusCode.OK); + } + catch (Exception ex) + { + // Return 200 even on unexpected errors — the agent must not retry the emergency channel + _logger.LogError(ex, "ReportAgentError: Unexpected error handling emergency report"); + return req.CreateResponse(HttpStatusCode.OK); + } + } + } +} diff --git a/src/Shared/AutopilotMonitor.Shared/Constants.cs b/src/Shared/AutopilotMonitor.Shared/Constants.cs index a40813ee..4f2424b6 100644 --- a/src/Shared/AutopilotMonitor.Shared/Constants.cs +++ b/src/Shared/AutopilotMonitor.Shared/Constants.cs @@ -73,6 +73,7 @@ namespace AutopilotMonitor.Shared public const string GatherRules = "/api/rules/gather"; public const string AnalyzeRules = "/api/rules/analyze"; public const string GetDiagnosticsUploadUrl = "/api/agent/upload-url"; + public const string ReportAgentError = "/api/agent/error"; public const string BlockDevice = "/api/devices/block"; public const string GetBlockedDevices = "/api/devices/blocked"; public const string ImeLogPatterns = "/api/rules/ime-log-patterns"; diff --git a/src/Shared/AutopilotMonitor.Shared/Models/AgentErrorReport.cs b/src/Shared/AutopilotMonitor.Shared/Models/AgentErrorReport.cs new file mode 100644 index 00000000..08706036 --- /dev/null +++ b/src/Shared/AutopilotMonitor.Shared/Models/AgentErrorReport.cs @@ -0,0 +1,78 @@ +using System; + +namespace AutopilotMonitor.Shared.Models +{ + /// + /// Classification of agent-side critical errors reported via the emergency channel. + /// + public enum AgentErrorType + { + /// + /// POST /api/agent/ingest returned a non-auth error (5xx / network failure). + /// Events remain in the spool but are not reaching the backend. + /// + IngestFailed = 0, + + /// + /// GET /api/agent/config failed — agent continues with defaults. + /// + ConfigFetchFailed = 1, + + /// + /// POST /api/agent/register-session failed — session may not be tracked correctly. + /// + RegisterSessionFailed = 2, + } + + /// + /// Lightweight payload sent by the agent to the emergency channel endpoint + /// when a critical backend communication failure occurs. + /// Never stored on disk; fire-and-forget from the agent. + /// + public class AgentErrorReport + { + /// + /// The agent's current session ID (from SessionPersistence). + /// Allows correlation with session data already in the backend. + /// + public string SessionId { get; set; } + + /// + /// Tenant ID, from the device's MDM enrollment registry key. + /// + public string TenantId { get; set; } + + /// + /// Classification of the error. + /// + public AgentErrorType ErrorType { get; set; } + + /// + /// Error message from the exception or HTTP response. + /// Kept short — no stack traces, no sensitive data. + /// + public string Message { get; set; } + + /// + /// HTTP status code returned by the backend, if applicable. + /// Null for network-level failures (timeout, DNS, TLS). + /// + public int? HttpStatusCode { get; set; } + + /// + /// The agent's current spool sequence number at the time of failure. + /// Useful for estimating how many events are at risk of being lost. + /// + public long? SequenceNumber { get; set; } + + /// + /// Agent version string for diagnosing version-specific issues. + /// + public string AgentVersion { get; set; } + + /// + /// UTC timestamp of when the error occurred on the agent. + /// + public DateTime Timestamp { get; set; } = DateTime.UtcNow; + } +}