From 7b975d46b626afda62ea5e7e287c5889bd65ee72 Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Mon, 27 Jul 2026 18:39:22 +0200 Subject: [PATCH] feat(backend): Azure Monitor alert webhook - platform alerts join the ops-event bus New anonymous POST ops/alert-webhook receives Common Alert Schema notifications from an Action Group webhook receiver and re-emits them as ops events (category Platform, type AzureMonitorAlert), so Azure Monitor alerts flow through the existing OpsEventService -> OpsAlertDispatchService fan-out (Telegram/Teams/Slack + portal ops-events list) like every internally raised operational event. The delivery path is independent of the portal: a SWA outage cannot suppress the notification; the action group's e-mail receiver stays as the backend-independent fallback. Security: action-group webhooks cannot send custom headers, so the shared secret travels as ?secret= and is compared in constant time (FixedTimeEquals) against the OpsAlertWebhookSecret app setting - fail-closed while unset. 64 KB payload cap, defensive essentials-only parsing (only alertRule is mandatory), severity mapped Sev0->Critical, Sev1->Error, Sev2->Warning, Sev3/4->Info with unknown values falling back to Info so schema drift can never escalate. Dual-registered per convention: EndpointAccessPolicyCatalog entry (PublicAnonymous), OPS_EVENT_TYPES in OpsAlertRulesSection (rule wiring) and Platform category in OpsEventsSection (badge + filter). Backend suite 3511 green (incl. new helper tests + catalog completeness), web tsc + vitest 651 green. Co-Authored-By: Claude Opus 5 (1M context) --- .../AzureMonitorAlertWebhookFunctionTests.cs | 172 ++++++++++++++ .../AzureMonitorAlertWebhookFunction.cs | 209 ++++++++++++++++++ .../Security/EndpointAccessPolicyCatalog.cs | 5 + .../Services/OpsEventService.cs | 18 ++ .../DataAccess/IOpsEventRepository.cs | 2 + .../admin/components/OpsAlertRulesSection.tsx | 10 + .../app/admin/components/OpsEventsSection.tsx | 3 +- 7 files changed, 418 insertions(+), 1 deletion(-) create mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/AzureMonitorAlertWebhookFunctionTests.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Functions/Infrastructure/AzureMonitorAlertWebhookFunction.cs diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/AzureMonitorAlertWebhookFunctionTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/AzureMonitorAlertWebhookFunctionTests.cs new file mode 100644 index 00000000..9cf48cb3 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/AzureMonitorAlertWebhookFunctionTests.cs @@ -0,0 +1,172 @@ +using AutopilotMonitor.Functions.Functions.Infrastructure; +using AutopilotMonitor.Shared.DataAccess; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Pins the pure helpers of — the anonymous, +/// secret-gated bridge that turns Azure Monitor Common Alert Schema notifications into ops +/// events. Load-bearing bits: the secret gate is fail-closed (an unset app setting rejects +/// everything), severity mapping can never escalate on schema drift (unknown → Info), and +/// parsing is defensive (only the alert rule name is mandatory). +/// +public class AzureMonitorAlertWebhookFunctionTests +{ + // ── SecretMatches ────────────────────────────────────────────────────── + + [Fact] + public void SecretMatches_accepts_equal_secrets() + { + Assert.True(AzureMonitorAlertWebhookFunction.SecretMatches("s3cret-value", "s3cret-value")); + } + + [Theory] + [InlineData("wrong", "right")] + [InlineData("right-but-longer", "right")] + [InlineData("", "configured")] + [InlineData(null, "configured")] + public void SecretMatches_rejects_mismatch_or_missing_provided(string? provided, string configured) + { + Assert.False(AzureMonitorAlertWebhookFunction.SecretMatches(provided, configured)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public void SecretMatches_fails_closed_when_not_configured(string? configured) + { + // While the app setting is unset, NOTHING matches — not even an empty provided value. + Assert.False(AzureMonitorAlertWebhookFunction.SecretMatches("anything", configured)); + Assert.False(AzureMonitorAlertWebhookFunction.SecretMatches("", configured)); + Assert.False(AzureMonitorAlertWebhookFunction.SecretMatches(null, configured)); + } + + // ── MapSeverity ──────────────────────────────────────────────────────── + + [Theory] + [InlineData("Sev0", OpsEventSeverity.Critical)] + [InlineData("Sev1", OpsEventSeverity.Error)] + [InlineData("Sev2", OpsEventSeverity.Warning)] + [InlineData("Sev3", OpsEventSeverity.Info)] + [InlineData("Sev4", OpsEventSeverity.Info)] + public void MapSeverity_maps_azure_scale_to_ops_scale(string azure, string expected) + { + Assert.Equal(expected, AzureMonitorAlertWebhookFunction.MapSeverity(azure)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData("Sev5")] + [InlineData("Critical")] // someone posting the ops scale directly must not pass through + public void MapSeverity_falls_back_to_info_on_unknown_values(string? azure) + { + Assert.Equal(OpsEventSeverity.Info, AzureMonitorAlertWebhookFunction.MapSeverity(azure)); + } + + // ── TryParse ─────────────────────────────────────────────────────────── + + private const string FullPayload = """ + { + "schemaId": "azureMonitorCommonAlertSchema", + "data": { + "essentials": { + "alertId": "/subscriptions/xxx/providers/Microsoft.AlertsManagement/alerts/abc", + "alertRule": "Portal RSC prefetch tail", + "severity": "Sev2", + "signalType": "Log", + "monitorCondition": "Fired", + "monitoringService": "Log Alerts V2", + "alertTargetIDs": [ + "/subscriptions/xxx/resourcegroups/rg-autopilotmonitor-prd-gwc/providers/microsoft.insights/components/autopilotmonitorweb" + ], + "description": "RSC prefetches exceeding 10s", + "firedDateTime": "2026-07-27T15:30:00.000Z" + }, + "alertContext": { + "condition": { + "windowSize": "PT1H", + "allOf": [ + { + "searchQuery": "dependencies | where name contains '_rsc='", + "metricValue": 18.0, + "operator": "GreaterThan", + "threshold": "0" + } + ] + } + } + } + } + """; + + [Fact] + public void TryParse_extracts_essentials_and_metric_value() + { + var alert = AzureMonitorAlertWebhookFunction.TryParse(FullPayload); + + Assert.NotNull(alert); + Assert.Equal("Portal RSC prefetch tail", alert!.AlertRule); + Assert.Equal("Sev2", alert.Severity); + Assert.Equal("Fired", alert.MonitorCondition); + Assert.Equal("RSC prefetches exceeding 10s", alert.Description); + Assert.Equal("Log Alerts V2", alert.MonitoringService); + // No targetResourceName in the payload → resource name segment of the first target ID. + Assert.Equal("autopilotmonitorweb", alert.TargetResource); + Assert.Equal(18.0, alert.MetricValue); + } + + [Fact] + public void TryParse_prefers_explicit_target_resource_name() + { + const string payload = """ + {"data":{"essentials":{"alertRule":"r","targetResourceName":"explicit-name", + "alertTargetIDs":["/sub/x/other-name"]}}} + """; + var alert = AzureMonitorAlertWebhookFunction.TryParse(payload); + Assert.Equal("explicit-name", alert!.TargetResource); + } + + [Fact] + public void TryParse_minimal_payload_needs_only_the_alert_rule() + { + var alert = AzureMonitorAlertWebhookFunction.TryParse( + """{"data":{"essentials":{"alertRule":"minimal"}}}"""); + + Assert.NotNull(alert); + Assert.Equal("minimal", alert!.AlertRule); + Assert.Null(alert.Severity); + Assert.Equal("Fired", alert.MonitorCondition); // default when absent + Assert.Null(alert.Description); + Assert.Null(alert.MetricValue); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + [InlineData("not json at all")] + [InlineData("{}")] // no data + [InlineData("""{"data":{}}""")] // no essentials + [InlineData("""{"data":{"essentials":{}}}""")] // no alertRule + [InlineData("""{"data":{"essentials":{"alertRule":""}}}""")] // empty alertRule + [InlineData("""{"data":{"essentials":{"alertRule":123}}}""")] // wrong type + public void TryParse_rejects_unusable_payloads(string? body) + { + Assert.Null(AzureMonitorAlertWebhookFunction.TryParse(body!)); + } + + [Fact] + public void TryParse_tolerates_malformed_alert_context() + { + // alertContext with unexpected shapes must not break essentials extraction. + const string payload = """ + {"data":{"essentials":{"alertRule":"r","severity":"Sev0"}, + "alertContext":{"condition":{"allOf":"not-an-array"}}}} + """; + var alert = AzureMonitorAlertWebhookFunction.TryParse(payload); + Assert.NotNull(alert); + Assert.Equal("Sev0", alert!.Severity); + Assert.Null(alert.MetricValue); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Infrastructure/AzureMonitorAlertWebhookFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Infrastructure/AzureMonitorAlertWebhookFunction.cs new file mode 100644 index 00000000..71d6c8a3 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Infrastructure/AzureMonitorAlertWebhookFunction.cs @@ -0,0 +1,209 @@ +using System; +using System.IO; +using System.Net; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Services; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Functions.Infrastructure +{ + /// + /// Receives Azure Monitor alert notifications (Common Alert Schema) from an Action Group + /// webhook receiver and re-emits them as ops events, so platform alerts flow through the + /// same OpsEventService → OpsAlertDispatchService fan-out (Telegram/Teams/Slack providers + + /// the portal ops-events list) as every internally raised operational event. + /// + /// The delivery path is deliberately independent of the portal: Azure Monitor → this + /// function → providers. A portal (SWA) outage cannot suppress these notifications — the + /// action group's e-mail receiver additionally stays as the backend-independent fallback, + /// because a self-hosted channel cannot alert on its own outage. + /// + /// Security: anonymous route (Action Group webhook receivers can only call a bare URI — no + /// custom headers), gated by a shared secret in the query string compared in constant time + /// against the OpsAlertWebhookSecret app setting. Fail-closed: while the setting is unset, + /// every request is rejected. Dispatch to providers only happens if an enabled ops alert + /// rule for EventType "AzureMonitorAlert" matches the mapped severity — the event itself is + /// always stored. + /// + public class AzureMonitorAlertWebhookFunction + { + internal const string SecretSettingName = "OpsAlertWebhookSecret"; + + // Common Alert Schema payloads for log-search alerts carry the search query, link URLs + // and dimension sets — a few KB in practice. 64 KB is a generous shape gate, not a throttle. + internal const int MaxContentLength = 64 * 1024; + + private readonly OpsEventService _opsEventService; + private readonly ILogger _logger; + + public AzureMonitorAlertWebhookFunction( + OpsEventService opsEventService, + ILogger logger) + { + _opsEventService = opsEventService; + _logger = logger; + } + + [Function("AzureMonitorAlertWebhook")] + public async Task Run( + [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "ops/alert-webhook")] HttpRequestData req) + { + var configuredSecret = Environment.GetEnvironmentVariable(SecretSettingName); + var providedSecret = System.Web.HttpUtility.ParseQueryString(req.Url.Query ?? string.Empty)["secret"]; + + if (!SecretMatches(providedSecret, configuredSecret)) + { + // Deliberately no detail in the response; the warning makes a misconfigured + // (unset) secret operator-visible in the logs. + _logger.LogWarning( + "Azure Monitor alert webhook rejected: {Reason}", + string.IsNullOrEmpty(configuredSecret) ? "secret not configured (fail-closed)" : "secret mismatch"); + return req.CreateResponse(HttpStatusCode.Unauthorized); + } + + if (req.Body.Length > MaxContentLength) + { + _logger.LogWarning("Azure Monitor alert webhook rejected: payload {Length} bytes exceeds cap", req.Body.Length); + return req.CreateResponse(HttpStatusCode.BadRequest); + } + + string body; + using (var reader = new StreamReader(req.Body, Encoding.UTF8)) + body = await reader.ReadToEndAsync(); + + var alert = TryParse(body); + if (alert == null) + { + _logger.LogWarning("Azure Monitor alert webhook rejected: body is not a Common Alert Schema payload"); + return req.CreateResponse(HttpStatusCode.BadRequest); + } + + await _opsEventService.RecordAzureMonitorAlertAsync( + alert.AlertRule, + MapSeverity(alert.Severity), + alert.MonitorCondition, + alert.Description, + alert.Severity, + alert.MonitoringService, + alert.TargetResource, + alert.MetricValue); + + _logger.LogInformation( + "Azure Monitor alert '{AlertRule}' {Condition} ({Severity}) recorded as ops event", + alert.AlertRule, alert.MonitorCondition, alert.Severity); + + return req.CreateResponse(HttpStatusCode.OK); + } + + /// + /// Constant-time secret comparison. Fail-closed: an unset/empty configured secret never + /// matches anything, so the endpoint stays dark until the app setting is provisioned. + /// + internal static bool SecretMatches(string? provided, string? configured) + { + if (string.IsNullOrEmpty(configured) || string.IsNullOrEmpty(provided)) + return false; + + return CryptographicOperations.FixedTimeEquals( + Encoding.UTF8.GetBytes(provided), + Encoding.UTF8.GetBytes(configured)); + } + + /// + /// Maps Azure Monitor severities (Sev0 = most severe … Sev4 = verbose) onto the ops event + /// severity scale. Unknown values fall back to Info so a schema drift can never escalate. + /// + internal static string MapSeverity(string? azureSeverity) => azureSeverity switch + { + "Sev0" => Shared.DataAccess.OpsEventSeverity.Critical, + "Sev1" => Shared.DataAccess.OpsEventSeverity.Error, + "Sev2" => Shared.DataAccess.OpsEventSeverity.Warning, + _ => Shared.DataAccess.OpsEventSeverity.Info, + }; + + /// + /// Parsed subset of the Common Alert Schema. Only is mandatory; + /// everything else is best-effort so a partial payload still produces a usable ops event. + /// + internal sealed record ParsedAlert( + string AlertRule, + string? Severity, + string MonitorCondition, + string? Description, + string? MonitoringService, + string? TargetResource, + double? MetricValue); + + /// + /// Defensively extracts data.essentials (+ the first condition's metricValue) from a + /// Common Alert Schema payload. Returns null when the body is not JSON or carries no + /// alert rule name — the two states in which no meaningful ops event can be formed. + /// + internal static ParsedAlert? TryParse(string body) + { + if (string.IsNullOrWhiteSpace(body)) + return null; + + try + { + using var doc = JsonDocument.Parse(body); + if (!doc.RootElement.TryGetProperty("data", out var data) || + !data.TryGetProperty("essentials", out var essentials)) + return null; + + var alertRule = GetString(essentials, "alertRule"); + if (string.IsNullOrWhiteSpace(alertRule)) + return null; + + var targetResource = GetString(essentials, "targetResourceName"); + if (string.IsNullOrEmpty(targetResource) && + essentials.TryGetProperty("alertTargetIDs", out var targets) && + targets.ValueKind == JsonValueKind.Array && targets.GetArrayLength() > 0 && + targets[0].ValueKind == JsonValueKind.String) + { + // Fall back to the resource name segment of the first target resource ID. + var id = targets[0].GetString() ?? string.Empty; + targetResource = id[(id.LastIndexOf('/') + 1)..]; + } + + // Log-search alerts carry the observed value under alertContext.condition.allOf[0]. + double? metricValue = null; + if (data.TryGetProperty("alertContext", out var ctx) && + ctx.ValueKind == JsonValueKind.Object && + ctx.TryGetProperty("condition", out var condition) && + condition.ValueKind == JsonValueKind.Object && + condition.TryGetProperty("allOf", out var allOf) && + allOf.ValueKind == JsonValueKind.Array && allOf.GetArrayLength() > 0 && + allOf[0].ValueKind == JsonValueKind.Object && + allOf[0].TryGetProperty("metricValue", out var mv) && + mv.ValueKind == JsonValueKind.Number) + { + metricValue = mv.GetDouble(); + } + + return new ParsedAlert( + alertRule!, + GetString(essentials, "severity"), + GetString(essentials, "monitorCondition") ?? "Fired", + GetString(essentials, "description"), + GetString(essentials, "monitoringService"), + targetResource, + metricValue); + } + catch (JsonException) + { + return null; + } + } + + private static string? GetString(JsonElement element, string property) + => element.TryGetProperty(property, out var value) && value.ValueKind == JsonValueKind.String + ? value.GetString() + : null; + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs b/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs index 7ee7f970..3ed1de13 100644 --- a/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs +++ b/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs @@ -170,6 +170,11 @@ public static class EndpointAccessPolicyCatalog // ticket is the sole authority — authz happened at mint time (diagnostics/download-ticket, // MemberRead). See DiagnosticsTicketDownloadFunction. new("GET", "diagnostics/download", EndpointPolicy.PublicAnonymous), + // Azure Monitor action-group webhook (Common Alert Schema → ops event). Action groups + // cannot send custom headers, so the shared secret travels as ?secret= and is compared + // in constant time against the OpsAlertWebhookSecret app setting (fail-closed while + // unset). See AzureMonitorAlertWebhookFunction. + new("POST", "ops/alert-webhook", EndpointPolicy.PublicAnonymous), // ── DeviceOrBootstrapAuth ─────────────────────────────────────── new("POST", "agent/register-session", EndpointPolicy.DeviceOrBootstrapAuth), diff --git a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs index 88dff9ba..68a3e44a 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs @@ -594,6 +594,24 @@ namespace AutopilotMonitor.Functions.Services null, "System.SlaEvaluation", new { tenantsEvaluated, breachesDetected, notificationsSent, durationMs }); + // ── Platform (Azure Monitor) ─────────────────────────────────────────── + + /// + /// Records an Azure Monitor alert notification delivered via the ops alert webhook + /// (AzureMonitorAlertWebhookFunction). severity is already mapped onto the ops scale; + /// azureSeverity keeps the original Sev0–Sev4 value for the details payload. + /// + public Task RecordAzureMonitorAlertAsync(string alertRule, string severity, string monitorCondition, + string? description, string? azureSeverity, string? monitoringService, string? targetResource, + double? metricValue) + { + var suffix = string.IsNullOrWhiteSpace(description) ? string.Empty : $": {description}"; + return WriteAsync(OpsEventCategory.Platform, "AzureMonitorAlert", severity, + $"Azure Monitor alert '{alertRule}' {monitorCondition}{suffix}", + null, "System.AzureMonitor", + new { alertRule, monitorCondition, azureSeverity, monitoringService, targetResource, metricValue }); + } + // ── Core write method ────────────────────────────────────────────────── private async Task WriteAsync(string category, string eventType, string severity, diff --git a/src/Shared/AutopilotMonitor.Shared/DataAccess/IOpsEventRepository.cs b/src/Shared/AutopilotMonitor.Shared/DataAccess/IOpsEventRepository.cs index 9eaab709..af0326e9 100644 --- a/src/Shared/AutopilotMonitor.Shared/DataAccess/IOpsEventRepository.cs +++ b/src/Shared/AutopilotMonitor.Shared/DataAccess/IOpsEventRepository.cs @@ -45,6 +45,8 @@ namespace AutopilotMonitor.Shared.DataAccess public const string Tenant = "Tenant"; public const string Agent = "Agent"; public const string Sla = "SLA"; + /// Platform infrastructure alerts relayed from Azure Monitor (ops alert webhook). + public const string Platform = "Platform"; } /// diff --git a/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx b/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx index 932af662..bcff312c 100644 --- a/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx @@ -110,6 +110,15 @@ const OPS_EVENT_TYPES: Record = { // feedback_ops_event_types_dual_register. "AgentEmergencyBreak", ], + Platform: [ + // Azure Monitor alerts relayed through POST ops/alert-webhook (Common Alert Schema → + // AzureMonitorAlertWebhookFunction → RecordAzureMonitorAlertAsync). One event type for ALL + // Azure Monitor alert rules — the rule name travels in the message/details. Severity is + // mapped Sev0→Critical, Sev1→Error, Sev2→Warning, Sev3/4→Info; both Fired and Resolved + // notifications carry the same mapped severity, so a rule here delivers recovery pings too. + // Dual-register per memory feedback_ops_event_types_dual_register. + "AzureMonitorAlert", + ], }; const SEVERITIES = ["Info", "Warning", "Error", "Critical"]; @@ -120,6 +129,7 @@ const CATEGORY_COLORS: Record = { Security: "text-orange-700 dark:text-orange-300", Tenant: "text-teal-700 dark:text-teal-300", Agent: "text-cyan-700 dark:text-cyan-300", + Platform: "text-sky-700 dark:text-sky-300", }; interface OpsAlertRulesSectionProps { diff --git a/src/Web/autopilot-monitor-web/app/admin/components/OpsEventsSection.tsx b/src/Web/autopilot-monitor-web/app/admin/components/OpsEventsSection.tsx index 6e94890e..2f201d05 100644 --- a/src/Web/autopilot-monitor-web/app/admin/components/OpsEventsSection.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/components/OpsEventsSection.tsx @@ -42,9 +42,10 @@ const CATEGORY_STYLES: Record = { Security: "bg-orange-100 text-orange-800 dark:bg-orange-900/40 dark:text-orange-300", Tenant: "bg-teal-100 text-teal-800 dark:bg-teal-900/40 dark:text-teal-300", Agent: "bg-cyan-100 text-cyan-800 dark:bg-cyan-900/40 dark:text-cyan-300", + Platform: "bg-sky-100 text-sky-800 dark:bg-sky-900/40 dark:text-sky-300", }; -const ALL_CATEGORIES = ["Consent", "Maintenance", "Security", "Tenant", "Agent"]; +const ALL_CATEGORIES = ["Consent", "Maintenance", "Security", "Tenant", "Agent", "Platform"]; const PAGE_SIZE = 20;