From 77fde56f30a92b2f0ba9b58300c7ef5dbb8d0179 Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Fri, 8 May 2026 16:07:06 +0200 Subject: [PATCH] feat(backend/security): cert-expiry watcher in maintenance with tiered OpsEvents Adds proactive monitoring of the embedded Intune CA bundle so the operator gets a Telegram ping months before agent mTLS would break. Bundle rotation (re-exporting fresh PEMs from a corporate-enrolled device) is the only fix and needs lead time - today's outage from 9af3de15 was rooted in exactly this blind spot. Hooks into the existing 2h MaintenanceService.RunAllAsync timer, iterates the embedded roots + intermediates, and emits tiered OpsEvents on the Security category: daysUntilExpiry | Severity | EventType -----------------+----------+---------------------------- <= 7 (or past) | Critical | EmbeddedCertExpired <= 30 | Error | EmbeddedCertExpiringUrgent <= 90 | Warning | EmbeddedCertExpiringSoon else | - | (silent) Plus a Critical EmbeddedCertBundleEmpty event if the loader returns no roots at all (server misconfiguration / failed deploy). Dedup is one event per cert per tier per UTC day, by querying today's Security OpsEvents and matching on EventType + thumbprint extracted from the Details JSON. Cold starts (Flex Consumption scale-to-zero) rebuild the dedup index from the table, so repeated process restarts do not multiply Telegram pings. Operator action required to receive Telegram alerts: configure four OpsAlertRules in admin UI - one per new EventType - with appropriate MinSeverity. The OpsEvents themselves land in the table regardless, visible in the Ops dashboard. Tests: 23 new (tier-classifier boundary + dedup-key extraction + JSON edge cases). Full suite 1098/1098 green. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../MaintenanceServiceCertExpiryTests.cs | 118 +++++++++++++++ .../Security/CertificateValidator.cs | 14 ++ .../Services/MaintenanceService.CertExpiry.cs | 141 ++++++++++++++++++ .../Services/MaintenanceService.cs | 1 + .../Services/OpsEventService.cs | 25 ++++ 5 files changed, 299 insertions(+) create mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceServiceCertExpiryTests.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.CertExpiry.cs diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceServiceCertExpiryTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceServiceCertExpiryTests.cs new file mode 100644 index 00000000..403feebd --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceServiceCertExpiryTests.cs @@ -0,0 +1,118 @@ +using AutopilotMonitor.Functions.Services; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Tier-boundary + dedup-key tests for the embedded-cert expiry watcher. +/// Full-service smoke tests would require constructing 14 dependencies; these +/// pure-function tests cover the boundary risks (off-by-one, wrong tier) that +/// are most likely to break the watcher silently. +/// +public class MaintenanceServiceCertExpiryTests +{ + // --- ClassifyCertExpiryTier --- + // Thresholds (per MaintenanceService.CertExpiry.cs): + // <= 7d -> EmbeddedCertExpired (Critical) + // <= 30d -> EmbeddedCertExpiringUrgent (Error) + // <= 90d -> EmbeddedCertExpiringSoon (Warning) + // else -> null (silent) + + [Theory] + [InlineData(91, null)] + [InlineData(180, null)] + [InlineData(3650, null)] + public void ClassifyCertExpiryTier_FarFromExpiry_ReturnsNull(int days, string? expected) + { + Assert.Equal(expected, MaintenanceService.ClassifyCertExpiryTier(days)); + } + + [Theory] + [InlineData(90)] + [InlineData(60)] + [InlineData(31)] + public void ClassifyCertExpiryTier_WithinSoonWindow_ReturnsExpiringSoon(int days) + { + Assert.Equal("EmbeddedCertExpiringSoon", MaintenanceService.ClassifyCertExpiryTier(days)); + } + + [Theory] + [InlineData(30)] + [InlineData(15)] + [InlineData(8)] + public void ClassifyCertExpiryTier_WithinUrgentWindow_ReturnsExpiringUrgent(int days) + { + Assert.Equal("EmbeddedCertExpiringUrgent", MaintenanceService.ClassifyCertExpiryTier(days)); + } + + [Theory] + [InlineData(7)] + [InlineData(1)] + [InlineData(0)] + [InlineData(-1)] + [InlineData(-365)] + public void ClassifyCertExpiryTier_WithinCriticalWindowOrExpired_ReturnsExpired(int days) + { + Assert.Equal("EmbeddedCertExpired", MaintenanceService.ClassifyCertExpiryTier(days)); + } + + [Fact] + public void ClassifyCertExpiryTier_BoundaryAt91_IsAboveSoonTier() + { + Assert.Null(MaintenanceService.ClassifyCertExpiryTier(91)); + } + + [Fact] + public void ClassifyCertExpiryTier_BoundaryAt31_IsSoonNotUrgent() + { + Assert.Equal("EmbeddedCertExpiringSoon", MaintenanceService.ClassifyCertExpiryTier(31)); + } + + [Fact] + public void ClassifyCertExpiryTier_BoundaryAt8_IsUrgentNotExpired() + { + Assert.Equal("EmbeddedCertExpiringUrgent", MaintenanceService.ClassifyCertExpiryTier(8)); + } + + // --- ExtractThumbprint (used to build the dedup key from existing OpsEvents) --- + + [Fact] + public void ExtractThumbprint_WithValidJson_ReturnsThumbprint() + { + var json = """{"role":"Root","subject":"CN=Foo","thumbprint":"ABC123","daysUntilExpiry":45}"""; + Assert.Equal("ABC123", MaintenanceService.ExtractThumbprint(json)); + } + + [Fact] + public void ExtractThumbprint_WithMissingProperty_ReturnsEmpty() + { + var json = """{"role":"Root","subject":"CN=Foo"}"""; + Assert.Equal("", MaintenanceService.ExtractThumbprint(json)); + } + + [Fact] + public void ExtractThumbprint_WithNull_ReturnsEmpty() + { + Assert.Equal("", MaintenanceService.ExtractThumbprint(null)); + } + + [Fact] + public void ExtractThumbprint_WithEmpty_ReturnsEmpty() + { + Assert.Equal("", MaintenanceService.ExtractThumbprint("")); + } + + [Fact] + public void ExtractThumbprint_WithMalformedJson_ReturnsEmpty() + { + Assert.Equal("", MaintenanceService.ExtractThumbprint("not json {")); + } + + [Fact] + public void ExtractThumbprint_WithNumericThumbprint_ReturnsEmpty() + { + // Defensive: real OpsEvent details always serialize thumbprint as string, + // but if a non-string value sneaks in we must not crash or mis-key. + var json = """{"thumbprint":12345}"""; + Assert.Equal("", MaintenanceService.ExtractThumbprint(json)); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Security/CertificateValidator.cs b/src/Backend/AutopilotMonitor.Functions/Security/CertificateValidator.cs index 01da004f..7b8b1676 100644 --- a/src/Backend/AutopilotMonitor.Functions/Security/CertificateValidator.cs +++ b/src/Backend/AutopilotMonitor.Functions/Security/CertificateValidator.cs @@ -40,6 +40,20 @@ namespace AutopilotMonitor.Functions.Security private static readonly Lazy _intuneIntermediateCerts = new(() => LoadIntuneCerts("intune-intermediate-")); + /// + /// Returns the embedded Intune root certificates loaded into the validator. + /// Used by maintenance jobs (e.g. cert-expiry monitor) that need to inspect + /// the bundle without re-loading PEM resources. Treat the returned array as + /// read-only; the chain validator caches it for process lifetime. + /// + internal static IReadOnlyList GetEmbeddedRoots() => _intuneRootCerts.Value; + + /// + /// Returns the embedded Intune intermediate certificates loaded into the validator. + /// See for usage notes. + /// + internal static IReadOnlyList GetEmbeddedIntermediates() => _intuneIntermediateCerts.Value; + private class CachedValidationResult { public CertificateValidationResult Result { get; set; } = null!; diff --git a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.CertExpiry.cs b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.CertExpiry.cs new file mode 100644 index 00000000..e31c90dd --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.CertExpiry.cs @@ -0,0 +1,141 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography.X509Certificates; +using System.Text.Json; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Security; +using AutopilotMonitor.Shared.DataAccess; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Services +{ + /// + /// Cert-expiry monitoring leg of MaintenanceService. Runs on the existing 2h + /// timer and emits tiered OpsEvents for embedded Intune CA certs nearing + /// NotAfter, so the operator gets a Telegram ping months before agent mTLS + /// would break - bundle rotation is the only fix and needs lead time to + /// source new PEMs from a corporate-enrolled device. + /// + public partial class MaintenanceService + { + // Tier thresholds, expressed as days remaining until NotAfter. + // Order matters: most-severe first. + internal const int CertExpiryCriticalThresholdDays = 7; + internal const int CertExpiryUrgentThresholdDays = 30; + internal const int CertExpirySoonThresholdDays = 90; + + // Emit-once-per-UTC-day dedup. Runtime caches by (eventType, thumbprint) + // tuple from today's Security ops events. A cold start re-builds from the + // table, so Flex Consumption scale-to-zero does not multiply pings. + internal const string CertExpiryEventTypePrefix = "EmbeddedCert"; + + /// + /// Pure tier classifier - returns the OpsEvent EventType for a given + /// daysUntilExpiry, or null if the cert is comfortably far from expiry. + /// Extracted so the tier boundaries are unit-testable without + /// constructing a full MaintenanceService. + /// + internal static string? ClassifyCertExpiryTier(int daysUntilExpiry) + { + if (daysUntilExpiry <= CertExpiryCriticalThresholdDays) return "EmbeddedCertExpired"; + if (daysUntilExpiry <= CertExpiryUrgentThresholdDays) return "EmbeddedCertExpiringUrgent"; + if (daysUntilExpiry <= CertExpirySoonThresholdDays) return "EmbeddedCertExpiringSoon"; + return null; + } + + /// + /// Inspects the embedded Intune CA bundle (roots + intermediates) and + /// emits Warning/Error/Critical OpsEvents for any cert near expiry. + /// Dedup: one event per cert per tier per UTC day, by querying today's + /// Security ops events. + /// + public async Task CheckEmbeddedCertExpiryAsync() + { + var roots = CertificateValidator.GetEmbeddedRoots(); + var intermediates = CertificateValidator.GetEmbeddedIntermediates(); + var seenToday = await BuildCertExpirySeenIndexAsync(); + + if (roots.Count == 0) + { + if (seenToday.Add($"EmbeddedCertBundleEmpty|")) + await _opsEventService.RecordEmbeddedCertBundleEmptyAsync(); + _logger.LogError("Embedded Intune root bundle is empty - agent mTLS will fail closed"); + return; + } + + await EvaluateCertBucketAsync(roots, "Root", seenToday); + await EvaluateCertBucketAsync(intermediates, "Intermediate", seenToday); + } + + private async Task EvaluateCertBucketAsync( + IReadOnlyList certs, string role, HashSet seenToday) + { + var now = DateTime.UtcNow; + foreach (var cert in certs) + { + var notAfterUtc = cert.NotAfter.ToUniversalTime(); + var daysLeft = (int)Math.Floor((notAfterUtc - now).TotalDays); + var eventType = ClassifyCertExpiryTier(daysLeft); + if (eventType is null) continue; + + var dedupKey = $"{eventType}|{cert.Thumbprint}"; + if (!seenToday.Add(dedupKey)) + { + _logger.LogDebug("Cert expiry already emitted today: {Key}", dedupKey); + continue; + } + + switch (eventType) + { + case "EmbeddedCertExpired": + await _opsEventService.RecordEmbeddedCertExpiredAsync( + role, cert.Subject, cert.Thumbprint, notAfterUtc, daysLeft); + break; + case "EmbeddedCertExpiringUrgent": + await _opsEventService.RecordEmbeddedCertExpiringUrgentAsync( + role, cert.Subject, cert.Thumbprint, notAfterUtc, daysLeft); + break; + case "EmbeddedCertExpiringSoon": + await _opsEventService.RecordEmbeddedCertExpiringSoonAsync( + role, cert.Subject, cert.Thumbprint, notAfterUtc, daysLeft); + break; + } + } + } + + private async Task> BuildCertExpirySeenIndexAsync() + { + var todayUtc = DateTime.UtcNow.Date; + var todaysSecurityEvents = await _opsEventRepo.GetOpsEventsAsync( + category: OpsEventCategory.Security, + dateFrom: todayUtc, + dateTo: todayUtc.AddDays(1)); + + return todaysSecurityEvents + .Where(e => e.EventType.StartsWith(CertExpiryEventTypePrefix, StringComparison.Ordinal)) + .Select(e => $"{e.EventType}|{ExtractThumbprint(e.Details)}") + .ToHashSet(StringComparer.OrdinalIgnoreCase); + } + + internal static string ExtractThumbprint(string? detailsJson) + { + if (string.IsNullOrEmpty(detailsJson)) return string.Empty; + try + { + using var doc = JsonDocument.Parse(detailsJson); + if (doc.RootElement.ValueKind == JsonValueKind.Object && + doc.RootElement.TryGetProperty("thumbprint", out var t) && + t.ValueKind == JsonValueKind.String) + { + return t.GetString() ?? string.Empty; + } + } + catch (JsonException) + { + // Malformed details JSON - dedup key falls back to empty thumbprint + } + return string.Empty; + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs index 38f7ec4b..0d143209 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs @@ -111,6 +111,7 @@ namespace AutopilotMonitor.Functions.Services await CleanupOldOpsEventsAsync(); await CleanupOrphanedEventsAsync(); await CheckAgentBlobStorageAsync(); + await CheckEmbeddedCertExpiryAsync(); await RecomputePlatformStatsAsync(); // Backfill and repair tasks run only via manual trigger (RunManualAsync) diff --git a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs index f38df64d..bfa0f1e6 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs @@ -87,6 +87,31 @@ namespace AutopilotMonitor.Functions.Services $"Agent version pattern '{pattern}' blocked", null, blockedBy, new { pattern }); + public Task RecordEmbeddedCertExpiringSoonAsync(string role, string subject, string thumbprint, DateTime notAfterUtc, int daysUntilExpiry) + => WriteAsync(OpsEventCategory.Security, "EmbeddedCertExpiringSoon", OpsEventSeverity.Warning, + $"Embedded Intune {role.ToLowerInvariant()} '{subject}' expires in {daysUntilExpiry}d ({notAfterUtc:u}) - rotate the embedded PEM bundle before {notAfterUtc:yyyy-MM-dd}", + null, "System.Maintenance", + new { role, subject, thumbprint, notAfterUtc = notAfterUtc.ToString("u"), daysUntilExpiry }); + + public Task RecordEmbeddedCertExpiringUrgentAsync(string role, string subject, string thumbprint, DateTime notAfterUtc, int daysUntilExpiry) + => WriteAsync(OpsEventCategory.Security, "EmbeddedCertExpiringUrgent", OpsEventSeverity.Error, + $"URGENT: embedded Intune {role.ToLowerInvariant()} '{subject}' expires in {daysUntilExpiry}d ({notAfterUtc:u}) - agent mTLS will break without bundle rotation", + null, "System.Maintenance", + new { role, subject, thumbprint, notAfterUtc = notAfterUtc.ToString("u"), daysUntilExpiry }); + + public Task RecordEmbeddedCertExpiredAsync(string role, string subject, string thumbprint, DateTime notAfterUtc, int daysUntilExpiry) + => WriteAsync(OpsEventCategory.Security, "EmbeddedCertExpired", OpsEventSeverity.Critical, + daysUntilExpiry < 0 + ? $"CRITICAL: embedded Intune {role.ToLowerInvariant()} '{subject}' EXPIRED {-daysUntilExpiry}d ago ({notAfterUtc:u}) - agent mTLS validation broken" + : $"CRITICAL: embedded Intune {role.ToLowerInvariant()} '{subject}' expires in {daysUntilExpiry}d ({notAfterUtc:u})", + null, "System.Maintenance", + new { role, subject, thumbprint, notAfterUtc = notAfterUtc.ToString("u"), daysUntilExpiry }); + + public Task RecordEmbeddedCertBundleEmptyAsync() + => WriteAsync(OpsEventCategory.Security, "EmbeddedCertBundleEmpty", OpsEventSeverity.Critical, + "No embedded Intune root certificates loaded - agent mTLS validator is failing closed for ALL clients", + null, "System.Maintenance", new { }); + // ── Tenant ───────────────────────────────────────────────────────────── public Task RecordTenantOffboardedAsync(string tenantId, string performedBy, Dictionary deletedCounts)