From ca26cb4f7cb5e87585994951de5e3cd87ca44a64 Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Tue, 18 Aug 2026 00:20:53 +0200 Subject: [PATCH] backend: auto-archive IME installer on first sighting of a new version When RecordImeVersionAsync reports a genuinely new IME version (~monthly), EventIngestProcessor now also enqueues an ime-msi-archive job. The worker (QueuePollingWorker, heartbeat, pause on ImeMsiArchivingEnabled=false) downloads the installer and stores it write-once (If-None-Match:*) in the permanent ime-archive container ({version}/IntuneWindowsAgent.msi + provenance.json), then merges MsiArchiveStatus/BlobPath/Sha256/Bytes/SourceUrl onto the ImeVersionHistory row - so every fleet-observed IME build stays available for decompilation/diffing after Microsoft's versionless CDN moves on. The download URL comes from agent telemetry (msiDownloadUrl enrichment) and is treated as untrusted: https-only *.manage.microsoft.com allowlist with exact filename match, canonical versionless fallback, no redirects, size cap (MaxImeMsiDownloadSizeMB, default 250, preflight + mid-stream) enforced by a SHA-256-hashing pass-through stream. Archiver never throws; the worker retries only transient statuses (download/timeout/error), permanent rejections complete with their Failed:* status persisted. Admin settings ImeMsiArchivingEnabled (default true) + MaxImeMsiDownloadSizeMB via the full 4-file web chain; archive fields on the version-history API are GA-only (non-GA projection unchanged, redaction tests extended). Shared manifests regenerated. Tests: archiver status branches via recording subclass (allowlist, caps, 409 idempotency, cancellation propagation), config defaults. Backend suite 4262 green, web tsc + vitest 762 green, solution 0 warnings. Co-Authored-By: Claude Fable 5 --- .../AdminConfigurationImeMsiArchivingTests.cs | 44 ++ .../GetImeVersionHistoryRedactionTests.cs | 32 +- .../ImeMsiArchiverTests.cs | 335 ++++++++++++++++ .../TableStorage/TableConfigRepository.cs | 4 + .../TableStorage/TableSessionRepository.cs | 4 + .../AutopilotMonitor.Functions/Program.cs | 20 + .../Services/EventIngestProcessor.cs | 26 +- .../Ime/AzureQueueImeMsiArchiveProducer.cs | 84 ++++ .../Services/Ime/IImeMsiArchiveProducer.cs | 16 + .../Services/Ime/ImeMsiArchiveQueueWorker.cs | 87 ++++ .../Services/Ime/ImeMsiArchiver.cs | 375 ++++++++++++++++++ .../Services/TableStorageService.Sessions.cs | 39 +- .../AutopilotMonitor.Shared/Constants.cs | 25 ++ .../DataAccess/ISessionRepository.cs | 8 + .../Models/Config/AdminConfiguration.cs | 18 + .../Models/ImeMsiArchiveEnvelope.cs | 44 ++ .../Models/SessionApiModels.cs | 20 + .../app/admin/AdminConfigContext.tsx | 16 +- .../components/AdminConfigSettingsSection.tsx | 49 +++ .../sections/SectionGlobalSettings.tsx | 6 + .../types/adminConfig.ts | 11 + .../utils/shared-manifests.generated.ts | 2 + .../utils/shared-manifests.json | 2 + 23 files changed, 1263 insertions(+), 4 deletions(-) create mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/AdminConfigurationImeMsiArchivingTests.cs create mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/ImeMsiArchiverTests.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/Ime/AzureQueueImeMsiArchiveProducer.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/Ime/IImeMsiArchiveProducer.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/Ime/ImeMsiArchiveQueueWorker.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/Ime/ImeMsiArchiver.cs create mode 100644 src/Shared/AutopilotMonitor.Shared/Models/ImeMsiArchiveEnvelope.cs diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/AdminConfigurationImeMsiArchivingTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/AdminConfigurationImeMsiArchivingTests.cs new file mode 100644 index 00000000..5a45fbe3 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/AdminConfigurationImeMsiArchivingTests.cs @@ -0,0 +1,44 @@ +using AutopilotMonitor.Shared.Models; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Default-value tests for the IME-installer archiving settings. The flag gates the +/// ime-msi-archive queue worker (pause, not drop); the size cap guards the download. +/// Defaulting the flag to TRUE is deliberate: archiving is operator infrastructure with +/// ~one small download per Microsoft IME release, and a version missed while the flag is +/// off is gone for good (the versionless CDN moves on). +/// +public class AdminConfigurationImeMsiArchivingTests +{ + [Fact] + public void ImeMsiArchivingEnabled_defaults_to_true_on_new_config() + { + var cfg = new AdminConfiguration(); + Assert.True(cfg.ImeMsiArchivingEnabled); + } + + [Fact] + public void ImeMsiArchivingEnabled_defaults_to_true_on_CreateDefault() + { + var cfg = AdminConfiguration.CreateDefault(); + Assert.True(cfg.ImeMsiArchivingEnabled); + } + + [Fact] + public void MaxImeMsiDownloadSizeMB_defaults_to_250() + { + // 250 MB: ~20x today's MSI so a legitimately grown installer is never missed, + // while a tampered URL still cannot stream gigabytes (Oliver, 2026-08-18). + Assert.Equal(250, new AdminConfiguration().MaxImeMsiDownloadSizeMB); + Assert.Equal(250, AdminConfiguration.CreateDefault().MaxImeMsiDownloadSizeMB); + } + + [Fact] + public void Settings_persist_on_the_config() + { + var cfg = new AdminConfiguration { ImeMsiArchivingEnabled = false, MaxImeMsiDownloadSizeMB = 50 }; + Assert.False(cfg.ImeMsiArchivingEnabled); + Assert.Equal(50, cfg.MaxImeMsiDownloadSizeMB); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/GetImeVersionHistoryRedactionTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/GetImeVersionHistoryRedactionTests.cs index a1e036a8..1922733a 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/GetImeVersionHistoryRedactionTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/GetImeVersionHistoryRedactionTests.cs @@ -39,6 +39,26 @@ public class GetImeVersionHistoryRedactionTests Assert.DoesNotContain("firstSeenSessionId", json, StringComparison.OrdinalIgnoreCase); } + /// + /// The MSI-archive columns (ImeMsiArchiver) are operator infrastructure — blob paths, + /// hashes and source URLs of the platform's own archive have no business in a tenant + /// member's payload. Pinned like the identifiers so the boundary survives refactors. + /// + [Fact] + public void NonGlobalCaller_DropsMsiArchiveFields() + { + var json = SerializeFor(hasGlobalScope: false, MakeEntry()); + + Assert.DoesNotContain("msiArchiveStatus", json, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("msiArchiveBlobPath", json, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("msiSha256", json, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("msiBytes", json, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("msiSourceUrl", json, StringComparison.OrdinalIgnoreCase); + // Values too, not just keys (same rationale as the identifier-value test below). + Assert.DoesNotContain("4d6b85fd3c49cf227ab4711308bca1f3c701cad9bff1799da4c6b6616066bb5e", json, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("imeswdb-afd-secondary", json, StringComparison.OrdinalIgnoreCase); + } + /// /// The values, not just the property names: a rename of the model property must not let the /// identifier survive under a different key. Asserting on the GUIDs catches that. @@ -101,7 +121,12 @@ public class GetImeVersionHistoryRedactionTests var keys = doc.RootElement[0].EnumerateObject().Select(p => p.Name).OrderBy(n => n).ToArray(); Assert.Equal( - new[] { "firstSeenAt", "firstSeenSessionId", "firstSeenTenantId", "lastSeenAt", "sessionCount", "version" }, + new[] + { + "firstSeenAt", "firstSeenSessionId", "firstSeenTenantId", "lastSeenAt", + "msiArchiveBlobPath", "msiArchiveStatus", "msiBytes", "msiSha256", "msiSourceUrl", + "sessionCount", "version", + }, keys); } @@ -185,5 +210,10 @@ public class GetImeVersionHistoryRedactionTests FirstSeenTenantId = "57f34dd1-6a42-47f9-9bc0-b921fa6caa30", LastSeenAt = Last, SessionCount = 3109, + MsiArchiveStatus = "Archived", + MsiArchiveBlobPath = $"{version}/IntuneWindowsAgent.msi", + MsiSha256 = "4d6b85fd3c49cf227ab4711308bca1f3c701cad9bff1799da4c6b6616066bb5e", + MsiBytes = 13_107_200, + MsiSourceUrl = "https://imeswdb-afd-secondary.manage.microsoft.com/IntuneWindowsAgent.msi", }; } diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/ImeMsiArchiverTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/ImeMsiArchiverTests.cs new file mode 100644 index 00000000..e6ec506c --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/ImeMsiArchiverTests.cs @@ -0,0 +1,335 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net.Http; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Services.Ime; +using AutopilotMonitor.Shared.Models; +using Azure; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// The IME-installer archiver downloads a newly sighted IME build into the permanent +/// ime-archive container. Its contract: NEVER throw (worker decides retry from the result), +/// treat the agent-supplied URL as untrusted (allowlist, else canonical fallback), enforce +/// the size cap both at Content-Length preflight and mid-stream, and stay idempotent under +/// queue re-delivery (409 on the write-once upload == archived). Tests pin every status +/// branch via a recording subclass — no Azurite, no HTTP. +/// +public class ImeMsiArchiverTests +{ + private const string Version = "1.104.102.0"; + private const string EventUrl = "https://imeswdb-afd-secondary.manage.microsoft.com/IntuneWindowsAgent.msi"; + + // ========================================================================= + // URL allowlist (static — the SSRF boundary) + // ========================================================================= + + [Theory] + [InlineData(EventUrl, true)] + [InlineData("https://swda01-mscdn.manage.microsoft.com/IntuneWindowsAgent.msi", true)] + [InlineData("https://manage.microsoft.com/IntuneWindowsAgent.msi", true)] // apex host + [InlineData("https://x.manage.microsoft.com/path/IntuneWindowsAgent.msi?sv=1&sig=2", true)] // query ignored + [InlineData("https://x.manage.microsoft.com/intunewindowsagent.MSI", true)] // case-insensitive + [InlineData("http://imeswdb-afd-secondary.manage.microsoft.com/IntuneWindowsAgent.msi", false)] // https only + [InlineData("https://evil.example.com/IntuneWindowsAgent.msi", false)] + [InlineData("https://manage.microsoft.com.evil.com/IntuneWindowsAgent.msi", false)] // suffix spoof + [InlineData("https://xmanage.microsoft.com/IntuneWindowsAgent.msi", false)] // needs the dot + [InlineData("https://x.manage.microsoft.com/SomethingElse.msi", false)] // wrong filename + [InlineData("https://x.manage.microsoft.com/IntuneWindowsAgent.msi.exe", false)] + [InlineData("not a url", false)] + [InlineData("", false)] + [InlineData(null, false)] + public void IsAllowedMsiUrl_cases(string? url, bool expected) + { + Assert.Equal(expected, ImeMsiArchiver.IsAllowedMsiUrl(url)); + } + + // ========================================================================= + // Version guard (blob path is built from untrusted input) + // ========================================================================= + + [Theory] + [InlineData("")] + [InlineData("1.104.102.0.1")] // too many components + [InlineData("1.104../escape")] + [InlineData("latest")] + [InlineData("1.104.102.0 ")] + public async Task ArchiveAsync_BadVersion_PermanentFailure_NoDownload(string version) + { + var archiver = new RecordingArchiver(); + + var result = await archiver.ArchiveAsync(Envelope(version: version)); + + Assert.False(result.Success); + Assert.Equal(ImeMsiArchiver.Statuses.FailedBadVersion, result.Status); + Assert.False(result.Retryable); + Assert.Equal(0, archiver.DownloadCalls + archiver.UploadCalls); + } + + // ========================================================================= + // URL selection + // ========================================================================= + + [Fact] + public async Task ArchiveAsync_AllowedEventUrl_IsUsed() + { + var archiver = new RecordingArchiver(); + + var result = await archiver.ArchiveAsync(Envelope(url: EventUrl)); + + Assert.True(result.Success); + Assert.Equal(EventUrl, archiver.DownloadLastUrl); + Assert.Equal(EventUrl, result.SourceUrl); + } + + [Theory] + [InlineData("https://evil.example.com/IntuneWindowsAgent.msi")] // allowlist reject + [InlineData(null)] // pre-enrichment agent + public async Task ArchiveAsync_MissingOrRejectedEventUrl_FallsBackToCanonical(string? url) + { + var archiver = new RecordingArchiver(); + + var result = await archiver.ArchiveAsync(Envelope(url: url)); + + Assert.True(result.Success); + Assert.Equal(ImeMsiArchiver.CanonicalMsiUrl, archiver.DownloadLastUrl); + Assert.Equal(ImeMsiArchiver.CanonicalMsiUrl, result.SourceUrl); + } + + // ========================================================================= + // Happy path + // ========================================================================= + + [Fact] + public async Task ArchiveAsync_HappyPath_UploadsMsiAndProvenance_WithHashAndSize() + { + var payload = new byte[3 * 1024 * 1024]; + new Random(42).NextBytes(payload); + var expectedSha = Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant(); + var archiver = new RecordingArchiver { Payload = payload }; + + var result = await archiver.ArchiveAsync(Envelope()); + + Assert.True(result.Success); + Assert.Equal(ImeMsiArchiver.Statuses.Archived, result.Status); + Assert.Equal($"{Version}/IntuneWindowsAgent.msi", result.BlobPath); + Assert.Equal(expectedSha, result.Sha256); + Assert.Equal(payload.Length, result.SizeBytes); + Assert.Equal(1, archiver.UploadCalls); + Assert.Equal($"{Version}/IntuneWindowsAgent.msi", archiver.UploadLastPath); + Assert.Equal(payload.Length, archiver.UploadedByteCount); + Assert.Equal(1, archiver.ProvenanceCalls); + Assert.Equal($"{Version}/provenance.json", archiver.ProvenanceLastPath); + Assert.Contains(expectedSha, archiver.ProvenanceLastJson); + Assert.Contains(Version, archiver.ProvenanceLastJson); + } + + // ========================================================================= + // Size cap — preflight and mid-stream + // ========================================================================= + + [Fact] + public async Task ArchiveAsync_ContentLengthAboveCap_Rejected_NoUpload() + { + var archiver = new RecordingArchiver + { + MaxDownloadSizeMB = 1, + Payload = new byte[2 * 1024 * 1024], + }; + + var result = await archiver.ArchiveAsync(Envelope()); + + Assert.False(result.Success); + Assert.Equal(ImeMsiArchiver.Statuses.FailedTooLarge, result.Status); + Assert.False(result.Retryable); + Assert.Equal(0, archiver.UploadCalls); + } + + [Fact] + public async Task ArchiveAsync_UnknownContentLength_CapStillEnforcedMidStream() + { + // A lying/absent Content-Length must not bypass the cap — the hashing wrapper + // aborts the stream as soon as the byte count crosses it. + var archiver = new RecordingArchiver + { + MaxDownloadSizeMB = 1, + Payload = new byte[2 * 1024 * 1024], + ReportContentLength = false, + }; + + var result = await archiver.ArchiveAsync(Envelope()); + + Assert.False(result.Success); + Assert.Equal(ImeMsiArchiver.Statuses.FailedTooLarge, result.Status); + Assert.False(result.Retryable); + Assert.Equal(0, archiver.ProvenanceCalls); + } + + [Fact] + public async Task ArchiveAsync_CapZero_MeansUnlimited() + { + var archiver = new RecordingArchiver + { + MaxDownloadSizeMB = 0, + Payload = new byte[2 * 1024 * 1024], + }; + + var result = await archiver.ArchiveAsync(Envelope()); + + Assert.True(result.Success); + } + + // ========================================================================= + // Idempotency under queue re-delivery + // ========================================================================= + + [Fact] + public async Task ArchiveAsync_BlobAlreadyExists_TreatedAsArchived_NotRetryable() + { + var archiver = new RecordingArchiver + { + UploadException = new RequestFailedException(409, "BlobAlreadyExists"), + }; + + var result = await archiver.ArchiveAsync(Envelope()); + + Assert.True(result.Success); + Assert.Equal(ImeMsiArchiver.Statuses.Archived, result.Status); + Assert.False(result.Retryable); + Assert.Equal($"{Version}/IntuneWindowsAgent.msi", result.BlobPath); + } + + // ========================================================================= + // Transient failures — retryable, never thrown + // ========================================================================= + + [Fact] + public async Task ArchiveAsync_DownloadFails_RetryableDownloadStatus() + { + var archiver = new RecordingArchiver + { + DownloadException = new HttpRequestException("boom"), + }; + + var result = await archiver.ArchiveAsync(Envelope()); + + Assert.False(result.Success); + Assert.Equal(ImeMsiArchiver.Statuses.FailedDownload, result.Status); + Assert.True(result.Retryable); + } + + [Fact] + public async Task ArchiveAsync_Timeout_RetryableTimeoutStatus() + { + var archiver = new RecordingArchiver + { + DownloadException = new OperationCanceledException(), + }; + + var result = await archiver.ArchiveAsync(Envelope()); + + Assert.False(result.Success); + Assert.Equal(ImeMsiArchiver.Statuses.FailedTimeout, result.Status); + Assert.True(result.Retryable); + } + + [Fact] + public async Task ArchiveAsync_UnexpectedUploadError_RetryableErrorStatus() + { + var archiver = new RecordingArchiver + { + UploadException = new InvalidOperationException("storage hiccup"), + }; + + var result = await archiver.ArchiveAsync(Envelope()); + + Assert.False(result.Success); + Assert.Equal(ImeMsiArchiver.Statuses.FailedError, result.Status); + Assert.True(result.Retryable); + } + + [Fact] + public async Task ArchiveAsync_CallerCancellation_Propagates() + { + // Host shutdown must reach the worker's dispatch loop so the message stays queued. + var archiver = new RecordingArchiver(); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + archiver.DownloadException = new OperationCanceledException(cts.Token); + + await Assert.ThrowsAnyAsync( + () => archiver.ArchiveAsync(Envelope(), cts.Token)); + } + + // ========================================================================= + // Helpers + // ========================================================================= + + private static ImeMsiArchiveEnvelope Envelope(string version = Version, string? url = EventUrl) => + new() + { + Version = version, + MsiDownloadUrl = url, + MsiMatchedBy = "productVersion", + TenantId = "11111111-1111-1111-1111-111111111111", + SessionId = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", + EnqueuedAt = DateTime.UtcNow, + }; + + private sealed class RecordingArchiver : ImeMsiArchiver + { + public byte[] Payload { get; set; } = new byte[2048]; + public bool ReportContentLength { get; set; } = true; + public int MaxDownloadSizeMB { get; set; } = 250; + public Exception? DownloadException { get; set; } + public Exception? UploadException { get; set; } + + public int DownloadCalls { get; private set; } + public string? DownloadLastUrl { get; private set; } + public int UploadCalls { get; private set; } + public string? UploadLastPath { get; private set; } + public long UploadedByteCount { get; private set; } + public int ProvenanceCalls { get; private set; } + public string? ProvenanceLastPath { get; private set; } + public string ProvenanceLastJson { get; private set; } = string.Empty; + + protected override Task GetAdminConfigurationAsync() + => Task.FromResult(new AdminConfiguration { MaxImeMsiDownloadSizeMB = MaxDownloadSizeMB }); + + protected override Task<(Stream Content, long ContentLength)> OpenDownloadAsync( + string url, CancellationToken cancellationToken) + { + DownloadCalls++; + DownloadLastUrl = url; + if (DownloadException is not null) throw DownloadException; + return Task.FromResult( + ((Stream)new MemoryStream(Payload), ReportContentLength ? Payload.LongLength : -1L)); + } + + protected override async Task UploadMsiAsync(string blobPath, Stream content, CancellationToken cancellationToken) + { + UploadCalls++; + UploadLastPath = blobPath; + // Drain the stream like the real blob upload would — this is what drives the + // hashing/cap wrapper (and mid-stream aborts surface here as exceptions). + var buffer = new byte[64 * 1024]; + int read; + while ((read = await content.ReadAsync(buffer, cancellationToken)) > 0) + UploadedByteCount += read; + if (UploadException is not null) throw UploadException; + } + + protected override Task UploadProvenanceAsync(string blobPath, string json, CancellationToken cancellationToken) + { + ProvenanceCalls++; + ProvenanceLastPath = blobPath; + ProvenanceLastJson = json; + return Task.CompletedTask; + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableConfigRepository.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableConfigRepository.cs index 1a4707bc..5f14f9f3 100644 --- a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableConfigRepository.cs +++ b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableConfigRepository.cs @@ -818,6 +818,8 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage { "VulnerabilityCorrelationEnabled", config.VulnerabilityCorrelationEnabled }, { "EnableIndexDualWrite", config.EnableIndexDualWrite }, { "AutoApproveNewTenants", config.AutoApproveNewTenants }, + { "ImeMsiArchivingEnabled", config.ImeMsiArchivingEnabled }, + { "MaxImeMsiDownloadSizeMB", config.MaxImeMsiDownloadSizeMB }, { "SelfServiceAppHomingEnabled", config.SelfServiceAppHomingEnabled }, { "SessionDeletionKillSwitch", config.SessionDeletionKillSwitch }, { "VulnerabilityDataLastSyncUtc", config.VulnerabilityDataLastSyncUtc }, @@ -885,6 +887,8 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage VulnerabilityCorrelationEnabled = entity.GetBoolean("VulnerabilityCorrelationEnabled") ?? true, EnableIndexDualWrite = entity.GetBoolean("EnableIndexDualWrite") ?? false, AutoApproveNewTenants = entity.GetBoolean("AutoApproveNewTenants") ?? false, + ImeMsiArchivingEnabled = entity.GetBoolean("ImeMsiArchivingEnabled") ?? true, + MaxImeMsiDownloadSizeMB = entity.GetInt32("MaxImeMsiDownloadSizeMB") ?? 250, SelfServiceAppHomingEnabled = entity.GetBoolean("SelfServiceAppHomingEnabled") ?? false, SessionDeletionKillSwitch = entity.GetBoolean("SessionDeletionKillSwitch") ?? false, VulnerabilityDataLastSyncUtc = entity.GetString("VulnerabilityDataLastSyncUtc"), diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSessionRepository.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSessionRepository.cs index ec0e6bd7..d2deb79d 100644 --- a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSessionRepository.cs +++ b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSessionRepository.cs @@ -172,6 +172,10 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage public Task> GetImeVersionHistoryAsync() => _storage.GetImeVersionHistoryAsync(); + public Task UpdateImeVersionArchiveInfoAsync( + string version, string status, string? blobPath, string? sha256, long? sizeBytes, string? sourceUrl) + => _storage.UpdateImeVersionArchiveInfoAsync(version, status, blobPath, sha256, sizeBytes, sourceUrl); + public async Task StoreEventAsync(EnrollmentEvent evt) { var result = await _storage.StoreEventAsync(evt); diff --git a/src/Backend/AutopilotMonitor.Functions/Program.cs b/src/Backend/AutopilotMonitor.Functions/Program.cs index 24fda8bb..81c6283f 100644 --- a/src/Backend/AutopilotMonitor.Functions/Program.cs +++ b/src/Backend/AutopilotMonitor.Functions/Program.cs @@ -334,6 +334,26 @@ builder.Services.AddSingleton< builder.Services.AddHostedService< AutopilotMonitor.Functions.Services.Vulnerability.VulnerabilityCorrelateQueueWorker>(); +// IME-installer archiving on first fleet-wide sighting of a new IME version (~monthly). +// Producer sits in EventIngestProcessor's new-version continuation; the worker downloads +// the MSI from the CSP-reported (allowlisted) URL into the permanent ime-archive container +// and merges the outcome onto the ImeVersionHistory row. Typed HttpClient: no redirects +// (SSRF posture — the allowlisted host must serve directly), long timeout owned by the +// archiver's own CTS, transient retry via the shared external-data policy. +builder.Services.AddHttpClient() + .ConfigurePrimaryHttpMessageHandler(() => new System.Net.Http.SocketsHttpHandler + { + AllowAutoRedirect = false, + ConnectTimeout = TimeSpan.FromSeconds(10), + }) + .ConfigureHttpClient(client => client.Timeout = Timeout.InfiniteTimeSpan) + .AddPolicyHandler((sp, _) => sp.GetRequiredService().ExternalDataApi); +builder.Services.AddSingleton< + AutopilotMonitor.Functions.Services.Ime.IImeMsiArchiveProducer, + AutopilotMonitor.Functions.Services.Ime.AzureQueueImeMsiArchiveProducer>(); +builder.Services.AddHostedService< + AutopilotMonitor.Functions.Services.Ime.ImeMsiArchiveQueueWorker>(); + // Delayed tenant auto-approve (public availability). Producer fires at first-login signup // with a ~1-minute visibility delay; the worker activates the tenant via the shared // TenantApprovalService only if AdminConfiguration.AutoApproveNewTenants is enabled at diff --git a/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.cs b/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.cs index 7033eb63..bb41f01a 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.cs @@ -43,6 +43,7 @@ namespace AutopilotMonitor.Functions.Services private readonly AutopilotMonitor.Functions.Services.Analyze.IAnalyzeOnEnrollmentEndProducer _analyzeProducer; private readonly AutopilotMonitor.Functions.Services.Analyze.InterimTriggerRegistry _interimTriggerRegistry; private readonly IVulnerabilityCorrelateProducer _vulnProducer; + private readonly Ime.IImeMsiArchiveProducer _imeMsiArchiveProducer; public EventIngestProcessor( ILogger logger, @@ -58,7 +59,8 @@ namespace AutopilotMonitor.Functions.Services TelemetryClient telemetryClient, AutopilotMonitor.Functions.Services.Analyze.IAnalyzeOnEnrollmentEndProducer analyzeProducer, AutopilotMonitor.Functions.Services.Analyze.InterimTriggerRegistry interimTriggerRegistry, - IVulnerabilityCorrelateProducer vulnProducer) + IVulnerabilityCorrelateProducer vulnProducer, + Ime.IImeMsiArchiveProducer imeMsiArchiveProducer) { _logger = logger; _sessionRepo = sessionRepo; @@ -74,6 +76,7 @@ namespace AutopilotMonitor.Functions.Services _analyzeProducer = analyzeProducer; _interimTriggerRegistry = interimTriggerRegistry; _vulnProducer = vulnProducer; + _imeMsiArchiveProducer = imeMsiArchiveProducer; } /// @@ -129,6 +132,14 @@ namespace AutopilotMonitor.Functions.Services .ContinueWith(t => _logger.LogWarning(t.Exception?.InnerException, "ImeAgentVersion update failed (non-fatal)"), TaskContinuationOptions.OnlyOnFaulted); + // Captured here (not inside the continuation) purely for clarity — the agent's + // CSP-registry enrichment; absent on agent builds before 2026-08-17. + string? msiDownloadUrl = null, msiMatchedBy = null; + if (imeVersionEvent.Data.TryGetValue("msiDownloadUrl", out var urlObj)) + msiDownloadUrl = urlObj?.ToString(); + if (imeVersionEvent.Data.TryGetValue("msiMatchedBy", out var matchedByObj)) + msiMatchedBy = matchedByObj?.ToString(); + _ = _sessionRepo.RecordImeVersionAsync(imeVersion, request.TenantId, request.SessionId) .ContinueWith(async t => { @@ -141,6 +152,19 @@ namespace AutopilotMonitor.Functions.Services { await _opsEventService.RecordNewImeVersionDetectedAsync( imeVersion, request.TenantId, request.SessionId); + + // First fleet-wide sighting → archive the installer binary while + // Microsoft's versionless CDN still serves exactly this build. + // Producer is fail-soft; worker pauses on ImeMsiArchivingEnabled=false. + await _imeMsiArchiveProducer.EnqueueAsync(new ImeMsiArchiveEnvelope + { + Version = imeVersion, + MsiDownloadUrl = msiDownloadUrl, + MsiMatchedBy = msiMatchedBy, + TenantId = request.TenantId, + SessionId = request.SessionId, + EnqueuedAt = DateTime.UtcNow, + }); } }, TaskScheduler.Default); } diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Ime/AzureQueueImeMsiArchiveProducer.cs b/src/Backend/AutopilotMonitor.Functions/Services/Ime/AzureQueueImeMsiArchiveProducer.cs new file mode 100644 index 00000000..8f422d98 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Ime/AzureQueueImeMsiArchiveProducer.cs @@ -0,0 +1,84 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Azure.Storage.Queues; +using AutopilotMonitor.Functions.Services.Queueing; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.Models; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +namespace AutopilotMonitor.Functions.Services.Ime +{ + /// + /// Azure Storage Queue implementation of . Mirrors + /// : Managed Identity + /// preferred, connection-string fallback, Base64 encoding (matches the consumer worker). + /// + /// Fault tolerance: queue-side exceptions are logged but never rethrown — the + /// caller is the ingest path's fire-and-forget continuation. A missed enqueue means the + /// version's installer is not auto-archived; the /ime-decompile skill's blob backfill + /// is the manual fallback. + /// + /// + public sealed class AzureQueueImeMsiArchiveProducer : IImeMsiArchiveProducer + { + private readonly QueueClient _queueClient; + private readonly ILogger _logger; + + private int _queueEnsured; // 0 = not yet ensured, 1 = CreateIfNotExistsAsync has run + + public AzureQueueImeMsiArchiveProducer( + QueueClientFactory queueFactory, + ILogger logger) + { + _logger = logger; + // Base64 encoding matches the consumer (BackgroundService) so messages round-trip. + _queueClient = queueFactory.Create(Constants.QueueNames.ImeMsiArchive); + } + + public async Task EnqueueAsync(ImeMsiArchiveEnvelope envelope, CancellationToken cancellationToken = default) + { + if (envelope is null) return; + if (string.IsNullOrEmpty(envelope.Version)) + { + _logger.LogWarning("ImeMsiArchive enqueue skipped — missing Version"); + return; + } + + await EnsureQueueExistsAsync(cancellationToken).ConfigureAwait(false); + + try + { + var body = JsonConvert.SerializeObject(envelope); + await _queueClient.SendMessageAsync(body, cancellationToken: cancellationToken).ConfigureAwait(false); + _logger.LogInformation( + "ImeMsiArchive enqueued (version={Version} urlPresent={UrlPresent} tenant={Tenant} session={Session})", + envelope.Version, !string.IsNullOrEmpty(envelope.MsiDownloadUrl), envelope.TenantId, envelope.SessionId); + } + catch (Exception ex) + { + // Fail-soft — ingest continuation. Skill-side blob backfill is the fallback. + _logger.LogWarning(ex, + "ImeMsiArchive enqueue failed (version={Version}) — /ime-decompile blob backfill is the fallback", + envelope.Version); + } + } + + private async Task EnsureQueueExistsAsync(CancellationToken cancellationToken) + { + if (_queueEnsured == 1) return; + + try + { + await _queueClient.CreateIfNotExistsAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + Interlocked.Exchange(ref _queueEnsured, 1); + } + catch (Exception ex) + { + // Leave _queueEnsured=0 so a later call retries. + _logger.LogWarning(ex, "ImeMsiArchive queue CreateIfNotExists failed — will retry next enqueue"); + } + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Ime/IImeMsiArchiveProducer.cs b/src/Backend/AutopilotMonitor.Functions/Services/Ime/IImeMsiArchiveProducer.cs new file mode 100644 index 00000000..d349fe66 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Ime/IImeMsiArchiveProducer.cs @@ -0,0 +1,16 @@ +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Shared.Models; + +namespace AutopilotMonitor.Functions.Services.Ime +{ + /// + /// Enqueues an IME-installer archive job when a new IME version is first sighted. + /// Implementations must be fail-soft: the caller sits on the ingest hot path's + /// fire-and-forget continuation and must never observe an exception. + /// + public interface IImeMsiArchiveProducer + { + Task EnqueueAsync(ImeMsiArchiveEnvelope envelope, CancellationToken cancellationToken = default); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Ime/ImeMsiArchiveQueueWorker.cs b/src/Backend/AutopilotMonitor.Functions/Services/Ime/ImeMsiArchiveQueueWorker.cs new file mode 100644 index 00000000..9508adad --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Ime/ImeMsiArchiveQueueWorker.cs @@ -0,0 +1,87 @@ +using AutopilotMonitor.Functions.Services.Queueing; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Services.Ime +{ + /// + /// Background worker for the ime-msi-archive queue: downloads a newly sighted IME + /// installer into the permanent ime-archive container and merges the outcome onto + /// the ImeVersionHistory row. Volume is ~one message per Microsoft IME release. + /// + /// Retry semantics: the archiver never throws — it classifies. Permanent rejections + /// (bad version, size cap) complete the message with their Failed:* status + /// persisted; transient failures (download/timeout) rethrow so the visibility-timeout + /// retry → poison ladder applies, with the last status visible on the row. + /// keeps the message invisible across the up-to-10-minute + /// download window. The status row-merge happens on EVERY attempt so operators can see + /// a stuck download as Failed:Download instead of nothing. + /// + /// + public sealed class ImeMsiArchiveQueueWorker : QueuePollingWorker + { + private readonly ImeMsiArchiver _archiver; + private readonly ISessionRepository _sessionRepo; + private readonly AdminConfigurationService _adminConfigService; + + public ImeMsiArchiveQueueWorker( + QueueClientFactory queueFactory, + ImeMsiArchiver archiver, + ISessionRepository sessionRepo, + AdminConfigurationService adminConfigService, + ILogger logger) + : base(queueFactory, Constants.QueueNames.ImeMsiArchive, logger) + { + _archiver = archiver ?? throw new ArgumentNullException(nameof(archiver)); + _sessionRepo = sessionRepo ?? throw new ArgumentNullException(nameof(sessionRepo)); + _adminConfigService = adminConfigService ?? throw new ArgumentNullException(nameof(adminConfigService)); + } + + /// Download can legitimately take minutes — extend visibility while working. + protected override bool UseHeartbeat => true; + + /// + /// Paused while is off — + /// messages stay parked (not dropped) and are processed when the flag returns. + /// Uses the cached config read: this is a pause, not a kill-switch, and the worker + /// only wakes for rare messages anyway. + /// + protected override async ValueTask ShouldPauseAsync(CancellationToken ct) + { + try + { + var config = await _adminConfigService.GetConfigurationAsync(); + return !config.ImeMsiArchivingEnabled; + } + catch + { + return false; // config unreadable → keep working (archive is fail-soft anyway) + } + } + + protected override bool TryValidate(ImeMsiArchiveEnvelope envelope) + => !string.IsNullOrEmpty(envelope.Version); + + protected override string DescribeForLog(ImeMsiArchiveEnvelope envelope) + => $"version {envelope.Version}"; + + protected override async Task HandleAsync(ImeMsiArchiveEnvelope envelope, CancellationToken ct) + { + var result = await _archiver.ArchiveAsync(envelope, ct); + + // Merge the outcome first — also for failures, so the row always tells the truth. + await _sessionRepo.UpdateImeVersionArchiveInfoAsync( + envelope.Version, result.Status, result.BlobPath, result.Sha256, result.SizeBytes, result.SourceUrl); + + if (!result.Success && result.Retryable) + { + // Visibility-timeout retry → poison after MaxDequeueCount. The thrown message + // lands in the worker's handler-failure log line. + throw new InvalidOperationException( + $"IME MSI archive attempt failed with {result.Status} for version {envelope.Version}"); + } + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Ime/ImeMsiArchiver.cs b/src/Backend/AutopilotMonitor.Functions/Services/Ime/ImeMsiArchiver.cs new file mode 100644 index 00000000..922c2629 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Ime/ImeMsiArchiver.cs @@ -0,0 +1,375 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.Models; +using Azure; +using Azure.Storage.Blobs.Models; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace AutopilotMonitor.Functions.Services.Ime +{ + /// + /// Outcome of an IME-installer archive attempt. is one of the + /// constants and is merged verbatim onto the + /// ImeVersionHistory row. tells the queue worker whether to + /// trigger a visibility-timeout retry (transient download problems) or complete the + /// message (permanent rejections). + /// + public sealed record ImeMsiArchiveResult( + bool Success, string Status, bool Retryable, + string? BlobPath, string? Sha256, long? SizeBytes, string? SourceUrl); + + /// + /// Downloads a newly sighted IME installer and archives it into the permanent + /// container + /// ({version}/IntuneWindowsAgent.msi + {version}/provenance.json). + /// Modeled on : + /// fail-soft by contract (never throws), Statuses constants persisted verbatim, + /// admin-configured size cap, typed catch ladder, protected virtual test seams. + /// + /// The download URL comes from agent telemetry and is therefore untrusted: only HTTPS + /// URLs on *.manage.microsoft.com whose path filename is exactly + /// IntuneWindowsAgent.msi are accepted; anything else falls back to the canonical + /// versionless CDN URL — safe for a NEW version, because "new" is by definition what + /// that URL currently serves. The blob write uses If-None-Match:* so a queue re-delivery + /// can never overwrite an existing archive entry. + /// + /// + public class ImeMsiArchiver + { + /// Persisted MsiArchiveStatus values. + public static class Statuses + { + public const string Archived = "Archived"; + public const string FailedBadVersion = "Failed:BadVersion"; + public const string FailedTooLarge = "Failed:TooLarge"; + public const string FailedDownload = "Failed:Download"; + public const string FailedTimeout = "Failed:Timeout"; + public const string FailedError = "Failed:Error"; + } + + /// + /// The CSP's versionless distribution endpoint — always serves the currently rolled-out + /// build. Fallback when the event carried no (acceptable) URL. + /// + public const string CanonicalMsiUrl = "https://imeswdb-afd-secondary.manage.microsoft.com/IntuneWindowsAgent.msi"; + + private const string MsiFileName = "IntuneWindowsAgent.msi"; + private const string AllowedHostSuffix = ".manage.microsoft.com"; + + /// Hard ceiling for one download attempt; the worker's heartbeat covers it. + private static readonly TimeSpan DownloadTimeout = TimeSpan.FromMinutes(10); + + /// Blob folder name = version — must be plain dotted digits (untrusted input). + internal static readonly Regex VersionRegex = new(@"^\d+(\.\d+){1,3}$", RegexOptions.Compiled); + + private readonly HttpClient _httpClient; + private readonly BlobStorageService _blobStorage; + private readonly AdminConfigurationService _adminConfigService; + private readonly ILogger _logger; + + public ImeMsiArchiver( + HttpClient httpClient, + BlobStorageService blobStorage, + AdminConfigurationService adminConfigService, + ILogger logger) + { + _httpClient = httpClient; + _blobStorage = blobStorage; + _adminConfigService = adminConfigService; + _logger = logger; + } + + /// + /// Test seam: Azure-free construction for recording subclasses that override the + /// virtual download/upload/config seams below. + /// + protected ImeMsiArchiver() + { + _httpClient = null!; + _blobStorage = null!; + _adminConfigService = null!; + _logger = NullLogger.Instance; + } + + /// + /// True when is an HTTPS URL on *.manage.microsoft.com + /// whose path filename is exactly IntuneWindowsAgent.msi (query ignored). + /// + public static bool IsAllowedMsiUrl(string? url) + { + if (string.IsNullOrEmpty(url)) return false; + if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)) return false; + if (uri.Scheme != Uri.UriSchemeHttps) return false; + + var host = uri.Host; + var hostAllowed = + host.EndsWith(AllowedHostSuffix, StringComparison.OrdinalIgnoreCase) || + host.Equals(AllowedHostSuffix.TrimStart('.'), StringComparison.OrdinalIgnoreCase); + if (!hostAllowed) return false; + + var fileName = uri.Segments.Length > 0 ? uri.Segments[^1] : string.Empty; + return fileName.Equals(MsiFileName, StringComparison.OrdinalIgnoreCase); + } + + public virtual async Task ArchiveAsync( + ImeMsiArchiveEnvelope envelope, CancellationToken cancellationToken = default) + { + if (envelope is null || !VersionRegex.IsMatch(envelope.Version ?? string.Empty)) + { + _logger.LogWarning( + "ImeMsiArchiver: rejecting version {Version} — not a plain dotted version string", + envelope?.Version ?? "(null)"); + return new ImeMsiArchiveResult(false, Statuses.FailedBadVersion, Retryable: false, + null, null, null, null); + } + + var version = envelope.Version!; + var blobPath = $"{version}/{MsiFileName}"; + + string sourceUrl; + if (IsAllowedMsiUrl(envelope.MsiDownloadUrl)) + { + sourceUrl = envelope.MsiDownloadUrl!; + } + else + { + if (!string.IsNullOrEmpty(envelope.MsiDownloadUrl)) + { + _logger.LogWarning( + "ImeMsiArchiver: event URL rejected by allowlist for version {Version} — using canonical URL", + version); + } + sourceUrl = CanonicalMsiUrl; + } + + try + { + var adminConfig = await GetAdminConfigurationAsync(); + var maxSizeBytes = (long)adminConfig.MaxImeMsiDownloadSizeMB * 1024 * 1024; + + using var cts = new CancellationTokenSource(DownloadTimeout); + using var linked = CancellationTokenSource.CreateLinkedTokenSource(cts.Token, cancellationToken); + + var (content, contentLength) = await OpenDownloadAsync(sourceUrl, linked.Token); + + string sha256; + long totalBytes; + using (content) + { + // Same fast-reject as the diagnostics proxy: cap of 0 means unlimited. + if (maxSizeBytes > 0 && contentLength > maxSizeBytes) + { + _logger.LogWarning( + "ImeMsiArchiver: download for {Version} rejected — Content-Length {SizeBytes} exceeds limit {MaxSizeBytes}", + version, contentLength, maxSizeBytes); + return new ImeMsiArchiveResult(false, Statuses.FailedTooLarge, Retryable: false, + null, null, contentLength, sourceUrl); + } + + // Hash + count while streaming into the blob; the wrapper enforces the cap + // mid-stream too (a lying/absent Content-Length cannot bypass it). + using var guarded = new HashingCappedReadStream(content, maxSizeBytes); + try + { + await UploadMsiAsync(blobPath, guarded, linked.Token); + } + catch (RequestFailedException ex) when (ex.Status == 409) + { + // Queue re-delivery after a successful upload: the archive already has + // this version — that IS the desired end state. + _logger.LogInformation( + "ImeMsiArchiver: blob {BlobPath} already exists — treating re-delivery as archived", + blobPath); + return new ImeMsiArchiveResult(true, Statuses.Archived, Retryable: false, + blobPath, null, null, sourceUrl); + } + sha256 = guarded.Sha256Hex; + totalBytes = guarded.TotalBytes; + } + + var provenance = new + { + version, + url = sourceUrl, + urlFromEvent = sourceUrl != CanonicalMsiUrl, + msiMatchedBy = envelope.MsiMatchedBy, + sha256, + msiBytes = totalBytes, + firstSeenTenantId = envelope.TenantId, + firstSeenSessionId = envelope.SessionId, + archivedAtUtc = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ"), + }; + await UploadProvenanceAsync( + $"{version}/provenance.json", + System.Text.Json.JsonSerializer.Serialize(provenance, + new System.Text.Json.JsonSerializerOptions { WriteIndented = true }), + linked.Token); + + _logger.LogInformation( + "ImeMsiArchiver: archived {Version} ({SizeBytes} bytes, sha256 {Sha256}) from {SourceUrl}", + version, totalBytes, sha256, sourceUrl); + return new ImeMsiArchiveResult(true, Statuses.Archived, Retryable: false, + blobPath, sha256, totalBytes, sourceUrl); + } + catch (SizeCapExceededException) + { + _logger.LogWarning( + "ImeMsiArchiver: download for {Version} exceeded the size cap mid-stream — aborted", + version); + return new ImeMsiArchiveResult(false, Statuses.FailedTooLarge, Retryable: false, + null, null, null, sourceUrl); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + _logger.LogWarning("ImeMsiArchiver: download for {Version} timed out", version); + return new ImeMsiArchiveResult(false, Statuses.FailedTimeout, Retryable: true, + null, null, null, sourceUrl); + } + catch (HttpRequestException ex) + { + _logger.LogWarning(ex, "ImeMsiArchiver: download for {Version} failed", version); + return new ImeMsiArchiveResult(false, Statuses.FailedDownload, Retryable: true, + null, null, null, sourceUrl); + } + catch (OperationCanceledException) + { + // Host shutdown — let the worker's dispatch loop see it and leave the message. + throw; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "ImeMsiArchiver: archiving {Version} failed", version); + return new ImeMsiArchiveResult(false, Statuses.FailedError, Retryable: true, + null, null, null, sourceUrl); + } + } + + // -------- Test seams (production defaults hit Azure / the network) -------- + + /// Fetches the platform admin configuration (size cap). + protected virtual Task GetAdminConfigurationAsync() + => _adminConfigService.GetConfigurationAsync(); + + /// Opens the installer download stream (headers-read, no buffering). + protected virtual async Task<(Stream Content, long ContentLength)> OpenDownloadAsync( + string url, CancellationToken cancellationToken) + { + var response = await _httpClient.GetAsync( + url, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + response.EnsureSuccessStatusCode(); + var content = await response.Content.ReadAsStreamAsync(cancellationToken); + return (content, response.Content.Headers.ContentLength ?? -1); + } + + /// + /// Uploads the installer stream write-once (If-None-Match:*). Implementations must + /// surface a conflict as with status 409. + /// + protected virtual async Task UploadMsiAsync(string blobPath, Stream content, CancellationToken cancellationToken) + { + var containerClient = _blobStorage.GetContainerClient(Constants.BlobContainers.ImeArchive); + await containerClient.CreateIfNotExistsAsync(cancellationToken: cancellationToken); + var blobClient = containerClient.GetBlobClient(blobPath); + await blobClient.UploadAsync(content, new BlobUploadOptions + { + HttpHeaders = new BlobHttpHeaders { ContentType = "application/octet-stream" }, + Conditions = new BlobRequestConditions { IfNoneMatch = ETag.All } + }, cancellationToken); + } + + /// Uploads the provenance sidecar (overwrite allowed — it is derived data). + protected virtual async Task UploadProvenanceAsync(string blobPath, string json, CancellationToken cancellationToken) + { + var containerClient = _blobStorage.GetContainerClient(Constants.BlobContainers.ImeArchive); + var blobClient = containerClient.GetBlobClient(blobPath); + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json)); + await blobClient.UploadAsync(stream, new BlobUploadOptions + { + HttpHeaders = new BlobHttpHeaders { ContentType = "application/json" } + }, cancellationToken); + } + + // -------- Streaming helpers -------- + + private sealed class SizeCapExceededException : Exception + { + } + + /// + /// Read-only pass-through stream that SHA-256-hashes and counts everything read and + /// throws once the byte count exceeds the cap (0 = unlimited). Non-seekable on + /// purpose — the blob SDK then stages blocks instead of demanding Length upfront. + /// + private sealed class HashingCappedReadStream : Stream + { + private readonly Stream _inner; + private readonly long _maxBytes; + private readonly IncrementalHash _hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + private string? _sha256Hex; + + public HashingCappedReadStream(Stream inner, long maxBytes) + { + _inner = inner; + _maxBytes = maxBytes; + } + + public long TotalBytes { get; private set; } + + public string Sha256Hex => _sha256Hex ??= + Convert.ToHexString(_hash.GetHashAndReset()).ToLowerInvariant(); + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position + { + get => TotalBytes; + set => throw new NotSupportedException(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + var read = _inner.Read(buffer, offset, count); + Account(buffer.AsSpan(offset, read)); + return read; + } + + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + var read = await _inner.ReadAsync(buffer, cancellationToken); + Account(buffer.Span[..read]); + return read; + } + + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => ReadAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); + + private void Account(ReadOnlySpan data) + { + if (data.IsEmpty) return; + TotalBytes += data.Length; + if (_maxBytes > 0 && TotalBytes > _maxBytes) + throw new SizeCapExceededException(); + _hash.AppendData(data); + } + + public override void Flush() + { + } + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (disposing) _hash.Dispose(); + base.Dispose(disposing); + } + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs index 50a506e5..16cb3e2b 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs @@ -3316,7 +3316,12 @@ namespace AutopilotMonitor.Functions.Services FirstSeenSessionId = entity.GetString("FirstSeenSessionId") ?? string.Empty, FirstSeenTenantId = entity.GetString("FirstSeenTenantId") ?? string.Empty, LastSeenAt = entity.GetDateTime("LastSeenAt") ?? DateTime.MinValue, - SessionCount = entity.GetInt32("SessionCount") ?? 0 + SessionCount = entity.GetInt32("SessionCount") ?? 0, + MsiArchiveStatus = entity.GetString("MsiArchiveStatus"), + MsiArchiveBlobPath = entity.GetString("MsiArchiveBlobPath"), + MsiSha256 = entity.GetString("MsiSha256"), + MsiBytes = entity.GetInt64("MsiBytes"), + MsiSourceUrl = entity.GetString("MsiSourceUrl") }); } } @@ -3328,6 +3333,38 @@ namespace AutopilotMonitor.Functions.Services return results.OrderByDescending(e => e.FirstSeenAt).ToList(); } + /// + /// Merges the installer-archiving outcome into the version's ImeVersionHistory row + /// (same Merge idiom as the LastSeen update above). Fail-soft: the archive job + /// already logged its own result; a lost status column must never fail the worker. + /// + public async Task UpdateImeVersionArchiveInfoAsync( + string version, string status, string? blobPath, string? sha256, long? sizeBytes, string? sourceUrl) + { + if (string.IsNullOrEmpty(version) || string.IsNullOrEmpty(status)) return; + + try + { + var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.ImeVersionHistory); + var mergeEntity = new TableEntity("Global", version) + { + ["MsiArchiveStatus"] = status + }; + if (!string.IsNullOrEmpty(blobPath)) mergeEntity["MsiArchiveBlobPath"] = blobPath; + if (!string.IsNullOrEmpty(sha256)) mergeEntity["MsiSha256"] = sha256; + if (sizeBytes.HasValue) mergeEntity["MsiBytes"] = sizeBytes.Value; + if (!string.IsNullOrEmpty(sourceUrl)) mergeEntity["MsiSourceUrl"] = sourceUrl; + + await tableClient.UpsertEntityAsync(mergeEntity, TableUpdateMode.Merge); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "Failed to update ImeVersionHistory archive info for version {Version} (status {Status})", + version, status); + } + } + #endregion #region Excessive-Event Detection diff --git a/src/Shared/AutopilotMonitor.Shared/Constants.cs b/src/Shared/AutopilotMonitor.Shared/Constants.cs index 52fe06cf..087873a2 100644 --- a/src/Shared/AutopilotMonitor.Shared/Constants.cs +++ b/src/Shared/AutopilotMonitor.Shared/Constants.cs @@ -1295,6 +1295,19 @@ namespace AutopilotMonitor.Shared /// settings (Soft-Delete + Versioning) provide defence-in-depth. /// public const string CriticalTableBackups = "critical-table-backups"; + + /// + /// Permanent platform archive of IME (Intune Management Extension) installer + /// binaries, one virtual folder per version: + /// {version}/IntuneWindowsAgent.msi + {version}/provenance.json. + /// Written by ImeMsiArchiver when a new IME version is first seen across + /// the fleet (companion of , which is + /// equally KeptByDesign). NOT tenant-scoped — blob paths are version-keyed, so + /// tenant offboarding (Phase 2.E) deliberately never touches this container. + /// No lifecycle rule: the archive is the point (build-to-build diffs of + /// Microsoft's rollouts). Private by default (SDK CreateIfNotExists). + /// + public const string ImeArchive = "ime-archive"; } /// @@ -1392,6 +1405,18 @@ namespace AutopilotMonitor.Shared /// Poison sibling of . public const string SessionDeletionMaintenancePoison = "session-deletion-maintenance-poison"; + + /// + /// IME-MSI archive fan-out. Producer = EventIngestProcessor when + /// RecordImeVersionAsync reports a genuinely NEW IME version (first sighting + /// across the fleet — roughly monthly); consumer = ImeMsiArchiveQueueWorker, + /// which downloads the installer from the CSP-reported + /// msiDownloadUrl (host-allowlisted, size-capped) into + /// and merges the outcome into the + /// row. Poison suffix -poison, + /// max-dequeue 5. + /// + public const string ImeMsiArchive = "ime-msi-archive"; } /// diff --git a/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs b/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs index c9d4d47f..28d92908 100644 --- a/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs +++ b/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs @@ -176,6 +176,14 @@ namespace AutopilotMonitor.Shared.DataAccess Task RecordImeVersionAsync(string version, string tenantId, string sessionId); Task> GetImeVersionHistoryAsync(); + /// + /// Merges the installer-archiving outcome (ImeMsiArchiver) into the version's + /// ImeVersionHistory row. Fail-soft: storage errors are logged, never thrown. + /// Blob path / hash / size / URL are only written on success (null leaves them absent). + /// + Task UpdateImeVersionArchiveInfoAsync( + string version, string status, string? blobPath, string? sha256, long? sizeBytes, string? sourceUrl); + // --- Events --- Task StoreEventAsync(EnrollmentEvent evt); Task> StoreEventsBatchAsync(List events); diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Config/AdminConfiguration.cs b/src/Shared/AutopilotMonitor.Shared/Models/Config/AdminConfiguration.cs index e40209ae..45631a15 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/Config/AdminConfiguration.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/Config/AdminConfiguration.cs @@ -460,6 +460,24 @@ namespace AutopilotMonitor.Shared.Models /// public bool AutoApproveNewTenants { get; set; } = false; + /// + /// When true, the first fleet-wide sighting of a new IME version enqueues an + /// archive job (ime-msi-archive queue) that downloads the installer from the + /// CSP-reported URL into the ime-archive blob container. Roughly one download + /// per Microsoft IME release (~monthly). Default true — flip off to pause archiving + /// (queued messages then wait, they are not dropped). Round-tripped via the 4-file + /// web chain (memory feedback_admin_config_ui_roundtrip). + /// + public bool ImeMsiArchivingEnabled { get; set; } = true; + + /// + /// Size cap for the IME installer download in MB (Content-Length preflight AND + /// streamed-byte guard). The MSI is ~12 MB today; the generous cap exists only so a + /// tampered/wrong URL cannot stream gigabytes into the archive while still never + /// missing a legitimately grown installer. 0 = no limit. Default: 250 MB. + /// + public int MaxImeMsiDownloadSizeMB { get; set; } = 250; + /// /// When true, the dual app-registration self-service homing flip is active: the Graph /// consent flow funnels legacy-homed tenants to the primary (new) app registration and diff --git a/src/Shared/AutopilotMonitor.Shared/Models/ImeMsiArchiveEnvelope.cs b/src/Shared/AutopilotMonitor.Shared/Models/ImeMsiArchiveEnvelope.cs new file mode 100644 index 00000000..4617370e --- /dev/null +++ b/src/Shared/AutopilotMonitor.Shared/Models/ImeMsiArchiveEnvelope.cs @@ -0,0 +1,44 @@ +using System; + +namespace AutopilotMonitor.Shared.Models +{ + /// + /// Queue-message envelope for the ime-msi-archive queue. Enqueued by + /// EventIngestProcessor when an ime_agent_version event carries a version + /// the platform has never seen before (RecordImeVersionAsync insert succeeded). The + /// worker downloads the IME installer and archives it into the ime-archive blob + /// container so every fleet-observed IME build stays available for later decompilation + /// and build-to-build diffing, even after Microsoft's versionless CDN URL has moved on. + /// + public sealed class ImeMsiArchiveEnvelope + { + /// Schema version — bump on breaking envelope changes so consumers can reject or migrate. + public string EnvelopeVersion { get; set; } = "1"; + + /// The newly sighted IME version, e.g. 1.104.102.0 (blob folder name). + public string Version { get; set; } = string.Empty; + + /// + /// The EnterpriseDesktopAppManagement-CSP CurrentDownloadUrl from the event's + /// msiDownloadUrl enrichment. Null when the reporting agent predates the + /// enrichment — the worker then falls back to the canonical versionless CDN URL + /// (safe: a brand-new version is by definition what that URL currently serves). + /// + public string? MsiDownloadUrl { get; set; } + + /// + /// The event's msiMatchedBy companion (productVersion = URL is + /// version-authoritative, fileName = registry drift). Provenance only. + /// + public string? MsiMatchedBy { get; set; } + + /// First-seen tenant — provenance only, never used for auth or paths. + public string TenantId { get; set; } = string.Empty; + + /// First-seen session — provenance only. + public string SessionId { get; set; } = string.Empty; + + /// UTC time the producer enqueued the message; useful for measuring queue lag. + public DateTime EnqueuedAt { get; set; } + } +} diff --git a/src/Shared/AutopilotMonitor.Shared/Models/SessionApiModels.cs b/src/Shared/AutopilotMonitor.Shared/Models/SessionApiModels.cs index 767043b8..ea39b92f 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/SessionApiModels.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/SessionApiModels.cs @@ -575,5 +575,25 @@ namespace AutopilotMonitor.Shared.Models public string FirstSeenTenantId { get; set; } = string.Empty; public DateTime LastSeenAt { get; set; } public int SessionCount { get; set; } + + /// + /// Outcome of the automatic installer archiving (ImeMsiArchiver): + /// Archived or a Failed:* status; null for versions sighted before the + /// feature existed or while the archive job is still queued. Like the FirstSeen* + /// identifiers, the archive fields are only serialized for Global Admin callers. + /// + public string? MsiArchiveStatus { get; set; } + + /// Blob path inside the ime-archive container, e.g. 1.104.102.0/IntuneWindowsAgent.msi. + public string? MsiArchiveBlobPath { get; set; } + + /// SHA-256 of the archived installer (hex, lowercase). + public string? MsiSha256 { get; set; } + + /// Size of the archived installer in bytes. + public long? MsiBytes { get; set; } + + /// The URL the installer was downloaded from (CSP-reported or canonical fallback). + public string? MsiSourceUrl { get; set; } } } diff --git a/src/Web/autopilot-monitor-web/app/admin/AdminConfigContext.tsx b/src/Web/autopilot-monitor-web/app/admin/AdminConfigContext.tsx index de9d3334..054ed08e 100644 --- a/src/Web/autopilot-monitor-web/app/admin/AdminConfigContext.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/AdminConfigContext.tsx @@ -50,6 +50,10 @@ interface AdminConfigContextValue { setAutoApproveNewTenants: (value: boolean) => void; selfServiceAppHomingEnabled: boolean; setSelfServiceAppHomingEnabled: (value: boolean) => void; + imeMsiArchivingEnabled: boolean; + setImeMsiArchivingEnabled: (value: boolean) => void; + maxImeMsiDownloadSizeMB: number; + setMaxImeMsiDownloadSizeMB: (value: number) => void; // Diagnostics log paths globalDiagPaths: DiagnosticsLogPath[]; @@ -152,6 +156,8 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode }) const [sessionDeletionKillSwitch, setSessionDeletionKillSwitch] = useState(false); const [autoApproveNewTenants, setAutoApproveNewTenants] = useState(false); const [selfServiceAppHomingEnabled, setSelfServiceAppHomingEnabled] = useState(false); + const [imeMsiArchivingEnabled, setImeMsiArchivingEnabled] = useState(true); + const [maxImeMsiDownloadSizeMB, setMaxImeMsiDownloadSizeMB] = useState(250); // Diagnostics Log Paths state const [globalDiagPaths, setGlobalDiagPaths] = useState([]); @@ -215,6 +221,8 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode }) setSessionDeletionKillSwitch(data.sessionDeletionKillSwitch ?? false); setAutoApproveNewTenants(data.autoApproveNewTenants ?? false); setSelfServiceAppHomingEnabled(data.selfServiceAppHomingEnabled ?? false); + setImeMsiArchivingEnabled(data.imeMsiArchivingEnabled ?? true); + setMaxImeMsiDownloadSizeMB(data.maxImeMsiDownloadSizeMB ?? 250); try { setGlobalDiagPaths(data.diagnosticsGlobalLogPathsJson ? JSON.parse(data.diagnosticsGlobalLogPathsJson) : []); } catch { @@ -332,6 +340,8 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode }) sessionDeletionKillSwitch, autoApproveNewTenants, selfServiceAppHomingEnabled, + imeMsiArchivingEnabled, + maxImeMsiDownloadSizeMB, }; const response = await authenticatedFetch(api.globalConfig.get(), getAccessToken, { @@ -358,7 +368,7 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode }) } finally { setSavingConfig(false); } - }, [isGlobalAdmin, adminConfig, globalRateLimit, userRateLimit, globalAdminRateLimit, platformStatsBlobSasUrl, agentMigrateApiBaseUrl, agentMigrateTenantOverridesJson, collectorIdleTimeoutMinutes, desktopDetectorNoCandidateTimeoutMinutes, opsEventRetentionDays, slaNotificationCooldownHours, allowAgentDowngrade, modernDeploymentHarmlessEventIds, enableIndexDualWrite, sessionDeletionKillSwitch, autoApproveNewTenants, selfServiceAppHomingEnabled, getAccessToken]); + }, [isGlobalAdmin, adminConfig, globalRateLimit, userRateLimit, globalAdminRateLimit, platformStatsBlobSasUrl, agentMigrateApiBaseUrl, agentMigrateTenantOverridesJson, collectorIdleTimeoutMinutes, desktopDetectorNoCandidateTimeoutMinutes, opsEventRetentionDays, slaNotificationCooldownHours, allowAgentDowngrade, modernDeploymentHarmlessEventIds, enableIndexDualWrite, sessionDeletionKillSwitch, autoApproveNewTenants, selfServiceAppHomingEnabled, imeMsiArchivingEnabled, maxImeMsiDownloadSizeMB, getAccessToken]); // Reset admin config const handleResetAdminConfig = useCallback(() => { @@ -379,6 +389,8 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode }) setSessionDeletionKillSwitch(adminConfig.sessionDeletionKillSwitch ?? false); setAutoApproveNewTenants(adminConfig.autoApproveNewTenants ?? false); setSelfServiceAppHomingEnabled(adminConfig.selfServiceAppHomingEnabled ?? false); + setImeMsiArchivingEnabled(adminConfig.imeMsiArchivingEnabled ?? true); + setMaxImeMsiDownloadSizeMB(adminConfig.maxImeMsiDownloadSizeMB ?? 250); try { setGlobalDiagPaths(adminConfig.diagnosticsGlobalLogPathsJson ? JSON.parse(adminConfig.diagnosticsGlobalLogPathsJson) : []); } catch { @@ -525,6 +537,8 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode }) sessionDeletionKillSwitch, setSessionDeletionKillSwitch, autoApproveNewTenants, setAutoApproveNewTenants, selfServiceAppHomingEnabled, setSelfServiceAppHomingEnabled, + imeMsiArchivingEnabled, setImeMsiArchivingEnabled, + maxImeMsiDownloadSizeMB, setMaxImeMsiDownloadSizeMB, globalDiagPaths, setGlobalDiagPaths, savingDiagPaths, opsAlertRules, setOpsAlertRules, opsAlertTelegramEnabled, setOpsAlertTelegramEnabled, diff --git a/src/Web/autopilot-monitor-web/app/admin/components/AdminConfigSettingsSection.tsx b/src/Web/autopilot-monitor-web/app/admin/components/AdminConfigSettingsSection.tsx index 503dc3d2..d9045c0e 100644 --- a/src/Web/autopilot-monitor-web/app/admin/components/AdminConfigSettingsSection.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/components/AdminConfigSettingsSection.tsx @@ -36,6 +36,10 @@ interface AdminConfigSettingsSectionProps { setAutoApproveNewTenants: (value: boolean) => void; selfServiceAppHomingEnabled: boolean; setSelfServiceAppHomingEnabled: (value: boolean) => void; + imeMsiArchivingEnabled: boolean; + setImeMsiArchivingEnabled: (value: boolean) => void; + maxImeMsiDownloadSizeMB: number; + setMaxImeMsiDownloadSizeMB: (value: number) => void; onSave: () => Promise; onReset: () => void; } @@ -74,6 +78,10 @@ export function AdminConfigSettingsSection({ setAutoApproveNewTenants, selfServiceAppHomingEnabled, setSelfServiceAppHomingEnabled, + imeMsiArchivingEnabled, + setImeMsiArchivingEnabled, + maxImeMsiDownloadSizeMB, + setMaxImeMsiDownloadSizeMB, onSave, onReset, }: AdminConfigSettingsSectionProps) { @@ -375,6 +383,47 @@ export function AdminConfigSettingsSection({ +
+ +
+ +
+ +
+