backend: auto-archive IME installer on first sighting of a new version

When RecordImeVersionAsync reports a genuinely new IME version (~monthly),
EventIngestProcessor now also enqueues an ime-msi-archive job. The worker
(QueuePollingWorker, heartbeat, pause on ImeMsiArchivingEnabled=false)
downloads the installer and stores it write-once (If-None-Match:*) in the
permanent ime-archive container ({version}/IntuneWindowsAgent.msi +
provenance.json), then merges MsiArchiveStatus/BlobPath/Sha256/Bytes/SourceUrl
onto the ImeVersionHistory row - so every fleet-observed IME build stays
available for decompilation/diffing after Microsoft's versionless CDN moves on.

The download URL comes from agent telemetry (msiDownloadUrl enrichment) and is
treated as untrusted: https-only *.manage.microsoft.com allowlist with exact
filename match, canonical versionless fallback, no redirects, size cap
(MaxImeMsiDownloadSizeMB, default 250, preflight + mid-stream) enforced by a
SHA-256-hashing pass-through stream. Archiver never throws; the worker retries
only transient statuses (download/timeout/error), permanent rejections
complete with their Failed:* status persisted.

Admin settings ImeMsiArchivingEnabled (default true) + MaxImeMsiDownloadSizeMB
via the full 4-file web chain; archive fields on the version-history API are
GA-only (non-GA projection unchanged, redaction tests extended). Shared
manifests regenerated. Tests: archiver status branches via recording subclass
(allowlist, caps, 409 idempotency, cancellation propagation), config defaults.
Backend suite 4262 green, web tsc + vitest 762 green, solution 0 warnings.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Oliver Kieselbach
2026-08-18 00:20:53 +02:00
parent 8ceab7a064
commit ca26cb4f7c
23 changed files with 1263 additions and 4 deletions
@@ -0,0 +1,44 @@
using AutopilotMonitor.Shared.Models;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Default-value tests for the IME-installer archiving settings. The flag gates the
/// ime-msi-archive queue worker (pause, not drop); the size cap guards the download.
/// Defaulting the flag to TRUE is deliberate: archiving is operator infrastructure with
/// ~one small download per Microsoft IME release, and a version missed while the flag is
/// off is gone for good (the versionless CDN moves on).
/// </summary>
public class AdminConfigurationImeMsiArchivingTests
{
[Fact]
public void ImeMsiArchivingEnabled_defaults_to_true_on_new_config()
{
var cfg = new AdminConfiguration();
Assert.True(cfg.ImeMsiArchivingEnabled);
}
[Fact]
public void ImeMsiArchivingEnabled_defaults_to_true_on_CreateDefault()
{
var cfg = AdminConfiguration.CreateDefault();
Assert.True(cfg.ImeMsiArchivingEnabled);
}
[Fact]
public void MaxImeMsiDownloadSizeMB_defaults_to_250()
{
// 250 MB: ~20x today's MSI so a legitimately grown installer is never missed,
// while a tampered URL still cannot stream gigabytes (Oliver, 2026-08-18).
Assert.Equal(250, new AdminConfiguration().MaxImeMsiDownloadSizeMB);
Assert.Equal(250, AdminConfiguration.CreateDefault().MaxImeMsiDownloadSizeMB);
}
[Fact]
public void Settings_persist_on_the_config()
{
var cfg = new AdminConfiguration { ImeMsiArchivingEnabled = false, MaxImeMsiDownloadSizeMB = 50 };
Assert.False(cfg.ImeMsiArchivingEnabled);
Assert.Equal(50, cfg.MaxImeMsiDownloadSizeMB);
}
}
@@ -39,6 +39,26 @@ public class GetImeVersionHistoryRedactionTests
Assert.DoesNotContain("firstSeenSessionId", json, StringComparison.OrdinalIgnoreCase);
}
/// <summary>
/// The MSI-archive columns (ImeMsiArchiver) are operator infrastructure — blob paths,
/// hashes and source URLs of the platform's own archive have no business in a tenant
/// member's payload. Pinned like the identifiers so the boundary survives refactors.
/// </summary>
[Fact]
public void NonGlobalCaller_DropsMsiArchiveFields()
{
var json = SerializeFor(hasGlobalScope: false, MakeEntry());
Assert.DoesNotContain("msiArchiveStatus", json, StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain("msiArchiveBlobPath", json, StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain("msiSha256", json, StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain("msiBytes", json, StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain("msiSourceUrl", json, StringComparison.OrdinalIgnoreCase);
// Values too, not just keys (same rationale as the identifier-value test below).
Assert.DoesNotContain("4d6b85fd3c49cf227ab4711308bca1f3c701cad9bff1799da4c6b6616066bb5e", json, StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain("imeswdb-afd-secondary", json, StringComparison.OrdinalIgnoreCase);
}
/// <summary>
/// The values, not just the property names: a rename of the model property must not let the
/// identifier survive under a different key. Asserting on the GUIDs catches that.
@@ -101,7 +121,12 @@ public class GetImeVersionHistoryRedactionTests
var keys = doc.RootElement[0].EnumerateObject().Select(p => p.Name).OrderBy(n => n).ToArray();
Assert.Equal(
new[] { "firstSeenAt", "firstSeenSessionId", "firstSeenTenantId", "lastSeenAt", "sessionCount", "version" },
new[]
{
"firstSeenAt", "firstSeenSessionId", "firstSeenTenantId", "lastSeenAt",
"msiArchiveBlobPath", "msiArchiveStatus", "msiBytes", "msiSha256", "msiSourceUrl",
"sessionCount", "version",
},
keys);
}
@@ -185,5 +210,10 @@ public class GetImeVersionHistoryRedactionTests
FirstSeenTenantId = "57f34dd1-6a42-47f9-9bc0-b921fa6caa30",
LastSeenAt = Last,
SessionCount = 3109,
MsiArchiveStatus = "Archived",
MsiArchiveBlobPath = $"{version}/IntuneWindowsAgent.msi",
MsiSha256 = "4d6b85fd3c49cf227ab4711308bca1f3c701cad9bff1799da4c6b6616066bb5e",
MsiBytes = 13_107_200,
MsiSourceUrl = "https://imeswdb-afd-secondary.manage.microsoft.com/IntuneWindowsAgent.msi",
};
}
@@ -0,0 +1,335 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Net.Http;
using System.Security.Cryptography;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services.Ime;
using AutopilotMonitor.Shared.Models;
using Azure;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// The IME-installer archiver downloads a newly sighted IME build into the permanent
/// ime-archive container. Its contract: NEVER throw (worker decides retry from the result),
/// treat the agent-supplied URL as untrusted (allowlist, else canonical fallback), enforce
/// the size cap both at Content-Length preflight and mid-stream, and stay idempotent under
/// queue re-delivery (409 on the write-once upload == archived). Tests pin every status
/// branch via a recording subclass — no Azurite, no HTTP.
/// </summary>
public class ImeMsiArchiverTests
{
private const string Version = "1.104.102.0";
private const string EventUrl = "https://imeswdb-afd-secondary.manage.microsoft.com/IntuneWindowsAgent.msi";
// =========================================================================
// URL allowlist (static — the SSRF boundary)
// =========================================================================
[Theory]
[InlineData(EventUrl, true)]
[InlineData("https://swda01-mscdn.manage.microsoft.com/IntuneWindowsAgent.msi", true)]
[InlineData("https://manage.microsoft.com/IntuneWindowsAgent.msi", true)] // apex host
[InlineData("https://x.manage.microsoft.com/path/IntuneWindowsAgent.msi?sv=1&sig=2", true)] // query ignored
[InlineData("https://x.manage.microsoft.com/intunewindowsagent.MSI", true)] // case-insensitive
[InlineData("http://imeswdb-afd-secondary.manage.microsoft.com/IntuneWindowsAgent.msi", false)] // https only
[InlineData("https://evil.example.com/IntuneWindowsAgent.msi", false)]
[InlineData("https://manage.microsoft.com.evil.com/IntuneWindowsAgent.msi", false)] // suffix spoof
[InlineData("https://xmanage.microsoft.com/IntuneWindowsAgent.msi", false)] // needs the dot
[InlineData("https://x.manage.microsoft.com/SomethingElse.msi", false)] // wrong filename
[InlineData("https://x.manage.microsoft.com/IntuneWindowsAgent.msi.exe", false)]
[InlineData("not a url", false)]
[InlineData("", false)]
[InlineData(null, false)]
public void IsAllowedMsiUrl_cases(string? url, bool expected)
{
Assert.Equal(expected, ImeMsiArchiver.IsAllowedMsiUrl(url));
}
// =========================================================================
// Version guard (blob path is built from untrusted input)
// =========================================================================
[Theory]
[InlineData("")]
[InlineData("1.104.102.0.1")] // too many components
[InlineData("1.104../escape")]
[InlineData("latest")]
[InlineData("1.104.102.0 ")]
public async Task ArchiveAsync_BadVersion_PermanentFailure_NoDownload(string version)
{
var archiver = new RecordingArchiver();
var result = await archiver.ArchiveAsync(Envelope(version: version));
Assert.False(result.Success);
Assert.Equal(ImeMsiArchiver.Statuses.FailedBadVersion, result.Status);
Assert.False(result.Retryable);
Assert.Equal(0, archiver.DownloadCalls + archiver.UploadCalls);
}
// =========================================================================
// URL selection
// =========================================================================
[Fact]
public async Task ArchiveAsync_AllowedEventUrl_IsUsed()
{
var archiver = new RecordingArchiver();
var result = await archiver.ArchiveAsync(Envelope(url: EventUrl));
Assert.True(result.Success);
Assert.Equal(EventUrl, archiver.DownloadLastUrl);
Assert.Equal(EventUrl, result.SourceUrl);
}
[Theory]
[InlineData("https://evil.example.com/IntuneWindowsAgent.msi")] // allowlist reject
[InlineData(null)] // pre-enrichment agent
public async Task ArchiveAsync_MissingOrRejectedEventUrl_FallsBackToCanonical(string? url)
{
var archiver = new RecordingArchiver();
var result = await archiver.ArchiveAsync(Envelope(url: url));
Assert.True(result.Success);
Assert.Equal(ImeMsiArchiver.CanonicalMsiUrl, archiver.DownloadLastUrl);
Assert.Equal(ImeMsiArchiver.CanonicalMsiUrl, result.SourceUrl);
}
// =========================================================================
// Happy path
// =========================================================================
[Fact]
public async Task ArchiveAsync_HappyPath_UploadsMsiAndProvenance_WithHashAndSize()
{
var payload = new byte[3 * 1024 * 1024];
new Random(42).NextBytes(payload);
var expectedSha = Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant();
var archiver = new RecordingArchiver { Payload = payload };
var result = await archiver.ArchiveAsync(Envelope());
Assert.True(result.Success);
Assert.Equal(ImeMsiArchiver.Statuses.Archived, result.Status);
Assert.Equal($"{Version}/IntuneWindowsAgent.msi", result.BlobPath);
Assert.Equal(expectedSha, result.Sha256);
Assert.Equal(payload.Length, result.SizeBytes);
Assert.Equal(1, archiver.UploadCalls);
Assert.Equal($"{Version}/IntuneWindowsAgent.msi", archiver.UploadLastPath);
Assert.Equal(payload.Length, archiver.UploadedByteCount);
Assert.Equal(1, archiver.ProvenanceCalls);
Assert.Equal($"{Version}/provenance.json", archiver.ProvenanceLastPath);
Assert.Contains(expectedSha, archiver.ProvenanceLastJson);
Assert.Contains(Version, archiver.ProvenanceLastJson);
}
// =========================================================================
// Size cap — preflight and mid-stream
// =========================================================================
[Fact]
public async Task ArchiveAsync_ContentLengthAboveCap_Rejected_NoUpload()
{
var archiver = new RecordingArchiver
{
MaxDownloadSizeMB = 1,
Payload = new byte[2 * 1024 * 1024],
};
var result = await archiver.ArchiveAsync(Envelope());
Assert.False(result.Success);
Assert.Equal(ImeMsiArchiver.Statuses.FailedTooLarge, result.Status);
Assert.False(result.Retryable);
Assert.Equal(0, archiver.UploadCalls);
}
[Fact]
public async Task ArchiveAsync_UnknownContentLength_CapStillEnforcedMidStream()
{
// A lying/absent Content-Length must not bypass the cap — the hashing wrapper
// aborts the stream as soon as the byte count crosses it.
var archiver = new RecordingArchiver
{
MaxDownloadSizeMB = 1,
Payload = new byte[2 * 1024 * 1024],
ReportContentLength = false,
};
var result = await archiver.ArchiveAsync(Envelope());
Assert.False(result.Success);
Assert.Equal(ImeMsiArchiver.Statuses.FailedTooLarge, result.Status);
Assert.False(result.Retryable);
Assert.Equal(0, archiver.ProvenanceCalls);
}
[Fact]
public async Task ArchiveAsync_CapZero_MeansUnlimited()
{
var archiver = new RecordingArchiver
{
MaxDownloadSizeMB = 0,
Payload = new byte[2 * 1024 * 1024],
};
var result = await archiver.ArchiveAsync(Envelope());
Assert.True(result.Success);
}
// =========================================================================
// Idempotency under queue re-delivery
// =========================================================================
[Fact]
public async Task ArchiveAsync_BlobAlreadyExists_TreatedAsArchived_NotRetryable()
{
var archiver = new RecordingArchiver
{
UploadException = new RequestFailedException(409, "BlobAlreadyExists"),
};
var result = await archiver.ArchiveAsync(Envelope());
Assert.True(result.Success);
Assert.Equal(ImeMsiArchiver.Statuses.Archived, result.Status);
Assert.False(result.Retryable);
Assert.Equal($"{Version}/IntuneWindowsAgent.msi", result.BlobPath);
}
// =========================================================================
// Transient failures — retryable, never thrown
// =========================================================================
[Fact]
public async Task ArchiveAsync_DownloadFails_RetryableDownloadStatus()
{
var archiver = new RecordingArchiver
{
DownloadException = new HttpRequestException("boom"),
};
var result = await archiver.ArchiveAsync(Envelope());
Assert.False(result.Success);
Assert.Equal(ImeMsiArchiver.Statuses.FailedDownload, result.Status);
Assert.True(result.Retryable);
}
[Fact]
public async Task ArchiveAsync_Timeout_RetryableTimeoutStatus()
{
var archiver = new RecordingArchiver
{
DownloadException = new OperationCanceledException(),
};
var result = await archiver.ArchiveAsync(Envelope());
Assert.False(result.Success);
Assert.Equal(ImeMsiArchiver.Statuses.FailedTimeout, result.Status);
Assert.True(result.Retryable);
}
[Fact]
public async Task ArchiveAsync_UnexpectedUploadError_RetryableErrorStatus()
{
var archiver = new RecordingArchiver
{
UploadException = new InvalidOperationException("storage hiccup"),
};
var result = await archiver.ArchiveAsync(Envelope());
Assert.False(result.Success);
Assert.Equal(ImeMsiArchiver.Statuses.FailedError, result.Status);
Assert.True(result.Retryable);
}
[Fact]
public async Task ArchiveAsync_CallerCancellation_Propagates()
{
// Host shutdown must reach the worker's dispatch loop so the message stays queued.
var archiver = new RecordingArchiver();
using var cts = new CancellationTokenSource();
cts.Cancel();
archiver.DownloadException = new OperationCanceledException(cts.Token);
await Assert.ThrowsAnyAsync<OperationCanceledException>(
() => archiver.ArchiveAsync(Envelope(), cts.Token));
}
// =========================================================================
// Helpers
// =========================================================================
private static ImeMsiArchiveEnvelope Envelope(string version = Version, string? url = EventUrl) =>
new()
{
Version = version,
MsiDownloadUrl = url,
MsiMatchedBy = "productVersion",
TenantId = "11111111-1111-1111-1111-111111111111",
SessionId = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
EnqueuedAt = DateTime.UtcNow,
};
private sealed class RecordingArchiver : ImeMsiArchiver
{
public byte[] Payload { get; set; } = new byte[2048];
public bool ReportContentLength { get; set; } = true;
public int MaxDownloadSizeMB { get; set; } = 250;
public Exception? DownloadException { get; set; }
public Exception? UploadException { get; set; }
public int DownloadCalls { get; private set; }
public string? DownloadLastUrl { get; private set; }
public int UploadCalls { get; private set; }
public string? UploadLastPath { get; private set; }
public long UploadedByteCount { get; private set; }
public int ProvenanceCalls { get; private set; }
public string? ProvenanceLastPath { get; private set; }
public string ProvenanceLastJson { get; private set; } = string.Empty;
protected override Task<AdminConfiguration> GetAdminConfigurationAsync()
=> Task.FromResult(new AdminConfiguration { MaxImeMsiDownloadSizeMB = MaxDownloadSizeMB });
protected override Task<(Stream Content, long ContentLength)> OpenDownloadAsync(
string url, CancellationToken cancellationToken)
{
DownloadCalls++;
DownloadLastUrl = url;
if (DownloadException is not null) throw DownloadException;
return Task.FromResult(
((Stream)new MemoryStream(Payload), ReportContentLength ? Payload.LongLength : -1L));
}
protected override async Task UploadMsiAsync(string blobPath, Stream content, CancellationToken cancellationToken)
{
UploadCalls++;
UploadLastPath = blobPath;
// Drain the stream like the real blob upload would — this is what drives the
// hashing/cap wrapper (and mid-stream aborts surface here as exceptions).
var buffer = new byte[64 * 1024];
int read;
while ((read = await content.ReadAsync(buffer, cancellationToken)) > 0)
UploadedByteCount += read;
if (UploadException is not null) throw UploadException;
}
protected override Task UploadProvenanceAsync(string blobPath, string json, CancellationToken cancellationToken)
{
ProvenanceCalls++;
ProvenanceLastPath = blobPath;
ProvenanceLastJson = json;
return Task.CompletedTask;
}
}
}
@@ -818,6 +818,8 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
{ "VulnerabilityCorrelationEnabled", config.VulnerabilityCorrelationEnabled },
{ "EnableIndexDualWrite", config.EnableIndexDualWrite },
{ "AutoApproveNewTenants", config.AutoApproveNewTenants },
{ "ImeMsiArchivingEnabled", config.ImeMsiArchivingEnabled },
{ "MaxImeMsiDownloadSizeMB", config.MaxImeMsiDownloadSizeMB },
{ "SelfServiceAppHomingEnabled", config.SelfServiceAppHomingEnabled },
{ "SessionDeletionKillSwitch", config.SessionDeletionKillSwitch },
{ "VulnerabilityDataLastSyncUtc", config.VulnerabilityDataLastSyncUtc },
@@ -885,6 +887,8 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
VulnerabilityCorrelationEnabled = entity.GetBoolean("VulnerabilityCorrelationEnabled") ?? true,
EnableIndexDualWrite = entity.GetBoolean("EnableIndexDualWrite") ?? false,
AutoApproveNewTenants = entity.GetBoolean("AutoApproveNewTenants") ?? false,
ImeMsiArchivingEnabled = entity.GetBoolean("ImeMsiArchivingEnabled") ?? true,
MaxImeMsiDownloadSizeMB = entity.GetInt32("MaxImeMsiDownloadSizeMB") ?? 250,
SelfServiceAppHomingEnabled = entity.GetBoolean("SelfServiceAppHomingEnabled") ?? false,
SessionDeletionKillSwitch = entity.GetBoolean("SessionDeletionKillSwitch") ?? false,
VulnerabilityDataLastSyncUtc = entity.GetString("VulnerabilityDataLastSyncUtc"),
@@ -172,6 +172,10 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
public Task<List<ImeVersionHistoryEntry>> GetImeVersionHistoryAsync()
=> _storage.GetImeVersionHistoryAsync();
public Task UpdateImeVersionArchiveInfoAsync(
string version, string status, string? blobPath, string? sha256, long? sizeBytes, string? sourceUrl)
=> _storage.UpdateImeVersionArchiveInfoAsync(version, status, blobPath, sha256, sizeBytes, sourceUrl);
public async Task<bool> StoreEventAsync(EnrollmentEvent evt)
{
var result = await _storage.StoreEventAsync(evt);
@@ -334,6 +334,26 @@ builder.Services.AddSingleton<
builder.Services.AddHostedService<
AutopilotMonitor.Functions.Services.Vulnerability.VulnerabilityCorrelateQueueWorker>();
// IME-installer archiving on first fleet-wide sighting of a new IME version (~monthly).
// Producer sits in EventIngestProcessor's new-version continuation; the worker downloads
// the MSI from the CSP-reported (allowlisted) URL into the permanent ime-archive container
// and merges the outcome onto the ImeVersionHistory row. Typed HttpClient: no redirects
// (SSRF posture — the allowlisted host must serve directly), long timeout owned by the
// archiver's own CTS, transient retry via the shared external-data policy.
builder.Services.AddHttpClient<AutopilotMonitor.Functions.Services.Ime.ImeMsiArchiver>()
.ConfigurePrimaryHttpMessageHandler(() => new System.Net.Http.SocketsHttpHandler
{
AllowAutoRedirect = false,
ConnectTimeout = TimeSpan.FromSeconds(10),
})
.ConfigureHttpClient(client => client.Timeout = Timeout.InfiniteTimeSpan)
.AddPolicyHandler((sp, _) => sp.GetRequiredService<ResiliencePolicies>().ExternalDataApi);
builder.Services.AddSingleton<
AutopilotMonitor.Functions.Services.Ime.IImeMsiArchiveProducer,
AutopilotMonitor.Functions.Services.Ime.AzureQueueImeMsiArchiveProducer>();
builder.Services.AddHostedService<
AutopilotMonitor.Functions.Services.Ime.ImeMsiArchiveQueueWorker>();
// Delayed tenant auto-approve (public availability). Producer fires at first-login signup
// with a ~1-minute visibility delay; the worker activates the tenant via the shared
// TenantApprovalService only if AdminConfiguration.AutoApproveNewTenants is enabled at
@@ -43,6 +43,7 @@ namespace AutopilotMonitor.Functions.Services
private readonly AutopilotMonitor.Functions.Services.Analyze.IAnalyzeOnEnrollmentEndProducer _analyzeProducer;
private readonly AutopilotMonitor.Functions.Services.Analyze.InterimTriggerRegistry _interimTriggerRegistry;
private readonly IVulnerabilityCorrelateProducer _vulnProducer;
private readonly Ime.IImeMsiArchiveProducer _imeMsiArchiveProducer;
public EventIngestProcessor(
ILogger<EventIngestProcessor> logger,
@@ -58,7 +59,8 @@ namespace AutopilotMonitor.Functions.Services
TelemetryClient telemetryClient,
AutopilotMonitor.Functions.Services.Analyze.IAnalyzeOnEnrollmentEndProducer analyzeProducer,
AutopilotMonitor.Functions.Services.Analyze.InterimTriggerRegistry interimTriggerRegistry,
IVulnerabilityCorrelateProducer vulnProducer)
IVulnerabilityCorrelateProducer vulnProducer,
Ime.IImeMsiArchiveProducer imeMsiArchiveProducer)
{
_logger = logger;
_sessionRepo = sessionRepo;
@@ -74,6 +76,7 @@ namespace AutopilotMonitor.Functions.Services
_analyzeProducer = analyzeProducer;
_interimTriggerRegistry = interimTriggerRegistry;
_vulnProducer = vulnProducer;
_imeMsiArchiveProducer = imeMsiArchiveProducer;
}
/// <summary>
@@ -129,6 +132,14 @@ namespace AutopilotMonitor.Functions.Services
.ContinueWith(t => _logger.LogWarning(t.Exception?.InnerException,
"ImeAgentVersion update failed (non-fatal)"), TaskContinuationOptions.OnlyOnFaulted);
// Captured here (not inside the continuation) purely for clarity — the agent's
// CSP-registry enrichment; absent on agent builds before 2026-08-17.
string? msiDownloadUrl = null, msiMatchedBy = null;
if (imeVersionEvent.Data.TryGetValue("msiDownloadUrl", out var urlObj))
msiDownloadUrl = urlObj?.ToString();
if (imeVersionEvent.Data.TryGetValue("msiMatchedBy", out var matchedByObj))
msiMatchedBy = matchedByObj?.ToString();
_ = _sessionRepo.RecordImeVersionAsync(imeVersion, request.TenantId, request.SessionId)
.ContinueWith(async t =>
{
@@ -141,6 +152,19 @@ namespace AutopilotMonitor.Functions.Services
{
await _opsEventService.RecordNewImeVersionDetectedAsync(
imeVersion, request.TenantId, request.SessionId);
// First fleet-wide sighting → archive the installer binary while
// Microsoft's versionless CDN still serves exactly this build.
// Producer is fail-soft; worker pauses on ImeMsiArchivingEnabled=false.
await _imeMsiArchiveProducer.EnqueueAsync(new ImeMsiArchiveEnvelope
{
Version = imeVersion,
MsiDownloadUrl = msiDownloadUrl,
MsiMatchedBy = msiMatchedBy,
TenantId = request.TenantId,
SessionId = request.SessionId,
EnqueuedAt = DateTime.UtcNow,
});
}
}, TaskScheduler.Default);
}
@@ -0,0 +1,84 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Azure.Storage.Queues;
using AutopilotMonitor.Functions.Services.Queueing;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.Models;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
namespace AutopilotMonitor.Functions.Services.Ime
{
/// <summary>
/// Azure Storage Queue implementation of <see cref="IImeMsiArchiveProducer"/>. Mirrors
/// <see cref="Analyze.AzureQueueAnalyzeOnEnrollmentEndProducer"/>: Managed Identity
/// preferred, connection-string fallback, Base64 encoding (matches the consumer worker).
/// <para>
/// <b>Fault tolerance:</b> queue-side exceptions are logged but never rethrown — the
/// caller is the ingest path's fire-and-forget continuation. A missed enqueue means the
/// version's installer is not auto-archived; the /ime-decompile skill's blob backfill
/// is the manual fallback.
/// </para>
/// </summary>
public sealed class AzureQueueImeMsiArchiveProducer : IImeMsiArchiveProducer
{
private readonly QueueClient _queueClient;
private readonly ILogger<AzureQueueImeMsiArchiveProducer> _logger;
private int _queueEnsured; // 0 = not yet ensured, 1 = CreateIfNotExistsAsync has run
public AzureQueueImeMsiArchiveProducer(
QueueClientFactory queueFactory,
ILogger<AzureQueueImeMsiArchiveProducer> logger)
{
_logger = logger;
// Base64 encoding matches the consumer (BackgroundService) so messages round-trip.
_queueClient = queueFactory.Create(Constants.QueueNames.ImeMsiArchive);
}
public async Task EnqueueAsync(ImeMsiArchiveEnvelope envelope, CancellationToken cancellationToken = default)
{
if (envelope is null) return;
if (string.IsNullOrEmpty(envelope.Version))
{
_logger.LogWarning("ImeMsiArchive enqueue skipped — missing Version");
return;
}
await EnsureQueueExistsAsync(cancellationToken).ConfigureAwait(false);
try
{
var body = JsonConvert.SerializeObject(envelope);
await _queueClient.SendMessageAsync(body, cancellationToken: cancellationToken).ConfigureAwait(false);
_logger.LogInformation(
"ImeMsiArchive enqueued (version={Version} urlPresent={UrlPresent} tenant={Tenant} session={Session})",
envelope.Version, !string.IsNullOrEmpty(envelope.MsiDownloadUrl), envelope.TenantId, envelope.SessionId);
}
catch (Exception ex)
{
// Fail-soft — ingest continuation. Skill-side blob backfill is the fallback.
_logger.LogWarning(ex,
"ImeMsiArchive enqueue failed (version={Version}) — /ime-decompile blob backfill is the fallback",
envelope.Version);
}
}
private async Task EnsureQueueExistsAsync(CancellationToken cancellationToken)
{
if (_queueEnsured == 1) return;
try
{
await _queueClient.CreateIfNotExistsAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
Interlocked.Exchange(ref _queueEnsured, 1);
}
catch (Exception ex)
{
// Leave _queueEnsured=0 so a later call retries.
_logger.LogWarning(ex, "ImeMsiArchive queue CreateIfNotExists failed — will retry next enqueue");
}
}
}
}
@@ -0,0 +1,16 @@
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Shared.Models;
namespace AutopilotMonitor.Functions.Services.Ime
{
/// <summary>
/// Enqueues an IME-installer archive job when a new IME version is first sighted.
/// Implementations must be fail-soft: the caller sits on the ingest hot path's
/// fire-and-forget continuation and must never observe an exception.
/// </summary>
public interface IImeMsiArchiveProducer
{
Task EnqueueAsync(ImeMsiArchiveEnvelope envelope, CancellationToken cancellationToken = default);
}
}
@@ -0,0 +1,87 @@
using AutopilotMonitor.Functions.Services.Queueing;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Services.Ime
{
/// <summary>
/// Background worker for the <c>ime-msi-archive</c> queue: downloads a newly sighted IME
/// installer into the permanent <c>ime-archive</c> container and merges the outcome onto
/// the ImeVersionHistory row. Volume is ~one message per Microsoft IME release.
/// <para>
/// Retry semantics: the archiver never throws — it classifies. Permanent rejections
/// (bad version, size cap) complete the message with their <c>Failed:*</c> status
/// persisted; transient failures (download/timeout) rethrow so the visibility-timeout
/// retry → poison ladder applies, with the last status visible on the row.
/// <see cref="UseHeartbeat"/> keeps the message invisible across the up-to-10-minute
/// download window. The status row-merge happens on EVERY attempt so operators can see
/// a stuck download as <c>Failed:Download</c> instead of nothing.
/// </para>
/// </summary>
public sealed class ImeMsiArchiveQueueWorker : QueuePollingWorker<ImeMsiArchiveEnvelope>
{
private readonly ImeMsiArchiver _archiver;
private readonly ISessionRepository _sessionRepo;
private readonly AdminConfigurationService _adminConfigService;
public ImeMsiArchiveQueueWorker(
QueueClientFactory queueFactory,
ImeMsiArchiver archiver,
ISessionRepository sessionRepo,
AdminConfigurationService adminConfigService,
ILogger<ImeMsiArchiveQueueWorker> logger)
: base(queueFactory, Constants.QueueNames.ImeMsiArchive, logger)
{
_archiver = archiver ?? throw new ArgumentNullException(nameof(archiver));
_sessionRepo = sessionRepo ?? throw new ArgumentNullException(nameof(sessionRepo));
_adminConfigService = adminConfigService ?? throw new ArgumentNullException(nameof(adminConfigService));
}
/// <summary>Download can legitimately take minutes — extend visibility while working.</summary>
protected override bool UseHeartbeat => true;
/// <summary>
/// Paused while <see cref="AdminConfiguration.ImeMsiArchivingEnabled"/> is off —
/// messages stay parked (not dropped) and are processed when the flag returns.
/// Uses the cached config read: this is a pause, not a kill-switch, and the worker
/// only wakes for rare messages anyway.
/// </summary>
protected override async ValueTask<bool> ShouldPauseAsync(CancellationToken ct)
{
try
{
var config = await _adminConfigService.GetConfigurationAsync();
return !config.ImeMsiArchivingEnabled;
}
catch
{
return false; // config unreadable → keep working (archive is fail-soft anyway)
}
}
protected override bool TryValidate(ImeMsiArchiveEnvelope envelope)
=> !string.IsNullOrEmpty(envelope.Version);
protected override string DescribeForLog(ImeMsiArchiveEnvelope envelope)
=> $"version {envelope.Version}";
protected override async Task HandleAsync(ImeMsiArchiveEnvelope envelope, CancellationToken ct)
{
var result = await _archiver.ArchiveAsync(envelope, ct);
// Merge the outcome first — also for failures, so the row always tells the truth.
await _sessionRepo.UpdateImeVersionArchiveInfoAsync(
envelope.Version, result.Status, result.BlobPath, result.Sha256, result.SizeBytes, result.SourceUrl);
if (!result.Success && result.Retryable)
{
// Visibility-timeout retry → poison after MaxDequeueCount. The thrown message
// lands in the worker's handler-failure log line.
throw new InvalidOperationException(
$"IME MSI archive attempt failed with {result.Status} for version {envelope.Version}");
}
}
}
}
@@ -0,0 +1,375 @@
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.Models;
using Azure;
using Azure.Storage.Blobs.Models;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
namespace AutopilotMonitor.Functions.Services.Ime
{
/// <summary>
/// Outcome of an IME-installer archive attempt. <see cref="Status"/> is one of the
/// <see cref="ImeMsiArchiver.Statuses"/> constants and is merged verbatim onto the
/// ImeVersionHistory row. <see cref="Retryable"/> tells the queue worker whether to
/// trigger a visibility-timeout retry (transient download problems) or complete the
/// message (permanent rejections).
/// </summary>
public sealed record ImeMsiArchiveResult(
bool Success, string Status, bool Retryable,
string? BlobPath, string? Sha256, long? SizeBytes, string? SourceUrl);
/// <summary>
/// Downloads a newly sighted IME installer and archives it into the permanent
/// <see cref="Constants.BlobContainers.ImeArchive"/> container
/// (<c>{version}/IntuneWindowsAgent.msi</c> + <c>{version}/provenance.json</c>).
/// Modeled on <see cref="Diagnostics.SessionReportDiagnosticsArchiveCopier"/>:
/// fail-soft by contract (never throws), Statuses constants persisted verbatim,
/// admin-configured size cap, typed catch ladder, <c>protected virtual</c> test seams.
/// <para>
/// The download URL comes from agent telemetry and is therefore untrusted: only HTTPS
/// URLs on <c>*.manage.microsoft.com</c> whose path filename is exactly
/// <c>IntuneWindowsAgent.msi</c> are accepted; anything else falls back to the canonical
/// versionless CDN URL — safe for a NEW version, because "new" is by definition what
/// that URL currently serves. The blob write uses If-None-Match:* so a queue re-delivery
/// can never overwrite an existing archive entry.
/// </para>
/// </summary>
public class ImeMsiArchiver
{
/// <summary>Persisted MsiArchiveStatus values.</summary>
public static class Statuses
{
public const string Archived = "Archived";
public const string FailedBadVersion = "Failed:BadVersion";
public const string FailedTooLarge = "Failed:TooLarge";
public const string FailedDownload = "Failed:Download";
public const string FailedTimeout = "Failed:Timeout";
public const string FailedError = "Failed:Error";
}
/// <summary>
/// The CSP's versionless distribution endpoint — always serves the currently rolled-out
/// build. Fallback when the event carried no (acceptable) URL.
/// </summary>
public const string CanonicalMsiUrl = "https://imeswdb-afd-secondary.manage.microsoft.com/IntuneWindowsAgent.msi";
private const string MsiFileName = "IntuneWindowsAgent.msi";
private const string AllowedHostSuffix = ".manage.microsoft.com";
/// <summary>Hard ceiling for one download attempt; the worker's heartbeat covers it.</summary>
private static readonly TimeSpan DownloadTimeout = TimeSpan.FromMinutes(10);
/// <summary>Blob folder name = version — must be plain dotted digits (untrusted input).</summary>
internal static readonly Regex VersionRegex = new(@"^\d+(\.\d+){1,3}$", RegexOptions.Compiled);
private readonly HttpClient _httpClient;
private readonly BlobStorageService _blobStorage;
private readonly AdminConfigurationService _adminConfigService;
private readonly ILogger<ImeMsiArchiver> _logger;
public ImeMsiArchiver(
HttpClient httpClient,
BlobStorageService blobStorage,
AdminConfigurationService adminConfigService,
ILogger<ImeMsiArchiver> logger)
{
_httpClient = httpClient;
_blobStorage = blobStorage;
_adminConfigService = adminConfigService;
_logger = logger;
}
/// <summary>
/// Test seam: Azure-free construction for recording subclasses that override the
/// virtual download/upload/config seams below.
/// </summary>
protected ImeMsiArchiver()
{
_httpClient = null!;
_blobStorage = null!;
_adminConfigService = null!;
_logger = NullLogger<ImeMsiArchiver>.Instance;
}
/// <summary>
/// True when <paramref name="url"/> is an HTTPS URL on <c>*.manage.microsoft.com</c>
/// whose path filename is exactly <c>IntuneWindowsAgent.msi</c> (query ignored).
/// </summary>
public static bool IsAllowedMsiUrl(string? url)
{
if (string.IsNullOrEmpty(url)) return false;
if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)) return false;
if (uri.Scheme != Uri.UriSchemeHttps) return false;
var host = uri.Host;
var hostAllowed =
host.EndsWith(AllowedHostSuffix, StringComparison.OrdinalIgnoreCase) ||
host.Equals(AllowedHostSuffix.TrimStart('.'), StringComparison.OrdinalIgnoreCase);
if (!hostAllowed) return false;
var fileName = uri.Segments.Length > 0 ? uri.Segments[^1] : string.Empty;
return fileName.Equals(MsiFileName, StringComparison.OrdinalIgnoreCase);
}
public virtual async Task<ImeMsiArchiveResult> ArchiveAsync(
ImeMsiArchiveEnvelope envelope, CancellationToken cancellationToken = default)
{
if (envelope is null || !VersionRegex.IsMatch(envelope.Version ?? string.Empty))
{
_logger.LogWarning(
"ImeMsiArchiver: rejecting version {Version} — not a plain dotted version string",
envelope?.Version ?? "(null)");
return new ImeMsiArchiveResult(false, Statuses.FailedBadVersion, Retryable: false,
null, null, null, null);
}
var version = envelope.Version!;
var blobPath = $"{version}/{MsiFileName}";
string sourceUrl;
if (IsAllowedMsiUrl(envelope.MsiDownloadUrl))
{
sourceUrl = envelope.MsiDownloadUrl!;
}
else
{
if (!string.IsNullOrEmpty(envelope.MsiDownloadUrl))
{
_logger.LogWarning(
"ImeMsiArchiver: event URL rejected by allowlist for version {Version} — using canonical URL",
version);
}
sourceUrl = CanonicalMsiUrl;
}
try
{
var adminConfig = await GetAdminConfigurationAsync();
var maxSizeBytes = (long)adminConfig.MaxImeMsiDownloadSizeMB * 1024 * 1024;
using var cts = new CancellationTokenSource(DownloadTimeout);
using var linked = CancellationTokenSource.CreateLinkedTokenSource(cts.Token, cancellationToken);
var (content, contentLength) = await OpenDownloadAsync(sourceUrl, linked.Token);
string sha256;
long totalBytes;
using (content)
{
// Same fast-reject as the diagnostics proxy: cap of 0 means unlimited.
if (maxSizeBytes > 0 && contentLength > maxSizeBytes)
{
_logger.LogWarning(
"ImeMsiArchiver: download for {Version} rejected — Content-Length {SizeBytes} exceeds limit {MaxSizeBytes}",
version, contentLength, maxSizeBytes);
return new ImeMsiArchiveResult(false, Statuses.FailedTooLarge, Retryable: false,
null, null, contentLength, sourceUrl);
}
// Hash + count while streaming into the blob; the wrapper enforces the cap
// mid-stream too (a lying/absent Content-Length cannot bypass it).
using var guarded = new HashingCappedReadStream(content, maxSizeBytes);
try
{
await UploadMsiAsync(blobPath, guarded, linked.Token);
}
catch (RequestFailedException ex) when (ex.Status == 409)
{
// Queue re-delivery after a successful upload: the archive already has
// this version — that IS the desired end state.
_logger.LogInformation(
"ImeMsiArchiver: blob {BlobPath} already exists — treating re-delivery as archived",
blobPath);
return new ImeMsiArchiveResult(true, Statuses.Archived, Retryable: false,
blobPath, null, null, sourceUrl);
}
sha256 = guarded.Sha256Hex;
totalBytes = guarded.TotalBytes;
}
var provenance = new
{
version,
url = sourceUrl,
urlFromEvent = sourceUrl != CanonicalMsiUrl,
msiMatchedBy = envelope.MsiMatchedBy,
sha256,
msiBytes = totalBytes,
firstSeenTenantId = envelope.TenantId,
firstSeenSessionId = envelope.SessionId,
archivedAtUtc = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ"),
};
await UploadProvenanceAsync(
$"{version}/provenance.json",
System.Text.Json.JsonSerializer.Serialize(provenance,
new System.Text.Json.JsonSerializerOptions { WriteIndented = true }),
linked.Token);
_logger.LogInformation(
"ImeMsiArchiver: archived {Version} ({SizeBytes} bytes, sha256 {Sha256}) from {SourceUrl}",
version, totalBytes, sha256, sourceUrl);
return new ImeMsiArchiveResult(true, Statuses.Archived, Retryable: false,
blobPath, sha256, totalBytes, sourceUrl);
}
catch (SizeCapExceededException)
{
_logger.LogWarning(
"ImeMsiArchiver: download for {Version} exceeded the size cap mid-stream — aborted",
version);
return new ImeMsiArchiveResult(false, Statuses.FailedTooLarge, Retryable: false,
null, null, null, sourceUrl);
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
{
_logger.LogWarning("ImeMsiArchiver: download for {Version} timed out", version);
return new ImeMsiArchiveResult(false, Statuses.FailedTimeout, Retryable: true,
null, null, null, sourceUrl);
}
catch (HttpRequestException ex)
{
_logger.LogWarning(ex, "ImeMsiArchiver: download for {Version} failed", version);
return new ImeMsiArchiveResult(false, Statuses.FailedDownload, Retryable: true,
null, null, null, sourceUrl);
}
catch (OperationCanceledException)
{
// Host shutdown — let the worker's dispatch loop see it and leave the message.
throw;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "ImeMsiArchiver: archiving {Version} failed", version);
return new ImeMsiArchiveResult(false, Statuses.FailedError, Retryable: true,
null, null, null, sourceUrl);
}
}
// -------- Test seams (production defaults hit Azure / the network) --------
/// <summary>Fetches the platform admin configuration (size cap).</summary>
protected virtual Task<AdminConfiguration> GetAdminConfigurationAsync()
=> _adminConfigService.GetConfigurationAsync();
/// <summary>Opens the installer download stream (headers-read, no buffering).</summary>
protected virtual async Task<(Stream Content, long ContentLength)> OpenDownloadAsync(
string url, CancellationToken cancellationToken)
{
var response = await _httpClient.GetAsync(
url, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
response.EnsureSuccessStatusCode();
var content = await response.Content.ReadAsStreamAsync(cancellationToken);
return (content, response.Content.Headers.ContentLength ?? -1);
}
/// <summary>
/// Uploads the installer stream write-once (If-None-Match:*). Implementations must
/// surface a conflict as <see cref="RequestFailedException"/> with status 409.
/// </summary>
protected virtual async Task UploadMsiAsync(string blobPath, Stream content, CancellationToken cancellationToken)
{
var containerClient = _blobStorage.GetContainerClient(Constants.BlobContainers.ImeArchive);
await containerClient.CreateIfNotExistsAsync(cancellationToken: cancellationToken);
var blobClient = containerClient.GetBlobClient(blobPath);
await blobClient.UploadAsync(content, new BlobUploadOptions
{
HttpHeaders = new BlobHttpHeaders { ContentType = "application/octet-stream" },
Conditions = new BlobRequestConditions { IfNoneMatch = ETag.All }
}, cancellationToken);
}
/// <summary>Uploads the provenance sidecar (overwrite allowed — it is derived data).</summary>
protected virtual async Task UploadProvenanceAsync(string blobPath, string json, CancellationToken cancellationToken)
{
var containerClient = _blobStorage.GetContainerClient(Constants.BlobContainers.ImeArchive);
var blobClient = containerClient.GetBlobClient(blobPath);
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json));
await blobClient.UploadAsync(stream, new BlobUploadOptions
{
HttpHeaders = new BlobHttpHeaders { ContentType = "application/json" }
}, cancellationToken);
}
// -------- Streaming helpers --------
private sealed class SizeCapExceededException : Exception
{
}
/// <summary>
/// Read-only pass-through stream that SHA-256-hashes and counts everything read and
/// throws once the byte count exceeds the cap (0 = unlimited). Non-seekable on
/// purpose — the blob SDK then stages blocks instead of demanding Length upfront.
/// </summary>
private sealed class HashingCappedReadStream : Stream
{
private readonly Stream _inner;
private readonly long _maxBytes;
private readonly IncrementalHash _hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
private string? _sha256Hex;
public HashingCappedReadStream(Stream inner, long maxBytes)
{
_inner = inner;
_maxBytes = maxBytes;
}
public long TotalBytes { get; private set; }
public string Sha256Hex => _sha256Hex ??=
Convert.ToHexString(_hash.GetHashAndReset()).ToLowerInvariant();
public override bool CanRead => true;
public override bool CanSeek => false;
public override bool CanWrite => false;
public override long Length => throw new NotSupportedException();
public override long Position
{
get => TotalBytes;
set => throw new NotSupportedException();
}
public override int Read(byte[] buffer, int offset, int count)
{
var read = _inner.Read(buffer, offset, count);
Account(buffer.AsSpan(offset, read));
return read;
}
public override async ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default)
{
var read = await _inner.ReadAsync(buffer, cancellationToken);
Account(buffer.Span[..read]);
return read;
}
public override Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
=> ReadAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask();
private void Account(ReadOnlySpan<byte> data)
{
if (data.IsEmpty) return;
TotalBytes += data.Length;
if (_maxBytes > 0 && TotalBytes > _maxBytes)
throw new SizeCapExceededException();
_hash.AppendData(data);
}
public override void Flush()
{
}
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
protected override void Dispose(bool disposing)
{
if (disposing) _hash.Dispose();
base.Dispose(disposing);
}
}
}
}
@@ -3316,7 +3316,12 @@ namespace AutopilotMonitor.Functions.Services
FirstSeenSessionId = entity.GetString("FirstSeenSessionId") ?? string.Empty,
FirstSeenTenantId = entity.GetString("FirstSeenTenantId") ?? string.Empty,
LastSeenAt = entity.GetDateTime("LastSeenAt") ?? DateTime.MinValue,
SessionCount = entity.GetInt32("SessionCount") ?? 0
SessionCount = entity.GetInt32("SessionCount") ?? 0,
MsiArchiveStatus = entity.GetString("MsiArchiveStatus"),
MsiArchiveBlobPath = entity.GetString("MsiArchiveBlobPath"),
MsiSha256 = entity.GetString("MsiSha256"),
MsiBytes = entity.GetInt64("MsiBytes"),
MsiSourceUrl = entity.GetString("MsiSourceUrl")
});
}
}
@@ -3328,6 +3333,38 @@ namespace AutopilotMonitor.Functions.Services
return results.OrderByDescending(e => e.FirstSeenAt).ToList();
}
/// <summary>
/// Merges the installer-archiving outcome into the version's ImeVersionHistory row
/// (same Merge idiom as the LastSeen update above). Fail-soft: the archive job
/// already logged its own result; a lost status column must never fail the worker.
/// </summary>
public async Task UpdateImeVersionArchiveInfoAsync(
string version, string status, string? blobPath, string? sha256, long? sizeBytes, string? sourceUrl)
{
if (string.IsNullOrEmpty(version) || string.IsNullOrEmpty(status)) return;
try
{
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.ImeVersionHistory);
var mergeEntity = new TableEntity("Global", version)
{
["MsiArchiveStatus"] = status
};
if (!string.IsNullOrEmpty(blobPath)) mergeEntity["MsiArchiveBlobPath"] = blobPath;
if (!string.IsNullOrEmpty(sha256)) mergeEntity["MsiSha256"] = sha256;
if (sizeBytes.HasValue) mergeEntity["MsiBytes"] = sizeBytes.Value;
if (!string.IsNullOrEmpty(sourceUrl)) mergeEntity["MsiSourceUrl"] = sourceUrl;
await tableClient.UpsertEntityAsync(mergeEntity, TableUpdateMode.Merge);
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"Failed to update ImeVersionHistory archive info for version {Version} (status {Status})",
version, status);
}
}
#endregion
#region Excessive-Event Detection
@@ -1295,6 +1295,19 @@ namespace AutopilotMonitor.Shared
/// settings (Soft-Delete + Versioning) provide defence-in-depth.
/// </summary>
public const string CriticalTableBackups = "critical-table-backups";
/// <summary>
/// Permanent platform archive of IME (Intune Management Extension) installer
/// binaries, one virtual folder per version:
/// <c>{version}/IntuneWindowsAgent.msi</c> + <c>{version}/provenance.json</c>.
/// Written by <c>ImeMsiArchiver</c> when a new IME version is first seen across
/// the fleet (companion of <see cref="TableNames.ImeVersionHistory"/>, which is
/// equally KeptByDesign). NOT tenant-scoped — blob paths are version-keyed, so
/// tenant offboarding (Phase 2.E) deliberately never touches this container.
/// No lifecycle rule: the archive is the point (build-to-build diffs of
/// Microsoft's rollouts). Private by default (SDK CreateIfNotExists).
/// </summary>
public const string ImeArchive = "ime-archive";
}
/// <summary>
@@ -1392,6 +1405,18 @@ namespace AutopilotMonitor.Shared
/// <summary>Poison sibling of <see cref="SessionDeletionMaintenance"/>.</summary>
public const string SessionDeletionMaintenancePoison = "session-deletion-maintenance-poison";
/// <summary>
/// IME-MSI archive fan-out. Producer = <c>EventIngestProcessor</c> when
/// <c>RecordImeVersionAsync</c> reports a genuinely NEW IME version (first sighting
/// across the fleet — roughly monthly); consumer = <c>ImeMsiArchiveQueueWorker</c>,
/// which downloads the installer from the CSP-reported
/// <c>msiDownloadUrl</c> (host-allowlisted, size-capped) into
/// <see cref="BlobContainers.ImeArchive"/> and merges the outcome into the
/// <see cref="TableNames.ImeVersionHistory"/> row. Poison suffix <c>-poison</c>,
/// max-dequeue 5.
/// </summary>
public const string ImeMsiArchive = "ime-msi-archive";
}
/// <summary>
@@ -176,6 +176,14 @@ namespace AutopilotMonitor.Shared.DataAccess
Task<bool> RecordImeVersionAsync(string version, string tenantId, string sessionId);
Task<List<ImeVersionHistoryEntry>> GetImeVersionHistoryAsync();
/// <summary>
/// Merges the installer-archiving outcome (<c>ImeMsiArchiver</c>) into the version's
/// <c>ImeVersionHistory</c> row. Fail-soft: storage errors are logged, never thrown.
/// Blob path / hash / size / URL are only written on success (null leaves them absent).
/// </summary>
Task UpdateImeVersionArchiveInfoAsync(
string version, string status, string? blobPath, string? sha256, long? sizeBytes, string? sourceUrl);
// --- Events ---
Task<bool> StoreEventAsync(EnrollmentEvent evt);
Task<List<EnrollmentEvent>> StoreEventsBatchAsync(List<EnrollmentEvent> events);
@@ -460,6 +460,24 @@ namespace AutopilotMonitor.Shared.Models
/// </summary>
public bool AutoApproveNewTenants { get; set; } = false;
/// <summary>
/// When true, the first fleet-wide sighting of a new IME version enqueues an
/// archive job (<c>ime-msi-archive</c> queue) that downloads the installer from the
/// CSP-reported URL into the <c>ime-archive</c> blob container. Roughly one download
/// per Microsoft IME release (~monthly). Default true — flip off to pause archiving
/// (queued messages then wait, they are not dropped). Round-tripped via the 4-file
/// web chain (memory <c>feedback_admin_config_ui_roundtrip</c>).
/// </summary>
public bool ImeMsiArchivingEnabled { get; set; } = true;
/// <summary>
/// Size cap for the IME installer download in MB (Content-Length preflight AND
/// streamed-byte guard). The MSI is ~12 MB today; the generous cap exists only so a
/// tampered/wrong URL cannot stream gigabytes into the archive while still never
/// missing a legitimately grown installer. 0 = no limit. Default: 250 MB.
/// </summary>
public int MaxImeMsiDownloadSizeMB { get; set; } = 250;
/// <summary>
/// When true, the dual app-registration self-service homing flip is active: the Graph
/// consent flow funnels legacy-homed tenants to the primary (new) app registration and
@@ -0,0 +1,44 @@
using System;
namespace AutopilotMonitor.Shared.Models
{
/// <summary>
/// Queue-message envelope for the <c>ime-msi-archive</c> queue. Enqueued by
/// <c>EventIngestProcessor</c> when an <c>ime_agent_version</c> event carries a version
/// the platform has never seen before (RecordImeVersionAsync insert succeeded). The
/// worker downloads the IME installer and archives it into the <c>ime-archive</c> blob
/// container so every fleet-observed IME build stays available for later decompilation
/// and build-to-build diffing, even after Microsoft's versionless CDN URL has moved on.
/// </summary>
public sealed class ImeMsiArchiveEnvelope
{
/// <summary>Schema version — bump on breaking envelope changes so consumers can reject or migrate.</summary>
public string EnvelopeVersion { get; set; } = "1";
/// <summary>The newly sighted IME version, e.g. <c>1.104.102.0</c> (blob folder name).</summary>
public string Version { get; set; } = string.Empty;
/// <summary>
/// The EnterpriseDesktopAppManagement-CSP <c>CurrentDownloadUrl</c> from the event's
/// <c>msiDownloadUrl</c> enrichment. Null when the reporting agent predates the
/// enrichment — the worker then falls back to the canonical versionless CDN URL
/// (safe: a brand-new version is by definition what that URL currently serves).
/// </summary>
public string? MsiDownloadUrl { get; set; }
/// <summary>
/// The event's <c>msiMatchedBy</c> companion (<c>productVersion</c> = URL is
/// version-authoritative, <c>fileName</c> = registry drift). Provenance only.
/// </summary>
public string? MsiMatchedBy { get; set; }
/// <summary>First-seen tenant — provenance only, never used for auth or paths.</summary>
public string TenantId { get; set; } = string.Empty;
/// <summary>First-seen session — provenance only.</summary>
public string SessionId { get; set; } = string.Empty;
/// <summary>UTC time the producer enqueued the message; useful for measuring queue lag.</summary>
public DateTime EnqueuedAt { get; set; }
}
}
@@ -575,5 +575,25 @@ namespace AutopilotMonitor.Shared.Models
public string FirstSeenTenantId { get; set; } = string.Empty;
public DateTime LastSeenAt { get; set; }
public int SessionCount { get; set; }
/// <summary>
/// Outcome of the automatic installer archiving (<c>ImeMsiArchiver</c>):
/// <c>Archived</c> or a <c>Failed:*</c> status; null for versions sighted before the
/// feature existed or while the archive job is still queued. Like the FirstSeen*
/// identifiers, the archive fields are only serialized for Global Admin callers.
/// </summary>
public string? MsiArchiveStatus { get; set; }
/// <summary>Blob path inside the <c>ime-archive</c> container, e.g. <c>1.104.102.0/IntuneWindowsAgent.msi</c>.</summary>
public string? MsiArchiveBlobPath { get; set; }
/// <summary>SHA-256 of the archived installer (hex, lowercase).</summary>
public string? MsiSha256 { get; set; }
/// <summary>Size of the archived installer in bytes.</summary>
public long? MsiBytes { get; set; }
/// <summary>The URL the installer was downloaded from (CSP-reported or canonical fallback).</summary>
public string? MsiSourceUrl { get; set; }
}
}
@@ -50,6 +50,10 @@ interface AdminConfigContextValue {
setAutoApproveNewTenants: (value: boolean) => void;
selfServiceAppHomingEnabled: boolean;
setSelfServiceAppHomingEnabled: (value: boolean) => void;
imeMsiArchivingEnabled: boolean;
setImeMsiArchivingEnabled: (value: boolean) => void;
maxImeMsiDownloadSizeMB: number;
setMaxImeMsiDownloadSizeMB: (value: number) => void;
// Diagnostics log paths
globalDiagPaths: DiagnosticsLogPath[];
@@ -152,6 +156,8 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode })
const [sessionDeletionKillSwitch, setSessionDeletionKillSwitch] = useState(false);
const [autoApproveNewTenants, setAutoApproveNewTenants] = useState(false);
const [selfServiceAppHomingEnabled, setSelfServiceAppHomingEnabled] = useState(false);
const [imeMsiArchivingEnabled, setImeMsiArchivingEnabled] = useState(true);
const [maxImeMsiDownloadSizeMB, setMaxImeMsiDownloadSizeMB] = useState(250);
// Diagnostics Log Paths state
const [globalDiagPaths, setGlobalDiagPaths] = useState<DiagnosticsLogPath[]>([]);
@@ -215,6 +221,8 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode })
setSessionDeletionKillSwitch(data.sessionDeletionKillSwitch ?? false);
setAutoApproveNewTenants(data.autoApproveNewTenants ?? false);
setSelfServiceAppHomingEnabled(data.selfServiceAppHomingEnabled ?? false);
setImeMsiArchivingEnabled(data.imeMsiArchivingEnabled ?? true);
setMaxImeMsiDownloadSizeMB(data.maxImeMsiDownloadSizeMB ?? 250);
try {
setGlobalDiagPaths(data.diagnosticsGlobalLogPathsJson ? JSON.parse(data.diagnosticsGlobalLogPathsJson) : []);
} catch {
@@ -332,6 +340,8 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode })
sessionDeletionKillSwitch,
autoApproveNewTenants,
selfServiceAppHomingEnabled,
imeMsiArchivingEnabled,
maxImeMsiDownloadSizeMB,
};
const response = await authenticatedFetch(api.globalConfig.get(), getAccessToken, {
@@ -358,7 +368,7 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode })
} finally {
setSavingConfig(false);
}
}, [isGlobalAdmin, adminConfig, globalRateLimit, userRateLimit, globalAdminRateLimit, platformStatsBlobSasUrl, agentMigrateApiBaseUrl, agentMigrateTenantOverridesJson, collectorIdleTimeoutMinutes, desktopDetectorNoCandidateTimeoutMinutes, opsEventRetentionDays, slaNotificationCooldownHours, allowAgentDowngrade, modernDeploymentHarmlessEventIds, enableIndexDualWrite, sessionDeletionKillSwitch, autoApproveNewTenants, selfServiceAppHomingEnabled, getAccessToken]);
}, [isGlobalAdmin, adminConfig, globalRateLimit, userRateLimit, globalAdminRateLimit, platformStatsBlobSasUrl, agentMigrateApiBaseUrl, agentMigrateTenantOverridesJson, collectorIdleTimeoutMinutes, desktopDetectorNoCandidateTimeoutMinutes, opsEventRetentionDays, slaNotificationCooldownHours, allowAgentDowngrade, modernDeploymentHarmlessEventIds, enableIndexDualWrite, sessionDeletionKillSwitch, autoApproveNewTenants, selfServiceAppHomingEnabled, imeMsiArchivingEnabled, maxImeMsiDownloadSizeMB, getAccessToken]);
// Reset admin config
const handleResetAdminConfig = useCallback(() => {
@@ -379,6 +389,8 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode })
setSessionDeletionKillSwitch(adminConfig.sessionDeletionKillSwitch ?? false);
setAutoApproveNewTenants(adminConfig.autoApproveNewTenants ?? false);
setSelfServiceAppHomingEnabled(adminConfig.selfServiceAppHomingEnabled ?? false);
setImeMsiArchivingEnabled(adminConfig.imeMsiArchivingEnabled ?? true);
setMaxImeMsiDownloadSizeMB(adminConfig.maxImeMsiDownloadSizeMB ?? 250);
try {
setGlobalDiagPaths(adminConfig.diagnosticsGlobalLogPathsJson ? JSON.parse(adminConfig.diagnosticsGlobalLogPathsJson) : []);
} catch {
@@ -525,6 +537,8 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode })
sessionDeletionKillSwitch, setSessionDeletionKillSwitch,
autoApproveNewTenants, setAutoApproveNewTenants,
selfServiceAppHomingEnabled, setSelfServiceAppHomingEnabled,
imeMsiArchivingEnabled, setImeMsiArchivingEnabled,
maxImeMsiDownloadSizeMB, setMaxImeMsiDownloadSizeMB,
globalDiagPaths, setGlobalDiagPaths, savingDiagPaths,
opsAlertRules, setOpsAlertRules,
opsAlertTelegramEnabled, setOpsAlertTelegramEnabled,
@@ -36,6 +36,10 @@ interface AdminConfigSettingsSectionProps {
setAutoApproveNewTenants: (value: boolean) => void;
selfServiceAppHomingEnabled: boolean;
setSelfServiceAppHomingEnabled: (value: boolean) => void;
imeMsiArchivingEnabled: boolean;
setImeMsiArchivingEnabled: (value: boolean) => void;
maxImeMsiDownloadSizeMB: number;
setMaxImeMsiDownloadSizeMB: (value: number) => void;
onSave: () => Promise<void>;
onReset: () => void;
}
@@ -74,6 +78,10 @@ export function AdminConfigSettingsSection({
setAutoApproveNewTenants,
selfServiceAppHomingEnabled,
setSelfServiceAppHomingEnabled,
imeMsiArchivingEnabled,
setImeMsiArchivingEnabled,
maxImeMsiDownloadSizeMB,
setMaxImeMsiDownloadSizeMB,
onSave,
onReset,
}: AdminConfigSettingsSectionProps) {
@@ -375,6 +383,47 @@ export function AdminConfigSettingsSection({
</label>
</div>
<div>
<label className="flex items-start space-x-3 cursor-pointer">
<input
type="checkbox"
checked={imeMsiArchivingEnabled}
onChange={(e) => setImeMsiArchivingEnabled(e.target.checked)}
className="mt-1 h-5 w-5 rounded border-indigo-300 dark:border-indigo-600 text-indigo-600 accent-indigo-600 focus:ring-indigo-500"
/>
<span>
<span className="text-indigo-900 dark:text-indigo-100 font-medium">IME installer archiving</span>
<p className="text-sm text-indigo-800 dark:text-gray-300 mt-1">
When <strong>on</strong>, the first fleet-wide sighting of a new IME version downloads
the installer (from the CSP-reported, allowlisted URL) into the permanent
<code className="ml-1 text-xs bg-indigo-100 dark:bg-indigo-900 dark:text-indigo-200 px-1 rounded">ime-archive</code> blob
container — roughly one small download per Microsoft IME release. The outcome is
visible per version in the IME version history. When <strong>off</strong>, queued
archive jobs wait (they are not dropped). Default <strong>on</strong>.
</p>
</span>
</label>
</div>
<div>
<label className="block">
<span className="text-indigo-900 dark:text-indigo-100 font-medium">IME installer download size cap (MB)</span>
<p className="text-sm text-indigo-800 dark:text-gray-300 mb-2">
Safety cap for the IME installer download (the MSI is ~12 MB today). Generous on
purpose so a legitimately grown installer is never missed, while a wrong URL can
still not stream gigabytes into the archive. 0 = unlimited. Default 250.
</p>
<input
type="number"
min="0"
max="10000"
value={maxImeMsiDownloadSizeMB}
onChange={(e) => setMaxImeMsiDownloadSizeMB(parseInt(e.target.value) || 0)}
className="mt-1 block w-full px-4 py-2 border border-indigo-300 dark:border-indigo-600 rounded-lg bg-white dark:bg-gray-700 text-gray-900 dark:text-gray-100 placeholder-gray-400 dark:placeholder-gray-500 focus:outline-none focus:ring-2 focus:ring-green-500 focus:border-green-500 transition-colors"
/>
</label>
</div>
<div>
<label className="flex items-start space-x-3 cursor-pointer">
<input
@@ -24,6 +24,8 @@ export function SectionGlobalSettings() {
sessionDeletionKillSwitch, setSessionDeletionKillSwitch,
autoApproveNewTenants, setAutoApproveNewTenants,
selfServiceAppHomingEnabled, setSelfServiceAppHomingEnabled,
imeMsiArchivingEnabled, setImeMsiArchivingEnabled,
maxImeMsiDownloadSizeMB, setMaxImeMsiDownloadSizeMB,
handleSaveAdminConfig, handleResetAdminConfig,
} = useAdminConfig();
@@ -66,6 +68,10 @@ export function SectionGlobalSettings() {
setAutoApproveNewTenants={setAutoApproveNewTenants}
selfServiceAppHomingEnabled={selfServiceAppHomingEnabled}
setSelfServiceAppHomingEnabled={setSelfServiceAppHomingEnabled}
imeMsiArchivingEnabled={imeMsiArchivingEnabled}
setImeMsiArchivingEnabled={setImeMsiArchivingEnabled}
maxImeMsiDownloadSizeMB={maxImeMsiDownloadSizeMB}
setMaxImeMsiDownloadSizeMB={setMaxImeMsiDownloadSizeMB}
onSave={handleSaveAdminConfig}
onReset={handleResetAdminConfig}
/>
@@ -99,6 +99,17 @@ export interface AdminConfiguration {
* only Global Admins can flip while it is off. Default false.
*/
selfServiceAppHomingEnabled?: boolean;
/**
* When true, the first fleet-wide sighting of a new IME version downloads the
* installer into the permanent ime-archive blob container (ime-msi-archive queue
* worker). When false, queued archive jobs wait — they are not dropped. Default true.
*/
imeMsiArchivingEnabled?: boolean;
/**
* Size cap for the IME installer download in MB (preflight + mid-stream guard).
* 0 = unlimited. Default 250.
*/
maxImeMsiDownloadSizeMB?: number;
}
export interface OpsAlertRule {
@@ -50,6 +50,8 @@ export const SHARED_MANIFEST = {
"vulnerabilityCorrelationEnabled",
"enableIndexDualWrite",
"autoApproveNewTenants",
"imeMsiArchivingEnabled",
"maxImeMsiDownloadSizeMB",
"selfServiceAppHomingEnabled",
"sessionDeletionKillSwitch",
"vulnerabilityDataLastSyncUtc",
@@ -46,6 +46,8 @@
"vulnerabilityCorrelationEnabled",
"enableIndexDualWrite",
"autoApproveNewTenants",
"imeMsiArchivingEnabled",
"maxImeMsiDownloadSizeMB",
"selfServiceAppHomingEnabled",
"sessionDeletionKillSwitch",
"vulnerabilityDataLastSyncUtc",