fix(ime-archive): heal provenance.json lost behind the write-once 409 path

Daily-review finding: the MSI upload is write-once (If-None-Match:*), so a first attempt that died between the MSI and the provenance upload funneled every retry into the 409 short-circuit — which returned "Archived" without ever writing provenance.json (permanently lost). The 409 path now checks for the sidecar and, when missing, rehashes the ARCHIVED bytes and writes it; heal failures ride the normal retryable ladder. Also closes the review's test gaps: ImeMsiArchiveQueueWorker + producer worker-level tests (internal QueueClient test ctors, SessionDeletionWorker pattern), CorporateIdentifierValidator null/whitespace + OData special-char + page-cap-transient tests, registry observer helper tests (TerminalOutcome/ChangedFields/ClassifyStatusServiceStatus). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-10-24 12:42:38 +00:00 · 2026-08-18 10:21:26 +02:00
parent 0ed1e6f542
commit e521fd5cf4
8 changed files with 756 additions and 24 deletions
@@ -59,6 +59,71 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
        public void ClassifyEnforcementState_bands(int state, string expected)
            => Assert.Equal(expected, ImeRegistryAppStateObserver.ClassifyEnforcementState(state));

+        [Theory]
+        [InlineData(1000, "installed")]
+        [InlineData(1999, "installed")]
+        [InlineData(3000, "failed")]
+        [InlineData(2000, "notApplicable")]
+        [InlineData(2999, "notApplicable")]
+        [InlineData(0, null)]
+        [InlineData(999, null)]
+        [InlineData(3001, null)]
+        [InlineData(4000, null)]
+        public void ClassifyStatusServiceStatus_bands(int status, string? expected)
+            => Assert.Equal(expected, ImeRegistryAppStateObserver.ClassifyStatusServiceStatus(status));
+
+        [Fact]
+        public void TerminalOutcome_prefers_enforcement_state_then_status_service()
+        {
+            // Enforcement state alone decides when terminal.
+            Assert.Equal("success", ImeRegistryAppStateObserver.TerminalOutcome(Entry(enforcementState: 1000)));
+            Assert.Equal("error", ImeRegistryAppStateObserver.TerminalOutcome(Entry(enforcementState: 5003)));
+            // Non-terminal enforcement bands defer to the StatusService pillar.
+            Assert.Null(ImeRegistryAppStateObserver.TerminalOutcome(Entry(enforcementState: 2009)));
+            Assert.Equal("error", ImeRegistryAppStateObserver.TerminalOutcome(
+                Entry(enforcementState: 2009, statusServiceStatus: 3000)));
+            Assert.Equal("success", ImeRegistryAppStateObserver.TerminalOutcome(Entry(statusServiceStatus: 1000)));
+            Assert.Equal("error", ImeRegistryAppStateObserver.TerminalOutcome(Entry(statusServiceStatus: 3000)));
+            // requirementsNotMet / notApplicable / empty entries are not judgeable.
+            Assert.Null(ImeRegistryAppStateObserver.TerminalOutcome(Entry(enforcementState: 3000)));
+            Assert.Null(ImeRegistryAppStateObserver.TerminalOutcome(Entry(statusServiceStatus: 2500)));
+            Assert.Null(ImeRegistryAppStateObserver.TerminalOutcome(Entry()));
+        }
+
+        [Fact]
+        public void ChangedFields_reports_each_field_and_ignores_espPhase_casing()
+        {
+            var prev = Entry(enforcementState: 2009, errorCode: null, exitCode: null,
+                statusServiceStatus: null, espTracked: false, espPhase: "DevicePreparation");
+            var next = Entry(enforcementState: 1000, errorCode: 0, exitCode: 0,
+                statusServiceStatus: 1000, espTracked: true, espPhase: "devicepreparation");
+
+            var changed = ImeRegistryAppStateObserver.ChangedFields(prev, next);
+
+            Assert.Equal(
+                new[] { "enforcementState", "errorCode", "exitCode", "statusServiceStatus", "espTracked" },
+                changed);
+            // Case-only espPhase difference is not a change; identity diffs to empty.
+            Assert.DoesNotContain("espPhase", changed);
+            Assert.Empty(ImeRegistryAppStateObserver.ChangedFields(next, next));
+        }
+
+        [Fact]
+        public void ChangedFields_null_previous_reports_populated_fields_of_new_entry()
+        {
+            var fresh = Entry(enforcementState: 2000, espTracked: true, espPhase: "AccountSetup");
+
+            var changed = ImeRegistryAppStateObserver.ChangedFields(null, fresh);
+
+            Assert.Contains("enforcementState", changed);
+            Assert.Contains("espTracked", changed);
+            Assert.Contains("espPhase", changed);
+            // Fields that are null on both sides are not "changes" for a brand-new entry.
+            Assert.DoesNotContain("errorCode", changed);
+            Assert.DoesNotContain("exitCode", changed);
+            Assert.DoesNotContain("statusServiceStatus", changed);
+        }
+
        [Fact]
        public void DiffSnapshots_reports_only_changed_fields()
        {
@@ -264,6 +329,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
            return (sink, clock, observer);
        }

+        private static AppRegistryEntry Entry(
+            int? enforcementState = null, long? errorCode = null, int? exitCode = null,
+            int? statusServiceStatus = null, bool espTracked = false, string? espPhase = null)
+        {
+            var entry = new AppRegistryEntry($"{ImeRegistrySnapshot.DeviceContext}|{App1}", ImeRegistrySnapshot.DeviceContext, App1);
+            entry.EnforcementState = enforcementState;
+            entry.ErrorCode = errorCode;
+            entry.ExitCode = exitCode;
+            entry.StatusServiceStatus = statusServiceStatus;
+            entry.EspTracked = espTracked;
+            entry.EspPhase = espPhase;
+            return entry;
+        }
+
        private static ImeRegistrySnapshot Snapshot(params (string appId, int enforcementState, long? errorCode)[] apps)
        {
            var snapshot = new ImeRegistrySnapshot();
@@ -0,0 +1,124 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Azure;
+using Azure.Storage.Queues;
+using Azure.Storage.Queues.Models;
+using AutopilotMonitor.Functions.Services.Ime;
+using AutopilotMonitor.Shared.Models;
+using Microsoft.Extensions.Logging.Abstractions;
+using Moq;
+using Newtonsoft.Json;
+using Xunit;
+
+namespace AutopilotMonitor.Functions.Tests;
+
+/// <summary>
+/// Producer tests for the ime-msi-archive queue (daily review 2026-08-18: previously
+/// untested). Contract: the caller is the ingest path's fire-and-forget continuation, so
+/// NOTHING may throw — version-less envelopes are skipped, send failures are swallowed,
+/// and a failed CreateIfNotExists is retried on the next enqueue instead of latching.
+/// </summary>
+public class AzureQueueImeMsiArchiveProducerTests
+{
+    private const string Version = "1.104.102.0";
+
+    [Fact]
+    public async Task EnqueueAsync_sends_envelope_json_and_ensures_queue_once()
+    {
+        var (producer, queue, sentBodies) = BuildSut();
+
+        await producer.EnqueueAsync(Envelope());
+        await producer.EnqueueAsync(Envelope());
+
+        Assert.Equal(2, sentBodies.Count);
+        var roundTripped = JsonConvert.DeserializeObject<ImeMsiArchiveEnvelope>(sentBodies[0])!;
+        Assert.Equal(Version, roundTripped.Version);
+        Assert.Equal(ImeMsiArchiver.CanonicalMsiUrl, roundTripped.MsiDownloadUrl);
+        Assert.Equal("11111111-1111-1111-1111-111111111111", roundTripped.TenantId);
+        // Queue existence is ensured once, then latched — not re-checked per message.
+        queue.Verify(q => q.CreateIfNotExistsAsync(
+            It.IsAny<IDictionary<string, string>>(), It.IsAny<CancellationToken>()),
+            Times.Once);
+    }
+
+    [Fact]
+    public async Task EnqueueAsync_skips_envelope_without_version()
+    {
+        var (producer, queue, sentBodies) = BuildSut();
+
+        await producer.EnqueueAsync(new ImeMsiArchiveEnvelope { Version = string.Empty });
+        await producer.EnqueueAsync(null!);
+
+        Assert.Empty(sentBodies);
+        queue.Verify(q => q.CreateIfNotExistsAsync(
+            It.IsAny<IDictionary<string, string>>(), It.IsAny<CancellationToken>()),
+            Times.Never);
+    }
+
+    [Fact]
+    public async Task EnqueueAsync_send_failure_is_swallowed()
+    {
+        var (producer, queue, _) = BuildSut();
+        queue.Setup(q => q.SendMessageAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()))
+            .ThrowsAsync(new RequestFailedException(503, "queue busy"));
+
+        // Fire-and-forget continuation on the ingest path — must never throw.
+        await producer.EnqueueAsync(Envelope());
+    }
+
+    [Fact]
+    public async Task EnqueueAsync_retries_queue_creation_after_failure()
+    {
+        var (producer, queue, sentBodies) = BuildSut();
+        queue.SetupSequence(q => q.CreateIfNotExistsAsync(
+                It.IsAny<IDictionary<string, string>>(), It.IsAny<CancellationToken>()))
+            .ThrowsAsync(new RequestFailedException(500, "storage down"))
+            .ReturnsAsync((Response?)null);
+
+        await producer.EnqueueAsync(Envelope()); // ensure fails — not latched
+        await producer.EnqueueAsync(Envelope()); // ensure retried and succeeds
+
+        queue.Verify(q => q.CreateIfNotExistsAsync(
+            It.IsAny<IDictionary<string, string>>(), It.IsAny<CancellationToken>()),
+            Times.Exactly(2));
+        // Both messages were still attempted (fail-soft send after a failed ensure).
+        Assert.Equal(2, sentBodies.Count);
+    }
+
+    // ============================================================ Helpers ====
+
+    private static ImeMsiArchiveEnvelope Envelope() => new()
+    {
+        Version = Version,
+        MsiDownloadUrl = ImeMsiArchiver.CanonicalMsiUrl,
+        MsiMatchedBy = "productVersion",
+        TenantId = "11111111-1111-1111-1111-111111111111",
+        SessionId = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+        EnqueuedAt = DateTime.UtcNow,
+    };
+
+    private static (AzureQueueImeMsiArchiveProducer Producer, Mock<QueueClient> Queue, List<string> SentBodies) BuildSut()
+    {
+        var queue = new Mock<QueueClient>();
+        var sentBodies = new List<string>();
+
+        queue.Setup(q => q.CreateIfNotExistsAsync(
+                It.IsAny<IDictionary<string, string>>(), It.IsAny<CancellationToken>()))
+            .ReturnsAsync((Response?)null);
+        queue.Setup(q => q.SendMessageAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()))
+            .Returns<string, CancellationToken>((body, _) =>
+            {
+                sentBodies.Add(body);
+                var receipt = QueuesModelFactory.SendReceipt(
+                    "msg", DateTimeOffset.UtcNow, DateTimeOffset.UtcNow.AddMinutes(7),
+                    "pop", DateTimeOffset.UtcNow);
+                return Task.FromResult(Response.FromValue(receipt, new Mock<Response>().Object));
+            });
+
+        var producer = new AzureQueueImeMsiArchiveProducer(
+            queue.Object, NullLogger<AzureQueueImeMsiArchiveProducer>.Instance);
+        return (producer, queue, sentBodies);
+    }
+}
@@ -247,4 +247,64 @@ public class CorporateIdentifierValidatorTests
        Assert.False(result.IsValid);
        Assert.True(result.IsTransient);
    }
+
+    [Theory]
+    [InlineData(null, "Virtual Machine", Serial)]
+    [InlineData("Microsoft Corporation", null, Serial)]
+    [InlineData("Microsoft Corporation", "Virtual Machine", null)]
+    [InlineData("", "Virtual Machine", Serial)]
+    [InlineData("Microsoft Corporation", "", Serial)]
+    [InlineData("Microsoft Corporation", "Virtual Machine", "")]
+    [InlineData("   ", "Virtual Machine", Serial)]
+    [InlineData("Microsoft Corporation", "Virtual Machine", "   ")]
+    public async Task ValidateAsync_missingHeaderComponent_isDefinitive_withoutGraphCall(
+        string? manufacturer, string? model, string? serial)
+    {
+        // Agents that fail to read a hardware property send the header empty — that is a
+        // definitive rejection (retries cannot conjure the value), and Graph must not be hit.
+        var (sut, handler) = BuildSut(Json("""{"value":[]}"""));
+
+        var result = await sut.ValidateAsync(TenantId, manufacturer, model, serial);
+
+        Assert.False(result.IsValid);
+        Assert.False(result.IsTransient);
+        Assert.Contains("not provided", result.ErrorMessage);
+        Assert.Empty(handler.Requests);
+    }
+
+    [Fact]
+    public async Task ValidateAsync_specialCharsInSerial_neverReachOdataFilter()
+    {
+        // The contains() narrowing uses the NORMALIZED (alphanumeric-only) serial, so an
+        // OData-breaking quote in a raw serial can neither corrupt nor inject into the filter.
+        const string hostileSerial = "O'Brien-123'&$top=9999";
+        var (sut, handler) = BuildSut(Json("""{"value":[]}"""));
+
+        var result = await sut.ValidateAsync(TenantId, "Microsoft Corporation", "Virtual Machine", hostileSerial);
+
+        Assert.False(result.IsValid);
+        var url = Uri.UnescapeDataString(Assert.Single(handler.Requests).RequestUri!.ToString());
+        Assert.Contains("contains(importedDeviceIdentifier,'OBRIEN123TOP9999')", url);
+        Assert.DoesNotContain("O'Brien", url);
+    }
+
+    [Fact]
+    public async Task ValidateAsync_pageCapExceeded_isTransient_andNotCached()
+    {
+        // Stopping mid-list must NOT be classified "not found" — a cached false negative would
+        // block a legitimately registered device for the negative-cache TTL.
+        var (sut, handler) = BuildSut(Json("""
+            {"value":[{"importedDeviceIdentityType":"manufacturerModelSerial","importedDeviceIdentifier":"DELL,XPS,OTHER1"}],
+             "@odata.nextLink":"https://graph.microsoft.com/beta/deviceManagement/importedDeviceIdentities?$skiptoken=more"}
+            """));
+
+        var result = await sut.ValidateAsync(TenantId, "Microsoft Corporation", "Virtual Machine", Serial);
+
+        Assert.False(result.IsValid);
+        Assert.True(result.IsTransient);
+        Assert.Contains("page budget", result.ErrorMessage);
+        // Filtered scan pages up to its cap of 5, and the in-call retry runs a second sweep —
+        // proof the transient path went through the retry loop instead of caching.
+        Assert.Equal(10, handler.Requests.Count);
+    }
 }
@@ -0,0 +1,261 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Azure;
+using Azure.Storage.Queues;
+using Azure.Storage.Queues.Models;
+using AutopilotMonitor.Functions.Services;
+using AutopilotMonitor.Functions.Services.Ime;
+using AutopilotMonitor.Shared.DataAccess;
+using AutopilotMonitor.Shared.Models;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Logging.Abstractions;
+using Moq;
+using Newtonsoft.Json;
+using Xunit;
+
+namespace AutopilotMonitor.Functions.Tests;
+
+/// <summary>
+/// Worker-level tests for the ime-msi-archive queue consumer (daily review 2026-08-18:
+/// previously untested). The worker's contract: merge the archiver's outcome onto the
+/// ImeVersionHistory row on EVERY attempt (also failures, so operators see a stuck
+/// download), delete the message for terminal outcomes, rethrow for retryable ones
+/// (visibility-timeout ladder), pause — not drop — while ImeMsiArchivingEnabled is off,
+/// and drop version-less envelopes without invoking the archiver.
+/// </summary>
+public class ImeMsiArchiveQueueWorkerTests
+{
+    private const string Version = "1.104.102.0";
+
+    [Fact]
+    public async Task Worker_success_merges_row_and_deletes_message()
+    {
+        var harness = new Harness();
+        harness.Archiver.Result = new ImeMsiArchiveResult(
+            true, ImeMsiArchiver.Statuses.Archived, Retryable: false,
+            $"{Version}/IntuneWindowsAgent.msi", "abc123", 42L, ImeMsiArchiver.CanonicalMsiUrl);
+        harness.EnqueueMessage(JsonConvert.SerializeObject(Envelope()));
+
+        await harness.RunUntilAsync(() => harness.MainQueueDeleted());
+
+        harness.Repo.Verify(r => r.UpdateImeVersionArchiveInfoAsync(
+            Version, ImeMsiArchiver.Statuses.Archived,
+            $"{Version}/IntuneWindowsAgent.msi", "abc123", 42L, ImeMsiArchiver.CanonicalMsiUrl),
+            Times.Once);
+        harness.MainQueue.Verify(q => q.DeleteMessageAsync(
+            It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
+            Times.Once);
+    }
+
+    [Fact]
+    public async Task Worker_retryable_failure_merges_row_but_leaves_message()
+    {
+        var harness = new Harness();
+        harness.Archiver.Result = new ImeMsiArchiveResult(
+            false, ImeMsiArchiver.Statuses.FailedDownload, Retryable: true,
+            null, null, null, ImeMsiArchiver.CanonicalMsiUrl);
+        harness.EnqueueMessage(JsonConvert.SerializeObject(Envelope()));
+
+        await harness.RunUntilAsync(() => harness.Archiver.Calls > 0);
+
+        // Row always tells the truth — the failure status IS merged...
+        harness.Repo.Verify(r => r.UpdateImeVersionArchiveInfoAsync(
+            Version, ImeMsiArchiver.Statuses.FailedDownload,
+            null, null, null, ImeMsiArchiver.CanonicalMsiUrl),
+            Times.AtLeastOnce);
+        // ...but the message stays for the visibility-timeout retry → poison ladder.
+        harness.MainQueue.Verify(q => q.DeleteMessageAsync(
+            It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
+            Times.Never);
+    }
+
+    [Fact]
+    public async Task Worker_permanent_failure_merges_row_and_completes_message()
+    {
+        var harness = new Harness();
+        harness.Archiver.Result = new ImeMsiArchiveResult(
+            false, ImeMsiArchiver.Statuses.FailedTooLarge, Retryable: false,
+            null, null, 999L, ImeMsiArchiver.CanonicalMsiUrl);
+        harness.EnqueueMessage(JsonConvert.SerializeObject(Envelope()));
+
+        await harness.RunUntilAsync(() => harness.MainQueueDeleted());
+
+        harness.Repo.Verify(r => r.UpdateImeVersionArchiveInfoAsync(
+            Version, ImeMsiArchiver.Statuses.FailedTooLarge,
+            null, null, 999L, ImeMsiArchiver.CanonicalMsiUrl),
+            Times.Once);
+        harness.MainQueue.Verify(q => q.DeleteMessageAsync(
+            It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
+            Times.Once);
+    }
+
+    [Fact]
+    public async Task Worker_pauses_while_archiving_disabled()
+    {
+        var harness = new Harness(archivingEnabled: false);
+        harness.EnqueueMessage(JsonConvert.SerializeObject(Envelope()));
+
+        await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
+
+        // Pause, not drop: no receive, no archive attempt — the message parks in the queue.
+        harness.MainQueue.Verify(q => q.ReceiveMessagesAsync(
+            It.IsAny<int>(), It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()),
+            Times.Never);
+        Assert.Equal(0, harness.Archiver.Calls);
+    }
+
+    [Fact]
+    public async Task Worker_drops_envelope_without_version_before_archiver()
+    {
+        var harness = new Harness();
+        harness.EnqueueMessage(JsonConvert.SerializeObject(new ImeMsiArchiveEnvelope
+        {
+            Version = string.Empty,
+            EnqueuedAt = DateTime.UtcNow,
+        }));
+
+        await harness.RunUntilAsync(() => harness.MainQueueDeleted());
+
+        Assert.Equal(0, harness.Archiver.Calls);
+        harness.Repo.Verify(r => r.UpdateImeVersionArchiveInfoAsync(
+            It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string?>(),
+            It.IsAny<string?>(), It.IsAny<long?>(), It.IsAny<string?>()),
+            Times.Never);
+    }
+
+    // ============================================================ Helpers ====
+
+    private static ImeMsiArchiveEnvelope Envelope() => new()
+    {
+        Version = Version,
+        MsiDownloadUrl = ImeMsiArchiver.CanonicalMsiUrl,
+        MsiMatchedBy = "productVersion",
+        TenantId = "11111111-1111-1111-1111-111111111111",
+        SessionId = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+        EnqueuedAt = DateTime.UtcNow,
+    };
+
+    /// <summary>Scripted archiver — the worker only consumes the classified result.</summary>
+    private sealed class StubArchiver : ImeMsiArchiver
+    {
+        private int _calls;
+
+        public ImeMsiArchiveResult Result { get; set; } = new(
+            true, Statuses.Archived, Retryable: false, null, null, null, null);
+
+        public int Calls => Volatile.Read(ref _calls);
+
+        public override Task<ImeMsiArchiveResult> ArchiveAsync(
+            ImeMsiArchiveEnvelope envelope, CancellationToken cancellationToken = default)
+        {
+            Interlocked.Increment(ref _calls);
+            return Task.FromResult(Result);
+        }
+    }
+
+    /// <summary>Mock-queue poll-loop harness, same shape as SessionDeletionWorkerTests.</summary>
+    private sealed class Harness
+    {
+        public Mock<QueueClient> MainQueue { get; }
+        public Mock<QueueClient> PoisonQueue { get; }
+        public StubArchiver Archiver { get; } = new();
+        public Mock<ISessionRepository> Repo { get; } = new();
+        public ImeMsiArchiveQueueWorker Sut { get; }
+
+        private readonly Queue<QueueMessage> _pendingMessages = new();
+
+        public Harness(bool archivingEnabled = true)
+        {
+            MainQueue = new Mock<QueueClient>();
+            PoisonQueue = new Mock<QueueClient>();
+
+            MainQueue.Setup(q => q.CreateIfNotExistsAsync(
+                    It.IsAny<IDictionary<string, string>>(), It.IsAny<CancellationToken>()))
+                .ReturnsAsync((Response?)null);
+            PoisonQueue.Setup(q => q.CreateIfNotExistsAsync(
+                    It.IsAny<IDictionary<string, string>>(), It.IsAny<CancellationToken>()))
+                .ReturnsAsync((Response?)null);
+
+            MainQueue.Setup(q => q.ReceiveMessagesAsync(
+                    It.IsAny<int>(), It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()))
+                .Returns<int, TimeSpan?, CancellationToken>((maxMessages, _, _) =>
+                {
+                    var batch = new List<QueueMessage>();
+                    lock (_pendingMessages)
+                    {
+                        while (batch.Count < maxMessages && _pendingMessages.Count > 0)
+                            batch.Add(_pendingMessages.Dequeue());
+                    }
+                    return Task.FromResult(Response.FromValue(batch.ToArray(), new Mock<Response>().Object));
+                });
+
+            MainQueue.Setup(q => q.DeleteMessageAsync(
+                    It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
+                .ReturnsAsync(new Mock<Response>().Object);
+
+            MainQueue.Setup(q => q.UpdateMessageAsync(
+                    It.IsAny<string>(), It.IsAny<string>(),
+                    It.IsAny<string>(), It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()))
+                .Returns<string, string, string, TimeSpan, CancellationToken>((_, _, _, vis, _) =>
+                {
+                    var updated = QueuesModelFactory.UpdateReceipt(
+                        "pop-extended-" + Guid.NewGuid().ToString("N"),
+                        DateTimeOffset.UtcNow.Add(vis));
+                    return Task.FromResult(Response.FromValue(updated, new Mock<Response>().Object));
+                });
+
+            var adminConfig = new Mock<AdminConfigurationService>(
+                Mock.Of<IConfigRepository>(),
+                NullLogger<AdminConfigurationService>.Instance,
+                new MemoryCache(new MemoryCacheOptions()));
+            adminConfig.Setup(a => a.GetConfigurationAsync())
+                .ReturnsAsync(new AdminConfiguration { ImeMsiArchivingEnabled = archivingEnabled });
+
+            Sut = new ImeMsiArchiveQueueWorker(
+                MainQueue.Object, PoisonQueue.Object,
+                Archiver, Repo.Object, adminConfig.Object,
+                NullLogger<ImeMsiArchiveQueueWorker>.Instance,
+                pollInterval: TimeSpan.FromMilliseconds(50));
+        }
+
+        public void EnqueueMessage(string body)
+        {
+            var msg = QueuesModelFactory.QueueMessage(
+                messageId: "msg-" + Guid.NewGuid().ToString("N"),
+                popReceipt: "pop-" + Guid.NewGuid().ToString("N"),
+                body: new BinaryData(body),
+                dequeueCount: 1);
+            lock (_pendingMessages) _pendingMessages.Enqueue(msg);
+        }
+
+        public async Task RunForAsync(TimeSpan duration)
+        {
+            using var cts = new CancellationTokenSource(duration);
+            try { await Sut.StartAsync(cts.Token); }
+            catch (OperationCanceledException) { }
+            try { await Task.Delay(duration, cts.Token); }
+            catch (OperationCanceledException) { }
+            try { await Sut.StopAsync(CancellationToken.None); }
+            catch (OperationCanceledException) { }
+        }
+
+        public async Task RunUntilAsync(Func<bool> condition, TimeSpan? timeout = null)
+        {
+            var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(10));
+            using var cts = new CancellationTokenSource();
+            try { await Sut.StartAsync(cts.Token); }
+            catch (OperationCanceledException) { }
+            while (!condition() && DateTime.UtcNow < deadline)
+                await Task.Delay(25);
+            cts.Cancel();
+            try { await Sut.StopAsync(CancellationToken.None); }
+            catch (OperationCanceledException) { }
+        }
+
+        public bool MainQueueDeleted() =>
+            MainQueue.Invocations.Any(i => i.Method.Name == nameof(QueueClient.DeleteMessageAsync));
+    }
+}
@@ -194,6 +194,7 @@ public class ImeMsiArchiverTests
        var archiver = new RecordingArchiver
        {
            UploadException = new RequestFailedException(409, "BlobAlreadyExists"),
+            ProvenanceExists = true,
        };

        var result = await archiver.ArchiveAsync(Envelope());
@@ -202,6 +203,101 @@ public class ImeMsiArchiverTests
        Assert.Equal(ImeMsiArchiver.Statuses.Archived, result.Status);
        Assert.False(result.Retryable);
        Assert.Equal($"{Version}/IntuneWindowsAgent.msi", result.BlobPath);
+        // Sidecar already there — the re-delivery must not rewrite it.
+        Assert.Equal(0, archiver.ProvenanceCalls);
+    }
+
+    // =========================================================================
+    // Provenance-write gap (daily review 2026-08-18): MSI upload is write-once,
+    // so a first attempt that dies between the MSI and the provenance upload
+    // funnels every retry into the 409 path — which must heal the sidecar.
+    // =========================================================================
+
+    [Fact]
+    public async Task ArchiveAsync_ProvenanceUploadFails_RetryableError()
+    {
+        var archiver = new RecordingArchiver
+        {
+            ProvenanceException = new RequestFailedException(500, "storage hiccup"),
+        };
+
+        var result = await archiver.ArchiveAsync(Envelope());
+
+        Assert.False(result.Success);
+        Assert.Equal(ImeMsiArchiver.Statuses.FailedError, result.Status);
+        Assert.True(result.Retryable);
+    }
+
+    [Fact]
+    public async Task ArchiveAsync_RedeliveryWithMissingProvenance_HealsSidecarFromArchivedBytes()
+    {
+        // The archived blob's content deliberately differs from this attempt's re-download —
+        // the healed provenance must describe the ARCHIVED bytes, not the fresh download.
+        var archivedPayload = new byte[1024];
+        new Random(7).NextBytes(archivedPayload);
+        var expectedSha = Convert.ToHexString(SHA256.HashData(archivedPayload)).ToLowerInvariant();
+        var archiver = new RecordingArchiver
+        {
+            UploadException = new RequestFailedException(409, "BlobAlreadyExists"),
+            ProvenanceExists = false,
+            ArchivedPayload = archivedPayload,
+        };
+
+        var result = await archiver.ArchiveAsync(Envelope());
+
+        Assert.True(result.Success);
+        Assert.Equal(ImeMsiArchiver.Statuses.Archived, result.Status);
+        Assert.False(result.Retryable);
+        Assert.Equal(expectedSha, result.Sha256);
+        Assert.Equal(archivedPayload.Length, result.SizeBytes);
+        Assert.Equal(1, archiver.ProvenanceCalls);
+        Assert.Equal($"{Version}/provenance.json", archiver.ProvenanceLastPath);
+        Assert.Contains(expectedSha, archiver.ProvenanceLastJson);
+        Assert.Contains(Version, archiver.ProvenanceLastJson);
+    }
+
+    [Fact]
+    public async Task ArchiveAsync_ProvenanceFailure_ThenRedelivery_EndsHealed()
+    {
+        // THE regression scenario end to end on one instance: attempt 1 uploads the MSI but
+        // the provenance write dies; attempt 2 (queue re-delivery) gets 409 on the write-once
+        // MSI and must still produce the sidecar.
+        var archiver = new RecordingArchiver
+        {
+            ProvenanceException = new RequestFailedException(500, "storage hiccup"),
+        };
+        var first = await archiver.ArchiveAsync(Envelope());
+        Assert.False(first.Success);
+        Assert.True(first.Retryable);
+
+        archiver.ProvenanceException = null;
+        archiver.UploadException = new RequestFailedException(409, "BlobAlreadyExists");
+        archiver.ProvenanceExists = false; // attempt 1 never landed the sidecar
+        var second = await archiver.ArchiveAsync(Envelope());
+
+        Assert.True(second.Success);
+        Assert.Equal(ImeMsiArchiver.Statuses.Archived, second.Status);
+        Assert.Equal($"{Version}/provenance.json", archiver.ProvenanceLastPath);
+        Assert.NotNull(second.Sha256);
+    }
+
+    [Fact]
+    public async Task ArchiveAsync_HealFails_RetryableError()
+    {
+        // Heal errors ride the normal retry ladder — the next re-delivery tries again.
+        var archiver = new RecordingArchiver
+        {
+            UploadException = new RequestFailedException(409, "BlobAlreadyExists"),
+            ProvenanceExists = false,
+            ArchivedOpenException = new RequestFailedException(500, "read failed"),
+        };
+
+        var result = await archiver.ArchiveAsync(Envelope());
+
+        Assert.False(result.Success);
+        Assert.Equal(ImeMsiArchiver.Statuses.FailedError, result.Status);
+        Assert.True(result.Retryable);
+        Assert.Equal(0, archiver.ProvenanceCalls);
    }

    // =========================================================================
@@ -288,6 +384,10 @@ public class ImeMsiArchiverTests
        public int MaxDownloadSizeMB { get; set; } = 250;
        public Exception? DownloadException { get; set; }
        public Exception? UploadException { get; set; }
+        public Exception? ProvenanceException { get; set; }
+        public bool ProvenanceExists { get; set; } = true;
+        public byte[]? ArchivedPayload { get; set; }
+        public Exception? ArchivedOpenException { get; set; }

        public int DownloadCalls { get; private set; }
        public string? DownloadLastUrl { get; private set; }
@@ -326,10 +426,20 @@ public class ImeMsiArchiverTests

        protected override Task UploadProvenanceAsync(string blobPath, string json, CancellationToken cancellationToken)
        {
+            if (ProvenanceException is not null) throw ProvenanceException;
            ProvenanceCalls++;
            ProvenanceLastPath = blobPath;
            ProvenanceLastJson = json;
            return Task.CompletedTask;
        }
+
+        protected override Task<bool> ProvenanceExistsAsync(string blobPath, CancellationToken cancellationToken)
+            => Task.FromResult(ProvenanceExists);
+
+        protected override Task<Stream> OpenArchivedMsiAsync(string blobPath, CancellationToken cancellationToken)
+        {
+            if (ArchivedOpenException is not null) throw ArchivedOpenException;
+            return Task.FromResult<Stream>(new MemoryStream(ArchivedPayload ?? Payload));
+        }
    }
 }
@@ -37,6 +37,18 @@ namespace AutopilotMonitor.Functions.Services.Ime
            _queueClient = queueFactory.Create(Constants.QueueNames.ImeMsiArchive);
        }

+        /// <summary>
+        /// Test seam: construct directly with a mock <see cref="QueueClient"/>. Mirrors
+        /// <see cref="Deletion.SessionDeletionProducer"/>'s internal test ctor.
+        /// </summary>
+        internal AzureQueueImeMsiArchiveProducer(
+            QueueClient queueClient,
+            ILogger<AzureQueueImeMsiArchiveProducer> logger)
+        {
+            _queueClient = queueClient;
+            _logger = logger;
+        }
+
        public async Task EnqueueAsync(ImeMsiArchiveEnvelope envelope, CancellationToken cancellationToken = default)
        {
            if (envelope is null) return;
@@ -39,6 +39,26 @@ namespace AutopilotMonitor.Functions.Services.Ime
            _adminConfigService = adminConfigService ?? throw new ArgumentNullException(nameof(adminConfigService));
        }

+        /// <summary>
+        /// Test seam: construct directly with mock <see cref="Azure.Storage.Queues.QueueClient"/>
+        /// instances so the poll loop can be exercised without Azurite. Mirrors
+        /// <see cref="Deletion.SessionDeletionWorker"/>'s internal test ctor.
+        /// </summary>
+        internal ImeMsiArchiveQueueWorker(
+            Azure.Storage.Queues.QueueClient mainQueue,
+            Azure.Storage.Queues.QueueClient poisonQueue,
+            ImeMsiArchiver archiver,
+            ISessionRepository sessionRepo,
+            AdminConfigurationService adminConfigService,
+            ILogger<ImeMsiArchiveQueueWorker> logger,
+            TimeSpan? pollInterval = null)
+            : base(mainQueue, poisonQueue, logger, pollInterval)
+        {
+            _archiver = archiver ?? throw new ArgumentNullException(nameof(archiver));
+            _sessionRepo = sessionRepo ?? throw new ArgumentNullException(nameof(sessionRepo));
+            _adminConfigService = adminConfigService ?? throw new ArgumentNullException(nameof(adminConfigService));
+        }
+
        /// <summary>Download can legitimately take minutes — extend visibility while working.</summary>
        protected override bool UseHeartbeat => true;

@@ -34,7 +34,9 @@ namespace AutopilotMonitor.Functions.Services.Ime
    /// <c>IntuneWindowsAgent.msi</c> are accepted; anything else falls back to the canonical
    /// versionless CDN URL — safe for a NEW version, because "new" is by definition what
    /// that URL currently serves. The blob write uses If-None-Match:* so a queue re-delivery
-    /// can never overwrite an existing archive entry.
+    /// can never overwrite an existing archive entry; the 409 path additionally heals a
+    /// provenance sidecar the first attempt failed to write (see
+    /// <see cref="CompleteExistingArchiveAsync"/>).
    /// </para>
    /// </summary>
    public class ImeMsiArchiver
@@ -179,34 +181,18 @@ namespace AutopilotMonitor.Functions.Services.Ime
                    catch (RequestFailedException ex) when (ex.Status == 409)
                    {
                        // Queue re-delivery after a successful upload: the archive already has
-                        // this version — that IS the desired end state.
-                        _logger.LogInformation(
-                            "ImeMsiArchiver: blob {BlobPath} already exists — treating re-delivery as archived",
-                            blobPath);
-                        return new ImeMsiArchiveResult(true, Statuses.Archived, Retryable: false,
-                            blobPath, null, null, sourceUrl);
+                        // this version — that IS the desired end state. But if the first
+                        // attempt died between the MSI and the provenance upload, the sidecar
+                        // is missing and this 409 pre-empts the happy path's provenance write
+                        // on every future retry — so it must be healed here.
+                        return await CompleteExistingArchiveAsync(
+                            envelope, version, blobPath, sourceUrl, linked.Token);
                    }
                    sha256 = guarded.Sha256Hex;
                    totalBytes = guarded.TotalBytes;
                }

-                var provenance = new
-                {
-                    version,
-                    url = sourceUrl,
-                    urlFromEvent = sourceUrl != CanonicalMsiUrl,
-                    msiMatchedBy = envelope.MsiMatchedBy,
-                    sha256,
-                    msiBytes = totalBytes,
-                    firstSeenTenantId = envelope.TenantId,
-                    firstSeenSessionId = envelope.SessionId,
-                    archivedAtUtc = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ"),
-                };
-                await UploadProvenanceAsync(
-                    $"{version}/provenance.json",
-                    System.Text.Json.JsonSerializer.Serialize(provenance,
-                        new System.Text.Json.JsonSerializerOptions { WriteIndented = true }),
-                    linked.Token);
+                await WriteProvenanceAsync(envelope, version, sourceUrl, sha256, totalBytes, linked.Token);

                _logger.LogInformation(
                    "ImeMsiArchiver: archived {Version} ({SizeBytes} bytes, sha256 {Sha256}) from {SourceUrl}",
@@ -247,6 +233,72 @@ namespace AutopilotMonitor.Functions.Services.Ime
            }
        }

+        /// <summary>
+        /// Re-delivery landed on an already-archived version (409 on the write-once MSI
+        /// upload). Normally the provenance sidecar exists too and there is nothing to do —
+        /// but when the first attempt died between the MSI and the provenance upload
+        /// (transient storage failure, host shutdown), the sidecar would otherwise be lost
+        /// FOREVER: the 409 short-circuit pre-empts the happy path's provenance write on
+        /// every retry. Hash and size are recomputed from the ARCHIVED bytes, not this
+        /// attempt's re-download, so the provenance describes the blob it sits next to.
+        /// Failures propagate into the caller's catch ladder (retryable), so the queue
+        /// retries the heal until it lands.
+        /// </summary>
+        private async Task<ImeMsiArchiveResult> CompleteExistingArchiveAsync(
+            ImeMsiArchiveEnvelope envelope, string version, string blobPath, string sourceUrl,
+            CancellationToken cancellationToken)
+        {
+            if (await ProvenanceExistsAsync($"{version}/provenance.json", cancellationToken))
+            {
+                _logger.LogInformation(
+                    "ImeMsiArchiver: blob {BlobPath} already exists — treating re-delivery as archived",
+                    blobPath);
+                return new ImeMsiArchiveResult(true, Statuses.Archived, Retryable: false,
+                    blobPath, null, null, sourceUrl);
+            }
+
+            string sha256;
+            long totalBytes;
+            using (var archived = await OpenArchivedMsiAsync(blobPath, cancellationToken))
+            using (var hashing = new HashingCappedReadStream(archived, maxBytes: 0))
+            {
+                await hashing.CopyToAsync(Stream.Null, cancellationToken);
+                sha256 = hashing.Sha256Hex;
+                totalBytes = hashing.TotalBytes;
+            }
+
+            await WriteProvenanceAsync(envelope, version, sourceUrl, sha256, totalBytes, cancellationToken);
+
+            _logger.LogInformation(
+                "ImeMsiArchiver: healed missing provenance for already-archived {Version} ({SizeBytes} bytes, sha256 {Sha256})",
+                version, totalBytes, sha256);
+            return new ImeMsiArchiveResult(true, Statuses.Archived, Retryable: false,
+                blobPath, sha256, totalBytes, sourceUrl);
+        }
+
+        private async Task WriteProvenanceAsync(
+            ImeMsiArchiveEnvelope envelope, string version, string sourceUrl,
+            string sha256, long totalBytes, CancellationToken cancellationToken)
+        {
+            var provenance = new
+            {
+                version,
+                url = sourceUrl,
+                urlFromEvent = sourceUrl != CanonicalMsiUrl,
+                msiMatchedBy = envelope.MsiMatchedBy,
+                sha256,
+                msiBytes = totalBytes,
+                firstSeenTenantId = envelope.TenantId,
+                firstSeenSessionId = envelope.SessionId,
+                archivedAtUtc = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ"),
+            };
+            await UploadProvenanceAsync(
+                $"{version}/provenance.json",
+                System.Text.Json.JsonSerializer.Serialize(provenance,
+                    new System.Text.Json.JsonSerializerOptions { WriteIndented = true }),
+                cancellationToken);
+        }
+
        // -------- Test seams (production defaults hit Azure / the network) --------

        /// <summary>Fetches the platform admin configuration (size cap).</summary>
@@ -280,6 +332,20 @@ namespace AutopilotMonitor.Functions.Services.Ime
            }, cancellationToken);
        }

+        /// <summary>True when the provenance sidecar already exists in the archive container.</summary>
+        protected virtual async Task<bool> ProvenanceExistsAsync(string blobPath, CancellationToken cancellationToken)
+        {
+            var containerClient = _blobStorage.GetContainerClient(Constants.BlobContainers.ImeArchive);
+            return (await containerClient.GetBlobClient(blobPath).ExistsAsync(cancellationToken)).Value;
+        }
+
+        /// <summary>Opens the already-archived installer blob for the provenance-heal rehash.</summary>
+        protected virtual async Task<Stream> OpenArchivedMsiAsync(string blobPath, CancellationToken cancellationToken)
+        {
+            var containerClient = _blobStorage.GetContainerClient(Constants.BlobContainers.ImeArchive);
+            return await containerClient.GetBlobClient(blobPath).OpenReadAsync(cancellationToken: cancellationToken);
+        }
+
        /// <summary>Uploads the provenance sidecar (overwrite allowed — it is derived data).</summary>
        protected virtual async Task UploadProvenanceAsync(string blobPath, string json, CancellationToken cancellationToken)
        {