diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/ImeRegistryAppStateObserverTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/ImeRegistryAppStateObserverTests.cs index 56fed968..6e7788f3 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/ImeRegistryAppStateObserverTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/ImeRegistryAppStateObserverTests.cs @@ -59,6 +59,71 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals public void ClassifyEnforcementState_bands(int state, string expected) => Assert.Equal(expected, ImeRegistryAppStateObserver.ClassifyEnforcementState(state)); + [Theory] + [InlineData(1000, "installed")] + [InlineData(1999, "installed")] + [InlineData(3000, "failed")] + [InlineData(2000, "notApplicable")] + [InlineData(2999, "notApplicable")] + [InlineData(0, null)] + [InlineData(999, null)] + [InlineData(3001, null)] + [InlineData(4000, null)] + public void ClassifyStatusServiceStatus_bands(int status, string? expected) + => Assert.Equal(expected, ImeRegistryAppStateObserver.ClassifyStatusServiceStatus(status)); + + [Fact] + public void TerminalOutcome_prefers_enforcement_state_then_status_service() + { + // Enforcement state alone decides when terminal. + Assert.Equal("success", ImeRegistryAppStateObserver.TerminalOutcome(Entry(enforcementState: 1000))); + Assert.Equal("error", ImeRegistryAppStateObserver.TerminalOutcome(Entry(enforcementState: 5003))); + // Non-terminal enforcement bands defer to the StatusService pillar. + Assert.Null(ImeRegistryAppStateObserver.TerminalOutcome(Entry(enforcementState: 2009))); + Assert.Equal("error", ImeRegistryAppStateObserver.TerminalOutcome( + Entry(enforcementState: 2009, statusServiceStatus: 3000))); + Assert.Equal("success", ImeRegistryAppStateObserver.TerminalOutcome(Entry(statusServiceStatus: 1000))); + Assert.Equal("error", ImeRegistryAppStateObserver.TerminalOutcome(Entry(statusServiceStatus: 3000))); + // requirementsNotMet / notApplicable / empty entries are not judgeable. + Assert.Null(ImeRegistryAppStateObserver.TerminalOutcome(Entry(enforcementState: 3000))); + Assert.Null(ImeRegistryAppStateObserver.TerminalOutcome(Entry(statusServiceStatus: 2500))); + Assert.Null(ImeRegistryAppStateObserver.TerminalOutcome(Entry())); + } + + [Fact] + public void ChangedFields_reports_each_field_and_ignores_espPhase_casing() + { + var prev = Entry(enforcementState: 2009, errorCode: null, exitCode: null, + statusServiceStatus: null, espTracked: false, espPhase: "DevicePreparation"); + var next = Entry(enforcementState: 1000, errorCode: 0, exitCode: 0, + statusServiceStatus: 1000, espTracked: true, espPhase: "devicepreparation"); + + var changed = ImeRegistryAppStateObserver.ChangedFields(prev, next); + + Assert.Equal( + new[] { "enforcementState", "errorCode", "exitCode", "statusServiceStatus", "espTracked" }, + changed); + // Case-only espPhase difference is not a change; identity diffs to empty. + Assert.DoesNotContain("espPhase", changed); + Assert.Empty(ImeRegistryAppStateObserver.ChangedFields(next, next)); + } + + [Fact] + public void ChangedFields_null_previous_reports_populated_fields_of_new_entry() + { + var fresh = Entry(enforcementState: 2000, espTracked: true, espPhase: "AccountSetup"); + + var changed = ImeRegistryAppStateObserver.ChangedFields(null, fresh); + + Assert.Contains("enforcementState", changed); + Assert.Contains("espTracked", changed); + Assert.Contains("espPhase", changed); + // Fields that are null on both sides are not "changes" for a brand-new entry. + Assert.DoesNotContain("errorCode", changed); + Assert.DoesNotContain("exitCode", changed); + Assert.DoesNotContain("statusServiceStatus", changed); + } + [Fact] public void DiffSnapshots_reports_only_changed_fields() { @@ -264,6 +329,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals return (sink, clock, observer); } + private static AppRegistryEntry Entry( + int? enforcementState = null, long? errorCode = null, int? exitCode = null, + int? statusServiceStatus = null, bool espTracked = false, string? espPhase = null) + { + var entry = new AppRegistryEntry($"{ImeRegistrySnapshot.DeviceContext}|{App1}", ImeRegistrySnapshot.DeviceContext, App1); + entry.EnforcementState = enforcementState; + entry.ErrorCode = errorCode; + entry.ExitCode = exitCode; + entry.StatusServiceStatus = statusServiceStatus; + entry.EspTracked = espTracked; + entry.EspPhase = espPhase; + return entry; + } + private static ImeRegistrySnapshot Snapshot(params (string appId, int enforcementState, long? errorCode)[] apps) { var snapshot = new ImeRegistrySnapshot(); diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/AzureQueueImeMsiArchiveProducerTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/AzureQueueImeMsiArchiveProducerTests.cs new file mode 100644 index 00000000..c55b11f4 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/AzureQueueImeMsiArchiveProducerTests.cs @@ -0,0 +1,124 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Storage.Queues; +using Azure.Storage.Queues.Models; +using AutopilotMonitor.Functions.Services.Ime; +using AutopilotMonitor.Shared.Models; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Newtonsoft.Json; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Producer tests for the ime-msi-archive queue (daily review 2026-08-18: previously +/// untested). Contract: the caller is the ingest path's fire-and-forget continuation, so +/// NOTHING may throw — version-less envelopes are skipped, send failures are swallowed, +/// and a failed CreateIfNotExists is retried on the next enqueue instead of latching. +/// +public class AzureQueueImeMsiArchiveProducerTests +{ + private const string Version = "1.104.102.0"; + + [Fact] + public async Task EnqueueAsync_sends_envelope_json_and_ensures_queue_once() + { + var (producer, queue, sentBodies) = BuildSut(); + + await producer.EnqueueAsync(Envelope()); + await producer.EnqueueAsync(Envelope()); + + Assert.Equal(2, sentBodies.Count); + var roundTripped = JsonConvert.DeserializeObject(sentBodies[0])!; + Assert.Equal(Version, roundTripped.Version); + Assert.Equal(ImeMsiArchiver.CanonicalMsiUrl, roundTripped.MsiDownloadUrl); + Assert.Equal("11111111-1111-1111-1111-111111111111", roundTripped.TenantId); + // Queue existence is ensured once, then latched — not re-checked per message. + queue.Verify(q => q.CreateIfNotExistsAsync( + It.IsAny>(), It.IsAny()), + Times.Once); + } + + [Fact] + public async Task EnqueueAsync_skips_envelope_without_version() + { + var (producer, queue, sentBodies) = BuildSut(); + + await producer.EnqueueAsync(new ImeMsiArchiveEnvelope { Version = string.Empty }); + await producer.EnqueueAsync(null!); + + Assert.Empty(sentBodies); + queue.Verify(q => q.CreateIfNotExistsAsync( + It.IsAny>(), It.IsAny()), + Times.Never); + } + + [Fact] + public async Task EnqueueAsync_send_failure_is_swallowed() + { + var (producer, queue, _) = BuildSut(); + queue.Setup(q => q.SendMessageAsync(It.IsAny(), It.IsAny())) + .ThrowsAsync(new RequestFailedException(503, "queue busy")); + + // Fire-and-forget continuation on the ingest path — must never throw. + await producer.EnqueueAsync(Envelope()); + } + + [Fact] + public async Task EnqueueAsync_retries_queue_creation_after_failure() + { + var (producer, queue, sentBodies) = BuildSut(); + queue.SetupSequence(q => q.CreateIfNotExistsAsync( + It.IsAny>(), It.IsAny())) + .ThrowsAsync(new RequestFailedException(500, "storage down")) + .ReturnsAsync((Response?)null); + + await producer.EnqueueAsync(Envelope()); // ensure fails — not latched + await producer.EnqueueAsync(Envelope()); // ensure retried and succeeds + + queue.Verify(q => q.CreateIfNotExistsAsync( + It.IsAny>(), It.IsAny()), + Times.Exactly(2)); + // Both messages were still attempted (fail-soft send after a failed ensure). + Assert.Equal(2, sentBodies.Count); + } + + // ============================================================ Helpers ==== + + private static ImeMsiArchiveEnvelope Envelope() => new() + { + Version = Version, + MsiDownloadUrl = ImeMsiArchiver.CanonicalMsiUrl, + MsiMatchedBy = "productVersion", + TenantId = "11111111-1111-1111-1111-111111111111", + SessionId = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", + EnqueuedAt = DateTime.UtcNow, + }; + + private static (AzureQueueImeMsiArchiveProducer Producer, Mock Queue, List SentBodies) BuildSut() + { + var queue = new Mock(); + var sentBodies = new List(); + + queue.Setup(q => q.CreateIfNotExistsAsync( + It.IsAny>(), It.IsAny())) + .ReturnsAsync((Response?)null); + queue.Setup(q => q.SendMessageAsync(It.IsAny(), It.IsAny())) + .Returns((body, _) => + { + sentBodies.Add(body); + var receipt = QueuesModelFactory.SendReceipt( + "msg", DateTimeOffset.UtcNow, DateTimeOffset.UtcNow.AddMinutes(7), + "pop", DateTimeOffset.UtcNow); + return Task.FromResult(Response.FromValue(receipt, new Mock().Object)); + }); + + var producer = new AzureQueueImeMsiArchiveProducer( + queue.Object, NullLogger.Instance); + return (producer, queue, sentBodies); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/CorporateIdentifierValidatorTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/CorporateIdentifierValidatorTests.cs index 5a421d86..1ae4affe 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/CorporateIdentifierValidatorTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/CorporateIdentifierValidatorTests.cs @@ -247,4 +247,64 @@ public class CorporateIdentifierValidatorTests Assert.False(result.IsValid); Assert.True(result.IsTransient); } + + [Theory] + [InlineData(null, "Virtual Machine", Serial)] + [InlineData("Microsoft Corporation", null, Serial)] + [InlineData("Microsoft Corporation", "Virtual Machine", null)] + [InlineData("", "Virtual Machine", Serial)] + [InlineData("Microsoft Corporation", "", Serial)] + [InlineData("Microsoft Corporation", "Virtual Machine", "")] + [InlineData(" ", "Virtual Machine", Serial)] + [InlineData("Microsoft Corporation", "Virtual Machine", " ")] + public async Task ValidateAsync_missingHeaderComponent_isDefinitive_withoutGraphCall( + string? manufacturer, string? model, string? serial) + { + // Agents that fail to read a hardware property send the header empty — that is a + // definitive rejection (retries cannot conjure the value), and Graph must not be hit. + var (sut, handler) = BuildSut(Json("""{"value":[]}""")); + + var result = await sut.ValidateAsync(TenantId, manufacturer, model, serial); + + Assert.False(result.IsValid); + Assert.False(result.IsTransient); + Assert.Contains("not provided", result.ErrorMessage); + Assert.Empty(handler.Requests); + } + + [Fact] + public async Task ValidateAsync_specialCharsInSerial_neverReachOdataFilter() + { + // The contains() narrowing uses the NORMALIZED (alphanumeric-only) serial, so an + // OData-breaking quote in a raw serial can neither corrupt nor inject into the filter. + const string hostileSerial = "O'Brien-123'&$top=9999"; + var (sut, handler) = BuildSut(Json("""{"value":[]}""")); + + var result = await sut.ValidateAsync(TenantId, "Microsoft Corporation", "Virtual Machine", hostileSerial); + + Assert.False(result.IsValid); + var url = Uri.UnescapeDataString(Assert.Single(handler.Requests).RequestUri!.ToString()); + Assert.Contains("contains(importedDeviceIdentifier,'OBRIEN123TOP9999')", url); + Assert.DoesNotContain("O'Brien", url); + } + + [Fact] + public async Task ValidateAsync_pageCapExceeded_isTransient_andNotCached() + { + // Stopping mid-list must NOT be classified "not found" — a cached false negative would + // block a legitimately registered device for the negative-cache TTL. + var (sut, handler) = BuildSut(Json(""" + {"value":[{"importedDeviceIdentityType":"manufacturerModelSerial","importedDeviceIdentifier":"DELL,XPS,OTHER1"}], + "@odata.nextLink":"https://graph.microsoft.com/beta/deviceManagement/importedDeviceIdentities?$skiptoken=more"} + """)); + + var result = await sut.ValidateAsync(TenantId, "Microsoft Corporation", "Virtual Machine", Serial); + + Assert.False(result.IsValid); + Assert.True(result.IsTransient); + Assert.Contains("page budget", result.ErrorMessage); + // Filtered scan pages up to its cap of 5, and the in-call retry runs a second sweep — + // proof the transient path went through the retry loop instead of caching. + Assert.Equal(10, handler.Requests.Count); + } } diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/ImeMsiArchiveQueueWorkerTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/ImeMsiArchiveQueueWorkerTests.cs new file mode 100644 index 00000000..acaf86e2 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/ImeMsiArchiveQueueWorkerTests.cs @@ -0,0 +1,261 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Storage.Queues; +using Azure.Storage.Queues.Models; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Functions.Services.Ime; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Newtonsoft.Json; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Worker-level tests for the ime-msi-archive queue consumer (daily review 2026-08-18: +/// previously untested). The worker's contract: merge the archiver's outcome onto the +/// ImeVersionHistory row on EVERY attempt (also failures, so operators see a stuck +/// download), delete the message for terminal outcomes, rethrow for retryable ones +/// (visibility-timeout ladder), pause — not drop — while ImeMsiArchivingEnabled is off, +/// and drop version-less envelopes without invoking the archiver. +/// +public class ImeMsiArchiveQueueWorkerTests +{ + private const string Version = "1.104.102.0"; + + [Fact] + public async Task Worker_success_merges_row_and_deletes_message() + { + var harness = new Harness(); + harness.Archiver.Result = new ImeMsiArchiveResult( + true, ImeMsiArchiver.Statuses.Archived, Retryable: false, + $"{Version}/IntuneWindowsAgent.msi", "abc123", 42L, ImeMsiArchiver.CanonicalMsiUrl); + harness.EnqueueMessage(JsonConvert.SerializeObject(Envelope())); + + await harness.RunUntilAsync(() => harness.MainQueueDeleted()); + + harness.Repo.Verify(r => r.UpdateImeVersionArchiveInfoAsync( + Version, ImeMsiArchiver.Statuses.Archived, + $"{Version}/IntuneWindowsAgent.msi", "abc123", 42L, ImeMsiArchiver.CanonicalMsiUrl), + Times.Once); + harness.MainQueue.Verify(q => q.DeleteMessageAsync( + It.IsAny(), It.IsAny(), It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Worker_retryable_failure_merges_row_but_leaves_message() + { + var harness = new Harness(); + harness.Archiver.Result = new ImeMsiArchiveResult( + false, ImeMsiArchiver.Statuses.FailedDownload, Retryable: true, + null, null, null, ImeMsiArchiver.CanonicalMsiUrl); + harness.EnqueueMessage(JsonConvert.SerializeObject(Envelope())); + + await harness.RunUntilAsync(() => harness.Archiver.Calls > 0); + + // Row always tells the truth — the failure status IS merged... + harness.Repo.Verify(r => r.UpdateImeVersionArchiveInfoAsync( + Version, ImeMsiArchiver.Statuses.FailedDownload, + null, null, null, ImeMsiArchiver.CanonicalMsiUrl), + Times.AtLeastOnce); + // ...but the message stays for the visibility-timeout retry → poison ladder. + harness.MainQueue.Verify(q => q.DeleteMessageAsync( + It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + } + + [Fact] + public async Task Worker_permanent_failure_merges_row_and_completes_message() + { + var harness = new Harness(); + harness.Archiver.Result = new ImeMsiArchiveResult( + false, ImeMsiArchiver.Statuses.FailedTooLarge, Retryable: false, + null, null, 999L, ImeMsiArchiver.CanonicalMsiUrl); + harness.EnqueueMessage(JsonConvert.SerializeObject(Envelope())); + + await harness.RunUntilAsync(() => harness.MainQueueDeleted()); + + harness.Repo.Verify(r => r.UpdateImeVersionArchiveInfoAsync( + Version, ImeMsiArchiver.Statuses.FailedTooLarge, + null, null, 999L, ImeMsiArchiver.CanonicalMsiUrl), + Times.Once); + harness.MainQueue.Verify(q => q.DeleteMessageAsync( + It.IsAny(), It.IsAny(), It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Worker_pauses_while_archiving_disabled() + { + var harness = new Harness(archivingEnabled: false); + harness.EnqueueMessage(JsonConvert.SerializeObject(Envelope())); + + await harness.RunForAsync(TimeSpan.FromMilliseconds(500)); + + // Pause, not drop: no receive, no archive attempt — the message parks in the queue. + harness.MainQueue.Verify(q => q.ReceiveMessagesAsync( + It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + Assert.Equal(0, harness.Archiver.Calls); + } + + [Fact] + public async Task Worker_drops_envelope_without_version_before_archiver() + { + var harness = new Harness(); + harness.EnqueueMessage(JsonConvert.SerializeObject(new ImeMsiArchiveEnvelope + { + Version = string.Empty, + EnqueuedAt = DateTime.UtcNow, + })); + + await harness.RunUntilAsync(() => harness.MainQueueDeleted()); + + Assert.Equal(0, harness.Archiver.Calls); + harness.Repo.Verify(r => r.UpdateImeVersionArchiveInfoAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + } + + // ============================================================ Helpers ==== + + private static ImeMsiArchiveEnvelope Envelope() => new() + { + Version = Version, + MsiDownloadUrl = ImeMsiArchiver.CanonicalMsiUrl, + MsiMatchedBy = "productVersion", + TenantId = "11111111-1111-1111-1111-111111111111", + SessionId = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", + EnqueuedAt = DateTime.UtcNow, + }; + + /// Scripted archiver — the worker only consumes the classified result. + private sealed class StubArchiver : ImeMsiArchiver + { + private int _calls; + + public ImeMsiArchiveResult Result { get; set; } = new( + true, Statuses.Archived, Retryable: false, null, null, null, null); + + public int Calls => Volatile.Read(ref _calls); + + public override Task ArchiveAsync( + ImeMsiArchiveEnvelope envelope, CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _calls); + return Task.FromResult(Result); + } + } + + /// Mock-queue poll-loop harness, same shape as SessionDeletionWorkerTests. + private sealed class Harness + { + public Mock MainQueue { get; } + public Mock PoisonQueue { get; } + public StubArchiver Archiver { get; } = new(); + public Mock Repo { get; } = new(); + public ImeMsiArchiveQueueWorker Sut { get; } + + private readonly Queue _pendingMessages = new(); + + public Harness(bool archivingEnabled = true) + { + MainQueue = new Mock(); + PoisonQueue = new Mock(); + + MainQueue.Setup(q => q.CreateIfNotExistsAsync( + It.IsAny>(), It.IsAny())) + .ReturnsAsync((Response?)null); + PoisonQueue.Setup(q => q.CreateIfNotExistsAsync( + It.IsAny>(), It.IsAny())) + .ReturnsAsync((Response?)null); + + MainQueue.Setup(q => q.ReceiveMessagesAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((maxMessages, _, _) => + { + var batch = new List(); + lock (_pendingMessages) + { + while (batch.Count < maxMessages && _pendingMessages.Count > 0) + batch.Add(_pendingMessages.Dequeue()); + } + return Task.FromResult(Response.FromValue(batch.ToArray(), new Mock().Object)); + }); + + MainQueue.Setup(q => q.DeleteMessageAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new Mock().Object); + + MainQueue.Setup(q => q.UpdateMessageAsync( + It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((_, _, _, vis, _) => + { + var updated = QueuesModelFactory.UpdateReceipt( + "pop-extended-" + Guid.NewGuid().ToString("N"), + DateTimeOffset.UtcNow.Add(vis)); + return Task.FromResult(Response.FromValue(updated, new Mock().Object)); + }); + + var adminConfig = new Mock( + Mock.Of(), + NullLogger.Instance, + new MemoryCache(new MemoryCacheOptions())); + adminConfig.Setup(a => a.GetConfigurationAsync()) + .ReturnsAsync(new AdminConfiguration { ImeMsiArchivingEnabled = archivingEnabled }); + + Sut = new ImeMsiArchiveQueueWorker( + MainQueue.Object, PoisonQueue.Object, + Archiver, Repo.Object, adminConfig.Object, + NullLogger.Instance, + pollInterval: TimeSpan.FromMilliseconds(50)); + } + + public void EnqueueMessage(string body) + { + var msg = QueuesModelFactory.QueueMessage( + messageId: "msg-" + Guid.NewGuid().ToString("N"), + popReceipt: "pop-" + Guid.NewGuid().ToString("N"), + body: new BinaryData(body), + dequeueCount: 1); + lock (_pendingMessages) _pendingMessages.Enqueue(msg); + } + + public async Task RunForAsync(TimeSpan duration) + { + using var cts = new CancellationTokenSource(duration); + try { await Sut.StartAsync(cts.Token); } + catch (OperationCanceledException) { } + try { await Task.Delay(duration, cts.Token); } + catch (OperationCanceledException) { } + try { await Sut.StopAsync(CancellationToken.None); } + catch (OperationCanceledException) { } + } + + public async Task RunUntilAsync(Func condition, TimeSpan? timeout = null) + { + var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(10)); + using var cts = new CancellationTokenSource(); + try { await Sut.StartAsync(cts.Token); } + catch (OperationCanceledException) { } + while (!condition() && DateTime.UtcNow < deadline) + await Task.Delay(25); + cts.Cancel(); + try { await Sut.StopAsync(CancellationToken.None); } + catch (OperationCanceledException) { } + } + + public bool MainQueueDeleted() => + MainQueue.Invocations.Any(i => i.Method.Name == nameof(QueueClient.DeleteMessageAsync)); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/ImeMsiArchiverTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/ImeMsiArchiverTests.cs index e6ec506c..d1c0b90a 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/ImeMsiArchiverTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/ImeMsiArchiverTests.cs @@ -194,6 +194,7 @@ public class ImeMsiArchiverTests var archiver = new RecordingArchiver { UploadException = new RequestFailedException(409, "BlobAlreadyExists"), + ProvenanceExists = true, }; var result = await archiver.ArchiveAsync(Envelope()); @@ -202,6 +203,101 @@ public class ImeMsiArchiverTests Assert.Equal(ImeMsiArchiver.Statuses.Archived, result.Status); Assert.False(result.Retryable); Assert.Equal($"{Version}/IntuneWindowsAgent.msi", result.BlobPath); + // Sidecar already there — the re-delivery must not rewrite it. + Assert.Equal(0, archiver.ProvenanceCalls); + } + + // ========================================================================= + // Provenance-write gap (daily review 2026-08-18): MSI upload is write-once, + // so a first attempt that dies between the MSI and the provenance upload + // funnels every retry into the 409 path — which must heal the sidecar. + // ========================================================================= + + [Fact] + public async Task ArchiveAsync_ProvenanceUploadFails_RetryableError() + { + var archiver = new RecordingArchiver + { + ProvenanceException = new RequestFailedException(500, "storage hiccup"), + }; + + var result = await archiver.ArchiveAsync(Envelope()); + + Assert.False(result.Success); + Assert.Equal(ImeMsiArchiver.Statuses.FailedError, result.Status); + Assert.True(result.Retryable); + } + + [Fact] + public async Task ArchiveAsync_RedeliveryWithMissingProvenance_HealsSidecarFromArchivedBytes() + { + // The archived blob's content deliberately differs from this attempt's re-download — + // the healed provenance must describe the ARCHIVED bytes, not the fresh download. + var archivedPayload = new byte[1024]; + new Random(7).NextBytes(archivedPayload); + var expectedSha = Convert.ToHexString(SHA256.HashData(archivedPayload)).ToLowerInvariant(); + var archiver = new RecordingArchiver + { + UploadException = new RequestFailedException(409, "BlobAlreadyExists"), + ProvenanceExists = false, + ArchivedPayload = archivedPayload, + }; + + var result = await archiver.ArchiveAsync(Envelope()); + + Assert.True(result.Success); + Assert.Equal(ImeMsiArchiver.Statuses.Archived, result.Status); + Assert.False(result.Retryable); + Assert.Equal(expectedSha, result.Sha256); + Assert.Equal(archivedPayload.Length, result.SizeBytes); + Assert.Equal(1, archiver.ProvenanceCalls); + Assert.Equal($"{Version}/provenance.json", archiver.ProvenanceLastPath); + Assert.Contains(expectedSha, archiver.ProvenanceLastJson); + Assert.Contains(Version, archiver.ProvenanceLastJson); + } + + [Fact] + public async Task ArchiveAsync_ProvenanceFailure_ThenRedelivery_EndsHealed() + { + // THE regression scenario end to end on one instance: attempt 1 uploads the MSI but + // the provenance write dies; attempt 2 (queue re-delivery) gets 409 on the write-once + // MSI and must still produce the sidecar. + var archiver = new RecordingArchiver + { + ProvenanceException = new RequestFailedException(500, "storage hiccup"), + }; + var first = await archiver.ArchiveAsync(Envelope()); + Assert.False(first.Success); + Assert.True(first.Retryable); + + archiver.ProvenanceException = null; + archiver.UploadException = new RequestFailedException(409, "BlobAlreadyExists"); + archiver.ProvenanceExists = false; // attempt 1 never landed the sidecar + var second = await archiver.ArchiveAsync(Envelope()); + + Assert.True(second.Success); + Assert.Equal(ImeMsiArchiver.Statuses.Archived, second.Status); + Assert.Equal($"{Version}/provenance.json", archiver.ProvenanceLastPath); + Assert.NotNull(second.Sha256); + } + + [Fact] + public async Task ArchiveAsync_HealFails_RetryableError() + { + // Heal errors ride the normal retry ladder — the next re-delivery tries again. + var archiver = new RecordingArchiver + { + UploadException = new RequestFailedException(409, "BlobAlreadyExists"), + ProvenanceExists = false, + ArchivedOpenException = new RequestFailedException(500, "read failed"), + }; + + var result = await archiver.ArchiveAsync(Envelope()); + + Assert.False(result.Success); + Assert.Equal(ImeMsiArchiver.Statuses.FailedError, result.Status); + Assert.True(result.Retryable); + Assert.Equal(0, archiver.ProvenanceCalls); } // ========================================================================= @@ -288,6 +384,10 @@ public class ImeMsiArchiverTests public int MaxDownloadSizeMB { get; set; } = 250; public Exception? DownloadException { get; set; } public Exception? UploadException { get; set; } + public Exception? ProvenanceException { get; set; } + public bool ProvenanceExists { get; set; } = true; + public byte[]? ArchivedPayload { get; set; } + public Exception? ArchivedOpenException { get; set; } public int DownloadCalls { get; private set; } public string? DownloadLastUrl { get; private set; } @@ -326,10 +426,20 @@ public class ImeMsiArchiverTests protected override Task UploadProvenanceAsync(string blobPath, string json, CancellationToken cancellationToken) { + if (ProvenanceException is not null) throw ProvenanceException; ProvenanceCalls++; ProvenanceLastPath = blobPath; ProvenanceLastJson = json; return Task.CompletedTask; } + + protected override Task ProvenanceExistsAsync(string blobPath, CancellationToken cancellationToken) + => Task.FromResult(ProvenanceExists); + + protected override Task OpenArchivedMsiAsync(string blobPath, CancellationToken cancellationToken) + { + if (ArchivedOpenException is not null) throw ArchivedOpenException; + return Task.FromResult(new MemoryStream(ArchivedPayload ?? Payload)); + } } } diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Ime/AzureQueueImeMsiArchiveProducer.cs b/src/Backend/AutopilotMonitor.Functions/Services/Ime/AzureQueueImeMsiArchiveProducer.cs index 8f422d98..8ada8ef8 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/Ime/AzureQueueImeMsiArchiveProducer.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/Ime/AzureQueueImeMsiArchiveProducer.cs @@ -37,6 +37,18 @@ namespace AutopilotMonitor.Functions.Services.Ime _queueClient = queueFactory.Create(Constants.QueueNames.ImeMsiArchive); } + /// + /// Test seam: construct directly with a mock . Mirrors + /// 's internal test ctor. + /// + internal AzureQueueImeMsiArchiveProducer( + QueueClient queueClient, + ILogger logger) + { + _queueClient = queueClient; + _logger = logger; + } + public async Task EnqueueAsync(ImeMsiArchiveEnvelope envelope, CancellationToken cancellationToken = default) { if (envelope is null) return; diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Ime/ImeMsiArchiveQueueWorker.cs b/src/Backend/AutopilotMonitor.Functions/Services/Ime/ImeMsiArchiveQueueWorker.cs index 9508adad..ff673f00 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/Ime/ImeMsiArchiveQueueWorker.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/Ime/ImeMsiArchiveQueueWorker.cs @@ -39,6 +39,26 @@ namespace AutopilotMonitor.Functions.Services.Ime _adminConfigService = adminConfigService ?? throw new ArgumentNullException(nameof(adminConfigService)); } + /// + /// Test seam: construct directly with mock + /// instances so the poll loop can be exercised without Azurite. Mirrors + /// 's internal test ctor. + /// + internal ImeMsiArchiveQueueWorker( + Azure.Storage.Queues.QueueClient mainQueue, + Azure.Storage.Queues.QueueClient poisonQueue, + ImeMsiArchiver archiver, + ISessionRepository sessionRepo, + AdminConfigurationService adminConfigService, + ILogger logger, + TimeSpan? pollInterval = null) + : base(mainQueue, poisonQueue, logger, pollInterval) + { + _archiver = archiver ?? throw new ArgumentNullException(nameof(archiver)); + _sessionRepo = sessionRepo ?? throw new ArgumentNullException(nameof(sessionRepo)); + _adminConfigService = adminConfigService ?? throw new ArgumentNullException(nameof(adminConfigService)); + } + /// Download can legitimately take minutes — extend visibility while working. protected override bool UseHeartbeat => true; diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Ime/ImeMsiArchiver.cs b/src/Backend/AutopilotMonitor.Functions/Services/Ime/ImeMsiArchiver.cs index 922c2629..659a75fe 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/Ime/ImeMsiArchiver.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/Ime/ImeMsiArchiver.cs @@ -34,7 +34,9 @@ namespace AutopilotMonitor.Functions.Services.Ime /// IntuneWindowsAgent.msi are accepted; anything else falls back to the canonical /// versionless CDN URL — safe for a NEW version, because "new" is by definition what /// that URL currently serves. The blob write uses If-None-Match:* so a queue re-delivery - /// can never overwrite an existing archive entry. + /// can never overwrite an existing archive entry; the 409 path additionally heals a + /// provenance sidecar the first attempt failed to write (see + /// ). /// /// public class ImeMsiArchiver @@ -179,34 +181,18 @@ namespace AutopilotMonitor.Functions.Services.Ime catch (RequestFailedException ex) when (ex.Status == 409) { // Queue re-delivery after a successful upload: the archive already has - // this version — that IS the desired end state. - _logger.LogInformation( - "ImeMsiArchiver: blob {BlobPath} already exists — treating re-delivery as archived", - blobPath); - return new ImeMsiArchiveResult(true, Statuses.Archived, Retryable: false, - blobPath, null, null, sourceUrl); + // this version — that IS the desired end state. But if the first + // attempt died between the MSI and the provenance upload, the sidecar + // is missing and this 409 pre-empts the happy path's provenance write + // on every future retry — so it must be healed here. + return await CompleteExistingArchiveAsync( + envelope, version, blobPath, sourceUrl, linked.Token); } sha256 = guarded.Sha256Hex; totalBytes = guarded.TotalBytes; } - var provenance = new - { - version, - url = sourceUrl, - urlFromEvent = sourceUrl != CanonicalMsiUrl, - msiMatchedBy = envelope.MsiMatchedBy, - sha256, - msiBytes = totalBytes, - firstSeenTenantId = envelope.TenantId, - firstSeenSessionId = envelope.SessionId, - archivedAtUtc = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ"), - }; - await UploadProvenanceAsync( - $"{version}/provenance.json", - System.Text.Json.JsonSerializer.Serialize(provenance, - new System.Text.Json.JsonSerializerOptions { WriteIndented = true }), - linked.Token); + await WriteProvenanceAsync(envelope, version, sourceUrl, sha256, totalBytes, linked.Token); _logger.LogInformation( "ImeMsiArchiver: archived {Version} ({SizeBytes} bytes, sha256 {Sha256}) from {SourceUrl}", @@ -247,6 +233,72 @@ namespace AutopilotMonitor.Functions.Services.Ime } } + /// + /// Re-delivery landed on an already-archived version (409 on the write-once MSI + /// upload). Normally the provenance sidecar exists too and there is nothing to do — + /// but when the first attempt died between the MSI and the provenance upload + /// (transient storage failure, host shutdown), the sidecar would otherwise be lost + /// FOREVER: the 409 short-circuit pre-empts the happy path's provenance write on + /// every retry. Hash and size are recomputed from the ARCHIVED bytes, not this + /// attempt's re-download, so the provenance describes the blob it sits next to. + /// Failures propagate into the caller's catch ladder (retryable), so the queue + /// retries the heal until it lands. + /// + private async Task CompleteExistingArchiveAsync( + ImeMsiArchiveEnvelope envelope, string version, string blobPath, string sourceUrl, + CancellationToken cancellationToken) + { + if (await ProvenanceExistsAsync($"{version}/provenance.json", cancellationToken)) + { + _logger.LogInformation( + "ImeMsiArchiver: blob {BlobPath} already exists — treating re-delivery as archived", + blobPath); + return new ImeMsiArchiveResult(true, Statuses.Archived, Retryable: false, + blobPath, null, null, sourceUrl); + } + + string sha256; + long totalBytes; + using (var archived = await OpenArchivedMsiAsync(blobPath, cancellationToken)) + using (var hashing = new HashingCappedReadStream(archived, maxBytes: 0)) + { + await hashing.CopyToAsync(Stream.Null, cancellationToken); + sha256 = hashing.Sha256Hex; + totalBytes = hashing.TotalBytes; + } + + await WriteProvenanceAsync(envelope, version, sourceUrl, sha256, totalBytes, cancellationToken); + + _logger.LogInformation( + "ImeMsiArchiver: healed missing provenance for already-archived {Version} ({SizeBytes} bytes, sha256 {Sha256})", + version, totalBytes, sha256); + return new ImeMsiArchiveResult(true, Statuses.Archived, Retryable: false, + blobPath, sha256, totalBytes, sourceUrl); + } + + private async Task WriteProvenanceAsync( + ImeMsiArchiveEnvelope envelope, string version, string sourceUrl, + string sha256, long totalBytes, CancellationToken cancellationToken) + { + var provenance = new + { + version, + url = sourceUrl, + urlFromEvent = sourceUrl != CanonicalMsiUrl, + msiMatchedBy = envelope.MsiMatchedBy, + sha256, + msiBytes = totalBytes, + firstSeenTenantId = envelope.TenantId, + firstSeenSessionId = envelope.SessionId, + archivedAtUtc = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ"), + }; + await UploadProvenanceAsync( + $"{version}/provenance.json", + System.Text.Json.JsonSerializer.Serialize(provenance, + new System.Text.Json.JsonSerializerOptions { WriteIndented = true }), + cancellationToken); + } + // -------- Test seams (production defaults hit Azure / the network) -------- /// Fetches the platform admin configuration (size cap). @@ -280,6 +332,20 @@ namespace AutopilotMonitor.Functions.Services.Ime }, cancellationToken); } + /// True when the provenance sidecar already exists in the archive container. + protected virtual async Task ProvenanceExistsAsync(string blobPath, CancellationToken cancellationToken) + { + var containerClient = _blobStorage.GetContainerClient(Constants.BlobContainers.ImeArchive); + return (await containerClient.GetBlobClient(blobPath).ExistsAsync(cancellationToken)).Value; + } + + /// Opens the already-archived installer blob for the provenance-heal rehash. + protected virtual async Task OpenArchivedMsiAsync(string blobPath, CancellationToken cancellationToken) + { + var containerClient = _blobStorage.GetContainerClient(Constants.BlobContainers.ImeArchive); + return await containerClient.GetBlobClient(blobPath).OpenReadAsync(cancellationToken: cancellationToken); + } + /// Uploads the provenance sidecar (overwrite allowed — it is derived data). protected virtual async Task UploadProvenanceAsync(string blobPath, string json, CancellationToken cancellationToken) {