From 12af1d6c76885658fda800f5347aed19e91a33ea Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Mon, 21 Sep 2026 20:58:49 +0200 Subject: [PATCH] fix(backend): IME archiver accepts an existing archive entry before walking the hosts A version whose installer was already in the archive could only turn Archived through the 409 on the write-once upload, which needs a distribution host that still serves the build. For a version no host serves any more the row stayed Failed:VersionMismatch next to its own blob and re-walked every host once a day. The archiver now checks the archive first: an entry whose ProductVersion is the observed version ends the job as Archived with the hash and size of the archived bytes and no download; a missing provenance sidecar is written without a source URL. An entry holding another build, or an unreadable one, is not accepted and the walk runs as before. Co-Authored-By: Claude Fable 5.1 --- .../ImeMsiArchiverTests.cs | 101 +++++++++++++++++- .../Services/Ime/ImeMsiArchiver.cs | 90 ++++++++++++---- 2 files changed, 168 insertions(+), 23 deletions(-) diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/ImeMsiArchiverTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/ImeMsiArchiverTests.cs index 34a49d6e..4559e398 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/ImeMsiArchiverTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/ImeMsiArchiverTests.cs @@ -441,6 +441,96 @@ public class ImeMsiArchiverTests Assert.Equal(0, archiver.ProvenanceCalls); } + // ========================================================================= + // Archive already holds the version before the walk (operator backfill of a + // build no host serves any more). The 409 path can never be reached for it — + // without the pre-walk check the row stays Failed:VersionMismatch forever. + // ========================================================================= + + [Fact] + public async Task ArchiveAsync_VersionAlreadyInArchive_NoHostServesIt_ArchivedWithoutDownload() + { + var archivedPayload = new byte[1024]; + new Random(11).NextBytes(archivedPayload); + var expectedSha = Convert.ToHexString(SHA256.HashData(archivedPayload)).ToLowerInvariant(); + var archiver = new RecordingArchiver + { + ArchivedMsiExists = true, + ArchivedPayload = archivedPayload, + DefaultProductVersion = "9.9.9.0", // every host has moved on + }; + archiver.ProductVersionByUrl[RecordingArchiver.ArchivedOrigin] = Version; + + var result = await archiver.ArchiveAsync(Envelope()); + + Assert.True(result.Success); + Assert.Equal(ImeMsiArchiver.Statuses.Archived, result.Status); + Assert.False(result.Retryable); + Assert.Equal($"{Version}/IntuneWindowsAgent.msi", result.BlobPath); + // The row must describe the archived bytes; the source URL of a backfill is unknown. + Assert.Equal(expectedSha, result.Sha256); + Assert.Equal(archivedPayload.Length, result.SizeBytes); + Assert.Null(result.SourceUrl); + Assert.Equal(0, archiver.DownloadCalls); + Assert.Equal(0, archiver.UploadCalls); + Assert.Equal(0, archiver.ProvenanceCalls); // sidecar came with the backfill + } + + [Fact] + public async Task ArchiveAsync_VersionAlreadyInArchive_MissingSidecar_IsWrittenWithoutSourceUrl() + { + var archiver = new RecordingArchiver { ArchivedMsiExists = true, ProvenanceExists = false }; + + var result = await archiver.ArchiveAsync(Envelope()); + + Assert.True(result.Success); + Assert.Equal(0, archiver.DownloadCalls); + Assert.Equal(1, archiver.ProvenanceCalls); + Assert.Equal($"{Version}/provenance.json", archiver.ProvenanceLastPath); + using var doc = System.Text.Json.JsonDocument.Parse(archiver.ProvenanceLastJson); + Assert.Equal(System.Text.Json.JsonValueKind.Null, doc.RootElement.GetProperty("url").ValueKind); + // A null source must not compare equal to a null event URL. + Assert.False(doc.RootElement.GetProperty("urlFromEvent").GetBoolean()); + Assert.Equal(0, doc.RootElement.GetProperty("candidates").GetArrayLength()); + } + + [Theory] + [InlineData("1.103.101.0")] // another ring's build filed under this version + [InlineData(null)] // not a readable package + public async Task ArchiveAsync_ArchiveEntryIsNotTheObservedVersion_NotAccepted_WalkRuns(string? archivedProductVersion) + { + var archiver = new RecordingArchiver + { + ArchivedMsiExists = true, + DefaultProductVersion = "9.9.9.0", + }; + archiver.ProductVersionByUrl[RecordingArchiver.ArchivedOrigin] = archivedProductVersion; + + var result = await archiver.ArchiveAsync(Envelope()); + + Assert.False(result.Success); + Assert.Equal(ImeMsiArchiver.Statuses.FailedVersionMismatch, result.Status); + Assert.Equal(ImeMsiArchiver.BuildCandidateUrls(EventUrl).Count, archiver.DownloadCalls); + Assert.Equal(0, archiver.ProvenanceCalls); + } + + [Fact] + public async Task ArchiveAsync_ArchiveCheckFails_RetryableError_NoDownload() + { + var archiver = new RecordingArchiver + { + ArchivedMsiExists = true, + ArchivedOpenException = new RequestFailedException(500, "read failed"), + }; + + var result = await archiver.ArchiveAsync(Envelope()); + + Assert.False(result.Success); + Assert.Equal(ImeMsiArchiver.Statuses.FailedError, result.Status); + Assert.True(result.Retryable); + Assert.Equal(0, archiver.DownloadCalls); + } + // ========================================================================= // Provenance-write gap (daily review 2026-08-18): MSI upload is write-once, // so a first attempt that dies between the MSI and the provenance upload @@ -660,6 +750,10 @@ public class ImeMsiArchiverTests public Exception? UploadException { get; set; } public Exception? ProvenanceException { get; set; } public bool ProvenanceExists { get; set; } = true; + /// Whether the archive already holds an installer for the version BEFORE the walk (operator backfill). + public bool ArchivedMsiExists { get; set; } + /// Key into for the archived bytes — they have no URL of their own. + public const string ArchivedOrigin = "archive://existing"; public byte[]? ArchivedPayload { get; set; } public Exception? ArchivedOpenException { get; set; } @@ -722,12 +816,15 @@ public class ImeMsiArchiverTests return Task.CompletedTask; } - protected override Task ProvenanceExistsAsync(string blobPath, CancellationToken cancellationToken) - => Task.FromResult(ProvenanceExists); + protected override Task ArchiveBlobExistsAsync(string blobPath, CancellationToken cancellationToken) + => Task.FromResult(blobPath.EndsWith("provenance.json", StringComparison.Ordinal) + ? ProvenanceExists + : ArchivedMsiExists); protected override Task OpenArchivedMsiAsync(string blobPath, CancellationToken cancellationToken) { if (ArchivedOpenException is not null) throw ArchivedOpenException; + _currentUrl = ArchivedOrigin; return Task.FromResult(new MemoryStream(ArchivedPayload ?? Payload)); } } diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Ime/ImeMsiArchiver.cs b/src/Backend/AutopilotMonitor.Functions/Services/Ime/ImeMsiArchiver.cs index 62afff5e..ae884bbc 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/Ime/ImeMsiArchiver.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/Ime/ImeMsiArchiver.cs @@ -202,7 +202,7 @@ namespace AutopilotMonitor.Functions.Services.Ime /// not permanently rejected as a bad version string), and the last archive activity is /// older than — or there never was any (versions sighted /// before the archiver existed, or archived by hand by an operator; the - /// 409 path then heals the row to Archived). + /// pre-walk archive check then turns the row to Archived without a download). /// public static bool ShouldRequeueOnSighting( ImeVersionSighting? sighting, string? msiDownloadUrl, string? msiMatchedBy, DateTime nowUtc) @@ -246,6 +246,13 @@ namespace AutopilotMonitor.Functions.Services.Ime try { + // The archive may already hold this version (operator backfill, earlier attempt). + // This must be checked BEFORE the walk: the 409 below is only reachable when a + // host still serves the build, which is never true for a version Microsoft has + // moved past — the row would stay Failed:VersionMismatch next to its own blob. + var existing = await TryCompleteFromArchiveAsync(envelope, version, blobPath, cancellationToken); + if (existing is not null) return existing; + var adminConfig = await GetAdminConfigurationAsync(); var maxSizeBytes = (long)adminConfig.MaxImeMsiDownloadSizeMB * 1024 * 1024; @@ -385,7 +392,7 @@ namespace AutopilotMonitor.Functions.Services.Ime ImeMsiArchiveEnvelope envelope, string version, string blobPath, string sourceUrl, IReadOnlyList attempts, CancellationToken cancellationToken) { - if (await ProvenanceExistsAsync($"{version}/provenance.json", cancellationToken)) + if (await ArchiveBlobExistsAsync($"{version}/provenance.json", cancellationToken)) { _logger.LogInformation( "ImeMsiArchiver: blob {BlobPath} already exists — treating re-delivery as archived", @@ -394,21 +401,7 @@ namespace AutopilotMonitor.Functions.Services.Ime blobPath, null, null, sourceUrl); } - string sha256; - long totalBytes; - string? archivedProductVersion; - using (var archived = await OpenArchivedMsiAsync(blobPath, cancellationToken)) - using (var spool = CreateSpool()) - { - using (var hashing = new HashingCappedReadStream(archived, maxBytes: 0)) - { - await hashing.CopyToAsync(spool, cancellationToken); - sha256 = hashing.Sha256Hex; - totalBytes = hashing.TotalBytes; - } - spool.Position = 0; - archivedProductVersion = ReadProductVersion(spool); - } + var (sha256, totalBytes, archivedProductVersion) = await InspectArchivedMsiAsync(blobPath, cancellationToken); await WriteProvenanceAsync(envelope, version, sourceUrl, sha256, totalBytes, archivedProductVersion, attempts, cancellationToken); @@ -419,8 +412,62 @@ namespace AutopilotMonitor.Functions.Services.Ime blobPath, sha256, totalBytes, sourceUrl); } + /// + /// Pre-walk check: when the archive already holds an installer for this version AND + /// its ProductVersion is the observed one, the job is done without touching the CDN. + /// Hash and size come from the archived bytes so the row describes the blob; a missing + /// sidecar is written (no source URL — the bytes did not come from this attempt). + /// An entry holding another build is NOT accepted: the folder invariant is "the bytes + /// are that version", so the walk runs and the row keeps telling the truth. Returns + /// null when there is no verified entry. Failures propagate (retryable), like the heal. + /// + private async Task TryCompleteFromArchiveAsync( + ImeMsiArchiveEnvelope envelope, string version, string blobPath, CancellationToken cancellationToken) + { + if (!await ArchiveBlobExistsAsync(blobPath, cancellationToken)) return null; + + var (sha256, totalBytes, archivedProductVersion) = await InspectArchivedMsiAsync(blobPath, cancellationToken); + if (!VersionsMatch(version, archivedProductVersion)) + { + _logger.LogWarning( + "ImeMsiArchiver: archive entry {BlobPath} holds ProductVersion {ProductVersion} — not the observed {Version}; not accepted, walking the hosts", + blobPath, archivedProductVersion ?? "(unreadable)", version); + return null; + } + + if (!await ArchiveBlobExistsAsync($"{version}/provenance.json", cancellationToken)) + { + await WriteProvenanceAsync(envelope, version, sourceUrl: null, sha256, totalBytes, archivedProductVersion, + Array.Empty(), cancellationToken); + } + + _logger.LogInformation( + "ImeMsiArchiver: {Version} is already archived ({SizeBytes} bytes, sha256 {Sha256}) — no download", + version, totalBytes, sha256); + return new ImeMsiArchiveResult(true, Statuses.Archived, Retryable: false, + blobPath, sha256, totalBytes, SourceUrl: null); + } + + /// Hash, size and ProductVersion of the installer that sits in the archive. + private async Task<(string Sha256, long TotalBytes, string? ProductVersion)> InspectArchivedMsiAsync( + string blobPath, CancellationToken cancellationToken) + { + using var archived = await OpenArchivedMsiAsync(blobPath, cancellationToken); + using var spool = CreateSpool(); + string sha256; + long totalBytes; + using (var hashing = new HashingCappedReadStream(archived, maxBytes: 0)) + { + await hashing.CopyToAsync(spool, cancellationToken); + sha256 = hashing.Sha256Hex; + totalBytes = hashing.TotalBytes; + } + spool.Position = 0; + return (sha256, totalBytes, ReadProductVersion(spool)); + } + private async Task WriteProvenanceAsync( - ImeMsiArchiveEnvelope envelope, string version, string sourceUrl, + ImeMsiArchiveEnvelope envelope, string version, string? sourceUrl, string sha256, long totalBytes, string? productVersion, IReadOnlyList attempts, CancellationToken cancellationToken) { @@ -429,7 +476,8 @@ namespace AutopilotMonitor.Functions.Services.Ime version, productVersion, url = sourceUrl, - urlFromEvent = string.Equals(sourceUrl, envelope.MsiDownloadUrl, StringComparison.OrdinalIgnoreCase), + urlFromEvent = sourceUrl is not null + && string.Equals(sourceUrl, envelope.MsiDownloadUrl, StringComparison.OrdinalIgnoreCase), msiMatchedBy = envelope.MsiMatchedBy, sha256, msiBytes = totalBytes, @@ -497,8 +545,8 @@ namespace AutopilotMonitor.Functions.Services.Ime }, cancellationToken); } - /// True when the provenance sidecar already exists in the archive container. - protected virtual async Task ProvenanceExistsAsync(string blobPath, CancellationToken cancellationToken) + /// True when the blob (installer or provenance sidecar) already exists in the archive container. + protected virtual async Task ArchiveBlobExistsAsync(string blobPath, CancellationToken cancellationToken) { var containerClient = _blobStorage.GetContainerClient(Constants.BlobContainers.ImeArchive); return (await containerClient.GetBlobClient(blobPath).ExistsAsync(cancellationToken)).Value;