Backend: one table-key sanitizer that drops control characters; inventory counter failures become one ops event per run (D-338)

- TableKeySanitizer replaces ten private and inline copies of the key rule (/, \, #, ? become _) and also removes control characters, which Table Storage rejects in keys and the HTTP layer rejects in a point read's URL. Keys without control characters are unchanged; a source ratchet forbids new copies. The dead UpsertSoftwareInventoryAsync is removed. - VulnerabilityInventoryLoader hands out inventory text without control characters, so CPE lookup, report text, side-row identity and counter keys all see the same text. - A correlation run whose SoftwareInventory counter writes or contributions side-row fail writes one SoftwareInventoryCounterWriteFailed ops event (Maintenance, Warning; counts, error type and status, up to five sample keys) instead of a per-key entry in the tenant audit trail. The counter work is awaited inside the invocation instead of a detached Task.Run. - The new type is listed in the alert-rule catalog; shared manifest and MCP vocabulary regenerated. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-23 20:26:10 +00:00 · 2026-10-09 21:33:00 +02:00
parent a6e8fe320f
commit 182ea2e220
24 changed files with 418 additions and 311 deletions
@@ -0,0 +1,82 @@
+using System.Text.RegularExpressions;
+using AutopilotMonitor.Functions.Helpers;
+
+namespace AutopilotMonitor.Functions.Tests;
+
+public class TableKeySanitizerTests
+{
+    [Fact]
+    public void Sanitize_RemovesControlCharacters()
+    {
+        // C0 and C1 controls: Table Storage rejects them in keys, and a point read carries the key
+        // in its URL, where the HTTP layer refuses them with "400 Invalid URL".
+        var input = $"a{(char)0}b\tc\r\nd{(char)0x7F}e{(char)0x85}f";
+        Assert.Equal("abcdef", TableKeySanitizer.Sanitize(input));
+    }
+
+    [Fact]
+    public void Sanitize_ReplacesTheCharactersTableStorageRejectsInKeys()
+    {
+        Assert.Equal("a_b_c_d_e", TableKeySanitizer.Sanitize("a/b\\c#d?e"));
+    }
+
+    [Fact]
+    public void Sanitize_NulPaddedInventoryTriple_YieldsTheCleanKey()
+    {
+        var padded = "contoso\0\0\0\0\0:widget\0\0\0\0\0:1.0";
+        Assert.Equal("contoso:widget:1.0", TableKeySanitizer.Sanitize(padded));
+    }
+
+    [Theory]
+    [InlineData("contoso:widget:1.0")]
+    [InlineData(":microsoft edge update:1.3.265.7")]
+    [InlineData("müller gmbh:übersicht:2.3")]
+    [InlineData("cve_cpe:2.3:a:contoso:widget:*:*:*:*:*:*:*:*")]
+    public void Sanitize_KeyWithoutControlOrIllegalCharacters_IsUnchanged(string key)
+    {
+        // Existing rows must stay reachable: the rule only changes keys that could never be read.
+        Assert.Equal(key, TableKeySanitizer.Sanitize(key));
+    }
+
+    [Fact]
+    public void Sanitize_IsIdempotent()
+    {
+        var once = TableKeySanitizer.Sanitize("a/b\0#c?\td");
+        Assert.Equal(once, TableKeySanitizer.Sanitize(once));
+    }
+
+    [Fact]
+    public void RemoveControlCharacters_KeepsKeyIllegalCharacters()
+    {
+        Assert.Equal("a/b#c", TableKeySanitizer.RemoveControlCharacters("a/b\0#\u0001c"));
+    }
+
+    [Fact]
+    public void No_key_builder_keeps_its_own_copy_of_the_rule()
+    {
+        // The rule existed ten times, and only one copy removed control characters. A new inline
+        // copy would map the same text to a different key than every other writer of that table.
+        var functionsDir = Path.Combine(RepoRoot(), "src", "Backend", "AutopilotMonitor.Functions");
+        var inlineCopy = new Regex(@"Replace\(\s*[""']#[""']\s*,\s*[""']_[""']\s*\)|static\s+string\s+SanitizeTableKey\s*\(");
+        var offenders = Directory
+            .EnumerateFiles(functionsDir, "*.cs", SearchOption.AllDirectories)
+            .Where(f => !f.Contains($"{Path.DirectorySeparatorChar}bin{Path.DirectorySeparatorChar}")
+                     && !f.Contains($"{Path.DirectorySeparatorChar}obj{Path.DirectorySeparatorChar}"))
+            .Where(f => inlineCopy.IsMatch(File.ReadAllText(f)))
+            .Select(f => Path.GetRelativePath(functionsDir, f))
+            .ToList();
+
+        Assert.True(offenders.Count == 0,
+            "Build table keys from free text through TableKeySanitizer.Sanitize, not a local copy: "
+            + string.Join(", ", offenders));
+    }
+
+    private static string RepoRoot()
+    {
+        var dir = new DirectoryInfo(AppContext.BaseDirectory);
+        while (dir != null && !File.Exists(Path.Combine(dir.FullName, "AutopilotMonitor.sln")))
+            dir = dir.Parent;
+        Assert.NotNull(dir);
+        return dir!.FullName;
+    }
+}
@@ -70,6 +70,36 @@ public class TableStorageInventoryCasTests
        harness.SoftwareInventory.Verify(t => t.UpdateEntityAsync(It.IsAny<TableEntity>(), It.IsAny<ETag>(), It.IsAny<TableUpdateMode>(), It.IsAny<CancellationToken>()), Times.Never);
    }

+    [Fact]
+    public async Task Increment_with_nul_padded_triple_reads_and_writes_the_clean_key()
+    {
+        // Installers that pad REG_SZ values with NULs used to put "%00" into the point-read URL,
+        // which the HTTP layer rejects ("400 Invalid URL") before Table Storage sees it.
+        var harness = new Harness();
+        string? readRowKey = null;
+        harness.SoftwareInventory.Setup(t =>
+            t.GetEntityAsync<TableEntity>(TenantId, It.IsAny<string>(), It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
+            .Callback<string, string, IEnumerable<string>, CancellationToken>((_, rk, _, _) => readRowKey = rk)
+            .ThrowsAsync(new RequestFailedException(404, "ResourceNotFound"));
+
+        TableEntity? addedEntity = null;
+        harness.SoftwareInventory.Setup(t => t.AddEntityAsync(It.IsAny<TableEntity>(), It.IsAny<CancellationToken>()))
+            .Returns<TableEntity, CancellationToken>((e, _) =>
+            {
+                addedEntity = e;
+                return Task.FromResult(new Mock<Response>().Object);
+            });
+
+        var item = new SoftwareInventoryItem
+        {
+            NormalizedVendor = "contoso\0\0\0", NormalizedName = "widget\0\0", NormalizedVersion = "1.0",
+        };
+        await harness.Sut.IncrementSoftwareInventoryEntryAsync(TenantId, item, SessionId);
+
+        Assert.Equal("contoso:widget:1.0", readRowKey);
+        Assert.Equal("contoso:widget:1.0", addedEntity!.RowKey);
+    }
+
    [Fact]
    public async Task Increment_does_etag_cas_plus_one_when_row_exists()
    {
@@ -1,10 +1,14 @@
 using System;
 using System.Collections.Generic;
 using System.Threading;
+using System.Text.Json;
 using System.Threading.Tasks;
+using Azure;
+using AutopilotMonitor.Functions.Services;
 using AutopilotMonitor.Functions.Services.Vulnerability;
 using AutopilotMonitor.Shared.DataAccess;
 using AutopilotMonitor.Shared.Models.Deletion;
+using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Logging.Abstractions;
 using Moq;
 using Xunit;
@@ -13,10 +17,9 @@ namespace AutopilotMonitor.Functions.Tests;

 /// <summary>
 /// Behaviour tests for the rewritten inventory-write path
-/// (<see cref="VulnerabilityCorrelationService.ApplyInventoryContributionsAsync"/>). The
-/// background <c>Task.Run</c> in <c>CorrelateAsync</c> just forwards to this method, so testing
-/// it directly is equivalent to testing the rewritten flow end-to-end without the timing
-/// uncertainty of a fire-and-forget Task.
+/// (<see cref="VulnerabilityCorrelationService.ApplyInventoryContributionsAsync"/>).
+/// <c>CorrelateAsync</c> awaits this method, so testing it directly covers the write flow
+/// without the NVD side of the correlation.
 /// </summary>
 public class VulnerabilityCorrelationServiceContributionsTests
 {
@@ -158,18 +161,19 @@ public class VulnerabilityCorrelationServiceContributionsTests
    }

    [Fact]
-    public async Task ApplyInventoryContributionsAsync_swallows_side_row_helper_exceptions_for_non_fatal_logging()
+    public async Task ApplyInventoryContributionsAsync_swallows_side_row_helper_exceptions_and_reports_one_ops_event()
    {
        // Helpers are fail-loud (per memory feedback_storage_helpers_fail_soft) but the caller
-        // intentionally logs+swallows the side-row CAS exhaustion so the primary VulnerabilityReport
-        // stays stored. Drift is the documented trade-off (§17.3 "Side-row write before counter writes").
+        // logs+swallows the side-row CAS exhaustion so the primary VulnerabilityReport stays
+        // stored. The operator still learns about it: one ops event, flagged sideRowFailed.
        var vulnRepo = new Mock<IVulnerabilityRepository>();
        vulnRepo
            .Setup(r => r.UpsertSessionInventoryContributionsAsync(
                TenantId, SessionId, It.IsAny<IReadOnlyList<SoftwareInventoryItem>>(), It.IsAny<CancellationToken>()))
            .ThrowsAsync(new InvalidOperationException("SessionInventoryContributions ETag-CAS exhausted ..."));

-        var service = NewService(vulnRepo);
+        var (opsEvents, saved) = OpsRig();
+        var service = NewService(vulnRepo, opsEvents);
        var raw = new List<Dictionary<string, object>>
        {
            new Dictionary<string, object> { ["normalizedPublisher"] = "v1", ["normalizedName"] = "n1", ["normalizedVersion"] = "1.0" },
@@ -181,15 +185,19 @@ public class VulnerabilityCorrelationServiceContributionsTests
        vulnRepo.Verify(r => r.IncrementSoftwareInventoryEntryAsync(
            It.IsAny<string>(), It.IsAny<SoftwareInventoryItem>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
            Times.Never);
+        var evt = Assert.Single(saved);
+        Assert.Equal(OpsEventTypes.SoftwareInventoryCounterWriteFailed, evt.EventType);
+        using var details = JsonDocument.Parse(evt.Details!);
+        Assert.True(details.RootElement.GetProperty("sideRowFailed").GetBoolean());
    }

    [Fact]
-    public async Task ApplyInventoryContributionsAsync_continues_after_per_counter_failure_and_audits_each()
+    public async Task ApplyInventoryContributionsAsync_continues_after_per_counter_failure_and_reports_one_ops_event_per_run()
    {
        // Codex Round 2: a single Increment failure must NOT abort the rest of the loop —
        // the side-row already promises the new key set, so subsequent runs would see an empty
-        // delta and never retry the under-counted keys. Per Plan §17.3 + §17.8, each failure
-        // emits an inventory_counter_write_failed audit + we keep processing the rest.
+        // delta and never retry the under-counted keys (Plan §17.3). The failures of the run are
+        // reported as ONE ops event — never per key, never in the tenant audit trail.
        var vulnRepo = new Mock<IVulnerabilityRepository>();
        vulnRepo
            .Setup(r => r.UpsertSessionInventoryContributionsAsync(
@@ -209,19 +217,18 @@ public class VulnerabilityCorrelationServiceContributionsTests
                },
            });

-        // Increment of (v1,n1,1.0) succeeds, (v2,n2,2.0) THROWS, (v3,n3,3.0) succeeds.
+        // Increments of (v2,n2) and (v3,n3) THROW, (v1,n1) succeeds; the decrement throws too.
        vulnRepo.Setup(r => r.IncrementSoftwareInventoryEntryAsync(
-                TenantId, It.Is<SoftwareInventoryItem>(i => i.NormalizedName == "n2"),
+                TenantId, It.Is<SoftwareInventoryItem>(i => i.NormalizedName == "n2" || i.NormalizedName == "n3"),
                SessionId, It.IsAny<CancellationToken>()))
-            .ThrowsAsync(new InvalidOperationException("SoftwareInventory increment ETag-CAS exhausted"));
+            .ThrowsAsync(new RequestFailedException(400,
+                "Service request failed.\nStatus: 400 (Bad Request)\n\nContent:\n<!DOCTYPE HTML><HTML>Bad Request - Invalid URL</HTML>"));
+        vulnRepo.Setup(r => r.DecrementSoftwareInventoryEntryAsync(
+                TenantId, "vOld", "nOld", "0.9", It.IsAny<CancellationToken>()))
+            .ThrowsAsync(new InvalidOperationException("Decrement ETag-CAS exhausted"));

-        var maintenanceRepo = new Mock<IMaintenanceRepository>();
-        maintenanceRepo.Setup(m => m.LogAuditEntryAsync(
-                It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
-                It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Dictionary<string, string>?>()))
-            .ReturnsAsync(true);
-
-        var service = NewService(vulnRepo, maintenanceRepo);
+        var (opsEvents, saved) = OpsRig();
+        var service = NewService(vulnRepo, opsEvents);
        var raw = new List<Dictionary<string, object>>
        {
            new Dictionary<string, object> { ["normalizedPublisher"] = "v1", ["normalizedName"] = "n1", ["normalizedVersion"] = "1.0" },
@@ -232,38 +239,33 @@ public class VulnerabilityCorrelationServiceContributionsTests
        // Should NOT throw.
        await service.ApplyInventoryContributionsAsync(TenantId, SessionId, raw, cpeMappings: null);

-        // All three increments attempted.
+        // Every counter write attempted.
        vulnRepo.Verify(r => r.IncrementSoftwareInventoryEntryAsync(
-            TenantId, It.Is<SoftwareInventoryItem>(i => i.NormalizedName == "n1"),
-            SessionId, It.IsAny<CancellationToken>()), Times.Once);
-        vulnRepo.Verify(r => r.IncrementSoftwareInventoryEntryAsync(
-            TenantId, It.Is<SoftwareInventoryItem>(i => i.NormalizedName == "n2"),
-            SessionId, It.IsAny<CancellationToken>()), Times.Once);
-        vulnRepo.Verify(r => r.IncrementSoftwareInventoryEntryAsync(
-            TenantId, It.Is<SoftwareInventoryItem>(i => i.NormalizedName == "n3"),
-            SessionId, It.IsAny<CancellationToken>()), Times.Once);
-        // Decrement still runs after the loop continues past the failed Increment.
+            TenantId, It.IsAny<SoftwareInventoryItem>(), SessionId, It.IsAny<CancellationToken>()), Times.Exactly(3));
        vulnRepo.Verify(r => r.DecrementSoftwareInventoryEntryAsync(
            TenantId, "vOld", "nOld", "0.9", It.IsAny<CancellationToken>()), Times.Once);

-        // Audit emitted exactly once for the failed (v2,n2,2.0) Increment.
-        maintenanceRepo.Verify(m => m.LogAuditEntryAsync(
-            TenantId,
-            "inventory_counter_write_failed",
-            "Session",
-            SessionId,
-            "system",
-            It.Is<Dictionary<string, string>?>(d =>
-                d != null
-                && d["operation"] == "increment"
-                && d["vendor"] == "v2"
-                && d["name"] == "n2"
-                && d["version"] == "2.0")),
-            Times.Once);
+        var evt = Assert.Single(saved);
+        Assert.Equal(OpsEventTypes.SoftwareInventoryCounterWriteFailed, evt.EventType);
+        Assert.Equal(OpsEventCategory.Maintenance, evt.Category);
+        Assert.Equal(OpsEventSeverity.Warning, evt.Severity);
+        Assert.Equal(TenantId, evt.TenantId);
+        using var details = JsonDocument.Parse(evt.Details!);
+        var root = details.RootElement;
+        Assert.Equal(SessionId, root.GetProperty("sessionId").GetString());
+        Assert.Equal(2, root.GetProperty("failedIncrements").GetInt32());
+        Assert.Equal(1, root.GetProperty("failedDecrements").GetInt32());
+        Assert.False(root.GetProperty("sideRowFailed").GetBoolean());
+        Assert.Equal(
+            new[] { "v2:n2:2.0", "v3:n3:3.0", "vOld:nOld:0.9" },
+            root.GetProperty("sampleKeys").EnumerateArray().Select(e => e.GetString()).ToArray());
+        // The storage failure is described by status only — never the raw response body.
+        Assert.Equal("RequestFailedException 400", root.GetProperty("error").GetString());
+        Assert.DoesNotContain("HTML", evt.Message, StringComparison.OrdinalIgnoreCase);
    }

    [Fact]
-    public async Task ApplyInventoryContributionsAsync_audits_decrement_failures_too()
+    public async Task ApplyInventoryContributionsAsync_writes_no_ops_event_when_every_counter_write_succeeds()
    {
        var vulnRepo = new Mock<IVulnerabilityRepository>();
        vulnRepo
@@ -271,24 +273,15 @@ public class VulnerabilityCorrelationServiceContributionsTests
                TenantId, SessionId, It.IsAny<IReadOnlyList<SoftwareInventoryItem>>(), It.IsAny<CancellationToken>()))
            .ReturnsAsync(new InventoryContributionsDelta
            {
-                AddedItems = new List<SoftwareInventoryItem>(),
-                RemovedKeys = new List<DeletionDecrementKey>
+                AddedItems = new List<SoftwareInventoryItem>
                {
-                    new DeletionDecrementKey { Vendor = "vDead", Name = "nDead", Version = "0.5" },
+                    new SoftwareInventoryItem { NormalizedVendor = "v1", NormalizedName = "n1", NormalizedVersion = "1.0" },
                },
+                RemovedKeys = new List<DeletionDecrementKey>(),
            });

-        vulnRepo.Setup(r => r.DecrementSoftwareInventoryEntryAsync(
-                TenantId, "vDead", "nDead", "0.5", It.IsAny<CancellationToken>()))
-            .ThrowsAsync(new InvalidOperationException("Decrement ETag-CAS exhausted"));
-
-        var maintenanceRepo = new Mock<IMaintenanceRepository>();
-        maintenanceRepo.Setup(m => m.LogAuditEntryAsync(
-                It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
-                It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Dictionary<string, string>?>()))
-            .ReturnsAsync(true);
-
-        var service = NewService(vulnRepo, maintenanceRepo);
+        var (opsEvents, saved) = OpsRig();
+        var service = NewService(vulnRepo, opsEvents);
        var raw = new List<Dictionary<string, object>>
        {
            new Dictionary<string, object> { ["normalizedPublisher"] = "v1", ["normalizedName"] = "n1", ["normalizedVersion"] = "1.0" },
@@ -296,23 +289,34 @@ public class VulnerabilityCorrelationServiceContributionsTests

        await service.ApplyInventoryContributionsAsync(TenantId, SessionId, raw, cpeMappings: null);

-        maintenanceRepo.Verify(m => m.LogAuditEntryAsync(
-            TenantId,
-            "inventory_counter_write_failed",
-            "Session",
-            SessionId,
-            "system",
-            It.Is<Dictionary<string, string>?>(d =>
-                d != null
-                && d["operation"] == "decrement"
-                && d["vendor"] == "vDead"
-                && d["name"] == "nDead"
-                && d["version"] == "0.5")),
-            Times.Once);
+        Assert.Empty(saved);
+    }
+
+    [Fact]
+    public void DescribeCounterFailure_keeps_the_first_line_of_a_non_storage_message()
+    {
+        Assert.Equal("InvalidOperationException: first line",
+            VulnerabilityCorrelationService.DescribeCounterFailure(new InvalidOperationException("first line\nsecond line")));
+        Assert.Equal("RequestFailedException 412 ConditionNotMet",
+            VulnerabilityCorrelationService.DescribeCounterFailure(new RequestFailedException(412, "body", "ConditionNotMet", null)));
+    }
+
+    private static (OpsEventService Service, List<OpsEventEntry> Saved) OpsRig()
+    {
+        var saved = new List<OpsEventEntry>();
+        var opsRepo = new Mock<IOpsEventRepository>();
+        opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>()))
+            .Callback<OpsEventEntry>(e => { lock (saved) saved.Add(e); })
+            .Returns(Task.CompletedTask);
+        var adminConfig = new Mock<AdminConfigurationService>(
+            Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance,
+            new MemoryCache(new MemoryCacheOptions())) { CallBase = false };
+        var alertDispatch = TestNotifications.InertOpsAlertDispatch(adminConfig.Object);
+        return (new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance, alertDispatch), saved);
    }

    private static VulnerabilityCorrelationService NewService(
-        Mock<IVulnerabilityRepository> vulnRepo, Mock<IMaintenanceRepository>? maintenanceRepo = null)
+        Mock<IVulnerabilityRepository> vulnRepo, OpsEventService? opsEvents = null)
    {
        return new VulnerabilityCorrelationService(
            vulnRepo.Object,
@@ -323,6 +327,6 @@ public class VulnerabilityCorrelationServiceContributionsTests
            adminConfig: null!,
            telemetryClient: null,
            logger: NullLogger<VulnerabilityCorrelationService>.Instance,
-            maintenanceRepo: maintenanceRepo?.Object);
+            opsEvents: opsEvents);
    }
 }
@@ -96,6 +96,43 @@ public class VulnerabilityInventoryLoaderTests
        Assert.Single(result);
    }

+    [Fact]
+    public async Task Loader_strips_control_characters_from_inventory_text()
+    {
+        // Installers can pad REG_SZ values with NULs. The vendor/name/version triple becomes a
+        // table key, so the loader hands out text without control characters; other values pass.
+        var evt = new EnrollmentEvent
+        {
+            TenantId  = TenantId,
+            SessionId = SessionId,
+            EventType = Constants.EventTypes.SoftwareInventoryAnalysis,
+            Data = new Dictionary<string, object>
+            {
+                ["triggered_at"] = "shutdown",
+                ["inventory"] = new List<object>
+                {
+                    new Dictionary<string, object>
+                    {
+                        ["displayName"]         = "Contoso Widget\0\0",
+                        ["normalizedName"]      = "contoso widget\0\0\0\0\0",
+                        ["normalizedPublisher"] = "contoso \0\0\0",
+                        ["normalizedVersion"]   = "1.0",
+                        ["isWindowsInstaller"]  = true,
+                    },
+                },
+            },
+        };
+
+        var result = await BuildLoader(new List<EnrollmentEvent> { evt }).LoadShutdownInventoryAsync(TenantId, SessionId);
+
+        var item = Assert.Single(result);
+        Assert.Equal("Contoso Widget", item["displayName"]);
+        Assert.Equal("contoso widget", item["normalizedName"]);
+        Assert.Equal("contoso", item["normalizedPublisher"]);
+        Assert.Equal("1.0", item["normalizedVersion"]);
+        Assert.Equal(true, item["isWindowsInstaller"]);
+    }
+
    // -------------------------------------------------------------------------
    // Phase filtering — guards against cross-WG-phase contamination
    // -------------------------------------------------------------------------
@@ -1,4 +1,5 @@
 using System.Text.Json;
+using AutopilotMonitor.Functions.Helpers;
 using AutopilotMonitor.Functions.Services;
 using AutopilotMonitor.Shared;
 using AutopilotMonitor.Shared.DataAccess;
@@ -431,9 +432,9 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
        {
            // Paths are a closed vocabulary plus rule ids (ANALYZE-...), statuses are enum names and
            // the group marker "*" - none contain table-key-hostile characters, but sanitize anyway.
-            var k = SanitizeTableKey((kind ?? string.Empty).Trim().ToLowerInvariant());
-            var p = SanitizeTableKey((verdictPath ?? string.Empty).Trim().ToLowerInvariant());
-            var s = SanitizeTableKey((status ?? string.Empty).Trim().ToLowerInvariant());
+            var k = TableKeySanitizer.Sanitize((kind ?? string.Empty).Trim().ToLowerInvariant());
+            var p = TableKeySanitizer.Sanitize((verdictPath ?? string.Empty).Trim().ToLowerInvariant());
+            var s = TableKeySanitizer.Sanitize((status ?? string.Empty).Trim().ToLowerInvariant());
            return $"{VerdictCalibrationRowKeyPrefix}{k}|{p}|{s}";
        }

@@ -579,25 +580,15 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
        /// </summary>
        internal static string BuildAppVersionRegressionRowKey(string? source, string appName, string version)
        {
-            var app = SanitizeTableKey((appName ?? string.Empty).Trim().ToLowerInvariant());
-            var ver = SanitizeTableKey((version ?? string.Empty).Trim().ToLowerInvariant());
+            var app = TableKeySanitizer.Sanitize((appName ?? string.Empty).Trim().ToLowerInvariant());
+            var ver = TableKeySanitizer.Sanitize((version ?? string.Empty).Trim().ToLowerInvariant());
            // IME keys predate channels and keep their shape; other channels prefix the app part.
            var channel = AppInstallSources.Normalize(source);
            if (channel != AppInstallSources.Ime)
-                app = $"{SanitizeTableKey(channel)}:{app}";
+                app = $"{TableKeySanitizer.Sanitize(channel)}:{app}";
            return $"{AppVersionRegressionRowKeyPrefix}{app}|{ver}";
        }

-        /// <summary>Replace characters not allowed in Azure Table Storage keys (/, \, #, ?).</summary>
-        private static string SanitizeTableKey(string key)
-        {
-            return key
-                .Replace("/", "_")
-                .Replace("\\", "_")
-                .Replace("#", "_")
-                .Replace("?", "_");
-        }
-
        // internal static: entity builder + mapper are pinned by round-trip unit tests
        // (table-storage-serialization rule — every property must survive Store→Map).
        internal static TableEntity BuildAppVersionRegressionEntity(string tenantId, AppVersionRegressionAlert alert)
@@ -48,10 +48,6 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage

        // --- Software Inventory ---

-        public Task UpsertSoftwareInventoryAsync(string tenantId, List<Dictionary<string, object>> inventoryItems,
-            string sessionId, Dictionary<string, string?>? cpeMappings = null)
-            => _storage.UpsertSoftwareInventoryAsync(tenantId, inventoryItems, sessionId, cpeMappings);
-
        public Task<List<Dictionary<string, object>>> GetSoftwareInventoryAsync(string tenantId)
            => _storage.GetSoftwareInventoryAsync(tenantId);

@@ -121,9 +121,8 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
                            item.SoftwareName);

                        // Save as custom mapping
-                        var rowKey = $"{(mapping.NormalizedVendor ?? "unknown")}:{mapping.NormalizedProduct}"
-                            .ToLowerInvariant()
-                            .Replace("/", "_").Replace("\\", "_").Replace("#", "_").Replace("?", "_");
+                        var rowKey = TableKeySanitizer.Sanitize(
+                            $"{(mapping.NormalizedVendor ?? "unknown")}:{mapping.NormalizedProduct}".ToLowerInvariant());

                        var properties = new Dictionary<string, object>
                        {
@@ -49,8 +49,8 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
                    return await req.BadRequestAsync("normalizedProduct is required");
                }

-                var rowKey = $"{(request.NormalizedVendor ?? "unknown")}:{request.NormalizedProduct}".ToLowerInvariant();
-                rowKey = rowKey.Replace("/", "_").Replace("\\", "_").Replace("#", "_").Replace("?", "_");
+                var rowKey = TableKeySanitizer.Sanitize(
+                    $"{(request.NormalizedVendor ?? "unknown")}:{request.NormalizedProduct}".ToLowerInvariant());

                // Check if mapping exists before deleting (repository silently ignores 404)
                var existingMappings = await _vulnRepo.GetCpeMappingsByPartitionAsync("cpe_map_custom");
@@ -85,9 +85,8 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
                {
                    if (string.IsNullOrWhiteSpace(item.SoftwareName)) continue;

-                    var rowKey = $"{(item.Publisher ?? "unknown")}:{item.SoftwareName}"
-                        .ToLowerInvariant()
-                        .Replace("/", "_").Replace("\\", "_").Replace("#", "_").Replace("?", "_");
+                    var rowKey = TableKeySanitizer.Sanitize(
+                        $"{(item.Publisher ?? "unknown")}:{item.SoftwareName}".ToLowerInvariant());

                    var properties = new Dictionary<string, object>
                    {
@@ -131,9 +130,8 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
                    return await req.BadRequestAsync("softwareName is required");
                }

-                var rowKey = $"{(request.Publisher ?? "unknown")}:{request.SoftwareName}"
-                    .ToLowerInvariant()
-                    .Replace("/", "_").Replace("\\", "_").Replace("#", "_").Replace("?", "_");
+                var rowKey = TableKeySanitizer.Sanitize(
+                    $"{(request.Publisher ?? "unknown")}:{request.SoftwareName}".ToLowerInvariant());

                await _vulnRepo.DeleteCpeMappingAsync(IgnorePartition, rowKey);

@@ -56,9 +56,8 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
                mapping.CpeUri = CpeUriNormalizer.Normalize(mapping.CpeUri);
                mapping.CpeProduct = CpeUriNormalizer.Normalize(mapping.CpeProduct);

-                var rowKey = $"{(mapping.NormalizedVendor ?? "unknown")}:{mapping.NormalizedProduct}".ToLowerInvariant();
-                // Sanitize RowKey (Table Storage doesn't allow /, \, #, ?)
-                rowKey = rowKey.Replace("/", "_").Replace("\\", "_").Replace("#", "_").Replace("?", "_");
+                var rowKey = TableKeySanitizer.Sanitize(
+                    $"{(mapping.NormalizedVendor ?? "unknown")}:{mapping.NormalizedProduct}".ToLowerInvariant());

                var properties = new Dictionary<string, object>
                {
@@ -32,7 +32,7 @@ public static class EndpointNormalizer
        // Replace GUIDs with {id}
        normalized = GuidPattern.Replace(normalized, "{id}");

-        return ReplaceIllegalKeyChars(normalized).ToLowerInvariant();
+        return TableKeySanitizer.Sanitize(normalized).ToLowerInvariant();
    }

    /// <summary>
@@ -53,11 +53,7 @@ public static class EndpointNormalizer
        // recognizes; the regex pass then drops any remaining control character or separator.
        var cleaned = toolName.Replace("\r", string.Empty).Replace("\n", string.Empty);
        cleaned = ControlChars.Replace(cleaned, string.Empty);
-        cleaned = ReplaceIllegalKeyChars(cleaned).Trim();
+        cleaned = TableKeySanitizer.Sanitize(cleaned).Trim();
        return cleaned.Length <= MaxToolNameLength ? cleaned : cleaned[..MaxToolNameLength];
    }
-
-    /// <summary>Replaces the characters Azure Table Storage rejects in keys (/, \, #, ?).</summary>
-    private static string ReplaceIllegalKeyChars(string value)
-        => value.Replace('/', '_').Replace('\\', '_').Replace('#', '_').Replace('?', '_');
 }
@@ -0,0 +1,45 @@
+using System.Text;
+
+namespace AutopilotMonitor.Functions.Helpers;
+
+/// <summary>
+/// The one rule for turning free text (device inventory, app names, admin input) into an Azure
+/// Table Storage key. Every builder of a key from free text uses it, so the same text always maps
+/// to the same key.
+/// <para>
+/// Control characters (U+0000–U+001F, U+007F–U+009F) are removed: Table Storage rejects them in
+/// keys, and a point read, update or delete carries the key in its request URL, which the HTTP layer
+/// in front of the service refuses outright ("400 Invalid URL"). The characters Table Storage
+/// rejects in keys (/, \, #, ?) become '_'. Text without control characters keeps its key byte for
+/// byte, so existing rows stay reachable.
+/// </para>
+/// </summary>
+public static class TableKeySanitizer
+{
+    public static string Sanitize(string key)
+    {
+        var sb = new StringBuilder(key.Length);
+        foreach (var c in key)
+        {
+            if (char.IsControl(c))
+                continue;
+            sb.Append(c is '/' or '\\' or '#' or '?' ? '_' : c);
+        }
+        return sb.ToString();
+    }
+
+    /// <summary>
+    /// Removes control characters only — for text that later becomes part of a key or an identity
+    /// compared against one (the software inventory triple).
+    /// </summary>
+    public static string RemoveControlCharacters(string text)
+    {
+        var sb = new StringBuilder(text.Length);
+        foreach (var c in text)
+        {
+            if (!char.IsControl(c))
+                sb.Append(c);
+        }
+        return sb.ToString();
+    }
+}
@@ -171,6 +171,23 @@ namespace AutopilotMonitor.Functions.Services
                });
        }

+        /// <summary>
+        /// One vulnerability-correlation run could not write every SoftwareInventory counter, or
+        /// its per-session contributions side-row (<paramref name="sideRowFailed"/>: then no
+        /// counter was touched). One event per run, never per key. The counters stay off by these
+        /// amounts (accepted drift, Plan §17), so a repeating event points at a systematic cause.
+        /// <paramref name="error"/> is the exception type plus a short reason — never a response body.
+        /// </summary>
+        public Task RecordSoftwareInventoryCounterWriteFailedAsync(
+            string tenantId, string sessionId, int failedIncrements, int failedDecrements, bool sideRowFailed,
+            string error, IReadOnlyList<string> sampleKeys)
+            => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SoftwareInventoryCounterWriteFailed, OpsEventSeverity.Warning,
+                sideRowFailed
+                    ? $"Software inventory contributions not written for session {sessionId}: {error}"
+                    : $"{failedIncrements + failedDecrements} software inventory counter writes failed for session {sessionId}: {error}",
+                tenantId, "System.VulnerabilityCorrelation",
+                new { sessionId, failedIncrements, failedDecrements, sideRowFailed, error, sampleKeys });
+
        // ── Cascade-Delete Maintenance (Plan §5 PR6 / §16 R14) ─────────────────
        // Event types dispatched by SessionDeletionMaintenanceFunction: Started, LongRunning,
        // LongRunningSevere, BudgetExceeded, SkippedLocked, Failed, Completed, FanoutSkipped
@@ -5,6 +5,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using Azure;
 using Azure.Data.Tables;
+using AutopilotMonitor.Functions.Helpers;
 using AutopilotMonitor.Functions.Security;
 using AutopilotMonitor.Shared;
 using AutopilotMonitor.Shared.Models.Deletion;
@@ -450,15 +451,13 @@ namespace AutopilotMonitor.Functions.Services
        // ============================================================ Helpers ====

        /// <summary>
-        /// SoftwareInventory RowKey format — preserves the same shape used by the legacy
-        /// <c>UpsertSoftwareInventoryAsync</c> path so existing rows remain reachable by the
-        /// new helpers. <c>SanitizeTableKey</c> lives in TableStorageService.Rules.cs as a
-        /// private static; partial classes share scope.
+        /// SoftwareInventory RowKey format: <c>vendor:name:version</c> through
+        /// <see cref="TableKeySanitizer"/>, capped at 512 characters.
        /// </summary>
        private static string BuildSoftwareInventoryRowKey(string vendor, string name, string version)
        {
            var raw = $"{vendor ?? string.Empty}:{name ?? string.Empty}:{version ?? string.Empty}";
-            var sanitized = SanitizeTableKey(raw);
+            var sanitized = TableKeySanitizer.Sanitize(raw);
            return sanitized.Length > 512 ? sanitized.Substring(0, 512) : sanitized;
        }
    }
@@ -249,7 +249,7 @@ namespace AutopilotMonitor.Functions.Services
        internal static string BuildAppInstallSummaryRowKey(AppInstallSummary summary)
        {
            var source = AppInstallSources.Normalize(summary.Source);
-            return SanitizeTableKey(source == AppInstallSources.Ime
+            return TableKeySanitizer.Sanitize(source == AppInstallSources.Ime
                ? $"{summary.SessionId}_{summary.AppName}"
                : $"{summary.SessionId}_{source}|{summary.InstallScope}|{summary.AppId}");
        }
@@ -1,6 +1,7 @@
 using System.Text.Json;
 using Azure.Data.Tables;
 using AutopilotMonitor.Functions.DataAccess.TableStorage;
+using AutopilotMonitor.Functions.Helpers;
 using AutopilotMonitor.Functions.Security;
 using AutopilotMonitor.Functions.Services.Caching;
 using AutopilotMonitor.Functions.Services.Vulnerability;
@@ -1044,104 +1045,6 @@ namespace AutopilotMonitor.Functions.Services

        // ===== SOFTWARE INVENTORY METHODS =====

-        /// <summary>
-        /// Upserts software inventory entries for a tenant.
-        /// PK = tenantId, RK = {normalizedVendor}:{normalizedProduct}:{normalizedVersion} (sanitized).
-        /// If an entry already exists, increments SessionCount and updates LastSeenAt/LastSessionId.
-        /// </summary>
-        public async Task UpsertSoftwareInventoryAsync(string tenantId, List<Dictionary<string, object>> inventoryItems, string sessionId, Dictionary<string, string?>? cpeMappings = null)
-        {
-            SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
-
-            try
-            {
-                var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.SoftwareInventory);
-
-                // Read all existing entities for this tenant to merge SessionCount
-                var existingEntities = new Dictionary<string, TableEntity>(StringComparer.OrdinalIgnoreCase);
-                var existingQuery = tableClient.QueryAsync<TableEntity>(filter: $"PartitionKey eq '{tenantId}'");
-                await foreach (var entity in existingQuery)
-                {
-                    existingEntities[entity.RowKey] = entity;
-                }
-
-                // Build the batch of entities to upsert (dictionary to deduplicate by RowKey)
-                var entitiesToUpsert = new Dictionary<string, TableEntity>(StringComparer.OrdinalIgnoreCase);
-
-                foreach (var item in inventoryItems)
-                {
-                    var normalizedName = item.ContainsKey("normalizedName") ? item["normalizedName"]?.ToString() ?? "" : "";
-                    var normalizedVendor = item.ContainsKey("normalizedPublisher") ? item["normalizedPublisher"]?.ToString() ?? "" : "";
-                    var normalizedVersion = item.ContainsKey("normalizedVersion") ? item["normalizedVersion"]?.ToString() ?? "" : "";
-                    var displayName = item.ContainsKey("displayName") ? item["displayName"]?.ToString() ?? "" : "";
-                    var publisher = item.ContainsKey("publisher") ? item["publisher"]?.ToString() ?? "" : "";
-                    var registrySource = item.ContainsKey("registrySource") ? item["registrySource"]?.ToString() ?? "" : "";
-                    var normalizationConfidence = item.ContainsKey("normalizationConfidence") ? item["normalizationConfidence"]?.ToString() ?? "" : "";
-
-                    var rowKey = SanitizeTableKey($"{normalizedVendor}:{normalizedName}:{normalizedVersion}");
-                    if (string.IsNullOrWhiteSpace(rowKey) || rowKey == "::")
-                        continue;
-
-                    // Truncate RowKey to Table Storage limit (1KB)
-                    if (rowKey.Length > 512)
-                        rowKey = rowKey.Substring(0, 512);
-
-                    var now = DateTime.UtcNow;
-                    TableEntity entity;
-
-                    if (existingEntities.TryGetValue(rowKey, out var existing))
-                    {
-                        // Update existing: increment SessionCount, update LastSeenAt
-                        existing["SessionCount"] = (existing.GetInt32("SessionCount") ?? 0) + 1;
-                        existing["LastSeenAt"] = now.ToString("o");
-                        existing["LastSessionId"] = sessionId;
-
-                        // Update CpeUri if we have a mapping and current is empty
-                        if (cpeMappings != null && cpeMappings.TryGetValue(normalizedName, out var cpeUri) && !string.IsNullOrEmpty(cpeUri))
-                        {
-                            existing["CpeUri"] = cpeUri;
-                        }
-
-                        entity = existing;
-                    }
-                    else
-                    {
-                        // Create new entry
-                        entity = new TableEntity(tenantId, rowKey)
-                        {
-                            ["DisplayName"] = displayName,
-                            ["NormalizedName"] = normalizedName,
-                            ["NormalizedVendor"] = normalizedVendor,
-                            ["NormalizedVersion"] = normalizedVersion,
-                            ["Publisher"] = publisher,
-                            ["RegistrySource"] = registrySource,
-                            ["NormalizationConfidence"] = normalizationConfidence,
-                            ["FirstSeenAt"] = now.ToString("o"),
-                            ["LastSeenAt"] = now.ToString("o"),
-                            ["FirstSessionId"] = sessionId,
-                            ["LastSessionId"] = sessionId,
-                            ["SessionCount"] = 1,
-                            ["CpeUri"] = cpeMappings != null && cpeMappings.TryGetValue(normalizedName, out var cpeUri) && !string.IsNullOrEmpty(cpeUri) ? cpeUri : ""
-                        };
-                    }
-
-                    entitiesToUpsert[rowKey] = entity;
-                }
-
-                // Batch write (byte-aware, all same PK)
-                var upsertList = entitiesToUpsert.Values.ToList();
-                foreach (var batch in TableTransactionBatcher.Split(upsertList, TableTransactionActionType.UpsertReplace))
-                    await tableClient.SubmitTransactionAsync(batch);
-
-                _logger.LogInformation("Upserted {Count} software inventory entries for tenant {TenantId}", upsertList.Count, tenantId);
-            }
-            catch (Exception ex)
-            {
-                _logger.LogWarning(ex, "Failed to upsert software inventory for tenant {TenantId}", tenantId);
-                throw;
-            }
-        }
-
        /// <summary>
        /// Gets all software inventory entries for a tenant.
        /// PK = tenantId. Returns all rows.
@@ -1185,19 +1088,6 @@ namespace AutopilotMonitor.Functions.Services
            }
        }

-        /// <summary>
-        /// Sanitize a string for use as an Azure Table Storage key.
-        /// Replaces /, \, #, ? with underscores.
-        /// </summary>
-        private static string SanitizeTableKey(string key)
-        {
-            return key
-                .Replace("/", "_")
-                .Replace("\\", "_")
-                .Replace("#", "_")
-                .Replace("?", "_");
-        }
-
        // ===== CPE MAPPING SEED IMPORT =====

        /// <summary>
@@ -1231,7 +1121,7 @@ namespace AutopilotMonitor.Functions.Services
            var entities = new List<TableEntity>();
            foreach (var mapping in seed.Mappings)
            {
-                var rowKey = SanitizeTableKey(
+                var rowKey = TableKeySanitizer.Sanitize(
                    $"{(mapping.NormalizedVendor ?? "unknown")}:{(mapping.NormalizedProduct ?? "unknown")}".ToLowerInvariant());

                if (rowKey.Length > 512)
@@ -1291,7 +1181,7 @@ namespace AutopilotMonitor.Functions.Services
            var entities = new List<TableEntity>();
            foreach (var mapping in seed.Mappings)
            {
-                var rowKey = SanitizeTableKey(
+                var rowKey = TableKeySanitizer.Sanitize(
                    $"{(mapping.NormalizedVendor ?? "unknown")}:{(mapping.NormalizedProduct ?? "unknown")}".ToLowerInvariant());

                if (rowKey.Length > 512)
@@ -6,6 +6,7 @@ using System.Text.Json.Serialization;
 using System.Threading;
 using System.Threading.Tasks;
 using AutopilotMonitor.Functions.DataAccess.TableStorage;
+using AutopilotMonitor.Functions.Helpers;
 using AutopilotMonitor.Shared.DataAccess;
 using AutopilotMonitor.Shared.Models.Deletion;
 using Microsoft.ApplicationInsights;
@@ -17,7 +18,7 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
    {
        private readonly IVulnerabilityRepository _vulnRepo;
        private readonly ISessionRepository _sessionRepo;
-        private readonly IMaintenanceRepository? _maintenanceRepo;
+        private readonly OpsEventService? _opsEvents;
        private readonly NvdApiClient _nvdApi;
        private readonly KevDataService _kevService;
        private readonly MsrcCorrelationService _msrcCorrelation;
@@ -48,12 +49,12 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
            AdminConfigurationService adminConfig,
            TelemetryClient? telemetryClient,
            ILogger<VulnerabilityCorrelationService> logger,
-            IMaintenanceRepository? maintenanceRepo = null,
+            OpsEventService? opsEvents = null,
            EpssApiClient? epssApi = null)
        {
            _vulnRepo = vulnRepo;
            _sessionRepo = sessionRepo;
-            _maintenanceRepo = maintenanceRepo;
+            _opsEvents = opsEvents;
            _nvdApi = nvdApi;
            _kevService = kevService;
            _msrcCorrelation = msrcCorrelation;
@@ -267,13 +268,13 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
                return ra.CompareTo(rb);
            });

-            // Persist software inventory via the contributions side-row (Plan §17).
-            // Fire-and-forget so the report still gets stored if the side-row write hits a
-            // transient storage failure — the report is the primary product, inventory is the
-            // secondary tenant-aggregate signal. PR3 wires SessionDeletionGuard above this
-            // layer (at VulnerabilityCorrelateHandler) so a session locked by a cascade never
-            // reaches this code path; PR2's side-row write proceeds unguarded.
-            _ = Task.Run(() => ApplyInventoryContributionsAsync(tenantId, sessionId, inventoryItems, cpeMappingsDict));
+            // Persist software inventory via the contributions side-row (Plan §17). Awaited inside
+            // the invocation: a detached task can be cut off by a scale-in after the side-row has
+            // already promised the key set, leaving the counters short with no trace. The method
+            // never throws, so the report — the primary product — is stored regardless. PR3 wires
+            // SessionDeletionGuard above this layer (at VulnerabilityCorrelateHandler) so a session
+            // locked by a cascade never reaches this code path.
+            await ApplyInventoryContributionsAsync(tenantId, sessionId, inventoryItems, cpeMappingsDict);

            // Summary log for production monitoring
            _logger.LogInformation(
@@ -536,9 +537,8 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
                {
                    try
                    {
-                        var rowKey = $"{(mapping.NormalizedVendor ?? "unknown")}:{mapping.NormalizedProduct}"
-                            .ToLowerInvariant()
-                            .Replace("/", "_").Replace("\\", "_").Replace("#", "_").Replace("?", "_");
+                        var rowKey = TableKeySanitizer.Sanitize(
+                            $"{(mapping.NormalizedVendor ?? "unknown")}:{mapping.NormalizedProduct}".ToLowerInvariant());

                        var properties = new Dictionary<string, object>
                        {
@@ -613,7 +613,7 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
            try
            {
                // 1. Try cache first
-                var cacheKey = SanitizeTableKey($"cve_{cpeUri}");
+                var cacheKey = TableKeySanitizer.Sanitize($"cve_{cpeUri}");
                var cachedResult = await GetCachedCvesAsync(cacheKey);

                if (cachedResult == null || cachedResult.IsHardExpired)
@@ -1076,13 +1076,13 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
        /// <c>+1</c> per added (with full metadata for first-insert) and <c>-1</c> per removed
        /// (clamp ≥ 0).
        /// <para>
-        /// Per Plan §17.3 "Side-row write before counter writes" + §17.8: individual counter
-        /// failures must NOT abort the rest of the loop, because the side-row already promises
-        /// the new key set — if we stopped on first failure, the next correlation run would see
-        /// an empty delta and never retry the under-counted keys. Each counter call is therefore
-        /// guarded individually; failures emit an <c>inventory_counter_write_failed</c> audit
-        /// entry (when the maintenance repo is wired) and continue processing. The wrapping
-        /// catch handles only the side-row CAS exhaustion + the parser path.
+        /// Per Plan §17.3 "Side-row write before counter writes": individual counter failures must
+        /// NOT abort the rest of the loop, because the side-row already promises the new key set —
+        /// if we stopped on first failure, the next correlation run would see an empty delta and
+        /// never retry the under-counted keys. Each counter call is therefore guarded individually;
+        /// the failures of a run are counted and reported as ONE
+        /// <c>SoftwareInventoryCounterWriteFailed</c> ops event — an operator signal, never an entry
+        /// in the tenant's audit trail. Never throws.
        /// </para>
        /// </summary>
        internal async Task ApplyInventoryContributionsAsync(
@@ -1102,9 +1102,21 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
                _logger.LogWarning(ex,
                    "Failed to upsert software inventory contributions side-row (non-fatal). tenant={TenantId} session={SessionId}",
                    tenantId, sessionId);
+                await ReportCounterFailuresAsync(tenantId, sessionId, 0, 0, sideRowFailed: true, ex, Array.Empty<string>());
                return;
            }

+            var failedIncrements = 0;
+            var failedDecrements = 0;
+            Exception? firstError = null;
+            var sampleKeys = new List<string>();
+            void Record(Exception ex, string vendor, string name, string version)
+            {
+                firstError ??= ex;
+                if (sampleKeys.Count < MaxCounterFailureSamples)
+                    sampleKeys.Add($"{vendor}:{name}:{version}");
+            }
+
            foreach (var added in delta.AddedItems)
            {
                try
@@ -1116,9 +1128,8 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
                    _logger.LogWarning(ex,
                        "SoftwareInventory increment failed (non-fatal — counter drift accepted). tenant={TenantId} session={SessionId} vendor={Vendor} name={Name} version={Version}",
                        tenantId, sessionId, added.NormalizedVendor, added.NormalizedName, added.NormalizedVersion);
-                    await TryAuditInventoryCounterFailureAsync(
-                        tenantId, sessionId, "increment",
-                        added.NormalizedVendor, added.NormalizedName, added.NormalizedVersion, ex);
+                    failedIncrements++;
+                    Record(ex, added.NormalizedVendor, added.NormalizedName, added.NormalizedVersion);
                }
            }

@@ -1134,46 +1145,43 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
                    _logger.LogWarning(ex,
                        "SoftwareInventory decrement failed (non-fatal — counter drift accepted). tenant={TenantId} session={SessionId} vendor={Vendor} name={Name} version={Version}",
                        tenantId, sessionId, removed.Vendor, removed.Name, removed.Version);
-                    await TryAuditInventoryCounterFailureAsync(
-                        tenantId, sessionId, "decrement",
-                        removed.Vendor, removed.Name, removed.Version, ex);
+                    failedDecrements++;
+                    Record(ex, removed.Vendor, removed.Name, removed.Version);
                }
            }
+
+            if (firstError != null)
+                await ReportCounterFailuresAsync(tenantId, sessionId, failedIncrements, failedDecrements, sideRowFailed: false, firstError, sampleKeys);
        }

+        private const int MaxCounterFailureSamples = 5;
+
+        private Task ReportCounterFailuresAsync(
+            string tenantId, string sessionId, int failedIncrements, int failedDecrements, bool sideRowFailed,
+            Exception firstError, IReadOnlyList<string> sampleKeys)
+            => _opsEvents?.RecordSoftwareInventoryCounterWriteFailedAsync(
+                   tenantId, sessionId, failedIncrements, failedDecrements, sideRowFailed,
+                   DescribeCounterFailure(firstError), sampleKeys)
+               ?? Task.CompletedTask;
+
        /// <summary>
-        /// Emits the <c>inventory_counter_write_failed</c> audit entry on a single counter
-        /// failure (Plan §17.8). Non-fatal — audit emission itself must not break the loop.
+        /// Exception type plus a short reason for the ops event. A storage failure is described by
+        /// status and error code only — its message embeds the raw response body and headers.
        /// </summary>
-        private async Task TryAuditInventoryCounterFailureAsync(
-            string tenantId, string sessionId, string operation,
-            string vendor, string name, string version, Exception failureCause)
+        internal static string DescribeCounterFailure(Exception ex)
        {
-            if (_maintenanceRepo == null) return;
-            try
+            if (ex is Azure.RequestFailedException rfe)
            {
-                await _maintenanceRepo.LogAuditEntryAsync(
-                    tenantId,
-                    action: "inventory_counter_write_failed",
-                    entityType: "Session",
-                    entityId: sessionId,
-                    performedBy: "system",
-                    details: new Dictionary<string, string>(StringComparer.Ordinal)
-                    {
-                        ["operation"] = operation,
-                        ["vendor"] = vendor,
-                        ["name"] = name,
-                        ["version"] = version,
-                        ["exceptionType"] = failureCause.GetType().Name,
-                        ["exceptionMessage"] = failureCause.Message,
-                    });
-            }
-            catch (Exception ex)
-            {
-                _logger.LogDebug(ex,
-                    "Failed to emit inventory_counter_write_failed audit entry (suppressed). tenant={TenantId} session={SessionId}",
-                    tenantId, sessionId);
+                return string.IsNullOrEmpty(rfe.ErrorCode)
+                    ? $"RequestFailedException {rfe.Status}"
+                    : $"RequestFailedException {rfe.Status} {rfe.ErrorCode}";
            }
+
+            var message = ex.Message;
+            var lineEnd = message.IndexOfAny(new[] { '\r', '\n' });
+            if (lineEnd >= 0) message = message.Substring(0, lineEnd);
+            if (message.Length > 200) message = message.Substring(0, 200);
+            return $"{ex.GetType().Name}: {message}";
        }

        /// <summary>
@@ -1226,18 +1234,6 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
            return items;
        }

-        /// <summary>
-        /// Replace characters not allowed in Azure Table Storage keys (/, \, #, ?).
-        /// </summary>
-        private static string SanitizeTableKey(string key)
-        {
-            return key
-                .Replace("/", "_")
-                .Replace("\\", "_")
-                .Replace("#", "_")
-                .Replace("?", "_");
-        }
-
        // -----------------------------------------------------------------------
        // CVE Cache Models
        // -----------------------------------------------------------------------
@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using AutopilotMonitor.Functions.Helpers;
 using AutopilotMonitor.Shared;
 using AutopilotMonitor.Shared.DataAccess;
 using Microsoft.Extensions.Logging;
@@ -70,13 +71,9 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
                    {
                        foreach (var entry in nativeList)
                        {
-                            if (entry is Dictionary<string, object> dict)
+                            if (entry is IDictionary<string, object> dict)
                            {
-                                items.Add(dict);
-                            }
-                            else if (entry is IDictionary<string, object> idict)
-                            {
-                                items.Add(new Dictionary<string, object>(idict));
+                                items.Add(WithoutControlCharacters(dict));
                            }
                        }
                    }
@@ -86,7 +83,7 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
                        {
                            if (item is Newtonsoft.Json.Linq.JObject obj)
                            {
-                                items.Add(obj.ToObject<Dictionary<string, object>>() ?? new Dictionary<string, object>());
+                                items.Add(WithoutControlCharacters(obj.ToObject<Dictionary<string, object>>() ?? new Dictionary<string, object>()));
                            }
                        }
                    }
@@ -104,6 +101,26 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
            return items;
        }

+        /// <summary>
+        /// Inventory text comes from the device registry, where some installers pad REG_SZ values
+        /// with NULs. The vendor/name/version triple built from it becomes a table key and a
+        /// side-row identity, and neither may carry a control character — so every text value
+        /// leaves the loader without them, trimmed. The source dictionary is not mutated.
+        /// </summary>
+        internal static Dictionary<string, object> WithoutControlCharacters(IDictionary<string, object> item)
+        {
+            var clean = item is Dictionary<string, object> typed
+                ? new Dictionary<string, object>(typed.Count, typed.Comparer)
+                : new Dictionary<string, object>(item.Count);
+            foreach (var (key, value) in item)
+            {
+                clean[key] = value is string text
+                    ? TableKeySanitizer.RemoveControlCharacters(text).Trim()
+                    : value;
+            }
+            return clean;
+        }
+
        /// <summary>
        /// Checks whether an event's <c>whiteglove_part</c> tag matches the requested phase.
        /// Numeric parsing is lenient because <c>EnrollmentEvent.Data</c> values can come
@@ -86,6 +86,7 @@ export const OPS_EVENT_TYPES = [
  "CriticalTableBackupSkippedLocked",
  "BackupRowRestored",
  "VerdictCalibrationDrift",
+  "SoftwareInventoryCounterWriteFailed",
  "DeviceBlocked",
  "VersionBlocked",
  "SessionTenantConflict",
@@ -23,8 +23,6 @@ namespace AutopilotMonitor.Shared.DataAccess
        Task DeleteVulnerabilityReportAsync(string tenantId, string sessionId);

        // --- Software Inventory ---
-        Task UpsertSoftwareInventoryAsync(string tenantId, List<Dictionary<string, object>> inventoryItems,
-            string sessionId, Dictionary<string, string?>? cpeMappings = null);
        Task<List<Dictionary<string, object>>> GetSoftwareInventoryAsync(string tenantId);
        /// <summary>
        /// Full cross-tenant scan of SoftwareInventory. <paramref name="selectColumns"/> projects the
@@ -54,6 +54,12 @@ namespace AutopilotMonitor.Shared.DataAccess
        public const string CriticalTableBackupSkippedLocked            = "CriticalTableBackupSkippedLocked";
        public const string BackupRowRestored                           = "BackupRowRestored";
        public const string VerdictCalibrationDrift                     = "VerdictCalibrationDrift";
+        /// <summary>
+        /// A vulnerability-correlation run could not write every SoftwareInventory counter, or the
+        /// per-session contributions side-row. Warning; one event per run with failure counts and
+        /// up to five sample keys, never one per key and never in the tenant audit trail.
+        /// </summary>
+        public const string SoftwareInventoryCounterWriteFailed         = "SoftwareInventoryCounterWriteFailed";

        // ── Security ── Blocks, ownership conflicts, certificate expiry, capacity + poison-queue alarms.
        public const string DeviceBlocked                      = "DeviceBlocked";
@@ -178,7 +184,7 @@ namespace AutopilotMonitor.Shared.DataAccess
        public static readonly IReadOnlyList<string> All = new[]
        {
            ConsentFlowStarted, ConsentFlowSuccess, ConsentFlowFailed, ConsentRedirectUriMismatch, AppHomingFlipped, AppHomingFlippedWithEntraRoles,
-            MaintenanceCompleted, MaintenanceFailed, MaintenanceLongRunning, MaintenanceStarted, MaintenanceSkippedLocked, SessionSweepCompleted, SessionSweepFailed, OpsEventCleanup, OrphanEventsCleaned, SessionDeletionMaintenanceStarted, SessionDeletionMaintenanceBudgetExceeded, SessionDeletionMaintenanceSkippedLocked, SessionDeletionMaintenanceLongRunning, SessionDeletionMaintenanceLongRunningSevere, SessionDeletionMaintenanceFailed, SessionDeletionStrandedQueued, SessionDeletionPoisoned, SessionDeletionMaintenanceCompleted, SessionDeletionMaintenanceFanoutSkipped, CriticalTableBackupCompleted, CriticalTableBackupPartial, CriticalTableBackupFailed, CriticalTableBackupSkippedLocked, BackupRowRestored, VerdictCalibrationDrift,
+            MaintenanceCompleted, MaintenanceFailed, MaintenanceLongRunning, MaintenanceStarted, MaintenanceSkippedLocked, SessionSweepCompleted, SessionSweepFailed, OpsEventCleanup, OrphanEventsCleaned, SessionDeletionMaintenanceStarted, SessionDeletionMaintenanceBudgetExceeded, SessionDeletionMaintenanceSkippedLocked, SessionDeletionMaintenanceLongRunning, SessionDeletionMaintenanceLongRunningSevere, SessionDeletionMaintenanceFailed, SessionDeletionStrandedQueued, SessionDeletionPoisoned, SessionDeletionMaintenanceCompleted, SessionDeletionMaintenanceFanoutSkipped, CriticalTableBackupCompleted, CriticalTableBackupPartial, CriticalTableBackupFailed, CriticalTableBackupSkippedLocked, BackupRowRestored, VerdictCalibrationDrift, SoftwareInventoryCounterWriteFailed,
            DeviceBlocked, VersionBlocked, SessionTenantConflict, SessionOwnerMismatch, KillSignalDelivered, EmbeddedCertExpiringSoon, EmbeddedCertExpiringUrgent, EmbeddedCertExpired, EmbeddedCertBundleEmpty, SignalRConnectionsHigh, SignalRConnectionsCritical, SignalRMessagesHigh, SignalRMessagesCritical, PoisonQueueBacklogHigh, PoisonQueueBacklogCritical, ExcessiveSessionEventsAutoActioned, PrivilegedRouteDenied, McpServicePrincipalFirstSeen,
            OffboardingFeedbackReceived, FeedbackReceived, TenantSignup, SessionReportSubmitted, DiagFilesReportSubmitted, RuleSubmissionReceived, TenantOffboarded, TenantOffboardingFailed, TenantAutoApproved, WelcomeEmailSent, WelcomeEmailSkipped, WelcomeEmailFailed, FarewellEmailSent, FarewellEmailSkipped, FarewellEmailFailed, TenantTrialStarted, TenantTrialExpiring, TenantTrialExpired, TenantPlanDowngraded, TenantRetentionGraceExpiring, TenantRetentionGraceEnded, RuleFrequencyRegression, AppVersionDurationRegression, CollectLogsQuickConfigEnabled, DiagnosticsUploadEnabled, DiagnosticsUploadDisabled,
            SessionActionQueued, SessionTimeouts, AgentEmergencyBreak, AgentBinaryIntegrityMismatch, CmTraceTimeSkewRegression, ExcessiveSessionEvents, NewImeVersionDetected, ImePatternDriftSuspected, BlobStorageMissing, BlobStorageUnreachable, TelemetryItemsRejected,
@@ -31,6 +31,10 @@ const OPS_EVENT_TYPES: Record<string, string[]> = {
    // fallthrough decides ≥20% of classifier verdicts. Warning-tier, once per episode
    // (tracker-deduped). Dual-register per memory feedback_ops_event_types_dual_register.
    "VerdictCalibrationDrift",
+    // A vulnerability-correlation run could not write every SoftwareInventory counter (or the
+    // per-session side-row). Warning, one per run with counts — replaced the per-key tenant audit
+    // entry inventory_counter_write_failed.
+    "SoftwareInventoryCounterWriteFailed",
    "MaintenanceCompleted",
    "MaintenanceFailed",
    // Soft watchdog (Warning): a maintenance run completed but exceeded the 10min threshold and is
@@ -620,6 +620,7 @@ export const SHARED_MANIFEST = {
    "CriticalTableBackupSkippedLocked",
    "BackupRowRestored",
    "VerdictCalibrationDrift",
+    "SoftwareInventoryCounterWriteFailed",
    "DeviceBlocked",
    "VersionBlocked",
    "SessionTenantConflict",
@@ -616,6 +616,7 @@
    "CriticalTableBackupSkippedLocked",
    "BackupRowRestored",
    "VerdictCalibrationDrift",
+    "SoftwareInventoryCounterWriteFailed",
    "DeviceBlocked",
    "VersionBlocked",
    "SessionTenantConflict",