feat(backend): surface genuine agent binary-integrity mismatches as ops events

IntegrityCheckFailed emergency reports used to land in App Insights
only — session e9753578's agent reported a hash mismatch and no product
surface showed it, which cost a day of mis-attributed root-causing.
They now materialize as an AgentBinaryIntegrityMismatch ops event
(Warning, Agent category, dual-registered in the web alert-rule
catalog per feedback_ops_event_types_dual_register).

With suppression for the KNOWN, accepted release race the operator
correctly called out: around every release, agents compare against
hash snapshots that do not belong to their own binary — an older agent
sees the new release's hash on its next config fetch, and a freshly
installed latest agent can receive the PREVIOUS release's hashes from
the 5-minute AdminConfiguration cache (e9753578: genuine 1410 exe
checked against the cached 1409 hash — proven by the 1409 archive
logging "verified OK" with exactly that hash). Both self-heal via the
forced self-update and are suppressed at materialization time against
the CURRENT config: reported running hash == published exe, or
reported version != latest. What remains — an agent claiming the
latest version whose binary is NOT the published one — is the real
signal: tamper, stale blob, or a build outside the release pipeline.
Missing config fails open to recording; visibility beats suppression.

Ops event only, deliberately no timeline event: the report is about
the binary, not the enrollment, and may precede session registration.

Backend suite 4320 green, web tsc clean.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

src/Backend/AutopilotMonitor.Functions.Tests/ReportAgentErrorFunctionTests.cs

@@ -200,4 +200,120 @@ public class ReportAgentErrorFunctionTests
 
         Assert.Empty(h.OpsEvents);
     }
+
+    // =========================================================================
+    // MaterializeIntegrityMismatchAsync — an IntegrityCheckFailed report becomes an
+    // AgentBinaryIntegrityMismatch ops event, UNLESS it is the known release race
+    // (agents comparing against hash snapshots that don't belong to their own binary;
+    // AdminConfig cache TTL 5 min). e9753578 proved the race: a genuine 1410 exe was
+    // compared against the cached 1409 hash and reported a "mismatch" that cost a day
+    // of mis-attributed root-causing while sitting invisible in App Insights.
+    // =========================================================================
+
+    private const string RunningExeSha = "5fca975e43fa5fca975e43fa5fca975e43fa5fca975e43fa5fca975e43fa0000";
+    private const string PublishedExeSha = "8ebaa060ad348ebaa060ad348ebaa060ad348ebaa060ad348ebaa060ad340000";
+
+    private static AgentErrorReport IntegrityReport(
+        string? sessionId = SessionId, string agentVersion = "2.0.1410", string? actualSha = null) => new()
+    {
+        SessionId = sessionId!,
+        TenantId = TenantId,
+        ErrorType = AgentErrorType.IntegrityCheckFailed,
+        Message = $"Running exe SHA-256 differs from backend-advertised hash. actual={actualSha ?? RunningExeSha}, expected={PublishedExeSha}",
+        AgentVersion = agentVersion,
+        Timestamp = Now,
+    };
+
+    private static Task RunIntegrityAsync(
+        Harness h, AgentErrorReport report,
+        string? latestVersion = "2.0.1410", string? latestExeSha = PublishedExeSha)
+        => ReportAgentErrorFunction.MaterializeIntegrityMismatchAsync(
+            report, TenantId, h.OpsService, latestVersion, latestExeSha,
+            NullLogger<ReportAgentErrorFunction>.Instance);
+
+    [Fact]
+    public async Task SameVersion_with_unknown_binary_records_ops_event()
+    {
+        // The real signal: an agent claiming the LATEST version whose running exe is NOT
+        // the published one — tamper, stale blob, or a build outside the release pipeline.
+        var h = new Harness();
+
+        await RunIntegrityAsync(h, IntegrityReport());
+
+        var ops = Assert.Single(h.OpsEvents);
+        Assert.Equal("AgentBinaryIntegrityMismatch", ops.EventType);
+        Assert.Equal(TenantId, ops.TenantId);
+        Assert.Contains("actual=5fca", ops.Message);
+        Assert.Contains("2.0.1410", ops.Message);
+    }
+
+    [Fact]
+    public async Task RunningHash_equal_to_published_exe_is_suppressed_as_stale_expectation()
+    {
+        // e9753578's exact shape: the device RUNS the published binary; only its cached
+        // expectation was the previous release's hash. Self-heals — must not alert.
+        var h = new Harness();
+
+        await RunIntegrityAsync(h, IntegrityReport(actualSha: PublishedExeSha));
+
+        Assert.Empty(h.OpsEvents);
+    }
+
+    [Fact]
+    public async Task Older_agent_version_than_latest_is_suppressed_as_update_race()
+    {
+        // The long-known direction: version N agent sees version N+1's hash right after a
+        // release; the forced self-update heals it.
+        var h = new Harness();
+
+        await RunIntegrityAsync(h, IntegrityReport(agentVersion: "2.0.1409+d43cfe2b"), latestVersion: "2.0.1410");
+
+        Assert.Empty(h.OpsEvents);
+    }
+
+    [Fact]
+    public async Task Version_with_commit_suffix_matches_latest_without_suffix()
+    {
+        // Agent versions carry "+<commit>", AdminConfiguration does not — the comparison
+        // must normalize, otherwise every genuine mismatch would look like a version race.
+        var h = new Harness();
+
+        await RunIntegrityAsync(h, IntegrityReport(agentVersion: "2.0.1410+3b59dab2df6c"));
+
+        Assert.Single(h.OpsEvents);
+    }
+
+    [Fact]
+    public async Task Missing_latest_config_fails_open_to_recording()
+    {
+        // No published-version info available → visibility wins over suppression.
+        var h = new Harness();
+
+        await RunIntegrityAsync(h, IntegrityReport(), latestVersion: null, latestExeSha: null);
+
+        Assert.Single(h.OpsEvents);
+    }
+
+    [Fact]
+    public async Task Integrity_mismatch_without_session_id_still_records_ops_event()
+    {
+        // The report can arrive before session registration — the binary evidence must
+        // never be dropped for lack of a session.
+        var h = new Harness();
+
+        await RunIntegrityAsync(h, IntegrityReport(sessionId: null));
+
+        var ops = Assert.Single(h.OpsEvents);
+        Assert.Equal("AgentBinaryIntegrityMismatch", ops.EventType);
+    }
+
+    [Fact]
+    public async Task Other_error_types_do_not_record_integrity_ops_event()
+    {
+        var h = new Harness();
+
+        await RunIntegrityAsync(h, Report(Now));
+
+        Assert.Empty(h.OpsEvents);
+    }
 }

src/Backend/AutopilotMonitor.Functions/Functions/Ingest/ReportAgentErrorFunction.cs

@@ -178,9 +178,90 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
 
             await MaterializeEmergencyBreakArtifactsAsync(report, tenantId, _sessionRepo, _opsEventService, _logger);
 
+            var adminConfig = await _adminConfigService.GetConfigurationAsync();
+            await MaterializeIntegrityMismatchAsync(
+                report, tenantId, _opsEventService,
+                adminConfig?.LatestAgentV2Version, adminConfig?.LatestAgentV2ExeSha256, _logger);
+
             return req.CreateResponse(HttpStatusCode.OK);
         }
 
+        /// <summary>
+        /// Regex over OUR OWN report format (AgentRuntimeConfig.VerifyBinaryIntegrity):
+        /// "... actual=&lt;sha256&gt;, expected=&lt;sha256&gt;". Parsed leniently — an unparseable
+        /// message records rather than suppresses.
+        /// </summary>
+        private static readonly System.Text.RegularExpressions.Regex ActualHashRegex =
+            new(@"actual=([0-9a-fA-F]{64})", System.Text.RegularExpressions.RegexOptions.Compiled);
+
+        /// <summary>
+        /// Surfaces an <c>IntegrityCheckFailed</c> report as an <c>AgentBinaryIntegrityMismatch</c>
+        /// ops event. Before 2026-08-20 this report went to App Insights only — session e9753578's
+        /// agent reported a hash mismatch and no product surface showed it, which cost a day of
+        /// mis-attributed root-causing.
+        ///
+        /// <para>
+        /// KNOWN, ACCEPTED RACE (must not spam the ops feed): around every release, agents compare
+        /// against a hash snapshot that does not belong to their own binary — a freshly installed
+        /// latest agent can receive the PREVIOUS release's hashes from the 5-minute
+        /// AdminConfiguration cache (e9753578: genuine 1410 exe vs cached 1409 hash), and an
+        /// older agent's next config fetch sees the NEW release's hash. Both self-heal via the
+        /// forced self-update. Suppression, evaluated against the CURRENT config at receive time:
+        /// the reported RUNNING hash equals the published exe (stale expectation), or the
+        /// reporting agent's version is not the latest (update in flight). What remains — an agent
+        /// claiming the latest version whose binary is NOT the published one — is the real signal:
+        /// tamper, stale blob, or a build that never came from the release pipeline.
+        /// </para>
+        ///
+        /// <para>
+        /// Ops event only, no timeline event: the report is about the BINARY, not the enrollment,
+        /// and may arrive before the session is registered. No idempotency read: the agent-side
+        /// trigger is single-shot per process. Best-effort — a failure here must never turn the
+        /// always-200 channel into a retry loop.
+        /// </para>
+        /// </summary>
+        internal static async Task MaterializeIntegrityMismatchAsync(
+            AgentErrorReport report,
+            string tenantId,
+            OpsEventService opsEventService,
+            string? latestAgentVersion,
+            string? latestAgentExeSha256,
+            ILogger logger)
+        {
+            if (report.ErrorType != AgentErrorType.IntegrityCheckFailed)
+            {
+                return;
+            }
+
+            try
+            {
+                // "2.0.1410+3b59dab2..." → "2.0.1410" (the manifest/AdminConfig carry no +commit).
+                var reportedVersion = (report.AgentVersion ?? string.Empty).Split('+')[0];
+                if (!string.IsNullOrEmpty(latestAgentVersion)
+                    && !string.IsNullOrEmpty(reportedVersion)
+                    && !string.Equals(reportedVersion, latestAgentVersion, StringComparison.OrdinalIgnoreCase))
+                {
+                    return; // older agent vs newer release — the forced self-update heals this.
+                }
+
+                var actualMatch = ActualHashRegex.Match(report.Message ?? string.Empty);
+                if (actualMatch.Success
+                    && !string.IsNullOrEmpty(latestAgentExeSha256)
+                    && string.Equals(actualMatch.Groups[1].Value, latestAgentExeSha256, StringComparison.OrdinalIgnoreCase))
+                {
+                    return; // the running binary IS the published one — the agent's expectation was stale.
+                }
+
+                await opsEventService.RecordAgentBinaryIntegrityMismatchAsync(
+                    tenantId, report.SessionId, report.AgentVersion, report.Message ?? string.Empty);
+            }
+            catch (Exception ex)
+            {
+                logger.LogWarning(ex,
+                    "ReportAgentError: failed to record AgentBinaryIntegrityMismatch ops event for session {SessionId}", report.SessionId);
+            }
+        }
+
         /// <summary>
         /// Materializes the agent's silent 48h emergency break as (1) a timeline event so it shows
         /// in the session and the timeout classifier can see it, (2) the cross-session EventType

src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs

@@ -608,6 +608,20 @@ namespace AutopilotMonitor.Functions.Services
                 $"Agent emergency break on session {sessionId} (agent {agentVersion ?? "?"}): {message}",
                 tenantId, "System.EmergencyChannel", new { sessionId, agentVersion });
 
+        /// <summary>
+        /// An agent reported that its RUNNING exe's SHA-256 differs from the hash the backend
+        /// advertises for its version — the binary in the field is not the published build
+        /// (tamper, stale blob, or a build that never came from a committed tree). Session
+        /// e9753578 (2026-08-20) carried exactly this report, but it landed only in App
+        /// Insights and was invisible to every product surface while the field damage was
+        /// mis-attributed to committed code for hours. Warning-tier so operators can wire a
+        /// Telegram rule; the message carries both hashes verbatim.
+        /// </summary>
+        public Task RecordAgentBinaryIntegrityMismatchAsync(string tenantId, string? sessionId, string? agentVersion, string message)
+            => WriteAsync(OpsEventCategory.Agent, "AgentBinaryIntegrityMismatch", OpsEventSeverity.Warning,
+                $"Agent binary integrity mismatch on session {sessionId ?? "?"} (agent {agentVersion ?? "?"}): {message}",
+                tenantId, "System.EmergencyChannel", new { sessionId, agentVersion });
+
         public Task RecordExcessiveSessionEventsAsync(string tenantId, string sessionId, int eventCount, int threshold)
             => WriteAsync(OpsEventCategory.Agent, "ExcessiveSessionEvents", OpsEventSeverity.Warning,
                 $"Session {sessionId} has {eventCount} events (threshold {threshold}) — likely agent loop bug",

src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx

@@ -146,6 +146,11 @@ const OPS_EVENT_TYPES: Record<string, string[]> = {
     // ReportAgentErrorFunction (RecordAgentEmergencyBreakAsync). Dual-register per memory
     // feedback_ops_event_types_dual_register.
     "AgentEmergencyBreak",
+    // The running exe's SHA-256 differs from the backend-advertised hash for its version —
+    // the field binary is not the published build (tamper / stale blob / uncommitted-tree
+    // build). Emitted by ReportAgentErrorFunction (RecordAgentBinaryIntegrityMismatchAsync).
+    // Dual-register per memory feedback_ops_event_types_dual_register.
+    "AgentBinaryIntegrityMismatch",
   ],
   Platform: [
     // Azure Monitor alerts relayed through POST ops/alert-webhook (Common Alert Schema →