From c728a0bd4149df160ba3bc06e7a7d5f927e16dd6 Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Thu, 20 Aug 2026 19:28:43 +0200 Subject: [PATCH] feat(backend): surface genuine agent binary-integrity mismatches as ops events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit IntegrityCheckFailed emergency reports used to land in App Insights only — session e9753578's agent reported a hash mismatch and no product surface showed it, which cost a day of mis-attributed root-causing. They now materialize as an AgentBinaryIntegrityMismatch ops event (Warning, Agent category, dual-registered in the web alert-rule catalog per feedback_ops_event_types_dual_register). With suppression for the KNOWN, accepted release race the operator correctly called out: around every release, agents compare against hash snapshots that do not belong to their own binary — an older agent sees the new release's hash on its next config fetch, and a freshly installed latest agent can receive the PREVIOUS release's hashes from the 5-minute AdminConfiguration cache (e9753578: genuine 1410 exe checked against the cached 1409 hash — proven by the 1409 archive logging "verified OK" with exactly that hash). Both self-heal via the forced self-update and are suppressed at materialization time against the CURRENT config: reported running hash == published exe, or reported version != latest. What remains — an agent claiming the latest version whose binary is NOT the published one — is the real signal: tamper, stale blob, or a build outside the release pipeline. Missing config fails open to recording; visibility beats suppression. Ops event only, deliberately no timeline event: the report is about the binary, not the enrollment, and may precede session registration. Backend suite 4320 green, web tsc clean. Co-Authored-By: Claude Fable 5 --- .../ReportAgentErrorFunctionTests.cs | 116 ++++++++++++++++++ .../Ingest/ReportAgentErrorFunction.cs | 81 ++++++++++++ .../Services/OpsEventService.cs | 14 +++ .../admin/components/OpsAlertRulesSection.tsx | 5 + 4 files changed, 216 insertions(+) diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/ReportAgentErrorFunctionTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/ReportAgentErrorFunctionTests.cs index 46d19b6a..3fb41f1e 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/ReportAgentErrorFunctionTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/ReportAgentErrorFunctionTests.cs @@ -200,4 +200,120 @@ public class ReportAgentErrorFunctionTests Assert.Empty(h.OpsEvents); } + + // ========================================================================= + // MaterializeIntegrityMismatchAsync — an IntegrityCheckFailed report becomes an + // AgentBinaryIntegrityMismatch ops event, UNLESS it is the known release race + // (agents comparing against hash snapshots that don't belong to their own binary; + // AdminConfig cache TTL 5 min). e9753578 proved the race: a genuine 1410 exe was + // compared against the cached 1409 hash and reported a "mismatch" that cost a day + // of mis-attributed root-causing while sitting invisible in App Insights. + // ========================================================================= + + private const string RunningExeSha = "5fca975e43fa5fca975e43fa5fca975e43fa5fca975e43fa5fca975e43fa0000"; + private const string PublishedExeSha = "8ebaa060ad348ebaa060ad348ebaa060ad348ebaa060ad348ebaa060ad340000"; + + private static AgentErrorReport IntegrityReport( + string? sessionId = SessionId, string agentVersion = "2.0.1410", string? actualSha = null) => new() + { + SessionId = sessionId!, + TenantId = TenantId, + ErrorType = AgentErrorType.IntegrityCheckFailed, + Message = $"Running exe SHA-256 differs from backend-advertised hash. actual={actualSha ?? RunningExeSha}, expected={PublishedExeSha}", + AgentVersion = agentVersion, + Timestamp = Now, + }; + + private static Task RunIntegrityAsync( + Harness h, AgentErrorReport report, + string? latestVersion = "2.0.1410", string? latestExeSha = PublishedExeSha) + => ReportAgentErrorFunction.MaterializeIntegrityMismatchAsync( + report, TenantId, h.OpsService, latestVersion, latestExeSha, + NullLogger.Instance); + + [Fact] + public async Task SameVersion_with_unknown_binary_records_ops_event() + { + // The real signal: an agent claiming the LATEST version whose running exe is NOT + // the published one — tamper, stale blob, or a build outside the release pipeline. + var h = new Harness(); + + await RunIntegrityAsync(h, IntegrityReport()); + + var ops = Assert.Single(h.OpsEvents); + Assert.Equal("AgentBinaryIntegrityMismatch", ops.EventType); + Assert.Equal(TenantId, ops.TenantId); + Assert.Contains("actual=5fca", ops.Message); + Assert.Contains("2.0.1410", ops.Message); + } + + [Fact] + public async Task RunningHash_equal_to_published_exe_is_suppressed_as_stale_expectation() + { + // e9753578's exact shape: the device RUNS the published binary; only its cached + // expectation was the previous release's hash. Self-heals — must not alert. + var h = new Harness(); + + await RunIntegrityAsync(h, IntegrityReport(actualSha: PublishedExeSha)); + + Assert.Empty(h.OpsEvents); + } + + [Fact] + public async Task Older_agent_version_than_latest_is_suppressed_as_update_race() + { + // The long-known direction: version N agent sees version N+1's hash right after a + // release; the forced self-update heals it. + var h = new Harness(); + + await RunIntegrityAsync(h, IntegrityReport(agentVersion: "2.0.1409+d43cfe2b"), latestVersion: "2.0.1410"); + + Assert.Empty(h.OpsEvents); + } + + [Fact] + public async Task Version_with_commit_suffix_matches_latest_without_suffix() + { + // Agent versions carry "+", AdminConfiguration does not — the comparison + // must normalize, otherwise every genuine mismatch would look like a version race. + var h = new Harness(); + + await RunIntegrityAsync(h, IntegrityReport(agentVersion: "2.0.1410+3b59dab2df6c")); + + Assert.Single(h.OpsEvents); + } + + [Fact] + public async Task Missing_latest_config_fails_open_to_recording() + { + // No published-version info available → visibility wins over suppression. + var h = new Harness(); + + await RunIntegrityAsync(h, IntegrityReport(), latestVersion: null, latestExeSha: null); + + Assert.Single(h.OpsEvents); + } + + [Fact] + public async Task Integrity_mismatch_without_session_id_still_records_ops_event() + { + // The report can arrive before session registration — the binary evidence must + // never be dropped for lack of a session. + var h = new Harness(); + + await RunIntegrityAsync(h, IntegrityReport(sessionId: null)); + + var ops = Assert.Single(h.OpsEvents); + Assert.Equal("AgentBinaryIntegrityMismatch", ops.EventType); + } + + [Fact] + public async Task Other_error_types_do_not_record_integrity_ops_event() + { + var h = new Harness(); + + await RunIntegrityAsync(h, Report(Now)); + + Assert.Empty(h.OpsEvents); + } } diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/ReportAgentErrorFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/ReportAgentErrorFunction.cs index c27a4e0c..dfae713a 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/ReportAgentErrorFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/ReportAgentErrorFunction.cs @@ -178,9 +178,90 @@ namespace AutopilotMonitor.Functions.Functions.Ingest await MaterializeEmergencyBreakArtifactsAsync(report, tenantId, _sessionRepo, _opsEventService, _logger); + var adminConfig = await _adminConfigService.GetConfigurationAsync(); + await MaterializeIntegrityMismatchAsync( + report, tenantId, _opsEventService, + adminConfig?.LatestAgentV2Version, adminConfig?.LatestAgentV2ExeSha256, _logger); + return req.CreateResponse(HttpStatusCode.OK); } + /// + /// Regex over OUR OWN report format (AgentRuntimeConfig.VerifyBinaryIntegrity): + /// "... actual=<sha256>, expected=<sha256>". Parsed leniently — an unparseable + /// message records rather than suppresses. + /// + private static readonly System.Text.RegularExpressions.Regex ActualHashRegex = + new(@"actual=([0-9a-fA-F]{64})", System.Text.RegularExpressions.RegexOptions.Compiled); + + /// + /// Surfaces an IntegrityCheckFailed report as an AgentBinaryIntegrityMismatch + /// ops event. Before 2026-08-20 this report went to App Insights only — session e9753578's + /// agent reported a hash mismatch and no product surface showed it, which cost a day of + /// mis-attributed root-causing. + /// + /// + /// KNOWN, ACCEPTED RACE (must not spam the ops feed): around every release, agents compare + /// against a hash snapshot that does not belong to their own binary — a freshly installed + /// latest agent can receive the PREVIOUS release's hashes from the 5-minute + /// AdminConfiguration cache (e9753578: genuine 1410 exe vs cached 1409 hash), and an + /// older agent's next config fetch sees the NEW release's hash. Both self-heal via the + /// forced self-update. Suppression, evaluated against the CURRENT config at receive time: + /// the reported RUNNING hash equals the published exe (stale expectation), or the + /// reporting agent's version is not the latest (update in flight). What remains — an agent + /// claiming the latest version whose binary is NOT the published one — is the real signal: + /// tamper, stale blob, or a build that never came from the release pipeline. + /// + /// + /// + /// Ops event only, no timeline event: the report is about the BINARY, not the enrollment, + /// and may arrive before the session is registered. No idempotency read: the agent-side + /// trigger is single-shot per process. Best-effort — a failure here must never turn the + /// always-200 channel into a retry loop. + /// + /// + internal static async Task MaterializeIntegrityMismatchAsync( + AgentErrorReport report, + string tenantId, + OpsEventService opsEventService, + string? latestAgentVersion, + string? latestAgentExeSha256, + ILogger logger) + { + if (report.ErrorType != AgentErrorType.IntegrityCheckFailed) + { + return; + } + + try + { + // "2.0.1410+3b59dab2..." → "2.0.1410" (the manifest/AdminConfig carry no +commit). + var reportedVersion = (report.AgentVersion ?? string.Empty).Split('+')[0]; + if (!string.IsNullOrEmpty(latestAgentVersion) + && !string.IsNullOrEmpty(reportedVersion) + && !string.Equals(reportedVersion, latestAgentVersion, StringComparison.OrdinalIgnoreCase)) + { + return; // older agent vs newer release — the forced self-update heals this. + } + + var actualMatch = ActualHashRegex.Match(report.Message ?? string.Empty); + if (actualMatch.Success + && !string.IsNullOrEmpty(latestAgentExeSha256) + && string.Equals(actualMatch.Groups[1].Value, latestAgentExeSha256, StringComparison.OrdinalIgnoreCase)) + { + return; // the running binary IS the published one — the agent's expectation was stale. + } + + await opsEventService.RecordAgentBinaryIntegrityMismatchAsync( + tenantId, report.SessionId, report.AgentVersion, report.Message ?? string.Empty); + } + catch (Exception ex) + { + logger.LogWarning(ex, + "ReportAgentError: failed to record AgentBinaryIntegrityMismatch ops event for session {SessionId}", report.SessionId); + } + } + /// /// Materializes the agent's silent 48h emergency break as (1) a timeline event so it shows /// in the session and the timeout classifier can see it, (2) the cross-session EventType diff --git a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs index 7926419d..91c386aa 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs @@ -608,6 +608,20 @@ namespace AutopilotMonitor.Functions.Services $"Agent emergency break on session {sessionId} (agent {agentVersion ?? "?"}): {message}", tenantId, "System.EmergencyChannel", new { sessionId, agentVersion }); + /// + /// An agent reported that its RUNNING exe's SHA-256 differs from the hash the backend + /// advertises for its version — the binary in the field is not the published build + /// (tamper, stale blob, or a build that never came from a committed tree). Session + /// e9753578 (2026-08-20) carried exactly this report, but it landed only in App + /// Insights and was invisible to every product surface while the field damage was + /// mis-attributed to committed code for hours. Warning-tier so operators can wire a + /// Telegram rule; the message carries both hashes verbatim. + /// + public Task RecordAgentBinaryIntegrityMismatchAsync(string tenantId, string? sessionId, string? agentVersion, string message) + => WriteAsync(OpsEventCategory.Agent, "AgentBinaryIntegrityMismatch", OpsEventSeverity.Warning, + $"Agent binary integrity mismatch on session {sessionId ?? "?"} (agent {agentVersion ?? "?"}): {message}", + tenantId, "System.EmergencyChannel", new { sessionId, agentVersion }); + public Task RecordExcessiveSessionEventsAsync(string tenantId, string sessionId, int eventCount, int threshold) => WriteAsync(OpsEventCategory.Agent, "ExcessiveSessionEvents", OpsEventSeverity.Warning, $"Session {sessionId} has {eventCount} events (threshold {threshold}) — likely agent loop bug", diff --git a/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx b/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx index b927ff94..9ab7f1b1 100644 --- a/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx @@ -146,6 +146,11 @@ const OPS_EVENT_TYPES: Record = { // ReportAgentErrorFunction (RecordAgentEmergencyBreakAsync). Dual-register per memory // feedback_ops_event_types_dual_register. "AgentEmergencyBreak", + // The running exe's SHA-256 differs from the backend-advertised hash for its version — + // the field binary is not the published build (tamper / stale blob / uncommitted-tree + // build). Emitted by ReportAgentErrorFunction (RecordAgentBinaryIntegrityMismatchAsync). + // Dual-register per memory feedback_ops_event_types_dual_register. + "AgentBinaryIntegrityMismatch", ], Platform: [ // Azure Monitor alerts relayed through POST ops/alert-webhook (Common Alert Schema →