From 6a9203905df78db0ef10a1861cadc19573be057f Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Tue, 21 Apr 2026 18:19:39 +0200 Subject: [PATCH] feat(agent): implement MaxAuthFailures watchdog + post-config binary integrity check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes three DTO-only fields on AgentConfigResponse that have been dead since the config schema was defined — neither V1 nor V2 ever consumed them. V2 now honours them end-to-end. 1. MaxAuthFailures + AuthFailureTimeoutMinutes — AuthFailureTracker New Security/AuthFailureTracker: thread-safe counter + time-window observer that fires a single-shot ThresholdExceeded event when either ceiling is crossed. Wired into: - RemoteConfigService.FetchConfigAsync: RecordSuccess on 2xx, RecordFailure on BackendAuthException (401/403 from /api/agent/config). - BackendTelemetryUploader.MapResponseAsync: RecordSuccess on 2xx, RecordFailure on 401/403 (moved from static to instance method). - Program.cs: tracker constructed before FetchConfig with CLI defaults; UpdateLimits called after RemoteConfigMerger.Merge so tenant policy wins; ThresholdExceeded handler sets the main shutdown signal so the agent exits cleanly instead of retrying a permanently-revoked certificate forever. Defaults: MaxAuthFailures=5, AuthFailureTimeoutMinutes=0 (time-window disabled). Both honour 0 = "feature off", and UpdateLimits can tighten or relax the ceilings mid-run when the tenant admin changes policy. 2. LatestAgentExeSha256 — BinaryIntegrityVerifier New Security/BinaryIntegrityVerifier: pure-static helper that computes SHA-256 over the running agent EXE and compares against the backend- advertised hash. Case-insensitive hex compare. Three outcomes: - Skipped: no hash provided / local path not resolvable → no-op. - Match: silent success path (debug-log only). - Mismatch: emits AgentErrorType.IntegrityCheckFailed via EmergencyReporter (the enum slot was already reserved — just never used). Does NOT terminate or auto-update; SelfUpdater's own hash-based trigger handles the "wrong EXE" case at next startup, and auto-terminating the agent on every mismatch would risk boot loops during rollouts. Wired in Program.cs immediately after RemoteConfigMerger.Merge. Context These are three of the four "DTO-only" fields I flagged in PR #45's body. User feedback: do not leave dead config schema entries. V1 never implemented any of these either (V1 has a ZIP-hash check in SelfUpdater via LatestAgentSha256, but no EXE-hash check and no auth-failure shutdown threshold) — so this is net-new functionality that makes V2 strictly better than V1 in these three dimensions, not a regression closure. Test coverage - AuthFailureTrackerTests (12 tests): count-based threshold, time-window threshold, RecordSuccess resets both, single-shot idempotence, handler-exception isolation, disabled-limits never fire, UpdateLimits retroactively tightens or relaxes, thread-safety under 8 concurrent callers, ctor null-guards. - BinaryIntegrityVerifierTests (6 tests): skipped on null/empty/missing file, match path, case-insensitive compare, mismatch path, and a known-vector test against SHA-256("abc") to pin the hex formatter. 464 / 464 V2.Core tests green (vs. 446 before). Not mapped / remaining dead-config EnableImeMatchLog stays DTO-only — it duplicates the CLI flag --ime-match-log with no semantic overlap to a backend switch, and mapping it would just overwrite a dev-only debug toggle. Left as documented non-feature. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../Security/AuthFailureTrackerTests.cs | 237 ++++++++++++++++++ .../Security/BinaryIntegrityVerifierTests.cs | 100 ++++++++ .../Configuration/RemoteConfigService.cs | 11 +- .../Security/AuthFailureTracker.cs | 142 +++++++++++ .../Security/BinaryIntegrityVerifier.cs | 122 +++++++++ .../Telemetry/BackendTelemetryUploader.cs | 15 +- .../AutopilotMonitor.Agent.V2/Program.cs | 41 ++- 7 files changed, 663 insertions(+), 5 deletions(-) create mode 100644 src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Security/AuthFailureTrackerTests.cs create mode 100644 src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Security/BinaryIntegrityVerifierTests.cs create mode 100644 src/Agent/AutopilotMonitor.Agent.V2.Core/Security/AuthFailureTracker.cs create mode 100644 src/Agent/AutopilotMonitor.Agent.V2.Core/Security/BinaryIntegrityVerifier.cs diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Security/AuthFailureTrackerTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Security/AuthFailureTrackerTests.cs new file mode 100644 index 00000000..f168e5a2 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Security/AuthFailureTrackerTests.cs @@ -0,0 +1,237 @@ +using System; +using System.Threading; +using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.Agent.V2.Core.Security; +using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using Xunit; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Security +{ + public sealed class AuthFailureTrackerTests + { + private static AgentLogger NewLogger() + { + var tmp = new TempDirectory(); + return new AgentLogger(System.IO.Path.Combine(tmp.Path, "logs"), AgentLogLevel.Debug); + } + + // ============================================================= Count-based threshold + + [Fact] + public void RecordFailure_fires_threshold_at_max_failures() + { + var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc)); + var tracker = new AuthFailureTracker(maxFailures: 3, timeoutMinutes: 0, clock, NewLogger()); + + AuthFailureThresholdEventArgs captured = null!; + tracker.ThresholdExceeded += (_, e) => captured = e; + + tracker.RecordFailure(401, "config"); + tracker.RecordFailure(401, "config"); + Assert.Null(captured); + + tracker.RecordFailure(401, "config"); + Assert.NotNull(captured); + Assert.Equal(3, captured.ConsecutiveFailures); + Assert.Equal(401, captured.LastStatusCode); + Assert.Equal("config", captured.LastOperation); + } + + [Fact] + public void RecordSuccess_resets_counter() + { + var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc)); + var tracker = new AuthFailureTracker(maxFailures: 3, timeoutMinutes: 0, clock, NewLogger()); + + int events = 0; + tracker.ThresholdExceeded += (_, _) => events++; + + tracker.RecordFailure(401, "config"); + tracker.RecordFailure(401, "config"); + tracker.RecordSuccess(); + tracker.RecordFailure(403, "upload"); + tracker.RecordFailure(403, "upload"); + + Assert.Equal(0, events); + Assert.Equal(2, tracker.ConsecutiveFailures); + } + + // ============================================================= Time-window threshold + + [Fact] + public void RecordFailure_fires_threshold_when_time_window_exceeded() + { + var start = new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc); + var clock = new FakeClock(start); + var tracker = new AuthFailureTracker(maxFailures: 0, timeoutMinutes: 10, clock, NewLogger()); + + int events = 0; + AuthFailureThresholdEventArgs captured = null!; + tracker.ThresholdExceeded += (_, e) => { events++; captured = e; }; + + tracker.RecordFailure(401, "config"); + Assert.Equal(0, events); + + clock.Now = start.AddMinutes(5); + tracker.RecordFailure(401, "config"); + Assert.Equal(0, events); + + clock.Now = start.AddMinutes(11); + tracker.RecordFailure(401, "config"); + + Assert.Equal(1, events); + Assert.Equal(start, captured.FirstFailureUtc); + Assert.Contains("time window", captured.Reason); + } + + [Fact] + public void RecordSuccess_resets_time_window_anchor() + { + var start = new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc); + var clock = new FakeClock(start); + var tracker = new AuthFailureTracker(maxFailures: 0, timeoutMinutes: 10, clock, NewLogger()); + + int events = 0; + tracker.ThresholdExceeded += (_, _) => events++; + + tracker.RecordFailure(401, "config"); + clock.Now = start.AddMinutes(9); + tracker.RecordSuccess(); + + // fresh window — 11 minutes later, first failure in this new window, no threshold. + clock.Now = start.AddMinutes(20); + tracker.RecordFailure(401, "config"); + + Assert.Equal(0, events); + } + + // ============================================================= Idempotence + + [Fact] + public void ThresholdExceeded_fires_only_once_even_with_many_failures() + { + var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc)); + var tracker = new AuthFailureTracker(maxFailures: 2, timeoutMinutes: 0, clock, NewLogger()); + + int events = 0; + tracker.ThresholdExceeded += (_, _) => events++; + + for (int i = 0; i < 10; i++) tracker.RecordFailure(401, "config"); + + Assert.Equal(1, events); + } + + [Fact] + public void ThresholdExceeded_handler_exception_does_not_propagate() + { + var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc)); + var tracker = new AuthFailureTracker(maxFailures: 1, timeoutMinutes: 0, clock, NewLogger()); + + tracker.ThresholdExceeded += (_, _) => throw new InvalidOperationException("listener exploded"); + + // Must not propagate — tracker is a best-effort observer. + tracker.RecordFailure(401, "config"); + } + + // ============================================================= Disabled limits + + [Fact] + public void RecordFailure_never_fires_when_both_limits_disabled() + { + var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc)); + var tracker = new AuthFailureTracker(maxFailures: 0, timeoutMinutes: 0, clock, NewLogger()); + + int events = 0; + tracker.ThresholdExceeded += (_, _) => events++; + + for (int i = 0; i < 100; i++) tracker.RecordFailure(401, "op"); + + Assert.Equal(0, events); + Assert.Equal(100, tracker.ConsecutiveFailures); + } + + // ============================================================= UpdateLimits (remote config override) + + [Fact] + public void UpdateLimits_tightens_threshold_retroactively() + { + var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc)); + var tracker = new AuthFailureTracker(maxFailures: 10, timeoutMinutes: 0, clock, NewLogger()); + + int events = 0; + tracker.ThresholdExceeded += (_, _) => events++; + + tracker.RecordFailure(401, "config"); + tracker.RecordFailure(401, "config"); + + // Tenant tightened the policy — the next failure should now fire. + tracker.UpdateLimits(maxFailures: 3, timeoutMinutes: 0); + tracker.RecordFailure(401, "config"); + + Assert.Equal(1, events); + } + + [Fact] + public void UpdateLimits_accepts_negative_max_as_disabled() + { + var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc)); + var tracker = new AuthFailureTracker(maxFailures: -1, timeoutMinutes: 0, clock, NewLogger()); + + int events = 0; + tracker.ThresholdExceeded += (_, _) => events++; + + for (int i = 0; i < 20; i++) tracker.RecordFailure(401, "op"); + Assert.Equal(0, events); + } + + // ============================================================= Thread-safety smoke + + [Fact] + public void RecordFailure_is_safe_under_concurrent_callers() + { + var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc)); + var tracker = new AuthFailureTracker(maxFailures: 50, timeoutMinutes: 0, clock, NewLogger()); + + int events = 0; + tracker.ThresholdExceeded += (_, _) => Interlocked.Increment(ref events); + + var threads = new Thread[8]; + for (int t = 0; t < threads.Length; t++) + { + threads[t] = new Thread(() => + { + for (int i = 0; i < 20; i++) tracker.RecordFailure(401, "concurrent"); + }); + threads[t].Start(); + } + foreach (var th in threads) th.Join(); + + // Many concurrent Increments cross the 50 ceiling — but ThresholdExceeded must fire exactly once. + Assert.Equal(1, events); + } + + // ============================================================= Guard + + [Fact] + public void Ctor_throws_on_null_clock() + { + Assert.Throws(() => + new AuthFailureTracker(1, 0, null!, NewLogger())); + } + + [Fact] + public void Ctor_throws_on_null_logger() + { + Assert.Throws(() => + new AuthFailureTracker(1, 0, new FakeClock(DateTime.UtcNow), null!)); + } + + private sealed class FakeClock : AutopilotMonitor.DecisionCore.Engine.IClock + { + public DateTime Now { get; set; } + public FakeClock(DateTime now) { Now = now; } + public DateTime UtcNow => Now; + public System.Threading.Tasks.Task Delay(TimeSpan delay, CancellationToken cancellationToken = default) => System.Threading.Tasks.Task.CompletedTask; + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Security/BinaryIntegrityVerifierTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Security/BinaryIntegrityVerifierTests.cs new file mode 100644 index 00000000..2bd59ded --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Security/BinaryIntegrityVerifierTests.cs @@ -0,0 +1,100 @@ +using System.IO; +using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.Agent.V2.Core.Security; +using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using Xunit; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Security +{ + public sealed class BinaryIntegrityVerifierTests + { + private static AgentLogger NewLogger(TempDirectory tmp) + => new AgentLogger(Path.Combine(tmp.Path, "logs"), AgentLogLevel.Debug); + + [Fact] + public void Check_returns_skipped_when_expected_hash_is_null_or_empty() + { + using var tmp = new TempDirectory(); + var logger = NewLogger(tmp); + + Assert.Equal(IntegrityCheckOutcome.Skipped, BinaryIntegrityVerifier.Check(null!, logger).Outcome); + Assert.Equal(IntegrityCheckOutcome.Skipped, BinaryIntegrityVerifier.Check("", logger).Outcome); + Assert.Equal(IntegrityCheckOutcome.Skipped, BinaryIntegrityVerifier.Check(" ", logger).Outcome); + } + + [Fact] + public void Check_returns_skipped_when_target_path_does_not_exist() + { + using var tmp = new TempDirectory(); + var logger = NewLogger(tmp); + var missing = Path.Combine(tmp.Path, "no-such.exe"); + + var result = BinaryIntegrityVerifier.Check("0".PadLeft(64, '0'), logger, exePath: missing); + + Assert.Equal(IntegrityCheckOutcome.Skipped, result.Outcome); + } + + [Fact] + public void Check_returns_match_when_hash_equals_actual() + { + using var tmp = new TempDirectory(); + var logger = NewLogger(tmp); + + var exe = Path.Combine(tmp.Path, "fake.exe"); + File.WriteAllBytes(exe, new byte[] { 0x01, 0x02, 0x03, 0x04 }); + var actual = BinaryIntegrityVerifier.ComputeSha256Hex(exe); + + var result = BinaryIntegrityVerifier.Check(actual, logger, exePath: exe); + + Assert.Equal(IntegrityCheckOutcome.Match, result.Outcome); + Assert.False(result.IsMismatch); + Assert.Equal(actual, result.ActualSha256); + } + + [Fact] + public void Check_is_case_insensitive_for_expected_hash() + { + using var tmp = new TempDirectory(); + var logger = NewLogger(tmp); + + var exe = Path.Combine(tmp.Path, "fake.exe"); + File.WriteAllBytes(exe, new byte[] { 0xAB, 0xCD }); + var actualLower = BinaryIntegrityVerifier.ComputeSha256Hex(exe); + + var result = BinaryIntegrityVerifier.Check(actualLower.ToUpperInvariant(), logger, exePath: exe); + + Assert.Equal(IntegrityCheckOutcome.Match, result.Outcome); + } + + [Fact] + public void Check_returns_mismatch_when_hash_differs() + { + using var tmp = new TempDirectory(); + var logger = NewLogger(tmp); + + var exe = Path.Combine(tmp.Path, "fake.exe"); + File.WriteAllBytes(exe, new byte[] { 0x01 }); + var wrongHash = new string('f', 64); + + var result = BinaryIntegrityVerifier.Check(wrongHash, logger, exePath: exe); + + Assert.Equal(IntegrityCheckOutcome.Mismatch, result.Outcome); + Assert.True(result.IsMismatch); + Assert.Equal(wrongHash, result.ExpectedSha256); + Assert.NotEqual(result.ExpectedSha256, result.ActualSha256); + } + + [Fact] + public void ComputeSha256Hex_matches_known_vector() + { + // SHA-256("abc") = ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad + using var tmp = new TempDirectory(); + var exe = Path.Combine(tmp.Path, "abc.bin"); + File.WriteAllText(exe, "abc"); + + var hash = BinaryIntegrityVerifier.ComputeSha256Hex(exe); + + Assert.Equal("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", hash); + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigService.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigService.cs index 8bd7fe2d..7915daa2 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigService.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigService.cs @@ -4,6 +4,7 @@ using System.Threading; using System.Threading.Tasks; using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport; using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.Agent.V2.Core.Security; using AutopilotMonitor.Shared.Models; using Newtonsoft.Json; @@ -21,6 +22,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration private readonly string _cacheFilePath; private readonly EmergencyReporter _emergencyReporter; private readonly DistressReporter _distressReporter; + private readonly AuthFailureTracker _authFailureTracker; private AgentConfigResponse _currentConfig; private readonly object _configLock = new object(); @@ -39,13 +41,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration } } - public RemoteConfigService(BackendApiClient apiClient, string tenantId, AgentLogger logger, EmergencyReporter emergencyReporter = null, DistressReporter distressReporter = null) + public RemoteConfigService(BackendApiClient apiClient, string tenantId, AgentLogger logger, EmergencyReporter emergencyReporter = null, DistressReporter distressReporter = null, AuthFailureTracker authFailureTracker = null) { _apiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient)); _tenantId = tenantId ?? throw new ArgumentNullException(nameof(tenantId)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _emergencyReporter = emergencyReporter; _distressReporter = distressReporter; + _authFailureTracker = authFailureTracker; var cacheDir = Environment.ExpandEnvironmentVariables(@"%ProgramData%\AutopilotMonitor\Config"); _cacheFilePath = Path.Combine(cacheDir, "remote-config.json"); @@ -70,6 +73,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration LogCollectorSettings(config.Collectors); SetConfig(config); CacheConfig(config); + _authFailureTracker?.RecordSuccess(); return config; } @@ -87,6 +91,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration ex.Message, httpStatusCode: ex.StatusCode); } + + // Feed the central auth-failure tracker so the agent can shut down cleanly + // after MaxAuthFailures / AuthFailureTimeoutMinutes is exceeded, instead of + // hammering the backend forever when the device certificate is revoked. + _authFailureTracker?.RecordFailure(ex.StatusCode, "agent/config"); } catch (Exception ex) { diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Security/AuthFailureTracker.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Security/AuthFailureTracker.cs new file mode 100644 index 00000000..801a8e5e --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Security/AuthFailureTracker.cs @@ -0,0 +1,142 @@ +using System; +using System.Threading; +using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.DecisionCore.Engine; + +namespace AutopilotMonitor.Agent.V2.Core.Security +{ + /// + /// Tracks consecutive 401/403 responses from the backend. When either the consecutive-count + /// ceiling (MaxAuthFailures) or the elapsed-time ceiling (AuthFailureTimeoutMinutes) + /// is exceeded, fires exactly once so Program.RunAgent + /// can trigger a soft shutdown. + /// + /// Without this tracker a device whose certificate is revoked (or whose tenant has been deleted) + /// would retry the config fetch and every telemetry upload indefinitely, flooding the backend + /// distress channel and the local log. The tracker is the missing enforcement for the + /// MaxAuthFailures and AuthFailureTimeoutMinutes knobs on + /// AgentConfigResponse (defaults: 5 consecutive, time window disabled). + /// + /// + /// Thread-safety: consecutive-count uses ; the first-failure timestamp + /// is protected by a single monitor lock. is raised outside the + /// lock to prevent handler re-entrancy deadlocks. + /// + /// + public sealed class AuthFailureTracker + { + private readonly IClock _clock; + private readonly AgentLogger _logger; + private readonly object _windowLock = new object(); + + private int _maxFailures; // 0 = disabled + private TimeSpan? _timeoutWindow; // null = disabled + private int _consecutiveFailures; + private DateTime? _firstFailureUtc; + private int _thresholdFired; // 0/1 — Interlocked exchange makes the event single-shot + + public AuthFailureTracker( + int maxFailures, + int timeoutMinutes, + IClock clock, + AgentLogger logger) + { + _clock = clock ?? throw new ArgumentNullException(nameof(clock)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + UpdateLimits(maxFailures, timeoutMinutes); + } + + /// Fires once when either threshold is crossed. Listener is responsible for shutting the agent down. + public event EventHandler ThresholdExceeded; + + /// Updates the limits from the merged remote config. Safe to call at any time; does not reset the counter. + public void UpdateLimits(int maxFailures, int timeoutMinutes) + { + _maxFailures = maxFailures < 0 ? 0 : maxFailures; + _timeoutWindow = timeoutMinutes > 0 ? (TimeSpan?)TimeSpan.FromMinutes(timeoutMinutes) : null; + } + + /// Current consecutive-failure count. Exposed for observability and tests. + public int ConsecutiveFailures => Volatile.Read(ref _consecutiveFailures); + + /// Reset counter + window anchor after a successful authenticated response. + public void RecordSuccess() + { + if (Interlocked.Exchange(ref _consecutiveFailures, 0) == 0) return; // no-op if already zero + lock (_windowLock) { _firstFailureUtc = null; } + } + + /// + /// Record a 401/403 from the given . Emits + /// the first time either ceiling is crossed; subsequent calls after termination is armed are no-ops. + /// + public void RecordFailure(int statusCode, string operation) + { + if (Volatile.Read(ref _thresholdFired) == 1) return; + + var count = Interlocked.Increment(ref _consecutiveFailures); + DateTime now = _clock.UtcNow; + DateTime firstFailureAt; + + lock (_windowLock) + { + if (_firstFailureUtc == null) _firstFailureUtc = now; + firstFailureAt = _firstFailureUtc.Value; + } + + bool countExceeded = _maxFailures > 0 && count >= _maxFailures; + bool windowExceeded = _timeoutWindow.HasValue && (now - firstFailureAt) >= _timeoutWindow.Value; + + if (!countExceeded && !windowExceeded) + { + _logger.Warning($"AuthFailureTracker: consecutive auth failure #{count} (http {statusCode}, {operation})."); + return; + } + + if (Interlocked.Exchange(ref _thresholdFired, 1) == 1) return; // another thread won the race + + var reason = countExceeded + ? $"consecutive auth failures reached limit ({count} >= {_maxFailures})" + : $"auth-failure time window exceeded ({(now - firstFailureAt).TotalMinutes:F0}min >= {_timeoutWindow.Value.TotalMinutes:F0}min)"; + + _logger.Error($"AuthFailureTracker: {reason} — signalling shutdown. Last operation: {operation}, statusCode={statusCode}."); + + try + { + ThresholdExceeded?.Invoke(this, new AuthFailureThresholdEventArgs( + consecutiveFailures: count, + firstFailureUtc: firstFailureAt, + lastOperation: operation, + lastStatusCode: statusCode, + reason: reason)); + } + catch (Exception ex) + { + _logger.Warning($"AuthFailureTracker: ThresholdExceeded handler threw: {ex.Message}"); + } + } + } + + public sealed class AuthFailureThresholdEventArgs : EventArgs + { + public AuthFailureThresholdEventArgs( + int consecutiveFailures, + DateTime firstFailureUtc, + string lastOperation, + int lastStatusCode, + string reason) + { + ConsecutiveFailures = consecutiveFailures; + FirstFailureUtc = firstFailureUtc; + LastOperation = lastOperation; + LastStatusCode = lastStatusCode; + Reason = reason; + } + + public int ConsecutiveFailures { get; } + public DateTime FirstFailureUtc { get; } + public string LastOperation { get; } + public int LastStatusCode { get; } + public string Reason { get; } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Security/BinaryIntegrityVerifier.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Security/BinaryIntegrityVerifier.cs new file mode 100644 index 00000000..202fe1a4 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Security/BinaryIntegrityVerifier.cs @@ -0,0 +1,122 @@ +using System; +using System.IO; +using System.Reflection; +using System.Security.Cryptography; +using System.Text; +using AutopilotMonitor.Agent.V2.Core.Logging; + +namespace AutopilotMonitor.Agent.V2.Core.Security +{ + /// + /// Post-config binary-integrity check. After the remote config is fetched, the agent compares + /// the SHA-256 of its running .exe against AgentConfigResponse.LatestAgentExeSha256. + /// A mismatch means one of: + /// + /// tampering — the on-disk binary was modified after deployment, + /// stale blob storage — the agent is running an older EXE than what the backend advertises, + /// self-update failed silently — a previous update swapped the ZIP but not the EXE. + /// + /// + /// We intentionally do not terminate or auto-update from this check: + /// already runs at startup and its hash-based trigger covers the "wrong EXE" case. All we do + /// here is emit an AgentErrorType.IntegrityCheckFailed report so operators see the drift. + /// + /// + public static class BinaryIntegrityVerifier + { + /// + /// Computes the SHA-256 of (or the currently executing assembly + /// when null) and compares it against . + /// Case-insensitive comparison (both lowercase and uppercase hex strings accepted). + /// + public static IntegrityCheckResult Check( + string expectedSha256, + AgentLogger logger, + string exePath = null) + { + if (logger == null) throw new ArgumentNullException(nameof(logger)); + + if (string.IsNullOrWhiteSpace(expectedSha256)) + { + logger.Debug("BinaryIntegrityVerifier: no expected hash provided — skipping check."); + return IntegrityCheckResult.Skipped(); + } + + var path = exePath ?? Assembly.GetExecutingAssembly().Location; + if (string.IsNullOrEmpty(path) || !File.Exists(path)) + { + logger.Warning($"BinaryIntegrityVerifier: executing assembly path not resolvable ({path ?? "null"}) — skipping check."); + return IntegrityCheckResult.Skipped(); + } + + string actualSha; + try + { + actualSha = ComputeSha256Hex(path); + } + catch (Exception ex) + { + logger.Warning($"BinaryIntegrityVerifier: failed to hash '{path}': {ex.Message}"); + return IntegrityCheckResult.Skipped(); + } + + if (string.Equals(actualSha, expectedSha256, StringComparison.OrdinalIgnoreCase)) + { + logger.Debug($"BinaryIntegrityVerifier: SHA-256 match (sha={Truncate(actualSha)})."); + return IntegrityCheckResult.Match(actualSha); + } + + logger.Warning( + $"BinaryIntegrityVerifier: SHA-256 MISMATCH — running exe={Truncate(actualSha)}, backend advertises={Truncate(expectedSha256)}. " + + "Possible tamper, stale blob, or failed self-update."); + return IntegrityCheckResult.Mismatch(actualSha, expectedSha256); + } + + internal static string ComputeSha256Hex(string filePath) + { + using (var stream = File.OpenRead(filePath)) + using (var sha = SHA256.Create()) + { + var bytes = sha.ComputeHash(stream); + var sb = new StringBuilder(bytes.Length * 2); + for (int i = 0; i < bytes.Length; i++) sb.Append(bytes[i].ToString("x2")); + return sb.ToString(); + } + } + + private static string Truncate(string hex) + => string.IsNullOrEmpty(hex) ? "(null)" : (hex.Length <= 12 ? hex : hex.Substring(0, 12) + "..."); + } + + public sealed class IntegrityCheckResult + { + public IntegrityCheckOutcome Outcome { get; } + public string ActualSha256 { get; } + public string ExpectedSha256 { get; } + + private IntegrityCheckResult(IntegrityCheckOutcome outcome, string actual, string expected) + { + Outcome = outcome; + ActualSha256 = actual; + ExpectedSha256 = expected; + } + + public static IntegrityCheckResult Skipped() => new IntegrityCheckResult(IntegrityCheckOutcome.Skipped, null, null); + public static IntegrityCheckResult Match(string actual) => new IntegrityCheckResult(IntegrityCheckOutcome.Match, actual, actual); + public static IntegrityCheckResult Mismatch(string actual, string expected) => new IntegrityCheckResult(IntegrityCheckOutcome.Mismatch, actual, expected); + + public bool IsMismatch => Outcome == IntegrityCheckOutcome.Mismatch; + } + + public enum IntegrityCheckOutcome + { + /// No expected hash provided, or the local path could not be hashed. + Skipped = 0, + + /// SHA-256 of the running EXE matches the backend-advertised hash. + Match = 1, + + /// SHA-256 differs — tamper / stale blob / failed self-update. + Mismatch = 2, + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/BackendTelemetryUploader.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/BackendTelemetryUploader.cs index 17d9f62a..d3a60e52 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/BackendTelemetryUploader.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/BackendTelemetryUploader.cs @@ -9,6 +9,7 @@ using System.Net.Http.Headers; using System.Text; using System.Threading; using System.Threading.Tasks; +using AutopilotMonitor.Agent.V2.Core.Security; using AutopilotMonitor.Shared; using AutopilotMonitor.Shared.Models; using Newtonsoft.Json; @@ -46,6 +47,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry private readonly string? _serialNumber; private readonly string? _bootstrapToken; private readonly string? _agentVersion; + private readonly AuthFailureTracker? _authFailureTracker; public BackendTelemetryUploader( HttpClient httpClient, @@ -55,7 +57,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry string? model = null, string? serialNumber = null, string? bootstrapToken = null, - string? agentVersion = null) + string? agentVersion = null, + AuthFailureTracker? authFailureTracker = null) { _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); if (string.IsNullOrEmpty(baseUrl)) throw new ArgumentException("BaseUrl is mandatory.", nameof(baseUrl)); @@ -68,6 +71,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry _serialNumber = serialNumber; _bootstrapToken = bootstrapToken; _agentVersion = agentVersion; + _authFailureTracker = authFailureTracker; } public async Task UploadBatchAsync( @@ -124,12 +128,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry } } - private static async Task MapResponseAsync(HttpResponseMessage response) + private async Task MapResponseAsync(HttpResponseMessage response) { var statusCode = (int)response.StatusCode; if (response.IsSuccessStatusCode) { + // Auth was accepted — reset the consecutive-failure counter so a transient + // backend hiccup does not eventually shut the agent down. + _authFailureTracker?.RecordSuccess(); + // M4.6.ε — parse backend-to-agent control signals from the 2xx response body. // Body is best-effort JSON; anything unparseable degrades cleanly to plain Ok(). return await TryReadControlSignalsAsync(response).ConfigureAwait(false); @@ -142,6 +150,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry { case HttpStatusCode.Unauthorized: // 401 case HttpStatusCode.Forbidden: // 403 + // Feed the central tracker so the agent can shut down after MaxAuthFailures + // instead of retrying a permanently-revoked certificate forever. + _authFailureTracker?.RecordFailure(statusCode, "agent/telemetry"); return UploadResult.Permanent($"unauthorized: {shortReason}"); case HttpStatusCode.RequestTimeout: // 408 diff --git a/src/Agent/AutopilotMonitor.Agent.V2/Program.cs b/src/Agent/AutopilotMonitor.Agent.V2/Program.cs index 835f7bab..ad4dea54 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2/Program.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2/Program.cs @@ -306,8 +306,18 @@ namespace AutopilotMonitor.Agent.V2 "MDM certificate not found in LocalMachine or CurrentUser store"); } + // Central observer for consecutive 401/403 responses. Initialised with the CLI/bootstrap + // defaults on AgentConfiguration; UpdateLimits is called after RemoteConfigMerger.Merge + // so tenant-policy overrides take effect. ThresholdExceeded is wired below, once the + // shutdown signal is available, to trigger a clean agent exit when the limit is hit. + var authFailureTracker = new AuthFailureTracker( + maxFailures: agentConfig.MaxAuthFailures, + timeoutMinutes: agentConfig.AuthFailureTimeoutMinutes, + clock: SystemClock.Instance, + logger: logger); + var remoteConfigService = new RemoteConfigService( - backendApiClient, agentConfig.TenantId, logger, emergencyReporter, distressReporter); + backendApiClient, agentConfig.TenantId, logger, emergencyReporter, distressReporter, authFailureTracker); var remoteConfig = remoteConfigService.FetchConfigAsync().GetAwaiter().GetResult(); // Project remote tenant-controlled knobs onto the runtime AgentConfiguration so that @@ -316,6 +326,9 @@ namespace AutopilotMonitor.Agent.V2 // actually respect the tenant admin settings. CLI flags stay authoritative over remote. RemoteConfigMerger.Merge(agentConfig, remoteConfig, args); + // Refresh tracker ceilings with the tenant-specific values we just merged in. + authFailureTracker.UpdateLimits(agentConfig.MaxAuthFailures, agentConfig.AuthFailureTimeoutMinutes); + logger.SetLogLevel(agentConfig.LogLevel); // Propagate the backend-expected SHA so the runtime hash-mismatch trigger (M4.6.α @@ -324,6 +337,18 @@ namespace AutopilotMonitor.Agent.V2 if (!string.IsNullOrEmpty(remoteConfig.LatestAgentSha256)) SelfUpdater.BackendExpectedSha256 = remoteConfig.LatestAgentSha256; + // Post-config binary-integrity check: verify the running EXE's SHA-256 against the + // value advertised by the backend. Mismatch does not terminate (SelfUpdater's own + // hash-based trigger covers the "wrong EXE" case at next startup) — we just emit an + // IntegrityCheckFailed error report so operators see the drift. + var integrityResult = BinaryIntegrityVerifier.Check(remoteConfig.LatestAgentExeSha256, logger); + if (integrityResult.IsMismatch) + { + _ = emergencyReporter.TrySendAsync( + AgentErrorType.IntegrityCheckFailed, + $"Running exe SHA-256 differs from backend-advertised hash. actual={integrityResult.ActualSha256}, expected={integrityResult.ExpectedSha256}"); + } + // H-2 mitigation: delete the persisted bootstrap-config.json once the MDM cert // proves it can authenticate. Non-blocking — any failure leaves the file for retry. try @@ -361,7 +386,8 @@ namespace AutopilotMonitor.Agent.V2 model: hardwareForReporters.Model, serialNumber: hardwareForReporters.SerialNumber, bootstrapToken: agentConfig.UseBootstrapTokenAuth ? agentConfig.BootstrapToken : null, - agentVersion: GetAgentVersion()); + agentVersion: GetAgentVersion(), + authFailureTracker: authFailureTracker); } catch (Exception ex) { @@ -498,6 +524,16 @@ namespace AutopilotMonitor.Agent.V2 AppDomain.CurrentDomain.ProcessExit += processExitHandler; orchestrator.Terminated += terminationHandler.Handle; + // Auth-failure watchdog: when MaxAuthFailures / AuthFailureTimeoutMinutes + // is exceeded the agent must shut down cleanly instead of hammering a + // backend that has definitely said no. Event fires at most once. + EventHandler authThresholdHandler = (s, e) => + { + logger.Error($"Auth-failure threshold exceeded ({e.Reason}) — initiating shutdown."); + shutdown.Set(); + }; + authFailureTracker.ThresholdExceeded += authThresholdHandler; + try { orchestrator.Start(); @@ -550,6 +586,7 @@ namespace AutopilotMonitor.Agent.V2 Console.CancelKeyPress -= cancelHandler; AppDomain.CurrentDomain.ProcessExit -= processExitHandler; orchestrator.Terminated -= terminationHandler.Handle; + authFailureTracker.ThresholdExceeded -= authThresholdHandler; try { orchestrator.Stop(); } catch (Exception ex) { logger.Error("Orchestrator stop failed.", ex); }