mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 20:53:06 +00:00
Merge pull request #46 from okieselbach/v2-feat-auth-tracker-binary-integrity
feat(agent): implement MaxAuthFailures watchdog + post-config binary integrity check
This commit is contained in:
@@ -0,0 +1,237 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
|
||||
{
|
||||
public sealed class AuthFailureTrackerTests
|
||||
{
|
||||
private static AgentLogger NewLogger()
|
||||
{
|
||||
var tmp = new TempDirectory();
|
||||
return new AgentLogger(System.IO.Path.Combine(tmp.Path, "logs"), AgentLogLevel.Debug);
|
||||
}
|
||||
|
||||
// ============================================================= Count-based threshold
|
||||
|
||||
[Fact]
|
||||
public void RecordFailure_fires_threshold_at_max_failures()
|
||||
{
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var tracker = new AuthFailureTracker(maxFailures: 3, timeoutMinutes: 0, clock, NewLogger());
|
||||
|
||||
AuthFailureThresholdEventArgs captured = null!;
|
||||
tracker.ThresholdExceeded += (_, e) => captured = e;
|
||||
|
||||
tracker.RecordFailure(401, "config");
|
||||
tracker.RecordFailure(401, "config");
|
||||
Assert.Null(captured);
|
||||
|
||||
tracker.RecordFailure(401, "config");
|
||||
Assert.NotNull(captured);
|
||||
Assert.Equal(3, captured.ConsecutiveFailures);
|
||||
Assert.Equal(401, captured.LastStatusCode);
|
||||
Assert.Equal("config", captured.LastOperation);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordSuccess_resets_counter()
|
||||
{
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var tracker = new AuthFailureTracker(maxFailures: 3, timeoutMinutes: 0, clock, NewLogger());
|
||||
|
||||
int events = 0;
|
||||
tracker.ThresholdExceeded += (_, _) => events++;
|
||||
|
||||
tracker.RecordFailure(401, "config");
|
||||
tracker.RecordFailure(401, "config");
|
||||
tracker.RecordSuccess();
|
||||
tracker.RecordFailure(403, "upload");
|
||||
tracker.RecordFailure(403, "upload");
|
||||
|
||||
Assert.Equal(0, events);
|
||||
Assert.Equal(2, tracker.ConsecutiveFailures);
|
||||
}
|
||||
|
||||
// ============================================================= Time-window threshold
|
||||
|
||||
[Fact]
|
||||
public void RecordFailure_fires_threshold_when_time_window_exceeded()
|
||||
{
|
||||
var start = new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc);
|
||||
var clock = new FakeClock(start);
|
||||
var tracker = new AuthFailureTracker(maxFailures: 0, timeoutMinutes: 10, clock, NewLogger());
|
||||
|
||||
int events = 0;
|
||||
AuthFailureThresholdEventArgs captured = null!;
|
||||
tracker.ThresholdExceeded += (_, e) => { events++; captured = e; };
|
||||
|
||||
tracker.RecordFailure(401, "config");
|
||||
Assert.Equal(0, events);
|
||||
|
||||
clock.Now = start.AddMinutes(5);
|
||||
tracker.RecordFailure(401, "config");
|
||||
Assert.Equal(0, events);
|
||||
|
||||
clock.Now = start.AddMinutes(11);
|
||||
tracker.RecordFailure(401, "config");
|
||||
|
||||
Assert.Equal(1, events);
|
||||
Assert.Equal(start, captured.FirstFailureUtc);
|
||||
Assert.Contains("time window", captured.Reason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordSuccess_resets_time_window_anchor()
|
||||
{
|
||||
var start = new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc);
|
||||
var clock = new FakeClock(start);
|
||||
var tracker = new AuthFailureTracker(maxFailures: 0, timeoutMinutes: 10, clock, NewLogger());
|
||||
|
||||
int events = 0;
|
||||
tracker.ThresholdExceeded += (_, _) => events++;
|
||||
|
||||
tracker.RecordFailure(401, "config");
|
||||
clock.Now = start.AddMinutes(9);
|
||||
tracker.RecordSuccess();
|
||||
|
||||
// fresh window — 11 minutes later, first failure in this new window, no threshold.
|
||||
clock.Now = start.AddMinutes(20);
|
||||
tracker.RecordFailure(401, "config");
|
||||
|
||||
Assert.Equal(0, events);
|
||||
}
|
||||
|
||||
// ============================================================= Idempotence
|
||||
|
||||
[Fact]
|
||||
public void ThresholdExceeded_fires_only_once_even_with_many_failures()
|
||||
{
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var tracker = new AuthFailureTracker(maxFailures: 2, timeoutMinutes: 0, clock, NewLogger());
|
||||
|
||||
int events = 0;
|
||||
tracker.ThresholdExceeded += (_, _) => events++;
|
||||
|
||||
for (int i = 0; i < 10; i++) tracker.RecordFailure(401, "config");
|
||||
|
||||
Assert.Equal(1, events);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ThresholdExceeded_handler_exception_does_not_propagate()
|
||||
{
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var tracker = new AuthFailureTracker(maxFailures: 1, timeoutMinutes: 0, clock, NewLogger());
|
||||
|
||||
tracker.ThresholdExceeded += (_, _) => throw new InvalidOperationException("listener exploded");
|
||||
|
||||
// Must not propagate — tracker is a best-effort observer.
|
||||
tracker.RecordFailure(401, "config");
|
||||
}
|
||||
|
||||
// ============================================================= Disabled limits
|
||||
|
||||
[Fact]
|
||||
public void RecordFailure_never_fires_when_both_limits_disabled()
|
||||
{
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var tracker = new AuthFailureTracker(maxFailures: 0, timeoutMinutes: 0, clock, NewLogger());
|
||||
|
||||
int events = 0;
|
||||
tracker.ThresholdExceeded += (_, _) => events++;
|
||||
|
||||
for (int i = 0; i < 100; i++) tracker.RecordFailure(401, "op");
|
||||
|
||||
Assert.Equal(0, events);
|
||||
Assert.Equal(100, tracker.ConsecutiveFailures);
|
||||
}
|
||||
|
||||
// ============================================================= UpdateLimits (remote config override)
|
||||
|
||||
[Fact]
|
||||
public void UpdateLimits_tightens_threshold_retroactively()
|
||||
{
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var tracker = new AuthFailureTracker(maxFailures: 10, timeoutMinutes: 0, clock, NewLogger());
|
||||
|
||||
int events = 0;
|
||||
tracker.ThresholdExceeded += (_, _) => events++;
|
||||
|
||||
tracker.RecordFailure(401, "config");
|
||||
tracker.RecordFailure(401, "config");
|
||||
|
||||
// Tenant tightened the policy — the next failure should now fire.
|
||||
tracker.UpdateLimits(maxFailures: 3, timeoutMinutes: 0);
|
||||
tracker.RecordFailure(401, "config");
|
||||
|
||||
Assert.Equal(1, events);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UpdateLimits_accepts_negative_max_as_disabled()
|
||||
{
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var tracker = new AuthFailureTracker(maxFailures: -1, timeoutMinutes: 0, clock, NewLogger());
|
||||
|
||||
int events = 0;
|
||||
tracker.ThresholdExceeded += (_, _) => events++;
|
||||
|
||||
for (int i = 0; i < 20; i++) tracker.RecordFailure(401, "op");
|
||||
Assert.Equal(0, events);
|
||||
}
|
||||
|
||||
// ============================================================= Thread-safety smoke
|
||||
|
||||
[Fact]
|
||||
public void RecordFailure_is_safe_under_concurrent_callers()
|
||||
{
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var tracker = new AuthFailureTracker(maxFailures: 50, timeoutMinutes: 0, clock, NewLogger());
|
||||
|
||||
int events = 0;
|
||||
tracker.ThresholdExceeded += (_, _) => Interlocked.Increment(ref events);
|
||||
|
||||
var threads = new Thread[8];
|
||||
for (int t = 0; t < threads.Length; t++)
|
||||
{
|
||||
threads[t] = new Thread(() =>
|
||||
{
|
||||
for (int i = 0; i < 20; i++) tracker.RecordFailure(401, "concurrent");
|
||||
});
|
||||
threads[t].Start();
|
||||
}
|
||||
foreach (var th in threads) th.Join();
|
||||
|
||||
// Many concurrent Increments cross the 50 ceiling — but ThresholdExceeded must fire exactly once.
|
||||
Assert.Equal(1, events);
|
||||
}
|
||||
|
||||
// ============================================================= Guard
|
||||
|
||||
[Fact]
|
||||
public void Ctor_throws_on_null_clock()
|
||||
{
|
||||
Assert.Throws<ArgumentNullException>(() =>
|
||||
new AuthFailureTracker(1, 0, null!, NewLogger()));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Ctor_throws_on_null_logger()
|
||||
{
|
||||
Assert.Throws<ArgumentNullException>(() =>
|
||||
new AuthFailureTracker(1, 0, new FakeClock(DateTime.UtcNow), null!));
|
||||
}
|
||||
|
||||
private sealed class FakeClock : AutopilotMonitor.DecisionCore.Engine.IClock
|
||||
{
|
||||
public DateTime Now { get; set; }
|
||||
public FakeClock(DateTime now) { Now = now; }
|
||||
public DateTime UtcNow => Now;
|
||||
public System.Threading.Tasks.Task Delay(TimeSpan delay, CancellationToken cancellationToken = default) => System.Threading.Tasks.Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
using System.IO;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
|
||||
{
|
||||
public sealed class BinaryIntegrityVerifierTests
|
||||
{
|
||||
private static AgentLogger NewLogger(TempDirectory tmp)
|
||||
=> new AgentLogger(Path.Combine(tmp.Path, "logs"), AgentLogLevel.Debug);
|
||||
|
||||
[Fact]
|
||||
public void Check_returns_skipped_when_expected_hash_is_null_or_empty()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = NewLogger(tmp);
|
||||
|
||||
Assert.Equal(IntegrityCheckOutcome.Skipped, BinaryIntegrityVerifier.Check(null!, logger).Outcome);
|
||||
Assert.Equal(IntegrityCheckOutcome.Skipped, BinaryIntegrityVerifier.Check("", logger).Outcome);
|
||||
Assert.Equal(IntegrityCheckOutcome.Skipped, BinaryIntegrityVerifier.Check(" ", logger).Outcome);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Check_returns_skipped_when_target_path_does_not_exist()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = NewLogger(tmp);
|
||||
var missing = Path.Combine(tmp.Path, "no-such.exe");
|
||||
|
||||
var result = BinaryIntegrityVerifier.Check("0".PadLeft(64, '0'), logger, exePath: missing);
|
||||
|
||||
Assert.Equal(IntegrityCheckOutcome.Skipped, result.Outcome);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Check_returns_match_when_hash_equals_actual()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = NewLogger(tmp);
|
||||
|
||||
var exe = Path.Combine(tmp.Path, "fake.exe");
|
||||
File.WriteAllBytes(exe, new byte[] { 0x01, 0x02, 0x03, 0x04 });
|
||||
var actual = BinaryIntegrityVerifier.ComputeSha256Hex(exe);
|
||||
|
||||
var result = BinaryIntegrityVerifier.Check(actual, logger, exePath: exe);
|
||||
|
||||
Assert.Equal(IntegrityCheckOutcome.Match, result.Outcome);
|
||||
Assert.False(result.IsMismatch);
|
||||
Assert.Equal(actual, result.ActualSha256);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Check_is_case_insensitive_for_expected_hash()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = NewLogger(tmp);
|
||||
|
||||
var exe = Path.Combine(tmp.Path, "fake.exe");
|
||||
File.WriteAllBytes(exe, new byte[] { 0xAB, 0xCD });
|
||||
var actualLower = BinaryIntegrityVerifier.ComputeSha256Hex(exe);
|
||||
|
||||
var result = BinaryIntegrityVerifier.Check(actualLower.ToUpperInvariant(), logger, exePath: exe);
|
||||
|
||||
Assert.Equal(IntegrityCheckOutcome.Match, result.Outcome);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Check_returns_mismatch_when_hash_differs()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = NewLogger(tmp);
|
||||
|
||||
var exe = Path.Combine(tmp.Path, "fake.exe");
|
||||
File.WriteAllBytes(exe, new byte[] { 0x01 });
|
||||
var wrongHash = new string('f', 64);
|
||||
|
||||
var result = BinaryIntegrityVerifier.Check(wrongHash, logger, exePath: exe);
|
||||
|
||||
Assert.Equal(IntegrityCheckOutcome.Mismatch, result.Outcome);
|
||||
Assert.True(result.IsMismatch);
|
||||
Assert.Equal(wrongHash, result.ExpectedSha256);
|
||||
Assert.NotEqual(result.ExpectedSha256, result.ActualSha256);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ComputeSha256Hex_matches_known_vector()
|
||||
{
|
||||
// SHA-256("abc") = ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad
|
||||
using var tmp = new TempDirectory();
|
||||
var exe = Path.Combine(tmp.Path, "abc.bin");
|
||||
File.WriteAllText(exe, "abc");
|
||||
|
||||
var hash = BinaryIntegrityVerifier.ComputeSha256Hex(exe);
|
||||
|
||||
Assert.Equal("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", hash);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
@@ -21,6 +22,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
private readonly string _cacheFilePath;
|
||||
private readonly EmergencyReporter _emergencyReporter;
|
||||
private readonly DistressReporter _distressReporter;
|
||||
private readonly AuthFailureTracker _authFailureTracker;
|
||||
|
||||
private AgentConfigResponse _currentConfig;
|
||||
private readonly object _configLock = new object();
|
||||
@@ -39,13 +41,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
}
|
||||
}
|
||||
|
||||
public RemoteConfigService(BackendApiClient apiClient, string tenantId, AgentLogger logger, EmergencyReporter emergencyReporter = null, DistressReporter distressReporter = null)
|
||||
public RemoteConfigService(BackendApiClient apiClient, string tenantId, AgentLogger logger, EmergencyReporter emergencyReporter = null, DistressReporter distressReporter = null, AuthFailureTracker authFailureTracker = null)
|
||||
{
|
||||
_apiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient));
|
||||
_tenantId = tenantId ?? throw new ArgumentNullException(nameof(tenantId));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_emergencyReporter = emergencyReporter;
|
||||
_distressReporter = distressReporter;
|
||||
_authFailureTracker = authFailureTracker;
|
||||
|
||||
var cacheDir = Environment.ExpandEnvironmentVariables(@"%ProgramData%\AutopilotMonitor\Config");
|
||||
_cacheFilePath = Path.Combine(cacheDir, "remote-config.json");
|
||||
@@ -70,6 +73,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
LogCollectorSettings(config.Collectors);
|
||||
SetConfig(config);
|
||||
CacheConfig(config);
|
||||
_authFailureTracker?.RecordSuccess();
|
||||
return config;
|
||||
}
|
||||
|
||||
@@ -87,6 +91,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
ex.Message,
|
||||
httpStatusCode: ex.StatusCode);
|
||||
}
|
||||
|
||||
// Feed the central auth-failure tracker so the agent can shut down cleanly
|
||||
// after MaxAuthFailures / AuthFailureTimeoutMinutes is exceeded, instead of
|
||||
// hammering the backend forever when the device certificate is revoked.
|
||||
_authFailureTracker?.RecordFailure(ex.StatusCode, "agent/config");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
{
|
||||
/// <summary>
|
||||
/// Tracks consecutive 401/403 responses from the backend. When either the consecutive-count
|
||||
/// ceiling (<c>MaxAuthFailures</c>) or the elapsed-time ceiling (<c>AuthFailureTimeoutMinutes</c>)
|
||||
/// is exceeded, <see cref="ThresholdExceeded"/> fires exactly once so <c>Program.RunAgent</c>
|
||||
/// can trigger a soft shutdown.
|
||||
/// <para>
|
||||
/// Without this tracker a device whose certificate is revoked (or whose tenant has been deleted)
|
||||
/// would retry the config fetch and every telemetry upload indefinitely, flooding the backend
|
||||
/// distress channel and the local log. The tracker is the missing enforcement for the
|
||||
/// <c>MaxAuthFailures</c> and <c>AuthFailureTimeoutMinutes</c> knobs on
|
||||
/// <c>AgentConfigResponse</c> (defaults: 5 consecutive, time window disabled).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Thread-safety: consecutive-count uses <see cref="Interlocked"/>; the first-failure timestamp
|
||||
/// is protected by a single monitor lock. <see cref="ThresholdExceeded"/> is raised outside the
|
||||
/// lock to prevent handler re-entrancy deadlocks.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class AuthFailureTracker
|
||||
{
|
||||
private readonly IClock _clock;
|
||||
private readonly AgentLogger _logger;
|
||||
private readonly object _windowLock = new object();
|
||||
|
||||
private int _maxFailures; // 0 = disabled
|
||||
private TimeSpan? _timeoutWindow; // null = disabled
|
||||
private int _consecutiveFailures;
|
||||
private DateTime? _firstFailureUtc;
|
||||
private int _thresholdFired; // 0/1 — Interlocked exchange makes the event single-shot
|
||||
|
||||
public AuthFailureTracker(
|
||||
int maxFailures,
|
||||
int timeoutMinutes,
|
||||
IClock clock,
|
||||
AgentLogger logger)
|
||||
{
|
||||
_clock = clock ?? throw new ArgumentNullException(nameof(clock));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
UpdateLimits(maxFailures, timeoutMinutes);
|
||||
}
|
||||
|
||||
/// <summary>Fires once when either threshold is crossed. Listener is responsible for shutting the agent down.</summary>
|
||||
public event EventHandler<AuthFailureThresholdEventArgs> ThresholdExceeded;
|
||||
|
||||
/// <summary>Updates the limits from the merged remote config. Safe to call at any time; does not reset the counter.</summary>
|
||||
public void UpdateLimits(int maxFailures, int timeoutMinutes)
|
||||
{
|
||||
_maxFailures = maxFailures < 0 ? 0 : maxFailures;
|
||||
_timeoutWindow = timeoutMinutes > 0 ? (TimeSpan?)TimeSpan.FromMinutes(timeoutMinutes) : null;
|
||||
}
|
||||
|
||||
/// <summary>Current consecutive-failure count. Exposed for observability and tests.</summary>
|
||||
public int ConsecutiveFailures => Volatile.Read(ref _consecutiveFailures);
|
||||
|
||||
/// <summary>Reset counter + window anchor after a successful authenticated response.</summary>
|
||||
public void RecordSuccess()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _consecutiveFailures, 0) == 0) return; // no-op if already zero
|
||||
lock (_windowLock) { _firstFailureUtc = null; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Record a 401/403 from the given <paramref name="operation"/>. Emits <see cref="ThresholdExceeded"/>
|
||||
/// the first time either ceiling is crossed; subsequent calls after termination is armed are no-ops.
|
||||
/// </summary>
|
||||
public void RecordFailure(int statusCode, string operation)
|
||||
{
|
||||
if (Volatile.Read(ref _thresholdFired) == 1) return;
|
||||
|
||||
var count = Interlocked.Increment(ref _consecutiveFailures);
|
||||
DateTime now = _clock.UtcNow;
|
||||
DateTime firstFailureAt;
|
||||
|
||||
lock (_windowLock)
|
||||
{
|
||||
if (_firstFailureUtc == null) _firstFailureUtc = now;
|
||||
firstFailureAt = _firstFailureUtc.Value;
|
||||
}
|
||||
|
||||
bool countExceeded = _maxFailures > 0 && count >= _maxFailures;
|
||||
bool windowExceeded = _timeoutWindow.HasValue && (now - firstFailureAt) >= _timeoutWindow.Value;
|
||||
|
||||
if (!countExceeded && !windowExceeded)
|
||||
{
|
||||
_logger.Warning($"AuthFailureTracker: consecutive auth failure #{count} (http {statusCode}, {operation}).");
|
||||
return;
|
||||
}
|
||||
|
||||
if (Interlocked.Exchange(ref _thresholdFired, 1) == 1) return; // another thread won the race
|
||||
|
||||
var reason = countExceeded
|
||||
? $"consecutive auth failures reached limit ({count} >= {_maxFailures})"
|
||||
: $"auth-failure time window exceeded ({(now - firstFailureAt).TotalMinutes:F0}min >= {_timeoutWindow.Value.TotalMinutes:F0}min)";
|
||||
|
||||
_logger.Error($"AuthFailureTracker: {reason} — signalling shutdown. Last operation: {operation}, statusCode={statusCode}.");
|
||||
|
||||
try
|
||||
{
|
||||
ThresholdExceeded?.Invoke(this, new AuthFailureThresholdEventArgs(
|
||||
consecutiveFailures: count,
|
||||
firstFailureUtc: firstFailureAt,
|
||||
lastOperation: operation,
|
||||
lastStatusCode: statusCode,
|
||||
reason: reason));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Warning($"AuthFailureTracker: ThresholdExceeded handler threw: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class AuthFailureThresholdEventArgs : EventArgs
|
||||
{
|
||||
public AuthFailureThresholdEventArgs(
|
||||
int consecutiveFailures,
|
||||
DateTime firstFailureUtc,
|
||||
string lastOperation,
|
||||
int lastStatusCode,
|
||||
string reason)
|
||||
{
|
||||
ConsecutiveFailures = consecutiveFailures;
|
||||
FirstFailureUtc = firstFailureUtc;
|
||||
LastOperation = lastOperation;
|
||||
LastStatusCode = lastStatusCode;
|
||||
Reason = reason;
|
||||
}
|
||||
|
||||
public int ConsecutiveFailures { get; }
|
||||
public DateTime FirstFailureUtc { get; }
|
||||
public string LastOperation { get; }
|
||||
public int LastStatusCode { get; }
|
||||
public string Reason { get; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Reflection;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
{
|
||||
/// <summary>
|
||||
/// Post-config binary-integrity check. After the remote config is fetched, the agent compares
|
||||
/// the SHA-256 of its running <c>.exe</c> against <c>AgentConfigResponse.LatestAgentExeSha256</c>.
|
||||
/// A mismatch means one of:
|
||||
/// <list type="bullet">
|
||||
/// <item>tampering — the on-disk binary was modified after deployment,</item>
|
||||
/// <item>stale blob storage — the agent is running an older EXE than what the backend advertises,</item>
|
||||
/// <item>self-update failed silently — a previous update swapped the ZIP but not the EXE.</item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// We intentionally do <b>not</b> terminate or auto-update from this check: <see cref="SelfUpdater"/>
|
||||
/// already runs at startup and its hash-based trigger covers the "wrong EXE" case. All we do
|
||||
/// here is emit an <c>AgentErrorType.IntegrityCheckFailed</c> report so operators see the drift.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static class BinaryIntegrityVerifier
|
||||
{
|
||||
/// <summary>
|
||||
/// Computes the SHA-256 of <paramref name="exePath"/> (or the currently executing assembly
|
||||
/// when <c>null</c>) and compares it against <paramref name="expectedSha256"/>.
|
||||
/// Case-insensitive comparison (both lowercase and uppercase hex strings accepted).
|
||||
/// </summary>
|
||||
public static IntegrityCheckResult Check(
|
||||
string expectedSha256,
|
||||
AgentLogger logger,
|
||||
string exePath = null)
|
||||
{
|
||||
if (logger == null) throw new ArgumentNullException(nameof(logger));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(expectedSha256))
|
||||
{
|
||||
logger.Debug("BinaryIntegrityVerifier: no expected hash provided — skipping check.");
|
||||
return IntegrityCheckResult.Skipped();
|
||||
}
|
||||
|
||||
var path = exePath ?? Assembly.GetExecutingAssembly().Location;
|
||||
if (string.IsNullOrEmpty(path) || !File.Exists(path))
|
||||
{
|
||||
logger.Warning($"BinaryIntegrityVerifier: executing assembly path not resolvable ({path ?? "null"}) — skipping check.");
|
||||
return IntegrityCheckResult.Skipped();
|
||||
}
|
||||
|
||||
string actualSha;
|
||||
try
|
||||
{
|
||||
actualSha = ComputeSha256Hex(path);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.Warning($"BinaryIntegrityVerifier: failed to hash '{path}': {ex.Message}");
|
||||
return IntegrityCheckResult.Skipped();
|
||||
}
|
||||
|
||||
if (string.Equals(actualSha, expectedSha256, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
logger.Debug($"BinaryIntegrityVerifier: SHA-256 match (sha={Truncate(actualSha)}).");
|
||||
return IntegrityCheckResult.Match(actualSha);
|
||||
}
|
||||
|
||||
logger.Warning(
|
||||
$"BinaryIntegrityVerifier: SHA-256 MISMATCH — running exe={Truncate(actualSha)}, backend advertises={Truncate(expectedSha256)}. " +
|
||||
"Possible tamper, stale blob, or failed self-update.");
|
||||
return IntegrityCheckResult.Mismatch(actualSha, expectedSha256);
|
||||
}
|
||||
|
||||
internal static string ComputeSha256Hex(string filePath)
|
||||
{
|
||||
using (var stream = File.OpenRead(filePath))
|
||||
using (var sha = SHA256.Create())
|
||||
{
|
||||
var bytes = sha.ComputeHash(stream);
|
||||
var sb = new StringBuilder(bytes.Length * 2);
|
||||
for (int i = 0; i < bytes.Length; i++) sb.Append(bytes[i].ToString("x2"));
|
||||
return sb.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
private static string Truncate(string hex)
|
||||
=> string.IsNullOrEmpty(hex) ? "(null)" : (hex.Length <= 12 ? hex : hex.Substring(0, 12) + "...");
|
||||
}
|
||||
|
||||
public sealed class IntegrityCheckResult
|
||||
{
|
||||
public IntegrityCheckOutcome Outcome { get; }
|
||||
public string ActualSha256 { get; }
|
||||
public string ExpectedSha256 { get; }
|
||||
|
||||
private IntegrityCheckResult(IntegrityCheckOutcome outcome, string actual, string expected)
|
||||
{
|
||||
Outcome = outcome;
|
||||
ActualSha256 = actual;
|
||||
ExpectedSha256 = expected;
|
||||
}
|
||||
|
||||
public static IntegrityCheckResult Skipped() => new IntegrityCheckResult(IntegrityCheckOutcome.Skipped, null, null);
|
||||
public static IntegrityCheckResult Match(string actual) => new IntegrityCheckResult(IntegrityCheckOutcome.Match, actual, actual);
|
||||
public static IntegrityCheckResult Mismatch(string actual, string expected) => new IntegrityCheckResult(IntegrityCheckOutcome.Mismatch, actual, expected);
|
||||
|
||||
public bool IsMismatch => Outcome == IntegrityCheckOutcome.Mismatch;
|
||||
}
|
||||
|
||||
public enum IntegrityCheckOutcome
|
||||
{
|
||||
/// <summary>No expected hash provided, or the local path could not be hashed.</summary>
|
||||
Skipped = 0,
|
||||
|
||||
/// <summary>SHA-256 of the running EXE matches the backend-advertised hash.</summary>
|
||||
Match = 1,
|
||||
|
||||
/// <summary>SHA-256 differs — tamper / stale blob / failed self-update.</summary>
|
||||
Mismatch = 2,
|
||||
}
|
||||
}
|
||||
+13
-2
@@ -9,6 +9,7 @@ using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Newtonsoft.Json;
|
||||
@@ -46,6 +47,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
private readonly string? _serialNumber;
|
||||
private readonly string? _bootstrapToken;
|
||||
private readonly string? _agentVersion;
|
||||
private readonly AuthFailureTracker? _authFailureTracker;
|
||||
|
||||
public BackendTelemetryUploader(
|
||||
HttpClient httpClient,
|
||||
@@ -55,7 +57,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
string? model = null,
|
||||
string? serialNumber = null,
|
||||
string? bootstrapToken = null,
|
||||
string? agentVersion = null)
|
||||
string? agentVersion = null,
|
||||
AuthFailureTracker? authFailureTracker = null)
|
||||
{
|
||||
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
|
||||
if (string.IsNullOrEmpty(baseUrl)) throw new ArgumentException("BaseUrl is mandatory.", nameof(baseUrl));
|
||||
@@ -68,6 +71,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
_serialNumber = serialNumber;
|
||||
_bootstrapToken = bootstrapToken;
|
||||
_agentVersion = agentVersion;
|
||||
_authFailureTracker = authFailureTracker;
|
||||
}
|
||||
|
||||
public async Task<UploadResult> UploadBatchAsync(
|
||||
@@ -124,12 +128,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<UploadResult> MapResponseAsync(HttpResponseMessage response)
|
||||
private async Task<UploadResult> MapResponseAsync(HttpResponseMessage response)
|
||||
{
|
||||
var statusCode = (int)response.StatusCode;
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
// Auth was accepted — reset the consecutive-failure counter so a transient
|
||||
// backend hiccup does not eventually shut the agent down.
|
||||
_authFailureTracker?.RecordSuccess();
|
||||
|
||||
// M4.6.ε — parse backend-to-agent control signals from the 2xx response body.
|
||||
// Body is best-effort JSON; anything unparseable degrades cleanly to plain Ok().
|
||||
return await TryReadControlSignalsAsync(response).ConfigureAwait(false);
|
||||
@@ -142,6 +150,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
{
|
||||
case HttpStatusCode.Unauthorized: // 401
|
||||
case HttpStatusCode.Forbidden: // 403
|
||||
// Feed the central tracker so the agent can shut down after MaxAuthFailures
|
||||
// instead of retrying a permanently-revoked certificate forever.
|
||||
_authFailureTracker?.RecordFailure(statusCode, "agent/telemetry");
|
||||
return UploadResult.Permanent($"unauthorized: {shortReason}");
|
||||
|
||||
case HttpStatusCode.RequestTimeout: // 408
|
||||
|
||||
@@ -306,8 +306,18 @@ namespace AutopilotMonitor.Agent.V2
|
||||
"MDM certificate not found in LocalMachine or CurrentUser store");
|
||||
}
|
||||
|
||||
// Central observer for consecutive 401/403 responses. Initialised with the CLI/bootstrap
|
||||
// defaults on AgentConfiguration; UpdateLimits is called after RemoteConfigMerger.Merge
|
||||
// so tenant-policy overrides take effect. ThresholdExceeded is wired below, once the
|
||||
// shutdown signal is available, to trigger a clean agent exit when the limit is hit.
|
||||
var authFailureTracker = new AuthFailureTracker(
|
||||
maxFailures: agentConfig.MaxAuthFailures,
|
||||
timeoutMinutes: agentConfig.AuthFailureTimeoutMinutes,
|
||||
clock: SystemClock.Instance,
|
||||
logger: logger);
|
||||
|
||||
var remoteConfigService = new RemoteConfigService(
|
||||
backendApiClient, agentConfig.TenantId, logger, emergencyReporter, distressReporter);
|
||||
backendApiClient, agentConfig.TenantId, logger, emergencyReporter, distressReporter, authFailureTracker);
|
||||
var remoteConfig = remoteConfigService.FetchConfigAsync().GetAwaiter().GetResult();
|
||||
|
||||
// Project remote tenant-controlled knobs onto the runtime AgentConfiguration so that
|
||||
@@ -316,6 +326,9 @@ namespace AutopilotMonitor.Agent.V2
|
||||
// actually respect the tenant admin settings. CLI flags stay authoritative over remote.
|
||||
RemoteConfigMerger.Merge(agentConfig, remoteConfig, args);
|
||||
|
||||
// Refresh tracker ceilings with the tenant-specific values we just merged in.
|
||||
authFailureTracker.UpdateLimits(agentConfig.MaxAuthFailures, agentConfig.AuthFailureTimeoutMinutes);
|
||||
|
||||
logger.SetLogLevel(agentConfig.LogLevel);
|
||||
|
||||
// Propagate the backend-expected SHA so the runtime hash-mismatch trigger (M4.6.α
|
||||
@@ -324,6 +337,18 @@ namespace AutopilotMonitor.Agent.V2
|
||||
if (!string.IsNullOrEmpty(remoteConfig.LatestAgentSha256))
|
||||
SelfUpdater.BackendExpectedSha256 = remoteConfig.LatestAgentSha256;
|
||||
|
||||
// Post-config binary-integrity check: verify the running EXE's SHA-256 against the
|
||||
// value advertised by the backend. Mismatch does not terminate (SelfUpdater's own
|
||||
// hash-based trigger covers the "wrong EXE" case at next startup) — we just emit an
|
||||
// IntegrityCheckFailed error report so operators see the drift.
|
||||
var integrityResult = BinaryIntegrityVerifier.Check(remoteConfig.LatestAgentExeSha256, logger);
|
||||
if (integrityResult.IsMismatch)
|
||||
{
|
||||
_ = emergencyReporter.TrySendAsync(
|
||||
AgentErrorType.IntegrityCheckFailed,
|
||||
$"Running exe SHA-256 differs from backend-advertised hash. actual={integrityResult.ActualSha256}, expected={integrityResult.ExpectedSha256}");
|
||||
}
|
||||
|
||||
// H-2 mitigation: delete the persisted bootstrap-config.json once the MDM cert
|
||||
// proves it can authenticate. Non-blocking — any failure leaves the file for retry.
|
||||
try
|
||||
@@ -361,7 +386,8 @@ namespace AutopilotMonitor.Agent.V2
|
||||
model: hardwareForReporters.Model,
|
||||
serialNumber: hardwareForReporters.SerialNumber,
|
||||
bootstrapToken: agentConfig.UseBootstrapTokenAuth ? agentConfig.BootstrapToken : null,
|
||||
agentVersion: GetAgentVersion());
|
||||
agentVersion: GetAgentVersion(),
|
||||
authFailureTracker: authFailureTracker);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -498,6 +524,16 @@ namespace AutopilotMonitor.Agent.V2
|
||||
AppDomain.CurrentDomain.ProcessExit += processExitHandler;
|
||||
orchestrator.Terminated += terminationHandler.Handle;
|
||||
|
||||
// Auth-failure watchdog: when MaxAuthFailures / AuthFailureTimeoutMinutes
|
||||
// is exceeded the agent must shut down cleanly instead of hammering a
|
||||
// backend that has definitely said no. Event fires at most once.
|
||||
EventHandler<AuthFailureThresholdEventArgs> authThresholdHandler = (s, e) =>
|
||||
{
|
||||
logger.Error($"Auth-failure threshold exceeded ({e.Reason}) — initiating shutdown.");
|
||||
shutdown.Set();
|
||||
};
|
||||
authFailureTracker.ThresholdExceeded += authThresholdHandler;
|
||||
|
||||
try
|
||||
{
|
||||
orchestrator.Start();
|
||||
@@ -550,6 +586,7 @@ namespace AutopilotMonitor.Agent.V2
|
||||
Console.CancelKeyPress -= cancelHandler;
|
||||
AppDomain.CurrentDomain.ProcessExit -= processExitHandler;
|
||||
orchestrator.Terminated -= terminationHandler.Handle;
|
||||
authFailureTracker.ThresholdExceeded -= authThresholdHandler;
|
||||
|
||||
try { orchestrator.Stop(); }
|
||||
catch (Exception ex) { logger.Error("Orchestrator stop failed.", ex); }
|
||||
|
||||
Reference in New Issue
Block a user