Merge pull request #46 from okieselbach/v2-feat-auth-tracker-binary-integrity

feat(agent): implement MaxAuthFailures watchdog + post-config binary integrity check
This commit is contained in:
Oliver Kieselbach
2026-04-21 18:20:26 +02:00
committed by GitHub
7 changed files with 663 additions and 5 deletions
@@ -0,0 +1,237 @@
using System;
using System.Threading;
using AutopilotMonitor.Agent.V2.Core.Logging;
using AutopilotMonitor.Agent.V2.Core.Security;
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
using Xunit;
namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
{
public sealed class AuthFailureTrackerTests
{
private static AgentLogger NewLogger()
{
var tmp = new TempDirectory();
return new AgentLogger(System.IO.Path.Combine(tmp.Path, "logs"), AgentLogLevel.Debug);
}
// ============================================================= Count-based threshold
[Fact]
public void RecordFailure_fires_threshold_at_max_failures()
{
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
var tracker = new AuthFailureTracker(maxFailures: 3, timeoutMinutes: 0, clock, NewLogger());
AuthFailureThresholdEventArgs captured = null!;
tracker.ThresholdExceeded += (_, e) => captured = e;
tracker.RecordFailure(401, "config");
tracker.RecordFailure(401, "config");
Assert.Null(captured);
tracker.RecordFailure(401, "config");
Assert.NotNull(captured);
Assert.Equal(3, captured.ConsecutiveFailures);
Assert.Equal(401, captured.LastStatusCode);
Assert.Equal("config", captured.LastOperation);
}
[Fact]
public void RecordSuccess_resets_counter()
{
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
var tracker = new AuthFailureTracker(maxFailures: 3, timeoutMinutes: 0, clock, NewLogger());
int events = 0;
tracker.ThresholdExceeded += (_, _) => events++;
tracker.RecordFailure(401, "config");
tracker.RecordFailure(401, "config");
tracker.RecordSuccess();
tracker.RecordFailure(403, "upload");
tracker.RecordFailure(403, "upload");
Assert.Equal(0, events);
Assert.Equal(2, tracker.ConsecutiveFailures);
}
// ============================================================= Time-window threshold
[Fact]
public void RecordFailure_fires_threshold_when_time_window_exceeded()
{
var start = new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc);
var clock = new FakeClock(start);
var tracker = new AuthFailureTracker(maxFailures: 0, timeoutMinutes: 10, clock, NewLogger());
int events = 0;
AuthFailureThresholdEventArgs captured = null!;
tracker.ThresholdExceeded += (_, e) => { events++; captured = e; };
tracker.RecordFailure(401, "config");
Assert.Equal(0, events);
clock.Now = start.AddMinutes(5);
tracker.RecordFailure(401, "config");
Assert.Equal(0, events);
clock.Now = start.AddMinutes(11);
tracker.RecordFailure(401, "config");
Assert.Equal(1, events);
Assert.Equal(start, captured.FirstFailureUtc);
Assert.Contains("time window", captured.Reason);
}
[Fact]
public void RecordSuccess_resets_time_window_anchor()
{
var start = new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc);
var clock = new FakeClock(start);
var tracker = new AuthFailureTracker(maxFailures: 0, timeoutMinutes: 10, clock, NewLogger());
int events = 0;
tracker.ThresholdExceeded += (_, _) => events++;
tracker.RecordFailure(401, "config");
clock.Now = start.AddMinutes(9);
tracker.RecordSuccess();
// fresh window — 11 minutes later, first failure in this new window, no threshold.
clock.Now = start.AddMinutes(20);
tracker.RecordFailure(401, "config");
Assert.Equal(0, events);
}
// ============================================================= Idempotence
[Fact]
public void ThresholdExceeded_fires_only_once_even_with_many_failures()
{
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
var tracker = new AuthFailureTracker(maxFailures: 2, timeoutMinutes: 0, clock, NewLogger());
int events = 0;
tracker.ThresholdExceeded += (_, _) => events++;
for (int i = 0; i < 10; i++) tracker.RecordFailure(401, "config");
Assert.Equal(1, events);
}
[Fact]
public void ThresholdExceeded_handler_exception_does_not_propagate()
{
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
var tracker = new AuthFailureTracker(maxFailures: 1, timeoutMinutes: 0, clock, NewLogger());
tracker.ThresholdExceeded += (_, _) => throw new InvalidOperationException("listener exploded");
// Must not propagate — tracker is a best-effort observer.
tracker.RecordFailure(401, "config");
}
// ============================================================= Disabled limits
[Fact]
public void RecordFailure_never_fires_when_both_limits_disabled()
{
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
var tracker = new AuthFailureTracker(maxFailures: 0, timeoutMinutes: 0, clock, NewLogger());
int events = 0;
tracker.ThresholdExceeded += (_, _) => events++;
for (int i = 0; i < 100; i++) tracker.RecordFailure(401, "op");
Assert.Equal(0, events);
Assert.Equal(100, tracker.ConsecutiveFailures);
}
// ============================================================= UpdateLimits (remote config override)
[Fact]
public void UpdateLimits_tightens_threshold_retroactively()
{
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
var tracker = new AuthFailureTracker(maxFailures: 10, timeoutMinutes: 0, clock, NewLogger());
int events = 0;
tracker.ThresholdExceeded += (_, _) => events++;
tracker.RecordFailure(401, "config");
tracker.RecordFailure(401, "config");
// Tenant tightened the policy — the next failure should now fire.
tracker.UpdateLimits(maxFailures: 3, timeoutMinutes: 0);
tracker.RecordFailure(401, "config");
Assert.Equal(1, events);
}
[Fact]
public void UpdateLimits_accepts_negative_max_as_disabled()
{
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
var tracker = new AuthFailureTracker(maxFailures: -1, timeoutMinutes: 0, clock, NewLogger());
int events = 0;
tracker.ThresholdExceeded += (_, _) => events++;
for (int i = 0; i < 20; i++) tracker.RecordFailure(401, "op");
Assert.Equal(0, events);
}
// ============================================================= Thread-safety smoke
[Fact]
public void RecordFailure_is_safe_under_concurrent_callers()
{
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
var tracker = new AuthFailureTracker(maxFailures: 50, timeoutMinutes: 0, clock, NewLogger());
int events = 0;
tracker.ThresholdExceeded += (_, _) => Interlocked.Increment(ref events);
var threads = new Thread[8];
for (int t = 0; t < threads.Length; t++)
{
threads[t] = new Thread(() =>
{
for (int i = 0; i < 20; i++) tracker.RecordFailure(401, "concurrent");
});
threads[t].Start();
}
foreach (var th in threads) th.Join();
// Many concurrent Increments cross the 50 ceiling — but ThresholdExceeded must fire exactly once.
Assert.Equal(1, events);
}
// ============================================================= Guard
[Fact]
public void Ctor_throws_on_null_clock()
{
Assert.Throws<ArgumentNullException>(() =>
new AuthFailureTracker(1, 0, null!, NewLogger()));
}
[Fact]
public void Ctor_throws_on_null_logger()
{
Assert.Throws<ArgumentNullException>(() =>
new AuthFailureTracker(1, 0, new FakeClock(DateTime.UtcNow), null!));
}
private sealed class FakeClock : AutopilotMonitor.DecisionCore.Engine.IClock
{
public DateTime Now { get; set; }
public FakeClock(DateTime now) { Now = now; }
public DateTime UtcNow => Now;
public System.Threading.Tasks.Task Delay(TimeSpan delay, CancellationToken cancellationToken = default) => System.Threading.Tasks.Task.CompletedTask;
}
}
}
@@ -0,0 +1,100 @@
using System.IO;
using AutopilotMonitor.Agent.V2.Core.Logging;
using AutopilotMonitor.Agent.V2.Core.Security;
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
using Xunit;
namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
{
public sealed class BinaryIntegrityVerifierTests
{
private static AgentLogger NewLogger(TempDirectory tmp)
=> new AgentLogger(Path.Combine(tmp.Path, "logs"), AgentLogLevel.Debug);
[Fact]
public void Check_returns_skipped_when_expected_hash_is_null_or_empty()
{
using var tmp = new TempDirectory();
var logger = NewLogger(tmp);
Assert.Equal(IntegrityCheckOutcome.Skipped, BinaryIntegrityVerifier.Check(null!, logger).Outcome);
Assert.Equal(IntegrityCheckOutcome.Skipped, BinaryIntegrityVerifier.Check("", logger).Outcome);
Assert.Equal(IntegrityCheckOutcome.Skipped, BinaryIntegrityVerifier.Check(" ", logger).Outcome);
}
[Fact]
public void Check_returns_skipped_when_target_path_does_not_exist()
{
using var tmp = new TempDirectory();
var logger = NewLogger(tmp);
var missing = Path.Combine(tmp.Path, "no-such.exe");
var result = BinaryIntegrityVerifier.Check("0".PadLeft(64, '0'), logger, exePath: missing);
Assert.Equal(IntegrityCheckOutcome.Skipped, result.Outcome);
}
[Fact]
public void Check_returns_match_when_hash_equals_actual()
{
using var tmp = new TempDirectory();
var logger = NewLogger(tmp);
var exe = Path.Combine(tmp.Path, "fake.exe");
File.WriteAllBytes(exe, new byte[] { 0x01, 0x02, 0x03, 0x04 });
var actual = BinaryIntegrityVerifier.ComputeSha256Hex(exe);
var result = BinaryIntegrityVerifier.Check(actual, logger, exePath: exe);
Assert.Equal(IntegrityCheckOutcome.Match, result.Outcome);
Assert.False(result.IsMismatch);
Assert.Equal(actual, result.ActualSha256);
}
[Fact]
public void Check_is_case_insensitive_for_expected_hash()
{
using var tmp = new TempDirectory();
var logger = NewLogger(tmp);
var exe = Path.Combine(tmp.Path, "fake.exe");
File.WriteAllBytes(exe, new byte[] { 0xAB, 0xCD });
var actualLower = BinaryIntegrityVerifier.ComputeSha256Hex(exe);
var result = BinaryIntegrityVerifier.Check(actualLower.ToUpperInvariant(), logger, exePath: exe);
Assert.Equal(IntegrityCheckOutcome.Match, result.Outcome);
}
[Fact]
public void Check_returns_mismatch_when_hash_differs()
{
using var tmp = new TempDirectory();
var logger = NewLogger(tmp);
var exe = Path.Combine(tmp.Path, "fake.exe");
File.WriteAllBytes(exe, new byte[] { 0x01 });
var wrongHash = new string('f', 64);
var result = BinaryIntegrityVerifier.Check(wrongHash, logger, exePath: exe);
Assert.Equal(IntegrityCheckOutcome.Mismatch, result.Outcome);
Assert.True(result.IsMismatch);
Assert.Equal(wrongHash, result.ExpectedSha256);
Assert.NotEqual(result.ExpectedSha256, result.ActualSha256);
}
[Fact]
public void ComputeSha256Hex_matches_known_vector()
{
// SHA-256("abc") = ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad
using var tmp = new TempDirectory();
var exe = Path.Combine(tmp.Path, "abc.bin");
File.WriteAllText(exe, "abc");
var hash = BinaryIntegrityVerifier.ComputeSha256Hex(exe);
Assert.Equal("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", hash);
}
}
}
@@ -4,6 +4,7 @@ using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
using AutopilotMonitor.Agent.V2.Core.Logging;
using AutopilotMonitor.Agent.V2.Core.Security;
using AutopilotMonitor.Shared.Models;
using Newtonsoft.Json;
@@ -21,6 +22,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
private readonly string _cacheFilePath;
private readonly EmergencyReporter _emergencyReporter;
private readonly DistressReporter _distressReporter;
private readonly AuthFailureTracker _authFailureTracker;
private AgentConfigResponse _currentConfig;
private readonly object _configLock = new object();
@@ -39,13 +41,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
}
}
public RemoteConfigService(BackendApiClient apiClient, string tenantId, AgentLogger logger, EmergencyReporter emergencyReporter = null, DistressReporter distressReporter = null)
public RemoteConfigService(BackendApiClient apiClient, string tenantId, AgentLogger logger, EmergencyReporter emergencyReporter = null, DistressReporter distressReporter = null, AuthFailureTracker authFailureTracker = null)
{
_apiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient));
_tenantId = tenantId ?? throw new ArgumentNullException(nameof(tenantId));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_emergencyReporter = emergencyReporter;
_distressReporter = distressReporter;
_authFailureTracker = authFailureTracker;
var cacheDir = Environment.ExpandEnvironmentVariables(@"%ProgramData%\AutopilotMonitor\Config");
_cacheFilePath = Path.Combine(cacheDir, "remote-config.json");
@@ -70,6 +73,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
LogCollectorSettings(config.Collectors);
SetConfig(config);
CacheConfig(config);
_authFailureTracker?.RecordSuccess();
return config;
}
@@ -87,6 +91,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
ex.Message,
httpStatusCode: ex.StatusCode);
}
// Feed the central auth-failure tracker so the agent can shut down cleanly
// after MaxAuthFailures / AuthFailureTimeoutMinutes is exceeded, instead of
// hammering the backend forever when the device certificate is revoked.
_authFailureTracker?.RecordFailure(ex.StatusCode, "agent/config");
}
catch (Exception ex)
{
@@ -0,0 +1,142 @@
using System;
using System.Threading;
using AutopilotMonitor.Agent.V2.Core.Logging;
using AutopilotMonitor.DecisionCore.Engine;
namespace AutopilotMonitor.Agent.V2.Core.Security
{
/// <summary>
/// Tracks consecutive 401/403 responses from the backend. When either the consecutive-count
/// ceiling (<c>MaxAuthFailures</c>) or the elapsed-time ceiling (<c>AuthFailureTimeoutMinutes</c>)
/// is exceeded, <see cref="ThresholdExceeded"/> fires exactly once so <c>Program.RunAgent</c>
/// can trigger a soft shutdown.
/// <para>
/// Without this tracker a device whose certificate is revoked (or whose tenant has been deleted)
/// would retry the config fetch and every telemetry upload indefinitely, flooding the backend
/// distress channel and the local log. The tracker is the missing enforcement for the
/// <c>MaxAuthFailures</c> and <c>AuthFailureTimeoutMinutes</c> knobs on
/// <c>AgentConfigResponse</c> (defaults: 5 consecutive, time window disabled).
/// </para>
/// <para>
/// Thread-safety: consecutive-count uses <see cref="Interlocked"/>; the first-failure timestamp
/// is protected by a single monitor lock. <see cref="ThresholdExceeded"/> is raised outside the
/// lock to prevent handler re-entrancy deadlocks.
/// </para>
/// </summary>
public sealed class AuthFailureTracker
{
private readonly IClock _clock;
private readonly AgentLogger _logger;
private readonly object _windowLock = new object();
private int _maxFailures; // 0 = disabled
private TimeSpan? _timeoutWindow; // null = disabled
private int _consecutiveFailures;
private DateTime? _firstFailureUtc;
private int _thresholdFired; // 0/1 — Interlocked exchange makes the event single-shot
public AuthFailureTracker(
int maxFailures,
int timeoutMinutes,
IClock clock,
AgentLogger logger)
{
_clock = clock ?? throw new ArgumentNullException(nameof(clock));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
UpdateLimits(maxFailures, timeoutMinutes);
}
/// <summary>Fires once when either threshold is crossed. Listener is responsible for shutting the agent down.</summary>
public event EventHandler<AuthFailureThresholdEventArgs> ThresholdExceeded;
/// <summary>Updates the limits from the merged remote config. Safe to call at any time; does not reset the counter.</summary>
public void UpdateLimits(int maxFailures, int timeoutMinutes)
{
_maxFailures = maxFailures < 0 ? 0 : maxFailures;
_timeoutWindow = timeoutMinutes > 0 ? (TimeSpan?)TimeSpan.FromMinutes(timeoutMinutes) : null;
}
/// <summary>Current consecutive-failure count. Exposed for observability and tests.</summary>
public int ConsecutiveFailures => Volatile.Read(ref _consecutiveFailures);
/// <summary>Reset counter + window anchor after a successful authenticated response.</summary>
public void RecordSuccess()
{
if (Interlocked.Exchange(ref _consecutiveFailures, 0) == 0) return; // no-op if already zero
lock (_windowLock) { _firstFailureUtc = null; }
}
/// <summary>
/// Record a 401/403 from the given <paramref name="operation"/>. Emits <see cref="ThresholdExceeded"/>
/// the first time either ceiling is crossed; subsequent calls after termination is armed are no-ops.
/// </summary>
public void RecordFailure(int statusCode, string operation)
{
if (Volatile.Read(ref _thresholdFired) == 1) return;
var count = Interlocked.Increment(ref _consecutiveFailures);
DateTime now = _clock.UtcNow;
DateTime firstFailureAt;
lock (_windowLock)
{
if (_firstFailureUtc == null) _firstFailureUtc = now;
firstFailureAt = _firstFailureUtc.Value;
}
bool countExceeded = _maxFailures > 0 && count >= _maxFailures;
bool windowExceeded = _timeoutWindow.HasValue && (now - firstFailureAt) >= _timeoutWindow.Value;
if (!countExceeded && !windowExceeded)
{
_logger.Warning($"AuthFailureTracker: consecutive auth failure #{count} (http {statusCode}, {operation}).");
return;
}
if (Interlocked.Exchange(ref _thresholdFired, 1) == 1) return; // another thread won the race
var reason = countExceeded
? $"consecutive auth failures reached limit ({count} >= {_maxFailures})"
: $"auth-failure time window exceeded ({(now - firstFailureAt).TotalMinutes:F0}min >= {_timeoutWindow.Value.TotalMinutes:F0}min)";
_logger.Error($"AuthFailureTracker: {reason} — signalling shutdown. Last operation: {operation}, statusCode={statusCode}.");
try
{
ThresholdExceeded?.Invoke(this, new AuthFailureThresholdEventArgs(
consecutiveFailures: count,
firstFailureUtc: firstFailureAt,
lastOperation: operation,
lastStatusCode: statusCode,
reason: reason));
}
catch (Exception ex)
{
_logger.Warning($"AuthFailureTracker: ThresholdExceeded handler threw: {ex.Message}");
}
}
}
public sealed class AuthFailureThresholdEventArgs : EventArgs
{
public AuthFailureThresholdEventArgs(
int consecutiveFailures,
DateTime firstFailureUtc,
string lastOperation,
int lastStatusCode,
string reason)
{
ConsecutiveFailures = consecutiveFailures;
FirstFailureUtc = firstFailureUtc;
LastOperation = lastOperation;
LastStatusCode = lastStatusCode;
Reason = reason;
}
public int ConsecutiveFailures { get; }
public DateTime FirstFailureUtc { get; }
public string LastOperation { get; }
public int LastStatusCode { get; }
public string Reason { get; }
}
}
@@ -0,0 +1,122 @@
using System;
using System.IO;
using System.Reflection;
using System.Security.Cryptography;
using System.Text;
using AutopilotMonitor.Agent.V2.Core.Logging;
namespace AutopilotMonitor.Agent.V2.Core.Security
{
/// <summary>
/// Post-config binary-integrity check. After the remote config is fetched, the agent compares
/// the SHA-256 of its running <c>.exe</c> against <c>AgentConfigResponse.LatestAgentExeSha256</c>.
/// A mismatch means one of:
/// <list type="bullet">
/// <item>tampering — the on-disk binary was modified after deployment,</item>
/// <item>stale blob storage — the agent is running an older EXE than what the backend advertises,</item>
/// <item>self-update failed silently — a previous update swapped the ZIP but not the EXE.</item>
/// </list>
/// <para>
/// We intentionally do <b>not</b> terminate or auto-update from this check: <see cref="SelfUpdater"/>
/// already runs at startup and its hash-based trigger covers the "wrong EXE" case. All we do
/// here is emit an <c>AgentErrorType.IntegrityCheckFailed</c> report so operators see the drift.
/// </para>
/// </summary>
public static class BinaryIntegrityVerifier
{
/// <summary>
/// Computes the SHA-256 of <paramref name="exePath"/> (or the currently executing assembly
/// when <c>null</c>) and compares it against <paramref name="expectedSha256"/>.
/// Case-insensitive comparison (both lowercase and uppercase hex strings accepted).
/// </summary>
public static IntegrityCheckResult Check(
string expectedSha256,
AgentLogger logger,
string exePath = null)
{
if (logger == null) throw new ArgumentNullException(nameof(logger));
if (string.IsNullOrWhiteSpace(expectedSha256))
{
logger.Debug("BinaryIntegrityVerifier: no expected hash provided — skipping check.");
return IntegrityCheckResult.Skipped();
}
var path = exePath ?? Assembly.GetExecutingAssembly().Location;
if (string.IsNullOrEmpty(path) || !File.Exists(path))
{
logger.Warning($"BinaryIntegrityVerifier: executing assembly path not resolvable ({path ?? "null"}) — skipping check.");
return IntegrityCheckResult.Skipped();
}
string actualSha;
try
{
actualSha = ComputeSha256Hex(path);
}
catch (Exception ex)
{
logger.Warning($"BinaryIntegrityVerifier: failed to hash '{path}': {ex.Message}");
return IntegrityCheckResult.Skipped();
}
if (string.Equals(actualSha, expectedSha256, StringComparison.OrdinalIgnoreCase))
{
logger.Debug($"BinaryIntegrityVerifier: SHA-256 match (sha={Truncate(actualSha)}).");
return IntegrityCheckResult.Match(actualSha);
}
logger.Warning(
$"BinaryIntegrityVerifier: SHA-256 MISMATCH — running exe={Truncate(actualSha)}, backend advertises={Truncate(expectedSha256)}. " +
"Possible tamper, stale blob, or failed self-update.");
return IntegrityCheckResult.Mismatch(actualSha, expectedSha256);
}
internal static string ComputeSha256Hex(string filePath)
{
using (var stream = File.OpenRead(filePath))
using (var sha = SHA256.Create())
{
var bytes = sha.ComputeHash(stream);
var sb = new StringBuilder(bytes.Length * 2);
for (int i = 0; i < bytes.Length; i++) sb.Append(bytes[i].ToString("x2"));
return sb.ToString();
}
}
private static string Truncate(string hex)
=> string.IsNullOrEmpty(hex) ? "(null)" : (hex.Length <= 12 ? hex : hex.Substring(0, 12) + "...");
}
public sealed class IntegrityCheckResult
{
public IntegrityCheckOutcome Outcome { get; }
public string ActualSha256 { get; }
public string ExpectedSha256 { get; }
private IntegrityCheckResult(IntegrityCheckOutcome outcome, string actual, string expected)
{
Outcome = outcome;
ActualSha256 = actual;
ExpectedSha256 = expected;
}
public static IntegrityCheckResult Skipped() => new IntegrityCheckResult(IntegrityCheckOutcome.Skipped, null, null);
public static IntegrityCheckResult Match(string actual) => new IntegrityCheckResult(IntegrityCheckOutcome.Match, actual, actual);
public static IntegrityCheckResult Mismatch(string actual, string expected) => new IntegrityCheckResult(IntegrityCheckOutcome.Mismatch, actual, expected);
public bool IsMismatch => Outcome == IntegrityCheckOutcome.Mismatch;
}
public enum IntegrityCheckOutcome
{
/// <summary>No expected hash provided, or the local path could not be hashed.</summary>
Skipped = 0,
/// <summary>SHA-256 of the running EXE matches the backend-advertised hash.</summary>
Match = 1,
/// <summary>SHA-256 differs — tamper / stale blob / failed self-update.</summary>
Mismatch = 2,
}
}
@@ -9,6 +9,7 @@ using System.Net.Http.Headers;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Agent.V2.Core.Security;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.Models;
using Newtonsoft.Json;
@@ -46,6 +47,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
private readonly string? _serialNumber;
private readonly string? _bootstrapToken;
private readonly string? _agentVersion;
private readonly AuthFailureTracker? _authFailureTracker;
public BackendTelemetryUploader(
HttpClient httpClient,
@@ -55,7 +57,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
string? model = null,
string? serialNumber = null,
string? bootstrapToken = null,
string? agentVersion = null)
string? agentVersion = null,
AuthFailureTracker? authFailureTracker = null)
{
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
if (string.IsNullOrEmpty(baseUrl)) throw new ArgumentException("BaseUrl is mandatory.", nameof(baseUrl));
@@ -68,6 +71,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
_serialNumber = serialNumber;
_bootstrapToken = bootstrapToken;
_agentVersion = agentVersion;
_authFailureTracker = authFailureTracker;
}
public async Task<UploadResult> UploadBatchAsync(
@@ -124,12 +128,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
}
}
private static async Task<UploadResult> MapResponseAsync(HttpResponseMessage response)
private async Task<UploadResult> MapResponseAsync(HttpResponseMessage response)
{
var statusCode = (int)response.StatusCode;
if (response.IsSuccessStatusCode)
{
// Auth was accepted — reset the consecutive-failure counter so a transient
// backend hiccup does not eventually shut the agent down.
_authFailureTracker?.RecordSuccess();
// M4.6.ε — parse backend-to-agent control signals from the 2xx response body.
// Body is best-effort JSON; anything unparseable degrades cleanly to plain Ok().
return await TryReadControlSignalsAsync(response).ConfigureAwait(false);
@@ -142,6 +150,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
{
case HttpStatusCode.Unauthorized: // 401
case HttpStatusCode.Forbidden: // 403
// Feed the central tracker so the agent can shut down after MaxAuthFailures
// instead of retrying a permanently-revoked certificate forever.
_authFailureTracker?.RecordFailure(statusCode, "agent/telemetry");
return UploadResult.Permanent($"unauthorized: {shortReason}");
case HttpStatusCode.RequestTimeout: // 408
+39 -2
View File
@@ -306,8 +306,18 @@ namespace AutopilotMonitor.Agent.V2
"MDM certificate not found in LocalMachine or CurrentUser store");
}
// Central observer for consecutive 401/403 responses. Initialised with the CLI/bootstrap
// defaults on AgentConfiguration; UpdateLimits is called after RemoteConfigMerger.Merge
// so tenant-policy overrides take effect. ThresholdExceeded is wired below, once the
// shutdown signal is available, to trigger a clean agent exit when the limit is hit.
var authFailureTracker = new AuthFailureTracker(
maxFailures: agentConfig.MaxAuthFailures,
timeoutMinutes: agentConfig.AuthFailureTimeoutMinutes,
clock: SystemClock.Instance,
logger: logger);
var remoteConfigService = new RemoteConfigService(
backendApiClient, agentConfig.TenantId, logger, emergencyReporter, distressReporter);
backendApiClient, agentConfig.TenantId, logger, emergencyReporter, distressReporter, authFailureTracker);
var remoteConfig = remoteConfigService.FetchConfigAsync().GetAwaiter().GetResult();
// Project remote tenant-controlled knobs onto the runtime AgentConfiguration so that
@@ -316,6 +326,9 @@ namespace AutopilotMonitor.Agent.V2
// actually respect the tenant admin settings. CLI flags stay authoritative over remote.
RemoteConfigMerger.Merge(agentConfig, remoteConfig, args);
// Refresh tracker ceilings with the tenant-specific values we just merged in.
authFailureTracker.UpdateLimits(agentConfig.MaxAuthFailures, agentConfig.AuthFailureTimeoutMinutes);
logger.SetLogLevel(agentConfig.LogLevel);
// Propagate the backend-expected SHA so the runtime hash-mismatch trigger (M4.6.α
@@ -324,6 +337,18 @@ namespace AutopilotMonitor.Agent.V2
if (!string.IsNullOrEmpty(remoteConfig.LatestAgentSha256))
SelfUpdater.BackendExpectedSha256 = remoteConfig.LatestAgentSha256;
// Post-config binary-integrity check: verify the running EXE's SHA-256 against the
// value advertised by the backend. Mismatch does not terminate (SelfUpdater's own
// hash-based trigger covers the "wrong EXE" case at next startup) — we just emit an
// IntegrityCheckFailed error report so operators see the drift.
var integrityResult = BinaryIntegrityVerifier.Check(remoteConfig.LatestAgentExeSha256, logger);
if (integrityResult.IsMismatch)
{
_ = emergencyReporter.TrySendAsync(
AgentErrorType.IntegrityCheckFailed,
$"Running exe SHA-256 differs from backend-advertised hash. actual={integrityResult.ActualSha256}, expected={integrityResult.ExpectedSha256}");
}
// H-2 mitigation: delete the persisted bootstrap-config.json once the MDM cert
// proves it can authenticate. Non-blocking — any failure leaves the file for retry.
try
@@ -361,7 +386,8 @@ namespace AutopilotMonitor.Agent.V2
model: hardwareForReporters.Model,
serialNumber: hardwareForReporters.SerialNumber,
bootstrapToken: agentConfig.UseBootstrapTokenAuth ? agentConfig.BootstrapToken : null,
agentVersion: GetAgentVersion());
agentVersion: GetAgentVersion(),
authFailureTracker: authFailureTracker);
}
catch (Exception ex)
{
@@ -498,6 +524,16 @@ namespace AutopilotMonitor.Agent.V2
AppDomain.CurrentDomain.ProcessExit += processExitHandler;
orchestrator.Terminated += terminationHandler.Handle;
// Auth-failure watchdog: when MaxAuthFailures / AuthFailureTimeoutMinutes
// is exceeded the agent must shut down cleanly instead of hammering a
// backend that has definitely said no. Event fires at most once.
EventHandler<AuthFailureThresholdEventArgs> authThresholdHandler = (s, e) =>
{
logger.Error($"Auth-failure threshold exceeded ({e.Reason}) — initiating shutdown.");
shutdown.Set();
};
authFailureTracker.ThresholdExceeded += authThresholdHandler;
try
{
orchestrator.Start();
@@ -550,6 +586,7 @@ namespace AutopilotMonitor.Agent.V2
Console.CancelKeyPress -= cancelHandler;
AppDomain.CurrentDomain.ProcessExit -= processExitHandler;
orchestrator.Terminated -= terminationHandler.Handle;
authFailureTracker.ThresholdExceeded -= authThresholdHandler;
try { orchestrator.Stop(); }
catch (Exception ex) { logger.Error("Orchestrator stop failed.", ex); }