diff --git a/src/Agent/AutopilotMonitor.Agent/buildcounter.txt b/src/Agent/AutopilotMonitor.Agent/buildcounter.txt index 67adbdb3..f8c40655 100644 --- a/src/Agent/AutopilotMonitor.Agent/buildcounter.txt +++ b/src/Agent/AutopilotMonitor.Agent/buildcounter.txt @@ -1 +1 @@ -724 +725 diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/EventTimestampValidationTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/EventTimestampValidationTests.cs new file mode 100644 index 00000000..bd3eeb54 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/EventTimestampValidationTests.cs @@ -0,0 +1,542 @@ +using AutopilotMonitor.Functions.Functions.Ingest; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Shared.Models; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Tests for timestamp validation and sanitization in the ingest pipeline. +/// +/// REGRESSION GUARD: Invalid agent-side timestamps (DateTime.MinValue, far-future dates, +/// clock-skewed values) previously flowed through to RowKey generation, duration calculations, +/// and Azure Table Storage writes without any validation — causing production issues. +/// +/// EventTimestampValidator clamps out-of-range timestamps to a safe range while preserving +/// the original value in OriginalTimestamp for troubleshooting. +/// +public class EventTimestampValidationTests +{ + // Fixed reference time for deterministic tests + private static readonly DateTime FixedUtcNow = new(2026, 3, 30, 12, 0, 0, DateTimeKind.Utc); + + private static readonly string ValidTenantId = "a1b2c3d4-e5f6-7890-abcd-ef1234567890"; + private static readonly string ValidSessionId = "b2c3d4e5-f6a7-8901-bcde-f12345678901"; + + // ========================================================================= + // SanitizeTimestamp + // ========================================================================= + + [Fact] + public void SanitizeTimestamp_ValidRecentUtcTimestamp_ReturnsSameValue() + { + var valid = new DateTime(2026, 3, 15, 10, 30, 0, DateTimeKind.Utc); + + var result = EventTimestampValidator.SanitizeTimestamp(valid, FixedUtcNow); + + Assert.Equal(valid, result); + } + + [Fact] + public void SanitizeTimestamp_DateTimeMinValue_ClampsToMinReasonable() + { + var result = EventTimestampValidator.SanitizeTimestamp(DateTime.MinValue, FixedUtcNow); + + Assert.Equal(EventTimestampValidator.MinReasonableTimestamp, result); + } + + [Fact] + public void SanitizeTimestamp_DateTimeMaxValue_ClampsToUtcNow() + { + var result = EventTimestampValidator.SanitizeTimestamp(DateTime.MaxValue, FixedUtcNow); + + Assert.Equal(FixedUtcNow, result); + } + + [Fact] + public void SanitizeTimestamp_FarPast1999_ClampsToMinReasonable() + { + var farPast = new DateTime(1999, 6, 15, 0, 0, 0, DateTimeKind.Utc); + + var result = EventTimestampValidator.SanitizeTimestamp(farPast, FixedUtcNow); + + Assert.Equal(EventTimestampValidator.MinReasonableTimestamp, result); + } + + [Fact] + public void SanitizeTimestamp_FarFuture2099_ClampsToUtcNow() + { + var farFuture = new DateTime(2099, 12, 31, 23, 59, 59, DateTimeKind.Utc); + + var result = EventTimestampValidator.SanitizeTimestamp(farFuture, FixedUtcNow); + + Assert.Equal(FixedUtcNow, result); + } + + [Fact] + public void SanitizeTimestamp_SlightFutureWithin24h_PassesThrough() + { + // Agent clock is 2 hours ahead — within tolerance + var slightFuture = FixedUtcNow.AddHours(2); + + var result = EventTimestampValidator.SanitizeTimestamp(slightFuture, FixedUtcNow); + + Assert.Equal(slightFuture, result); + } + + [Fact] + public void SanitizeTimestamp_ExactlyAt24hBoundary_PassesThrough() + { + var boundary = FixedUtcNow.AddHours(24); + + var result = EventTimestampValidator.SanitizeTimestamp(boundary, FixedUtcNow); + + Assert.Equal(boundary, result); + } + + [Fact] + public void SanitizeTimestamp_25HoursInFuture_ClampsToUtcNow() + { + var tooFar = FixedUtcNow.AddHours(25); + + var result = EventTimestampValidator.SanitizeTimestamp(tooFar, FixedUtcNow); + + Assert.Equal(FixedUtcNow, result); + } + + [Fact] + public void SanitizeTimestamp_LocalKind_ConvertsToUtcThenValidates() + { + // A recent Local kind timestamp — should be converted to UTC, then pass validation + var local = new DateTime(2026, 3, 15, 10, 0, 0, DateTimeKind.Local); + var expectedUtc = local.ToUniversalTime(); + + var result = EventTimestampValidator.SanitizeTimestamp(local, FixedUtcNow); + + Assert.Equal(DateTimeKind.Utc, result.Kind); + Assert.Equal(expectedUtc, result); + } + + [Fact] + public void SanitizeTimestamp_UnspecifiedKind_TreatedAsUtc() + { + var unspecified = new DateTime(2026, 3, 15, 10, 0, 0, DateTimeKind.Unspecified); + + var result = EventTimestampValidator.SanitizeTimestamp(unspecified, FixedUtcNow); + + Assert.Equal(DateTimeKind.Utc, result.Kind); + Assert.Equal(2026, result.Year); + Assert.Equal(3, result.Month); + Assert.Equal(15, result.Day); + } + + [Fact] + public void SanitizeTimestamp_ExactMinBound_PassesThrough() + { + var result = EventTimestampValidator.SanitizeTimestamp( + EventTimestampValidator.MinReasonableTimestamp, FixedUtcNow); + + Assert.Equal(EventTimestampValidator.MinReasonableTimestamp, result); + } + + // ========================================================================= + // IsReasonableTimestamp + // ========================================================================= + + [Fact] + public void IsReasonableTimestamp_ValidDate_ReturnsTrue() + { + var valid = new DateTime(2026, 3, 15, 10, 0, 0, DateTimeKind.Utc); + + Assert.True(EventTimestampValidator.IsReasonableTimestamp(valid, FixedUtcNow)); + } + + [Fact] + public void IsReasonableTimestamp_MinValue_ReturnsFalse() + { + Assert.False(EventTimestampValidator.IsReasonableTimestamp(DateTime.MinValue, FixedUtcNow)); + } + + [Fact] + public void IsReasonableTimestamp_MaxValue_ReturnsFalse() + { + Assert.False(EventTimestampValidator.IsReasonableTimestamp(DateTime.MaxValue, FixedUtcNow)); + } + + [Fact] + public void IsReasonableTimestamp_FarFuture_ReturnsFalse() + { + var farFuture = new DateTime(2099, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + Assert.False(EventTimestampValidator.IsReasonableTimestamp(farFuture, FixedUtcNow)); + } + + [Fact] + public void IsReasonableTimestamp_JustBeforeMin_ReturnsFalse() + { + var justBefore = new DateTime(2019, 12, 31, 23, 59, 59, DateTimeKind.Utc); + + Assert.False(EventTimestampValidator.IsReasonableTimestamp(justBefore, FixedUtcNow)); + } + + // ========================================================================= + // SafeDurationSeconds + // ========================================================================= + + [Fact] + public void SafeDurationSeconds_NormalRange_ReturnsCorrectDuration() + { + var start = new DateTime(2026, 3, 30, 10, 0, 0, DateTimeKind.Utc); + var end = start.AddSeconds(300); + + Assert.Equal(300, EventTimestampValidator.SafeDurationSeconds(start, end)); + } + + [Fact] + public void SafeDurationSeconds_EndBeforeStart_ReturnsZero() + { + var start = new DateTime(2026, 3, 30, 10, 5, 0, DateTimeKind.Utc); + var end = new DateTime(2026, 3, 30, 10, 0, 0, DateTimeKind.Utc); + + Assert.Equal(0, EventTimestampValidator.SafeDurationSeconds(start, end)); + } + + [Fact] + public void SafeDurationSeconds_MinValueToMaxValue_ClampsToMax() + { + // This would overflow int if not clamped: ~315 billion seconds + var result = EventTimestampValidator.SafeDurationSeconds(DateTime.MinValue, DateTime.MaxValue); + + Assert.Equal(EventTimestampValidator.DefaultMaxDurationSeconds, result); + } + + [Fact] + public void SafeDurationSeconds_SameTimestamp_ReturnsZero() + { + var ts = new DateTime(2026, 3, 30, 10, 0, 0, DateTimeKind.Utc); + + Assert.Equal(0, EventTimestampValidator.SafeDurationSeconds(ts, ts)); + } + + [Fact] + public void SafeDurationSeconds_ExceedsMaxDuration_ClampsToMax() + { + var start = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var end = new DateTime(2026, 2, 1, 0, 0, 0, DateTimeKind.Utc); // 31 days apart + + var result = EventTimestampValidator.SafeDurationSeconds(start, end); + + Assert.Equal(EventTimestampValidator.DefaultMaxDurationSeconds, result); // 7 days max + } + + [Fact] + public void SafeDurationSeconds_CustomMaxDuration_Respected() + { + var start = new DateTime(2026, 3, 30, 10, 0, 0, DateTimeKind.Utc); + var end = start.AddSeconds(500); + + var result = EventTimestampValidator.SafeDurationSeconds(start, end, maxDurationSeconds: 100); + + Assert.Equal(100, result); + } + + // ========================================================================= + // SafeRowKeyTimestamp + // ========================================================================= + + [Fact] + public void SafeRowKeyTimestamp_ValidDate_FormatsCorrectly() + { + var ts = new DateTime(2026, 3, 30, 14, 25, 30, 123, DateTimeKind.Utc); + + var result = EventTimestampValidator.SafeRowKeyTimestamp(ts, FixedUtcNow); + + Assert.Equal("20260330142530123", result); + } + + [Fact] + public void SafeRowKeyTimestamp_MinValue_ProducesMinBoundKey() + { + // DateTime.MinValue must be clamped — never produce "00010101000000000" + var result = EventTimestampValidator.SafeRowKeyTimestamp(DateTime.MinValue, FixedUtcNow); + + Assert.Equal("20200101000000000", result); + Assert.DoesNotContain("0001", result); + } + + [Fact] + public void SafeRowKeyTimestamp_MaxValue_ProducesReasonableKey() + { + // DateTime.MaxValue must be clamped — never produce "99991231235959999" + var result = EventTimestampValidator.SafeRowKeyTimestamp(DateTime.MaxValue, FixedUtcNow); + + Assert.DoesNotContain("9999", result); + Assert.StartsWith("2026", result); // Clamped to FixedUtcNow year + } + + // ========================================================================= + // SanitizeEventTimestamps (pipeline integration) + // ========================================================================= + + [Fact] + public void SanitizeEventTimestamps_ValidTimestamp_NoFlagsSet() + { + var events = new List + { + new() { Timestamp = new DateTime(2026, 3, 15, 10, 0, 0, DateTimeKind.Utc) } + }; + + IngestEventsFunction.SanitizeEventTimestamps(events, FixedUtcNow); + + Assert.Null(events[0].OriginalTimestamp); + Assert.False(events[0].TimestampClamped); + } + + [Fact] + public void SanitizeEventTimestamps_InvalidTimestamp_PreservesOriginalAndSetsFlag() + { + var badTimestamp = DateTime.MinValue; + var events = new List + { + new() { Timestamp = badTimestamp } + }; + + IngestEventsFunction.SanitizeEventTimestamps(events, FixedUtcNow); + + Assert.True(events[0].TimestampClamped); + Assert.Equal(badTimestamp, events[0].OriginalTimestamp); + Assert.Equal(EventTimestampValidator.MinReasonableTimestamp, events[0].Timestamp); + } + + [Fact] + public void SanitizeEventTimestamps_MixedValidAndInvalid_OnlyInvalidGetFlag() + { + var validTs = new DateTime(2026, 3, 15, 10, 0, 0, DateTimeKind.Utc); + var events = new List + { + new() { Timestamp = validTs, EventType = "valid_event" }, + new() { Timestamp = DateTime.MinValue, EventType = "bad_past_event" }, + new() { Timestamp = new DateTime(2099, 1, 1, 0, 0, 0, DateTimeKind.Utc), EventType = "bad_future_event" }, + }; + + IngestEventsFunction.SanitizeEventTimestamps(events, FixedUtcNow); + + // Valid event: unchanged + Assert.False(events[0].TimestampClamped); + Assert.Null(events[0].OriginalTimestamp); + Assert.Equal(validTs, events[0].Timestamp); + + // Bad past: clamped to min, original preserved + Assert.True(events[1].TimestampClamped); + Assert.Equal(DateTime.MinValue, events[1].OriginalTimestamp); + Assert.Equal(EventTimestampValidator.MinReasonableTimestamp, events[1].Timestamp); + + // Bad future: clamped to utcNow, original preserved + Assert.True(events[2].TimestampClamped); + Assert.Equal(new DateTime(2099, 1, 1, 0, 0, 0, DateTimeKind.Utc), events[2].OriginalTimestamp); + Assert.Equal(FixedUtcNow, events[2].Timestamp); + } + + [Fact] + public void SanitizeEventTimestamps_NoEventsDropped() + { + var events = new List + { + new() { Timestamp = DateTime.MinValue }, + new() { Timestamp = DateTime.MaxValue }, + new() { Timestamp = new DateTime(2026, 3, 15, 10, 0, 0, DateTimeKind.Utc) }, + }; + + IngestEventsFunction.SanitizeEventTimestamps(events, FixedUtcNow); + + Assert.Equal(3, events.Count); // All events preserved + } + + [Fact] + public void SanitizeEventTimestamps_EmptyList_DoesNotThrow() + { + var ex = Record.Exception(() => + IngestEventsFunction.SanitizeEventTimestamps(new List(), FixedUtcNow)); + Assert.Null(ex); + } + + // ========================================================================= + // RecalculateAppDurations (exposed as internal static for testing) + // ========================================================================= + + [Fact] + public void RecalculateAppDurations_NormalInstall_CorrectDuration() + { + var start = new DateTime(2026, 3, 30, 10, 0, 0, DateTimeKind.Utc); + var end = start.AddSeconds(60); + + var state = new AppInstallAggregationState + { + Summary = new AppInstallSummary + { + AppName = "TestApp", + StartedAt = start, + CompletedAt = end, + Status = "Succeeded" + } + }; + + IngestEventsFunction.RecalculateAppDurations(state); + + Assert.Equal(60, state.Summary.DurationSeconds); + } + + [Fact] + public void RecalculateAppDurations_DownloadThenInstall_CorrectDownloadDuration() + { + var downloadStart = new DateTime(2026, 3, 30, 10, 0, 0, DateTimeKind.Utc); + var installStart = downloadStart.AddSeconds(30); + var completed = installStart.AddSeconds(45); + + var state = new AppInstallAggregationState + { + DownloadStartedAt = downloadStart, + InstallStartedAt = installStart, + Summary = new AppInstallSummary + { + AppName = "TestApp", + StartedAt = downloadStart, + CompletedAt = completed, + Status = "Succeeded" + } + }; + + IngestEventsFunction.RecalculateAppDurations(state); + + Assert.Equal(30, state.Summary.DownloadDurationSeconds); // download→install gap + Assert.Equal(75, state.Summary.DurationSeconds); // full duration + } + + [Fact] + public void RecalculateAppDurations_ExtremeTimestampGap_DurationClamped() + { + // Even if timestamps somehow bypass sanitization, SafeDurationSeconds prevents overflow + var start = new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var end = new DateTime(2026, 12, 31, 23, 59, 59, DateTimeKind.Utc); // ~7 years apart + + var state = new AppInstallAggregationState + { + Summary = new AppInstallSummary + { + AppName = "TestApp", + StartedAt = start, + CompletedAt = end, + Status = "Succeeded" + } + }; + + IngestEventsFunction.RecalculateAppDurations(state); + + // Should be clamped to max 7 days (604800 seconds), not ~220 million seconds + Assert.Equal(EventTimestampValidator.DefaultMaxDurationSeconds, state.Summary.DurationSeconds); + } + + // ========================================================================= + // End-to-End Pipeline: NDJSON parse → stamp → sanitize → safe for storage + // ========================================================================= + + [Fact] + public void FullPipeline_ParseStampSanitize_TimestampsAreSafe() + { + // Simulate NDJSON payload with events containing bad timestamps + var ndjsonLines = new[] + { + Newtonsoft.Json.JsonConvert.SerializeObject( + new { SessionId = ValidSessionId, TenantId = ValidTenantId }), + Newtonsoft.Json.JsonConvert.SerializeObject( + new EnrollmentEvent + { + EventType = "phase_changed", + Timestamp = new DateTime(2026, 3, 15, 10, 0, 0, DateTimeKind.Utc) // valid + }), + Newtonsoft.Json.JsonConvert.SerializeObject( + new EnrollmentEvent + { + EventType = "bad_clock_event", + Timestamp = DateTime.MinValue // invalid + }), + }; + var ndjson = string.Join('\n', ndjsonLines); + + // Step 1: Parse + var (sessionId, tenantId, events) = NdjsonParser.ParseNdjson(ndjson); + + // Step 2: Stamp server fields + IngestEventsFunction.StampServerFields(events, tenantId, sessionId, FixedUtcNow); + + // Step 3: Sanitize timestamps + IngestEventsFunction.SanitizeEventTimestamps(events, FixedUtcNow); + + // Verify: all timestamps are within valid range + foreach (var evt in events) + { + Assert.True(EventTimestampValidator.IsReasonableTimestamp(evt.Timestamp, FixedUtcNow), + $"Event {evt.EventType} has unreasonable timestamp: {evt.Timestamp}"); + } + + // Verify: no events were dropped + Assert.Equal(2, events.Count); + } + + [Fact] + public void FullPipeline_MinValueTimestamp_ClampedWithOriginalPreserved() + { + var events = new List + { + new() + { + EventType = "bad_event", + Timestamp = DateTime.MinValue, + TenantId = ValidTenantId, + SessionId = ValidSessionId + } + }; + + IngestEventsFunction.StampServerFields(events, ValidTenantId, ValidSessionId, FixedUtcNow); + IngestEventsFunction.SanitizeEventTimestamps(events, FixedUtcNow); + + var evt = events[0]; + + // Clamped timestamp produces valid RowKey format + var rowKey = $"{evt.Timestamp:yyyyMMddHHmmssfff}_{evt.Sequence:D10}"; + Assert.StartsWith("2020", rowKey); // Clamped to MinReasonableTimestamp + Assert.DoesNotContain("0001", rowKey); + + // Original preserved for troubleshooting + Assert.True(evt.TimestampClamped); + Assert.Equal(DateTime.MinValue, evt.OriginalTimestamp); + } + + [Fact] + public void FullPipeline_MixedTimestamps_AllEventsPreservedFlagsCorrect() + { + var validTs = new DateTime(2026, 3, 25, 8, 0, 0, DateTimeKind.Utc); + var events = new List + { + new() { EventType = "valid", Timestamp = validTs }, + new() { EventType = "past", Timestamp = new DateTime(1990, 1, 1, 0, 0, 0, DateTimeKind.Utc) }, + new() { EventType = "future", Timestamp = new DateTime(2099, 6, 1, 0, 0, 0, DateTimeKind.Utc) }, + new() { EventType = "also_valid", Timestamp = FixedUtcNow.AddHours(1) }, + }; + + IngestEventsFunction.StampServerFields(events, ValidTenantId, ValidSessionId, FixedUtcNow); + IngestEventsFunction.SanitizeEventTimestamps(events, FixedUtcNow); + + // All 4 events preserved + Assert.Equal(4, events.Count); + + // Valid: no flags + Assert.False(events[0].TimestampClamped); + Assert.False(events[3].TimestampClamped); + + // Invalid: flags set, originals preserved + Assert.True(events[1].TimestampClamped); + Assert.True(events[2].TimestampClamped); + Assert.NotNull(events[1].OriginalTimestamp); + Assert.NotNull(events[2].OriginalTimestamp); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestEventsFunction.Helpers.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestEventsFunction.Helpers.cs index b2fe68fc..31fd7148 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestEventsFunction.Helpers.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestEventsFunction.Helpers.cs @@ -95,14 +95,14 @@ namespace AutopilotMonitor.Functions.Functions.Ingest summary.Status = "Succeeded"; summary.CompletedAt = evt.Timestamp; if (summary.StartedAt != DateTime.MinValue) - summary.DurationSeconds = Math.Max(1, (int)(evt.Timestamp - summary.StartedAt).TotalSeconds); + summary.DurationSeconds = Math.Max(1, EventTimestampValidator.SafeDurationSeconds(summary.StartedAt, evt.Timestamp)); break; case "app_install_failed": summary.Status = "Failed"; summary.CompletedAt = evt.Timestamp; if (summary.StartedAt != DateTime.MinValue) - summary.DurationSeconds = Math.Max(1, (int)(evt.Timestamp - summary.StartedAt).TotalSeconds); + summary.DurationSeconds = Math.Max(1, EventTimestampValidator.SafeDurationSeconds(summary.StartedAt, evt.Timestamp)); // Agent does not send errorCode/errorMessage in Data — use the event message summary.FailureCode = evt.Data?.ContainsKey("errorCode") == true ? evt.Data["errorCode"]?.ToString() ?? string.Empty : string.Empty; @@ -158,7 +158,7 @@ namespace AutopilotMonitor.Functions.Functions.Ingest RecalculateAppDurations(state); } - private static void RecalculateAppDurations(AppInstallAggregationState state) + internal static void RecalculateAppDurations(AppInstallAggregationState state) { var summary = state.Summary; @@ -185,14 +185,16 @@ namespace AutopilotMonitor.Functions.Functions.Ingest if (state.DownloadStartedAt.HasValue && state.InstallStartedAt.HasValue && state.InstallStartedAt.Value >= state.DownloadStartedAt.Value) { - summary.DownloadDurationSeconds = (int)(state.InstallStartedAt.Value - state.DownloadStartedAt.Value).TotalSeconds; + summary.DownloadDurationSeconds = EventTimestampValidator.SafeDurationSeconds( + state.DownloadStartedAt.Value, state.InstallStartedAt.Value); } // Full duration: from effective start to completion/failure. if (summary.CompletedAt.HasValue && summary.StartedAt != DateTime.MinValue && summary.CompletedAt.Value >= summary.StartedAt) { - summary.DurationSeconds = (int)(summary.CompletedAt.Value - summary.StartedAt).TotalSeconds; + summary.DurationSeconds = EventTimestampValidator.SafeDurationSeconds( + summary.StartedAt, summary.CompletedAt.Value); } } diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestEventsFunction.Processing.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestEventsFunction.Processing.cs index 95f5c6ae..c9d47cb8 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestEventsFunction.Processing.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestEventsFunction.Processing.cs @@ -128,6 +128,10 @@ namespace AutopilotMonitor.Functions.Functions.Ingest var receivedAt = DateTime.UtcNow; StampServerFields(request.Events, request.TenantId, request.SessionId, receivedAt); + // Sanitize agent-side timestamps: clamp out-of-range values to prevent storage/sorting issues. + // Original timestamps are preserved in OriginalTimestamp for troubleshooting. + SanitizeEventTimestamps(request.Events, receivedAt); + // Store events in Azure Table Storage (batch write for efficiency) var storedEvents = await _sessionRepo.StoreEventsBatchAsync(request.Events); int processedCount = storedEvents.Count; @@ -742,6 +746,27 @@ namespace AutopilotMonitor.Functions.Functions.Ingest evt.SessionId = sessionId; } } + + /// + /// Sanitizes agent-side timestamps on all events by clamping out-of-range values. + /// When a timestamp is clamped, the original value is preserved in OriginalTimestamp + /// and TimestampClamped is set to true — keeping the raw data available for + /// troubleshooting and root-cause analysis of clock issues on devices. + /// Exposed as internal for unit testing. + /// + internal static void SanitizeEventTimestamps(List events, DateTime utcNow) + { + foreach (var evt in events) + { + var sanitized = EventTimestampValidator.SanitizeTimestamp(evt.Timestamp, utcNow); + if (sanitized != evt.Timestamp) + { + evt.OriginalTimestamp = evt.Timestamp; + evt.TimestampClamped = true; + evt.Timestamp = sanitized; + } + } + } } /// diff --git a/src/Backend/AutopilotMonitor.Functions/Services/EventTimestampValidator.cs b/src/Backend/AutopilotMonitor.Functions/Services/EventTimestampValidator.cs new file mode 100644 index 00000000..367a0d43 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/EventTimestampValidator.cs @@ -0,0 +1,96 @@ +using System; + +namespace AutopilotMonitor.Functions.Services +{ + /// + /// Validates and sanitizes event timestamps from agent devices. + /// Timestamps outside the reasonable range are clamped (not rejected) to preserve event data. + /// When clamping occurs, callers should set OriginalTimestamp + TimestampClamped on the event + /// so that the original value remains available for troubleshooting and root-cause analysis. + /// + internal static class EventTimestampValidator + { + /// + /// Earliest reasonable timestamp — the product did not exist before this date. + /// + internal static readonly DateTime MinReasonableTimestamp = new(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + /// + /// Maximum clock skew tolerance for agent-side timestamps ahead of server time. + /// Agents may have slightly fast clocks; 24 hours covers timezone misconfiguration too. + /// + internal const int MaxFutureToleranceHours = 24; + + /// + /// Default maximum duration in seconds (7 days) for SafeDurationSeconds clamping. + /// + internal const int DefaultMaxDurationSeconds = 604800; + + /// + /// Ensures a DateTime has UTC kind. Mirrors the EnsureUtc logic in TableStorageService + /// but is available without an instance. Called internally by SanitizeTimestamp. + /// + internal static DateTime EnsureUtc(DateTime dt) => dt.Kind switch + { + DateTimeKind.Utc => dt, + DateTimeKind.Local => dt.ToUniversalTime(), + _ => DateTime.SpecifyKind(dt, DateTimeKind.Utc) + }; + + /// + /// Checks whether a timestamp falls within the reasonable range [MinReasonableTimestamp, utcNow + 24h]. + /// + internal static bool IsReasonableTimestamp(DateTime timestamp, DateTime utcNow) + { + var utcTs = EnsureUtc(timestamp); + return utcTs >= MinReasonableTimestamp && utcTs <= utcNow.AddHours(MaxFutureToleranceHours); + } + + /// + /// Sanitizes a timestamp by ensuring UTC kind and clamping to the valid range. + /// - Below MinReasonableTimestamp → clamped to MinReasonableTimestamp + /// - Above utcNow + 24h → clamped to utcNow (server receive time as best-effort fallback) + /// Returns the original value unchanged if it is within the valid range. + /// + internal static DateTime SanitizeTimestamp(DateTime timestamp, DateTime utcNow) + { + var utcTs = EnsureUtc(timestamp); + + if (utcTs < MinReasonableTimestamp) + return MinReasonableTimestamp; + + if (utcTs > utcNow.AddHours(MaxFutureToleranceHours)) + return EnsureUtc(utcNow); + + return utcTs; + } + + /// + /// Computes the duration in seconds between two timestamps, clamped to [0, maxDurationSeconds]. + /// Prevents int overflow from extreme timestamp differences (e.g. DateTime.MinValue to DateTime.MaxValue). + /// Returns 0 if end is before start. + /// + internal static int SafeDurationSeconds(DateTime start, DateTime end, int maxDurationSeconds = DefaultMaxDurationSeconds) + { + if (end <= start) + return 0; + + var totalSeconds = (end - start).TotalSeconds; + + if (totalSeconds > maxDurationSeconds) + return maxDurationSeconds; + + return (int)totalSeconds; + } + + /// + /// Produces a safe RowKey timestamp segment by sanitizing the timestamp first, + /// then formatting as yyyyMMddHHmmssfff. + /// + internal static string SafeRowKeyTimestamp(DateTime timestamp, DateTime utcNow) + { + var sanitized = SanitizeTimestamp(timestamp, utcNow); + return sanitized.ToString("yyyyMMddHHmmssfff"); + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs index 954cfa99..1eb18bc3 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs @@ -467,9 +467,13 @@ namespace AutopilotMonitor.Functions.Services ? JsonConvert.SerializeObject(evt.Data) : string.Empty, "DataJson", evt.EventId), - ["ReceivedAt"] = evt.ReceivedAt + ["ReceivedAt"] = evt.ReceivedAt, + ["TimestampClamped"] = evt.TimestampClamped }; + if (evt.OriginalTimestamp.HasValue) + entity["OriginalTimestamp"] = EnsureUtc(evt.OriginalTimestamp.Value); + await tableClient.UpsertEntityAsync(entity); _logger.LogDebug($"Stored event {evt.EventId}"); @@ -540,9 +544,13 @@ namespace AutopilotMonitor.Functions.Services ? JsonConvert.SerializeObject(evt.Data) : string.Empty, "DataJson", evt.EventId), - ["ReceivedAt"] = evt.ReceivedAt + ["ReceivedAt"] = evt.ReceivedAt, + ["TimestampClamped"] = evt.TimestampClamped }; + if (evt.OriginalTimestamp.HasValue) + entity["OriginalTimestamp"] = EnsureUtc(evt.OriginalTimestamp.Value); + return new TableTransactionAction(TableTransactionActionType.UpsertReplace, entity); }).ToList(); @@ -1520,7 +1528,9 @@ namespace AutopilotMonitor.Functions.Services Sequence = entity.GetInt64("Sequence") ?? 0, Data = DeserializeEventData(entity.GetString("DataJson")), RowKey = entity.RowKey, - ReceivedAt = entity.GetDateTimeOffset("ReceivedAt")?.UtcDateTime + ReceivedAt = entity.GetDateTimeOffset("ReceivedAt")?.UtcDateTime, + OriginalTimestamp = entity.GetDateTimeOffset("OriginalTimestamp")?.UtcDateTime, + TimestampClamped = entity.GetBoolean("TimestampClamped") ?? false }; } diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Enrollment/EnrollmentEvent.cs b/src/Shared/AutopilotMonitor.Shared/Models/Enrollment/EnrollmentEvent.cs index c3f5f61a..a39230d6 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/Enrollment/EnrollmentEvent.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/Enrollment/EnrollmentEvent.cs @@ -123,6 +123,20 @@ namespace AutopilotMonitor.Shared.Models /// public long Sequence { get; set; } + /// + /// Original agent-side timestamp preserved when the backend clamps an out-of-range value. + /// Null when the timestamp was within the valid range and no correction was needed. + /// Use this for troubleshooting and root-cause analysis of clock issues on devices. + /// + public DateTime? OriginalTimestamp { get; set; } + + /// + /// True when the backend had to clamp the agent-side Timestamp to a valid range. + /// When set, OriginalTimestamp contains the raw value the agent sent. + /// AI analysis and UI should treat clamped timestamps with caution. + /// + public bool TimestampClamped { get; set; } + /// /// Azure Table Storage RowKey — format: {Timestamp:yyyyMMddHHmmssfff}_{Sequence:D10} /// Represents the exact sort key used in storage.