From 5d058bf84ca0e7a0372e33252f61bfc5577b4223 Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Mon, 2 Mar 2026 13:20:11 +0100 Subject: [PATCH] fix DateTime Kind=Local causing NotSupportedException on Azure Table Storage writes Windows EventLog record.TimeCreated returns DateTime with Kind=Local. The Azure Data Tables SDK rejects Local DateTimes with NotSupportedException, causing UpdateSessionStatusAsync to fail deterministically for every WhiteGlove session (Shell-Core EventID 62407 = whiteglove_complete). This was the root cause of the systematic 6/6 WhiteGlove failures, not ETag conflicts or transient Azure errors. Agent fix: convert record.TimeCreated to UTC at all 3 EventLog timestamp sources. Backend fix: EnsureUtc() safety net on all 11 DateTime writes to the sessions table. Co-Authored-By: Claude Opus 4.6 --- .../Collectors/EspAndHelloTracker.cs | 6 ++-- .../Services/TableStorageService.Sessions.cs | 34 +++++++++++++------ 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.cs b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.cs index 871c557a..a1f6d0f5 100644 --- a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.cs +++ b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.cs @@ -388,7 +388,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors ProcessHelloEvent( eventId, - record.TimeCreated ?? DateTime.UtcNow, + (record.TimeCreated ?? DateTime.UtcNow).ToUniversalTime(), record.ProviderName ?? "", isBackfill: false); } @@ -683,7 +683,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors return; } - var eventTimestamp = record.TimeCreated ?? DateTime.UtcNow; + var eventTimestamp = (record.TimeCreated ?? DateTime.UtcNow).ToUniversalTime(); _onEventCollected(new EnrollmentEvent { @@ -979,7 +979,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors if (latestEventId.HasValue) { _logger.Info($"Backfill found recent Hello terminal event: EventID {latestEventId.Value} at {latestTimestamp:O}"); - ProcessHelloEvent(latestEventId.Value, latestTimestamp == DateTime.MinValue ? DateTime.UtcNow : latestTimestamp, latestProvider, isBackfill: true); + ProcessHelloEvent(latestEventId.Value, (latestTimestamp == DateTime.MinValue ? DateTime.UtcNow : latestTimestamp).ToUniversalTime(), latestProvider, isBackfill: true); } else { diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs index 3f63dbd0..ab1706c4 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs @@ -12,6 +12,18 @@ namespace AutopilotMonitor.Functions.Services { // ===== SESSION MANAGEMENT METHODS ===== + /// + /// Ensures a DateTime value has DateTimeKind.Utc before writing to Azure Table Storage. + /// The Azure Data Tables SDK throws NotSupportedException for DateTime values with Kind=Local. + /// This guards against timestamps that lost their UTC kind during JSON round-trips (e.g. agent spool). + /// + private static DateTime EnsureUtc(DateTime dt) => dt.Kind switch + { + DateTimeKind.Utc => dt, + DateTimeKind.Local => dt.ToUniversalTime(), + _ => DateTime.SpecifyKind(dt, DateTimeKind.Utc) // Unspecified: assume UTC + }; + /// /// Stores a session registration /// @@ -93,7 +105,7 @@ namespace AutopilotMonitor.Functions.Services ["OsLanguage"] = registration.OsLanguage ?? string.Empty, ["IsUserDriven"] = registration.IsUserDriven, ["IsPreProvisioned"] = isPreProvisioned, - ["StartedAt"] = startedAt, + ["StartedAt"] = EnsureUtc(startedAt), ["AgentVersion"] = registration.AgentVersion ?? string.Empty, ["EnrollmentType"] = registration.EnrollmentType ?? "v1", ["CurrentPhase"] = currentPhase, @@ -102,13 +114,13 @@ namespace AutopilotMonitor.Functions.Services }; if (completedAt.HasValue) - entity["CompletedAt"] = completedAt.Value; + entity["CompletedAt"] = EnsureUtc(completedAt.Value); if (!string.IsNullOrWhiteSpace(failureReason)) entity["FailureReason"] = failureReason; if (lastEventAt.HasValue) - entity["LastEventAt"] = lastEventAt.Value; + entity["LastEventAt"] = EnsureUtc(lastEventAt.Value); if (durationSeconds.HasValue) entity["DurationSeconds"] = durationSeconds.Value; @@ -418,14 +430,14 @@ namespace AutopilotMonitor.Functions.Services var currentStartedAt = session.GetDateTimeOffset("StartedAt")?.UtcDateTime ?? DateTime.MaxValue; if (earliestEventTimestamp.HasValue && earliestEventTimestamp.Value < currentStartedAt) { - update["StartedAt"] = earliestEventTimestamp.Value; + update["StartedAt"] = EnsureUtc(earliestEventTimestamp.Value); } // Set completion time if succeeded or failed if (status == SessionStatus.Succeeded || status == SessionStatus.Failed) { // Use the provided completedAt timestamp (from event) if available, otherwise use current time - var effectiveCompletedAt = completedAt ?? DateTime.UtcNow; + var effectiveCompletedAt = EnsureUtc(completedAt ?? DateTime.UtcNow); update["CompletedAt"] = effectiveCompletedAt; // Read earliest event from Events table — authoritative source, immune to @@ -445,7 +457,7 @@ namespace AutopilotMonitor.Functions.Services { var currentLastEventAt = session.GetDateTimeOffset("LastEventAt")?.UtcDateTime; if (!currentLastEventAt.HasValue || latestEventTimestamp.Value > currentLastEventAt.Value) - update["LastEventAt"] = latestEventTimestamp.Value; + update["LastEventAt"] = EnsureUtc(latestEventTimestamp.Value); } // Set failure reason if failed @@ -503,11 +515,11 @@ namespace AutopilotMonitor.Functions.Services var freshStartedAt = freshSession.GetDateTimeOffset("StartedAt")?.UtcDateTime ?? DateTime.MaxValue; if (earliestEventTimestamp.HasValue && earliestEventTimestamp.Value < freshStartedAt) - forceUpdate["StartedAt"] = earliestEventTimestamp.Value; + forceUpdate["StartedAt"] = EnsureUtc(earliestEventTimestamp.Value); if (status == SessionStatus.Succeeded || status == SessionStatus.Failed) { - var effectiveCompletedAt = completedAt ?? DateTime.UtcNow; + var effectiveCompletedAt = EnsureUtc(completedAt ?? DateTime.UtcNow); forceUpdate["CompletedAt"] = effectiveCompletedAt; var earliestStoredEvent = await GetEarliestSessionEventTimestampAsync(tenantId, sessionId); @@ -523,7 +535,7 @@ namespace AutopilotMonitor.Functions.Services { var freshLastEventAt = freshSession.GetDateTimeOffset("LastEventAt")?.UtcDateTime; if (!freshLastEventAt.HasValue || latestEventTimestamp.Value > freshLastEventAt.Value) - forceUpdate["LastEventAt"] = latestEventTimestamp.Value; + forceUpdate["LastEventAt"] = EnsureUtc(latestEventTimestamp.Value); } if (status == SessionStatus.Failed && !string.IsNullOrEmpty(failureReason)) @@ -613,7 +625,7 @@ namespace AutopilotMonitor.Functions.Services var currentStartedAt = entity.Value.GetDateTimeOffset("StartedAt")?.UtcDateTime ?? DateTime.MaxValue; if (earliestEventTimestamp.HasValue && earliestEventTimestamp.Value < currentStartedAt) { - update["StartedAt"] = earliestEventTimestamp.Value; + update["StartedAt"] = EnsureUtc(earliestEventTimestamp.Value); } // Track the most recent event timestamp for excessive data sender detection @@ -621,7 +633,7 @@ namespace AutopilotMonitor.Functions.Services { var currentLastEventAt = entity.Value.GetDateTimeOffset("LastEventAt")?.UtcDateTime; if (!currentLastEventAt.HasValue || latestEventTimestamp.Value > currentLastEventAt.Value) - update["LastEventAt"] = latestEventTimestamp.Value; + update["LastEventAt"] = EnsureUtc(latestEventTimestamp.Value); } await tableClient.UpdateEntityAsync(update, entity.Value.ETag, TableUpdateMode.Merge);