gsadmin/AutopilotMonitor
1
0
Fork 0
mirror of https://github.com/okieselbach/AutopilotMonitor.git synced 2026-10-24 12:42:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix DateTime Kind=Local causing NotSupportedException on Azure Table Storage writes

Browse Source 
Windows EventLog record.TimeCreated returns DateTime with Kind=Local. The Azure
Data Tables SDK rejects Local DateTimes with NotSupportedException, causing
UpdateSessionStatusAsync to fail deterministically for every WhiteGlove session
(Shell-Core EventID 62407 = whiteglove_complete). This was the root cause of the
systematic 6/6 WhiteGlove failures, not ETag conflicts or transient Azure errors.

Agent fix: convert record.TimeCreated to UTC at all 3 EventLog timestamp sources.
Backend fix: EnsureUtc() safety net on all 11 DateTime writes to the sessions table.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Oliver Kieselbach
2026-03-02 13:20:11 +01:00
parent 8a8002491d
commit 5d058bf84c
2 changed files with 26 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.cs
+3 -3
View File
@@ -388,7 +388,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors
ProcessHelloEvent(
eventId,
record.TimeCreated ?? DateTime.UtcNow,
(record.TimeCreated ?? DateTime.UtcNow).ToUniversalTime(),
record.ProviderName ?? "",
isBackfill: false);
}
@@ -683,7 +683,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors
return;
}
var eventTimestamp = record.TimeCreated ?? DateTime.UtcNow;
var eventTimestamp = (record.TimeCreated ?? DateTime.UtcNow).ToUniversalTime();
_onEventCollected(new EnrollmentEvent
{
@@ -979,7 +979,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors
if (latestEventId.HasValue)
{
_logger.Info($"Backfill found recent Hello terminal event: EventID {latestEventId.Value} at {latestTimestamp:O}");
ProcessHelloEvent(latestEventId.Value, latestTimestamp == DateTime.MinValue ? DateTime.UtcNow : latestTimestamp, latestProvider, isBackfill: true);
ProcessHelloEvent(latestEventId.Value, (latestTimestamp == DateTime.MinValue ? DateTime.UtcNow : latestTimestamp).ToUniversalTime(), latestProvider, isBackfill: true);
}
else
{
src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs
+23 -11
View File
@@ -12,6 +12,18 @@ namespace AutopilotMonitor.Functions.Services
{
// ===== SESSION MANAGEMENT METHODS =====
/// <summary>
/// Ensures a DateTime value has DateTimeKind.Utc before writing to Azure Table Storage.
/// The Azure Data Tables SDK throws NotSupportedException for DateTime values with Kind=Local.
/// This guards against timestamps that lost their UTC kind during JSON round-trips (e.g. agent spool).
/// </summary>
private static DateTime EnsureUtc(DateTime dt) => dt.Kind switch
{
DateTimeKind.Utc => dt,
DateTimeKind.Local => dt.ToUniversalTime(),
_ => DateTime.SpecifyKind(dt, DateTimeKind.Utc) // Unspecified: assume UTC
};
/// <summary>
/// Stores a session registration
/// </summary>
@@ -93,7 +105,7 @@ namespace AutopilotMonitor.Functions.Services
["OsLanguage"] = registration.OsLanguage ?? string.Empty,
["IsUserDriven"] = registration.IsUserDriven,
["IsPreProvisioned"] = isPreProvisioned,
["StartedAt"] = startedAt,
["StartedAt"] = EnsureUtc(startedAt),
["AgentVersion"] = registration.AgentVersion ?? string.Empty,
["EnrollmentType"] = registration.EnrollmentType ?? "v1",
["CurrentPhase"] = currentPhase,
@@ -102,13 +114,13 @@ namespace AutopilotMonitor.Functions.Services
};
if (completedAt.HasValue)
entity["CompletedAt"] = completedAt.Value;
entity["CompletedAt"] = EnsureUtc(completedAt.Value);
if (!string.IsNullOrWhiteSpace(failureReason))
entity["FailureReason"] = failureReason;
if (lastEventAt.HasValue)
entity["LastEventAt"] = lastEventAt.Value;
entity["LastEventAt"] = EnsureUtc(lastEventAt.Value);
if (durationSeconds.HasValue)
entity["DurationSeconds"] = durationSeconds.Value;
@@ -418,14 +430,14 @@ namespace AutopilotMonitor.Functions.Services
var currentStartedAt = session.GetDateTimeOffset("StartedAt")?.UtcDateTime ?? DateTime.MaxValue;
if (earliestEventTimestamp.HasValue && earliestEventTimestamp.Value < currentStartedAt)
{
update["StartedAt"] = earliestEventTimestamp.Value;
update["StartedAt"] = EnsureUtc(earliestEventTimestamp.Value);
}
// Set completion time if succeeded or failed
if (status == SessionStatus.Succeeded || status == SessionStatus.Failed)
{
// Use the provided completedAt timestamp (from event) if available, otherwise use current time
var effectiveCompletedAt = completedAt ?? DateTime.UtcNow;
var effectiveCompletedAt = EnsureUtc(completedAt ?? DateTime.UtcNow);
update["CompletedAt"] = effectiveCompletedAt;
// Read earliest event from Events table — authoritative source, immune to
@@ -445,7 +457,7 @@ namespace AutopilotMonitor.Functions.Services
{
var currentLastEventAt = session.GetDateTimeOffset("LastEventAt")?.UtcDateTime;
if (!currentLastEventAt.HasValue || latestEventTimestamp.Value > currentLastEventAt.Value)
update["LastEventAt"] = latestEventTimestamp.Value;
update["LastEventAt"] = EnsureUtc(latestEventTimestamp.Value);
}
// Set failure reason if failed
@@ -503,11 +515,11 @@ namespace AutopilotMonitor.Functions.Services
var freshStartedAt = freshSession.GetDateTimeOffset("StartedAt")?.UtcDateTime ?? DateTime.MaxValue;
if (earliestEventTimestamp.HasValue && earliestEventTimestamp.Value < freshStartedAt)
forceUpdate["StartedAt"] = earliestEventTimestamp.Value;
forceUpdate["StartedAt"] = EnsureUtc(earliestEventTimestamp.Value);
if (status == SessionStatus.Succeeded || status == SessionStatus.Failed)
{
var effectiveCompletedAt = completedAt ?? DateTime.UtcNow;
var effectiveCompletedAt = EnsureUtc(completedAt ?? DateTime.UtcNow);
forceUpdate["CompletedAt"] = effectiveCompletedAt;
var earliestStoredEvent = await GetEarliestSessionEventTimestampAsync(tenantId, sessionId);
@@ -523,7 +535,7 @@ namespace AutopilotMonitor.Functions.Services
{
var freshLastEventAt = freshSession.GetDateTimeOffset("LastEventAt")?.UtcDateTime;
if (!freshLastEventAt.HasValue || latestEventTimestamp.Value > freshLastEventAt.Value)
forceUpdate["LastEventAt"] = latestEventTimestamp.Value;
forceUpdate["LastEventAt"] = EnsureUtc(latestEventTimestamp.Value);
}
if (status == SessionStatus.Failed && !string.IsNullOrEmpty(failureReason))
@@ -613,7 +625,7 @@ namespace AutopilotMonitor.Functions.Services
var currentStartedAt = entity.Value.GetDateTimeOffset("StartedAt")?.UtcDateTime ?? DateTime.MaxValue;
if (earliestEventTimestamp.HasValue && earliestEventTimestamp.Value < currentStartedAt)
{
update["StartedAt"] = earliestEventTimestamp.Value;
update["StartedAt"] = EnsureUtc(earliestEventTimestamp.Value);
}
// Track the most recent event timestamp for excessive data sender detection
@@ -621,7 +633,7 @@ namespace AutopilotMonitor.Functions.Services
{
var currentLastEventAt = entity.Value.GetDateTimeOffset("LastEventAt")?.UtcDateTime;
if (!currentLastEventAt.HasValue || latestEventTimestamp.Value > currentLastEventAt.Value)
update["LastEventAt"] = latestEventTimestamp.Value;
update["LastEventAt"] = EnsureUtc(latestEventTimestamp.Value);
}
await tableClient.UpdateEntityAsync(update, entity.Value.ETag, TableUpdateMode.Merge);