From 6c444c1c73fd4df2d22e58aae9e4579ca16faaad Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Tue, 10 Feb 2026 15:35:02 +0100 Subject: [PATCH] Analyze Rule phase_change fix --- .../Services/BuiltInAnalyzeRules.cs | 12 +- .../Services/RuleEngine.cs | 115 +++++++++++++----- .../Services/TableStorageService.cs | 48 +++++++- .../app/fleet-health/page.tsx | 2 +- 4 files changed, 142 insertions(+), 35 deletions(-) diff --git a/src/Backend/AutopilotMonitor.Functions/Services/BuiltInAnalyzeRules.cs b/src/Backend/AutopilotMonitor.Functions/Services/BuiltInAnalyzeRules.cs index 3b739288..9dc71861 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/BuiltInAnalyzeRules.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/BuiltInAnalyzeRules.cs @@ -174,19 +174,23 @@ namespace AutopilotMonitor.Functions.Services { RuleId = "ANALYZE-ESP-001", Title = "ESP Blocking App Timeout", - Description = "Detects when the ESP is waiting too long for a blocking app to install.", + Description = "Detects when the DeviceSetup phase of the ESP took longer than 30 minutes, indicating a stuck or very slow blocking app.", Severity = "high", Category = "esp", - BaseConfidence = 60, + // BaseConfidence is intentionally below ConfidenceThreshold so the rule only fires + // when the duration factor (phase_duration > 1800s) also matches. + BaseConfidence = 50, Conditions = new List { new RuleCondition { Signal = "esp_stalled", Source = "phase_duration", EventType = "esp_phase_changed", DataField = "espPhase", Operator = "equals", Value = "DeviceSetup", Required = true } }, ConfidenceFactors = new List { - new ConfidenceFactor { Signal = "long_esp", Condition = "phase_duration > 1800", Weight = 30 } + new ConfidenceFactor { Signal = "long_esp", Condition = "phase_duration > 1800", Weight = 40 } }, - Explanation = "The Enrollment Status Page has been waiting for a blocking app for an extended period. This often indicates the app is stuck installing or the ESP timeout is about to trigger.", + // Only fire when duration factor also matched (50 base + 40 factor = 90 > 70 threshold) + ConfidenceThreshold = 70, + Explanation = "The ESP DeviceSetup phase took more than 30 minutes. This indicates a blocking app is stuck installing, downloading very slowly, or has entered a retry loop.", Remediation = new List { new RemediationStep { Title = "Investigate blocking app", Steps = new List { "Identify which app is blocking (check ESP events)", "Verify app content is downloadable", "Consider reducing the number of blocking apps" } } diff --git a/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.cs b/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.cs index 75340809..1ced8fff 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.cs @@ -178,14 +178,30 @@ namespace AutopilotMonitor.Functions.Services var fieldValue = GetDataFieldValue(evt, condition.DataField); if (fieldValue != null && MatchesOperator(fieldValue, condition.Operator, condition.Value)) { - return (true, new { eventType = evt.EventType, field = condition.DataField, value = fieldValue }); + return (true, new Dictionary + { + ["eventId"] = evt.EventId, + ["sequence"] = evt.Sequence, + ["timestamp"] = evt.Timestamp, + ["eventType"] = evt.EventType, + ["field"] = condition.DataField, + ["value"] = fieldValue + }); } } return (false, "data field not matched"); } - // Just check if event type exists - return (true, new { eventType = condition.EventType, count = matchingEvents.Count }); + // Just check if event type exists — return first matching event for reference + var first = matchingEvents[0]; + return (true, new Dictionary + { + ["eventId"] = first.EventId, + ["sequence"] = first.Sequence, + ["timestamp"] = first.Timestamp, + ["eventType"] = condition.EventType, + ["count"] = matchingEvents.Count + }); } private (bool matched, object evidence) EvaluateEventDataCondition(RuleCondition condition, List events) @@ -197,7 +213,14 @@ namespace AutopilotMonitor.Functions.Services var fieldValue = GetDataFieldValue(evt, condition.DataField); if (fieldValue != null && MatchesOperator(fieldValue, condition.Operator, condition.Value)) { - return (true, new { field = condition.DataField, value = fieldValue }); + return (true, new Dictionary + { + ["eventId"] = evt.EventId, + ["sequence"] = evt.Sequence, + ["timestamp"] = evt.Timestamp, + ["field"] = condition.DataField, + ["value"] = fieldValue + }); } } @@ -206,18 +229,29 @@ namespace AutopilotMonitor.Functions.Services private (bool matched, object evidence) EvaluateEventCountCondition(RuleCondition condition, List events) { - var count = events.Count(e => MatchesEventType(e, condition.EventType)); + var matchingEvents = events.Where(e => MatchesEventType(e, condition.EventType)).ToList(); + var count = matchingEvents.Count; if (condition.Operator == "count_gte" && int.TryParse(condition.Value, out var threshold)) { if (count >= threshold) - return (true, new { count, threshold }); + { + var first = matchingEvents[0]; + return (true, new Dictionary + { + ["eventId"] = first.EventId, + ["sequence"] = first.Sequence, + ["timestamp"] = first.Timestamp, + ["count"] = count, + ["threshold"] = threshold + }); + } } - return (false, new { count }); + return (false, new Dictionary { ["count"] = count }); } - private (bool matched, object evidence) EvaluatePhaseDurationCondition(RuleCondition condition, List events) + private static (bool matched, object evidence) EvaluatePhaseDurationCondition(RuleCondition condition, List events) { // Find phase change events to calculate phase duration var phaseEvents = events @@ -228,48 +262,71 @@ namespace AutopilotMonitor.Functions.Services if (!phaseEvents.Any()) return (false, "no phase events"); - // Check if a specific phase took too long + // condition.DataField = field to look up (e.g. "espPhase"), condition.Value = target phase name (e.g. "DeviceSetup") var targetPhase = condition.Value; + var lookupField = string.IsNullOrEmpty(condition.DataField) ? "espPhase" : condition.DataField; + for (int i = 0; i < phaseEvents.Count; i++) { - var phaseData = phaseEvents[i].Data; - var currentPhase = phaseData?.ContainsKey("espPhase") == true - ? phaseData["espPhase"]?.ToString() + var evt = phaseEvents[i]; + var currentPhase = evt.Data?.ContainsKey(lookupField) == true + ? evt.Data[lookupField]?.ToString() : null; - if (currentPhase == targetPhase) - { - // Calculate how long this phase lasted - DateTime phaseEnd; - if (i + 1 < phaseEvents.Count) - { - phaseEnd = phaseEvents[i + 1].Timestamp; - } - else - { - phaseEnd = DateTime.UtcNow; // Phase is still active - } + if (!string.Equals(currentPhase, targetPhase, StringComparison.OrdinalIgnoreCase)) + continue; - var duration = (phaseEnd - phaseEvents[i].Timestamp).TotalSeconds; - return (true, new { phase = targetPhase, durationSeconds = duration }); + // Calculate how long this phase lasted + DateTime phaseEnd; + string? phaseEndEventId = null; + if (i + 1 < phaseEvents.Count) + { + phaseEnd = phaseEvents[i + 1].Timestamp; + phaseEndEventId = phaseEvents[i + 1].EventId; } + else + { + phaseEnd = DateTime.UtcNow; // Phase is still active + } + + var durationSeconds = (phaseEnd - evt.Timestamp).TotalSeconds; + + return (true, new Dictionary + { + ["eventId"] = evt.EventId, + ["sequence"] = evt.Sequence, + ["phaseStartTimestamp"] = evt.Timestamp, + ["phaseEndEventId"] = phaseEndEventId ?? "(still active)", + ["phase"] = targetPhase, + ["durationSeconds"] = durationSeconds, + ["durationFormatted"] = FormatDuration(durationSeconds) + }); } return (false, "phase not found"); } + private static string FormatDuration(double totalSeconds) + { + var ts = TimeSpan.FromSeconds(totalSeconds); + if (ts.TotalHours >= 1) + return $"{(int)ts.TotalHours}h {ts.Minutes}m {ts.Seconds}s"; + if (ts.TotalMinutes >= 1) + return $"{ts.Minutes}m {ts.Seconds}s"; + return $"{ts.Seconds}s"; + } + private bool EvaluateConfidenceFactor(ConfidenceFactor factor, List events, Dictionary matchedConditions) { - // Simple confidence factor evaluation if (factor.Condition.StartsWith("phase_duration >")) { if (int.TryParse(factor.Condition.Replace("phase_duration >", "").Trim(), out var threshold)) { foreach (var mc in matchedConditions.Values) { - if (mc is IDictionary dict && dict.ContainsKey("durationSeconds")) + if (mc is Dictionary dict && dict.TryGetValue("durationSeconds", out var rawDuration)) { - var duration = Convert.ToDouble(dict["durationSeconds"]); + var duration = Convert.ToDouble(rawDuration); return duration > threshold; } } diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.cs index 1877a50c..ed0d1756 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.cs @@ -1018,7 +1018,7 @@ namespace AutopilotMonitor.Functions.Services Explanation = entity.GetString("Explanation") ?? string.Empty, Remediation = DeserializeJson>(entity.GetString("RemediationJson")), RelatedDocs = DeserializeJson>(entity.GetString("RelatedDocsJson")), - MatchedConditions = DeserializeJson>(entity.GetString("MatchedConditionsJson")), + MatchedConditions = DeserializeMatchedConditions(entity.GetString("MatchedConditionsJson")), DetectedAt = entity.GetDateTimeOffset("DetectedAt")?.UtcDateTime ?? DateTime.UtcNow }); } @@ -1757,6 +1757,52 @@ namespace AutopilotMonitor.Functions.Services } } + /// + /// Deserializes MatchedConditions JSON and normalizes nested JObject/JArray values + /// to plain Dictionary/List so System.Text.Json can serialize them correctly. + /// + private Dictionary DeserializeMatchedConditions(string? json) + { + if (string.IsNullOrEmpty(json)) + return new Dictionary(); + + try + { + var raw = JsonConvert.DeserializeObject>(json) + ?? new Dictionary(); + + var result = new Dictionary(); + foreach (var kv in raw) + result[kv.Key] = NormalizeJToken(kv.Value); + return result; + } + catch + { + return new Dictionary(); + } + } + + private static object NormalizeJToken(object? value) + { + if (value is Newtonsoft.Json.Linq.JObject jObj) + { + var dict = new Dictionary(); + foreach (var prop in jObj.Properties()) + dict[prop.Name] = NormalizeJToken(prop.Value); + return dict; + } + if (value is Newtonsoft.Json.Linq.JArray jArr) + { + var list = new List(); + foreach (var item in jArr) + list.Add(NormalizeJToken(item)); + return list; + } + if (value is Newtonsoft.Json.Linq.JValue jVal) + return jVal.Value ?? string.Empty; + return value ?? string.Empty; + } + private string[] DeserializeJsonArray(string? json) { if (string.IsNullOrEmpty(json)) diff --git a/src/Web/autopilot-monitor-web/app/fleet-health/page.tsx b/src/Web/autopilot-monitor-web/app/fleet-health/page.tsx index 5dcc300f..01b58be3 100644 --- a/src/Web/autopilot-monitor-web/app/fleet-health/page.tsx +++ b/src/Web/autopilot-monitor-web/app/fleet-health/page.tsx @@ -364,7 +364,7 @@ export default function FleetHealthPage() { />

- FLEET HEALTH + Feet Health