From 2235170a577b85a535e1728252211adaf5767775 Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Tue, 24 Feb 2026 13:38:46 +0100 Subject: [PATCH] add generic event_correlation source for flexible cross-event rule correlation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the hardcoded app_state_regression source with a generic event_correlation source that allows correlating any two event types by a shared field (e.g. appId), with optional time windows and filters on both events — all configurable via JSON without backend code changes. RuleCondition gains 5 new nullable properties: - CorrelateEventType: second event type (Event B) - JoinField: field to match across both events (e.g. "appId") - TimeWindowSeconds: max time between Event A and B - EventAFilterField/Operator/Value: optional filter on Event A ANALYZE-APP-010 migrated to use event_correlation source. Co-Authored-By: Claude Sonnet 4.6 --- .../Services/BuiltInAnalyzeRules.cs | 12 +- .../Services/RuleEngine.cs | 200 ++++++++++++------ .../Models/AnalyzeRule.cs | 52 ++++- 3 files changed, 195 insertions(+), 69 deletions(-) diff --git a/src/Backend/AutopilotMonitor.Functions/Services/BuiltInAnalyzeRules.cs b/src/Backend/AutopilotMonitor.Functions/Services/BuiltInAnalyzeRules.cs index 154961cb..1b38fef6 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/BuiltInAnalyzeRules.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/BuiltInAnalyzeRules.cs @@ -569,7 +569,8 @@ namespace AutopilotMonitor.Functions.Services /// /// Detects when an app was initially reported as successfully installed but then /// transitioned back to an error state (IME-ERROR-REPORT), correlated by appId. - /// Uses the app_state_regression source which matches the exact same app across both events. + /// Uses the generic event_correlation source to match the exact same app across + /// app_install_completed and app_install_failed events. /// This is a silent failure that can leave apps broken post-enrollment. /// private static AnalyzeRule CreateAppPostSuccessFailureRule() => new AnalyzeRule @@ -583,12 +584,15 @@ namespace AutopilotMonitor.Functions.Services BaseConfidence = 75, Conditions = new List { - // Single condition: finds apps where completed -> failed regression exists for the same appId, - // and the failed event has errorPatternId = IME-ERROR-REPORT + // Correlate app_install_completed (Event A) with app_install_failed (Event B) + // for the same appId, where Event B has errorPatternId = IME-ERROR-REPORT new RuleCondition { Signal = "app_regression", - Source = "app_state_regression", + Source = "event_correlation", + EventType = "app_install_completed", + CorrelateEventType = "app_install_failed", + JoinField = "appId", DataField = "errorPatternId", Operator = "equals", Value = "IME-ERROR-REPORT", diff --git a/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.cs b/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.cs index 8e3b1137..64259abb 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.cs @@ -161,8 +161,8 @@ namespace AutopilotMonitor.Functions.Services case "app_install_duration": return EvaluateAppInstallDurationCondition(condition, events); - case "app_state_regression": - return EvaluateAppStateRegressionCondition(condition, events); + case "event_correlation": + return EvaluateEventCorrelationCondition(condition, events); default: return (false, "unknown source"); @@ -517,69 +517,149 @@ namespace AutopilotMonitor.Functions.Services } /// - /// Detects apps that regressed: first seen as app_install_completed, then as app_install_failed, - /// correlated by appId. Optionally filters the failed event by a DataField/Operator/Value condition. - /// Returns evidence with appId, appName, and both event details. + /// Generic event correlation: finds pairs of events (A before B) sharing + /// the same value in a join field, with optional filters on each event + /// and optional time window constraint. + /// + /// Condition fields used: + /// EventType = Event A type (required) + /// CorrelateEventType = Event B type (required) + /// JoinField = field that must match in both events (required, e.g. "appId") + /// TimeWindowSeconds = max seconds between A and B (optional, 0/null = unlimited) + /// DataField/Operator/Value = optional filter on Event B + /// EventAFilterField/Operator/Value = optional filter on Event A /// - private (bool matched, object evidence) EvaluateAppStateRegressionCondition(RuleCondition condition, List events) + private (bool matched, object evidence) EvaluateEventCorrelationCondition(RuleCondition condition, List events) { - var sortedEvents = events.OrderBy(e => e.Timestamp).ThenBy(e => e.Sequence).ToList(); - - var completedEvents = sortedEvents - .Where(e => string.Equals(e.EventType, "app_install_completed", StringComparison.OrdinalIgnoreCase)) - .ToList(); - - var failedEvents = sortedEvents - .Where(e => string.Equals(e.EventType, "app_install_failed", StringComparison.OrdinalIgnoreCase)) - .ToList(); - - if (!completedEvents.Any() || !failedEvents.Any()) - return (false, "no completed/failed event pair found"); - - foreach (var failedEvent in failedEvents) + if (string.IsNullOrEmpty(condition.EventType) || + string.IsNullOrEmpty(condition.CorrelateEventType) || + string.IsNullOrEmpty(condition.JoinField)) { - // Apply optional DataField filter on the failed event - if (!string.IsNullOrEmpty(condition.DataField) && !string.IsNullOrEmpty(condition.Operator)) - { - var fieldValue = GetDataFieldValue(failedEvent, condition.DataField); - if (fieldValue == null || !MatchesOperator(fieldValue, condition.Operator, condition.Value ?? "")) - continue; - } - - var failedAppId = GetDataFieldValue(failedEvent, "appId"); - var failedAppName = GetDataFieldValue(failedEvent, "appName"); - var appKey = !string.IsNullOrWhiteSpace(failedAppId) ? failedAppId : failedAppName; - - if (string.IsNullOrWhiteSpace(appKey)) - continue; - - // Find a completed event for the same app that occurred before the failed event - var matchingCompleted = completedEvents.LastOrDefault(c => - c.Timestamp <= failedEvent.Timestamp && - string.Equals( - GetDataFieldValue(c, "appId") ?? GetDataFieldValue(c, "appName"), - appKey, - StringComparison.OrdinalIgnoreCase)); - - if (matchingCompleted == null) - continue; - - return (true, new Dictionary - { - ["appId"] = failedAppId ?? string.Empty, - ["appName"] = failedAppName ?? appKey, - ["completedEventId"] = matchingCompleted.EventId ?? string.Empty, - ["completedSequence"] = matchingCompleted.Sequence, - ["completedTimestamp"] = matchingCompleted.Timestamp, - ["failedEventId"] = failedEvent.EventId ?? string.Empty, - ["failedSequence"] = failedEvent.Sequence, - ["failedTimestamp"] = failedEvent.Timestamp, - ["errorPatternId"] = GetDataFieldValue(failedEvent, "errorPatternId") ?? string.Empty, - ["errorDetail"] = GetDataFieldValue(failedEvent, "errorDetail") ?? failedEvent.Message ?? string.Empty - }); + _logger.LogWarning($"event_correlation condition '{condition.Signal}' missing required fields (EventType, CorrelateEventType, JoinField)"); + return (false, "event_correlation requires EventType, CorrelateEventType, and JoinField"); } - return (false, "no app found with completed-then-failed regression"); + var sortedEvents = events.OrderBy(e => e.Timestamp).ThenBy(e => e.Sequence).ToList(); + + // Collect Event A candidates with optional filter + var eventAList = sortedEvents + .Where(e => MatchesEventType(e, condition.EventType)) + .Where(e => + { + if (string.IsNullOrEmpty(condition.EventAFilterField)) + return true; + var val = GetDataFieldValue(e, condition.EventAFilterField); + return val != null && MatchesOperator(val, condition.EventAFilterOperator ?? "exists", condition.EventAFilterValue ?? ""); + }) + .ToList(); + + // Collect Event B candidates with optional DataField/Operator/Value filter + var eventBList = sortedEvents + .Where(e => MatchesEventType(e, condition.CorrelateEventType)) + .Where(e => + { + if (string.IsNullOrEmpty(condition.DataField)) + return true; + var val = GetDataFieldValue(e, condition.DataField); + return val != null && MatchesOperator(val, condition.Operator ?? "exists", condition.Value ?? ""); + }) + .ToList(); + + if (!eventAList.Any() || !eventBList.Any()) + return (false, "no matching event pairs found"); + + // Index Event A by join field value for O(1) lookup + var eventAByJoinKey = eventAList + .Select(e => new { Event = e, Key = GetDataFieldValue(e, condition.JoinField) }) + .Where(x => !string.IsNullOrEmpty(x.Key)) + .GroupBy(x => x.Key!, StringComparer.OrdinalIgnoreCase) + .ToDictionary(g => g.Key, g => g.Select(x => x.Event).ToList(), StringComparer.OrdinalIgnoreCase); + + // For each Event B, find a matching Event A (same join key, A before B, within time window) + var matchedPairs = new List>(); + var matchedJoinKeys = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var eventB in eventBList) + { + var bKey = GetDataFieldValue(eventB, condition.JoinField); + if (string.IsNullOrEmpty(bKey) || !eventAByJoinKey.ContainsKey(bKey)) + continue; + + // Find the latest Event A that occurred before Event B + var matchingA = eventAByJoinKey[bKey] + .Where(a => a.Timestamp < eventB.Timestamp || + (a.Timestamp == eventB.Timestamp && a.Sequence < eventB.Sequence)) + .Where(a => + { + if (condition.TimeWindowSeconds == null || condition.TimeWindowSeconds <= 0) + return true; + return (eventB.Timestamp - a.Timestamp).TotalSeconds <= condition.TimeWindowSeconds.Value; + }) + .OrderByDescending(a => a.Timestamp) + .ThenByDescending(a => a.Sequence) + .FirstOrDefault(); + + if (matchingA == null) + continue; + + // One match per join key to avoid duplicates + if (matchedJoinKeys.Contains(bKey)) + continue; + matchedJoinKeys.Add(bKey); + + // Build evidence for this correlated pair + var pair = new Dictionary + { + ["joinField"] = condition.JoinField, + ["joinValue"] = bKey, + ["eventA_eventId"] = matchingA.EventId ?? string.Empty, + ["eventA_eventType"] = matchingA.EventType ?? string.Empty, + ["eventA_timestamp"] = matchingA.Timestamp, + ["eventA_sequence"] = matchingA.Sequence, + ["eventA_message"] = matchingA.Message ?? string.Empty, + ["eventB_eventId"] = eventB.EventId ?? string.Empty, + ["eventB_eventType"] = eventB.EventType ?? string.Empty, + ["eventB_timestamp"] = eventB.Timestamp, + ["eventB_sequence"] = eventB.Sequence, + ["eventB_message"] = eventB.Message ?? string.Empty, + ["timeDeltaSeconds"] = (eventB.Timestamp - matchingA.Timestamp).TotalSeconds + }; + + // Add well-known data fields from both events + AddDataFieldsToEvidence(pair, matchingA, "eventA_", condition.JoinField); + AddDataFieldsToEvidence(pair, eventB, "eventB_", condition.JoinField); + + matchedPairs.Add(pair); + } + + if (!matchedPairs.Any()) + return (false, "no correlated event pairs found"); + + // Single match: flat dictionary; multiple: first as primary + allMatches list + var primary = matchedPairs[0]; + primary["totalMatches"] = matchedPairs.Count; + if (matchedPairs.Count > 1) + { + primary["allMatches"] = matchedPairs; + } + return (true, primary); + } + + /// + /// Adds well-known data fields from an event to the evidence dictionary with a prefix. + /// + private void AddDataFieldsToEvidence(Dictionary evidence, EnrollmentEvent evt, string prefix, string joinField) + { + if (evt.Data == null) return; + + var knownFields = new[] { "appId", "appName", "errorPatternId", "errorDetail", "errorCode", "status" }; + + foreach (var field in knownFields) + { + var val = GetDataFieldValue(evt, field); + if (val != null) + evidence[$"{prefix}{field}"] = val; + } } } } diff --git a/src/Shared/AutopilotMonitor.Shared/Models/AnalyzeRule.cs b/src/Shared/AutopilotMonitor.Shared/Models/AnalyzeRule.cs index e18a56ed..7e0a2062 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/AnalyzeRule.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/AnalyzeRule.cs @@ -134,28 +134,34 @@ namespace AutopilotMonitor.Shared.Models public string Signal { get; set; } /// - /// Source of the signal: "event_type", "event_data", "phase_duration", "event_count", "app_install_duration", "app_state_regression" + /// Source of the signal: "event_type", "event_data", "phase_duration", "event_count", "app_install_duration", "event_correlation" /// public string Source { get; set; } /// - /// Event type to match on (when Source = "event_type" or "event_data") + /// Event type to match on. + /// For "event_type"/"event_data": the event type to match. + /// For "event_correlation": the FIRST event type (Event A). /// public string EventType { get; set; } /// - /// Data field to match on (when Source = "event_data") - /// Uses dot notation for nested fields (e.g., "data.errorCode") + /// Data field to match on. + /// For "event_data": field to check with Operator/Value. + /// For "event_correlation": optional filter field on Event B (the second event). + /// Uses dot notation for nested fields (e.g., "data.errorCode"). /// public string DataField { get; set; } /// /// Comparison operator: "equals", "contains", "regex", "gt", "lt", "gte", "lte", "exists", "count_gte" + /// For "event_correlation": operator for the Event B filter (applied to DataField). /// public string Operator { get; set; } /// - /// Value to compare against + /// Value to compare against. + /// For "event_correlation": value for the Event B filter. /// public string Value { get; set; } @@ -164,6 +170,42 @@ namespace AutopilotMonitor.Shared.Models /// If false, it only contributes to confidence scoring /// public bool Required { get; set; } = false; + + // ===== Event Correlation Properties ===== + // Used only when Source = "event_correlation" + + /// + /// The second event type to correlate with (Event B). + /// Example: "app_install_failed" + /// + public string CorrelateEventType { get; set; } + + /// + /// The data field to join on — must have the same value in both Event A and Event B. + /// Example: "appId" means both events must share the same appId value. + /// + public string JoinField { get; set; } + + /// + /// Maximum time in seconds between Event A and Event B. Null or 0 means no time limit. + /// + public int? TimeWindowSeconds { get; set; } + + /// + /// Optional filter field on Event A (the first event). + /// Combined with EventAFilterOperator and EventAFilterValue. + /// + public string EventAFilterField { get; set; } + + /// + /// Operator for the Event A filter. Uses same operators as the main Operator field. + /// + public string EventAFilterOperator { get; set; } + + /// + /// Value for the Event A filter. + /// + public string EventAFilterValue { get; set; } } ///