added rule stats

This commit is contained in:
Oliver Kieselbach
2026-04-13 14:12:19 +02:00
parent 37cc7540bf
commit 62918c9cbf
17 changed files with 858 additions and 20 deletions
@@ -1 +1 @@
915
918
@@ -61,5 +61,19 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
public Task<List<object>> GetMetricsSummaryAsync(string? tenantId)
=> _storage.GetMetricsSummaryAsync(tenantId);
public Task IncrementRuleStatAsync(string date, string tenantId, string ruleId, string ruleType,
string ruleTitle, string category, string severity, bool fired, int? confidenceScore)
=> _storage.IncrementRuleStatAsync(date, tenantId, ruleId, ruleType, ruleTitle, category, severity, fired, confidenceScore);
public Task<bool> SaveRuleStatsEntryAsync(RuleStatsEntry entry)
=> _storage.SaveRuleStatsEntryAsync(entry);
public Task<List<RuleStatsEntry>> GetRuleStatsAsync(string? tenantId = null, string? startDate = null,
string? endDate = null, string? ruleType = null, int maxResults = 500)
=> _storage.GetRuleStatsAsync(tenantId, startDate, endDate, ruleType, maxResults);
public Task<int> DeleteRuleStatsOlderThanAsync(DateTime cutoffDate)
=> _storage.DeleteRuleStatsOlderThanAsync(cutoffDate);
}
}
@@ -266,30 +266,33 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
try
{
var ruleEngine = new RuleEngine(_analyzeRuleService, _ruleRepo, _sessionRepo, _logger);
var results = await ruleEngine.AnalyzeSessionAsync(ruleTenantId, ruleSessionId);
var outcome = await ruleEngine.AnalyzeSessionAsync(ruleTenantId, ruleSessionId);
foreach (var result in results)
foreach (var result in outcome.Results)
{
await _ruleRepo.StoreRuleResultAsync(result);
}
if (results.Count > 0)
if (outcome.Results.Count > 0)
{
_logger.LogInformation("{Prefix} Enrollment-end analysis (async): {Count} issue(s) detected",
rulePrefix, results.Count);
rulePrefix, outcome.Results.Count);
// Push SignalR notification so the UI fetches results immediately
await _signalRNotification.NotifyRuleResultsAvailableAsync(
ruleTenantId, ruleSessionId, results.Count);
ruleTenantId, ruleSessionId, outcome.Results.Count);
}
// Increment platform stats for detected issues
if (results.Count > 0)
if (outcome.Results.Count > 0)
{
_ = _metricsRepo.IncrementPlatformStatAsync("IssuesDetected", results.Count)
_ = _metricsRepo.IncrementPlatformStatAsync("IssuesDetected", outcome.Results.Count)
.ContinueWith(t => _logger.LogWarning(t.Exception?.InnerException,
"Fire-and-forget IncrementPlatformStatAsync failed"), TaskContinuationOptions.OnlyOnFaulted);
}
// Record rule telemetry: increment stats for every evaluated rule
_ = RecordAnalyzeRuleStatsAsync(ruleTenantId, outcome);
}
catch (Exception ruleEx)
{
@@ -447,6 +450,9 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
_ = _metricsRepo.IncrementPlatformStatAsync("SuccessfulEnrollments")
.ContinueWith(t => _logger.LogWarning(t.Exception?.InnerException, "Fire-and-forget IncrementPlatformStatAsync failed"), TaskContinuationOptions.OnlyOnFaulted);
// Record gather rule telemetry for events that carry ruleId in their data
_ = RecordGatherRuleStatsAsync(request.TenantId, storedEvents);
// Store diagnostics blob name on session (if agent uploaded a diagnostics package)
if (classification.DiagnosticsUploadedEvent != null)
{
@@ -973,6 +979,93 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
maxFutureDriftHours);
}
}
/// <summary>
/// Records gather rule telemetry for events that carry a ruleId in their data.
/// Fire-and-forget — failures are logged but never propagated.
/// Gather rules have no evaluation count (they run on the agent); we only track fires.
/// </summary>
private async Task RecordGatherRuleStatsAsync(string tenantId, List<EnrollmentEvent> events)
{
try
{
var today = DateTime.UtcNow.ToString("yyyy-MM-dd");
// Gather rule results carry ruleId in their event data
var gatherEvents = events.Where(e =>
e.Data != null &&
e.Data.ContainsKey("ruleId") &&
e.Source == "GatherRuleExecutor").ToList();
if (gatherEvents.Count == 0) return;
// Deduplicate: count each unique ruleId once per ingest batch
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var evt in gatherEvents)
{
var ruleId = evt.Data!["ruleId"]?.ToString();
if (string.IsNullOrEmpty(ruleId) || !seen.Add(ruleId)) continue;
var ruleTitle = evt.Data.ContainsKey("ruleTitle") ? evt.Data["ruleTitle"]?.ToString() ?? "" : "";
// Tenant-specific row
await _metricsRepo.IncrementRuleStatAsync(
today, tenantId, ruleId, "gather",
ruleTitle, "", "",
fired: true, confidenceScore: null);
// Global aggregate row
await _metricsRepo.IncrementRuleStatAsync(
today, "global", ruleId, "gather",
ruleTitle, "", "",
fired: true, confidenceScore: null);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to record gather rule stats (non-fatal)");
}
}
/// <summary>
/// Records rule telemetry for all evaluated analyze rules.
/// Fire-and-forget — failures are logged but never propagated.
/// </summary>
private async Task RecordAnalyzeRuleStatsAsync(string tenantId, AnalysisOutcome outcome)
{
try
{
var today = DateTime.UtcNow.ToString("yyyy-MM-dd");
var firedRuleIds = new HashSet<string>(outcome.Results.Select(r => r.RuleId));
foreach (var rule in outcome.EvaluatedRules)
{
var fired = firedRuleIds.Contains(rule.RuleId);
int? confidence = null;
if (fired)
{
var result = outcome.Results.FirstOrDefault(r => r.RuleId == rule.RuleId);
confidence = result?.ConfidenceScore;
}
// Tenant-specific row
await _metricsRepo.IncrementRuleStatAsync(
today, tenantId, rule.RuleId, "analyze",
rule.Title, rule.Category, rule.Severity,
fired, confidence);
// Global aggregate row
await _metricsRepo.IncrementRuleStatAsync(
today, "global", rule.RuleId, "analyze",
rule.Title, rule.Category, rule.Severity,
fired, confidence);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to record analyze rule stats (non-fatal)");
}
}
}
/// <summary>
@@ -0,0 +1,189 @@
using System.Net;
using AutopilotMonitor.Functions.Helpers;
using AutopilotMonitor.Shared.DataAccess;
using Microsoft.Azure.Functions.Worker;
using Microsoft.Azure.Functions.Worker.Http;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Functions.Metrics
{
/// <summary>
/// Functions for retrieving rule telemetry stats.
/// Tenant-facing: per-tenant rule effectiveness metrics.
/// Global admin: cross-tenant rule stats and adoption summaries.
/// </summary>
public class RuleStatsFunction
{
private readonly ILogger<RuleStatsFunction> _logger;
private readonly IMetricsRepository _metricsRepo;
public RuleStatsFunction(
ILogger<RuleStatsFunction> logger,
IMetricsRepository metricsRepo)
{
_logger = logger;
_metricsRepo = metricsRepo;
}
/// <summary>
/// GET /api/metrics/rule-stats?startDate=...&endDate=...&ruleType=analyze
/// Returns rule stats for the caller's tenant (MemberRead).
/// </summary>
[Function("GetRuleStats")]
public async Task<HttpResponseData> GetTenantRuleStats(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "metrics/rule-stats")]
HttpRequestData req)
{
try
{
string tenantId = TenantHelper.GetTenantId(req);
var startDate = req.Query["startDate"] ?? DateTime.UtcNow.AddDays(-30).ToString("yyyy-MM-dd");
var endDate = req.Query["endDate"] ?? DateTime.UtcNow.ToString("yyyy-MM-dd");
var ruleType = req.Query["ruleType"];
_logger.LogInformation("Rule stats requested for tenant {TenantId} ({StartDate} to {EndDate})",
tenantId, startDate, endDate);
var entries = await _metricsRepo.GetRuleStatsAsync(tenantId, startDate, endDate, ruleType);
// Aggregate across dates per ruleId
var aggregated = entries
.GroupBy(e => e.RuleId)
.Select(g => new
{
ruleId = g.Key,
ruleType = g.First().RuleType,
ruleTitle = g.First().RuleTitle,
category = g.First().Category,
severity = g.First().Severity,
fireCount = g.Sum(e => e.FireCount),
evaluationCount = g.Sum(e => e.EvaluationCount),
sessionsEvaluated = g.Sum(e => e.SessionsEvaluated),
hitRate = g.Sum(e => e.EvaluationCount) > 0
? Math.Round(100.0 * g.Sum(e => e.FireCount) / g.Sum(e => e.EvaluationCount), 1)
: 0.0,
avgConfidenceScore = g.Sum(e => e.FireCount) > 0
? Math.Round((double)g.Sum(e => e.ConfidenceScoreSum) / g.Sum(e => e.FireCount), 1)
: 0.0,
trend = g.OrderBy(e => e.Date).Select(e => new
{
date = e.Date,
fireCount = e.FireCount,
evaluationCount = e.EvaluationCount
})
})
.OrderByDescending(r => r.fireCount)
.ToList();
var totalEvaluations = aggregated.Sum(r => r.evaluationCount);
var totalFires = aggregated.Sum(r => r.fireCount);
var result = new
{
rules = aggregated,
summary = new
{
totalEvaluations,
totalFires,
overallHitRate = totalEvaluations > 0
? Math.Round(100.0 * totalFires / totalEvaluations, 1)
: 0.0,
topRuleByFireCount = aggregated.FirstOrDefault()?.ruleId,
period = new { start = startDate, end = endDate }
}
};
var response = req.CreateResponse(HttpStatusCode.OK);
await response.WriteAsJsonAsync(result);
return response;
}
catch (Exception ex)
{
_logger.LogError(ex, "Error fetching tenant rule stats");
var errorResponse = req.CreateResponse(HttpStatusCode.InternalServerError);
await errorResponse.WriteAsJsonAsync(new { success = false, message = "Failed to fetch rule stats" });
return errorResponse;
}
}
/// <summary>
/// GET /api/global/metrics/rule-stats?startDate=...&endDate=...&ruleType=analyze
/// Returns cross-tenant global rule stats (Global Admin only).
/// </summary>
[Function("GetGlobalRuleStats")]
public async Task<HttpResponseData> GetGlobalRuleStats(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "global/metrics/rule-stats")]
HttpRequestData req)
{
try
{
var startDate = req.Query["startDate"] ?? DateTime.UtcNow.AddDays(-30).ToString("yyyy-MM-dd");
var endDate = req.Query["endDate"] ?? DateTime.UtcNow.ToString("yyyy-MM-dd");
var ruleType = req.Query["ruleType"];
_logger.LogInformation("Global rule stats requested ({StartDate} to {EndDate})", startDate, endDate);
// Fetch global aggregate rows
var entries = await _metricsRepo.GetRuleStatsAsync("global", startDate, endDate, ruleType);
var aggregated = entries
.GroupBy(e => e.RuleId)
.Select(g => new
{
ruleId = g.Key,
ruleType = g.First().RuleType,
ruleTitle = g.First().RuleTitle,
category = g.First().Category,
severity = g.First().Severity,
fireCount = g.Sum(e => e.FireCount),
evaluationCount = g.Sum(e => e.EvaluationCount),
sessionsEvaluated = g.Sum(e => e.SessionsEvaluated),
hitRate = g.Sum(e => e.EvaluationCount) > 0
? Math.Round(100.0 * g.Sum(e => e.FireCount) / g.Sum(e => e.EvaluationCount), 1)
: 0.0,
avgConfidenceScore = g.Sum(e => e.FireCount) > 0
? Math.Round((double)g.Sum(e => e.ConfidenceScoreSum) / g.Sum(e => e.FireCount), 1)
: 0.0,
trend = g.OrderBy(e => e.Date).Select(e => new
{
date = e.Date,
fireCount = e.FireCount,
evaluationCount = e.EvaluationCount
})
})
.OrderByDescending(r => r.fireCount)
.ToList();
var totalEvaluations = aggregated.Sum(r => r.evaluationCount);
var totalFires = aggregated.Sum(r => r.fireCount);
var result = new
{
rules = aggregated,
summary = new
{
totalEvaluations,
totalFires,
overallHitRate = totalEvaluations > 0
? Math.Round(100.0 * totalFires / totalEvaluations, 1)
: 0.0,
topRuleByFireCount = aggregated.FirstOrDefault()?.ruleId,
uniqueRules = aggregated.Count,
period = new { start = startDate, end = endDate }
}
};
var response = req.CreateResponse(HttpStatusCode.OK);
await response.WriteAsJsonAsync(result);
return response;
}
catch (Exception ex)
{
_logger.LogError(ex, "Error fetching global rule stats");
var errorResponse = req.CreateResponse(HttpStatusCode.InternalServerError);
await errorResponse.WriteAsJsonAsync(new { success = false, message = "Failed to fetch global rule stats" });
return errorResponse;
}
}
}
}
@@ -62,17 +62,17 @@ namespace AutopilotMonitor.Functions.Functions.Rules
try
{
var ruleEngine = new RuleEngine(_analyzeRuleService, _ruleRepo, _sessionRepo, _logger);
var freshResults = await ruleEngine.AnalyzeSessionAsync(effectiveTenantId, sessionId, reanalyze: true);
var outcome = await ruleEngine.AnalyzeSessionAsync(effectiveTenantId, sessionId, reanalyze: true);
// Delete existing results so stale entries don't persist after re-analysis
await _maintenanceRepo.DeleteSessionRuleResultsAsync(effectiveTenantId, sessionId);
foreach (var result in freshResults)
foreach (var result in outcome.Results)
{
await _ruleRepo.StoreRuleResultAsync(result);
}
_logger.LogInformation($"On-demand re-analysis for session {sessionId}: {freshResults.Count} issue(s) detected");
_logger.LogInformation($"On-demand re-analysis for session {sessionId}: {outcome.Results.Count} issue(s) detected");
}
catch (Exception ex)
{
@@ -144,6 +144,7 @@ public static class EndpointAccessPolicyCatalog
new("GET", "apps/{appName}/analytics", EndpointPolicy.MemberRead),
new("GET", "apps/{appName}/sessions", EndpointPolicy.MemberRead),
new("GET", "metrics/usage", EndpointPolicy.MemberRead),
new("GET", "metrics/rule-stats", EndpointPolicy.MemberRead),
new("GET", "metrics/geographic", EndpointPolicy.MemberRead),
new("GET", "metrics/mcp-usage/user/{userId}", EndpointPolicy.TenantAdminOrGA),
new("GET", "metrics/geographic/sessions", EndpointPolicy.MemberRead),
@@ -227,6 +228,7 @@ public static class EndpointAccessPolicyCatalog
new("GET", "global/metrics/geographic", EndpointPolicy.GlobalAdminOnly),
new("GET", "global/metrics/geographic/sessions", EndpointPolicy.GlobalAdminOnly),
new("GET", "global/metrics/usage", EndpointPolicy.GlobalAdminOnly),
new("GET", "global/metrics/rule-stats", EndpointPolicy.GlobalAdminOnly),
new("GET", "global/metrics/mcp-usage", EndpointPolicy.GlobalAdminOnly),
new("GET", "global/metrics/mcp-usage/daily", EndpointPolicy.GlobalAdminOnly),
new("GET", "global/distress-reports", EndpointPolicy.GlobalAdminOnly),
@@ -81,6 +81,9 @@ namespace AutopilotMonitor.Functions.Services
await _metricsRepo.SaveUsageMetricsSnapshotAsync(tenantMetrics);
}
// Aggregate rule stats from RuleResults table for this date
await AggregateRuleStatsForDateAsync(targetDate, targetDateSessions);
aggregateStart.Stop();
_logger.LogInformation($"Aggregated metrics for {targetDateSessions.Count} sessions from {targetDateStr} in {aggregateStart.ElapsedMilliseconds}ms");
}
@@ -256,6 +259,68 @@ namespace AutopilotMonitor.Functions.Services
}
}
/// <summary>
/// Reconciles rule stats for a given date by computing global aggregate rows
/// from per-tenant rows. This ensures consistency even if real-time global
/// increments were missed (e.g. during transient failures).
/// </summary>
private async Task AggregateRuleStatsForDateAsync(DateTime targetDate, List<SessionSummary> sessions)
{
try
{
var dateStr = targetDate.ToString("yyyy-MM-dd");
// Fetch all tenant-specific rows for this date (excluding existing global rows)
var allEntries = await _metricsRepo.GetRuleStatsAsync(startDate: dateStr, endDate: dateStr);
var tenantEntries = allEntries.Where(e => e.TenantId != "global").ToList();
if (tenantEntries.Count == 0)
{
_logger.LogInformation("No tenant rule stats for {Date}, skipping rule stats aggregation", dateStr);
return;
}
// Group by RuleId to compute global aggregates
var groups = tenantEntries.GroupBy(e => e.RuleId);
int written = 0;
foreach (var group in groups)
{
var first = group.First();
var totalFire = group.Sum(e => e.FireCount);
var totalEval = group.Sum(e => e.EvaluationCount);
var totalSessions = group.Sum(e => e.SessionsEvaluated);
var totalConfSum = group.Sum(e => e.ConfidenceScoreSum);
var globalEntry = new RuleStatsEntry
{
Date = dateStr,
TenantId = "global",
RuleId = first.RuleId,
RuleType = first.RuleType,
RuleTitle = first.RuleTitle,
Category = first.Category,
Severity = first.Severity,
FireCount = totalFire,
EvaluationCount = totalEval,
SessionsEvaluated = totalSessions,
ConfidenceScoreSum = totalConfSum,
AvgConfidenceScore = totalFire > 0 ? Math.Round((double)totalConfSum / totalFire, 1) : 0,
UpdatedAt = DateTime.UtcNow
};
await _metricsRepo.SaveRuleStatsEntryAsync(globalEntry);
written++;
}
_logger.LogInformation("Rule stats aggregation for {Date}: reconciled {Count} global rule entries from {TenantEntries} tenant entries",
dateStr, written, tenantEntries.Count);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to aggregate rule stats for {Date} (non-fatal)", targetDate.ToString("yyyy-MM-dd"));
}
}
/// <summary>
/// Deletes usage tracking records older than 90 days from UserUsageLog.
/// </summary>
@@ -273,6 +338,20 @@ namespace AutopilotMonitor.Functions.Services
{
_logger.LogWarning(ex, "Failed to cleanup old usage data");
}
// Rule stats retention: delete entries older than 90 days
try
{
var ruleStatsCutoff = DateTime.UtcNow.AddDays(-90);
var deletedRuleStats = await _metricsRepo.DeleteRuleStatsOlderThanAsync(ruleStatsCutoff);
if (deletedRuleStats > 0)
_logger.LogInformation("Rule stats cleanup: deleted {Count} entries older than 90 days", deletedRuleStats);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to cleanup old rule stats");
}
}
/// <summary>
@@ -29,10 +29,11 @@ namespace AutopilotMonitor.Functions.Services
/// Evaluates ALL active analyze rules against the full session event stream.
/// Called once at enrollment end or on-demand. Fetches events internally.
/// When reanalyze=true, all rules are re-evaluated regardless of existing results.
/// Returns both the fired results and metadata about all rules that were evaluated (for telemetry).
/// </summary>
public async Task<List<RuleResult>> AnalyzeSessionAsync(string tenantId, string sessionId, bool reanalyze = false)
public async Task<AnalysisOutcome> AnalyzeSessionAsync(string tenantId, string sessionId, bool reanalyze = false)
{
var results = new List<RuleResult>();
var outcome = new AnalysisOutcome();
try
{
@@ -42,7 +43,7 @@ namespace AutopilotMonitor.Functions.Services
if (allEvents.Count == 0)
{
_logger.LogInformation($"No events found for session {sessionId}, skipping analysis");
return results;
return outcome;
}
// On reanalyze: skip deduplication so all rules are re-evaluated from scratch
@@ -68,12 +69,15 @@ namespace AutopilotMonitor.Functions.Services
if (existingRuleIds.Contains(rule.RuleId))
continue;
// Track that this rule was evaluated (for telemetry)
outcome.EvaluatedRules.Add(rule);
var result = EvaluateRule(rule, allEvents);
if (result != null)
{
result.SessionId = sessionId;
result.TenantId = tenantId;
results.Add(result);
outcome.Results.Add(result);
_logger.LogInformation($"Rule {rule.RuleId} ({rule.Trigger}) fired for session {sessionId} with confidence {result.ConfidenceScore}%");
}
}
@@ -88,7 +92,7 @@ namespace AutopilotMonitor.Functions.Services
_logger.LogError(ex, "Error analyzing session");
}
return results;
return outcome;
}
/// <summary>
@@ -161,4 +165,16 @@ namespace AutopilotMonitor.Functions.Services
/// Evaluates a single condition against the event stream
/// </summary>
}
/// <summary>
/// Return type for AnalyzeSessionAsync — includes both fired results and evaluation metadata for telemetry.
/// </summary>
public class AnalysisOutcome
{
/// <summary>Rules that fired (produced a result)</summary>
public List<RuleResult> Results { get; set; } = new List<RuleResult>();
/// <summary>All rules that were evaluated in this pass (includes rules that didn't fire)</summary>
public List<AnalyzeRule> EvaluatedRules { get; set; } = new List<AnalyzeRule>();
}
}
@@ -688,5 +688,220 @@ namespace AutopilotMonitor.Functions.Services
return (0, 0);
}
}
// ===== RULE STATS METHODS =====
/// <summary>
/// Increments rule stats counters atomically for a single rule evaluation.
/// Creates the row if it doesn't exist. Uses read-modify-write pattern.
/// Called once per rule per session evaluation (for both tenant-specific and global rows).
/// </summary>
public async Task IncrementRuleStatAsync(
string date, string tenantId, string ruleId, string ruleType,
string ruleTitle, string category, string severity,
bool fired, int? confidenceScore)
{
try
{
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.RuleStats);
var rowKey = $"{tenantId}_{ruleId}";
TableEntity entity;
try
{
var response = await tableClient.GetEntityAsync<TableEntity>(date, rowKey);
entity = response.Value;
}
catch (RequestFailedException ex) when (ex.Status == 404)
{
entity = new TableEntity(date, rowKey)
{
["RuleId"] = ruleId,
["RuleType"] = ruleType,
["RuleTitle"] = ruleTitle,
["Category"] = category,
["Severity"] = severity,
["FireCount"] = 0,
["EvaluationCount"] = 0,
["SessionsEvaluated"] = 0,
["ConfidenceScoreSum"] = 0L,
["AvgConfidenceScore"] = 0.0,
["UpdatedAt"] = DateTime.UtcNow
};
}
// Always increment evaluation and session counters
entity["EvaluationCount"] = (entity.GetInt32("EvaluationCount") ?? 0) + 1;
entity["SessionsEvaluated"] = (entity.GetInt32("SessionsEvaluated") ?? 0) + 1;
if (fired)
{
var newFireCount = (entity.GetInt32("FireCount") ?? 0) + 1;
entity["FireCount"] = newFireCount;
if (confidenceScore.HasValue)
{
var newSum = (entity.GetInt64("ConfidenceScoreSum") ?? 0) + confidenceScore.Value;
entity["ConfidenceScoreSum"] = newSum;
entity["AvgConfidenceScore"] = newFireCount > 0 ? (double)newSum / newFireCount : 0.0;
}
}
// Keep metadata fresh
entity["RuleTitle"] = ruleTitle;
entity["Category"] = category;
entity["Severity"] = severity;
entity["UpdatedAt"] = DateTime.UtcNow;
await tableClient.UpsertEntityAsync(entity);
}
catch (Exception ex)
{
// Non-fatal: don't break the caller if stats update fails
_logger.LogWarning(ex, "Failed to increment rule stat for {RuleId} / {TenantId}", ruleId, tenantId);
}
}
/// <summary>
/// Saves a fully computed rule stats entry (used by daily aggregation).
/// </summary>
public async Task<bool> SaveRuleStatsEntryAsync(RuleStatsEntry entry)
{
try
{
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.RuleStats);
var rowKey = $"{entry.TenantId}_{entry.RuleId}";
var entity = new TableEntity(entry.Date, rowKey)
{
["RuleId"] = entry.RuleId,
["RuleType"] = entry.RuleType,
["RuleTitle"] = entry.RuleTitle,
["Category"] = entry.Category,
["Severity"] = entry.Severity,
["FireCount"] = entry.FireCount,
["EvaluationCount"] = entry.EvaluationCount,
["SessionsEvaluated"] = entry.SessionsEvaluated,
["AvgConfidenceScore"] = entry.AvgConfidenceScore,
["ConfidenceScoreSum"] = entry.ConfidenceScoreSum,
["UpdatedAt"] = entry.UpdatedAt
};
await tableClient.UpsertEntityAsync(entity);
return true;
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to save rule stats entry for {RuleId} / {TenantId} / {Date}",
entry.RuleId, entry.TenantId, entry.Date);
return false;
}
}
/// <summary>
/// Gets rule stats entries for a date range, optionally filtered by tenant and/or rule type.
/// </summary>
public async Task<List<RuleStatsEntry>> GetRuleStatsAsync(
string? tenantId = null, string? startDate = null, string? endDate = null,
string? ruleType = null, int maxResults = 500)
{
try
{
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.RuleStats);
var filters = new List<string>();
if (!string.IsNullOrEmpty(startDate))
filters.Add($"PartitionKey ge '{startDate}'");
if (!string.IsNullOrEmpty(endDate))
filters.Add($"PartitionKey le '{endDate}'");
// Filter by tenant via RowKey prefix
if (!string.IsNullOrEmpty(tenantId))
{
filters.Add($"RowKey ge '{tenantId}_'");
filters.Add($"RowKey lt '{tenantId}_~'"); // ~ is after all printable ASCII
}
if (!string.IsNullOrEmpty(ruleType))
filters.Add($"RuleType eq '{ruleType}'");
var filter = filters.Count > 0 ? string.Join(" and ", filters) : null;
var query = tableClient.QueryAsync<TableEntity>(filter: filter);
var results = new List<RuleStatsEntry>();
await foreach (var entity in query)
{
results.Add(MapToRuleStatsEntry(entity));
if (results.Count >= maxResults) break;
}
return results;
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to get rule stats");
return new List<RuleStatsEntry>();
}
}
/// <summary>
/// Deletes rule stats entries older than a given date (retention cleanup).
/// </summary>
public async Task<int> DeleteRuleStatsOlderThanAsync(DateTime cutoffDate)
{
try
{
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.RuleStats);
var cutoffStr = cutoffDate.ToString("yyyy-MM-dd");
var filter = $"PartitionKey lt '{cutoffStr}'";
var query = tableClient.QueryAsync<TableEntity>(filter: filter, select: new[] { "PartitionKey", "RowKey" });
int deleted = 0;
await foreach (var entity in query)
{
await tableClient.DeleteEntityAsync(entity.PartitionKey, entity.RowKey);
deleted++;
}
if (deleted > 0)
_logger.LogInformation("Deleted {Count} rule stats entries older than {Cutoff}", deleted, cutoffStr);
return deleted;
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to delete old rule stats entries");
return 0;
}
}
private static RuleStatsEntry MapToRuleStatsEntry(TableEntity entity)
{
var rowKey = entity.RowKey ?? string.Empty;
var separatorIndex = rowKey.IndexOf('_');
var tenantId = separatorIndex > 0 ? rowKey.Substring(0, separatorIndex) : rowKey;
// RuleId may contain underscores, so take everything after the first underscore
var ruleId = separatorIndex > 0 && separatorIndex < rowKey.Length - 1
? rowKey.Substring(separatorIndex + 1)
: entity.GetString("RuleId") ?? string.Empty;
return new RuleStatsEntry
{
Date = entity.PartitionKey ?? string.Empty,
TenantId = tenantId,
RuleId = entity.GetString("RuleId") ?? ruleId,
RuleType = entity.GetString("RuleType") ?? string.Empty,
RuleTitle = entity.GetString("RuleTitle") ?? string.Empty,
Category = entity.GetString("Category") ?? string.Empty,
Severity = entity.GetString("Severity") ?? string.Empty,
FireCount = entity.GetInt32("FireCount") ?? 0,
EvaluationCount = entity.GetInt32("EvaluationCount") ?? 0,
SessionsEvaluated = entity.GetInt32("SessionsEvaluated") ?? 0,
AvgConfidenceScore = entity.GetDouble("AvgConfidenceScore") ?? 0,
ConfidenceScoreSum = entity.GetInt64("ConfidenceScoreSum") ?? 0,
UpdatedAt = entity.GetDateTimeOffset("UpdatedAt")?.UtcDateTime ?? DateTime.UtcNow
};
}
}
}
@@ -396,4 +396,36 @@ export function registerAdminTools(server: McpServer): void {
}
}
);
// Tool: get_rule_stats
server.tool(
'get_rule_stats',
'Get rule firing statistics for analyze and gather rules. Shows which rules fire most often, ' +
'their hit rates (fires/evaluations), and daily trends. Use to identify commonly triggered rules, ' +
'optimize rule definitions, or understand tenant-specific failure patterns. ' +
'Without tenantId returns global stats (cross-tenant). With tenantId returns tenant-specific stats.',
{
tenantId: z.string().optional().describe('Filter by tenant ID. Omit for global (cross-tenant) stats.'),
ruleType: z.enum(['analyze', 'gather']).optional().describe('Filter by rule type'),
startDate: z.string().optional().describe('Start date (YYYY-MM-DD). Defaults to 30 days ago.'),
endDate: z.string().optional().describe('End date (YYYY-MM-DD). Defaults to today.'),
},
READ_ONLY,
async (args) => {
try {
const params = buildQuery({
startDate: args.startDate,
endDate: args.endDate,
ruleType: args.ruleType,
});
const path = args.tenantId
? `/api/metrics/rule-stats${params}`
: `/api/global/metrics/rule-stats${params}`;
const data = await apiFetch(path);
return { content: [{ type: 'text' as const, text: JSON.stringify(data, null, 2) }] };
} catch (error: unknown) {
return toolError('get_rule_stats', args, error);
}
}
);
}
@@ -260,6 +260,9 @@ namespace AutopilotMonitor.Shared
public const string AppInstallSummaries = "AppInstallSummaries";
public const string PlatformStats = "PlatformStats";
// Rule telemetry (daily per-rule fire/evaluation counters)
public const string RuleStats = "RuleStats";
// Configuration tables
public const string TenantConfiguration = "TenantConfiguration";
public const string AdminConfiguration = "AdminConfiguration";
@@ -353,7 +356,8 @@ namespace AutopilotMonitor.Shared
DistressReports,
UserUsageLog,
OpsEvents,
ImeVersionHistory
ImeVersionHistory,
RuleStats
};
}
}
@@ -35,6 +35,14 @@ namespace AutopilotMonitor.Shared.DataAccess
// --- Metrics Summary (Agent API) ---
Task<List<object>> GetMetricsSummaryAsync(string? tenantId);
// --- Rule Stats ---
Task IncrementRuleStatAsync(string date, string tenantId, string ruleId, string ruleType,
string ruleTitle, string category, string severity, bool fired, int? confidenceScore);
Task<bool> SaveRuleStatsEntryAsync(RuleStatsEntry entry);
Task<List<RuleStatsEntry>> GetRuleStatsAsync(string? tenantId = null, string? startDate = null,
string? endDate = null, string? ruleType = null, int maxResults = 500);
Task<int> DeleteRuleStatsOlderThanAsync(DateTime cutoffDate);
}
public class UserActivityMetrics
@@ -0,0 +1,79 @@
using System;
namespace AutopilotMonitor.Shared.Models
{
/// <summary>
/// Daily per-rule telemetry entry.
/// Stored in Azure Table Storage for rule effectiveness analysis and tenant dashboards.
/// PartitionKey = Date (YYYY-MM-DD format)
/// RowKey = "{TenantId}_{RuleId}" (tenant-scoped) or "global_{RuleId}" (cross-tenant aggregate)
/// </summary>
public class RuleStatsEntry
{
/// <summary>
/// Date of the stats entry (YYYY-MM-DD) — maps to PartitionKey
/// </summary>
public string Date { get; set; } = string.Empty;
/// <summary>
/// Tenant ID or "global" for cross-tenant — part of RowKey
/// </summary>
public string TenantId { get; set; } = string.Empty;
/// <summary>
/// Rule identifier (e.g., "ANALYZE-NET-001" or "GATHER-REG-001")
/// </summary>
public string RuleId { get; set; } = string.Empty;
/// <summary>
/// "analyze" or "gather"
/// </summary>
public string RuleType { get; set; } = string.Empty;
/// <summary>
/// Human-readable rule title (denormalized for display)
/// </summary>
public string RuleTitle { get; set; } = string.Empty;
/// <summary>
/// Rule category (network, identity, enrollment, apps, esp, device)
/// </summary>
public string Category { get; set; } = string.Empty;
/// <summary>
/// Rule severity (info, warning, high, critical)
/// </summary>
public string Severity { get; set; } = string.Empty;
/// <summary>
/// Number of times this rule fired (produced a result/event) on this day
/// </summary>
public int FireCount { get; set; }
/// <summary>
/// Number of times this rule was evaluated on this day (analyze rules only; 0 for gather rules)
/// </summary>
public int EvaluationCount { get; set; }
/// <summary>
/// Number of distinct sessions where this rule was evaluated
/// </summary>
public int SessionsEvaluated { get; set; }
/// <summary>
/// Average confidence score when the rule fired (analyze rules only; 0 for gather rules)
/// </summary>
public double AvgConfidenceScore { get; set; }
/// <summary>
/// Sum of confidence scores — used for incremental average computation.
/// AvgConfidenceScore = ConfidenceScoreSum / FireCount
/// </summary>
public long ConfidenceScoreSum { get; set; }
/// <summary>
/// When this entry was last updated
/// </summary>
public DateTime UpdatedAt { get; set; } = DateTime.UtcNow;
}
}
@@ -34,6 +34,8 @@ interface AnalyzeRuleCardProps {
templateCopyExists?: boolean;
templateCopyRuleId?: string;
onScrollToCopy?: (ruleId: string) => void;
hitRate?: number | null;
fireCount?: number | null;
}
export default function AnalyzeRuleCard({
@@ -45,6 +47,7 @@ export default function AnalyzeRuleCard({
onSetJsonModeEdit, onSetJsonText, onSetJsonError,
readOnly = false,
onConfigureTemplate, templateCopyExists, templateCopyRuleId, onScrollToCopy,
hitRate, fireCount,
}: AnalyzeRuleCardProps) {
const [showJson, setShowJson] = useState(false);
const sevColor = getSeverityColor(rule.severity);
@@ -104,6 +107,18 @@ export default function AnalyzeRuleCard({
</span>
)}
<span className="text-xs text-gray-500 flex-shrink-0 hidden md:inline" title="Confidence Threshold">Threshold: {rule.confidenceThreshold}%</span>
{hitRate != null && hitRate > 0 && (
<span
className={`inline-flex items-center px-2 py-0.5 rounded text-xs font-medium flex-shrink-0 ${
hitRate >= 20 ? "bg-red-50 text-red-700 border border-red-200" :
hitRate >= 5 ? "bg-amber-50 text-amber-700 border border-amber-200" :
"bg-gray-50 text-gray-600 border border-gray-200"
}`}
title={`Fires on ${hitRate}% of evaluated sessions (${fireCount ?? 0} total fires in last 30 days)`}
>
{hitRate}% hit rate
</span>
)}
<svg className={`w-5 h-5 text-gray-400 transition-transform flex-shrink-0 ${isExpanded ? "rotate-180" : ""}`} fill="none" stroke="currentColor" viewBox="0 0 24 24"><path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M19 9l-7 7-7-7" /></svg>
</div>
</div>
@@ -76,6 +76,9 @@ export default function AnalyzeRulesPage() {
const [configureTemplateRule, setConfigureTemplateRule] = useState<AnalyzeRule | null>(null);
const [creatingFromTemplate, setCreatingFromTemplate] = useState(false);
// Rule telemetry stats (hit rates)
const [ruleStatsMap, setRuleStatsMap] = useState<Record<string, { hitRate: number; fireCount: number }>>({});
// Global admin mode
const { globalAdminMode } = useAdminMode();
const [tenants, setTenants] = useState<TenantInfo[]>([]);
@@ -132,6 +135,32 @@ export default function AnalyzeRulesPage() {
fetchRules();
}, [fetchRules]);
// Fetch rule telemetry stats (hit rates for last 30 days)
useEffect(() => {
if (!effectiveTenantId) return;
const fetchStats = async () => {
try {
const statsUrl = globalAdminMode && user?.isGlobalAdmin
? api.metrics.globalRuleStats(undefined, undefined, "analyze")
: api.metrics.ruleStats(undefined, undefined, "analyze");
const response = await authenticatedFetch(statsUrl, getAccessToken);
if (response.ok) {
const data = await response.json();
const map: Record<string, { hitRate: number; fireCount: number }> = {};
if (data.rules && Array.isArray(data.rules)) {
for (const r of data.rules) {
map[r.ruleId] = { hitRate: r.hitRate ?? 0, fireCount: r.fireCount ?? 0 };
}
}
setRuleStatsMap(map);
}
} catch {
// Non-critical: don't break the page if stats fail to load
}
};
fetchStats();
}, [effectiveTenantId, globalAdminMode, user?.isGlobalAdmin, getAccessToken]);
// Toggle rule enabled/disabled
const handleToggleRule = async (rule: AnalyzeRule) => {
// Intercept: if this is a template rule being enabled, open the config modal instead
@@ -456,6 +485,25 @@ export default function AnalyzeRulesPage() {
<StatCard label="Info" value={infoCount} borderColor="border-blue-400" valueColor="text-blue-600" />
</div>
{/* Telemetry Stats (only shown when data is available) */}
{Object.keys(ruleStatsMap).length > 0 && (() => {
const stats = Object.values(ruleStatsMap);
const totalFires = stats.reduce((sum, s) => sum + s.fireCount, 0);
const rulesWithHits = stats.filter(s => s.fireCount > 0).length;
const highHitRules = stats.filter(s => s.hitRate >= 20).length;
const avgHitRate = stats.length > 0
? Math.round(stats.reduce((sum, s) => sum + s.hitRate, 0) / stats.length * 10) / 10
: 0;
return (
<div className="grid grid-cols-2 md:grid-cols-4 gap-4">
<StatCard label="Total Fires (30d)" value={totalFires} borderColor="border-indigo-400" valueColor="text-indigo-600" />
<StatCard label="Rules Firing" value={rulesWithHits} borderColor="border-indigo-400" valueColor="text-indigo-600" />
<StatCard label="Avg Hit Rate" value={`${avgHitRate}%`} borderColor="border-indigo-400" valueColor="text-indigo-600" />
<StatCard label="High Hit Rate (>20%)" value={highHitRules} borderColor="border-red-400" valueColor="text-red-600" />
</div>
);
})()}
{/* Filter Bar + Create Button */}
<RuleFilterBar
searchQuery={searchQuery}
@@ -615,6 +663,8 @@ export default function AnalyzeRulesPage() {
templateCopyExists={templateCopyMap.has(rule.ruleId)}
templateCopyRuleId={templateCopyMap.get(rule.ruleId)}
onScrollToCopy={(copyId) => setExpandedRuleId(copyId)}
hitRate={ruleStatsMap[rule.ruleId]?.hitRate ?? null}
fireCount={ruleStatsMap[rule.ruleId]?.fireCount ?? null}
/>
))}
</div>
@@ -1,8 +1,12 @@
"use client";
import { useState } from "react";
import { useState, useEffect } from "react";
import { RuleResult } from "@/types";
import { formatInlineMarkdown } from "@/lib/formatInlineMarkdown";
import { api } from "@/lib/api";
import { authenticatedFetch } from "@/lib/authenticatedFetch";
import { useAuth } from "@/contexts/AuthContext";
import { useTenant } from "@/contexts/TenantContext";
interface AnalysisResultsSectionProps {
analysisResults: RuleResult[];
@@ -19,6 +23,35 @@ export default function AnalysisResultsSection({
setAnalysisExpanded,
onReanalyze,
}: AnalysisResultsSectionProps) {
const { getAccessToken } = useAuth();
const { tenantId } = useTenant();
const [ruleHitRates, setRuleHitRates] = useState<Record<string, number>>({});
useEffect(() => {
if (!tenantId || analysisResults.length === 0) return;
const fetchStats = async () => {
try {
const response = await authenticatedFetch(
api.metrics.ruleStats(undefined, undefined, "analyze"),
getAccessToken
);
if (response.ok) {
const data = await response.json();
const map: Record<string, number> = {};
if (data.rules && Array.isArray(data.rules)) {
for (const r of data.rules) {
if (r.hitRate > 0) map[r.ruleId] = r.hitRate;
}
}
setRuleHitRates(map);
}
} catch {
// Non-critical
}
};
fetchStats();
}, [tenantId, analysisResults.length, getAccessToken]);
return (
<div className="bg-white shadow rounded-lg p-6 mb-6">
<div
@@ -94,7 +127,7 @@ export default function AnalysisResultsSection({
) : (
<div className="space-y-3">
{analysisResults.map((result) => (
<AnalysisResultCard key={result.ruleId} result={result} />
<AnalysisResultCard key={result.ruleId} result={result} hitRate={ruleHitRates[result.ruleId]} />
))}
</div>
)}
@@ -104,7 +137,7 @@ export default function AnalysisResultsSection({
);
}
function AnalysisResultCard({ result }: { result: RuleResult }) {
function AnalysisResultCard({ result, hitRate }: { result: RuleResult; hitRate?: number }) {
const [expanded, setExpanded] = useState(false);
const severityColors: Record<string, string> = {
@@ -151,6 +184,11 @@ function AnalysisResultCard({ result }: { result: RuleResult }) {
</div>
<span className="text-xs font-medium text-gray-700">{result.confidenceScore}%</span>
</div>
{hitRate != null && hitRate > 0 && (
<span className="text-xs text-gray-500 italic" title="Based on rule telemetry from the last 30 days">
Fires on {hitRate}% of enrollments in your tenant
</span>
)}
</div>
</div>
<svg className={`w-5 h-5 text-gray-400 ml-2 transition-transform duration-200 flex-shrink-0 ${expanded ? 'rotate-90' : ''}`} fill="none" stroke="currentColor" viewBox="0 0 24 24">
+4
View File
@@ -145,6 +145,10 @@ export const api = {
globalGeographicSessions: (days: number, groupBy: string, locationKey: string) =>
`${API_BASE_URL}/api/global/metrics/geographic/sessions${qs({ days: String(days), groupBy, locationKey })}`,
platform: () => `${API_BASE_URL}/api/global/metrics/platform`,
ruleStats: (startDate?: string, endDate?: string, ruleType?: string) =>
`${API_BASE_URL}/api/metrics/rule-stats${qs({ startDate, endDate, ruleType })}`,
globalRuleStats: (startDate?: string, endDate?: string, ruleType?: string) =>
`${API_BASE_URL}/api/global/metrics/rule-stats${qs({ startDate, endDate, ruleType })}`,
},
// ── Apps Dashboard ────────────────────────────────────────────────────────