mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 12:42:38 +00:00
perf(backend): project index scans + bound AppInstall reads to window
Two classes of read-efficiency fixes on hot/aggregation paths. No response shape or value changes — DTOs untouched, all 2541 backend tests pass. P2 — select projections on full-mirror SessionsIndex scans (bandwidth only): - GetMetricsSummaryAsync: cross-partition status tally now projects [PartitionKey, Status] instead of the ~40-column mirror row (incl. the large FailureSnapshotJson) -> ~10-20x less transfer. - QuickSearchSessionsAsync: typeahead scan projects only the matched fields (+ RowKey for the SessionId fallback) instead of the full row, for up to QuickSearchMaxScannedRows rows per request. P3 — push the existing sinceUtc window floor server-side at 8 AppInstall call sites across 5 files (previously each dematerialized the whole table / partition and filtered to the window in memory): - UsageMetricsService (global + tenant): now.AddDays(-days) - MaintenanceService.Aggregation (global + tenant, per aggregated day): targetDate - SlaBreachEvaluationService (current ISO week): now.AddDays(-8) margin - SlaMetricsService: startDate (matches the in-memory lower bound exactly) - Get(Global)GeographicLocationSessionsFunction: cutoff Each floor equals the session-window floor the apps are joined against (app.StartedAt >= session.StartedAt >= floor), so results are unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+2
-1
@@ -69,7 +69,8 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
|
|||||||
? FilterSessionsByLocation(sessions, locationKey, groupBy)
|
? FilterSessionsByLocation(sessions, locationKey, groupBy)
|
||||||
: FilterSessionsByFields(sessions, country!, query["region"], query["city"]);
|
: FilterSessionsByFields(sessions, country!, query["region"], query["city"]);
|
||||||
|
|
||||||
var appSummaries = await _metricsRepo.GetAppInstallSummariesByTenantAsync(tenantId);
|
// Same window as the sessions above (cutoff); BuildRows joins apps to those sessions.
|
||||||
|
var appSummaries = await _metricsRepo.GetAppInstallSummariesByTenantAsync(tenantId, cutoff);
|
||||||
var rows = BuildRows(filtered, appSummaries);
|
var rows = BuildRows(filtered, appSummaries);
|
||||||
|
|
||||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||||
|
|||||||
+3
-2
@@ -67,9 +67,10 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
|
|||||||
? GetGeographicLocationSessionsFunction.FilterSessionsByLocation(sessions, locationKey, groupBy)
|
? GetGeographicLocationSessionsFunction.FilterSessionsByLocation(sessions, locationKey, groupBy)
|
||||||
: GetGeographicLocationSessionsFunction.FilterSessionsByFields(sessions, country!, query["region"], query["city"]);
|
: GetGeographicLocationSessionsFunction.FilterSessionsByFields(sessions, country!, query["region"], query["city"]);
|
||||||
|
|
||||||
|
// Same window as the sessions above (cutoff); BuildRows joins apps to those sessions.
|
||||||
var appSummaries = string.IsNullOrEmpty(filterTenantId)
|
var appSummaries = string.IsNullOrEmpty(filterTenantId)
|
||||||
? await _metricsRepo.GetAllAppInstallSummariesAsync()
|
? await _metricsRepo.GetAllAppInstallSummariesAsync(cutoff)
|
||||||
: await _metricsRepo.GetAppInstallSummariesByTenantAsync(filterTenantId);
|
: await _metricsRepo.GetAppInstallSummariesByTenantAsync(filterTenantId, cutoff);
|
||||||
var rows = GetGeographicLocationSessionsFunction.BuildRows(filtered, appSummaries);
|
var rows = GetGeographicLocationSessionsFunction.BuildRows(filtered, appSummaries);
|
||||||
|
|
||||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||||
|
|||||||
@@ -139,10 +139,13 @@ namespace AutopilotMonitor.Functions.Services
|
|||||||
// App metrics: count apps per session from AppInstallSummaries table
|
// App metrics: count apps per session from AppInstallSummaries table
|
||||||
var sessionIdSet = new HashSet<string>(sessions.Select(s => s.SessionId));
|
var sessionIdSet = new HashSet<string>(sessions.Select(s => s.SessionId));
|
||||||
List<AppInstallSummary> appSummaries;
|
List<AppInstallSummary> appSummaries;
|
||||||
|
// Bound the AppInstall scan to this snapshot's day. The session set is already
|
||||||
|
// [targetDate, targetDate+1) and apps install during their session (StartedAt >= targetDate),
|
||||||
|
// so relevantApps is unchanged; without sinceUtc this scanned the whole table per aggregated day.
|
||||||
if (tenantId == "global")
|
if (tenantId == "global")
|
||||||
appSummaries = await _metricsRepo.GetAllAppInstallSummariesAsync();
|
appSummaries = await _metricsRepo.GetAllAppInstallSummariesAsync(targetDate);
|
||||||
else
|
else
|
||||||
appSummaries = await _metricsRepo.GetAppInstallSummariesByTenantAsync(tenantId);
|
appSummaries = await _metricsRepo.GetAppInstallSummariesByTenantAsync(tenantId, targetDate);
|
||||||
|
|
||||||
var relevantApps = appSummaries.Where(a => sessionIdSet.Contains(a.SessionId)).ToList();
|
var relevantApps = appSummaries.Where(a => sessionIdSet.Contains(a.SessionId)).ToList();
|
||||||
var appsPerSession = relevantApps.GroupBy(a => a.SessionId).Select(g => g.Count()).ToList();
|
var appsPerSession = relevantApps.GroupBy(a => a.SessionId).Select(g => g.Count()).ToList();
|
||||||
|
|||||||
@@ -186,7 +186,10 @@ namespace AutopilotMonitor.Functions.Services
|
|||||||
if (config.SlaNotifyOnAppInstallBreach && config.SlaTargetAppInstallSuccessRate.HasValue)
|
if (config.SlaNotifyOnAppInstallBreach && config.SlaTargetAppInstallSuccessRate.HasValue)
|
||||||
{
|
{
|
||||||
var currentWeekKey = SlaMetricsService.GetIsoWeekKey(now);
|
var currentWeekKey = SlaMetricsService.GetIsoWeekKey(now);
|
||||||
var raw = await _metricsRepo.GetAppInstallSummariesByTenantAsync(config.TenantId);
|
// Bound to a safe margin before the current ISO week (an ISO week spans <=7 days, so
|
||||||
|
// now-8d never excludes a current-week row). The exact week match happens in-memory
|
||||||
|
// below; this just stops the query from scanning the tenant's entire history every 2h.
|
||||||
|
var raw = await _metricsRepo.GetAppInstallSummariesByTenantAsync(config.TenantId, now.AddDays(-8));
|
||||||
appInstalls = raw
|
appInstalls = raw
|
||||||
.Where(a => (a.Status == "Succeeded" || a.Status == "Failed")
|
.Where(a => (a.Status == "Succeeded" || a.Status == "Failed")
|
||||||
&& SlaMetricsService.GetIsoWeekKey(a.StartedAt) == currentWeekKey)
|
&& SlaMetricsService.GetIsoWeekKey(a.StartedAt) == currentWeekKey)
|
||||||
|
|||||||
@@ -125,7 +125,9 @@ namespace AutopilotMonitor.Functions.Services
|
|||||||
List<AppInstallSummary>? appInstalls = null;
|
List<AppInstallSummary>? appInstalls = null;
|
||||||
if (config?.SlaTargetAppInstallSuccessRate != null)
|
if (config?.SlaTargetAppInstallSuccessRate != null)
|
||||||
{
|
{
|
||||||
appInstalls = await _metricsRepo.GetAppInstallSummariesByTenantAsync(tenantId);
|
// Push the window's lower bound server-side (startDate); the in-memory filter below
|
||||||
|
// still applies the upper bound. Without sinceUtc this read the tenant's full history.
|
||||||
|
appInstalls = await _metricsRepo.GetAppInstallSummariesByTenantAsync(tenantId, startDate);
|
||||||
// Filter to the time window
|
// Filter to the time window
|
||||||
appInstalls = appInstalls
|
appInstalls = appInstalls
|
||||||
.Where(a => a.StartedAt >= startDate && a.StartedAt <= endDate)
|
.Where(a => a.StartedAt >= startDate && a.StartedAt <= endDate)
|
||||||
|
|||||||
@@ -338,7 +338,11 @@ namespace AutopilotMonitor.Functions.Services
|
|||||||
: windowBound;
|
: windowBound;
|
||||||
|
|
||||||
var scannedRows = 0;
|
var scannedRows = 0;
|
||||||
await foreach (var entity in indexTableClient.QueryAsync<TableEntity>(filter: filter))
|
// Project to only the typeahead-matched fields (+ RowKey for the SessionId fallback at
|
||||||
|
// ExtractSessionIdFromIndexRowKey). Avoids pulling the full ~40-column SessionsIndex
|
||||||
|
// mirror for up to QuickSearchMaxScannedRows rows on every keystroke-driven request.
|
||||||
|
await foreach (var entity in indexTableClient.QueryAsync<TableEntity>(
|
||||||
|
filter: filter, select: new[] { "RowKey", "SessionId", "SerialNumber", "DeviceName", "Status", "StartedAt" }))
|
||||||
{
|
{
|
||||||
if (++scannedRows > QuickSearchMaxScannedRows)
|
if (++scannedRows > QuickSearchMaxScannedRows)
|
||||||
break;
|
break;
|
||||||
@@ -1261,7 +1265,11 @@ namespace AutopilotMonitor.Functions.Services
|
|||||||
// longer vanish from the summary). See SessionStatusBuckets in MetricsMath.cs.
|
// longer vanish from the summary). See SessionStatusBuckets in MetricsMath.cs.
|
||||||
var groups = new Dictionary<string, SessionStatusBuckets>(StringComparer.OrdinalIgnoreCase);
|
var groups = new Dictionary<string, SessionStatusBuckets>(StringComparer.OrdinalIgnoreCase);
|
||||||
|
|
||||||
await foreach (var entity in indexTableClient.QueryAsync<TableEntity>(filter: oDataFilter))
|
// Project to the two columns the tally consumes. The SessionsIndex row is a full
|
||||||
|
// ~40-column mirror (incl. the large FailureSnapshotJson), so reading it whole for a
|
||||||
|
// status count multiplied transfer ~10-20x for nothing on this cross-partition scan.
|
||||||
|
await foreach (var entity in indexTableClient.QueryAsync<TableEntity>(
|
||||||
|
filter: oDataFilter, select: new[] { "PartitionKey", "Status" }))
|
||||||
{
|
{
|
||||||
var pk = entity.PartitionKey;
|
var pk = entity.PartitionKey;
|
||||||
var statusStr = entity.GetString("Status") ?? "InProgress";
|
var statusStr = entity.GetString("Status") ?? "InProgress";
|
||||||
|
|||||||
@@ -217,7 +217,11 @@ namespace AutopilotMonitor.Functions.Services
|
|||||||
};
|
};
|
||||||
|
|
||||||
// App & Script Metrics
|
// App & Script Metrics
|
||||||
var allAppSummaries = await _metricsRepo.GetAllAppInstallSummariesAsync();
|
// Bound the AppInstall scan to the same window as the sessions above — relevantApps is
|
||||||
|
// filtered to sessionIdSet anyway, and an app installs during its session so its StartedAt
|
||||||
|
// is >= the session's. Without sinceUtc this dematerialized the entire AppInstallSummaries
|
||||||
|
// table on every cache miss just to discard everything outside the window.
|
||||||
|
var allAppSummaries = await _metricsRepo.GetAllAppInstallSummariesAsync(DateTime.UtcNow.AddDays(-days));
|
||||||
var sessionIdSet = new HashSet<string>(allSessions.Select(s => s.SessionId));
|
var sessionIdSet = new HashSet<string>(allSessions.Select(s => s.SessionId));
|
||||||
var relevantApps = allAppSummaries.Where(a => sessionIdSet.Contains(a.SessionId)).ToList();
|
var relevantApps = allAppSummaries.Where(a => sessionIdSet.Contains(a.SessionId)).ToList();
|
||||||
var appsPerSessionList = relevantApps.GroupBy(a => a.SessionId).Select(g => g.Count()).ToList();
|
var appsPerSessionList = relevantApps.GroupBy(a => a.SessionId).Select(g => g.Count()).ToList();
|
||||||
@@ -366,7 +370,8 @@ namespace AutopilotMonitor.Functions.Services
|
|||||||
};
|
};
|
||||||
|
|
||||||
// App & Script Metrics
|
// App & Script Metrics
|
||||||
var tenantAppSummaries = await _metricsRepo.GetAppInstallSummariesByTenantAsync(tenantId);
|
// Bound to the window (see ComputeUsageMetricsInternalAsync) — joined to tenantSessionIdSet below.
|
||||||
|
var tenantAppSummaries = await _metricsRepo.GetAppInstallSummariesByTenantAsync(tenantId, DateTime.UtcNow.AddDays(-days));
|
||||||
var tenantSessionIdSet = new HashSet<string>(tenantSessions.Select(s => s.SessionId));
|
var tenantSessionIdSet = new HashSet<string>(tenantSessions.Select(s => s.SessionId));
|
||||||
var relevantTenantApps = tenantAppSummaries.Where(a => tenantSessionIdSet.Contains(a.SessionId)).ToList();
|
var relevantTenantApps = tenantAppSummaries.Where(a => tenantSessionIdSet.Contains(a.SessionId)).ToList();
|
||||||
var tenantAppsPerSession = relevantTenantApps.GroupBy(a => a.SessionId).Select(g => g.Count()).ToList();
|
var tenantAppsPerSession = relevantTenantApps.GroupBy(a => a.SessionId).Select(g => g.Count()).ToList();
|
||||||
|
|||||||
Reference in New Issue
Block a user