From 863291ff7b54ae5ef7398937e79d409fc897cfc9 Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Tue, 19 May 2026 01:23:21 +0200 Subject: [PATCH] feat(backend,web): App Insights telemetry for optional Graph add-on feature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So we can answer "which tenants actively have this on" and "how many people are using it" later — without bolting on a separate metrics store. Backend events - `GraphAddOnPermissionsDetected` (GraphFeatureDetector, fresh-acquire path): fires at most every ~55min per tenant (cache TTL). Props: TenantId, GrantedRoleCount, HasScriptDisplayNames, TokenCacheTtlMinutes. → distinct count over time = tenants who have the optional permission turned on. - `ScriptDisplayNamesResolved` (ScriptDisplayNameResolver, only when permission is active and the resolver actually did work): per session-view with scripts. Props: TenantId, RefsRequested, RefsResolved, CacheHits, TriggeredFullPull. → real usage pulse. - `GraphAddOnStatusChecked` (GET status endpoint): admin engagement with the capability page. Props: TenantId, UserId, IsTransient, ScriptDisplayNamesGranted. - `GraphAddOnRefreshTriggered` (POST refresh endpoint): high-signal "tenant just granted" event. Props: TenantId, UserId. Web events (via `trackEvent` -- tenantId auto-attached by the telemetry initializer) - `GraphAddOnStatusViewed` on Settings → Optional Graph capabilities mount. - `GraphAddOnGrantCommandCopied` on Copy command / Copy permissions click. Notes - All emits wrapped in try/catch with debug-level logging — telemetry failures must never poison the acquire / resolve hot paths. - Tests pass a no-op `TelemetryClient(new TelemetryConfiguration())` (same pattern as SlaTenantStatusLifecycleTests). 1718 backend / 364 web tests green. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../ScriptDisplayNameResolverTests.cs | 3 ++ .../GetGraphPermissionsStatusFunction.cs | 30 +++++++++++++- .../Graph/RefreshGraphPermissionsFunction.cs | 23 ++++++++++- .../GraphResolution/GraphFeatureDetector.cs | 36 ++++++++++++++++- .../ScriptDisplayNameResolver.cs | 40 ++++++++++++++++++- .../SectionOptionalGraphCapabilities.tsx | 12 ++++++ 6 files changed, 138 insertions(+), 6 deletions(-) diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/GraphResolution/ScriptDisplayNameResolverTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/GraphResolution/ScriptDisplayNameResolverTests.cs index f26bb369..6cc4cfe4 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/GraphResolution/ScriptDisplayNameResolverTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/GraphResolution/ScriptDisplayNameResolverTests.cs @@ -4,6 +4,8 @@ using System.Net; using System.Threading.Tasks; using AutopilotMonitor.Functions.Services.GraphResolution; using AutopilotMonitor.Shared.Models.Graph; +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.Extensibility; using Microsoft.Extensions.Logging.Abstractions; using Xunit; @@ -246,6 +248,7 @@ public class ScriptDisplayNameResolverTests Sut = new ScriptDisplayNameResolver( Detector, Repo, HttpFactory, NullLogger.Instance, + new TelemetryClient(new TelemetryConfiguration()), new FakeTimeProvider(FixedNow)); } diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Graph/GetGraphPermissionsStatusFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Graph/GetGraphPermissionsStatusFunction.cs index f81d0ae7..765bc787 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Graph/GetGraphPermissionsStatusFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Graph/GetGraphPermissionsStatusFunction.cs @@ -1,10 +1,13 @@ using System; +using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Threading; using System.Threading.Tasks; using AutopilotMonitor.Functions.Helpers; using AutopilotMonitor.Functions.Services.GraphResolution; using AutopilotMonitor.Shared.Models.Graph; +using Microsoft.ApplicationInsights; using Microsoft.Azure.Functions.Worker; using Microsoft.Azure.Functions.Worker.Http; using Microsoft.Extensions.Configuration; @@ -38,15 +41,18 @@ public class GetGraphPermissionsStatusFunction private readonly ILogger _logger; private readonly IGraphFeatureDetector _detector; private readonly IConfiguration _configuration; + private readonly TelemetryClient _telemetry; public GetGraphPermissionsStatusFunction( ILogger logger, IGraphFeatureDetector detector, - IConfiguration configuration) + IConfiguration configuration, + TelemetryClient telemetry) { _logger = logger; _detector = detector; _configuration = configuration; + _telemetry = telemetry; } [Function("GetGraphPermissionsStatus")] @@ -79,6 +85,8 @@ public class GetGraphPermissionsStatusFunction requiredPermissions = GraphFeatureCatalog.RequiredPermissions(featureName), }).ToList(); + EmitStatusChecked(requestCtx.TargetTenantId, requestCtx.UserPrincipalName, snapshot); + return await req.OkAsync(new { clientId = _configuration["EntraId:ClientId"] ?? string.Empty, @@ -93,4 +101,24 @@ public class GetGraphPermissionsStatusFunction $"Get graph-permissions status for tenant '{tenantId}'"); } } + + private void EmitStatusChecked(string tenantId, string userPrincipalName, GraphPermissionSnapshot snapshot) + { + try + { + _telemetry.TrackEvent("GraphAddOnStatusChecked", new Dictionary + { + ["TenantId"] = tenantId, + ["UserId"] = userPrincipalName, + ["IsTransient"] = snapshot.IsTransient.ToString(CultureInfo.InvariantCulture), + ["ScriptDisplayNamesGranted"] = GraphFeatureCatalog + .IsFeatureGranted(GraphFeatureCatalog.FeatureScriptDisplayNames, snapshot.GrantedRoles) + .ToString(CultureInfo.InvariantCulture), + }); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "GraphAddOnStatusChecked telemetry emit failed"); + } + } } diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Graph/RefreshGraphPermissionsFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Graph/RefreshGraphPermissionsFunction.cs index 58bf0896..7786719f 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Graph/RefreshGraphPermissionsFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Graph/RefreshGraphPermissionsFunction.cs @@ -1,7 +1,9 @@ using System; +using System.Collections.Generic; using System.Threading.Tasks; using AutopilotMonitor.Functions.Helpers; using AutopilotMonitor.Functions.Services.GraphResolution; +using Microsoft.ApplicationInsights; using Microsoft.Azure.Functions.Worker; using Microsoft.Azure.Functions.Worker.Http; using Microsoft.Extensions.Logging; @@ -18,13 +20,16 @@ public class RefreshGraphPermissionsFunction { private readonly ILogger _logger; private readonly IGraphFeatureDetector _detector; + private readonly TelemetryClient _telemetry; public RefreshGraphPermissionsFunction( ILogger logger, - IGraphFeatureDetector detector) + IGraphFeatureDetector detector, + TelemetryClient telemetry) { _logger = logger; _detector = detector; + _telemetry = telemetry; } [Function("RefreshGraphPermissions")] @@ -39,6 +44,22 @@ public class RefreshGraphPermissionsFunction _detector.InvalidateTenant(requestCtx.TargetTenantId); _logger.LogInformation("Graph-permissions cache invalidated for tenant {Tenant} by request", requestCtx.TargetTenantId); + + try + { + // High-signal event: admin just ran the grant script and asked us to pick + // up the new appRoleAssignment. Pulses cluster around tenants newly opting in. + _telemetry.TrackEvent("GraphAddOnRefreshTriggered", new Dictionary + { + ["TenantId"] = requestCtx.TargetTenantId, + ["UserId"] = requestCtx.UserPrincipalName, + }); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "GraphAddOnRefreshTriggered telemetry emit failed"); + } + return await req.OkAsync(new { success = true }); } catch (Exception ex) diff --git a/src/Backend/AutopilotMonitor.Functions/Services/GraphResolution/GraphFeatureDetector.cs b/src/Backend/AutopilotMonitor.Functions/Services/GraphResolution/GraphFeatureDetector.cs index 557d99c8..3bc56dda 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/GraphResolution/GraphFeatureDetector.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/GraphResolution/GraphFeatureDetector.cs @@ -1,10 +1,13 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.IdentityModel.Tokens.Jwt; using System.Linq; using System.Threading; using System.Threading.Tasks; using AutopilotMonitor.Functions.Security; +using AutopilotMonitor.Shared.Models.Graph; +using Microsoft.ApplicationInsights; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; @@ -24,13 +27,15 @@ public sealed class GraphFeatureDetector : IGraphFeatureDetector private readonly GraphTokenService _tokenService; private readonly IMemoryCache _cache; private readonly ILogger _logger; + private readonly TelemetryClient _telemetry; private readonly TimeProvider _time; public GraphFeatureDetector( GraphTokenService tokenService, IMemoryCache cache, - ILogger logger) - : this(tokenService, cache, logger, TimeProvider.System) + ILogger logger, + TelemetryClient telemetry) + : this(tokenService, cache, logger, telemetry, TimeProvider.System) { } @@ -39,11 +44,13 @@ public sealed class GraphFeatureDetector : IGraphFeatureDetector GraphTokenService tokenService, IMemoryCache cache, ILogger logger, + TelemetryClient telemetry, TimeProvider time) { _tokenService = tokenService; _cache = cache; _logger = logger; + _telemetry = telemetry; _time = time; } @@ -127,9 +134,34 @@ public sealed class GraphFeatureDetector : IGraphFeatureDetector _logger.LogDebug( "GraphFeatureDetector: cached token for tenant {TenantId} with {RoleCount} roles, TTL={Ttl}", tenantId, roles.Count, ttl); + + // Telemetry: fires only on FRESH acquire (cache miss). Frequency is bounded by the + // cache TTL (~55 min per tenant), so this is the right pulse for "which tenants + // currently have the optional Graph permissions on" without spamming AppInsights. + EmitPermissionsDetected(tenantId, roles, ttl); return (ctx, false); } + private void EmitPermissionsDetected(string tenantId, IReadOnlySet roles, TimeSpan ttl) + { + try + { + _telemetry.TrackEvent("GraphAddOnPermissionsDetected", new Dictionary + { + ["TenantId"] = tenantId, + ["GrantedRoleCount"] = roles.Count.ToString(CultureInfo.InvariantCulture), + ["HasScriptDisplayNames"] = roles.Contains(GraphAppPermissions.DeviceManagementScriptsReadAll) + .ToString(CultureInfo.InvariantCulture), + ["TokenCacheTtlMinutes"] = ((int)ttl.TotalMinutes).ToString(CultureInfo.InvariantCulture), + }); + } + catch (Exception ex) + { + // Never let a telemetry failure poison the acquire hot path. + _logger.LogDebug(ex, "GraphAddOnPermissionsDetected telemetry emit failed"); + } + } + public void InvalidateTenant(string tenantId) { if (string.IsNullOrWhiteSpace(tenantId)) return; diff --git a/src/Backend/AutopilotMonitor.Functions/Services/GraphResolution/ScriptDisplayNameResolver.cs b/src/Backend/AutopilotMonitor.Functions/Services/GraphResolution/ScriptDisplayNameResolver.cs index 2dd11230..37ec63b7 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/GraphResolution/ScriptDisplayNameResolver.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/GraphResolution/ScriptDisplayNameResolver.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Net; using System.Net.Http; @@ -8,6 +9,7 @@ using System.Threading; using System.Threading.Tasks; using AutopilotMonitor.Shared.DataAccess; using AutopilotMonitor.Shared.Models.Graph; +using Microsoft.ApplicationInsights; using Microsoft.Extensions.Logging; using Newtonsoft.Json.Linq; @@ -42,14 +44,16 @@ public sealed class ScriptDisplayNameResolver : IScriptDisplayNameResolver private readonly IScriptNameCacheRepository _cache; private readonly IHttpClientFactory _httpClientFactory; private readonly ILogger _logger; + private readonly TelemetryClient _telemetry; private readonly TimeProvider _time; public ScriptDisplayNameResolver( IGraphFeatureDetector detector, IScriptNameCacheRepository cache, IHttpClientFactory httpClientFactory, - ILogger logger) - : this(detector, cache, httpClientFactory, logger, TimeProvider.System) + ILogger logger, + TelemetryClient telemetry) + : this(detector, cache, httpClientFactory, logger, telemetry, TimeProvider.System) { } @@ -59,12 +63,14 @@ public sealed class ScriptDisplayNameResolver : IScriptDisplayNameResolver IScriptNameCacheRepository cache, IHttpClientFactory httpClientFactory, ILogger logger, + TelemetryClient telemetry, TimeProvider time) { _detector = detector; _cache = cache; _httpClientFactory = httpClientFactory; _logger = logger; + _telemetry = telemetry; _time = time; } @@ -113,11 +119,14 @@ public sealed class ScriptDisplayNameResolver : IScriptDisplayNameResolver cached = new Dictionary(); } + var cacheHits = 0; foreach (var kv in cached) { if (!kv.Value.IsNotFound) result[kv.Key] = kv.Value.DisplayName; // NotFound rows: leave result[ref] = null (already initialised). + cacheHits++; } + var triggeredFullPull = false; var missing = refs.Where(r => !cached.ContainsKey(r)).ToList(); if (missing.Count == 0) return result; @@ -137,6 +146,7 @@ public sealed class ScriptDisplayNameResolver : IScriptDisplayNameResolver if (metaStale) { + triggeredFullPull = true; try { await FullPullAsync(tenantId, kind, ctx.AccessToken, result, budget).ConfigureAwait(false); @@ -219,9 +229,35 @@ public sealed class ScriptDisplayNameResolver : IScriptDisplayNameResolver } } + // Usage telemetry — fires only when the optional permission was active AND the + // resolver actually did work. Volume = once per session-view that has scripts, + // which is a meaningful "tenant is using this feature" pulse. + EmitResolved(tenantId, refs.Count, result, cacheHits, triggeredFullPull); + return result; } + private void EmitResolved(string tenantId, int refsRequested, Dictionary result, int cacheHits, bool triggeredFullPull) + { + try + { + var resolved = 0; + foreach (var v in result.Values) { if (v != null) resolved++; } + _telemetry.TrackEvent("ScriptDisplayNamesResolved", new Dictionary + { + ["TenantId"] = tenantId, + ["RefsRequested"] = refsRequested.ToString(CultureInfo.InvariantCulture), + ["RefsResolved"] = resolved.ToString(CultureInfo.InvariantCulture), + ["CacheHits"] = cacheHits.ToString(CultureInfo.InvariantCulture), + ["TriggeredFullPull"] = triggeredFullPull.ToString(CultureInfo.InvariantCulture), + }); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "ScriptDisplayNamesResolved telemetry emit failed"); + } + } + // ── Graph HTTP plumbing ────────────────────────────────────────────────── private async Task FullPullAsync( diff --git a/src/Web/autopilot-monitor-web/app/settings/tenant/sections/SectionOptionalGraphCapabilities.tsx b/src/Web/autopilot-monitor-web/app/settings/tenant/sections/SectionOptionalGraphCapabilities.tsx index 381357c3..0e613489 100644 --- a/src/Web/autopilot-monitor-web/app/settings/tenant/sections/SectionOptionalGraphCapabilities.tsx +++ b/src/Web/autopilot-monitor-web/app/settings/tenant/sections/SectionOptionalGraphCapabilities.tsx @@ -5,6 +5,7 @@ import { useAuth } from "../../../../contexts/AuthContext"; import { useTenant } from "../../../../contexts/TenantContext"; import { api } from "@/lib/api"; import { authenticatedFetch } from "@/lib/authenticatedFetch"; +import { trackEvent } from "@/lib/appInsights"; interface FeatureStatus { name: string; @@ -44,6 +45,15 @@ export function SectionOptionalGraphCapabilities() { } const body = (await response.json()) as StatusResponse; setStatus(body); + // Tracks distinct admins/tenants looking at the capability page. The + // backend's GraphAddOnStatusChecked event covers the same signal server-side + // (more authoritative), but the client-side flavour also captures admins who + // bounce off the page before the round-trip lands. + trackEvent("GraphAddOnStatusViewed", { + isTransient: !!body.isTransient, + scriptDisplayNamesGranted: !!body.features.find(f => f.name === "ScriptDisplayNames")?.granted, + grantedRoleCount: body.grantedRoles?.length ?? 0, + }); } catch (err) { setError(err instanceof Error ? err.message : "Failed to load status."); setStatus(null); @@ -82,6 +92,8 @@ export function SectionOptionalGraphCapabilities() { await navigator.clipboard.writeText(text); setCopied(label); window.setTimeout(() => setCopied(null), 2000); + // High-intent signal: admin is about to run (or share) the grant command. + trackEvent("GraphAddOnGrantCommandCopied", { kind: label }); } catch { // Older browsers / iframes — best effort only. }