using System.Diagnostics; using Microsoft.ApplicationInsights; using Microsoft.ApplicationInsights.Channel; using Microsoft.ApplicationInsights.DataContracts; using Microsoft.Azure.Functions.Worker; using Microsoft.Azure.Functions.Worker.Middleware; using AutopilotMonitor.Functions.Extensions; using AutopilotMonitor.Functions.Helpers; using AutopilotMonitor.Functions.Security; namespace AutopilotMonitor.Functions.Middleware; /// /// Emits worker-side with business context (TenantId, UserId, /// CorrelationId, UserRole) so the Application Insights requests table is queryable /// by tenant, user, and correlation ID. Runs first in the pipeline to capture accurate /// duration including auth and policy evaluation. Non-HTTP triggers are skipped. /// public class RequestTelemetryMiddleware : IFunctionsWorkerMiddleware { private readonly TelemetryClient _telemetryClient; // Exact allowlist of device endpoints that READ + validate the X-Tenant-Id header (cert auth // for agent/*, bootstrap-token context for bootstrap/*). ONLY these may seed the requests-table // TenantId tag from the header. A prefix match (/api/agent/* or /api/bootstrap/*) would also // catch routes that never validate the header — agent/config (takes ?tenantId=), // agent/register-session + agent/upload-url (tenant from body), bootstrap/validate/{code}, // bootstrap/sessions{,/{code}}, bootstrap/config, bootstrap/register-session — where a client // could send X-Tenant-Id: and pollute tenant attribution in telemetry. private static readonly HashSet TenantHeaderTrustedPaths = new(StringComparer.OrdinalIgnoreCase) { "/api/agent/telemetry", "/api/agent/error", "/api/agent/distress", "/api/bootstrap/error", }; public RequestTelemetryMiddleware(TelemetryClient telemetryClient) { _telemetryClient = telemetryClient; } public async Task Invoke(FunctionContext context, FunctionExecutionDelegate next) { var httpContext = context.GetHttpContext(); if (httpContext == null) { // Non-HTTP trigger (timer, queue) — nothing to track await next(context); return; } var sw = Stopwatch.StartNew(); var startTime = DateTimeOffset.UtcNow; Exception? caughtException = null; try { await next(context); } catch (Exception ex) { caughtException = ex; throw; } finally { sw.Stop(); var statusCode = caughtException != null ? 500 : httpContext.Response.StatusCode; var functionName = context.FunctionDefinition.Name; var method = httpContext.Request.Method; var url = $"{httpContext.Request.Scheme}://{httpContext.Request.Host}{httpContext.Request.Path}{httpContext.Request.QueryString}"; var requestTelemetry = new RequestTelemetry { Name = $"{method} {functionName}", Timestamp = startTime, Duration = sw.Elapsed, ResponseCode = statusCode.ToString(), Success = statusCode < 500, Url = new Uri(url), }; // Distributed trace correlation var activity = Activity.Current; if (activity != null) { requestTelemetry.Context.Operation.Id = activity.RootId; requestTelemetry.Context.Operation.ParentId = activity.Id; } // NOTE: client_IP is deliberately NOT populated here. App Insights uses the request IP // only for geo-lookup and then masks the stored client_IP to 0.0.0.0 (verified: every // request in the resource shows 0.0.0.0), so setting Context.Location.Ip has no effect // on the column. On top of that, the only IP reaching the isolated worker is an Azure- // infra hop (West Europe), not the device's real egress IP — the device IP would need a // non-masked custom property AND the correct forwarded header (Front-Door-topology // dependent). Not worth it: TenantId (above) + CorrelationId already isolate a device. // See memory project_ingest_request_multiplication_429 for the full diagnosis. // Business context from downstream middleware requestTelemetry.Properties["Source"] = "WorkerMiddleware"; requestTelemetry.Properties["FunctionName"] = functionName; requestTelemetry.Properties["HttpMethod"] = method; requestTelemetry.Properties["HttpPath"] = httpContext.Request.Path.Value ?? ""; var clientSource = httpContext.Request.Headers["X-Client-Source"].FirstOrDefault(); if (!string.IsNullOrEmpty(clientSource)) requestTelemetry.Properties["ClientSource"] = clientSource; var mcpToolName = httpContext.Request.Headers["X-MCP-Tool-Name"].FirstOrDefault(); if (!string.IsNullOrEmpty(mcpToolName)) requestTelemetry.Properties["McpToolName"] = mcpToolName; // APPLICATION PRINCIPALS — the request-row dimension that lets KQL separate automation // (app-only tokens) from people: "app" plus the calling client id, nothing for a person. if (context.GetUser() is { } principal && principal.IsApplicationPrincipal()) { requestTelemetry.Properties["PrincipalKind"] = "app"; var applicationId = principal.GetApplicationId(); if (applicationId != null) requestTelemetry.Properties["ApplicationId"] = applicationId; } if (context.Items.TryGetValue("CorrelationId", out var corrId) && corrId is string correlationId) requestTelemetry.Properties["CorrelationId"] = correlationId; // CERT-TENANT-BINDING-SHADOW — set by SecurityValidator.ObserveCertTenantBinding. // Carried on the request row rather than a trace line: worker-side LogInformation never // reaches App Insights (provider default rule is Warning+), so the bulk "Match" outcome // would be invisible and the shadow telemetry would have no denominator. This row already // exists per request and is unsampled, so the outcome costs no additional telemetry. if (context.Items.TryGetValue(CertTenantBinding.RequestItemKey, out var binding) && binding is string bindingOutcome) requestTelemetry.Properties["CertTenantBinding"] = bindingOutcome; // SESSION-OWNER-BINDING-SHADOW — set by SessionOwnerBindingObserver on register / // telemetry / error-report requests. Same carrier and same reasoning as above. if (context.Items.TryGetValue(SessionOwnershipPolicy.RequestItemKey, out var ownerBinding) && ownerBinding is string ownerOutcome) requestTelemetry.Properties["SessionOwnerBinding"] = ownerOutcome; // DEVICE-IDENTITY-BLOCK-BINDING — set by the kill-switch call sites (telemetry / config). if (context.Items.TryGetValue(DeviceIdentityBinding.RequestItemKey, out var identityBinding) && identityBinding is string identityOutcome) requestTelemetry.Properties[DeviceIdentityBinding.RequestItemKey] = identityOutcome; var reqCtx = context.GetRequestContext(); var tenantId = reqCtx.TenantId; if (string.IsNullOrEmpty(tenantId)) { // Device ingest endpoints do not run through PolicyEnforcementMiddleware, so // reqCtx.TenantId is empty — but they carry a cert/token-validated tenant in the // X-Tenant-Id header. Only honor the header on the exact routes that actually // validate it (TenantHeaderTrustedPaths) and only when it is a well-formed GUID, so // other/anonymous routes cannot pollute the requests table with arbitrary tenant ids. var path = httpContext.Request.Path.Value ?? string.Empty; if (TenantHeaderTrustedPaths.Contains(path)) { var headerTenant = httpContext.Request.Headers["X-Tenant-Id"].FirstOrDefault(); if (!string.IsNullOrEmpty(headerTenant) && Guid.TryParse(headerTenant, out _)) tenantId = headerTenant; } } if (!string.IsNullOrEmpty(tenantId)) requestTelemetry.Properties["TenantId"] = tenantId; if (!string.IsNullOrEmpty(reqCtx.UserPrincipalName)) requestTelemetry.Properties["UserId"] = reqCtx.UserPrincipalName; if (!string.IsNullOrEmpty(reqCtx.UserRole)) requestTelemetry.Properties["UserRole"] = reqCtx.UserRole; if (caughtException != null) requestTelemetry.Properties["ExceptionType"] = caughtException.GetType().Name; // Bypass the worker's adaptive sampling for this item. The SDK sampling processor // passes through any item whose SamplingPercentage is already set, so this is the // per-item opt-out that host.json "excludedTypes" (host process only) cannot provide. // Live-verified 2026-08-23 BEFORE this change: worker request rows carried // ItemCount 2-5 on weekdays (~30% of individual requests missing), so count()-based // queries undercounted. This copy is the single canonical request record — the host // duplicate is dropped by a workspace DCR transformation on AppRequests. ((ISupportSampling)requestTelemetry).SamplingPercentage = 100; try { _telemetryClient.TrackRequest(requestTelemetry); } catch { // Never let telemetry failures mask the original exception or crash the pipeline } } } }