AutopilotMonitor/src/Backend/AutopilotMonitor.Functions/Middleware/RequestTelemetryMiddleware.cs

using System.Diagnostics;
using Microsoft.ApplicationInsights;
using Microsoft.ApplicationInsights.Channel;
using Microsoft.ApplicationInsights.DataContracts;
using Microsoft.Azure.Functions.Worker;
using Microsoft.Azure.Functions.Worker.Middleware;
using AutopilotMonitor.Functions.Extensions;
using AutopilotMonitor.Functions.Helpers;
using AutopilotMonitor.Functions.Security;

namespace AutopilotMonitor.Functions.Middleware;

/// <summary>
/// Emits worker-side <see cref="RequestTelemetry"/> with business context (TenantId, UserId,
/// CorrelationId, UserRole) so the Application Insights <c>requests</c> table is queryable
/// by tenant, user, and correlation ID. Runs first in the pipeline to capture accurate
/// duration including auth and policy evaluation. Non-HTTP triggers are skipped.
/// </summary>
public class RequestTelemetryMiddleware : IFunctionsWorkerMiddleware
{
    private readonly TelemetryClient _telemetryClient;

    // Exact allowlist of device endpoints that READ + validate the X-Tenant-Id header (cert auth
    // for agent/*, bootstrap-token context for bootstrap/*). ONLY these may seed the requests-table
    // TenantId tag from the header. A prefix match (/api/agent/* or /api/bootstrap/*) would also
    // catch routes that never validate the header — agent/config (takes ?tenantId=),
    // agent/register-session + agent/upload-url (tenant from body), bootstrap/validate/{code},
    // bootstrap/sessions{,/{code}}, bootstrap/config, bootstrap/register-session — where a client
    // could send X-Tenant-Id:<valid-guid> and pollute tenant attribution in telemetry.
    private static readonly HashSet<string> TenantHeaderTrustedPaths = new(StringComparer.OrdinalIgnoreCase)
    {
        "/api/agent/telemetry",
        "/api/agent/error",
        "/api/agent/distress",
        "/api/bootstrap/error",
    };

    public RequestTelemetryMiddleware(TelemetryClient telemetryClient)
    {
        _telemetryClient = telemetryClient;
    }

    public async Task Invoke(FunctionContext context, FunctionExecutionDelegate next)
    {
        var httpContext = context.GetHttpContext();
        if (httpContext == null)
        {
            // Non-HTTP trigger (timer, queue) — nothing to track
            await next(context);
            return;
        }

        var sw = Stopwatch.StartNew();
        var startTime = DateTimeOffset.UtcNow;
        Exception? caughtException = null;

        try
        {
            await next(context);
        }
        catch (Exception ex)
        {
            caughtException = ex;
            throw;
        }
        finally
        {
            sw.Stop();

            var statusCode = caughtException != null ? 500 : httpContext.Response.StatusCode;
            var functionName = context.FunctionDefinition.Name;
            var method = httpContext.Request.Method;
            var url = $"{httpContext.Request.Scheme}://{httpContext.Request.Host}{httpContext.Request.Path}{httpContext.Request.QueryString}";

            var requestTelemetry = new RequestTelemetry
            {
                Name = $"{method} {functionName}",
                Timestamp = startTime,
                Duration = sw.Elapsed,
                ResponseCode = statusCode.ToString(),
                Success = statusCode < 500,
                Url = new Uri(url),
            };

            // Distributed trace correlation
            var activity = Activity.Current;
            if (activity != null)
            {
                requestTelemetry.Context.Operation.Id = activity.RootId;
                requestTelemetry.Context.Operation.ParentId = activity.Id;
            }

            // NOTE: client_IP is deliberately NOT populated here. App Insights uses the request IP
            // only for geo-lookup and then masks the stored client_IP to 0.0.0.0 (verified: every
            // request in the resource shows 0.0.0.0), so setting Context.Location.Ip has no effect
            // on the column. On top of that, the only IP reaching the isolated worker is an Azure-
            // infra hop (West Europe), not the device's real egress IP — the device IP would need a
            // non-masked custom property AND the correct forwarded header (Front-Door-topology
            // dependent). Not worth it: TenantId (above) + CorrelationId already isolate a device.
            // See memory project_ingest_request_multiplication_429 for the full diagnosis.

            // Business context from downstream middleware
            requestTelemetry.Properties["Source"] = "WorkerMiddleware";
            requestTelemetry.Properties["FunctionName"] = functionName;
            requestTelemetry.Properties["HttpMethod"] = method;
            requestTelemetry.Properties["HttpPath"] = httpContext.Request.Path.Value ?? "";

            var clientSource = httpContext.Request.Headers["X-Client-Source"].FirstOrDefault();
            if (!string.IsNullOrEmpty(clientSource))
                requestTelemetry.Properties["ClientSource"] = clientSource;

            var mcpToolName = httpContext.Request.Headers["X-MCP-Tool-Name"].FirstOrDefault();
            if (!string.IsNullOrEmpty(mcpToolName))
                requestTelemetry.Properties["McpToolName"] = mcpToolName;

            // APPLICATION PRINCIPALS — the request-row dimension that lets KQL separate automation
            // (app-only tokens) from people: "app" plus the calling client id, nothing for a person.
            if (context.GetUser() is { } principal && principal.IsApplicationPrincipal())
            {
                requestTelemetry.Properties["PrincipalKind"] = "app";
                var applicationId = principal.GetApplicationId();
                if (applicationId != null)
                    requestTelemetry.Properties["ApplicationId"] = applicationId;
            }

            if (context.Items.TryGetValue("CorrelationId", out var corrId) && corrId is string correlationId)
                requestTelemetry.Properties["CorrelationId"] = correlationId;

            // CERT-TENANT-BINDING-SHADOW — set by SecurityValidator.ObserveCertTenantBinding.
            // Carried on the request row rather than a trace line: worker-side LogInformation never
            // reaches App Insights (provider default rule is Warning+), so the bulk "Match" outcome
            // would be invisible and the shadow telemetry would have no denominator. This row already
            // exists per request and is unsampled, so the outcome costs no additional telemetry.
            if (context.Items.TryGetValue(CertTenantBinding.RequestItemKey, out var binding) && binding is string bindingOutcome)
                requestTelemetry.Properties["CertTenantBinding"] = bindingOutcome;

            // SESSION-OWNER-BINDING-SHADOW — set by SessionOwnerBindingObserver on register /
            // telemetry / error-report requests. Same carrier and same reasoning as above.
            if (context.Items.TryGetValue(SessionOwnershipPolicy.RequestItemKey, out var ownerBinding) && ownerBinding is string ownerOutcome)
                requestTelemetry.Properties["SessionOwnerBinding"] = ownerOutcome;

            // DEVICE-IDENTITY-BLOCK-BINDING — set by the kill-switch call sites (telemetry / config).
            if (context.Items.TryGetValue(DeviceIdentityBinding.RequestItemKey, out var identityBinding) && identityBinding is string identityOutcome)
                requestTelemetry.Properties[DeviceIdentityBinding.RequestItemKey] = identityOutcome;

            var reqCtx = context.GetRequestContext();
            var tenantId = reqCtx.TenantId;
            if (string.IsNullOrEmpty(tenantId))
            {
                // Device ingest endpoints do not run through PolicyEnforcementMiddleware, so
                // reqCtx.TenantId is empty — but they carry a cert/token-validated tenant in the
                // X-Tenant-Id header. Only honor the header on the exact routes that actually
                // validate it (TenantHeaderTrustedPaths) and only when it is a well-formed GUID, so
                // other/anonymous routes cannot pollute the requests table with arbitrary tenant ids.
                var path = httpContext.Request.Path.Value ?? string.Empty;
                if (TenantHeaderTrustedPaths.Contains(path))
                {
                    var headerTenant = httpContext.Request.Headers["X-Tenant-Id"].FirstOrDefault();
                    if (!string.IsNullOrEmpty(headerTenant) && Guid.TryParse(headerTenant, out _))
                        tenantId = headerTenant;
                }
            }
            if (!string.IsNullOrEmpty(tenantId))
                requestTelemetry.Properties["TenantId"] = tenantId;
            if (!string.IsNullOrEmpty(reqCtx.UserPrincipalName))
                requestTelemetry.Properties["UserId"] = reqCtx.UserPrincipalName;
            if (!string.IsNullOrEmpty(reqCtx.UserRole))
                requestTelemetry.Properties["UserRole"] = reqCtx.UserRole;

            if (caughtException != null)
                requestTelemetry.Properties["ExceptionType"] = caughtException.GetType().Name;

            // Bypass the worker's adaptive sampling for this item. The SDK sampling processor
            // passes through any item whose SamplingPercentage is already set, so this is the
            // per-item opt-out that host.json "excludedTypes" (host process only) cannot provide.
            // Live-verified 2026-08-23 BEFORE this change: worker request rows carried
            // ItemCount 2-5 on weekdays (~30% of individual requests missing), so count()-based
            // queries undercounted. This copy is the single canonical request record — the host
            // duplicate is dropped by a workspace DCR transformation on AppRequests.
            ((ISupportSampling)requestTelemetry).SamplingPercentage = 100;

            try
            {
                _telemetryClient.TrackRequest(requestTelemetry);
            }
            catch
            {
                // Never let telemetry failures mask the original exception or crash the pipeline
            }
        }
    }
}