mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 04:32:32 +00:00
backend: blob proxy lifetime and timing, one delegation key per process, shared KEV instance, single-flight config misses (audit F11/F12)
- DiagnosticsBlobStreamer disposes the download result on every exit path (it was disposed only on the 413 branch), measures open and transfer separately and fires DiagnosticsDownloadProxied after the copy with Completed=true/false. - UserDelegationKeyCache: one user delegation key per process (24 h, re-mint under a lock with 2 h margin or when a SAS would outlive it) instead of one storage round-trip per download or upload SAS. - KevDataService is resolved as one singleton over the typed-client handler pipeline; the cache warmer used to hydrate a throwaway transient instance while the correlation service held a cold one. - Admin and tenant configuration services single-flight concurrent cache misses (Lazy<Task> per key); the MemoryCache and the in-place projection semantics stay untouched. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,83 @@
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Shared.Models.Config;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Concurrent cache misses on the two configuration services share one repository read
|
||||
/// (audit 2026-09-05 F12). The MemoryCache stays: the hit path and the in-place projection
|
||||
/// semantics are untouched, only the miss path is single-flighted.
|
||||
/// </summary>
|
||||
public class ConfigurationServiceSingleFlightTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Admin_config_concurrent_misses_issue_one_repository_read()
|
||||
{
|
||||
var gate = new TaskCompletionSource<AdminConfiguration?>();
|
||||
var repo = new Mock<IConfigRepository>();
|
||||
repo.Setup(r => r.GetAdminConfigurationAsync()).Returns(gate.Task);
|
||||
var sut = new AdminConfigurationService(repo.Object, NullLogger<AdminConfigurationService>.Instance, new MemoryCache(new MemoryCacheOptions()));
|
||||
|
||||
var calls = Enumerable.Range(0, 10).Select(_ => sut.GetConfigurationAsync()).ToArray();
|
||||
gate.SetResult(AdminConfiguration.CreateDefault());
|
||||
var results = await Task.WhenAll(calls);
|
||||
|
||||
repo.Verify(r => r.GetAdminConfigurationAsync(), Times.Once);
|
||||
Assert.All(results, r => Assert.Same(results[0], r));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Admin_config_after_completion_the_next_miss_reads_again()
|
||||
{
|
||||
var repo = new Mock<IConfigRepository>();
|
||||
repo.Setup(r => r.GetAdminConfigurationAsync()).ReturnsAsync(AdminConfiguration.CreateDefault());
|
||||
var sut = new AdminConfigurationService(repo.Object, NullLogger<AdminConfigurationService>.Instance, new MemoryCache(new MemoryCacheOptions()));
|
||||
|
||||
await sut.GetConfigurationAsync();
|
||||
sut.InvalidateCache();
|
||||
await sut.GetConfigurationAsync();
|
||||
|
||||
repo.Verify(r => r.GetAdminConfigurationAsync(), Times.Exactly(2));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Tenant_config_concurrent_misses_for_one_tenant_issue_one_repository_read()
|
||||
{
|
||||
const string tenantId = "11111111-1111-1111-1111-111111111111";
|
||||
var gate = new TaskCompletionSource<TenantConfiguration?>();
|
||||
var repo = new Mock<IConfigRepository>();
|
||||
repo.Setup(r => r.GetTenantConfigurationAsync(tenantId)).Returns(gate.Task);
|
||||
var sut = new TenantConfigurationService(repo.Object, NullLogger<TenantConfigurationService>.Instance, new MemoryCache(new MemoryCacheOptions()));
|
||||
|
||||
var calls = Enumerable.Range(0, 10).Select(_ => sut.GetConfigurationAsync(tenantId)).ToArray();
|
||||
gate.SetResult(TenantConfiguration.CreateDefault(tenantId));
|
||||
var results = await Task.WhenAll(calls);
|
||||
|
||||
repo.Verify(r => r.GetTenantConfigurationAsync(tenantId), Times.Once);
|
||||
Assert.All(results, r => Assert.Same(results[0], r));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Tenant_config_different_tenants_do_not_share_a_flight()
|
||||
{
|
||||
const string a = "11111111-1111-1111-1111-111111111111";
|
||||
const string b = "22222222-2222-2222-2222-222222222222";
|
||||
var repo = new Mock<IConfigRepository>();
|
||||
repo.Setup(r => r.GetTenantConfigurationAsync(It.IsAny<string>()))
|
||||
.Returns<string>(id => Task.FromResult<TenantConfiguration?>(TenantConfiguration.CreateDefault(id)));
|
||||
var sut = new TenantConfigurationService(repo.Object, NullLogger<TenantConfigurationService>.Instance, new MemoryCache(new MemoryCacheOptions()));
|
||||
|
||||
await Task.WhenAll(sut.GetConfigurationAsync(a), sut.GetConfigurationAsync(b));
|
||||
|
||||
repo.Verify(r => r.GetTenantConfigurationAsync(a), Times.Once);
|
||||
repo.Verify(r => r.GetTenantConfigurationAsync(b), Times.Once);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure;
|
||||
using Azure.Storage.Blobs;
|
||||
using Azure.Storage.Blobs.Models;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
public class UserDelegationKeyCacheTests
|
||||
{
|
||||
private static (UserDelegationKeyCache cache, Mock<BlobServiceClient> client, Func<DateTimeOffset> now, Action<TimeSpan> advance) Build()
|
||||
{
|
||||
var clock = new DateTimeOffset(2026, 9, 5, 12, 0, 0, TimeSpan.Zero);
|
||||
var client = new Mock<BlobServiceClient>();
|
||||
client
|
||||
.Setup(c => c.GetUserDelegationKeyAsync(It.IsAny<DateTimeOffset?>(), It.IsAny<DateTimeOffset>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<DateTimeOffset?, DateTimeOffset, CancellationToken>((start, expires, _) =>
|
||||
{
|
||||
var key = BlobsModelFactory.UserDelegationKey("oid", "tid", start ?? clock, expires, "b", "2024-11-04", "c2lnbmVk");
|
||||
return Task.FromResult(Response.FromValue(key, Mock.Of<Response>()));
|
||||
});
|
||||
var cache = new UserDelegationKeyCache(client.Object, () => clock);
|
||||
return (cache, client, () => clock, delta => clock += delta);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Second_sas_reuses_the_key_instead_of_minting_again()
|
||||
{
|
||||
var (cache, client, now, _) = Build();
|
||||
|
||||
var first = await cache.GetAsync(now() + TimeSpan.FromMinutes(15));
|
||||
var second = await cache.GetAsync(now() + TimeSpan.FromMinutes(60));
|
||||
|
||||
Assert.Same(first, second);
|
||||
client.Verify(c => c.GetUserDelegationKeyAsync(It.IsAny<DateTimeOffset?>(), It.IsAny<DateTimeOffset>(), It.IsAny<CancellationToken>()), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Key_is_minted_for_the_full_lifetime_with_clock_skew_on_the_start()
|
||||
{
|
||||
var (cache, _, now, _) = Build();
|
||||
|
||||
var key = await cache.GetAsync(now() + TimeSpan.FromMinutes(15));
|
||||
|
||||
Assert.Equal(now() - TimeSpan.FromMinutes(5), key.SignedStartsOn);
|
||||
Assert.Equal(now() + UserDelegationKeyCache.KeyLifetime, key.SignedExpiresOn);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Key_is_re_minted_once_the_refresh_margin_is_reached()
|
||||
{
|
||||
var (cache, client, now, advance) = Build();
|
||||
await cache.GetAsync(now() + TimeSpan.FromMinutes(15));
|
||||
|
||||
advance(UserDelegationKeyCache.KeyLifetime - UserDelegationKeyCache.RefreshMargin + TimeSpan.FromMinutes(1));
|
||||
await cache.GetAsync(now() + TimeSpan.FromMinutes(15));
|
||||
|
||||
client.Verify(c => c.GetUserDelegationKeyAsync(It.IsAny<DateTimeOffset?>(), It.IsAny<DateTimeOffset>(), It.IsAny<CancellationToken>()), Times.Exactly(2));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Key_is_re_minted_when_the_sas_would_outlive_it()
|
||||
{
|
||||
var (cache, client, now, _) = Build();
|
||||
await cache.GetAsync(now() + TimeSpan.FromMinutes(15));
|
||||
|
||||
await cache.GetAsync(now() + UserDelegationKeyCache.KeyLifetime + TimeSpan.FromMinutes(1));
|
||||
|
||||
client.Verify(c => c.GetUserDelegationKeyAsync(It.IsAny<DateTimeOffset?>(), It.IsAny<DateTimeOffset>(), It.IsAny<CancellationToken>()), Times.Exactly(2));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Concurrent_first_callers_share_one_mint()
|
||||
{
|
||||
var (cache, client, now, _) = Build();
|
||||
|
||||
var tasks = new Task<UserDelegationKey>[8];
|
||||
for (var i = 0; i < tasks.Length; i++)
|
||||
tasks[i] = Task.Run(() => cache.GetAsync(now() + TimeSpan.FromMinutes(15)));
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
client.Verify(c => c.GetUserDelegationKeyAsync(It.IsAny<DateTimeOffset?>(), It.IsAny<DateTimeOffset>(), It.IsAny<CancellationToken>()), Times.Once);
|
||||
}
|
||||
}
|
||||
@@ -410,8 +410,17 @@ builder.Services.AddSingleton<ISignalRNotificationService>(sp => sp.GetRequiredS
|
||||
// Vulnerability correlation services
|
||||
builder.Services.AddHttpClient<AutopilotMonitor.Functions.Services.Vulnerability.NvdApiClient>()
|
||||
.AddPolicyHandler((sp, _) => sp.GetRequiredService<ResiliencePolicies>().ExternalDataApi);
|
||||
// KevDataService keeps the hydrated catalog as instance state, but AddHttpClient<T> registers T
|
||||
// transient: the cache warmer hydrated one instance while the correlation service and the sync
|
||||
// functions each held a cold one (audit 2026-09-05 F12). The typed-client registration stays for
|
||||
// the named handler pipeline; the singleton below is what every consumer resolves.
|
||||
builder.Services.AddHttpClient<AutopilotMonitor.Functions.Services.Vulnerability.KevDataService>()
|
||||
.AddPolicyHandler((sp, _) => sp.GetRequiredService<ResiliencePolicies>().ExternalDataApi);
|
||||
.AddPolicyHandler((sp, _) => sp.GetRequiredService<ResiliencePolicies>().ExternalDataApi)
|
||||
.SetHandlerLifetime(Timeout.InfiniteTimeSpan);
|
||||
builder.Services.AddSingleton(sp => new AutopilotMonitor.Functions.Services.Vulnerability.KevDataService(
|
||||
sp.GetRequiredService<IHttpClientFactory>().CreateClient(nameof(AutopilotMonitor.Functions.Services.Vulnerability.KevDataService)),
|
||||
sp.GetRequiredService<BlobStorageService>(),
|
||||
sp.GetRequiredService<ILogger<AutopilotMonitor.Functions.Services.Vulnerability.KevDataService>>()));
|
||||
builder.Services.AddHttpClient<AutopilotMonitor.Functions.Services.Vulnerability.MsrcApiClient>()
|
||||
.AddPolicyHandler((sp, _) => sp.GetRequiredService<ResiliencePolicies>().ExternalDataApi);
|
||||
builder.Services.AddHttpClient<AutopilotMonitor.Functions.Services.Vulnerability.EpssApiClient>()
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
@@ -22,6 +23,10 @@ namespace AutopilotMonitor.Functions.Services
|
||||
private const string CacheKey = "admin-config";
|
||||
private static readonly TimeSpan CacheDuration = TimeSpan.FromMinutes(5);
|
||||
|
||||
// Concurrent misses (cold start, after invalidation) share one repository read instead of
|
||||
// each issuing their own (audit 2026-09-05 F12). Lazy guarantees a single factory run.
|
||||
private readonly ConcurrentDictionary<string, Lazy<Task<AdminConfiguration>>> _inFlight = new();
|
||||
|
||||
public AdminConfigurationService(IConfigRepository configRepo, ILogger<AdminConfigurationService> logger, IMemoryCache cache)
|
||||
{
|
||||
_configRepo = configRepo;
|
||||
@@ -43,6 +48,19 @@ namespace AutopilotMonitor.Functions.Services
|
||||
return cachedConfig;
|
||||
}
|
||||
|
||||
var load = _inFlight.GetOrAdd(CacheKey, _ => new Lazy<Task<AdminConfiguration>>(LoadAndCacheAsync));
|
||||
try
|
||||
{
|
||||
return await load.Value;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_inFlight.TryRemove(new KeyValuePair<string, Lazy<Task<AdminConfiguration>>>(CacheKey, load));
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<AdminConfiguration> LoadAndCacheAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
// Load from repository
|
||||
@@ -213,6 +231,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
public void InvalidateCache()
|
||||
{
|
||||
_cache.Remove(CacheKey);
|
||||
_inFlight.TryRemove(CacheKey, out _);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
private readonly BlobServiceClient _blobServiceClient;
|
||||
private readonly ILogger<BlobStorageService> _logger;
|
||||
private readonly bool _usesManagedIdentity;
|
||||
private readonly UserDelegationKeyCache _delegationKeys;
|
||||
|
||||
public BlobStorageService(IConfiguration configuration, ILogger<BlobStorageService> logger)
|
||||
{
|
||||
@@ -54,6 +55,8 @@ namespace AutopilotMonitor.Functions.Services
|
||||
throw new InvalidOperationException(
|
||||
"Blob Storage not configured. Set either 'AzureStorageAccountName' (for Managed Identity) or 'AzureBlobStorageConnectionString'.");
|
||||
}
|
||||
|
||||
_delegationKeys = new UserDelegationKeyCache(_blobServiceClient);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -67,6 +70,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
_blobServiceClient = blobServiceClient;
|
||||
_logger = logger;
|
||||
_usesManagedIdentity = usesManagedIdentity;
|
||||
_delegationKeys = new UserDelegationKeyCache(_blobServiceClient);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -121,8 +125,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
if (_usesManagedIdentity)
|
||||
{
|
||||
// Generate User Delegation SAS (no account key needed)
|
||||
var delegationKey = await _blobServiceClient.GetUserDelegationKeyAsync(
|
||||
DateTimeOffset.UtcNow.AddMinutes(-5), expiresOn);
|
||||
var delegationKey = await _delegationKeys.GetAsync(expiresOn);
|
||||
|
||||
var sasBuilder = new BlobSasBuilder
|
||||
{
|
||||
|
||||
+70
-50
@@ -4,6 +4,7 @@ using AutopilotMonitor.Functions.Functions.Diagnostics;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Shared;
|
||||
using Azure.Storage.Blobs;
|
||||
using Azure.Storage.Blobs.Models;
|
||||
using Microsoft.ApplicationInsights;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -81,15 +82,12 @@ namespace AutopilotMonitor.Functions.Services.Diagnostics
|
||||
? new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds))
|
||||
: new CancellationTokenSource();
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
long contentLength;
|
||||
Stream content;
|
||||
var openWatch = Stopwatch.StartNew();
|
||||
BlobDownloadStreamingResult download;
|
||||
|
||||
if (destination == DiagnosticsDownloadFunction.BlobDestination.Hosted)
|
||||
{
|
||||
var download = await _hostedDiagnostics.OpenReadAsync(blobName, cts.Token);
|
||||
contentLength = download.Value.Details.ContentLength;
|
||||
content = download.Value.Content;
|
||||
download = (await _hostedDiagnostics.OpenReadAsync(blobName, cts.Token)).Value;
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -101,57 +99,79 @@ namespace AutopilotMonitor.Functions.Services.Diagnostics
|
||||
|
||||
var blobUrl = BuildCustomerBlobUrl(tenantConfig.DiagnosticsBlobSasUrl, blobName);
|
||||
var blobClient = new BlobClient(new Uri(blobUrl));
|
||||
var download = await blobClient.DownloadStreamingAsync(cancellationToken: cts.Token);
|
||||
contentLength = download.Value.Details.ContentLength;
|
||||
content = download.Value.Content;
|
||||
download = (await blobClient.DownloadStreamingAsync(cancellationToken: cts.Token)).Value;
|
||||
}
|
||||
sw.Stop();
|
||||
openWatch.Stop();
|
||||
|
||||
var destinationLabel = destination == DiagnosticsDownloadFunction.BlobDestination.Hosted
|
||||
? "Hosted" : "CustomerSas";
|
||||
|
||||
// Enforce size cap before streaming (fast reject).
|
||||
if (maxSizeBytes > 0 && contentLength > maxSizeBytes)
|
||||
// The download result owns the network stream; disposing it on EVERY exit path
|
||||
// (413, copy failure, client abort, success) returns the connection to the pool.
|
||||
using (download)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"DiagnosticsBlobStreamer: Blob {BlobName} for tenant {TenantId} (destination={Destination}) rejected — size {SizeBytes} exceeds limit {MaxSizeBytes}",
|
||||
blobName, tenantId, destinationLabel, contentLength, maxSizeBytes);
|
||||
content.Dispose();
|
||||
var contentLength = download.Details.ContentLength;
|
||||
var destinationLabel = destination == DiagnosticsDownloadFunction.BlobDestination.Hosted
|
||||
? "Hosted"
|
||||
: "CustomerSas";
|
||||
|
||||
return await req.ErrorAsync(HttpStatusCode.RequestEntityTooLarge, Constants.ApiErrorCodes.PayloadTooLarge,
|
||||
$"Diagnostics package size ({contentLength / (1024 * 1024)} MB) exceeds the maximum allowed size ({adminConfig.MaxDiagnosticsDownloadSizeMB} MB).");
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"DiagnosticsBlobStreamer: Proxying blob {BlobName} for tenant {TenantId} (destination={Destination}), size {SizeBytes} bytes, fetch took {DurationMs}ms",
|
||||
blobName, tenantId, destinationLabel, contentLength, sw.ElapsedMilliseconds);
|
||||
|
||||
var props = new Dictionary<string, string>
|
||||
{
|
||||
["TenantId"] = tenantId,
|
||||
["BlobName"] = blobName,
|
||||
["Destination"] = destinationLabel,
|
||||
};
|
||||
if (extraTelemetryProps != null)
|
||||
foreach (var kv in extraTelemetryProps) props[kv.Key] = kv.Value;
|
||||
|
||||
_telemetryClient.TrackEvent("DiagnosticsDownloadProxied",
|
||||
properties: props,
|
||||
metrics: new Dictionary<string, double>
|
||||
if (maxSizeBytes > 0 && contentLength > maxSizeBytes)
|
||||
{
|
||||
["BlobSizeBytes"] = contentLength,
|
||||
["DurationMs"] = sw.ElapsedMilliseconds,
|
||||
});
|
||||
_logger.LogWarning(
|
||||
"DiagnosticsBlobStreamer: Download of {BlobName} for tenant {TenantId} rejected — size {SizeBytes} exceeds limit {MaxSizeBytes}",
|
||||
blobName, tenantId, contentLength, maxSizeBytes);
|
||||
return await req.ErrorAsync(HttpStatusCode.RequestEntityTooLarge, Constants.ApiErrorCodes.PayloadTooLarge,
|
||||
$"Diagnostics package size ({contentLength / (1024 * 1024)} MB) exceeds the configured maximum ({adminConfig.MaxDiagnosticsDownloadSizeMB} MB).");
|
||||
}
|
||||
|
||||
var downloadFilename = ExtractDownloadFilename(blobName);
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
response.Headers.Add("Content-Type", "application/octet-stream");
|
||||
response.Headers.Add("Content-Disposition", $"attachment; filename=\"{downloadFilename}\"");
|
||||
if (contentLength > 0)
|
||||
response.Headers.Add("Content-Length", contentLength.ToString());
|
||||
var downloadFilename = ExtractDownloadFilename(blobName);
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
response.Headers.Add("Content-Type", "application/octet-stream");
|
||||
response.Headers.Add("Content-Disposition", $"attachment; filename=\"{downloadFilename}\"");
|
||||
if (contentLength > 0)
|
||||
response.Headers.Add("Content-Length", contentLength.ToString());
|
||||
|
||||
await content.CopyToAsync(response.Body, cts.Token);
|
||||
return response;
|
||||
// Open and transfer are measured separately: a slow open is the storage account,
|
||||
// a slow transfer is the client or the proxy; the event fires after the copy so
|
||||
// "proxied" means the bytes actually left, not that the blob merely opened.
|
||||
var transferWatch = Stopwatch.StartNew();
|
||||
var completed = false;
|
||||
try
|
||||
{
|
||||
await download.Content.CopyToAsync(response.Body, cts.Token);
|
||||
completed = true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
transferWatch.Stop();
|
||||
|
||||
_logger.LogInformation(
|
||||
"DiagnosticsBlobStreamer: Proxied blob {BlobName} for tenant {TenantId} (destination={Destination}), size {SizeBytes} bytes, open {OpenMs}ms, transfer {TransferMs}ms, completed={Completed}",
|
||||
blobName, tenantId, destinationLabel, contentLength, openWatch.ElapsedMilliseconds, transferWatch.ElapsedMilliseconds, completed);
|
||||
|
||||
var props = new Dictionary<string, string>
|
||||
{
|
||||
["TenantId"] = tenantId,
|
||||
["BlobName"] = blobName,
|
||||
["Destination"] = destinationLabel,
|
||||
["Completed"] = completed ? "true" : "false",
|
||||
};
|
||||
if (extraTelemetryProps != null)
|
||||
{
|
||||
foreach (var kv in extraTelemetryProps)
|
||||
props[kv.Key] = kv.Value;
|
||||
}
|
||||
|
||||
_telemetryClient.TrackEvent("DiagnosticsDownloadProxied",
|
||||
properties: props,
|
||||
metrics: new Dictionary<string, double>
|
||||
{
|
||||
["BlobSizeBytes"] = contentLength,
|
||||
["OpenMs"] = openWatch.ElapsedMilliseconds,
|
||||
["TransferMs"] = transferWatch.ElapsedMilliseconds,
|
||||
["DurationMs"] = openWatch.ElapsedMilliseconds + transferWatch.ElapsedMilliseconds,
|
||||
});
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
+5
-2
@@ -53,6 +53,7 @@ namespace AutopilotMonitor.Functions.Services.Diagnostics
|
||||
private readonly BlobServiceClient _blobServiceClient;
|
||||
private readonly ILogger<HostedDiagnosticsBlobService> _logger;
|
||||
private readonly bool _usesManagedIdentity;
|
||||
private readonly UserDelegationKeyCache _delegationKeys;
|
||||
private int _containerEnsured;
|
||||
|
||||
public HostedDiagnosticsBlobService(IConfiguration configuration, ILogger<HostedDiagnosticsBlobService> logger)
|
||||
@@ -84,6 +85,8 @@ namespace AutopilotMonitor.Functions.Services.Diagnostics
|
||||
throw new InvalidOperationException(
|
||||
"HostedDiagnosticsBlobService not configured. Set either 'AzureStorageAccountName' (Managed Identity) or 'AzureBlobStorageConnectionString'.");
|
||||
}
|
||||
|
||||
_delegationKeys = new UserDelegationKeyCache(_blobServiceClient);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -99,6 +102,7 @@ namespace AutopilotMonitor.Functions.Services.Diagnostics
|
||||
_blobServiceClient = blobServiceClient ?? throw new ArgumentNullException(nameof(blobServiceClient));
|
||||
_logger = logger;
|
||||
_usesManagedIdentity = usesManagedIdentity;
|
||||
_delegationKeys = new UserDelegationKeyCache(_blobServiceClient);
|
||||
}
|
||||
|
||||
// -------- Public API --------
|
||||
@@ -330,8 +334,7 @@ namespace AutopilotMonitor.Functions.Services.Diagnostics
|
||||
|
||||
if (UsesManagedIdentity)
|
||||
{
|
||||
var delegationKey = await _blobServiceClient.GetUserDelegationKeyAsync(
|
||||
DateTimeOffset.UtcNow.AddMinutes(-5), expiresOn, cancellationToken);
|
||||
var delegationKey = await _delegationKeys.GetAsync(expiresOn, cancellationToken);
|
||||
|
||||
var sasUri = new BlobUriBuilder(blobClient.Uri)
|
||||
{
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
@@ -27,6 +28,10 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
private static readonly TimeSpan CacheDuration = TimeSpan.FromMinutes(5);
|
||||
|
||||
// Concurrent misses for one tenant (cold start, after invalidation) share one repository
|
||||
// read instead of each issuing their own (audit 2026-09-05 F12).
|
||||
private readonly ConcurrentDictionary<string, Lazy<Task<TenantConfiguration>>> _inFlight = new();
|
||||
|
||||
public TenantConfigurationService(
|
||||
IConfigRepository configRepo,
|
||||
ILogger<TenantConfigurationService> logger,
|
||||
@@ -75,6 +80,19 @@ namespace AutopilotMonitor.Functions.Services
|
||||
return (await ProjectAsync(cachedConfig))!;
|
||||
}
|
||||
|
||||
var load = _inFlight.GetOrAdd(cacheKey, _ => new Lazy<Task<TenantConfiguration>>(() => LoadAndCacheAsync(tenantId, cacheKey)));
|
||||
try
|
||||
{
|
||||
return await load.Value;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_inFlight.TryRemove(new KeyValuePair<string, Lazy<Task<TenantConfiguration>>>(cacheKey, load));
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<TenantConfiguration> LoadAndCacheAsync(string tenantId, string cacheKey)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Load from repository
|
||||
@@ -210,6 +228,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
public void InvalidateCache(string tenantId)
|
||||
{
|
||||
_cache.Remove($"tenant-config:{tenantId}");
|
||||
_inFlight.TryRemove($"tenant-config:{tenantId}", out _);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure.Storage.Blobs;
|
||||
using Azure.Storage.Blobs.Models;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
/// <summary>
|
||||
/// One user delegation key per process instead of one per SAS. Under Managed Identity every
|
||||
/// blob SAS needs a delegation key from the service; minting it per download or upload URL
|
||||
/// costs a storage round-trip and an AAD token exchange on the hot path (audit 2026-09-05
|
||||
/// F11). A key is valid for up to seven days; this cache mints one for
|
||||
/// <see cref="KeyLifetime"/>, hands it out while at least <see cref="RefreshMargin"/> of
|
||||
/// validity remains and the requested SAS expiry fits inside it, and re-mints under a lock
|
||||
/// otherwise. The SAS itself keeps its own short lifetime and blob scope.
|
||||
/// </summary>
|
||||
public sealed class UserDelegationKeyCache
|
||||
{
|
||||
public static readonly TimeSpan KeyLifetime = TimeSpan.FromHours(24);
|
||||
public static readonly TimeSpan RefreshMargin = TimeSpan.FromHours(2);
|
||||
|
||||
/// <summary>Clock-skew tolerance on the key start, mirrors the SAS <c>StartsOn</c>.</summary>
|
||||
private static readonly TimeSpan StartSkew = TimeSpan.FromMinutes(5);
|
||||
|
||||
private readonly BlobServiceClient _client;
|
||||
private readonly Func<DateTimeOffset> _utcNow;
|
||||
private readonly SemaphoreSlim _gate = new(1, 1);
|
||||
private UserDelegationKey? _key;
|
||||
|
||||
public UserDelegationKeyCache(BlobServiceClient client, Func<DateTimeOffset>? utcNow = null)
|
||||
{
|
||||
_client = client ?? throw new ArgumentNullException(nameof(client));
|
||||
_utcNow = utcNow ?? (() => DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a key that covers a SAS expiring at <paramref name="sasExpiresOn"/>, minting a
|
||||
/// fresh one when the cached key is missing, close to expiry or too short for the SAS.
|
||||
/// </summary>
|
||||
public async Task<UserDelegationKey> GetAsync(DateTimeOffset sasExpiresOn, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var cached = _key;
|
||||
if (Covers(cached, sasExpiresOn)) return cached!;
|
||||
|
||||
await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
cached = _key;
|
||||
if (Covers(cached, sasExpiresOn)) return cached!;
|
||||
|
||||
var now = _utcNow();
|
||||
var fresh = await _client.GetUserDelegationKeyAsync(now - StartSkew, now + KeyLifetime, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
_key = fresh.Value;
|
||||
return fresh.Value;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_gate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private bool Covers(UserDelegationKey? key, DateTimeOffset sasExpiresOn)
|
||||
{
|
||||
if (key == null) return false;
|
||||
var now = _utcNow();
|
||||
return key.SignedExpiresOn - now >= RefreshMargin && sasExpiresOn <= key.SignedExpiresOn;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user