mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-25 13:10:13 +00:00
Compare commits
44 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7fb78a7e29 | |||
| bfd8d58142 | |||
| 8ad472d1bc | |||
| fc5e2cd751 | |||
| 9540791441 | |||
| 058da1627f | |||
| cb9b3fe404 | |||
| 12315b2f2d | |||
| 0a4a051866 | |||
| f47d961947 | |||
| c701d0d694 | |||
| d4a3edf849 | |||
| e164eac1eb | |||
| c81591b498 | |||
| d8c03d9d95 | |||
| ffb133f0b9 | |||
| b3fe321f0d | |||
| 69d210c51d | |||
| 06a88e9897 | |||
| aee30f68e8 | |||
| 3db9a884de | |||
| 0037b8cb12 | |||
| 205b13dbb9 | |||
| b938a03367 | |||
| d474c2be9f | |||
| 96390beeb9 | |||
| fe63a0a352 | |||
| 2e36e9c234 | |||
| b83aa6d97d | |||
| f42fcbcb3c | |||
| f70ba354f8 | |||
| fc17047290 | |||
| 7bbc2cbddf | |||
| 9a78daa880 | |||
| 5d532d3dbc | |||
| a22c26eb9d | |||
| b84c467255 | |||
| 53124e94a6 | |||
| 838068fd31 | |||
| 71282f3556 | |||
| 7080dcc45a | |||
| 5a62714dc2 | |||
| c930d18f1e | |||
| 583c81c9a8 |
+1
-1
Submodule .claude/commands updated: 13cfe49e35...315ad0e736
+1
-1
Submodule infra updated: 742249e7e3...8a8f8c66dd
+1
-1
Submodule internal updated: 6c492edd11...1e6b548904
+193
@@ -0,0 +1,193 @@
|
||||
using System;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Gather;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Gather
|
||||
{
|
||||
/// <summary>
|
||||
/// A configured diagnostics entry is resolved to the folder and pattern that get
|
||||
/// enumerated, and the guard judges exactly that folder. The cases below pin the
|
||||
/// property that matters: no spelling of an allowed entry ever selects its parent.
|
||||
/// </summary>
|
||||
public sealed class DiagnosticsPathGuardsTests
|
||||
{
|
||||
private const string TestProfile = @"C:\Users\TestUser";
|
||||
|
||||
private static readonly Func<string, bool> IsDirectory = _ => true;
|
||||
private static readonly Func<string, bool> IsNotDirectory = _ => false;
|
||||
|
||||
private static DiagnosticsPathGuards.CollectionTarget Resolve(
|
||||
string path,
|
||||
bool includeSubfolders = false,
|
||||
bool unrestrictedMode = false,
|
||||
string? userProfilePath = null,
|
||||
Func<string, bool>? directoryExists = null)
|
||||
{
|
||||
Assert.True(DiagnosticsPathGuards.TryResolveCollectionTarget(
|
||||
path, includeSubfolders, unrestrictedMode, userProfilePath!, directoryExists ?? IsDirectory, out var target));
|
||||
return target;
|
||||
}
|
||||
|
||||
private static void AssertRefused(
|
||||
string path,
|
||||
bool unrestrictedMode = false,
|
||||
string? userProfilePath = null,
|
||||
Func<string, bool>? directoryExists = null)
|
||||
{
|
||||
Assert.False(DiagnosticsPathGuards.TryResolveCollectionTarget(
|
||||
path, true, unrestrictedMode, userProfilePath!, directoryExists ?? IsDirectory, out var target));
|
||||
Assert.Null(target);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------ folder entries
|
||||
|
||||
[Theory]
|
||||
[InlineData(@"C:\Windows\Logs")]
|
||||
[InlineData(@"C:\Windows\Logs\")]
|
||||
[InlineData(@"C:\Windows\SetupDiag")]
|
||||
[InlineData(@"C:\ProgramData\AutopilotMonitor")]
|
||||
[InlineData(@"C:\ProgramData\Microsoft\DiagnosticLogCSP")]
|
||||
public void Folder_entry_collects_from_itself_never_from_its_parent(string configured)
|
||||
{
|
||||
var target = Resolve(configured, includeSubfolders: true);
|
||||
|
||||
Assert.Equal(configured.TrimEnd('\\'), target.Folder, ignoreCase: true);
|
||||
Assert.Equal("*", target.Pattern);
|
||||
Assert.True(target.Recurse);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Trailing_separator_means_folder_even_when_the_folder_does_not_exist_yet()
|
||||
{
|
||||
var target = Resolve(@"C:\Windows\Logs\NotThereYet\", directoryExists: IsNotDirectory);
|
||||
|
||||
Assert.Equal(@"C:\Windows\Logs\NotThereYet", target.Folder, ignoreCase: true);
|
||||
Assert.Equal("*", target.Pattern);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Folder_name_with_a_dot_is_still_a_folder()
|
||||
{
|
||||
var target = Resolve(@"C:\Windows\Logs\vendor.v2");
|
||||
|
||||
Assert.Equal(@"C:\Windows\Logs\vendor.v2", target.Folder, ignoreCase: true);
|
||||
Assert.Equal("*", target.Pattern);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------ wildcard entries
|
||||
|
||||
[Fact]
|
||||
public void Wildcard_entry_collects_the_pattern_from_its_validated_parent()
|
||||
{
|
||||
var target = Resolve(@"C:\Windows\Panther\*.log", includeSubfolders: true);
|
||||
|
||||
Assert.Equal(@"C:\Windows\Panther", target.Folder, ignoreCase: true);
|
||||
Assert.Equal("*.log", target.Pattern);
|
||||
Assert.True(target.Recurse);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Wildcard_beside_an_allowlisted_file_is_refused()
|
||||
{
|
||||
// Only ReportingEvents.log is on the allowlist, not the folder it lives in.
|
||||
AssertRefused(@"C:\Windows\SoftwareDistribution\*.log");
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------ file entries
|
||||
|
||||
[Fact]
|
||||
public void File_entry_never_recurses_because_its_folder_was_not_validated()
|
||||
{
|
||||
var target = Resolve(
|
||||
@"C:\Windows\SoftwareDistribution\ReportingEvents.log",
|
||||
includeSubfolders: true,
|
||||
directoryExists: IsNotDirectory);
|
||||
|
||||
Assert.Equal(@"C:\Windows\SoftwareDistribution", target.Folder, ignoreCase: true);
|
||||
Assert.Equal("ReportingEvents.log", target.Pattern);
|
||||
Assert.False(target.Recurse);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Missing_path_without_a_dot_is_one_file_name_not_a_wildcard_on_the_parent()
|
||||
{
|
||||
var target = Resolve(@"C:\Windows\Logs\nodot", includeSubfolders: true, directoryExists: IsNotDirectory);
|
||||
|
||||
Assert.Equal(@"C:\Windows\Logs", target.Folder, ignoreCase: true);
|
||||
Assert.Equal("nodot", target.Pattern);
|
||||
Assert.False(target.Recurse);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------ refusals
|
||||
|
||||
[Theory]
|
||||
[InlineData(@"C:\Windows")]
|
||||
[InlineData(@"C:\ProgramData")]
|
||||
[InlineData(@"C:\ProgramData\Microsoft")]
|
||||
[InlineData(@"C:\Windows\Logs\..\System32")]
|
||||
[InlineData(@"C:\Windows\LogsEvil")]
|
||||
[InlineData("")]
|
||||
public void Paths_outside_the_allowlist_are_refused(string configured)
|
||||
{
|
||||
AssertRefused(configured);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(@"C:\Users")]
|
||||
[InlineData(@"C:\Users\TestUser\Documents")]
|
||||
[InlineData(@"C:\Windows\System32\config")]
|
||||
[InlineData(@"C:\Windows\System32\config\*")]
|
||||
public void Hard_blocks_hold_in_unrestricted_mode(string configured)
|
||||
{
|
||||
AssertRefused(configured, unrestrictedMode: true);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------ user-profile token
|
||||
|
||||
[Fact]
|
||||
public void Expanded_token_path_outside_AppData_is_refused_even_in_unrestricted_mode()
|
||||
{
|
||||
// The service expands %LOGGED_ON_USER_PROFILE% before the guard sees the path; an
|
||||
// unexpanded token is not under C:\Users lexically and used to slip past the block.
|
||||
AssertRefused(TestProfile + @"\Documents", unrestrictedMode: true, userProfilePath: TestProfile);
|
||||
AssertRefused(TestProfile + @"\Documents\*.txt", unrestrictedMode: true, userProfilePath: TestProfile);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Expanded_token_path_on_the_profile_allowlist_is_admitted_in_restricted_mode()
|
||||
{
|
||||
var target = Resolve(
|
||||
TestProfile + @"\AppData\Local\RealmJoin\Logs\*.log",
|
||||
userProfilePath: TestProfile);
|
||||
|
||||
Assert.Equal(TestProfile + @"\AppData\Local\RealmJoin\Logs", target.Folder, ignoreCase: true);
|
||||
Assert.Equal("*.log", target.Pattern);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Profile_path_without_the_token_stays_blocked()
|
||||
{
|
||||
AssertRefused(TestProfile + @"\AppData\Local\RealmJoin\Logs\*.log", userProfilePath: null);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------ recursion guard
|
||||
|
||||
[Theory]
|
||||
[InlineData(@"C:\Users")]
|
||||
[InlineData(@"C:\Users\TestUser")]
|
||||
[InlineData(@"C:\Windows\System32\config")]
|
||||
[InlineData(@"C:\Windows\System32\config\RegBack")]
|
||||
public void Recursion_never_enters_a_hard_blocked_subtree_in_unrestricted_mode(string directory)
|
||||
{
|
||||
Assert.False(DiagnosticsPathGuards.IsDirectoryEnterable(directory, unrestrictedMode: true, userProfilePath: null));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Recursion_enters_ordinary_subfolders()
|
||||
{
|
||||
Assert.True(DiagnosticsPathGuards.IsDirectoryEnterable(@"C:\Windows\System32\drivers", unrestrictedMode: true, userProfilePath: null));
|
||||
Assert.True(DiagnosticsPathGuards.IsDirectoryEnterable(@"C:\Windows\Logs\CBS", unrestrictedMode: false, userProfilePath: null));
|
||||
Assert.False(DiagnosticsPathGuards.IsDirectoryEnterable(@"C:\Windows\System32\drivers", unrestrictedMode: false, userProfilePath: null));
|
||||
}
|
||||
}
|
||||
}
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Ime
|
||||
{
|
||||
/// <summary>
|
||||
/// The poll loop's timing maxima (session c3ecb568: the loop was not scheduled for 26 s at
|
||||
/// 100 % VM CPU and nothing in the telemetry could show it): the longest pass and the
|
||||
/// longest pause between two passes, on a monotonic clock, restart-safe like the other
|
||||
/// health counters.
|
||||
/// </summary>
|
||||
public sealed class ImeLogTrackerPollTimingTests
|
||||
{
|
||||
/// <summary>Monotonic readings the tracker takes at the start and the end of every pass.</summary>
|
||||
private sealed class Clock
|
||||
{
|
||||
private readonly Queue<long> _readings = new Queue<long>();
|
||||
public void Enqueue(params long[] ms) { foreach (var m in ms) _readings.Enqueue(m); }
|
||||
public long Next() => _readings.Dequeue();
|
||||
}
|
||||
|
||||
private static ImeLogTracker Build(TempDirectory tmp, Clock clock)
|
||||
{
|
||||
var tracker = new ImeLogTracker(tmp.Path, new List<ImeLogPattern>(), new AgentLogger(tmp.Path, AgentLogLevel.Info), stateDirectory: tmp.Path);
|
||||
tracker.MonotonicMillisProvider = clock.Next;
|
||||
return tracker;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Longest_pass_and_longest_pause_are_kept_as_session_maxima()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var clock = new Clock();
|
||||
var tracker = Build(tmp, clock);
|
||||
|
||||
// Pass 1 takes 30 ms; the loop is not scheduled again for 26.1 s; pass 2 takes 10 ms,
|
||||
// pass 3 follows after the normal 100 ms.
|
||||
clock.Enqueue(0, 30, 26130, 26140, 26240, 26245);
|
||||
await tracker.RunPollPassAsync(CancellationToken.None);
|
||||
await tracker.RunPollPassAsync(CancellationToken.None);
|
||||
await tracker.RunPollPassAsync(CancellationToken.None);
|
||||
|
||||
var health = tracker.GetHealthSnapshot();
|
||||
Assert.Equal(30, health.PassMaxMs);
|
||||
Assert.Equal(26100, health.PassGapMaxMs);
|
||||
tracker.Dispose();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_fresh_tracker_reports_zero()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var tracker = Build(tmp, new Clock());
|
||||
|
||||
var health = tracker.GetHealthSnapshot();
|
||||
Assert.Equal(0, health.PassMaxMs);
|
||||
Assert.Equal(0, health.PassGapMaxMs);
|
||||
tracker.Dispose();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Maxima_survive_a_restart_and_are_never_lowered()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var clock = new Clock();
|
||||
var tracker1 = Build(tmp, clock);
|
||||
clock.Enqueue(0, 40, 5140, 5150);
|
||||
await tracker1.RunPollPassAsync(CancellationToken.None);
|
||||
await tracker1.RunPollPassAsync(CancellationToken.None);
|
||||
tracker1.SaveStateForTest();
|
||||
tracker1.Dispose();
|
||||
|
||||
var tracker2 = Build(tmp, clock);
|
||||
tracker2.LoadStateForTest();
|
||||
Assert.Equal(40, tracker2.GetHealthSnapshot().PassMaxMs);
|
||||
Assert.Equal(5100, tracker2.GetHealthSnapshot().PassGapMaxMs);
|
||||
|
||||
// The first pass after a restart has no pause to measure; a calmer session afterwards
|
||||
// never lowers what the session already saw.
|
||||
clock.Enqueue(6000, 6010, 6110, 6115);
|
||||
await tracker2.RunPollPassAsync(CancellationToken.None);
|
||||
await tracker2.RunPollPassAsync(CancellationToken.None);
|
||||
var health = tracker2.GetHealthSnapshot();
|
||||
Assert.Equal(40, health.PassMaxMs);
|
||||
Assert.Equal(5100, health.PassGapMaxMs);
|
||||
tracker2.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
+204
@@ -0,0 +1,204 @@
|
||||
#nullable enable
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Gather;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Runtime
|
||||
{
|
||||
/// <summary>
|
||||
/// The hard-blocked event-log channels (Security, PowerShell, Sysmon) are unreadable through
|
||||
/// the diagnostics package as well — by path, by wildcard, by resolved channel, and in
|
||||
/// unrestricted mode. Before this the block held only in the event-log collector.
|
||||
/// </summary>
|
||||
public sealed class DiagnosticsEventLogBlockTests
|
||||
{
|
||||
private const string WinevtLogs = @"C:\Windows\System32\winevt\Logs";
|
||||
|
||||
// A channel no machine has: the export fails and the packager falls back to the raw copy.
|
||||
private const string HarmlessChannel = "AutopilotMonitor-Tests-NoSuchChannel/Operational";
|
||||
|
||||
[Theory]
|
||||
[InlineData("Security.evtx")]
|
||||
[InlineData("security.EVTX")]
|
||||
[InlineData("Microsoft-Windows-PowerShell%4Operational.evtx")]
|
||||
[InlineData("Microsoft-Windows-PowerShell%4Admin.evtx")]
|
||||
[InlineData("Windows PowerShell.evtx")]
|
||||
[InlineData("Microsoft-Windows-Sysmon%4Operational.evtx")]
|
||||
[InlineData("Archive-Security-2026-09-21-10-15-30-123.evtx")]
|
||||
[InlineData("Archive-Microsoft-Windows-PowerShell%4Operational-2026-09-21-10-15-30-123.evtx")]
|
||||
public void Hard_blocked_channel_files_are_recognised_by_name(string fileName)
|
||||
{
|
||||
Assert.True(DiagnosticsPathGuards.IsHardBlockedEventLogFile(Path.Combine(WinevtLogs, fileName)));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("System.evtx")]
|
||||
[InlineData("Application.evtx")]
|
||||
[InlineData("SecurityAudit.evtx")]
|
||||
[InlineData("Security.log")]
|
||||
[InlineData("Microsoft-Windows-Security-Mitigations%4KernelMode.evtx")]
|
||||
// Same boundary as the collector: only '/' continues a blocked channel name.
|
||||
[InlineData("Microsoft-Windows-PowerShell-DesiredStateConfiguration-FileDownloadManager%4Operational.evtx")]
|
||||
[InlineData("Archive-System-2026-09-21-10-15-30-123.evtx")]
|
||||
public void Other_files_are_not_mistaken_for_a_blocked_channel(string fileName)
|
||||
{
|
||||
Assert.False(DiagnosticsPathGuards.IsHardBlockedEventLogFile(Path.Combine(WinevtLogs, fileName)));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public void Path_guard_refuses_a_blocked_channel_file_in_every_mode(bool unrestrictedMode)
|
||||
{
|
||||
Assert.False(DiagnosticsPathGuards.IsDiagnosticsPathAllowed(
|
||||
WinevtLogs + @"\Security.evtx", unrestrictedMode));
|
||||
Assert.False(DiagnosticsPathGuards.IsDiagnosticsPathAllowed(
|
||||
WinevtLogs + @"\Microsoft-Windows-PowerShell%4Operational.evtx", unrestrictedMode));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Path_guard_still_admits_other_channels_and_wildcards()
|
||||
{
|
||||
Assert.True(DiagnosticsPathGuards.IsDiagnosticsPathAllowed(WinevtLogs + @"\System.evtx"));
|
||||
// Undecidable on the string — the packager filters per file (tests below).
|
||||
Assert.True(DiagnosticsPathGuards.IsDiagnosticsPathAllowed(WinevtLogs + @"\*.evtx"));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("Security", true)]
|
||||
[InlineData("Microsoft-Windows-PowerShell/Operational", true)]
|
||||
[InlineData(" Windows PowerShell ", true)]
|
||||
[InlineData("Microsoft-Windows-Sysmon/Operational", true)]
|
||||
[InlineData("System", false)]
|
||||
[InlineData("", false)]
|
||||
[InlineData(null, false)]
|
||||
public void Channel_block_is_one_matcher_for_collector_and_package(string? channel, bool blocked)
|
||||
{
|
||||
Assert.Equal(blocked, GatherRuleGuards.IsHardBlockedEventLogChannel(channel!));
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------ packager ----
|
||||
|
||||
// %TEMP% lives under C:\Users, which the path guard blocks in every mode, so the
|
||||
// configured folder sits beside the test assembly and the run is unrestricted — which
|
||||
// is also the strongest form of the claim: not even unrestricted mode collects these.
|
||||
private sealed class EvtxFolder : System.IDisposable
|
||||
{
|
||||
public string Path { get; } = System.IO.Path.Combine(
|
||||
System.AppContext.BaseDirectory, "diag-evtx-" + System.Guid.NewGuid().ToString("N"));
|
||||
|
||||
public EvtxFolder(params string[] fileNames)
|
||||
{
|
||||
Directory.CreateDirectory(Path);
|
||||
foreach (var name in fileNames)
|
||||
File.WriteAllBytes(System.IO.Path.Combine(Path, name), new byte[] { 1, 2, 3 });
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
try { Directory.Delete(Path, recursive: true); } catch { }
|
||||
}
|
||||
}
|
||||
|
||||
private static DiagnosticsPackageService BuildService(
|
||||
TempDirectory scratch, string configuredPath, System.Func<string, string> channelResolver)
|
||||
{
|
||||
var logger = new AgentLogger(Directory.CreateDirectory(Path.Combine(scratch.Path, "logs")).FullName);
|
||||
var apiClient = new BackendApiClient(
|
||||
httpClient: new System.Net.Http.HttpClient(),
|
||||
baseUrl: "http://localhost",
|
||||
manufacturer: string.Empty,
|
||||
model: string.Empty,
|
||||
serialNumber: string.Empty,
|
||||
useBootstrapTokenAuth: false,
|
||||
bootstrapToken: null,
|
||||
agentVersion: "0.0.0",
|
||||
logger: logger);
|
||||
|
||||
var cfg = new AgentConfiguration
|
||||
{
|
||||
SessionId = "S1",
|
||||
TenantId = "T1",
|
||||
ApiBaseUrl = "http://localhost",
|
||||
UnrestrictedMode = true,
|
||||
};
|
||||
cfg.DiagnosticsLogPaths.Add(new DiagnosticsLogPath { Path = configuredPath });
|
||||
|
||||
string Sub(string name) => Directory.CreateDirectory(Path.Combine(scratch.Path, name)).FullName;
|
||||
return new DiagnosticsPackageService(
|
||||
cfg,
|
||||
logger,
|
||||
apiClient,
|
||||
agentLogFolderOverride: Sub("agentlogs"),
|
||||
imeLogFolderOverride: Sub("ime"),
|
||||
agentStateFolderOverride: Sub("state"),
|
||||
agentSpoolFolderOverride: Sub("spool"),
|
||||
agentDataFolderOverride: Sub("data"),
|
||||
sectionFolderOverrides: null,
|
||||
devicePreparationProbe: () => false)
|
||||
{
|
||||
EventLogChannelResolver = channelResolver,
|
||||
};
|
||||
}
|
||||
|
||||
private static List<string> ZipEntryNames(byte[] bytes)
|
||||
{
|
||||
using var ms = new MemoryStream(bytes);
|
||||
using var archive = new ZipArchive(ms, ZipArchiveMode.Read);
|
||||
return archive.Entries.Select(e => e.FullName).ToList();
|
||||
}
|
||||
|
||||
private static string ReadManifest(byte[] bytes)
|
||||
{
|
||||
using var ms = new MemoryStream(bytes);
|
||||
using var archive = new ZipArchive(ms, ZipArchiveMode.Read);
|
||||
using var reader = new StreamReader(archive.GetEntry("package-manifest.txt")!.Open());
|
||||
return reader.ReadToEnd();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Wildcard_entry_skips_blocked_channel_files_and_keeps_the_rest()
|
||||
{
|
||||
using var scratch = new TempDirectory();
|
||||
using var folder = new EvtxFolder(
|
||||
"Security.evtx", "Microsoft-Windows-PowerShell%4Operational.evtx", "Setup.evtx");
|
||||
|
||||
var service = BuildService(scratch, Path.Combine(folder.Path, "*.evtx"), _ => HarmlessChannel);
|
||||
var bytes = service.BuildArchiveBytes(enrollmentSucceeded: false);
|
||||
|
||||
var entries = ZipEntryNames(bytes);
|
||||
Assert.Contains(entries, e => e.EndsWith("/Setup.evtx"));
|
||||
Assert.DoesNotContain(entries, e => e.EndsWith("Security.evtx"));
|
||||
Assert.DoesNotContain(entries, e => e.Contains("PowerShell"));
|
||||
|
||||
var manifest = ReadManifest(bytes);
|
||||
Assert.Contains("BLOCKED (event-log channel guard", manifest);
|
||||
Assert.Contains("Security.evtx", manifest);
|
||||
Assert.Equal(2, service.LastPackaging!.ProblemsByKind["path-guard"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void File_with_a_harmless_name_is_blocked_when_it_resolves_to_a_blocked_channel()
|
||||
{
|
||||
// A channel is free to write to any file name; and the resolver's naming fallback
|
||||
// maps a stray file onto a live channel. The export must never read that channel.
|
||||
using var scratch = new TempDirectory();
|
||||
using var folder = new EvtxFolder("Harmless.evtx");
|
||||
|
||||
var service = BuildService(scratch, Path.Combine(folder.Path, "Harmless.evtx"), _ => "Security");
|
||||
var bytes = service.BuildArchiveBytes(enrollmentSucceeded: false);
|
||||
|
||||
Assert.DoesNotContain(ZipEntryNames(bytes), e => e.EndsWith("Harmless.evtx"));
|
||||
Assert.Contains("BLOCKED (event-log channel guard, channel 'Security')", ReadManifest(bytes));
|
||||
}
|
||||
}
|
||||
}
|
||||
+85
@@ -206,6 +206,91 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Runtime
|
||||
Assert.Contains($"BLOCKED (path guard): {configuredPath}", manifest);
|
||||
}
|
||||
|
||||
// %TEMP% lives under C:\Users, which the guard blocks in every mode — the positive
|
||||
// configured-path cases run beside the test assembly, in unrestricted mode.
|
||||
private sealed class ConfiguredPathTree : System.IDisposable
|
||||
{
|
||||
public string Root { get; } = Path.Combine(
|
||||
System.AppContext.BaseDirectory, "diag-paths-" + System.Guid.NewGuid().ToString("N"));
|
||||
public string Logs => Path.Combine(Root, "Logs");
|
||||
|
||||
public ConfiguredPathTree()
|
||||
{
|
||||
Directory.CreateDirectory(Path.Combine(Logs, "sub"));
|
||||
File.WriteAllText(Path.Combine(Root, "parent-only.txt"), "not configured");
|
||||
File.WriteAllText(Path.Combine(Logs, "app.log"), "configured");
|
||||
File.WriteAllText(Path.Combine(Logs, "sub", "app.log"), "below");
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
try { Directory.Delete(Root, recursive: true); } catch { /* best effort */ }
|
||||
}
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("")]
|
||||
[InlineData("\\")]
|
||||
public void BuildArchiveBytes_folder_entry_collects_the_folder_not_its_parent(string trailing)
|
||||
{
|
||||
using var rig = new Rig();
|
||||
using var tree = new ConfiguredPathTree();
|
||||
|
||||
var bytes = rig.Build(cfg =>
|
||||
{
|
||||
cfg.UnrestrictedMode = true;
|
||||
cfg.DiagnosticsLogPaths.Add(new AutopilotMonitor.Shared.Models.DiagnosticsLogPath
|
||||
{
|
||||
Path = tree.Logs + trailing,
|
||||
IncludeSubfolders = true,
|
||||
});
|
||||
})
|
||||
.BuildArchiveBytes(enrollmentSucceeded: true);
|
||||
|
||||
var entries = ZipEntryNames(bytes);
|
||||
Assert.Contains("AdditionalLogs/Logs/app.log", entries);
|
||||
Assert.Contains("AdditionalLogs/Logs/sub/app.log", entries);
|
||||
Assert.DoesNotContain(entries, e => e.EndsWith("parent-only.txt"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildArchiveBytes_file_entry_ignores_include_subfolders()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
using var tree = new ConfiguredPathTree();
|
||||
|
||||
var bytes = rig.Build(cfg =>
|
||||
{
|
||||
cfg.UnrestrictedMode = true;
|
||||
cfg.DiagnosticsLogPaths.Add(new AutopilotMonitor.Shared.Models.DiagnosticsLogPath
|
||||
{
|
||||
Path = Path.Combine(tree.Logs, "app.log"),
|
||||
IncludeSubfolders = true,
|
||||
});
|
||||
})
|
||||
.BuildArchiveBytes(enrollmentSucceeded: true);
|
||||
|
||||
var entries = ZipEntryNames(bytes);
|
||||
Assert.Contains("AdditionalLogs/Logs/app.log", entries);
|
||||
Assert.DoesNotContain("AdditionalLogs/Logs/sub/app.log", entries);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EnumerateFilesNoReparseDirs_asks_the_guard_before_entering_a_subfolder()
|
||||
{
|
||||
using var tree = new ConfiguredPathTree();
|
||||
var refused = new List<string>();
|
||||
|
||||
var files = DiagnosticsPackageService.EnumerateFilesNoReparseDirs(
|
||||
tree.Root, "*", recurse: true,
|
||||
mayEnterDirectory: dir => !dir.EndsWith("sub"),
|
||||
refusedDirectories: refused);
|
||||
|
||||
Assert.Contains(Path.Combine(tree.Logs, "app.log"), files);
|
||||
Assert.DoesNotContain(Path.Combine(tree.Logs, "sub", "app.log"), files);
|
||||
Assert.Equal(new[] { Path.Combine(tree.Logs, "sub") }, refused);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildArchiveBytes_excludes_top_level_session_id_and_bootstrap_config()
|
||||
{
|
||||
|
||||
@@ -302,6 +302,114 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
Assert.False(File.Exists(path));
|
||||
}
|
||||
|
||||
// ================================================================= Multi-instance guard
|
||||
|
||||
private static AutopilotMonitor.Agent.V2.Program.AgentProcessFact Proc(int pid, int sessionId)
|
||||
=> new AutopilotMonitor.Agent.V2.Program.AgentProcessFact(pid, sessionId);
|
||||
|
||||
[Fact]
|
||||
public void FindSiblingAgentPid_ignores_a_same_named_process_in_a_user_session()
|
||||
{
|
||||
// The agent (pid 100, session 0) starts while a standard user keeps a process with
|
||||
// the agent's image name resident in their own session. It must not block the start.
|
||||
var matches = new[] { Proc(100, 0), Proc(200, 1) };
|
||||
|
||||
Assert.Equal(0, AutopilotMonitor.Agent.V2.Program.FindSiblingAgentPid(matches, ownPid: 100));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(1)]
|
||||
[InlineData(2)]
|
||||
[InlineData(7)]
|
||||
[InlineData(-1)]
|
||||
public void FindSiblingAgentPid_ignores_every_session_but_zero(int sessionId)
|
||||
{
|
||||
var matches = new[] { Proc(100, 0), Proc(200, sessionId) };
|
||||
|
||||
Assert.Equal(0, AutopilotMonitor.Agent.V2.Program.FindSiblingAgentPid(matches, ownPid: 100));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FindSiblingAgentPid_reports_a_second_runtime_in_session_zero()
|
||||
{
|
||||
// Self-update restart while the old runtime is still alive — the single-agent
|
||||
// invariant the guard exists for.
|
||||
var matches = new[] { Proc(100, 0), Proc(300, 0) };
|
||||
|
||||
Assert.Equal(300, AutopilotMonitor.Agent.V2.Program.FindSiblingAgentPid(matches, ownPid: 100));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FindSiblingAgentPid_never_reports_the_caller_itself()
|
||||
{
|
||||
var matches = new[] { Proc(100, 0) };
|
||||
|
||||
Assert.Equal(0, AutopilotMonitor.Agent.V2.Program.FindSiblingAgentPid(matches, ownPid: 100));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FindSiblingAgentPid_skips_user_session_processes_listed_before_the_runtime()
|
||||
{
|
||||
// Install-mode launch verification: a user-session process enumerated first must not
|
||||
// be reported as the launched runtime.
|
||||
var matches = new[] { Proc(200, 1), Proc(201, 2), Proc(100, 0), Proc(300, 0) };
|
||||
|
||||
Assert.Equal(300, AutopilotMonitor.Agent.V2.Program.FindSiblingAgentPid(matches, ownPid: 100));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FindSiblingAgentPid_blocks_an_interactive_start_next_to_the_session_zero_runtime()
|
||||
{
|
||||
// --console run from an admin prompt (session 1) while the Scheduled Task runtime is up.
|
||||
var matches = new[] { Proc(100, 1), Proc(300, 0) };
|
||||
|
||||
Assert.Equal(300, AutopilotMonitor.Agent.V2.Program.FindSiblingAgentPid(matches, ownPid: 100));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FindSiblingAgentPid_returns_zero_for_no_matches()
|
||||
{
|
||||
Assert.Equal(0, AutopilotMonitor.Agent.V2.Program.FindSiblingAgentPid(
|
||||
new AutopilotMonitor.Agent.V2.Program.AgentProcessFact[0], ownPid: 100));
|
||||
Assert.Equal(0, AutopilotMonitor.Agent.V2.Program.FindSiblingAgentPid(null, ownPid: 100));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SnapshotProcessesByName_reads_pid_and_session_of_the_running_test_host()
|
||||
{
|
||||
using var self = System.Diagnostics.Process.GetCurrentProcess();
|
||||
|
||||
var facts = AutopilotMonitor.Agent.V2.Program.SnapshotProcessesByName(self.ProcessName);
|
||||
|
||||
var own = Assert.Single(facts, f => f.Pid == self.Id);
|
||||
Assert.Equal(self.SessionId, own.SessionId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SnapshotProcessesByName_reads_the_session_of_system_owned_processes_without_access_rights()
|
||||
{
|
||||
// Positive control for the fail-open direction: the session id comes from the process
|
||||
// snapshot, not from an opened handle, so even an unprivileged caller reads it for
|
||||
// SYSTEM-owned service hosts. A skipped process would let a real sibling go unseen.
|
||||
var expected = System.Diagnostics.Process.GetProcessesByName("svchost");
|
||||
foreach (var p in expected) p.Dispose();
|
||||
|
||||
var facts = AutopilotMonitor.Agent.V2.Program.SnapshotProcessesByName("svchost");
|
||||
|
||||
Assert.NotEmpty(facts);
|
||||
Assert.All(facts, f => Assert.True(f.Pid > 0));
|
||||
Assert.Contains(facts, f => f.SessionId == 0);
|
||||
// Service hosts come and go; a wholesale access failure would drop (nearly) all of them.
|
||||
Assert.True(facts.Count >= expected.Length / 2,
|
||||
$"Snapshot returned {facts.Count} of ~{expected.Length} svchost processes.");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SnapshotProcessesByName_returns_empty_for_an_unknown_name()
|
||||
{
|
||||
Assert.Empty(AutopilotMonitor.Agent.V2.Program.SnapshotProcessesByName("no-such-process-" + Guid.NewGuid().ToString("N")));
|
||||
}
|
||||
|
||||
// ================================================================= Crash log writer
|
||||
|
||||
[Fact]
|
||||
|
||||
+45
-12
@@ -336,7 +336,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
private int _unanchoredPatterns;
|
||||
private int _filesTailed;
|
||||
private long _backlogBytes;
|
||||
// Poll-loop timing (restart-safe maxima on a monotonic clock): the longest pass and the
|
||||
// longest pause between passes — see ImeTrackerHealth.PassMaxMs / PassGapMaxMs.
|
||||
private long _passMaxMs;
|
||||
private long _passGapMaxMs;
|
||||
private long _lastPassEndMs = -1;
|
||||
private string _imeAgentVersionSeen;
|
||||
|
||||
/// <summary>Monotonic milliseconds for the poll-loop timing; tests substitute a counter.</summary>
|
||||
internal Func<long> MonotonicMillisProvider { get; set; } = () => Stopwatch.GetTimestamp() * 1000 / Stopwatch.Frequency;
|
||||
private bool _trackerDegradedFired;
|
||||
private readonly Dictionary<string, int> _patternHits = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
@@ -370,6 +378,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
OverwriteBytesReprocessed = _overwriteBytesReprocessed,
|
||||
VerifyPasses = _verifyPasses,
|
||||
VerifiedBytes = _verifiedBytes,
|
||||
PassMaxMs = _passMaxMs,
|
||||
PassGapMaxMs = _passGapMaxMs,
|
||||
PatternHits = new Dictionary<string, int>(_patternHits, StringComparer.OrdinalIgnoreCase),
|
||||
};
|
||||
}
|
||||
@@ -954,18 +964,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
{
|
||||
try
|
||||
{
|
||||
await CheckLogFilesAsync(token);
|
||||
|
||||
// Platform-script completions that wait for the other file: an IME result
|
||||
// held for its executor end block, and the exit-code fallback for a
|
||||
// result IME never logged within the grace period. Runs on this same loop
|
||||
// thread (no locking) so it observes the buffer right after parsing.
|
||||
FlushPendingPlatformScriptResults(UtcNowProvider());
|
||||
|
||||
// Token-failure grace window: fires only when the failure stayed
|
||||
// unresolved (no IME-TOKEN-SUCCESS) for the whole window. Runs after
|
||||
// the drain so a success later in the same batch clears first.
|
||||
CheckPendingTokenFailure(DateTime.UtcNow);
|
||||
await RunPollPassAsync(token);
|
||||
}
|
||||
catch (OperationCanceledException) { break; }
|
||||
catch (Exception ex)
|
||||
@@ -1006,6 +1005,36 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
}, token);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One poll pass, timed: the file reads, then the platform-script completions that wait
|
||||
/// for the other file (an IME result held for its executor end block, the exit-code
|
||||
/// fallback for a result IME never logged), then the token-failure grace window — all on
|
||||
/// this one loop thread, no locking, so the flushes observe the buffers right after
|
||||
/// parsing. The longest pass and the longest pause since the previous pass ended are the
|
||||
/// session's <see cref="ImeTrackerHealth.PassMaxMs"/> / <see cref="ImeTrackerHealth.PassGapMaxMs"/>.
|
||||
/// </summary>
|
||||
internal async Task RunPollPassAsync(CancellationToken token)
|
||||
{
|
||||
var startMs = MonotonicMillisProvider();
|
||||
if (_lastPassEndMs >= 0 && startMs - _lastPassEndMs > _passGapMaxMs)
|
||||
{
|
||||
_passGapMaxMs = startMs - _lastPassEndMs;
|
||||
_stateDirty = true;
|
||||
}
|
||||
|
||||
await CheckLogFilesAsync(token);
|
||||
FlushPendingPlatformScriptResults(UtcNowProvider());
|
||||
CheckPendingTokenFailure(UtcNowProvider());
|
||||
|
||||
var endMs = MonotonicMillisProvider();
|
||||
if (endMs - startMs > _passMaxMs)
|
||||
{
|
||||
_passMaxMs = endMs - startMs;
|
||||
_stateDirty = true;
|
||||
}
|
||||
_lastPassEndMs = endMs;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stops the background polling task
|
||||
/// </summary>
|
||||
@@ -1155,6 +1184,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
_overwriteBytesReprocessed = state.Health.OverwriteBytesReprocessed;
|
||||
_verifyPasses = state.Health.VerifyPasses;
|
||||
_verifiedBytes = state.Health.VerifiedBytes;
|
||||
_passMaxMs = state.Health.PassMaxMs;
|
||||
_passGapMaxMs = state.Health.PassGapMaxMs;
|
||||
}
|
||||
if (state.PatternHits != null)
|
||||
{
|
||||
@@ -1228,6 +1259,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
OverwriteBytesReprocessed = _overwriteBytesReprocessed,
|
||||
VerifyPasses = _verifyPasses,
|
||||
VerifiedBytes = _verifiedBytes,
|
||||
PassMaxMs = _passMaxMs,
|
||||
PassGapMaxMs = _passGapMaxMs,
|
||||
};
|
||||
state.PatternHits = new Dictionary<string, int>(_patternHits, StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
+18
@@ -63,6 +63,24 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
/// <summary>Bytes re-read for verification, guard-zone checks included — the cost of the overwrite detection.</summary>
|
||||
public long VerifiedBytes { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Longest single poll pass this session (one <c>CheckLogFilesAsync</c> plus the
|
||||
/// flushes), milliseconds on a monotonic clock. A pass spends its time in the file
|
||||
/// reads, the ledger verifications and the regex matching; the fleet baseline is well
|
||||
/// under 100 ms.
|
||||
/// </summary>
|
||||
public long PassMaxMs { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Longest pause between the end of one poll pass and the start of the next this
|
||||
/// session, milliseconds. The loop sleeps 100 ms between passes and saves its state now
|
||||
/// and then; anything far above that is the loop not being scheduled — session c3ecb568
|
||||
/// lost 26 s this way at 100 % VM CPU while the rest of the agent kept running. Late
|
||||
/// reading, not lost data: every event keeps its log line's own timestamp, and a
|
||||
/// result read before its executor end block waits for it.
|
||||
/// </summary>
|
||||
public long PassGapMaxMs { get; set; }
|
||||
|
||||
/// <summary>Match count per enabled pattern ID — every enabled pattern is present, zeros included.</summary>
|
||||
public IReadOnlyDictionary<string, int> PatternHits { get; set; }
|
||||
|
||||
|
||||
+3
@@ -177,6 +177,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
public long OverwriteBytesReprocessed { get; set; }
|
||||
public long VerifyPasses { get; set; }
|
||||
public long VerifiedBytes { get; set; }
|
||||
// Poll-loop timing maxima (null-defaulting on older state files).
|
||||
public long PassMaxMs { get; set; }
|
||||
public long PassGapMaxMs { get; set; }
|
||||
}
|
||||
|
||||
public class PackageStateData
|
||||
|
||||
+70
-28
@@ -99,7 +99,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
|
||||
/// <summary>
|
||||
/// Packaging problems that are not cap skips, per kind: <c>path-guard</c> (configured path
|
||||
/// refused), <c>no-user-session</c> (user-profile token without a signed-in user),
|
||||
/// or hard-blocked event log refused),<c>no-user-session</c> (user-profile token without a signed-in user),
|
||||
/// <c>folder-rejected</c>, <c>enumerate</c>, <c>open</c>, <c>copy</c>.
|
||||
/// </summary>
|
||||
public IReadOnlyDictionary<string, int> ProblemsByKind { get; set; } = Empty;
|
||||
@@ -145,6 +145,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
// exact point production is exposed. Null in production.
|
||||
internal Action<string> BeforeSourceFileOpen { get; set; }
|
||||
|
||||
// Test seam: which channel writes to an .evtx file. Production asks the registered
|
||||
// channels; a test cannot register one.
|
||||
internal Func<string, string> EventLogChannelResolver { get; set; } = ResolveChannelForEvtxFile;
|
||||
|
||||
// Tracks per-build inclusion totals + skip reasons. Threaded through every
|
||||
// AddLogFiles call so caps are global across all sections, not per section.
|
||||
private sealed class BudgetTracker
|
||||
@@ -462,17 +466,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
// 3. Configured additional log paths (global + tenant, validated by guards)
|
||||
foreach (var entry in _configuration.DiagnosticsLogPaths ?? new System.Collections.Generic.List<Shared.Models.DiagnosticsLogPath>())
|
||||
{
|
||||
// Resolve %LOGGED_ON_USER_PROFILE% token and get profile path for guard exception
|
||||
// The profile path only lifts the C:\Users block for entries that use the token.
|
||||
var userProfilePath = UserProfileResolver.ContainsUserProfileToken(entry.Path)
|
||||
? UserProfileResolver.GetLoggedOnUserProfilePath() : null;
|
||||
|
||||
if (!DiagnosticsPathGuards.IsDiagnosticsPathAllowed(entry.Path, _configuration.UnrestrictedMode, userProfilePath))
|
||||
{
|
||||
_logger.Warning($"Diagnostics path blocked by guard: {entry.Path}");
|
||||
ManifestLine($"BLOCKED (path guard): {entry.Path}");
|
||||
tracker.RecordProblem("path-guard");
|
||||
continue;
|
||||
}
|
||||
// Expand BEFORE the guard: the guard must judge the path that is read,
|
||||
// and an unresolved token is not a path it can judge.
|
||||
var expandedPath = UserProfileResolver.ExpandCustomTokens(entry.Path);
|
||||
if (expandedPath == null)
|
||||
{
|
||||
@@ -481,13 +480,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
tracker.RecordProblem("no-user-session");
|
||||
continue;
|
||||
}
|
||||
var folder = Path.GetDirectoryName(expandedPath);
|
||||
var pattern = Path.GetFileName(expandedPath);
|
||||
if (string.IsNullOrEmpty(folder)) continue;
|
||||
if (string.IsNullOrEmpty(pattern) || !pattern.Contains(".")) pattern = "*";
|
||||
var zipFolder = $"AdditionalLogs/{Path.GetFileName(folder)}";
|
||||
ManifestLine($"CONFIGURED PATH: '{entry.Path}' -> folder='{folder}' pattern='{pattern}'");
|
||||
AddLogFiles(archive, folder, zipFolder, pattern, tracker, entry.IncludeSubfolders);
|
||||
|
||||
// The guard validates the folder that gets enumerated — never a path
|
||||
// derived from the validated one afterwards.
|
||||
if (!DiagnosticsPathGuards.TryResolveCollectionTarget(
|
||||
expandedPath, entry.IncludeSubfolders, _configuration.UnrestrictedMode, userProfilePath, out var target))
|
||||
{
|
||||
_logger.Warning($"Diagnostics path blocked by guard: {entry.Path}");
|
||||
ManifestLine($"BLOCKED (path guard): {entry.Path}");
|
||||
tracker.RecordProblem("path-guard");
|
||||
continue;
|
||||
}
|
||||
|
||||
var unrestrictedMode = _configuration.UnrestrictedMode;
|
||||
var zipFolder = $"AdditionalLogs/{Path.GetFileName(target.Folder)}";
|
||||
ManifestLine($"CONFIGURED PATH: '{entry.Path}' -> folder='{target.Folder}' pattern='{target.Pattern}' recursive={target.Recurse}");
|
||||
AddLogFiles(archive, target.Folder, zipFolder, target.Pattern, tracker, target.Recurse,
|
||||
dir => DiagnosticsPathGuards.IsDirectoryEnterable(dir, unrestrictedMode, userProfilePath));
|
||||
}
|
||||
|
||||
// Packaging manifest — always written, even when empty of problems, so its
|
||||
@@ -617,14 +626,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
// Recursive enumeration that skips reparse-point directories. Materializes the
|
||||
// result into a list so the caller can iterate without holding a directory handle.
|
||||
// Errors during enumeration are swallowed (logged via _logger by the caller).
|
||||
private static List<string> EnumerateFilesNoReparseDirs(string folder, string pattern, bool recurse)
|
||||
// mayEnterDirectory (configured paths only) is asked for every subdirectory before the
|
||||
// walk descends: a hard-blocked subtree can sit below a folder that passed the guard.
|
||||
internal static List<string> EnumerateFilesNoReparseDirs(string folder, string pattern, bool recurse,
|
||||
Func<string, bool> mayEnterDirectory, List<string> refusedDirectories)
|
||||
{
|
||||
var result = new List<string>();
|
||||
CollectFilesNoReparseDirs(folder, pattern, recurse, result);
|
||||
CollectFilesNoReparseDirs(folder, pattern, recurse, mayEnterDirectory, refusedDirectories, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
private static void CollectFilesNoReparseDirs(string folder, string pattern, bool recurse, List<string> result)
|
||||
private static void CollectFilesNoReparseDirs(string folder, string pattern, bool recurse,
|
||||
Func<string, bool> mayEnterDirectory, List<string> refusedDirectories, List<string> result)
|
||||
{
|
||||
string[] files;
|
||||
try { files = Directory.GetFiles(folder, pattern, SearchOption.TopDirectoryOnly); }
|
||||
@@ -643,12 +656,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
try { attrs = File.GetAttributes(sub); }
|
||||
catch { continue; }
|
||||
if (IsReparsePoint(attrs)) continue;
|
||||
CollectFilesNoReparseDirs(sub, pattern, recurse: true, result);
|
||||
if (mayEnterDirectory != null && !mayEnterDirectory(sub))
|
||||
{
|
||||
refusedDirectories.Add(sub);
|
||||
continue;
|
||||
}
|
||||
CollectFilesNoReparseDirs(sub, pattern, recurse: true, mayEnterDirectory, refusedDirectories, result);
|
||||
}
|
||||
}
|
||||
|
||||
private void AddLogFiles(ZipArchive archive, string sourceFolder, string zipFolder, string searchPattern,
|
||||
BudgetTracker tracker, bool includeSubfolders = false)
|
||||
BudgetTracker tracker, bool includeSubfolders = false, Func<string, bool> mayEnterDirectory = null)
|
||||
{
|
||||
// The folder stays pinned (open handle) for this whole section. The path guards and
|
||||
// the enumeration below only ever judge path strings; every byte that reaches the
|
||||
@@ -673,9 +691,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
}
|
||||
|
||||
List<string> files;
|
||||
var refusedDirectories = new List<string>();
|
||||
try
|
||||
{
|
||||
files = EnumerateFilesNoReparseDirs(pinned.LexicalPath, searchPattern, includeSubfolders);
|
||||
files = EnumerateFilesNoReparseDirs(pinned.LexicalPath, searchPattern, includeSubfolders,
|
||||
mayEnterDirectory, refusedDirectories);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -685,6 +705,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
return;
|
||||
}
|
||||
|
||||
// Expected outcome, not a packaging problem: the subtree is off limits by design.
|
||||
foreach (var refused in refusedDirectories)
|
||||
{
|
||||
_logger.Info($"Diagnostics subfolder skipped by guard: {refused}");
|
||||
ManifestLine($"SKIPPED DIR (path guard): {refused}");
|
||||
}
|
||||
|
||||
if (files.Count == 0)
|
||||
ManifestLine($"NO MATCH: {sourceFolder} pattern '{searchPattern}'");
|
||||
|
||||
@@ -704,7 +731,22 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
// the tenant-configured BootstrapperAgent channel never made it into the ZIP.
|
||||
if (file.EndsWith(".evtx", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
tempExport = TryExportEventLogChannel(file);
|
||||
// Hard-blocked channels are refused for every source and in every mode,
|
||||
// by file name AND by the channel the export would read: the path guard
|
||||
// cannot judge what a wildcard or folder entry resolves to, and the %4
|
||||
// fallback of the resolver turns a "Security.evtx" in any allowed folder
|
||||
// into the live Security channel.
|
||||
var channel = EventLogChannelResolver(file);
|
||||
if (DiagnosticsPathGuards.IsHardBlockedEventLogFile(file) ||
|
||||
GatherRuleGuards.IsHardBlockedEventLogChannel(channel))
|
||||
{
|
||||
_logger.Warning($"Event log blocked by channel guard (channel '{channel}'): {file}");
|
||||
ManifestLine($"BLOCKED (event-log channel guard, channel '{channel}'): {file}");
|
||||
tracker.RecordProblem("path-guard");
|
||||
return;
|
||||
}
|
||||
|
||||
tempExport = TryExportEventLogChannel(file, channel);
|
||||
if (tempExport == null)
|
||||
{
|
||||
// Export failed (channel not resolvable / wevtutil error) — fall through
|
||||
@@ -850,16 +892,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
/// <summary>
|
||||
/// Exports the event-log channel behind a winevt .evtx file to a temp copy via
|
||||
/// <c>wevtutil epl</c> (active channels hold their file exclusively locked, so a raw
|
||||
/// copy fails). Channel resolution happens against the registered channels — see
|
||||
/// <see cref="ResolveChannelForEvtxFile"/>. Returns the temp file path (caller deletes)
|
||||
/// or null when the export failed.
|
||||
/// copy fails). The caller resolved <paramref name="channel"/> against the registered
|
||||
/// channels — see <see cref="ResolveChannelForEvtxFile"/> — and checked it against the
|
||||
/// hard-blocked channels. Returns the temp file path (caller deletes) or null when the
|
||||
/// export failed.
|
||||
/// </summary>
|
||||
private string TryExportEventLogChannel(string evtxPath)
|
||||
private string TryExportEventLogChannel(string evtxPath, string channel)
|
||||
{
|
||||
string tempPath = null;
|
||||
try
|
||||
{
|
||||
var channel = ResolveChannelForEvtxFile(evtxPath);
|
||||
tempPath = Path.Combine(Path.GetTempPath(), $"am-evtx-{Guid.NewGuid():N}.evtx");
|
||||
var psi = new System.Diagnostics.ProcessStartInfo
|
||||
{
|
||||
|
||||
+209
-45
@@ -2,6 +2,7 @@ using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Gather
|
||||
{
|
||||
@@ -124,51 +125,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Gather
|
||||
if (IsSymlinkOrReparsePoint(normalizedDir))
|
||||
return false;
|
||||
|
||||
// C:\Users block always applies (even in unrestricted mode)
|
||||
// Exception: paths under <userProfilePath>\AppData\Local or AppData\Roaming
|
||||
// are allowed when the %LOGGED_ON_USER_PROFILE% token was used.
|
||||
if (normalizedDir.StartsWith(BlockedUsersPrefix, StringComparison.OrdinalIgnoreCase) &&
|
||||
(normalizedDir.Length == BlockedUsersPrefix.Length ||
|
||||
normalizedDir[BlockedUsersPrefix.Length] == Path.DirectorySeparatorChar))
|
||||
{
|
||||
if (!GatherRuleGuards.IsUserProfileSubpathAllowed(normalizedDir, userProfilePath))
|
||||
return false;
|
||||
}
|
||||
|
||||
// Additional hard-blocked paths (even in unrestricted mode)
|
||||
foreach (var blocked in AdditionalHardBlockedPrefixes)
|
||||
{
|
||||
var normalizedBlocked = Path.GetFullPath(blocked);
|
||||
if (normalizedDir.StartsWith(normalizedBlocked, StringComparison.OrdinalIgnoreCase) &&
|
||||
(normalizedDir.Length == normalizedBlocked.Length ||
|
||||
normalizedDir[normalizedBlocked.Length] == Path.DirectorySeparatorChar))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// In unrestricted mode, everything except hard-blocked paths is allowed
|
||||
if (unrestrictedMode)
|
||||
return true;
|
||||
|
||||
foreach (var prefix in AllowedDiagnosticsPathPrefixes)
|
||||
{
|
||||
var normalizedPrefix = Path.GetFullPath(prefix);
|
||||
if (normalizedDir.StartsWith(normalizedPrefix, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
// Segment-bounded: next char must be '\' or end-of-string
|
||||
if (normalizedDir.Length == normalizedPrefix.Length ||
|
||||
normalizedDir[normalizedPrefix.Length] == Path.DirectorySeparatorChar)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Vendor folders under the signed-in user's profile — same allowlist the file
|
||||
// collectors use, since it answers the same question: which folders below a
|
||||
// profile carry enrollment-relevant logs rather than the user's own data.
|
||||
return GatherRuleGuards.IsUserProfilePathOnAllowlist(normalizedDir, userProfilePath);
|
||||
return IsNormalizedPathAllowed(normalizedDir, unrestrictedMode, userProfilePath);
|
||||
}
|
||||
catch
|
||||
{
|
||||
@@ -177,6 +134,213 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Gather
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// What the package actually reads for one configured entry: every file matching
|
||||
/// <see cref="Pattern"/> in <see cref="Folder"/>, optionally below it.
|
||||
/// </summary>
|
||||
public sealed class CollectionTarget
|
||||
{
|
||||
public CollectionTarget(string folder, string pattern, bool recurse)
|
||||
{
|
||||
Folder = folder;
|
||||
Pattern = pattern;
|
||||
Recurse = recurse;
|
||||
}
|
||||
|
||||
/// <summary>Full path without a trailing separator (drive roots keep theirs).</summary>
|
||||
public string Folder { get; }
|
||||
public string Pattern { get; }
|
||||
public bool Recurse { get; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resolves a configured entry to the folder and pattern that get enumerated, and
|
||||
/// validates exactly that. Returns false when the guard refuses the entry.
|
||||
///
|
||||
/// <paramref name="expandedPath"/> must already have the %LOGGED_ON_USER_PROFILE% token
|
||||
/// resolved: validating the raw token would judge a different path than the one read.
|
||||
///
|
||||
/// A wildcard in the last segment selects files in its parent folder. A trailing
|
||||
/// separator or an existing directory selects every file in that folder. Anything else
|
||||
/// names one file — validated as that file and never recursed, because its parent
|
||||
/// folder has not passed the guard. The filesystem decides folder-vs-file, never the
|
||||
/// spelling of the last segment.
|
||||
/// </summary>
|
||||
public static bool TryResolveCollectionTarget(
|
||||
string expandedPath,
|
||||
bool includeSubfolders,
|
||||
bool unrestrictedMode,
|
||||
string userProfilePath,
|
||||
out CollectionTarget target)
|
||||
=> TryResolveCollectionTarget(
|
||||
expandedPath, includeSubfolders, unrestrictedMode, userProfilePath, Directory.Exists, out target);
|
||||
|
||||
internal static bool TryResolveCollectionTarget(
|
||||
string expandedPath,
|
||||
bool includeSubfolders,
|
||||
bool unrestrictedMode,
|
||||
string userProfilePath,
|
||||
Func<string, bool> directoryExists,
|
||||
out CollectionTarget target)
|
||||
{
|
||||
target = null;
|
||||
if (string.IsNullOrWhiteSpace(expandedPath))
|
||||
return false;
|
||||
|
||||
try
|
||||
{
|
||||
var expanded = Environment.ExpandEnvironmentVariables(expandedPath.Trim());
|
||||
var lastSegment = Path.GetFileName(expanded);
|
||||
|
||||
if (lastSegment.Contains('*') || lastSegment.Contains('?'))
|
||||
{
|
||||
var parent = Path.GetDirectoryName(expanded);
|
||||
if (string.IsNullOrEmpty(parent))
|
||||
return false;
|
||||
var folder = TrimTrailingSeparators(Path.GetFullPath(parent));
|
||||
if (!IsDiagnosticsPathAllowed(folder, unrestrictedMode, userProfilePath))
|
||||
return false;
|
||||
target = new CollectionTarget(folder, lastSegment, includeSubfolders);
|
||||
return true;
|
||||
}
|
||||
|
||||
var full = TrimTrailingSeparators(Path.GetFullPath(expanded));
|
||||
if (!IsDiagnosticsPathAllowed(full, unrestrictedMode, userProfilePath))
|
||||
return false;
|
||||
|
||||
if (lastSegment.Length == 0 || directoryExists(full))
|
||||
{
|
||||
target = new CollectionTarget(full, "*", includeSubfolders);
|
||||
return true;
|
||||
}
|
||||
|
||||
var fileFolder = Path.GetDirectoryName(full);
|
||||
if (string.IsNullOrEmpty(fileFolder))
|
||||
return false;
|
||||
target = new CollectionTarget(fileFolder, Path.GetFileName(full), recurse: false);
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
target = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Guard for every subdirectory a recursive collection is about to enter. The start
|
||||
/// folder passed <see cref="IsDiagnosticsPathAllowed(string, bool, string)"/>, but a
|
||||
/// hard-blocked subtree can sit below an allowed folder (unrestricted mode, drive
|
||||
/// root). Lexical only: reparse points are skipped by the enumeration and refused
|
||||
/// again on the handle that is read.
|
||||
/// </summary>
|
||||
public static bool IsDirectoryEnterable(string directory, bool unrestrictedMode, string userProfilePath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(directory))
|
||||
return false;
|
||||
|
||||
try
|
||||
{
|
||||
return IsNormalizedPathAllowed(Path.GetFullPath(directory), unrestrictedMode, userProfilePath);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static string TrimTrailingSeparators(string fullPath)
|
||||
{
|
||||
var root = Path.GetPathRoot(fullPath);
|
||||
var trimmed = fullPath.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
|
||||
return root != null && trimmed.Length < root.Length ? root : trimmed;
|
||||
}
|
||||
|
||||
// Hard blocks, then (restricted mode) the allowlists — on an already normalized path.
|
||||
private static bool IsNormalizedPathAllowed(string normalizedDir, bool unrestrictedMode, string userProfilePath)
|
||||
{
|
||||
// C:\Users block always applies (even in unrestricted mode)
|
||||
// Exception: paths under <userProfilePath>\AppData\Local or AppData\Roaming
|
||||
// are allowed when the %LOGGED_ON_USER_PROFILE% token was used.
|
||||
if (normalizedDir.StartsWith(BlockedUsersPrefix, StringComparison.OrdinalIgnoreCase) &&
|
||||
(normalizedDir.Length == BlockedUsersPrefix.Length ||
|
||||
normalizedDir[BlockedUsersPrefix.Length] == Path.DirectorySeparatorChar))
|
||||
{
|
||||
if (!GatherRuleGuards.IsUserProfileSubpathAllowed(normalizedDir, userProfilePath))
|
||||
return false;
|
||||
}
|
||||
|
||||
// Additional hard-blocked paths (even in unrestricted mode)
|
||||
foreach (var blocked in AdditionalHardBlockedPrefixes)
|
||||
{
|
||||
var normalizedBlocked = Path.GetFullPath(blocked);
|
||||
if (normalizedDir.StartsWith(normalizedBlocked, StringComparison.OrdinalIgnoreCase) &&
|
||||
(normalizedDir.Length == normalizedBlocked.Length ||
|
||||
normalizedDir[normalizedBlocked.Length] == Path.DirectorySeparatorChar))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Hard-blocked event logs (even in unrestricted mode). Only a path that names the
|
||||
// file is caught here; a wildcard or folder entry is judged per file by
|
||||
// DiagnosticsPackageService, which is the enforcement point.
|
||||
if (IsHardBlockedEventLogFile(normalizedDir))
|
||||
return false;
|
||||
|
||||
// In unrestricted mode, everything except hard-blocked paths is allowed
|
||||
if (unrestrictedMode)
|
||||
return true;
|
||||
|
||||
foreach (var prefix in AllowedDiagnosticsPathPrefixes)
|
||||
{
|
||||
var normalizedPrefix = Path.GetFullPath(prefix);
|
||||
if (normalizedDir.StartsWith(normalizedPrefix, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
// Segment-bounded: next char must be '\' or end-of-string
|
||||
if (normalizedDir.Length == normalizedPrefix.Length ||
|
||||
normalizedDir[normalizedPrefix.Length] == Path.DirectorySeparatorChar)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Vendor folders under the signed-in user's profile — same allowlist the file
|
||||
// collectors use, since it answers the same question: which folders below a
|
||||
// profile carry enrollment-relevant logs rather than the user's own data.
|
||||
return GatherRuleGuards.IsUserProfilePathOnAllowlist(normalizedDir, userProfilePath);
|
||||
}
|
||||
|
||||
// Windows archives a full channel as "Archive-<file name>-yyyy-MM-dd-HH-mm-ss-fff.evtx".
|
||||
private static readonly Regex ArchivedEventLogRegex = new Regex(
|
||||
@"^Archive-(.+)-\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}-\d{3}$",
|
||||
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
|
||||
|
||||
/// <summary>
|
||||
/// True when the .evtx file name belongs to one of
|
||||
/// <see cref="GatherRuleGuards.HardBlockedEventLogChannels"/> — live file
|
||||
/// ("Microsoft-Windows-PowerShell%4Operational.evtx", where %4 encodes '/') or
|
||||
/// archived copy. Judges the NAME only; a channel may write to a file that does not
|
||||
/// follow the convention, so the packager also checks the channel it resolved.
|
||||
/// </summary>
|
||||
internal static bool IsHardBlockedEventLogFile(string path)
|
||||
{
|
||||
if (string.IsNullOrEmpty(path))
|
||||
return false;
|
||||
|
||||
var name = Path.GetFileName(path);
|
||||
if (!name.EndsWith(".evtx", StringComparison.OrdinalIgnoreCase))
|
||||
return false;
|
||||
|
||||
var stem = name.Substring(0, name.Length - ".evtx".Length);
|
||||
var archived = ArchivedEventLogRegex.Match(stem);
|
||||
if (archived.Success)
|
||||
stem = archived.Groups[1].Value;
|
||||
|
||||
return GatherRuleGuards.IsHardBlockedEventLogChannel(stem.Replace("%4", "/"));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks whether any component of the path (directories or file) is a symlink,
|
||||
/// junction, or other reparse point. Returns true if a reparse point is detected.
|
||||
|
||||
+16
-5
@@ -451,11 +451,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Gather
|
||||
var trimmed = channel.Trim();
|
||||
|
||||
// Hard blocks apply even in unrestricted mode
|
||||
foreach (var blocked in HardBlockedEventLogChannels)
|
||||
{
|
||||
if (MatchesChannel(trimmed, blocked))
|
||||
return false;
|
||||
}
|
||||
if (IsHardBlockedEventLogChannel(trimmed))
|
||||
return false;
|
||||
|
||||
if (unrestrictedMode)
|
||||
return true;
|
||||
@@ -463,6 +460,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Gather
|
||||
return AllowedEventLogChannels.Any(allowed => MatchesChannel(trimmed, allowed));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True when the channel is one of <see cref="HardBlockedEventLogChannels"/>. Every
|
||||
/// path that can read an event log asks here — the event-log collector and the
|
||||
/// diagnostics package alike — so the block cannot hold in one and leak through another.
|
||||
/// </summary>
|
||||
internal static bool IsHardBlockedEventLogChannel(string channel)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(channel))
|
||||
return false;
|
||||
|
||||
var trimmed = channel.Trim();
|
||||
return HardBlockedEventLogChannels.Any(blocked => MatchesChannel(trimmed, blocked));
|
||||
}
|
||||
|
||||
private static bool MatchesChannel(string channel, string prefix)
|
||||
=> channel.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) &&
|
||||
(channel.Length == prefix.Length || channel[prefix.Length] == '/');
|
||||
|
||||
+2
@@ -213,6 +213,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
data["ime_overwrite_bytes_reprocessed"] = health.OverwriteBytesReprocessed;
|
||||
data["ime_verify_passes"] = health.VerifyPasses;
|
||||
data["ime_verified_bytes"] = health.VerifiedBytes;
|
||||
data["ime_pass_max_ms"] = health.PassMaxMs;
|
||||
data["ime_pass_gap_max_ms"] = health.PassGapMaxMs;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
|
||||
@@ -588,6 +588,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
["overwriteBytesReprocessed"] = health.OverwriteBytesReprocessed,
|
||||
["verifyPasses"] = health.VerifyPasses,
|
||||
["verifiedBytes"] = health.VerifiedBytes,
|
||||
["passMaxMs"] = health.PassMaxMs,
|
||||
["passGapMaxMs"] = health.PassGapMaxMs,
|
||||
["agentVersion"] = _agentVersion ?? string.Empty,
|
||||
};
|
||||
if (!string.IsNullOrEmpty(health.ImeAgentVersion))
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics.Eventing.Reader;
|
||||
using System.IO;
|
||||
@@ -41,17 +42,31 @@ namespace AutopilotMonitor.Agent.V2
|
||||
|
||||
// ----------------------------------------------------------------- Multi-instance guard
|
||||
|
||||
/// <summary>Pid and terminal-services session of one name-matched process.</summary>
|
||||
internal readonly struct AgentProcessFact
|
||||
{
|
||||
public AgentProcessFact(int pid, int sessionId)
|
||||
{
|
||||
Pid = pid;
|
||||
SessionId = sessionId;
|
||||
}
|
||||
|
||||
public int Pid { get; }
|
||||
public int SessionId { get; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <c>true</c> when another process with the same filename (module name) is alive.
|
||||
/// Race-safe by design: we count processes sharing our name; if 1, only we exist.
|
||||
/// <c>true</c> when another agent runtime is alive. A process name is not a protected
|
||||
/// namespace — any user session can run a binary named like the agent — so a name match
|
||||
/// only counts in session 0. Every runtime start (Scheduled Task, WMI launch, self-update
|
||||
/// restart) lands there, and a standard user cannot start a process in it.
|
||||
/// </summary>
|
||||
internal static bool IsAnotherAgentInstanceRunning()
|
||||
{
|
||||
try
|
||||
{
|
||||
var self = Process.GetCurrentProcess();
|
||||
var siblings = Process.GetProcessesByName(self.ProcessName);
|
||||
return siblings.Length > 1;
|
||||
using (var self = Process.GetCurrentProcess())
|
||||
return FindSiblingAgentPid(SnapshotProcessesByName(self.ProcessName), self.Id) != 0;
|
||||
}
|
||||
catch
|
||||
{
|
||||
@@ -61,6 +76,40 @@ namespace AutopilotMonitor.Agent.V2
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pid of the first name-matched process that is a real agent runtime — session 0 and not
|
||||
/// the caller itself — or <c>0</c> when there is none. Shared by the multi-instance guard
|
||||
/// and install mode's launch verification so both trust the same identity.
|
||||
/// </summary>
|
||||
internal static int FindSiblingAgentPid(IEnumerable<AgentProcessFact> nameMatches, int ownPid)
|
||||
{
|
||||
if (nameMatches == null) return 0;
|
||||
foreach (var candidate in nameMatches)
|
||||
{
|
||||
if (candidate.Pid == ownPid) continue;
|
||||
if (candidate.SessionId != 0) continue;
|
||||
return candidate.Pid;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads pid and session of every process with the given name. A process that vanished
|
||||
/// between enumeration and read is skipped: it is not a live sibling, and failing open
|
||||
/// means the worst case is two agents, never none.
|
||||
/// </summary>
|
||||
internal static List<AgentProcessFact> SnapshotProcessesByName(string processName)
|
||||
{
|
||||
var facts = new List<AgentProcessFact>();
|
||||
foreach (var process in Process.GetProcessesByName(processName))
|
||||
{
|
||||
try { facts.Add(new AgentProcessFact(process.Id, process.SessionId)); }
|
||||
catch { /* exited mid-read */ }
|
||||
finally { process.Dispose(); }
|
||||
}
|
||||
return facts;
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------- clean-exit marker
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -731,7 +731,8 @@ namespace AutopilotMonitor.Agent.V2
|
||||
/// 10s / 500ms contract as the bootstrap script's Get-Process probe. The install-mode
|
||||
/// process shares the runtime's exe name, so the own pid is excluded; a pre-existing
|
||||
/// foreign runtime cannot be running here because install mode bails earlier on the
|
||||
/// Deployed marker.
|
||||
/// Deployed marker. Same identity rule as the multi-instance guard: a same-named process
|
||||
/// in a user session is not the runtime and must not satisfy the verification.
|
||||
/// </summary>
|
||||
private static int WaitForDetachedRuntimePid(string exePath, AgentLogger logger)
|
||||
{
|
||||
@@ -742,16 +743,8 @@ namespace AutopilotMonitor.Agent.V2
|
||||
{
|
||||
try
|
||||
{
|
||||
var candidates = Process.GetProcessesByName(processName);
|
||||
try
|
||||
{
|
||||
var runtime = candidates.FirstOrDefault(p => p.Id != ownPid);
|
||||
if (runtime != null) return runtime.Id;
|
||||
}
|
||||
finally
|
||||
{
|
||||
foreach (var p in candidates) p.Dispose();
|
||||
}
|
||||
var runtimePid = FindSiblingAgentPid(SnapshotProcessesByName(processName), ownPid);
|
||||
if (runtimePid != 0) return runtimePid;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
@@ -447,41 +447,27 @@ public class AdminWireParityTests
|
||||
// ---- TriggerMaintenance --------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void TriggerMaintenanceResponse_matches_the_manual_trigger_shape()
|
||||
public void TriggerMaintenanceResponse_matches_the_queued_trigger_shape()
|
||||
{
|
||||
var result = new MaintenanceResult
|
||||
{
|
||||
Success = true,
|
||||
TriggeredBy = "admin@contoso.com",
|
||||
TriggeredAt = new DateTime(2026, 8, 30, 13, 10, 0, DateTimeKind.Utc),
|
||||
DurationMs = 1234,
|
||||
StalledSessionsChecked = true,
|
||||
MetricsAggregated = true,
|
||||
AggregatedDate = "2026-08-29",
|
||||
DataCleanupExecuted = false,
|
||||
PlatformStatsRecomputed = true,
|
||||
DevicesBlockedForExcessiveData = 0,
|
||||
ContactEmailsBackfilled = 2,
|
||||
};
|
||||
var userEmail = "admin@contoso.com";
|
||||
var triggeredAt = new DateTime(2026, 8, 30, 13, 10, 5, DateTimeKind.Utc);
|
||||
|
||||
AssertParity(
|
||||
new
|
||||
{
|
||||
success = true,
|
||||
message = "Maintenance tasks completed",
|
||||
result = result,
|
||||
message = "Maintenance run queued",
|
||||
triggeredBy = userEmail,
|
||||
triggeredAt
|
||||
triggeredAt,
|
||||
targetDate = "2026-08-29",
|
||||
aggregateOnly = true
|
||||
},
|
||||
new TriggerMaintenanceResponse
|
||||
{
|
||||
Success = true,
|
||||
Message = "Maintenance tasks completed",
|
||||
Result = result,
|
||||
Message = "Maintenance run queued",
|
||||
TriggeredBy = userEmail,
|
||||
TriggeredAt = triggeredAt
|
||||
TriggeredAt = triggeredAt,
|
||||
TargetDate = "2026-08-29",
|
||||
AggregateOnly = true
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -27,6 +27,13 @@ public class AppVersionRegressionRadarTests
|
||||
private const string App = "Contoso VPN";
|
||||
private static readonly DateTime Now = new(2026, 8, 13, 12, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
/// <summary>
|
||||
/// For the analytics tests only: AppsAnalyticsHelper windows off the REAL clock (DateTime.UtcNow − days),
|
||||
/// so their rows must be dated relative to it — a row dated off the fixed <see cref="Now"/> falls out of
|
||||
/// the 30-day window once the calendar moves on (it did on 2026-09-11).
|
||||
/// </summary>
|
||||
private static readonly DateTime Recent = DateTime.UtcNow.AddDays(-1);
|
||||
|
||||
private static int _sessionCounter;
|
||||
|
||||
private static AppInstallSummary Install(
|
||||
@@ -357,10 +364,10 @@ public class AppVersionRegressionRadarTests
|
||||
|
||||
var rows = new List<AppInstallSummary>
|
||||
{
|
||||
Install("1.0", 100), Install("1.0", 200), Install("1.0", 300), Install("1.0", 400),
|
||||
Install("1.0", 0), // unmeasured — excluded from stats
|
||||
Install("1.0", 50, terminalState: "Skipped"), // skip — excluded
|
||||
Install("1.0", 50, status: "Failed"), // failed — excluded from durations
|
||||
Install("1.0", 100, Recent), Install("1.0", 200, Recent), Install("1.0", 300, Recent), Install("1.0", 400, Recent),
|
||||
Install("1.0", 0, Recent), // unmeasured — excluded from stats
|
||||
Install("1.0", 50, Recent, terminalState: "Skipped"), // skip — excluded
|
||||
Install("1.0", 50, Recent, status: "Failed"), // failed — excluded from durations
|
||||
};
|
||||
|
||||
var root = TestWire.SerializeToElement(await AppsAnalyticsHelper.BuildAnalyticsResponseAsync(
|
||||
@@ -393,7 +400,7 @@ public class AppVersionRegressionRadarTests
|
||||
Assert.Equal(1, emptyRoot.GetProperty("versionRegressions").GetArrayLength());
|
||||
|
||||
var root = TestWire.SerializeToElement(await AppsAnalyticsHelper.BuildAnalyticsResponseAsync(
|
||||
new List<AppInstallSummary> { Install("1.0", 100) }, sessionRepo.Object, App, days: 30,
|
||||
new List<AppInstallSummary> { Install("1.0", 100, Recent) }, sessionRepo.Object, App, days: 30,
|
||||
new List<AppVersionRegressionAlert> { alert }));
|
||||
|
||||
var regression = root.GetProperty("versionRegressions")[0];
|
||||
|
||||
@@ -219,6 +219,8 @@ public class AuthFunctionSideEffectTests
|
||||
// OnboardedBy is the immutable copy of the first-login UPN that auto-promote on
|
||||
// preview approval reads — UpdatedBy may later be clobbered by background syncs.
|
||||
Assert.Equal(Upn, config.OnboardedBy);
|
||||
// The DPA version rides along in the same onboarding write (D-252).
|
||||
Assert.Equal(CurrentDpa, config.DpaVersion);
|
||||
_tenantConfigMock.Verify(x => x.SaveConfigurationAsync(config, It.IsAny<string?>(), It.IsAny<string?>()), Times.Once);
|
||||
_telegramMock.Verify(x => x.SendNewTenantSignupAsync(TenantId, Upn), Times.Once);
|
||||
_globalNotificationMock.Verify(x => x.CreateNotificationAsync(
|
||||
@@ -304,6 +306,23 @@ public class AuthFunctionSideEffectTests
|
||||
_tenantConfigMock.Verify(x => x.SaveConfigurationAsync(It.IsAny<TenantConfiguration>(), It.IsAny<string?>(), It.IsAny<string?>()), Times.Never);
|
||||
}
|
||||
|
||||
private const string CurrentDpa = AutopilotMonitor.Shared.Constants.CurrentDpaVersion;
|
||||
|
||||
[Fact]
|
||||
public async Task HandleNewTenantDomain_WhenDomainAlreadySet_NeverStampsDpaVersion()
|
||||
{
|
||||
// The DPA version is written once, with the onboarding (D-252): an existing tenant's
|
||||
// login writes nothing — no stamp, no save.
|
||||
var config = DefaultConfig();
|
||||
config.DomainName = "contoso.com";
|
||||
config.DpaVersion = null;
|
||||
|
||||
await _sut.HandleNewTenantDomainAsync(config, TenantId, Upn);
|
||||
|
||||
Assert.Null(config.DpaVersion);
|
||||
_tenantConfigMock.Verify(x => x.SaveConfigurationAsync(It.IsAny<TenantConfiguration>(), It.IsAny<string?>(), It.IsAny<string?>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HandleNewTenantDomain_PreservesExistingOnboardedBy()
|
||||
{
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using AutopilotMonitor.Functions.Functions.Config;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Shared.Models.Config;
|
||||
@@ -619,12 +620,26 @@ public class ConfigWireParityTests
|
||||
// keys omitted on the wire).
|
||||
TenantDailyRequestLimit = 3000,
|
||||
TenantMonthlyRequestLimit = 60000,
|
||||
// Per purchased delegation slot (the community tier leaves these unset too).
|
||||
SlotDailyRequestLimit = 300,
|
||||
SlotMonthlyRequestLimit = 6000,
|
||||
SlotTenantDailyRequestLimit = 900,
|
||||
SlotTenantMonthlyRequestLimit = 18000,
|
||||
},
|
||||
};
|
||||
|
||||
AssertParity(
|
||||
new { tiers },
|
||||
new PlanTierDefinitionsResponse { Tiers = tiers });
|
||||
new
|
||||
{
|
||||
tiers,
|
||||
// The built-in fallbacks per edition — what a blank definition field means.
|
||||
catalog = new[]
|
||||
{
|
||||
new { edition = "community", dailyRequestLimit = 100, monthlyRequestLimit = 3000, tenantDailyRequestLimit = 300, tenantMonthlyRequestLimit = 9000, includedDelegatedSlots = 0, slotDailyRequestLimit = 0, slotMonthlyRequestLimit = 0, slotTenantDailyRequestLimit = 0, slotTenantMonthlyRequestLimit = 0 },
|
||||
new { edition = "pro", dailyRequestLimit = 1000, monthlyRequestLimit = 20000, tenantDailyRequestLimit = 3000, tenantMonthlyRequestLimit = 60000, includedDelegatedSlots = 2, slotDailyRequestLimit = 300, slotMonthlyRequestLimit = 6000, slotTenantDailyRequestLimit = 900, slotTenantMonthlyRequestLimit = 18000 },
|
||||
},
|
||||
},
|
||||
new PlanTierDefinitionsResponse { Tiers = tiers, Catalog = PlanManagementFunction.BuildUsagePlanCatalog() });
|
||||
}
|
||||
|
||||
// ---- TestWebhookNotification (dual-purpose: success carries the delivery verdict) ----
|
||||
|
||||
@@ -66,16 +66,12 @@ public class DelegatedAdminServiceTests
|
||||
private static void ReturnsGroups(Mock<IAdminRepository> repo, params TenantGroupAssignment[] assignments) =>
|
||||
repo.Setup(r => r.GetGroupAssignmentsForUpnAsync(It.IsAny<string>())).ReturnsAsync(assignments.ToList());
|
||||
|
||||
/// <summary>Stubs the hot-path membership read (tenants from one partition read).</summary>
|
||||
private static void GroupTenants(Mock<IAdminRepository> repo, string groupId, params string[] tenantIds) =>
|
||||
GroupMembership(repo, groupId, chargeHomeTenantQuota: false, tenantIds);
|
||||
|
||||
/// <summary>Stubs the hot-path membership read (tenants + meta flags from one partition read).</summary>
|
||||
private static void GroupMembership(Mock<IAdminRepository> repo, string groupId, bool chargeHomeTenantQuota, params string[] tenantIds) =>
|
||||
repo.Setup(r => r.GetGroupMembershipAsync(groupId)).ReturnsAsync(new TenantGroupMembership
|
||||
{
|
||||
GroupId = groupId,
|
||||
TenantIds = tenantIds.ToList(),
|
||||
ChargeHomeTenantQuota = chargeHomeTenantQuota,
|
||||
});
|
||||
|
||||
private static DelegatedAdminEntry Row(
|
||||
@@ -390,96 +386,6 @@ public class DelegatedAdminServiceTests
|
||||
repo.Verify(r => r.GetGroupMembershipAsync(GroupId1), Times.Once);
|
||||
}
|
||||
|
||||
// --- ChargeHomeTenantQuota: MCP billing attribution rides in the scope (never authorization) ---
|
||||
|
||||
[Fact]
|
||||
public async Task GetScope_FlaggedGroup_MarksItsTenantsHomeCharged()
|
||||
{
|
||||
var (svc, repo) = Build();
|
||||
ReturnsGroups(repo, Assignment(GroupId1));
|
||||
GroupMembership(repo, GroupId1, chargeHomeTenantQuota: true, TenantA, TenantB);
|
||||
|
||||
var scope = await svc.GetScopeAsync(Id(Upn));
|
||||
|
||||
Assert.True(scope.Covers(TenantA));
|
||||
Assert.True(scope.ChargesHome(TenantA));
|
||||
Assert.True(scope.ChargesHome(TenantB));
|
||||
Assert.Equal(2, scope.HomeChargedTenantIds.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetScope_UnflaggedGroup_NothingIsHomeCharged()
|
||||
{
|
||||
var (svc, repo) = Build();
|
||||
ReturnsGroups(repo, Assignment(GroupId1));
|
||||
GroupTenants(repo, GroupId1, TenantA);
|
||||
|
||||
var scope = await svc.GetScopeAsync(Id(Upn));
|
||||
|
||||
Assert.True(scope.Covers(TenantA));
|
||||
Assert.False(scope.ChargesHome(TenantA));
|
||||
Assert.Empty(scope.HomeChargedTenantIds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetScope_DirectGrantAndFlaggedGroup_TheFlagWins()
|
||||
{
|
||||
// The operator's explicit group flag is the more privileged choice — a parallel direct grant on the
|
||||
// same tenant does not re-attribute the reads to the managed tenant.
|
||||
var (svc, repo) = Build();
|
||||
Returns(repo, Row(TenantA, Constants.DelegatedRoles.DelegatedAdmin));
|
||||
ReturnsGroups(repo, Assignment(GroupId1));
|
||||
GroupMembership(repo, GroupId1, chargeHomeTenantQuota: true, TenantA);
|
||||
|
||||
var scope = await svc.GetScopeAsync(Id(Upn));
|
||||
|
||||
Assert.Equal(Constants.DelegatedRoles.DelegatedAdmin, scope.RoleFor(TenantA)); // stronger role kept
|
||||
Assert.True(scope.ChargesHome(TenantA));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetScope_DisabledAssignmentToFlaggedGroup_ConfersNothing()
|
||||
{
|
||||
var (svc, repo) = Build();
|
||||
ReturnsGroups(repo, Assignment(GroupId1, enabled: false));
|
||||
GroupMembership(repo, GroupId1, chargeHomeTenantQuota: true, TenantA);
|
||||
|
||||
var scope = await svc.GetScopeAsync(Id(Upn));
|
||||
|
||||
Assert.False(scope.Covers(TenantA));
|
||||
Assert.False(scope.ChargesHome(TenantA));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SetChargeHomeTenantQuota_InvalidatesEveryAssignee()
|
||||
{
|
||||
var (svc, repo) = Build();
|
||||
ReturnsGroups(repo, Assignment(GroupId1));
|
||||
GroupMembership(repo, GroupId1, chargeHomeTenantQuota: false, TenantA);
|
||||
repo.Setup(r => r.SetTenantGroupChargeHomeTenantQuotaAsync(GroupId1, true)).ReturnsAsync(true);
|
||||
repo.Setup(r => r.GetGroupAssigneesAsync(GroupId1)).ReturnsAsync(new List<TenantGroupAssignment> { Assignment(GroupId1) });
|
||||
|
||||
await svc.GetScopeAsync(Id(Upn));
|
||||
var ok = await svc.SetChargeHomeTenantQuotaAsync(GroupId1, true);
|
||||
await svc.GetScopeAsync(Id(Upn));
|
||||
|
||||
Assert.True(ok);
|
||||
// The flag rides in the cached scope, so the flip forces a re-resolution for the assignee.
|
||||
repo.Verify(r => r.GetGroupMembershipAsync(GroupId1), Times.Exactly(2));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SetChargeHomeTenantQuota_UnknownGroup_IsFalse_AndInvalidatesNobody()
|
||||
{
|
||||
var (svc, repo) = Build();
|
||||
repo.Setup(r => r.SetTenantGroupChargeHomeTenantQuotaAsync(It.IsAny<string>(), It.IsAny<bool>())).ReturnsAsync(false);
|
||||
|
||||
var ok = await svc.SetChargeHomeTenantQuotaAsync("no-such-group", true);
|
||||
|
||||
Assert.False(ok);
|
||||
repo.Verify(r => r.GetGroupAssigneesAsync(It.IsAny<string>()), Times.Never);
|
||||
}
|
||||
|
||||
// --- Group mutations go through the service and invalidate cached scope (no stale auth) ---
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -45,7 +45,10 @@ public class DelegationSelfServiceTests
|
||||
public required Mock<ProConferralService> ProConferral { get; init; }
|
||||
}
|
||||
|
||||
private static Harness Build(TenantEdition homeEdition = TenantEdition.Pro, int? slotOverride = null)
|
||||
/// <summary>Pro by conferral only (managed by a Pro MSP): the shape that must never delegate onward.</summary>
|
||||
private static readonly EditionResolution ConferredPro = new(TenantEdition.Pro, EditionSource.Msp, OwnPro: false);
|
||||
|
||||
private static Harness Build(TenantEdition homeEdition = TenantEdition.Pro, int? slotOverride = null, EditionResolution? homeResolution = null)
|
||||
{
|
||||
var rows = new List<DelegationInvitation>();
|
||||
var groupTenants = new List<string>();
|
||||
@@ -104,7 +107,9 @@ public class DelegationSelfServiceTests
|
||||
|
||||
var cache = new MemoryCache(new MemoryCacheOptions());
|
||||
var time = new TestTimeProvider(Now);
|
||||
var entitlements = new StubTenantEntitlementService(t => t == Home ? homeEdition : TenantEdition.Community);
|
||||
var entitlements = homeResolution is EditionResolution resolution
|
||||
? new StubTenantEntitlementService(t => t == Home ? resolution : new EditionResolution(TenantEdition.Community, EditionSource.Community, OwnPro: false), null)
|
||||
: new StubTenantEntitlementService(t => t == Home ? homeEdition : TenantEdition.Community);
|
||||
var delegatedAdmins = new DelegatedAdminService(repo.Object, new StubAdminIdentityBindingService(bound: true), entitlements, cache, NullLogger<DelegatedAdminService>.Instance);
|
||||
var slots = new DelegatedSlotService(repo.Object, configRepo.Object, invitations.Object, cache, NullLogger<DelegatedSlotService>.Instance, time);
|
||||
var audit = new Mock<IMaintenanceRepository>();
|
||||
@@ -199,6 +204,18 @@ public class DelegationSelfServiceTests
|
||||
Assert.Empty(h.Rows);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CreateInvitation_ConferredProHome_403_NoTransitiveDelegation()
|
||||
{
|
||||
// The would-be manager is Pro only because a Pro MSP manages IT — conferred Pro carries no delegation right.
|
||||
var h = Build(homeResolution: ConferredPro);
|
||||
var r = await h.Svc.CreateInvitationAsync(Home, MspAdmin);
|
||||
Assert.False(r.Ok);
|
||||
Assert.Equal(403, r.Failure!.Status);
|
||||
Assert.Equal(Constants.DelegationCodes.DelegatedAdminNotAllowed, r.Failure.Code);
|
||||
Assert.Empty(h.Rows);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CreateInvitation_NoFreeSlot_409_WithViolation()
|
||||
{
|
||||
@@ -322,6 +339,17 @@ public class DelegationSelfServiceTests
|
||||
Assert.Equal(Constants.DelegationCodes.ManagerNotEntitled, r.Failure!.Code);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Accept_ManagerConferredProOnly_409()
|
||||
{
|
||||
// The inviting tenant lost its own Pro and is now Pro by conferral: its pending link is not redeemable.
|
||||
var h = Build(homeResolution: ConferredPro);
|
||||
h.Rows.Add(Pending());
|
||||
var r = await h.Svc.AcceptAsync(Token(), Customer, CustomerAdmin);
|
||||
Assert.Equal(Constants.DelegationCodes.ManagerNotEntitled, r.Failure!.Code);
|
||||
Assert.Equal(Constants.DelegationInvitationStatus.Pending, h.Rows[0].Status); // untouched
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Accept_LimitLoweredSinceInvitation_409_SlotViolation()
|
||||
{
|
||||
@@ -453,6 +481,16 @@ public class DelegationSelfServiceTests
|
||||
h.Repo.Verify(x => x.EnsureOwnedTenantGroupAsync(GroupId, It.IsAny<string>(), Home), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Assign_ConferredProHome_403()
|
||||
{
|
||||
var h = Build(homeResolution: ConferredPro);
|
||||
var r = await h.Svc.AssignAsync(Home, "Analyst@Partner.Example", MspAdmin);
|
||||
Assert.Equal(403, r.Failure!.Status);
|
||||
Assert.Equal(Constants.DelegationCodes.DelegatedAdminNotAllowed, r.Failure.Code);
|
||||
h.Repo.Verify(x => x.AssignGroupAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>(), It.IsAny<string>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Unassign_NotAssigned_404()
|
||||
{
|
||||
|
||||
@@ -106,12 +106,11 @@ public class DelegationWireParityTests
|
||||
// ---- Self-service delegation ------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void ManagedTenantListResponse_matches_the_managed_shape_and_omits_null_usage()
|
||||
public void ManagedTenantListResponse_matches_the_managed_shape_and_omits_null_domain_and_since()
|
||||
{
|
||||
var since = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc);
|
||||
var slots = new DelegatedSlotUsageResponse { HomeTenantId = Home, Limit = 2, CatalogLimit = 2, Used = 1, ManagedTenantIds = new[] { TenantA }, PendingInvitations = 0, Holds = Array.Empty<DelegatedSlotHold>() };
|
||||
string? domainB = null;
|
||||
ManagedTenantQuotaUsage? usageB = null;
|
||||
DateTime? sinceB = null;
|
||||
|
||||
AssertParity(
|
||||
@@ -121,9 +120,8 @@ public class DelegationWireParityTests
|
||||
slots,
|
||||
tenants = new[]
|
||||
{
|
||||
new { tenantId = TenantA, domain = (string?)"customer.example", source = "self-service", sinceUtc = (DateTime?)since, removable = true,
|
||||
usage = (ManagedTenantQuotaUsage?)new ManagedTenantQuotaUsage { TenantPlan = "community", TenantDailyLimit = 300, TenantMonthlyLimit = 9000, TenantDailyUsed = 12, TenantMonthlyUsed = 340 } },
|
||||
new { tenantId = TenantB, domain = domainB, source = "operator", sinceUtc = sinceB, removable = false, usage = usageB },
|
||||
new { tenantId = TenantA, domain = (string?)"customer.example", source = "self-service", sinceUtc = (DateTime?)since, removable = true },
|
||||
new { tenantId = TenantB, domain = domainB, source = "operator", sinceUtc = sinceB, removable = false },
|
||||
},
|
||||
},
|
||||
new ManagedTenantListResponse
|
||||
@@ -132,8 +130,7 @@ public class DelegationWireParityTests
|
||||
Slots = slots,
|
||||
Tenants = new List<ManagedTenantItem>
|
||||
{
|
||||
new() { TenantId = TenantA, Domain = "customer.example", Source = "self-service", SinceUtc = since, Removable = true,
|
||||
Usage = new ManagedTenantQuotaUsage { TenantPlan = "community", TenantDailyLimit = 300, TenantMonthlyLimit = 9000, TenantDailyUsed = 12, TenantMonthlyUsed = 340 } },
|
||||
new() { TenantId = TenantA, Domain = "customer.example", Source = "self-service", SinceUtc = since, Removable = true },
|
||||
new() { TenantId = TenantB, Source = "operator", Removable = false },
|
||||
},
|
||||
});
|
||||
|
||||
@@ -158,6 +158,11 @@ public class FeatureEntitlementCatalogTests
|
||||
Assert.Equal(pro.McpMonthlyRequestLimit, conferred.McpMonthlyRequestLimit);
|
||||
Assert.Equal(pro.McpTenantDailyRequestLimit, conferred.McpTenantDailyRequestLimit);
|
||||
Assert.Equal(pro.McpTenantMonthlyRequestLimit, conferred.McpTenantMonthlyRequestLimit);
|
||||
// …except the slot growth, which follows the delegation right.
|
||||
Assert.Equal(0, conferred.McpSlotDailyRequestLimit);
|
||||
Assert.Equal(0, conferred.McpSlotMonthlyRequestLimit);
|
||||
Assert.Equal(0, conferred.McpSlotTenantDailyRequestLimit);
|
||||
Assert.Equal(0, conferred.McpSlotTenantMonthlyRequestLimit);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -207,6 +212,10 @@ public class FeatureEntitlementCatalogTests
|
||||
Assert.Equal(3000, e.McpMonthlyRequestLimit);
|
||||
Assert.Equal(300, e.McpTenantDailyRequestLimit);
|
||||
Assert.Equal(9000, e.McpTenantMonthlyRequestLimit);
|
||||
Assert.Equal(0, e.McpSlotDailyRequestLimit);
|
||||
Assert.Equal(0, e.McpSlotMonthlyRequestLimit);
|
||||
Assert.Equal(0, e.McpSlotTenantDailyRequestLimit);
|
||||
Assert.Equal(0, e.McpSlotTenantMonthlyRequestLimit);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -224,6 +233,11 @@ public class FeatureEntitlementCatalogTests
|
||||
Assert.Equal(20000, e.McpMonthlyRequestLimit);
|
||||
Assert.Equal(3000, e.McpTenantDailyRequestLimit);
|
||||
Assert.Equal(60000, e.McpTenantMonthlyRequestLimit);
|
||||
// Per purchased delegation slot: 30 % of the Pro windows (an additional slot costs 30 % of Pro).
|
||||
Assert.Equal(300, e.McpSlotDailyRequestLimit);
|
||||
Assert.Equal(6000, e.McpSlotMonthlyRequestLimit);
|
||||
Assert.Equal(900, e.McpSlotTenantDailyRequestLimit);
|
||||
Assert.Equal(18000, e.McpSlotTenantMonthlyRequestLimit);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The IME pattern self-seed behind the newest-source-wins gate (<see cref="RuleCatalogSeedGate"/>):
|
||||
/// a GitHub reseed newer than the build owns the table (nothing written, nothing deleted), an
|
||||
/// older one hands it back to the binary (missing ids written, retired ids sunset), an empty
|
||||
/// table is always seeded, and the code reseed stamps <c>embedded</c>. The retired id stands in
|
||||
/// for PS-SCRIPT-EXITCODE, which the deployed binary re-inserted on 2026-09-11 after the reseed
|
||||
/// had deleted it.
|
||||
/// </summary>
|
||||
public class ImeLogPatternServiceSeedTests
|
||||
{
|
||||
private const string RetiredId = "IME-RETIRED-TEST";
|
||||
private static readonly DateTime Build = new(2026, 9, 10, 19, 19, 57, DateTimeKind.Utc);
|
||||
private static readonly BackendBuildInfo BuildInfo = new("1.5.1205", "e3b2113", Build);
|
||||
|
||||
private static RuleCatalogStamp GitHubStamp(DateTime at) => new()
|
||||
{
|
||||
Kind = RuleCatalogStamp.KindIme, Source = RuleCatalogStamp.SourceGitHub, StampedAt = at, Count = 82,
|
||||
};
|
||||
|
||||
/// <summary>Global partition = the embedded catalog minus its first id plus one retired built-in row.</summary>
|
||||
private static (List<ImeLogPattern> table, string missingId) DriftedTable()
|
||||
{
|
||||
var embedded = BuiltInImeLogPatterns.GetAll();
|
||||
var missing = embedded[0].PatternId;
|
||||
var table = embedded.Skip(1).Select(p => p.Clone()).ToList();
|
||||
table.Add(new ImeLogPattern
|
||||
{
|
||||
PatternId = RetiredId, Category = "always", Pattern = "^retired", Action = "scriptExitCode",
|
||||
Parameters = new Dictionary<string, string>(), Enabled = true, Description = "retired", IsBuiltIn = true,
|
||||
});
|
||||
return (table, missing);
|
||||
}
|
||||
|
||||
private sealed class Rig
|
||||
{
|
||||
public readonly Mock<IRuleRepository> Repo = new();
|
||||
public readonly List<string> Stored = new();
|
||||
public readonly List<string> Deleted = new();
|
||||
|
||||
public Rig(List<ImeLogPattern> global, RuleCatalogStamp? stamp)
|
||||
{
|
||||
Repo.Setup(r => r.GetImeLogPatternsAsync("global")).ReturnsAsync(() => global.Select(p => p.Clone()).ToList());
|
||||
Repo.Setup(r => r.GetImeLogPatternsAsync(It.Is<string>(k => k != "global"))).ReturnsAsync(new List<ImeLogPattern>());
|
||||
Repo.Setup(r => r.GetRuleCatalogStampAsync(RuleCatalogStamp.KindIme)).ReturnsAsync(stamp);
|
||||
Repo.Setup(r => r.StoreImeLogPatternAsync(It.IsAny<ImeLogPattern>(), "global"))
|
||||
.Callback<ImeLogPattern, string>((p, _) => Stored.Add(p.PatternId)).ReturnsAsync(true);
|
||||
Repo.Setup(r => r.DeleteImeLogPatternAsync("global", It.IsAny<string>()))
|
||||
.Callback<string, string>((_, id) => Deleted.Add(id)).ReturnsAsync(true);
|
||||
Repo.Setup(r => r.SetRuleCatalogStampAsync(It.IsAny<RuleCatalogStamp>())).ReturnsAsync(true);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GitHub_reseed_newer_than_the_build_owns_the_table()
|
||||
{
|
||||
var (table, missing) = DriftedTable();
|
||||
var rig = new Rig(table, GitHubStamp(Build.AddHours(6)));
|
||||
var service = new ImeLogPatternService(rig.Repo.Object, NullLogger<ImeLogPatternService>.Instance, BuildInfo);
|
||||
|
||||
var catalog = await service.GetGlobalCatalogAsync();
|
||||
|
||||
Assert.Empty(rig.Stored);
|
||||
Assert.Empty(rig.Deleted);
|
||||
Assert.Contains(catalog, p => p.PatternId == RetiredId); // the reseed's table, untouched
|
||||
Assert.DoesNotContain(catalog, p => p.PatternId == missing);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GitHub_reseed_older_than_the_build_hands_the_table_to_the_binary()
|
||||
{
|
||||
var (table, missing) = DriftedTable();
|
||||
var rig = new Rig(table, GitHubStamp(Build.AddHours(-1)));
|
||||
var service = new ImeLogPatternService(rig.Repo.Object, NullLogger<ImeLogPatternService>.Instance, BuildInfo);
|
||||
|
||||
await service.GetGlobalCatalogAsync();
|
||||
|
||||
Assert.Contains(missing, rig.Stored); // added from the embedded catalog
|
||||
Assert.Equal(new[] { RetiredId }, rig.Deleted); // sunset: no longer shipped
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Without_build_info_the_gate_is_open_and_the_stamp_is_never_read()
|
||||
{
|
||||
var (table, missing) = DriftedTable();
|
||||
var rig = new Rig(table, GitHubStamp(Build.AddHours(6)));
|
||||
var service = new ImeLogPatternService(rig.Repo.Object, NullLogger<ImeLogPatternService>.Instance);
|
||||
|
||||
await service.GetGlobalCatalogAsync();
|
||||
|
||||
Assert.Contains(missing, rig.Stored);
|
||||
rig.Repo.Verify(r => r.GetRuleCatalogStampAsync(It.IsAny<string>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Empty_table_is_seeded_even_behind_a_newer_reseed()
|
||||
{
|
||||
var rig = new Rig(new List<ImeLogPattern>(), GitHubStamp(Build.AddHours(6)));
|
||||
var service = new ImeLogPatternService(rig.Repo.Object, NullLogger<ImeLogPatternService>.Instance, BuildInfo);
|
||||
|
||||
await service.GetGlobalCatalogAsync();
|
||||
|
||||
Assert.Equal(BuiltInImeLogPatterns.GetAll().Count, rig.Stored.Count);
|
||||
Assert.Empty(rig.Deleted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Seed_runs_once_per_instance_also_when_the_gate_is_closed()
|
||||
{
|
||||
var (table, _) = DriftedTable();
|
||||
var rig = new Rig(table, GitHubStamp(Build.AddHours(6)));
|
||||
var service = new ImeLogPatternService(rig.Repo.Object, NullLogger<ImeLogPatternService>.Instance, BuildInfo);
|
||||
|
||||
await service.GetGlobalCatalogAsync();
|
||||
await service.GetGlobalCatalogAsync();
|
||||
|
||||
rig.Repo.Verify(r => r.GetRuleCatalogStampAsync(RuleCatalogStamp.KindIme), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Code_reseed_stamps_the_embedded_source()
|
||||
{
|
||||
var (table, _) = DriftedTable();
|
||||
var rig = new Rig(table, GitHubStamp(Build.AddHours(6)));
|
||||
var service = new ImeLogPatternService(rig.Repo.Object, NullLogger<ImeLogPatternService>.Instance, BuildInfo);
|
||||
|
||||
var (deleted, written) = await service.ReseedBuiltInPatternsAsync();
|
||||
|
||||
Assert.Equal(table.Count, deleted);
|
||||
Assert.Equal(BuiltInImeLogPatterns.GetAll().Count, written);
|
||||
rig.Repo.Verify(r => r.SetRuleCatalogStampAsync(It.Is<RuleCatalogStamp>(s =>
|
||||
s.Kind == RuleCatalogStamp.KindIme && s.Source == RuleCatalogStamp.SourceEmbedded && s.Count == written)), Times.Once);
|
||||
}
|
||||
}
|
||||
@@ -93,8 +93,9 @@ public class ImePatternHealthTests
|
||||
["PS-AGENT-OUTPUT"] = "not-a-number",
|
||||
};
|
||||
var data = new Dictionary<string, object> { ["hits"] = hits, ["imeVersion"] = "1.105.103.0" };
|
||||
var shipped = new HashSet<string>(new[] { "IME-ESP-PHASE", "IME-STARTED", "IME-DO-TEL", "PS-AGENT-OUTPUT" }, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var result = ImePatternHealthService.ExtractBuiltInHits(data);
|
||||
var result = ImePatternHealthService.ExtractBuiltInHits(data, shipped);
|
||||
|
||||
Assert.Equal(12, result["IME-ESP-PHASE"]);
|
||||
Assert.Equal(1, result["IME-STARTED"]);
|
||||
@@ -106,8 +107,9 @@ public class ImePatternHealthTests
|
||||
[Fact]
|
||||
public void ExtractBuiltInHits_without_a_histogram_is_empty()
|
||||
{
|
||||
Assert.Empty(ImePatternHealthService.ExtractBuiltInHits(null));
|
||||
Assert.Empty(ImePatternHealthService.ExtractBuiltInHits(new Dictionary<string, object> { ["linesRead"] = 5 }));
|
||||
var shipped = new HashSet<string>(new[] { "IME-ESP-PHASE" }, StringComparer.OrdinalIgnoreCase);
|
||||
Assert.Empty(ImePatternHealthService.ExtractBuiltInHits(null, shipped));
|
||||
Assert.Empty(ImePatternHealthService.ExtractBuiltInHits(new Dictionary<string, object> { ["linesRead"] = 5 }, shipped));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -128,9 +130,13 @@ public class ImePatternHealthTests
|
||||
new() { PatternId = "IME-NEW-THING", Category = "currentPhase", Enabled = true }, // no data yet
|
||||
};
|
||||
|
||||
var r = ImePatternHealthService.BuildResponse(stats, history, catalog, flaggedAt);
|
||||
var stamp = new RuleCatalogStamp { Kind = RuleCatalogStamp.KindIme, Source = RuleCatalogStamp.SourceGitHub, StampedAt = flaggedAt.AddDays(-1), Count = 3 };
|
||||
var r = ImePatternHealthService.BuildResponse(stats, history, catalog, stamp, flaggedAt);
|
||||
|
||||
Assert.Equal("1.104.102.0", r.BaselineVersion);
|
||||
Assert.Equal("github", r.Catalog.Source);
|
||||
Assert.Equal(flaggedAt.AddDays(-1), r.Catalog.StampedAt);
|
||||
Assert.Equal(3, r.Catalog.PatternCount);
|
||||
Assert.Equal("1.105.103.0", r.Versions[0].Version); // newest first
|
||||
Assert.Equal(30, r.Versions[0].Sessions);
|
||||
Assert.Equal(31, r.Versions[0].FleetSessions);
|
||||
@@ -150,11 +156,13 @@ public class ImePatternHealthTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuiltInPatternIds_matches_the_embedded_catalog()
|
||||
public void BuildResponse_without_a_stamp_reports_the_embedded_source()
|
||||
{
|
||||
var ids = BuiltInImeLogPatterns.BuiltInPatternIds.Value;
|
||||
Assert.Contains("IME-ESP-PHASE", ids);
|
||||
Assert.Contains("PS-SCRIPT-RESULT", ids);
|
||||
Assert.Equal(BuiltInImeLogPatterns.GetAll().Count, ids.Count);
|
||||
var r = ImePatternHealthService.BuildResponse(new List<ImePatternStatsEntry>(), new List<ImeVersionHistoryEntry>(),
|
||||
new List<ImeLogPattern> { new() { PatternId = "IME-ESP-PHASE", Category = "always", Enabled = true } }, null, DateTime.UtcNow);
|
||||
|
||||
Assert.Equal("embedded", r.Catalog.Source);
|
||||
Assert.Null(r.Catalog.StampedAt);
|
||||
Assert.Equal(1, r.Catalog.PatternCount);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -656,7 +656,8 @@ public class InfrastructureWireParityTests
|
||||
// DELIBERATE wire change (error-envelope pass): the body now leads with the envelope prefix
|
||||
// error/code/correlationId (error carries the former `message`); quotaExceeded stays as the
|
||||
// MCP's discriminator. These literals pin the NEW shape (correlationId is stamped by the writer,
|
||||
// so the builder leaves it empty).
|
||||
// so the builder leaves it empty). Both windows belong to the caller's own tenant since 2026-09-11
|
||||
// (the budget follows the delegating tenant) — there is no managed-tenant shape any more.
|
||||
|
||||
[Fact]
|
||||
public void McpQuotaExceededResponse_matches_the_daily_blocked_shape()
|
||||
@@ -745,62 +746,6 @@ public class InfrastructureWireParityTests
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void McpQuotaExceededResponse_matches_the_managed_tenant_blocked_shape()
|
||||
{
|
||||
// A delegated (MSP) read charged to a MANAGED Community tenant whose organization window is spent:
|
||||
// level=tenant, the managed tenant is named (label + targetTenantId) and the upgrade path is spelled out.
|
||||
var decision = McpQuotaService.BuildDecision(
|
||||
new McpPlanLimits("pro", 1000, 20000, "community", 300, 9000),
|
||||
dailyUsed: 12, monthlyUsed: 340, tenantDailyUsed: 300, tenantMonthlyUsed: 2100,
|
||||
new DateTime(2026, 8, 30, 15, 0, 0, DateTimeKind.Utc),
|
||||
targetTenantId: "7aa20c11-0002-4b7c-a1d2-52f3aaaa0002");
|
||||
|
||||
AssertParity(
|
||||
new
|
||||
{
|
||||
error = "MCP daily request quota of the managed tenant 'customer.example' exceeded (tenant plan 'community', shared by all its members and delegated admins). That tenant is on the Community plan, which is sized for occasional use; its own plan governs this window, not yours. Upgrading that tenant to Pro lifts its organization windows. Resets at 2026-08-31T00:00:00Z.",
|
||||
code = "QuotaExceeded",
|
||||
correlationId = "",
|
||||
quotaExceeded = true,
|
||||
plan = "pro",
|
||||
scope = "daily",
|
||||
level = "tenant",
|
||||
limit = 300,
|
||||
used = 300L,
|
||||
resetUtc = "2026-08-31T00:00:00Z",
|
||||
targetTenantId = "7aa20c11-0002-4b7c-a1d2-52f3aaaa0002"
|
||||
},
|
||||
McpQuotaEnforcementMiddleware.BuildExceededResponse(decision, "customer.example"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void McpQuotaExceededResponse_matches_the_all_managed_tenants_exhausted_shape()
|
||||
{
|
||||
// The fleet aggregate where EVERY managed tenant is spent: no single target, the count is named.
|
||||
var decision = McpQuotaService.BuildDecision(
|
||||
new McpPlanLimits("pro", 1000, 20000, "community", 300, 9000),
|
||||
dailyUsed: 12, monthlyUsed: 340, tenantDailyUsed: 300, tenantMonthlyUsed: 2100,
|
||||
new DateTime(2026, 8, 30, 15, 0, 0, DateTimeKind.Utc),
|
||||
targetTenantId: "7aa20c11-0002-4b7c-a1d2-52f3aaaa0002");
|
||||
|
||||
AssertParity(
|
||||
new
|
||||
{
|
||||
error = "MCP request quota exceeded for all 2 managed tenants in scope (each managed tenant's own plan governs its organization windows; upgrading a managed tenant to Pro lifts them). Earliest reset at 2026-08-31T00:00:00Z.",
|
||||
code = "QuotaExceeded",
|
||||
correlationId = "",
|
||||
quotaExceeded = true,
|
||||
plan = "pro",
|
||||
scope = "daily",
|
||||
level = "tenant",
|
||||
limit = 300,
|
||||
used = 300L,
|
||||
resetUtc = "2026-08-31T00:00:00Z"
|
||||
},
|
||||
McpQuotaEnforcementMiddleware.BuildExceededResponse(decision, exhaustedTenantCount: 2));
|
||||
}
|
||||
|
||||
private static void AssertParity(object anonymousLiteral, IApiResponse typed)
|
||||
=> ApiResponseWireParityTests.AssertWireIdentical(anonymousLiteral, typed);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Backup;
|
||||
using AutopilotMonitor.Functions.Services.Maintenance;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Azure.Storage.Blobs.Specialized;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Single-flight of the platform maintenance run: the 2h timer and the manual trigger enter
|
||||
/// through one gate, so a second run is skipped instead of aggregating, cleaning up and
|
||||
/// backfilling the same tables in parallel — and a skip never looks like an active run.
|
||||
/// </summary>
|
||||
public class MaintenanceRunGateTests
|
||||
{
|
||||
private sealed class Harness
|
||||
{
|
||||
public Mock<MaintenanceRunLockStore> LockStore { get; }
|
||||
public Mock<BlobLeaseClient> Lease { get; } = new();
|
||||
public List<string> OpsEvents { get; } = new();
|
||||
public MaintenanceRunGate Sut { get; }
|
||||
|
||||
public Harness()
|
||||
{
|
||||
LockStore = new Mock<MaintenanceRunLockStore>(new SessionDeletionMaintenanceFunctionTests.MaintenanceBlobStub());
|
||||
LockStore.Setup(l => l.AcquireLeaseAsync(It.IsAny<TimeSpan?>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(Lease.Object);
|
||||
|
||||
var opsRepo = new Mock<IOpsEventRepository>();
|
||||
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>()))
|
||||
.Returns<OpsEventEntry>(e => { lock (OpsEvents) OpsEvents.Add(e.EventType); return Task.CompletedTask; });
|
||||
var adminConfig = new Mock<AdminConfigurationService>(
|
||||
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance, new MemoryCache(new MemoryCacheOptions()));
|
||||
var opsService = new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance,
|
||||
TestNotifications.InertOpsAlertDispatch(adminConfig.Object));
|
||||
|
||||
Sut = new MaintenanceRunGate(LockStore.Object, opsService, NullLogger<MaintenanceRunGate>.Instance);
|
||||
}
|
||||
|
||||
public void HoldLease() =>
|
||||
LockStore.Setup(l => l.AcquireLeaseAsync(It.IsAny<TimeSpan?>(), It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new LeaseHeldException("held", new Exception("409")));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_held_lease_skips_the_run_and_never_reports_it_as_started()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.HoldLease();
|
||||
var ran = false;
|
||||
|
||||
var result = await h.Sut.RunExclusiveAsync("Timer", () => { ran = true; return Task.CompletedTask; });
|
||||
|
||||
Assert.False(result);
|
||||
Assert.False(ran);
|
||||
Assert.Equal(new[] { OpsEventTypes.MaintenanceSkippedLocked }, h.OpsEvents);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Started_is_written_under_the_lease_before_the_body_and_the_lease_is_released_last()
|
||||
{
|
||||
var h = new Harness();
|
||||
var order = new List<string>();
|
||||
h.Lease.Setup(l => l.ReleaseAsync(It.IsAny<Azure.RequestConditions>(), It.IsAny<CancellationToken>()))
|
||||
.Callback(() => order.Add("release"))
|
||||
.ReturnsAsync(Mock.Of<Azure.Response<Azure.Storage.Blobs.Models.ReleasedObjectInfo>>());
|
||||
|
||||
var result = await h.Sut.RunExclusiveAsync("admin@example.com", () =>
|
||||
{
|
||||
order.Add($"body after [{string.Join(",", h.OpsEvents)}]");
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
|
||||
Assert.True(result);
|
||||
Assert.Equal(new[] { $"body after [{OpsEventTypes.MaintenanceStarted}]", "release" }, order);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_lease_is_released_when_the_body_throws()
|
||||
{
|
||||
var h = new Harness();
|
||||
var released = false;
|
||||
h.Lease.Setup(l => l.ReleaseAsync(It.IsAny<Azure.RequestConditions>(), It.IsAny<CancellationToken>()))
|
||||
.Callback(() => released = true)
|
||||
.ReturnsAsync(Mock.Of<Azure.Response<Azure.Storage.Blobs.Models.ReleasedObjectInfo>>());
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() =>
|
||||
h.Sut.RunExclusiveAsync("Timer", () => throw new InvalidOperationException("boom")));
|
||||
|
||||
Assert.True(released);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_reports_an_active_run_while_the_lease_is_held()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.HoldLease();
|
||||
|
||||
Assert.True(await h.Sut.IsRunActiveAsync());
|
||||
Assert.Empty(h.OpsEvents);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_releases_its_lease_and_reports_no_active_run()
|
||||
{
|
||||
var h = new Harness();
|
||||
var released = false;
|
||||
h.Lease.Setup(l => l.ReleaseAsync(It.IsAny<Azure.RequestConditions>(), It.IsAny<CancellationToken>()))
|
||||
.Callback(() => released = true)
|
||||
.ReturnsAsync(Mock.Of<Azure.Response<Azure.Storage.Blobs.Models.ReleasedObjectInfo>>());
|
||||
|
||||
Assert.False(await h.Sut.IsRunActiveAsync());
|
||||
Assert.True(released);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_lets_a_failed_release_surface_so_nothing_is_queued_behind_a_stuck_lease()
|
||||
{
|
||||
// A queued run picked up seconds later would find the probe's own lease still held,
|
||||
// skip as "locked" and delete the message — the operator's run would silently vanish.
|
||||
var h = new Harness();
|
||||
h.Lease.Setup(l => l.ReleaseAsync(It.IsAny<Azure.RequestConditions>(), It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new Azure.RequestFailedException(500, "storage down"));
|
||||
|
||||
await Assert.ThrowsAsync<Azure.RequestFailedException>(() => h.Sut.IsRunActiveAsync());
|
||||
}
|
||||
}
|
||||
+50
-35
@@ -7,47 +7,43 @@ namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for the pure folds behind GET metrics/mcp-usage/organization (and its global twin): one item per
|
||||
/// account, the delegated (MSP) marker from the row's home tenant, the three windows (today / month / range),
|
||||
/// and the tenant's organization windows over every account.
|
||||
/// account, the three windows (today / month / range), and the tenant's organization windows over every
|
||||
/// account including the purchased-slot breakdown.
|
||||
/// </summary>
|
||||
public class McpOrganizationUsageAggregationTests
|
||||
{
|
||||
private const string Tenant = "11111111-1111-1111-1111-111111111111";
|
||||
private const string MspHome = "99999999-9999-9999-9999-999999999999";
|
||||
private const string Member = "aaaaaaaa-0000-0000-0000-000000000001";
|
||||
private const string MspAdmin = "bbbbbbbb-0000-0000-0000-000000000002";
|
||||
private const string OtherMember = "bbbbbbbb-0000-0000-0000-000000000002";
|
||||
|
||||
private static TenantUsageRecord Row(string user, string date, long count, string upn = "", string home = "", DateTime? last = null)
|
||||
=> new() { TenantId = Tenant, UserId = user, Date = date, RequestCount = count, UserPrincipalName = upn, HomeTenantId = home, LastRequestAt = last };
|
||||
private static TenantUsageRecord Row(string user, string date, long count, string upn = "", DateTime? last = null)
|
||||
=> new() { TenantId = Tenant, UserId = user, Date = date, RequestCount = count, UserPrincipalName = upn, LastRequestAt = last };
|
||||
|
||||
[Fact]
|
||||
public void Aggregate_MarksDelegatedAccounts_ByForeignHomeTenant()
|
||||
public void Aggregate_OneItemPerAccount_KeepsTheLatestUpn()
|
||||
{
|
||||
var rows = new[]
|
||||
{
|
||||
Row(Member, "20260902", 10, "alice@contoso.com"),
|
||||
Row(MspAdmin, "20260902", 40, "msp@partner.example", MspHome),
|
||||
Row(Member, "20260901", 10), // written before the UPN column existed
|
||||
Row(Member, "20260902", 5, "alice@contoso.com"),
|
||||
Row(OtherMember, "20260902", 40, "bob@contoso.com"),
|
||||
};
|
||||
|
||||
var items = McpUsageMetricsFunction.AggregateOrganizationUsage(rows, Tenant, "20260901", "20260902", "20260902", "20260901");
|
||||
var items = McpUsageMetricsFunction.AggregateOrganizationUsage(rows, "20260901", "20260902", "20260902", "20260901");
|
||||
|
||||
var msp = Assert.Single(items, i => i.UserId == MspAdmin);
|
||||
Assert.True(msp.Delegated);
|
||||
Assert.Equal(MspHome, msp.HomeTenantId);
|
||||
Assert.Equal("msp@partner.example", msp.UserPrincipalName);
|
||||
var member = Assert.Single(items, i => i.UserId == Member);
|
||||
Assert.False(member.Delegated);
|
||||
Assert.Null(member.HomeTenantId);
|
||||
Assert.Equal(2, items.Count);
|
||||
var alice = Assert.Single(items, i => i.UserId == Member);
|
||||
Assert.Equal("alice@contoso.com", alice.UserPrincipalName);
|
||||
Assert.Equal(15, alice.RequestsThisMonth);
|
||||
var bob = Assert.Single(items, i => i.UserId == OtherMember);
|
||||
Assert.Equal("bob@contoso.com", bob.UserPrincipalName);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Aggregate_OwnTenantAsHome_IsNotDelegated()
|
||||
public void Aggregate_RowsWithoutUpn_LeaveItNull()
|
||||
{
|
||||
// Legacy/own rows may carry the tenant itself (or nothing) as home — both mean "own member".
|
||||
var rows = new[] { Row(Member, "20260902", 1, home: Tenant), Row(Member, "20260901", 1) };
|
||||
var items = McpUsageMetricsFunction.AggregateOrganizationUsage(rows, Tenant, "20260901", "20260902", "20260902", "20260901");
|
||||
var item = Assert.Single(items);
|
||||
Assert.False(item.Delegated);
|
||||
var rows = new[] { Row(Member, "20260902", 1), Row(Member, "20260901", 1) };
|
||||
var item = Assert.Single(McpUsageMetricsFunction.AggregateOrganizationUsage(rows, "20260901", "20260902", "20260902", "20260901"));
|
||||
Assert.Null(item.UserPrincipalName);
|
||||
}
|
||||
|
||||
@@ -62,7 +58,7 @@ public class McpOrganizationUsageAggregationTests
|
||||
Row(Member, "20260903", 7), // read window past the range end (never counted in range)
|
||||
};
|
||||
|
||||
var items = McpUsageMetricsFunction.AggregateOrganizationUsage(rows, Tenant, "20260815", "20260902", "20260902", "20260901");
|
||||
var items = McpUsageMetricsFunction.AggregateOrganizationUsage(rows, "20260815", "20260902", "20260902", "20260901");
|
||||
|
||||
var item = Assert.Single(items);
|
||||
Assert.Equal(5, item.RequestsToday);
|
||||
@@ -79,12 +75,12 @@ public class McpOrganizationUsageAggregationTests
|
||||
{
|
||||
Row(Member, "20260901", 3, last: newer),
|
||||
Row(Member, "20260902", 1, last: older),
|
||||
Row(MspAdmin, "20260902", 50, home: MspHome, last: older),
|
||||
Row(OtherMember, "20260902", 50, last: older),
|
||||
};
|
||||
|
||||
var items = McpUsageMetricsFunction.AggregateOrganizationUsage(rows, Tenant, "20260901", "20260902", "20260902", "20260901");
|
||||
var items = McpUsageMetricsFunction.AggregateOrganizationUsage(rows, "20260901", "20260902", "20260902", "20260901");
|
||||
|
||||
Assert.Equal(new[] { MspAdmin, Member }, items.Select(i => i.UserId));
|
||||
Assert.Equal(new[] { OtherMember, Member }, items.Select(i => i.UserId));
|
||||
Assert.Equal(newer, items[1].LastRequestAt);
|
||||
}
|
||||
|
||||
@@ -93,11 +89,11 @@ public class McpOrganizationUsageAggregationTests
|
||||
{
|
||||
var rows = new[]
|
||||
{
|
||||
Row(Member, "20260815", 100), // last month: neither window
|
||||
Row(Member, "20260901", 20), // month
|
||||
Row(Member, "20260902", 2), // today + month
|
||||
Row(MspAdmin, "20260902", 5, home: MspHome), // delegated reads count against the same windows
|
||||
Row(Member, "20260903", 7), // read window past today (range only)
|
||||
Row(Member, "20260815", 100), // last month: neither window
|
||||
Row(Member, "20260901", 20), // month
|
||||
Row(Member, "20260902", 2), // today + month
|
||||
Row(OtherMember, "20260902", 5), // another account counts against the same windows
|
||||
Row(Member, "20260903", 7), // read window past today (range only)
|
||||
};
|
||||
|
||||
var quota = McpUsageMetricsFunction.BuildOrganizationQuota(
|
||||
@@ -108,6 +104,25 @@ public class McpOrganizationUsageAggregationTests
|
||||
Assert.Equal(15000, quota.MonthlyLimit);
|
||||
Assert.Equal(7, quota.DailyUsed);
|
||||
Assert.Equal(27, quota.MonthlyUsed);
|
||||
// No purchased slot: the breakdown stays off the wire.
|
||||
Assert.Null(quota.PurchasedDelegatedSlots);
|
||||
Assert.Null(quota.SlotDailyLimit);
|
||||
Assert.Null(quota.SlotMonthlyLimit);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OrganizationQuota_PurchasedSlots_CarryTheBreakdown()
|
||||
{
|
||||
// 3 000 + 3 × 900 = 5 700 — the limits arrive grown; the slot fields let the page show the sum.
|
||||
var quota = McpUsageMetricsFunction.BuildOrganizationQuota(
|
||||
new[] { Row(Member, "20260902", 3) },
|
||||
new McpQuotaService.TenantPlanLimits("pro", 5700, 114000, PurchasedDelegatedSlots: 3, SlotTenantDailyLimit: 900, SlotTenantMonthlyLimit: 18000),
|
||||
"20260902", "20260901");
|
||||
|
||||
Assert.Equal(5700, quota.DailyLimit);
|
||||
Assert.Equal(3, quota.PurchasedDelegatedSlots);
|
||||
Assert.Equal(900, quota.SlotDailyLimit);
|
||||
Assert.Equal(18000, quota.SlotMonthlyLimit);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -115,11 +130,11 @@ public class McpOrganizationUsageAggregationTests
|
||||
{
|
||||
var rows = new[]
|
||||
{
|
||||
Row(Member, "20260815", 100), // before the range
|
||||
Row(Member, "20260815", 100), // before the range
|
||||
Row(Member, "20260902", 2),
|
||||
Row(MspAdmin, "20260902", 5, home: MspHome), // same day, delegated — one bar
|
||||
Row(OtherMember, "20260902", 5), // same day, another account — one bar
|
||||
Row(Member, "20260901", 20),
|
||||
Row(Member, "20260903", 7), // after the range (read window)
|
||||
Row(Member, "20260903", 7), // after the range (read window)
|
||||
};
|
||||
|
||||
var daily = McpUsageMetricsFunction.BuildOrganizationDaily(rows, "20260901", "20260902");
|
||||
|
||||
+32
-218
@@ -6,252 +6,77 @@ using Xunit;
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for the pure seams of <see cref="McpQuotaEnforcementMiddleware"/>: which tenant a request is
|
||||
/// charged to ("the budget follows the data"), the bound-narrowing isolation invariant, and the 429 texts
|
||||
/// that name whose budget is exhausted.
|
||||
/// Tests for the pure seams of <see cref="McpQuotaEnforcementMiddleware"/>: who bypasses the check (the Global
|
||||
/// Admin only — counted, never blocked) and the 429 texts that name whose window is exhausted. Every request is
|
||||
/// charged to the caller's HOME tenant ("the budget follows the delegating tenant"), so no seam names a target
|
||||
/// tenant any more.
|
||||
/// </summary>
|
||||
public class McpQuotaEnforcementMiddlewareTests
|
||||
{
|
||||
private const string Home = "aaaaaaaa-0000-0000-0000-000000000001";
|
||||
private const string TenantB = "bbbbbbbb-0000-0000-0000-000000000002";
|
||||
private const string TenantC = "cccccccc-0000-0000-0000-000000000003";
|
||||
private const string TenantD = "dddddddd-0000-0000-0000-000000000004";
|
||||
private static readonly DateTime Now = new(2026, 9, 2, 15, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
// ── ResolveChargeScope ─────────────────────────────────────────────────────────
|
||||
// ── IsExempt — the Global Admin runs, the Global Reader is quota'd ─────────────
|
||||
|
||||
[Fact]
|
||||
public void Member_ReadingOwnTenant_ChargesHome()
|
||||
public void GlobalAdmin_IsExempt_EvenOnAForeignTarget()
|
||||
{
|
||||
var ctx = new RequestContext { TenantId = Home, TargetTenantId = Home };
|
||||
var scope = McpQuotaEnforcementMiddleware.ResolveChargeScope(ctx, Home);
|
||||
Assert.Equal(McpChargeKind.Home, scope.Kind);
|
||||
Assert.Null(scope.TargetTenantId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Delegated_SingleManagedTarget_ChargesThatTarget()
|
||||
{
|
||||
var ctx = new RequestContext
|
||||
{
|
||||
TenantId = Home, TargetTenantId = TenantB, IsDelegatedReader = true,
|
||||
AllowedTenantIds = new[] { TenantB, TenantC },
|
||||
};
|
||||
var scope = McpQuotaEnforcementMiddleware.ResolveChargeScope(ctx, Home);
|
||||
Assert.Equal(McpChargeKind.SingleTarget, scope.Kind);
|
||||
Assert.Equal(TenantB, scope.TargetTenantId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Delegated_ReadingHomeTenant_ChargesHome()
|
||||
{
|
||||
// The subset tier admits a delegated caller for ?tenantId=<home> too (crossTenant=false) — that read
|
||||
// is the caller's OWN tenant and draws on its own budget.
|
||||
var ctx = new RequestContext
|
||||
{
|
||||
TenantId = Home, TargetTenantId = Home, IsDelegatedReader = true,
|
||||
AllowedTenantIds = new[] { TenantB },
|
||||
};
|
||||
var scope = McpQuotaEnforcementMiddleware.ResolveChargeScope(ctx, Home);
|
||||
Assert.Equal(McpChargeKind.Home, scope.Kind);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Delegated_TargetReachedThroughHomeChargedGroup_ChargesHome()
|
||||
{
|
||||
var ctx = new RequestContext
|
||||
{
|
||||
TenantId = Home, TargetTenantId = TenantC, IsDelegatedReader = true,
|
||||
AllowedTenantIds = new[] { TenantB, TenantC },
|
||||
HomeChargedTenantIds = new[] { TenantC },
|
||||
};
|
||||
var scope = McpQuotaEnforcementMiddleware.ResolveChargeScope(ctx, Home);
|
||||
Assert.Equal(McpChargeKind.Home, scope.Kind);
|
||||
Assert.Null(scope.TargetTenantId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Delegated_FleetAggregate_ChargesEveryManagedTenant()
|
||||
{
|
||||
var ctx = new RequestContext
|
||||
{
|
||||
TenantId = Home, TargetTenantId = Home, IsDelegatedReader = true, IsDelegatedAggregate = true,
|
||||
AllowedTenantIds = new[] { TenantB, TenantC },
|
||||
};
|
||||
var scope = McpQuotaEnforcementMiddleware.ResolveChargeScope(ctx, Home);
|
||||
Assert.Equal(McpChargeKind.BoundedAggregate, scope.Kind);
|
||||
Assert.Equal(new[] { TenantB, TenantC }, scope.ChargeMap!.Keys.OrderBy(k => k));
|
||||
Assert.Equal(new[] { TenantB }, scope.TargetsOf(new[] { TenantB }));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Delegated_FleetAggregate_FoldsHomeChargedTenantsOntoHome()
|
||||
{
|
||||
var ctx = new RequestContext
|
||||
{
|
||||
TenantId = Home, TargetTenantId = Home, IsDelegatedReader = true, IsDelegatedAggregate = true,
|
||||
AllowedTenantIds = new[] { TenantB, TenantC, TenantD },
|
||||
HomeChargedTenantIds = new[] { TenantC, TenantD },
|
||||
};
|
||||
var scope = McpQuotaEnforcementMiddleware.ResolveChargeScope(ctx, Home);
|
||||
Assert.Equal(McpChargeKind.BoundedAggregate, scope.Kind);
|
||||
Assert.Equal(new[] { Home, TenantB }, scope.ChargeMap!.Keys.OrderBy(k => k));
|
||||
// Home is charged ONCE for both operator-managed tenants; they are served or dropped together.
|
||||
Assert.Equal(new[] { TenantC, TenantD }, scope.TargetsOf(new[] { Home }).OrderBy(t => t));
|
||||
Assert.Equal(new[] { TenantB }, scope.TargetsOf(new[] { TenantB }));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Delegated_SubsetTierDirectoryListing_IsNotAnAggregate_ChargesHome()
|
||||
{
|
||||
// config/all: subset tier but TenantScoping.None → the policy middleware never marks it an aggregate.
|
||||
var ctx = new RequestContext
|
||||
{
|
||||
TenantId = Home, TargetTenantId = Home, IsDelegatedReader = true, IsDelegatedAggregate = false,
|
||||
AllowedTenantIds = new[] { TenantB, TenantC },
|
||||
};
|
||||
var scope = McpQuotaEnforcementMiddleware.ResolveChargeScope(ctx, Home);
|
||||
Assert.Equal(McpChargeKind.Home, scope.Kind);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GlobalAdmin_IsNeverDelegated_ResolvesHome()
|
||||
{
|
||||
// The policy middleware short-circuits delegated resolution for platform scope, so a GA reading
|
||||
// ?tenantId=<customer> carries no delegated flags — the charge scope is Home (and Invoke tracks
|
||||
// GA on home only, exempt from the check).
|
||||
// Platform operations must never be blocked by a budget; Invoke still counts the GA on home.
|
||||
var ctx = new RequestContext { TenantId = Home, TargetTenantId = TenantB, IsGlobalAdmin = true };
|
||||
var scope = McpQuotaEnforcementMiddleware.ResolveChargeScope(ctx, Home);
|
||||
Assert.Equal(McpChargeKind.Home, scope.Kind);
|
||||
}
|
||||
|
||||
// ── Narrow — ISOLATION INVARIANT ───────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void Narrow_WithoutBound_LeavesContextUntouched()
|
||||
{
|
||||
// null = unbounded (GA/Reader). Narrowing must neither introduce a bound nor touch the context.
|
||||
var ctx = new RequestContext { TenantId = Home, AllowedTenantIds = null };
|
||||
var narrowed = McpQuotaEnforcementMiddleware.Narrow(ctx, new[] { TenantB }, new[] { TenantC });
|
||||
Assert.Same(ctx, narrowed);
|
||||
Assert.Null(narrowed.AllowedTenantIds);
|
||||
Assert.Null(narrowed.QuotaExcludedTenantIds);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(new[] { TenantB, TenantC }, new[] { TenantB }, new[] { TenantC })]
|
||||
[InlineData(new[] { TenantB, TenantC }, new string[0], new[] { TenantB, TenantC })]
|
||||
[InlineData(new[] { TenantB, TenantC }, new[] { TenantB, TenantC }, new string[0])]
|
||||
[InlineData(new[] { TenantB }, new[] { TenantB, TenantD }, new[] { TenantD })]
|
||||
[InlineData(new[] { TenantB, TenantC, TenantD }, new[] { TenantD, TenantB }, new[] { TenantC })]
|
||||
public void Narrow_NeverWidens_NeverNull(string[] bound, string[] admitted, string[] excluded)
|
||||
{
|
||||
var ctx = new RequestContext { TenantId = Home, AllowedTenantIds = bound, IsDelegatedReader = true };
|
||||
|
||||
var narrowed = McpQuotaEnforcementMiddleware.Narrow(ctx, admitted, excluded);
|
||||
|
||||
Assert.NotNull(narrowed.AllowedTenantIds);
|
||||
var boundSet = new HashSet<string>(bound, StringComparer.OrdinalIgnoreCase);
|
||||
Assert.All(narrowed.AllowedTenantIds!, t => Assert.Contains(t, boundSet));
|
||||
Assert.True(narrowed.AllowedTenantIds!.Count <= bound.Length);
|
||||
// A foreign tenant sneaking into "admitted" never widens the bound (TenantD ∉ bound in row 4).
|
||||
Assert.DoesNotContain(narrowed.AllowedTenantIds!, t => !boundSet.Contains(t));
|
||||
if (narrowed.QuotaExcludedTenantIds != null)
|
||||
Assert.All(narrowed.QuotaExcludedTenantIds, t => Assert.Contains(t, boundSet));
|
||||
// The rest of the context survives the copy.
|
||||
Assert.True(narrowed.IsDelegatedReader);
|
||||
Assert.Equal(Home, narrowed.TenantId);
|
||||
Assert.True(McpQuotaEnforcementMiddleware.IsExempt(ctx));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Narrow_EmptyAdmittedSet_PublishesEmptyBound_NotNull()
|
||||
public void GlobalReader_IsNotExempt()
|
||||
{
|
||||
// Empty ⇒ the repository serves an empty page. NEVER null (= all tenants).
|
||||
var ctx = new RequestContext { TenantId = Home, AllowedTenantIds = new[] { TenantB, TenantC } };
|
||||
var narrowed = McpQuotaEnforcementMiddleware.Narrow(ctx, Array.Empty<string>(), new[] { TenantB, TenantC });
|
||||
Assert.NotNull(narrowed.AllowedTenantIds);
|
||||
Assert.Empty(narrowed.AllowedTenantIds!);
|
||||
Assert.Equal(new[] { TenantB, TenantC }, narrowed.QuotaExcludedTenantIds!.OrderBy(t => t));
|
||||
// A customer-facing read role: it stays within its home tenant's windows like every member.
|
||||
var ctx = new RequestContext { TenantId = Home, TargetTenantId = TenantB, IsGlobalReader = true };
|
||||
Assert.False(McpQuotaEnforcementMiddleware.IsExempt(ctx));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Narrow_NothingExcluded_OmitsTheExcludedSet()
|
||||
public void Member_And_DelegatedReader_AreNotExempt()
|
||||
{
|
||||
var ctx = new RequestContext { TenantId = Home, AllowedTenantIds = new[] { TenantB, TenantC } };
|
||||
var narrowed = McpQuotaEnforcementMiddleware.Narrow(ctx, new[] { TenantB, TenantC }, Array.Empty<string>());
|
||||
Assert.Equal(2, narrowed.AllowedTenantIds!.Count);
|
||||
Assert.Null(narrowed.QuotaExcludedTenantIds);
|
||||
Assert.False(McpQuotaEnforcementMiddleware.IsExempt(new RequestContext { TenantId = Home, TargetTenantId = Home }));
|
||||
Assert.False(McpQuotaEnforcementMiddleware.IsExempt(new RequestContext
|
||||
{
|
||||
TenantId = Home, TargetTenantId = TenantB, IsDelegatedReader = true, AllowedTenantIds = new[] { TenantB },
|
||||
}));
|
||||
}
|
||||
|
||||
// ── BuildExceededResponse — whose budget ───────────────────────────────────────
|
||||
|
||||
private static McpQuotaDecision Blocked(string tenantPlan, string? target, long tenantDailyUsed = 300, int tenantDaily = 300)
|
||||
private static McpQuotaDecision Blocked(string tenantPlan, long tenantDailyUsed = 300, int tenantDaily = 300)
|
||||
=> McpQuotaService.BuildDecision(
|
||||
new McpPlanLimits("pro", 1000, 20000, tenantPlan, tenantDaily, 60000),
|
||||
dailyUsed: 5, monthlyUsed: 50, tenantDailyUsed: tenantDailyUsed, tenantMonthlyUsed: 400,
|
||||
Now, target);
|
||||
Now);
|
||||
|
||||
[Fact]
|
||||
public void ManagedCommunityTarget_NamesTheTenant_AndTheUpgradePath()
|
||||
public void OrganizationBlock_KeepsTheOrganizationWording()
|
||||
{
|
||||
var body = McpQuotaEnforcementMiddleware.BuildExceededResponse(Blocked("community", TenantB), "customer.example");
|
||||
|
||||
var body = McpQuotaEnforcementMiddleware.BuildExceededResponse(Blocked("community"));
|
||||
Assert.Equal("tenant", body.Level);
|
||||
Assert.Equal(TenantB, body.TargetTenantId);
|
||||
Assert.Contains("managed tenant 'customer.example'", body.Error);
|
||||
Assert.Contains("tenant plan 'community'", body.Error);
|
||||
Assert.Contains("Upgrading that tenant to Pro", body.Error);
|
||||
Assert.DoesNotContain("your organization", body.Error);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ManagedTarget_WithoutLabel_FallsBackToTheId()
|
||||
{
|
||||
var body = McpQuotaEnforcementMiddleware.BuildExceededResponse(Blocked("community", TenantB));
|
||||
Assert.Contains($"managed tenant '{TenantB}'", body.Error);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ManagedProTarget_OmitsTheUpgradeHint()
|
||||
{
|
||||
var body = McpQuotaEnforcementMiddleware.BuildExceededResponse(Blocked("pro", TenantB, 3000, 3000), "customer.example");
|
||||
Assert.Equal(TenantB, body.TargetTenantId);
|
||||
Assert.DoesNotContain("Upgrading", body.Error);
|
||||
Assert.Contains("shared by all its members and delegated admins", body.Error);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OwnTenant_KeepsTheOrganizationWording_AndNoTarget()
|
||||
{
|
||||
var body = McpQuotaEnforcementMiddleware.BuildExceededResponse(Blocked("community", target: null));
|
||||
Assert.Null(body.TargetTenantId);
|
||||
Assert.Contains("of your organization", body.Error);
|
||||
Assert.Contains("tenant plan 'community'", body.Error);
|
||||
Assert.DoesNotContain("managed tenant", body.Error);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OwnCommunityTenant_NamesTheUpgradePath()
|
||||
public void OrganizationBlock_OnCommunity_NamesTheUpgradePath()
|
||||
{
|
||||
// The quota is the upgrade lever: a member blocked by their own Community organization window is told
|
||||
// that Community is sized for occasional use and that Pro lifts it — Pro members never see the hint.
|
||||
var community = McpQuotaEnforcementMiddleware.BuildExceededResponse(Blocked("community", target: null));
|
||||
var community = McpQuotaEnforcementMiddleware.BuildExceededResponse(Blocked("community"));
|
||||
Assert.Contains("sized for occasional use", community.Error);
|
||||
Assert.Contains("upgrading your organization to Pro", community.Error);
|
||||
Assert.EndsWith("Resets at 2026-09-03T00:00:00Z.", community.Error);
|
||||
|
||||
var pro = McpQuotaEnforcementMiddleware.BuildExceededResponse(Blocked("pro", target: null, 3000, 3000));
|
||||
var pro = McpQuotaEnforcementMiddleware.BuildExceededResponse(Blocked("pro", 3000, 3000));
|
||||
Assert.DoesNotContain("upgrading", pro.Error);
|
||||
Assert.DoesNotContain("occasional use", pro.Error);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ManagedCommunityTarget_SaysWhosePlanGoverns()
|
||||
{
|
||||
// The delegated admin must learn that the CUSTOMER's plan (not their own Pro/MSP plan) governs the window.
|
||||
var body = McpQuotaEnforcementMiddleware.BuildExceededResponse(Blocked("community", TenantB), "customer.example");
|
||||
Assert.Contains("on the Community plan", body.Error);
|
||||
Assert.Contains("its own plan governs this window, not yours", body.Error);
|
||||
Assert.Contains("Upgrading that tenant to Pro", body.Error);
|
||||
Assert.Contains("shared by all its members", pro.Error);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
@@ -272,25 +97,14 @@ public class McpQuotaEnforcementMiddlewareTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AllManagedTenantsExhausted_NamesTheCount_AndNoSingleTarget()
|
||||
public void UserLevelBlock_ReportsTheUserWindow()
|
||||
{
|
||||
var body = McpQuotaEnforcementMiddleware.BuildExceededResponse(Blocked("community", TenantB), exhaustedTenantCount: 3);
|
||||
Assert.Null(body.TargetTenantId);
|
||||
Assert.Equal("tenant", body.Level);
|
||||
Assert.Contains("all 3 managed tenants", body.Error);
|
||||
Assert.Contains("Earliest reset at 2026-09-03T00:00:00Z", body.Error);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UserLevelBlock_IgnoresTargetAndCount()
|
||||
{
|
||||
// The caller's OWN budget is exhausted — the target tenant is irrelevant and must not be named.
|
||||
var decision = McpQuotaService.BuildDecision(
|
||||
new McpPlanLimits("community", 100, 3000, "pro", 3000, 60000),
|
||||
dailyUsed: 100, monthlyUsed: 500, tenantDailyUsed: 1, tenantMonthlyUsed: 1, Now, TenantB);
|
||||
var body = McpQuotaEnforcementMiddleware.BuildExceededResponse(decision, "customer.example", exhaustedTenantCount: 2);
|
||||
Assert.Equal("user", body.Level);
|
||||
Assert.Null(body.TargetTenantId);
|
||||
dailyUsed: 100, monthlyUsed: 500, tenantDailyUsed: 1, tenantMonthlyUsed: 1, Now);
|
||||
var body = McpQuotaEnforcementMiddleware.BuildExceededResponse(decision);
|
||||
Assert.Equal(100, body.Limit);
|
||||
Assert.Equal(100, body.Used);
|
||||
Assert.Equal(
|
||||
"MCP daily request quota exceeded for plan 'community'. The Community plan is sized for occasional use; Pro raises your daily and monthly windows. Resets at 2026-09-03T00:00:00Z.",
|
||||
body.Error);
|
||||
|
||||
@@ -11,7 +11,8 @@ namespace AutopilotMonitor.Functions.Tests;
|
||||
/// <summary>
|
||||
/// Tests for <see cref="McpQuotaService"/>: window math (daily/monthly, reset times), plan
|
||||
/// precedence (per-user override → tenant edition; SectionUsagePlans definition → catalog
|
||||
/// fallback), and the fail-open contract on counter errors.
|
||||
/// fallback), the growth of both windows by purchased delegation slots, and the fail-open
|
||||
/// contract on counter errors.
|
||||
/// </summary>
|
||||
public class McpQuotaServiceTests
|
||||
{
|
||||
@@ -127,7 +128,9 @@ public class McpQuotaServiceTests
|
||||
IMemoryCache? cache = null,
|
||||
bool bound = true,
|
||||
Func<string?, TenantEdition>? editionResolver = null,
|
||||
Func<string?, string?>? tenantPlanOverrideResolver = null)
|
||||
Func<string?, string?>? tenantPlanOverrideResolver = null,
|
||||
Func<string?, int>? purchasedSlotsResolver = null,
|
||||
Func<string?, EditionResolution>? resolutionResolver = null)
|
||||
{
|
||||
var adminRepo = new Mock<IAdminRepository>();
|
||||
adminRepo.Setup(r => r.GetMcpUserAsync(It.IsAny<string>()))
|
||||
@@ -150,11 +153,15 @@ public class McpQuotaServiceTests
|
||||
var adminConfigService = new AdminConfigurationService(
|
||||
configRepo.Object, NullLogger<AdminConfigurationService>.Instance, cache);
|
||||
|
||||
var entitlements = resolutionResolver != null
|
||||
? new StubTenantEntitlementService(resolutionResolver, tenantPlanOverrideResolver, purchasedSlotsResolver)
|
||||
: new StubTenantEntitlementService(editionResolver ?? (_ => edition), tenantPlanOverrideResolver, purchasedSlotsResolver);
|
||||
|
||||
return new McpQuotaService(
|
||||
usageRepo.Object,
|
||||
mcpUserService,
|
||||
adminConfigService,
|
||||
new StubTenantEntitlementService(editionResolver ?? (_ => edition), tenantPlanOverrideResolver),
|
||||
entitlements,
|
||||
cache,
|
||||
NullLogger<McpQuotaService>.Instance,
|
||||
new TestTimeProvider(Now));
|
||||
@@ -338,25 +345,18 @@ public class McpQuotaServiceTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Check_DelegatedTarget_MspHomeOverride_LiftsTheCallerNotTheCustomer()
|
||||
public async Task Check_TenantOverride_ComposesWithPurchasedSlots()
|
||||
{
|
||||
// The MSP's home tenant carries the "msp" override; the managed Community customer does not.
|
||||
// The caller's own windows are lifted, the customer's organization window still bites.
|
||||
var repo = UsageRepo(("20260707", 1200));
|
||||
TenantRows(repo, Target, ("20260707", 300));
|
||||
var svc = Build(repo, planDefinitionsJson: MspPlanJson,
|
||||
editionResolver: HomeProTargetsCommunity,
|
||||
tenantPlanOverrideResolver: t => t == TenantId ? "msp" : null);
|
||||
// The "msp" definition carries no slot fields → the Pro catalog slot values grow its windows.
|
||||
var svc = Build(UsageRepo(("20260707", 1)), planDefinitionsJson: MspPlanJson,
|
||||
edition: TenantEdition.Pro, tenantPlanOverrideResolver: _ => "msp", purchasedSlotsResolver: _ => 2);
|
||||
|
||||
var d = await svc.CheckAsync(Oid, Upn, homeTenantId: TenantId, chargeTenantId: Target);
|
||||
var d = await svc.CheckAsync(Oid, Upn, TenantId);
|
||||
|
||||
Assert.False(d.Allowed);
|
||||
Assert.Equal("tenant", d.Level);
|
||||
Assert.Equal("msp", d.Plan);
|
||||
Assert.Equal(1500, d.DailyLimit);
|
||||
Assert.Equal("community", d.TenantPlan);
|
||||
Assert.Equal(300, d.TenantDailyLimit);
|
||||
Assert.Equal(Target, d.TargetTenantId);
|
||||
Assert.Equal(2100, d.DailyLimit); // 1500 + 2 × 300
|
||||
Assert.Equal("msp", d.TenantPlan);
|
||||
Assert.Equal(6800, d.TenantDailyLimit); // 5000 + 2 × 900
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -560,155 +560,176 @@ public class McpQuotaServiceTests
|
||||
repo.Verify(r => r.GetTenantUsageAsync(It.IsAny<string>(), It.IsAny<string?>(), It.IsAny<string?>()), Times.Exactly(2));
|
||||
}
|
||||
|
||||
// ── Delegated (MSP) reads: the budget follows the data ───────────────────────
|
||||
// The CHARGED tenant (the managed customer) supplies the organization windows and counters; the
|
||||
// caller's own windows still follow their HOME tenant's plan (and their per-user override).
|
||||
// ── Purchased delegation slots grow both windows ─────────────────────────────
|
||||
// The home tenant's windows (user AND organization) grow by purchased slots × per-slot value; the slot
|
||||
// values come from the governing plan definition, else the home edition's catalog. Community and
|
||||
// conferred Pro carry catalog slot values of 0, so a stray slot count grows nothing there.
|
||||
|
||||
private const string Target = "22222222-2222-2222-2222-222222222222";
|
||||
private const string Target2 = "33333333-3333-3333-3333-333333333333";
|
||||
private const string OtherOid = "00000000-0000-0000-0000-000000000002";
|
||||
|
||||
private static void TenantRows(Mock<IUserUsageRepository> repo, string tenantId, params (string Date, long Count)[] rows)
|
||||
=> repo.Setup(r => r.GetTenantUsageAsync(tenantId, It.IsAny<string?>(), It.IsAny<string?>()))
|
||||
.ReturnsAsync(rows.Select(r => new TenantUsageRecord
|
||||
{
|
||||
TenantId = tenantId,
|
||||
UserId = "some-member-or-msp",
|
||||
Date = r.Date,
|
||||
RequestCount = r.Count
|
||||
}).ToList());
|
||||
[Fact]
|
||||
public async Task Check_PurchasedSlots_GrowBothWindows()
|
||||
{
|
||||
// Pro home with 3 purchased slots: 1000 + 3 × 300 per account, 3000 + 3 × 900 for the organization.
|
||||
var svc = Build(UsageRepo(("20260707", 1)), edition: TenantEdition.Pro, purchasedSlotsResolver: _ => 3);
|
||||
|
||||
private static TenantEdition HomeProTargetsCommunity(string? tenantId)
|
||||
=> tenantId == TenantId ? TenantEdition.Pro : TenantEdition.Community;
|
||||
var d = await svc.CheckAsync(Oid, Upn, TenantId);
|
||||
|
||||
Assert.Equal(1900, d.DailyLimit);
|
||||
Assert.Equal(38000, d.MonthlyLimit);
|
||||
Assert.Equal(5700, d.TenantDailyLimit);
|
||||
Assert.Equal(114000, d.TenantMonthlyLimit);
|
||||
Assert.Equal(3, d.PurchasedDelegatedSlots);
|
||||
Assert.Equal(300, d.SlotDailyLimit);
|
||||
Assert.Equal(6000, d.SlotMonthlyLimit);
|
||||
Assert.Equal(900, d.SlotTenantDailyLimit);
|
||||
Assert.Equal(18000, d.SlotTenantMonthlyLimit);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Check_DelegatedTarget_ChargesTheManagedTenantsPlanAndCounters()
|
||||
public async Task Check_IncludedSlotsOnly_NoGrowth()
|
||||
{
|
||||
// Pro MSP reads a Community customer whose organization window (300/day) other readers burned.
|
||||
var repo = UsageRepo(("20260707", 5));
|
||||
TenantRows(repo, Target, ("20260707", 300));
|
||||
var svc = Build(repo, editionResolver: HomeProTargetsCommunity);
|
||||
// A Pro tenant on its two included slots (purchased = 0) keeps the plain Pro windows.
|
||||
var svc = Build(UsageRepo(("20260707", 1)), edition: TenantEdition.Pro, purchasedSlotsResolver: _ => 0);
|
||||
|
||||
var d = await svc.CheckAsync(Oid, Upn, homeTenantId: TenantId, chargeTenantId: Target);
|
||||
var d = await svc.CheckAsync(Oid, Upn, TenantId);
|
||||
|
||||
Assert.Equal(1000, d.DailyLimit);
|
||||
Assert.Equal(3000, d.TenantDailyLimit);
|
||||
Assert.Equal(0, d.PurchasedDelegatedSlots);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Check_ConferredProHome_GrowsNothing_EvenWithASlotCount()
|
||||
{
|
||||
// Conferred Pro (managed by a Pro MSP, not Pro itself): the catalog slot values are 0, so even a slot
|
||||
// count leaking through (the entitlement service gates it to 0 anyway) grows neither window.
|
||||
var conferred = new EditionResolution(TenantEdition.Pro, EditionSource.Msp, OwnPro: false);
|
||||
var svc = Build(UsageRepo(("20260707", 1)), resolutionResolver: _ => conferred, purchasedSlotsResolver: _ => 3);
|
||||
|
||||
var d = await svc.CheckAsync(Oid, Upn, TenantId);
|
||||
|
||||
Assert.False(d.Allowed);
|
||||
Assert.Equal("tenant", d.Level);
|
||||
Assert.Equal("daily", d.Scope);
|
||||
Assert.Equal("community", d.TenantPlan);
|
||||
Assert.Equal(300, d.TenantDailyLimit);
|
||||
Assert.Equal(Target, d.TargetTenantId);
|
||||
// The caller's OWN windows still come from the Pro home tenant.
|
||||
Assert.Equal("pro", d.Plan);
|
||||
Assert.Equal(1000, d.DailyLimit);
|
||||
repo.Verify(r => r.GetTenantUsageAsync(TenantId, It.IsAny<string?>(), It.IsAny<string?>()), Times.Never);
|
||||
Assert.Equal(3000, d.TenantDailyLimit);
|
||||
Assert.Equal(0, d.SlotDailyLimit);
|
||||
Assert.Equal(0, d.SlotTenantDailyLimit);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Check_DelegatedTarget_UserBudgetStillFollowsHome()
|
||||
public async Task Check_UserOverridePlan_StillGrowsWithTheHomeSlots()
|
||||
{
|
||||
// Community home (100/day) reading a Pro customer: the caller's own daily window bites first.
|
||||
var repo = UsageRepo(("20260707", 100));
|
||||
TenantRows(repo, Target, ("20260707", 1));
|
||||
var svc = Build(repo, editionResolver: t => t == Target ? TenantEdition.Pro : TenantEdition.Community);
|
||||
// The per-user "power" plan carries no slot fields → the HOME edition's catalog slot values apply,
|
||||
// so the override account grows with the tenant's purchased slots like everyone else.
|
||||
var json = """[{"name":"power","dailyRequestLimit":10000,"monthlyRequestLimit":100000,"description":""}]""";
|
||||
var svc = Build(UsageRepo(("20260707", 1)), planDefinitionsJson: json, edition: TenantEdition.Pro,
|
||||
mcpUserPlanOverride: "power", purchasedSlotsResolver: _ => 2);
|
||||
|
||||
var d = await svc.CheckAsync(Oid, Upn, homeTenantId: TenantId, chargeTenantId: Target);
|
||||
var d = await svc.CheckAsync(Oid, Upn, TenantId);
|
||||
|
||||
Assert.Equal("power", d.Plan);
|
||||
Assert.Equal(10600, d.DailyLimit);
|
||||
Assert.Equal(112000, d.MonthlyLimit);
|
||||
Assert.Equal(4800, d.TenantDailyLimit); // 3000 + 2 × 900 — the organization follows the tenant plan
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Check_SlotValues_DefinitionZero_DisablesTheGrowth()
|
||||
{
|
||||
var json = """[{"name":"pro","dailyRequestLimit":1000,"monthlyRequestLimit":20000,"slotDailyRequestLimit":0,"slotMonthlyRequestLimit":0,"slotTenantDailyRequestLimit":0,"slotTenantMonthlyRequestLimit":0,"description":""}]""";
|
||||
var svc = Build(UsageRepo(("20260707", 1)), planDefinitionsJson: json, edition: TenantEdition.Pro, purchasedSlotsResolver: _ => 5);
|
||||
|
||||
var d = await svc.CheckAsync(Oid, Upn, TenantId);
|
||||
|
||||
Assert.Equal(1000, d.DailyLimit);
|
||||
Assert.Equal(3000, d.TenantDailyLimit);
|
||||
Assert.Equal(5, d.PurchasedDelegatedSlots);
|
||||
Assert.Equal(0, d.SlotDailyLimit);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Check_SlotValues_DefinitionSet_OverridesTheCatalog()
|
||||
{
|
||||
var json = """[{"name":"pro","dailyRequestLimit":1000,"monthlyRequestLimit":20000,"slotDailyRequestLimit":50,"slotMonthlyRequestLimit":500,"slotTenantDailyRequestLimit":100,"slotTenantMonthlyRequestLimit":1000,"description":""}]""";
|
||||
var svc = Build(UsageRepo(("20260707", 1)), planDefinitionsJson: json, edition: TenantEdition.Pro, purchasedSlotsResolver: _ => 2);
|
||||
|
||||
var d = await svc.CheckAsync(Oid, Upn, TenantId);
|
||||
|
||||
Assert.Equal(1100, d.DailyLimit);
|
||||
Assert.Equal(21000, d.MonthlyLimit);
|
||||
Assert.Equal(3200, d.TenantDailyLimit);
|
||||
Assert.Equal(62000, d.TenantMonthlyLimit);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Check_SlotValues_DefinitionNull_UsesTheCatalog()
|
||||
{
|
||||
// A pre-existing definition without slot fields → the edition's catalog slot values.
|
||||
var json = """[{"name":"pro","dailyRequestLimit":1000,"monthlyRequestLimit":20000,"description":""}]""";
|
||||
var svc = Build(UsageRepo(("20260707", 1)), planDefinitionsJson: json, edition: TenantEdition.Pro, purchasedSlotsResolver: _ => 1);
|
||||
|
||||
var d = await svc.CheckAsync(Oid, Upn, TenantId);
|
||||
|
||||
Assert.Equal(1300, d.DailyLimit);
|
||||
Assert.Equal(3900, d.TenantDailyLimit);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Check_LiftedWindow_StaysUnlimited_DespiteSlots()
|
||||
{
|
||||
// 0 = unlimited must never turn into "0 + slots × value" — a limit again. 30 000 requests today would
|
||||
// exceed any grown daily window (and the plain 20 000 monthly one) but stay under 20 000 + 4 × 6 000.
|
||||
var json = """[{"name":"pro","dailyRequestLimit":0,"monthlyRequestLimit":20000,"tenantDailyRequestLimit":0,"tenantMonthlyRequestLimit":60000,"description":""}]""";
|
||||
var svc = Build(UsageRepo(("20260707", 30000)), planDefinitionsJson: json, edition: TenantEdition.Pro, purchasedSlotsResolver: _ => 4);
|
||||
|
||||
var d = await svc.CheckAsync(Oid, Upn, TenantId);
|
||||
|
||||
Assert.True(d.Allowed);
|
||||
Assert.Equal(0, d.DailyLimit);
|
||||
Assert.Equal(0, d.TenantDailyLimit);
|
||||
Assert.Equal(44000, d.MonthlyLimit);
|
||||
Assert.Equal(30000, d.MonthlyUsed);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(1000, 3, 300, 1900)]
|
||||
[InlineData(1000, 0, 300, 1000)]
|
||||
[InlineData(1000, 3, 0, 1000)]
|
||||
[InlineData(0, 3, 300, 0)]
|
||||
[InlineData(-1, 3, 300, -1)]
|
||||
public void Grow_AddsSlotsTimesValue_AndKeepsALiftedWindowLifted(int baseLimit, int slots, int perSlot, int expected)
|
||||
=> Assert.Equal(expected, McpQuotaService.Grow(baseLimit, slots, perSlot));
|
||||
|
||||
[Fact]
|
||||
public async Task Check_GrownWindow_IsTheEnforcedOne()
|
||||
{
|
||||
// Pro 1000 + 1 × 300 = 1300: the 1300th request of the day is the first blocked one.
|
||||
var below = Build(UsageRepo(("20260707", 1299)), edition: TenantEdition.Pro, purchasedSlotsResolver: _ => 1);
|
||||
Assert.True((await below.CheckAsync(Oid, Upn, TenantId)).Allowed);
|
||||
|
||||
var at = Build(UsageRepo(("20260707", 1300)), edition: TenantEdition.Pro, purchasedSlotsResolver: _ => 1);
|
||||
var d = await at.CheckAsync(Oid, Upn, TenantId);
|
||||
Assert.False(d.Allowed);
|
||||
Assert.Equal("daily", d.Scope);
|
||||
Assert.Equal("user", d.Level);
|
||||
Assert.Equal("community", d.Plan);
|
||||
Assert.Equal("pro", d.TenantPlan);
|
||||
Assert.Equal(Target, d.TargetTenantId);
|
||||
Assert.Equal(1300, d.ExceededLimit);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Check_OwnTenant_NamesNoTarget()
|
||||
{
|
||||
var svc = Build(UsageRepo(("20260707", 1)));
|
||||
var legacy = await svc.CheckAsync(Oid, Upn, TenantId);
|
||||
var explicitHome = await svc.CheckAsync(Oid, Upn, TenantId, TenantId);
|
||||
Assert.Null(legacy.TargetTenantId);
|
||||
Assert.Null(explicitHome.TargetTenantId);
|
||||
}
|
||||
// ── One tenant, many callers ─────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task Check_TenantSnapshot_IsSharedAcrossCallersOfTheSameTenant()
|
||||
public async Task Check_TenantSnapshot_IsSharedAcrossMembersOfTheSameTenant()
|
||||
{
|
||||
// Two different callers charged to the same customer read its organization counters ONCE per TTL.
|
||||
var repo = UsageRepo(("20260707", 1));
|
||||
// Two members of one tenant read its organization counters ONCE per TTL.
|
||||
var repo = UsageRepo(new[] { ("20260707", 1L) }, tenantRows: new[] { ("20260707", 10L) });
|
||||
repo.Setup(r => r.GetUsageByUserAsync(OtherOid, It.IsAny<string?>(), It.IsAny<string?>()))
|
||||
.ReturnsAsync(new List<UserUsageRecord>());
|
||||
TenantRows(repo, Target, ("20260707", 10));
|
||||
var svc = Build(repo, editionResolver: HomeProTargetsCommunity);
|
||||
|
||||
await svc.CheckAsync(Oid, Upn, TenantId, Target);
|
||||
await svc.CheckAsync(OtherOid, "bob@contoso.com", TenantId, Target);
|
||||
|
||||
repo.Verify(r => r.GetTenantUsageAsync(Target, It.IsAny<string?>(), It.IsAny<string?>()), Times.Once);
|
||||
repo.Verify(r => r.GetUsageByUserAsync(It.IsAny<string>(), It.IsAny<string?>(), It.IsAny<string?>()), Times.Exactly(2));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CheckMany_ExcludesExhaustedTenants_AdmitsTheRest()
|
||||
{
|
||||
var repo = UsageRepo(("20260707", 1));
|
||||
TenantRows(repo, Target, ("20260707", 300));
|
||||
TenantRows(repo, Target2, ("20260707", 10));
|
||||
var svc = Build(repo, editionResolver: HomeProTargetsCommunity);
|
||||
|
||||
var r = await svc.CheckManyAsync(Oid, Upn, TenantId, new[] { Target, Target2 });
|
||||
|
||||
Assert.True(r.Allowed);
|
||||
Assert.Null(r.BlockingDecision);
|
||||
Assert.True(r.UserDecision.Allowed);
|
||||
Assert.Equal(new[] { Target2 }, r.AdmittedTenantIds);
|
||||
Assert.Equal(new[] { Target }, r.ExcludedTenantIds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CheckMany_AllExhausted_BlocksWithTheEarliestReset()
|
||||
{
|
||||
var repo = UsageRepo(("20260707", 1));
|
||||
TenantRows(repo, Target, ("20260707", 300)); // daily window → resets tomorrow
|
||||
TenantRows(repo, Target2, ("20260701", 9000), ("20260707", 1)); // monthly window → resets on the 1st
|
||||
var svc = Build(repo, editionResolver: HomeProTargetsCommunity);
|
||||
|
||||
var r = await svc.CheckManyAsync(Oid, Upn, TenantId, new[] { Target, Target2 });
|
||||
|
||||
Assert.False(r.Allowed);
|
||||
Assert.Empty(r.AdmittedTenantIds);
|
||||
Assert.Equal(2, r.ExcludedTenantIds.Count);
|
||||
Assert.Equal("tenant", r.BlockingDecision!.Level);
|
||||
Assert.Equal("daily", r.BlockingDecision.Scope);
|
||||
Assert.Equal(new DateTime(2026, 7, 8, 0, 0, 0, DateTimeKind.Utc), r.BlockingDecision.ResetUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CheckMany_UserExhausted_SkipsEveryTenantRead()
|
||||
{
|
||||
var repo = UsageRepo(("20260707", 100)); // Community home: 100/day
|
||||
var svc = Build(repo);
|
||||
|
||||
var r = await svc.CheckManyAsync(Oid, Upn, TenantId, new[] { Target, Target2 });
|
||||
await svc.CheckAsync(Oid, Upn, TenantId);
|
||||
await svc.CheckAsync(OtherOid, "bob@contoso.com", TenantId);
|
||||
|
||||
Assert.False(r.Allowed);
|
||||
Assert.Equal("user", r.BlockingDecision!.Level);
|
||||
Assert.Equal(2, r.ExcludedTenantIds.Count);
|
||||
repo.Verify(r2 => r2.GetTenantUsageAsync(It.IsAny<string>(), It.IsAny<string?>(), It.IsAny<string?>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CheckMany_TenantReadFails_AdmitsThatTenant_FailOpen()
|
||||
{
|
||||
var repo = UsageRepo(("20260707", 1));
|
||||
repo.Setup(r => r.GetTenantUsageAsync(Target, It.IsAny<string?>(), It.IsAny<string?>()))
|
||||
.ThrowsAsync(new InvalidOperationException("storage down"));
|
||||
TenantRows(repo, Target2, ("20260707", 10));
|
||||
var svc = Build(repo, editionResolver: HomeProTargetsCommunity);
|
||||
|
||||
var r = await svc.CheckManyAsync(Oid, Upn, TenantId, new[] { Target, Target2 });
|
||||
|
||||
Assert.True(r.Allowed);
|
||||
Assert.Equal(new[] { Target, Target2 }, r.AdmittedTenantIds.OrderBy(t => t));
|
||||
Assert.Empty(r.ExcludedTenantIds);
|
||||
repo.Verify(r => r.GetTenantUsageAsync(TenantId, It.IsAny<string?>(), It.IsAny<string?>()), Times.Once);
|
||||
repo.Verify(r => r.GetUsageByUserAsync(It.IsAny<string>(), It.IsAny<string?>(), It.IsAny<string?>()), Times.Exactly(2));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -407,6 +407,12 @@ public class MetricsWireParityTests
|
||||
var tenantMonthlyLimit = 9000;
|
||||
long tenantDailyUsed = 120;
|
||||
long tenantMonthlyUsed = 4100;
|
||||
// Three purchased delegation slots: the limits above already grew by them; the breakdown rides along.
|
||||
int? purchasedDelegatedSlots = 3;
|
||||
int? slotDailyLimit = 300;
|
||||
int? slotMonthlyLimit = 6000;
|
||||
int? slotTenantDailyLimit = 900;
|
||||
int? slotTenantMonthlyLimit = 18000;
|
||||
var records = SampleUsageRecords(userId);
|
||||
|
||||
AssertParity(
|
||||
@@ -427,7 +433,12 @@ public class MetricsWireParityTests
|
||||
tenantDailyLimit,
|
||||
tenantMonthlyLimit,
|
||||
tenantDailyUsed,
|
||||
tenantMonthlyUsed
|
||||
tenantMonthlyUsed,
|
||||
purchasedDelegatedSlots,
|
||||
slotDailyLimit,
|
||||
slotMonthlyLimit,
|
||||
slotTenantDailyLimit,
|
||||
slotTenantMonthlyLimit
|
||||
},
|
||||
records
|
||||
},
|
||||
@@ -448,7 +459,12 @@ public class MetricsWireParityTests
|
||||
TenantDailyLimit = tenantDailyLimit,
|
||||
TenantMonthlyLimit = tenantMonthlyLimit,
|
||||
TenantDailyUsed = tenantDailyUsed,
|
||||
TenantMonthlyUsed = tenantMonthlyUsed
|
||||
TenantMonthlyUsed = tenantMonthlyUsed,
|
||||
PurchasedDelegatedSlots = purchasedDelegatedSlots,
|
||||
SlotDailyLimit = slotDailyLimit,
|
||||
SlotMonthlyLimit = slotMonthlyLimit,
|
||||
SlotTenantDailyLimit = slotTenantDailyLimit,
|
||||
SlotTenantMonthlyLimit = slotTenantMonthlyLimit
|
||||
},
|
||||
Records = records
|
||||
});
|
||||
@@ -457,10 +473,16 @@ public class MetricsWireParityTests
|
||||
[Fact]
|
||||
public void GetMyMcpUsageResponse_omits_null_upn_and_usagePlan()
|
||||
{
|
||||
// …and the slot breakdown, which is absent when no delegation slot was purchased.
|
||||
var userId = "7aa20c11-0002-4b7c-a1d2-52f3aaaa0008";
|
||||
string? upn = null;
|
||||
string? usagePlan = null;
|
||||
var effectivePlan = "free";
|
||||
int? purchasedDelegatedSlots = null;
|
||||
int? slotDailyLimit = null;
|
||||
int? slotMonthlyLimit = null;
|
||||
int? slotTenantDailyLimit = null;
|
||||
int? slotTenantMonthlyLimit = null;
|
||||
var records = new List<UserUsageRecord>();
|
||||
|
||||
AssertParity(
|
||||
@@ -481,7 +503,12 @@ public class MetricsWireParityTests
|
||||
tenantDailyLimit = 0,
|
||||
tenantMonthlyLimit = 0,
|
||||
tenantDailyUsed = 0L,
|
||||
tenantMonthlyUsed = 0L
|
||||
tenantMonthlyUsed = 0L,
|
||||
purchasedDelegatedSlots,
|
||||
slotDailyLimit,
|
||||
slotMonthlyLimit,
|
||||
slotTenantDailyLimit,
|
||||
slotTenantMonthlyLimit
|
||||
},
|
||||
records
|
||||
},
|
||||
@@ -672,7 +699,6 @@ public class MetricsWireParityTests
|
||||
{
|
||||
var tenantId = "7aa20c11-0002-4b7c-a1d2-52f3aaaa0001";
|
||||
var userId = "7aa20c11-0002-4b7c-a1d2-52f3aaaa0009";
|
||||
var homeTenantId = "7aa20c11-0002-4b7c-a1d2-52f3aaaa0002";
|
||||
var lastRequestAt = new DateTime(2026, 9, 2, 14, 5, 0, DateTimeKind.Utc);
|
||||
|
||||
AssertParity(
|
||||
@@ -686,9 +712,7 @@ public class MetricsWireParityTests
|
||||
new
|
||||
{
|
||||
userId,
|
||||
userPrincipalName = "msp@partner.example",
|
||||
delegated = true,
|
||||
homeTenantId,
|
||||
userPrincipalName = "admin@contoso.com",
|
||||
requestsToday = 4L,
|
||||
requestsThisMonth = 120L,
|
||||
requestsInRange = 120L,
|
||||
@@ -706,9 +730,7 @@ public class MetricsWireParityTests
|
||||
new()
|
||||
{
|
||||
UserId = userId,
|
||||
UserPrincipalName = "msp@partner.example",
|
||||
Delegated = true,
|
||||
HomeTenantId = homeTenantId,
|
||||
UserPrincipalName = "admin@contoso.com",
|
||||
RequestsToday = 4,
|
||||
RequestsThisMonth = 120,
|
||||
RequestsInRange = 120,
|
||||
@@ -719,13 +741,12 @@ public class MetricsWireParityTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetMcpOrganizationUsageResponse_omits_null_upn_home_and_lastRequest()
|
||||
public void GetMcpOrganizationUsageResponse_omits_null_upn_and_lastRequest()
|
||||
{
|
||||
// A row written before the attribution columns existed: own member, no UPN, no timestamp.
|
||||
// A row written before the attribution columns existed: no UPN, no timestamp.
|
||||
var tenantId = "7aa20c11-0002-4b7c-a1d2-52f3aaaa0001";
|
||||
var userId = "7aa20c11-0002-4b7c-a1d2-52f3aaaa0009";
|
||||
string? userPrincipalName = null;
|
||||
string? homeTenantId = null;
|
||||
DateTime? lastRequestAt = null;
|
||||
|
||||
AssertParity(
|
||||
@@ -740,8 +761,6 @@ public class MetricsWireParityTests
|
||||
{
|
||||
userId,
|
||||
userPrincipalName,
|
||||
delegated = false,
|
||||
homeTenantId,
|
||||
requestsToday = 0L,
|
||||
requestsThisMonth = 3L,
|
||||
requestsInRange = 3L,
|
||||
@@ -756,11 +775,63 @@ public class MetricsWireParityTests
|
||||
DateTo = "20260902",
|
||||
Users = new List<McpOrganizationUsageItem>
|
||||
{
|
||||
new() { UserId = userId, Delegated = false, RequestsToday = 0, RequestsThisMonth = 3, RequestsInRange = 3 },
|
||||
new() { UserId = userId, RequestsToday = 0, RequestsThisMonth = 3, RequestsInRange = 3 },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetMcpOrganizationUsageResponse_quota_carries_the_slot_breakdown()
|
||||
{
|
||||
// Three purchased slots: 3 000 + 3 × 900 arrive as the grown limit; the breakdown lets the page show the sum.
|
||||
var tenantId = "7aa20c11-0002-4b7c-a1d2-52f3aaaa0001";
|
||||
var daily = new List<McpOrganizationDailyItem>
|
||||
{
|
||||
new() { Date = "20260901", Requests = 12 },
|
||||
new() { Date = "20260902", Requests = 7 },
|
||||
};
|
||||
|
||||
AssertParity(
|
||||
new
|
||||
{
|
||||
tenantId,
|
||||
dateFrom = "20260901",
|
||||
dateTo = "20260902",
|
||||
users = Array.Empty<object>(),
|
||||
quota = new
|
||||
{
|
||||
tenantPlan = "pro",
|
||||
dailyLimit = 5700,
|
||||
monthlyLimit = 114000,
|
||||
dailyUsed = 7L,
|
||||
monthlyUsed = 19L,
|
||||
purchasedDelegatedSlots = 3,
|
||||
slotDailyLimit = 900,
|
||||
slotMonthlyLimit = 18000,
|
||||
},
|
||||
daily,
|
||||
},
|
||||
new GetMcpOrganizationUsageResponse
|
||||
{
|
||||
TenantId = tenantId,
|
||||
DateFrom = "20260901",
|
||||
DateTo = "20260902",
|
||||
Users = new List<McpOrganizationUsageItem>(),
|
||||
Quota = new McpOrganizationQuotaNode
|
||||
{
|
||||
TenantPlan = "pro",
|
||||
DailyLimit = 5700,
|
||||
MonthlyLimit = 114000,
|
||||
DailyUsed = 7,
|
||||
MonthlyUsed = 19,
|
||||
PurchasedDelegatedSlots = 3,
|
||||
SlotDailyLimit = 900,
|
||||
SlotMonthlyLimit = 18000,
|
||||
},
|
||||
Daily = daily,
|
||||
});
|
||||
}
|
||||
|
||||
private static void AssertParity(object anonymousLiteral, IApiResponse typed)
|
||||
=> ApiResponseWireParityTests.AssertWireIdentical(anonymousLiteral, typed);
|
||||
}
|
||||
|
||||
+1
-1
@@ -414,7 +414,7 @@ public class OffboardingMarkerCleanupFunctionTests
|
||||
{
|
||||
}
|
||||
|
||||
public override Task<int> WipeByExactPartitionAsync(string tableName, string normalizedTenantId, CancellationToken ct = default)
|
||||
protected override Task<int> WipeByExactPartitionCoreAsync(string tableName, string normalizedTenantId, string filter, CancellationToken ct)
|
||||
{
|
||||
WipeCalls.Add((tableName, normalizedTenantId));
|
||||
if (ThrowOnNextWipe is { } ex)
|
||||
|
||||
+9
-9
@@ -593,9 +593,9 @@ public class TenantOffboardingHandlerDrainContractTests
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="SafeWipeService"/> that records call counts without performing any storage IO.
|
||||
/// Tests use this to assert that fail-closed paths NEVER touch the wipe methods. Now that
|
||||
/// <see cref="SafeWipeService"/> exposes the wipe methods as <c>virtual</c>, the production
|
||||
/// dispatch lands here.
|
||||
/// Tests use this to assert that fail-closed paths NEVER touch the wipe methods. Only the
|
||||
/// protected cores are overridden, so the tenant-GUID guard of the public entry points still
|
||||
/// runs against whatever the handler passes in.
|
||||
/// </summary>
|
||||
private sealed class CountingSafeWipeService : SafeWipeService
|
||||
{
|
||||
@@ -610,37 +610,37 @@ public class TenantOffboardingHandlerDrainContractTests
|
||||
{
|
||||
}
|
||||
|
||||
public override Task<int> WipeByExactPartitionAsync(string tableName, string normalizedTenantId, CancellationToken ct = default)
|
||||
protected override Task<int> WipeByExactPartitionCoreAsync(string tableName, string normalizedTenantId, string filter, CancellationToken ct)
|
||||
{
|
||||
WipeCallCount++;
|
||||
return Task.FromResult(0);
|
||||
}
|
||||
|
||||
public override Task<int> WipeByCompositePartitionRangeAsync(string tableName, string normalizedTenantId, CancellationToken ct = default)
|
||||
protected override Task<int> WipeByCompositePartitionRangeCoreAsync(string tableName, string normalizedTenantId, CancellationToken ct)
|
||||
{
|
||||
WipeCallCount++;
|
||||
return Task.FromResult(0);
|
||||
}
|
||||
|
||||
public override Task<int> WipeByDiscriminatorAndTenantPropertyAsync(string tableName, string discriminator, string normalizedTenantId, CancellationToken ct = default)
|
||||
protected override Task<int> WipeByDiscriminatorAndTenantPropertyCoreAsync(string tableName, string discriminator, string normalizedTenantId, CancellationToken ct)
|
||||
{
|
||||
WipeCallCount++;
|
||||
return Task.FromResult(0);
|
||||
}
|
||||
|
||||
public override Task<int> WipeByTenantIdPropertyAsync(string tableName, string normalizedTenantId, CancellationToken ct = default)
|
||||
protected override Task<int> WipeByTenantIdPropertyCoreAsync(string tableName, string normalizedTenantId, CancellationToken ct)
|
||||
{
|
||||
WipeCallCount++;
|
||||
return Task.FromResult(0);
|
||||
}
|
||||
|
||||
public override Task<int> WipeByRowKeyAsync(string tableName, string normalizedTenantId, CancellationToken ct = default)
|
||||
protected override Task<int> WipeByRowKeyCoreAsync(string tableName, string normalizedTenantId, string filter, CancellationToken ct)
|
||||
{
|
||||
WipeCallCount++;
|
||||
return Task.FromResult(0);
|
||||
}
|
||||
|
||||
public override Task<int> WipeBlobsByTenantPrefixAsync(string containerName, string normalizedTenantId, CancellationToken ct = default)
|
||||
protected override Task<int> WipeBlobsByTenantPrefixCoreAsync(string containerName, string normalizedTenantId, CancellationToken ct)
|
||||
{
|
||||
WipeCallCount++;
|
||||
return Task.FromResult(0);
|
||||
|
||||
+26
-6
@@ -320,6 +320,22 @@ public class TenantOffboardingHandlerOrderingTests
|
||||
Assert.Contains(Constants.TableNames.ScriptNameCache, harness.SafeWipeProbe.ExactPartitionWipes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PostDrain_OwnedGroupWipe_AnchorsOnSelfServiceGroupId()
|
||||
{
|
||||
// The tenant's own self-service group lives under "msp-{tenantId}" — not a GUID. The wipe
|
||||
// must go through the owned-group entry points (tenant GUID in, group id derived inside);
|
||||
// handing the group id to a tenant-GUID entry point throws and poisons every offboarding.
|
||||
var harness = Harness.New();
|
||||
var ownedGroupId = Constants.TenantGroupIds.ForHomeTenant(TenantId);
|
||||
|
||||
await harness.Sut.HandleAsync(harness.Envelope());
|
||||
|
||||
Assert.Contains((Constants.TableNames.TenantGroups, ownedGroupId), harness.SafeWipeProbe.ExactPartitionAnchors);
|
||||
Assert.Contains((Constants.TableNames.TenantGroupAssignments, ownedGroupId), harness.SafeWipeProbe.RowKeyAnchors);
|
||||
Assert.Equal("Completed", harness.Repo.History[HistoryRowKey].Status);
|
||||
}
|
||||
|
||||
// ── Harness (copied minimal — only what these tests need) ───────────────────
|
||||
|
||||
private sealed class Harness
|
||||
@@ -470,12 +486,16 @@ public class TenantOffboardingHandlerOrderingTests
|
||||
new BlobStorageService(new BlobServiceClient("UseDevelopmentStorage=true"),
|
||||
NullLogger<BlobStorageService>.Instance, usesManagedIdentity: false),
|
||||
NullLogger<SafeWipeService>.Instance) { }
|
||||
public override Task<int> WipeByExactPartitionAsync(string t, string i, CancellationToken c = default) { WipeCallCount++; ExactPartitionWipes.Add(t); return Task.FromResult(0); }
|
||||
public override Task<int> WipeByCompositePartitionRangeAsync(string t, string i, CancellationToken c = default) { WipeCallCount++; return Task.FromResult(0); }
|
||||
public override Task<int> WipeByDiscriminatorAndTenantPropertyAsync(string t, string d, string i, CancellationToken c = default) { WipeCallCount++; return Task.FromResult(0); }
|
||||
public override Task<int> WipeByTenantIdPropertyAsync(string t, string i, CancellationToken c = default) { WipeCallCount++; PropertyOnlyWipes.Add(t); return Task.FromResult(0); }
|
||||
public override Task<int> WipeByRowKeyAsync(string t, string i, CancellationToken c = default) { WipeCallCount++; RowKeyWipes.Add(t); return Task.FromResult(0); }
|
||||
public override Task<int> WipeBlobsByTenantPrefixAsync(string c, string i, CancellationToken ct = default) { WipeCallCount++; return Task.FromResult(0); }
|
||||
/// <summary>(table, partition key) pairs of the exact-partition wipe — the anchor the delete would run against.</summary>
|
||||
public List<(string Table, string Anchor)> ExactPartitionAnchors { get; } = new();
|
||||
/// <summary>(table, row key) pairs of the RowKey-anchored wipe.</summary>
|
||||
public List<(string Table, string Anchor)> RowKeyAnchors { get; } = new();
|
||||
protected override Task<int> WipeByExactPartitionCoreAsync(string t, string pk, string filter, CancellationToken c) { WipeCallCount++; ExactPartitionWipes.Add(t); ExactPartitionAnchors.Add((t, pk)); return Task.FromResult(0); }
|
||||
protected override Task<int> WipeByCompositePartitionRangeCoreAsync(string t, string i, CancellationToken c) { WipeCallCount++; return Task.FromResult(0); }
|
||||
protected override Task<int> WipeByDiscriminatorAndTenantPropertyCoreAsync(string t, string d, string i, CancellationToken c) { WipeCallCount++; return Task.FromResult(0); }
|
||||
protected override Task<int> WipeByTenantIdPropertyCoreAsync(string t, string i, CancellationToken c) { WipeCallCount++; PropertyOnlyWipes.Add(t); return Task.FromResult(0); }
|
||||
protected override Task<int> WipeByRowKeyCoreAsync(string t, string rk, string filter, CancellationToken c) { WipeCallCount++; RowKeyWipes.Add(t); RowKeyAnchors.Add((t, rk)); return Task.FromResult(0); }
|
||||
protected override Task<int> WipeBlobsByTenantPrefixCoreAsync(string c, string i, CancellationToken ct) { WipeCallCount++; return Task.FromResult(0); }
|
||||
}
|
||||
|
||||
private sealed class NoopCustomsArchivingOrderingHandler : TenantOffboardingHandler
|
||||
|
||||
+130
-19
@@ -111,6 +111,71 @@ public class TenantOffboardingWorkerPoisonTests
|
||||
Assert.Equal("InProgress", harness.Repo.History[HistoryRowKey].Status);
|
||||
}
|
||||
|
||||
// ── Superseded dead letters ─────────────────────────────────────────────────
|
||||
//
|
||||
// A retry after a poisoned run is a new envelope, so nothing else ever removes the dead
|
||||
// letter of the failed run. The completed run sweeps it.
|
||||
|
||||
private const string OtherTenantId = "22222222-2222-2222-2222-222222222222";
|
||||
|
||||
[Fact]
|
||||
public async Task Completed_RemovesOnlyTheTenantsDeadLetters()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.Repo.History[HistoryRowKey].Status = "Completed";
|
||||
var own = harness.AddDeadLetter(TenantId.ToUpperInvariant()); // tenant ids compare case-insensitively
|
||||
var foreign = harness.AddDeadLetter(OtherTenantId);
|
||||
var malformed = harness.AddDeadLetter(tenantId: null, bodyOverride: "{ not json at all");
|
||||
|
||||
await harness.InvokeHandleAsync();
|
||||
|
||||
Assert.Equal(new[] { own }, harness.PoisonDeletedIds);
|
||||
Assert.DoesNotContain(foreign, harness.PoisonDeletedIds);
|
||||
Assert.DoesNotContain(malformed, harness.PoisonDeletedIds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Completed_SweepsPastTheFirstBatch()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.Repo.History[HistoryRowKey].Status = "Completed";
|
||||
for (var i = 0; i < TenantOffboardingWorker.PoisonSweepBatchSize; i++)
|
||||
harness.AddDeadLetter(OtherTenantId);
|
||||
var own = harness.AddDeadLetter(TenantId);
|
||||
|
||||
await harness.InvokeHandleAsync();
|
||||
|
||||
Assert.Equal(new[] { own }, harness.PoisonDeletedIds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task NotCompleted_LeavesThePoisonQueueAlone()
|
||||
{
|
||||
// A pickup that finds the run Failed must not touch the dead letter — it is the only
|
||||
// queue-side trace of a failure nobody has resolved yet.
|
||||
var harness = new Harness();
|
||||
harness.Repo.History[HistoryRowKey].Status = "Failed";
|
||||
harness.AddDeadLetter(TenantId);
|
||||
|
||||
await harness.InvokeHandleAsync();
|
||||
|
||||
Assert.Equal(0, harness.PoisonReceiveCount);
|
||||
Assert.Empty(harness.PoisonDeletedIds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Completed_SweepFailure_DoesNotFailTheRun()
|
||||
{
|
||||
// A throw here would keep the completed envelope on the main queue and finally move a
|
||||
// Completed run to poison.
|
||||
var harness = new Harness(poisonReceiveThrows: true);
|
||||
harness.Repo.History[HistoryRowKey].Status = "Completed";
|
||||
|
||||
await harness.InvokeHandleAsync();
|
||||
|
||||
Assert.Equal(1, harness.PoisonReceiveCount);
|
||||
}
|
||||
|
||||
// ── Harness ─────────────────────────────────────────────────────────────────
|
||||
|
||||
private sealed class Harness
|
||||
@@ -121,11 +186,38 @@ public class TenantOffboardingWorkerPoisonTests
|
||||
public Mock<QueueClient> PoisonQueue { get; } = new();
|
||||
public int PoisonSendCount { get; private set; }
|
||||
public int MainDeleteCount { get; private set; }
|
||||
public int PoisonReceiveCount { get; private set; }
|
||||
public List<string> PoisonDeletedIds { get; } = new();
|
||||
|
||||
public Harness(bool poisonSendThrows = false)
|
||||
// Poison-queue content for the sweep tests. A received message turns invisible, like the
|
||||
// real queue, so a second receive returns the next batch instead of the same one.
|
||||
private readonly List<QueueMessage> _deadLetters = new();
|
||||
private readonly HashSet<string> _hiddenDeadLetters = new();
|
||||
|
||||
public Harness(bool poisonSendThrows = false, bool poisonReceiveThrows = false)
|
||||
{
|
||||
SeedAudit();
|
||||
|
||||
PoisonQueue.Setup(q => q.ReceiveMessagesAsync(It.IsAny<int?>(), It.IsAny<TimeSpan?>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<int?, TimeSpan?, CancellationToken>((max, _, _) =>
|
||||
{
|
||||
PoisonReceiveCount++;
|
||||
if (poisonReceiveThrows) throw new InvalidOperationException("simulated poison-queue 503");
|
||||
var batch = _deadLetters
|
||||
.Where(m => !_hiddenDeadLetters.Contains(m.MessageId))
|
||||
.Take(max ?? 1)
|
||||
.ToArray();
|
||||
foreach (var m in batch) _hiddenDeadLetters.Add(m.MessageId);
|
||||
return Task.FromResult(Response.FromValue(batch, new Mock<Response>().Object));
|
||||
});
|
||||
|
||||
PoisonQueue.Setup(q => q.DeleteMessageAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<string, string, CancellationToken>((id, _, _) =>
|
||||
{
|
||||
PoisonDeletedIds.Add(id);
|
||||
return Task.FromResult(new Mock<Response>().Object);
|
||||
});
|
||||
|
||||
// Cascade-handler dependencies. The handler is real — it talks to FakeRepo +
|
||||
// OpsEvent + (unused) other deps. Only MarkEnvelopeFailedFromPoisonAsync is
|
||||
// exercised, so most deps are noise.
|
||||
@@ -160,24 +252,43 @@ public class TenantOffboardingWorkerPoisonTests
|
||||
}
|
||||
|
||||
public QueueMessage BuildMessage(int dequeueCount, string? bodyOverride = null)
|
||||
{
|
||||
var envelope = new TenantOffboardingEnvelope
|
||||
{
|
||||
TenantId = TenantId,
|
||||
HistoryPartitionKey = Constants.OffboardingPartitionKeys.History,
|
||||
HistoryRowKey = HistoryRowKey,
|
||||
InitiatedBy = "alice@contoso.invalid",
|
||||
InitiatedAt = DateTime.UtcNow.AddMinutes(-1),
|
||||
EnqueuedAt = DateTime.UtcNow,
|
||||
DrainPollCount = 0,
|
||||
};
|
||||
return QueuesModelFactory.QueueMessage(
|
||||
=> QueuesModelFactory.QueueMessage(
|
||||
messageId: Guid.NewGuid().ToString(),
|
||||
popReceipt: "fake-receipt",
|
||||
body: new BinaryData(bodyOverride ?? JsonConvert.SerializeObject(envelope)),
|
||||
body: new BinaryData(bodyOverride ?? JsonConvert.SerializeObject(BuildEnvelope(TenantId))),
|
||||
dequeueCount: dequeueCount);
|
||||
|
||||
/// <summary>Puts a dead letter into the poison queue and returns its message id.</summary>
|
||||
public string AddDeadLetter(string? tenantId, string? bodyOverride = null)
|
||||
{
|
||||
var msg = QueuesModelFactory.QueueMessage(
|
||||
messageId: Guid.NewGuid().ToString(),
|
||||
popReceipt: "fake-receipt",
|
||||
body: new BinaryData(bodyOverride ?? JsonConvert.SerializeObject(BuildEnvelope(tenantId!))),
|
||||
dequeueCount: 1);
|
||||
_deadLetters.Add(msg);
|
||||
return msg.MessageId;
|
||||
}
|
||||
|
||||
public Task InvokeHandleAsync()
|
||||
{
|
||||
var method = typeof(TenantOffboardingWorker)
|
||||
.GetMethod("HandleAsync", BindingFlags.NonPublic | BindingFlags.Instance);
|
||||
Assert.NotNull(method);
|
||||
return (Task)method!.Invoke(Worker, new object[] { BuildEnvelope(TenantId), default(CancellationToken) })!;
|
||||
}
|
||||
|
||||
private static TenantOffboardingEnvelope BuildEnvelope(string tenantId) => new()
|
||||
{
|
||||
TenantId = tenantId,
|
||||
HistoryPartitionKey = Constants.OffboardingPartitionKeys.History,
|
||||
HistoryRowKey = HistoryRowKey,
|
||||
InitiatedBy = "alice@contoso.invalid",
|
||||
InitiatedAt = DateTime.UtcNow.AddMinutes(-1),
|
||||
EnqueuedAt = DateTime.UtcNow,
|
||||
DrainPollCount = 0,
|
||||
};
|
||||
|
||||
public Task InvokeMoveToPoisonAsync(QueueMessage msg)
|
||||
{
|
||||
// MoveToPoisonAsync is private — invoke via reflection. Avoids spinning up the
|
||||
@@ -288,10 +399,10 @@ public class TenantOffboardingWorkerPoisonTests
|
||||
new BlobStorageService(new BlobServiceClient("UseDevelopmentStorage=true"),
|
||||
NullLogger<BlobStorageService>.Instance, usesManagedIdentity: false),
|
||||
NullLogger<SafeWipeService>.Instance) { }
|
||||
public override Task<int> WipeByExactPartitionAsync(string t, string i, CancellationToken c = default) => Task.FromResult(0);
|
||||
public override Task<int> WipeByCompositePartitionRangeAsync(string t, string i, CancellationToken c = default) => Task.FromResult(0);
|
||||
public override Task<int> WipeByDiscriminatorAndTenantPropertyAsync(string t, string d, string i, CancellationToken c = default) => Task.FromResult(0);
|
||||
public override Task<int> WipeByTenantIdPropertyAsync(string t, string i, CancellationToken c = default) => Task.FromResult(0);
|
||||
public override Task<int> WipeBlobsByTenantPrefixAsync(string c, string i, CancellationToken ct = default) => Task.FromResult(0);
|
||||
protected override Task<int> WipeByExactPartitionCoreAsync(string t, string i, string filter, CancellationToken c) => Task.FromResult(0);
|
||||
protected override Task<int> WipeByCompositePartitionRangeCoreAsync(string t, string i, CancellationToken c) => Task.FromResult(0);
|
||||
protected override Task<int> WipeByDiscriminatorAndTenantPropertyCoreAsync(string t, string d, string i, CancellationToken c) => Task.FromResult(0);
|
||||
protected override Task<int> WipeByTenantIdPropertyCoreAsync(string t, string i, CancellationToken c) => Task.FromResult(0);
|
||||
protected override Task<int> WipeBlobsByTenantPrefixCoreAsync(string c, string i, CancellationToken ct) => Task.FromResult(0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -116,6 +116,23 @@ public class OffboardingFiltersTests
|
||||
Assert.DoesNotContain("PartitionKey", filter);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OwnedGroupFilters_AnchorOnTheDerivedSelfServiceGroupId()
|
||||
{
|
||||
Assert.Equal($"PartitionKey eq 'msp-{ValidTenant}'", OffboardingFilters.OwnedGroupPartition(ValidTenant));
|
||||
Assert.Equal($"RowKey eq 'msp-{ValidTenant}'", OffboardingFilters.OwnedGroupRowKey(ValidTenant));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("not-a-guid")]
|
||||
[InlineData("msp-11111111-1111-1111-1111-111111111111")] // the group id itself is not an input
|
||||
[InlineData("")]
|
||||
public void OwnedGroupFilters_ThrowForNonGuid(string input)
|
||||
{
|
||||
Assert.Throws<ArgumentException>(() => OffboardingFilters.OwnedGroupPartition(input));
|
||||
Assert.Throws<ArgumentException>(() => OffboardingFilters.OwnedGroupRowKey(input));
|
||||
}
|
||||
|
||||
// ── Injection neutralization (defense in depth even after EnsureValidGuid) ──
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -1,129 +0,0 @@
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Pins the "since release" semantics of the platform-stats recompute: every cumulative
|
||||
/// counter is a monotonic high-water-mark. The tables the recompute scans are pruned by
|
||||
/// retention, so a raw recompute only sees the retention window — persisting that verbatim
|
||||
/// regressed the public landing-page figures after every cleanup.
|
||||
/// </summary>
|
||||
public class PlatformStatsMonotonicTests
|
||||
{
|
||||
private static readonly DateTime Now = new(2026, 07, 11, 12, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
[Fact]
|
||||
public void Cumulative_counters_never_regress_below_persisted_values()
|
||||
{
|
||||
// Retention pruned old sessions: the recompute sees far less than the persisted
|
||||
// counters accumulated. Every cumulative figure must keep its high-water-mark.
|
||||
var existing = new PlatformStats
|
||||
{
|
||||
TotalEnrollments = 50_000,
|
||||
SuccessfulEnrollments = 40_000,
|
||||
TotalEventsProcessed = 9_000_000,
|
||||
TotalUsers = 800,
|
||||
TotalTenants = 120,
|
||||
UniqueDeviceModels = 450,
|
||||
IssuesDetected = 6_500,
|
||||
TotalSignedUpTenants = 150
|
||||
};
|
||||
|
||||
var stats = MaintenanceService.BuildMonotonicPlatformStats(
|
||||
recomputedEnrollments: 7_000,
|
||||
recomputedSuccessful: 6_000,
|
||||
recomputedEvents: 1_200_000,
|
||||
recomputedUsers: 90,
|
||||
recomputedActiveTenants: 80,
|
||||
recomputedUniqueModels: 200,
|
||||
signedUpTenants: 155,
|
||||
existing: existing,
|
||||
nowUtc: Now);
|
||||
|
||||
Assert.Equal(50_000, stats.TotalEnrollments);
|
||||
Assert.Equal(40_000, stats.SuccessfulEnrollments);
|
||||
Assert.Equal(9_000_000, stats.TotalEventsProcessed);
|
||||
Assert.Equal(800, stats.TotalUsers);
|
||||
Assert.Equal(120, stats.TotalTenants);
|
||||
Assert.Equal(450, stats.UniqueDeviceModels);
|
||||
Assert.Equal(6_500, stats.IssuesDetected);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Recompute_raises_counters_when_live_data_exceeds_persisted_values()
|
||||
{
|
||||
// Self-heal: lost increments (fire-and-forget failures, pre-feature history) are
|
||||
// recovered as long as the sessions are still within retention.
|
||||
var existing = new PlatformStats
|
||||
{
|
||||
TotalEnrollments = 1_000,
|
||||
SuccessfulEnrollments = 700,
|
||||
TotalEventsProcessed = 100_000,
|
||||
TotalUsers = 50,
|
||||
TotalTenants = 10,
|
||||
UniqueDeviceModels = 30
|
||||
};
|
||||
|
||||
var stats = MaintenanceService.BuildMonotonicPlatformStats(
|
||||
recomputedEnrollments: 1_500,
|
||||
recomputedSuccessful: 1_100,
|
||||
recomputedEvents: 250_000,
|
||||
recomputedUsers: 80,
|
||||
recomputedActiveTenants: 14,
|
||||
recomputedUniqueModels: 42,
|
||||
signedUpTenants: 20,
|
||||
existing: existing,
|
||||
nowUtc: Now);
|
||||
|
||||
Assert.Equal(1_500, stats.TotalEnrollments);
|
||||
Assert.Equal(1_100, stats.SuccessfulEnrollments);
|
||||
Assert.Equal(250_000, stats.TotalEventsProcessed);
|
||||
Assert.Equal(80, stats.TotalUsers);
|
||||
Assert.Equal(14, stats.TotalTenants);
|
||||
Assert.Equal(42, stats.UniqueDeviceModels);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SignedUpTenants_is_current_state_and_may_drop()
|
||||
{
|
||||
// Deliberate exception: TenantConfiguration is not retention-pruned, so its count is
|
||||
// authoritative — a drop reflects real offboarding, not data loss.
|
||||
var existing = new PlatformStats { TotalSignedUpTenants = 150 };
|
||||
|
||||
var stats = MaintenanceService.BuildMonotonicPlatformStats(
|
||||
recomputedEnrollments: 0, recomputedSuccessful: 0, recomputedEvents: 0,
|
||||
recomputedUsers: 0, recomputedActiveTenants: 0, recomputedUniqueModels: 0,
|
||||
signedUpTenants: 140,
|
||||
existing: existing,
|
||||
nowUtc: Now);
|
||||
|
||||
Assert.Equal(140, stats.TotalSignedUpTenants);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void First_run_without_persisted_row_uses_recomputed_values()
|
||||
{
|
||||
var stats = MaintenanceService.BuildMonotonicPlatformStats(
|
||||
recomputedEnrollments: 10,
|
||||
recomputedSuccessful: 8,
|
||||
recomputedEvents: 500,
|
||||
recomputedUsers: 3,
|
||||
recomputedActiveTenants: 2,
|
||||
recomputedUniqueModels: 4,
|
||||
signedUpTenants: 5,
|
||||
existing: null,
|
||||
nowUtc: Now);
|
||||
|
||||
Assert.Equal(10, stats.TotalEnrollments);
|
||||
Assert.Equal(8, stats.SuccessfulEnrollments);
|
||||
Assert.Equal(500, stats.TotalEventsProcessed);
|
||||
Assert.Equal(3, stats.TotalUsers);
|
||||
Assert.Equal(2, stats.TotalTenants);
|
||||
Assert.Equal(4, stats.UniqueDeviceModels);
|
||||
Assert.Equal(0, stats.IssuesDetected);
|
||||
Assert.Equal(5, stats.TotalSignedUpTenants);
|
||||
Assert.Equal(Now, stats.LastFullCompute);
|
||||
Assert.Equal(Now, stats.LastUpdated);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Pins the "since release" semantics of the platform-stats rollup: a cumulative counter is
|
||||
/// the persisted total plus the growth of its source since the previous run. A total derived
|
||||
/// from an absolute figure stops moving for good once that figure shrinks below it — retention
|
||||
/// prunes sessions, offboarding deletes a tenant's counters — which froze the public
|
||||
/// landing-page figures while enrollments kept arriving.
|
||||
/// </summary>
|
||||
public class PlatformStatsRollupTests
|
||||
{
|
||||
private static readonly DateTime Now = new(2026, 07, 11, 12, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private static PlatformRollupSources Sources(long enrollments, long successful = 0, long events = 0, long tenants = 0, long models = 0)
|
||||
=> new()
|
||||
{
|
||||
TotalEnrollments = enrollments,
|
||||
SuccessfulEnrollments = successful,
|
||||
TotalEventsProcessed = events,
|
||||
ActiveTenants = tenants,
|
||||
SeenDeviceModels = models
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void Counters_grow_by_the_source_delta_although_the_source_is_far_below_the_total()
|
||||
{
|
||||
// The freeze scenario: the persisted total (19 312) is above anything the source can
|
||||
// show, yet 40 enrollments arrived since the last run. Max(source, total) never moves.
|
||||
var existing = new PlatformStats
|
||||
{
|
||||
TotalEnrollments = 19_312,
|
||||
SuccessfulEnrollments = 10_289,
|
||||
TotalEventsProcessed = 5_779_212,
|
||||
TotalTenants = 84,
|
||||
UniqueDeviceModels = 486
|
||||
};
|
||||
|
||||
var stats = MaintenanceService.BuildPlatformStatsRollup(
|
||||
existing,
|
||||
baseline: Sources(14_540, successful: 10_240, events: 4_000_000, tenants: 80, models: 430),
|
||||
sources: Sources(14_580, successful: 10_275, events: 4_012_000, tenants: 81, models: 433),
|
||||
recomputedUsers: 0, signedUpTenants: 0, nowUtc: Now);
|
||||
|
||||
Assert.Equal(19_352, stats.TotalEnrollments);
|
||||
Assert.Equal(10_324, stats.SuccessfulEnrollments);
|
||||
Assert.Equal(5_791_212, stats.TotalEventsProcessed);
|
||||
Assert.Equal(85, stats.TotalTenants);
|
||||
Assert.Equal(489, stats.UniqueDeviceModels);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_shrunken_source_never_lowers_the_total_and_the_next_run_counts_again()
|
||||
{
|
||||
// Offboarding deleted a tenant's counters (−3 000) while 20 enrollments arrived: the
|
||||
// delta of this interval is clamped, the baseline follows, the next interval is exact.
|
||||
var existing = new PlatformStats { TotalEnrollments = 50_000 };
|
||||
var afterOffboarding = Sources(37_020);
|
||||
|
||||
var clamped = MaintenanceService.BuildPlatformStatsRollup(
|
||||
existing, baseline: Sources(40_000), sources: afterOffboarding,
|
||||
recomputedUsers: 0, signedUpTenants: 0, nowUtc: Now);
|
||||
Assert.Equal(50_000, clamped.TotalEnrollments);
|
||||
|
||||
var next = MaintenanceService.BuildPlatformStatsRollup(
|
||||
clamped, baseline: afterOffboarding, sources: Sources(37_055),
|
||||
recomputedUsers: 0, signedUpTenants: 0, nowUtc: Now);
|
||||
Assert.Equal(50_035, next.TotalEnrollments);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void First_run_without_a_baseline_treats_the_source_as_a_lower_bound_only()
|
||||
{
|
||||
// No baseline yet: whatever the sources show is already contained in the persisted
|
||||
// totals (or exceeds them and heals them) — it must never be added on top.
|
||||
var existing = new PlatformStats { TotalEnrollments = 19_312, SuccessfulEnrollments = 10_289 };
|
||||
|
||||
var stats = MaintenanceService.BuildPlatformStatsRollup(
|
||||
existing, baseline: null, sources: Sources(21_687, successful: 10_275),
|
||||
recomputedUsers: 0, signedUpTenants: 0, nowUtc: Now);
|
||||
|
||||
Assert.Equal(21_687, stats.TotalEnrollments);
|
||||
Assert.Equal(10_289, stats.SuccessfulEnrollments);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void First_run_without_a_persisted_row_starts_from_the_sources()
|
||||
{
|
||||
var stats = MaintenanceService.BuildPlatformStatsRollup(
|
||||
existing: null, baseline: null,
|
||||
sources: Sources(10, successful: 8, events: 500, tenants: 2, models: 4),
|
||||
recomputedUsers: 3, signedUpTenants: 5, nowUtc: Now);
|
||||
|
||||
Assert.Equal(10, stats.TotalEnrollments);
|
||||
Assert.Equal(8, stats.SuccessfulEnrollments);
|
||||
Assert.Equal(500, stats.TotalEventsProcessed);
|
||||
Assert.Equal(3, stats.TotalUsers);
|
||||
Assert.Equal(2, stats.TotalTenants);
|
||||
Assert.Equal(4, stats.UniqueDeviceModels);
|
||||
Assert.Equal(0, stats.IssuesDetected);
|
||||
Assert.Equal(5, stats.TotalSignedUpTenants);
|
||||
Assert.Equal(Now, stats.LastFullCompute);
|
||||
Assert.Equal(Now, stats.LastUpdated);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Users_stay_a_high_water_mark_and_issues_are_carried_over()
|
||||
{
|
||||
var existing = new PlatformStats { TotalUsers = 800, IssuesDetected = 6_500 };
|
||||
|
||||
var pruned = MaintenanceService.BuildPlatformStatsRollup(
|
||||
existing, baseline: Sources(0), sources: Sources(0),
|
||||
recomputedUsers: 90, signedUpTenants: 0, nowUtc: Now);
|
||||
Assert.Equal(800, pruned.TotalUsers);
|
||||
Assert.Equal(6_500, pruned.IssuesDetected);
|
||||
|
||||
var grown = MaintenanceService.BuildPlatformStatsRollup(
|
||||
existing, baseline: Sources(0), sources: Sources(0),
|
||||
recomputedUsers: 950, signedUpTenants: 0, nowUtc: Now);
|
||||
Assert.Equal(950, grown.TotalUsers);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SignedUpTenants_is_current_state_and_may_drop()
|
||||
{
|
||||
// Deliberate exception: TenantConfiguration is not retention-pruned, so its count is
|
||||
// authoritative — a drop reflects real offboarding, not data loss.
|
||||
var existing = new PlatformStats { TotalSignedUpTenants = 150 };
|
||||
|
||||
var stats = MaintenanceService.BuildPlatformStatsRollup(
|
||||
existing, baseline: Sources(0), sources: Sources(0),
|
||||
recomputedUsers: 0, signedUpTenants: 140, nowUtc: Now);
|
||||
|
||||
Assert.Equal(140, stats.TotalSignedUpTenants);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure;
|
||||
using Azure.Data.Tables;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Storage side of the platform rollup: the platform row is merged with the ETag it was read
|
||||
/// with and never carries IssuesDetected, the tenant floors land in one write, and the reads
|
||||
/// that feed the rollup baseline are scoped to exactly the rows they mean.
|
||||
/// </summary>
|
||||
public class PlatformStatsStorageTests
|
||||
{
|
||||
private const string Tenant = "11111111-1111-1111-1111-111111111111";
|
||||
|
||||
private sealed class Harness
|
||||
{
|
||||
public Mock<TableClient> Table { get; } = new();
|
||||
public List<string> Filters { get; } = new();
|
||||
public List<List<TableTransactionAction>> Submitted { get; } = new();
|
||||
public Func<string, TableEntity[]> RowsFor { get; set; } = _ => Array.Empty<TableEntity>();
|
||||
public Queue<Exception> SubmitFailures { get; } = new();
|
||||
public TableStorageService Sut { get; }
|
||||
|
||||
public Harness()
|
||||
{
|
||||
Table.Setup(c => c.QueryAsync<TableEntity>(It.IsAny<string>(), It.IsAny<int?>(), It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.Returns((string filter, int? _, IEnumerable<string> _, CancellationToken _) =>
|
||||
{
|
||||
Filters.Add(filter);
|
||||
var page = Page<TableEntity>.FromValues(RowsFor(filter), null, Mock.Of<Response>());
|
||||
return AsyncPageable<TableEntity>.FromPages(new[] { page });
|
||||
});
|
||||
Table.Setup(c => c.SubmitTransactionAsync(It.IsAny<IEnumerable<TableTransactionAction>>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<IEnumerable<TableTransactionAction>, CancellationToken>((actions, _) =>
|
||||
{
|
||||
var snapshot = actions.ToList();
|
||||
Submitted.Add(snapshot);
|
||||
if (SubmitFailures.Count > 0) throw SubmitFailures.Dequeue();
|
||||
var inner = (IReadOnlyList<Response>)snapshot.Select(_ => Mock.Of<Response>()).ToList();
|
||||
return Task.FromResult(Response.FromValue(inner, Mock.Of<Response>()));
|
||||
});
|
||||
|
||||
var service = new Mock<TableServiceClient>();
|
||||
service.Setup(s => s.GetTableClient(Constants.TableNames.PlatformStats)).Returns(Table.Object);
|
||||
Sut = new TableStorageService(service.Object, NullLogger<TableStorageService>.Instance);
|
||||
}
|
||||
}
|
||||
|
||||
private static TableEntity PlatformRow(long enrollments, long issues, string etag)
|
||||
{
|
||||
var row = new TableEntity("global", "current") { ["TotalEnrollments"] = enrollments, ["IssuesDetected"] = issues };
|
||||
row.ETag = new ETag(etag);
|
||||
return row;
|
||||
}
|
||||
|
||||
private static TableEntity BaselineRow(long enrollments)
|
||||
=> new("global", "rollup-baseline") { ["TotalEnrollments"] = enrollments };
|
||||
|
||||
// ---------- rollup write ----------
|
||||
|
||||
[Fact]
|
||||
public async Task Rollup_merges_the_platform_row_with_its_read_etag_and_leaves_IssuesDetected_alone()
|
||||
{
|
||||
var h = new Harness { RowsFor = _ => new[] { PlatformRow(19_312, issues: 35_000, etag: "e1"), BaselineRow(21_000) } };
|
||||
PlatformRollupSources? seenBaseline = null;
|
||||
|
||||
var result = await h.Sut.RollupPlatformStatsAsync(
|
||||
(existing, baseline) =>
|
||||
{
|
||||
seenBaseline = baseline;
|
||||
return new PlatformStats { TotalEnrollments = existing!.TotalEnrollments + 40, IssuesDetected = existing.IssuesDetected };
|
||||
},
|
||||
new PlatformRollupSources { TotalEnrollments = 21_040 });
|
||||
|
||||
Assert.Equal(21_000, seenBaseline!.TotalEnrollments);
|
||||
Assert.Equal(19_352, result!.TotalEnrollments);
|
||||
Assert.Equal("PartitionKey eq 'global'", Assert.Single(h.Filters));
|
||||
|
||||
var batch = Assert.Single(h.Submitted);
|
||||
var platform = batch.Single(a => a.Entity.RowKey == "current");
|
||||
Assert.Equal(TableTransactionActionType.UpdateMerge, platform.ActionType);
|
||||
Assert.Equal(new ETag("e1"), platform.ETag);
|
||||
Assert.Equal(19_352L, ((TableEntity)platform.Entity).GetInt64("TotalEnrollments"));
|
||||
Assert.False(((TableEntity)platform.Entity).ContainsKey("IssuesDetected"),
|
||||
"a write-back of IssuesDetected undoes every increment that landed since the read");
|
||||
|
||||
var baselineWrite = batch.Single(a => a.Entity.RowKey == "rollup-baseline");
|
||||
Assert.Equal(TableTransactionActionType.UpsertReplace, baselineWrite.ActionType);
|
||||
Assert.Equal(21_040L, ((TableEntity)baselineWrite.Entity).GetInt64("TotalEnrollments"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Rollup_re_reads_and_re_merges_when_the_platform_row_changed_in_between()
|
||||
{
|
||||
var reads = 0;
|
||||
var h = new Harness
|
||||
{
|
||||
RowsFor = _ => ++reads == 1
|
||||
? new[] { PlatformRow(19_312, issues: 1, etag: "e1"), BaselineRow(100) }
|
||||
: new[] { PlatformRow(21_800, issues: 2, etag: "e2"), BaselineRow(100) } // corrected in between
|
||||
};
|
||||
h.SubmitFailures.Enqueue(new RequestFailedException(412, "precondition failed"));
|
||||
|
||||
var result = await h.Sut.RollupPlatformStatsAsync(
|
||||
(existing, _) => new PlatformStats { TotalEnrollments = existing!.TotalEnrollments + 5 },
|
||||
new PlatformRollupSources { TotalEnrollments = 105 });
|
||||
|
||||
Assert.Equal(21_805, result!.TotalEnrollments);
|
||||
Assert.Equal(2, h.Submitted.Count);
|
||||
Assert.Equal(new ETag("e2"), h.Submitted[1].Single(a => a.Entity.RowKey == "current").ETag);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Rollup_adds_a_missing_platform_row_instead_of_upserting_it()
|
||||
{
|
||||
var h = new Harness();
|
||||
|
||||
await h.Sut.RollupPlatformStatsAsync(
|
||||
(existing, baseline) =>
|
||||
{
|
||||
Assert.Null(existing);
|
||||
Assert.Null(baseline);
|
||||
return new PlatformStats { TotalEnrollments = 10 };
|
||||
},
|
||||
new PlatformRollupSources { TotalEnrollments = 10 });
|
||||
|
||||
var platform = Assert.Single(h.Submitted).Single(a => a.Entity.RowKey == "current");
|
||||
Assert.Equal(TableTransactionActionType.Add, platform.ActionType);
|
||||
Assert.True(((TableEntity)platform.Entity).ContainsKey("IssuesDetected"));
|
||||
}
|
||||
|
||||
// ---------- tenant counters ----------
|
||||
|
||||
private static (Mock<TableClient> Table, TableStorageService Sut, List<(TableEntity Entity, ETag ETag, TableUpdateMode Mode)> Updates) TenantRowHarness(TableEntity row)
|
||||
{
|
||||
var table = new Mock<TableClient>();
|
||||
table.Setup(t => t.GetEntityAsync<TableEntity>(Tenant, "current", It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(Response.FromValue(row, Mock.Of<Response>()));
|
||||
var updates = new List<(TableEntity, ETag, TableUpdateMode)>();
|
||||
table.Setup(t => t.UpdateEntityAsync(It.IsAny<TableEntity>(), It.IsAny<ETag>(), It.IsAny<TableUpdateMode>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<TableEntity, ETag, TableUpdateMode, CancellationToken>((e, tag, mode, _) => { updates.Add((e, tag, mode)); return Task.FromResult(Mock.Of<Response>()); });
|
||||
var service = new Mock<TableServiceClient>();
|
||||
service.Setup(s => s.GetTableClient(Constants.TableNames.PlatformStats)).Returns(table.Object);
|
||||
return (table, new TableStorageService(service.Object, NullLogger<TableStorageService>.Instance), updates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Floors_raise_only_the_counters_below_their_floor_in_one_merge()
|
||||
{
|
||||
var row = new TableEntity(Tenant, "current") { ["TotalEnrollments"] = 500L, ["SuccessfulEnrollments"] = 10L };
|
||||
row.ETag = new ETag("t1");
|
||||
var (_, sut, updates) = TenantRowHarness(row);
|
||||
|
||||
var inPlace = await sut.EnsureTenantStatFloorsAsync(Tenant, new Dictionary<string, long>
|
||||
{
|
||||
["TotalEnrollments"] = 120, // below the counter: retention pruned, must not lower
|
||||
["SuccessfulEnrollments"] = 90, // above: heal
|
||||
["TotalEventsProcessed"] = 30_000 // field missing on the row: seed
|
||||
});
|
||||
|
||||
Assert.True(inPlace);
|
||||
var update = Assert.Single(updates);
|
||||
Assert.Equal(new ETag("t1"), update.ETag);
|
||||
Assert.Equal(TableUpdateMode.Merge, update.Mode);
|
||||
Assert.False(update.Entity.ContainsKey("TotalEnrollments"));
|
||||
Assert.Equal(90L, update.Entity.GetInt64("SuccessfulEnrollments"));
|
||||
Assert.Equal(30_000L, update.Entity.GetInt64("TotalEventsProcessed"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Floors_already_met_write_nothing_and_still_count_as_in_place()
|
||||
{
|
||||
var row = new TableEntity(Tenant, "current") { ["TotalEnrollments"] = 500L };
|
||||
row.ETag = new ETag("t1");
|
||||
var (_, sut, updates) = TenantRowHarness(row);
|
||||
|
||||
var inPlace = await sut.EnsureTenantStatFloorsAsync(Tenant, new Dictionary<string, long> { ["TotalEnrollments"] = 120 });
|
||||
|
||||
Assert.True(inPlace);
|
||||
Assert.Empty(updates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Floors_that_cannot_be_written_report_false()
|
||||
{
|
||||
var row = new TableEntity(Tenant, "current") { ["TotalEnrollments"] = 1L };
|
||||
row.ETag = new ETag("t1");
|
||||
var (table, sut, _) = TenantRowHarness(row);
|
||||
table.Setup(t => t.UpdateEntityAsync(It.IsAny<TableEntity>(), It.IsAny<ETag>(), It.IsAny<TableUpdateMode>(), It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new RequestFailedException(500, "storage down"));
|
||||
|
||||
Assert.False(await sut.EnsureTenantStatFloorsAsync(Tenant, new Dictionary<string, long> { ["TotalEnrollments"] = 120 }));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Increment_adds_to_the_one_field()
|
||||
{
|
||||
var row = new TableEntity(Tenant, "current") { ["TotalEnrollments"] = 500L, ["TotalEventsProcessed"] = 1_000L };
|
||||
row.ETag = new ETag("t1");
|
||||
var (_, sut, updates) = TenantRowHarness(row);
|
||||
|
||||
await sut.IncrementTenantStatAsync(Tenant, "TotalEventsProcessed", 37);
|
||||
|
||||
var update = Assert.Single(updates);
|
||||
Assert.Equal(1_037L, update.Entity.GetInt64("TotalEventsProcessed"));
|
||||
Assert.False(update.Entity.ContainsKey("TotalEnrollments"), "the patch must not carry fields it did not change");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task All_tenant_stats_read_the_tenant_rows_only()
|
||||
{
|
||||
var h = new Harness
|
||||
{
|
||||
RowsFor = _ => new[]
|
||||
{
|
||||
new TableEntity(Tenant, "current") { ["TotalEnrollments"] = 7L, ["SuccessfulEnrollments"] = 5L, ["TotalEventsProcessed"] = 900L },
|
||||
}
|
||||
};
|
||||
|
||||
var stats = Assert.Single(await h.Sut.GetAllTenantStatsAsync());
|
||||
|
||||
Assert.Equal("RowKey eq 'current' and PartitionKey ne 'global'", Assert.Single(h.Filters));
|
||||
Assert.Equal(7, stats.TotalEnrollments);
|
||||
Assert.Equal(5, stats.SuccessfulEnrollments);
|
||||
Assert.Equal(900, stats.TotalEventsProcessed);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_failing_tenant_stats_read_throws_instead_of_reporting_zero()
|
||||
{
|
||||
// Zero would become the rollup baseline and the next run would count every tenant
|
||||
// counter as growth a second time.
|
||||
var h = new Harness { RowsFor = _ => throw new RequestFailedException(503, "busy") };
|
||||
|
||||
await Assert.ThrowsAsync<RequestFailedException>(() => h.Sut.GetAllTenantStatsAsync());
|
||||
}
|
||||
|
||||
// ---------- seen device models ----------
|
||||
|
||||
[Fact]
|
||||
public async Task Seen_models_adds_only_unknown_models_and_returns_the_size_of_the_set()
|
||||
{
|
||||
var known = TableStorageService.SeenDeviceModelRowKey("LENOVO ThinkPad T14 Gen 6");
|
||||
var h = new Harness { RowsFor = _ => new[] { new TableEntity("models", known), new TableEntity("models", "PRUNEDMODELHASH") } };
|
||||
|
||||
var size = await h.Sut.RecordSeenDeviceModelsAsync(new[]
|
||||
{
|
||||
"lenovo thinkpad t14 gen 6", // known, different casing
|
||||
"HP EliteBook 840 14 inch G10/#1", // new, with characters a table key cannot hold
|
||||
" ",
|
||||
});
|
||||
|
||||
Assert.Equal(3, size);
|
||||
Assert.Equal("PartitionKey eq 'models'", Assert.Single(h.Filters));
|
||||
var added = Assert.Single(Assert.Single(h.Submitted));
|
||||
Assert.Equal("models", added.Entity.PartitionKey);
|
||||
Assert.Equal("HP EliteBook 840 14 inch G10/#1", ((TableEntity)added.Entity).GetString("Model"));
|
||||
Assert.Matches("^[0-9A-F]{64}$", added.Entity.RowKey);
|
||||
}
|
||||
}
|
||||
@@ -937,13 +937,10 @@ public class PolicyEnforcementMiddlewareTests
|
||||
Assert.False(rc.IsGlobalReader);
|
||||
Assert.NotNull(rc.AllowedTenantIds);
|
||||
Assert.Contains(TenantB.ToLowerInvariant(), rc.AllowedTenantIds!);
|
||||
// …and it is the DATA aggregate the MCP quota layer may narrow (charged per managed tenant).
|
||||
Assert.True(rc.IsDelegatedAggregate);
|
||||
Assert.Null(rc.HomeChargedTenantIds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Delegated_GlobalSessions_WithTenantId_IsADrill_NotAnAggregate()
|
||||
public async Task Delegated_GlobalSessions_WithTenantId_IsADrill()
|
||||
{
|
||||
const string upn = "msp@partner.example";
|
||||
var h = BuildHarness();
|
||||
@@ -955,15 +952,12 @@ public class PolicyEnforcementMiddlewareTests
|
||||
var rc = result.Context!;
|
||||
Assert.True(rc.IsDelegatedReader);
|
||||
Assert.Equal(TenantB, rc.TargetTenantId);
|
||||
Assert.False(rc.IsDelegatedAggregate);
|
||||
Assert.Null(rc.HomeChargedTenantIds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Delegated_ConfigAll_IsADirectoryListing_NotAnAggregate()
|
||||
public async Task Delegated_ConfigAll_IsADirectoryListing_AdmittedAndBounded()
|
||||
{
|
||||
// Subset tier with TenantScoping.None: admitted + bounded, but no data fan-out — the quota layer
|
||||
// charges it to the caller's home tenant, never to the managed tenants.
|
||||
// Subset tier with TenantScoping.None: admitted + bounded, but no data fan-out.
|
||||
const string upn = "msp@partner.example";
|
||||
var h = BuildHarness();
|
||||
h.AsDelegated(TenantB);
|
||||
@@ -974,54 +968,6 @@ public class PolicyEnforcementMiddlewareTests
|
||||
var rc = result.Context!;
|
||||
Assert.True(rc.IsDelegatedReader);
|
||||
Assert.NotNull(rc.AllowedTenantIds);
|
||||
Assert.False(rc.IsDelegatedAggregate);
|
||||
}
|
||||
|
||||
private static void AsHomeChargedGroupMember(Harness h, string groupId, string upn, params string[] tenantIds)
|
||||
{
|
||||
h.Repo.Setup(r => r.GetGroupAssignmentsForUpnAsync(It.IsAny<string>()))
|
||||
.ReturnsAsync(new List<TenantGroupAssignment>
|
||||
{
|
||||
new() { Upn = upn, GroupId = groupId, Role = Constants.DelegatedRoles.DelegatedReader, IsEnabled = true, AssignedBy = "ga@vendor.example" },
|
||||
});
|
||||
h.Repo.Setup(r => r.GetGroupMembershipAsync(groupId))
|
||||
.ReturnsAsync(new TenantGroupMembership
|
||||
{
|
||||
GroupId = groupId,
|
||||
TenantIds = tenantIds.Select(t => t.ToLowerInvariant()).ToList(),
|
||||
ChargeHomeTenantQuota = true,
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Delegated_HomeChargedGroup_MarksTheDrillTarget()
|
||||
{
|
||||
const string upn = "ops@vendor.example";
|
||||
var h = BuildHarness();
|
||||
AsHomeChargedGroupMember(h, "grp-managed-service", upn, TenantB);
|
||||
|
||||
var result = await h.Middleware.DecideAsync("GET", "/api/global/sessions", TenantB, AuthedPrincipal(TenantA, upn));
|
||||
|
||||
Assert.True(result.Allowed);
|
||||
var rc = result.Context!;
|
||||
Assert.True(rc.IsDelegatedReader);
|
||||
Assert.Equal(new[] { TenantB.ToLowerInvariant() }, rc.HomeChargedTenantIds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Delegated_HomeChargedGroup_MarksTheAggregateMembers()
|
||||
{
|
||||
const string upn = "ops@vendor.example";
|
||||
var h = BuildHarness();
|
||||
AsHomeChargedGroupMember(h, "grp-managed-service", upn, TenantB);
|
||||
|
||||
var result = await h.Middleware.DecideAsync("GET", "/api/global/sessions", null, AuthedPrincipal(TenantA, upn));
|
||||
|
||||
Assert.True(result.Allowed);
|
||||
var rc = result.Context!;
|
||||
Assert.True(rc.IsDelegatedAggregate);
|
||||
Assert.Contains(TenantB.ToLowerInvariant(), rc.AllowedTenantIds!);
|
||||
Assert.Equal(new[] { TenantB.ToLowerInvariant() }, rc.HomeChargedTenantIds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -1769,7 +1715,6 @@ public class PolicyEnforcementMiddlewareTests
|
||||
Assert.True(mcp.Allowed);
|
||||
Assert.NotNull(mcp.Context!.AllowedTenantIds);
|
||||
Assert.Empty(mcp.Context!.AllowedTenantIds!);
|
||||
Assert.True(mcp.Context!.IsDelegatedAggregate);
|
||||
Assert.Contains(TenantB.ToLowerInvariant(), portal.Context!.AllowedTenantIds!);
|
||||
}
|
||||
|
||||
|
||||
@@ -101,51 +101,6 @@ public class RevokeEnforcementTests
|
||||
ManagedTenantProIndex.None,
|
||||
NullLogger<ProConferralService>.Instance) { CallBase = false };
|
||||
|
||||
// --- ChargeHomeTenantQuota flip: audited under every tenant, no-op on an unchanged value ---
|
||||
|
||||
[Fact]
|
||||
public async Task SetChargeMode_UnknownGroup_False_NoAudit()
|
||||
{
|
||||
var (fn, _, audit, _) = BuildGroupFn();
|
||||
var ok = await fn.SetChargeModeCoreAsync("no-such-group", true, "ga@vendor.example");
|
||||
Assert.False(ok);
|
||||
VerifyNoAudit(audit);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SetChargeMode_UnchangedValue_IsNoOp_NoAudit()
|
||||
{
|
||||
var (fn, repo, audit, _) = BuildGroupFn();
|
||||
repo.Setup(r => r.GetTenantGroupAsync(GroupId)).ReturnsAsync(
|
||||
new TenantGroup { GroupId = GroupId, Name = "Managed Service", TenantIds = { TenantA }, ChargeHomeTenantQuota = true });
|
||||
|
||||
var ok = await fn.SetChargeModeCoreAsync(GroupId, true, "ga@vendor.example");
|
||||
|
||||
Assert.True(ok);
|
||||
repo.Verify(r => r.SetTenantGroupChargeHomeTenantQuotaAsync(It.IsAny<string>(), It.IsAny<bool>()), Times.Never);
|
||||
VerifyNoAudit(audit);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SetChargeMode_RealChange_AuditsUnderEveryTenantInTheGroup()
|
||||
{
|
||||
var (fn, repo, audit, _) = BuildGroupFn();
|
||||
repo.Setup(r => r.GetTenantGroupAsync(GroupId)).ReturnsAsync(
|
||||
new TenantGroup { GroupId = GroupId, Name = "Managed Service", TenantIds = { TenantA, TenantB } });
|
||||
repo.Setup(r => r.SetTenantGroupChargeHomeTenantQuotaAsync(GroupId, true)).ReturnsAsync(true);
|
||||
|
||||
var ok = await fn.SetChargeModeCoreAsync(GroupId, true, "ga@vendor.example");
|
||||
|
||||
Assert.True(ok);
|
||||
foreach (var tenant in new[] { TenantA, TenantB })
|
||||
{
|
||||
audit.Verify(a => a.LogAuditEntryAsync(tenant, "UPDATE", "DelegatedGroupAccess", "*", "ga@vendor.example",
|
||||
It.Is<Dictionary<string, string>?>(d => d != null
|
||||
&& d["Reason"] == "quota-charge-mode-changed"
|
||||
&& d["ChargeHomeTenantQuota"] == "true")), Times.Once);
|
||||
}
|
||||
}
|
||||
|
||||
// --- Delegated grant validation (fail-closed) ---
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The gate in front of the embedded self-seed: only a GitHub reseed that is NEWER than the
|
||||
/// deployed binary closes it. The dates are the 2026-09-11 incident — a backend built on
|
||||
/// 2026-09-10 19:19 UTC re-inserted a pattern that the reseed of 2026-09-11 01:20 had deleted.
|
||||
/// </summary>
|
||||
public class RuleCatalogStampTests
|
||||
{
|
||||
private static readonly DateTime Build = new(2026, 9, 10, 19, 19, 57, DateTimeKind.Utc);
|
||||
|
||||
private static RuleCatalogStamp Stamp(string source, DateTime at) => new()
|
||||
{
|
||||
Kind = RuleCatalogStamp.KindIme, Source = source, StampedAt = at, Count = 82,
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void No_stamp_lets_the_embedded_seed_run()
|
||||
{
|
||||
Assert.True(RuleCatalogStamp.EmbeddedSeedAllowed(null, Build));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GitHub_reseed_after_the_build_closes_the_gate()
|
||||
{
|
||||
var reseed = new DateTime(2026, 9, 11, 1, 20, 42, DateTimeKind.Utc);
|
||||
Assert.False(RuleCatalogStamp.EmbeddedSeedAllowed(Stamp(RuleCatalogStamp.SourceGitHub, reseed), Build));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GitHub_reseed_before_or_at_the_build_keeps_it_open()
|
||||
{
|
||||
Assert.True(RuleCatalogStamp.EmbeddedSeedAllowed(Stamp(RuleCatalogStamp.SourceGitHub, Build.AddHours(-1)), Build));
|
||||
Assert.True(RuleCatalogStamp.EmbeddedSeedAllowed(Stamp(RuleCatalogStamp.SourceGitHub, Build), Build));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Embedded_reseed_never_closes_the_gate()
|
||||
{
|
||||
Assert.True(RuleCatalogStamp.EmbeddedSeedAllowed(Stamp(RuleCatalogStamp.SourceEmbedded, Build.AddDays(3)), Build));
|
||||
Assert.False(RuleCatalogStamp.EmbeddedSeedAllowed(Stamp("GitHub", Build.AddDays(3)), Build)); // source is compared case-insensitively
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ using Azure.Data.Tables;
|
||||
using Azure.Storage.Blobs;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Offboarding;
|
||||
using AutopilotMonitor.Shared;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
@@ -205,6 +206,75 @@ public class SafeWipeServiceTests
|
||||
Assert.Empty(harness.SubmittedBatches);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OwnedGroupPartition_DeletesRowsUnderSelfServiceGroupId()
|
||||
{
|
||||
// TenantGroups: PK = "msp-{tenantId}". The entry point takes the tenant GUID and derives
|
||||
// the group id — the rows under that partition are the ones deleted.
|
||||
var groupId = Constants.TenantGroupIds.ForHomeTenant(TenantId);
|
||||
var harness = new Harness(
|
||||
queriedRows: new[]
|
||||
{
|
||||
new TableEntity(groupId, "meta"),
|
||||
new TableEntity(groupId, OtherTenant),
|
||||
});
|
||||
|
||||
var deleted = await harness.Sut.WipeByOwnedGroupPartitionAsync("FakeTable", TenantId);
|
||||
|
||||
Assert.Equal(2, deleted);
|
||||
Assert.Equal($"PartitionKey eq '{groupId}'", Assert.Single(harness.Filters));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OwnedGroupPartition_AbortsWhenForeignGroupReturned()
|
||||
{
|
||||
var harness = new Harness(
|
||||
queriedRows: new[]
|
||||
{
|
||||
new TableEntity(Constants.TenantGroupIds.ForHomeTenant(TenantId), "meta"),
|
||||
new TableEntity(Constants.TenantGroupIds.ForHomeTenant(OtherTenant), "meta"), // <-- another manager's group
|
||||
});
|
||||
|
||||
await Assert.ThrowsAsync<SafeWipeVerificationException>(() =>
|
||||
harness.Sut.WipeByOwnedGroupPartitionAsync("FakeTable", TenantId));
|
||||
|
||||
Assert.Empty(harness.SubmittedBatches);
|
||||
Assert.Empty(harness.PerRowDeletes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OwnedGroupRowKey_AbortsWhenForeignGroupAssignmentReturned()
|
||||
{
|
||||
// TenantGroupAssignments: PK = managed tenant, RK = groupId. An assignment pointing at
|
||||
// another manager's group must never be deleted.
|
||||
var harness = new Harness(
|
||||
queriedRows: new[]
|
||||
{
|
||||
new TableEntity(OtherTenant, Constants.TenantGroupIds.ForHomeTenant(TenantId)),
|
||||
new TableEntity(OtherTenant, Constants.TenantGroupIds.ForHomeTenant(OtherTenant)),
|
||||
});
|
||||
|
||||
await Assert.ThrowsAsync<SafeWipeVerificationException>(() =>
|
||||
harness.Sut.WipeByOwnedGroupRowKeyAsync("FakeTable", TenantId));
|
||||
|
||||
Assert.Empty(harness.SubmittedBatches);
|
||||
Assert.Empty(harness.PerRowDeletes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TenantGuidEntryPoints_RejectTheDerivedGroupId()
|
||||
{
|
||||
// The production bug: the handler passed "msp-{tenantId}" into the tenant-GUID entry
|
||||
// points and every offboarding poisoned. Those entry points stay GUID-only.
|
||||
var harness = new Harness(queriedRows: Array.Empty<TableEntity>());
|
||||
var groupId = Constants.TenantGroupIds.ForHomeTenant(TenantId);
|
||||
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => harness.Sut.WipeByExactPartitionAsync("t", groupId));
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => harness.Sut.WipeByRowKeyAsync("t", groupId));
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => harness.Sut.WipeByOwnedGroupPartitionAsync("t", groupId));
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => harness.Sut.WipeByOwnedGroupRowKeyAsync("t", groupId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Variants_RejectNonGuidTenantId()
|
||||
{
|
||||
@@ -228,6 +298,8 @@ public class SafeWipeServiceTests
|
||||
{
|
||||
public List<List<TableTransactionAction>> SubmittedBatches { get; } = new();
|
||||
public List<(string Pk, string Rk, ETag Etag)> PerRowDeletes { get; } = new();
|
||||
/// <summary>OData filters the service sent to the table — the server-side anchor.</summary>
|
||||
public List<string> Filters { get; } = new();
|
||||
public SafeWipeService Sut { get; }
|
||||
|
||||
public Harness(
|
||||
@@ -244,6 +316,7 @@ public class SafeWipeServiceTests
|
||||
It.IsAny<int?>(),
|
||||
It.IsAny<IEnumerable<string>>(),
|
||||
It.IsAny<CancellationToken>()))
|
||||
.Callback<string, int?, IEnumerable<string>, CancellationToken>((filter, _, _, _) => Filters.Add(filter))
|
||||
.Returns(AsAsyncPageable(queriedRows));
|
||||
|
||||
mockTableClient
|
||||
|
||||
@@ -850,45 +850,6 @@ public class SessionsWireParityTests
|
||||
});
|
||||
}
|
||||
|
||||
// ---- Delegated (MSP) MCP fleet aggregate: quota-excluded tenants ---------------------
|
||||
|
||||
[Fact]
|
||||
public void SessionListResponse_carries_quotaExcludedTenants_on_a_narrowed_fleet_aggregate()
|
||||
{
|
||||
IReadOnlyList<SessionSummary> items = new List<SessionSummary> { SampleSummary("0b6f7a37-1111-4d61-9c93-0aa111111111") };
|
||||
string? nextLink = null;
|
||||
var quotaExcludedTenants = new[] { "7aa20c11-0002-4b7c-a1d2-52f3aaaa0002" };
|
||||
|
||||
AssertParity(
|
||||
new
|
||||
{
|
||||
success = true,
|
||||
count = items.Count,
|
||||
sessions = items,
|
||||
nextLink,
|
||||
quotaExcludedTenants,
|
||||
},
|
||||
new SessionListResponse
|
||||
{
|
||||
Success = true,
|
||||
Count = items.Count,
|
||||
Sessions = items,
|
||||
NextLink = null,
|
||||
QuotaExcludedTenants = quotaExcludedTenants,
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SessionStatsResponse_carries_quotaExcludedTenants_on_a_narrowed_fleet_aggregate()
|
||||
{
|
||||
var stats = new SessionStats { Days = 7, ComputedAt = new DateTime(2026, 8, 30, 12, 0, 0, DateTimeKind.Utc) };
|
||||
var quotaExcludedTenants = new[] { "7aa20c11-0002-4b7c-a1d2-52f3aaaa0002", "7aa20c11-0002-4b7c-a1d2-52f3aaaa0003" };
|
||||
|
||||
AssertParity(
|
||||
new { success = true, stats, quotaExcludedTenants },
|
||||
new SessionStatsResponse { Success = true, Stats = stats, QuotaExcludedTenants = quotaExcludedTenants });
|
||||
}
|
||||
|
||||
private static void AssertParity(object anonymousLiteral, IApiResponse typed)
|
||||
=> ApiResponseWireParityTests.AssertWireIdentical(anonymousLiteral, typed);
|
||||
}
|
||||
|
||||
@@ -17,6 +17,10 @@ public class SlaMetricsServiceTests
|
||||
// Use unique tenant IDs per test to avoid static cache collisions
|
||||
private static string NewTenantId() => $"sla-test-{Guid.NewGuid():N}";
|
||||
|
||||
// Fixed clock: Wednesday 2026-09-23, ISO week 39. Mid-week and mid-month, so neither a week
|
||||
// nor a month boundary can move a fixture out of its period.
|
||||
private static readonly DateTime Now = new(2026, 9, 23, 12, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private static (SlaMetricsService Service, string TenantId) CreateService(
|
||||
List<SessionSummary> sessions,
|
||||
TenantConfiguration? config = null,
|
||||
@@ -44,7 +48,7 @@ public class SlaMetricsServiceTests
|
||||
|
||||
var logger = NullLogger<SlaMetricsService>.Instance;
|
||||
|
||||
return (new SlaMetricsService(maintenanceRepo.Object, metricsRepo.Object, configService.Object, logger), tenantId);
|
||||
return (new SlaMetricsService(maintenanceRepo.Object, metricsRepo.Object, configService.Object, logger, () => Now), tenantId);
|
||||
}
|
||||
|
||||
private static TenantConfiguration CreateDefaultConfig(
|
||||
@@ -65,15 +69,7 @@ public class SlaMetricsServiceTests
|
||||
int? durationSeconds = null,
|
||||
DateTime? startedAt = null)
|
||||
{
|
||||
// The CurrentWeek asserts require the session to fall into the CURRENT ISO week
|
||||
// (SlaMetricsService groups by GetIsoWeekKey, weeks start Monday 00:00 UTC).
|
||||
// A plain "now - 1h" crosses into LAST week during the first hour of every
|
||||
// Monday (UTC) and empties CurrentWeek — exactly the CI failure on
|
||||
// Mon 2026-08-31 00:20Z. Clamp the anchor to the week's Monday instead.
|
||||
var now = DateTime.UtcNow;
|
||||
var isoWeekMonday = now.Date.AddDays(-(((int)now.DayOfWeek + 6) % 7));
|
||||
var anHourAgo = now.AddHours(-1);
|
||||
var started = startedAt ?? (anHourAgo >= isoWeekMonday ? anHourAgo : isoWeekMonday);
|
||||
var started = startedAt ?? Now.AddHours(-1);
|
||||
return new SessionSummary
|
||||
{
|
||||
SessionId = Guid.NewGuid().ToString(),
|
||||
@@ -134,6 +130,96 @@ public class SlaMetricsServiceTests
|
||||
Assert.Equal(0, result.CurrentWeek.SuccessRate);
|
||||
Assert.Equal(0, result.CurrentWeek.TotalCompleted);
|
||||
Assert.Empty(result.Violators);
|
||||
|
||||
// An empty period is "no data": it must not read as a breached target.
|
||||
foreach (var snapshot in new[] { result.CurrentWeek, result.EvaluationPeriod })
|
||||
{
|
||||
Assert.False(snapshot.HasData);
|
||||
Assert.True(snapshot.SuccessRateMet);
|
||||
Assert.True(snapshot.DurationTargetMet);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ComputeSlaMetrics_EmptyWeekFilledWindow_HeadlineCarriesTheEvaluationWindow()
|
||||
{
|
||||
// The evaluation window holds finished enrollments, the current ISO week none — the
|
||||
// headline snapshot (what the breach notification evaluates) reports them, the week stays empty.
|
||||
var twoWeeksAgo = new DateTime(2026, 9, 8, 10, 0, 0, DateTimeKind.Utc); // W37
|
||||
var sessions = new List<SessionSummary>();
|
||||
for (var i = 0; i < 32; i++) sessions.Add(CreateSession(SessionStatus.Succeeded, 1800, twoWeeksAgo));
|
||||
for (var i = 0; i < 9; i++) sessions.Add(CreateSession(SessionStatus.Failed, 1800, twoWeeksAgo));
|
||||
|
||||
var (service, tenantId) = CreateService(sessions);
|
||||
var result = await service.ComputeSlaMetricsAsync(tenantId, 1);
|
||||
|
||||
Assert.Equal(30, result.EvaluationWindowDays);
|
||||
Assert.Equal("last-30-days", result.EvaluationPeriod.Period);
|
||||
Assert.Equal("", result.EvaluationPeriod.Week);
|
||||
Assert.True(result.EvaluationPeriod.HasData);
|
||||
Assert.Equal(41, result.EvaluationPeriod.TotalCompleted);
|
||||
Assert.Equal(78, result.EvaluationPeriod.SuccessRate);
|
||||
Assert.False(result.EvaluationPeriod.SuccessRateMet);
|
||||
|
||||
Assert.Equal("2026-W39", result.CurrentWeek.Period);
|
||||
Assert.Equal("2026-W39", result.CurrentWeek.Week);
|
||||
Assert.False(result.CurrentWeek.HasData);
|
||||
Assert.True(result.CurrentWeek.SuccessRateMet);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ComputeSlaMetrics_EvaluationWindow_IsRolling_NotTheCalendarMonth()
|
||||
{
|
||||
// Now = 2026-09-23 12:00 ⇒ the window starts 2026-08-24 12:00.
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
CreateSession(SessionStatus.Succeeded, 1800),
|
||||
// previous calendar month, inside the rolling window
|
||||
CreateSession(SessionStatus.Failed, 1800, new DateTime(2026, 8, 25, 9, 0, 0, DateTimeKind.Utc)),
|
||||
// just outside the window
|
||||
CreateSession(SessionStatus.Failed, 1800, new DateTime(2026, 8, 24, 11, 59, 59, DateTimeKind.Utc)),
|
||||
CreateSession(SessionStatus.Failed, 1800, new DateTime(2026, 7, 15, 9, 0, 0, DateTimeKind.Utc)),
|
||||
};
|
||||
|
||||
var (service, tenantId) = CreateService(sessions);
|
||||
var result = await service.ComputeSlaMetricsAsync(tenantId, 3);
|
||||
|
||||
Assert.Equal(2, result.EvaluationPeriod.TotalCompleted);
|
||||
Assert.Equal(50, result.EvaluationPeriod.SuccessRate);
|
||||
Assert.Equal(4, result.WeeklyTrend.Sum(w => w.TotalCompleted));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ComputeSlaMetrics_OneMonthSelection_HeadlineStillReachesBack_TrendDoesNot()
|
||||
{
|
||||
// "1 Month" starts on the first; the evaluation window reaches into August. The headline
|
||||
// must count the August session, the trend and the violators must not grow by it.
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
CreateSession(SessionStatus.Succeeded, 1800),
|
||||
CreateSession(SessionStatus.Failed, 1800, new DateTime(2026, 8, 28, 9, 0, 0, DateTimeKind.Utc)),
|
||||
};
|
||||
|
||||
var (service, tenantId) = CreateService(sessions);
|
||||
var result = await service.ComputeSlaMetricsAsync(tenantId, 1);
|
||||
|
||||
Assert.Equal(2, result.EvaluationPeriod.TotalCompleted);
|
||||
Assert.Equal(1, result.WeeklyTrend.Sum(w => w.TotalCompleted));
|
||||
Assert.Empty(result.Violators);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ComputeSlaMetrics_SessionsWithoutDuration_DurationTargetIsNotJudged()
|
||||
{
|
||||
var config = CreateDefaultConfig(targetMaxDuration: 30);
|
||||
var sessions = new List<SessionSummary> { CreateSession(SessionStatus.Failed) };
|
||||
|
||||
var (service, tenantId) = CreateService(sessions, config);
|
||||
var result = await service.ComputeSlaMetricsAsync(tenantId, 1);
|
||||
|
||||
Assert.True(result.EvaluationPeriod.HasData);
|
||||
Assert.False(result.EvaluationPeriod.SuccessRateMet);
|
||||
Assert.True(result.EvaluationPeriod.DurationTargetMet);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -213,6 +213,14 @@ public class SlaTenantStatusLifecycleTests
|
||||
.ReturnsAsync(sessions);
|
||||
}
|
||||
|
||||
/// <summary>Like the real repository: only sessions started inside the queried range come back.</summary>
|
||||
public void SetSessionsHonoringDateRange(string tenantId, List<SessionSummary> sessions)
|
||||
{
|
||||
_maintenanceRepo.Setup(r => r.GetSessionsByDateRangeAsync(It.IsAny<DateTime>(), It.IsAny<DateTime>(), tenantId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((DateTime start, DateTime end, string? _, CancellationToken _) =>
|
||||
sessions.Where(s => s.StartedAt >= start && s.StartedAt < end).ToList());
|
||||
}
|
||||
|
||||
public void SetRecentSessions(string tenantId, List<SessionSummary> sessions)
|
||||
{
|
||||
_sessionRepo.Setup(r => r.GetSessionsPageAsync(tenantId, It.IsAny<int?>(), It.IsAny<int>(), It.IsAny<string?>()))
|
||||
@@ -291,7 +299,7 @@ public class SlaTenantStatusLifecycleTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReBreach_AfterCooldown_SendsAnotherNotification()
|
||||
public async Task ReBreach_AfterCooldown_WithNewlyFinishedSession_SendsAnotherNotification()
|
||||
{
|
||||
var h = new Harness();
|
||||
var config = CreateConfig(successRate: true);
|
||||
@@ -307,6 +315,15 @@ public class SlaTenantStatusLifecycleTests
|
||||
row!.SuccessRate_LastNotifiedAt = DateTime.UtcNow.AddHours(-25);
|
||||
await h.StatusRepo.UpsertAsync(row);
|
||||
|
||||
// A session that finished after the last notification: the population changed.
|
||||
var sessions = Sessions(succeeded: 8, failed: 4);
|
||||
sessions.Add(new SessionSummary
|
||||
{
|
||||
TenantId = TenantId, SessionId = Guid.NewGuid().ToString(), Status = SessionStatus.Failed,
|
||||
StartedAt = DateTime.UtcNow.AddHours(-26), CompletedAt = DateTime.UtcNow.AddHours(-1), DurationSeconds = 1800,
|
||||
});
|
||||
h.SetTerminalSessions(TenantId, sessions);
|
||||
|
||||
await h.Service.EvaluateAllTenantsAsync();
|
||||
|
||||
Assert.Equal(2, h.NotificationsSent.Count(n => n.Type == "sla_breach"));
|
||||
@@ -314,6 +331,67 @@ public class SlaTenantStatusLifecycleTests
|
||||
Assert.True((DateTime.UtcNow - afterRow!.SuccessRate_LastNotifiedAt!.Value).TotalMinutes < 1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A lasting breach without any newly finished enrollment is not repeated, however long the
|
||||
/// cooldown has expired — a tenant that enrolls nothing must not get the same number every day.
|
||||
/// Includes a session that finished BEFORE the last notification (an imported or merely old
|
||||
/// row): it is not new evidence.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ReBreach_AfterCooldown_WithoutNewSession_StaysSilent()
|
||||
{
|
||||
var h = new Harness();
|
||||
var config = CreateConfig(successRate: true, duration: true, targetMaxDurationMinutes: 10);
|
||||
h.SetTenants(config);
|
||||
var sessions = Sessions(succeeded: 8, failed: 4); // 66.7 % and P95 30 min: both breach
|
||||
sessions.Add(new SessionSummary
|
||||
{
|
||||
TenantId = TenantId, SessionId = Guid.NewGuid().ToString(), Status = SessionStatus.Failed,
|
||||
StartedAt = DateTime.UtcNow.AddDays(-9), CompletedAt = DateTime.UtcNow.AddDays(-9).AddMinutes(30), DurationSeconds = 1800,
|
||||
});
|
||||
h.SetTerminalSessions(TenantId, sessions);
|
||||
|
||||
await h.Service.EvaluateAllTenantsAsync();
|
||||
Assert.Equal(2, h.NotificationsSent.Count(n => n.Type == "sla_breach"));
|
||||
|
||||
var row = await h.StatusRepo.GetAsync(TenantId);
|
||||
row!.SuccessRate_LastNotifiedAt = DateTime.UtcNow.AddDays(-5);
|
||||
row.Duration_LastNotifiedAt = DateTime.UtcNow.AddDays(-5);
|
||||
await h.StatusRepo.UpsertAsync(row);
|
||||
|
||||
await h.Service.EvaluateAllTenantsAsync();
|
||||
|
||||
Assert.Equal(2, h.NotificationsSent.Count(n => n.Type == "sla_breach"));
|
||||
var afterRow = await h.StatusRepo.GetAsync(TenantId);
|
||||
Assert.True(afterRow!.SuccessRate_IsActive); // still breaching, still tracked
|
||||
Assert.True(afterRow.Duration_IsActive);
|
||||
Assert.NotNull(afterRow.SuccessRate_LastBreachAt);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A new breach episode after a resolve notifies without new evidence: the rolling window can
|
||||
/// re-breach by good sessions aging out, and the admin was last told "resolved".
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task NewBreachEpisode_AfterResolve_NotifiesWithoutNewSession()
|
||||
{
|
||||
var h = new Harness();
|
||||
var config = CreateConfig(successRate: true);
|
||||
h.SetTenants(config);
|
||||
await h.StatusRepo.UpsertAsync(new SlaTenantStatus
|
||||
{
|
||||
TenantId = TenantId,
|
||||
SuccessRate_IsActive = false,
|
||||
SuccessRate_LastNotifiedAt = DateTime.UtcNow.AddDays(-3),
|
||||
SuccessRate_ResolvedAt = DateTime.UtcNow.AddDays(-2),
|
||||
});
|
||||
h.SetTerminalSessions(TenantId, Sessions(succeeded: 8, failed: 4)); // no CompletedAt after the watermark
|
||||
|
||||
await h.Service.EvaluateAllTenantsAsync();
|
||||
|
||||
Assert.Single(h.NotificationsSent, n => n.Type == "sla_breach");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task BreachResolved_SetsIsActiveFalseAndSendsResolvedNotification()
|
||||
{
|
||||
@@ -773,6 +851,68 @@ public class SlaTenantStatusLifecycleTests
|
||||
Assert.DoesNotContain(h.NotificationsSent, n => n.Type == "sla_breach");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The breach notification links to the SLA dashboard, so both must report the same number
|
||||
/// for the same target. Same sessions through the evaluator and through SlaMetricsService:
|
||||
/// the persisted breach values equal the dashboard's evaluation-period snapshot. The fixture
|
||||
/// holds sessions of the previous calendar month inside AND outside the rolling window and an
|
||||
/// empty current ISO week on purpose — a headline computed on a calendar month, on the ISO
|
||||
/// week or on the selected months diverges here. Both month selections run: with "1 Month"
|
||||
/// the dashboard's own range starts after the window does.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(1)]
|
||||
[InlineData(3)]
|
||||
public async Task Evaluator_And_DashboardEvaluationPeriod_ReportTheSameValues(int months)
|
||||
{
|
||||
var clock = new DateTime(2026, 9, 21, 12, 0, 0, DateTimeKind.Utc); // Monday, ISO week 39; window starts 08-22 12:00
|
||||
SessionSummary At(DateTime startedAt, SessionStatus status, int durationSec) => new()
|
||||
{
|
||||
TenantId = TenantId, SessionId = Guid.NewGuid().ToString(), Status = status,
|
||||
StartedAt = startedAt, DurationSeconds = durationSec,
|
||||
};
|
||||
|
||||
var sessions = new List<SessionSummary>();
|
||||
for (int i = 0; i < 4; i++) // previous month, outside the rolling window
|
||||
sessions.Add(At(new DateTime(2026, 8, 20, 9, 0, 0, DateTimeKind.Utc), SessionStatus.Failed, 9000));
|
||||
for (int i = 0; i < 2; i++) // previous month, inside the rolling window
|
||||
sessions.Add(At(new DateTime(2026, 8, 28, 9, 0, 0, DateTimeKind.Utc), SessionStatus.Failed, 5400));
|
||||
for (int i = 0; i < 7; i++) // this month, earlier ISO weeks
|
||||
sessions.Add(At(new DateTime(2026, 9, 3 + i, 9, 0, 0, DateTimeKind.Utc), SessionStatus.Succeeded, 900 + i * 600));
|
||||
sessions.Add(At(new DateTime(2026, 9, 9, 9, 0, 0, DateTimeKind.Utc), SessionStatus.Failed, 7200));
|
||||
sessions.Add(At(new DateTime(2026, 9, 10, 9, 0, 0, DateTimeKind.Utc), SessionStatus.Failed, 300));
|
||||
|
||||
var h = new Harness(clock: clock);
|
||||
var config = CreateConfig(successRate: true, duration: true, targetMaxDurationMinutes: 30);
|
||||
h.SetTenants(config);
|
||||
h.SetSessionsHonoringDateRange(TenantId, sessions);
|
||||
|
||||
await h.Service.EvaluateAllTenantsAsync();
|
||||
var row = await h.StatusRepo.GetAsync(TenantId);
|
||||
|
||||
var maintenanceRepo = new Mock<IMaintenanceRepository>();
|
||||
maintenanceRepo.Setup(r => r.GetSessionsByDateRangeAsync(It.IsAny<DateTime>(), It.IsAny<DateTime>(), TenantId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((DateTime start, DateTime end, string? _, CancellationToken _) =>
|
||||
sessions.Where(s => s.StartedAt >= start && s.StartedAt < end).ToList());
|
||||
var configService = new Mock<TenantConfigurationService>(
|
||||
Mock.Of<IConfigRepository>(), NullLogger<TenantConfigurationService>.Instance, new MemoryCache(new MemoryCacheOptions()));
|
||||
configService.Setup(c => c.GetConfigurationAsync(TenantId)).ReturnsAsync(config);
|
||||
var dashboard = new SlaMetricsService(
|
||||
maintenanceRepo.Object, Mock.Of<IMetricsRepository>(), configService.Object,
|
||||
NullLogger<SlaMetricsService>.Instance, () => clock);
|
||||
|
||||
var metrics = await dashboard.ComputeSlaMetricsAsync(TenantId, months, fresh: true);
|
||||
|
||||
Assert.Equal(11, metrics.EvaluationPeriod.TotalCompleted);
|
||||
Assert.Equal(63.6, metrics.EvaluationPeriod.SuccessRate);
|
||||
Assert.Equal(row!.SuccessRate_CurrentValue, metrics.EvaluationPeriod.SuccessRate);
|
||||
Assert.Equal(row.SuccessRate_TotalSessions, metrics.EvaluationPeriod.TotalCompleted);
|
||||
Assert.Equal(row.Duration_CurrentP95Minutes, metrics.EvaluationPeriod.P95DurationMinutes);
|
||||
Assert.Equal(row.SuccessRate_IsActive, !metrics.EvaluationPeriod.SuccessRateMet);
|
||||
Assert.Equal(row.Duration_IsActive, !metrics.EvaluationPeriod.DurationTargetMet);
|
||||
Assert.False(metrics.CurrentWeek.HasData);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Review-finding regression: AppInstall notification + ops event must carry the actual
|
||||
/// installer totals (not 0/0). Without this, Application Insights / Ops dashboards see
|
||||
@@ -877,8 +1017,8 @@ public class SlaTenantStatusLifecycleTests
|
||||
/// ops dashboards even though they were evaluated on the current ISO week.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData("SuccessRate", "CurrentMonth")]
|
||||
[InlineData("Duration", "CurrentMonth")]
|
||||
[InlineData("SuccessRate", "Last30Days")]
|
||||
[InlineData("Duration", "Last30Days")]
|
||||
[InlineData("AppInstall", "CurrentWeek")]
|
||||
public void PeriodForBreachType_MatchesEvaluationWindow(string breachType, string expectedPeriod)
|
||||
{
|
||||
|
||||
@@ -13,6 +13,7 @@ internal sealed class StubTenantEntitlementService : TenantEntitlementService
|
||||
{
|
||||
private readonly Func<string?, EditionResolution> _resolver;
|
||||
private readonly Func<string?, string?> _planOverrideResolver;
|
||||
private readonly Func<string?, int> _purchasedSlotsResolver;
|
||||
|
||||
public StubTenantEntitlementService(TenantEdition edition) : this(_ => edition)
|
||||
{
|
||||
@@ -23,18 +24,27 @@ internal sealed class StubTenantEntitlementService : TenantEntitlementService
|
||||
/// The tenant-wide MCP usage-plan override (TenantConfiguration.McpUsagePlanOverride) per tenant id;
|
||||
/// null = no override (the edition's plan name applies). Defaults to "no override anywhere".
|
||||
/// </param>
|
||||
public StubTenantEntitlementService(Func<string?, TenantEdition> resolver, Func<string?, string?>? planOverrideResolver = null)
|
||||
: this(tenantId => AsOwnResolution(resolver(tenantId)), planOverrideResolver)
|
||||
/// <param name="purchasedSlotsResolver">Purchased delegation slots per tenant id; defaults to 0 everywhere.</param>
|
||||
public StubTenantEntitlementService(
|
||||
Func<string?, TenantEdition> resolver,
|
||||
Func<string?, string?>? planOverrideResolver = null,
|
||||
Func<string?, int>? purchasedSlotsResolver = null)
|
||||
: this(tenantId => AsOwnResolution(resolver(tenantId)), planOverrideResolver, purchasedSlotsResolver)
|
||||
{
|
||||
}
|
||||
|
||||
/// <param name="resolver">Full resolution per tenant id (edition, source, own standing) — for conferred-Pro cases.</param>
|
||||
/// <param name="planOverrideResolver">See the edition-based constructor.</param>
|
||||
public StubTenantEntitlementService(Func<string?, EditionResolution> resolver, Func<string?, string?>? planOverrideResolver)
|
||||
/// <param name="purchasedSlotsResolver">See the edition-based constructor.</param>
|
||||
public StubTenantEntitlementService(
|
||||
Func<string?, EditionResolution> resolver,
|
||||
Func<string?, string?>? planOverrideResolver,
|
||||
Func<string?, int>? purchasedSlotsResolver = null)
|
||||
: base(configService: null!, logger: NullLogger<TenantEntitlementService>.Instance)
|
||||
{
|
||||
_resolver = resolver;
|
||||
_planOverrideResolver = planOverrideResolver ?? (_ => null);
|
||||
_purchasedSlotsResolver = purchasedSlotsResolver ?? (_ => 0);
|
||||
}
|
||||
|
||||
private static EditionResolution AsOwnResolution(TenantEdition edition) => edition == TenantEdition.Pro
|
||||
@@ -49,4 +59,7 @@ internal sealed class StubTenantEntitlementService : TenantEntitlementService
|
||||
=> Task.FromResult(
|
||||
NormalizePlanName(_planOverrideResolver(tenantId))
|
||||
?? FeatureEntitlementCatalog.Get(_resolver(tenantId)).McpUsagePlanName);
|
||||
|
||||
public override Task<int> GetPurchasedDelegatedSlotsAsync(string? tenantId)
|
||||
=> Task.FromResult(_purchasedSlotsResolver(tenantId));
|
||||
}
|
||||
|
||||
@@ -122,17 +122,18 @@ public class TableUserUsageRepositoryCasTests
|
||||
public async Task Tenant_increment_merges_only_the_counter_and_attribution_columns()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.Tenants.Read = () => TableHarness.Row("tid-customer", "20260906_oid-1", 41, "t1");
|
||||
h.Tenants.Read = () => TableHarness.Row("tid-home", "20260906_oid-1", 41, "t1");
|
||||
|
||||
await h.Repo.IncrementTenantUsageAsync("tid-customer", "oid-1", "msp@example.test", "tid-home");
|
||||
await h.Repo.IncrementTenantUsageAsync("tid-home", "oid-1", "admin@example.test");
|
||||
|
||||
var (entity, etag, mode) = Assert.Single(h.Tenants.Updates);
|
||||
Assert.Equal(TableUpdateMode.Merge, mode);
|
||||
Assert.Equal(new ETag("t1"), etag);
|
||||
Assert.Equal(42L, entity.GetInt64("RequestCount"));
|
||||
Assert.Equal("msp@example.test", entity.GetString("UserPrincipalName"));
|
||||
Assert.Equal("tid-home", entity.GetString("HomeTenantId"));
|
||||
Assert.Equal("admin@example.test", entity.GetString("UserPrincipalName"));
|
||||
Assert.NotNull(entity["LastRequestAt"]);
|
||||
// The counter is the caller's HOME tenant's; there is no foreign-home attribution any more.
|
||||
Assert.False(entity.ContainsKey("HomeTenantId"));
|
||||
// A merge patch never re-sends the identity columns of the row.
|
||||
Assert.False(entity.ContainsKey("Date"));
|
||||
Assert.False(entity.ContainsKey("UserId"));
|
||||
@@ -143,8 +144,8 @@ public class TableUserUsageRepositoryCasTests
|
||||
{
|
||||
var h = new Harness();
|
||||
|
||||
await h.Repo.IncrementTenantUsageAsync("", "oid-1", null, null);
|
||||
await h.Repo.IncrementTenantUsageAsync("tid-customer", " ", null, null);
|
||||
await h.Repo.IncrementTenantUsageAsync("", "oid-1", null);
|
||||
await h.Repo.IncrementTenantUsageAsync("tid-home", " ", null);
|
||||
|
||||
Assert.Empty(h.Tenants.Updates);
|
||||
Assert.Empty(h.Tenants.Added);
|
||||
|
||||
@@ -30,7 +30,7 @@ public class TenantConfigProjectionTests
|
||||
"tenantId", "domainName", "planTier", "trialExpiresUtc", "trialConsumed", "maxDelegatedTenantsOverride",
|
||||
"mcpUsagePlanOverride", "payingCustomer", "managedByProTenantId",
|
||||
"disabled", "disabledReason", "mcpDisabled", "mcpDisabledReason",
|
||||
"onboardedAt", "onboardedBy", "lastUpdated", "dataRetentionDays",
|
||||
"onboardedAt", "onboardedBy", "dpaVersion", "lastUpdated", "dataRetentionDays",
|
||||
"homedAppClientId", "lastAuthClientId", "lastAuthClientIdSince",
|
||||
};
|
||||
|
||||
|
||||
@@ -332,6 +332,54 @@ public class TenantEntitlementServiceTests
|
||||
Assert.Equal(7, await svc.GetMaxDelegatedTenantsAsync(TenantId));
|
||||
}
|
||||
|
||||
// ── Purchased delegation slots (beyond the edition's included ones) ──────────
|
||||
|
||||
[Theory]
|
||||
[InlineData("pro", null, 0)] // the two included slots earn nothing
|
||||
[InlineData("pro", 2, 0)]
|
||||
[InlineData("pro", 5, 3)]
|
||||
[InlineData("pro", 0, 0)] // a lowered limit never goes negative
|
||||
[InlineData("community", null, 0)]
|
||||
[InlineData("community", 4, 0)] // pre-provisioned override without the delegation right: no growth
|
||||
public void GetPurchasedDelegatedSlots_CountsOnlySlotsBeyondTheIncludedOnes(string tier, int? overrideLimit, int expected)
|
||||
{
|
||||
var config = new TenantConfiguration { TenantId = TenantId, PlanTier = tier, MaxDelegatedTenantsOverride = overrideLimit };
|
||||
Assert.Equal(expected, TenantEntitlementService.GetPurchasedDelegatedSlots(config, Now));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetPurchasedDelegatedSlots_ConferredPro_IsZero_DespiteAnOverride()
|
||||
{
|
||||
// Pro by conferral only: no delegation right, so no slot can have been bought — nothing grows.
|
||||
var config = new TenantConfiguration
|
||||
{
|
||||
TenantId = TenantId, PlanTier = "community", ManagedByProTenantId = "99999999-9999-9999-9999-999999999999", MaxDelegatedTenantsOverride = 3,
|
||||
};
|
||||
Assert.Equal(0, TenantEntitlementService.GetPurchasedDelegatedSlots(config, Now));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetPurchasedDelegatedSlots_ActiveTrial_CountsLikePro()
|
||||
{
|
||||
var config = new TenantConfiguration { TenantId = TenantId, PlanTier = "free", TrialExpiresUtc = Now.AddDays(3), MaxDelegatedTenantsOverride = 3 };
|
||||
Assert.Equal(1, TenantEntitlementService.GetPurchasedDelegatedSlots(config, Now));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetPurchasedDelegatedSlotsAsync_NoRow_FailsClosedToZero()
|
||||
{
|
||||
var (svc, _) = Build(config: null);
|
||||
Assert.Equal(0, await svc.GetPurchasedDelegatedSlotsAsync(TenantId));
|
||||
Assert.Equal(0, await svc.GetPurchasedDelegatedSlotsAsync(null));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetPurchasedDelegatedSlotsAsync_ReadsTheOverride()
|
||||
{
|
||||
var (svc, _) = Build(new TenantConfiguration { TenantId = TenantId, PlanTier = "pro", MaxDelegatedTenantsOverride = 7 });
|
||||
Assert.Equal(5, await svc.GetPurchasedDelegatedSlotsAsync(TenantId));
|
||||
}
|
||||
|
||||
// ── MCP usage plan (tenant-wide override) ────────────────────────────────────
|
||||
|
||||
[Theory]
|
||||
|
||||
@@ -1046,4 +1046,210 @@ public sealed class TenantOffboardFunctionTests
|
||||
// No Save call (the tombstone was already correct).
|
||||
configRepoMock.Verify(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>(), It.IsAny<string?>(), It.IsAny<string?>()), Times.Never);
|
||||
}
|
||||
|
||||
// ── RetryFailedOffboardingAsync (operator retry of a Failed offboarding) ──────────
|
||||
|
||||
private const string HistoryRowKey = "20260101000000000_" + TenantId;
|
||||
|
||||
private static FakeOffboardingAuditRepository SeedFailedOffboarding(
|
||||
string failedPhase = "max_dequeue", DateTime? drainCompletedAt = null, int retryCount = 3)
|
||||
{
|
||||
var repo = new FakeOffboardingAuditRepository();
|
||||
repo.Markers[TenantId] = new OffboardingMarkerEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
|
||||
RowKey = TenantId, TenantId = TenantId,
|
||||
OffboardingHistoryRowKey = HistoryRowKey,
|
||||
InitiatedAt = DateTime.UtcNow.AddHours(-3),
|
||||
InitiatedBy = "alice@contoso.com",
|
||||
Status = "Failed",
|
||||
FailedAt = DateTime.UtcNow.AddHours(-2),
|
||||
FailedPhase = failedPhase,
|
||||
};
|
||||
repo.History[HistoryRowKey] = new OffboardingHistoryEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.History,
|
||||
RowKey = HistoryRowKey, TenantId = TenantId,
|
||||
Status = "Failed",
|
||||
ErrorMessage = "Poison queue: max dequeue count reached",
|
||||
RetryCount = retryCount,
|
||||
DrainCompletedAt = drainCompletedAt,
|
||||
EarliestProcessingAt = DateTime.UtcNow.AddHours(-3),
|
||||
};
|
||||
return repo;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Retry_Failed_ResetsHistoryAndMarker_KeepsDrainAnchor_ReEnqueues()
|
||||
{
|
||||
var drained = DateTime.UtcNow.AddHours(-2).AddMinutes(-30);
|
||||
var repo = SeedFailedOffboarding(drainCompletedAt: drained, retryCount: 5);
|
||||
var enqueuer = new RecordingEnqueuer();
|
||||
var sut = Build(repo, enqueuer);
|
||||
|
||||
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
|
||||
|
||||
Assert.Equal(RetryOutcomeKind.Retried, outcome.Kind);
|
||||
Assert.Equal(HistoryRowKey, outcome.HistoryRowKey);
|
||||
Assert.Contains("max_dequeue", outcome.Message);
|
||||
|
||||
var history = repo.History[HistoryRowKey];
|
||||
Assert.Equal("Initiated", history.Status);
|
||||
Assert.Equal(6, history.RetryCount);
|
||||
Assert.Null(history.ErrorMessage);
|
||||
Assert.Equal(drained, history.DrainCompletedAt); // the worker's resume anchor survives
|
||||
Assert.Equal(HistoryRowKey, history.RowKey); // same run, no new history row
|
||||
Assert.Single(repo.History);
|
||||
|
||||
var marker = repo.Markers[TenantId];
|
||||
Assert.Equal("Initiated", marker.Status);
|
||||
Assert.Null(marker.FailedAt);
|
||||
Assert.Null(marker.FailedPhase);
|
||||
|
||||
// History is written before the marker (worker authority first, HTTP anchor second).
|
||||
Assert.Equal(new[] { "Initiated" }, repo.HistoryWrites);
|
||||
Assert.Equal(new[] { "Initiated" }, repo.MarkerWrites);
|
||||
|
||||
var envelope = Assert.Single(enqueuer.Sent);
|
||||
Assert.Equal(TenantId, envelope.TenantId);
|
||||
Assert.Equal(HistoryRowKey, envelope.HistoryRowKey);
|
||||
Assert.Equal("ops@contoso.com", envelope.InitiatedBy);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Retry_Completed_Conflict_NoWrites()
|
||||
{
|
||||
var repo = SeedFailedOffboarding();
|
||||
repo.Markers[TenantId].Status = "Completed";
|
||||
repo.Markers[TenantId].FailedPhase = null;
|
||||
var enqueuer = new RecordingEnqueuer();
|
||||
var sut = Build(repo, enqueuer);
|
||||
|
||||
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
|
||||
|
||||
Assert.Equal(RetryOutcomeKind.NotFailed, outcome.Kind);
|
||||
Assert.Contains("already completed", outcome.Message);
|
||||
Assert.Empty(repo.HistoryWrites);
|
||||
Assert.Empty(repo.MarkerWrites);
|
||||
Assert.Empty(enqueuer.Sent);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("Initiated")]
|
||||
[InlineData("InProgress")]
|
||||
public async Task Retry_InFlight_Conflict_PointsAtTheOffboardResume(string status)
|
||||
{
|
||||
var repo = SeedFailedOffboarding();
|
||||
repo.Markers[TenantId].Status = status;
|
||||
var enqueuer = new RecordingEnqueuer();
|
||||
var sut = Build(repo, enqueuer);
|
||||
|
||||
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
|
||||
|
||||
Assert.Equal(RetryOutcomeKind.NotFailed, outcome.Kind);
|
||||
Assert.Contains(status, outcome.Message);
|
||||
Assert.Equal("Failed", repo.History[HistoryRowKey].Status);
|
||||
Assert.Empty(repo.MarkerWrites);
|
||||
Assert.Empty(enqueuer.Sent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Retry_NoMarker_NotFound()
|
||||
{
|
||||
var enqueuer = new RecordingEnqueuer();
|
||||
var sut = Build(new FakeOffboardingAuditRepository(), enqueuer);
|
||||
|
||||
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
|
||||
|
||||
Assert.Equal(RetryOutcomeKind.NotFound, outcome.Kind);
|
||||
Assert.Empty(enqueuer.Sent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Retry_HistoryRowMissing_FailsBeforeTouchingTheMarker()
|
||||
{
|
||||
var repo = SeedFailedOffboarding();
|
||||
repo.History.Clear();
|
||||
var enqueuer = new RecordingEnqueuer();
|
||||
var sut = Build(repo, enqueuer);
|
||||
|
||||
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
|
||||
|
||||
Assert.Equal(RetryOutcomeKind.HistoryMissing, outcome.Kind);
|
||||
Assert.Equal("Failed", repo.Markers[TenantId].Status);
|
||||
Assert.Empty(repo.MarkerWrites);
|
||||
Assert.Empty(enqueuer.Sent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Retry_EnqueueThrows_LeavesMarkerInitiated_ForTheOffboardResume()
|
||||
{
|
||||
var repo = SeedFailedOffboarding();
|
||||
var enqueuer = new RecordingEnqueuer { ThrowOnEnqueue = new InvalidOperationException("queue down") };
|
||||
var sut = Build(repo, enqueuer);
|
||||
|
||||
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
|
||||
|
||||
Assert.Equal(RetryOutcomeKind.ReEnqueueFailed, outcome.Kind);
|
||||
// Rows are reset: the ordinary DELETE offboard re-click now hits the Initiated resume path.
|
||||
Assert.Equal("Initiated", repo.Markers[TenantId].Status);
|
||||
Assert.Equal("Initiated", repo.History[HistoryRowKey].Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Retry_ResumesThroughTheDisabledGate_WithRemainingBarrierDelay()
|
||||
{
|
||||
// Tombstone written 2 minutes ago → the resume path must wait out the remaining
|
||||
// 4 minutes of the 6-minute drain barrier, not restart it and not skip it.
|
||||
var repo = SeedFailedOffboarding();
|
||||
repo.History[HistoryRowKey].EarliestProcessingAt = DateTime.UtcNow.AddHours(-3);
|
||||
var configRepoMock = new Mock<IConfigRepository>();
|
||||
var tombstone = TenantConfiguration.CreateDefault(TenantId);
|
||||
tombstone.Disabled = true;
|
||||
tombstone.DisabledReason = "Offboarding in progress";
|
||||
tombstone.LastUpdated = DateTime.UtcNow.AddMinutes(-2);
|
||||
configRepoMock.Setup(r => r.GetTenantConfigurationAsync(It.IsAny<string>())).ReturnsAsync(tombstone);
|
||||
var cache = new MemoryCache(new MemoryCacheOptions());
|
||||
var service = new TenantConfigurationService(configRepoMock.Object, NullLogger<TenantConfigurationService>.Instance, cache);
|
||||
var enqueuer = new RecordingEnqueuer();
|
||||
var sut = new TenantOffboardFunction(
|
||||
logger: NullLogger<TenantOffboardFunction>.Instance,
|
||||
configRepo: configRepoMock.Object,
|
||||
tenantConfigService: service,
|
||||
maintenanceRepo: Mock.Of<IMaintenanceRepository>(),
|
||||
offboardingRepo: repo,
|
||||
offboardingEnqueuer: enqueuer,
|
||||
previewWhitelistService: BuildPreviewWhitelistService(configRepoMock));
|
||||
|
||||
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
|
||||
|
||||
Assert.Equal(RetryOutcomeKind.Retried, outcome.Kind);
|
||||
var delay = Assert.Single(enqueuer.VisibilityDelays);
|
||||
Assert.NotNull(delay);
|
||||
Assert.InRange(delay!.Value, TimeSpan.FromMinutes(3.5), TimeSpan.FromMinutes(4.1));
|
||||
configRepoMock.Verify(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>(), It.IsAny<string>(), It.IsAny<string>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildStatusResponse_ProjectsMarkerStatus_AndHistoryDetails()
|
||||
{
|
||||
var repo = SeedFailedOffboarding(retryCount: 5);
|
||||
var marker = repo.Markers[TenantId];
|
||||
var history = repo.History[HistoryRowKey];
|
||||
|
||||
var dto = TenantOffboardFunction.BuildStatusResponse(TenantId, marker, history);
|
||||
|
||||
Assert.Equal("Failed", dto.Status);
|
||||
Assert.Equal("max_dequeue", dto.FailedPhase);
|
||||
Assert.Equal(marker.FailedAt, dto.FailedAt);
|
||||
Assert.Equal(5, dto.RetryCount);
|
||||
Assert.Equal(history.ErrorMessage, dto.ErrorMessage);
|
||||
Assert.Equal(HistoryRowKey, dto.HistoryRowKey);
|
||||
|
||||
// History row gone (should not happen, but the record must still render).
|
||||
var bare = TenantOffboardFunction.BuildStatusResponse(TenantId, marker, null);
|
||||
Assert.Equal("Failed", bare.Status);
|
||||
Assert.Equal(0, bare.RetryCount);
|
||||
Assert.Null(bare.ErrorMessage);
|
||||
}
|
||||
}
|
||||
|
||||
+71
@@ -888,4 +888,75 @@ public class TypedContractCompletionWireParityTests
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// ── global/tenants/{tenantId}/offboarding ────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void TenantOffboardingStatusResponse_pins_the_failed_record_shape()
|
||||
{
|
||||
var initiated = new DateTime(2026, 9, 18, 6, 0, 0, DateTimeKind.Utc);
|
||||
var failed = initiated.AddMinutes(30);
|
||||
AssertWireIdentical(
|
||||
new
|
||||
{
|
||||
tenantId = "11111111-1111-1111-1111-111111111111",
|
||||
status = "Failed",
|
||||
historyRowKey = "20260918060000000_11111111-1111-1111-1111-111111111111",
|
||||
initiatedAt = initiated,
|
||||
initiatedBy = "alice@contoso.com",
|
||||
retryCount = 5,
|
||||
earliestProcessingAt = (DateTime?)initiated.AddMinutes(6),
|
||||
drainCompletedAt = (DateTime?)initiated.AddMinutes(8),
|
||||
completedAt = (DateTime?)null,
|
||||
failedAt = (DateTime?)failed,
|
||||
failedPhase = (string?)"max_dequeue",
|
||||
errorMessage = (string?)"Poison queue: max dequeue count reached",
|
||||
},
|
||||
new TenantOffboardingStatusResponse
|
||||
{
|
||||
TenantId = "11111111-1111-1111-1111-111111111111",
|
||||
Status = "Failed",
|
||||
HistoryRowKey = "20260918060000000_11111111-1111-1111-1111-111111111111",
|
||||
InitiatedAt = initiated,
|
||||
InitiatedBy = "alice@contoso.com",
|
||||
RetryCount = 5,
|
||||
EarliestProcessingAt = initiated.AddMinutes(6),
|
||||
DrainCompletedAt = initiated.AddMinutes(8),
|
||||
CompletedAt = null,
|
||||
FailedAt = failed,
|
||||
FailedPhase = "max_dequeue",
|
||||
ErrorMessage = "Poison queue: max dequeue count reached",
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TenantOffboardingStatusResponse_keeps_null_slots_for_an_in_flight_record()
|
||||
{
|
||||
var initiated = new DateTime(2026, 9, 18, 6, 0, 0, DateTimeKind.Utc);
|
||||
AssertWireIdentical(
|
||||
new
|
||||
{
|
||||
tenantId = "11111111-1111-1111-1111-111111111111",
|
||||
status = "InProgress",
|
||||
historyRowKey = "20260918060000000_11111111-1111-1111-1111-111111111111",
|
||||
initiatedAt = initiated,
|
||||
initiatedBy = "alice@contoso.com",
|
||||
retryCount = 0,
|
||||
earliestProcessingAt = (DateTime?)initiated.AddMinutes(6),
|
||||
drainCompletedAt = (DateTime?)null,
|
||||
completedAt = (DateTime?)null,
|
||||
failedAt = (DateTime?)null,
|
||||
failedPhase = (string?)null,
|
||||
errorMessage = (string?)null,
|
||||
},
|
||||
new TenantOffboardingStatusResponse
|
||||
{
|
||||
TenantId = "11111111-1111-1111-1111-111111111111",
|
||||
Status = "InProgress",
|
||||
HistoryRowKey = "20260918060000000_11111111-1111-1111-1111-111111111111",
|
||||
InitiatedAt = initiated,
|
||||
InitiatedBy = "alice@contoso.com",
|
||||
EarliestProcessingAt = initiated.AddMinutes(6),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
-24
@@ -520,28 +520,6 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<bool> SetTenantGroupChargeHomeTenantQuotaAsync(string groupId, bool chargeHomeTenantQuota)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(groupId))
|
||||
return false;
|
||||
|
||||
try
|
||||
{
|
||||
var result = await _tenantGroupsTableClient.GetEntityAsync<TenantGroupEntity>(
|
||||
groupId, TenantGroupEntity.MetaRowKey);
|
||||
var entity = result.Value;
|
||||
if (entity == null) return false;
|
||||
|
||||
entity.ChargeHomeTenantQuota = chargeHomeTenantQuota;
|
||||
await _tenantGroupsTableClient.UpdateEntityAsync(entity, ETag.All);
|
||||
return true;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 404)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<bool> DeleteTenantGroupAsync(string groupId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(groupId))
|
||||
@@ -694,7 +672,6 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
if (entity.RowKey == TenantGroupEntity.MetaRowKey)
|
||||
{
|
||||
metaBacked = true;
|
||||
membership.ChargeHomeTenantQuota = entity.ChargeHomeTenantQuota ?? false;
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -1044,7 +1021,6 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
group.Name = entity.Name;
|
||||
group.CreatedBy = entity.CreatedBy;
|
||||
group.CreatedAt = entity.CreatedDate ?? default;
|
||||
group.ChargeHomeTenantQuota = entity.ChargeHomeTenantQuota ?? false;
|
||||
group.OwnerTenantId = string.IsNullOrEmpty(entity.OwnerTenantId) ? null : entity.OwnerTenantId;
|
||||
return true;
|
||||
}
|
||||
|
||||
+2
@@ -662,6 +662,7 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
{ "LastUpdated", config.LastUpdated },
|
||||
{ "UpdatedBy", config.UpdatedBy },
|
||||
{ "OnboardedBy", config.OnboardedBy },
|
||||
{ "DpaVersion", config.DpaVersion },
|
||||
{ "ContactEmail", config.ContactEmail },
|
||||
{ "CompanyName", config.CompanyName },
|
||||
{ "Disabled", config.Disabled },
|
||||
@@ -775,6 +776,7 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
LastUpdated = entity.GetDateTime("LastUpdated") ?? DateTime.UtcNow,
|
||||
UpdatedBy = entity.GetString("UpdatedBy") ?? "Unknown",
|
||||
OnboardedBy = entity.GetString("OnboardedBy"),
|
||||
DpaVersion = entity.GetString("DpaVersion"),
|
||||
ContactEmail = entity.GetString("ContactEmail"),
|
||||
CompanyName = entity.GetString("CompanyName"),
|
||||
Disabled = entity.GetBoolean("Disabled") ?? false,
|
||||
|
||||
+11
-4
@@ -56,20 +56,27 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
public Task<PlatformStats?> GetPlatformStatsAsync()
|
||||
=> _storage.GetPlatformStatsAsync();
|
||||
|
||||
public Task<bool> SavePlatformStatsAsync(PlatformStats stats)
|
||||
=> _storage.SavePlatformStatsAsync(stats);
|
||||
public Task<PlatformStats?> RollupPlatformStatsAsync(
|
||||
Func<PlatformStats?, PlatformRollupSources?, PlatformStats> merge, PlatformRollupSources sources)
|
||||
=> _storage.RollupPlatformStatsAsync(merge, sources);
|
||||
|
||||
public Task IncrementPlatformStatAsync(string field, long amount = 1)
|
||||
=> _storage.IncrementPlatformStatAsync(field, amount);
|
||||
|
||||
public Task<long> RecordSeenDeviceModelsAsync(IReadOnlyCollection<string> models)
|
||||
=> _storage.RecordSeenDeviceModelsAsync(models);
|
||||
|
||||
public Task<TenantStats?> GetTenantStatsAsync(string tenantId)
|
||||
=> _storage.GetTenantStatsAsync(tenantId);
|
||||
|
||||
public Task<List<TenantStats>> GetAllTenantStatsAsync()
|
||||
=> _storage.GetAllTenantStatsAsync();
|
||||
|
||||
public Task IncrementTenantStatAsync(string tenantId, string field, long amount = 1)
|
||||
=> _storage.IncrementTenantStatAsync(tenantId, field, amount);
|
||||
|
||||
public Task EnsureTenantStatFloorAsync(string tenantId, string field, long floor)
|
||||
=> _storage.EnsureTenantStatFloorAsync(tenantId, field, floor);
|
||||
public Task<bool> EnsureTenantStatFloorsAsync(string tenantId, IReadOnlyDictionary<string, long> floors)
|
||||
=> _storage.EnsureTenantStatFloorsAsync(tenantId, floors);
|
||||
|
||||
public Task RecordUserLoginAsync(string tenantId, string upn, string? displayName, string? objectId)
|
||||
=> _storage.RecordUserLoginAsync(tenantId, upn, displayName, objectId);
|
||||
|
||||
@@ -77,5 +77,11 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
|
||||
public Task<bool> DeleteImeLogPatternAsync(string tenantId, string patternId)
|
||||
=> _storage.DeleteImeLogPatternAsync(tenantId, patternId);
|
||||
|
||||
public Task<RuleCatalogStamp?> GetRuleCatalogStampAsync(string kind)
|
||||
=> _storage.GetRuleCatalogStampAsync(kind);
|
||||
|
||||
public Task<bool> SetRuleCatalogStampAsync(RuleCatalogStamp stamp)
|
||||
=> _storage.SetRuleCatalogStampAsync(stamp);
|
||||
}
|
||||
}
|
||||
|
||||
+6
@@ -126,6 +126,12 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
// update the chain entry's status. Fail-soft + idempotent; the maintenance
|
||||
// sweep self-heals any miss.
|
||||
await _storage.UpdateDeviceHistoryForSessionAsync(tenantId, sessionId);
|
||||
|
||||
// Cumulative success counter, at the one seam every Succeeded writer passes:
|
||||
// transitioned means the session was not Succeeded before, so each session
|
||||
// counts once. Fail-soft.
|
||||
if (status == SessionStatus.Succeeded)
|
||||
await _storage.IncrementTenantStatAsync(tenantId, nameof(TenantStats.SuccessfulEnrollments));
|
||||
}
|
||||
|
||||
return transitioned;
|
||||
|
||||
+5
-7
@@ -204,7 +204,7 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
|
||||
// ---- McpTenantUsage (organization-wide quota counters) ----
|
||||
|
||||
public Task IncrementTenantUsageAsync(string tenantId, string userId, string? userPrincipalName, string? homeTenantId)
|
||||
public Task IncrementTenantUsageAsync(string tenantId, string userId, string? userPrincipalName)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tenantId) || string.IsNullOrWhiteSpace(userId))
|
||||
return Task.CompletedTask;
|
||||
@@ -220,10 +220,10 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
{
|
||||
["RequestCount"] = ReadCount(read) + 1,
|
||||
["LastRequestAt"] = DateTimeOffset.UtcNow,
|
||||
// Attribution columns (added 2026-09): the UPN as last seen, and the caller's home
|
||||
// tenant when this row was charged by a delegated (MSP) read. Refreshed on every
|
||||
// increment so a row created before the columns existed heals on its next hit.
|
||||
["HomeTenantId"] = homeTenantId ?? string.Empty,
|
||||
// Attribution column (added 2026-09): the UPN as last seen, refreshed on every increment
|
||||
// so a row created before the column existed heals on its next hit. Rows written before
|
||||
// 2026-09-11 may carry a HomeTenantId column (delegated reads used to be charged to the
|
||||
// managed tenant); it is neither written nor read any more and ages out with retention.
|
||||
};
|
||||
if (!string.IsNullOrEmpty(userPrincipalName))
|
||||
patch["UserPrincipalName"] = userPrincipalName;
|
||||
@@ -234,7 +234,6 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
["Date"] = date,
|
||||
["UserId"] = userId,
|
||||
["UserPrincipalName"] = userPrincipalName ?? string.Empty,
|
||||
["HomeTenantId"] = homeTenantId ?? string.Empty,
|
||||
["RequestCount"] = 1L,
|
||||
["LastRequestAt"] = DateTimeOffset.UtcNow,
|
||||
},
|
||||
@@ -258,7 +257,6 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
TenantId = entity.PartitionKey,
|
||||
UserId = entity.GetString("UserId") ?? string.Empty,
|
||||
UserPrincipalName = entity.GetString("UserPrincipalName") ?? string.Empty,
|
||||
HomeTenantId = entity.GetString("HomeTenantId") ?? string.Empty,
|
||||
Date = entity.GetString("Date") ?? entity.RowKey.Split('_')[0],
|
||||
RequestCount = entity.TryGetValue("RequestCount", out var rc) ? Convert.ToInt64(rc) : 0L,
|
||||
LastRequestAt = entity.GetDateTimeOffset("LastRequestAt")?.UtcDateTime,
|
||||
|
||||
+2
-46
@@ -2,7 +2,6 @@ using System.Net;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
@@ -19,27 +18,18 @@ namespace AutopilotMonitor.Functions.Functions.Admin;
|
||||
/// </summary>
|
||||
public class DelegationSelfServiceFunction
|
||||
{
|
||||
/// <summary>Managed-tenant MCP usage is resolved for at most this many tenants per listing (Global Admin overrides can be large).</summary>
|
||||
internal const int ManagedUsageCap = 25;
|
||||
|
||||
private readonly ILogger<DelegationSelfServiceFunction> _logger;
|
||||
private readonly DelegationSelfService _svc;
|
||||
private readonly DelegatedSlotService _slots;
|
||||
private readonly McpQuotaService _quota;
|
||||
private readonly IUserUsageRepository _usage;
|
||||
|
||||
public DelegationSelfServiceFunction(
|
||||
ILogger<DelegationSelfServiceFunction> logger,
|
||||
DelegationSelfService svc,
|
||||
DelegatedSlotService slots,
|
||||
McpQuotaService quota,
|
||||
IUserUsageRepository usage)
|
||||
DelegatedSlotService slots)
|
||||
{
|
||||
_logger = logger;
|
||||
_svc = svc;
|
||||
_slots = slots;
|
||||
_quota = quota;
|
||||
_usage = usage;
|
||||
}
|
||||
|
||||
// ── Slots + managed tenants ──────────────────────────────────────────────────
|
||||
@@ -54,7 +44,7 @@ public class DelegationSelfServiceFunction
|
||||
return await OkAsync(req, DelegatedSlotManagementFunction.ToResponse(usage));
|
||||
}
|
||||
|
||||
/// <summary>GET /api/delegations/managed — the tenants the caller's tenant manages, with their MCP organization budget.</summary>
|
||||
/// <summary>GET /api/delegations/managed — the tenants the caller's tenant manages.</summary>
|
||||
[Function("GetDelegationManagedTenants")]
|
||||
[Authorize]
|
||||
public async Task<HttpResponseData> GetManaged(
|
||||
@@ -64,15 +54,8 @@ public class DelegationSelfServiceFunction
|
||||
var view = await _svc.ListManagedAsync(home);
|
||||
|
||||
var items = new List<ManagedTenantItem>(view.Tenants.Count);
|
||||
var resolved = 0;
|
||||
foreach (var t in view.Tenants)
|
||||
{
|
||||
ManagedTenantQuotaUsage? usage = null;
|
||||
if (resolved < ManagedUsageCap)
|
||||
{
|
||||
usage = await ReadManagedUsageAsync(t.TenantId);
|
||||
resolved++;
|
||||
}
|
||||
items.Add(new ManagedTenantItem
|
||||
{
|
||||
TenantId = t.TenantId,
|
||||
@@ -80,7 +63,6 @@ public class DelegationSelfServiceFunction
|
||||
Source = t.Source,
|
||||
SinceUtc = t.SinceUtc,
|
||||
Removable = t.Removable,
|
||||
Usage = usage,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -268,32 +250,6 @@ public class DelegationSelfServiceFunction
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>The managed tenant's organization MCP budget — its own plan's windows and counters.</summary>
|
||||
private async Task<ManagedTenantQuotaUsage?> ReadManagedUsageAsync(string tenantId)
|
||||
{
|
||||
try
|
||||
{
|
||||
var limits = await _quota.ResolvePlanAsync(null, tenantId);
|
||||
var nowUtc = DateTime.UtcNow;
|
||||
var monthStart = new DateTime(nowUtc.Year, nowUtc.Month, 1, 0, 0, 0, DateTimeKind.Utc).ToString("yyyyMMdd");
|
||||
var today = nowUtc.ToString("yyyyMMdd");
|
||||
var rows = await _usage.GetTenantUsageAsync(tenantId, monthStart, today);
|
||||
return new ManagedTenantQuotaUsage
|
||||
{
|
||||
TenantPlan = limits.TenantPlan,
|
||||
TenantDailyLimit = limits.TenantDailyLimit,
|
||||
TenantMonthlyLimit = limits.TenantMonthlyLimit,
|
||||
TenantDailyUsed = rows.Where(r => r.Date == today).Sum(r => r.RequestCount),
|
||||
TenantMonthlyUsed = rows.Sum(r => r.RequestCount),
|
||||
};
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "[Delegation] Managed tenant usage lookup failed for {TenantId}", tenantId);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
internal static async Task<HttpResponseData> FailAsync(HttpRequestData req, DelegationFailure failure)
|
||||
{
|
||||
if (failure.SlotViolation != null)
|
||||
|
||||
@@ -199,6 +199,8 @@ namespace AutopilotMonitor.Functions.Functions.Admin
|
||||
_logger.LogInformation(
|
||||
"GitHub reseed gather: {Deleted} deleted, {Written} written, {OrphanCount} orphan per-tenant RuleState(s) cleaned across {SunsetTotal} sunset rule(s) ({Skipped} skipped on failure for retry)",
|
||||
deleted, rules.Count, orphanStatesGcd, sunset.Count, sunsetSkipped);
|
||||
await _ruleRepo.SetRuleCatalogStampAsync(AutopilotMonitor.Functions.Services.RuleCatalogSeedGate.Stamp(
|
||||
AutopilotMonitor.Shared.Models.RuleCatalogStamp.KindGather, AutopilotMonitor.Shared.Models.RuleCatalogStamp.SourceGitHub, rules.Count));
|
||||
return (deleted, rules.Count, orphanStatesGcd, sunsetSkipped);
|
||||
}
|
||||
|
||||
@@ -279,6 +281,8 @@ namespace AutopilotMonitor.Functions.Functions.Admin
|
||||
_logger.LogInformation(
|
||||
"GitHub reseed analyze: {Deleted} deleted, {Written} written, {OrphanCount} orphan per-tenant RuleState(s) cleaned across {SunsetTotal} sunset rule(s) ({Skipped} skipped on failure for retry)",
|
||||
deleted, rules.Count, orphanStatesGcd, sunset.Count, sunsetSkipped);
|
||||
await _ruleRepo.SetRuleCatalogStampAsync(AutopilotMonitor.Functions.Services.RuleCatalogSeedGate.Stamp(
|
||||
AutopilotMonitor.Shared.Models.RuleCatalogStamp.KindAnalyze, AutopilotMonitor.Shared.Models.RuleCatalogStamp.SourceGitHub, rules.Count));
|
||||
return (deleted, rules.Count, orphanStatesGcd, sunsetSkipped);
|
||||
}
|
||||
|
||||
@@ -299,6 +303,8 @@ namespace AutopilotMonitor.Functions.Functions.Admin
|
||||
}
|
||||
|
||||
_logger.LogInformation($"GitHub reseed IME: {deleted} deleted, {patterns.Count} written");
|
||||
await _ruleRepo.SetRuleCatalogStampAsync(AutopilotMonitor.Functions.Services.RuleCatalogSeedGate.Stamp(
|
||||
AutopilotMonitor.Shared.Models.RuleCatalogStamp.KindIme, AutopilotMonitor.Shared.Models.RuleCatalogStamp.SourceGitHub, patterns.Count));
|
||||
return (deleted, patterns.Count);
|
||||
}
|
||||
|
||||
|
||||
+6
-44
@@ -100,10 +100,8 @@ public class TenantGroupManagementFunction
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// PATCH /api/global/tenant-groups/{groupId} — update group metadata. GlobalAdminOnly.
|
||||
/// Body: { "name"?: "...", "chargeHomeTenantQuota"?: true|false } — at least one field. A rename has no
|
||||
/// scope effect; the quota flag re-attributes every assignee's MCP reads into the group's tenants (charged
|
||||
/// to the assignee's home tenant instead of the managed tenant) and is audited under each tenant.
|
||||
/// PATCH /api/global/tenant-groups/{groupId} — rename a group. GlobalAdminOnly.
|
||||
/// Body: { "name": "..." }. A rename has no scope effect.
|
||||
/// </summary>
|
||||
[Function("UpdateTenantGroup")]
|
||||
[Authorize]
|
||||
@@ -116,52 +114,16 @@ public class TenantGroupManagementFunction
|
||||
var read = await req.ReadAsync<UpdateTenantGroupRequest>();
|
||||
if (read.Error != null) return read.Error;
|
||||
var body = read.Value!;
|
||||
if (string.IsNullOrWhiteSpace(body.Name) && body.ChargeHomeTenantQuota == null)
|
||||
return await Bad(req, "name and/or chargeHomeTenantQuota is required");
|
||||
if (string.IsNullOrWhiteSpace(body.Name))
|
||||
return await Bad(req, "name is required");
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(body.Name))
|
||||
{
|
||||
if (!await _delegatedAdminService.RenameGroupAsync(groupId, body.Name))
|
||||
return await NotFound(req);
|
||||
_logger.LogInformation("Tenant group renamed: {GroupId} -> '{Name}' by {By}", groupId, body.Name, currentUpn);
|
||||
}
|
||||
|
||||
if (body.ChargeHomeTenantQuota is bool chargeHomeTenantQuota
|
||||
&& !await SetChargeModeCoreAsync(groupId, chargeHomeTenantQuota, currentUpn))
|
||||
if (!await _delegatedAdminService.RenameGroupAsync(groupId, body.Name))
|
||||
return await NotFound(req);
|
||||
_logger.LogInformation("Tenant group renamed: {GroupId} -> '{Name}' by {By}", groupId, body.Name, currentUpn);
|
||||
|
||||
return await req.OkAsync(new MessageResponse { Message = "Group updated" });
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Testable core of the quota-charge-mode flip (no HTTP plumbing). Returns false when the group does not
|
||||
/// exist. An unchanged value is a no-op (no audit); a real change is audited under EVERY tenant in the group
|
||||
/// — the customer's trail records whose budget the group's assignees draw on from now on.
|
||||
/// </summary>
|
||||
internal async Task<bool> SetChargeModeCoreAsync(string groupId, bool chargeHomeTenantQuota, string? currentUpn)
|
||||
{
|
||||
var group = await _delegatedAdminService.GetGroupAsync(groupId);
|
||||
if (group == null)
|
||||
return false;
|
||||
if (group.ChargeHomeTenantQuota == chargeHomeTenantQuota)
|
||||
return true;
|
||||
|
||||
if (!await _delegatedAdminService.SetChargeHomeTenantQuotaAsync(groupId, chargeHomeTenantQuota))
|
||||
return false;
|
||||
|
||||
await AuditPerTenantAsync(group.TenantIds, "UPDATE", "*", currentUpn,
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
{ "Group", group.Name },
|
||||
{ "GroupId", groupId },
|
||||
{ "Reason", "quota-charge-mode-changed" },
|
||||
{ "ChargeHomeTenantQuota", chargeHomeTenantQuota ? "true" : "false" },
|
||||
});
|
||||
|
||||
_logger.LogInformation("Tenant group {GroupId} chargeHomeTenantQuota={Flag} by {By}", groupId, chargeHomeTenantQuota, currentUpn);
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>DELETE /api/global/tenant-groups/{groupId} — delete group + all its assignments. GlobalAdminOnly.</summary>
|
||||
[Function("DeleteTenantGroup")]
|
||||
[Authorize]
|
||||
|
||||
@@ -506,6 +506,194 @@ public class TenantOffboardFunction
|
||||
EarliestProcessingAt: earliestProcessingAtForResponse);
|
||||
}
|
||||
|
||||
// ── Offboarding record + operator retry (Global Admin tooling) ────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// GET /api/global/tenants/{tenantId}/offboarding — the tenant's current offboarding record
|
||||
/// (marker + history) for the admin tenant editor. 404 without a marker: the tenant was
|
||||
/// never offboarded, or the Completed marker is already cleaned up
|
||||
/// (<see cref="OffboardingMarkerCleanupFunction"/>, 15-min grace) — the normal end state,
|
||||
/// nothing for an operator to do. Global Admin / Global Reader only (middleware-enforced).
|
||||
/// </summary>
|
||||
[Function("GetTenantOffboardingStatus")]
|
||||
[Authorize]
|
||||
public async Task<HttpResponseData> GetTenantOffboardingStatus(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "global/tenants/{tenantId}/offboarding")] HttpRequestData req,
|
||||
string tenantId,
|
||||
FunctionContext context)
|
||||
{
|
||||
var targetTenantId = context.GetRequestContext().TargetTenantId;
|
||||
if (string.IsNullOrEmpty(targetTenantId) || !SecurityValidator.IsValidGuid(targetTenantId))
|
||||
{
|
||||
return await req.BadRequestAsync("tenantId must be a valid GUID");
|
||||
}
|
||||
|
||||
var normalizedTenantId = targetTenantId.ToLowerInvariant();
|
||||
var marker = await _offboardingRepo.TryGetMarkerAsync(normalizedTenantId);
|
||||
if (marker == null)
|
||||
{
|
||||
return await req.NotFoundAsync("No offboarding on record for this tenant");
|
||||
}
|
||||
|
||||
var history = await _offboardingRepo.TryGetHistoryAsync(marker.OffboardingHistoryRowKey);
|
||||
return await req.OkAsync(BuildStatusResponse(normalizedTenantId, marker, history));
|
||||
}
|
||||
|
||||
internal static TenantOffboardingStatusResponse BuildStatusResponse(
|
||||
string normalizedTenantId, OffboardingMarkerEntry marker, OffboardingHistoryEntry? history)
|
||||
=> new()
|
||||
{
|
||||
TenantId = normalizedTenantId,
|
||||
Status = marker.Status,
|
||||
HistoryRowKey = marker.OffboardingHistoryRowKey,
|
||||
InitiatedAt = marker.InitiatedAt,
|
||||
InitiatedBy = marker.InitiatedBy,
|
||||
RetryCount = history?.RetryCount ?? 0,
|
||||
EarliestProcessingAt = history?.EarliestProcessingAt,
|
||||
DrainCompletedAt = history?.DrainCompletedAt,
|
||||
CompletedAt = marker.CompletedAt,
|
||||
FailedAt = marker.FailedAt,
|
||||
FailedPhase = marker.FailedPhase,
|
||||
ErrorMessage = history?.ErrorMessage,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// POST /api/global/tenants/{tenantId}/offboarding/retry — re-drives an offboarding whose
|
||||
/// marker is Failed. Failed is fail-closed by design (the worker returns, the offboard
|
||||
/// re-click refuses to re-enqueue), so this is the explicit operator action that design
|
||||
/// asks for; nothing retries on its own. History and marker go back to Initiated — the
|
||||
/// History row keeps its RowKey and <c>DrainCompletedAt</c> (the worker's resume anchor)
|
||||
/// and counts the retry — and the ordinary resume path re-affirms the Disabled-gate,
|
||||
/// computes the remaining drain barrier and enqueues the envelope. Global Admin only.
|
||||
/// </summary>
|
||||
[Function("RetryTenantOffboarding")]
|
||||
[Authorize]
|
||||
public async Task<HttpResponseData> RetryTenantOffboarding(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "global/tenants/{tenantId}/offboarding/retry")] HttpRequestData req,
|
||||
string tenantId,
|
||||
FunctionContext context)
|
||||
{
|
||||
var requestCtx = context.GetRequestContext();
|
||||
var upn = requestCtx.UserPrincipalName;
|
||||
var targetTenantId = requestCtx.TargetTenantId;
|
||||
|
||||
if (string.IsNullOrEmpty(targetTenantId) || !SecurityValidator.IsValidGuid(targetTenantId))
|
||||
{
|
||||
return await req.BadRequestAsync("tenantId must be a valid GUID");
|
||||
}
|
||||
|
||||
var normalizedTenantId = targetTenantId.ToLowerInvariant();
|
||||
var retry = await RetryFailedOffboardingAsync(normalizedTenantId, upn);
|
||||
|
||||
switch (retry.Kind)
|
||||
{
|
||||
case RetryOutcomeKind.Retried:
|
||||
{
|
||||
var response = req.CreateResponse(HttpStatusCode.Accepted);
|
||||
await response.WriteAsJsonAsync(new OffboardResponse
|
||||
{
|
||||
TenantId = normalizedTenantId,
|
||||
Status = "Queued",
|
||||
HistoryPartitionKey = Constants.OffboardingPartitionKeys.History,
|
||||
HistoryRowKey = retry.HistoryRowKey ?? string.Empty,
|
||||
EarliestProcessingAt = retry.EarliestProcessingAt,
|
||||
Message = retry.Message,
|
||||
});
|
||||
return response;
|
||||
}
|
||||
case RetryOutcomeKind.NotFound:
|
||||
return await req.NotFoundAsync(retry.Message);
|
||||
case RetryOutcomeKind.NotFailed:
|
||||
return await req.ConflictAsync(retry.Message);
|
||||
default:
|
||||
return await Build500Async(req, retry.Message);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pure retry decision behind <see cref="RetryTenantOffboarding"/>; internal for unit tests.
|
||||
/// Write order matters: History first (the worker's authority — a Failed history makes
|
||||
/// it return early), Marker second (the HTTP idempotency anchor). A crash in between
|
||||
/// leaves History=Initiated + Marker=Failed, and the next retry click resolves it the
|
||||
/// same way. The worker's first pickup then flips History/Pointer/Marker to InProgress.
|
||||
/// </summary>
|
||||
internal async Task<RetryOutcome> RetryFailedOffboardingAsync(string normalizedTenantId, string upn)
|
||||
{
|
||||
var marker = await _offboardingRepo.TryGetMarkerAsync(normalizedTenantId);
|
||||
if (marker == null)
|
||||
{
|
||||
return new RetryOutcome(RetryOutcomeKind.NotFound, "No offboarding on record for this tenant");
|
||||
}
|
||||
|
||||
if (!string.Equals(marker.Status, "Failed", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var message = string.Equals(marker.Status, "Completed", StringComparison.OrdinalIgnoreCase)
|
||||
? "Tenant offboarding already completed — nothing to retry"
|
||||
: $"Tenant offboarding is {marker.Status} — not Failed; re-run the offboard action to resume it";
|
||||
_logger.LogInformation(
|
||||
"TenantOffboard retry for {TenantId} refused: marker status={Status}", normalizedTenantId, marker.Status);
|
||||
return new RetryOutcome(RetryOutcomeKind.NotFailed, message, marker.OffboardingHistoryRowKey);
|
||||
}
|
||||
|
||||
var history = await _offboardingRepo.TryGetHistoryAsync(marker.OffboardingHistoryRowKey);
|
||||
if (history == null)
|
||||
{
|
||||
_logger.LogError(
|
||||
"TenantOffboard retry for {TenantId}: History row {History} is missing — operator inspection required",
|
||||
normalizedTenantId, marker.OffboardingHistoryRowKey);
|
||||
return new RetryOutcome(
|
||||
RetryOutcomeKind.HistoryMissing,
|
||||
"Offboarding history row is missing — operator inspection required",
|
||||
marker.OffboardingHistoryRowKey);
|
||||
}
|
||||
|
||||
var failedPhase = marker.FailedPhase ?? "unknown";
|
||||
|
||||
history.Status = "Initiated";
|
||||
history.RetryCount += 1;
|
||||
history.ErrorMessage = null;
|
||||
await _offboardingRepo.UpsertHistoryAsync(history);
|
||||
|
||||
marker.Status = "Initiated";
|
||||
marker.FailedAt = null;
|
||||
marker.FailedPhase = null;
|
||||
await _offboardingRepo.UpsertMarkerAsync(marker);
|
||||
|
||||
_logger.LogWarning(
|
||||
"TENANT OFFBOARD RETRY for {TenantId} by {Upn}: previous failure phase={Phase}; historyRowKey={History} retryCount={RetryCount}",
|
||||
normalizedTenantId, upn, failedPhase, history.RowKey, history.RetryCount);
|
||||
|
||||
// Global audit row (survives the TenantConfiguration wipe like the initiation's DELETE row).
|
||||
await _maintenanceRepo.LogAuditEntryAsync(
|
||||
Constants.AuditGlobalTenantId,
|
||||
"RETRY",
|
||||
"TenantOffboarding",
|
||||
normalizedTenantId,
|
||||
upn,
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["failedPhase"] = failedPhase,
|
||||
["historyRowKey"] = history.RowKey,
|
||||
["retryCount"] = history.RetryCount.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
});
|
||||
|
||||
var resume = await ResumeExistingMarkerAsync(marker, upn, normalizedTenantId, isRace: false);
|
||||
if (resume.Kind == ResumeOutcomeKind.ReEnqueueFailed)
|
||||
{
|
||||
// Marker is Initiated now, so the ordinary offboard re-click resumes it.
|
||||
return new RetryOutcome(
|
||||
RetryOutcomeKind.ReEnqueueFailed,
|
||||
"Failed to re-enqueue tenant offboarding — re-run the offboard action to resume it",
|
||||
history.RowKey);
|
||||
}
|
||||
|
||||
return new RetryOutcome(
|
||||
RetryOutcomeKind.Retried,
|
||||
$"Tenant offboarding retried after the failure in phase '{failedPhase}'",
|
||||
history.RowKey,
|
||||
resume.EarliestProcessingAt);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Magic-string DisabledReason that identifies a TenantConfiguration row as the
|
||||
/// offboarding tombstone. Used by <c>EnsureTenantDisabledAsync</c> for the idempotent
|
||||
@@ -775,3 +963,24 @@ internal enum ResumeOutcomeKind
|
||||
/// <summary>Marker classified as in-flight but re-enqueue threw; surface 500.</summary>
|
||||
ReEnqueueFailed,
|
||||
}
|
||||
|
||||
/// <summary>Result of <see cref="TenantOffboardFunction.RetryFailedOffboardingAsync"/>.</summary>
|
||||
internal sealed record RetryOutcome(
|
||||
RetryOutcomeKind Kind,
|
||||
string Message,
|
||||
string? HistoryRowKey = null,
|
||||
DateTime? EarliestProcessingAt = null);
|
||||
|
||||
internal enum RetryOutcomeKind
|
||||
{
|
||||
/// <summary>Marker was Failed; history + marker reset to Initiated and the envelope re-enqueued (202).</summary>
|
||||
Retried,
|
||||
/// <summary>No marker on record (404).</summary>
|
||||
NotFound,
|
||||
/// <summary>Marker is Completed or in flight — nothing to retry (409).</summary>
|
||||
NotFailed,
|
||||
/// <summary>Marker points at a History row that no longer exists (500).</summary>
|
||||
HistoryMissing,
|
||||
/// <summary>Rows were reset but the re-enqueue threw (500); the offboard re-click resumes.</summary>
|
||||
ReEnqueueFailed,
|
||||
}
|
||||
|
||||
+34
-32
@@ -2,7 +2,7 @@ using System;
|
||||
using System.Net;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Maintenance;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
@@ -11,24 +11,29 @@ using Microsoft.Extensions.Logging;
|
||||
namespace AutopilotMonitor.Functions.Functions.Admin
|
||||
{
|
||||
/// <summary>
|
||||
/// Allows Global Admins to manually trigger maintenance tasks
|
||||
/// <c>POST /api/maintenance/trigger</c> — manual Global Admin trigger of the platform
|
||||
/// maintenance run. The run takes minutes, so the request only queues it:
|
||||
/// <c>202 Accepted</c> when queued, <c>409</c> while a run (timer or manual) is active,
|
||||
/// <c>500</c> when the enqueue failed — never a hollow success. Progress and the run report
|
||||
/// surface as <c>Maintenance*</c> ops events.
|
||||
/// </summary>
|
||||
public class TriggerMaintenanceFunction
|
||||
{
|
||||
private readonly ILogger<TriggerMaintenanceFunction> _logger;
|
||||
private readonly MaintenanceService _maintenanceService;
|
||||
private readonly IMaintenanceTriggerProducer _producer;
|
||||
private readonly MaintenanceRunGate _runGate;
|
||||
|
||||
public TriggerMaintenanceFunction(
|
||||
ILogger<TriggerMaintenanceFunction> logger,
|
||||
MaintenanceService maintenanceService)
|
||||
IMaintenanceTriggerProducer producer,
|
||||
MaintenanceRunGate runGate)
|
||||
{
|
||||
_logger = logger;
|
||||
_maintenanceService = maintenanceService;
|
||||
_producer = producer;
|
||||
_runGate = runGate;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// POST /api/maintenance/trigger
|
||||
/// Manually trigger maintenance tasks (Global Admin only)
|
||||
/// Query parameters:
|
||||
/// - date: Optional date to aggregate (yyyy-MM-dd). If not provided, uses yesterday.
|
||||
/// - aggregateOnly: If true, only runs aggregation (skips timeout and cleanup)
|
||||
@@ -38,45 +43,42 @@ namespace AutopilotMonitor.Functions.Functions.Admin
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "maintenance/trigger")] HttpRequestData req,
|
||||
FunctionContext context)
|
||||
{
|
||||
_logger.LogInformation("Manual maintenance trigger requested");
|
||||
|
||||
// Authentication + GlobalAdminOnly authorization enforced by PolicyEnforcementMiddleware
|
||||
var userEmail = TenantHelper.GetUserIdentifier(req);
|
||||
|
||||
_logger.LogInformation($"Maintenance trigger initiated by Global Admin: {userEmail}");
|
||||
var userEmail = TenantHelper.GetUserIdentifier(req) ?? "GlobalAdmin";
|
||||
var ct = context.CancellationToken;
|
||||
|
||||
try
|
||||
{
|
||||
// Parse query parameters
|
||||
var dateParam = req.Query["date"];
|
||||
var aggregateOnlyParam = req.Query["aggregateOnly"];
|
||||
|
||||
if (!QueryParams.TryUtcInstant(dateParam, "date", out var dateInstant, out _))
|
||||
if (!QueryParams.TryUtcInstant(req.Query["date"], "date", out var dateInstant, out _))
|
||||
{
|
||||
return await req.BadRequestAsync("Invalid date format. Use yyyy-MM-dd");
|
||||
}
|
||||
DateTime? targetDate = dateInstant?.Date;
|
||||
if (targetDate.HasValue)
|
||||
bool aggregateOnly = req.Query["aggregateOnly"]?.ToLower() == "true";
|
||||
|
||||
if (await _runGate.IsRunActiveAsync(ct))
|
||||
{
|
||||
_logger.LogInformation($"Manual maintenance for date: {targetDate:yyyy-MM-dd}");
|
||||
return await req.ConflictAsync("A maintenance run is already active.");
|
||||
}
|
||||
|
||||
bool aggregateOnly = aggregateOnlyParam?.ToLower() == "true";
|
||||
|
||||
// Execute maintenance tasks
|
||||
var result = await _maintenanceService.RunManualAsync(targetDate, aggregateOnly, userEmail);
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
await response.WriteAsJsonAsync(new TriggerMaintenanceResponse
|
||||
await _producer.EnqueueAsync(new MaintenanceTriggerEnvelope
|
||||
{
|
||||
Success = true,
|
||||
Message = "Maintenance tasks completed",
|
||||
Result = result,
|
||||
TriggeredBy = userEmail,
|
||||
TriggeredAt = DateTime.UtcNow
|
||||
});
|
||||
TargetDate = targetDate,
|
||||
AggregateOnly = aggregateOnly,
|
||||
}, ct);
|
||||
|
||||
return response;
|
||||
_logger.LogInformation("TriggerMaintenance: run queued (triggeredBy={TriggeredBy}, date={Date:yyyy-MM-dd}, aggregateOnly={AggregateOnly})",
|
||||
userEmail, targetDate, aggregateOnly);
|
||||
|
||||
return await req.JsonAsync(HttpStatusCode.Accepted, new TriggerMaintenanceResponse
|
||||
{
|
||||
Message = "Maintenance run queued",
|
||||
TriggeredBy = userEmail,
|
||||
TriggeredAt = DateTime.UtcNow,
|
||||
TargetDate = targetDate?.ToString("yyyy-MM-dd"),
|
||||
AggregateOnly = aggregateOnly,
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
@@ -528,9 +528,31 @@ namespace AutopilotMonitor.Functions.Functions.Config
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The built-in catalog values per edition — what a blank field of the same-named plan definition
|
||||
/// falls back to (McpQuotaService). Sent with the definitions so the operator sees what "unset" means.
|
||||
/// </summary>
|
||||
internal static IReadOnlyList<UsagePlanCatalogDefaults> BuildUsagePlanCatalog()
|
||||
=> new[] { TenantEdition.Community, TenantEdition.Pro }
|
||||
.Select(FeatureEntitlementCatalog.Get)
|
||||
.Select(e => new UsagePlanCatalogDefaults
|
||||
{
|
||||
Edition = e.McpUsagePlanName,
|
||||
DailyRequestLimit = e.McpDailyRequestLimit,
|
||||
MonthlyRequestLimit = e.McpMonthlyRequestLimit,
|
||||
TenantDailyRequestLimit = e.McpTenantDailyRequestLimit,
|
||||
TenantMonthlyRequestLimit = e.McpTenantMonthlyRequestLimit,
|
||||
IncludedDelegatedSlots = e.MaxDelegatedTenants,
|
||||
SlotDailyRequestLimit = e.McpSlotDailyRequestLimit,
|
||||
SlotMonthlyRequestLimit = e.McpSlotMonthlyRequestLimit,
|
||||
SlotTenantDailyRequestLimit = e.McpSlotTenantDailyRequestLimit,
|
||||
SlotTenantMonthlyRequestLimit = e.McpSlotTenantMonthlyRequestLimit,
|
||||
})
|
||||
.ToList();
|
||||
|
||||
/// <summary>
|
||||
/// GET /api/global/config/plan-tiers
|
||||
/// Returns plan tier definitions from AdminConfiguration.PlanTierDefinitionsJson.
|
||||
/// Returns plan tier definitions from AdminConfiguration.PlanTierDefinitionsJson plus the catalog fallbacks.
|
||||
/// </summary>
|
||||
[Function("GetPlanTierDefinitions")]
|
||||
public async Task<HttpResponseData> GetPlanTierDefinitions(
|
||||
@@ -542,7 +564,7 @@ namespace AutopilotMonitor.Functions.Functions.Config
|
||||
var tiers = PlanTierDefinitionParser.Parse(config.PlanTierDefinitionsJson);
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
await response.WriteAsJsonAsync(new PlanTierDefinitionsResponse { Tiers = tiers });
|
||||
await response.WriteAsJsonAsync(new PlanTierDefinitionsResponse { Tiers = tiers, Catalog = BuildUsagePlanCatalog() });
|
||||
return response;
|
||||
}
|
||||
catch (Exception ex)
|
||||
@@ -585,7 +607,7 @@ namespace AutopilotMonitor.Functions.Functions.Config
|
||||
await _adminConfigService.SaveConfigurationAsync(config);
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
await response.WriteAsJsonAsync(new PlanTierDefinitionsResponse { Tiers = body.Tiers });
|
||||
await response.WriteAsJsonAsync(new PlanTierDefinitionsResponse { Tiers = body.Tiers, Catalog = BuildUsagePlanCatalog() });
|
||||
return response;
|
||||
}
|
||||
catch (Exception ex)
|
||||
|
||||
@@ -387,6 +387,10 @@ public class AuthFunction
|
||||
// sync jobs that mutate UpdatedBy cannot leak sentinel strings into TenantAdmins.
|
||||
if (string.IsNullOrWhiteSpace(tenantConfig.OnboardedBy))
|
||||
tenantConfig.OnboardedBy = upn;
|
||||
// The DPA is accepted with the onboarding itself (D-251/D-252): record the version in force,
|
||||
// once, in this same first write — later logins never touch it.
|
||||
if (string.IsNullOrWhiteSpace(tenantConfig.DpaVersion))
|
||||
tenantConfig.DpaVersion = Constants.CurrentDpaVersion;
|
||||
// Dual app-reg window: home a NEW tenant on the primary app ONLY when its first login
|
||||
// actually arrived via the primary app. A first login via the legacy app leaves the field
|
||||
// null (= legacy) — keeps the "null = legacy" invariant clean and never routes a tenant
|
||||
|
||||
+19
-15
@@ -85,7 +85,12 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
|
||||
TenantDailyLimit = quota.TenantDailyLimit,
|
||||
TenantMonthlyLimit = quota.TenantMonthlyLimit,
|
||||
TenantDailyUsed = quota.TenantDailyUsed,
|
||||
TenantMonthlyUsed = quota.TenantMonthlyUsed
|
||||
TenantMonthlyUsed = quota.TenantMonthlyUsed,
|
||||
PurchasedDelegatedSlots = WithSlots(quota.PurchasedDelegatedSlots, quota.PurchasedDelegatedSlots),
|
||||
SlotDailyLimit = WithSlots(quota.PurchasedDelegatedSlots, quota.SlotDailyLimit),
|
||||
SlotMonthlyLimit = WithSlots(quota.PurchasedDelegatedSlots, quota.SlotMonthlyLimit),
|
||||
SlotTenantDailyLimit = WithSlots(quota.PurchasedDelegatedSlots, quota.SlotTenantDailyLimit),
|
||||
SlotTenantMonthlyLimit = WithSlots(quota.PurchasedDelegatedSlots, quota.SlotTenantMonthlyLimit),
|
||||
},
|
||||
Records = records
|
||||
});
|
||||
@@ -99,10 +104,10 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
|
||||
|
||||
/// <summary>
|
||||
/// GET /api/metrics/mcp-usage/organization?dateFrom=&dateTo= — the caller's OWN tenant's organization
|
||||
/// budget broken down by account: its members and any delegated (MSP) administrators whose reads were
|
||||
/// charged to this tenant (flagged, with their home tenant). Always the JWT tenant (TenantScoping.None)
|
||||
/// — a delegated caller can never list a managed tenant's accounts through this route. Reads the
|
||||
/// organization counters (McpTenantUsage, one partition range), never the per-user log.
|
||||
/// budget broken down by account (every MCP request of a member is charged here, reads into managed
|
||||
/// tenants included). Always the JWT tenant (TenantScoping.None) — a delegated caller can never list a
|
||||
/// managed tenant's accounts through this route. Reads the organization counters (McpTenantUsage, one
|
||||
/// partition range), never the per-user log.
|
||||
/// </summary>
|
||||
[Function("GetMcpOrganizationUsage")]
|
||||
public async Task<HttpResponseData> GetOrganizationUsage(
|
||||
@@ -158,7 +163,7 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
|
||||
TenantId = tenantId,
|
||||
DateFrom = dateFrom,
|
||||
DateTo = dateTo,
|
||||
Users = AggregateOrganizationUsage(records, tenantId, dateFrom, dateTo, today, monthStart),
|
||||
Users = AggregateOrganizationUsage(records, dateFrom, dateTo, today, monthStart),
|
||||
Quota = BuildOrganizationQuota(records, limits, today, monthStart),
|
||||
Daily = BuildOrganizationDaily(records, dateFrom, dateTo),
|
||||
});
|
||||
@@ -207,15 +212,20 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
|
||||
MonthlyLimit = limits.TenantMonthlyLimit,
|
||||
DailyUsed = dailyUsed,
|
||||
MonthlyUsed = monthlyUsed,
|
||||
PurchasedDelegatedSlots = WithSlots(limits.PurchasedDelegatedSlots, limits.PurchasedDelegatedSlots),
|
||||
SlotDailyLimit = WithSlots(limits.PurchasedDelegatedSlots, limits.SlotTenantDailyLimit),
|
||||
SlotMonthlyLimit = WithSlots(limits.PurchasedDelegatedSlots, limits.SlotTenantMonthlyLimit),
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>The slot breakdown is only worth sending when a slot was actually purchased.</summary>
|
||||
private static int? WithSlots(int purchasedSlots, int value) => purchasedSlots > 0 ? value : null;
|
||||
|
||||
/// <summary>
|
||||
/// Pure: folds the organization counter rows into one item per account. A row whose HomeTenantId is set
|
||||
/// and differs from the tenant was charged by a delegated (MSP) read. Ordered by this month's usage.
|
||||
/// Pure: folds the organization counter rows into one item per account, ordered by this month's usage.
|
||||
/// </summary>
|
||||
internal static List<McpOrganizationUsageItem> AggregateOrganizationUsage(
|
||||
IEnumerable<TenantUsageRecord> records, string tenantId, string dateFrom, string dateTo, string today, string monthStart)
|
||||
IEnumerable<TenantUsageRecord> records, string dateFrom, string dateTo, string today, string monthStart)
|
||||
{
|
||||
var byUser = new Dictionary<string, McpOrganizationUsageItem>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var record in records)
|
||||
@@ -230,12 +240,6 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
|
||||
|
||||
if (!string.IsNullOrEmpty(record.UserPrincipalName))
|
||||
item.UserPrincipalName = record.UserPrincipalName;
|
||||
if (!string.IsNullOrEmpty(record.HomeTenantId)
|
||||
&& !string.Equals(record.HomeTenantId, tenantId, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
item.Delegated = true;
|
||||
item.HomeTenantId = record.HomeTenantId;
|
||||
}
|
||||
if (record.LastRequestAt is DateTime last && (item.LastRequestAt == null || last > item.LastRequestAt))
|
||||
item.LastRequestAt = last;
|
||||
|
||||
|
||||
-2
@@ -67,8 +67,6 @@ namespace AutopilotMonitor.Functions.Functions.Sessions
|
||||
{
|
||||
Success = true,
|
||||
Stats = stats,
|
||||
// MCP fleet aggregate only: managed tenants the quota layer dropped (see GetAllSessions).
|
||||
QuotaExcludedTenants = req.GetRequestContext().QuotaExcludedTenantIds?.ToArray(),
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
|
||||
@@ -99,9 +99,6 @@ namespace AutopilotMonitor.Functions.Functions.Sessions
|
||||
Count = page.Items.Count,
|
||||
Sessions = page.Items,
|
||||
NextLink = nextLink,
|
||||
// MCP fleet aggregate only: managed tenants the quota layer dropped (exhausted organization
|
||||
// budget). The bound above is already narrowed; this just tells the caller what is missing.
|
||||
QuotaExcludedTenants = req.GetRequestContext().QuotaExcludedTenantIds?.ToArray(),
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
|
||||
+3
-3
@@ -228,11 +228,11 @@ namespace AutopilotMonitor.Functions.Functions.Sessions
|
||||
// registrations count: agent restarts and WhiteGlove Part 2 resumes re-register the
|
||||
// same session and must not inflate it. The per-tenant counter has no corrective
|
||||
// recompute (only a raise-only floor), so gating on fresh registrations is
|
||||
// load-bearing. The platform-wide TotalEnrollments is recomputed every two hours from
|
||||
// live data (D-198) and no longer incremented here.
|
||||
// load-bearing. The platform-wide TotalEnrollments is rolled up from these tenant
|
||||
// counters by the maintenance run.
|
||||
if (isFreshRegistration)
|
||||
{
|
||||
_ = _metricsRepo.IncrementTenantStatAsync(registration.TenantId, "TotalEnrollments")
|
||||
_ = _metricsRepo.IncrementTenantStatAsync(registration.TenantId, nameof(TenantStats.TotalEnrollments))
|
||||
.ContinueWith(t => _logger.LogWarning(t.Exception?.InnerException, "Fire-and-forget IncrementTenantStatAsync failed"), TaskContinuationOptions.OnlyOnFaulted);
|
||||
}
|
||||
|
||||
|
||||
@@ -88,34 +88,10 @@ public sealed record RequestContext
|
||||
/// The set of tenant IDs (lowercase) the caller is authorized for via delegated assignments, or null
|
||||
/// when the caller is not a delegated admin (or full platform scope, which is unbounded = "all tenants").
|
||||
/// Populated on cross-tenant delegated reads; consumed by cross-tenant aggregation to bound the fan-out
|
||||
/// to the delegated subset (Phase 2). Null ⇒ no delegated bound applies. For an MCP fleet aggregate
|
||||
/// McpQuotaEnforcementMiddleware may NARROW this set (never widen it, never null it) to the managed
|
||||
/// tenants whose organization budget is not exhausted — see <see cref="QuotaExcludedTenantIds"/>.
|
||||
/// to the delegated subset (Phase 2). Null ⇒ no delegated bound applies.
|
||||
/// </summary>
|
||||
public IReadOnlyCollection<string>? AllowedTenantIds { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// True when a delegated caller was admitted to a subset-tier DATA aggregate (GlobalReadOrDelegatedSubset +
|
||||
/// TenantScoping.QueryParam, no tenantId named): the handler fans out over <see cref="AllowedTenantIds"/>.
|
||||
/// False for config/all (TenantScoping.None — a directory listing) and for every single-target drill.
|
||||
/// </summary>
|
||||
public bool IsDelegatedAggregate { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Managed tenants dropped from <see cref="AllowedTenantIds"/> by McpQuotaEnforcementMiddleware because
|
||||
/// their organization MCP budget is exhausted. Null when nothing was excluded (or not an MCP aggregate).
|
||||
/// Handlers echo it as <c>quotaExcludedTenants</c>.
|
||||
/// </summary>
|
||||
public IReadOnlyCollection<string>? QuotaExcludedTenantIds { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The subset of this request's managed target tenants (the single target, or members of
|
||||
/// <see cref="AllowedTenantIds"/>) that were reached through a Tenant Group flagged
|
||||
/// <c>ChargeHomeTenantQuota</c>: MCP reads into them are charged to the caller's HOME tenant instead of
|
||||
/// the managed tenant. Null when none apply. Billing attribution only — never an authorization input.
|
||||
/// </summary>
|
||||
public IReadOnlyCollection<string>? HomeChargedTenantIds { get; init; }
|
||||
|
||||
/// <summary>True if the user is a Tenant Admin of their own tenant.</summary>
|
||||
public bool IsTenantAdmin { get; init; }
|
||||
|
||||
|
||||
@@ -39,6 +39,8 @@ namespace AutopilotMonitor.Functions.Helpers
|
||||
("mcpDisabledReason", c => c.McpDisabledReason),
|
||||
("onboardedAt", c => c.OnboardedAt),
|
||||
("onboardedBy", c => c.OnboardedBy),
|
||||
// DPA version accepted at onboarding — a version label, not a secret.
|
||||
("dpaVersion", c => c.DpaVersion),
|
||||
("lastUpdated", c => c.LastUpdated),
|
||||
("dataRetentionDays", c => c.DataRetentionDays),
|
||||
// App-reg migration observability: which app the tenant is homed on (null = legacy)
|
||||
|
||||
+52
-250
@@ -14,27 +14,20 @@ namespace AutopilotMonitor.Functions.Middleware;
|
||||
|
||||
/// <summary>
|
||||
/// Enforces the MCP daily/monthly request quota — the caller's own budget AND the organization-wide
|
||||
/// budget of the CHARGED tenant (both counted here, both must be free). Runs after
|
||||
/// budget of the caller's HOME tenant (both counted here, both must be free). Runs after
|
||||
/// <see cref="UserRateLimitMiddleware"/> (per-minute burst control) — this is the budget layer on
|
||||
/// top. Applies ONLY to HTTP requests marked <c>X-Client-Source: mcp</c> with an authenticated
|
||||
/// principal (oid); everything else passes through untouched.
|
||||
///
|
||||
/// <b>Which tenant is charged ("the budget follows the data"):</b>
|
||||
/// <list type="bullet">
|
||||
/// <item>A member reading their own tenant: the home tenant (JWT tid).</item>
|
||||
/// <item>A delegated (MSP) admin reading ONE managed tenant (<c>?tenantId=</c>): that managed tenant — its
|
||||
/// plan governs the organization windows, and its admins see the consumption on their MCP Usage page.
|
||||
/// Exception: a tenant reached through a Tenant Group flagged <c>ChargeHomeTenantQuota</c> (operator-run
|
||||
/// managed service) is charged to the admin's HOME tenant instead.</item>
|
||||
/// <item>A delegated admin's bounded fleet aggregate (no tenantId on a subset-tier route): every charged
|
||||
/// tenant in the managed set. Tenants whose budget is exhausted are EXCLUDED — <see cref="RequestContext.AllowedTenantIds"/>
|
||||
/// is NARROWED to the admitted ones and the dropped set is published as <see cref="RequestContext.QuotaExcludedTenantIds"/>
|
||||
/// for the handler to echo — instead of failing the whole request; only "everything exhausted" is a 429.</item>
|
||||
/// <item>Global Admins are exempt from the quota and tracked on their HOME tenant only — never on a
|
||||
/// target tenant's counters (they read every tenant for platform operations).</item>
|
||||
/// </list>
|
||||
/// <b>Isolation invariant:</b> narrowing only ever SHRINKS the published bound and never turns it into
|
||||
/// <c>null</c> (null = unbounded in the repository). See <see cref="Narrow"/>.
|
||||
/// <b>Which tenant is charged: always the caller's home tenant (JWT tid)</b> — "the budget follows the
|
||||
/// delegating tenant". A member reading their own tenant, a delegated (MSP) admin reading a managed tenant,
|
||||
/// a fleet aggregate over every managed tenant: one check, one charge, on home. A managed tenant is never
|
||||
/// charged and never blocked by its manager's reads; what the manager gets in return is the slot growth of
|
||||
/// its own windows (see <see cref="McpQuotaService"/>).
|
||||
///
|
||||
/// Global Admins are exempt from the check (platform operations must never be blocked by a budget) but
|
||||
/// counted like everyone else — on their home tenant — so their usage stays measurable. Global Readers are
|
||||
/// NOT exempt; see <see cref="IsExempt"/>.
|
||||
///
|
||||
/// This middleware also OWNS the usage-counter increment (moved here from AuthenticationMiddleware,
|
||||
/// Codex finding 2026-07-07): check-then-increment, and only for requests that are actually
|
||||
@@ -43,7 +36,7 @@ namespace AutopilotMonitor.Functions.Middleware;
|
||||
/// fire-and-forget (never blocks the request path).
|
||||
///
|
||||
/// <b>The quota boundary is deliberately SOFT, not exact.</b> McpQuotaService caches the usage
|
||||
/// snapshots for 60 seconds (per instance; per user and per charged tenant), so an ALLOWED decision keeps
|
||||
/// snapshots for 60 seconds (per instance; per user and per tenant), so an ALLOWED decision keeps
|
||||
/// admitting requests inside that window even after the async increments push the stored counters past
|
||||
/// the limit. Worst-case overshoot is bounded: ~60s × the request rate against that counter (× instance
|
||||
/// count on scaled-out Flex Consumption) — the same deliberate posture as the sliding-window
|
||||
@@ -60,18 +53,15 @@ public class McpQuotaEnforcementMiddleware : IFunctionsWorkerMiddleware
|
||||
{
|
||||
private readonly McpQuotaService _quotaService;
|
||||
private readonly IUserUsageRepository _userUsageRepo;
|
||||
private readonly TenantConfigurationService _tenantConfigService;
|
||||
private readonly ILogger<McpQuotaEnforcementMiddleware> _logger;
|
||||
|
||||
public McpQuotaEnforcementMiddleware(
|
||||
McpQuotaService quotaService,
|
||||
IUserUsageRepository userUsageRepo,
|
||||
TenantConfigurationService tenantConfigService,
|
||||
ILogger<McpQuotaEnforcementMiddleware> logger)
|
||||
{
|
||||
_quotaService = quotaService;
|
||||
_userUsageRepo = userUsageRepo;
|
||||
_tenantConfigService = tenantConfigService;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
@@ -101,189 +91,65 @@ public class McpQuotaEnforcementMiddleware : IFunctionsWorkerMiddleware
|
||||
|
||||
var upn = principal.GetUserPrincipalName();
|
||||
var homeTenantId = principal.GetTenantId() ?? string.Empty;
|
||||
var requestContext = context.GetRequestContext();
|
||||
|
||||
// GA is exempt from the quota and is tracked on its HOME tenant only — never on a target tenant's
|
||||
// counters (platform operations must not consume, or be blocked by, a customer's budget).
|
||||
if (requestContext.IsGlobalAdmin)
|
||||
// Exempt callers skip the check but are still counted on their home tenant — the usage of a platform
|
||||
// operator is what the operator wants to see, not what may block them.
|
||||
if (IsExempt(context.GetRequestContext()))
|
||||
{
|
||||
TrackUsage(httpContext, oid, upn, homeTenantId, new[] { homeTenantId });
|
||||
TrackUsage(httpContext, oid, upn, homeTenantId);
|
||||
await next(context);
|
||||
return;
|
||||
}
|
||||
|
||||
var scope = ResolveChargeScope(requestContext, homeTenantId);
|
||||
if (scope.Kind == McpChargeKind.BoundedAggregate)
|
||||
{
|
||||
await InvokeAggregateAsync(context, next, httpContext, requestContext, oid, upn, homeTenantId, scope);
|
||||
return;
|
||||
}
|
||||
|
||||
var chargeTenantId = scope.TargetTenantId ?? homeTenantId;
|
||||
McpQuotaDecision decision;
|
||||
try
|
||||
{
|
||||
decision = await _quotaService.CheckAsync(oid, upn, homeTenantId, chargeTenantId);
|
||||
decision = await _quotaService.CheckAsync(oid, upn, homeTenantId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Belt-and-braces fail-open: the quota layer must never take MCP down.
|
||||
_logger.LogError(ex, "[McpQuota] Quota check threw for oid={Oid} — allowing request (fail-open)", oid);
|
||||
TrackUsage(httpContext, oid, upn, homeTenantId, new[] { chargeTenantId });
|
||||
TrackUsage(httpContext, oid, upn, homeTenantId);
|
||||
await next(context);
|
||||
return;
|
||||
}
|
||||
|
||||
StampQuotaHeaders(httpContext, decision, chargeTenantId);
|
||||
StampQuotaHeaders(httpContext, decision, homeTenantId);
|
||||
|
||||
if (decision.Allowed)
|
||||
{
|
||||
// Check-then-increment: the decision above reflects previously SERVED requests only.
|
||||
TrackUsage(httpContext, oid, upn, homeTenantId, new[] { chargeTenantId });
|
||||
TrackUsage(httpContext, oid, upn, homeTenantId);
|
||||
await next(context);
|
||||
return;
|
||||
}
|
||||
|
||||
LogBlocked(oid, chargeTenantId, decision);
|
||||
var targetLabel = decision.TargetTenantId != null ? await ResolveTenantLabelAsync(decision.TargetTenantId) : null;
|
||||
await WriteExceededAsync(context, httpContext, decision, BuildExceededResponse(decision, targetLabel));
|
||||
LogBlocked(oid, homeTenantId, decision);
|
||||
await WriteExceededAsync(context, httpContext, decision, BuildExceededResponse(decision));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The bounded fleet aggregate: every charged tenant is checked; exhausted ones are dropped from the
|
||||
/// published bound (never the whole request), and the handler echoes them as quotaExcludedTenants.
|
||||
/// Pure: who bypasses the check. Only the Global Admin — platform operations must never be blocked by a
|
||||
/// budget. The Global Reader is a customer-facing read role and stays within its home tenant's windows.
|
||||
/// Exempt callers are still COUNTED (see <see cref="Invoke"/>) so their usage remains measurable.
|
||||
/// </summary>
|
||||
private async Task InvokeAggregateAsync(
|
||||
FunctionContext context, FunctionExecutionDelegate next, HttpContext httpContext, RequestContext requestContext,
|
||||
string oid, string? upn, string homeTenantId, McpChargeScope scope)
|
||||
{
|
||||
var chargeTenantIds = scope.ChargeMap!.Keys.ToList();
|
||||
McpAggregateQuotaResult result;
|
||||
try
|
||||
{
|
||||
result = await _quotaService.CheckManyAsync(oid, upn, homeTenantId, chargeTenantIds);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "[McpQuota] Aggregate quota check threw for oid={Oid} — allowing request (fail-open)", oid);
|
||||
TrackUsage(httpContext, oid, upn, homeTenantId, chargeTenantIds);
|
||||
await next(context);
|
||||
return;
|
||||
}
|
||||
|
||||
var excludedTargets = scope.TargetsOf(result.ExcludedTenantIds);
|
||||
if (!result.Allowed)
|
||||
{
|
||||
var blocking = result.BlockingDecision!;
|
||||
StampQuotaHeaders(httpContext, blocking, chargeTenantId: null);
|
||||
LogBlocked(oid, $"aggregate({chargeTenantIds.Count})", blocking);
|
||||
// A user-level block names the caller's own plan; "every managed tenant exhausted" names the count.
|
||||
var exhausted = blocking.Level == McpQuotaLevel.Tenant ? excludedTargets.Count : (int?)null;
|
||||
await WriteExceededAsync(context, httpContext, blocking, BuildExceededResponse(blocking, exhaustedTenantCount: exhausted));
|
||||
return;
|
||||
}
|
||||
|
||||
var admittedTargets = scope.TargetsOf(result.AdmittedTenantIds);
|
||||
context.Items[RequestContext.ItemsKey] = Narrow(requestContext, admittedTargets, excludedTargets);
|
||||
|
||||
StampQuotaHeaders(httpContext, result.UserDecision, chargeTenantId: null, includeTenantWindows: false);
|
||||
httpContext.Response.Headers["X-MCP-Quota-Excluded-Tenants"] = excludedTargets.Count.ToString();
|
||||
|
||||
TrackUsage(httpContext, oid, upn, homeTenantId, result.AdmittedTenantIds);
|
||||
await next(context);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pure: which tenant(s) this request draws on. Not delegated ⇒ home. A delegated single-target read ⇒
|
||||
/// that target, unless it was reached through a home-charged group ⇒ home. A delegated bounded aggregate ⇒
|
||||
/// one charge per managed tenant, home-charged ones mapped onto the home tenant (checked and counted
|
||||
/// ONCE, never excluded on the managed tenant's behalf).
|
||||
/// </summary>
|
||||
internal static McpChargeScope ResolveChargeScope(RequestContext ctx, string homeTenantId)
|
||||
{
|
||||
if (!ctx.IsDelegated)
|
||||
return McpChargeScope.Home(homeTenantId);
|
||||
|
||||
var homeCharged = new HashSet<string>(ctx.HomeChargedTenantIds ?? Array.Empty<string>(), StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
if (ctx.IsDelegatedAggregate && ctx.AllowedTenantIds is { Count: > 0 })
|
||||
{
|
||||
var map = new Dictionary<string, List<string>>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var target in ctx.AllowedTenantIds)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(target))
|
||||
continue;
|
||||
var charge = homeCharged.Contains(target) ? homeTenantId : target;
|
||||
if (!map.TryGetValue(charge, out var targets))
|
||||
map[charge] = targets = new List<string>();
|
||||
targets.Add(target);
|
||||
}
|
||||
if (map.Count > 0)
|
||||
return new McpChargeScope(McpChargeKind.BoundedAggregate, homeTenantId, null, map);
|
||||
}
|
||||
|
||||
var single = ctx.TargetTenantId;
|
||||
if (!string.IsNullOrEmpty(single)
|
||||
&& !string.Equals(single, homeTenantId, StringComparison.OrdinalIgnoreCase)
|
||||
&& !homeCharged.Contains(single))
|
||||
return new McpChargeScope(McpChargeKind.SingleTarget, homeTenantId, single, null);
|
||||
|
||||
return McpChargeScope.Home(homeTenantId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pure. ISOLATION INVARIANT: the published bound is only ever NARROWED — the result is a non-null
|
||||
/// list that is a subset of the input bound (empty allowed: the repository then serves an empty page),
|
||||
/// and a context without a bound (null = GA/Reader, unbounded) is returned untouched. Never produce
|
||||
/// null here: null means "all tenants" to the repository.
|
||||
/// </summary>
|
||||
internal static RequestContext Narrow(RequestContext ctx, IReadOnlyCollection<string> admittedTargets, IReadOnlyCollection<string> excludedTargets)
|
||||
{
|
||||
if (ctx.AllowedTenantIds == null)
|
||||
return ctx;
|
||||
|
||||
var bound = new HashSet<string>(ctx.AllowedTenantIds, StringComparer.OrdinalIgnoreCase);
|
||||
var narrowed = admittedTargets.Where(bound.Contains).Distinct(StringComparer.OrdinalIgnoreCase).ToList();
|
||||
var excluded = excludedTargets.Where(bound.Contains).Distinct(StringComparer.OrdinalIgnoreCase).ToList();
|
||||
return ctx with
|
||||
{
|
||||
AllowedTenantIds = narrowed,
|
||||
QuotaExcludedTenantIds = excluded.Count > 0 ? excluded : null,
|
||||
};
|
||||
}
|
||||
internal static bool IsExempt(RequestContext ctx) => ctx.IsGlobalAdmin;
|
||||
|
||||
/// <summary>
|
||||
/// The 429 body for a blocked decision. The message names WHOSE budget is exhausted — a member hitting the
|
||||
/// organization-wide window must not conclude their own plan is too small, and a delegated admin hitting
|
||||
/// a MANAGED tenant's window must learn that the managed tenant's plan (not their own) governs it.
|
||||
/// Every Community block also says that Community is sized for occasional use and that Pro lifts the
|
||||
/// window: the quota is the upgrade lever ("the budget follows the data" — a managed Community customer
|
||||
/// is read with Community windows, so the fix is on the CUSTOMER's plan, never the MSP's).
|
||||
/// organization-wide window must not conclude their own plan is too small. Both windows belong to the
|
||||
/// caller's own tenant (a delegated read never draws on a managed tenant), so the fix is always on the
|
||||
/// caller's side: every Community block says that Community is sized for occasional use and that Pro
|
||||
/// lifts the window.
|
||||
/// </summary>
|
||||
/// <param name="targetTenantLabel">Display label (domain) of <see cref="McpQuotaDecision.TargetTenantId"/>; falls back to the id.</param>
|
||||
/// <param name="exhaustedTenantCount">Set on the aggregate block where EVERY managed tenant is exhausted.</param>
|
||||
internal static McpQuotaExceededResponse BuildExceededResponse(
|
||||
McpQuotaDecision decision, string? targetTenantLabel = null, int? exhaustedTenantCount = null)
|
||||
internal static McpQuotaExceededResponse BuildExceededResponse(McpQuotaDecision decision)
|
||||
{
|
||||
var resetStamp = $"{decision.ResetUtc:yyyy-MM-ddTHH:mm:ss}Z";
|
||||
var reset = $"Resets at {resetStamp}.";
|
||||
string message;
|
||||
string? targetTenantId = null;
|
||||
|
||||
if (exhaustedTenantCount is int exhausted && decision.Level == McpQuotaLevel.Tenant)
|
||||
{
|
||||
message = $"MCP request quota exceeded for all {exhausted} managed tenants in scope (each managed tenant's own plan governs its organization windows; upgrading a managed tenant to Pro lifts them). Earliest reset at {resetStamp}.";
|
||||
}
|
||||
else if (decision.Level == McpQuotaLevel.Tenant && decision.TargetTenantId != null)
|
||||
{
|
||||
targetTenantId = decision.TargetTenantId;
|
||||
var label = string.IsNullOrWhiteSpace(targetTenantLabel) ? decision.TargetTenantId : targetTenantLabel;
|
||||
var upgrade = FeatureEntitlementCatalog.IsPermanentProTier(decision.TenantPlan)
|
||||
? string.Empty
|
||||
: " That tenant is on the Community plan, which is sized for occasional use; its own plan governs this window, not yours. Upgrading that tenant to Pro lifts its organization windows.";
|
||||
message = $"MCP {decision.Scope} request quota of the managed tenant '{label}' exceeded (tenant plan '{decision.TenantPlan}', shared by all its members and delegated admins).{upgrade} {reset}";
|
||||
}
|
||||
else if (decision.Level == McpQuotaLevel.Tenant)
|
||||
if (decision.Level == McpQuotaLevel.Tenant)
|
||||
{
|
||||
var upgrade = FeatureEntitlementCatalog.IsPermanentProTier(decision.TenantPlan)
|
||||
? string.Empty
|
||||
@@ -310,30 +176,14 @@ public class McpQuotaEnforcementMiddleware : IFunctionsWorkerMiddleware
|
||||
Limit = decision.ExceededLimit,
|
||||
Used = decision.ExceededUsed,
|
||||
ResetUtc = resetStamp,
|
||||
TargetTenantId = targetTenantId,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>Blocked-path only (no hot-path read): the managed tenant's domain for the 429 text, else its id.</summary>
|
||||
private async Task<string> ResolveTenantLabelAsync(string tenantId)
|
||||
{
|
||||
try
|
||||
{
|
||||
var config = await _tenantConfigService.GetConfigurationIfExistsAsync(tenantId);
|
||||
return string.IsNullOrWhiteSpace(config?.DomainName) ? tenantId : config!.DomainName;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "[McpQuota] Tenant label lookup failed for {TenantId}", LogSanitizer.Clean(tenantId));
|
||||
return tenantId;
|
||||
}
|
||||
}
|
||||
|
||||
private void LogBlocked(string oid, string chargedTenant, McpQuotaDecision decision)
|
||||
private void LogBlocked(string oid, string tenantId, McpQuotaDecision decision)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"[McpQuota] BLOCKED oid={Oid} charged={ChargedTenant} level={Level} scope={Scope} plan={Plan} daily={DailyUsed}/{DailyLimit} monthly={MonthlyUsed}/{MonthlyLimit} tenantDaily={TenantDailyUsed}/{TenantDailyLimit} tenantMonthly={TenantMonthlyUsed}/{TenantMonthlyLimit}",
|
||||
oid, LogSanitizer.Clean(chargedTenant), decision.Level, decision.Scope, decision.Plan, decision.DailyUsed, decision.DailyLimit, decision.MonthlyUsed, decision.MonthlyLimit,
|
||||
oid, LogSanitizer.Clean(tenantId), decision.Level, decision.Scope, decision.Plan, decision.DailyUsed, decision.DailyLimit, decision.MonthlyUsed, decision.MonthlyLimit,
|
||||
decision.TenantDailyUsed, decision.TenantDailyLimit, decision.TenantMonthlyUsed, decision.TenantMonthlyLimit);
|
||||
}
|
||||
|
||||
@@ -345,13 +195,11 @@ public class McpQuotaEnforcementMiddleware : IFunctionsWorkerMiddleware
|
||||
|
||||
/// <summary>
|
||||
/// Fire-and-forget usage increment (same posture as the previous AuthenticationMiddleware
|
||||
/// tracking — never blocks or fails the request path). The per-user row (UserUsageLog) is ALWAYS
|
||||
/// attributed to the caller's HOME tenant — so a delegated admin's own rows never surface under a
|
||||
/// customer's per-user usage views — while the organization counter (McpTenantUsage) is written once
|
||||
/// per CHARGED tenant, carrying the caller's home tenant when it differs. Endpoint is normalized and
|
||||
/// prefixed with the X-MCP-Tool-Name when the MCP server supplies it.
|
||||
/// tracking — never blocks or fails the request path). Both rows belong to the caller's HOME tenant:
|
||||
/// the per-user row (UserUsageLog) and the organization counter (McpTenantUsage). Endpoint is normalized
|
||||
/// and prefixed with the X-MCP-Tool-Name when the MCP server supplies it.
|
||||
/// </summary>
|
||||
private void TrackUsage(HttpContext httpContext, string oid, string? upn, string homeTenantId, IReadOnlyCollection<string> chargeTenantIds)
|
||||
private void TrackUsage(HttpContext httpContext, string oid, string? upn, string homeTenantId)
|
||||
{
|
||||
var normalizedEndpoint = EndpointNormalizer.Normalize(httpContext.Request.Path.Value ?? string.Empty);
|
||||
// Key-safe copy of the header: the value becomes part of the UserUsageLog row key.
|
||||
@@ -362,7 +210,6 @@ public class McpQuotaEnforcementMiddleware : IFunctionsWorkerMiddleware
|
||||
var repo = _userUsageRepo;
|
||||
var logger = _logger;
|
||||
var upnValue = upn ?? "unknown";
|
||||
var charges = chargeTenantIds.Where(t => !string.IsNullOrEmpty(t)).Distinct(StringComparer.OrdinalIgnoreCase).ToList();
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
@@ -374,28 +221,24 @@ public class McpQuotaEnforcementMiddleware : IFunctionsWorkerMiddleware
|
||||
logger.LogWarning(ex, "[McpQuota] Failed to record usage: user={UserId}, endpoint={Endpoint}", LogSanitizer.Clean(oid), LogSanitizer.Clean(normalizedEndpoint));
|
||||
}
|
||||
|
||||
// Organization-wide counters (the tenant quota's source). Separate try per tenant so a failure on
|
||||
// one row never starves the others.
|
||||
foreach (var chargeTenantId in charges)
|
||||
if (string.IsNullOrEmpty(homeTenantId))
|
||||
return;
|
||||
|
||||
// Organization-wide counter (the tenant quota's source). Separate try so a failure here never
|
||||
// hides the per-user write above.
|
||||
try
|
||||
{
|
||||
var isForeign = !string.Equals(chargeTenantId, homeTenantId, StringComparison.OrdinalIgnoreCase);
|
||||
try
|
||||
{
|
||||
await repo.IncrementTenantUsageAsync(chargeTenantId, oid, upn, isForeign ? homeTenantId : null);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogWarning(ex, "[McpQuota] Failed to record tenant usage: tenant={TenantId}, user={UserId}", LogSanitizer.Clean(chargeTenantId), LogSanitizer.Clean(oid));
|
||||
}
|
||||
await repo.IncrementTenantUsageAsync(homeTenantId, oid, upn);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogWarning(ex, "[McpQuota] Failed to record tenant usage: tenant={TenantId}, user={UserId}", LogSanitizer.Clean(homeTenantId), LogSanitizer.Clean(oid));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Quota headers. The tenant windows describe the CHARGED tenant (named in X-MCP-Quota-Tenant-Id); on
|
||||
/// the aggregate path no single tenant applies, so only the caller's own windows are stamped.
|
||||
/// </summary>
|
||||
private static void StampQuotaHeaders(HttpContext httpContext, McpQuotaDecision decision, string? chargeTenantId, bool includeTenantWindows = true)
|
||||
/// <summary>Quota headers: the caller's own windows and the home tenant's organization windows (named in X-MCP-Quota-Tenant-Id).</summary>
|
||||
private static void StampQuotaHeaders(HttpContext httpContext, McpQuotaDecision decision, string tenantId)
|
||||
{
|
||||
// Direct-write pattern — same as UserRateLimitMiddleware's X-RateLimit-* headers.
|
||||
httpContext.Response.Headers["X-MCP-Quota-Plan"] = decision.Plan;
|
||||
@@ -407,11 +250,8 @@ public class McpQuotaEnforcementMiddleware : IFunctionsWorkerMiddleware
|
||||
httpContext.Response.Headers["X-MCP-Quota-Monthly-Used"] = decision.MonthlyUsed.ToString();
|
||||
}
|
||||
|
||||
if (!includeTenantWindows)
|
||||
return;
|
||||
|
||||
if (!string.IsNullOrEmpty(chargeTenantId))
|
||||
httpContext.Response.Headers["X-MCP-Quota-Tenant-Id"] = chargeTenantId;
|
||||
if (!string.IsNullOrEmpty(tenantId))
|
||||
httpContext.Response.Headers["X-MCP-Quota-Tenant-Id"] = tenantId;
|
||||
httpContext.Response.Headers["X-MCP-Quota-Tenant-Plan"] = decision.TenantPlan;
|
||||
httpContext.Response.Headers["X-MCP-Quota-Tenant-Daily-Limit"] = decision.TenantDailyLimit.ToString();
|
||||
httpContext.Response.Headers["X-MCP-Quota-Tenant-Monthly-Limit"] = decision.TenantMonthlyLimit.ToString();
|
||||
@@ -422,41 +262,3 @@ public class McpQuotaEnforcementMiddleware : IFunctionsWorkerMiddleware
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>How an MCP request is charged — see <see cref="McpQuotaEnforcementMiddleware.ResolveChargeScope"/>.</summary>
|
||||
internal enum McpChargeKind
|
||||
{
|
||||
/// <summary>The caller's home tenant (JWT tid).</summary>
|
||||
Home,
|
||||
/// <summary>One managed tenant of a delegated (MSP) read.</summary>
|
||||
SingleTarget,
|
||||
/// <summary>A delegated bounded fleet aggregate: one charge per managed tenant (home-charged ones folded onto home).</summary>
|
||||
BoundedAggregate,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pure charge plan for one request. <see cref="ChargeMap"/> (aggregate only) maps each CHARGED tenant to
|
||||
/// the managed TARGET tenants it pays for — a home-charged group's tenants all map onto the home tenant.
|
||||
/// </summary>
|
||||
internal sealed record McpChargeScope(
|
||||
McpChargeKind Kind,
|
||||
string HomeTenantId,
|
||||
string? TargetTenantId,
|
||||
IReadOnlyDictionary<string, List<string>>? ChargeMap)
|
||||
{
|
||||
public static McpChargeScope Home(string homeTenantId) => new(McpChargeKind.Home, homeTenantId, null, null);
|
||||
|
||||
/// <summary>The managed target tenants paid for by the given charged tenants (aggregate only).</summary>
|
||||
public IReadOnlyList<string> TargetsOf(IEnumerable<string> chargeTenantIds)
|
||||
{
|
||||
if (ChargeMap == null)
|
||||
return Array.Empty<string>();
|
||||
var targets = new List<string>();
|
||||
foreach (var charge in chargeTenantIds)
|
||||
{
|
||||
if (ChargeMap.TryGetValue(charge, out var mapped))
|
||||
targets.AddRange(mapped);
|
||||
}
|
||||
return targets;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -246,7 +246,6 @@ public class PolicyEnforcementMiddleware : IFunctionsWorkerMiddleware
|
||||
// keeps the common authenticated path free of the extra table read.
|
||||
string? delegatedRole = null;
|
||||
IReadOnlyCollection<string>? allowedTenantIds = null;
|
||||
IReadOnlyCollection<string>? homeChargedTenantIds = null;
|
||||
var isScopedRoute = catalogEntry.TenantScoping is TenantScoping.RouteParam or TenantScoping.QueryParam;
|
||||
if (!hasGlobalScope && !string.IsNullOrEmpty(upn) && isScopedRoute && crossTenant)
|
||||
{
|
||||
@@ -255,12 +254,7 @@ public class PolicyEnforcementMiddleware : IFunctionsWorkerMiddleware
|
||||
var scope = await _delegatedAdminService.GetScopeAsync(AdminIdentity.FromPrincipal(principal));
|
||||
delegatedRole = scope.RoleFor(namedTarget);
|
||||
if (delegatedRole != null)
|
||||
{
|
||||
allowedTenantIds = scope.TenantIds;
|
||||
// Billing attribution for the MCP quota layer: a target reached through a home-charged group.
|
||||
if (scope.ChargesHome(namedTarget))
|
||||
homeChargedTenantIds = new[] { namedTarget!.ToLowerInvariant() };
|
||||
}
|
||||
}
|
||||
// A delegated grant only applies to the READ tiers — never to write tiers (TenantAdminOrGA,
|
||||
// GlobalAdminOnly, …). This is the single fact that keeps delegation read-only in this phase and
|
||||
@@ -384,12 +378,6 @@ public class PolicyEnforcementMiddleware : IFunctionsWorkerMiddleware
|
||||
IsDelegatedReader = delegatedRole == Constants.DelegatedRoles.DelegatedReader || admittedViaDelegatedSubset,
|
||||
IsDelegatedAdmin = delegatedRole == Constants.DelegatedRoles.DelegatedAdmin,
|
||||
AllowedTenantIds = effectiveAllowedTenantIds,
|
||||
// The bounded DATA aggregate: subset tier on a QueryParam route with no tenantId named. config/all
|
||||
// (subset tier, TenantScoping.None) is a directory listing, not an aggregate — it stays false there.
|
||||
IsDelegatedAggregate = admittedViaDelegatedSubset
|
||||
&& namedTarget == null
|
||||
&& catalogEntry.TenantScoping == TenantScoping.QueryParam,
|
||||
HomeChargedTenantIds = homeChargedTenantIds ?? decision.HomeChargedTenantIds,
|
||||
// Additive: a caller's tenant-admin status is its OWN-TENANT role, independent of any
|
||||
// platform role. Evaluators that admit via a platform-role branch still surface the
|
||||
// resolved tenant role on decision.TenantRole; pure-tenant branches leave it null and
|
||||
@@ -749,8 +737,7 @@ public class PolicyEnforcementMiddleware : IFunctionsWorkerMiddleware
|
||||
if (!scope.IsEmpty)
|
||||
return CatalogDecisionResult.Allow(
|
||||
userIdentifier, Constants.DelegatedRoles.DelegatedReader, "DelegatedSubset",
|
||||
allowedTenantIds: scope.TenantIds,
|
||||
homeChargedTenantIds: scope.HomeChargedTenantIds.Count > 0 ? scope.HomeChargedTenantIds : null);
|
||||
allowedTenantIds: scope.TenantIds);
|
||||
|
||||
return CatalogDecisionResult.Deny(userIdentifier, "NonGlobal", "NotGlobalOrDelegated");
|
||||
}
|
||||
@@ -832,15 +819,9 @@ public class PolicyEnforcementMiddleware : IFunctionsWorkerMiddleware
|
||||
/// </summary>
|
||||
public IReadOnlyCollection<string>? AllowedTenantIds { get; private init; }
|
||||
|
||||
/// <summary>
|
||||
/// Subset tier only: the members of <see cref="AllowedTenantIds"/> reached through a home-charged
|
||||
/// Tenant Group (MCP quota billing attribution, see RequestContext.HomeChargedTenantIds). Null when none.
|
||||
/// </summary>
|
||||
public IReadOnlyCollection<string>? HomeChargedTenantIds { get; private init; }
|
||||
|
||||
public static CatalogDecisionResult Allow(string user, string role, string reason, string? tenantRole = null,
|
||||
IReadOnlyCollection<string>? allowedTenantIds = null, IReadOnlyCollection<string>? homeChargedTenantIds = null)
|
||||
=> new() { IsAllowed = true, UserIdentifier = user, UserRole = role, Reason = reason, TenantRole = tenantRole, AllowedTenantIds = allowedTenantIds, HomeChargedTenantIds = homeChargedTenantIds };
|
||||
IReadOnlyCollection<string>? allowedTenantIds = null)
|
||||
=> new() { IsAllowed = true, UserIdentifier = user, UserRole = role, Reason = reason, TenantRole = tenantRole, AllowedTenantIds = allowedTenantIds };
|
||||
|
||||
public static CatalogDecisionResult Deny(string user, string role, string reason)
|
||||
=> new() { IsAllowed = false, UserIdentifier = user, UserRole = role, Reason = reason };
|
||||
|
||||
@@ -283,7 +283,17 @@ builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Analyze.Interi
|
||||
builder.Services.AddSingleton<ImeLogPatternService>();
|
||||
builder.Services.AddHttpClient<GitHubRuleRepository>()
|
||||
.AddPolicyHandler((sp, _) => sp.GetRequiredService<ResiliencePolicies>().ExternalDataApi);
|
||||
// Platform maintenance run: blob-lease gate (serializes the 2h timer, the manual trigger and a
|
||||
// second host instance), fail-hard queue producer for POST /api/maintenance/trigger, and the
|
||||
// BackgroundService worker that runs a queued manual run.
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Maintenance.MaintenanceRunLockStore>();
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Maintenance.MaintenanceRunGate>();
|
||||
builder.Services.AddSingleton<
|
||||
AutopilotMonitor.Functions.Services.Maintenance.IMaintenanceTriggerProducer,
|
||||
AutopilotMonitor.Functions.Services.Maintenance.AzureQueueMaintenanceTriggerProducer>();
|
||||
builder.Services.AddSingleton<MaintenanceService>();
|
||||
builder.Services.AddHostedService<
|
||||
AutopilotMonitor.Functions.Services.Maintenance.MaintenanceTriggerQueueWorker>();
|
||||
builder.Services.AddSingleton<LegacyReclassificationService>();
|
||||
builder.Services.AddSingleton<OccurredUtcBackfillService>();
|
||||
builder.Services.AddSingleton<
|
||||
|
||||
@@ -428,6 +428,11 @@ public static class EndpointAccessPolicyCatalog
|
||||
new("GET", "global/sessions/{sessionId}/deletion-manifest", EndpointPolicy.GlobalReadOrAdmin),
|
||||
// Platform-operational (cascade-delete restore prep) — GA + read-only Reader only, NOT delegated.
|
||||
new("GET", "global/tenants/{tenantId}/deletion-manifests", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.RouteParam, excludeDelegated: true),
|
||||
// Offboarding record (marker + history) behind the tenant editor's status card, and the
|
||||
// operator retry of a Failed offboarding. Platform-operational: GA + Reader read, GA-only
|
||||
// retry, never delegated (an MSP manages active tenants, not the platform's offboarding).
|
||||
new("GET", "global/tenants/{tenantId}/offboarding", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.RouteParam, excludeDelegated: true),
|
||||
new("POST", "global/tenants/{tenantId}/offboarding/retry", EndpointPolicy.GlobalAdminOnly, TenantScoping.RouteParam, excludeDelegated: true),
|
||||
new("GET", "global/session-deletions", EndpointPolicy.GlobalReadOrAdmin),
|
||||
// Manual maintenance-run trigger (GC sweeps + retention fanout) — mutating, GA only.
|
||||
new("POST", "global/session-deletions/maintenance/trigger", EndpointPolicy.GlobalAdminOnly),
|
||||
|
||||
@@ -127,6 +127,23 @@ namespace AutopilotMonitor.Functions.Security
|
||||
|
||||
/// <summary>Organization-wide MCP monthly limit — see <see cref="McpTenantDailyRequestLimit"/>.</summary>
|
||||
public int McpTenantMonthlyRequestLimit { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Per-user daily MCP requests every purchased delegation slot (beyond <see cref="MaxDelegatedTenants"/>)
|
||||
/// adds to each account of the tenant. Fallback when the plan's SectionUsagePlans row sets no slot value.
|
||||
/// Zero wherever <see cref="DelegatedAdminAllowed"/> is false — a tenant without the delegation right
|
||||
/// earns nothing from a pre-provisioned slot override.
|
||||
/// </summary>
|
||||
public int McpSlotDailyRequestLimit { get; init; }
|
||||
|
||||
/// <summary>Per-user monthly growth per purchased slot — see <see cref="McpSlotDailyRequestLimit"/>.</summary>
|
||||
public int McpSlotMonthlyRequestLimit { get; init; }
|
||||
|
||||
/// <summary>Organization-wide daily growth per purchased slot — see <see cref="McpSlotDailyRequestLimit"/>.</summary>
|
||||
public int McpSlotTenantDailyRequestLimit { get; init; }
|
||||
|
||||
/// <summary>Organization-wide monthly growth per purchased slot — see <see cref="McpSlotDailyRequestLimit"/>.</summary>
|
||||
public int McpSlotTenantMonthlyRequestLimit { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -176,7 +193,11 @@ namespace AutopilotMonitor.Functions.Security
|
||||
McpDailyRequestLimit = 100,
|
||||
McpMonthlyRequestLimit = 3000,
|
||||
McpTenantDailyRequestLimit = 300,
|
||||
McpTenantMonthlyRequestLimit = 9000
|
||||
McpTenantMonthlyRequestLimit = 9000,
|
||||
McpSlotDailyRequestLimit = 0,
|
||||
McpSlotMonthlyRequestLimit = 0,
|
||||
McpSlotTenantDailyRequestLimit = 0,
|
||||
McpSlotTenantMonthlyRequestLimit = 0
|
||||
};
|
||||
|
||||
private static readonly EditionEntitlements Pro = new()
|
||||
@@ -193,13 +214,19 @@ namespace AutopilotMonitor.Functions.Security
|
||||
McpDailyRequestLimit = 1000,
|
||||
McpMonthlyRequestLimit = 20000,
|
||||
McpTenantDailyRequestLimit = 3000,
|
||||
McpTenantMonthlyRequestLimit = 60000
|
||||
McpTenantMonthlyRequestLimit = 60000,
|
||||
// Per purchased slot: 30 % of the Pro windows — the share an additional managed tenant costs
|
||||
// relative to Pro itself. Operator-editable per plan in SectionUsagePlans; these are the fallbacks.
|
||||
McpSlotDailyRequestLimit = 300,
|
||||
McpSlotMonthlyRequestLimit = 6000,
|
||||
McpSlotTenantDailyRequestLimit = 900,
|
||||
McpSlotTenantMonthlyRequestLimit = 18000
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Pro conferred by a managing tenant: every Pro value EXCEPT the delegation right. A managed
|
||||
/// tenant that is not Pro in its own right cannot invite or manage tenants itself — the
|
||||
/// delegation right is what the managing tenant paid for, and conferral must not chain.
|
||||
/// Pro conferred by a managing tenant: every Pro value EXCEPT the delegation right and, following it,
|
||||
/// the slot growth. A managed tenant that is not Pro in its own right cannot invite or manage tenants
|
||||
/// itself — the delegation right is what the managing tenant paid for, and conferral must not chain.
|
||||
/// </summary>
|
||||
private static readonly EditionEntitlements ProViaMsp = new()
|
||||
{
|
||||
@@ -215,7 +242,11 @@ namespace AutopilotMonitor.Functions.Security
|
||||
McpDailyRequestLimit = Pro.McpDailyRequestLimit,
|
||||
McpMonthlyRequestLimit = Pro.McpMonthlyRequestLimit,
|
||||
McpTenantDailyRequestLimit = Pro.McpTenantDailyRequestLimit,
|
||||
McpTenantMonthlyRequestLimit = Pro.McpTenantMonthlyRequestLimit
|
||||
McpTenantMonthlyRequestLimit = Pro.McpTenantMonthlyRequestLimit,
|
||||
McpSlotDailyRequestLimit = 0,
|
||||
McpSlotMonthlyRequestLimit = 0,
|
||||
McpSlotTenantDailyRequestLimit = 0,
|
||||
McpSlotTenantMonthlyRequestLimit = 0
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -36,6 +36,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
private readonly IRuleRepository _ruleRepo;
|
||||
private readonly ILogger<AnalyzeRuleService> _logger;
|
||||
private bool _seeded = false;
|
||||
private readonly BackendBuildInfo? _buildInfo;
|
||||
|
||||
// Cached set of currently-shipped built-in rule IDs from
|
||||
// <see cref="BuiltInAnalyzeRules.GetAll"/>. Used by the runtime sunset filter
|
||||
@@ -48,10 +49,11 @@ namespace AutopilotMonitor.Functions.Services
|
||||
private HashSet<string>? _liveCatalogIds;
|
||||
private readonly object _liveCatalogLock = new();
|
||||
|
||||
public AnalyzeRuleService(IRuleRepository ruleRepo, ILogger<AnalyzeRuleService> logger)
|
||||
public AnalyzeRuleService(IRuleRepository ruleRepo, ILogger<AnalyzeRuleService> logger, BackendBuildInfo? buildInfo = null)
|
||||
{
|
||||
_ruleRepo = ruleRepo;
|
||||
_logger = logger;
|
||||
_buildInfo = buildInfo;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -505,6 +507,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
await _ruleRepo.StoreAnalyzeRuleAsync(rule, "global");
|
||||
}
|
||||
_logger.LogInformation($"Written {builtInRules.Count} built-in analyze rules from code");
|
||||
await _ruleRepo.SetRuleCatalogStampAsync(RuleCatalogSeedGate.Stamp(RuleCatalogStamp.KindAnalyze, RuleCatalogStamp.SourceEmbedded, builtInRules.Count));
|
||||
|
||||
_seeded = false;
|
||||
|
||||
@@ -540,6 +543,14 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
else
|
||||
{
|
||||
// A GitHub reseed newer than this build owns the table: the embedded catalog is
|
||||
// the older source (RuleCatalogSeedGate).
|
||||
if (!await RuleCatalogSeedGate.AllowedAsync(_ruleRepo, _buildInfo, RuleCatalogStamp.KindAnalyze, _logger))
|
||||
{
|
||||
_seeded = true;
|
||||
return;
|
||||
}
|
||||
|
||||
var existingLookup = existingRules.ToDictionary(r => r.RuleId, r => r);
|
||||
var updated = 0;
|
||||
|
||||
|
||||
@@ -36,6 +36,14 @@ namespace AutopilotMonitor.Functions.Services
|
||||
BuildUtc = ResolveBuildUtc(asm);
|
||||
}
|
||||
|
||||
/// <summary>Test seam: a build identity with a chosen build time.</summary>
|
||||
internal BackendBuildInfo(string version, string commitHash, DateTime buildUtc)
|
||||
{
|
||||
Version = version;
|
||||
CommitHash = commitHash;
|
||||
BuildUtc = buildUtc;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads the <c>BuildTimestampUtc</c> AssemblyMetadata attribute. Falls back to
|
||||
/// <c>DateTime.UtcNow</c> only when the attribute is missing or unparseable —
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Newtonsoft.Json;
|
||||
@@ -15,15 +13,6 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// </summary>
|
||||
public static class BuiltInImeLogPatterns
|
||||
{
|
||||
/// <summary>
|
||||
/// IDs of the shipped patterns, resolved once. The ingest path filters the agent's
|
||||
/// <c>ime_pattern_hits</c> histogram against this set per batch — a device may only
|
||||
/// CLAIM an ID, and tenant custom IDs never reach the global statistics.
|
||||
/// </summary>
|
||||
public static readonly Lazy<HashSet<string>> BuiltInPatternIds = new(() =>
|
||||
new HashSet<string>(GetAll().Select(p => p.PatternId).Where(id => !string.IsNullOrEmpty(id)),
|
||||
StringComparer.OrdinalIgnoreCase));
|
||||
|
||||
public static List<ImeLogPattern> GetAll()
|
||||
{
|
||||
var assembly = Assembly.GetExecutingAssembly();
|
||||
|
||||
@@ -119,11 +119,6 @@ public class DelegatedAdminService
|
||||
// set on the group. Same fail-closed + stronger-role-wins merge as direct grants; a tenant
|
||||
// present both directly and via a group keeps the stronger role. Membership changes converge
|
||||
// within the cache TTL (no separate per-group cache by design).
|
||||
// A group flagged ChargeHomeTenantQuota (operator-run managed service) marks its tenants as
|
||||
// home-charged for the MCP quota layer: reads into them draw on the assignee's HOME tenant budget,
|
||||
// never the managed tenant's. The flag wins over a parallel direct grant on the same tenant —
|
||||
// it is the operator's explicit, more privileged choice. Billing attribution only, never authz.
|
||||
var homeCharged = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var groupAssignments = await _adminRepo.GetGroupAssignmentsForUpnAsync(upn);
|
||||
foreach (var assignment in groupAssignments)
|
||||
{
|
||||
@@ -134,7 +129,7 @@ public class DelegatedAdminService
|
||||
if (role == null)
|
||||
continue;
|
||||
|
||||
// Members + meta flags from ONE partition read (meta-backed: a missing meta row ⇒ no group, no scope).
|
||||
// Members from ONE partition read (meta-backed: a missing meta row ⇒ no group, no scope).
|
||||
var membership = await _adminRepo.GetGroupMembershipAsync(assignment.GroupId);
|
||||
if (membership == null)
|
||||
continue;
|
||||
@@ -142,8 +137,6 @@ public class DelegatedAdminService
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tenantId))
|
||||
continue;
|
||||
if (membership.ChargeHomeTenantQuota)
|
||||
homeCharged.Add(tenantId);
|
||||
if (tenantRoles.TryGetValue(tenantId, out var existing) && IsStronger(existing, role))
|
||||
continue;
|
||||
tenantRoles[tenantId] = role;
|
||||
@@ -156,7 +149,7 @@ public class DelegatedAdminService
|
||||
// NOT of the managed targets. A Pro MSP may manage Community customers. Resolution
|
||||
// failure / unknown home tenant counts as Community (fail-closed inside the entitlement
|
||||
// service), which empties the scope. Grant rows stay untouched — inert until upgrade/trial.
|
||||
var scope = new DelegatedScope(tenantRoles, homeCharged);
|
||||
var scope = new DelegatedScope(tenantRoles);
|
||||
_cache.Set(cacheKey, scope, _cacheDuration);
|
||||
return scope;
|
||||
}
|
||||
@@ -283,20 +276,6 @@ public class DelegatedAdminService
|
||||
public Task EnsureOwnedGroupAsync(string groupId, string name, string ownerTenantId)
|
||||
=> _adminRepo.EnsureOwnedTenantGroupAsync(NormalizeGroupId(groupId), name, ownerTenantId);
|
||||
|
||||
/// <summary>
|
||||
/// Sets the group's <see cref="TenantGroup.ChargeHomeTenantQuota"/> flag. The flag rides in every
|
||||
/// assignee's cached scope (<see cref="DelegatedScope.HomeChargedTenantIds"/>), so every current assignee
|
||||
/// is invalidated. Returns false if the group does not exist.
|
||||
/// </summary>
|
||||
public async Task<bool> SetChargeHomeTenantQuotaAsync(string groupId, bool chargeHomeTenantQuota)
|
||||
{
|
||||
groupId = NormalizeGroupId(groupId);
|
||||
var ok = await _adminRepo.SetTenantGroupChargeHomeTenantQuotaAsync(groupId, chargeHomeTenantQuota);
|
||||
if (ok)
|
||||
await InvalidateGroupAssigneesAsync(groupId);
|
||||
return ok;
|
||||
}
|
||||
|
||||
/// <summary>Deletes a group and all its assignments; invalidates every (former) assignee's scope.</summary>
|
||||
public async Task<bool> DeleteGroupAsync(string groupId)
|
||||
{
|
||||
@@ -435,17 +414,10 @@ public sealed class DelegatedScope
|
||||
public static readonly DelegatedScope Empty = new(new Dictionary<string, string>());
|
||||
|
||||
private readonly IReadOnlyDictionary<string, string> _tenantRoles;
|
||||
private readonly HashSet<string> _homeCharged;
|
||||
|
||||
public DelegatedScope(IReadOnlyDictionary<string, string> tenantRoles)
|
||||
: this(tenantRoles, null)
|
||||
{
|
||||
}
|
||||
|
||||
public DelegatedScope(IReadOnlyDictionary<string, string> tenantRoles, IReadOnlyCollection<string>? homeChargedTenantIds)
|
||||
{
|
||||
_tenantRoles = tenantRoles;
|
||||
_homeCharged = new HashSet<string>(homeChargedTenantIds ?? Array.Empty<string>(), StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
/// <summary>Tenant IDs (lowercase) this scope grants access to.</summary>
|
||||
@@ -457,19 +429,9 @@ public sealed class DelegatedScope
|
||||
if (_tenantRoles.Values.All(r => r == Constants.DelegatedRoles.DelegatedReader))
|
||||
return this;
|
||||
var readerRoles = _tenantRoles.Keys.ToDictionary(t => t, _ => Constants.DelegatedRoles.DelegatedReader, StringComparer.OrdinalIgnoreCase);
|
||||
return new DelegatedScope(readerRoles, _homeCharged);
|
||||
return new DelegatedScope(readerRoles);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The covered tenants reached through a Tenant Group flagged <see cref="TenantGroup.ChargeHomeTenantQuota"/>:
|
||||
/// MCP reads into them are charged to the caller's HOME tenant. Billing attribution only — never authz.
|
||||
/// </summary>
|
||||
public IReadOnlyCollection<string> HomeChargedTenantIds => _homeCharged;
|
||||
|
||||
/// <summary>True if MCP reads into the given tenant are charged to the caller's home tenant.</summary>
|
||||
public bool ChargesHome(string? tenantId)
|
||||
=> !string.IsNullOrEmpty(tenantId) && _homeCharged.Contains(tenantId);
|
||||
|
||||
public bool IsEmpty => _tenantRoles.Count == 0;
|
||||
|
||||
/// <summary>True if the scope grants access to the given tenant (any delegated role).</summary>
|
||||
@@ -557,12 +519,6 @@ public class TenantGroupEntity : ITableEntity
|
||||
/// <summary>The managed tenant ID (lowercase) — denormalized copy of RowKey on membership rows; empty on meta.</summary>
|
||||
public string TenantId { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Meta row only (null/omitted on membership rows): see <see cref="TenantGroup.ChargeHomeTenantQuota"/>.
|
||||
/// Nullable so a membership-row write never carries a spurious false column.
|
||||
/// </summary>
|
||||
public bool? ChargeHomeTenantQuota { get; set; }
|
||||
|
||||
/// <summary>Meta row only: the managing tenant that owns this self-service group (see <see cref="Constants.TenantGroupIds"/>); null on operator-created groups.</summary>
|
||||
public string? OwnerTenantId { get; set; }
|
||||
}
|
||||
|
||||
+7
-77
@@ -1,12 +1,6 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services.Backup;
|
||||
using AutopilotMonitor.Functions.Services.Locking;
|
||||
using AutopilotMonitor.Shared;
|
||||
using Azure;
|
||||
using Azure.Storage.Blobs.Models;
|
||||
using Azure.Storage.Blobs.Specialized;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
@@ -19,89 +13,25 @@ namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
/// Deliberately a DEDICATED sentinel (<see cref="LockBlobName"/> in the
|
||||
/// <c>deletion-manifests</c> container) rather than the backup sentinel in
|
||||
/// <see cref="BlobBackupStore"/> — sharing that lease would serialize backups against the
|
||||
/// retention fanout, which have no reason to exclude each other. Lease mechanics
|
||||
/// (60s lease + <see cref="MaintenanceLeaseHolder"/> 45s renewal, <see cref="LeaseHeldException"/>
|
||||
/// on contention) are identical to the backup path.
|
||||
/// retention fanout, which have no reason to exclude each other.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The <c>deletion-manifests</c> container has a 30-day lifecycle policy that may delete an
|
||||
/// idle sentinel; <see cref="AcquireLeaseAsync"/> re-ensures it on every acquire, so the
|
||||
/// lock self-heals. The manifest-TTL sweep skips the sentinel because <c>_lock/…</c> does
|
||||
/// not parse as a manifest blob name.
|
||||
/// idle sentinel; the base class re-ensures it on every acquire, so the lock self-heals.
|
||||
/// The manifest-TTL sweep skips the sentinel because <c>_lock/…</c> does not parse as a
|
||||
/// manifest blob name.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class SessionDeletionMaintenanceLockStore
|
||||
public class SessionDeletionMaintenanceLockStore : BlobLeaseLockStore
|
||||
{
|
||||
/// <summary>Sentinel blob path — leased by the timer / manual-trigger worker to serialize runs.</summary>
|
||||
public const string LockBlobName = "_lock/session-deletion-maintenance.lock";
|
||||
|
||||
/// <summary>Lease duration (Azure spec allows 15..60s for renewable leases).</summary>
|
||||
public static readonly TimeSpan LeaseDuration = TimeSpan.FromSeconds(60);
|
||||
|
||||
private readonly BlobStorageService _blobs;
|
||||
private readonly ILogger<SessionDeletionMaintenanceLockStore> _logger;
|
||||
private int _containerEnsured;
|
||||
|
||||
public SessionDeletionMaintenanceLockStore(
|
||||
BlobStorageService blobs,
|
||||
ILogger<SessionDeletionMaintenanceLockStore> logger)
|
||||
: base(blobs, Constants.BlobContainers.DeletionManifests, LockBlobName, "session-deletion maintenance")
|
||||
{
|
||||
_blobs = blobs;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Attempts to acquire the session-deletion maintenance lease. Throws
|
||||
/// <see cref="LeaseHeldException"/> on 409 (another holder); other errors propagate.
|
||||
/// Caller is responsible for renewing (via <see cref="MaintenanceLeaseHolder"/>) and
|
||||
/// releasing. <c>virtual</c> so tests can stub it.
|
||||
/// </summary>
|
||||
public virtual async Task<BlobLeaseClient> AcquireLeaseAsync(TimeSpan? leaseDuration = null, CancellationToken ct = default)
|
||||
{
|
||||
await EnsureSentinelAsync(ct).ConfigureAwait(false);
|
||||
|
||||
var blob = _blobs.GetContainerClient(Constants.BlobContainers.DeletionManifests).GetBlobClient(LockBlobName);
|
||||
var leaseClient = blob.GetBlobLeaseClient();
|
||||
try
|
||||
{
|
||||
await leaseClient.AcquireAsync(leaseDuration ?? LeaseDuration, conditions: null, ct).ConfigureAwait(false);
|
||||
return leaseClient;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 409)
|
||||
{
|
||||
throw new LeaseHeldException("session-deletion maintenance lease is held by another run", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Idempotently ensures the sentinel blob exists — an AcquireLease on a missing blob
|
||||
/// throws 404. If-None-Match=* lets concurrent callers cooperate; both 409
|
||||
/// BlobAlreadyExists and 412 ConditionNotMet are the steady state.
|
||||
/// </summary>
|
||||
private async Task EnsureSentinelAsync(CancellationToken ct)
|
||||
{
|
||||
var container = _blobs.GetContainerClient(Constants.BlobContainers.DeletionManifests);
|
||||
if (Interlocked.CompareExchange(ref _containerEnsured, 0, 0) == 0)
|
||||
{
|
||||
await container.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false);
|
||||
Interlocked.Exchange(ref _containerEnsured, 1);
|
||||
}
|
||||
|
||||
var blob = container.GetBlobClient(LockBlobName);
|
||||
var options = new BlobUploadOptions
|
||||
{
|
||||
HttpHeaders = new BlobHttpHeaders { ContentType = "application/octet-stream" },
|
||||
Conditions = new BlobRequestConditions { IfNoneMatch = ETag.All },
|
||||
};
|
||||
try
|
||||
{
|
||||
using var empty = new MemoryStream(Array.Empty<byte>());
|
||||
await blob.UploadAsync(empty, options, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 409 || ex.Status == 412)
|
||||
{
|
||||
// Sentinel exists already — steady state.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -239,22 +239,22 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
|
||||
// IME pattern-drift loop: the agent's session-end histogram (every enabled pattern,
|
||||
// zeros included) folds into the global per-IME-version statistics. Version from the
|
||||
// event (the tracker saw IME's own "Agent version is:" line) with the session row as
|
||||
// fallback — the ime_agent_version write above is fire-and-forget, so the row may lag
|
||||
// in the same batch. Fire-and-forget, never affects the ingest response.
|
||||
// zeros included) folds into the global per-IME-version statistics, filtered to the
|
||||
// shipped catalog inside the service. Version from the event (the tracker saw IME's
|
||||
// own "Agent version is:" line) with the session row as fallback — the
|
||||
// ime_agent_version write above is fire-and-forget, so the row may lag in the same
|
||||
// batch. Fire-and-forget, never affects the ingest response.
|
||||
var patternHitsEvent = request.Events.FirstOrDefault(e =>
|
||||
e.EventType == AutopilotMonitor.Shared.Constants.EventTypes.ImePatternHits && e.Data != null);
|
||||
if (patternHitsEvent != null)
|
||||
{
|
||||
var builtInHits = Ime.ImePatternHealthService.ExtractBuiltInHits(patternHitsEvent.Data);
|
||||
string? hitsImeVersion = null;
|
||||
if (patternHitsEvent.Data!.TryGetValue("imeVersion", out var imeVersionObj))
|
||||
hitsImeVersion = imeVersionObj?.ToString();
|
||||
if (string.IsNullOrWhiteSpace(hitsImeVersion))
|
||||
hitsImeVersion = updatedSession?.ImeAgentVersion;
|
||||
|
||||
_ = _imePatternHealth.RecordSessionHitsAsync(hitsImeVersion, builtInHits, request.TenantId, request.SessionId);
|
||||
_ = _imePatternHealth.RecordSessionHitsAsync(hitsImeVersion, patternHitsEvent.Data, request.TenantId, request.SessionId);
|
||||
}
|
||||
|
||||
// Authoritative counter reconcile (EventCount + RebootCount): the LAST counter write
|
||||
@@ -395,8 +395,12 @@ namespace AutopilotMonitor.Functions.Services
|
||||
});
|
||||
}
|
||||
|
||||
// TotalEventsProcessed / SuccessfulEnrollments are recomputed every two hours from
|
||||
// live data (D-198); no per-batch increment on the one global PlatformStats row.
|
||||
// Cumulative event counter on the TENANT row: the platform-wide figure is rolled up
|
||||
// from the tenant rows, so no batch ever touches the one global PlatformStats row.
|
||||
if (processedCount > 0)
|
||||
_ = _metricsRepo.IncrementTenantStatAsync(request.TenantId, nameof(TenantStats.TotalEventsProcessed), processedCount)
|
||||
.ContinueWith(t => _logger.LogWarning(t.Exception?.InnerException,
|
||||
"Fire-and-forget IncrementTenantStatAsync failed"), TaskContinuationOptions.OnlyOnFaulted);
|
||||
|
||||
_ = RecordGatherRuleStatsAsync(request.TenantId, storedEvents)
|
||||
.ContinueWith(t => _logger.LogWarning(t.Exception?.InnerException,
|
||||
|
||||
@@ -17,6 +17,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
private readonly IRuleRepository _ruleRepo;
|
||||
private readonly ILogger<GatherRuleService> _logger;
|
||||
private volatile bool _seeded = false;
|
||||
private readonly BackendBuildInfo? _buildInfo;
|
||||
private readonly SemaphoreSlim _seedGate = new(1, 1);
|
||||
|
||||
// Cached set of currently-shipped built-in gather rule IDs from
|
||||
@@ -28,10 +29,11 @@ namespace AutopilotMonitor.Functions.Services
|
||||
private HashSet<string>? _liveCatalogIds;
|
||||
private readonly object _liveCatalogLock = new();
|
||||
|
||||
public GatherRuleService(IRuleRepository ruleRepo, ILogger<GatherRuleService> logger)
|
||||
public GatherRuleService(IRuleRepository ruleRepo, ILogger<GatherRuleService> logger, BackendBuildInfo? buildInfo = null)
|
||||
{
|
||||
_ruleRepo = ruleRepo;
|
||||
_logger = logger;
|
||||
_buildInfo = buildInfo;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -349,6 +351,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
await _ruleRepo.StoreGatherRuleAsync(rule, "global");
|
||||
}
|
||||
_logger.LogInformation($"Written {builtInRules.Count} built-in gather rules from code");
|
||||
await _ruleRepo.SetRuleCatalogStampAsync(RuleCatalogSeedGate.Stamp(RuleCatalogStamp.KindGather, RuleCatalogStamp.SourceEmbedded, builtInRules.Count));
|
||||
|
||||
_seeded = false;
|
||||
|
||||
@@ -399,6 +402,14 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
else
|
||||
{
|
||||
// A GitHub reseed newer than this build owns the table: the embedded catalog is
|
||||
// the older source (RuleCatalogSeedGate).
|
||||
if (!await RuleCatalogSeedGate.AllowedAsync(_ruleRepo, _buildInfo, RuleCatalogStamp.KindGather, _logger))
|
||||
{
|
||||
_seeded = true;
|
||||
return;
|
||||
}
|
||||
|
||||
var existingLookup = existingRules.ToDictionary(r => r.RuleId, r => r);
|
||||
var updated = 0;
|
||||
|
||||
|
||||
@@ -9,9 +9,9 @@ namespace AutopilotMonitor.Functions.Services.Ime
|
||||
/// <para>
|
||||
/// <b>Ingest</b> (<see cref="RecordSessionHitsAsync"/>, fire-and-forget from
|
||||
/// <c>EventIngestProcessor</c>): the agent's session-end <c>ime_pattern_hits</c> histogram is
|
||||
/// filtered to shipped pattern IDs (a device may only claim an ID; tenant custom patterns
|
||||
/// never reach the global table), folded into <c>ImePatternStats</c> for the session's IME
|
||||
/// version, then the version is evaluated for drift against the fleet baseline
|
||||
/// filtered to the shipped pattern IDs of the global catalog (a device may only claim an ID;
|
||||
/// tenant custom patterns never reach the global table), folded into <c>ImePatternStats</c>
|
||||
/// for the session's IME version, then the version is evaluated for drift against the fleet baseline
|
||||
/// (<see cref="ImePatternDriftEvaluator"/>). A finding stamps the cell once and raises
|
||||
/// <c>ImePatternDriftSuspected</c>.
|
||||
/// </para>
|
||||
@@ -20,6 +20,12 @@ namespace AutopilotMonitor.Functions.Services.Ime
|
||||
/// page and the MCP tool. Stats are cached for <see cref="StatsCacheTtl"/> for the drift
|
||||
/// evaluation only — the read side always queries fresh.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The pattern catalog on both sides is the seeded <c>global</c> partition
|
||||
/// (<see cref="ImeLogPatternService.GetGlobalCatalogAsync"/>), never the embedded resource:
|
||||
/// a GitHub reseed changes what agents run before the next backend deploy, and the health
|
||||
/// view must follow it.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class ImePatternHealthService
|
||||
{
|
||||
@@ -27,6 +33,7 @@ namespace AutopilotMonitor.Functions.Services.Ime
|
||||
|
||||
private readonly IMetricsRepository _metricsRepo;
|
||||
private readonly ISessionRepository _sessionRepo;
|
||||
private readonly ImeLogPatternService _patterns;
|
||||
private readonly OpsEventService _opsEvents;
|
||||
private readonly ILogger<ImePatternHealthService> _logger;
|
||||
|
||||
@@ -37,29 +44,30 @@ namespace AutopilotMonitor.Functions.Services.Ime
|
||||
public ImePatternHealthService(
|
||||
IMetricsRepository metricsRepo,
|
||||
ISessionRepository sessionRepo,
|
||||
ImeLogPatternService patterns,
|
||||
OpsEventService opsEvents,
|
||||
ILogger<ImePatternHealthService> logger)
|
||||
{
|
||||
_metricsRepo = metricsRepo;
|
||||
_sessionRepo = sessionRepo;
|
||||
_patterns = patterns;
|
||||
_opsEvents = opsEvents;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extracts the histogram from an <c>ime_pattern_hits</c> event payload: the nested
|
||||
/// <c>hits</c> object (patternId → count), restricted to shipped pattern IDs. Returns an
|
||||
/// empty map when the payload has no usable histogram.
|
||||
/// <c>hits</c> object (patternId → count), restricted to <paramref name="shippedIds"/>.
|
||||
/// Returns an empty map when the payload has no usable histogram.
|
||||
/// </summary>
|
||||
public static Dictionary<string, int> ExtractBuiltInHits(IReadOnlyDictionary<string, object>? data)
|
||||
public static Dictionary<string, int> ExtractBuiltInHits(IReadOnlyDictionary<string, object>? data, ISet<string> shippedIds)
|
||||
{
|
||||
var result = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
|
||||
if (data == null || !data.TryGetValue("hits", out var hitsObj) || hitsObj == null) return result;
|
||||
|
||||
var builtIn = BuiltInImeLogPatterns.BuiltInPatternIds.Value;
|
||||
foreach (var (patternId, value) in EnumerateHits(hitsObj))
|
||||
{
|
||||
if (!builtIn.Contains(patternId)) continue;
|
||||
if (!shippedIds.Contains(patternId)) continue;
|
||||
result[patternId] = value;
|
||||
}
|
||||
return result;
|
||||
@@ -102,13 +110,17 @@ namespace AutopilotMonitor.Functions.Services.Ime
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Folds one session's histogram into the version statistics and evaluates drift.
|
||||
/// Never throws — the caller runs it fire-and-forget behind the ingest response.
|
||||
/// Folds one session's histogram (the raw <c>ime_pattern_hits</c> payload) into the
|
||||
/// version statistics and evaluates drift. Never throws — the caller runs it
|
||||
/// fire-and-forget behind the ingest response.
|
||||
/// </summary>
|
||||
public async Task RecordSessionHitsAsync(string? imeVersion, IReadOnlyDictionary<string, int> builtInHits, string tenantId, string sessionId)
|
||||
public async Task RecordSessionHitsAsync(string? imeVersion, IReadOnlyDictionary<string, object>? data, string tenantId, string sessionId)
|
||||
{
|
||||
try
|
||||
{
|
||||
var catalog = await _patterns.GetGlobalCatalogAsync();
|
||||
var shipped = new HashSet<string>(catalog.Select(p => p.PatternId), StringComparer.OrdinalIgnoreCase);
|
||||
var builtInHits = ExtractBuiltInHits(data, shipped);
|
||||
if (builtInHits.Count == 0) return;
|
||||
if (!ImeMsiArchiver.IsPlausibleVersion(imeVersion))
|
||||
{
|
||||
@@ -202,7 +214,9 @@ namespace AutopilotMonitor.Functions.Services.Ime
|
||||
{
|
||||
var stats = await _metricsRepo.GetImePatternStatsAsync();
|
||||
var history = await _sessionRepo.GetImeVersionHistoryAsync();
|
||||
return BuildResponse(stats, history, BuiltInImeLogPatterns.GetAll(), DateTime.UtcNow);
|
||||
var catalog = await _patterns.GetGlobalCatalogAsync();
|
||||
var stamp = await _patterns.GetCatalogStampAsync();
|
||||
return BuildResponse(stats, history, catalog, stamp, DateTime.UtcNow);
|
||||
}
|
||||
|
||||
/// <summary>Pure projection — testable without storage.</summary>
|
||||
@@ -210,6 +224,7 @@ namespace AutopilotMonitor.Functions.Services.Ime
|
||||
IReadOnlyCollection<ImePatternStatsEntry> stats,
|
||||
IReadOnlyCollection<ImeVersionHistoryEntry> history,
|
||||
IReadOnlyCollection<ImeLogPattern> catalog,
|
||||
RuleCatalogStamp? catalogStamp,
|
||||
DateTime nowUtc)
|
||||
{
|
||||
var baseline = ImePatternDriftEvaluator.SelectBaseline(stats, candidateVersion: null);
|
||||
@@ -290,6 +305,12 @@ namespace AutopilotMonitor.Functions.Services.Ime
|
||||
MinBaselineSessions = ImePatternDriftEvaluator.MinBaselineSessions,
|
||||
ExpectedHitRate = ImePatternDriftEvaluator.ExpectedHitRate,
|
||||
MinCandidateSessions = ImePatternDriftEvaluator.MinCandidateSessions,
|
||||
Catalog = new ImePatternHealthCatalog
|
||||
{
|
||||
Source = catalogStamp?.Source ?? RuleCatalogStamp.SourceEmbedded,
|
||||
StampedAt = catalogStamp?.StampedAt,
|
||||
PatternCount = catalog.Count,
|
||||
},
|
||||
Versions = versions,
|
||||
Patterns = patterns,
|
||||
Cells = cells,
|
||||
|
||||
@@ -15,11 +15,13 @@ namespace AutopilotMonitor.Functions.Services
|
||||
private readonly ILogger<ImeLogPatternService> _logger;
|
||||
private volatile bool _seeded = false;
|
||||
private readonly SemaphoreSlim _seedGate = new(1, 1);
|
||||
private readonly BackendBuildInfo? _buildInfo;
|
||||
|
||||
public ImeLogPatternService(IRuleRepository ruleRepo, ILogger<ImeLogPatternService> logger)
|
||||
public ImeLogPatternService(IRuleRepository ruleRepo, ILogger<ImeLogPatternService> logger, BackendBuildInfo? buildInfo = null)
|
||||
{
|
||||
_ruleRepo = ruleRepo;
|
||||
_logger = logger;
|
||||
_buildInfo = buildInfo;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -38,6 +40,21 @@ namespace AutopilotMonitor.Functions.Services
|
||||
public Task<List<ImeLogPattern>> GetAllPatternsForTenantAsync(string tenantId)
|
||||
=> LoadMergedPatternsAsync(tenantId);
|
||||
|
||||
/// <summary>
|
||||
/// The shipped catalog as the agents receive it: the seeded <c>global</c> partition
|
||||
/// (per-instance repository cache). The pattern-drift loop reads this, never the embedded
|
||||
/// resource, so the health view and the ingest filter follow every reseed.
|
||||
/// </summary>
|
||||
public async Task<List<ImeLogPattern>> GetGlobalCatalogAsync()
|
||||
{
|
||||
await EnsureBuiltInPatternsSeededAsync();
|
||||
return await _ruleRepo.GetImeLogPatternsAsync("global");
|
||||
}
|
||||
|
||||
/// <summary>Which reseed last wrote the global catalog, if any was stamped.</summary>
|
||||
public Task<RuleCatalogStamp?> GetCatalogStampAsync()
|
||||
=> _ruleRepo.GetRuleCatalogStampAsync(RuleCatalogStamp.KindIme);
|
||||
|
||||
/// <summary>
|
||||
/// Global built-in patterns with the tenant's overrides applied (same PatternId wins),
|
||||
/// followed by the tenant's custom patterns. The global and tenant partitions are read
|
||||
@@ -147,6 +164,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
await _ruleRepo.StoreImeLogPatternAsync(pattern, "global");
|
||||
}
|
||||
_logger.LogInformation($"Written {builtInPatterns.Count} built-in IME log patterns from code");
|
||||
await _ruleRepo.SetRuleCatalogStampAsync(RuleCatalogSeedGate.Stamp(RuleCatalogStamp.KindIme, RuleCatalogStamp.SourceEmbedded, builtInPatterns.Count));
|
||||
|
||||
_seeded = false;
|
||||
|
||||
@@ -192,6 +210,14 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
else
|
||||
{
|
||||
// A GitHub reseed newer than this build owns the table: the embedded catalog is
|
||||
// the older source (RuleCatalogSeedGate).
|
||||
if (!await RuleCatalogSeedGate.AllowedAsync(_ruleRepo, _buildInfo, RuleCatalogStamp.KindIme, _logger))
|
||||
{
|
||||
_seeded = true;
|
||||
return;
|
||||
}
|
||||
|
||||
var existingLookup = existingPatterns.ToDictionary(p => p.PatternId, p => p);
|
||||
var updated = 0;
|
||||
|
||||
@@ -220,6 +246,22 @@ namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
_logger.LogInformation($"Updated {updated} built-in IME log patterns from code definitions");
|
||||
}
|
||||
|
||||
// Sunset: built-in rows the embedded catalog no longer ships. Patterns carry no
|
||||
// per-tenant RuleState, so this is a plain delete (the gather/analyze sunset minus
|
||||
// the GC). A failed delete leaves _seeded=false so the next call retries.
|
||||
var catalogIds = new HashSet<string>(builtInPatterns.Select(p => p.PatternId), StringComparer.OrdinalIgnoreCase);
|
||||
var sunset = existingPatterns.Where(p => p.IsBuiltIn && !catalogIds.Contains(p.PatternId)).ToList();
|
||||
var sunsetFailed = 0;
|
||||
foreach (var pattern in sunset)
|
||||
{
|
||||
if (!await _ruleRepo.DeleteImeLogPatternAsync("global", pattern.PatternId)) sunsetFailed++;
|
||||
}
|
||||
if (sunset.Count > 0)
|
||||
{
|
||||
_logger.LogInformation("Sunset {Count} built-in IME log pattern(s) no longer in the embedded catalog ({Failed} failed)", sunset.Count, sunsetFailed);
|
||||
}
|
||||
if (sunsetFailed > 0) return;
|
||||
}
|
||||
|
||||
_seeded = true;
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services.Backup;
|
||||
using Azure;
|
||||
using Azure.Storage.Blobs.Models;
|
||||
using Azure.Storage.Blobs.Specialized;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Locking
|
||||
{
|
||||
/// <summary>
|
||||
/// Blob-lease lock on a sentinel blob: serializes a run against every other entry point of
|
||||
/// the same work (timer, manual trigger, a second host instance). One subclass per lock —
|
||||
/// each names its own sentinel, because two kinds of work that have no reason to exclude
|
||||
/// each other must not share a lease.
|
||||
/// <para>
|
||||
/// Lease mechanics: 60s lease, renewed by <see cref="MaintenanceLeaseHolder"/> every 45s,
|
||||
/// <see cref="LeaseHeldException"/> on contention. The sentinel is re-ensured on every
|
||||
/// acquire, so a lifecycle policy that deletes an idle sentinel cannot break the lock.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public abstract class BlobLeaseLockStore
|
||||
{
|
||||
/// <summary>Lease duration (Azure spec allows 15..60s for renewable leases).</summary>
|
||||
public static readonly TimeSpan LeaseDuration = TimeSpan.FromSeconds(60);
|
||||
|
||||
private readonly BlobStorageService _blobs;
|
||||
private readonly string _containerName;
|
||||
private readonly string _lockBlobName;
|
||||
private readonly string _description;
|
||||
private int _containerEnsured;
|
||||
|
||||
protected BlobLeaseLockStore(BlobStorageService blobs, string containerName, string lockBlobName, string description)
|
||||
{
|
||||
_blobs = blobs;
|
||||
_containerName = containerName;
|
||||
_lockBlobName = lockBlobName;
|
||||
_description = description;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Attempts to acquire the lease. Throws <see cref="LeaseHeldException"/> on 409 (another
|
||||
/// holder); other errors propagate. Caller is responsible for renewing (via
|
||||
/// <see cref="MaintenanceLeaseHolder"/>) and releasing. <c>virtual</c> so tests can stub it.
|
||||
/// </summary>
|
||||
public virtual async Task<BlobLeaseClient> AcquireLeaseAsync(TimeSpan? leaseDuration = null, CancellationToken ct = default)
|
||||
{
|
||||
await EnsureSentinelAsync(ct).ConfigureAwait(false);
|
||||
|
||||
var blob = _blobs.GetContainerClient(_containerName).GetBlobClient(_lockBlobName);
|
||||
var leaseClient = blob.GetBlobLeaseClient();
|
||||
try
|
||||
{
|
||||
await leaseClient.AcquireAsync(leaseDuration ?? LeaseDuration, conditions: null, ct).ConfigureAwait(false);
|
||||
return leaseClient;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 409)
|
||||
{
|
||||
throw new LeaseHeldException($"{_description} lease is held by another run", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Idempotently ensures the sentinel blob exists — an AcquireLease on a missing blob
|
||||
/// throws 404. If-None-Match=* lets concurrent callers cooperate; both 409
|
||||
/// BlobAlreadyExists and 412 ConditionNotMet are the steady state.
|
||||
/// </summary>
|
||||
private async Task EnsureSentinelAsync(CancellationToken ct)
|
||||
{
|
||||
var container = _blobs.GetContainerClient(_containerName);
|
||||
if (Interlocked.CompareExchange(ref _containerEnsured, 0, 0) == 0)
|
||||
{
|
||||
await container.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false);
|
||||
Interlocked.Exchange(ref _containerEnsured, 1);
|
||||
}
|
||||
|
||||
var blob = container.GetBlobClient(_lockBlobName);
|
||||
var options = new BlobUploadOptions
|
||||
{
|
||||
HttpHeaders = new BlobHttpHeaders { ContentType = "application/octet-stream" },
|
||||
Conditions = new BlobRequestConditions { IfNoneMatch = ETag.All },
|
||||
};
|
||||
try
|
||||
{
|
||||
using var empty = new MemoryStream(Array.Empty<byte>());
|
||||
await blob.UploadAsync(empty, options, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 409 || ex.Status == 412)
|
||||
{
|
||||
// Sentinel exists already — steady state.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services.Queueing;
|
||||
using AutopilotMonitor.Shared;
|
||||
using Azure.Storage.Queues;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Maintenance
|
||||
{
|
||||
/// <summary>
|
||||
/// Azure-Storage-Queue producer for the manual maintenance trigger. Same shape as the
|
||||
/// session-deletion and backup trigger producers: <see cref="QueueClientFactory"/>
|
||||
/// (Managed Identity preferred), fail-hard.
|
||||
/// </summary>
|
||||
public sealed class AzureQueueMaintenanceTriggerProducer : IMaintenanceTriggerProducer
|
||||
{
|
||||
private readonly QueueClient _queueClient;
|
||||
private readonly ILogger<AzureQueueMaintenanceTriggerProducer> _logger;
|
||||
private int _queueEnsured;
|
||||
|
||||
public AzureQueueMaintenanceTriggerProducer(
|
||||
QueueClientFactory queueFactory,
|
||||
ILogger<AzureQueueMaintenanceTriggerProducer> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
_queueClient = queueFactory.Create(Constants.QueueNames.MaintenanceTrigger);
|
||||
}
|
||||
|
||||
public async Task EnqueueAsync(MaintenanceTriggerEnvelope envelope, CancellationToken ct = default)
|
||||
{
|
||||
if (envelope == null) throw new ArgumentNullException(nameof(envelope));
|
||||
if (string.IsNullOrEmpty(envelope.TriggeredBy)) throw new ArgumentException("TriggeredBy required", nameof(envelope));
|
||||
|
||||
await EnsureQueueExistsAsync(ct).ConfigureAwait(false);
|
||||
|
||||
// Fail-hard: do NOT swallow. Caller (HTTP trigger) catches and returns 500.
|
||||
var body = JsonConvert.SerializeObject(envelope);
|
||||
await _queueClient.SendMessageAsync(body, ct).ConfigureAwait(false);
|
||||
_logger.LogInformation("Maintenance trigger enqueued (triggeredBy={TriggeredBy})", envelope.TriggeredBy);
|
||||
}
|
||||
|
||||
private async Task EnsureQueueExistsAsync(CancellationToken ct)
|
||||
{
|
||||
if (_queueEnsured == 1) return;
|
||||
try
|
||||
{
|
||||
await _queueClient.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false);
|
||||
Interlocked.Exchange(ref _queueEnsured, 1);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Ensure failure during enqueue is itself a fail-hard signal — surface to the caller.
|
||||
_logger.LogError(ex, "Maintenance trigger queue CreateIfNotExists failed");
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Maintenance
|
||||
{
|
||||
/// <summary>
|
||||
/// Queue envelope of a manually triggered maintenance run. There is no per-run state row —
|
||||
/// the Maintenance* ops events are the status surface.
|
||||
/// </summary>
|
||||
public sealed class MaintenanceTriggerEnvelope
|
||||
{
|
||||
public string TriggeredBy { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Date to aggregate (UTC date); null = yesterday.</summary>
|
||||
public DateTime? TargetDate { get; set; }
|
||||
|
||||
public bool AggregateOnly { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Producer for the manual maintenance trigger. <b>Fail-hard</b>: send failures propagate so
|
||||
/// the HTTP trigger answers 5xx instead of a hollow 202.
|
||||
/// </summary>
|
||||
public interface IMaintenanceTriggerProducer
|
||||
{
|
||||
Task EnqueueAsync(MaintenanceTriggerEnvelope envelope, CancellationToken ct = default);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services.Backup;
|
||||
using Azure.Storage.Blobs.Specialized;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Maintenance
|
||||
{
|
||||
/// <summary>
|
||||
/// Single-flight gate for the platform maintenance run: the 2h timer, the manual trigger
|
||||
/// and a second host instance all enter through <see cref="RunExclusiveAsync"/>, so two
|
||||
/// runs never aggregate, clean up and backfill the same tables at the same time.
|
||||
/// </summary>
|
||||
public class MaintenanceRunGate
|
||||
{
|
||||
private readonly MaintenanceRunLockStore _lockStore;
|
||||
private readonly OpsEventService _opsEvents;
|
||||
private readonly ILogger<MaintenanceRunGate> _logger;
|
||||
|
||||
public MaintenanceRunGate(MaintenanceRunLockStore lockStore, OpsEventService opsEvents, ILogger<MaintenanceRunGate> logger)
|
||||
{
|
||||
_lockStore = lockStore;
|
||||
_opsEvents = opsEvents;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Acquire+release probe for the manual trigger: a held lease means a run is active RIGHT
|
||||
/// NOW, so the operator is told instead of queuing a message that would only produce a
|
||||
/// SkippedLocked event. Racing the probe (a run starts between probe and worker pickup)
|
||||
/// is benign: the worker's own acquire lands on the SkippedLocked path.
|
||||
/// </summary>
|
||||
public virtual async Task<bool> IsRunActiveAsync(CancellationToken ct = default)
|
||||
{
|
||||
BlobLeaseClient lease;
|
||||
try
|
||||
{
|
||||
lease = await _lockStore.AcquireLeaseAsync(ct: ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (LeaseHeldException)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
// A probe lease that cannot be released would still be held when the worker picks the
|
||||
// message up seconds later — the run would be skipped as "locked" and the message
|
||||
// deleted. Let the failure surface: the trigger answers 5xx and nothing is queued.
|
||||
await lease.ReleaseAsync(cancellationToken: ct).ConfigureAwait(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs <paramref name="body"/> under the maintenance-run lease. Returns false without
|
||||
/// running it when another run holds the lease (SkippedLocked ops event). The Started
|
||||
/// event is written AFTER the lease is held, so a skip never looks like an active run,
|
||||
/// and the lease is released LAST, so the body's terminal ops event is still exclusive.
|
||||
/// </summary>
|
||||
public virtual async Task<bool> RunExclusiveAsync(string triggeredBy, Func<Task> body)
|
||||
{
|
||||
BlobLeaseClient lease;
|
||||
try
|
||||
{
|
||||
lease = await _lockStore.AcquireLeaseAsync().ConfigureAwait(false);
|
||||
}
|
||||
catch (LeaseHeldException)
|
||||
{
|
||||
_logger.LogWarning("Maintenance: lease held by another run — skipping (triggeredBy={TriggeredBy})", triggeredBy);
|
||||
await _opsEvents.RecordMaintenanceSkippedLockedAsync(triggeredBy).ConfigureAwait(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
// The maintenance body takes no cancellation token; a lost lease is reported, not
|
||||
// enforced — the run is minutes long and the lease renews every 45s.
|
||||
using var leaseLost = new CancellationTokenSource();
|
||||
var leaseHolder = new MaintenanceLeaseHolder(lease, leaseLost, _logger);
|
||||
try
|
||||
{
|
||||
await _opsEvents.RecordMaintenanceStartedAsync(triggeredBy).ConfigureAwait(false);
|
||||
await body().ConfigureAwait(false);
|
||||
return true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (leaseHolder.RenewalFailureReason != null)
|
||||
_logger.LogWarning("Maintenance: the run lease could not be renewed during the run ({Reason}) — another run may have overlapped",
|
||||
leaseHolder.RenewalFailureReason);
|
||||
await leaseHolder.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
using AutopilotMonitor.Functions.Services.Locking;
|
||||
using AutopilotMonitor.Shared;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Maintenance
|
||||
{
|
||||
/// <summary>
|
||||
/// Blob-lease lock for the platform maintenance run (2h timer and manual trigger). Its own
|
||||
/// sentinel: the session-deletion maintenance and the critical-table backup have no reason
|
||||
/// to exclude this run, so they keep their own leases.
|
||||
/// </summary>
|
||||
public class MaintenanceRunLockStore : BlobLeaseLockStore
|
||||
{
|
||||
/// <summary>Sentinel blob path. <c>_lock/…</c> does not parse as a manifest blob name, so the manifest-TTL sweep skips it.</summary>
|
||||
public const string LockBlobName = "_lock/platform-maintenance.lock";
|
||||
|
||||
public MaintenanceRunLockStore(BlobStorageService blobs)
|
||||
: base(blobs, Constants.BlobContainers.DeletionManifests, LockBlobName, "platform maintenance")
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services.Queueing;
|
||||
using AutopilotMonitor.Shared;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Maintenance
|
||||
{
|
||||
/// <summary>
|
||||
/// Background worker for the <c>maintenance-trigger</c> queue — runs the manually triggered
|
||||
/// maintenance off the HTTP request, which would otherwise have to stay open for minutes
|
||||
/// and dies at the gateway limit.
|
||||
/// <para>
|
||||
/// A self-polling worker, not a queue-triggered function, because the run holds a lease
|
||||
/// for its whole lifetime. Concurrency against the 2h timer is handled inside
|
||||
/// <see cref="MaintenanceService.RunManualAsync"/> by the run gate: a lease-held run
|
||||
/// returns normally (SkippedLocked ops event), so the message is deleted rather than
|
||||
/// retried — the operator asked for "a run", and one is already active. A failing run
|
||||
/// reports MaintenanceFailed and returns as well: maintenance is not retried blindly.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class MaintenanceTriggerQueueWorker : QueuePollingWorker<MaintenanceTriggerEnvelope>
|
||||
{
|
||||
private readonly MaintenanceService _maintenance;
|
||||
|
||||
public MaintenanceTriggerQueueWorker(
|
||||
QueueClientFactory queueFactory,
|
||||
MaintenanceService maintenance,
|
||||
ILogger<MaintenanceTriggerQueueWorker> logger)
|
||||
: base(queueFactory, Constants.QueueNames.MaintenanceTrigger, logger, Constants.QueueNames.MaintenanceTriggerPoison)
|
||||
{
|
||||
_maintenance = maintenance ?? throw new ArgumentNullException(nameof(maintenance));
|
||||
}
|
||||
|
||||
// The host functionTimeout (60min) is the upper bound of a run; the message must not
|
||||
// reappear while its run is still going.
|
||||
protected override TimeSpan VisibilityTimeout => TimeSpan.FromMinutes(60);
|
||||
|
||||
protected override bool TryValidate(MaintenanceTriggerEnvelope envelope)
|
||||
=> !string.IsNullOrEmpty(envelope.TriggeredBy);
|
||||
|
||||
protected override string DescribeForLog(MaintenanceTriggerEnvelope envelope)
|
||||
=> $"triggeredBy={envelope.TriggeredBy} targetDate={envelope.TargetDate:yyyy-MM-dd} aggregateOnly={envelope.AggregateOnly}";
|
||||
|
||||
protected override Task HandleAsync(MaintenanceTriggerEnvelope envelope, CancellationToken ct)
|
||||
=> _maintenance.RunManualAsync(envelope.TargetDate, envelope.AggregateOnly, envelope.TriggeredBy);
|
||||
}
|
||||
}
|
||||
@@ -638,10 +638,10 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Recomputes platform-wide stats from all tables. Used on the public landing page (no
|
||||
/// auth required). Since D-198 this is the ONLY writer of TotalEnrollments,
|
||||
/// SuccessfulEnrollments and TotalEventsProcessed — the hot paths no longer increment the
|
||||
/// global row — so the figures move in two-hour steps.
|
||||
/// Rolls the platform-wide stats (public landing page, no auth) up from the per-tenant
|
||||
/// counters. The hot paths increment the tenant rows only; this run adds the growth of
|
||||
/// their sums since the previous run to the platform row, so the figures move in
|
||||
/// two-hour steps.
|
||||
/// </summary>
|
||||
private async Task RecomputePlatformStatsAsync()
|
||||
{
|
||||
@@ -652,65 +652,76 @@ namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
var tenantIds = await _maintenanceRepo.GetAllTenantIdsAsync();
|
||||
var allConfigs = await _tenantConfigService.GetAllConfigurationsAsync();
|
||||
long totalEnrollments = 0;
|
||||
long successfulEnrollments = 0;
|
||||
long totalEvents = 0;
|
||||
long totalUsers = 0;
|
||||
// "Active tenants" = tenants that have actually produced at least one enrollment
|
||||
// session. tenantIds comes from the TenantConfiguration table (every registered
|
||||
// tenant, including those that never granted consent and can never send data), so
|
||||
// it equals TotalSignedUpTenants and must NOT be used for the active count. Count
|
||||
// the tenants whose session query returns rows instead.
|
||||
int activeTenants = 0;
|
||||
var uniqueModels = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var liveModels = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var floorsInPlace = true;
|
||||
|
||||
foreach (var tid in tenantIds)
|
||||
{
|
||||
var sessions = await _sessionRepo.GetSessionsAsync(tid);
|
||||
if (sessions.Count > 0)
|
||||
activeTenants++;
|
||||
totalEnrollments += sessions.Count;
|
||||
successfulEnrollments += sessions.Count(s => s.Status == SessionStatus.Succeeded);
|
||||
long liveEvents = 0;
|
||||
|
||||
foreach (var s in sessions)
|
||||
{
|
||||
var modelKey = $"{s.Manufacturer} {s.Model}".Trim();
|
||||
if (!string.IsNullOrEmpty(modelKey))
|
||||
uniqueModels.Add(modelKey);
|
||||
totalEvents += s.EventCount;
|
||||
liveModels.Add(modelKey);
|
||||
liveEvents += s.EventCount;
|
||||
}
|
||||
|
||||
var userMetrics = await _metricsRepo.GetUserActivityMetricsAsync(tid);
|
||||
totalUsers += userMetrics.TotalUniqueUsers;
|
||||
|
||||
// Seed / self-heal the cumulative per-tenant enrollment counter: the live
|
||||
// session count (within retention) is a lower bound for "since signup".
|
||||
// Raise-only — retention prunes sessions, so recomputing/overwriting would
|
||||
// regress the counter (same reasoning as the TotalUsers clamp below).
|
||||
// Seed / self-heal the cumulative per-tenant counters: the live figures
|
||||
// (within retention) are a lower bound for "since signup". Raise-only —
|
||||
// retention prunes sessions, so overwriting would regress the counters.
|
||||
if (sessions.Count > 0)
|
||||
await _metricsRepo.EnsureTenantStatFloorAsync(tid, "TotalEnrollments", sessions.Count);
|
||||
{
|
||||
floorsInPlace &= await _metricsRepo.EnsureTenantStatFloorsAsync(tid, new Dictionary<string, long>
|
||||
{
|
||||
[nameof(TenantStats.TotalEnrollments)] = sessions.Count,
|
||||
[nameof(TenantStats.SuccessfulEnrollments)] = sessions.Count(s => s.Status == SessionStatus.Succeeded),
|
||||
[nameof(TenantStats.TotalEventsProcessed)] = liveEvents,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
var existingStats = await _metricsRepo.GetPlatformStatsAsync();
|
||||
// A floor that lands one run late would be counted as growth although the
|
||||
// platform row already contains it. Skip the rollup; the baseline stays, so the
|
||||
// next run adds the whole growth since the last successful one.
|
||||
if (!floorsInPlace)
|
||||
{
|
||||
_logger.LogWarning("Platform stats rollup skipped: a tenant counter floor could not be written");
|
||||
return;
|
||||
}
|
||||
|
||||
var stats = BuildMonotonicPlatformStats(
|
||||
recomputedEnrollments: totalEnrollments,
|
||||
recomputedSuccessful: successfulEnrollments,
|
||||
recomputedEvents: totalEvents,
|
||||
recomputedUsers: totalUsers,
|
||||
recomputedActiveTenants: activeTenants,
|
||||
recomputedUniqueModels: uniqueModels.Count,
|
||||
signedUpTenants: allConfigs.Count,
|
||||
existing: existingStats,
|
||||
nowUtc: DateTime.UtcNow);
|
||||
var tenantStats = await _metricsRepo.GetAllTenantStatsAsync();
|
||||
var sources = new PlatformRollupSources
|
||||
{
|
||||
TotalEnrollments = tenantStats.Sum(t => t.TotalEnrollments),
|
||||
SuccessfulEnrollments = tenantStats.Sum(t => t.SuccessfulEnrollments),
|
||||
TotalEventsProcessed = tenantStats.Sum(t => t.TotalEventsProcessed),
|
||||
ActiveTenants = tenantStats.Count(t => t.TotalEnrollments > 0),
|
||||
SeenDeviceModels = await _metricsRepo.RecordSeenDeviceModelsAsync(liveModels),
|
||||
};
|
||||
|
||||
var nowUtc = DateTime.UtcNow;
|
||||
var signedUpTenants = allConfigs.Count;
|
||||
var stats = await _metricsRepo.RollupPlatformStatsAsync(
|
||||
(existing, baseline) => BuildPlatformStatsRollup(existing, baseline, sources, totalUsers, signedUpTenants, nowUtc),
|
||||
sources);
|
||||
if (stats == null)
|
||||
{
|
||||
_logger.LogWarning("Platform stats rollup was not persisted; publish skipped");
|
||||
return;
|
||||
}
|
||||
|
||||
await _metricsRepo.SavePlatformStatsAsync(stats);
|
||||
await TryPublishPlatformStatsJsonAsync(stats);
|
||||
|
||||
sw.Stop();
|
||||
_logger.LogInformation($"Platform stats recomputed in {sw.ElapsedMilliseconds}ms: " +
|
||||
_logger.LogInformation($"Platform stats rolled up in {sw.ElapsedMilliseconds}ms: " +
|
||||
$"{stats.TotalEnrollments} enrollments, {stats.TotalUsers} users, {tenantIds.Count} tenants, " +
|
||||
$"{stats.UniqueDeviceModels} models (all cumulative high-water-marks)");
|
||||
$"{stats.UniqueDeviceModels} models");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -719,35 +730,43 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merges a fresh platform-stats recompute with the previously persisted row. Platform
|
||||
/// stats are "since release" counters (public landing page): the tables the recompute
|
||||
/// scans are pruned by session/user-activity retention, so a raw recompute can only see
|
||||
/// the retention window and would regress the figures after every cleanup. Every
|
||||
/// cumulative counter is therefore a monotonic high-water-mark — the recompute may raise
|
||||
/// it (self-heal for lost increments), never lower it. IssuesDetected is increment-only
|
||||
/// (no recompute source) and is carried over verbatim. TotalSignedUpTenants is the one
|
||||
/// deliberate exception: the TenantConfiguration table is not retention-pruned, so its
|
||||
/// count is authoritative current state and a drop reflects real offboarding, not data loss.
|
||||
/// One cumulative platform counter: the persisted total plus the growth of its source
|
||||
/// since the previous run. The total is never derived from the source's absolute value —
|
||||
/// the sources shrink (retention prunes sessions, offboarding deletes a tenant's
|
||||
/// counters), and a high-water-mark over a shrinking source stops moving for good. A
|
||||
/// shrunken source costs the growth of that one interval. Without a baseline (first run)
|
||||
/// the source is a lower bound only, so nothing already contained in the total is added
|
||||
/// a second time.
|
||||
/// </summary>
|
||||
internal static PlatformStats BuildMonotonicPlatformStats(
|
||||
long recomputedEnrollments,
|
||||
long recomputedSuccessful,
|
||||
long recomputedEvents,
|
||||
long recomputedUsers,
|
||||
long recomputedActiveTenants,
|
||||
long recomputedUniqueModels,
|
||||
long signedUpTenants,
|
||||
internal static long Accumulate(long existingTotal, long? previousSource, long currentSource)
|
||||
=> previousSource is null
|
||||
? Math.Max(existingTotal, currentSource)
|
||||
: existingTotal + Math.Max(0, currentSource - previousSource.Value);
|
||||
|
||||
/// <summary>
|
||||
/// Merges one rollup run into the persisted platform row ("since release" counters for the
|
||||
/// public landing page). TotalUsers stays a high-water-mark over the live count: a
|
||||
/// persisted set of users would outlive the user-activity retention. IssuesDetected is
|
||||
/// increment-only and carried over for the published JSON; the storage write leaves it
|
||||
/// alone. TotalSignedUpTenants is current state — the TenantConfiguration table is not
|
||||
/// retention-pruned, so a drop reflects real offboarding, not data loss.
|
||||
/// </summary>
|
||||
internal static PlatformStats BuildPlatformStatsRollup(
|
||||
PlatformStats? existing,
|
||||
PlatformRollupSources? baseline,
|
||||
PlatformRollupSources sources,
|
||||
long recomputedUsers,
|
||||
long signedUpTenants,
|
||||
DateTime nowUtc)
|
||||
{
|
||||
return new PlatformStats
|
||||
{
|
||||
TotalEnrollments = Math.Max(recomputedEnrollments, existing?.TotalEnrollments ?? 0),
|
||||
SuccessfulEnrollments = Math.Max(recomputedSuccessful, existing?.SuccessfulEnrollments ?? 0),
|
||||
TotalEventsProcessed = Math.Max(recomputedEvents, existing?.TotalEventsProcessed ?? 0),
|
||||
TotalEnrollments = Accumulate(existing?.TotalEnrollments ?? 0, baseline?.TotalEnrollments, sources.TotalEnrollments),
|
||||
SuccessfulEnrollments = Accumulate(existing?.SuccessfulEnrollments ?? 0, baseline?.SuccessfulEnrollments, sources.SuccessfulEnrollments),
|
||||
TotalEventsProcessed = Accumulate(existing?.TotalEventsProcessed ?? 0, baseline?.TotalEventsProcessed, sources.TotalEventsProcessed),
|
||||
TotalTenants = Accumulate(existing?.TotalTenants ?? 0, baseline?.ActiveTenants, sources.ActiveTenants),
|
||||
UniqueDeviceModels = Accumulate(existing?.UniqueDeviceModels ?? 0, baseline?.SeenDeviceModels, sources.SeenDeviceModels),
|
||||
TotalUsers = Math.Max(recomputedUsers, existing?.TotalUsers ?? 0),
|
||||
TotalTenants = Math.Max(recomputedActiveTenants, existing?.TotalTenants ?? 0),
|
||||
UniqueDeviceModels = Math.Max(recomputedUniqueModels, existing?.UniqueDeviceModels ?? 0),
|
||||
TotalSignedUpTenants = signedUpTenants,
|
||||
IssuesDetected = existing?.IssuesDetected ?? 0,
|
||||
LastFullCompute = nowUtc,
|
||||
|
||||
@@ -55,6 +55,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly PreviewWhitelistService _previewWhitelistService;
|
||||
private readonly IStorageInitializer _storageInitializer;
|
||||
private readonly Maintenance.MaintenanceRunGate _runGate;
|
||||
private readonly ILogger<MaintenanceService> _logger;
|
||||
|
||||
private const string PlatformStatsAliasFileName = "platform-stats.json";
|
||||
@@ -90,6 +91,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
IHttpClientFactory httpClientFactory,
|
||||
PreviewWhitelistService previewWhitelistService,
|
||||
IStorageInitializer storageInitializer,
|
||||
Maintenance.MaintenanceRunGate runGate,
|
||||
ILogger<MaintenanceService> logger)
|
||||
{
|
||||
_maintenanceRepo = maintenanceRepo;
|
||||
@@ -120,6 +122,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
_httpClientFactory = httpClientFactory;
|
||||
_previewWhitelistService = previewWhitelistService;
|
||||
_storageInitializer = storageInitializer;
|
||||
_runGate = runGate;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
@@ -136,9 +139,12 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs all maintenance tasks (used by the daily timer trigger)
|
||||
/// Runs all maintenance tasks (used by the 2h timer trigger). Single-flight with the
|
||||
/// manual trigger: skipped when another run holds the maintenance-run lease.
|
||||
/// </summary>
|
||||
public async Task RunAllAsync()
|
||||
public Task RunAllAsync() => _runGate.RunExclusiveAsync("Timer", RunAllCoreAsync);
|
||||
|
||||
private async Task RunAllCoreAsync()
|
||||
{
|
||||
_logger.LogInformation($"Daily maintenance started at {DateTime.UtcNow}");
|
||||
var maintenanceStart = Stopwatch.StartNew();
|
||||
@@ -246,9 +252,27 @@ namespace AutopilotMonitor.Functions.Services
|
||||
=> window.Where(s => s.StartedAt >= windowStart).ToList();
|
||||
|
||||
/// <summary>
|
||||
/// Manually triggered maintenance with flexible date selection
|
||||
/// Manually triggered maintenance with flexible date selection. Single-flight with the
|
||||
/// timer: when another run holds the maintenance-run lease, nothing runs and the result
|
||||
/// says so.
|
||||
/// </summary>
|
||||
public async Task<MaintenanceResult> RunManualAsync(DateTime? targetDate = null, bool aggregateOnly = false, string triggeredBy = "Unknown")
|
||||
{
|
||||
MaintenanceResult? result = null;
|
||||
var ran = await _runGate.RunExclusiveAsync(triggeredBy,
|
||||
async () => result = await RunManualCoreAsync(targetDate, aggregateOnly, triggeredBy));
|
||||
return ran && result != null
|
||||
? result
|
||||
: new MaintenanceResult
|
||||
{
|
||||
TriggeredBy = triggeredBy,
|
||||
TriggeredAt = DateTime.UtcNow,
|
||||
Success = false,
|
||||
Error = "Another maintenance run is active",
|
||||
};
|
||||
}
|
||||
|
||||
private async Task<MaintenanceResult> RunManualCoreAsync(DateTime? targetDate, bool aggregateOnly, string triggeredBy)
|
||||
{
|
||||
_logger.LogInformation($"Manual maintenance triggered by {triggeredBy} at {DateTime.UtcNow}");
|
||||
var maintenanceStart = Stopwatch.StartNew();
|
||||
@@ -329,7 +353,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
result.Success = true;
|
||||
|
||||
_logger.LogInformation($"Manual maintenance completed in {maintenanceStart.ElapsedMilliseconds}ms");
|
||||
await _opsEventService.RecordMaintenanceCompletedAsync(result.DurationMs, triggeredBy);
|
||||
await _opsEventService.RecordMaintenanceCompletedAsync(result.DurationMs, triggeredBy, result);
|
||||
return result;
|
||||
}
|
||||
catch (Exception ex)
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
@@ -14,35 +12,40 @@ namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
/// <summary>
|
||||
/// Resolves and enforces the MCP request quota: a per-USER budget (daily + monthly) AND a per-TENANT
|
||||
/// budget (daily + monthly) that every request charged to that tenant counts against. Both must be free
|
||||
/// for a request to pass; the tenant budget is what makes "just create ten more accounts" pointless.
|
||||
/// budget (daily + monthly) that every request of that tenant's accounts counts against. Both must be
|
||||
/// free for a request to pass; the tenant budget is what makes "just create ten more accounts" pointless.
|
||||
///
|
||||
/// Both budgets belong to the caller's HOME tenant (the token's tid) — "the budget follows the
|
||||
/// delegating tenant": a delegated (MSP) read into a managed tenant draws on the MSP's own windows, never
|
||||
/// on the managed tenant's, so a managed tenant is neither charged nor blocked by its manager's reads.
|
||||
///
|
||||
/// User plan precedence: explicit per-user override (McpUsers.UsagePlan — honoured only for the identity
|
||||
/// the UPN is bound to, tid + oid, see McpUserService.GetBoundMcpUserAsync) → the caller's HOME-tenant
|
||||
/// edition default (FeatureEntitlementCatalog.McpUsagePlanName). Limits come from the admin-editable
|
||||
/// SectionUsagePlans definitions (AdminConfiguration.PlanTierDefinitionsJson); when no definition matches
|
||||
/// the plan name, the static catalog fallbacks apply. An override naming a plan that exists nowhere
|
||||
/// resolves to the Community fallback (fail-closed).
|
||||
/// the UPN is bound to, tid + oid, see McpUserService.GetBoundMcpUserAsync) → the home tenant's usage plan
|
||||
/// (TenantConfiguration.McpUsagePlanOverride, else the edition's catalog plan name). Limits come from the
|
||||
/// admin-editable SectionUsagePlans definitions (AdminConfiguration.PlanTierDefinitionsJson); when no
|
||||
/// definition matches the plan name, the static catalog fallbacks apply. An override naming a plan that
|
||||
/// exists nowhere resolves to the Community fallback (fail-closed).
|
||||
///
|
||||
/// Tenant plan: ALWAYS the CHARGED tenant's edition plan — a per-user override lifts that person's own
|
||||
/// budget, never an organization's. The charged tenant is the caller's own tenant, or, for a delegated
|
||||
/// (MSP) read, the MANAGED tenant whose data is read ("the budget follows the data": a Community customer
|
||||
/// is read with Community windows even by a Pro MSP). A definition without tenant limits falls back to
|
||||
/// the edition's catalog tenant limits; an explicit 0 lifts that window.
|
||||
/// Tenant plan: the home tenant's usage plan — a per-user override lifts that person's own budget, never
|
||||
/// the organization's. A definition without tenant limits falls back to the edition's catalog tenant
|
||||
/// limits; an explicit 0 lifts that window.
|
||||
///
|
||||
/// Purchased delegation slots grow BOTH budgets: every slot beyond the edition's included ones
|
||||
/// (TenantEntitlementService.GetPurchasedDelegatedSlots — entitlement-gated, so a Community or a
|
||||
/// conferred-Pro tenant earns nothing) adds the plan definition's slot values (else the edition's catalog
|
||||
/// slot values) to each window. A window lifted to 0 (unlimited) stays unlimited.
|
||||
///
|
||||
/// Counters: user counters ride on the UserUsageLog table (PK = oid), tenant counters on the
|
||||
/// McpTenantUsage table (PK = charged tenantId, RK = {yyyyMMdd}_{oid} — one partition read per check, no
|
||||
/// McpTenantUsage table (PK = home tenantId, RK = {yyyyMMdd}_{oid} — one partition read per check, no
|
||||
/// hot row); both are written fire-and-forget by McpQuotaEnforcementMiddleware for X-Client-Source: mcp
|
||||
/// requests. Daily = today's rows, monthly = the sum over the month. The usage SNAPSHOTS are cached for
|
||||
/// 60 seconds — per user (oid) and per charged tenant — so the worst-case overshoot is bounded
|
||||
/// 60 seconds — per user (oid) and per tenant — so the worst-case overshoot is bounded
|
||||
/// (limit + 60s × request-rate) — the same posture as the sliding-window rate limiter. Limits are
|
||||
/// re-resolved per check from services that carry their own caches, so a plan change applies at once.
|
||||
/// </summary>
|
||||
public class McpQuotaService
|
||||
{
|
||||
private static readonly TimeSpan CacheDuration = TimeSpan.FromSeconds(60);
|
||||
/// <summary>Concurrent tenant-snapshot reads for one bounded (fleet) aggregate check.</summary>
|
||||
private const int AggregateParallelism = 8;
|
||||
|
||||
internal static string UserCacheKey(string oid) => $"mcp-quota:user:{oid}";
|
||||
internal static string TenantCacheKey(string tenantId) => $"mcp-quota:tenant:{tenantId.ToLowerInvariant()}";
|
||||
@@ -86,133 +89,60 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks the caller against their own budget and their OWN tenant's organization budget
|
||||
/// (<paramref name="tenantId"/> = the token's tid, charged and homed alike). The self-service usage
|
||||
/// endpoint and every non-delegated MCP request use this shape.
|
||||
/// Resolves the caller's effective plans + limits and checks the current user AND tenant usage against
|
||||
/// them. <paramref name="tenantId"/> is the token's tid — the identity the per-user override is bound to
|
||||
/// and the tenant whose organization budget every request of this caller draws on, a delegated read
|
||||
/// included. Fail-open on counter/storage errors (a broken quota check must not take down MCP);
|
||||
/// fail-closed on plan resolution (unknown plan → Community).
|
||||
/// </summary>
|
||||
public virtual Task<McpQuotaDecision> CheckAsync(string oid, string? upn, string? tenantId)
|
||||
=> CheckAsync(oid, upn, tenantId, tenantId);
|
||||
|
||||
/// <summary>
|
||||
/// Resolves the caller's effective plans + limits and checks the current user AND charged-tenant usage
|
||||
/// against them. <paramref name="homeTenantId"/> is the token's tid (the identity the per-user override
|
||||
/// is bound to, and the edition the user's own plan follows); <paramref name="chargeTenantId"/> is the
|
||||
/// tenant whose organization budget this request draws on — the home tenant, or the MANAGED tenant of a
|
||||
/// delegated (MSP) read. Fail-open on counter/storage errors (a broken quota check must not take down
|
||||
/// MCP); fail-closed on plan resolution (unknown plan → Community).
|
||||
/// </summary>
|
||||
public virtual async Task<McpQuotaDecision> CheckAsync(string oid, string? upn, string? homeTenantId, string? chargeTenantId)
|
||||
public virtual async Task<McpQuotaDecision> CheckAsync(string oid, string? upn, string? tenantId)
|
||||
{
|
||||
var nowUtc = _time.GetUtcNow().UtcDateTime;
|
||||
var limits = await ResolvePlanAsync(AdminIdentity.Create(upn, homeTenantId, oid), homeTenantId, chargeTenantId);
|
||||
var targetTenantId = TargetOf(homeTenantId, chargeTenantId);
|
||||
var limits = await ResolvePlanAsync(AdminIdentity.Create(upn, tenantId, oid), tenantId);
|
||||
|
||||
var user = await ReadUserUsageAsync(oid, nowUtc);
|
||||
if (user == null)
|
||||
return McpQuotaDecision.FailOpen(limits, targetTenantId);
|
||||
return McpQuotaDecision.FailOpen(limits);
|
||||
|
||||
var tenant = await ReadTenantUsageAsync(chargeTenantId, limits, nowUtc);
|
||||
var tenant = await ReadTenantUsageAsync(tenantId, limits, nowUtc);
|
||||
if (tenant == null)
|
||||
return McpQuotaDecision.FailOpen(limits, targetTenantId);
|
||||
return McpQuotaDecision.FailOpen(limits);
|
||||
|
||||
return BuildDecision(limits, user.DailyUsed, user.MonthlyUsed, tenant.DailyUsed, tenant.MonthlyUsed, nowUtc, targetTenantId);
|
||||
return BuildDecision(limits, user.DailyUsed, user.MonthlyUsed, tenant.DailyUsed, tenant.MonthlyUsed, nowUtc);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Bounded (fleet) aggregate check for a delegated (MSP) caller: the caller's own budget once, then
|
||||
/// every charged tenant's organization budget. A tenant whose budget is exhausted is EXCLUDED (soft —
|
||||
/// the aggregate proceeds over the rest); the result is blocked only when the caller's own budget is
|
||||
/// exhausted or when every charged tenant is. A tenant whose counters cannot be read is admitted
|
||||
/// (fail-open per tenant, nothing cached). Reads run with bounded parallelism.
|
||||
/// </summary>
|
||||
public virtual async Task<McpAggregateQuotaResult> CheckManyAsync(
|
||||
string oid, string? upn, string? homeTenantId, IReadOnlyCollection<string> chargeTenantIds)
|
||||
{
|
||||
var nowUtc = _time.GetUtcNow().UtcDateTime;
|
||||
var identity = AdminIdentity.Create(upn, homeTenantId, oid);
|
||||
var (userLimits, definitions) = await ResolveUserLimitsAsync(identity, homeTenantId);
|
||||
var userOnly = Compose(userLimits, new TenantPlanLimits(userLimits.PlanName, 0, 0));
|
||||
|
||||
var user = await ReadUserUsageAsync(oid, nowUtc);
|
||||
var userDecision = user == null
|
||||
? McpQuotaDecision.FailOpen(userOnly)
|
||||
: BuildDecision(userOnly, user.DailyUsed, user.MonthlyUsed, 0, 0, nowUtc);
|
||||
if (!userDecision.Allowed)
|
||||
{
|
||||
return new McpAggregateQuotaResult
|
||||
{
|
||||
Allowed = false,
|
||||
UserDecision = userDecision,
|
||||
BlockingDecision = userDecision,
|
||||
ExcludedTenantIds = chargeTenantIds.Distinct(StringComparer.OrdinalIgnoreCase).ToList(),
|
||||
};
|
||||
}
|
||||
|
||||
var decisions = new ConcurrentDictionary<string, McpQuotaDecision>(StringComparer.OrdinalIgnoreCase);
|
||||
using var gate = new SemaphoreSlim(AggregateParallelism);
|
||||
await Task.WhenAll(chargeTenantIds.Distinct(StringComparer.OrdinalIgnoreCase).Select(async tenantId =>
|
||||
{
|
||||
await gate.WaitAsync();
|
||||
try
|
||||
{
|
||||
var limits = Compose(userLimits, await ResolveTenantLimitsAsync(tenantId, definitions));
|
||||
var targetTenantId = TargetOf(homeTenantId, tenantId);
|
||||
var tenant = await ReadTenantUsageAsync(tenantId, limits, nowUtc);
|
||||
decisions[tenantId] = tenant == null
|
||||
? McpQuotaDecision.FailOpen(limits, targetTenantId)
|
||||
: BuildDecision(limits, user?.DailyUsed ?? -1, user?.MonthlyUsed ?? -1,
|
||||
tenant.DailyUsed, tenant.MonthlyUsed, nowUtc, targetTenantId);
|
||||
}
|
||||
finally
|
||||
{
|
||||
gate.Release();
|
||||
}
|
||||
}));
|
||||
|
||||
var admitted = decisions.Where(kv => kv.Value.Allowed).Select(kv => kv.Key).ToList();
|
||||
var excluded = decisions.Where(kv => !kv.Value.Allowed).Select(kv => kv.Key).ToList();
|
||||
var allExhausted = admitted.Count == 0 && excluded.Count > 0;
|
||||
|
||||
return new McpAggregateQuotaResult
|
||||
{
|
||||
Allowed = !allExhausted,
|
||||
UserDecision = userDecision,
|
||||
// Retry-After for "everything is exhausted" is the moment at least ONE tenant becomes usable again.
|
||||
BlockingDecision = allExhausted ? excluded.Select(t => decisions[t]).MinBy(d => d.ResetUtc) : null,
|
||||
AdmittedTenantIds = admitted,
|
||||
ExcludedTenantIds = excluded,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Plan resolution only (no counter read) for a caller whose charged tenant IS their home tenant —
|
||||
/// used by the self-service usage endpoint. See the four-argument overload.
|
||||
/// </summary>
|
||||
public virtual Task<McpPlanLimits> ResolvePlanAsync(AdminIdentity? identity, string? tenantId)
|
||||
=> ResolvePlanAsync(identity, tenantId, tenantId);
|
||||
|
||||
/// <summary>
|
||||
/// Plan resolution only (no counter read). <paramref name="identity"/> is the caller's validated
|
||||
/// (upn, tid, oid); the per-user override applies only when that identity IS the one the McpUsers UPN
|
||||
/// is bound to — a null / unbound identity gets the home tenant's edition default. The user windows
|
||||
/// follow <paramref name="homeTenantId"/>'s edition; the tenant windows ALWAYS follow the CHARGED
|
||||
/// tenant's edition plan (<paramref name="chargeTenantId"/>).
|
||||
/// is bound to — a null / unbound identity gets the home tenant's plan. Both window sets follow
|
||||
/// <paramref name="tenantId"/>, the caller's home tenant.
|
||||
/// </summary>
|
||||
public virtual async Task<McpPlanLimits> ResolvePlanAsync(AdminIdentity? identity, string? homeTenantId, string? chargeTenantId)
|
||||
public virtual async Task<McpPlanLimits> ResolvePlanAsync(AdminIdentity? identity, string? tenantId)
|
||||
{
|
||||
var (userLimits, definitions) = await ResolveUserLimitsAsync(identity, homeTenantId);
|
||||
return Compose(userLimits, await ResolveTenantLimitsAsync(chargeTenantId, definitions));
|
||||
var home = await ResolveHomeAsync(tenantId);
|
||||
return Compose(await ResolveUserLimitsAsync(identity, home), ResolveTenantLimits(home));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One charged tenant's organization windows alone (plan name + tenant limits; no counters, no per-user
|
||||
/// part) — for the organization usage report.
|
||||
/// One tenant's organization windows alone (plan name + tenant limits with the slot growth applied; no
|
||||
/// counters, no per-user part) — for the organization usage report.
|
||||
/// </summary>
|
||||
public virtual async Task<TenantPlanLimits> ResolveTenantPlanAsync(string chargeTenantId)
|
||||
=> await ResolveTenantLimitsAsync(chargeTenantId, await LoadDefinitionsAsync());
|
||||
public virtual async Task<TenantPlanLimits> ResolveTenantPlanAsync(string tenantId)
|
||||
=> ResolveTenantLimits(await ResolveHomeAsync(tenantId));
|
||||
|
||||
private async Task<(UserPlanLimits Limits, List<PlanTierDefinition> Definitions)> ResolveUserLimitsAsync(
|
||||
AdminIdentity? identity, string? homeTenantId)
|
||||
/// <summary>
|
||||
/// Everything about the home tenant a resolution needs, read once per check (each entitlement read is
|
||||
/// served from the tenant-config cache): the plan definitions, the entitlement set, the usage plan name
|
||||
/// and the purchased delegation slots.
|
||||
/// </summary>
|
||||
private async Task<HomeTenantPlan> ResolveHomeAsync(string? tenantId)
|
||||
=> new(
|
||||
await LoadDefinitionsAsync(),
|
||||
await _entitlementService.GetEntitlementsAsync(tenantId),
|
||||
await _entitlementService.GetMcpUsagePlanNameAsync(tenantId),
|
||||
await _entitlementService.GetPurchasedDelegatedSlotsAsync(tenantId));
|
||||
|
||||
private async Task<UserPlanLimits> ResolveUserLimitsAsync(AdminIdentity? identity, HomeTenantPlan home)
|
||||
{
|
||||
// 1. Per-user override wins when set — for the BOUND identity only.
|
||||
string? overridePlan = null;
|
||||
@@ -229,41 +159,55 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Home-tenant plan: the tenant-wide GA override (TenantConfiguration.McpUsagePlanOverride) when
|
||||
// set, else the edition default (fail-closed → Community inside the entitlement service).
|
||||
var planName = overridePlan ?? await _entitlementService.GetMcpUsagePlanNameAsync(homeTenantId);
|
||||
// 2. Else the home tenant's usage plan (the tenant-wide GA override, else the edition default).
|
||||
var planName = overridePlan ?? home.PlanName;
|
||||
|
||||
// 3. Limits: admin-edited SectionUsagePlans definitions, else catalog fallbacks.
|
||||
var definitions = await LoadDefinitionsAsync();
|
||||
|
||||
// User limits: the definition for the (possibly overridden) plan name, else the catalog fallback
|
||||
// for the edition plans, else Community (fail-closed for overrides naming a plan that exists nowhere).
|
||||
var userDefinition = Find(definitions, planName);
|
||||
var userFallback = FeatureEntitlementCatalog.IsPermanentProTier(planName)
|
||||
// 3. User limits: the definition for the (possibly overridden) plan name, else the catalog fallback
|
||||
// for the edition plans, else Community (fail-closed for overrides naming a plan that exists nowhere).
|
||||
var definition = Find(home.Definitions, planName);
|
||||
var fallback = FeatureEntitlementCatalog.IsPermanentProTier(planName)
|
||||
? FeatureEntitlementCatalog.Get(TenantEdition.Pro)
|
||||
: FeatureEntitlementCatalog.Get(TenantEdition.Community);
|
||||
var dailyLimit = userDefinition?.DailyRequestLimit ?? userFallback.McpDailyRequestLimit;
|
||||
var monthlyLimit = userDefinition?.MonthlyRequestLimit ?? userFallback.McpMonthlyRequestLimit;
|
||||
|
||||
return (new UserPlanLimits(planName, dailyLimit, monthlyLimit), definitions);
|
||||
// 4. Slot growth: the definition's slot values, else the HOME tenant's catalog values — a per-user
|
||||
// override plan without slot fields still grows with the home tenant's purchased slots.
|
||||
var slotDaily = definition?.SlotDailyRequestLimit ?? home.Entitlements.McpSlotDailyRequestLimit;
|
||||
var slotMonthly = definition?.SlotMonthlyRequestLimit ?? home.Entitlements.McpSlotMonthlyRequestLimit;
|
||||
|
||||
return new UserPlanLimits(
|
||||
planName,
|
||||
Grow(definition?.DailyRequestLimit ?? fallback.McpDailyRequestLimit, home.PurchasedSlots, slotDaily),
|
||||
Grow(definition?.MonthlyRequestLimit ?? fallback.McpMonthlyRequestLimit, home.PurchasedSlots, slotMonthly),
|
||||
home.PurchasedSlots, slotDaily, slotMonthly);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The CHARGED tenant's organization windows: its tenant plan (the tenant-wide GA override, else the
|
||||
/// edition plan) definition when that carries tenant limits (null = not set → the edition's catalog
|
||||
/// tenant limits; an explicit 0 lifts the window), never a per-user override.
|
||||
/// The home tenant's organization windows: its usage plan definition when that carries tenant limits
|
||||
/// (null = not set → the edition's catalog tenant limits; an explicit 0 lifts the window), never a
|
||||
/// per-user override, plus the slot growth (definition's slot values, else the catalog's).
|
||||
/// </summary>
|
||||
private async Task<TenantPlanLimits> ResolveTenantLimitsAsync(string? chargeTenantId, List<PlanTierDefinition> definitions)
|
||||
private static TenantPlanLimits ResolveTenantLimits(HomeTenantPlan home)
|
||||
{
|
||||
var entitlements = await _entitlementService.GetEntitlementsAsync(chargeTenantId);
|
||||
var tenantPlan = await _entitlementService.GetMcpUsagePlanNameAsync(chargeTenantId);
|
||||
var tenantDefinition = Find(definitions, tenantPlan);
|
||||
var definition = Find(home.Definitions, home.PlanName);
|
||||
var slotDaily = definition?.SlotTenantDailyRequestLimit ?? home.Entitlements.McpSlotTenantDailyRequestLimit;
|
||||
var slotMonthly = definition?.SlotTenantMonthlyRequestLimit ?? home.Entitlements.McpSlotTenantMonthlyRequestLimit;
|
||||
|
||||
return new TenantPlanLimits(
|
||||
tenantPlan,
|
||||
tenantDefinition?.TenantDailyRequestLimit ?? entitlements.McpTenantDailyRequestLimit,
|
||||
tenantDefinition?.TenantMonthlyRequestLimit ?? entitlements.McpTenantMonthlyRequestLimit);
|
||||
home.PlanName,
|
||||
Grow(definition?.TenantDailyRequestLimit ?? home.Entitlements.McpTenantDailyRequestLimit, home.PurchasedSlots, slotDaily),
|
||||
Grow(definition?.TenantMonthlyRequestLimit ?? home.Entitlements.McpTenantMonthlyRequestLimit, home.PurchasedSlots, slotMonthly),
|
||||
home.PurchasedSlots, slotDaily, slotMonthly);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pure: a window grows by purchased slots × per-slot value. A lifted window (0 = unlimited) stays
|
||||
/// lifted — growth must never turn "unlimited" into a limit.
|
||||
/// </summary>
|
||||
internal static int Grow(int baseLimit, int purchasedSlots, int perSlot)
|
||||
=> baseLimit <= 0 || purchasedSlots <= 0 || perSlot <= 0
|
||||
? baseLimit
|
||||
: baseLimit + purchasedSlots * perSlot;
|
||||
|
||||
private async Task<List<PlanTierDefinition>> LoadDefinitionsAsync()
|
||||
{
|
||||
try
|
||||
@@ -304,23 +248,23 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The charged tenant's organization counters (cached snapshot, shared by every caller charged to that
|
||||
/// tenant). Skipped — zero counters, no read, nothing cached — when nothing could ever block on it (no
|
||||
/// tenant, or both windows lifted). Null = read failed (fail-open, nothing cached).
|
||||
/// The tenant's organization counters (cached snapshot, shared by every member of that tenant).
|
||||
/// Skipped — zero counters, no read, nothing cached — when nothing could ever block on it (no tenant,
|
||||
/// or both windows lifted). Null = read failed (fail-open, nothing cached).
|
||||
/// </summary>
|
||||
private async Task<UsageSnapshot?> ReadTenantUsageAsync(string? chargeTenantId, McpPlanLimits limits, DateTime nowUtc)
|
||||
private async Task<UsageSnapshot?> ReadTenantUsageAsync(string? tenantId, McpPlanLimits limits, DateTime nowUtc)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(chargeTenantId) || (limits.TenantDailyLimit <= 0 && limits.TenantMonthlyLimit <= 0))
|
||||
if (string.IsNullOrWhiteSpace(tenantId) || (limits.TenantDailyLimit <= 0 && limits.TenantMonthlyLimit <= 0))
|
||||
return UsageSnapshot.Zero;
|
||||
|
||||
var cacheKey = TenantCacheKey(chargeTenantId);
|
||||
var cacheKey = TenantCacheKey(tenantId);
|
||||
if (_cache.TryGetValue<UsageSnapshot>(cacheKey, out var cached) && cached != null)
|
||||
return cached;
|
||||
|
||||
try
|
||||
{
|
||||
var (monthStart, today) = Window(nowUtc);
|
||||
var records = await _usageRepo.GetTenantUsageAsync(chargeTenantId, monthStart, today);
|
||||
var records = await _usageRepo.GetTenantUsageAsync(tenantId, monthStart, today);
|
||||
var snapshot = new UsageSnapshot(
|
||||
records.Where(r => r.Date == today).Sum(r => r.RequestCount),
|
||||
records.Sum(r => r.RequestCount));
|
||||
@@ -329,7 +273,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "[McpQuota] Tenant usage lookup failed for tenant={TenantId} — allowing (fail-open)", chargeTenantId);
|
||||
_logger.LogWarning(ex, "[McpQuota] Tenant usage lookup failed for tenant={TenantId} — allowing (fail-open)", tenantId);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -338,15 +282,9 @@ namespace AutopilotMonitor.Functions.Services
|
||||
=> (new DateTime(nowUtc.Year, nowUtc.Month, 1, 0, 0, 0, DateTimeKind.Utc).ToString("yyyyMMdd"),
|
||||
nowUtc.ToString("yyyyMMdd"));
|
||||
|
||||
/// <summary>The managed tenant a decision names — null when the charged tenant IS the home tenant.</summary>
|
||||
private static string? TargetOf(string? homeTenantId, string? chargeTenantId)
|
||||
=> !string.IsNullOrWhiteSpace(chargeTenantId)
|
||||
&& !string.Equals(chargeTenantId, homeTenantId, StringComparison.OrdinalIgnoreCase)
|
||||
? chargeTenantId
|
||||
: null;
|
||||
|
||||
private static McpPlanLimits Compose(UserPlanLimits user, TenantPlanLimits tenant)
|
||||
=> new(user.PlanName, user.DailyLimit, user.MonthlyLimit, tenant.TenantPlan, tenant.TenantDailyLimit, tenant.TenantMonthlyLimit);
|
||||
=> new(user.PlanName, user.DailyLimit, user.MonthlyLimit, tenant.TenantPlan, tenant.TenantDailyLimit, tenant.TenantMonthlyLimit,
|
||||
user.PurchasedDelegatedSlots, user.SlotDailyLimit, user.SlotMonthlyLimit, tenant.SlotTenantDailyLimit, tenant.SlotTenantMonthlyLimit);
|
||||
|
||||
private static PlanTierDefinition? Find(List<PlanTierDefinition> definitions, string planName)
|
||||
=> definitions.FirstOrDefault(t => string.Equals(t.Name, planName, StringComparison.OrdinalIgnoreCase));
|
||||
@@ -354,8 +292,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
internal static McpQuotaDecision BuildDecision(
|
||||
McpPlanLimits limits,
|
||||
long dailyUsed, long monthlyUsed, long tenantDailyUsed, long tenantMonthlyUsed,
|
||||
DateTime nowUtc,
|
||||
string? targetTenantId = null)
|
||||
DateTime nowUtc)
|
||||
{
|
||||
// Daily windows reset at midnight UTC; monthly on the 1st. 0/negative limit = unlimited
|
||||
// for that window (an operator can deliberately lift a window via SectionUsagePlans).
|
||||
@@ -387,32 +324,50 @@ namespace AutopilotMonitor.Functions.Services
|
||||
TenantDailyUsed = tenantDailyUsed,
|
||||
TenantMonthlyUsed = tenantMonthlyUsed,
|
||||
ResetUtc = resetUtc,
|
||||
TargetTenantId = targetTenantId,
|
||||
PurchasedDelegatedSlots = limits.PurchasedDelegatedSlots,
|
||||
SlotDailyLimit = limits.SlotDailyLimit,
|
||||
SlotMonthlyLimit = limits.SlotMonthlyLimit,
|
||||
SlotTenantDailyLimit = limits.SlotTenantDailyLimit,
|
||||
SlotTenantMonthlyLimit = limits.SlotTenantMonthlyLimit,
|
||||
};
|
||||
}
|
||||
|
||||
private static bool Exceeded(int limit, long used) => limit > 0 && used >= limit;
|
||||
|
||||
/// <summary>Cached counter pair (today / this month) for one user or one charged tenant.</summary>
|
||||
/// <summary>Cached counter pair (today / this month) for one user or one tenant.</summary>
|
||||
private sealed record UsageSnapshot(long DailyUsed, long MonthlyUsed)
|
||||
{
|
||||
public static readonly UsageSnapshot Zero = new(0, 0);
|
||||
}
|
||||
|
||||
private sealed record UserPlanLimits(string PlanName, int DailyLimit, int MonthlyLimit);
|
||||
/// <summary>The home tenant's plan facts a resolution needs, read once per check.</summary>
|
||||
private sealed record HomeTenantPlan(
|
||||
List<PlanTierDefinition> Definitions, EditionEntitlements Entitlements, string PlanName, int PurchasedSlots);
|
||||
|
||||
/// <summary>The charged tenant's plan name and organization-wide windows (0 = unlimited).</summary>
|
||||
public sealed record TenantPlanLimits(string TenantPlan, int TenantDailyLimit, int TenantMonthlyLimit);
|
||||
private sealed record UserPlanLimits(
|
||||
string PlanName, int DailyLimit, int MonthlyLimit,
|
||||
int PurchasedDelegatedSlots, int SlotDailyLimit, int SlotMonthlyLimit);
|
||||
|
||||
/// <summary>
|
||||
/// The tenant's plan name and organization-wide windows (0 = unlimited; the slot growth is already
|
||||
/// applied) plus the growth breakdown: purchased slots and the per-slot values.
|
||||
/// </summary>
|
||||
public sealed record TenantPlanLimits(
|
||||
string TenantPlan, int TenantDailyLimit, int TenantMonthlyLimit,
|
||||
int PurchasedDelegatedSlots = 0, int SlotTenantDailyLimit = 0, int SlotTenantMonthlyLimit = 0);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resolved plan names and limits for one caller against one charged tenant: the user's plan (override
|
||||
/// or home edition) with the user windows, and the charged tenant's edition plan with the
|
||||
/// organization-wide windows. 0 = unlimited.
|
||||
/// Resolved plan names and limits for one caller: the user's plan (override or home plan) with the user
|
||||
/// windows, and the home tenant's plan with the organization-wide windows. 0 = unlimited. The limits
|
||||
/// already contain the growth from purchased delegation slots; the trailing fields carry the breakdown.
|
||||
/// </summary>
|
||||
public sealed record McpPlanLimits(
|
||||
string PlanName, int DailyLimit, int MonthlyLimit,
|
||||
string TenantPlan, int TenantDailyLimit, int TenantMonthlyLimit);
|
||||
string TenantPlan, int TenantDailyLimit, int TenantMonthlyLimit,
|
||||
int PurchasedDelegatedSlots = 0,
|
||||
int SlotDailyLimit = 0, int SlotMonthlyLimit = 0,
|
||||
int SlotTenantDailyLimit = 0, int SlotTenantMonthlyLimit = 0);
|
||||
|
||||
/// <summary>Whose budget a blocked decision names — wire vocabulary of <c>level</c>.</summary>
|
||||
public static class McpQuotaLevel
|
||||
@@ -434,7 +389,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
public int MonthlyLimit { get; init; }
|
||||
public long DailyUsed { get; init; }
|
||||
public long MonthlyUsed { get; init; }
|
||||
/// <summary>The CHARGED tenant's edition plan — the organization-wide windows follow it, never the override.</summary>
|
||||
/// <summary>The home tenant's usage plan — the organization-wide windows follow it, never the override.</summary>
|
||||
public string TenantPlan { get; init; } = string.Empty;
|
||||
public int TenantDailyLimit { get; init; }
|
||||
public int TenantMonthlyLimit { get; init; }
|
||||
@@ -442,11 +397,12 @@ namespace AutopilotMonitor.Functions.Services
|
||||
public long TenantMonthlyUsed { get; init; }
|
||||
/// <summary>When the exceeded (or daily, when allowed) window resets.</summary>
|
||||
public DateTime ResetUtc { get; init; }
|
||||
/// <summary>
|
||||
/// The MANAGED tenant whose organization windows this decision reflects (a delegated read charged to
|
||||
/// the managed tenant); null when the charged tenant is the caller's own home tenant.
|
||||
/// </summary>
|
||||
public string? TargetTenantId { get; init; }
|
||||
/// <summary>Delegation slots bought beyond the edition's included ones (0 = none; the limits above already grew by them).</summary>
|
||||
public int PurchasedDelegatedSlots { get; init; }
|
||||
public int SlotDailyLimit { get; init; }
|
||||
public int SlotMonthlyLimit { get; init; }
|
||||
public int SlotTenantDailyLimit { get; init; }
|
||||
public int SlotTenantMonthlyLimit { get; init; }
|
||||
|
||||
/// <summary>Limit of the exceeded window (0 when allowed).</summary>
|
||||
public int ExceededLimit => Level == McpQuotaLevel.Tenant
|
||||
@@ -458,7 +414,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
? (Scope == "monthly" ? TenantMonthlyUsed : TenantDailyUsed)
|
||||
: (Scope == "monthly" ? MonthlyUsed : DailyUsed);
|
||||
|
||||
public static McpQuotaDecision FailOpen(McpPlanLimits limits, string? targetTenantId = null) => new()
|
||||
public static McpQuotaDecision FailOpen(McpPlanLimits limits) => new()
|
||||
{
|
||||
Allowed = true,
|
||||
Plan = limits.PlanName,
|
||||
@@ -472,25 +428,11 @@ namespace AutopilotMonitor.Functions.Services
|
||||
TenantDailyUsed = -1,
|
||||
TenantMonthlyUsed = -1,
|
||||
ResetUtc = DateTime.MinValue,
|
||||
TargetTenantId = targetTenantId,
|
||||
PurchasedDelegatedSlots = limits.PurchasedDelegatedSlots,
|
||||
SlotDailyLimit = limits.SlotDailyLimit,
|
||||
SlotMonthlyLimit = limits.SlotMonthlyLimit,
|
||||
SlotTenantDailyLimit = limits.SlotTenantDailyLimit,
|
||||
SlotTenantMonthlyLimit = limits.SlotTenantMonthlyLimit,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Outcome of a bounded (fleet) aggregate check: which charged tenants may be served and which are
|
||||
/// dropped because their organization budget is exhausted. <see cref="Allowed"/> is false only when the
|
||||
/// caller's own budget is exhausted (then <see cref="BlockingDecision"/> is user-level and every tenant is
|
||||
/// excluded) or when EVERY charged tenant is exhausted (then it is the excluded decision with the earliest
|
||||
/// reset).
|
||||
/// </summary>
|
||||
public sealed class McpAggregateQuotaResult
|
||||
{
|
||||
public bool Allowed { get; init; }
|
||||
/// <summary>The caller's own windows (tenant windows zero) — for the response headers.</summary>
|
||||
public McpQuotaDecision UserDecision { get; init; } = default!;
|
||||
/// <summary>Set when <see cref="Allowed"/> is false.</summary>
|
||||
public McpQuotaDecision? BlockingDecision { get; init; }
|
||||
public IReadOnlyList<string> AdmittedTenantIds { get; init; } = Array.Empty<string>();
|
||||
public IReadOnlyList<string> ExcludedTenantIds { get; init; } = Array.Empty<string>();
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -311,7 +311,7 @@ namespace AutopilotMonitor.Functions.Services.Notifications
|
||||
label = "Success Rate";
|
||||
title = $"\ud83d\udfe1 SLA Breach: Success Rate {currentRate:F1}%";
|
||||
summary = $"SLA breach: success rate {currentRate:F1}% is below target {targetRate:F1}%";
|
||||
period = "Current Month";
|
||||
period = $"Last {SlaEvaluationWindow.WindowDays} days";
|
||||
facts.Add(new NotificationFact { Name = "Breach Type", Value = label });
|
||||
facts.Add(new NotificationFact { Name = "Current Rate", Value = $"{currentRate:F1}%" });
|
||||
facts.Add(new NotificationFact { Name = "Target Rate", Value = $"{targetRate:F1}%" });
|
||||
@@ -323,7 +323,7 @@ namespace AutopilotMonitor.Functions.Services.Notifications
|
||||
label = "Duration (P95)";
|
||||
title = "\ud83d\udfe1 SLA Breach: P95 Duration Exceeds Target";
|
||||
summary = $"SLA breach: P95 duration {currentRate:F1}min exceeds target {targetRate:F0}min";
|
||||
period = "Current Month";
|
||||
period = $"Last {SlaEvaluationWindow.WindowDays} days";
|
||||
facts.Add(new NotificationFact { Name = "Breach Type", Value = label });
|
||||
facts.Add(new NotificationFact { Name = "Current P95", Value = $"{currentRate:F1} min" });
|
||||
facts.Add(new NotificationFact { Name = "Target Max", Value = $"{targetRate:F0} min" });
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Shared;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Offboarding
|
||||
{
|
||||
@@ -77,5 +78,29 @@ namespace AutopilotMonitor.Functions.Services.Offboarding
|
||||
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
|
||||
return $"RowKey eq '{ODataSanitizer.EscapeValue(normalizedTenantId)}'";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <c>PartitionKey eq 'msp-{tenantId}'</c> — the tenant's own self-service Tenant Group
|
||||
/// (TenantGroups: PK=groupId). The group id is derived from the validated tenant GUID here,
|
||||
/// never passed in.
|
||||
/// </summary>
|
||||
public static string OwnedGroupPartition(string normalizedTenantId)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
|
||||
var groupId = Constants.TenantGroupIds.ForHomeTenant(normalizedTenantId);
|
||||
return $"PartitionKey eq '{ODataSanitizer.EscapeValue(groupId)}'";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <c>RowKey eq 'msp-{tenantId}'</c> — assignment rows pointing at the tenant's own
|
||||
/// self-service Tenant Group (TenantGroupAssignments: PK=managed tenant, RK=groupId).
|
||||
/// Full-table scan, same reservation as <see cref="RowKeyEquals"/>.
|
||||
/// </summary>
|
||||
public static string OwnedGroupRowKey(string normalizedTenantId)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
|
||||
var groupId = Constants.TenantGroupIds.ForHomeTenant(normalizedTenantId);
|
||||
return $"RowKey eq '{ODataSanitizer.EscapeValue(groupId)}'";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,21 +38,46 @@ namespace AutopilotMonitor.Functions.Services.Offboarding
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
// Every public entry point validates the tenant GUID and builds the filter, then hands
|
||||
// over to a protected virtual core. Test fakes override the cores only, so the guard runs
|
||||
// under every fake — a caller that passes anything but the tenant GUID fails in tests.
|
||||
|
||||
/// <summary>
|
||||
/// Variant A (exact tenant PartitionKey). Used for tables where rows live under
|
||||
/// <c>PartitionKey == normalizedTenantId</c> (AuditLogs, UsageMetrics, …).
|
||||
/// </summary>
|
||||
public virtual Task<int> WipeByExactPartitionAsync(
|
||||
public Task<int> WipeByExactPartitionAsync(
|
||||
string tableName, string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
|
||||
var filter = OffboardingFilters.ExactPartition(normalizedTenantId);
|
||||
return WipeByExactPartitionCoreAsync(
|
||||
tableName, normalizedTenantId, OffboardingFilters.ExactPartition(normalizedTenantId), ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Variant A for the tenant's own self-service Tenant Group: rows live under
|
||||
/// <c>PartitionKey == "msp-{tenantId}"</c>. Takes the tenant GUID and derives the group id
|
||||
/// itself, so the anchor is never a caller-supplied string.
|
||||
/// </summary>
|
||||
public Task<int> WipeByOwnedGroupPartitionAsync(
|
||||
string tableName, string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
|
||||
return WipeByExactPartitionCoreAsync(
|
||||
tableName,
|
||||
Constants.TenantGroupIds.ForHomeTenant(normalizedTenantId),
|
||||
OffboardingFilters.OwnedGroupPartition(normalizedTenantId),
|
||||
ct);
|
||||
}
|
||||
|
||||
protected virtual Task<int> WipeByExactPartitionCoreAsync(
|
||||
string tableName, string partitionKey, string filter, CancellationToken ct)
|
||||
{
|
||||
return RunFetchVerifyDeleteAsync(
|
||||
tableName,
|
||||
filter,
|
||||
expectedAnchor: normalizedTenantId,
|
||||
verifyRow: e => string.Equals(e.PartitionKey, normalizedTenantId, StringComparison.Ordinal),
|
||||
expectedAnchor: partitionKey,
|
||||
verifyRow: e => string.Equals(e.PartitionKey, partitionKey, StringComparison.Ordinal),
|
||||
ct);
|
||||
}
|
||||
|
||||
@@ -60,10 +85,16 @@ namespace AutopilotMonitor.Functions.Services.Offboarding
|
||||
/// Variant A range (composite tenant PartitionKey). Used for tables where rows live
|
||||
/// under <c>{tenantId}_{sessionId|discriminator}</c>.
|
||||
/// </summary>
|
||||
public virtual Task<int> WipeByCompositePartitionRangeAsync(
|
||||
public Task<int> WipeByCompositePartitionRangeAsync(
|
||||
string tableName, string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
|
||||
return WipeByCompositePartitionRangeCoreAsync(tableName, normalizedTenantId, ct);
|
||||
}
|
||||
|
||||
protected virtual Task<int> WipeByCompositePartitionRangeCoreAsync(
|
||||
string tableName, string normalizedTenantId, CancellationToken ct)
|
||||
{
|
||||
var prefix = normalizedTenantId + "_";
|
||||
var filter = OffboardingFilters.CompositePartitionRange(normalizedTenantId);
|
||||
|
||||
@@ -81,12 +112,17 @@ namespace AutopilotMonitor.Functions.Services.Offboarding
|
||||
/// <c>TenantId</c> property identifies tenant ownership. Verify-step MUST check the
|
||||
/// property because the PK is non-anchored.
|
||||
/// </summary>
|
||||
public virtual Task<int> WipeByDiscriminatorAndTenantPropertyAsync(
|
||||
public Task<int> WipeByDiscriminatorAndTenantPropertyAsync(
|
||||
string tableName, string discriminator, string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
|
||||
if (string.IsNullOrEmpty(discriminator)) throw new ArgumentException("Discriminator required", nameof(discriminator));
|
||||
return WipeByDiscriminatorAndTenantPropertyCoreAsync(tableName, discriminator, normalizedTenantId, ct);
|
||||
}
|
||||
|
||||
protected virtual Task<int> WipeByDiscriminatorAndTenantPropertyCoreAsync(
|
||||
string tableName, string discriminator, string normalizedTenantId, CancellationToken ct)
|
||||
{
|
||||
var filter = OffboardingFilters.DiscriminatorWithTenantProp(discriminator, normalizedTenantId);
|
||||
var expectedAnchor = $"{discriminator}+TenantId={normalizedTenantId}";
|
||||
|
||||
@@ -104,10 +140,16 @@ namespace AutopilotMonitor.Functions.Services.Offboarding
|
||||
/// PartitionKey is not tenant-anchored (e.g. <c>UserUsageLog</c> keyed by userOid).
|
||||
/// Full-table scan — DO NOT use on high-volume tables without consideration.
|
||||
/// </summary>
|
||||
public virtual Task<int> WipeByTenantIdPropertyAsync(
|
||||
public Task<int> WipeByTenantIdPropertyAsync(
|
||||
string tableName, string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
|
||||
return WipeByTenantIdPropertyCoreAsync(tableName, normalizedTenantId, ct);
|
||||
}
|
||||
|
||||
protected virtual Task<int> WipeByTenantIdPropertyCoreAsync(
|
||||
string tableName, string normalizedTenantId, CancellationToken ct)
|
||||
{
|
||||
var filter = OffboardingFilters.TenantIdProperty(normalizedTenantId);
|
||||
|
||||
return RunFetchVerifyDeleteAsync(
|
||||
@@ -124,17 +166,38 @@ namespace AutopilotMonitor.Functions.Services.Offboarding
|
||||
/// PK=date, RK=tenantId|"global"). Full-table scan — reserve for retention-bounded
|
||||
/// tables.
|
||||
/// </summary>
|
||||
public virtual Task<int> WipeByRowKeyAsync(
|
||||
public Task<int> WipeByRowKeyAsync(
|
||||
string tableName, string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
|
||||
var filter = OffboardingFilters.RowKeyEquals(normalizedTenantId);
|
||||
return WipeByRowKeyCoreAsync(
|
||||
tableName, normalizedTenantId, OffboardingFilters.RowKeyEquals(normalizedTenantId), ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Variant D for the tenant's own self-service Tenant Group: rows whose
|
||||
/// <c>RowKey == "msp-{tenantId}"</c> (TenantGroupAssignments: PK=managed tenant). Takes
|
||||
/// the tenant GUID and derives the group id itself.
|
||||
/// </summary>
|
||||
public Task<int> WipeByOwnedGroupRowKeyAsync(
|
||||
string tableName, string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
|
||||
return WipeByRowKeyCoreAsync(
|
||||
tableName,
|
||||
Constants.TenantGroupIds.ForHomeTenant(normalizedTenantId),
|
||||
OffboardingFilters.OwnedGroupRowKey(normalizedTenantId),
|
||||
ct);
|
||||
}
|
||||
|
||||
protected virtual Task<int> WipeByRowKeyCoreAsync(
|
||||
string tableName, string rowKey, string filter, CancellationToken ct)
|
||||
{
|
||||
return RunFetchVerifyDeleteAsync(
|
||||
tableName,
|
||||
filter,
|
||||
expectedAnchor: $"RowKey={normalizedTenantId}",
|
||||
verifyRow: e => string.Equals(e.RowKey, normalizedTenantId, StringComparison.Ordinal),
|
||||
expectedAnchor: $"RowKey={rowKey}",
|
||||
verifyRow: e => string.Equals(e.RowKey, rowKey, StringComparison.Ordinal),
|
||||
ct);
|
||||
}
|
||||
|
||||
@@ -144,12 +207,17 @@ namespace AutopilotMonitor.Functions.Services.Offboarding
|
||||
/// per-blob (DeleteIfExistsAsync) so the helper stays idempotent across crash/resume.
|
||||
/// Mismatch → abort before any delete. Returns the number of blobs deleted.
|
||||
/// </summary>
|
||||
public virtual async Task<int> WipeBlobsByTenantPrefixAsync(
|
||||
public Task<int> WipeBlobsByTenantPrefixAsync(
|
||||
string containerName, string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
|
||||
if (string.IsNullOrEmpty(containerName)) throw new ArgumentException("Container required", nameof(containerName));
|
||||
return WipeBlobsByTenantPrefixCoreAsync(containerName, normalizedTenantId, ct);
|
||||
}
|
||||
|
||||
protected virtual async Task<int> WipeBlobsByTenantPrefixCoreAsync(
|
||||
string containerName, string normalizedTenantId, CancellationToken ct)
|
||||
{
|
||||
var prefix = normalizedTenantId + "/";
|
||||
var container = _blobs.GetContainerClient(containerName);
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user