// Load benchmark: the real backend (hosted in this process against a live NATS server, SQLite) // and N simulated agents in a child process, sealed (envelope v2) or unsealed (v1). // // LoadBench run --devices 5000 [--sealed true] [--connections 16] [--steady 180] [--lines 20] [--nats-pid 1234] [--out results.jsonl] [--provider sqlite|postgres] // [--burst true|false] [--upgrade true|false] [--start-within 1] [--settings-retry true|false] // LoadBench micro [--count 20000] // // Phases: the connect storm (ready event, settings pull, inventory and presence from every device // within 20 s; the pull is tried once unless --settings-retry, which retries as the agent does), // steady state, with --upgrade every device upgraded through the scheduler's paced batches // (SCHEDULE_WORKER on, StartWithinMinutes --start-within; each pass's sends timed), and unless // --burst false a script to every device through the job runner. // // NATS_URL picks the server (default nats://127.0.0.1:4222; no auth, JetStream on). Not part of CI. // --provider postgres runs on a throwaway database (created and dropped) on the server PGHOST, PGPORT, // PGUSER and PGPASSWORD name. // The JetStream streams are kept in memory storage, as the integration tests keep them on a shared // server (production uses file storage). "cleanup" removes a stopped run's consumers. using System.Collections.Concurrent; using System.Diagnostics; using System.Globalization; using System.Net.Http.Json; using System.Security.Cryptography; using System.Text; using System.Text.Json; using Agent.Enrollment; using Agent.Identity; using Agent.Inventory; using Agent.Messaging; using Backend.Data; using Backend.Jobs; using Backend.Nats; using Backend.Security.Messages; using Backend.Upgrades; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.AspNetCore.TestHost; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using NATS.Client.Core; using NATS.Client.JetStream; using NATS.NKeys; using Shared.Contracts.Api; using Shared.Contracts.Messages; using Shared.Contracts.Models; using Shared.Contracts.Serialization; using Shared.Infrastructure.Constants; using Shared.Infrastructure.Nats; using Shared.Infrastructure.Security; using Shared.Infrastructure.Security.Messaging; namespace LoadBench; public static class Program { public static string NatsUrl => Environment.GetEnvironmentVariable("NATS_URL") ?? "nats://127.0.0.1:4222"; public static async Task Main(string[] args) { var options = Options.Parse(args); return args.FirstOrDefault() switch { "run" => await Bench.RunAsync(options), "agents" => await Simulator.RunAsync(options), "micro" => await Micro.RunAsync(options), "cleanup" => await CleanupAsync(), _ => Usage(), }; } /// Deletes the consumers (and queued commands) a run that did not finish left on the shared NATS. private static async Task CleanupAsync() { await using var connection = new NatsConnection(NatsOpts.Default with { Url = NatsUrl }); var js = new NatsJSContext(connection); var stream = await js.GetStreamAsync(JetStreamDefaults.CommandStreamName); var removed = 0; await foreach (var name in stream.ListConsumerNamesAsync()) { var clientId = name.Replace("agent-commands-", string.Empty, StringComparison.Ordinal); if (name.StartsWith("agent-commands-", StringComparison.Ordinal) && clientId.Length == 22 && clientId[0] is 'l' or 'm') { await stream.DeleteConsumerAsync(name); await stream.PurgeAsync(new NATS.Client.JetStream.Models.StreamPurgeRequest { Filter = NatsSubjects.AgentCommandWildcard(clientId) }); removed++; } } Console.WriteLine($"removed {removed} benchmark consumers"); // The backend provisions file storage (as in production, so a run measures that); the // shared test server is left with the memory streams the integration tests use. foreach (var (name, subjects, maxAge) in new[] { (JetStreamDefaults.CommandStreamName, JetStreamDefaults.CommandStreamSubjects, JetStreamDefaults.CommandMaxAge), (JetStreamDefaults.JobEventsStreamName, JetStreamDefaults.JobEventsStreamSubjects, JetStreamDefaults.JobEventsMaxAge), }) { try { var existing = await js.GetStreamAsync(name); if (existing.Info.Config.Storage == NATS.Client.JetStream.Models.StreamConfigStorage.File) { await js.DeleteStreamAsync(name); await js.CreateStreamAsync(new NATS.Client.JetStream.Models.StreamConfig(name, subjects) { Storage = NATS.Client.JetStream.Models.StreamConfigStorage.Memory, MaxAge = maxAge, }); Console.WriteLine($"{name}: file storage replaced by memory storage"); } } catch (NatsJSApiException exception) when (exception.Error.Code == 404) { // Not there. } } return 0; } private static int Usage() { Console.Error.WriteLine("LoadBench run --devices N [--sealed true|false] [--connections K] [--steady S] [--lines L] [--nats-pid P] [--out F] | micro [--count N]"); return 2; } } public sealed class Options(Dictionary values) { public static Options Parse(string[] args) { var values = new Dictionary(StringComparer.OrdinalIgnoreCase); for (var i = 1; i < args.Length - 1; i += 2) { values[args[i].TrimStart('-')] = args[i + 1]; } return new Options(values); } public int Int(string name, int fallback) => values.TryGetValue(name, out var v) ? int.Parse(v, CultureInfo.InvariantCulture) : fallback; public bool Bool(string name, bool fallback) => values.TryGetValue(name, out var v) ? bool.Parse(v) : fallback; public string? Text(string name) => values.TryGetValue(name, out var v) ? v : null; } /// One simulated device's identity as the child process gets it. public sealed record DeviceCredentials(string ClientId, string NkeySeed, string? MessageKey, ServerMessageKeys? ServerKeys); /// When each upgrade command's publish returned (the scheduler's sends, timed as they go out). public sealed class UpgradeSendLog { public ConcurrentQueue<(Guid UpgradeId, string ClientId, DateTimeOffset DoneUtc, bool Sent)> Sends { get; } = new(); } /// The real sender, each send recorded in . public sealed class TimedUpgradeSender(IAgentUpgradeSender inner, UpgradeSendLog log) : IAgentUpgradeSender { public async Task DispatchAgentUpgradeAsync(string clientId, AgentUpgradeCommandPayload command, CancellationToken cancellationToken) { var sent = await inner.DispatchAgentUpgradeAsync(clientId, command, cancellationToken); log.Sends.Enqueue((command.UpgradeId, clientId, DateTimeOffset.UtcNow, sent)); return sent; } } /// The backend, hosted in this process against the live NATS (as the integration tests host it). public sealed class BenchBackend(string root, string? postgres = null, bool scheduleWorker = false) : WebApplicationFactory { private readonly KeyPair account = KeyPair.CreatePair(PrefixByte.Account); public UpgradeSendLog UpgradeSends { get; } = new(); protected override void ConfigureWebHost(IWebHostBuilder builder) { builder.ConfigureTestServices(services => { // The upgrade scheduler's sends, timed (the scheduler runs only with SCHEDULE_WORKER on). services.AddSingleton(sp => new TimedUpgradeSender(sp.GetRequiredService(), UpgradeSends)); }); builder.UseSetting("DATABASE_PATH", Path.Combine(root, "app.db")); if (postgres is not null) { builder.UseSetting("DATABASE_PROVIDER", "postgres"); builder.UseSetting("DATABASE_CONNECTION_STRING", postgres); } builder.UseSetting("DATA_PATH", Path.Combine(root, "data")); builder.UseSetting("DATAPROTECTION_PATH", Path.Combine(root, "keys")); builder.UseSetting("LOG_DIRECTORY", Path.Combine(root, "logs")); builder.UseSetting("LOG_LEVEL", Environment.GetEnvironmentVariable("BENCH_LOG_LEVEL") ?? "Warning"); builder.UseSetting("AGENT_CATALOG_SYNC", "0"); builder.UseSetting("S3_EMBEDDED", "0"); builder.UseSetting("NATS_ACCOUNT_SIGNING_SEED", account.GetSeed()); builder.UseSetting("NATS_ACCOUNT_PUBLIC_KEY", account.GetPublicKey()); builder.UseSetting("NATS_URL", Program.NatsUrl); builder.UseSetting("RATE_LIMIT_PER_MINUTE", "100000000"); builder.UseSetting("SCHEDULE_WORKER", scheduleWorker ? "1" : "0"); builder.UseSetting("TLS_SELF_SIGNED", "0"); builder.UseEnvironment("Production"); } public async Task WithDbAsync(Func> action) { using var scope = Services.CreateScope(); return await action(scope.ServiceProvider.GetRequiredService()); } public async Task TokenAsync() { var token = TokenGenerator.CreateUrlSafeToken(32); await WithDbAsync(async db => { db.BootstrapTokens.Add(new BootstrapTokenEntity { Id = Guid.NewGuid(), TokenHash = TokenHashing.Sha256Hex(token), CreatedUtc = DateTimeOffset.UtcNow, ExpiresUtc = DateTimeOffset.UtcNow + TimeSpan.FromDays(1), MaxUses = 0, }); return await db.SaveChangesAsync(); }); return token; } /// Enrolls devices through the real bootstrap endpoint, at a time. public async Task> EnrollAsync(int count, bool seals, string prefix, int parallel = 16) { var token = await TokenAsync(); var result = new ConcurrentBag(); using var http = CreateClient(); http.Timeout = TimeSpan.FromMinutes(5); await Parallel.ForEachAsync(Enumerable.Range(0, count), new ParallelOptions { MaxDegreeOfParallelism = parallel }, async (i, ct) => { var clientId = prefix + Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(10)); using var nkey = KeyPair.CreatePair(PrefixByte.User); using var ecdh = MessageKeys.Create(); var response = await http.PostAsJsonAsync(ApiRoutes.Bootstrap, new BootstrapRequest( token, clientId, nkey.GetPublicKey(), "2026.11.06.0000", new EnrollDeviceInfo("PC-" + clientId[2..10], "Contoso", "Model", "Windows 11", "10.0.26200"), seals ? MessageKeys.PublicKeyOf(ecdh) : null), JsonDefaults.Options, ct); response.EnsureSuccessStatusCode(); var answer = (await response.Content.ReadFromJsonAsync(JsonDefaults.Options, ct))!; result.Add(new DeviceCredentials(clientId, nkey.GetSeed(), seals ? MessageKeys.PrivateKeyOf(ecdh) : null, answer.ServerKeys)); }); return [.. result]; } protected override void Dispose(bool disposing) { base.Dispose(disposing); account.Dispose(); } } /// CPU and memory of one process between samples. public sealed class ProcessMeter(Process process, string name) { private TimeSpan lastCpu = Cpu(process); private long lastTicks = Stopwatch.GetTimestamp(); public string Name => name; private static TimeSpan Cpu(Process p) { try { p.Refresh(); return p.TotalProcessorTime; } catch (Exception) { return TimeSpan.Zero; } } /// Cores busy since the last call, working set and private bytes (MB). public (double Cores, double WorkingSetMb, double PrivateMb) Sample() { var cpu = Cpu(process); var now = Stopwatch.GetTimestamp(); var wall = Stopwatch.GetElapsedTime(lastTicks, now); var cores = wall.TotalSeconds > 0 ? (cpu - lastCpu).TotalSeconds / wall.TotalSeconds : 0; lastCpu = cpu; lastTicks = now; try { return (cores, process.WorkingSet64 / 1048576.0, process.PrivateMemorySize64 / 1048576.0); } catch (Exception) { return (cores, 0, 0); } } } /// The orchestrator: hosts the backend, enrolls, starts the simulator, measures the three phases. public static class Bench { private sealed record PhaseResult( string Phase, double Seconds, double ServerCores, double ServerPeakCores, double ServerWorkingSetMb, double ServerPrivateMb, double AgentsCores, double NatsCores, double NatsWorkingSetMb, double MessagesPerSecond, long Dropped, string Detail); public static async Task RunAsync(Options options) { var devices = options.Int("devices", 1000); var seals = options.Bool("sealed", true); var connections = options.Int("connections", Math.Max(1, Math.Min(32, devices / 500))); var steady = options.Int("steady", 180); var lines = options.Int("lines", 20); var outFile = options.Text("out") ?? Path.Combine(AppContext.BaseDirectory, "results.jsonl"); var root = Directory.CreateTempSubdirectory("loadbench-").FullName; var provider = options.Text("provider") ?? "sqlite"; var upgrade = options.Bool("upgrade", false); var jobBurst = options.Bool("burst", true); var settingsRetry = options.Bool("settings-retry", false); await using var database = provider == "postgres" ? await ThrowawayPostgres.CreateAsync() : null; Log($"{devices} devices, {(seals ? "sealed (v2)" : "unsealed (v1)")}, {connections} NATS connections for the agents, {provider}{(database is null ? string.Empty : $" ({database.Name})")}, data in {root}"); Process? nats = options.Int("nats-pid", 0) is > 0 and var pid ? Process.GetProcessById(pid) : FindNats(); // The streams in memory storage, as the integration tests keep them on a shared server (a // run that switched them to file storage had them deleted under it by a test run). The // backend's own file-storage provisioning then only warns; its job event consumer is made here. await using (var setup = new NatsConnection(NatsOpts.Default with { Url = Program.NatsUrl })) { var js = new NatsJSContext(setup); foreach (var (name, subjects, maxAge) in new[] { (JetStreamDefaults.CommandStreamName, JetStreamDefaults.CommandStreamSubjects, JetStreamDefaults.CommandMaxAge), (JetStreamDefaults.JobEventsStreamName, JetStreamDefaults.JobEventsStreamSubjects, JetStreamDefaults.JobEventsMaxAge), }) { var config = new NATS.Client.JetStream.Models.StreamConfig(name, subjects) { Storage = NATS.Client.JetStream.Models.StreamConfigStorage.Memory, MaxAge = maxAge, }; try { await js.CreateOrUpdateStreamAsync(config); } catch (NatsJSApiException) { await js.DeleteStreamAsync(name); await js.CreateOrUpdateStreamAsync(config); } } await js.CreateOrUpdateConsumerAsync(JetStreamDefaults.JobEventsStreamName, new NATS.Client.JetStream.Models.ConsumerConfig(JetStreamDefaults.BackendJobEventsConsumerName) { DurableName = JetStreamDefaults.BackendJobEventsConsumerName, AckPolicy = NATS.Client.JetStream.Models.ConsumerConfigAckPolicy.Explicit, AckWait = JetStreamDefaults.CommandAckWait, FilterSubject = JetStreamDefaults.JobEventsStreamSubjects[0], }); } await using var backend = new BenchBackend(root, database?.ConnectionString, scheduleWorker: upgrade); using (backend.CreateClient()) { } var hub = backend.Services.GetRequiredService().Connection!; long dropped = 0; var droppedBySubject = new ConcurrentDictionary(); hub.MessageDropped += (_, e) => { Interlocked.Increment(ref dropped); droppedBySubject.AddOrUpdate(Kind(e.Subject), 1, (_, n) => n + 1); return ValueTask.CompletedTask; }; await backend.Services.GetRequiredService().Keys.EnsureLoadedAsync(CancellationToken.None); // Enrollment through the real endpoint (keys recorded, consumers made). var enrollWatch = Stopwatch.StartNew(); var credentials = await backend.EnrollAsync(devices, seals, "lb"); enrollWatch.Stop(); Log($"enrolled {devices} in {enrollWatch.Elapsed.TotalSeconds:0.0}s ({devices / enrollWatch.Elapsed.TotalSeconds:0}/s)"); var credentialsFile = Path.Combine(root, "devices.json"); await File.WriteAllTextAsync(credentialsFile, JsonSerializer.Serialize(credentials, JsonDefaults.Options)); // The simulator, in its own process (its CPU is the agents', not the server's). var child = Process.Start(new ProcessStartInfo(Environment.ProcessPath!, $"agents --file \"{credentialsFile}\" --connections {connections} --lines {lines} --ramp 20 --settings-retry {settingsRetry}") { RedirectStandardOutput = true, UseShellExecute = false, })!; var stats = new SimulatorStats(); _ = Task.Run(async () => { while (await child.StandardOutput.ReadLineAsync() is { } line) { stats.Take(line); } }); var self = new ProcessMeter(Process.GetCurrentProcess(), "server"); var agentsMeter = new ProcessMeter(child, "agents"); var natsMeter = nats is null ? null : new ProcessMeter(nats, "nats"); var results = new List(); var startedUtc = DateTimeOffset.UtcNow; // Phase 1: every agent connects within 20 s (a fleet coming back after an outage): // ready event, settings pull, inventory and presence each. var storm = await MeasureAsync("connect storm", self, agentsMeter, natsMeter, stats, () => dropped, TimeSpan.FromMinutes(15), async () => { var inventories = await backend.WithDbAsync(db => db.DeviceInventory.CountAsync(i => i.UpdatedUtc >= startedUtc)); var seen = await backend.WithDbAsync(db => db.Devices.CountAsync(d => d.LastSeenUtc >= startedUtc)); var ready = await backend.WithDbAsync(db => db.Devices.CountAsync(d => d.ReadyUtc >= startedUtc)); return (inventories >= devices && seen >= devices && ready >= devices && stats.SettingsAnswered + stats.SettingsFailed >= devices, $"inventories {inventories}/{devices}, seen {seen}/{devices}, ready {ready}/{devices}, settings answered {stats.SettingsAnswered} " + $"(first try {stats.Get("settingsFirstOk")}, retries {stats.Get("settingsRetries")}) failed {stats.SettingsFailed}, " + $"reply ms p50 {stats.Get("settingsP50")} p95 {stats.Get("settingsP95")} max {stats.Get("settingsMax")}"); }); results.Add(storm); Log($"connect storm: every device's settings decided after {storm.Seconds:0.0}s: answered {stats.SettingsAnswered}/{devices}, first try {stats.Get("settingsFirstOk")}, failed {stats.SettingsFailed}"); // Phase 2: steady state (presence every 30 s, inventory every 15 min, spread out). var steadyStart = stats.Snapshot(); var steadyResult = await MeasureAsync("steady", self, agentsMeter, natsMeter, stats, () => dropped, TimeSpan.FromSeconds(steady), async () => { var recent = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(40); var fresh = await backend.WithDbAsync(db => db.Devices.CountAsync(d => d.LastSeenUtc >= recent)); return (false, $"seen in the last 40 s {fresh}/{devices}"); }, fixedDuration: true); results.Add(steadyResult); // Phase 3 (with --upgrade): every device upgraded through the upgrade scheduler's paced batches. object? upgradeSummary = upgrade ? await UpgradeFanOutAsync(backend, devices, options.Int("start-within", 1), stats, self, agentsMeter, natsMeter, () => dropped, results) : null; // Phase 4 (unless --burst false): a script to every device (~L output lines and two states each), through the job runner. double? statesSeconds = null; long outputRows = 0; var dispatchWatch = new Stopwatch(); if (jobBurst) { await JobBurstAsync(); } async Task JobBurstAsync() { var runId = Guid.NewGuid(); dispatchWatch.Start(); var runWatch = Stopwatch.StartNew(); var dispatchTask = Task.Run(async () => { using var scope = backend.Services.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var runner = scope.ServiceProvider.GetRequiredService(); var script = new ScriptEntity { Id = Guid.NewGuid(), Name = "Load burst", Language = "powershell", Content = $"1..{lines} | ForEach-Object {{ Write-Output \"line $_ of the load burst, about eighty characters of output each time\" }}", CreatedUtc = DateTimeOffset.UtcNow, UpdatedUtc = DateTimeOffset.UtcNow, }; db.Scripts.Add(script); await db.SaveChangesAsync(); var targets = await db.Devices.AsNoTracking().ToListAsync(); await runner.RecordRunAsync(runId, script, "loadbench", "manual", "all devices", [], CancellationToken.None); var prepared = await runner.PrepareAsync(script, CancellationToken.None); await runner.StartRunAsync(runId, prepared, null, targets, "loadbench", 0, 0, d => $"load burst on {d.Hostname}", CancellationToken.None); dispatchWatch.Stop(); }); // Sampled every second: when the last state was stored (every job succeeded) is the burst's completion. var burst = await MeasureAsync("job burst", self, agentsMeter, natsMeter, stats, () => dropped, TimeSpan.FromMinutes(30), async () => { var done = await backend.WithDbAsync(db => db.Jobs.CountAsync(j => j.RunId == runId && j.State == JobState.Succeeded)); var created = await backend.WithDbAsync(db => db.Jobs.CountAsync(j => j.RunId == runId)); var output = await backend.WithDbAsync(db => db.JobOutput.CountAsync(o => db.Jobs.Any(j => j.RunId == runId && j.JobId == o.JobId))); var dispatched = dispatchTask.IsCompleted ? $"dispatched in {dispatchWatch.Elapsed.TotalSeconds:0.0}s" : $"dispatching ({created} created)"; if (done >= devices && statesSeconds is null) { statesSeconds = runWatch.Elapsed.TotalSeconds; } return (dispatchTask.IsCompleted && done >= devices && stats.Outputs >= (long)devices * lines, $"{dispatched}; succeeded {done}/{devices}; output rows {output}/{(long)devices * lines}; agents ran {stats.Commands}"); }, interval: TimeSpan.FromSeconds(1)); results.Add(burst); Log($"burst: dispatched in {dispatchWatch.Elapsed.TotalSeconds:0.0}s ({devices / dispatchWatch.Elapsed.TotalSeconds:0}/s), every job's final state stored after {statesSeconds:0.0}s"); // The output still being written after the last state. var settle = Stopwatch.StartNew(); for (var i = 0; i < 120; i++) { var before = outputRows; outputRows = await backend.WithDbAsync(db => db.JobOutput.LongCountAsync(o => db.Jobs.Any(j => j.RunId == runId && j.JobId == o.JobId))); if (outputRows >= (long)devices * lines || (i > 3 && outputRows == before)) { break; } await Task.Delay(5000); } Log($"output rows stored {outputRows}/{(long)devices * lines} ({settle.Elapsed.TotalSeconds:0}s after the last state)"); } Log($"dropped by the backend's subscriptions: {string.Join(", ", droppedBySubject.Select(p => $"{p.Key} {p.Value}"))}"); child.Kill(entireProcessTree: true); await child.WaitForExitAsync(); var summary = new { utc = DateTimeOffset.UtcNow, devices, sealedMode = seals, connections, enrollSeconds = enrollWatch.Elapsed.TotalSeconds, provider, dispatchSeconds = dispatchWatch.Elapsed.TotalSeconds, statesSeconds, outputRows, expectedOutputRows = (long)devices * lines, dropped, droppedBySubject, phases = results, upgrade = upgradeSummary, }; await File.AppendAllTextAsync(outFile, JsonSerializer.Serialize(summary) + Environment.NewLine); foreach (var r in results) { Log($"{r.Phase,-14} {r.Seconds,7:0.0}s server {r.ServerCores,5:0.00} cores (peak {r.ServerPeakCores:0.00}) ws {r.ServerWorkingSetMb,6:0}MB priv {r.ServerPrivateMb,6:0}MB | agents {r.AgentsCores,5:0.00} | nats {r.NatsCores,5:0.00} cores {r.NatsWorkingSetMb,6:0}MB | {r.MessagesPerSecond,7:0} msg/s from agents | dropped {r.Dropped} | {r.Detail}"); } // Leave the shared NATS as it was: this run's consumers and queued commands go. var provisioner = backend.Services.GetRequiredService(); await Parallel.ForEachAsync(credentials, new ParallelOptions { MaxDegreeOfParallelism = 16 }, async (c, ct) => { try { await provisioner.RemoveAgentCommandsAsync(c.ClientId, ct); } catch (Exception) { // Best effort. } }); return 0; } /// /// Every device queued for an upgrade at once (an MSI made up for the catalog), the scheduler /// running as on the server with SCHEDULE_WORKER on and StartWithinMinutes set to /// : until every agent took its command. Each pass's sends are /// timed from the pass's start (the rows' SentUtc) to the last publish that returned. /// private static async Task UpgradeFanOutAsync( BenchBackend backend, int devices, int startWithin, SimulatorStats stats, ProcessMeter self, ProcessMeter agents, ProcessMeter? nats, Func dropped, List results) { const string Target = "2026.11.07.0000"; var folder = Directory.CreateTempSubdirectory("loadbench-package-").FullName; var msi = Path.Combine(folder, "ExecuteR.Agent.msi"); await File.WriteAllBytesAsync(msi, RandomNumberGenerator.GetBytes(256 * 1024)); AgentPackageEntity package; using (var scope = backend.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var row = await Backend.Auth.SettingsDocuments.LoadAsync(db, AgentUpgradeSettingsStore.Key, CancellationToken.None); Backend.Auth.SettingsDocuments.Stage( db, row, AgentUpgradeSettingsStore.Key, null, AgentUpgradeSettingsStore.Defaults with { StartWithinMinutes = startWithin }, "loadbench", always: true); await db.SaveChangesAsync(); scope.ServiceProvider.GetRequiredService().Invalidate(); package = await scope.ServiceProvider.GetRequiredService() .ImportAsync(AgentPackageTypes.Msi, msi, Target, CancellationToken.None) ?? throw new InvalidOperationException("The bench package was not imported"); } var queuedUtc = DateTimeOffset.UtcNow; using (var scope = backend.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var upgrades = scope.ServiceProvider.GetRequiredService(); foreach (var device in await db.Devices.AsNoTracking().ToListAsync()) { upgrades.Queue(device, package, AgentUpgradeSources.Manual, "loadbench"); } await db.SaveChangesAsync(); } Log($"upgrade: {devices} devices queued for {Target}, StartWithinMinutes {startWithin}"); var phase = await MeasureAsync("upgrade fan-out", self, agents, nats, stats, dropped, TimeSpan.FromMinutes(startWithin + 10), async () => { var sent = await backend.WithDbAsync(db => db.AgentUpgrades.CountAsync(u => u.State != AgentUpgradeStates.Queued)); var received = stats.Get("upgrades"); return (received >= devices && sent >= devices, $"rows sent {sent}/{devices}; agents took {received}, a second command for the same version {stats.Get("upgradeDuplicates")}"); }, interval: TimeSpan.FromSeconds(1)); results.Add(phase); var sentAt = await backend.WithDbAsync(db => db.AgentUpgrades.AsNoTracking() .Where(u => u.SentUtc != null) .Select(u => new { u.Id, u.SentUtc }) .ToListAsync()); var passOf = sentAt.ToDictionary(u => u.Id, u => u.SentUtc!.Value); var sends = backend.UpgradeSends.Sends.ToList(); var passes = sends .Where(s => passOf.ContainsKey(s.UpgradeId)) .GroupBy(s => passOf[s.UpgradeId]) .OrderBy(g => g.Key) .Select(g => new { atSeconds = Math.Round((g.Key - queuedUtc).TotalSeconds, 1), sent = g.Count(s => s.Sent), seconds = Math.Round((g.Max(s => s.DoneUtc) - g.Key).TotalSeconds, 2), }) .ToList(); var lastSend = sends.Count == 0 ? queuedUtc : sends.Max(s => s.DoneUtc); foreach (var pass in passes) { Log($" pass at {pass.atSeconds,6:0.0}s: {pass.sent,5} sent in {pass.seconds,6:0.00}s"); } var summary = new { startWithinMinutes = startWithin, passes, failedSends = sends.Count(s => !s.Sent), lastCommandSeconds = Math.Round((lastSend - queuedUtc).TotalSeconds, 1), received = stats.Get("upgrades"), duplicates = stats.Get("upgradeDuplicates"), }; Log($"upgrade: {summary.received}/{devices} agents took their command, the last {summary.lastCommandSeconds:0.0}s after queueing " + $"({passes.Count} passes, the longest {(passes.Count == 0 ? 0 : passes.Max(p => p.seconds)):0.00}s); " + $"failed sends {summary.failedSends}, a second command for the same version {summary.duplicates}"); return summary; } private static string Kind(string subject) { var parts = subject.Split('.'); return parts.Length > 0 ? parts[0] + "." + parts[^1] : subject; } private static async Task MeasureAsync( string phase, ProcessMeter server, ProcessMeter agents, ProcessMeter? nats, SimulatorStats stats, Func dropped, TimeSpan limit, Func> check, bool fixedDuration = false, TimeSpan? interval = null) { Log($"{phase}…"); server.Sample(); agents.Sample(); nats?.Sample(); var watch = Stopwatch.StartNew(); var published = stats.Published; var droppedBefore = dropped(); double serverCpu = 0, peak = 0, agentsCpu = 0, natsCpu = 0, ws = 0, priv = 0, natsWs = 0; var samples = 0; var detail = string.Empty; while (true) { await Task.Delay(interval ?? TimeSpan.FromSeconds(5)); var s = server.Sample(); var a = agents.Sample(); var n = nats?.Sample() ?? (0, 0, 0); samples++; serverCpu += s.Cores; peak = Math.Max(peak, s.Cores); agentsCpu += a.Cores; natsCpu += n.Cores; ws = Math.Max(ws, s.WorkingSetMb); priv = Math.Max(priv, s.PrivateMb); natsWs = Math.Max(natsWs, n.WorkingSetMb); var (done, text) = await check(); detail = text; Log($" {watch.Elapsed.TotalSeconds,6:0}s server {s.Cores:0.00} cores {s.WorkingSetMb:0}MB, agents {a.Cores:0.00}, nats {n.Cores:0.00} {n.WorkingSetMb:0}MB, dropped {dropped() - droppedBefore}, {text}"); if ((done && !fixedDuration) || watch.Elapsed >= limit) { break; } } var seconds = watch.Elapsed.TotalSeconds; return new PhaseResult( phase, seconds, serverCpu / samples, peak, ws, priv, agentsCpu / samples, natsCpu / samples, natsWs, (stats.Published - published) / seconds, dropped() - droppedBefore, detail); } private static Process? FindNats() { var candidates = Process.GetProcessesByName("nats-server"); return candidates.Length == 1 ? candidates[0] : null; } public static void Log(string text) => Console.WriteLine($"{DateTime.Now:HH:mm:ss} {text}"); } /// The simulator's counters as it prints them every few seconds. public sealed class SimulatorStats { private readonly ConcurrentDictionary values = new(); public long Published => Get("published"); public long Outputs => Get("outputs"); public long Commands => Get("commands"); public long SettingsAnswered => Get("settingsOk"); public long SettingsFailed => Get("settingsFailed"); public long Get(string name) => values.TryGetValue(name, out var v) ? v : 0; public Dictionary Snapshot() => new(values); public void Take(string line) { if (!line.StartsWith("STAT ", StringComparison.Ordinal)) { Bench.Log("agents: " + line); return; } foreach (var pair in line[5..].Split(' ', StringSplitOptions.RemoveEmptyEntries)) { var at = pair.IndexOf('='); values[pair[..at]] = long.Parse(pair[(at + 1)..], CultureInfo.InvariantCulture); } } } /// /// The simulated agents: each device with the agent's own message security and publisher /// (sealing and signing as the agent does), multiplexed over a few NATS connections, each /// pulling its own durable command consumer as the agent does. /// public static class Simulator { private static long published, presence, inventory, events, outputs, states, commands, refused, settingsOk, settingsFailed, bound, errors; private static long settingsFirstOk, settingsRetries, upgrades, upgradeDuplicates; private static readonly ConcurrentQueue SettingsLatencies = new(); private static bool retrySettings; /// The p-th percentile (0–100) of the settings replies' times so far, in ms. private static long Percentile(int[] sorted, double p) => sorted.Length == 0 ? 0 : sorted[Math.Min(sorted.Length - 1, (int)(sorted.Length * p / 100))]; public static async Task RunAsync(Options options) { var credentials = JsonSerializer.Deserialize>(await File.ReadAllTextAsync(options.Text("file")!), JsonDefaults.Options)!; var connectionCount = options.Int("connections", 8); var lines = options.Int("lines", 20); retrySettings = options.Bool("settings-retry", false); var ramp = TimeSpan.FromSeconds(options.Int("ramp", 20)); var connections = new List(); for (var i = 0; i < connectionCount; i++) { var connection = new NatsConnection(NatsOpts.Default with { Url = Program.NatsUrl, Name = $"loadbench-agents-{i}", MaxReconnectRetry = -1, SubPendingChannelCapacity = 65536, }); await connection.ConnectAsync(); connections.Add(connection); } var template = new InventoryCollector(new FixedIdentity()).Collect("template"); Console.WriteLine($"inventory report {JsonDefaults.SerializeToUtf8Bytes(new InventoryPayload(template)).Length} bytes"); var seen = new ReplayWindow(10_000_000); var devices = credentials.Select((c, i) => new SimDevice(c, connections[i % connections.Count], seen, template, lines)).ToList(); var stop = new CancellationTokenSource(); var tasks = devices.Select(d => Task.Run(() => d.RunAsync(ramp, stop.Token))).ToList(); _ = Task.Run(async () => { while (!stop.IsCancellationRequested) { await Task.Delay(2000); var latencies = SettingsLatencies.ToArray(); Array.Sort(latencies); Console.WriteLine( $"STAT settingsFirstOk={Interlocked.Read(ref settingsFirstOk)} settingsRetries={Interlocked.Read(ref settingsRetries)} " + $"settingsP50={Percentile(latencies, 50)} settingsP95={Percentile(latencies, 95)} settingsMax={(latencies.Length == 0 ? 0 : latencies[^1])} " + $"upgrades={Interlocked.Read(ref upgrades)} upgradeDuplicates={Interlocked.Read(ref upgradeDuplicates)}"); Console.WriteLine( $"STAT published={Interlocked.Read(ref published)} presence={Interlocked.Read(ref presence)} inventory={Interlocked.Read(ref inventory)} " + $"events={Interlocked.Read(ref events)} outputs={Interlocked.Read(ref outputs)} states={Interlocked.Read(ref states)} " + $"commands={Interlocked.Read(ref commands)} refused={Interlocked.Read(ref refused)} settingsOk={Interlocked.Read(ref settingsOk)} " + $"settingsFailed={Interlocked.Read(ref settingsFailed)} bound={Interlocked.Read(ref bound)} errors={Interlocked.Read(ref errors)}"); } }); await Task.WhenAll(tasks); return 0; } private sealed class FixedIdentity : IHardwareIdentitySource { public HardwareIdentity Read() => new("uuid", "bios", "board", "ident", "Contoso", "Model"); } private sealed class MemoryStore(AgentCredentials credentials) : ICredentialStore { private AgentCredentials? current = credentials; public AgentCredentials? Load() => current; public void Save(AgentCredentials value) => current = value; public void Delete() => current = null; } private sealed class SimDevice { private readonly DeviceCredentials credentials; private readonly NatsConnection connection; private readonly AgentMessageSecurity security; private readonly AgentPublisher publisher; private readonly InventoryReport report; private readonly int lines; private readonly string hostname; private readonly DateTimeOffset started = DateTimeOffset.UtcNow; private readonly HashSet upgradeVersions = new(StringComparer.Ordinal); public SimDevice(DeviceCredentials credentials, NatsConnection connection, ISeenMessageStore seen, InventoryReport template, int lines) { this.credentials = credentials; this.connection = connection; this.lines = lines; hostname = "PC-" + credentials.ClientId[2..10]; security = new AgentMessageSecurity( new MemoryStore(new AgentCredentials( AgentCredentials.CurrentVersion, credentials.ClientId, Guid.NewGuid(), "jwt", credentials.NkeySeed, null, credentials.MessageKey, credentials.ServerKeys)), seen, NullLogger.Instance); security.Reload(); publisher = new AgentPublisher(connection, credentials.ClientId, security); report = template with { Identity = template.Identity with { ClientId = credentials.ClientId, Hostname = hostname }, MessageSecurity = security.Report(), }; } public async Task RunAsync(TimeSpan ramp, CancellationToken cancellationToken) { var random = new Random(credentials.ClientId.GetHashCode()); await Task.Delay(TimeSpan.FromMilliseconds(random.NextDouble() * ramp.TotalMilliseconds), cancellationToken); _ = Task.Run(() => CommandsAsync(cancellationToken), cancellationToken); await Guard(() => PublishAsync(NatsSubjects.AgentEvent(credentials.ClientId), MessageTypes.AgentEvent, new AgentEventPayload(credentials.ClientId, AgentEventKinds.Ready, "command consumer bound", DateTimeOffset.UtcNow), ref events)); _ = Task.Run(() => PullSettingsAsync(cancellationToken), cancellationToken); await Guard(() => PublishAsync(NatsSubjects.AgentInventory(credentials.ClientId), MessageTypes.AgentInventory, new InventoryPayload(report), ref inventory)); var nextInventory = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(random.NextDouble() * 900); while (!cancellationToken.IsCancellationRequested) { await Guard(() => PublishAsync(NatsSubjects.AgentPresence(credentials.ClientId), MessageTypes.AgentPresence, new PresencePayload(credentials.ClientId, hostname, "2026.11.06.0000", "online", DateTimeOffset.UtcNow, (long)(DateTimeOffset.UtcNow - started).TotalSeconds, 0, "win-x64", "msi"), ref presence)); if (DateTimeOffset.UtcNow >= nextInventory) { nextInventory += TimeSpan.FromSeconds(900); await Guard(() => PublishAsync(NatsSubjects.AgentInventory(credentials.ClientId), MessageTypes.AgentInventory, new InventoryPayload(report with { CollectedUtc = DateTimeOffset.UtcNow }), ref inventory)); } await Task.Delay(ProtocolDefaults.PresenceInterval, cancellationToken); } } private static async Task Guard(Func action) { try { await action(); } catch (OperationCanceledException) { throw; } catch (Exception) { Interlocked.Increment(ref errors); } } private ValueTask PublishAsync(string subject, string type, T payload, ref long counter) { Interlocked.Increment(ref counter); Interlocked.Increment(ref published); return publisher.PublishAsync(subject, type, payload); } /// /// As AgentSettingsClient pulls: sealed request on the event subject, sealed reply. With /// --settings-retry, a request that timed out is tried again as the agent does; without it, /// one try (what reaches the device at its connect). /// /// The agent's own pause before each retry (a few, so a run that does not answer still ends). private static TimeSpan? SettingsRetryDelay(int attempt) => attempt < 6 ? AgentSettingsClient.RetryDelay(attempt, Random.Shared) : null; private async Task PullSettingsAsync(CancellationToken cancellationToken) { var subject = NatsSubjects.AgentEvent(credentials.ClientId); for (var attempt = 0; ; attempt++) { var envelope = MessageEnvelope.Create( MessageTypes.AgentSettingsRequest, publisher.Source, new AgentSettingsRequestPayload(credentials.ClientId, 0)); var watch = Stopwatch.StartNew(); byte[]? data = null; try { Interlocked.Increment(ref events); Interlocked.Increment(ref published); var answer = await connection.RequestAsync( subject, AgentPublisher.Seal(security, subject, envelope.MessageType, envelope.ToUtf8Bytes()), replyOpts: new NatsSubOpts { Timeout = AgentSettingsClient.RequestTimeout }, cancellationToken: cancellationToken); data = answer.Data is { Length: > 0 } d ? security.Open(answer.Subject, d, MessageReceipt.Request)?.Payload : null; } catch (Exception) when (!cancellationToken.IsCancellationRequested) { // Timed out: no answer in time. } if (data is not null) { SettingsLatencies.Enqueue((int)watch.ElapsedMilliseconds); Interlocked.Increment(ref settingsOk); if (attempt == 0) { Interlocked.Increment(ref settingsFirstOk); } return; } if (!retrySettings || SettingsRetryDelay(attempt) is not { } delay) { Interlocked.Increment(ref settingsFailed); return; } Interlocked.Increment(ref settingsRetries); await Task.Delay(delay, cancellationToken); } } private async Task CommandsAsync(CancellationToken cancellationToken) { var js = new NatsJSContext(connection); while (!cancellationToken.IsCancellationRequested) { try { var consumer = await js.GetConsumerAsync( JetStreamDefaults.CommandStreamName, JetStreamDefaults.AgentCommandConsumerName(credentials.ClientId), cancellationToken); Interlocked.Increment(ref bound); await foreach (var message in consumer.ConsumeAsync(opts: new NatsJSConsumeOpts { MaxMsgs = 10 }, cancellationToken: cancellationToken)) { var opened = message.Data is { Length: > 0 } data ? security.Open(message.Subject, data, MessageReceipt.Durable, message.Metadata?.Timestamp, out _) : null; await message.AckAsync(cancellationToken: cancellationToken); if (opened is null) { Interlocked.Increment(ref refused); continue; } security.Remember(opened); var commandEnvelope = MessageEnvelope.FromUtf8Bytes(opened.Payload); if (commandEnvelope?.MessageType == MessageTypes.CommandAgentUpgrade) { // Taken and kept: no download, no report (the row stays Sent). var upgrade = commandEnvelope.GetPayload(); Interlocked.Increment(ref upgrades); if (upgrade is not null && !upgradeVersions.Add(upgrade.Version)) { Interlocked.Increment(ref upgradeDuplicates); } continue; } var command = commandEnvelope?.GetPayload(); if (command is null) { continue; } Interlocked.Increment(ref commands); _ = Task.Run(() => RunJobAsync(command, cancellationToken), cancellationToken); } } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { return; } catch (Exception) { Interlocked.Increment(ref errors); await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); } } } private async Task RunJobAsync(ScriptCommandPayload command, CancellationToken cancellationToken) { var target = new EndpointRef(EndpointKinds.Job, command.JobId.ToString("D")); await Guard(() => PublishAsync(NatsSubjects.JobState(command.JobId), MessageTypes.JobState, new JobStatePayload(command.JobId, credentials.ClientId, JobState.Running, null, DateTimeOffset.UtcNow, null), ref states)); for (var i = 1; i <= lines; i++) { await Guard(() => PublishAsync(NatsSubjects.JobOutput(command.JobId), MessageTypes.JobOutput, new JobOutputPayload(command.JobId, credentials.ClientId, i, OutputStreamKind.Output, DateTimeOffset.UtcNow, $"line {i} of the load burst, about eighty characters of output each time"), ref outputs)); } await Guard(() => PublishAsync(NatsSubjects.JobState(command.JobId), MessageTypes.JobState, new JobStatePayload(command.JobId, credentials.ClientId, JobState.Succeeded, 0, DateTimeOffset.UtcNow, "done"), ref states)); } } } /// Per-message cost of sealing and opening, against the real server security (one thread). public static class Micro { private const int Warm = 500; public static async Task RunAsync(Options options) { var count = options.Int("count", 20000); var root = Directory.CreateTempSubdirectory("loadbench-micro-").FullName; await using var backend = new BenchBackend(root); using (backend.CreateClient()) { } var security = backend.Services.GetRequiredService(); await security.Keys.EnsureLoadedAsync(CancellationToken.None); const int Devices = 200; var sealedDevices = await backend.EnrollAsync(Devices, seals: true, "mb"); var plainDevices = await backend.EnrollAsync(Devices, seals: false, "mp"); var agents = sealedDevices.Select(c => Agent(c)).ToList(); var plainAgents = plainDevices.Select(c => Agent(c)).ToList(); Bench.Log($"per message, {count} each, one thread (µs, KB allocated)"); var presence = (AgentMessageSecurity a, string id) => MessageEnvelope.Create( MessageTypes.AgentPresence, new EndpointRef(EndpointKinds.Agent, id), new PresencePayload(id, "PC-000001", "2026.11.06.0000", "online", DateTimeOffset.UtcNow, 100, 0, "win-x64", "msi")).ToUtf8Bytes(); var output = (AgentMessageSecurity a, string id) => MessageEnvelope.Create( MessageTypes.JobOutput, new EndpointRef(EndpointKinds.Agent, id), new JobOutputPayload(Guid.NewGuid(), id, 1, OutputStreamKind.Output, DateTimeOffset.UtcNow, "line 1 of the load burst, about eighty characters of output each time")).ToUtf8Bytes(); foreach (var (name, make) in new[] { ("presence", presence), ("job output", output) }) { // Agent side: seal (encrypt + sign). var plain = Enumerable.Range(0, count + Warm).Select(i => make(agents[i % Devices].Security, agents[i % Devices].Id)).ToList(); var wire = new byte[count + Warm][]; Measure($"agent seal {name}", count, i => { var a = agents[i % Devices]; wire[i] = a.Security.Seal($"agent.{a.Id}.presence", MessageTypes.AgentPresence, plain[i]); return ValueTask.CompletedTask; }); Bench.Log($" {name}: v1 {plain[0].Length} bytes, sealed {wire[0].Length} bytes"); // Server side: open (verify, check, decrypt, replay window) sealed vs the v1 acceptance check. await MeasureAsync($"server open sealed {name}", count, async i => { var a = agents[i % Devices]; var result = await security.OpenFromAgentAsync($"agent.{a.Id}.presence", wire[i], a.Id, ServerReceipt.Live); if (result.Message is null) { throw new InvalidOperationException("refused"); } }); var plainWire = Enumerable.Range(0, count + Warm).Select(i => make(plainAgents[i % Devices].Security, plainAgents[i % Devices].Id)).ToList(); await MeasureAsync($"server open v1 {name}", count, async i => { var a = plainAgents[i % Devices]; var result = await security.OpenFromAgentAsync($"agent.{a.Id}.presence", plainWire[i], a.Id, ServerReceipt.Live); if (result.Message is null) { throw new InvalidOperationException("refused"); } }); Measure($"parse envelope {name}", count, i => { _ = MessageEnvelope.FromUtf8Bytes(plainWire[i]); return ValueTask.CompletedTask; }); } // Where an open's time goes: the Ed25519 signature (NATS.NKeys), the key agreement (cached), the AEAD. { var a = agents[0]; var bytes = a.Security.Seal($"agent.{a.Id}.presence", MessageTypes.AgentPresence, presence(a.Security, a.Id)); var envelope = MessageSealing.Parse(bytes)!; var nkeyPublic = KeyPair.FromSeed(sealedDevices[0].NkeySeed).GetPublicKey(); var secret = security.Keys.Current.Secret(MessageKeys.PublicKeyOf(MessageKeys.FromPrivateKey(sealedDevices[0].MessageKey!))); using var signer = new NkeySigner(sealedDevices[0].NkeySeed); var input = MessageSealing.SignatureInput(envelope); Measure(" parse sealed envelope (JSON)", count, _ => { MessageSealing.Parse(bytes); return ValueTask.CompletedTask; }); Measure(" Ed25519 verify (NATS.NKeys)", count / 4, _ => { MessageSealing.VerifySignature(envelope, nkeyPublic); return ValueTask.CompletedTask; }); Measure(" Ed25519 sign (NATS.NKeys)", count / 4, _ => { signer.Sign(input); return ValueTask.CompletedTask; }); Measure(" HKDF + AES-GCM decrypt", count, _ => { MessageSealing.TryDecrypt(envelope, secret, MessageDirections.AgentToServer); return ValueTask.CompletedTask; }); Measure(" ECDH P-256 agreement (uncached)", count / 20, _ => { MessageKeys.Agree(MessageKeys.FromPrivateKey(sealedDevices[1].MessageKey!), security.Keys.Current.EcdhPublic); return ValueTask.CompletedTask; }); } // Server side: a script command sealed for a device (fresh keys from the database, as the dispatcher does). var commandPlain = MessageEnvelope.Create(MessageTypes.CommandScript, new EndpointRef(EndpointKinds.Backend, "backend"), new ScriptCommandPayload(Guid.NewGuid(), Guid.NewGuid(), "Load burst", "1..20 | ForEach-Object { Write-Output \"line $_\" }", 3600)).ToUtf8Bytes(); await MeasureAsync("server seal command (cached keys)", count, async i => { var a = agents[i % Devices]; await security.SealForAgentAsync(a.Id, $"agent.{a.Id}.command.script", MessageTypes.CommandScript, commandPlain, MessageSecurityDefaults.DurableCommandLifetime); }); await MeasureAsync("server seal command (fresh: DB read)", count / 4, async i => { var a = agents[i % Devices]; await security.SealForAgentAsync(a.Id, $"agent.{a.Id}.command.script", MessageTypes.CommandScript, commandPlain, MessageSecurityDefaults.DurableCommandLifetime, fresh: true); }); await MeasureAsync("server v1 command (fresh: DB read)", count / 4, async i => { var a = plainAgents[i % Devices]; await security.SealForAgentAsync(a.Id, $"agent.{a.Id}.command.script", MessageTypes.CommandScript, commandPlain, MessageSecurityDefaults.DurableCommandLifetime, fresh: true); }); // The job runner's fan-out by itself (nothing answers, nothing else writes): per device. foreach (var prefix in new[] { "mb", "mp" }) { using var scope = backend.Services.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var runner = scope.ServiceProvider.GetRequiredService(); var script = new ScriptEntity { Id = Guid.NewGuid(), Name = "Micro", Language = "powershell", Content = "Write-Output 1", CreatedUtc = DateTimeOffset.UtcNow, UpdatedUtc = DateTimeOffset.UtcNow, }; db.Scripts.Add(script); await db.SaveChangesAsync(); var targets = await db.Devices.AsNoTracking().Where(d => d.ClientId.StartsWith(prefix)).ToListAsync(); var runId = Guid.NewGuid(); await runner.RecordRunAsync(runId, script, "micro", "manual", "all", [], CancellationToken.None); var prepared = await runner.PrepareAsync(script, CancellationToken.None); var watch = Stopwatch.StartNew(); await runner.StartRunAsync(runId, prepared, null, targets, "micro", 0, 0, d => d.Hostname, CancellationToken.None); Bench.Log($" job runner fan-out ({(prefix == "mb" ? "sealed" : "v1")}) {watch.Elapsed.TotalMilliseconds / targets.Count,8:0.0} ms per device ({targets.Count} devices)"); // Its two halves, per device: the job row (and history pruning), then the sealed publish and its state. double create = 0, dispatch = 0, publish = 0, prune = 0; var js = new NatsJSContext(backend.Services.GetRequiredService().Connection!); foreach (var device in targets) { var t = Stopwatch.StartNew(); var job = await runner.CreateJobAsync(script, device.ClientId, "micro", CancellationToken.None, runId); create += t.Elapsed.TotalMilliseconds; t.Restart(); await runner.DispatchAsync(prepared, job, null, device.Hostname, CancellationToken.None); dispatch += t.Elapsed.TotalMilliseconds; t.Restart(); (await js.PublishAsync($"agent.{device.ClientId}.command.inventory", commandPlain)).EnsureSuccess(); publish += t.Elapsed.TotalMilliseconds; t.Restart(); await runner.PruneHistoryAsync(device.ClientId, CancellationToken.None); prune += t.Elapsed.TotalMilliseconds; } Bench.Log($" create job {create / targets.Count:0.0} ms, dispatch {dispatch / targets.Count:0.0} ms, a bare JetStream publish {publish / targets.Count:0.0} ms, history pruning alone {prune / targets.Count:0.0} ms (per device)"); } var provisioner = backend.Services.GetRequiredService(); foreach (var c in sealedDevices.Concat(plainDevices)) { await provisioner.RemoveAgentCommandsAsync(c.ClientId, CancellationToken.None); } return 0; } private static (AgentMessageSecurity Security, string Id) Agent(DeviceCredentials c) { var security = new AgentMessageSecurity( new StaticStore(new AgentCredentials(AgentCredentials.CurrentVersion, c.ClientId, Guid.NewGuid(), "jwt", c.NkeySeed, null, c.MessageKey, c.ServerKeys)), new ReplayWindow(), NullLogger.Instance); security.Reload(); return (security, c.ClientId); } private sealed class StaticStore(AgentCredentials credentials) : ICredentialStore { public AgentCredentials? Load() => credentials; public void Save(AgentCredentials value) { } public void Delete() { } } private static void Measure(string name, int count, Func action) => MeasureAsync(name, count, i => action(i).AsTask()).GetAwaiter().GetResult(); private static async Task MeasureAsync(string name, int count, Func action) { // Warm-up on its own indices (count..count+Warm), so the measured messages are new. for (var i = 0; i < Warm; i++) { await action(count + i); } GC.Collect(); var allocated = GC.GetTotalAllocatedBytes(precise: true); var cpu = Process.GetCurrentProcess().TotalProcessorTime; var watch = Stopwatch.StartNew(); for (var i = 0; i < count; i++) { await action(i); } watch.Stop(); var used = Process.GetCurrentProcess().TotalProcessorTime - cpu; Bench.Log($" {name,-40} {watch.Elapsed.TotalMicroseconds / count,8:0.0} µs wall {used.TotalMicroseconds / count,8:0.0} µs cpu {(GC.GetTotalAllocatedBytes(precise: true) - allocated) / 1024.0 / count,6:0.0} KB"); } } /// A database of its own on the PostgreSQL server the PG* variables name, dropped afterwards. public sealed class ThrowawayPostgres : IAsyncDisposable { private readonly string admin; private ThrowawayPostgres(string admin, string name, string connectionString) { this.admin = admin; Name = name; ConnectionString = connectionString; } public string Name { get; } public string ConnectionString { get; } public static async Task CreateAsync() { var builder = new Npgsql.NpgsqlConnectionStringBuilder { Host = Environment.GetEnvironmentVariable("PGHOST") ?? "127.0.0.1", Port = int.TryParse(Environment.GetEnvironmentVariable("PGPORT"), out var port) ? port : 5432, Username = Environment.GetEnvironmentVariable("PGUSER") ?? "postgres", Password = Environment.GetEnvironmentVariable("PGPASSWORD"), Database = "postgres", MaxPoolSize = 200, }; var admin = builder.ConnectionString; var name = "loadbench_" + Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(6)); await using (var connection = new Npgsql.NpgsqlConnection(admin)) { await connection.OpenAsync(); await using var command = new Npgsql.NpgsqlCommand($"CREATE DATABASE {name}", connection); await command.ExecuteNonQueryAsync(); } builder.Database = name; return new ThrowawayPostgres(admin, name, builder.ConnectionString); } public async ValueTask DisposeAsync() { Npgsql.NpgsqlConnection.ClearAllPools(); await using var connection = new Npgsql.NpgsqlConnection(admin); await connection.OpenAsync(); await using var command = new Npgsql.NpgsqlCommand($"DROP DATABASE IF EXISTS {Name} WITH (FORCE)", connection); await command.ExecuteNonQueryAsync(); Bench.Log($"dropped database {Name}"); } }