Files
Alphaeus Mote 23ddf5c665 test(loadbench): the connect storm's settings answers and ready events, and an upgrade fan-out [skip ci]
- Connect storm: settings answers counted by first try and retries, with
  their reply times (p50, p95, max); the phase also waits for every
  device's ready event to be stored. --settings-retry retries a missed
  pull as the agent does (default: one try, what reaches a device at once).
- --upgrade: an MSI made up for the catalog, every device queued for it,
  the scheduler running as with SCHEDULE_WORKER on and StartWithinMinutes
  from --start-within (default 1); the simulated agents count upgrade
  commands and a second one for the same version, and each pass's sends are
  timed from the pass's start to the last publish returned.
- --burst false skips the job burst.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-07 11:02:40 -05:00

1285 lines
63 KiB
C#
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Load benchmark: the real backend (hosted in this process against a live NATS server, SQLite)
// and N simulated agents in a child process, sealed (envelope v2) or unsealed (v1).
//
// LoadBench run --devices 5000 [--sealed true] [--connections 16] [--steady 180] [--lines 20] [--nats-pid 1234] [--out results.jsonl] [--provider sqlite|postgres]
// [--burst true|false] [--upgrade true|false] [--start-within 1] [--settings-retry true|false]
// LoadBench micro [--count 20000]
//
// Phases: the connect storm (ready event, settings pull, inventory and presence from every device
// within 20 s; the pull is tried once unless --settings-retry, which retries as the agent does),
// steady state, with --upgrade every device upgraded through the scheduler's paced batches
// (SCHEDULE_WORKER on, StartWithinMinutes --start-within; each pass's sends timed), and unless
// --burst false a script to every device through the job runner.
//
// NATS_URL picks the server (default nats://127.0.0.1:4222; no auth, JetStream on). Not part of CI.
// --provider postgres runs on a throwaway database (created and dropped) on the server PGHOST, PGPORT,
// PGUSER and PGPASSWORD name.
// The JetStream streams are kept in memory storage, as the integration tests keep them on a shared
// server (production uses file storage). "cleanup" removes a stopped run's consumers.
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Globalization;
using System.Net.Http.Json;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using Agent.Enrollment;
using Agent.Identity;
using Agent.Inventory;
using Agent.Messaging;
using Backend.Data;
using Backend.Jobs;
using Backend.Nats;
using Backend.Security.Messages;
using Backend.Upgrades;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.AspNetCore.TestHost;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using NATS.Client.Core;
using NATS.Client.JetStream;
using NATS.NKeys;
using Shared.Contracts.Api;
using Shared.Contracts.Messages;
using Shared.Contracts.Models;
using Shared.Contracts.Serialization;
using Shared.Infrastructure.Constants;
using Shared.Infrastructure.Nats;
using Shared.Infrastructure.Security;
using Shared.Infrastructure.Security.Messaging;
namespace LoadBench;
public static class Program
{
public static string NatsUrl => Environment.GetEnvironmentVariable("NATS_URL") ?? "nats://127.0.0.1:4222";
public static async Task<int> Main(string[] args)
{
var options = Options.Parse(args);
return args.FirstOrDefault() switch
{
"run" => await Bench.RunAsync(options),
"agents" => await Simulator.RunAsync(options),
"micro" => await Micro.RunAsync(options),
"cleanup" => await CleanupAsync(),
_ => Usage(),
};
}
/// <summary>Deletes the consumers (and queued commands) a run that did not finish left on the shared NATS.</summary>
private static async Task<int> CleanupAsync()
{
await using var connection = new NatsConnection(NatsOpts.Default with { Url = NatsUrl });
var js = new NatsJSContext(connection);
var stream = await js.GetStreamAsync(JetStreamDefaults.CommandStreamName);
var removed = 0;
await foreach (var name in stream.ListConsumerNamesAsync())
{
var clientId = name.Replace("agent-commands-", string.Empty, StringComparison.Ordinal);
if (name.StartsWith("agent-commands-", StringComparison.Ordinal) && clientId.Length == 22 && clientId[0] is 'l' or 'm')
{
await stream.DeleteConsumerAsync(name);
await stream.PurgeAsync(new NATS.Client.JetStream.Models.StreamPurgeRequest { Filter = NatsSubjects.AgentCommandWildcard(clientId) });
removed++;
}
}
Console.WriteLine($"removed {removed} benchmark consumers");
// The backend provisions file storage (as in production, so a run measures that); the
// shared test server is left with the memory streams the integration tests use.
foreach (var (name, subjects, maxAge) in new[]
{
(JetStreamDefaults.CommandStreamName, JetStreamDefaults.CommandStreamSubjects, JetStreamDefaults.CommandMaxAge),
(JetStreamDefaults.JobEventsStreamName, JetStreamDefaults.JobEventsStreamSubjects, JetStreamDefaults.JobEventsMaxAge),
})
{
try
{
var existing = await js.GetStreamAsync(name);
if (existing.Info.Config.Storage == NATS.Client.JetStream.Models.StreamConfigStorage.File)
{
await js.DeleteStreamAsync(name);
await js.CreateStreamAsync(new NATS.Client.JetStream.Models.StreamConfig(name, subjects)
{
Storage = NATS.Client.JetStream.Models.StreamConfigStorage.Memory,
MaxAge = maxAge,
});
Console.WriteLine($"{name}: file storage replaced by memory storage");
}
}
catch (NatsJSApiException exception) when (exception.Error.Code == 404)
{
// Not there.
}
}
return 0;
}
private static int Usage()
{
Console.Error.WriteLine("LoadBench run --devices N [--sealed true|false] [--connections K] [--steady S] [--lines L] [--nats-pid P] [--out F] | micro [--count N]");
return 2;
}
}
public sealed class Options(Dictionary<string, string> values)
{
public static Options Parse(string[] args)
{
var values = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
for (var i = 1; i < args.Length - 1; i += 2)
{
values[args[i].TrimStart('-')] = args[i + 1];
}
return new Options(values);
}
public int Int(string name, int fallback) => values.TryGetValue(name, out var v) ? int.Parse(v, CultureInfo.InvariantCulture) : fallback;
public bool Bool(string name, bool fallback) => values.TryGetValue(name, out var v) ? bool.Parse(v) : fallback;
public string? Text(string name) => values.TryGetValue(name, out var v) ? v : null;
}
/// <summary>One simulated device's identity as the child process gets it.</summary>
public sealed record DeviceCredentials(string ClientId, string NkeySeed, string? MessageKey, ServerMessageKeys? ServerKeys);
/// <summary>When each upgrade command's publish returned (the scheduler's sends, timed as they go out).</summary>
public sealed class UpgradeSendLog
{
public ConcurrentQueue<(Guid UpgradeId, string ClientId, DateTimeOffset DoneUtc, bool Sent)> Sends { get; } = new();
}
/// <summary>The real sender, each send recorded in <see cref="UpgradeSendLog"/>.</summary>
public sealed class TimedUpgradeSender(IAgentUpgradeSender inner, UpgradeSendLog log) : IAgentUpgradeSender
{
public async Task<bool> DispatchAgentUpgradeAsync(string clientId, AgentUpgradeCommandPayload command, CancellationToken cancellationToken)
{
var sent = await inner.DispatchAgentUpgradeAsync(clientId, command, cancellationToken);
log.Sends.Enqueue((command.UpgradeId, clientId, DateTimeOffset.UtcNow, sent));
return sent;
}
}
/// <summary>The backend, hosted in this process against the live NATS (as the integration tests host it).</summary>
public sealed class BenchBackend(string root, string? postgres = null, bool scheduleWorker = false) : WebApplicationFactory<global::Program>
{
private readonly KeyPair account = KeyPair.CreatePair(PrefixByte.Account);
public UpgradeSendLog UpgradeSends { get; } = new();
protected override void ConfigureWebHost(IWebHostBuilder builder)
{
builder.ConfigureTestServices(services =>
{
// The upgrade scheduler's sends, timed (the scheduler runs only with SCHEDULE_WORKER on).
services.AddSingleton<IAgentUpgradeSender>(sp => new TimedUpgradeSender(sp.GetRequiredService<CommandDispatcher>(), UpgradeSends));
});
builder.UseSetting("DATABASE_PATH", Path.Combine(root, "app.db"));
if (postgres is not null)
{
builder.UseSetting("DATABASE_PROVIDER", "postgres");
builder.UseSetting("DATABASE_CONNECTION_STRING", postgres);
}
builder.UseSetting("DATA_PATH", Path.Combine(root, "data"));
builder.UseSetting("DATAPROTECTION_PATH", Path.Combine(root, "keys"));
builder.UseSetting("LOG_DIRECTORY", Path.Combine(root, "logs"));
builder.UseSetting("LOG_LEVEL", Environment.GetEnvironmentVariable("BENCH_LOG_LEVEL") ?? "Warning");
builder.UseSetting("AGENT_CATALOG_SYNC", "0");
builder.UseSetting("S3_EMBEDDED", "0");
builder.UseSetting("NATS_ACCOUNT_SIGNING_SEED", account.GetSeed());
builder.UseSetting("NATS_ACCOUNT_PUBLIC_KEY", account.GetPublicKey());
builder.UseSetting("NATS_URL", Program.NatsUrl);
builder.UseSetting("RATE_LIMIT_PER_MINUTE", "100000000");
builder.UseSetting("SCHEDULE_WORKER", scheduleWorker ? "1" : "0");
builder.UseSetting("TLS_SELF_SIGNED", "0");
builder.UseEnvironment("Production");
}
public async Task<T> WithDbAsync<T>(Func<AppDbContext, Task<T>> action)
{
using var scope = Services.CreateScope();
return await action(scope.ServiceProvider.GetRequiredService<AppDbContext>());
}
public async Task<string> TokenAsync()
{
var token = TokenGenerator.CreateUrlSafeToken(32);
await WithDbAsync(async db =>
{
db.BootstrapTokens.Add(new BootstrapTokenEntity
{
Id = Guid.NewGuid(),
TokenHash = TokenHashing.Sha256Hex(token),
CreatedUtc = DateTimeOffset.UtcNow,
ExpiresUtc = DateTimeOffset.UtcNow + TimeSpan.FromDays(1),
MaxUses = 0,
});
return await db.SaveChangesAsync();
});
return token;
}
/// <summary>Enrolls devices through the real bootstrap endpoint, <paramref name="parallel"/> at a time.</summary>
public async Task<List<DeviceCredentials>> EnrollAsync(int count, bool seals, string prefix, int parallel = 16)
{
var token = await TokenAsync();
var result = new ConcurrentBag<DeviceCredentials>();
using var http = CreateClient();
http.Timeout = TimeSpan.FromMinutes(5);
await Parallel.ForEachAsync(Enumerable.Range(0, count), new ParallelOptions { MaxDegreeOfParallelism = parallel }, async (i, ct) =>
{
var clientId = prefix + Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(10));
using var nkey = KeyPair.CreatePair(PrefixByte.User);
using var ecdh = MessageKeys.Create();
var response = await http.PostAsJsonAsync(ApiRoutes.Bootstrap, new BootstrapRequest(
token, clientId, nkey.GetPublicKey(), "2026.11.06.0000",
new EnrollDeviceInfo("PC-" + clientId[2..10], "Contoso", "Model", "Windows 11", "10.0.26200"),
seals ? MessageKeys.PublicKeyOf(ecdh) : null), JsonDefaults.Options, ct);
response.EnsureSuccessStatusCode();
var answer = (await response.Content.ReadFromJsonAsync<BootstrapResponse>(JsonDefaults.Options, ct))!;
result.Add(new DeviceCredentials(clientId, nkey.GetSeed(), seals ? MessageKeys.PrivateKeyOf(ecdh) : null, answer.ServerKeys));
});
return [.. result];
}
protected override void Dispose(bool disposing)
{
base.Dispose(disposing);
account.Dispose();
}
}
/// <summary>CPU and memory of one process between samples.</summary>
public sealed class ProcessMeter(Process process, string name)
{
private TimeSpan lastCpu = Cpu(process);
private long lastTicks = Stopwatch.GetTimestamp();
public string Name => name;
private static TimeSpan Cpu(Process p)
{
try
{
p.Refresh();
return p.TotalProcessorTime;
}
catch (Exception)
{
return TimeSpan.Zero;
}
}
/// <summary>Cores busy since the last call, working set and private bytes (MB).</summary>
public (double Cores, double WorkingSetMb, double PrivateMb) Sample()
{
var cpu = Cpu(process);
var now = Stopwatch.GetTimestamp();
var wall = Stopwatch.GetElapsedTime(lastTicks, now);
var cores = wall.TotalSeconds > 0 ? (cpu - lastCpu).TotalSeconds / wall.TotalSeconds : 0;
lastCpu = cpu;
lastTicks = now;
try
{
return (cores, process.WorkingSet64 / 1048576.0, process.PrivateMemorySize64 / 1048576.0);
}
catch (Exception)
{
return (cores, 0, 0);
}
}
}
/// <summary>The orchestrator: hosts the backend, enrolls, starts the simulator, measures the three phases.</summary>
public static class Bench
{
private sealed record PhaseResult(
string Phase, double Seconds, double ServerCores, double ServerPeakCores, double ServerWorkingSetMb, double ServerPrivateMb,
double AgentsCores, double NatsCores, double NatsWorkingSetMb, double MessagesPerSecond, long Dropped, string Detail);
public static async Task<int> RunAsync(Options options)
{
var devices = options.Int("devices", 1000);
var seals = options.Bool("sealed", true);
var connections = options.Int("connections", Math.Max(1, Math.Min(32, devices / 500)));
var steady = options.Int("steady", 180);
var lines = options.Int("lines", 20);
var outFile = options.Text("out") ?? Path.Combine(AppContext.BaseDirectory, "results.jsonl");
var root = Directory.CreateTempSubdirectory("loadbench-").FullName;
var provider = options.Text("provider") ?? "sqlite";
var upgrade = options.Bool("upgrade", false);
var jobBurst = options.Bool("burst", true);
var settingsRetry = options.Bool("settings-retry", false);
await using var database = provider == "postgres" ? await ThrowawayPostgres.CreateAsync() : null;
Log($"{devices} devices, {(seals ? "sealed (v2)" : "unsealed (v1)")}, {connections} NATS connections for the agents, {provider}{(database is null ? string.Empty : $" ({database.Name})")}, data in {root}");
Process? nats = options.Int("nats-pid", 0) is > 0 and var pid ? Process.GetProcessById(pid) : FindNats();
// The streams in memory storage, as the integration tests keep them on a shared server (a
// run that switched them to file storage had them deleted under it by a test run). The
// backend's own file-storage provisioning then only warns; its job event consumer is made here.
await using (var setup = new NatsConnection(NatsOpts.Default with { Url = Program.NatsUrl }))
{
var js = new NatsJSContext(setup);
foreach (var (name, subjects, maxAge) in new[]
{
(JetStreamDefaults.CommandStreamName, JetStreamDefaults.CommandStreamSubjects, JetStreamDefaults.CommandMaxAge),
(JetStreamDefaults.JobEventsStreamName, JetStreamDefaults.JobEventsStreamSubjects, JetStreamDefaults.JobEventsMaxAge),
})
{
var config = new NATS.Client.JetStream.Models.StreamConfig(name, subjects)
{
Storage = NATS.Client.JetStream.Models.StreamConfigStorage.Memory,
MaxAge = maxAge,
};
try
{
await js.CreateOrUpdateStreamAsync(config);
}
catch (NatsJSApiException)
{
await js.DeleteStreamAsync(name);
await js.CreateOrUpdateStreamAsync(config);
}
}
await js.CreateOrUpdateConsumerAsync(JetStreamDefaults.JobEventsStreamName, new NATS.Client.JetStream.Models.ConsumerConfig(JetStreamDefaults.BackendJobEventsConsumerName)
{
DurableName = JetStreamDefaults.BackendJobEventsConsumerName,
AckPolicy = NATS.Client.JetStream.Models.ConsumerConfigAckPolicy.Explicit,
AckWait = JetStreamDefaults.CommandAckWait,
FilterSubject = JetStreamDefaults.JobEventsStreamSubjects[0],
});
}
await using var backend = new BenchBackend(root, database?.ConnectionString, scheduleWorker: upgrade);
using (backend.CreateClient())
{
}
var hub = backend.Services.GetRequiredService<NatsHub>().Connection!;
long dropped = 0;
var droppedBySubject = new ConcurrentDictionary<string, long>();
hub.MessageDropped += (_, e) =>
{
Interlocked.Increment(ref dropped);
droppedBySubject.AddOrUpdate(Kind(e.Subject), 1, (_, n) => n + 1);
return ValueTask.CompletedTask;
};
await backend.Services.GetRequiredService<ServerMessageSecurity>().Keys.EnsureLoadedAsync(CancellationToken.None);
// Enrollment through the real endpoint (keys recorded, consumers made).
var enrollWatch = Stopwatch.StartNew();
var credentials = await backend.EnrollAsync(devices, seals, "lb");
enrollWatch.Stop();
Log($"enrolled {devices} in {enrollWatch.Elapsed.TotalSeconds:0.0}s ({devices / enrollWatch.Elapsed.TotalSeconds:0}/s)");
var credentialsFile = Path.Combine(root, "devices.json");
await File.WriteAllTextAsync(credentialsFile, JsonSerializer.Serialize(credentials, JsonDefaults.Options));
// The simulator, in its own process (its CPU is the agents', not the server's).
var child = Process.Start(new ProcessStartInfo(Environment.ProcessPath!, $"agents --file \"{credentialsFile}\" --connections {connections} --lines {lines} --ramp 20 --settings-retry {settingsRetry}")
{
RedirectStandardOutput = true,
UseShellExecute = false,
})!;
var stats = new SimulatorStats();
_ = Task.Run(async () =>
{
while (await child.StandardOutput.ReadLineAsync() is { } line)
{
stats.Take(line);
}
});
var self = new ProcessMeter(Process.GetCurrentProcess(), "server");
var agentsMeter = new ProcessMeter(child, "agents");
var natsMeter = nats is null ? null : new ProcessMeter(nats, "nats");
var results = new List<PhaseResult>();
var startedUtc = DateTimeOffset.UtcNow;
// Phase 1: every agent connects within 20 s (a fleet coming back after an outage):
// ready event, settings pull, inventory and presence each.
var storm = await MeasureAsync("connect storm", self, agentsMeter, natsMeter, stats, () => dropped, TimeSpan.FromMinutes(15), async () =>
{
var inventories = await backend.WithDbAsync(db => db.DeviceInventory.CountAsync(i => i.UpdatedUtc >= startedUtc));
var seen = await backend.WithDbAsync(db => db.Devices.CountAsync(d => d.LastSeenUtc >= startedUtc));
var ready = await backend.WithDbAsync(db => db.Devices.CountAsync(d => d.ReadyUtc >= startedUtc));
return (inventories >= devices && seen >= devices && ready >= devices && stats.SettingsAnswered + stats.SettingsFailed >= devices,
$"inventories {inventories}/{devices}, seen {seen}/{devices}, ready {ready}/{devices}, settings answered {stats.SettingsAnswered} " +
$"(first try {stats.Get("settingsFirstOk")}, retries {stats.Get("settingsRetries")}) failed {stats.SettingsFailed}, " +
$"reply ms p50 {stats.Get("settingsP50")} p95 {stats.Get("settingsP95")} max {stats.Get("settingsMax")}");
});
results.Add(storm);
Log($"connect storm: every device's settings decided after {storm.Seconds:0.0}s: answered {stats.SettingsAnswered}/{devices}, first try {stats.Get("settingsFirstOk")}, failed {stats.SettingsFailed}");
// Phase 2: steady state (presence every 30 s, inventory every 15 min, spread out).
var steadyStart = stats.Snapshot();
var steadyResult = await MeasureAsync("steady", self, agentsMeter, natsMeter, stats, () => dropped, TimeSpan.FromSeconds(steady), async () =>
{
var recent = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(40);
var fresh = await backend.WithDbAsync(db => db.Devices.CountAsync(d => d.LastSeenUtc >= recent));
return (false, $"seen in the last 40 s {fresh}/{devices}");
}, fixedDuration: true);
results.Add(steadyResult);
// Phase 3 (with --upgrade): every device upgraded through the upgrade scheduler's paced batches.
object? upgradeSummary = upgrade
? await UpgradeFanOutAsync(backend, devices, options.Int("start-within", 1), stats, self, agentsMeter, natsMeter, () => dropped, results)
: null;
// Phase 4 (unless --burst false): a script to every device (~L output lines and two states each), through the job runner.
double? statesSeconds = null;
long outputRows = 0;
var dispatchWatch = new Stopwatch();
if (jobBurst)
{
await JobBurstAsync();
}
async Task JobBurstAsync()
{
var runId = Guid.NewGuid();
dispatchWatch.Start();
var runWatch = Stopwatch.StartNew();
var dispatchTask = Task.Run(async () =>
{
using var scope = backend.Services.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
var runner = scope.ServiceProvider.GetRequiredService<JobRunner>();
var script = new ScriptEntity
{
Id = Guid.NewGuid(),
Name = "Load burst",
Language = "powershell",
Content = $"1..{lines} | ForEach-Object {{ Write-Output \"line $_ of the load burst, about eighty characters of output each time\" }}",
CreatedUtc = DateTimeOffset.UtcNow,
UpdatedUtc = DateTimeOffset.UtcNow,
};
db.Scripts.Add(script);
await db.SaveChangesAsync();
var targets = await db.Devices.AsNoTracking().ToListAsync();
await runner.RecordRunAsync(runId, script, "loadbench", "manual", "all devices", [], CancellationToken.None);
var prepared = await runner.PrepareAsync(script, CancellationToken.None);
await runner.StartRunAsync(runId, prepared, null, targets, "loadbench", 0, 0, d => $"load burst on {d.Hostname}", CancellationToken.None);
dispatchWatch.Stop();
});
// Sampled every second: when the last state was stored (every job succeeded) is the burst's completion.
var burst = await MeasureAsync("job burst", self, agentsMeter, natsMeter, stats, () => dropped, TimeSpan.FromMinutes(30), async () =>
{
var done = await backend.WithDbAsync(db => db.Jobs.CountAsync(j => j.RunId == runId && j.State == JobState.Succeeded));
var created = await backend.WithDbAsync(db => db.Jobs.CountAsync(j => j.RunId == runId));
var output = await backend.WithDbAsync(db => db.JobOutput.CountAsync(o => db.Jobs.Any(j => j.RunId == runId && j.JobId == o.JobId)));
var dispatched = dispatchTask.IsCompleted ? $"dispatched in {dispatchWatch.Elapsed.TotalSeconds:0.0}s" : $"dispatching ({created} created)";
if (done >= devices && statesSeconds is null)
{
statesSeconds = runWatch.Elapsed.TotalSeconds;
}
return (dispatchTask.IsCompleted && done >= devices && stats.Outputs >= (long)devices * lines,
$"{dispatched}; succeeded {done}/{devices}; output rows {output}/{(long)devices * lines}; agents ran {stats.Commands}");
}, interval: TimeSpan.FromSeconds(1));
results.Add(burst);
Log($"burst: dispatched in {dispatchWatch.Elapsed.TotalSeconds:0.0}s ({devices / dispatchWatch.Elapsed.TotalSeconds:0}/s), every job's final state stored after {statesSeconds:0.0}s");
// The output still being written after the last state.
var settle = Stopwatch.StartNew();
for (var i = 0; i < 120; i++)
{
var before = outputRows;
outputRows = await backend.WithDbAsync(db => db.JobOutput.LongCountAsync(o => db.Jobs.Any(j => j.RunId == runId && j.JobId == o.JobId)));
if (outputRows >= (long)devices * lines || (i > 3 && outputRows == before))
{
break;
}
await Task.Delay(5000);
}
Log($"output rows stored {outputRows}/{(long)devices * lines} ({settle.Elapsed.TotalSeconds:0}s after the last state)");
}
Log($"dropped by the backend's subscriptions: {string.Join(", ", droppedBySubject.Select(p => $"{p.Key} {p.Value}"))}");
child.Kill(entireProcessTree: true);
await child.WaitForExitAsync();
var summary = new
{
utc = DateTimeOffset.UtcNow,
devices,
sealedMode = seals,
connections,
enrollSeconds = enrollWatch.Elapsed.TotalSeconds,
provider,
dispatchSeconds = dispatchWatch.Elapsed.TotalSeconds,
statesSeconds,
outputRows,
expectedOutputRows = (long)devices * lines,
dropped,
droppedBySubject,
phases = results,
upgrade = upgradeSummary,
};
await File.AppendAllTextAsync(outFile, JsonSerializer.Serialize(summary) + Environment.NewLine);
foreach (var r in results)
{
Log($"{r.Phase,-14} {r.Seconds,7:0.0}s server {r.ServerCores,5:0.00} cores (peak {r.ServerPeakCores:0.00}) ws {r.ServerWorkingSetMb,6:0}MB priv {r.ServerPrivateMb,6:0}MB | agents {r.AgentsCores,5:0.00} | nats {r.NatsCores,5:0.00} cores {r.NatsWorkingSetMb,6:0}MB | {r.MessagesPerSecond,7:0} msg/s from agents | dropped {r.Dropped} | {r.Detail}");
}
// Leave the shared NATS as it was: this run's consumers and queued commands go.
var provisioner = backend.Services.GetRequiredService<JetStreamProvisioner>();
await Parallel.ForEachAsync(credentials, new ParallelOptions { MaxDegreeOfParallelism = 16 }, async (c, ct) =>
{
try
{
await provisioner.RemoveAgentCommandsAsync(c.ClientId, ct);
}
catch (Exception)
{
// Best effort.
}
});
return 0;
}
/// <summary>
/// Every device queued for an upgrade at once (an MSI made up for the catalog), the scheduler
/// running as on the server with SCHEDULE_WORKER on and StartWithinMinutes set to
/// <paramref name="startWithin"/>: until every agent took its command. Each pass's sends are
/// timed from the pass's start (the rows' SentUtc) to the last publish that returned.
/// </summary>
private static async Task<object> UpgradeFanOutAsync(
BenchBackend backend, int devices, int startWithin, SimulatorStats stats, ProcessMeter self, ProcessMeter agents, ProcessMeter? nats,
Func<long> dropped, List<PhaseResult> results)
{
const string Target = "2026.11.07.0000";
var folder = Directory.CreateTempSubdirectory("loadbench-package-").FullName;
var msi = Path.Combine(folder, "ExecuteR.Agent.msi");
await File.WriteAllBytesAsync(msi, RandomNumberGenerator.GetBytes(256 * 1024));
AgentPackageEntity package;
using (var scope = backend.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
var row = await Backend.Auth.SettingsDocuments.LoadAsync(db, AgentUpgradeSettingsStore.Key, CancellationToken.None);
Backend.Auth.SettingsDocuments.Stage(
db, row, AgentUpgradeSettingsStore.Key, null, AgentUpgradeSettingsStore.Defaults with { StartWithinMinutes = startWithin },
"loadbench", always: true);
await db.SaveChangesAsync();
scope.ServiceProvider.GetRequiredService<AgentUpgradeSettingsStore>().Invalidate();
package = await scope.ServiceProvider.GetRequiredService<AgentCatalogService>()
.ImportAsync(AgentPackageTypes.Msi, msi, Target, CancellationToken.None)
?? throw new InvalidOperationException("The bench package was not imported");
}
var queuedUtc = DateTimeOffset.UtcNow;
using (var scope = backend.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
var upgrades = scope.ServiceProvider.GetRequiredService<AgentUpgradeService>();
foreach (var device in await db.Devices.AsNoTracking().ToListAsync())
{
upgrades.Queue(device, package, AgentUpgradeSources.Manual, "loadbench");
}
await db.SaveChangesAsync();
}
Log($"upgrade: {devices} devices queued for {Target}, StartWithinMinutes {startWithin}");
var phase = await MeasureAsync("upgrade fan-out", self, agents, nats, stats, dropped, TimeSpan.FromMinutes(startWithin + 10), async () =>
{
var sent = await backend.WithDbAsync(db => db.AgentUpgrades.CountAsync(u => u.State != AgentUpgradeStates.Queued));
var received = stats.Get("upgrades");
return (received >= devices && sent >= devices,
$"rows sent {sent}/{devices}; agents took {received}, a second command for the same version {stats.Get("upgradeDuplicates")}");
}, interval: TimeSpan.FromSeconds(1));
results.Add(phase);
var sentAt = await backend.WithDbAsync(db => db.AgentUpgrades.AsNoTracking()
.Where(u => u.SentUtc != null)
.Select(u => new { u.Id, u.SentUtc })
.ToListAsync());
var passOf = sentAt.ToDictionary(u => u.Id, u => u.SentUtc!.Value);
var sends = backend.UpgradeSends.Sends.ToList();
var passes = sends
.Where(s => passOf.ContainsKey(s.UpgradeId))
.GroupBy(s => passOf[s.UpgradeId])
.OrderBy(g => g.Key)
.Select(g => new
{
atSeconds = Math.Round((g.Key - queuedUtc).TotalSeconds, 1),
sent = g.Count(s => s.Sent),
seconds = Math.Round((g.Max(s => s.DoneUtc) - g.Key).TotalSeconds, 2),
})
.ToList();
var lastSend = sends.Count == 0 ? queuedUtc : sends.Max(s => s.DoneUtc);
foreach (var pass in passes)
{
Log($" pass at {pass.atSeconds,6:0.0}s: {pass.sent,5} sent in {pass.seconds,6:0.00}s");
}
var summary = new
{
startWithinMinutes = startWithin,
passes,
failedSends = sends.Count(s => !s.Sent),
lastCommandSeconds = Math.Round((lastSend - queuedUtc).TotalSeconds, 1),
received = stats.Get("upgrades"),
duplicates = stats.Get("upgradeDuplicates"),
};
Log($"upgrade: {summary.received}/{devices} agents took their command, the last {summary.lastCommandSeconds:0.0}s after queueing " +
$"({passes.Count} passes, the longest {(passes.Count == 0 ? 0 : passes.Max(p => p.seconds)):0.00}s); " +
$"failed sends {summary.failedSends}, a second command for the same version {summary.duplicates}");
return summary;
}
private static string Kind(string subject)
{
var parts = subject.Split('.');
return parts.Length > 0 ? parts[0] + "." + parts[^1] : subject;
}
private static async Task<PhaseResult> MeasureAsync(
string phase, ProcessMeter server, ProcessMeter agents, ProcessMeter? nats, SimulatorStats stats, Func<long> dropped, TimeSpan limit,
Func<Task<(bool Done, string Detail)>> check, bool fixedDuration = false, TimeSpan? interval = null)
{
Log($"{phase}…");
server.Sample();
agents.Sample();
nats?.Sample();
var watch = Stopwatch.StartNew();
var published = stats.Published;
var droppedBefore = dropped();
double serverCpu = 0, peak = 0, agentsCpu = 0, natsCpu = 0, ws = 0, priv = 0, natsWs = 0;
var samples = 0;
var detail = string.Empty;
while (true)
{
await Task.Delay(interval ?? TimeSpan.FromSeconds(5));
var s = server.Sample();
var a = agents.Sample();
var n = nats?.Sample() ?? (0, 0, 0);
samples++;
serverCpu += s.Cores;
peak = Math.Max(peak, s.Cores);
agentsCpu += a.Cores;
natsCpu += n.Cores;
ws = Math.Max(ws, s.WorkingSetMb);
priv = Math.Max(priv, s.PrivateMb);
natsWs = Math.Max(natsWs, n.WorkingSetMb);
var (done, text) = await check();
detail = text;
Log($" {watch.Elapsed.TotalSeconds,6:0}s server {s.Cores:0.00} cores {s.WorkingSetMb:0}MB, agents {a.Cores:0.00}, nats {n.Cores:0.00} {n.WorkingSetMb:0}MB, dropped {dropped() - droppedBefore}, {text}");
if ((done && !fixedDuration) || watch.Elapsed >= limit)
{
break;
}
}
var seconds = watch.Elapsed.TotalSeconds;
return new PhaseResult(
phase, seconds, serverCpu / samples, peak, ws, priv, agentsCpu / samples, natsCpu / samples, natsWs,
(stats.Published - published) / seconds, dropped() - droppedBefore, detail);
}
private static Process? FindNats()
{
var candidates = Process.GetProcessesByName("nats-server");
return candidates.Length == 1 ? candidates[0] : null;
}
public static void Log(string text) => Console.WriteLine($"{DateTime.Now:HH:mm:ss} {text}");
}
/// <summary>The simulator's counters as it prints them every few seconds.</summary>
public sealed class SimulatorStats
{
private readonly ConcurrentDictionary<string, long> values = new();
public long Published => Get("published");
public long Outputs => Get("outputs");
public long Commands => Get("commands");
public long SettingsAnswered => Get("settingsOk");
public long SettingsFailed => Get("settingsFailed");
public long Get(string name) => values.TryGetValue(name, out var v) ? v : 0;
public Dictionary<string, long> Snapshot() => new(values);
public void Take(string line)
{
if (!line.StartsWith("STAT ", StringComparison.Ordinal))
{
Bench.Log("agents: " + line);
return;
}
foreach (var pair in line[5..].Split(' ', StringSplitOptions.RemoveEmptyEntries))
{
var at = pair.IndexOf('=');
values[pair[..at]] = long.Parse(pair[(at + 1)..], CultureInfo.InvariantCulture);
}
}
}
/// <summary>
/// The simulated agents: each device with the agent's own message security and publisher
/// (sealing and signing as the agent does), multiplexed over a few NATS connections, each
/// pulling its own durable command consumer as the agent does.
/// </summary>
public static class Simulator
{
private static long published, presence, inventory, events, outputs, states, commands, refused, settingsOk, settingsFailed, bound, errors;
private static long settingsFirstOk, settingsRetries, upgrades, upgradeDuplicates;
private static readonly ConcurrentQueue<int> SettingsLatencies = new();
private static bool retrySettings;
/// <summary>The p-th percentile (0–100) of the settings replies' times so far, in ms.</summary>
private static long Percentile(int[] sorted, double p) => sorted.Length == 0 ? 0 : sorted[Math.Min(sorted.Length - 1, (int)(sorted.Length * p / 100))];
public static async Task<int> RunAsync(Options options)
{
var credentials = JsonSerializer.Deserialize<List<DeviceCredentials>>(await File.ReadAllTextAsync(options.Text("file")!), JsonDefaults.Options)!;
var connectionCount = options.Int("connections", 8);
var lines = options.Int("lines", 20);
retrySettings = options.Bool("settings-retry", false);
var ramp = TimeSpan.FromSeconds(options.Int("ramp", 20));
var connections = new List<NatsConnection>();
for (var i = 0; i < connectionCount; i++)
{
var connection = new NatsConnection(NatsOpts.Default with
{
Url = Program.NatsUrl,
Name = $"loadbench-agents-{i}",
MaxReconnectRetry = -1,
SubPendingChannelCapacity = 65536,
});
await connection.ConnectAsync();
connections.Add(connection);
}
var template = new InventoryCollector(new FixedIdentity()).Collect("template");
Console.WriteLine($"inventory report {JsonDefaults.SerializeToUtf8Bytes(new InventoryPayload(template)).Length} bytes");
var seen = new ReplayWindow(10_000_000);
var devices = credentials.Select((c, i) => new SimDevice(c, connections[i % connections.Count], seen, template, lines)).ToList();
var stop = new CancellationTokenSource();
var tasks = devices.Select(d => Task.Run(() => d.RunAsync(ramp, stop.Token))).ToList();
_ = Task.Run(async () =>
{
while (!stop.IsCancellationRequested)
{
await Task.Delay(2000);
var latencies = SettingsLatencies.ToArray();
Array.Sort(latencies);
Console.WriteLine(
$"STAT settingsFirstOk={Interlocked.Read(ref settingsFirstOk)} settingsRetries={Interlocked.Read(ref settingsRetries)} " +
$"settingsP50={Percentile(latencies, 50)} settingsP95={Percentile(latencies, 95)} settingsMax={(latencies.Length == 0 ? 0 : latencies[^1])} " +
$"upgrades={Interlocked.Read(ref upgrades)} upgradeDuplicates={Interlocked.Read(ref upgradeDuplicates)}");
Console.WriteLine(
$"STAT published={Interlocked.Read(ref published)} presence={Interlocked.Read(ref presence)} inventory={Interlocked.Read(ref inventory)} " +
$"events={Interlocked.Read(ref events)} outputs={Interlocked.Read(ref outputs)} states={Interlocked.Read(ref states)} " +
$"commands={Interlocked.Read(ref commands)} refused={Interlocked.Read(ref refused)} settingsOk={Interlocked.Read(ref settingsOk)} " +
$"settingsFailed={Interlocked.Read(ref settingsFailed)} bound={Interlocked.Read(ref bound)} errors={Interlocked.Read(ref errors)}");
}
});
await Task.WhenAll(tasks);
return 0;
}
private sealed class FixedIdentity : IHardwareIdentitySource
{
public HardwareIdentity Read() => new("uuid", "bios", "board", "ident", "Contoso", "Model");
}
private sealed class MemoryStore(AgentCredentials credentials) : ICredentialStore
{
private AgentCredentials? current = credentials;
public AgentCredentials? Load() => current;
public void Save(AgentCredentials value) => current = value;
public void Delete() => current = null;
}
private sealed class SimDevice
{
private readonly DeviceCredentials credentials;
private readonly NatsConnection connection;
private readonly AgentMessageSecurity security;
private readonly AgentPublisher publisher;
private readonly InventoryReport report;
private readonly int lines;
private readonly string hostname;
private readonly DateTimeOffset started = DateTimeOffset.UtcNow;
private readonly HashSet<string> upgradeVersions = new(StringComparer.Ordinal);
public SimDevice(DeviceCredentials credentials, NatsConnection connection, ISeenMessageStore seen, InventoryReport template, int lines)
{
this.credentials = credentials;
this.connection = connection;
this.lines = lines;
hostname = "PC-" + credentials.ClientId[2..10];
security = new AgentMessageSecurity(
new MemoryStore(new AgentCredentials(
AgentCredentials.CurrentVersion, credentials.ClientId, Guid.NewGuid(), "jwt", credentials.NkeySeed, null,
credentials.MessageKey, credentials.ServerKeys)),
seen, NullLogger.Instance);
security.Reload();
publisher = new AgentPublisher(connection, credentials.ClientId, security);
report = template with
{
Identity = template.Identity with { ClientId = credentials.ClientId, Hostname = hostname },
MessageSecurity = security.Report(),
};
}
public async Task RunAsync(TimeSpan ramp, CancellationToken cancellationToken)
{
var random = new Random(credentials.ClientId.GetHashCode());
await Task.Delay(TimeSpan.FromMilliseconds(random.NextDouble() * ramp.TotalMilliseconds), cancellationToken);
_ = Task.Run(() => CommandsAsync(cancellationToken), cancellationToken);
await Guard(() => PublishAsync(NatsSubjects.AgentEvent(credentials.ClientId), MessageTypes.AgentEvent,
new AgentEventPayload(credentials.ClientId, AgentEventKinds.Ready, "command consumer bound", DateTimeOffset.UtcNow), ref events));
_ = Task.Run(() => PullSettingsAsync(cancellationToken), cancellationToken);
await Guard(() => PublishAsync(NatsSubjects.AgentInventory(credentials.ClientId), MessageTypes.AgentInventory, new InventoryPayload(report), ref inventory));
var nextInventory = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(random.NextDouble() * 900);
while (!cancellationToken.IsCancellationRequested)
{
await Guard(() => PublishAsync(NatsSubjects.AgentPresence(credentials.ClientId), MessageTypes.AgentPresence,
new PresencePayload(credentials.ClientId, hostname, "2026.11.06.0000", "online", DateTimeOffset.UtcNow,
(long)(DateTimeOffset.UtcNow - started).TotalSeconds, 0, "win-x64", "msi"), ref presence));
if (DateTimeOffset.UtcNow >= nextInventory)
{
nextInventory += TimeSpan.FromSeconds(900);
await Guard(() => PublishAsync(NatsSubjects.AgentInventory(credentials.ClientId), MessageTypes.AgentInventory,
new InventoryPayload(report with { CollectedUtc = DateTimeOffset.UtcNow }), ref inventory));
}
await Task.Delay(ProtocolDefaults.PresenceInterval, cancellationToken);
}
}
private static async Task Guard(Func<ValueTask> action)
{
try
{
await action();
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception)
{
Interlocked.Increment(ref errors);
}
}
private ValueTask PublishAsync<T>(string subject, string type, T payload, ref long counter)
{
Interlocked.Increment(ref counter);
Interlocked.Increment(ref published);
return publisher.PublishAsync(subject, type, payload);
}
/// <summary>
/// As AgentSettingsClient pulls: sealed request on the event subject, sealed reply. With
/// --settings-retry, a request that timed out is tried again as the agent does; without it,
/// one try (what reaches the device at its connect).
/// </summary>
/// <summary>The agent's own pause before each retry (a few, so a run that does not answer still ends).</summary>
private static TimeSpan? SettingsRetryDelay(int attempt)
=> attempt < 6 ? AgentSettingsClient.RetryDelay(attempt, Random.Shared) : null;
private async Task PullSettingsAsync(CancellationToken cancellationToken)
{
var subject = NatsSubjects.AgentEvent(credentials.ClientId);
for (var attempt = 0; ; attempt++)
{
var envelope = MessageEnvelope.Create(
MessageTypes.AgentSettingsRequest, publisher.Source, new AgentSettingsRequestPayload(credentials.ClientId, 0));
var watch = Stopwatch.StartNew();
byte[]? data = null;
try
{
Interlocked.Increment(ref events);
Interlocked.Increment(ref published);
var answer = await connection.RequestAsync<byte[], byte[]>(
subject, AgentPublisher.Seal(security, subject, envelope.MessageType, envelope.ToUtf8Bytes()),
replyOpts: new NatsSubOpts { Timeout = AgentSettingsClient.RequestTimeout }, cancellationToken: cancellationToken);
data = answer.Data is { Length: > 0 } d ? security.Open(answer.Subject, d, MessageReceipt.Request)?.Payload : null;
}
catch (Exception) when (!cancellationToken.IsCancellationRequested)
{
// Timed out: no answer in time.
}
if (data is not null)
{
SettingsLatencies.Enqueue((int)watch.ElapsedMilliseconds);
Interlocked.Increment(ref settingsOk);
if (attempt == 0)
{
Interlocked.Increment(ref settingsFirstOk);
}
return;
}
if (!retrySettings || SettingsRetryDelay(attempt) is not { } delay)
{
Interlocked.Increment(ref settingsFailed);
return;
}
Interlocked.Increment(ref settingsRetries);
await Task.Delay(delay, cancellationToken);
}
}
private async Task CommandsAsync(CancellationToken cancellationToken)
{
var js = new NatsJSContext(connection);
while (!cancellationToken.IsCancellationRequested)
{
try
{
var consumer = await js.GetConsumerAsync(
JetStreamDefaults.CommandStreamName, JetStreamDefaults.AgentCommandConsumerName(credentials.ClientId), cancellationToken);
Interlocked.Increment(ref bound);
await foreach (var message in consumer.ConsumeAsync<byte[]>(opts: new NatsJSConsumeOpts { MaxMsgs = 10 }, cancellationToken: cancellationToken))
{
var opened = message.Data is { Length: > 0 } data
? security.Open(message.Subject, data, MessageReceipt.Durable, message.Metadata?.Timestamp, out _)
: null;
await message.AckAsync(cancellationToken: cancellationToken);
if (opened is null)
{
Interlocked.Increment(ref refused);
continue;
}
security.Remember(opened);
var commandEnvelope = MessageEnvelope.FromUtf8Bytes(opened.Payload);
if (commandEnvelope?.MessageType == MessageTypes.CommandAgentUpgrade)
{
// Taken and kept: no download, no report (the row stays Sent).
var upgrade = commandEnvelope.GetPayload<AgentUpgradeCommandPayload>();
Interlocked.Increment(ref upgrades);
if (upgrade is not null && !upgradeVersions.Add(upgrade.Version))
{
Interlocked.Increment(ref upgradeDuplicates);
}
continue;
}
var command = commandEnvelope?.GetPayload<ScriptCommandPayload>();
if (command is null)
{
continue;
}
Interlocked.Increment(ref commands);
_ = Task.Run(() => RunJobAsync(command, cancellationToken), cancellationToken);
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
return;
}
catch (Exception)
{
Interlocked.Increment(ref errors);
await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
}
}
}
private async Task RunJobAsync(ScriptCommandPayload command, CancellationToken cancellationToken)
{
var target = new EndpointRef(EndpointKinds.Job, command.JobId.ToString("D"));
await Guard(() => PublishAsync(NatsSubjects.JobState(command.JobId), MessageTypes.JobState,
new JobStatePayload(command.JobId, credentials.ClientId, JobState.Running, null, DateTimeOffset.UtcNow, null), ref states));
for (var i = 1; i <= lines; i++)
{
await Guard(() => PublishAsync(NatsSubjects.JobOutput(command.JobId), MessageTypes.JobOutput,
new JobOutputPayload(command.JobId, credentials.ClientId, i, OutputStreamKind.Output, DateTimeOffset.UtcNow,
$"line {i} of the load burst, about eighty characters of output each time"), ref outputs));
}
await Guard(() => PublishAsync(NatsSubjects.JobState(command.JobId), MessageTypes.JobState,
new JobStatePayload(command.JobId, credentials.ClientId, JobState.Succeeded, 0, DateTimeOffset.UtcNow, "done"), ref states));
}
}
}
/// <summary>Per-message cost of sealing and opening, against the real server security (one thread).</summary>
public static class Micro
{
private const int Warm = 500;
public static async Task<int> RunAsync(Options options)
{
var count = options.Int("count", 20000);
var root = Directory.CreateTempSubdirectory("loadbench-micro-").FullName;
await using var backend = new BenchBackend(root);
using (backend.CreateClient())
{
}
var security = backend.Services.GetRequiredService<ServerMessageSecurity>();
await security.Keys.EnsureLoadedAsync(CancellationToken.None);
const int Devices = 200;
var sealedDevices = await backend.EnrollAsync(Devices, seals: true, "mb");
var plainDevices = await backend.EnrollAsync(Devices, seals: false, "mp");
var agents = sealedDevices.Select(c => Agent(c)).ToList();
var plainAgents = plainDevices.Select(c => Agent(c)).ToList();
Bench.Log($"per message, {count} each, one thread (µs, KB allocated)");
var presence = (AgentMessageSecurity a, string id) => MessageEnvelope.Create(
MessageTypes.AgentPresence, new EndpointRef(EndpointKinds.Agent, id),
new PresencePayload(id, "PC-000001", "2026.11.06.0000", "online", DateTimeOffset.UtcNow, 100, 0, "win-x64", "msi")).ToUtf8Bytes();
var output = (AgentMessageSecurity a, string id) => MessageEnvelope.Create(
MessageTypes.JobOutput, new EndpointRef(EndpointKinds.Agent, id),
new JobOutputPayload(Guid.NewGuid(), id, 1, OutputStreamKind.Output, DateTimeOffset.UtcNow, "line 1 of the load burst, about eighty characters of output each time")).ToUtf8Bytes();
foreach (var (name, make) in new[] { ("presence", presence), ("job output", output) })
{
// Agent side: seal (encrypt + sign).
var plain = Enumerable.Range(0, count + Warm).Select(i => make(agents[i % Devices].Security, agents[i % Devices].Id)).ToList();
var wire = new byte[count + Warm][];
Measure($"agent seal {name}", count, i =>
{
var a = agents[i % Devices];
wire[i] = a.Security.Seal($"agent.{a.Id}.presence", MessageTypes.AgentPresence, plain[i]);
return ValueTask.CompletedTask;
});
Bench.Log($" {name}: v1 {plain[0].Length} bytes, sealed {wire[0].Length} bytes");
// Server side: open (verify, check, decrypt, replay window) sealed vs the v1 acceptance check.
await MeasureAsync($"server open sealed {name}", count, async i =>
{
var a = agents[i % Devices];
var result = await security.OpenFromAgentAsync($"agent.{a.Id}.presence", wire[i], a.Id, ServerReceipt.Live);
if (result.Message is null)
{
throw new InvalidOperationException("refused");
}
});
var plainWire = Enumerable.Range(0, count + Warm).Select(i => make(plainAgents[i % Devices].Security, plainAgents[i % Devices].Id)).ToList();
await MeasureAsync($"server open v1 {name}", count, async i =>
{
var a = plainAgents[i % Devices];
var result = await security.OpenFromAgentAsync($"agent.{a.Id}.presence", plainWire[i], a.Id, ServerReceipt.Live);
if (result.Message is null)
{
throw new InvalidOperationException("refused");
}
});
Measure($"parse envelope {name}", count, i =>
{
_ = MessageEnvelope.FromUtf8Bytes(plainWire[i]);
return ValueTask.CompletedTask;
});
}
// Where an open's time goes: the Ed25519 signature (NATS.NKeys), the key agreement (cached), the AEAD.
{
var a = agents[0];
var bytes = a.Security.Seal($"agent.{a.Id}.presence", MessageTypes.AgentPresence, presence(a.Security, a.Id));
var envelope = MessageSealing.Parse(bytes)!;
var nkeyPublic = KeyPair.FromSeed(sealedDevices[0].NkeySeed).GetPublicKey();
var secret = security.Keys.Current.Secret(MessageKeys.PublicKeyOf(MessageKeys.FromPrivateKey(sealedDevices[0].MessageKey!)));
using var signer = new NkeySigner(sealedDevices[0].NkeySeed);
var input = MessageSealing.SignatureInput(envelope);
Measure(" parse sealed envelope (JSON)", count, _ => { MessageSealing.Parse(bytes); return ValueTask.CompletedTask; });
Measure(" Ed25519 verify (NATS.NKeys)", count / 4, _ => { MessageSealing.VerifySignature(envelope, nkeyPublic); return ValueTask.CompletedTask; });
Measure(" Ed25519 sign (NATS.NKeys)", count / 4, _ => { signer.Sign(input); return ValueTask.CompletedTask; });
Measure(" HKDF + AES-GCM decrypt", count, _ => { MessageSealing.TryDecrypt(envelope, secret, MessageDirections.AgentToServer); return ValueTask.CompletedTask; });
Measure(" ECDH P-256 agreement (uncached)", count / 20, _ =>
{
MessageKeys.Agree(MessageKeys.FromPrivateKey(sealedDevices[1].MessageKey!), security.Keys.Current.EcdhPublic);
return ValueTask.CompletedTask;
});
}
// Server side: a script command sealed for a device (fresh keys from the database, as the dispatcher does).
var commandPlain = MessageEnvelope.Create(MessageTypes.CommandScript, new EndpointRef(EndpointKinds.Backend, "backend"),
new ScriptCommandPayload(Guid.NewGuid(), Guid.NewGuid(), "Load burst", "1..20 | ForEach-Object { Write-Output \"line $_\" }", 3600)).ToUtf8Bytes();
await MeasureAsync("server seal command (cached keys)", count, async i =>
{
var a = agents[i % Devices];
await security.SealForAgentAsync(a.Id, $"agent.{a.Id}.command.script", MessageTypes.CommandScript, commandPlain, MessageSecurityDefaults.DurableCommandLifetime);
});
await MeasureAsync("server seal command (fresh: DB read)", count / 4, async i =>
{
var a = agents[i % Devices];
await security.SealForAgentAsync(a.Id, $"agent.{a.Id}.command.script", MessageTypes.CommandScript, commandPlain, MessageSecurityDefaults.DurableCommandLifetime, fresh: true);
});
await MeasureAsync("server v1 command (fresh: DB read)", count / 4, async i =>
{
var a = plainAgents[i % Devices];
await security.SealForAgentAsync(a.Id, $"agent.{a.Id}.command.script", MessageTypes.CommandScript, commandPlain, MessageSecurityDefaults.DurableCommandLifetime, fresh: true);
});
// The job runner's fan-out by itself (nothing answers, nothing else writes): per device.
foreach (var prefix in new[] { "mb", "mp" })
{
using var scope = backend.Services.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
var runner = scope.ServiceProvider.GetRequiredService<JobRunner>();
var script = new ScriptEntity
{
Id = Guid.NewGuid(), Name = "Micro", Language = "powershell", Content = "Write-Output 1",
CreatedUtc = DateTimeOffset.UtcNow, UpdatedUtc = DateTimeOffset.UtcNow,
};
db.Scripts.Add(script);
await db.SaveChangesAsync();
var targets = await db.Devices.AsNoTracking().Where(d => d.ClientId.StartsWith(prefix)).ToListAsync();
var runId = Guid.NewGuid();
await runner.RecordRunAsync(runId, script, "micro", "manual", "all", [], CancellationToken.None);
var prepared = await runner.PrepareAsync(script, CancellationToken.None);
var watch = Stopwatch.StartNew();
await runner.StartRunAsync(runId, prepared, null, targets, "micro", 0, 0, d => d.Hostname, CancellationToken.None);
Bench.Log($" job runner fan-out ({(prefix == "mb" ? "sealed" : "v1")}) {watch.Elapsed.TotalMilliseconds / targets.Count,8:0.0} ms per device ({targets.Count} devices)");
// Its two halves, per device: the job row (and history pruning), then the sealed publish and its state.
double create = 0, dispatch = 0, publish = 0, prune = 0;
var js = new NatsJSContext(backend.Services.GetRequiredService<NatsHub>().Connection!);
foreach (var device in targets)
{
var t = Stopwatch.StartNew();
var job = await runner.CreateJobAsync(script, device.ClientId, "micro", CancellationToken.None, runId);
create += t.Elapsed.TotalMilliseconds;
t.Restart();
await runner.DispatchAsync(prepared, job, null, device.Hostname, CancellationToken.None);
dispatch += t.Elapsed.TotalMilliseconds;
t.Restart();
(await js.PublishAsync($"agent.{device.ClientId}.command.inventory", commandPlain)).EnsureSuccess();
publish += t.Elapsed.TotalMilliseconds;
t.Restart();
await runner.PruneHistoryAsync(device.ClientId, CancellationToken.None);
prune += t.Elapsed.TotalMilliseconds;
}
Bench.Log($" create job {create / targets.Count:0.0} ms, dispatch {dispatch / targets.Count:0.0} ms, a bare JetStream publish {publish / targets.Count:0.0} ms, history pruning alone {prune / targets.Count:0.0} ms (per device)");
}
var provisioner = backend.Services.GetRequiredService<JetStreamProvisioner>();
foreach (var c in sealedDevices.Concat(plainDevices))
{
await provisioner.RemoveAgentCommandsAsync(c.ClientId, CancellationToken.None);
}
return 0;
}
private static (AgentMessageSecurity Security, string Id) Agent(DeviceCredentials c)
{
var security = new AgentMessageSecurity(
new StaticStore(new AgentCredentials(AgentCredentials.CurrentVersion, c.ClientId, Guid.NewGuid(), "jwt", c.NkeySeed, null, c.MessageKey, c.ServerKeys)),
new ReplayWindow(), NullLogger.Instance);
security.Reload();
return (security, c.ClientId);
}
private sealed class StaticStore(AgentCredentials credentials) : ICredentialStore
{
public AgentCredentials? Load() => credentials;
public void Save(AgentCredentials value)
{
}
public void Delete()
{
}
}
private static void Measure(string name, int count, Func<int, ValueTask> action)
=> MeasureAsync(name, count, i => action(i).AsTask()).GetAwaiter().GetResult();
private static async Task MeasureAsync(string name, int count, Func<int, Task> action)
{
// Warm-up on its own indices (count..count+Warm), so the measured messages are new.
for (var i = 0; i < Warm; i++)
{
await action(count + i);
}
GC.Collect();
var allocated = GC.GetTotalAllocatedBytes(precise: true);
var cpu = Process.GetCurrentProcess().TotalProcessorTime;
var watch = Stopwatch.StartNew();
for (var i = 0; i < count; i++)
{
await action(i);
}
watch.Stop();
var used = Process.GetCurrentProcess().TotalProcessorTime - cpu;
Bench.Log($" {name,-40} {watch.Elapsed.TotalMicroseconds / count,8:0.0} µs wall {used.TotalMicroseconds / count,8:0.0} µs cpu {(GC.GetTotalAllocatedBytes(precise: true) - allocated) / 1024.0 / count,6:0.0} KB");
}
}
/// <summary>A database of its own on the PostgreSQL server the PG* variables name, dropped afterwards.</summary>
public sealed class ThrowawayPostgres : IAsyncDisposable
{
private readonly string admin;
private ThrowawayPostgres(string admin, string name, string connectionString)
{
this.admin = admin;
Name = name;
ConnectionString = connectionString;
}
public string Name { get; }
public string ConnectionString { get; }
public static async Task<ThrowawayPostgres> CreateAsync()
{
var builder = new Npgsql.NpgsqlConnectionStringBuilder
{
Host = Environment.GetEnvironmentVariable("PGHOST") ?? "127.0.0.1",
Port = int.TryParse(Environment.GetEnvironmentVariable("PGPORT"), out var port) ? port : 5432,
Username = Environment.GetEnvironmentVariable("PGUSER") ?? "postgres",
Password = Environment.GetEnvironmentVariable("PGPASSWORD"),
Database = "postgres",
MaxPoolSize = 200,
};
var admin = builder.ConnectionString;
var name = "loadbench_" + Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(6));
await using (var connection = new Npgsql.NpgsqlConnection(admin))
{
await connection.OpenAsync();
await using var command = new Npgsql.NpgsqlCommand($"CREATE DATABASE {name}", connection);
await command.ExecuteNonQueryAsync();
}
builder.Database = name;
return new ThrowawayPostgres(admin, name, builder.ConnectionString);
}
public async ValueTask DisposeAsync()
{
Npgsql.NpgsqlConnection.ClearAllPools();
await using var connection = new Npgsql.NpgsqlConnection(admin);
await connection.OpenAsync();
await using var command = new Npgsql.NpgsqlCommand($"DROP DATABASE IF EXISTS {Name} WITH (FORCE)", connection);
await command.ExecuteNonQueryAsync();
Bench.Log($"dropped database {Name}");
}
}