23ddf5c665
- Connect storm: settings answers counted by first try and retries, with their reply times (p50, p95, max); the phase also waits for every device's ready event to be stored. --settings-retry retries a missed pull as the agent does (default: one try, what reaches a device at once). - --upgrade: an MSI made up for the catalog, every device queued for it, the scheduler running as with SCHEDULE_WORKER on and StartWithinMinutes from --start-within (default 1); the simulated agents count upgrade commands and a second one for the same version, and each pass's sends are timed from the pass's start to the last publish returned. - --burst false skips the job burst. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1285 lines
63 KiB
C#
1285 lines
63 KiB
C#
// Load benchmark: the real backend (hosted in this process against a live NATS server, SQLite)
|
||
// and N simulated agents in a child process, sealed (envelope v2) or unsealed (v1).
|
||
//
|
||
// LoadBench run --devices 5000 [--sealed true] [--connections 16] [--steady 180] [--lines 20] [--nats-pid 1234] [--out results.jsonl] [--provider sqlite|postgres]
|
||
// [--burst true|false] [--upgrade true|false] [--start-within 1] [--settings-retry true|false]
|
||
// LoadBench micro [--count 20000]
|
||
//
|
||
// Phases: the connect storm (ready event, settings pull, inventory and presence from every device
|
||
// within 20 s; the pull is tried once unless --settings-retry, which retries as the agent does),
|
||
// steady state, with --upgrade every device upgraded through the scheduler's paced batches
|
||
// (SCHEDULE_WORKER on, StartWithinMinutes --start-within; each pass's sends timed), and unless
|
||
// --burst false a script to every device through the job runner.
|
||
//
|
||
// NATS_URL picks the server (default nats://127.0.0.1:4222; no auth, JetStream on). Not part of CI.
|
||
// --provider postgres runs on a throwaway database (created and dropped) on the server PGHOST, PGPORT,
|
||
// PGUSER and PGPASSWORD name.
|
||
// The JetStream streams are kept in memory storage, as the integration tests keep them on a shared
|
||
// server (production uses file storage). "cleanup" removes a stopped run's consumers.
|
||
using System.Collections.Concurrent;
|
||
using System.Diagnostics;
|
||
using System.Globalization;
|
||
using System.Net.Http.Json;
|
||
using System.Security.Cryptography;
|
||
using System.Text;
|
||
using System.Text.Json;
|
||
using Agent.Enrollment;
|
||
using Agent.Identity;
|
||
using Agent.Inventory;
|
||
using Agent.Messaging;
|
||
using Backend.Data;
|
||
using Backend.Jobs;
|
||
using Backend.Nats;
|
||
using Backend.Security.Messages;
|
||
using Backend.Upgrades;
|
||
using Microsoft.AspNetCore.Hosting;
|
||
using Microsoft.AspNetCore.Mvc.Testing;
|
||
using Microsoft.AspNetCore.TestHost;
|
||
using Microsoft.EntityFrameworkCore;
|
||
using Microsoft.Extensions.DependencyInjection;
|
||
using Microsoft.Extensions.Logging;
|
||
using Microsoft.Extensions.Logging.Abstractions;
|
||
using NATS.Client.Core;
|
||
using NATS.Client.JetStream;
|
||
using NATS.NKeys;
|
||
using Shared.Contracts.Api;
|
||
using Shared.Contracts.Messages;
|
||
using Shared.Contracts.Models;
|
||
using Shared.Contracts.Serialization;
|
||
using Shared.Infrastructure.Constants;
|
||
using Shared.Infrastructure.Nats;
|
||
using Shared.Infrastructure.Security;
|
||
using Shared.Infrastructure.Security.Messaging;
|
||
|
||
namespace LoadBench;
|
||
|
||
public static class Program
|
||
{
|
||
public static string NatsUrl => Environment.GetEnvironmentVariable("NATS_URL") ?? "nats://127.0.0.1:4222";
|
||
|
||
public static async Task<int> Main(string[] args)
|
||
{
|
||
var options = Options.Parse(args);
|
||
return args.FirstOrDefault() switch
|
||
{
|
||
"run" => await Bench.RunAsync(options),
|
||
"agents" => await Simulator.RunAsync(options),
|
||
"micro" => await Micro.RunAsync(options),
|
||
"cleanup" => await CleanupAsync(),
|
||
_ => Usage(),
|
||
};
|
||
}
|
||
|
||
/// <summary>Deletes the consumers (and queued commands) a run that did not finish left on the shared NATS.</summary>
|
||
private static async Task<int> CleanupAsync()
|
||
{
|
||
await using var connection = new NatsConnection(NatsOpts.Default with { Url = NatsUrl });
|
||
var js = new NatsJSContext(connection);
|
||
var stream = await js.GetStreamAsync(JetStreamDefaults.CommandStreamName);
|
||
var removed = 0;
|
||
await foreach (var name in stream.ListConsumerNamesAsync())
|
||
{
|
||
var clientId = name.Replace("agent-commands-", string.Empty, StringComparison.Ordinal);
|
||
if (name.StartsWith("agent-commands-", StringComparison.Ordinal) && clientId.Length == 22 && clientId[0] is 'l' or 'm')
|
||
{
|
||
await stream.DeleteConsumerAsync(name);
|
||
await stream.PurgeAsync(new NATS.Client.JetStream.Models.StreamPurgeRequest { Filter = NatsSubjects.AgentCommandWildcard(clientId) });
|
||
removed++;
|
||
}
|
||
}
|
||
|
||
Console.WriteLine($"removed {removed} benchmark consumers");
|
||
|
||
// The backend provisions file storage (as in production, so a run measures that); the
|
||
// shared test server is left with the memory streams the integration tests use.
|
||
foreach (var (name, subjects, maxAge) in new[]
|
||
{
|
||
(JetStreamDefaults.CommandStreamName, JetStreamDefaults.CommandStreamSubjects, JetStreamDefaults.CommandMaxAge),
|
||
(JetStreamDefaults.JobEventsStreamName, JetStreamDefaults.JobEventsStreamSubjects, JetStreamDefaults.JobEventsMaxAge),
|
||
})
|
||
{
|
||
try
|
||
{
|
||
var existing = await js.GetStreamAsync(name);
|
||
if (existing.Info.Config.Storage == NATS.Client.JetStream.Models.StreamConfigStorage.File)
|
||
{
|
||
await js.DeleteStreamAsync(name);
|
||
await js.CreateStreamAsync(new NATS.Client.JetStream.Models.StreamConfig(name, subjects)
|
||
{
|
||
Storage = NATS.Client.JetStream.Models.StreamConfigStorage.Memory,
|
||
MaxAge = maxAge,
|
||
});
|
||
Console.WriteLine($"{name}: file storage replaced by memory storage");
|
||
}
|
||
}
|
||
catch (NatsJSApiException exception) when (exception.Error.Code == 404)
|
||
{
|
||
// Not there.
|
||
}
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
private static int Usage()
|
||
{
|
||
Console.Error.WriteLine("LoadBench run --devices N [--sealed true|false] [--connections K] [--steady S] [--lines L] [--nats-pid P] [--out F] | micro [--count N]");
|
||
return 2;
|
||
}
|
||
}
|
||
|
||
public sealed class Options(Dictionary<string, string> values)
|
||
{
|
||
public static Options Parse(string[] args)
|
||
{
|
||
var values = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||
for (var i = 1; i < args.Length - 1; i += 2)
|
||
{
|
||
values[args[i].TrimStart('-')] = args[i + 1];
|
||
}
|
||
|
||
return new Options(values);
|
||
}
|
||
|
||
public int Int(string name, int fallback) => values.TryGetValue(name, out var v) ? int.Parse(v, CultureInfo.InvariantCulture) : fallback;
|
||
|
||
public bool Bool(string name, bool fallback) => values.TryGetValue(name, out var v) ? bool.Parse(v) : fallback;
|
||
|
||
public string? Text(string name) => values.TryGetValue(name, out var v) ? v : null;
|
||
}
|
||
|
||
/// <summary>One simulated device's identity as the child process gets it.</summary>
|
||
public sealed record DeviceCredentials(string ClientId, string NkeySeed, string? MessageKey, ServerMessageKeys? ServerKeys);
|
||
|
||
/// <summary>When each upgrade command's publish returned (the scheduler's sends, timed as they go out).</summary>
|
||
public sealed class UpgradeSendLog
|
||
{
|
||
public ConcurrentQueue<(Guid UpgradeId, string ClientId, DateTimeOffset DoneUtc, bool Sent)> Sends { get; } = new();
|
||
}
|
||
|
||
/// <summary>The real sender, each send recorded in <see cref="UpgradeSendLog"/>.</summary>
|
||
public sealed class TimedUpgradeSender(IAgentUpgradeSender inner, UpgradeSendLog log) : IAgentUpgradeSender
|
||
{
|
||
public async Task<bool> DispatchAgentUpgradeAsync(string clientId, AgentUpgradeCommandPayload command, CancellationToken cancellationToken)
|
||
{
|
||
var sent = await inner.DispatchAgentUpgradeAsync(clientId, command, cancellationToken);
|
||
log.Sends.Enqueue((command.UpgradeId, clientId, DateTimeOffset.UtcNow, sent));
|
||
return sent;
|
||
}
|
||
}
|
||
|
||
/// <summary>The backend, hosted in this process against the live NATS (as the integration tests host it).</summary>
|
||
public sealed class BenchBackend(string root, string? postgres = null, bool scheduleWorker = false) : WebApplicationFactory<global::Program>
|
||
{
|
||
private readonly KeyPair account = KeyPair.CreatePair(PrefixByte.Account);
|
||
|
||
public UpgradeSendLog UpgradeSends { get; } = new();
|
||
|
||
protected override void ConfigureWebHost(IWebHostBuilder builder)
|
||
{
|
||
builder.ConfigureTestServices(services =>
|
||
{
|
||
// The upgrade scheduler's sends, timed (the scheduler runs only with SCHEDULE_WORKER on).
|
||
services.AddSingleton<IAgentUpgradeSender>(sp => new TimedUpgradeSender(sp.GetRequiredService<CommandDispatcher>(), UpgradeSends));
|
||
});
|
||
|
||
builder.UseSetting("DATABASE_PATH", Path.Combine(root, "app.db"));
|
||
if (postgres is not null)
|
||
{
|
||
builder.UseSetting("DATABASE_PROVIDER", "postgres");
|
||
builder.UseSetting("DATABASE_CONNECTION_STRING", postgres);
|
||
}
|
||
|
||
builder.UseSetting("DATA_PATH", Path.Combine(root, "data"));
|
||
builder.UseSetting("DATAPROTECTION_PATH", Path.Combine(root, "keys"));
|
||
builder.UseSetting("LOG_DIRECTORY", Path.Combine(root, "logs"));
|
||
builder.UseSetting("LOG_LEVEL", Environment.GetEnvironmentVariable("BENCH_LOG_LEVEL") ?? "Warning");
|
||
builder.UseSetting("AGENT_CATALOG_SYNC", "0");
|
||
builder.UseSetting("S3_EMBEDDED", "0");
|
||
builder.UseSetting("NATS_ACCOUNT_SIGNING_SEED", account.GetSeed());
|
||
builder.UseSetting("NATS_ACCOUNT_PUBLIC_KEY", account.GetPublicKey());
|
||
builder.UseSetting("NATS_URL", Program.NatsUrl);
|
||
builder.UseSetting("RATE_LIMIT_PER_MINUTE", "100000000");
|
||
builder.UseSetting("SCHEDULE_WORKER", scheduleWorker ? "1" : "0");
|
||
builder.UseSetting("TLS_SELF_SIGNED", "0");
|
||
builder.UseEnvironment("Production");
|
||
}
|
||
|
||
public async Task<T> WithDbAsync<T>(Func<AppDbContext, Task<T>> action)
|
||
{
|
||
using var scope = Services.CreateScope();
|
||
return await action(scope.ServiceProvider.GetRequiredService<AppDbContext>());
|
||
}
|
||
|
||
public async Task<string> TokenAsync()
|
||
{
|
||
var token = TokenGenerator.CreateUrlSafeToken(32);
|
||
await WithDbAsync(async db =>
|
||
{
|
||
db.BootstrapTokens.Add(new BootstrapTokenEntity
|
||
{
|
||
Id = Guid.NewGuid(),
|
||
TokenHash = TokenHashing.Sha256Hex(token),
|
||
CreatedUtc = DateTimeOffset.UtcNow,
|
||
ExpiresUtc = DateTimeOffset.UtcNow + TimeSpan.FromDays(1),
|
||
MaxUses = 0,
|
||
});
|
||
return await db.SaveChangesAsync();
|
||
});
|
||
return token;
|
||
}
|
||
|
||
/// <summary>Enrolls devices through the real bootstrap endpoint, <paramref name="parallel"/> at a time.</summary>
|
||
public async Task<List<DeviceCredentials>> EnrollAsync(int count, bool seals, string prefix, int parallel = 16)
|
||
{
|
||
var token = await TokenAsync();
|
||
var result = new ConcurrentBag<DeviceCredentials>();
|
||
using var http = CreateClient();
|
||
http.Timeout = TimeSpan.FromMinutes(5);
|
||
await Parallel.ForEachAsync(Enumerable.Range(0, count), new ParallelOptions { MaxDegreeOfParallelism = parallel }, async (i, ct) =>
|
||
{
|
||
var clientId = prefix + Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(10));
|
||
using var nkey = KeyPair.CreatePair(PrefixByte.User);
|
||
using var ecdh = MessageKeys.Create();
|
||
var response = await http.PostAsJsonAsync(ApiRoutes.Bootstrap, new BootstrapRequest(
|
||
token, clientId, nkey.GetPublicKey(), "2026.11.06.0000",
|
||
new EnrollDeviceInfo("PC-" + clientId[2..10], "Contoso", "Model", "Windows 11", "10.0.26200"),
|
||
seals ? MessageKeys.PublicKeyOf(ecdh) : null), JsonDefaults.Options, ct);
|
||
response.EnsureSuccessStatusCode();
|
||
var answer = (await response.Content.ReadFromJsonAsync<BootstrapResponse>(JsonDefaults.Options, ct))!;
|
||
result.Add(new DeviceCredentials(clientId, nkey.GetSeed(), seals ? MessageKeys.PrivateKeyOf(ecdh) : null, answer.ServerKeys));
|
||
});
|
||
return [.. result];
|
||
}
|
||
|
||
protected override void Dispose(bool disposing)
|
||
{
|
||
base.Dispose(disposing);
|
||
account.Dispose();
|
||
}
|
||
}
|
||
|
||
/// <summary>CPU and memory of one process between samples.</summary>
|
||
public sealed class ProcessMeter(Process process, string name)
|
||
{
|
||
private TimeSpan lastCpu = Cpu(process);
|
||
private long lastTicks = Stopwatch.GetTimestamp();
|
||
|
||
public string Name => name;
|
||
|
||
private static TimeSpan Cpu(Process p)
|
||
{
|
||
try
|
||
{
|
||
p.Refresh();
|
||
return p.TotalProcessorTime;
|
||
}
|
||
catch (Exception)
|
||
{
|
||
return TimeSpan.Zero;
|
||
}
|
||
}
|
||
|
||
/// <summary>Cores busy since the last call, working set and private bytes (MB).</summary>
|
||
public (double Cores, double WorkingSetMb, double PrivateMb) Sample()
|
||
{
|
||
var cpu = Cpu(process);
|
||
var now = Stopwatch.GetTimestamp();
|
||
var wall = Stopwatch.GetElapsedTime(lastTicks, now);
|
||
var cores = wall.TotalSeconds > 0 ? (cpu - lastCpu).TotalSeconds / wall.TotalSeconds : 0;
|
||
lastCpu = cpu;
|
||
lastTicks = now;
|
||
try
|
||
{
|
||
return (cores, process.WorkingSet64 / 1048576.0, process.PrivateMemorySize64 / 1048576.0);
|
||
}
|
||
catch (Exception)
|
||
{
|
||
return (cores, 0, 0);
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>The orchestrator: hosts the backend, enrolls, starts the simulator, measures the three phases.</summary>
|
||
public static class Bench
|
||
{
|
||
private sealed record PhaseResult(
|
||
string Phase, double Seconds, double ServerCores, double ServerPeakCores, double ServerWorkingSetMb, double ServerPrivateMb,
|
||
double AgentsCores, double NatsCores, double NatsWorkingSetMb, double MessagesPerSecond, long Dropped, string Detail);
|
||
|
||
public static async Task<int> RunAsync(Options options)
|
||
{
|
||
var devices = options.Int("devices", 1000);
|
||
var seals = options.Bool("sealed", true);
|
||
var connections = options.Int("connections", Math.Max(1, Math.Min(32, devices / 500)));
|
||
var steady = options.Int("steady", 180);
|
||
var lines = options.Int("lines", 20);
|
||
var outFile = options.Text("out") ?? Path.Combine(AppContext.BaseDirectory, "results.jsonl");
|
||
var root = Directory.CreateTempSubdirectory("loadbench-").FullName;
|
||
var provider = options.Text("provider") ?? "sqlite";
|
||
var upgrade = options.Bool("upgrade", false);
|
||
var jobBurst = options.Bool("burst", true);
|
||
var settingsRetry = options.Bool("settings-retry", false);
|
||
await using var database = provider == "postgres" ? await ThrowawayPostgres.CreateAsync() : null;
|
||
Log($"{devices} devices, {(seals ? "sealed (v2)" : "unsealed (v1)")}, {connections} NATS connections for the agents, {provider}{(database is null ? string.Empty : $" ({database.Name})")}, data in {root}");
|
||
|
||
Process? nats = options.Int("nats-pid", 0) is > 0 and var pid ? Process.GetProcessById(pid) : FindNats();
|
||
// The streams in memory storage, as the integration tests keep them on a shared server (a
|
||
// run that switched them to file storage had them deleted under it by a test run). The
|
||
// backend's own file-storage provisioning then only warns; its job event consumer is made here.
|
||
await using (var setup = new NatsConnection(NatsOpts.Default with { Url = Program.NatsUrl }))
|
||
{
|
||
var js = new NatsJSContext(setup);
|
||
foreach (var (name, subjects, maxAge) in new[]
|
||
{
|
||
(JetStreamDefaults.CommandStreamName, JetStreamDefaults.CommandStreamSubjects, JetStreamDefaults.CommandMaxAge),
|
||
(JetStreamDefaults.JobEventsStreamName, JetStreamDefaults.JobEventsStreamSubjects, JetStreamDefaults.JobEventsMaxAge),
|
||
})
|
||
{
|
||
var config = new NATS.Client.JetStream.Models.StreamConfig(name, subjects)
|
||
{
|
||
Storage = NATS.Client.JetStream.Models.StreamConfigStorage.Memory,
|
||
MaxAge = maxAge,
|
||
};
|
||
try
|
||
{
|
||
await js.CreateOrUpdateStreamAsync(config);
|
||
}
|
||
catch (NatsJSApiException)
|
||
{
|
||
await js.DeleteStreamAsync(name);
|
||
await js.CreateOrUpdateStreamAsync(config);
|
||
}
|
||
}
|
||
|
||
await js.CreateOrUpdateConsumerAsync(JetStreamDefaults.JobEventsStreamName, new NATS.Client.JetStream.Models.ConsumerConfig(JetStreamDefaults.BackendJobEventsConsumerName)
|
||
{
|
||
DurableName = JetStreamDefaults.BackendJobEventsConsumerName,
|
||
AckPolicy = NATS.Client.JetStream.Models.ConsumerConfigAckPolicy.Explicit,
|
||
AckWait = JetStreamDefaults.CommandAckWait,
|
||
FilterSubject = JetStreamDefaults.JobEventsStreamSubjects[0],
|
||
});
|
||
}
|
||
|
||
await using var backend = new BenchBackend(root, database?.ConnectionString, scheduleWorker: upgrade);
|
||
using (backend.CreateClient())
|
||
{
|
||
}
|
||
|
||
var hub = backend.Services.GetRequiredService<NatsHub>().Connection!;
|
||
long dropped = 0;
|
||
var droppedBySubject = new ConcurrentDictionary<string, long>();
|
||
hub.MessageDropped += (_, e) =>
|
||
{
|
||
Interlocked.Increment(ref dropped);
|
||
droppedBySubject.AddOrUpdate(Kind(e.Subject), 1, (_, n) => n + 1);
|
||
return ValueTask.CompletedTask;
|
||
};
|
||
await backend.Services.GetRequiredService<ServerMessageSecurity>().Keys.EnsureLoadedAsync(CancellationToken.None);
|
||
|
||
// Enrollment through the real endpoint (keys recorded, consumers made).
|
||
var enrollWatch = Stopwatch.StartNew();
|
||
var credentials = await backend.EnrollAsync(devices, seals, "lb");
|
||
enrollWatch.Stop();
|
||
Log($"enrolled {devices} in {enrollWatch.Elapsed.TotalSeconds:0.0}s ({devices / enrollWatch.Elapsed.TotalSeconds:0}/s)");
|
||
var credentialsFile = Path.Combine(root, "devices.json");
|
||
await File.WriteAllTextAsync(credentialsFile, JsonSerializer.Serialize(credentials, JsonDefaults.Options));
|
||
|
||
// The simulator, in its own process (its CPU is the agents', not the server's).
|
||
var child = Process.Start(new ProcessStartInfo(Environment.ProcessPath!, $"agents --file \"{credentialsFile}\" --connections {connections} --lines {lines} --ramp 20 --settings-retry {settingsRetry}")
|
||
{
|
||
RedirectStandardOutput = true,
|
||
UseShellExecute = false,
|
||
})!;
|
||
var stats = new SimulatorStats();
|
||
_ = Task.Run(async () =>
|
||
{
|
||
while (await child.StandardOutput.ReadLineAsync() is { } line)
|
||
{
|
||
stats.Take(line);
|
||
}
|
||
});
|
||
|
||
var self = new ProcessMeter(Process.GetCurrentProcess(), "server");
|
||
var agentsMeter = new ProcessMeter(child, "agents");
|
||
var natsMeter = nats is null ? null : new ProcessMeter(nats, "nats");
|
||
var results = new List<PhaseResult>();
|
||
var startedUtc = DateTimeOffset.UtcNow;
|
||
|
||
// Phase 1: every agent connects within 20 s (a fleet coming back after an outage):
|
||
// ready event, settings pull, inventory and presence each.
|
||
var storm = await MeasureAsync("connect storm", self, agentsMeter, natsMeter, stats, () => dropped, TimeSpan.FromMinutes(15), async () =>
|
||
{
|
||
var inventories = await backend.WithDbAsync(db => db.DeviceInventory.CountAsync(i => i.UpdatedUtc >= startedUtc));
|
||
var seen = await backend.WithDbAsync(db => db.Devices.CountAsync(d => d.LastSeenUtc >= startedUtc));
|
||
var ready = await backend.WithDbAsync(db => db.Devices.CountAsync(d => d.ReadyUtc >= startedUtc));
|
||
return (inventories >= devices && seen >= devices && ready >= devices && stats.SettingsAnswered + stats.SettingsFailed >= devices,
|
||
$"inventories {inventories}/{devices}, seen {seen}/{devices}, ready {ready}/{devices}, settings answered {stats.SettingsAnswered} " +
|
||
$"(first try {stats.Get("settingsFirstOk")}, retries {stats.Get("settingsRetries")}) failed {stats.SettingsFailed}, " +
|
||
$"reply ms p50 {stats.Get("settingsP50")} p95 {stats.Get("settingsP95")} max {stats.Get("settingsMax")}");
|
||
});
|
||
results.Add(storm);
|
||
Log($"connect storm: every device's settings decided after {storm.Seconds:0.0}s: answered {stats.SettingsAnswered}/{devices}, first try {stats.Get("settingsFirstOk")}, failed {stats.SettingsFailed}");
|
||
|
||
// Phase 2: steady state (presence every 30 s, inventory every 15 min, spread out).
|
||
var steadyStart = stats.Snapshot();
|
||
var steadyResult = await MeasureAsync("steady", self, agentsMeter, natsMeter, stats, () => dropped, TimeSpan.FromSeconds(steady), async () =>
|
||
{
|
||
var recent = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(40);
|
||
var fresh = await backend.WithDbAsync(db => db.Devices.CountAsync(d => d.LastSeenUtc >= recent));
|
||
return (false, $"seen in the last 40 s {fresh}/{devices}");
|
||
}, fixedDuration: true);
|
||
results.Add(steadyResult);
|
||
|
||
// Phase 3 (with --upgrade): every device upgraded through the upgrade scheduler's paced batches.
|
||
object? upgradeSummary = upgrade
|
||
? await UpgradeFanOutAsync(backend, devices, options.Int("start-within", 1), stats, self, agentsMeter, natsMeter, () => dropped, results)
|
||
: null;
|
||
|
||
// Phase 4 (unless --burst false): a script to every device (~L output lines and two states each), through the job runner.
|
||
double? statesSeconds = null;
|
||
long outputRows = 0;
|
||
var dispatchWatch = new Stopwatch();
|
||
if (jobBurst)
|
||
{
|
||
await JobBurstAsync();
|
||
}
|
||
|
||
async Task JobBurstAsync()
|
||
{
|
||
var runId = Guid.NewGuid();
|
||
dispatchWatch.Start();
|
||
var runWatch = Stopwatch.StartNew();
|
||
var dispatchTask = Task.Run(async () =>
|
||
{
|
||
using var scope = backend.Services.CreateScope();
|
||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||
var runner = scope.ServiceProvider.GetRequiredService<JobRunner>();
|
||
var script = new ScriptEntity
|
||
{
|
||
Id = Guid.NewGuid(),
|
||
Name = "Load burst",
|
||
Language = "powershell",
|
||
Content = $"1..{lines} | ForEach-Object {{ Write-Output \"line $_ of the load burst, about eighty characters of output each time\" }}",
|
||
CreatedUtc = DateTimeOffset.UtcNow,
|
||
UpdatedUtc = DateTimeOffset.UtcNow,
|
||
};
|
||
db.Scripts.Add(script);
|
||
await db.SaveChangesAsync();
|
||
var targets = await db.Devices.AsNoTracking().ToListAsync();
|
||
await runner.RecordRunAsync(runId, script, "loadbench", "manual", "all devices", [], CancellationToken.None);
|
||
var prepared = await runner.PrepareAsync(script, CancellationToken.None);
|
||
await runner.StartRunAsync(runId, prepared, null, targets, "loadbench", 0, 0, d => $"load burst on {d.Hostname}", CancellationToken.None);
|
||
dispatchWatch.Stop();
|
||
});
|
||
// Sampled every second: when the last state was stored (every job succeeded) is the burst's completion.
|
||
var burst = await MeasureAsync("job burst", self, agentsMeter, natsMeter, stats, () => dropped, TimeSpan.FromMinutes(30), async () =>
|
||
{
|
||
var done = await backend.WithDbAsync(db => db.Jobs.CountAsync(j => j.RunId == runId && j.State == JobState.Succeeded));
|
||
var created = await backend.WithDbAsync(db => db.Jobs.CountAsync(j => j.RunId == runId));
|
||
var output = await backend.WithDbAsync(db => db.JobOutput.CountAsync(o => db.Jobs.Any(j => j.RunId == runId && j.JobId == o.JobId)));
|
||
var dispatched = dispatchTask.IsCompleted ? $"dispatched in {dispatchWatch.Elapsed.TotalSeconds:0.0}s" : $"dispatching ({created} created)";
|
||
if (done >= devices && statesSeconds is null)
|
||
{
|
||
statesSeconds = runWatch.Elapsed.TotalSeconds;
|
||
}
|
||
|
||
return (dispatchTask.IsCompleted && done >= devices && stats.Outputs >= (long)devices * lines,
|
||
$"{dispatched}; succeeded {done}/{devices}; output rows {output}/{(long)devices * lines}; agents ran {stats.Commands}");
|
||
}, interval: TimeSpan.FromSeconds(1));
|
||
results.Add(burst);
|
||
Log($"burst: dispatched in {dispatchWatch.Elapsed.TotalSeconds:0.0}s ({devices / dispatchWatch.Elapsed.TotalSeconds:0}/s), every job's final state stored after {statesSeconds:0.0}s");
|
||
|
||
// The output still being written after the last state.
|
||
var settle = Stopwatch.StartNew();
|
||
for (var i = 0; i < 120; i++)
|
||
{
|
||
var before = outputRows;
|
||
outputRows = await backend.WithDbAsync(db => db.JobOutput.LongCountAsync(o => db.Jobs.Any(j => j.RunId == runId && j.JobId == o.JobId)));
|
||
if (outputRows >= (long)devices * lines || (i > 3 && outputRows == before))
|
||
{
|
||
break;
|
||
}
|
||
|
||
await Task.Delay(5000);
|
||
}
|
||
|
||
Log($"output rows stored {outputRows}/{(long)devices * lines} ({settle.Elapsed.TotalSeconds:0}s after the last state)");
|
||
}
|
||
|
||
Log($"dropped by the backend's subscriptions: {string.Join(", ", droppedBySubject.Select(p => $"{p.Key} {p.Value}"))}");
|
||
|
||
child.Kill(entireProcessTree: true);
|
||
await child.WaitForExitAsync();
|
||
|
||
var summary = new
|
||
{
|
||
utc = DateTimeOffset.UtcNow,
|
||
devices,
|
||
sealedMode = seals,
|
||
connections,
|
||
enrollSeconds = enrollWatch.Elapsed.TotalSeconds,
|
||
provider,
|
||
dispatchSeconds = dispatchWatch.Elapsed.TotalSeconds,
|
||
statesSeconds,
|
||
outputRows,
|
||
expectedOutputRows = (long)devices * lines,
|
||
dropped,
|
||
droppedBySubject,
|
||
phases = results,
|
||
upgrade = upgradeSummary,
|
||
};
|
||
await File.AppendAllTextAsync(outFile, JsonSerializer.Serialize(summary) + Environment.NewLine);
|
||
foreach (var r in results)
|
||
{
|
||
Log($"{r.Phase,-14} {r.Seconds,7:0.0}s server {r.ServerCores,5:0.00} cores (peak {r.ServerPeakCores:0.00}) ws {r.ServerWorkingSetMb,6:0}MB priv {r.ServerPrivateMb,6:0}MB | agents {r.AgentsCores,5:0.00} | nats {r.NatsCores,5:0.00} cores {r.NatsWorkingSetMb,6:0}MB | {r.MessagesPerSecond,7:0} msg/s from agents | dropped {r.Dropped} | {r.Detail}");
|
||
}
|
||
|
||
// Leave the shared NATS as it was: this run's consumers and queued commands go.
|
||
var provisioner = backend.Services.GetRequiredService<JetStreamProvisioner>();
|
||
await Parallel.ForEachAsync(credentials, new ParallelOptions { MaxDegreeOfParallelism = 16 }, async (c, ct) =>
|
||
{
|
||
try
|
||
{
|
||
await provisioner.RemoveAgentCommandsAsync(c.ClientId, ct);
|
||
}
|
||
catch (Exception)
|
||
{
|
||
// Best effort.
|
||
}
|
||
});
|
||
|
||
return 0;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Every device queued for an upgrade at once (an MSI made up for the catalog), the scheduler
|
||
/// running as on the server with SCHEDULE_WORKER on and StartWithinMinutes set to
|
||
/// <paramref name="startWithin"/>: until every agent took its command. Each pass's sends are
|
||
/// timed from the pass's start (the rows' SentUtc) to the last publish that returned.
|
||
/// </summary>
|
||
private static async Task<object> UpgradeFanOutAsync(
|
||
BenchBackend backend, int devices, int startWithin, SimulatorStats stats, ProcessMeter self, ProcessMeter agents, ProcessMeter? nats,
|
||
Func<long> dropped, List<PhaseResult> results)
|
||
{
|
||
const string Target = "2026.11.07.0000";
|
||
var folder = Directory.CreateTempSubdirectory("loadbench-package-").FullName;
|
||
var msi = Path.Combine(folder, "ExecuteR.Agent.msi");
|
||
await File.WriteAllBytesAsync(msi, RandomNumberGenerator.GetBytes(256 * 1024));
|
||
AgentPackageEntity package;
|
||
using (var scope = backend.Services.CreateScope())
|
||
{
|
||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||
var row = await Backend.Auth.SettingsDocuments.LoadAsync(db, AgentUpgradeSettingsStore.Key, CancellationToken.None);
|
||
Backend.Auth.SettingsDocuments.Stage(
|
||
db, row, AgentUpgradeSettingsStore.Key, null, AgentUpgradeSettingsStore.Defaults with { StartWithinMinutes = startWithin },
|
||
"loadbench", always: true);
|
||
await db.SaveChangesAsync();
|
||
scope.ServiceProvider.GetRequiredService<AgentUpgradeSettingsStore>().Invalidate();
|
||
package = await scope.ServiceProvider.GetRequiredService<AgentCatalogService>()
|
||
.ImportAsync(AgentPackageTypes.Msi, msi, Target, CancellationToken.None)
|
||
?? throw new InvalidOperationException("The bench package was not imported");
|
||
}
|
||
|
||
var queuedUtc = DateTimeOffset.UtcNow;
|
||
using (var scope = backend.Services.CreateScope())
|
||
{
|
||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||
var upgrades = scope.ServiceProvider.GetRequiredService<AgentUpgradeService>();
|
||
foreach (var device in await db.Devices.AsNoTracking().ToListAsync())
|
||
{
|
||
upgrades.Queue(device, package, AgentUpgradeSources.Manual, "loadbench");
|
||
}
|
||
|
||
await db.SaveChangesAsync();
|
||
}
|
||
|
||
Log($"upgrade: {devices} devices queued for {Target}, StartWithinMinutes {startWithin}");
|
||
var phase = await MeasureAsync("upgrade fan-out", self, agents, nats, stats, dropped, TimeSpan.FromMinutes(startWithin + 10), async () =>
|
||
{
|
||
var sent = await backend.WithDbAsync(db => db.AgentUpgrades.CountAsync(u => u.State != AgentUpgradeStates.Queued));
|
||
var received = stats.Get("upgrades");
|
||
return (received >= devices && sent >= devices,
|
||
$"rows sent {sent}/{devices}; agents took {received}, a second command for the same version {stats.Get("upgradeDuplicates")}");
|
||
}, interval: TimeSpan.FromSeconds(1));
|
||
results.Add(phase);
|
||
|
||
var sentAt = await backend.WithDbAsync(db => db.AgentUpgrades.AsNoTracking()
|
||
.Where(u => u.SentUtc != null)
|
||
.Select(u => new { u.Id, u.SentUtc })
|
||
.ToListAsync());
|
||
var passOf = sentAt.ToDictionary(u => u.Id, u => u.SentUtc!.Value);
|
||
var sends = backend.UpgradeSends.Sends.ToList();
|
||
var passes = sends
|
||
.Where(s => passOf.ContainsKey(s.UpgradeId))
|
||
.GroupBy(s => passOf[s.UpgradeId])
|
||
.OrderBy(g => g.Key)
|
||
.Select(g => new
|
||
{
|
||
atSeconds = Math.Round((g.Key - queuedUtc).TotalSeconds, 1),
|
||
sent = g.Count(s => s.Sent),
|
||
seconds = Math.Round((g.Max(s => s.DoneUtc) - g.Key).TotalSeconds, 2),
|
||
})
|
||
.ToList();
|
||
var lastSend = sends.Count == 0 ? queuedUtc : sends.Max(s => s.DoneUtc);
|
||
foreach (var pass in passes)
|
||
{
|
||
Log($" pass at {pass.atSeconds,6:0.0}s: {pass.sent,5} sent in {pass.seconds,6:0.00}s");
|
||
}
|
||
|
||
var summary = new
|
||
{
|
||
startWithinMinutes = startWithin,
|
||
passes,
|
||
failedSends = sends.Count(s => !s.Sent),
|
||
lastCommandSeconds = Math.Round((lastSend - queuedUtc).TotalSeconds, 1),
|
||
received = stats.Get("upgrades"),
|
||
duplicates = stats.Get("upgradeDuplicates"),
|
||
};
|
||
Log($"upgrade: {summary.received}/{devices} agents took their command, the last {summary.lastCommandSeconds:0.0}s after queueing " +
|
||
$"({passes.Count} passes, the longest {(passes.Count == 0 ? 0 : passes.Max(p => p.seconds)):0.00}s); " +
|
||
$"failed sends {summary.failedSends}, a second command for the same version {summary.duplicates}");
|
||
return summary;
|
||
}
|
||
|
||
private static string Kind(string subject)
|
||
{
|
||
var parts = subject.Split('.');
|
||
return parts.Length > 0 ? parts[0] + "." + parts[^1] : subject;
|
||
}
|
||
|
||
private static async Task<PhaseResult> MeasureAsync(
|
||
string phase, ProcessMeter server, ProcessMeter agents, ProcessMeter? nats, SimulatorStats stats, Func<long> dropped, TimeSpan limit,
|
||
Func<Task<(bool Done, string Detail)>> check, bool fixedDuration = false, TimeSpan? interval = null)
|
||
{
|
||
Log($"{phase}…");
|
||
server.Sample();
|
||
agents.Sample();
|
||
nats?.Sample();
|
||
var watch = Stopwatch.StartNew();
|
||
var published = stats.Published;
|
||
var droppedBefore = dropped();
|
||
double serverCpu = 0, peak = 0, agentsCpu = 0, natsCpu = 0, ws = 0, priv = 0, natsWs = 0;
|
||
var samples = 0;
|
||
var detail = string.Empty;
|
||
while (true)
|
||
{
|
||
await Task.Delay(interval ?? TimeSpan.FromSeconds(5));
|
||
var s = server.Sample();
|
||
var a = agents.Sample();
|
||
var n = nats?.Sample() ?? (0, 0, 0);
|
||
samples++;
|
||
serverCpu += s.Cores;
|
||
peak = Math.Max(peak, s.Cores);
|
||
agentsCpu += a.Cores;
|
||
natsCpu += n.Cores;
|
||
ws = Math.Max(ws, s.WorkingSetMb);
|
||
priv = Math.Max(priv, s.PrivateMb);
|
||
natsWs = Math.Max(natsWs, n.WorkingSetMb);
|
||
var (done, text) = await check();
|
||
detail = text;
|
||
Log($" {watch.Elapsed.TotalSeconds,6:0}s server {s.Cores:0.00} cores {s.WorkingSetMb:0}MB, agents {a.Cores:0.00}, nats {n.Cores:0.00} {n.WorkingSetMb:0}MB, dropped {dropped() - droppedBefore}, {text}");
|
||
if ((done && !fixedDuration) || watch.Elapsed >= limit)
|
||
{
|
||
break;
|
||
}
|
||
}
|
||
|
||
var seconds = watch.Elapsed.TotalSeconds;
|
||
return new PhaseResult(
|
||
phase, seconds, serverCpu / samples, peak, ws, priv, agentsCpu / samples, natsCpu / samples, natsWs,
|
||
(stats.Published - published) / seconds, dropped() - droppedBefore, detail);
|
||
}
|
||
|
||
private static Process? FindNats()
|
||
{
|
||
var candidates = Process.GetProcessesByName("nats-server");
|
||
return candidates.Length == 1 ? candidates[0] : null;
|
||
}
|
||
|
||
public static void Log(string text) => Console.WriteLine($"{DateTime.Now:HH:mm:ss} {text}");
|
||
}
|
||
|
||
/// <summary>The simulator's counters as it prints them every few seconds.</summary>
|
||
public sealed class SimulatorStats
|
||
{
|
||
private readonly ConcurrentDictionary<string, long> values = new();
|
||
|
||
public long Published => Get("published");
|
||
public long Outputs => Get("outputs");
|
||
public long Commands => Get("commands");
|
||
public long SettingsAnswered => Get("settingsOk");
|
||
public long SettingsFailed => Get("settingsFailed");
|
||
|
||
public long Get(string name) => values.TryGetValue(name, out var v) ? v : 0;
|
||
|
||
public Dictionary<string, long> Snapshot() => new(values);
|
||
|
||
public void Take(string line)
|
||
{
|
||
if (!line.StartsWith("STAT ", StringComparison.Ordinal))
|
||
{
|
||
Bench.Log("agents: " + line);
|
||
return;
|
||
}
|
||
|
||
foreach (var pair in line[5..].Split(' ', StringSplitOptions.RemoveEmptyEntries))
|
||
{
|
||
var at = pair.IndexOf('=');
|
||
values[pair[..at]] = long.Parse(pair[(at + 1)..], CultureInfo.InvariantCulture);
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// The simulated agents: each device with the agent's own message security and publisher
|
||
/// (sealing and signing as the agent does), multiplexed over a few NATS connections, each
|
||
/// pulling its own durable command consumer as the agent does.
|
||
/// </summary>
|
||
public static class Simulator
|
||
{
|
||
private static long published, presence, inventory, events, outputs, states, commands, refused, settingsOk, settingsFailed, bound, errors;
|
||
private static long settingsFirstOk, settingsRetries, upgrades, upgradeDuplicates;
|
||
private static readonly ConcurrentQueue<int> SettingsLatencies = new();
|
||
private static bool retrySettings;
|
||
|
||
/// <summary>The p-th percentile (0–100) of the settings replies' times so far, in ms.</summary>
|
||
private static long Percentile(int[] sorted, double p) => sorted.Length == 0 ? 0 : sorted[Math.Min(sorted.Length - 1, (int)(sorted.Length * p / 100))];
|
||
|
||
public static async Task<int> RunAsync(Options options)
|
||
{
|
||
var credentials = JsonSerializer.Deserialize<List<DeviceCredentials>>(await File.ReadAllTextAsync(options.Text("file")!), JsonDefaults.Options)!;
|
||
var connectionCount = options.Int("connections", 8);
|
||
var lines = options.Int("lines", 20);
|
||
retrySettings = options.Bool("settings-retry", false);
|
||
var ramp = TimeSpan.FromSeconds(options.Int("ramp", 20));
|
||
var connections = new List<NatsConnection>();
|
||
for (var i = 0; i < connectionCount; i++)
|
||
{
|
||
var connection = new NatsConnection(NatsOpts.Default with
|
||
{
|
||
Url = Program.NatsUrl,
|
||
Name = $"loadbench-agents-{i}",
|
||
MaxReconnectRetry = -1,
|
||
SubPendingChannelCapacity = 65536,
|
||
});
|
||
await connection.ConnectAsync();
|
||
connections.Add(connection);
|
||
}
|
||
|
||
var template = new InventoryCollector(new FixedIdentity()).Collect("template");
|
||
Console.WriteLine($"inventory report {JsonDefaults.SerializeToUtf8Bytes(new InventoryPayload(template)).Length} bytes");
|
||
var seen = new ReplayWindow(10_000_000);
|
||
var devices = credentials.Select((c, i) => new SimDevice(c, connections[i % connections.Count], seen, template, lines)).ToList();
|
||
var stop = new CancellationTokenSource();
|
||
var tasks = devices.Select(d => Task.Run(() => d.RunAsync(ramp, stop.Token))).ToList();
|
||
_ = Task.Run(async () =>
|
||
{
|
||
while (!stop.IsCancellationRequested)
|
||
{
|
||
await Task.Delay(2000);
|
||
var latencies = SettingsLatencies.ToArray();
|
||
Array.Sort(latencies);
|
||
Console.WriteLine(
|
||
$"STAT settingsFirstOk={Interlocked.Read(ref settingsFirstOk)} settingsRetries={Interlocked.Read(ref settingsRetries)} " +
|
||
$"settingsP50={Percentile(latencies, 50)} settingsP95={Percentile(latencies, 95)} settingsMax={(latencies.Length == 0 ? 0 : latencies[^1])} " +
|
||
$"upgrades={Interlocked.Read(ref upgrades)} upgradeDuplicates={Interlocked.Read(ref upgradeDuplicates)}");
|
||
Console.WriteLine(
|
||
$"STAT published={Interlocked.Read(ref published)} presence={Interlocked.Read(ref presence)} inventory={Interlocked.Read(ref inventory)} " +
|
||
$"events={Interlocked.Read(ref events)} outputs={Interlocked.Read(ref outputs)} states={Interlocked.Read(ref states)} " +
|
||
$"commands={Interlocked.Read(ref commands)} refused={Interlocked.Read(ref refused)} settingsOk={Interlocked.Read(ref settingsOk)} " +
|
||
$"settingsFailed={Interlocked.Read(ref settingsFailed)} bound={Interlocked.Read(ref bound)} errors={Interlocked.Read(ref errors)}");
|
||
}
|
||
});
|
||
await Task.WhenAll(tasks);
|
||
return 0;
|
||
}
|
||
|
||
private sealed class FixedIdentity : IHardwareIdentitySource
|
||
{
|
||
public HardwareIdentity Read() => new("uuid", "bios", "board", "ident", "Contoso", "Model");
|
||
}
|
||
|
||
private sealed class MemoryStore(AgentCredentials credentials) : ICredentialStore
|
||
{
|
||
private AgentCredentials? current = credentials;
|
||
|
||
public AgentCredentials? Load() => current;
|
||
|
||
public void Save(AgentCredentials value) => current = value;
|
||
|
||
public void Delete() => current = null;
|
||
}
|
||
|
||
private sealed class SimDevice
|
||
{
|
||
private readonly DeviceCredentials credentials;
|
||
private readonly NatsConnection connection;
|
||
private readonly AgentMessageSecurity security;
|
||
private readonly AgentPublisher publisher;
|
||
private readonly InventoryReport report;
|
||
private readonly int lines;
|
||
private readonly string hostname;
|
||
private readonly DateTimeOffset started = DateTimeOffset.UtcNow;
|
||
private readonly HashSet<string> upgradeVersions = new(StringComparer.Ordinal);
|
||
|
||
public SimDevice(DeviceCredentials credentials, NatsConnection connection, ISeenMessageStore seen, InventoryReport template, int lines)
|
||
{
|
||
this.credentials = credentials;
|
||
this.connection = connection;
|
||
this.lines = lines;
|
||
hostname = "PC-" + credentials.ClientId[2..10];
|
||
security = new AgentMessageSecurity(
|
||
new MemoryStore(new AgentCredentials(
|
||
AgentCredentials.CurrentVersion, credentials.ClientId, Guid.NewGuid(), "jwt", credentials.NkeySeed, null,
|
||
credentials.MessageKey, credentials.ServerKeys)),
|
||
seen, NullLogger.Instance);
|
||
security.Reload();
|
||
publisher = new AgentPublisher(connection, credentials.ClientId, security);
|
||
report = template with
|
||
{
|
||
Identity = template.Identity with { ClientId = credentials.ClientId, Hostname = hostname },
|
||
MessageSecurity = security.Report(),
|
||
};
|
||
}
|
||
|
||
public async Task RunAsync(TimeSpan ramp, CancellationToken cancellationToken)
|
||
{
|
||
var random = new Random(credentials.ClientId.GetHashCode());
|
||
await Task.Delay(TimeSpan.FromMilliseconds(random.NextDouble() * ramp.TotalMilliseconds), cancellationToken);
|
||
_ = Task.Run(() => CommandsAsync(cancellationToken), cancellationToken);
|
||
await Guard(() => PublishAsync(NatsSubjects.AgentEvent(credentials.ClientId), MessageTypes.AgentEvent,
|
||
new AgentEventPayload(credentials.ClientId, AgentEventKinds.Ready, "command consumer bound", DateTimeOffset.UtcNow), ref events));
|
||
_ = Task.Run(() => PullSettingsAsync(cancellationToken), cancellationToken);
|
||
await Guard(() => PublishAsync(NatsSubjects.AgentInventory(credentials.ClientId), MessageTypes.AgentInventory, new InventoryPayload(report), ref inventory));
|
||
var nextInventory = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(random.NextDouble() * 900);
|
||
while (!cancellationToken.IsCancellationRequested)
|
||
{
|
||
await Guard(() => PublishAsync(NatsSubjects.AgentPresence(credentials.ClientId), MessageTypes.AgentPresence,
|
||
new PresencePayload(credentials.ClientId, hostname, "2026.11.06.0000", "online", DateTimeOffset.UtcNow,
|
||
(long)(DateTimeOffset.UtcNow - started).TotalSeconds, 0, "win-x64", "msi"), ref presence));
|
||
if (DateTimeOffset.UtcNow >= nextInventory)
|
||
{
|
||
nextInventory += TimeSpan.FromSeconds(900);
|
||
await Guard(() => PublishAsync(NatsSubjects.AgentInventory(credentials.ClientId), MessageTypes.AgentInventory,
|
||
new InventoryPayload(report with { CollectedUtc = DateTimeOffset.UtcNow }), ref inventory));
|
||
}
|
||
|
||
await Task.Delay(ProtocolDefaults.PresenceInterval, cancellationToken);
|
||
}
|
||
}
|
||
|
||
private static async Task Guard(Func<ValueTask> action)
|
||
{
|
||
try
|
||
{
|
||
await action();
|
||
}
|
||
catch (OperationCanceledException)
|
||
{
|
||
throw;
|
||
}
|
||
catch (Exception)
|
||
{
|
||
Interlocked.Increment(ref errors);
|
||
}
|
||
}
|
||
|
||
private ValueTask PublishAsync<T>(string subject, string type, T payload, ref long counter)
|
||
{
|
||
Interlocked.Increment(ref counter);
|
||
Interlocked.Increment(ref published);
|
||
return publisher.PublishAsync(subject, type, payload);
|
||
}
|
||
|
||
/// <summary>
|
||
/// As AgentSettingsClient pulls: sealed request on the event subject, sealed reply. With
|
||
/// --settings-retry, a request that timed out is tried again as the agent does; without it,
|
||
/// one try (what reaches the device at its connect).
|
||
/// </summary>
|
||
/// <summary>The agent's own pause before each retry (a few, so a run that does not answer still ends).</summary>
|
||
private static TimeSpan? SettingsRetryDelay(int attempt)
|
||
=> attempt < 6 ? AgentSettingsClient.RetryDelay(attempt, Random.Shared) : null;
|
||
|
||
private async Task PullSettingsAsync(CancellationToken cancellationToken)
|
||
{
|
||
var subject = NatsSubjects.AgentEvent(credentials.ClientId);
|
||
for (var attempt = 0; ; attempt++)
|
||
{
|
||
var envelope = MessageEnvelope.Create(
|
||
MessageTypes.AgentSettingsRequest, publisher.Source, new AgentSettingsRequestPayload(credentials.ClientId, 0));
|
||
var watch = Stopwatch.StartNew();
|
||
byte[]? data = null;
|
||
try
|
||
{
|
||
Interlocked.Increment(ref events);
|
||
Interlocked.Increment(ref published);
|
||
var answer = await connection.RequestAsync<byte[], byte[]>(
|
||
subject, AgentPublisher.Seal(security, subject, envelope.MessageType, envelope.ToUtf8Bytes()),
|
||
replyOpts: new NatsSubOpts { Timeout = AgentSettingsClient.RequestTimeout }, cancellationToken: cancellationToken);
|
||
data = answer.Data is { Length: > 0 } d ? security.Open(answer.Subject, d, MessageReceipt.Request)?.Payload : null;
|
||
}
|
||
catch (Exception) when (!cancellationToken.IsCancellationRequested)
|
||
{
|
||
// Timed out: no answer in time.
|
||
}
|
||
|
||
if (data is not null)
|
||
{
|
||
SettingsLatencies.Enqueue((int)watch.ElapsedMilliseconds);
|
||
Interlocked.Increment(ref settingsOk);
|
||
if (attempt == 0)
|
||
{
|
||
Interlocked.Increment(ref settingsFirstOk);
|
||
}
|
||
|
||
return;
|
||
}
|
||
|
||
if (!retrySettings || SettingsRetryDelay(attempt) is not { } delay)
|
||
{
|
||
Interlocked.Increment(ref settingsFailed);
|
||
return;
|
||
}
|
||
|
||
Interlocked.Increment(ref settingsRetries);
|
||
await Task.Delay(delay, cancellationToken);
|
||
}
|
||
}
|
||
|
||
private async Task CommandsAsync(CancellationToken cancellationToken)
|
||
{
|
||
var js = new NatsJSContext(connection);
|
||
while (!cancellationToken.IsCancellationRequested)
|
||
{
|
||
try
|
||
{
|
||
var consumer = await js.GetConsumerAsync(
|
||
JetStreamDefaults.CommandStreamName, JetStreamDefaults.AgentCommandConsumerName(credentials.ClientId), cancellationToken);
|
||
Interlocked.Increment(ref bound);
|
||
await foreach (var message in consumer.ConsumeAsync<byte[]>(opts: new NatsJSConsumeOpts { MaxMsgs = 10 }, cancellationToken: cancellationToken))
|
||
{
|
||
var opened = message.Data is { Length: > 0 } data
|
||
? security.Open(message.Subject, data, MessageReceipt.Durable, message.Metadata?.Timestamp, out _)
|
||
: null;
|
||
await message.AckAsync(cancellationToken: cancellationToken);
|
||
if (opened is null)
|
||
{
|
||
Interlocked.Increment(ref refused);
|
||
continue;
|
||
}
|
||
|
||
security.Remember(opened);
|
||
var commandEnvelope = MessageEnvelope.FromUtf8Bytes(opened.Payload);
|
||
if (commandEnvelope?.MessageType == MessageTypes.CommandAgentUpgrade)
|
||
{
|
||
// Taken and kept: no download, no report (the row stays Sent).
|
||
var upgrade = commandEnvelope.GetPayload<AgentUpgradeCommandPayload>();
|
||
Interlocked.Increment(ref upgrades);
|
||
if (upgrade is not null && !upgradeVersions.Add(upgrade.Version))
|
||
{
|
||
Interlocked.Increment(ref upgradeDuplicates);
|
||
}
|
||
|
||
continue;
|
||
}
|
||
|
||
var command = commandEnvelope?.GetPayload<ScriptCommandPayload>();
|
||
if (command is null)
|
||
{
|
||
continue;
|
||
}
|
||
|
||
Interlocked.Increment(ref commands);
|
||
_ = Task.Run(() => RunJobAsync(command, cancellationToken), cancellationToken);
|
||
}
|
||
}
|
||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||
{
|
||
return;
|
||
}
|
||
catch (Exception)
|
||
{
|
||
Interlocked.Increment(ref errors);
|
||
await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
|
||
}
|
||
}
|
||
}
|
||
|
||
private async Task RunJobAsync(ScriptCommandPayload command, CancellationToken cancellationToken)
|
||
{
|
||
var target = new EndpointRef(EndpointKinds.Job, command.JobId.ToString("D"));
|
||
await Guard(() => PublishAsync(NatsSubjects.JobState(command.JobId), MessageTypes.JobState,
|
||
new JobStatePayload(command.JobId, credentials.ClientId, JobState.Running, null, DateTimeOffset.UtcNow, null), ref states));
|
||
for (var i = 1; i <= lines; i++)
|
||
{
|
||
await Guard(() => PublishAsync(NatsSubjects.JobOutput(command.JobId), MessageTypes.JobOutput,
|
||
new JobOutputPayload(command.JobId, credentials.ClientId, i, OutputStreamKind.Output, DateTimeOffset.UtcNow,
|
||
$"line {i} of the load burst, about eighty characters of output each time"), ref outputs));
|
||
}
|
||
|
||
await Guard(() => PublishAsync(NatsSubjects.JobState(command.JobId), MessageTypes.JobState,
|
||
new JobStatePayload(command.JobId, credentials.ClientId, JobState.Succeeded, 0, DateTimeOffset.UtcNow, "done"), ref states));
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>Per-message cost of sealing and opening, against the real server security (one thread).</summary>
|
||
public static class Micro
|
||
{
|
||
private const int Warm = 500;
|
||
|
||
public static async Task<int> RunAsync(Options options)
|
||
{
|
||
var count = options.Int("count", 20000);
|
||
var root = Directory.CreateTempSubdirectory("loadbench-micro-").FullName;
|
||
await using var backend = new BenchBackend(root);
|
||
using (backend.CreateClient())
|
||
{
|
||
}
|
||
|
||
var security = backend.Services.GetRequiredService<ServerMessageSecurity>();
|
||
await security.Keys.EnsureLoadedAsync(CancellationToken.None);
|
||
const int Devices = 200;
|
||
var sealedDevices = await backend.EnrollAsync(Devices, seals: true, "mb");
|
||
var plainDevices = await backend.EnrollAsync(Devices, seals: false, "mp");
|
||
var agents = sealedDevices.Select(c => Agent(c)).ToList();
|
||
var plainAgents = plainDevices.Select(c => Agent(c)).ToList();
|
||
|
||
Bench.Log($"per message, {count} each, one thread (µs, KB allocated)");
|
||
var presence = (AgentMessageSecurity a, string id) => MessageEnvelope.Create(
|
||
MessageTypes.AgentPresence, new EndpointRef(EndpointKinds.Agent, id),
|
||
new PresencePayload(id, "PC-000001", "2026.11.06.0000", "online", DateTimeOffset.UtcNow, 100, 0, "win-x64", "msi")).ToUtf8Bytes();
|
||
var output = (AgentMessageSecurity a, string id) => MessageEnvelope.Create(
|
||
MessageTypes.JobOutput, new EndpointRef(EndpointKinds.Agent, id),
|
||
new JobOutputPayload(Guid.NewGuid(), id, 1, OutputStreamKind.Output, DateTimeOffset.UtcNow, "line 1 of the load burst, about eighty characters of output each time")).ToUtf8Bytes();
|
||
|
||
foreach (var (name, make) in new[] { ("presence", presence), ("job output", output) })
|
||
{
|
||
// Agent side: seal (encrypt + sign).
|
||
var plain = Enumerable.Range(0, count + Warm).Select(i => make(agents[i % Devices].Security, agents[i % Devices].Id)).ToList();
|
||
var wire = new byte[count + Warm][];
|
||
Measure($"agent seal {name}", count, i =>
|
||
{
|
||
var a = agents[i % Devices];
|
||
wire[i] = a.Security.Seal($"agent.{a.Id}.presence", MessageTypes.AgentPresence, plain[i]);
|
||
return ValueTask.CompletedTask;
|
||
});
|
||
Bench.Log($" {name}: v1 {plain[0].Length} bytes, sealed {wire[0].Length} bytes");
|
||
|
||
// Server side: open (verify, check, decrypt, replay window) sealed vs the v1 acceptance check.
|
||
await MeasureAsync($"server open sealed {name}", count, async i =>
|
||
{
|
||
var a = agents[i % Devices];
|
||
var result = await security.OpenFromAgentAsync($"agent.{a.Id}.presence", wire[i], a.Id, ServerReceipt.Live);
|
||
if (result.Message is null)
|
||
{
|
||
throw new InvalidOperationException("refused");
|
||
}
|
||
});
|
||
var plainWire = Enumerable.Range(0, count + Warm).Select(i => make(plainAgents[i % Devices].Security, plainAgents[i % Devices].Id)).ToList();
|
||
await MeasureAsync($"server open v1 {name}", count, async i =>
|
||
{
|
||
var a = plainAgents[i % Devices];
|
||
var result = await security.OpenFromAgentAsync($"agent.{a.Id}.presence", plainWire[i], a.Id, ServerReceipt.Live);
|
||
if (result.Message is null)
|
||
{
|
||
throw new InvalidOperationException("refused");
|
||
}
|
||
});
|
||
Measure($"parse envelope {name}", count, i =>
|
||
{
|
||
_ = MessageEnvelope.FromUtf8Bytes(plainWire[i]);
|
||
return ValueTask.CompletedTask;
|
||
});
|
||
}
|
||
|
||
// Where an open's time goes: the Ed25519 signature (NATS.NKeys), the key agreement (cached), the AEAD.
|
||
{
|
||
var a = agents[0];
|
||
var bytes = a.Security.Seal($"agent.{a.Id}.presence", MessageTypes.AgentPresence, presence(a.Security, a.Id));
|
||
var envelope = MessageSealing.Parse(bytes)!;
|
||
var nkeyPublic = KeyPair.FromSeed(sealedDevices[0].NkeySeed).GetPublicKey();
|
||
var secret = security.Keys.Current.Secret(MessageKeys.PublicKeyOf(MessageKeys.FromPrivateKey(sealedDevices[0].MessageKey!)));
|
||
using var signer = new NkeySigner(sealedDevices[0].NkeySeed);
|
||
var input = MessageSealing.SignatureInput(envelope);
|
||
Measure(" parse sealed envelope (JSON)", count, _ => { MessageSealing.Parse(bytes); return ValueTask.CompletedTask; });
|
||
Measure(" Ed25519 verify (NATS.NKeys)", count / 4, _ => { MessageSealing.VerifySignature(envelope, nkeyPublic); return ValueTask.CompletedTask; });
|
||
Measure(" Ed25519 sign (NATS.NKeys)", count / 4, _ => { signer.Sign(input); return ValueTask.CompletedTask; });
|
||
Measure(" HKDF + AES-GCM decrypt", count, _ => { MessageSealing.TryDecrypt(envelope, secret, MessageDirections.AgentToServer); return ValueTask.CompletedTask; });
|
||
Measure(" ECDH P-256 agreement (uncached)", count / 20, _ =>
|
||
{
|
||
MessageKeys.Agree(MessageKeys.FromPrivateKey(sealedDevices[1].MessageKey!), security.Keys.Current.EcdhPublic);
|
||
return ValueTask.CompletedTask;
|
||
});
|
||
}
|
||
|
||
// Server side: a script command sealed for a device (fresh keys from the database, as the dispatcher does).
|
||
var commandPlain = MessageEnvelope.Create(MessageTypes.CommandScript, new EndpointRef(EndpointKinds.Backend, "backend"),
|
||
new ScriptCommandPayload(Guid.NewGuid(), Guid.NewGuid(), "Load burst", "1..20 | ForEach-Object { Write-Output \"line $_\" }", 3600)).ToUtf8Bytes();
|
||
await MeasureAsync("server seal command (cached keys)", count, async i =>
|
||
{
|
||
var a = agents[i % Devices];
|
||
await security.SealForAgentAsync(a.Id, $"agent.{a.Id}.command.script", MessageTypes.CommandScript, commandPlain, MessageSecurityDefaults.DurableCommandLifetime);
|
||
});
|
||
await MeasureAsync("server seal command (fresh: DB read)", count / 4, async i =>
|
||
{
|
||
var a = agents[i % Devices];
|
||
await security.SealForAgentAsync(a.Id, $"agent.{a.Id}.command.script", MessageTypes.CommandScript, commandPlain, MessageSecurityDefaults.DurableCommandLifetime, fresh: true);
|
||
});
|
||
await MeasureAsync("server v1 command (fresh: DB read)", count / 4, async i =>
|
||
{
|
||
var a = plainAgents[i % Devices];
|
||
await security.SealForAgentAsync(a.Id, $"agent.{a.Id}.command.script", MessageTypes.CommandScript, commandPlain, MessageSecurityDefaults.DurableCommandLifetime, fresh: true);
|
||
});
|
||
|
||
// The job runner's fan-out by itself (nothing answers, nothing else writes): per device.
|
||
foreach (var prefix in new[] { "mb", "mp" })
|
||
{
|
||
using var scope = backend.Services.CreateScope();
|
||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||
var runner = scope.ServiceProvider.GetRequiredService<JobRunner>();
|
||
var script = new ScriptEntity
|
||
{
|
||
Id = Guid.NewGuid(), Name = "Micro", Language = "powershell", Content = "Write-Output 1",
|
||
CreatedUtc = DateTimeOffset.UtcNow, UpdatedUtc = DateTimeOffset.UtcNow,
|
||
};
|
||
db.Scripts.Add(script);
|
||
await db.SaveChangesAsync();
|
||
var targets = await db.Devices.AsNoTracking().Where(d => d.ClientId.StartsWith(prefix)).ToListAsync();
|
||
var runId = Guid.NewGuid();
|
||
await runner.RecordRunAsync(runId, script, "micro", "manual", "all", [], CancellationToken.None);
|
||
var prepared = await runner.PrepareAsync(script, CancellationToken.None);
|
||
var watch = Stopwatch.StartNew();
|
||
await runner.StartRunAsync(runId, prepared, null, targets, "micro", 0, 0, d => d.Hostname, CancellationToken.None);
|
||
Bench.Log($" job runner fan-out ({(prefix == "mb" ? "sealed" : "v1")}) {watch.Elapsed.TotalMilliseconds / targets.Count,8:0.0} ms per device ({targets.Count} devices)");
|
||
|
||
// Its two halves, per device: the job row (and history pruning), then the sealed publish and its state.
|
||
double create = 0, dispatch = 0, publish = 0, prune = 0;
|
||
var js = new NatsJSContext(backend.Services.GetRequiredService<NatsHub>().Connection!);
|
||
foreach (var device in targets)
|
||
{
|
||
var t = Stopwatch.StartNew();
|
||
var job = await runner.CreateJobAsync(script, device.ClientId, "micro", CancellationToken.None, runId);
|
||
create += t.Elapsed.TotalMilliseconds;
|
||
t.Restart();
|
||
await runner.DispatchAsync(prepared, job, null, device.Hostname, CancellationToken.None);
|
||
dispatch += t.Elapsed.TotalMilliseconds;
|
||
t.Restart();
|
||
(await js.PublishAsync($"agent.{device.ClientId}.command.inventory", commandPlain)).EnsureSuccess();
|
||
publish += t.Elapsed.TotalMilliseconds;
|
||
t.Restart();
|
||
await runner.PruneHistoryAsync(device.ClientId, CancellationToken.None);
|
||
prune += t.Elapsed.TotalMilliseconds;
|
||
}
|
||
|
||
Bench.Log($" create job {create / targets.Count:0.0} ms, dispatch {dispatch / targets.Count:0.0} ms, a bare JetStream publish {publish / targets.Count:0.0} ms, history pruning alone {prune / targets.Count:0.0} ms (per device)");
|
||
}
|
||
|
||
var provisioner = backend.Services.GetRequiredService<JetStreamProvisioner>();
|
||
foreach (var c in sealedDevices.Concat(plainDevices))
|
||
{
|
||
await provisioner.RemoveAgentCommandsAsync(c.ClientId, CancellationToken.None);
|
||
}
|
||
|
||
return 0;
|
||
}
|
||
|
||
private static (AgentMessageSecurity Security, string Id) Agent(DeviceCredentials c)
|
||
{
|
||
var security = new AgentMessageSecurity(
|
||
new StaticStore(new AgentCredentials(AgentCredentials.CurrentVersion, c.ClientId, Guid.NewGuid(), "jwt", c.NkeySeed, null, c.MessageKey, c.ServerKeys)),
|
||
new ReplayWindow(), NullLogger.Instance);
|
||
security.Reload();
|
||
return (security, c.ClientId);
|
||
}
|
||
|
||
private sealed class StaticStore(AgentCredentials credentials) : ICredentialStore
|
||
{
|
||
public AgentCredentials? Load() => credentials;
|
||
|
||
public void Save(AgentCredentials value)
|
||
{
|
||
}
|
||
|
||
public void Delete()
|
||
{
|
||
}
|
||
}
|
||
|
||
private static void Measure(string name, int count, Func<int, ValueTask> action)
|
||
=> MeasureAsync(name, count, i => action(i).AsTask()).GetAwaiter().GetResult();
|
||
|
||
private static async Task MeasureAsync(string name, int count, Func<int, Task> action)
|
||
{
|
||
// Warm-up on its own indices (count..count+Warm), so the measured messages are new.
|
||
for (var i = 0; i < Warm; i++)
|
||
{
|
||
await action(count + i);
|
||
}
|
||
|
||
GC.Collect();
|
||
var allocated = GC.GetTotalAllocatedBytes(precise: true);
|
||
var cpu = Process.GetCurrentProcess().TotalProcessorTime;
|
||
var watch = Stopwatch.StartNew();
|
||
for (var i = 0; i < count; i++)
|
||
{
|
||
await action(i);
|
||
}
|
||
|
||
watch.Stop();
|
||
var used = Process.GetCurrentProcess().TotalProcessorTime - cpu;
|
||
Bench.Log($" {name,-40} {watch.Elapsed.TotalMicroseconds / count,8:0.0} µs wall {used.TotalMicroseconds / count,8:0.0} µs cpu {(GC.GetTotalAllocatedBytes(precise: true) - allocated) / 1024.0 / count,6:0.0} KB");
|
||
}
|
||
}
|
||
|
||
/// <summary>A database of its own on the PostgreSQL server the PG* variables name, dropped afterwards.</summary>
|
||
public sealed class ThrowawayPostgres : IAsyncDisposable
|
||
{
|
||
private readonly string admin;
|
||
|
||
private ThrowawayPostgres(string admin, string name, string connectionString)
|
||
{
|
||
this.admin = admin;
|
||
Name = name;
|
||
ConnectionString = connectionString;
|
||
}
|
||
|
||
public string Name { get; }
|
||
|
||
public string ConnectionString { get; }
|
||
|
||
public static async Task<ThrowawayPostgres> CreateAsync()
|
||
{
|
||
var builder = new Npgsql.NpgsqlConnectionStringBuilder
|
||
{
|
||
Host = Environment.GetEnvironmentVariable("PGHOST") ?? "127.0.0.1",
|
||
Port = int.TryParse(Environment.GetEnvironmentVariable("PGPORT"), out var port) ? port : 5432,
|
||
Username = Environment.GetEnvironmentVariable("PGUSER") ?? "postgres",
|
||
Password = Environment.GetEnvironmentVariable("PGPASSWORD"),
|
||
Database = "postgres",
|
||
MaxPoolSize = 200,
|
||
};
|
||
var admin = builder.ConnectionString;
|
||
var name = "loadbench_" + Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(6));
|
||
await using (var connection = new Npgsql.NpgsqlConnection(admin))
|
||
{
|
||
await connection.OpenAsync();
|
||
await using var command = new Npgsql.NpgsqlCommand($"CREATE DATABASE {name}", connection);
|
||
await command.ExecuteNonQueryAsync();
|
||
}
|
||
|
||
builder.Database = name;
|
||
return new ThrowawayPostgres(admin, name, builder.ConnectionString);
|
||
}
|
||
|
||
public async ValueTask DisposeAsync()
|
||
{
|
||
Npgsql.NpgsqlConnection.ClearAllPools();
|
||
await using var connection = new Npgsql.NpgsqlConnection(admin);
|
||
await connection.OpenAsync();
|
||
await using var command = new Npgsql.NpgsqlCommand($"DROP DATABASE IF EXISTS {Name} WITH (FORCE)", connection);
|
||
await command.ExecuteNonQueryAsync();
|
||
Bench.Log($"dropped database {Name}");
|
||
}
|
||
}
|