Files
ExecuteR/tests/Integration.Tests/ParallelRunIntegrationTests.cs
Alphaeus Mote 084f569c31 perf(jobs): parallel dispatch for large runs, job rows written in batches [skip ci]
A run without a stagger now writes its jobs 500 per transaction and seals and
publishes each batch's commands JOB_DISPATCH_PARALLELISM at a time (default 16,
at most 128) while the next batch is written and the last one's Dispatched
states and history pruning are stored. A batch's rows commit before any of its
commands goes out, and a batch is sent only once the one before has been, so an
agent's first state always finds its job and a device's jobs leave in order.
The job dispatcher (staggered slots, retries) sends a pass's due jobs the same
way: only what is due, so Spread Over and Devices at a Time hold as before.

- A device's failed send stays with that device: it is left queued for the
  dispatcher, and its later jobs in the batch wait behind it.
- A script command carries its job id as the JetStream message id, so a resend
  inside the stream's duplicate window (a publish that was not confirmed, a run
  stopped part way) is dropped by the stream: a job is never delivered twice.
- A run stopped part way hands what was written but not sent to the dispatcher.
- Staggered runs write their rows a batch per transaction after the snapshot.
- History pruning for a batch: one grouped count, deletes in one transaction.
- On SQLite, new rows go through one prepared INSERT (BulkInsert): SaveChanges
  cost about 0.4 ms a row there. PostgreSQL and SQL Server keep SaveChanges.
- SqliteWriteGate: the batch writers (a run's rows and states, and the
  messaging workers' in the next commit) take SQLite's one writer in turn,
  awaited, instead of meeting at its lock, where Microsoft.Data.Sqlite retries
  after 150 ms sleeps. It does nothing on PostgreSQL and SQL Server.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-07 06:49:02 -05:00

142 lines
6.0 KiB
C#

using Backend.Data;
using Backend.Jobs;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using NATS.Client.JetStream;
using NATS.Client.JetStream.Models;
using Shared.Contracts.Messages;
using Shared.Contracts.Models;
using Shared.Infrastructure.Nats;
using Xunit;
namespace Integration.Tests;
/// <summary>
/// A run to several sealed devices through the job runner over the live NATS server: its commands
/// sealed and published side by side, every device's states stored in batches, each job sent once.
/// Skipped when NATS is not reachable at NATS_URL.
/// </summary>
[Collection(SealedNatsCollection.Name)]
public sealed class ParallelRunIntegrationTests(SealedBackend backend)
{
private static CancellationToken Ct => TestContext.Current.CancellationToken;
private void RequireNats()
{
if (!backend.Available)
{
Assert.Skip($"No NATS server reachable at {SealedBackend.NatsUrl}; set NATS_URL to run this test.");
}
}
private static async Task WaitUntilAsync(Func<Task<bool>> done, TimeSpan timeout, string what)
{
var until = DateTime.UtcNow + timeout;
while (!await done())
{
if (DateTime.UtcNow > until)
{
throw new TimeoutException($"Timed out waiting for {what}.");
}
await Task.Delay(100, Ct);
}
}
[Fact]
public async Task ARunToSeveralSealedDevicesGoesOutOnceEachAndEveryStateIsStored()
{
RequireNats();
var agents = new List<TestAgent>();
try
{
for (var i = 0; i < 4; i++)
{
agents.Add(await TestAgent.StartAsync(backend));
}
var ids = agents.Select(a => a.ClientId).ToList();
await using var tap = await WireTap.StartAsync(backend.Raw!, [.. ids.Select(NatsSubjects.AgentCommandScript)]);
var runId = Guid.NewGuid();
using (var scope = backend.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
var runner = scope.ServiceProvider.GetRequiredService<JobRunner>();
var script = new ScriptEntity
{
Id = Guid.NewGuid(),
Name = "Parallel run",
Language = "powershell",
Content = "1..3 | ForEach-Object { Write-Output \"parallel line $_\" }",
CreatedUtc = DateTimeOffset.UtcNow,
UpdatedUtc = DateTimeOffset.UtcNow,
};
db.Scripts.Add(script);
await db.SaveChangesAsync(Ct);
var devices = await db.Devices.AsNoTracking().Where(d => ids.Contains(d.ClientId)).ToListAsync(Ct);
await runner.RecordRunAsync(runId, script, "integration", RunSources.Manual, "four devices", [], Ct);
var prepared = await runner.PrepareAsync(script, Ct);
var jobs = await runner.StartRunAsync(runId, prepared, null, devices, "integration", 0, 0, d => d.Hostname, Ct);
Assert.Equal(4, jobs.Count);
}
await WaitUntilAsync(
() => backend.WithDbAsync(db => db.Jobs.AsNoTracking().CountAsync(j => j.RunId == runId && j.State == JobState.Succeeded, Ct)).ContinueWith(t => t.Result == 4, TaskScheduler.Default),
TimeSpan.FromSeconds(90), "every job to succeed");
var stored = await backend.WithDbAsync(db => db.Jobs.AsNoTracking().Where(j => j.RunId == runId).ToListAsync(Ct));
Assert.All(stored, j =>
{
Assert.Equal(0, j.ExitCode);
Assert.NotNull(j.StartedUtc);
Assert.NotNull(j.CompletedUtc);
});
var jobIds = stored.Select(j => j.JobId).ToList();
await WaitUntilAsync(
() => backend.WithDbAsync(db => db.JobOutput.AsNoTracking().CountAsync(o => jobIds.Contains(o.JobId), Ct)).ContinueWith(t => t.Result >= 12, TaskScheduler.Default),
TimeSpan.FromSeconds(10), "the output to be stored");
// Each device was sent its command once.
Assert.All(ids, id => Assert.Single(tap.On(NatsSubjects.AgentCommandScript(id))));
Assert.All(agents, a => Assert.Equal(0, a.Security.Refused));
}
finally
{
foreach (var agent in agents)
{
await agent.DisposeAsync();
}
}
}
[Fact]
public async Task ASecondSendOfTheSameJobIsDroppedByTheCommandStream()
{
RequireNats();
// A sealed device that takes no commands: what reaches the stream stays there to be counted.
await using var agent = await TestAgent.StartAsync(backend, commands: false);
var scripts = backend.Services.GetRequiredService<IScriptCommandSender>();
var clientId = agent.ClientId;
var subject = NatsSubjects.AgentCommandScript(clientId);
var jobId = Guid.NewGuid();
var command = new ScriptCommandPayload(jobId, Guid.NewGuid(), "Once", "Write-Output once", 60);
var js = new NatsJSContext(backend.Raw!);
try
{
// A retry after a publish that was not confirmed: the stream keeps the first.
// The second is taken as sent (the stream has it), so the dispatcher does not keep retrying it.
Assert.True(await scripts.DispatchScriptAsync(clientId, command, Ct));
Assert.True(await scripts.DispatchScriptAsync(clientId, command, Ct));
Assert.True(await scripts.DispatchScriptAsync(clientId, command with { JobId = Guid.NewGuid() }, Ct));
var stream = await js.GetStreamAsync(JetStreamDefaults.CommandStreamName, new StreamInfoRequest { SubjectsFilter = subject }, Ct);
Assert.Equal(2L, stream.Info.State.Subjects![subject]);
}
finally
{
await js.PurgeStreamAsync(JetStreamDefaults.CommandStreamName, new StreamPurgeRequest { Filter = subject }, Ct);
}
}
}