Files
ExecuteR/tests/Backend.Tests/ParallelDispatchTests.cs
Alphaeus Mote 4cb34ce7e5 Merge parallel dispatch and batched job-state writes [skip ci]
Batched final states keep Restart After Run (Requested/Refused) in the same conditional update, without replacing one the restart request stored; the restart request keeps the in-memory ApplyJobState. Dispatch test send delay raised so overlap holds on a loaded runner.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-07 07:18:28 -05:00

373 lines
18 KiB
C#
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using System.Collections.Concurrent;
using System.Net.Http.Json;
using Backend.Data;
using Backend.Jobs;
using Backend.Messaging;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Shared.Contracts.Api;
using Shared.Contracts.Messages;
using Shared.Contracts.Models;
using Shared.Contracts.Serialization;
using Shared.Infrastructure.Nats;
using Xunit;
namespace Backend.Tests;
/// <summary>
/// Sends script commands the way messaging does, slowly enough for sends to overlap: records how many
/// ran at once, what each device got and in which order, and fails (or throws for) the devices it is told to.
/// </summary>
public sealed class ConcurrentScriptSender(TimeSpan delay) : IScriptCommandSender
{
private int current;
public int MaxConcurrent;
public ConcurrentQueue<(string ClientId, Guid JobId, long Started, long Ended)> Sent { get; } = new();
public HashSet<string> Refuse { get; } = [];
public HashSet<string> Throw { get; } = [];
/// <summary>Called as each send starts (the test checks what is stored by then).</summary>
public Func<string, Guid, Task>? OnSend { get; set; }
public async Task<bool> DispatchScriptAsync(string clientId, ScriptCommandPayload command, CancellationToken cancellationToken)
{
var now = Interlocked.Increment(ref current);
InterlockedMax(ref MaxConcurrent, now);
var started = System.Diagnostics.Stopwatch.GetTimestamp();
try
{
if (OnSend is not null)
{
await OnSend(clientId, command.JobId);
}
await Task.Delay(delay, cancellationToken);
if (Throw.Contains(clientId))
{
throw new InvalidOperationException("the publish blew up");
}
if (Refuse.Contains(clientId))
{
return false;
}
Sent.Enqueue((clientId, command.JobId, started, System.Diagnostics.Stopwatch.GetTimestamp()));
return true;
}
finally
{
Interlocked.Decrement(ref current);
}
}
private static void InterlockedMax(ref int target, int value)
{
int seen;
while ((seen = Volatile.Read(ref target)) < value && Interlocked.CompareExchange(ref target, value, seen) != seen)
{
}
}
}
public class ParallelDispatchTests(BackendTestFixture fixture) : IClassFixture<BackendTestFixture>
{
/// <summary>A job runner on a scope of its own, sending through <paramref name="sender"/>, <paramref name="parallelism"/> at a time.</summary>
private static (JobRunner Runner, AppDbContext Db, IServiceScope Scope) Runner(
BackendTestFixture fixture, IScriptCommandSender sender, int parallelism)
{
var scope = fixture.Services.CreateScope();
var services = scope.ServiceProvider;
var configuration = new ConfigurationBuilder()
.AddConfiguration(services.GetRequiredService<IConfiguration>())
.AddInMemoryCollection(new Dictionary<string, string?> { ["JOB_DISPATCH_PARALLELISM"] = parallelism.ToString(System.Globalization.CultureInfo.InvariantCulture) })
.Build();
var db = services.GetRequiredService<AppDbContext>();
var runner = new JobRunner(
db, sender, services.GetRequiredService<BrowserEventBroadcaster>(),
services.GetRequiredService<Backend.Secrets.SecretsProviderService>(),
services.GetRequiredService<Backend.Files.S3PresignService>(),
services.GetRequiredService<RunSnapshots>(), configuration, NullLogger<JobRunner>.Instance,
services.GetRequiredService<SqliteWriteGate>());
return (runner, db, scope);
}
private async Task<List<DeviceEntity>> SeedDevicesAsync(string prefix, int count)
{
var devices = Enumerable.Range(0, count).Select(i => new DeviceEntity
{
ClientId = $"{prefix}{i:00000}",
Hostname = $"{prefix}-{i}",
NkeyPublic = $"U{Guid.NewGuid():N}",
EnrolledUtc = DateTimeOffset.UtcNow,
}).ToList();
await fixture.WithDbAsync(async db =>
{
db.Devices.AddRange(devices);
await db.SaveChangesAsync();
});
return devices;
}
private async Task<(Guid RunId, PreparedScript Prepared)> RecordRunAsync(JobRunner runner, AppDbContext db, string name)
{
var scriptId = await fixture.SeedScriptAsync(name);
var script = await db.Scripts.AsNoTracking().SingleAsync(s => s.Id == scriptId);
var runId = Guid.NewGuid();
await runner.RecordRunAsync(runId, script, "test", RunSources.Manual, name, [], CancellationToken.None);
return (runId, await runner.PrepareAsync(script, CancellationToken.None));
}
private Task<List<JobEntity>> JobsAsync(Guid runId)
=> fixture.WithDbAsync(db => db.Jobs.AsNoTracking().Where(j => j.RunId == runId).ToListAsync());
[Fact]
public async Task ALargeRunIsWrittenInBatchesAndSentSeveralAtOnceEachJobOnce()
{
var devices = await SeedDevicesAsync("bulk", 1_200);
// Long enough a send that two overlap even on a loaded machine (at 2 ms a busy CI runner
// could finish each before the next began).
var sender = new ConcurrentScriptSender(TimeSpan.FromMilliseconds(15));
var (runner, db, scope) = Runner(fixture, sender, parallelism: 8);
using var _ = scope;
var (runId, prepared) = await RecordRunAsync(runner, db, "bulk");
// Each command goes out only once its job is stored (an agent's first state must find it).
var missing = 0;
sender.OnSend = async (_, jobId) =>
{
if (!await fixture.WithDbAsync(d => d.Jobs.AsNoTracking().AnyAsync(j => j.JobId == jobId)))
{
Interlocked.Increment(ref missing);
}
};
var jobs = await runner.StartRunAsync(runId, prepared, null, devices, "test", 0, 0, d => d.Hostname, CancellationToken.None);
Assert.Equal(1_200, jobs.Count);
Assert.Equal(0, missing);
var sent = sender.Sent.ToList();
Assert.Equal(1_200, sent.Count);
Assert.Equal(1_200, sent.Select(s => s.JobId).Distinct().Count());
Assert.Equal(devices.Select(d => d.ClientId).Order(), sent.Select(s => s.ClientId).Order());
Assert.InRange(sender.MaxConcurrent, 2, 8);
var stored = await JobsAsync(runId);
Assert.Equal(1_200, stored.Count);
// Written a batch at a time (a batch's rows share their time).
Assert.Equal([500, 500, 200], stored.GroupBy(j => j.CreatedUtc).OrderBy(g => g.Key).Select(g => g.Count()));
Assert.All(stored, j =>
{
Assert.Equal(JobState.Dispatched, j.State);
Assert.Null(j.NotBeforeUtc);
});
Assert.All(jobs, j => Assert.Equal(JobState.Dispatched, j.State));
Assert.Null((await fixture.WithDbAsync(d => d.Runs.AsNoTracking().SingleAsync(r => r.RunId == runId))).SnapshotProtected);
}
[Fact]
public async Task ADeviceWhoseSendFailsWaitsForTheDispatcherAndTheOthersGo()
{
var devices = await SeedDevicesAsync("isolated", 40);
var sender = new ConcurrentScriptSender(TimeSpan.FromMilliseconds(1));
sender.Refuse.UnionWith([devices[3].ClientId, devices[17].ClientId]);
sender.Throw.Add(devices[25].ClientId);
var (runner, db, scope) = Runner(fixture, sender, parallelism: 16);
using var _ = scope;
var (runId, prepared) = await RecordRunAsync(runner, db, "isolated");
await runner.StartRunAsync(runId, prepared, null, devices, "test", 0, 0, d => d.Hostname, CancellationToken.None);
var failed = new[] { devices[3].ClientId, devices[17].ClientId, devices[25].ClientId };
Assert.Equal(37, sender.Sent.Count);
Assert.DoesNotContain(sender.Sent, s => failed.Contains(s.ClientId));
var jobs = await JobsAsync(runId);
Assert.All(jobs.Where(j => !failed.Contains(j.ClientId)), j => Assert.Equal(JobState.Dispatched, j.State));
Assert.All(jobs.Where(j => failed.Contains(j.ClientId)), j =>
{
Assert.Equal(JobState.Queued, j.State);
Assert.NotNull(j.NotBeforeUtc);
});
Assert.NotNull((await fixture.WithDbAsync(d => d.Runs.AsNoTracking().SingleAsync(r => r.RunId == runId))).SnapshotProtected);
// The dispatcher sends those three, and nothing that already went.
fixture.Scripts.Accept = true;
var dispatcher = new JobDispatcher(fixture.Services.GetRequiredService<IServiceScopeFactory>(), NullLogger<JobDispatcher>.Instance);
await dispatcher.RunPassAsync(DateTimeOffset.UtcNow.AddSeconds(1), CancellationToken.None);
List<Guid> again;
lock (fixture.Scripts.Sent)
{
again = fixture.Scripts.Sent.Where(s => devices.Any(d => d.ClientId == s.ClientId)).Select(s => s.Command.JobId).ToList();
}
Assert.Equal(jobs.Where(j => failed.Contains(j.ClientId)).Select(j => j.JobId).Order(), again.Order());
Assert.Empty(again.Intersect(sender.Sent.Select(s => s.JobId)));
Assert.All(await JobsAsync(runId), j => Assert.Equal(JobState.Dispatched, j.State));
}
[Fact]
public async Task ADevicesJobsGoOneAfterAnotherAndWaitBehindOneThatFailed()
{
var devices = await SeedDevicesAsync("ordered", 3);
var sender = new ConcurrentScriptSender(TimeSpan.FromMilliseconds(15));
var (runner, db, scope) = Runner(fixture, sender, parallelism: 16);
using var _ = scope;
var (_, prepared) = await RecordRunAsync(runner, db, "ordered");
JobRunner.Outgoing Job(DeviceEntity device) => new(new JobEntity
{
JobId = Guid.NewGuid(), ClientId = device.ClientId, ScriptName = "ordered", State = JobState.Queued,
}, device.Hostname);
// Device 0 twice, device 1 three times, device 2 twice with its first refused.
var batch = new[] { Job(devices[0]), Job(devices[1]), Job(devices[0]), Job(devices[1]), Job(devices[2]), Job(devices[1]), Job(devices[2]) };
sender.Refuse.Add(devices[2].ClientId);
var sent = await runner.SendAsync(prepared, batch, null, null, null, stopOnFailure: false, CancellationToken.None);
Assert.Equal([true, true, true, true, false, true, false], sent);
foreach (var device in devices.Take(2))
{
var mine = batch.Select((o, i) => (o.Job.JobId, o.Job.ClientId)).Where(o => o.ClientId == device.ClientId).Select(o => o.JobId).ToList();
var order = sender.Sent.Where(s => s.ClientId == device.ClientId).ToList();
Assert.Equal(mine, order.Select(s => s.JobId));
for (var i = 1; i < order.Count; i++)
{
Assert.True(order[i].Started >= order[i - 1].Ended, "a device's sends overlapped");
}
}
// The devices themselves went side by side.
Assert.True(sender.MaxConcurrent >= 2);
}
[Fact]
public async Task TheDispatcherSendsOnlyWhatIsDueThoughItSendsSeveralAtOnce()
{
var devices = await SeedDevicesAsync("slots", 200);
var (runner, db, scope) = Runner(fixture, fixture.Scripts, parallelism: 16);
using var _ = scope;
var (runId, prepared) = await RecordRunAsync(runner, db, "slots");
fixture.Scripts.Accept = true;
await runner.StartRunAsync(runId, prepared, null, devices, "test", 600, 50, d => d.Hostname, CancellationToken.None);
var jobs = await JobsAsync(runId);
var start = jobs.Min(j => j.NotBeforeUtc)!.Value;
Assert.Equal(4, jobs.Select(j => j.NotBeforeUtc).Distinct().Count());
var dispatcher = new JobDispatcher(fixture.Services.GetRequiredService<IServiceScopeFactory>(), NullLogger<JobDispatcher>.Instance);
List<Guid> SentHere()
{
lock (fixture.Scripts.Sent)
{
var ids = jobs.Select(j => j.JobId).ToHashSet();
return fixture.Scripts.Sent.Where(s => ids.Contains(s.Command.JobId)).Select(s => s.Command.JobId).ToList();
}
}
foreach (var (seconds, expected) in new[] { (1, 50), (199, 50), (201, 100), (401, 150), (601, 200), (900, 200) })
{
await dispatcher.RunPassAsync(start.AddSeconds(seconds), CancellationToken.None);
var sent = SentHere();
Assert.Equal(expected, sent.Count);
Assert.Equal(sent.Count, sent.Distinct().Count());
var due = jobs.Where(j => j.NotBeforeUtc <= start.AddSeconds(seconds)).Select(j => j.JobId).ToHashSet();
Assert.All(sent, id => Assert.Contains(id, due));
}
Assert.All(await JobsAsync(runId), j => Assert.Equal(JobState.Dispatched, j.State));
}
[Fact]
public async Task HistoryIsPrunedForEveryDeviceOfABatch()
{
var devices = await SeedDevicesAsync("pruned", 3);
var sender = new ConcurrentScriptSender(TimeSpan.Zero);
var (runner, db, scope) = Runner(fixture, sender, parallelism: 4);
using var _ = scope;
var (oldRun, prepared) = await RecordRunAsync(runner, db, "pruned");
// 200 old jobs each (the default limit) with output.
await fixture.WithDbAsync(async d =>
{
foreach (var device in devices)
{
for (var i = 0; i < 200; i++)
{
var id = Guid.NewGuid();
d.Jobs.Add(new JobEntity
{
JobId = id, RunId = oldRun, ClientId = device.ClientId, ScriptName = "old", State = JobState.Succeeded,
CreatedUtc = DateTimeOffset.UtcNow.AddDays(-1).AddMinutes(i),
});
d.JobOutput.Add(new JobOutputEntity { JobId = id, Sequence = 1, Stream = OutputStreamKind.Output, TimestampUtc = DateTimeOffset.UtcNow, Text = "x" });
}
}
await d.SaveChangesAsync();
});
var oldest = await fixture.WithDbAsync(d => d.Jobs.AsNoTracking()
.Where(j => j.RunId == oldRun).GroupBy(j => j.ClientId).Select(g => g.OrderBy(j => j.CreatedUtc).First().JobId).ToListAsync());
var (runId, _) = await RecordRunAsync(runner, db, "pruned-new");
await runner.StartRunAsync(runId, prepared, null, devices, "test", 0, 0, d => d.Hostname, CancellationToken.None);
var counts = await fixture.WithDbAsync(d => d.Jobs.AsNoTracking()
.Where(j => devices.Select(x => x.ClientId).Contains(j.ClientId)).GroupBy(j => j.ClientId).Select(g => g.Count()).ToListAsync());
Assert.Equal([200, 200, 200], counts);
Assert.False(await fixture.WithDbAsync(d => d.Jobs.AnyAsync(j => oldest.Contains(j.JobId))));
Assert.False(await fixture.WithDbAsync(d => d.JobOutput.AnyAsync(o => oldest.Contains(o.JobId))));
}
}
public class BulkInsertTests(BackendTestFixture fixture) : IClassFixture<BackendTestFixture>
{
[Fact]
public async Task RowsWrittenTogetherReadBackAsSaveChangesWouldHaveStoredThem()
{
var device = await fixture.SeedDeviceAsync("bulk-insert");
var at = new DateTimeOffset(2026, 11, 7, 12, 30, 15, 123, TimeSpan.Zero).AddTicks(4567);
var full = new JobEntity
{
JobId = Guid.NewGuid(), RunId = Guid.NewGuid(), ClientId = device, ScriptId = Guid.NewGuid(), ScriptName = "every column",
State = JobState.TimedOut, CreatedUtc = at, StartedUtc = at.AddSeconds(1), CompletedUtc = at.AddSeconds(2), ExitCode = -3,
RequestedBy = "someone", OutputSummary = "summary ‘quoted’", NotBeforeUtc = at.AddMinutes(5),
ErrorType = "System.Exception", ErrorId = "Id", ErrorMessage = "message",
};
var sparse = new JobEntity { JobId = Guid.NewGuid(), ClientId = device, ScriptName = "nulls", State = JobState.Queued, CreatedUtc = at };
var others = Enumerable.Range(0, 1_100)
.Select(i => new JobEntity { JobId = Guid.NewGuid(), ClientId = device, ScriptName = $"n{i}", State = JobState.Dispatched, CreatedUtc = at.AddTicks(i) })
.ToList();
await fixture.WithDbAsync(db => db.InsertRowsAsync([full, sparse, .. others], CancellationToken.None));
await fixture.WithDbAsync(db => db.InsertRowsAsync(
[new JobOutputEntity { JobId = full.JobId, Sequence = 7, Stream = OutputStreamKind.Error, TimestampUtc = at, Text = "line" }],
CancellationToken.None));
var stored = await fixture.WithDbAsync(db => db.Jobs.AsNoTracking().Where(j => j.ClientId == device).ToDictionaryAsync(j => j.JobId));
Assert.Equal(1_102, stored.Count);
Assert.Equivalent(full, stored[full.JobId], strict: true);
Assert.Equivalent(sparse, stored[sparse.JobId], strict: true);
// What queries on those columns find (state as text, times as the provider keeps them).
Assert.Equal(1_100, await fixture.WithDbAsync(db => db.Jobs.CountAsync(j => j.ClientId == device && j.State == JobState.Dispatched)));
Assert.Equal(1, await fixture.WithDbAsync(db => db.Jobs.CountAsync(j => j.ClientId == device && j.NotBeforeUtc > at)));
var line = await fixture.WithDbAsync(db => db.JobOutput.AsNoTracking().SingleAsync(o => o.JobId == full.JobId));
Assert.Equal((7, OutputStreamKind.Error, at, "line"), (line.Sequence, line.Stream, line.TimestampUtc, line.Text));
Assert.True(line.Id > 0);
// A row already there fails the whole call, and nothing of it stays.
var again = new JobEntity { JobId = Guid.NewGuid(), ClientId = device, ScriptName = "again", State = JobState.Queued, CreatedUtc = at };
await Assert.ThrowsAnyAsync<Exception>(() => fixture.WithDbAsync(db => db.InsertRowsAsync([again, full], CancellationToken.None)));
Assert.False(await fixture.WithDbAsync(db => db.Jobs.AnyAsync(j => j.JobId == again.JobId)));
}
}