4cb34ce7e5
Batched final states keep Restart After Run (Requested/Refused) in the same conditional update, without replacing one the restart request stored; the restart request keeps the in-memory ApplyJobState. Dispatch test send delay raised so overlap holds on a loaded runner. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
373 lines
18 KiB
C#
373 lines
18 KiB
C#
using System.Collections.Concurrent;
|
||
using System.Net.Http.Json;
|
||
using Backend.Data;
|
||
using Backend.Jobs;
|
||
using Backend.Messaging;
|
||
using Microsoft.EntityFrameworkCore;
|
||
using Microsoft.Extensions.Configuration;
|
||
using Microsoft.Extensions.DependencyInjection;
|
||
using Microsoft.Extensions.Hosting;
|
||
using Microsoft.Extensions.Logging;
|
||
using Microsoft.Extensions.Logging.Abstractions;
|
||
using Shared.Contracts.Api;
|
||
using Shared.Contracts.Messages;
|
||
using Shared.Contracts.Models;
|
||
using Shared.Contracts.Serialization;
|
||
using Shared.Infrastructure.Nats;
|
||
using Xunit;
|
||
|
||
namespace Backend.Tests;
|
||
|
||
|
||
/// <summary>
|
||
/// Sends script commands the way messaging does, slowly enough for sends to overlap: records how many
|
||
/// ran at once, what each device got and in which order, and fails (or throws for) the devices it is told to.
|
||
/// </summary>
|
||
public sealed class ConcurrentScriptSender(TimeSpan delay) : IScriptCommandSender
|
||
{
|
||
private int current;
|
||
|
||
public int MaxConcurrent;
|
||
|
||
public ConcurrentQueue<(string ClientId, Guid JobId, long Started, long Ended)> Sent { get; } = new();
|
||
|
||
public HashSet<string> Refuse { get; } = [];
|
||
|
||
public HashSet<string> Throw { get; } = [];
|
||
|
||
/// <summary>Called as each send starts (the test checks what is stored by then).</summary>
|
||
public Func<string, Guid, Task>? OnSend { get; set; }
|
||
|
||
public async Task<bool> DispatchScriptAsync(string clientId, ScriptCommandPayload command, CancellationToken cancellationToken)
|
||
{
|
||
var now = Interlocked.Increment(ref current);
|
||
InterlockedMax(ref MaxConcurrent, now);
|
||
var started = System.Diagnostics.Stopwatch.GetTimestamp();
|
||
try
|
||
{
|
||
if (OnSend is not null)
|
||
{
|
||
await OnSend(clientId, command.JobId);
|
||
}
|
||
|
||
await Task.Delay(delay, cancellationToken);
|
||
if (Throw.Contains(clientId))
|
||
{
|
||
throw new InvalidOperationException("the publish blew up");
|
||
}
|
||
|
||
if (Refuse.Contains(clientId))
|
||
{
|
||
return false;
|
||
}
|
||
|
||
Sent.Enqueue((clientId, command.JobId, started, System.Diagnostics.Stopwatch.GetTimestamp()));
|
||
return true;
|
||
}
|
||
finally
|
||
{
|
||
Interlocked.Decrement(ref current);
|
||
}
|
||
}
|
||
|
||
private static void InterlockedMax(ref int target, int value)
|
||
{
|
||
int seen;
|
||
while ((seen = Volatile.Read(ref target)) < value && Interlocked.CompareExchange(ref target, value, seen) != seen)
|
||
{
|
||
}
|
||
}
|
||
}
|
||
|
||
public class ParallelDispatchTests(BackendTestFixture fixture) : IClassFixture<BackendTestFixture>
|
||
{
|
||
/// <summary>A job runner on a scope of its own, sending through <paramref name="sender"/>, <paramref name="parallelism"/> at a time.</summary>
|
||
private static (JobRunner Runner, AppDbContext Db, IServiceScope Scope) Runner(
|
||
BackendTestFixture fixture, IScriptCommandSender sender, int parallelism)
|
||
{
|
||
var scope = fixture.Services.CreateScope();
|
||
var services = scope.ServiceProvider;
|
||
var configuration = new ConfigurationBuilder()
|
||
.AddConfiguration(services.GetRequiredService<IConfiguration>())
|
||
.AddInMemoryCollection(new Dictionary<string, string?> { ["JOB_DISPATCH_PARALLELISM"] = parallelism.ToString(System.Globalization.CultureInfo.InvariantCulture) })
|
||
.Build();
|
||
var db = services.GetRequiredService<AppDbContext>();
|
||
var runner = new JobRunner(
|
||
db, sender, services.GetRequiredService<BrowserEventBroadcaster>(),
|
||
services.GetRequiredService<Backend.Secrets.SecretsProviderService>(),
|
||
services.GetRequiredService<Backend.Files.S3PresignService>(),
|
||
services.GetRequiredService<RunSnapshots>(), configuration, NullLogger<JobRunner>.Instance,
|
||
services.GetRequiredService<SqliteWriteGate>());
|
||
return (runner, db, scope);
|
||
}
|
||
|
||
private async Task<List<DeviceEntity>> SeedDevicesAsync(string prefix, int count)
|
||
{
|
||
var devices = Enumerable.Range(0, count).Select(i => new DeviceEntity
|
||
{
|
||
ClientId = $"{prefix}{i:00000}",
|
||
Hostname = $"{prefix}-{i}",
|
||
NkeyPublic = $"U{Guid.NewGuid():N}",
|
||
EnrolledUtc = DateTimeOffset.UtcNow,
|
||
}).ToList();
|
||
await fixture.WithDbAsync(async db =>
|
||
{
|
||
db.Devices.AddRange(devices);
|
||
await db.SaveChangesAsync();
|
||
});
|
||
return devices;
|
||
}
|
||
|
||
private async Task<(Guid RunId, PreparedScript Prepared)> RecordRunAsync(JobRunner runner, AppDbContext db, string name)
|
||
{
|
||
var scriptId = await fixture.SeedScriptAsync(name);
|
||
var script = await db.Scripts.AsNoTracking().SingleAsync(s => s.Id == scriptId);
|
||
var runId = Guid.NewGuid();
|
||
await runner.RecordRunAsync(runId, script, "test", RunSources.Manual, name, [], CancellationToken.None);
|
||
return (runId, await runner.PrepareAsync(script, CancellationToken.None));
|
||
}
|
||
|
||
private Task<List<JobEntity>> JobsAsync(Guid runId)
|
||
=> fixture.WithDbAsync(db => db.Jobs.AsNoTracking().Where(j => j.RunId == runId).ToListAsync());
|
||
|
||
[Fact]
|
||
public async Task ALargeRunIsWrittenInBatchesAndSentSeveralAtOnceEachJobOnce()
|
||
{
|
||
var devices = await SeedDevicesAsync("bulk", 1_200);
|
||
// Long enough a send that two overlap even on a loaded machine (at 2 ms a busy CI runner
|
||
// could finish each before the next began).
|
||
var sender = new ConcurrentScriptSender(TimeSpan.FromMilliseconds(15));
|
||
var (runner, db, scope) = Runner(fixture, sender, parallelism: 8);
|
||
using var _ = scope;
|
||
var (runId, prepared) = await RecordRunAsync(runner, db, "bulk");
|
||
|
||
// Each command goes out only once its job is stored (an agent's first state must find it).
|
||
var missing = 0;
|
||
sender.OnSend = async (_, jobId) =>
|
||
{
|
||
if (!await fixture.WithDbAsync(d => d.Jobs.AsNoTracking().AnyAsync(j => j.JobId == jobId)))
|
||
{
|
||
Interlocked.Increment(ref missing);
|
||
}
|
||
};
|
||
|
||
var jobs = await runner.StartRunAsync(runId, prepared, null, devices, "test", 0, 0, d => d.Hostname, CancellationToken.None);
|
||
|
||
Assert.Equal(1_200, jobs.Count);
|
||
Assert.Equal(0, missing);
|
||
var sent = sender.Sent.ToList();
|
||
Assert.Equal(1_200, sent.Count);
|
||
Assert.Equal(1_200, sent.Select(s => s.JobId).Distinct().Count());
|
||
Assert.Equal(devices.Select(d => d.ClientId).Order(), sent.Select(s => s.ClientId).Order());
|
||
Assert.InRange(sender.MaxConcurrent, 2, 8);
|
||
var stored = await JobsAsync(runId);
|
||
Assert.Equal(1_200, stored.Count);
|
||
|
||
// Written a batch at a time (a batch's rows share their time).
|
||
Assert.Equal([500, 500, 200], stored.GroupBy(j => j.CreatedUtc).OrderBy(g => g.Key).Select(g => g.Count()));
|
||
Assert.All(stored, j =>
|
||
{
|
||
Assert.Equal(JobState.Dispatched, j.State);
|
||
Assert.Null(j.NotBeforeUtc);
|
||
});
|
||
Assert.All(jobs, j => Assert.Equal(JobState.Dispatched, j.State));
|
||
Assert.Null((await fixture.WithDbAsync(d => d.Runs.AsNoTracking().SingleAsync(r => r.RunId == runId))).SnapshotProtected);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task ADeviceWhoseSendFailsWaitsForTheDispatcherAndTheOthersGo()
|
||
{
|
||
var devices = await SeedDevicesAsync("isolated", 40);
|
||
var sender = new ConcurrentScriptSender(TimeSpan.FromMilliseconds(1));
|
||
sender.Refuse.UnionWith([devices[3].ClientId, devices[17].ClientId]);
|
||
sender.Throw.Add(devices[25].ClientId);
|
||
var (runner, db, scope) = Runner(fixture, sender, parallelism: 16);
|
||
using var _ = scope;
|
||
var (runId, prepared) = await RecordRunAsync(runner, db, "isolated");
|
||
|
||
await runner.StartRunAsync(runId, prepared, null, devices, "test", 0, 0, d => d.Hostname, CancellationToken.None);
|
||
|
||
var failed = new[] { devices[3].ClientId, devices[17].ClientId, devices[25].ClientId };
|
||
Assert.Equal(37, sender.Sent.Count);
|
||
Assert.DoesNotContain(sender.Sent, s => failed.Contains(s.ClientId));
|
||
var jobs = await JobsAsync(runId);
|
||
Assert.All(jobs.Where(j => !failed.Contains(j.ClientId)), j => Assert.Equal(JobState.Dispatched, j.State));
|
||
Assert.All(jobs.Where(j => failed.Contains(j.ClientId)), j =>
|
||
{
|
||
Assert.Equal(JobState.Queued, j.State);
|
||
Assert.NotNull(j.NotBeforeUtc);
|
||
});
|
||
Assert.NotNull((await fixture.WithDbAsync(d => d.Runs.AsNoTracking().SingleAsync(r => r.RunId == runId))).SnapshotProtected);
|
||
|
||
// The dispatcher sends those three, and nothing that already went.
|
||
fixture.Scripts.Accept = true;
|
||
var dispatcher = new JobDispatcher(fixture.Services.GetRequiredService<IServiceScopeFactory>(), NullLogger<JobDispatcher>.Instance);
|
||
await dispatcher.RunPassAsync(DateTimeOffset.UtcNow.AddSeconds(1), CancellationToken.None);
|
||
|
||
List<Guid> again;
|
||
lock (fixture.Scripts.Sent)
|
||
{
|
||
again = fixture.Scripts.Sent.Where(s => devices.Any(d => d.ClientId == s.ClientId)).Select(s => s.Command.JobId).ToList();
|
||
}
|
||
|
||
Assert.Equal(jobs.Where(j => failed.Contains(j.ClientId)).Select(j => j.JobId).Order(), again.Order());
|
||
Assert.Empty(again.Intersect(sender.Sent.Select(s => s.JobId)));
|
||
Assert.All(await JobsAsync(runId), j => Assert.Equal(JobState.Dispatched, j.State));
|
||
}
|
||
|
||
[Fact]
|
||
public async Task ADevicesJobsGoOneAfterAnotherAndWaitBehindOneThatFailed()
|
||
{
|
||
var devices = await SeedDevicesAsync("ordered", 3);
|
||
var sender = new ConcurrentScriptSender(TimeSpan.FromMilliseconds(15));
|
||
var (runner, db, scope) = Runner(fixture, sender, parallelism: 16);
|
||
using var _ = scope;
|
||
var (_, prepared) = await RecordRunAsync(runner, db, "ordered");
|
||
JobRunner.Outgoing Job(DeviceEntity device) => new(new JobEntity
|
||
{
|
||
JobId = Guid.NewGuid(), ClientId = device.ClientId, ScriptName = "ordered", State = JobState.Queued,
|
||
}, device.Hostname);
|
||
|
||
// Device 0 twice, device 1 three times, device 2 twice with its first refused.
|
||
var batch = new[] { Job(devices[0]), Job(devices[1]), Job(devices[0]), Job(devices[1]), Job(devices[2]), Job(devices[1]), Job(devices[2]) };
|
||
sender.Refuse.Add(devices[2].ClientId);
|
||
|
||
var sent = await runner.SendAsync(prepared, batch, null, null, null, stopOnFailure: false, CancellationToken.None);
|
||
|
||
Assert.Equal([true, true, true, true, false, true, false], sent);
|
||
foreach (var device in devices.Take(2))
|
||
{
|
||
var mine = batch.Select((o, i) => (o.Job.JobId, o.Job.ClientId)).Where(o => o.ClientId == device.ClientId).Select(o => o.JobId).ToList();
|
||
var order = sender.Sent.Where(s => s.ClientId == device.ClientId).ToList();
|
||
Assert.Equal(mine, order.Select(s => s.JobId));
|
||
for (var i = 1; i < order.Count; i++)
|
||
{
|
||
Assert.True(order[i].Started >= order[i - 1].Ended, "a device's sends overlapped");
|
||
}
|
||
}
|
||
|
||
// The devices themselves went side by side.
|
||
Assert.True(sender.MaxConcurrent >= 2);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task TheDispatcherSendsOnlyWhatIsDueThoughItSendsSeveralAtOnce()
|
||
{
|
||
var devices = await SeedDevicesAsync("slots", 200);
|
||
var (runner, db, scope) = Runner(fixture, fixture.Scripts, parallelism: 16);
|
||
using var _ = scope;
|
||
var (runId, prepared) = await RecordRunAsync(runner, db, "slots");
|
||
fixture.Scripts.Accept = true;
|
||
|
||
await runner.StartRunAsync(runId, prepared, null, devices, "test", 600, 50, d => d.Hostname, CancellationToken.None);
|
||
var jobs = await JobsAsync(runId);
|
||
var start = jobs.Min(j => j.NotBeforeUtc)!.Value;
|
||
Assert.Equal(4, jobs.Select(j => j.NotBeforeUtc).Distinct().Count());
|
||
var dispatcher = new JobDispatcher(fixture.Services.GetRequiredService<IServiceScopeFactory>(), NullLogger<JobDispatcher>.Instance);
|
||
List<Guid> SentHere()
|
||
{
|
||
lock (fixture.Scripts.Sent)
|
||
{
|
||
var ids = jobs.Select(j => j.JobId).ToHashSet();
|
||
return fixture.Scripts.Sent.Where(s => ids.Contains(s.Command.JobId)).Select(s => s.Command.JobId).ToList();
|
||
}
|
||
}
|
||
|
||
foreach (var (seconds, expected) in new[] { (1, 50), (199, 50), (201, 100), (401, 150), (601, 200), (900, 200) })
|
||
{
|
||
await dispatcher.RunPassAsync(start.AddSeconds(seconds), CancellationToken.None);
|
||
var sent = SentHere();
|
||
Assert.Equal(expected, sent.Count);
|
||
Assert.Equal(sent.Count, sent.Distinct().Count());
|
||
var due = jobs.Where(j => j.NotBeforeUtc <= start.AddSeconds(seconds)).Select(j => j.JobId).ToHashSet();
|
||
Assert.All(sent, id => Assert.Contains(id, due));
|
||
}
|
||
|
||
Assert.All(await JobsAsync(runId), j => Assert.Equal(JobState.Dispatched, j.State));
|
||
}
|
||
|
||
[Fact]
|
||
public async Task HistoryIsPrunedForEveryDeviceOfABatch()
|
||
{
|
||
var devices = await SeedDevicesAsync("pruned", 3);
|
||
var sender = new ConcurrentScriptSender(TimeSpan.Zero);
|
||
var (runner, db, scope) = Runner(fixture, sender, parallelism: 4);
|
||
using var _ = scope;
|
||
var (oldRun, prepared) = await RecordRunAsync(runner, db, "pruned");
|
||
|
||
// 200 old jobs each (the default limit) with output.
|
||
await fixture.WithDbAsync(async d =>
|
||
{
|
||
foreach (var device in devices)
|
||
{
|
||
for (var i = 0; i < 200; i++)
|
||
{
|
||
var id = Guid.NewGuid();
|
||
d.Jobs.Add(new JobEntity
|
||
{
|
||
JobId = id, RunId = oldRun, ClientId = device.ClientId, ScriptName = "old", State = JobState.Succeeded,
|
||
CreatedUtc = DateTimeOffset.UtcNow.AddDays(-1).AddMinutes(i),
|
||
});
|
||
d.JobOutput.Add(new JobOutputEntity { JobId = id, Sequence = 1, Stream = OutputStreamKind.Output, TimestampUtc = DateTimeOffset.UtcNow, Text = "x" });
|
||
}
|
||
}
|
||
|
||
await d.SaveChangesAsync();
|
||
});
|
||
var oldest = await fixture.WithDbAsync(d => d.Jobs.AsNoTracking()
|
||
.Where(j => j.RunId == oldRun).GroupBy(j => j.ClientId).Select(g => g.OrderBy(j => j.CreatedUtc).First().JobId).ToListAsync());
|
||
|
||
var (runId, _) = await RecordRunAsync(runner, db, "pruned-new");
|
||
await runner.StartRunAsync(runId, prepared, null, devices, "test", 0, 0, d => d.Hostname, CancellationToken.None);
|
||
|
||
var counts = await fixture.WithDbAsync(d => d.Jobs.AsNoTracking()
|
||
.Where(j => devices.Select(x => x.ClientId).Contains(j.ClientId)).GroupBy(j => j.ClientId).Select(g => g.Count()).ToListAsync());
|
||
Assert.Equal([200, 200, 200], counts);
|
||
Assert.False(await fixture.WithDbAsync(d => d.Jobs.AnyAsync(j => oldest.Contains(j.JobId))));
|
||
Assert.False(await fixture.WithDbAsync(d => d.JobOutput.AnyAsync(o => oldest.Contains(o.JobId))));
|
||
}
|
||
}
|
||
|
||
public class BulkInsertTests(BackendTestFixture fixture) : IClassFixture<BackendTestFixture>
|
||
{
|
||
[Fact]
|
||
public async Task RowsWrittenTogetherReadBackAsSaveChangesWouldHaveStoredThem()
|
||
{
|
||
var device = await fixture.SeedDeviceAsync("bulk-insert");
|
||
var at = new DateTimeOffset(2026, 11, 7, 12, 30, 15, 123, TimeSpan.Zero).AddTicks(4567);
|
||
var full = new JobEntity
|
||
{
|
||
JobId = Guid.NewGuid(), RunId = Guid.NewGuid(), ClientId = device, ScriptId = Guid.NewGuid(), ScriptName = "every column",
|
||
State = JobState.TimedOut, CreatedUtc = at, StartedUtc = at.AddSeconds(1), CompletedUtc = at.AddSeconds(2), ExitCode = -3,
|
||
RequestedBy = "someone", OutputSummary = "summary ‘quoted’", NotBeforeUtc = at.AddMinutes(5),
|
||
ErrorType = "System.Exception", ErrorId = "Id", ErrorMessage = "message",
|
||
};
|
||
var sparse = new JobEntity { JobId = Guid.NewGuid(), ClientId = device, ScriptName = "nulls", State = JobState.Queued, CreatedUtc = at };
|
||
var others = Enumerable.Range(0, 1_100)
|
||
.Select(i => new JobEntity { JobId = Guid.NewGuid(), ClientId = device, ScriptName = $"n{i}", State = JobState.Dispatched, CreatedUtc = at.AddTicks(i) })
|
||
.ToList();
|
||
|
||
await fixture.WithDbAsync(db => db.InsertRowsAsync([full, sparse, .. others], CancellationToken.None));
|
||
await fixture.WithDbAsync(db => db.InsertRowsAsync(
|
||
[new JobOutputEntity { JobId = full.JobId, Sequence = 7, Stream = OutputStreamKind.Error, TimestampUtc = at, Text = "line" }],
|
||
CancellationToken.None));
|
||
|
||
var stored = await fixture.WithDbAsync(db => db.Jobs.AsNoTracking().Where(j => j.ClientId == device).ToDictionaryAsync(j => j.JobId));
|
||
Assert.Equal(1_102, stored.Count);
|
||
Assert.Equivalent(full, stored[full.JobId], strict: true);
|
||
Assert.Equivalent(sparse, stored[sparse.JobId], strict: true);
|
||
|
||
// What queries on those columns find (state as text, times as the provider keeps them).
|
||
Assert.Equal(1_100, await fixture.WithDbAsync(db => db.Jobs.CountAsync(j => j.ClientId == device && j.State == JobState.Dispatched)));
|
||
Assert.Equal(1, await fixture.WithDbAsync(db => db.Jobs.CountAsync(j => j.ClientId == device && j.NotBeforeUtc > at)));
|
||
var line = await fixture.WithDbAsync(db => db.JobOutput.AsNoTracking().SingleAsync(o => o.JobId == full.JobId));
|
||
Assert.Equal((7, OutputStreamKind.Error, at, "line"), (line.Sequence, line.Stream, line.TimestampUtc, line.Text));
|
||
Assert.True(line.Id > 0);
|
||
|
||
// A row already there fails the whole call, and nothing of it stays.
|
||
var again = new JobEntity { JobId = Guid.NewGuid(), ClientId = device, ScriptName = "again", State = JobState.Queued, CreatedUtc = at };
|
||
await Assert.ThrowsAnyAsync<Exception>(() => fixture.WithDbAsync(db => db.InsertRowsAsync([again, full], CancellationToken.None)));
|
||
Assert.False(await fixture.WithDbAsync(db => db.Jobs.AnyAsync(j => j.JobId == again.JobId)));
|
||
}
|
||
}
|