using System.Net; using System.Net.Http.Json; using Backend.Data; using Backend.Files; using Backend.Jobs; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; using Shared.Contracts.Api; using Shared.Contracts.Messages; using Shared.Contracts.Models; using Shared.Contracts.Serialization; using Xunit; namespace Backend.Tests; /// How a script's file selection, Download To and Clean Up Downloads are stored and checked, and what is signed for a run. public class ScriptFileSetTests { [Fact] public void TheFormFromBeforeTheRootReadsAsKeptAbsoluteDestinations() { var set = ScriptFileSet.Read("""[{"name":"Tool","objectKey":"tools/a.exe","destinationPath":"%TEMP%\\a.exe"}]"""); Assert.Equal("%TEMP%\\a.exe", Assert.Single(set.Bindings).DestinationPath); Assert.Null(set.Root); Assert.False(set.CleanUp); } [Fact] public void TheOptionsRoundTripAndTheDefaultsStoreNothing() { var set = new ScriptFileSet([new ScriptFileBinding(" Tool ", "tools/a.exe", " ")], @"D:\Drop", CleanUp: false); var read = ScriptFileSet.Read(set.Write()); Assert.Equal(@"D:\Drop", read.Root); Assert.False(read.CleanUp); Assert.Equal(new ScriptFileBinding("Tool", "tools/a.exe"), Assert.Single(read.Bindings)); Assert.Null(ScriptFileSet.Empty.Write()); Assert.Null(new ScriptFileSet([], ScriptFileOptions.DefaultRoot).Write()); Assert.NotNull(new ScriptFileSet([], CleanUp: false).Write()); Assert.True(ScriptFileSet.Read(null).CleanUp); } [Theory] [InlineData("Tool", "tools/a.exe", null, false, null, null)] [InlineData("Tool", "tools/", null, true, null, null)] [InlineData("Tool", "tools/a.exe", "sub/a.exe", false, null, null)] [InlineData("Tool", "tools/a.exe", "%ProgramData%\\Vendor\\a.exe", false, null, null)] [InlineData("Tool", "tools/a.exe", "C:\\Vendor\\a.exe", false, "$ExecuteRAgentDir\\Drop", null)] [InlineData("Tool", "tools/a.exe", null, false, "/opt/drop", null)] [InlineData("1Tool", "tools/a.exe", null, false, null, "name usable")] [InlineData("Tool-X", "tools/a.exe", null, false, null, "name usable")] [InlineData("Tool", "", null, false, null, "valid S3 key")] [InlineData("Tool", "../a.exe", null, false, null, "valid S3 key")] [InlineData("Tool", "tools/", null, false, null, "mark it as a folder")] [InlineData("Tool", "tools/a.exe", "..\\a.exe", false, null, "“..”")] [InlineData("Tool", "tools/a.exe", "sub/../../a.exe", false, null, "“..”")] [InlineData("Tool", "tools/", "dest/setup.msi", true, null, "not a file path")] [InlineData("Tool", "tools/a.exe", null, false, "$ExecuteRJobDir\\..\\x", "“..”")] [InlineData("Tool", "tools/a.exe", null, false, "relative\\root", "absolute path or start with a variable")] public void BindingsAndTheRootAreChecked(string name, string key, string? destination, bool folder, string? root, string? problem) { var found = ScriptFiles.Validate([new ScriptFileBinding(name, key, destination, folder)], root); if (problem is null) { Assert.Null(found); } else { Assert.Contains(problem, found); } } [Fact] public void NamesAreUniqueIgnoringCase() => Assert.Contains("unique", ScriptFiles.Validate( [new ScriptFileBinding("Tool", "a.exe"), new ScriptFileBinding("TOOL", "b.exe")], null)); /// A bucket with more in it than the selection; only the selected keys may be listed or signed. private sealed class FakeBucket(params (string Key, long Size)[] objects) { public List Listed { get; } = []; public List Signed { get; } = []; public Task> ListAsync(string prefix, int maxKeys, CancellationToken _) { Listed.Add(prefix); return Task.FromResult>( [.. objects.Where(o => o.Key.StartsWith(prefix, StringComparison.Ordinal)).OrderBy(o => o.Key, StringComparer.Ordinal).Take(maxKeys)]); } public Task PresignAsync(string key, CancellationToken _) { Signed.Add(key); return Task.FromResult($"https://s3.example/{key}?sig"); } } [Fact] public async Task OnlyTheSelectedKeysAreSigned() { var bucket = new FakeBucket( ("installers/app.msi", 12), ("installers/app.msi.bak", 3), ("installers/other.msi", 5), ("tools/run.cmd", 7), ("tools/lib/helper.dll", 6), ("toolsets/not-this.txt", 1), ("secret/keys.txt", 9)); var deliveries = await ScriptFiles.ResolveAsync( [new ScriptFileBinding("App", "installers/app.msi"), new ScriptFileBinding("Tools", "tools", "Kit", Directory: true)], bucket.ListAsync, bucket.PresignAsync, CancellationToken.None); Assert.Equal(["installers/app.msi", "tools/lib/helper.dll", "tools/run.cmd"], bucket.Signed.Order(StringComparer.Ordinal)); Assert.Equal(["installers/app.msi", "tools/"], bucket.Listed); var app = deliveries![0]; Assert.Equal("app.msi", app.DestinationPath); Assert.Equal(12, Assert.Single(app.Items).SizeBytes); var tools = deliveries[1]; Assert.Equal("Kit", tools.DestinationPath); Assert.Equal(["lib/helper.dll", "run.cmd"], tools.Items.Select(i => i.RelativePath)); Assert.Equal([6L, 7L], tools.Items.Select(i => i.SizeBytes ?? 0)); } [Fact] public async Task AFolderTooLargeIsRefusedNotCutShort() { var bucket = new FakeBucket([.. Enumerable.Range(0, ScriptFiles.MaxFolderObjects + 1).Select(i => ($"big/{i:D5}", 1L))]); var problem = await Assert.ThrowsAsync(() => ScriptFiles.ResolveAsync( [new ScriptFileBinding("Big", "big/", Directory: true)], bucket.ListAsync, bucket.PresignAsync, CancellationToken.None)); Assert.Contains($"more than {ScriptFiles.MaxFolderObjects}", problem.Message); Assert.Empty(bucket.Signed); } [Fact] public async Task AMissingFileFailsUnlessItContinuesOnFailure() { var bucket = new FakeBucket(("installers/app.msi.bak", 3)); await Assert.ThrowsAsync(() => ScriptFiles.ResolveAsync( [new ScriptFileBinding("App", "installers/app.msi")], bucket.ListAsync, bucket.PresignAsync, CancellationToken.None)); var optional = await ScriptFiles.ResolveAsync( [new ScriptFileBinding("App", "installers/app.msi", ContinueOnFailure: true)], bucket.ListAsync, bucket.PresignAsync, CancellationToken.None); Assert.Contains("is not in S3", Assert.Single(optional!).Error); Assert.Empty(bucket.Signed); } [Fact] public async Task AKeyThatMayNotListStillDownloadsWithoutASize() { var signed = new List(); var deliveries = await ScriptFiles.ResolveAsync( [new ScriptFileBinding("App", "installers/app.msi")], (_, _, _) => throw new InvalidOperationException("Access Denied"), (key, _) => { signed.Add(key); return Task.FromResult("u"); }, CancellationToken.None); Assert.Null(Assert.Single(Assert.Single(deliveries!).Items).SizeBytes); Assert.Equal(["installers/app.msi"], signed); } } /// The options through the API: stored on the script, validated, and kept by an older console that leaves them out. public class ScriptFileEndpointTests(BackendTestFixture fixture) : IClassFixture { private static async Task SaveAsync(HttpClient client, SaveScriptRequest request, Guid? id = null) => id is null ? await client.PostAsJsonAsync(ApiRoutes.Scripts, request, JsonDefaults.Options) : await client.PutAsJsonAsync(ApiRoutes.For.Script(id.Value), request, JsonDefaults.Options); private static async Task ReadAsync(HttpResponseMessage response) { response.EnsureSuccessStatusCode(); return (await response.Content.ReadFromJsonAsync(JsonDefaults.Options))!; } [Fact] public async Task ANewScriptUsesTheDefaultRootAndCleansUp() { using var client = await fixture.SignedInAsync("files-admin"); var created = await ReadAsync(await SaveAsync(client, new SaveScriptRequest( "files-defaults", null, "'x'", FileBindings: [new ScriptFileBinding("App", "installers/app.msi")]))); Assert.Null(created.FilesRoot); Assert.True(created.CleanUpFiles); Assert.Null(Assert.Single(created.FileBindings!).DestinationPath); } [Fact] public async Task TheOptionsAreSavedAndLeftOutKeepsThem() { using var client = await fixture.SignedInAsync("files-admin"); var bindings = new[] { new ScriptFileBinding("Kit", "tools/", "Kit", Directory: true) }; var created = await ReadAsync(await SaveAsync(client, new SaveScriptRequest( "files-saved", null, "'x'", FileBindings: bindings, FilesRoot: "%ProgramData%\\Drop", CleanUpFiles: false))); Assert.Equal(("%ProgramData%\\Drop", false), (created.FilesRoot, created.CleanUpFiles)); // An older console sends the bindings only. var kept = await ReadAsync(await SaveAsync(client, new SaveScriptRequest("files-saved", null, "'x'", FileBindings: bindings), created.Id)); Assert.Equal(("%ProgramData%\\Drop", false), (kept.FilesRoot, kept.CleanUpFiles)); // Back to the default root (empty), cleanup on. var reset = await ReadAsync(await SaveAsync(client, new SaveScriptRequest( "files-saved", null, "'x'", FileBindings: bindings, FilesRoot: "", CleanUpFiles: true), created.Id)); Assert.Equal((null, true), (reset.FilesRoot, reset.CleanUpFiles)); var stored = await fixture.WithDbAsync(db => db.Scripts.AsNoTracking().SingleAsync(s => s.Id == created.Id)); Assert.StartsWith("{", stored.FileBindingsJson); } [Theory] [InlineData("""[{"name":"A","objectKey":"a.txt","destinationPath":"x\\..\\..\\y"}]""", null)] [InlineData("""[{"name":"A","objectKey":"a.txt"},{"name":"a","objectKey":"b.txt"}]""", null)] [InlineData("""[{"name":"A","objectKey":"a.txt"}]""", "Drop")] public async Task ABadSelectionOrRootIsRefused(string bindingsJson, string? root) { using var client = await fixture.SignedInAsync("files-admin"); var bindings = JsonDefaults.Deserialize>(bindingsJson)!; var response = await SaveAsync(client, new SaveScriptRequest("files-bad", null, "'x'", FileBindings: bindings, FilesRoot: root)); Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); } [Fact] public async Task AScriptSavedBeforeTheRootKeepsItsBindingsAndDoesNotCleanUp() { using var client = await fixture.SignedInAsync("files-admin"); var created = await ReadAsync(await SaveAsync(client, new SaveScriptRequest("files-legacy", null, "'x'"))); await fixture.WithDbAsync(async db => { var script = await db.Scripts.SingleAsync(s => s.Id == created.Id); script.FileBindingsJson = """[{"name":"Tool","objectKey":"tools/a.exe","destinationPath":"C:\\Tools\\a.exe"}]"""; await db.SaveChangesAsync(); }); var read = (await client.GetFromJsonAsync(ApiRoutes.For.Script(created.Id), JsonDefaults.Options))!; Assert.Equal("C:\\Tools\\a.exe", Assert.Single(read.FileBindings!).DestinationPath); Assert.False(read.CleanUpFiles); } } /// On its own fixture with an external S3 store (a fake one): what a run sends, its own selection, and its snapshot. public sealed class ScriptFileDispatchTests : IAsyncLifetime { private readonly FakeS3 s3 = new(["installers/app.msi", "installers/app.msi.bak", "tools/run.cmd", "tools/lib/helper.dll", "other/secret.txt"]); private readonly BackendTestFixture fixture = new(); public ValueTask InitializeAsync() { fixture.Settings["S3_ENABLED"] = "1"; fixture.Settings["S3_ENDPOINT"] = s3.Url; fixture.Settings["S3_BUCKET"] = "b"; fixture.Settings["S3_ACCESS_KEY"] = "access"; fixture.Settings["S3_SECRET_KEY"] = "secret-key"; fixture.Settings["S3_FORCE_PATH_STYLE"] = "1"; fixture.Scripts.Accept = true; return ValueTask.CompletedTask; } public async ValueTask DisposeAsync() { await fixture.DisposeAsync(); await s3.DisposeAsync(); } private async Task CreateAsync(HttpClient client, string name, string? root = null, bool? cleanUp = null) { var response = await client.PostAsJsonAsync(ApiRoutes.Scripts, new SaveScriptRequest( name, null, "'x'", FileBindings: [new ScriptFileBinding("App", "installers/app.msi"), new ScriptFileBinding("Tools", "tools/", Directory: true)], FilesRoot: root, CleanUpFiles: cleanUp), JsonDefaults.Options); response.EnsureSuccessStatusCode(); return (await response.Content.ReadFromJsonAsync(JsonDefaults.Options))!; } private List Sent(string clientId) { lock (fixture.Scripts.Sent) { return [.. fixture.Scripts.Sent.Where(s => s.ClientId == clientId).Select(s => s.Command)]; } } private static IEnumerable Keys(ScriptCommandPayload command) => command.Files!.SelectMany(f => f.Items).Select(i => new Uri(i.Url).AbsolutePath.TrimStart('/')["b/".Length..]); [Fact] public async Task TheCommandCarriesTheSelectionSignedAndTheOptions() { using var client = await fixture.SignedInAsync("dispatch-admin"); var script = await CreateAsync(client, "dispatch-files", "$ExecuteRAgentDir\\Drop", cleanUp: false); var device = await fixture.SeedDeviceAsync("FILES-DISPATCH"); (await client.PostAsJsonAsync(ApiRoutes.Jobs, new CreateJobRequest(device, script.Id), JsonDefaults.Options)).EnsureSuccessStatusCode(); var command = Assert.Single(Sent(device)); Assert.Equal(new ScriptFileOptions("$ExecuteRAgentDir\\Drop", false), command.FileOptions); Assert.Equal(["app.msi", "tools"], command.Files!.Select(f => f.DestinationPath)); Assert.Equal(["installers/app.msi", "tools/lib/helper.dll", "tools/run.cmd"], Keys(command).Order(StringComparer.Ordinal)); Assert.Equal(7, command.Files![0].Items[0].SizeBytes); } [Fact] public async Task ARunsOwnSelectionReplacesTheScriptsAndKeepsItsOptions() { using var client = await fixture.SignedInAsync("dispatch-admin"); var script = await CreateAsync(client, "dispatch-override", "D:\\Drop"); var device = await fixture.SeedDeviceAsync("FILES-OVERRIDE"); (await client.PostAsJsonAsync( ApiRoutes.Jobs, new CreateJobRequest(device, script.Id, Files: [new ScriptFileBinding("Lib", "tools/lib/helper.dll", "bin")]), JsonDefaults.Options)).EnsureSuccessStatusCode(); var command = Assert.Single(Sent(device)); Assert.Equal(["tools/lib/helper.dll"], Keys(command)); Assert.Equal("bin", Assert.Single(command.Files!).DestinationPath); Assert.Equal("D:\\Drop", command.FileOptions!.Root); // The script itself is unchanged. var stored = await client.GetFromJsonAsync(ApiRoutes.For.Script(script.Id), JsonDefaults.Options); Assert.Equal(2, stored!.FileBindings!.Count); } [Fact] public async Task ARunsOwnSelectionIsCheckedAndNeedsToSendFiles() { using var admin = await fixture.SignedInAsync("dispatch-admin"); var script = await CreateAsync(admin, "dispatch-guarded"); var device = await fixture.SeedDeviceAsync("FILES-GUARDED"); var bad = await admin.PostAsJsonAsync( ApiRoutes.Jobs, new CreateJobRequest(device, script.Id, Files: [new ScriptFileBinding("X", "../escape")]), JsonDefaults.Options); Assert.Equal(HttpStatusCode.BadRequest, bad.StatusCode); var roleId = await fixture.WithDbAsync(async db => { var role = new RoleEntity { Id = Guid.NewGuid(), Name = $"runner-{Guid.NewGuid():N}", PermissionsJson = JsonDefaults.Serialize(new[] { "jobs.run", "devices.view", "scripts.view", "jobs.view" }), CreatedUtc = DateTimeOffset.UtcNow, UpdatedUtc = DateTimeOffset.UtcNow, }; db.Roles.Add(role); await db.SaveChangesAsync(); return role.Id; }); using var runner = await fixture.SignedInAsync("dispatch-runner", roleId: roleId); var refused = await runner.PostAsJsonAsync( ApiRoutes.Jobs, new CreateJobRequest(device, script.Id, Files: [new ScriptFileBinding("X", "other/secret.txt")]), JsonDefaults.Options); Assert.Equal(HttpStatusCode.Forbidden, refused.StatusCode); (await runner.PostAsJsonAsync(ApiRoutes.Jobs, new CreateJobRequest(device, script.Id), JsonDefaults.Options)).EnsureSuccessStatusCode(); Assert.DoesNotContain(Sent(device).SelectMany(Keys), key => key == "other/secret.txt"); } [Fact] public async Task AStaggeredRunKeepsTheSelectionAndOptionsItStartedWith() { using var client = await fixture.SignedInAsync("dispatch-admin"); var script = await CreateAsync(client, "dispatch-snapshot", "D:\\Before"); var devices = new[] { await fixture.SeedDeviceAsync("FILES-SNAP-1"), await fixture.SeedDeviceAsync("FILES-SNAP-2") }; var response = await client.PostAsJsonAsync( ApiRoutes.Jobs, new CreateJobRequest(null, script.Id, ClientIds: devices, StaggerSeconds: 600), JsonDefaults.Options); response.EnsureSuccessStatusCode(); var run = (await response.Content.ReadFromJsonAsync(JsonDefaults.Options))!; (await client.PutAsJsonAsync(ApiRoutes.For.Script(script.Id), new SaveScriptRequest( "dispatch-snapshot", null, "'x'", FileBindings: [new ScriptFileBinding("Other", "other/secret.txt")], FilesRoot: "D:\\After", CleanUpFiles: false), JsonDefaults.Options)).EnsureSuccessStatusCode(); var latest = await fixture.WithDbAsync(db => db.Jobs.AsNoTracking().Where(j => j.RunId == run.RunId).MaxAsync(j => j.NotBeforeUtc)); await new JobDispatcher(fixture.Services.GetRequiredService(), NullLogger.Instance) .RunPassAsync(latest!.Value.AddSeconds(1), CancellationToken.None); var commands = devices.SelectMany(Sent).ToList(); Assert.Equal(2, commands.Count); Assert.All(commands, c => { Assert.Equal(new ScriptFileOptions("D:\\Before", true), c.FileOptions); Assert.Equal(["installers/app.msi", "tools/lib/helper.dll", "tools/run.cmd"], Keys(c).Order(StringComparer.Ordinal)); }); } } /// /// The built-in store end to end: objects uploaded through the API, a script binding a file and a folder, /// a run's command, and every link in it downloading what was uploaded, with the size the command states /// and, when the ETag is a plain MD5, that MD5 (what the agent checks). Needs the store program /// (scripts/Get-ObjectStore.ps1); skipped otherwise. /// public sealed class ScriptFileBuiltInStoreTests(EmbeddedS3StoreTests.Fixture store) : IClassFixture { [Fact] public async Task ARunDownloadsTheSelectionFromTheBuiltInStore() { Assert.SkipWhen(store.Program is null, "No store program under artifacts/object-store; run scripts/Get-ObjectStore.ps1."); var ct = TestContext.Current.CancellationToken; await using var server = new BackendTestFixture(); server.Settings["S3_EMBEDDED"] = "1"; server.Settings["S3_EMBEDDED_PROGRAM"] = store.Program; server.Settings["S3_EMBEDDED_DATA"] = Path.Combine(store.Folder, Guid.NewGuid().ToString("N")); server.Settings["PUBLIC_SERVER_URL"] = string.Empty; server.Scripts.Accept = true; using var admin = await server.SignedInAsync("files-store-admin"); using var anonymous = server.CreateClient(); var deadline = DateTime.UtcNow.AddSeconds(90); while ((await admin.GetFromJsonAsync(ApiRoutes.S3Settings, ct))!.Embedded?.State != S3EmbeddedStates.Running) { Assert.True(DateTime.UtcNow < deadline, "The store did not start."); await Task.Delay(250, ct); } var objects = new Dictionary { ["installers/app.msi"] = System.Security.Cryptography.RandomNumberGenerator.GetBytes(300_000), ["kit/run.cmd"] = "echo hi"u8.ToArray(), ["kit/lib/deep/helper.dll"] = System.Security.Cryptography.RandomNumberGenerator.GetBytes(70_000), }; foreach (var (key, body) in objects) { Assert.Equal(HttpStatusCode.NoContent, (await admin.PutAsync( $"{ApiRoutes.S3ObjectContent}?key={Uri.EscapeDataString(key)}&overwrite=false", new ByteArrayContent(body), ct)).StatusCode); } var created = await admin.PostAsJsonAsync(ApiRoutes.Scripts, new SaveScriptRequest( "files-built-in", null, "'x'", FileBindings: [new ScriptFileBinding("App", "installers/app.msi"), new ScriptFileBinding("Kit", "kit/", Directory: true)]), JsonDefaults.Options, ct); var script = (await created.Content.ReadFromJsonAsync(JsonDefaults.Options, ct))!; var device = await server.SeedDeviceAsync("FILES-BUILT-IN"); (await admin.PostAsJsonAsync(ApiRoutes.Jobs, new CreateJobRequest(device, script.Id), JsonDefaults.Options, ct)).EnsureSuccessStatusCode(); ScriptCommandPayload command; lock (server.Scripts.Sent) { command = server.Scripts.Sent.Single(s => s.ClientId == device).Command; } Assert.Equal(new ScriptFileOptions(null, true), command.FileOptions); var downloaded = new Dictionary(); foreach (var delivery in command.Files!) { foreach (var item in delivery.Items) { Assert.StartsWith("/api/s3/store/", item.Url); using var response = await anonymous.GetAsync(item.Url, ct); response.EnsureSuccessStatusCode(); var body = await response.Content.ReadAsByteArrayAsync(ct); Assert.Equal(item.SizeBytes, body.LongLength); var etag = response.Headers.ETag?.Tag.Trim('"'); if (etag is { Length: 32 } && etag.All(char.IsAsciiHexDigit)) { Assert.Equal(Convert.ToHexStringLower(System.Security.Cryptography.MD5.HashData(body)), etag.ToLowerInvariant()); } downloaded[delivery.Directory ? $"kit/{item.RelativePath}" : "installers/app.msi"] = body; } } Assert.Equal(objects.Keys.Order(), downloaded.Keys.Order()); Assert.All(objects, o => Assert.Equal(o.Value, downloaded[o.Key])); Assert.Equal(["app.msi", "kit"], command.Files!.Select(f => f.DestinationPath)); } }