using System.Net;
using System.Net.Http.Json;
using Backend.Data;
using Backend.Files;
using Backend.Jobs;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Shared.Contracts.Api;
using Shared.Contracts.Messages;
using Shared.Contracts.Models;
using Shared.Contracts.Serialization;
using Xunit;
namespace Backend.Tests;
/// How a script's file selection, Download To and Clean Up Downloads are stored and checked, and what is signed for a run.
public class ScriptFileSetTests
{
[Fact]
public void TheFormFromBeforeTheRootReadsAsKeptAbsoluteDestinations()
{
var set = ScriptFileSet.Read("""[{"name":"Tool","objectKey":"tools/a.exe","destinationPath":"%TEMP%\\a.exe"}]""");
Assert.Equal("%TEMP%\\a.exe", Assert.Single(set.Bindings).DestinationPath);
Assert.Null(set.Root);
Assert.False(set.CleanUp);
}
[Fact]
public void TheOptionsRoundTripAndTheDefaultsStoreNothing()
{
var set = new ScriptFileSet([new ScriptFileBinding(" Tool ", "tools/a.exe", " ")], @"D:\Drop", CleanUp: false);
var read = ScriptFileSet.Read(set.Write());
Assert.Equal(@"D:\Drop", read.Root);
Assert.False(read.CleanUp);
Assert.Equal(new ScriptFileBinding("Tool", "tools/a.exe"), Assert.Single(read.Bindings));
Assert.Null(ScriptFileSet.Empty.Write());
Assert.Null(new ScriptFileSet([], ScriptFileOptions.DefaultRoot).Write());
Assert.NotNull(new ScriptFileSet([], CleanUp: false).Write());
Assert.True(ScriptFileSet.Read(null).CleanUp);
}
[Theory]
[InlineData("Tool", "tools/a.exe", null, false, null, null)]
[InlineData("Tool", "tools/", null, true, null, null)]
[InlineData("Tool", "tools/a.exe", "sub/a.exe", false, null, null)]
[InlineData("Tool", "tools/a.exe", "%ProgramData%\\Vendor\\a.exe", false, null, null)]
[InlineData("Tool", "tools/a.exe", "C:\\Vendor\\a.exe", false, "$ExecuteRAgentDir\\Drop", null)]
[InlineData("Tool", "tools/a.exe", null, false, "/opt/drop", null)]
[InlineData("1Tool", "tools/a.exe", null, false, null, "name usable")]
[InlineData("Tool-X", "tools/a.exe", null, false, null, "name usable")]
[InlineData("Tool", "", null, false, null, "valid S3 key")]
[InlineData("Tool", "../a.exe", null, false, null, "valid S3 key")]
[InlineData("Tool", "tools/", null, false, null, "mark it as a folder")]
[InlineData("Tool", "tools/a.exe", "..\\a.exe", false, null, "“..”")]
[InlineData("Tool", "tools/a.exe", "sub/../../a.exe", false, null, "“..”")]
[InlineData("Tool", "tools/", "dest/setup.msi", true, null, "not a file path")]
[InlineData("Tool", "tools/a.exe", null, false, "$ExecuteRJobDir\\..\\x", "“..”")]
[InlineData("Tool", "tools/a.exe", null, false, "relative\\root", "absolute path or start with a variable")]
public void BindingsAndTheRootAreChecked(string name, string key, string? destination, bool folder, string? root, string? problem)
{
var found = ScriptFiles.Validate([new ScriptFileBinding(name, key, destination, folder)], root);
if (problem is null)
{
Assert.Null(found);
}
else
{
Assert.Contains(problem, found);
}
}
[Fact]
public void NamesAreUniqueIgnoringCase()
=> Assert.Contains("unique", ScriptFiles.Validate(
[new ScriptFileBinding("Tool", "a.exe"), new ScriptFileBinding("TOOL", "b.exe")], null));
/// A bucket with more in it than the selection; only the selected keys may be listed or signed.
private sealed class FakeBucket(params (string Key, long Size)[] objects)
{
public List Listed { get; } = [];
public List Signed { get; } = [];
public Task> ListAsync(string prefix, int maxKeys, CancellationToken _)
{
Listed.Add(prefix);
return Task.FromResult>(
[.. objects.Where(o => o.Key.StartsWith(prefix, StringComparison.Ordinal)).OrderBy(o => o.Key, StringComparer.Ordinal).Take(maxKeys)]);
}
public Task PresignAsync(string key, CancellationToken _)
{
Signed.Add(key);
return Task.FromResult($"https://s3.example/{key}?sig");
}
}
[Fact]
public async Task OnlyTheSelectedKeysAreSigned()
{
var bucket = new FakeBucket(
("installers/app.msi", 12), ("installers/app.msi.bak", 3), ("installers/other.msi", 5),
("tools/run.cmd", 7), ("tools/lib/helper.dll", 6), ("toolsets/not-this.txt", 1), ("secret/keys.txt", 9));
var deliveries = await ScriptFiles.ResolveAsync(
[new ScriptFileBinding("App", "installers/app.msi"), new ScriptFileBinding("Tools", "tools", "Kit", Directory: true)],
bucket.ListAsync, bucket.PresignAsync, CancellationToken.None);
Assert.Equal(["installers/app.msi", "tools/lib/helper.dll", "tools/run.cmd"], bucket.Signed.Order(StringComparer.Ordinal));
Assert.Equal(["installers/app.msi", "tools/"], bucket.Listed);
var app = deliveries![0];
Assert.Equal("app.msi", app.DestinationPath);
Assert.Equal(12, Assert.Single(app.Items).SizeBytes);
var tools = deliveries[1];
Assert.Equal("Kit", tools.DestinationPath);
Assert.Equal(["lib/helper.dll", "run.cmd"], tools.Items.Select(i => i.RelativePath));
Assert.Equal([6L, 7L], tools.Items.Select(i => i.SizeBytes ?? 0));
}
[Fact]
public async Task AFolderTooLargeIsRefusedNotCutShort()
{
var bucket = new FakeBucket([.. Enumerable.Range(0, ScriptFiles.MaxFolderObjects + 1).Select(i => ($"big/{i:D5}", 1L))]);
var problem = await Assert.ThrowsAsync(() => ScriptFiles.ResolveAsync(
[new ScriptFileBinding("Big", "big/", Directory: true)], bucket.ListAsync, bucket.PresignAsync, CancellationToken.None));
Assert.Contains($"more than {ScriptFiles.MaxFolderObjects}", problem.Message);
Assert.Empty(bucket.Signed);
}
[Fact]
public async Task AMissingFileFailsUnlessItContinuesOnFailure()
{
var bucket = new FakeBucket(("installers/app.msi.bak", 3));
await Assert.ThrowsAsync(() => ScriptFiles.ResolveAsync(
[new ScriptFileBinding("App", "installers/app.msi")], bucket.ListAsync, bucket.PresignAsync, CancellationToken.None));
var optional = await ScriptFiles.ResolveAsync(
[new ScriptFileBinding("App", "installers/app.msi", ContinueOnFailure: true)], bucket.ListAsync, bucket.PresignAsync, CancellationToken.None);
Assert.Contains("is not in S3", Assert.Single(optional!).Error);
Assert.Empty(bucket.Signed);
}
[Fact]
public async Task AKeyThatMayNotListStillDownloadsWithoutASize()
{
var signed = new List();
var deliveries = await ScriptFiles.ResolveAsync(
[new ScriptFileBinding("App", "installers/app.msi")],
(_, _, _) => throw new InvalidOperationException("Access Denied"),
(key, _) =>
{
signed.Add(key);
return Task.FromResult("u");
},
CancellationToken.None);
Assert.Null(Assert.Single(Assert.Single(deliveries!).Items).SizeBytes);
Assert.Equal(["installers/app.msi"], signed);
}
}
/// The options through the API: stored on the script, validated, and kept by an older console that leaves them out.
public class ScriptFileEndpointTests(BackendTestFixture fixture) : IClassFixture
{
private static async Task SaveAsync(HttpClient client, SaveScriptRequest request, Guid? id = null)
=> id is null
? await client.PostAsJsonAsync(ApiRoutes.Scripts, request, JsonDefaults.Options)
: await client.PutAsJsonAsync(ApiRoutes.For.Script(id.Value), request, JsonDefaults.Options);
private static async Task ReadAsync(HttpResponseMessage response)
{
response.EnsureSuccessStatusCode();
return (await response.Content.ReadFromJsonAsync(JsonDefaults.Options))!;
}
[Fact]
public async Task ANewScriptUsesTheDefaultRootAndCleansUp()
{
using var client = await fixture.SignedInAsync("files-admin");
var created = await ReadAsync(await SaveAsync(client, new SaveScriptRequest(
"files-defaults", null, "'x'", FileBindings: [new ScriptFileBinding("App", "installers/app.msi")])));
Assert.Null(created.FilesRoot);
Assert.True(created.CleanUpFiles);
Assert.Null(Assert.Single(created.FileBindings!).DestinationPath);
}
[Fact]
public async Task TheOptionsAreSavedAndLeftOutKeepsThem()
{
using var client = await fixture.SignedInAsync("files-admin");
var bindings = new[] { new ScriptFileBinding("Kit", "tools/", "Kit", Directory: true) };
var created = await ReadAsync(await SaveAsync(client, new SaveScriptRequest(
"files-saved", null, "'x'", FileBindings: bindings, FilesRoot: "%ProgramData%\\Drop", CleanUpFiles: false)));
Assert.Equal(("%ProgramData%\\Drop", false), (created.FilesRoot, created.CleanUpFiles));
// An older console sends the bindings only.
var kept = await ReadAsync(await SaveAsync(client, new SaveScriptRequest("files-saved", null, "'x'", FileBindings: bindings), created.Id));
Assert.Equal(("%ProgramData%\\Drop", false), (kept.FilesRoot, kept.CleanUpFiles));
// Back to the default root (empty), cleanup on.
var reset = await ReadAsync(await SaveAsync(client, new SaveScriptRequest(
"files-saved", null, "'x'", FileBindings: bindings, FilesRoot: "", CleanUpFiles: true), created.Id));
Assert.Equal((null, true), (reset.FilesRoot, reset.CleanUpFiles));
var stored = await fixture.WithDbAsync(db => db.Scripts.AsNoTracking().SingleAsync(s => s.Id == created.Id));
Assert.StartsWith("{", stored.FileBindingsJson);
}
[Theory]
[InlineData("""[{"name":"A","objectKey":"a.txt","destinationPath":"x\\..\\..\\y"}]""", null)]
[InlineData("""[{"name":"A","objectKey":"a.txt"},{"name":"a","objectKey":"b.txt"}]""", null)]
[InlineData("""[{"name":"A","objectKey":"a.txt"}]""", "Drop")]
public async Task ABadSelectionOrRootIsRefused(string bindingsJson, string? root)
{
using var client = await fixture.SignedInAsync("files-admin");
var bindings = JsonDefaults.Deserialize>(bindingsJson)!;
var response = await SaveAsync(client, new SaveScriptRequest("files-bad", null, "'x'", FileBindings: bindings, FilesRoot: root));
Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode);
}
[Fact]
public async Task AScriptSavedBeforeTheRootKeepsItsBindingsAndDoesNotCleanUp()
{
using var client = await fixture.SignedInAsync("files-admin");
var created = await ReadAsync(await SaveAsync(client, new SaveScriptRequest("files-legacy", null, "'x'")));
await fixture.WithDbAsync(async db =>
{
var script = await db.Scripts.SingleAsync(s => s.Id == created.Id);
script.FileBindingsJson = """[{"name":"Tool","objectKey":"tools/a.exe","destinationPath":"C:\\Tools\\a.exe"}]""";
await db.SaveChangesAsync();
});
var read = (await client.GetFromJsonAsync(ApiRoutes.For.Script(created.Id), JsonDefaults.Options))!;
Assert.Equal("C:\\Tools\\a.exe", Assert.Single(read.FileBindings!).DestinationPath);
Assert.False(read.CleanUpFiles);
}
}
/// On its own fixture with an external S3 store (a fake one): what a run sends, its own selection, and its snapshot.
public sealed class ScriptFileDispatchTests : IAsyncLifetime
{
private readonly FakeS3 s3 = new(["installers/app.msi", "installers/app.msi.bak", "tools/run.cmd", "tools/lib/helper.dll", "other/secret.txt"]);
private readonly BackendTestFixture fixture = new();
public ValueTask InitializeAsync()
{
fixture.Settings["S3_ENABLED"] = "1";
fixture.Settings["S3_ENDPOINT"] = s3.Url;
fixture.Settings["S3_BUCKET"] = "b";
fixture.Settings["S3_ACCESS_KEY"] = "access";
fixture.Settings["S3_SECRET_KEY"] = "secret-key";
fixture.Settings["S3_FORCE_PATH_STYLE"] = "1";
fixture.Scripts.Accept = true;
return ValueTask.CompletedTask;
}
public async ValueTask DisposeAsync()
{
await fixture.DisposeAsync();
await s3.DisposeAsync();
}
private async Task CreateAsync(HttpClient client, string name, string? root = null, bool? cleanUp = null)
{
var response = await client.PostAsJsonAsync(ApiRoutes.Scripts, new SaveScriptRequest(
name, null, "'x'",
FileBindings: [new ScriptFileBinding("App", "installers/app.msi"), new ScriptFileBinding("Tools", "tools/", Directory: true)],
FilesRoot: root, CleanUpFiles: cleanUp), JsonDefaults.Options);
response.EnsureSuccessStatusCode();
return (await response.Content.ReadFromJsonAsync(JsonDefaults.Options))!;
}
private List Sent(string clientId)
{
lock (fixture.Scripts.Sent)
{
return [.. fixture.Scripts.Sent.Where(s => s.ClientId == clientId).Select(s => s.Command)];
}
}
private static IEnumerable Keys(ScriptCommandPayload command)
=> command.Files!.SelectMany(f => f.Items).Select(i => new Uri(i.Url).AbsolutePath.TrimStart('/')["b/".Length..]);
[Fact]
public async Task TheCommandCarriesTheSelectionSignedAndTheOptions()
{
using var client = await fixture.SignedInAsync("dispatch-admin");
var script = await CreateAsync(client, "dispatch-files", "$ExecuteRAgentDir\\Drop", cleanUp: false);
var device = await fixture.SeedDeviceAsync("FILES-DISPATCH");
(await client.PostAsJsonAsync(ApiRoutes.Jobs, new CreateJobRequest(device, script.Id), JsonDefaults.Options)).EnsureSuccessStatusCode();
var command = Assert.Single(Sent(device));
Assert.Equal(new ScriptFileOptions("$ExecuteRAgentDir\\Drop", false), command.FileOptions);
Assert.Equal(["app.msi", "tools"], command.Files!.Select(f => f.DestinationPath));
Assert.Equal(["installers/app.msi", "tools/lib/helper.dll", "tools/run.cmd"], Keys(command).Order(StringComparer.Ordinal));
Assert.Equal(7, command.Files![0].Items[0].SizeBytes);
}
[Fact]
public async Task ARunsOwnSelectionReplacesTheScriptsAndKeepsItsOptions()
{
using var client = await fixture.SignedInAsync("dispatch-admin");
var script = await CreateAsync(client, "dispatch-override", "D:\\Drop");
var device = await fixture.SeedDeviceAsync("FILES-OVERRIDE");
(await client.PostAsJsonAsync(
ApiRoutes.Jobs,
new CreateJobRequest(device, script.Id, Files: [new ScriptFileBinding("Lib", "tools/lib/helper.dll", "bin")]),
JsonDefaults.Options)).EnsureSuccessStatusCode();
var command = Assert.Single(Sent(device));
Assert.Equal(["tools/lib/helper.dll"], Keys(command));
Assert.Equal("bin", Assert.Single(command.Files!).DestinationPath);
Assert.Equal("D:\\Drop", command.FileOptions!.Root);
// The script itself is unchanged.
var stored = await client.GetFromJsonAsync(ApiRoutes.For.Script(script.Id), JsonDefaults.Options);
Assert.Equal(2, stored!.FileBindings!.Count);
}
[Fact]
public async Task ARunsOwnSelectionIsCheckedAndNeedsToSendFiles()
{
using var admin = await fixture.SignedInAsync("dispatch-admin");
var script = await CreateAsync(admin, "dispatch-guarded");
var device = await fixture.SeedDeviceAsync("FILES-GUARDED");
var bad = await admin.PostAsJsonAsync(
ApiRoutes.Jobs, new CreateJobRequest(device, script.Id, Files: [new ScriptFileBinding("X", "../escape")]), JsonDefaults.Options);
Assert.Equal(HttpStatusCode.BadRequest, bad.StatusCode);
var roleId = await fixture.WithDbAsync(async db =>
{
var role = new RoleEntity
{
Id = Guid.NewGuid(), Name = $"runner-{Guid.NewGuid():N}",
PermissionsJson = JsonDefaults.Serialize(new[] { "jobs.run", "devices.view", "scripts.view", "jobs.view" }),
CreatedUtc = DateTimeOffset.UtcNow, UpdatedUtc = DateTimeOffset.UtcNow,
};
db.Roles.Add(role);
await db.SaveChangesAsync();
return role.Id;
});
using var runner = await fixture.SignedInAsync("dispatch-runner", roleId: roleId);
var refused = await runner.PostAsJsonAsync(
ApiRoutes.Jobs, new CreateJobRequest(device, script.Id, Files: [new ScriptFileBinding("X", "other/secret.txt")]), JsonDefaults.Options);
Assert.Equal(HttpStatusCode.Forbidden, refused.StatusCode);
(await runner.PostAsJsonAsync(ApiRoutes.Jobs, new CreateJobRequest(device, script.Id), JsonDefaults.Options)).EnsureSuccessStatusCode();
Assert.DoesNotContain(Sent(device).SelectMany(Keys), key => key == "other/secret.txt");
}
[Fact]
public async Task AStaggeredRunKeepsTheSelectionAndOptionsItStartedWith()
{
using var client = await fixture.SignedInAsync("dispatch-admin");
var script = await CreateAsync(client, "dispatch-snapshot", "D:\\Before");
var devices = new[] { await fixture.SeedDeviceAsync("FILES-SNAP-1"), await fixture.SeedDeviceAsync("FILES-SNAP-2") };
var response = await client.PostAsJsonAsync(
ApiRoutes.Jobs, new CreateJobRequest(null, script.Id, ClientIds: devices, StaggerSeconds: 600), JsonDefaults.Options);
response.EnsureSuccessStatusCode();
var run = (await response.Content.ReadFromJsonAsync(JsonDefaults.Options))!;
(await client.PutAsJsonAsync(ApiRoutes.For.Script(script.Id), new SaveScriptRequest(
"dispatch-snapshot", null, "'x'", FileBindings: [new ScriptFileBinding("Other", "other/secret.txt")], FilesRoot: "D:\\After", CleanUpFiles: false),
JsonDefaults.Options)).EnsureSuccessStatusCode();
var latest = await fixture.WithDbAsync(db => db.Jobs.AsNoTracking().Where(j => j.RunId == run.RunId).MaxAsync(j => j.NotBeforeUtc));
await new JobDispatcher(fixture.Services.GetRequiredService(), NullLogger.Instance)
.RunPassAsync(latest!.Value.AddSeconds(1), CancellationToken.None);
var commands = devices.SelectMany(Sent).ToList();
Assert.Equal(2, commands.Count);
Assert.All(commands, c =>
{
Assert.Equal(new ScriptFileOptions("D:\\Before", true), c.FileOptions);
Assert.Equal(["installers/app.msi", "tools/lib/helper.dll", "tools/run.cmd"], Keys(c).Order(StringComparer.Ordinal));
});
}
}
///
/// The built-in store end to end: objects uploaded through the API, a script binding a file and a folder,
/// a run's command, and every link in it downloading what was uploaded, with the size the command states
/// and, when the ETag is a plain MD5, that MD5 (what the agent checks). Needs the store program
/// (scripts/Get-ObjectStore.ps1); skipped otherwise.
///
public sealed class ScriptFileBuiltInStoreTests(EmbeddedS3StoreTests.Fixture store) : IClassFixture
{
[Fact]
public async Task ARunDownloadsTheSelectionFromTheBuiltInStore()
{
Assert.SkipWhen(store.Program is null, "No store program under artifacts/object-store; run scripts/Get-ObjectStore.ps1.");
var ct = TestContext.Current.CancellationToken;
await using var server = new BackendTestFixture();
server.Settings["S3_EMBEDDED"] = "1";
server.Settings["S3_EMBEDDED_PROGRAM"] = store.Program;
server.Settings["S3_EMBEDDED_DATA"] = Path.Combine(store.Folder, Guid.NewGuid().ToString("N"));
server.Settings["PUBLIC_SERVER_URL"] = string.Empty;
server.Scripts.Accept = true;
using var admin = await server.SignedInAsync("files-store-admin");
using var anonymous = server.CreateClient();
var deadline = DateTime.UtcNow.AddSeconds(90);
while ((await admin.GetFromJsonAsync(ApiRoutes.S3Settings, ct))!.Embedded?.State != S3EmbeddedStates.Running)
{
Assert.True(DateTime.UtcNow < deadline, "The store did not start.");
await Task.Delay(250, ct);
}
var objects = new Dictionary
{
["installers/app.msi"] = System.Security.Cryptography.RandomNumberGenerator.GetBytes(300_000),
["kit/run.cmd"] = "echo hi"u8.ToArray(),
["kit/lib/deep/helper.dll"] = System.Security.Cryptography.RandomNumberGenerator.GetBytes(70_000),
};
foreach (var (key, body) in objects)
{
Assert.Equal(HttpStatusCode.NoContent, (await admin.PutAsync(
$"{ApiRoutes.S3ObjectContent}?key={Uri.EscapeDataString(key)}&overwrite=false", new ByteArrayContent(body), ct)).StatusCode);
}
var created = await admin.PostAsJsonAsync(ApiRoutes.Scripts, new SaveScriptRequest(
"files-built-in", null, "'x'",
FileBindings: [new ScriptFileBinding("App", "installers/app.msi"), new ScriptFileBinding("Kit", "kit/", Directory: true)]),
JsonDefaults.Options, ct);
var script = (await created.Content.ReadFromJsonAsync(JsonDefaults.Options, ct))!;
var device = await server.SeedDeviceAsync("FILES-BUILT-IN");
(await admin.PostAsJsonAsync(ApiRoutes.Jobs, new CreateJobRequest(device, script.Id), JsonDefaults.Options, ct)).EnsureSuccessStatusCode();
ScriptCommandPayload command;
lock (server.Scripts.Sent)
{
command = server.Scripts.Sent.Single(s => s.ClientId == device).Command;
}
Assert.Equal(new ScriptFileOptions(null, true), command.FileOptions);
var downloaded = new Dictionary();
foreach (var delivery in command.Files!)
{
foreach (var item in delivery.Items)
{
Assert.StartsWith("/api/s3/store/", item.Url);
using var response = await anonymous.GetAsync(item.Url, ct);
response.EnsureSuccessStatusCode();
var body = await response.Content.ReadAsByteArrayAsync(ct);
Assert.Equal(item.SizeBytes, body.LongLength);
var etag = response.Headers.ETag?.Tag.Trim('"');
if (etag is { Length: 32 } && etag.All(char.IsAsciiHexDigit))
{
Assert.Equal(Convert.ToHexStringLower(System.Security.Cryptography.MD5.HashData(body)), etag.ToLowerInvariant());
}
downloaded[delivery.Directory ? $"kit/{item.RelativePath}" : "installers/app.msi"] = body;
}
}
Assert.Equal(objects.Keys.Order(), downloaded.Keys.Order());
Assert.All(objects, o => Assert.Equal(o.Value, downloaded[o.Key]));
Assert.Equal(["app.msi", "kit"], command.Files!.Select(f => f.DestinationPath));
}
}