using System.Text; using Agent.Files; using Agent.PowerShell; using Shared.Contracts.Messages; using Shared.Contracts.Models; using Xunit; namespace Agent.Tests; /// /// The Download To root and what lands under it: directory creation, refused /// traversal, folder structure, parallel and verified downloads with retries, /// progress lines, and the ledger Clean Up Downloads works from. /// public sealed class ScriptFileDeliveryTests : IDisposable { private readonly string work = Path.Combine(Path.GetTempPath(), "executer-delivery-" + Guid.NewGuid().ToString("N")); private readonly TestFileServer server = new(); private readonly List records = []; private readonly List progress = []; public ScriptFileDeliveryTests() => Directory.CreateDirectory(work); public void Dispose() { server.Dispose(); try { Directory.Delete(work, recursive: true); } catch (IOException) { } } private string JobDir => Path.Combine(work, "job"); private static readonly ScriptFileFetcher.FetchSettings Fast = new() { RetryDelay = TimeSpan.FromMilliseconds(10) }; private Task FetchAsync( IReadOnlyList deliveries, ScriptFileOptions? options = null, ScriptFileFetcher.FetchSettings? settings = null, CancellationToken cancellationToken = default) { Directory.CreateDirectory(JobDir); var expander = new FilePathExpander(JobDir); return ScriptFileFetcher.FetchAsync( deliveries, options ?? new ScriptFileOptions(), JobDir, new ServerDownloads(null, false), expander.Expand, r => { lock (records) records.Add(r); }, cancellationToken, p => { lock (progress) progress.Add(p); }, settings ?? Fast); } private ScriptFileDelivery File(string name, string path, string destination = "", long? size = null) => new(name, destination.Length == 0 ? path.TrimStart('/') : destination, false, false, [new ScriptFileItem("", server.Url(path), size)]); private ScriptFileDelivery Folder(string name, string destination, params string[] relativePaths) => new(name, destination, true, false, [.. relativePaths.Select(r => new ScriptFileItem(r, server.Url($"/{destination}/{r}")))]); private IEnumerable Lines(OutputStreamKind stream) { lock (records) { return [.. records.Where(r => r.Stream == stream).Select(r => r.Text)]; } } [Fact] public async Task TheDefaultRootIsCreatedInsideTheJobFolderAndFoldersKeepTheirStructure() { server.Add("/setup.msi", "installer"); server.Add("/Tools/a.txt", "alpha"); server.Add("/Tools/sub/deeper/b.txt", "beta"); var result = await FetchAsync([File("Installer", "/setup.msi"), Folder("Tools", "Tools", "a.txt", "sub/deeper/b.txt")]); Assert.Null(result.FailureMessage); var root = Path.Combine(JobDir, "Files"); Assert.Equal(root, result.Root!.FullName); Assert.Equal("installer", await System.IO.File.ReadAllTextAsync(Path.Combine(root, "setup.msi"))); Assert.Equal("beta", await System.IO.File.ReadAllTextAsync(Path.Combine(root, "Tools", "sub", "deeper", "b.txt"))); Assert.Equal(Path.Combine(root, "setup.msi"), Assert.IsType(result.Files["installer"]).FullName); Assert.Equal(Path.Combine(root, "Tools"), Assert.IsType(result.Files["TOOLS"]).FullName); Assert.True(result.Ledger!.InsideJobFolder); Assert.Contains(Path.Combine(root, "Tools", "sub", "deeper"), result.Ledger.CreatedDirectories); // Progress in the output: the plan, each selection, the summary. var info = Lines(OutputStreamKind.Information).ToList(); Assert.StartsWith("Downloading 1 file, 1 folder (3 objects", info[0]); Assert.Contains(info, l => l.StartsWith("Downloaded Installer: setup.msi (9 bytes) → ")); Assert.Contains(info, l => l.StartsWith("Downloaded Tools: 2 files (9 bytes) → ")); Assert.Matches(@"^Downloaded 1 file, 1 folder, 18 bytes in [\d.]+ s \(.+/s\)\.$", info[^1]); Assert.Equal(100, progress[^1].PercentComplete); } [Fact] public async Task AMissingUserRootIsCreatedWithItsParents() { server.Add("/a.txt", "a"); var root = Path.Combine(work, "made", "for", "this"); var result = await FetchAsync([File("A", "/a.txt")], new ScriptFileOptions(root)); Assert.Null(result.FailureMessage); Assert.True(System.IO.File.Exists(Path.Combine(root, "a.txt"))); Assert.Equal( [Path.Combine(work, "made"), Path.Combine(work, "made", "for"), root], result.Ledger!.CreatedDirectories); Assert.False(result.Ledger.InsideJobFolder); } [Fact] public async Task AnAbsoluteDestinationIsUsedAsIs() { server.Add("/a.txt", "a"); var elsewhere = Path.Combine(work, "elsewhere", "a.txt"); var result = await FetchAsync([File("A", "/a.txt", destination: elsewhere)]); Assert.Null(result.FailureMessage); Assert.True(System.IO.File.Exists(elsewhere)); } [Theory] [InlineData(@"..\escape.txt")] [InlineData("sub/../../escape.txt")] public async Task ADestinationMayNotClimbOutOfTheRoot(string destination) { server.Add("/a.txt", "a"); var result = await FetchAsync([File("A", "/a.txt", destination: destination)]); Assert.Contains("traversal", result.FailureMessage); Assert.False(System.IO.File.Exists(Path.Combine(JobDir, "escape.txt"))); Assert.Equal(0, server.Requests); } [Fact] public async Task AFolderObjectMayNotClimbOutOfItsFolder() { var result = await FetchAsync([new ScriptFileDelivery("Tools", "Tools", true, false, [new ScriptFileItem("../../evil.txt", server.Url("/x"))])]); Assert.Contains("traversal", result.FailureMessage); Assert.Equal(0, server.Requests); } [Theory] [InlineData(@"$ExecuteRJobDir\..\Shared", "“..”")] [InlineData("Files", "absolute")] [InlineData(@"%NOT_DEFINED_ON_THIS_DEVICE%\x", "not defined")] public async Task ARootThatCannotBeUsedFailsBeforeAnyDownload(string root, string why) { server.Add("/a.txt", "a"); var result = await FetchAsync([File("A", "/a.txt")], new ScriptFileOptions(root)); Assert.StartsWith($"Download To “{root}”", result.FailureMessage); Assert.Contains(why, result.FailureMessage); Assert.Equal(0, server.Requests); } [Fact] public async Task SeveralFilesDownloadAtOnceUpToTheLimit() { for (var i = 0; i < 8; i++) { server.Add($"/f{i}.bin", new byte[1000]).Delay = TimeSpan.FromMilliseconds(300); } var result = await FetchAsync( [.. Enumerable.Range(0, 8).Select(i => File($"F{i}", $"/f{i}.bin"))], settings: Fast with { Parallelism = 3 }); Assert.Null(result.FailureMessage); Assert.InRange(server.MaxConcurrent, 2, 3); Assert.Equal(8, Directory.GetFiles(result.Root!.FullName).Length); } [Fact] public async Task AShortDownloadIsRetriedThenFails() { server.Add("/a.bin", new byte[100]); var result = await FetchAsync([File("A", "/a.bin", size: 150)], settings: Fast with { Attempts = 2 }); Assert.Contains("100 bytes arrived of 150 bytes", result.FailureMessage); Assert.Contains(Lines(OutputStreamKind.Warning), l => l.StartsWith("Retrying a.bin (attempt 2 of 2)")); Assert.Equal(2, server.Requests); Assert.Empty(Directory.GetFiles(result.Root!.FullName)); } [Fact] public async Task ContentThatDoesNotMatchTheETagIsRefused() { server.Add("/a.bin", "content").ETag = new string('0', 32); var result = await FetchAsync([File("A", "/a.bin")], settings: Fast with { Attempts = 1 }); Assert.Contains("checksum", result.FailureMessage); } [Fact] public async Task AMatchingETagAndSizePass() { server.Add("/a.bin", "content"); var result = await FetchAsync([File("A", "/a.bin", size: 7)]); Assert.Null(result.FailureMessage); } [Fact] public async Task AServerErrorIsRetried() { server.Add("/a.bin", "content").FailFirst = 2; var result = await FetchAsync([File("A", "/a.bin")]); Assert.Null(result.FailureMessage); Assert.Equal(2, Lines(OutputStreamKind.Warning).Count(l => l.StartsWith("Retrying a.bin"))); Assert.Equal(3, server.Requests); } [Fact] public async Task AMissingObjectIsNotRetried() { var result = await FetchAsync([File("Missing", "/missing.bin")]); Assert.Contains("“Missing” failed to download: the store answered 404", result.FailureMessage); Assert.Equal(1, server.Requests); } [Fact] public async Task ARequiredFailureStopsTheRestButAnOptionalOneOnlyWarns() { server.Add("/ok.bin", "ok"); var optional = new ScriptFileDelivery("Optional", "optional.bin", false, true, [new ScriptFileItem("", server.Url("/nope"))]); var result = await FetchAsync([optional, File("Ok", "/ok.bin")]); Assert.Null(result.FailureMessage); Assert.Contains(Lines(OutputStreamKind.Warning), l => l.Contains("Optional") && l.EndsWith("continuing.")); Assert.False(Assert.IsType(result.Files["Optional"]).Exists); Assert.Matches("^Downloaded 1 file, ", Lines(OutputStreamKind.Information).Last()); } [Fact] public async Task CancellingStopsTheDownloadAndLeavesNoPartialFile() { var slow = server.Add("/slow.bin", new byte[100_000]); slow.Chunks = 50; slow.ChunkDelay = TimeSpan.FromMilliseconds(100); using var cancel = new CancellationTokenSource(TimeSpan.FromMilliseconds(500)); var result = await FetchAsync([File("Slow", "/slow.bin")], cancellationToken: cancel.Token); Assert.True(result.Cancelled); Assert.Empty(Directory.GetFiles(result.Root!.FullName)); } [Fact] public void ProgressLinesComeAtEachTenthAndOtherwiseOnlyAsAHeartbeat() { var now = TimeSpan.Zero; var lines = new List(); var bars = new List(); var reporter = new DownloadProgress( 1000, 2, r => lines.Add(r.Text), bars.Add, () => now, new DownloadProgress.Throttle(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5))); reporter.Started("big.iso"); // 1% every 100 ms for 10 s: a line per 10%, no more than one a second. for (var i = 0; i < 100; i++) { now += TimeSpan.FromMilliseconds(100); reporter.Add(10); } Assert.InRange(lines.Count, 9, 10); Assert.Contains("Downloading 0/2: 100 bytes / 1000 bytes (10%), 100 bytes/s — big.iso", lines); Assert.InRange(bars.Count, 10, 11); // Stalled: nothing at all for under 5 s, then a heartbeat. lines.Clear(); now += TimeSpan.FromSeconds(4); reporter.Add(0); Assert.Empty(lines); now += TimeSpan.FromSeconds(1.5); reporter.Add(0); Assert.Single(lines); reporter.Finish("2 files"); Assert.Matches(@"^Downloaded 2 files, 1000 bytes in 16 s \(64 bytes/s\)\.$", lines[^1]); } [Fact] public void TheLedgerRemovesOnlyWhatTheJobCreated() { var root = Path.Combine(work, "user-root"); Directory.CreateDirectory(root); var kept = Path.Combine(root, "kept.txt"); System.IO.File.WriteAllText(kept, "was here"); var overwritten = Path.Combine(root, "overwritten.txt"); System.IO.File.WriteAllText(overwritten, "was here too"); var ledger = new DownloadLedger(JobDir); ledger.EnsureDirectory(root); var created = Path.Combine(root, "new", "deeper"); ledger.EnsureDirectory(created); var downloaded = Path.Combine(created, "a.txt"); System.IO.File.WriteAllText(downloaded, "ours"); ledger.RecordFile(downloaded, existedBefore: false); ledger.RecordFile(overwritten, existedBefore: true); var scriptMade = Path.Combine(root, "new", "script-output.log"); System.IO.File.WriteAllText(scriptMade, "the script's"); Assert.Equal(1, ledger.CleanUp()); Assert.False(System.IO.File.Exists(downloaded)); Assert.False(Directory.Exists(created)); Assert.True(System.IO.File.Exists(kept)); Assert.True(System.IO.File.Exists(overwritten)); // A folder it made that the script put something in stays, with that. Assert.True(System.IO.File.Exists(scriptMade)); Assert.True(Directory.Exists(root)); } }