Files
ExecuteR/tests/Agent.Tests/ScriptFileDeliveryTests.cs
Alphaeus Mote ea60dc7848 feat(scripts): pick S3 files and folders, Download To root, live progress and cleanup [skip ci]
- Script editor: Add From S3 opens the S3 browser in multi-select mode
  (files and folders, search, breadcrumb); each pick gets a unique
  identifier name. Download To (default $ExecuteRJobDir\Files) with the
  variables on hover, per-binding Path inside it, Clean Up Downloads (on).
- Run Script: Review > Files > Change for This Run overrides the selection
  for one run (needs scripts.manage or files.send), sealed into the
  run's snapshot.
- Storage: FileBindingsJson becomes {root, cleanUp, bindings}; the older
  array form still reads (absolute destinations, nothing removed). No
  schema change.
- Server: only the selected keys are listed and presigned; single files
  carry their listed size; a folder over 2,000 objects is refused.
- Agent: paths expand on the device ($ExecuteRJobDir, $ExecuteRAgentDir,
  $ExecuteRAgentPath, %VAR%, $env:VAR, $VAR/${VAR} on Unix, $(...));
  undefined variables and ".." fail before the script runs. Missing
  folders are created (inheriting the parent's ACL). Four downloads at
  once, streamed, retried, checked against size and plain-MD5 ETags,
  timeout/cancel aware. Information lines and progress records while it
  runs, and a summary. $Files (FileInfo/DirectoryInfo, ordered,
  case-insensitive) and $FilesRoot in PowerShell 7 and 5.1;
  EXECUTER_FILE_<NAME> and EXECUTER_FILES_ROOT in Bash. Clean Up
  Downloads removes only what the job wrote, on success, failure,
  timeout or cancel.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-07 06:17:41 -05:00

326 lines
13 KiB
C#

using System.Text;
using Agent.Files;
using Agent.PowerShell;
using Shared.Contracts.Messages;
using Shared.Contracts.Models;
using Xunit;
namespace Agent.Tests;
/// <summary>
/// The Download To root and what lands under it: directory creation, refused
/// traversal, folder structure, parallel and verified downloads with retries,
/// progress lines, and the ledger Clean Up Downloads works from.
/// </summary>
public sealed class ScriptFileDeliveryTests : IDisposable
{
private readonly string work = Path.Combine(Path.GetTempPath(), "executer-delivery-" + Guid.NewGuid().ToString("N"));
private readonly TestFileServer server = new();
private readonly List<NormalizedOutputRecord> records = [];
private readonly List<NormalizedProgressRecord> progress = [];
public ScriptFileDeliveryTests() => Directory.CreateDirectory(work);
public void Dispose()
{
server.Dispose();
try
{
Directory.Delete(work, recursive: true);
}
catch (IOException)
{
}
}
private string JobDir => Path.Combine(work, "job");
private static readonly ScriptFileFetcher.FetchSettings Fast = new() { RetryDelay = TimeSpan.FromMilliseconds(10) };
private Task<ScriptFileFetcher.FetchResult> FetchAsync(
IReadOnlyList<ScriptFileDelivery> deliveries,
ScriptFileOptions? options = null,
ScriptFileFetcher.FetchSettings? settings = null,
CancellationToken cancellationToken = default)
{
Directory.CreateDirectory(JobDir);
var expander = new FilePathExpander(JobDir);
return ScriptFileFetcher.FetchAsync(
deliveries, options ?? new ScriptFileOptions(), JobDir, new ServerDownloads(null, false), expander.Expand,
r => { lock (records) records.Add(r); }, cancellationToken, p => { lock (progress) progress.Add(p); }, settings ?? Fast);
}
private ScriptFileDelivery File(string name, string path, string destination = "", long? size = null)
=> new(name, destination.Length == 0 ? path.TrimStart('/') : destination, false, false, [new ScriptFileItem("", server.Url(path), size)]);
private ScriptFileDelivery Folder(string name, string destination, params string[] relativePaths)
=> new(name, destination, true, false, [.. relativePaths.Select(r => new ScriptFileItem(r, server.Url($"/{destination}/{r}")))]);
private IEnumerable<string> Lines(OutputStreamKind stream)
{
lock (records)
{
return [.. records.Where(r => r.Stream == stream).Select(r => r.Text)];
}
}
[Fact]
public async Task TheDefaultRootIsCreatedInsideTheJobFolderAndFoldersKeepTheirStructure()
{
server.Add("/setup.msi", "installer");
server.Add("/Tools/a.txt", "alpha");
server.Add("/Tools/sub/deeper/b.txt", "beta");
var result = await FetchAsync([File("Installer", "/setup.msi"), Folder("Tools", "Tools", "a.txt", "sub/deeper/b.txt")]);
Assert.Null(result.FailureMessage);
var root = Path.Combine(JobDir, "Files");
Assert.Equal(root, result.Root!.FullName);
Assert.Equal("installer", await System.IO.File.ReadAllTextAsync(Path.Combine(root, "setup.msi")));
Assert.Equal("beta", await System.IO.File.ReadAllTextAsync(Path.Combine(root, "Tools", "sub", "deeper", "b.txt")));
Assert.Equal(Path.Combine(root, "setup.msi"), Assert.IsType<FileInfo>(result.Files["installer"]).FullName);
Assert.Equal(Path.Combine(root, "Tools"), Assert.IsType<DirectoryInfo>(result.Files["TOOLS"]).FullName);
Assert.True(result.Ledger!.InsideJobFolder);
Assert.Contains(Path.Combine(root, "Tools", "sub", "deeper"), result.Ledger.CreatedDirectories);
// Progress in the output: the plan, each selection, the summary.
var info = Lines(OutputStreamKind.Information).ToList();
Assert.StartsWith("Downloading 1 file, 1 folder (3 objects", info[0]);
Assert.Contains(info, l => l.StartsWith("Downloaded Installer: setup.msi (9 bytes) → "));
Assert.Contains(info, l => l.StartsWith("Downloaded Tools: 2 files (9 bytes) → "));
Assert.Matches(@"^Downloaded 1 file, 1 folder, 18 bytes in [\d.]+ s \(.+/s\)\.$", info[^1]);
Assert.Equal(100, progress[^1].PercentComplete);
}
[Fact]
public async Task AMissingUserRootIsCreatedWithItsParents()
{
server.Add("/a.txt", "a");
var root = Path.Combine(work, "made", "for", "this");
var result = await FetchAsync([File("A", "/a.txt")], new ScriptFileOptions(root));
Assert.Null(result.FailureMessage);
Assert.True(System.IO.File.Exists(Path.Combine(root, "a.txt")));
Assert.Equal(
[Path.Combine(work, "made"), Path.Combine(work, "made", "for"), root],
result.Ledger!.CreatedDirectories);
Assert.False(result.Ledger.InsideJobFolder);
}
[Fact]
public async Task AnAbsoluteDestinationIsUsedAsIs()
{
server.Add("/a.txt", "a");
var elsewhere = Path.Combine(work, "elsewhere", "a.txt");
var result = await FetchAsync([File("A", "/a.txt", destination: elsewhere)]);
Assert.Null(result.FailureMessage);
Assert.True(System.IO.File.Exists(elsewhere));
}
[Theory]
[InlineData(@"..\escape.txt")]
[InlineData("sub/../../escape.txt")]
public async Task ADestinationMayNotClimbOutOfTheRoot(string destination)
{
server.Add("/a.txt", "a");
var result = await FetchAsync([File("A", "/a.txt", destination: destination)]);
Assert.Contains("traversal", result.FailureMessage);
Assert.False(System.IO.File.Exists(Path.Combine(JobDir, "escape.txt")));
Assert.Equal(0, server.Requests);
}
[Fact]
public async Task AFolderObjectMayNotClimbOutOfItsFolder()
{
var result = await FetchAsync([new ScriptFileDelivery("Tools", "Tools", true, false, [new ScriptFileItem("../../evil.txt", server.Url("/x"))])]);
Assert.Contains("traversal", result.FailureMessage);
Assert.Equal(0, server.Requests);
}
[Theory]
[InlineData(@"$ExecuteRJobDir\..\Shared", "“..”")]
[InlineData("Files", "absolute")]
[InlineData(@"%NOT_DEFINED_ON_THIS_DEVICE%\x", "not defined")]
public async Task ARootThatCannotBeUsedFailsBeforeAnyDownload(string root, string why)
{
server.Add("/a.txt", "a");
var result = await FetchAsync([File("A", "/a.txt")], new ScriptFileOptions(root));
Assert.StartsWith($"Download To “{root}”", result.FailureMessage);
Assert.Contains(why, result.FailureMessage);
Assert.Equal(0, server.Requests);
}
[Fact]
public async Task SeveralFilesDownloadAtOnceUpToTheLimit()
{
for (var i = 0; i < 8; i++)
{
server.Add($"/f{i}.bin", new byte[1000]).Delay = TimeSpan.FromMilliseconds(300);
}
var result = await FetchAsync(
[.. Enumerable.Range(0, 8).Select(i => File($"F{i}", $"/f{i}.bin"))], settings: Fast with { Parallelism = 3 });
Assert.Null(result.FailureMessage);
Assert.InRange(server.MaxConcurrent, 2, 3);
Assert.Equal(8, Directory.GetFiles(result.Root!.FullName).Length);
}
[Fact]
public async Task AShortDownloadIsRetriedThenFails()
{
server.Add("/a.bin", new byte[100]);
var result = await FetchAsync([File("A", "/a.bin", size: 150)], settings: Fast with { Attempts = 2 });
Assert.Contains("100 bytes arrived of 150 bytes", result.FailureMessage);
Assert.Contains(Lines(OutputStreamKind.Warning), l => l.StartsWith("Retrying a.bin (attempt 2 of 2)"));
Assert.Equal(2, server.Requests);
Assert.Empty(Directory.GetFiles(result.Root!.FullName));
}
[Fact]
public async Task ContentThatDoesNotMatchTheETagIsRefused()
{
server.Add("/a.bin", "content").ETag = new string('0', 32);
var result = await FetchAsync([File("A", "/a.bin")], settings: Fast with { Attempts = 1 });
Assert.Contains("checksum", result.FailureMessage);
}
[Fact]
public async Task AMatchingETagAndSizePass()
{
server.Add("/a.bin", "content");
var result = await FetchAsync([File("A", "/a.bin", size: 7)]);
Assert.Null(result.FailureMessage);
}
[Fact]
public async Task AServerErrorIsRetried()
{
server.Add("/a.bin", "content").FailFirst = 2;
var result = await FetchAsync([File("A", "/a.bin")]);
Assert.Null(result.FailureMessage);
Assert.Equal(2, Lines(OutputStreamKind.Warning).Count(l => l.StartsWith("Retrying a.bin")));
Assert.Equal(3, server.Requests);
}
[Fact]
public async Task AMissingObjectIsNotRetried()
{
var result = await FetchAsync([File("Missing", "/missing.bin")]);
Assert.Contains("“Missing” failed to download: the store answered 404", result.FailureMessage);
Assert.Equal(1, server.Requests);
}
[Fact]
public async Task ARequiredFailureStopsTheRestButAnOptionalOneOnlyWarns()
{
server.Add("/ok.bin", "ok");
var optional = new ScriptFileDelivery("Optional", "optional.bin", false, true, [new ScriptFileItem("", server.Url("/nope"))]);
var result = await FetchAsync([optional, File("Ok", "/ok.bin")]);
Assert.Null(result.FailureMessage);
Assert.Contains(Lines(OutputStreamKind.Warning), l => l.Contains("Optional") && l.EndsWith("continuing."));
Assert.False(Assert.IsType<FileInfo>(result.Files["Optional"]).Exists);
Assert.Matches("^Downloaded 1 file, ", Lines(OutputStreamKind.Information).Last());
}
[Fact]
public async Task CancellingStopsTheDownloadAndLeavesNoPartialFile()
{
var slow = server.Add("/slow.bin", new byte[100_000]);
slow.Chunks = 50;
slow.ChunkDelay = TimeSpan.FromMilliseconds(100);
using var cancel = new CancellationTokenSource(TimeSpan.FromMilliseconds(500));
var result = await FetchAsync([File("Slow", "/slow.bin")], cancellationToken: cancel.Token);
Assert.True(result.Cancelled);
Assert.Empty(Directory.GetFiles(result.Root!.FullName));
}
[Fact]
public void ProgressLinesComeAtEachTenthAndOtherwiseOnlyAsAHeartbeat()
{
var now = TimeSpan.Zero;
var lines = new List<string>();
var bars = new List<NormalizedProgressRecord>();
var reporter = new DownloadProgress(
1000, 2, r => lines.Add(r.Text), bars.Add, () => now,
new DownloadProgress.Throttle(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5)));
reporter.Started("big.iso");
// 1% every 100 ms for 10 s: a line per 10%, no more than one a second.
for (var i = 0; i < 100; i++)
{
now += TimeSpan.FromMilliseconds(100);
reporter.Add(10);
}
Assert.InRange(lines.Count, 9, 10);
Assert.Contains("Downloading 0/2: 100 bytes / 1000 bytes (10%), 100 bytes/s — big.iso", lines);
Assert.InRange(bars.Count, 10, 11);
// Stalled: nothing at all for under 5 s, then a heartbeat.
lines.Clear();
now += TimeSpan.FromSeconds(4);
reporter.Add(0);
Assert.Empty(lines);
now += TimeSpan.FromSeconds(1.5);
reporter.Add(0);
Assert.Single(lines);
reporter.Finish("2 files");
Assert.Matches(@"^Downloaded 2 files, 1000 bytes in 16 s \(64 bytes/s\)\.$", lines[^1]);
}
[Fact]
public void TheLedgerRemovesOnlyWhatTheJobCreated()
{
var root = Path.Combine(work, "user-root");
Directory.CreateDirectory(root);
var kept = Path.Combine(root, "kept.txt");
System.IO.File.WriteAllText(kept, "was here");
var overwritten = Path.Combine(root, "overwritten.txt");
System.IO.File.WriteAllText(overwritten, "was here too");
var ledger = new DownloadLedger(JobDir);
ledger.EnsureDirectory(root);
var created = Path.Combine(root, "new", "deeper");
ledger.EnsureDirectory(created);
var downloaded = Path.Combine(created, "a.txt");
System.IO.File.WriteAllText(downloaded, "ours");
ledger.RecordFile(downloaded, existedBefore: false);
ledger.RecordFile(overwritten, existedBefore: true);
var scriptMade = Path.Combine(root, "new", "script-output.log");
System.IO.File.WriteAllText(scriptMade, "the script's");
Assert.Equal(1, ledger.CleanUp());
Assert.False(System.IO.File.Exists(downloaded));
Assert.False(Directory.Exists(created));
Assert.True(System.IO.File.Exists(kept));
Assert.True(System.IO.File.Exists(overwritten));
// A folder it made that the script put something in stays, with that.
Assert.True(System.IO.File.Exists(scriptMade));
Assert.True(Directory.Exists(root));
}
}