ea60dc7848
- Script editor: Add From S3 opens the S3 browser in multi-select mode
(files and folders, search, breadcrumb); each pick gets a unique
identifier name. Download To (default $ExecuteRJobDir\Files) with the
variables on hover, per-binding Path inside it, Clean Up Downloads (on).
- Run Script: Review > Files > Change for This Run overrides the selection
for one run (needs scripts.manage or files.send), sealed into the
run's snapshot.
- Storage: FileBindingsJson becomes {root, cleanUp, bindings}; the older
array form still reads (absolute destinations, nothing removed). No
schema change.
- Server: only the selected keys are listed and presigned; single files
carry their listed size; a folder over 2,000 objects is refused.
- Agent: paths expand on the device ($ExecuteRJobDir, $ExecuteRAgentDir,
$ExecuteRAgentPath, %VAR%, $env:VAR, $VAR/${VAR} on Unix, $(...));
undefined variables and ".." fail before the script runs. Missing
folders are created (inheriting the parent's ACL). Four downloads at
once, streamed, retried, checked against size and plain-MD5 ETags,
timeout/cancel aware. Information lines and progress records while it
runs, and a summary. $Files (FileInfo/DirectoryInfo, ordered,
case-insensitive) and $FilesRoot in PowerShell 7 and 5.1;
EXECUTER_FILE_<NAME> and EXECUTER_FILES_ROOT in Bash. Clean Up
Downloads removes only what the job wrote, on success, failure,
timeout or cancel.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
326 lines
13 KiB
C#
326 lines
13 KiB
C#
using System.Text;
|
|
using Agent.Files;
|
|
using Agent.PowerShell;
|
|
using Shared.Contracts.Messages;
|
|
using Shared.Contracts.Models;
|
|
using Xunit;
|
|
|
|
namespace Agent.Tests;
|
|
|
|
/// <summary>
|
|
/// The Download To root and what lands under it: directory creation, refused
|
|
/// traversal, folder structure, parallel and verified downloads with retries,
|
|
/// progress lines, and the ledger Clean Up Downloads works from.
|
|
/// </summary>
|
|
public sealed class ScriptFileDeliveryTests : IDisposable
|
|
{
|
|
private readonly string work = Path.Combine(Path.GetTempPath(), "executer-delivery-" + Guid.NewGuid().ToString("N"));
|
|
private readonly TestFileServer server = new();
|
|
private readonly List<NormalizedOutputRecord> records = [];
|
|
private readonly List<NormalizedProgressRecord> progress = [];
|
|
|
|
public ScriptFileDeliveryTests() => Directory.CreateDirectory(work);
|
|
|
|
public void Dispose()
|
|
{
|
|
server.Dispose();
|
|
try
|
|
{
|
|
Directory.Delete(work, recursive: true);
|
|
}
|
|
catch (IOException)
|
|
{
|
|
}
|
|
}
|
|
|
|
private string JobDir => Path.Combine(work, "job");
|
|
|
|
private static readonly ScriptFileFetcher.FetchSettings Fast = new() { RetryDelay = TimeSpan.FromMilliseconds(10) };
|
|
|
|
private Task<ScriptFileFetcher.FetchResult> FetchAsync(
|
|
IReadOnlyList<ScriptFileDelivery> deliveries,
|
|
ScriptFileOptions? options = null,
|
|
ScriptFileFetcher.FetchSettings? settings = null,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
Directory.CreateDirectory(JobDir);
|
|
var expander = new FilePathExpander(JobDir);
|
|
return ScriptFileFetcher.FetchAsync(
|
|
deliveries, options ?? new ScriptFileOptions(), JobDir, new ServerDownloads(null, false), expander.Expand,
|
|
r => { lock (records) records.Add(r); }, cancellationToken, p => { lock (progress) progress.Add(p); }, settings ?? Fast);
|
|
}
|
|
|
|
private ScriptFileDelivery File(string name, string path, string destination = "", long? size = null)
|
|
=> new(name, destination.Length == 0 ? path.TrimStart('/') : destination, false, false, [new ScriptFileItem("", server.Url(path), size)]);
|
|
|
|
private ScriptFileDelivery Folder(string name, string destination, params string[] relativePaths)
|
|
=> new(name, destination, true, false, [.. relativePaths.Select(r => new ScriptFileItem(r, server.Url($"/{destination}/{r}")))]);
|
|
|
|
private IEnumerable<string> Lines(OutputStreamKind stream)
|
|
{
|
|
lock (records)
|
|
{
|
|
return [.. records.Where(r => r.Stream == stream).Select(r => r.Text)];
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task TheDefaultRootIsCreatedInsideTheJobFolderAndFoldersKeepTheirStructure()
|
|
{
|
|
server.Add("/setup.msi", "installer");
|
|
server.Add("/Tools/a.txt", "alpha");
|
|
server.Add("/Tools/sub/deeper/b.txt", "beta");
|
|
|
|
var result = await FetchAsync([File("Installer", "/setup.msi"), Folder("Tools", "Tools", "a.txt", "sub/deeper/b.txt")]);
|
|
|
|
Assert.Null(result.FailureMessage);
|
|
var root = Path.Combine(JobDir, "Files");
|
|
Assert.Equal(root, result.Root!.FullName);
|
|
Assert.Equal("installer", await System.IO.File.ReadAllTextAsync(Path.Combine(root, "setup.msi")));
|
|
Assert.Equal("beta", await System.IO.File.ReadAllTextAsync(Path.Combine(root, "Tools", "sub", "deeper", "b.txt")));
|
|
Assert.Equal(Path.Combine(root, "setup.msi"), Assert.IsType<FileInfo>(result.Files["installer"]).FullName);
|
|
Assert.Equal(Path.Combine(root, "Tools"), Assert.IsType<DirectoryInfo>(result.Files["TOOLS"]).FullName);
|
|
Assert.True(result.Ledger!.InsideJobFolder);
|
|
Assert.Contains(Path.Combine(root, "Tools", "sub", "deeper"), result.Ledger.CreatedDirectories);
|
|
|
|
// Progress in the output: the plan, each selection, the summary.
|
|
var info = Lines(OutputStreamKind.Information).ToList();
|
|
Assert.StartsWith("Downloading 1 file, 1 folder (3 objects", info[0]);
|
|
Assert.Contains(info, l => l.StartsWith("Downloaded Installer: setup.msi (9 bytes) → "));
|
|
Assert.Contains(info, l => l.StartsWith("Downloaded Tools: 2 files (9 bytes) → "));
|
|
Assert.Matches(@"^Downloaded 1 file, 1 folder, 18 bytes in [\d.]+ s \(.+/s\)\.$", info[^1]);
|
|
Assert.Equal(100, progress[^1].PercentComplete);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task AMissingUserRootIsCreatedWithItsParents()
|
|
{
|
|
server.Add("/a.txt", "a");
|
|
var root = Path.Combine(work, "made", "for", "this");
|
|
|
|
var result = await FetchAsync([File("A", "/a.txt")], new ScriptFileOptions(root));
|
|
|
|
Assert.Null(result.FailureMessage);
|
|
Assert.True(System.IO.File.Exists(Path.Combine(root, "a.txt")));
|
|
Assert.Equal(
|
|
[Path.Combine(work, "made"), Path.Combine(work, "made", "for"), root],
|
|
result.Ledger!.CreatedDirectories);
|
|
Assert.False(result.Ledger.InsideJobFolder);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task AnAbsoluteDestinationIsUsedAsIs()
|
|
{
|
|
server.Add("/a.txt", "a");
|
|
var elsewhere = Path.Combine(work, "elsewhere", "a.txt");
|
|
|
|
var result = await FetchAsync([File("A", "/a.txt", destination: elsewhere)]);
|
|
|
|
Assert.Null(result.FailureMessage);
|
|
Assert.True(System.IO.File.Exists(elsewhere));
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(@"..\escape.txt")]
|
|
[InlineData("sub/../../escape.txt")]
|
|
public async Task ADestinationMayNotClimbOutOfTheRoot(string destination)
|
|
{
|
|
server.Add("/a.txt", "a");
|
|
|
|
var result = await FetchAsync([File("A", "/a.txt", destination: destination)]);
|
|
|
|
Assert.Contains("traversal", result.FailureMessage);
|
|
Assert.False(System.IO.File.Exists(Path.Combine(JobDir, "escape.txt")));
|
|
Assert.Equal(0, server.Requests);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task AFolderObjectMayNotClimbOutOfItsFolder()
|
|
{
|
|
var result = await FetchAsync([new ScriptFileDelivery("Tools", "Tools", true, false, [new ScriptFileItem("../../evil.txt", server.Url("/x"))])]);
|
|
|
|
Assert.Contains("traversal", result.FailureMessage);
|
|
Assert.Equal(0, server.Requests);
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(@"$ExecuteRJobDir\..\Shared", "“..”")]
|
|
[InlineData("Files", "absolute")]
|
|
[InlineData(@"%NOT_DEFINED_ON_THIS_DEVICE%\x", "not defined")]
|
|
public async Task ARootThatCannotBeUsedFailsBeforeAnyDownload(string root, string why)
|
|
{
|
|
server.Add("/a.txt", "a");
|
|
|
|
var result = await FetchAsync([File("A", "/a.txt")], new ScriptFileOptions(root));
|
|
|
|
Assert.StartsWith($"Download To “{root}”", result.FailureMessage);
|
|
Assert.Contains(why, result.FailureMessage);
|
|
Assert.Equal(0, server.Requests);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task SeveralFilesDownloadAtOnceUpToTheLimit()
|
|
{
|
|
for (var i = 0; i < 8; i++)
|
|
{
|
|
server.Add($"/f{i}.bin", new byte[1000]).Delay = TimeSpan.FromMilliseconds(300);
|
|
}
|
|
|
|
var result = await FetchAsync(
|
|
[.. Enumerable.Range(0, 8).Select(i => File($"F{i}", $"/f{i}.bin"))], settings: Fast with { Parallelism = 3 });
|
|
|
|
Assert.Null(result.FailureMessage);
|
|
Assert.InRange(server.MaxConcurrent, 2, 3);
|
|
Assert.Equal(8, Directory.GetFiles(result.Root!.FullName).Length);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task AShortDownloadIsRetriedThenFails()
|
|
{
|
|
server.Add("/a.bin", new byte[100]);
|
|
|
|
var result = await FetchAsync([File("A", "/a.bin", size: 150)], settings: Fast with { Attempts = 2 });
|
|
|
|
Assert.Contains("100 bytes arrived of 150 bytes", result.FailureMessage);
|
|
Assert.Contains(Lines(OutputStreamKind.Warning), l => l.StartsWith("Retrying a.bin (attempt 2 of 2)"));
|
|
Assert.Equal(2, server.Requests);
|
|
Assert.Empty(Directory.GetFiles(result.Root!.FullName));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ContentThatDoesNotMatchTheETagIsRefused()
|
|
{
|
|
server.Add("/a.bin", "content").ETag = new string('0', 32);
|
|
|
|
var result = await FetchAsync([File("A", "/a.bin")], settings: Fast with { Attempts = 1 });
|
|
|
|
Assert.Contains("checksum", result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task AMatchingETagAndSizePass()
|
|
{
|
|
server.Add("/a.bin", "content");
|
|
|
|
var result = await FetchAsync([File("A", "/a.bin", size: 7)]);
|
|
|
|
Assert.Null(result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task AServerErrorIsRetried()
|
|
{
|
|
server.Add("/a.bin", "content").FailFirst = 2;
|
|
|
|
var result = await FetchAsync([File("A", "/a.bin")]);
|
|
|
|
Assert.Null(result.FailureMessage);
|
|
Assert.Equal(2, Lines(OutputStreamKind.Warning).Count(l => l.StartsWith("Retrying a.bin")));
|
|
Assert.Equal(3, server.Requests);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task AMissingObjectIsNotRetried()
|
|
{
|
|
var result = await FetchAsync([File("Missing", "/missing.bin")]);
|
|
|
|
Assert.Contains("“Missing” failed to download: the store answered 404", result.FailureMessage);
|
|
Assert.Equal(1, server.Requests);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ARequiredFailureStopsTheRestButAnOptionalOneOnlyWarns()
|
|
{
|
|
server.Add("/ok.bin", "ok");
|
|
var optional = new ScriptFileDelivery("Optional", "optional.bin", false, true, [new ScriptFileItem("", server.Url("/nope"))]);
|
|
|
|
var result = await FetchAsync([optional, File("Ok", "/ok.bin")]);
|
|
|
|
Assert.Null(result.FailureMessage);
|
|
Assert.Contains(Lines(OutputStreamKind.Warning), l => l.Contains("Optional") && l.EndsWith("continuing."));
|
|
Assert.False(Assert.IsType<FileInfo>(result.Files["Optional"]).Exists);
|
|
Assert.Matches("^Downloaded 1 file, ", Lines(OutputStreamKind.Information).Last());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task CancellingStopsTheDownloadAndLeavesNoPartialFile()
|
|
{
|
|
var slow = server.Add("/slow.bin", new byte[100_000]);
|
|
slow.Chunks = 50;
|
|
slow.ChunkDelay = TimeSpan.FromMilliseconds(100);
|
|
using var cancel = new CancellationTokenSource(TimeSpan.FromMilliseconds(500));
|
|
|
|
var result = await FetchAsync([File("Slow", "/slow.bin")], cancellationToken: cancel.Token);
|
|
|
|
Assert.True(result.Cancelled);
|
|
Assert.Empty(Directory.GetFiles(result.Root!.FullName));
|
|
}
|
|
|
|
[Fact]
|
|
public void ProgressLinesComeAtEachTenthAndOtherwiseOnlyAsAHeartbeat()
|
|
{
|
|
var now = TimeSpan.Zero;
|
|
var lines = new List<string>();
|
|
var bars = new List<NormalizedProgressRecord>();
|
|
var reporter = new DownloadProgress(
|
|
1000, 2, r => lines.Add(r.Text), bars.Add, () => now,
|
|
new DownloadProgress.Throttle(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5)));
|
|
reporter.Started("big.iso");
|
|
|
|
// 1% every 100 ms for 10 s: a line per 10%, no more than one a second.
|
|
for (var i = 0; i < 100; i++)
|
|
{
|
|
now += TimeSpan.FromMilliseconds(100);
|
|
reporter.Add(10);
|
|
}
|
|
|
|
Assert.InRange(lines.Count, 9, 10);
|
|
Assert.Contains("Downloading 0/2: 100 bytes / 1000 bytes (10%), 100 bytes/s — big.iso", lines);
|
|
Assert.InRange(bars.Count, 10, 11);
|
|
|
|
// Stalled: nothing at all for under 5 s, then a heartbeat.
|
|
lines.Clear();
|
|
now += TimeSpan.FromSeconds(4);
|
|
reporter.Add(0);
|
|
Assert.Empty(lines);
|
|
now += TimeSpan.FromSeconds(1.5);
|
|
reporter.Add(0);
|
|
Assert.Single(lines);
|
|
|
|
reporter.Finish("2 files");
|
|
Assert.Matches(@"^Downloaded 2 files, 1000 bytes in 16 s \(64 bytes/s\)\.$", lines[^1]);
|
|
}
|
|
|
|
[Fact]
|
|
public void TheLedgerRemovesOnlyWhatTheJobCreated()
|
|
{
|
|
var root = Path.Combine(work, "user-root");
|
|
Directory.CreateDirectory(root);
|
|
var kept = Path.Combine(root, "kept.txt");
|
|
System.IO.File.WriteAllText(kept, "was here");
|
|
var overwritten = Path.Combine(root, "overwritten.txt");
|
|
System.IO.File.WriteAllText(overwritten, "was here too");
|
|
|
|
var ledger = new DownloadLedger(JobDir);
|
|
ledger.EnsureDirectory(root);
|
|
var created = Path.Combine(root, "new", "deeper");
|
|
ledger.EnsureDirectory(created);
|
|
var downloaded = Path.Combine(created, "a.txt");
|
|
System.IO.File.WriteAllText(downloaded, "ours");
|
|
ledger.RecordFile(downloaded, existedBefore: false);
|
|
ledger.RecordFile(overwritten, existedBefore: true);
|
|
var scriptMade = Path.Combine(root, "new", "script-output.log");
|
|
System.IO.File.WriteAllText(scriptMade, "the script's");
|
|
|
|
Assert.Equal(1, ledger.CleanUp());
|
|
|
|
Assert.False(System.IO.File.Exists(downloaded));
|
|
Assert.False(Directory.Exists(created));
|
|
Assert.True(System.IO.File.Exists(kept));
|
|
Assert.True(System.IO.File.Exists(overwritten));
|
|
// A folder it made that the script put something in stays, with that.
|
|
Assert.True(System.IO.File.Exists(scriptMade));
|
|
Assert.True(Directory.Exists(root));
|
|
}
|
|
}
|