Files
Alphaeus Mote ea60dc7848 feat(scripts): pick S3 files and folders, Download To root, live progress and cleanup [skip ci]
- Script editor: Add From S3 opens the S3 browser in multi-select mode
  (files and folders, search, breadcrumb); each pick gets a unique
  identifier name. Download To (default $ExecuteRJobDir\Files) with the
  variables on hover, per-binding Path inside it, Clean Up Downloads (on).
- Run Script: Review > Files > Change for This Run overrides the selection
  for one run (needs scripts.manage or files.send), sealed into the
  run's snapshot.
- Storage: FileBindingsJson becomes {root, cleanUp, bindings}; the older
  array form still reads (absolute destinations, nothing removed). No
  schema change.
- Server: only the selected keys are listed and presigned; single files
  carry their listed size; a folder over 2,000 objects is refused.
- Agent: paths expand on the device ($ExecuteRJobDir, $ExecuteRAgentDir,
  $ExecuteRAgentPath, %VAR%, $env:VAR, $VAR/${VAR} on Unix, $(...));
  undefined variables and ".." fail before the script runs. Missing
  folders are created (inheriting the parent's ACL). Four downloads at
  once, streamed, retried, checked against size and plain-MD5 ETags,
  timeout/cancel aware. Information lines and progress records while it
  runs, and a summary. $Files (FileInfo/DirectoryInfo, ordered,
  case-insensitive) and $FilesRoot in PowerShell 7 and 5.1;
  EXECUTER_FILE_<NAME> and EXECUTER_FILES_ROOT in Bash. Clean Up
  Downloads removes only what the job wrote, on success, failure,
  timeout or cancel.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-07 06:17:41 -05:00

243 lines
9.7 KiB
C#

using Agent.PowerShell;
using Agent.Storage;
using Microsoft.Extensions.Logging.Abstractions;
using Shared.Contracts.Messages;
using Shared.Contracts.Models;
using Shared.Infrastructure.Configuration;
using Xunit;
namespace Agent.Tests;
/// <summary>
/// File deliveries through a whole job: $Files as FileInfo / DirectoryInfo and
/// $FilesRoot in every engine, Bash's variables, and Clean Up Downloads after a
/// success, a failure, a timeout and a cancellation, never touching what was
/// already in a chosen root.
/// </summary>
public sealed class ScriptFileJobTests : IDisposable
{
private readonly string work = Path.Combine(Path.GetTempPath(), "executer-file-jobs-" + Guid.NewGuid().ToString("N"));
private readonly TestFileServer server = new();
private readonly RecordingPublisher publisher = new(() => DateTimeOffset.UtcNow);
private readonly JobHistoryStore history;
public ScriptFileJobTests()
{
Directory.CreateDirectory(work);
history = new JobHistoryStore(Path.Combine(work, "jobs.db"));
server.Add("/config.json", "{\"a\":1}");
server.Add("/tools/run.cmd", "echo hi");
server.Add("/tools/lib/helper.dll", "binary");
}
public void Dispose()
{
history.Dispose();
server.Dispose();
try
{
Directory.Delete(work, recursive: true);
}
catch (IOException)
{
}
}
private string JobRootPath => Path.Combine(work, "jobs");
private PowerShellJobHandler Handler() => new(
new PowerShellExecutionService(), publisher, history, new Agent.Files.ServerDownloads(null, false), NullLogger.Instance,
jobRoot: new JobRootInfo(JobRootPath, ConfigurationSource.Default));
private IReadOnlyList<ScriptFileDelivery> Deliveries() =>
[
new("Config", "config.json", false, false, [new ScriptFileItem("", server.Url("/config.json"), 7)]),
new("Tools", "tools", true, false,
[
new ScriptFileItem("run.cmd", server.Url("/tools/run.cmd"), 7),
new ScriptFileItem("lib/helper.dll", server.Url("/tools/lib/helper.dll"), 6),
]),
];
private async Task<(ScriptCommandPayload Command, JobStatePayload State, List<string> Output)> RunAsync(
string content,
ScriptEngine engine = ScriptEngine.PowerShell,
ScriptFileOptions? options = null,
bool cleanUpAfterRun = false,
int timeoutSeconds = 120,
CancellationToken cancellationToken = default,
IReadOnlyList<ScriptFileDelivery>? deliveries = null)
{
var command = new ScriptCommandPayload(
Guid.NewGuid(), Guid.NewGuid(), "Files Test", content, timeoutSeconds, Engine: engine,
Files: deliveries ?? Deliveries(), FileOptions: options ?? new ScriptFileOptions(), CleanUpAfterRun: cleanUpAfterRun);
await Handler().RunJobAsync(command, Guid.NewGuid(), null, cancellationToken);
var state = publisher.States.Last();
var output = publisher.Output.Where(o => o.JobId == command.JobId).OrderBy(o => o.Sequence).Select(o => o.Text).ToList();
return (command, state, output);
}
private string JobDir(ScriptCommandPayload command) => Path.Combine(JobRootPath, command.JobId.ToString("D"));
private const string TypesScript = """
"config=$($Files.Config.GetType().FullName)"
"tools=$($Files['tools'].GetType().FullName)"
"root=$($FilesRoot.GetType().FullName)"
"rootpath=$($FilesRoot.FullName)"
"configpath=$($Files.Config.FullName)"
"content=$(Get-Content -Raw -LiteralPath $Files.Config.FullName)"
"helper=$(Test-Path -LiteralPath (Join-Path $Files.Tools.FullName 'lib\helper.dll'))"
"order=$($Files.Keys -join ',')"
""";
private void AssertTypes(ScriptCommandPayload command, List<string> output)
{
var root = Path.Combine(JobDir(command), "Files");
Assert.Contains("config=System.IO.FileInfo", output);
Assert.Contains("tools=System.IO.DirectoryInfo", output);
Assert.Contains("root=System.IO.DirectoryInfo", output);
Assert.Contains($"rootpath={root}", output);
Assert.Contains($"configpath={Path.Combine(root, "config.json")}", output);
Assert.Contains("content={\"a\":1}", output);
Assert.Contains("helper=True", output);
Assert.Contains("order=Config,Tools", output);
}
[Fact]
public async Task PowerShell7SeesFileInfoDirectoryInfoAndTheRoot()
{
var (command, state, output) = await RunAsync(TypesScript);
Assert.Equal(JobState.Succeeded, state.State);
AssertTypes(command, output);
Assert.Contains(output, l => l.StartsWith("Downloading 1 file, 1 folder (3 objects, 20 bytes) to "));
Assert.Contains(output, l => l.StartsWith("Downloaded 1 file, 1 folder, 20 bytes in "));
}
[Fact]
public async Task WindowsPowerShellSeesTheSame()
{
Assert.SkipUnless(OperatingSystem.IsWindows(), "powershell.exe is Windows-only");
var (command, state, output) = await RunAsync(TypesScript, ScriptEngine.WindowsPowerShell);
Assert.Equal(JobState.Succeeded, state.State);
AssertTypes(command, output);
}
[Fact]
public async Task BashGetsThePathsAsVariables()
{
Assert.SkipUnless(ExternalScriptRunner.IsAvailable(ScriptEngine.Bash), "bash is not available on this machine");
var (command, state, output) = await RunAsync(
"""
echo "named=$Config"
echo "prefixed=$EXECUTER_FILE_CONFIG"
echo "tools=$EXECUTER_FILE_TOOLS"
echo "root=$EXECUTER_FILES_ROOT"
""",
ScriptEngine.Bash);
Assert.Equal(JobState.Succeeded, state.State);
var root = Path.Combine(JobDir(command), "Files");
Assert.Contains($"named={Path.Combine(root, "config.json")}", output);
Assert.Contains($"prefixed={Path.Combine(root, "config.json")}", output);
Assert.Contains($"tools={Path.Combine(root, "tools")}", output);
Assert.Contains($"root={root}", output);
}
[Fact]
public async Task AFailedDownloadFailsTheJobBeforeTheScriptRuns()
{
var deliveries = new List<ScriptFileDelivery>
{
new("Missing", "missing.bin", false, false, [new ScriptFileItem("", server.Url("/missing.bin"))]),
};
var (_, state, output) = await RunAsync("'the script ran'", deliveries: deliveries);
Assert.Equal(JobState.Failed, state.State);
Assert.Contains("“Missing” failed to download", state.Message);
Assert.DoesNotContain("the script ran", output);
}
[Fact]
public async Task DownloadsInsideTheJobFolderGoAfterASuccess()
{
var (command, state, _) = await RunAsync("'ok'");
Assert.Equal(JobState.Succeeded, state.State);
Assert.True(Directory.Exists(JobDir(command)));
Assert.False(Directory.Exists(Path.Combine(JobDir(command), "Files")));
}
[Fact]
public async Task CleanUpOffKeepsThem()
{
var (command, _, _) = await RunAsync("'ok'", options: new ScriptFileOptions(CleanUp: false));
Assert.True(File.Exists(Path.Combine(JobDir(command), "Files", "tools", "lib", "helper.dll")));
}
[Theory]
[InlineData("failure")]
[InlineData("timeout")]
[InlineData("cancel")]
[InlineData("success")]
public async Task AChosenRootLosesOnlyWhatTheJobDownloaded(string outcome)
{
var root = Path.Combine(work, "drop");
Directory.CreateDirectory(root);
File.WriteAllText(Path.Combine(root, "before.txt"), "was here");
Directory.CreateDirectory(Path.Combine(root, "tools"));
File.WriteAllText(Path.Combine(root, "tools", "mine.txt"), "was here too");
using var cancel = new CancellationTokenSource();
var script = outcome switch
{
"failure" => "throw 'broken'",
"timeout" or "cancel" => "Start-Sleep -Seconds 30",
_ => "'fine'",
};
if (outcome == "cancel")
{
cancel.CancelAfter(TimeSpan.FromSeconds(3));
}
var (command, state, _) = await RunAsync(
script, options: new ScriptFileOptions(root), timeoutSeconds: outcome == "timeout" ? 3 : 120, cancellationToken: cancel.Token);
Assert.Equal(
outcome switch { "failure" => JobState.Failed, "timeout" => JobState.TimedOut, "cancel" => JobState.Cancelled, _ => JobState.Succeeded },
state.State);
Assert.False(File.Exists(Path.Combine(root, "config.json")));
Assert.False(File.Exists(Path.Combine(root, "tools", "run.cmd")));
Assert.False(Directory.Exists(Path.Combine(root, "tools", "lib")));
Assert.True(File.Exists(Path.Combine(root, "before.txt")));
Assert.True(File.Exists(Path.Combine(root, "tools", "mine.txt")));
_ = command;
}
[Fact]
public async Task CancellingWhileDownloadingCancelsTheJobAndCleansUp()
{
var slow = server.Add("/slow.bin", new byte[200_000]);
slow.Chunks = 100;
slow.ChunkDelay = TimeSpan.FromMilliseconds(100);
var root = Path.Combine(work, "slow-root");
using var cancel = new CancellationTokenSource(TimeSpan.FromSeconds(1));
var (_, state, output) = await RunAsync(
"'the script ran'",
options: new ScriptFileOptions(root),
cancellationToken: cancel.Token,
deliveries: [new("Slow", "slow.bin", false, false, [new ScriptFileItem("", server.Url("/slow.bin"), 200_000)])]);
Assert.Equal(JobState.Cancelled, state.State);
Assert.Contains("while downloading", state.Message);
Assert.DoesNotContain("the script ran", output);
Assert.False(Directory.Exists(root));
}
}