Files
Alphaeus Mote ea60dc7848 feat(scripts): pick S3 files and folders, Download To root, live progress and cleanup [skip ci]
- Script editor: Add From S3 opens the S3 browser in multi-select mode
  (files and folders, search, breadcrumb); each pick gets a unique
  identifier name. Download To (default $ExecuteRJobDir\Files) with the
  variables on hover, per-binding Path inside it, Clean Up Downloads (on).
- Run Script: Review > Files > Change for This Run overrides the selection
  for one run (needs scripts.manage or files.send), sealed into the
  run's snapshot.
- Storage: FileBindingsJson becomes {root, cleanUp, bindings}; the older
  array form still reads (absolute destinations, nothing removed). No
  schema change.
- Server: only the selected keys are listed and presigned; single files
  carry their listed size; a folder over 2,000 objects is refused.
- Agent: paths expand on the device ($ExecuteRJobDir, $ExecuteRAgentDir,
  $ExecuteRAgentPath, %VAR%, $env:VAR, $VAR/${VAR} on Unix, $(...));
  undefined variables and ".." fail before the script runs. Missing
  folders are created (inheriting the parent's ACL). Four downloads at
  once, streamed, retried, checked against size and plain-MD5 ETags,
  timeout/cancel aware. Information lines and progress records while it
  runs, and a summary. $Files (FileInfo/DirectoryInfo, ordered,
  case-insensitive) and $FilesRoot in PowerShell 7 and 5.1;
  EXECUTER_FILE_<NAME> and EXECUTER_FILES_ROOT in Bash. Clean Up
  Downloads removes only what the job wrote, on success, failure,
  timeout or cancel.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-07 06:17:41 -05:00

165 lines
4.9 KiB
C#

using System.Collections.Concurrent;
using System.Net;
using System.Security.Cryptography;
using System.Text;
namespace Agent.Tests;
/// <summary>
/// A loopback HTTP server for real download round trips: objects with an MD5 ETag
/// (or a wrong one), requests that fail before they succeed, slow responses, and
/// a count of how many are served at once.
/// </summary>
internal sealed class TestFileServer : IDisposable
{
private readonly HttpListener listener = new();
private readonly string baseUrl;
private readonly ConcurrentDictionary<string, Entry> entries = new();
private int inFlight;
private int maxInFlight;
private int requests;
public sealed class Entry
{
public required byte[] Body { get; init; }
public string? ETag { get; set; }
/// <summary>Requests answered 500 before the object is served.</summary>
public int FailFirst { get; set; }
/// <summary>A wait before the headers.</summary>
public TimeSpan Delay { get; set; }
/// <summary>The body in this many pieces, with this wait between them.</summary>
public int Chunks { get; set; } = 1;
public TimeSpan ChunkDelay { get; set; }
}
public TestFileServer()
{
for (var attempt = 0; ; attempt++)
{
var probe = new System.Net.Sockets.TcpListener(IPAddress.Loopback, 0);
probe.Start();
var port = ((IPEndPoint)probe.LocalEndpoint).Port;
probe.Stop();
baseUrl = $"http://127.0.0.1:{port}";
listener.Prefixes.Clear();
listener.Prefixes.Add(baseUrl + "/");
try
{
listener.Start();
break;
}
catch (HttpListenerException) when (attempt < 5)
{
}
}
_ = Task.Run(async () =>
{
while (listener.IsListening)
{
HttpListenerContext context;
try
{
context = await listener.GetContextAsync();
}
catch (Exception)
{
return;
}
_ = Task.Run(() => ServeAsync(context));
}
});
}
public int MaxConcurrent => Volatile.Read(ref maxInFlight);
public int Requests => Volatile.Read(ref requests);
public Entry Add(string path, string content, bool etag = true) => Add(path, Encoding.UTF8.GetBytes(content), etag);
public Entry Add(string path, byte[] body, bool etag = true)
{
var entry = new Entry { Body = body, ETag = etag ? Convert.ToHexStringLower(MD5.HashData(body)) : null };
entries[path] = entry;
return entry;
}
public string Url(string path) => baseUrl + path;
private async Task ServeAsync(HttpListenerContext context)
{
Interlocked.Increment(ref requests);
var now = Interlocked.Increment(ref inFlight);
for (var seen = Volatile.Read(ref maxInFlight); now > seen; seen = Volatile.Read(ref maxInFlight))
{
Interlocked.CompareExchange(ref maxInFlight, now, seen);
}
try
{
if (!entries.TryGetValue(context.Request.Url!.AbsolutePath, out var entry))
{
context.Response.StatusCode = 404;
return;
}
if (entry.Delay > TimeSpan.Zero)
{
await Task.Delay(entry.Delay);
}
lock (entry)
{
if (entry.FailFirst > 0)
{
entry.FailFirst--;
context.Response.StatusCode = 500;
return;
}
}
context.Response.StatusCode = 200;
context.Response.ContentLength64 = entry.Body.Length;
if (entry.ETag is not null)
{
context.Response.AddHeader("ETag", $"\"{entry.ETag}\"");
}
var size = Math.Max(1, (entry.Body.Length + entry.Chunks - 1) / entry.Chunks);
for (var offset = 0; offset < entry.Body.Length; offset += size)
{
await context.Response.OutputStream.WriteAsync(entry.Body.AsMemory(offset, Math.Min(size, entry.Body.Length - offset)));
await context.Response.OutputStream.FlushAsync();
if (entry.ChunkDelay > TimeSpan.Zero)
{
await Task.Delay(entry.ChunkDelay);
}
}
}
catch (Exception)
{
// The client went away (a cancelled download).
}
finally
{
Interlocked.Decrement(ref inFlight);
try
{
context.Response.Close();
}
catch (Exception)
{
}
}
}
public void Dispose() => listener.Stop();
}