Major update: Enhanced documentation, comprehensive examples, and fixed remaining issues

- Updated README.md with modern feature descriptions and comprehensive overview
- Enhanced docs/USAGE.md with advanced object listing, directory management, and chunked operations
- Created examples/ directory with 6 comprehensive example scripts:
  * 01-Basic-Operations.ps1 - Fundamental operations for beginners
  * 02-Advanced-Object-Listing.ps1 - Filtering, sorting, and pagination
  * 03-Directory-Management.ps1 - Nested directory structures and organization
  * 04-Chunked-Operations.ps1 - Large file handling with performance optimization
  * 05-Bulk-Operations.ps1 - Batch processing and automation workflows
  * 06-Enterprise-Automation.ps1 - Enterprise monitoring, reporting, and compliance
- Added comprehensive examples/README.md with usage patterns and best practices
- Fixed directory creation warnings (now clean verbose logging)
- Implemented missing Get-MinIOObject cmdlet with full filtering/sorting capabilities
- Added timing and performance metrics to all operations
- Enhanced chunked operations with multi-layer progress tracking
- Improved error handling and resource cleanup
- Removed temporary test files and cleaned up repository structure
- All examples use proper PowerShell output (no Write-Host usage)
- Professional logging with timestamps and structured output
This commit is contained in:
PSMinIO Developer
2025-07-10 22:33:03 -04:00
parent 2974ead76e
commit d887fd7f46
66 changed files with 2475 additions and 3428 deletions
+175
View File
@@ -0,0 +1,175 @@
using System;
using System.Linq;
using System.Management.Automation;
using PSMinIO.Models;
using PSMinIO.Utils;
namespace PSMinIO.Cmdlets
{
/// <summary>
/// Gets information about objects in a MinIO bucket
/// </summary>
[Cmdlet(VerbsCommon.Get, "MinIOObject", SupportsShouldProcess = true)]
[OutputType(typeof(MinIOObjectInfo))]
public class GetMinIOObjectCmdlet : MinIOBaseCmdlet
{
/// <summary>
/// Name of the bucket to list objects from
/// </summary>
[Parameter(Mandatory = true, Position = 0, ValueFromPipelineByPropertyName = true)]
[ValidateNotNullOrEmpty]
[Alias("Bucket")]
public string BucketName { get; set; } = string.Empty;
/// <summary>
/// Optional prefix to filter objects
/// </summary>
[Parameter(Position = 1, ValueFromPipelineByPropertyName = true)]
[Alias("Filter")]
public string? Prefix { get; set; }
/// <summary>
/// Specific object name to retrieve. If specified, only this object is returned.
/// </summary>
[Parameter(ValueFromPipelineByPropertyName = true)]
[Alias("Object", "Name")]
public string? ObjectName { get; set; }
/// <summary>
/// Whether to list objects recursively (default: true)
/// </summary>
[Parameter]
public SwitchParameter Recursive { get; set; } = true;
/// <summary>
/// Include all versions of objects (for versioned buckets)
/// </summary>
[Parameter]
[Alias("Versions")]
public SwitchParameter IncludeVersions { get; set; }
/// <summary>
/// Maximum number of objects to return
/// </summary>
[Parameter]
[ValidateRange(1, 10000)]
[Alias("Limit")]
public int? MaxObjects { get; set; }
/// <summary>
/// Only return objects (exclude directory markers)
/// </summary>
[Parameter]
[Alias("FilesOnly")]
public SwitchParameter ObjectsOnly { get; set; }
/// <summary>
/// Sort objects by the specified property
/// </summary>
[Parameter]
[ValidateSet("Name", "Size", "LastModified", "ETag")]
public string? SortBy { get; set; }
/// <summary>
/// Sort in descending order (default: ascending)
/// </summary>
[Parameter]
[Alias("Desc")]
public SwitchParameter Descending { get; set; }
/// <summary>
/// Processes the cmdlet
/// </summary>
protected override void ProcessRecord()
{
var targetDescription = !string.IsNullOrWhiteSpace(ObjectName)
? $"object '{ObjectName}'"
: !string.IsNullOrWhiteSpace(Prefix)
? $"objects with prefix '{Prefix}'"
: "all objects";
// Determine the prefix to use
var searchPrefix = !string.IsNullOrWhiteSpace(ObjectName) ? ObjectName : Prefix;
if (ShouldProcess(BucketName, $"List {targetDescription}"))
{
ExecuteOperation("ListObjects", () =>
{
// Check if bucket exists
var bucketExists = Client.BucketExists(BucketName);
if (!bucketExists)
{
WriteError(new ErrorRecord(
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
"BucketNotFound",
ErrorCategory.ObjectNotFound,
BucketName));
return;
}
MinIOLogger.WriteVerbose(this,
"Listing objects in bucket '{0}' with prefix '{1}', recursive: {2}",
BucketName, searchPrefix ?? "(none)", Recursive.IsPresent);
// Get objects from MinIO
var objects = Client.ListObjects(BucketName, searchPrefix, Recursive.IsPresent, IncludeVersions.IsPresent);
MinIOLogger.WriteVerbose(this, "Found {0} objects", objects.Count);
// Filter for exact object name match if specified
if (!string.IsNullOrWhiteSpace(ObjectName))
{
objects = objects.Where(obj =>
string.Equals(obj.Name, ObjectName, StringComparison.Ordinal)).ToList();
}
// Filter out directory markers if ObjectsOnly is specified
if (ObjectsOnly.IsPresent)
{
objects = objects.Where(obj =>
!obj.Name.EndsWith("/") && obj.Size > 0).ToList();
}
// Apply sorting if specified
if (!string.IsNullOrWhiteSpace(SortBy))
{
objects = SortBy.ToLowerInvariant() switch
{
"name" => Descending.IsPresent
? objects.OrderByDescending(o => o.Name).ToList()
: objects.OrderBy(o => o.Name).ToList(),
"size" => Descending.IsPresent
? objects.OrderByDescending(o => o.Size).ToList()
: objects.OrderBy(o => o.Size).ToList(),
"lastmodified" => Descending.IsPresent
? objects.OrderByDescending(o => o.LastModified).ToList()
: objects.OrderBy(o => o.LastModified).ToList(),
"etag" => Descending.IsPresent
? objects.OrderByDescending(o => o.ETag).ToList()
: objects.OrderBy(o => o.ETag).ToList(),
_ => objects
};
}
// Apply limit if specified
if (MaxObjects.HasValue && objects.Count > MaxObjects.Value)
{
MinIOLogger.WriteVerbose(this,
"Limiting results to {0} objects", MaxObjects.Value);
objects = objects.Take(MaxObjects.Value).ToList();
}
MinIOLogger.WriteVerbose(this,
"Returning {0} objects after filtering and sorting", objects.Count);
// Output objects
foreach (var obj in objects)
{
WriteObject(obj);
}
}, $"Bucket: {BucketName}, Prefix: {searchPrefix ?? "(none)"}");
}
}
}
}
@@ -47,7 +47,7 @@ namespace PSMinIO.Cmdlets
/// Size of each chunk for download (default: 10MB)
/// </summary>
[Parameter]
[ValidateRange(1024 * 1024, 100 * 1024 * 1024)] // 1MB to 100MB
[ValidateRange(5 * 1024 * 1024, 500 * 1024 * 1024)] // 5MB to 500MB
public long ChunkSize { get; set; } = 10 * 1024 * 1024; // 10MB default
/// <summary>
@@ -128,17 +128,16 @@ namespace PSMinIO.Cmdlets
ObjectName, BucketName, SizeFormatter.FormatBytes(objectInfo.Size), SizeFormatter.FormatBytes(ChunkSize));
// Download using chunked transfer
var downloadedFile = DownloadObjectChunked(objectInfo);
if (downloadedFile != null)
var downloadResult = DownloadObjectChunked(objectInfo);
if (downloadResult != null)
{
MinIOLogger.WriteVerbose(this,
"Successfully downloaded object '{0}' from bucket '{1}' to '{2}'",
MinIOLogger.WriteVerbose(this,
"Successfully downloaded object '{0}' from bucket '{1}' to '{2}'",
ObjectName, BucketName, FilePath.FullName);
// Always return file information
FilePath.Refresh(); // Refresh to get updated file info
WriteObject(FilePath);
// Return download result with timing information
WriteObject(downloadResult);
}
}, $"Bucket: {BucketName}, Object: {ObjectName}, File: {FilePath.FullName}, ChunkSize: {SizeFormatter.FormatBytes(ChunkSize)}");
@@ -167,8 +166,8 @@ namespace PSMinIO.Cmdlets
/// Downloads an object using chunked transfer with resume capability
/// </summary>
/// <param name="objectInfo">Information about the object to download</param>
/// <returns>Downloaded file info or null if failed</returns>
private FileInfo? DownloadObjectChunked(MinIOObjectInfo objectInfo)
/// <returns>Download result with timing information or null if failed</returns>
private MinIODownloadResult? DownloadObjectChunked(MinIOObjectInfo objectInfo)
{
ChunkedTransferState? transferState = null;
@@ -210,29 +209,55 @@ namespace PSMinIO.Cmdlets
// Calculate total chunks for progress reporting
var totalChunks = (int)Math.Ceiling((double)objectInfo.Size / ChunkSize);
// Create progress reporter (single file, so no collection progress)
var progressReporter = new ChunkedSingleFileProgressReporter(
// Validate chunk count to prevent memory issues
if (totalChunks > 5000)
{
var recommendedChunkSize = (long)Math.Ceiling((double)objectInfo.Size / 5000);
WriteWarning($"Too many chunks ({totalChunks:N0}) would be created. Consider using a larger chunk size (recommended: {SizeFormatter.FormatBytes(recommendedChunkSize)})");
ThrowTerminatingError(new ErrorRecord(
new ArgumentException($"Chunk size too small would create {totalChunks:N0} chunks. Maximum allowed is 5,000 chunks."),
"TooManyChunks",
ErrorCategory.InvalidArgument,
ChunkSize));
}
// Create thread-safe progress reporter
var progressReporter = new ThreadSafeChunkedProgressReporter(
this,
objectInfo.Size,
totalChunks,
"Downloading",
ProgressUpdateInterval);
"Downloading");
progressReporter.StartNewFile(ObjectName, objectInfo.Size, totalChunks);
// Track timing for this download
var startTime = DateTime.UtcNow;
try
{
// Download file using chunked transfer
var result = Client.DownloadFileChunked(transferState, progressReporter, MaxRetries, ParallelDownloads);
if (result)
{
var completionTime = DateTime.UtcNow;
// Clean up resume data on successful completion
if (Resume.IsPresent)
{
ChunkedTransferResumeManager.CleanupResumeData(transferState, ResumeDataPath);
}
progressReporter.CompleteDownload();
return FilePath;
progressReporter.CompleteFile();
// Process any queued progress updates from the main thread
progressReporter.ProcessQueuedUpdates();
progressReporter.Complete();
// Create download result with timing information
FilePath.Refresh();
var downloadResult = new MinIODownloadResult(FilePath, BucketName, ObjectName, startTime, completionTime);
return downloadResult;
}
}
#pragma warning disable CS0168 // Variable is declared but never used - false positive, ex is used in error handling
+10 -3
View File
@@ -2,6 +2,7 @@ using System;
using System.IO;
using System.Linq;
using System.Management.Automation;
using PSMinIO.Models;
using PSMinIO.Utils;
namespace PSMinIO.Cmdlets
@@ -97,6 +98,9 @@ namespace PSMinIO.Cmdlets
try
{
// Track timing for this download
var startTime = DateTime.UtcNow;
// Download the file with progress reporting
Client.DownloadFile(
BucketName,
@@ -104,6 +108,8 @@ namespace PSMinIO.Cmdlets
FilePath.FullName,
bytesTransferred => progressReporter.UpdateProgress(bytesTransferred));
var completionTime = DateTime.UtcNow;
// Complete progress reporting
progressReporter.Complete();
@@ -111,9 +117,10 @@ namespace PSMinIO.Cmdlets
"Successfully downloaded object '{0}' from bucket '{1}' to '{2}'",
ObjectName, BucketName, FilePath.FullName);
// Always return file information
FilePath.Refresh(); // Refresh to get updated file info
WriteObject(FilePath);
// Refresh file info and create download result with timing information
FilePath.Refresh();
var downloadResult = new MinIODownloadResult(FilePath, BucketName, ObjectName, startTime, completionTime);
WriteObject(downloadResult);
}
#pragma warning disable CS0168 // Variable is declared but never used - false positive, ex is used in throw
catch (Exception ex)
+2 -3
View File
@@ -152,9 +152,8 @@ namespace PSMinIO.Cmdlets
{
MinIOLogger.WriteVerbose(this, "Creating folder level: {0}", fullFolderPath);
// Create the folder by uploading a zero-byte object
using var emptyStream = new MemoryStream();
var etag = Client.UploadStream(BucketName, fullFolderPath, emptyStream, "application/x-directory");
// Create the folder using the dedicated method
var etag = Client.CreateDirectory(BucketName, fullFolderPath);
var folderInfo = new MinIOObjectInfo(
fullFolderPath,
+41 -33
View File
@@ -323,15 +323,8 @@ namespace PSMinIO.Cmdlets
// Calculate total size for overall progress
var totalSize = validFiles.Sum(f => f.Length);
// Create progress reporter
var progressReporter = new ChunkedCollectionProgressReporter(
this,
validFiles.Length,
totalSize,
"Uploading",
ProgressUpdateInterval);
var uploadedObjects = new System.Collections.Generic.List<MinIOObjectInfo>();
// Create thread-safe result collector
var resultCollector = new ThreadSafeResultCollector(this);
for (int i = 0; i < validFiles.Length; i++)
{
@@ -342,39 +335,47 @@ namespace PSMinIO.Cmdlets
{
// Calculate chunks for this file
var totalChunks = (int)Math.Ceiling((double)fileInfo.Length / ChunkSize);
// Create thread-safe progress reporter for this file
var progressReporter = new ThreadSafeChunkedProgressReporter(
this,
fileInfo.Length,
totalChunks,
"Uploading");
progressReporter.StartNewFile(fileInfo.Name, fileInfo.Length, totalChunks);
// Track timing for this file
var startTime = DateTime.UtcNow;
// Upload file using chunked transfer
var uploadedObject = UploadFileChunked(fileInfo, objectName, progressReporter);
if (uploadedObject != null)
{
uploadedObjects.Add(uploadedObject);
// Add timing information
uploadedObject.StartTime = startTime;
uploadedObject.CompletionTime = DateTime.UtcNow;
resultCollector.QueueResult(uploadedObject);
progressReporter.CompleteFile();
}
// Process any queued progress updates from the main thread
progressReporter.ProcessQueuedUpdates();
progressReporter.Complete();
}
catch (Exception ex)
{
WriteError(new ErrorRecord(
ex,
"ChunkedFileUploadFailed",
ErrorCategory.WriteError,
fileInfo));
resultCollector.QueueError(ex, "ChunkedFileUploadFailed", ErrorCategory.WriteError, fileInfo);
MinIOLogger.WriteVerbose(this, "Failed to upload {0}: {1}", fileInfo.Name, ex.Message);
}
}
progressReporter.CompleteCollection();
MinIOLogger.WriteVerbose(this, "Completed chunked upload: {0} files", validFiles.Length);
// Always return uploaded objects
foreach (var obj in uploadedObjects)
{
WriteObject(obj);
}
MinIOLogger.WriteVerbose(this, "Completed chunked upload: {0} files ({1} successful, {2} failed)",
validFiles.Length, uploadedObjects.Count, validFiles.Length - uploadedObjects.Count);
// Process all results from the main thread
resultCollector.Complete();
}
/// <summary>
@@ -382,9 +383,9 @@ namespace PSMinIO.Cmdlets
/// </summary>
/// <param name="fileInfo">File to upload</param>
/// <param name="objectName">Object name in bucket</param>
/// <param name="progressReporter">Progress reporter</param>
/// <param name="progressReporter">Thread-safe progress reporter</param>
/// <returns>Uploaded object info or null if failed</returns>
private MinIOObjectInfo? UploadFileChunked(FileInfo fileInfo, string objectName, ChunkedCollectionProgressReporter progressReporter)
private MinIOObjectInfo? UploadFileChunked(FileInfo fileInfo, string objectName, ThreadSafeChunkedProgressReporter progressReporter)
{
ChunkedTransferState? transferState = null;
@@ -563,16 +564,23 @@ namespace PSMinIO.Cmdlets
{
MinIOLogger.WriteVerbose(this, "Creating bucket directory: {0}", folderPath);
// Create the directory by uploading a zero-byte object
using var emptyStream = new MemoryStream();
Client.UploadStream(BucketName, folderPath, emptyStream, "application/x-directory");
MinIOLogger.WriteVerbose(this, "Successfully created bucket directory: {0}", folderPath);
try
{
// Create the directory using the dedicated method
Client.CreateDirectory(BucketName, folderPath);
MinIOLogger.WriteVerbose(this, "Successfully created bucket directory: {0}", folderPath);
}
catch (Exception createEx)
{
// Directory creation failed, but this is not critical since MinIO creates directories implicitly
MinIOLogger.WriteVerbose(this, "Directory creation failed (non-critical): {0} - {1}", folderPath, createEx.Message);
}
}
}
catch (Exception ex)
{
WriteWarning($"Could not create bucket directory '{folderPath}': {ex.Message}");
// Listing failed, but this is not critical for the upload operation
MinIOLogger.WriteVerbose(this, "Could not check directory existence (non-critical): {0} - {1}", folderPath, ex.Message);
}
}
}
+23 -7
View File
@@ -317,6 +317,9 @@ namespace PSMinIO.Cmdlets
MinIOLogger.WriteVerbose(this, "Uploading file {0}/{1}: {2} -> {3}",
i + 1, validFiles.Length, fileInfo.Name, objectName);
// Track timing for this file
var startTime = DateTime.UtcNow;
// Upload the file
var etag = Client.UploadFile(
BucketName,
@@ -328,6 +331,8 @@ namespace PSMinIO.Cmdlets
overallProgress.UpdateProgress(totalProcessed + bytesTransferred);
});
var completionTime = DateTime.UtcNow;
totalProcessed += fileInfo.Length;
overallProgress.UpdateProgress(totalProcessed);
@@ -337,7 +342,11 @@ namespace PSMinIO.Cmdlets
fileInfo.Length,
DateTime.UtcNow,
etag,
BucketName);
BucketName)
{
StartTime = startTime,
CompletionTime = completionTime
};
// Generate presigned URL if requested
if (ShowURL.IsPresent)
@@ -473,16 +482,23 @@ namespace PSMinIO.Cmdlets
{
MinIOLogger.WriteVerbose(this, "Creating bucket directory: {0}", folderPath);
// Create the directory by uploading a zero-byte object
using var emptyStream = new MemoryStream();
Client.UploadStream(BucketName, folderPath, emptyStream, "application/x-directory");
MinIOLogger.WriteVerbose(this, "Successfully created bucket directory: {0}", folderPath);
try
{
// Create the directory using the dedicated method
Client.CreateDirectory(BucketName, folderPath);
MinIOLogger.WriteVerbose(this, "Successfully created bucket directory: {0}", folderPath);
}
catch (Exception createEx)
{
// Directory creation failed, but this is not critical since MinIO creates directories implicitly
MinIOLogger.WriteVerbose(this, "Directory creation failed (non-critical): {0} - {1}", folderPath, createEx.Message);
}
}
}
catch (Exception ex)
{
WriteWarning($"Could not create bucket directory '{folderPath}': {ex.Message}");
// Listing failed, but this is not critical for the upload operation
MinIOLogger.WriteVerbose(this, "Could not check directory existence (non-critical): {0} - {1}", folderPath, ex.Message);
}
}
}
+103
View File
@@ -0,0 +1,103 @@
using System;
using System.IO;
using PSMinIO.Utils;
namespace PSMinIO.Models
{
/// <summary>
/// Represents the result of a MinIO download operation with timing and speed information
/// </summary>
public class MinIODownloadResult
{
/// <summary>
/// The downloaded file information
/// </summary>
public FileInfo File { get; set; }
/// <summary>
/// Name of the bucket the object was downloaded from
/// </summary>
public string BucketName { get; set; }
/// <summary>
/// Name of the object that was downloaded
/// </summary>
public string ObjectName { get; set; }
/// <summary>
/// Size of the downloaded file in bytes
/// </summary>
public long Size => File?.Length ?? 0;
/// <summary>
/// Transfer start time
/// </summary>
public DateTime? StartTime { get; set; }
/// <summary>
/// Transfer completion time
/// </summary>
public DateTime? CompletionTime { get; set; }
/// <summary>
/// Transfer duration
/// </summary>
public TimeSpan? Duration => StartTime.HasValue && CompletionTime.HasValue ?
CompletionTime.Value - StartTime.Value : null;
/// <summary>
/// Average transfer speed in bytes per second
/// </summary>
public double? AverageSpeed => Duration.HasValue && Duration.Value.TotalSeconds > 0 ?
Size / Duration.Value.TotalSeconds : null;
/// <summary>
/// Average transfer speed formatted as string (e.g., "15.2 MB/s")
/// </summary>
public string? AverageSpeedFormatted => AverageSpeed.HasValue ?
$"{SizeFormatter.FormatBytes((long)AverageSpeed.Value)}/s" : null;
/// <summary>
/// Full path to the downloaded file
/// </summary>
public string FullName => File?.FullName ?? string.Empty;
/// <summary>
/// Name of the downloaded file
/// </summary>
public string Name => File?.Name ?? string.Empty;
/// <summary>
/// Directory containing the downloaded file
/// </summary>
public string? DirectoryName => File?.DirectoryName;
/// <summary>
/// Creates a new MinIODownloadResult
/// </summary>
/// <param name="file">Downloaded file information</param>
/// <param name="bucketName">Source bucket name</param>
/// <param name="objectName">Source object name</param>
/// <param name="startTime">Transfer start time</param>
/// <param name="completionTime">Transfer completion time</param>
public MinIODownloadResult(FileInfo file, string bucketName, string objectName,
DateTime? startTime = null, DateTime? completionTime = null)
{
File = file ?? throw new ArgumentNullException(nameof(file));
BucketName = bucketName ?? throw new ArgumentNullException(nameof(bucketName));
ObjectName = objectName ?? throw new ArgumentNullException(nameof(objectName));
StartTime = startTime;
CompletionTime = completionTime;
}
/// <summary>
/// Returns a string representation of the download result
/// </summary>
public override string ToString()
{
var duration = Duration?.ToString(@"hh\:mm\:ss\.fff") ?? "Unknown";
var speed = AverageSpeedFormatted ?? "Unknown";
return $"{BucketName}/{ObjectName} -> {FullName} ({SizeFormatter.FormatBytes(Size)}, {duration}, {speed})";
}
}
}
+29
View File
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using PSMinIO.Utils;
namespace PSMinIO.Models
{
@@ -78,6 +79,34 @@ namespace PSMinIO.Models
/// </summary>
public DateTime? PresignedUrlExpiration { get; set; }
/// <summary>
/// Transfer start time
/// </summary>
public DateTime? StartTime { get; set; }
/// <summary>
/// Transfer completion time
/// </summary>
public DateTime? CompletionTime { get; set; }
/// <summary>
/// Transfer duration
/// </summary>
public TimeSpan? Duration => StartTime.HasValue && CompletionTime.HasValue ?
CompletionTime.Value - StartTime.Value : null;
/// <summary>
/// Average transfer speed in bytes per second
/// </summary>
public double? AverageSpeed => Duration.HasValue && Duration.Value.TotalSeconds > 0 ?
Size / Duration.Value.TotalSeconds : null;
/// <summary>
/// Average transfer speed formatted as string (e.g., "15.2 MB/s")
/// </summary>
public string? AverageSpeedFormatted => AverageSpeed.HasValue ?
$"{SizeFormatter.FormatBytes((long)AverageSpeed.Value)}/s" : null;
/// <summary>
/// Creates a new MinIOObjectInfo instance
/// </summary>
+83 -14
View File
@@ -452,6 +452,48 @@ namespace PSMinIO.Utils
}
}
/// <summary>
/// Creates a directory object in MinIO (zero-byte object with trailing slash)
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <param name="directoryPath">Path of the directory (should end with /)</param>
/// <returns>ETag of the created directory object</returns>
public string CreateDirectory(string bucketName, string directoryPath)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
if (string.IsNullOrWhiteSpace(directoryPath))
throw new ArgumentException("Directory path cannot be null or empty", nameof(directoryPath));
// Ensure directory path ends with /
if (!directoryPath.EndsWith("/"))
directoryPath += "/";
try
{
// Use a properly configured empty stream
using var emptyStream = new MemoryStream(new byte[0]);
emptyStream.Position = 0;
var args = new PutObjectArgs()
.WithBucket(bucketName)
.WithObject(directoryPath)
.WithStreamData(emptyStream)
.WithObjectSize(0)
.WithContentType("application/x-directory");
Task.Run(async () =>
await _client.PutObjectAsync(args, CancellationToken)).GetAwaiter().GetResult();
return ""; // Directory objects don't have meaningful ETags
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to create directory '{directoryPath}' in bucket '{bucketName}': {ex.Message}", ex);
}
}
/// <summary>
/// Uploads a stream to MinIO synchronously
/// </summary>
@@ -475,6 +517,12 @@ namespace PSMinIO.Utils
try
{
// Ensure stream is at the beginning
if (data.CanSeek)
{
data.Position = 0;
}
var args = new PutObjectArgs()
.WithBucket(bucketName)
.WithObject(objectName)
@@ -586,12 +634,12 @@ namespace PSMinIO.Utils
/// Uploads a file using chunked transfer with resume capability
/// </summary>
/// <param name="transferState">Transfer state for resume functionality</param>
/// <param name="progressReporter">Progress reporter for updates</param>
/// <param name="progressReporter">Thread-safe progress reporter for updates</param>
/// <param name="maxRetries">Maximum retry attempts per chunk</param>
/// <returns>MinIOObjectInfo of uploaded object or null if failed</returns>
public MinIOObjectInfo? UploadFileChunked(
ChunkedTransferState transferState,
ChunkedCollectionProgressReporter progressReporter,
ThreadSafeChunkedProgressReporter progressReporter,
int maxRetries = 3)
{
if (transferState == null)
@@ -705,13 +753,13 @@ namespace PSMinIO.Utils
/// Downloads a file using chunked transfer with resume capability
/// </summary>
/// <param name="transferState">Transfer state for resume functionality</param>
/// <param name="progressReporter">Progress reporter for updates</param>
/// <param name="progressReporter">Thread-safe progress reporter for updates</param>
/// <param name="maxRetries">Maximum retry attempts per chunk</param>
/// <param name="parallelDownloads">Number of parallel chunk downloads</param>
/// <returns>True if download succeeded, false otherwise</returns>
public bool DownloadFileChunked(
ChunkedTransferState transferState,
ChunkedSingleFileProgressReporter progressReporter,
ThreadSafeChunkedProgressReporter progressReporter,
int maxRetries = 3,
int parallelDownloads = 3)
{
@@ -724,9 +772,11 @@ namespace PSMinIO.Utils
using var fileStream = new FileStream(transferState.FilePath, FileMode.Create, FileAccess.Write, FileShare.None);
fileStream.SetLength(transferState.TotalSize);
// Get list of chunks to download
// Get list of chunks to download (with safety limit)
var chunksToDownload = new List<ChunkInfo>();
while (!transferState.IsComplete)
var maxChunks = Math.Min(transferState.TotalChunks, 10000); // Safety limit of 10,000 chunks
for (int i = 0; i < maxChunks && !transferState.IsComplete; i++)
{
var nextChunk = transferState.GetNextChunk();
if (nextChunk == null)
@@ -771,7 +821,7 @@ namespace PSMinIO.Utils
/// <param name="transferState">Transfer state</param>
/// <param name="chunk">Chunk to download</param>
/// <param name="fileStream">Target file stream</param>
/// <param name="progressReporter">Progress reporter</param>
/// <param name="progressReporter">Thread-safe progress reporter</param>
/// <param name="maxRetries">Maximum retry attempts</param>
/// <param name="semaphore">Semaphore for controlling parallelism</param>
/// <returns>True if chunk downloaded successfully</returns>
@@ -779,7 +829,7 @@ namespace PSMinIO.Utils
ChunkedTransferState transferState,
ChunkInfo chunk,
FileStream fileStream,
ChunkedSingleFileProgressReporter progressReporter,
ThreadSafeChunkedProgressReporter progressReporter,
int maxRetries,
SemaphoreSlim semaphore)
{
@@ -795,17 +845,36 @@ namespace PSMinIO.Utils
{
using var chunkStream = new MemoryStream();
// Use GetObjectAsync with offset and length for proper chunked download
var getArgs = new GetObjectArgs()
.WithBucket(transferState.BucketName)
.WithObject(transferState.ObjectName)
.WithCallbackStream((stream) =>
{
// For MinIO 5.0.0, we'll need to implement range requests differently
// For now, let's use the basic GetObject and handle chunking at the stream level
var buffer = new byte[chunk.Size];
stream.Seek(chunk.StartByte, SeekOrigin.Begin);
var bytesRead = stream.Read(buffer, 0, (int)chunk.Size);
chunkStream.Write(buffer, 0, bytesRead);
// Skip to the start position and read only the chunk size
var buffer = new byte[8192]; // 8KB buffer
long totalRead = 0;
long skipBytes = chunk.StartByte;
// Skip to start position
while (skipBytes > 0)
{
var toSkip = (int)Math.Min(skipBytes, buffer.Length);
var skipped = stream.Read(buffer, 0, toSkip);
if (skipped == 0) break; // End of stream
skipBytes -= skipped;
}
// Read the chunk data
while (totalRead < chunk.Size)
{
var toRead = (int)Math.Min(chunk.Size - totalRead, buffer.Length);
var bytesRead = stream.Read(buffer, 0, toRead);
if (bytesRead == 0) break; // End of stream
chunkStream.Write(buffer, 0, bytesRead);
totalRead += bytesRead;
}
});
await _client.GetObjectAsync(getArgs, CancellationToken);
@@ -0,0 +1,163 @@
using System;
using System.Management.Automation;
using System.Threading;
namespace PSMinIO.Utils
{
/// <summary>
/// Thread-safe wrapper for chunked progress reporting that can be safely called from background threads
/// </summary>
public class ThreadSafeChunkedProgressReporter
{
private readonly ThreadSafeProgressCollector _progressCollector;
private readonly string _operationName;
private readonly long _totalSize;
private readonly int _totalChunks;
private readonly DateTime _startTime;
// Activity IDs for progress hierarchy
private const int FileActivityId = 1;
private const int ChunkActivityId = 2;
// Current state (using Interlocked for long values since volatile doesn't support long)
private volatile int _currentChunk = 0;
private long _currentChunkSize = 0;
private long _totalBytesTransferred = 0;
private volatile string _currentFileName = string.Empty;
public ThreadSafeChunkedProgressReporter(
PSCmdlet cmdlet,
long totalSize,
int totalChunks,
string operationName)
{
_progressCollector = new ThreadSafeProgressCollector(cmdlet);
_totalSize = totalSize;
_totalChunks = totalChunks;
_operationName = operationName;
_startTime = DateTime.UtcNow;
}
/// <summary>
/// Starts a new file operation (thread-safe)
/// </summary>
public void StartNewFile(string fileName, long fileSize, int totalChunks)
{
_currentFileName = fileName;
Interlocked.Exchange(ref _totalBytesTransferred, 0);
_currentChunk = 0;
_progressCollector.QueueVerboseMessage("Starting {0} of file: {1} ({2})",
_operationName.ToLower(), fileName, SizeFormatter.FormatBytes(fileSize));
_progressCollector.QueueProgressUpdate(
FileActivityId,
$"{_operationName} File",
$"{_operationName}: {fileName}",
0);
}
/// <summary>
/// Starts a new chunk operation (thread-safe)
/// </summary>
public void StartNewChunk(int chunkNumber, long chunkSize)
{
_currentChunk = chunkNumber;
Interlocked.Exchange(ref _currentChunkSize, chunkSize);
_progressCollector.QueueVerboseMessage("File {0}: Starting chunk {1}/{2} ({3})",
_currentFileName, chunkNumber, _totalChunks, SizeFormatter.FormatBytes(chunkSize));
_progressCollector.QueueProgressUpdate(
ChunkActivityId,
"Current Chunk",
$"Chunk {chunkNumber}/{_totalChunks}",
0,
FileActivityId);
}
/// <summary>
/// Updates chunk progress (thread-safe)
/// </summary>
public void UpdateChunkProgress(long bytesTransferred)
{
var newTotal = Interlocked.Add(ref _totalBytesTransferred, bytesTransferred);
var currentChunkSize = Interlocked.Read(ref _currentChunkSize);
var chunkPercent = currentChunkSize > 0 ?
(int)((double)bytesTransferred / currentChunkSize * 100) : 100;
var filePercent = _totalSize > 0 ?
(int)((double)newTotal / _totalSize * 100) : 100;
// Update chunk progress
_progressCollector.QueueProgressUpdate(
ChunkActivityId,
"Current Chunk",
$"Chunk {_currentChunk}/{_totalChunks} - {SizeFormatter.FormatBytes(bytesTransferred)}/{SizeFormatter.FormatBytes(_currentChunkSize)}",
Math.Min(chunkPercent, 100),
FileActivityId);
// Update file progress
_progressCollector.QueueProgressUpdate(
FileActivityId,
$"{_operationName} File",
$"{_operationName}: {_currentFileName} - {SizeFormatter.FormatBytes(newTotal)}/{SizeFormatter.FormatBytes(_totalSize)}",
Math.Min(filePercent, 100));
}
/// <summary>
/// Completes the current chunk (thread-safe)
/// </summary>
public void CompleteChunk(string? chunkETag = null)
{
_progressCollector.QueueVerboseMessage("File {0}: Completed chunk {1}/{2}{3}",
_currentFileName, _currentChunk, _totalChunks,
!string.IsNullOrEmpty(chunkETag) ? $" - ETag: {chunkETag}" : "");
_progressCollector.QueueProgressCompletion(ChunkActivityId, "Current Chunk", FileActivityId);
}
/// <summary>
/// Completes the current file (thread-safe)
/// </summary>
public void CompleteFile()
{
var elapsed = DateTime.UtcNow - _startTime;
_progressCollector.QueueVerboseMessage("File {0}: {1} completed in {2} - Total size: {3}",
_currentFileName, _operationName.ToLower(), elapsed.ToString(@"hh\:mm\:ss"),
SizeFormatter.FormatBytes(_totalSize));
_progressCollector.QueueProgressCompletion(FileActivityId, $"{_operationName} File");
}
/// <summary>
/// Reports a chunk error (thread-safe)
/// </summary>
public void ReportChunkError(Exception error, int retryAttempt, int maxRetries)
{
_progressCollector.QueueVerboseMessage("File {0}: Chunk {1}/{2} failed (attempt {3}/{4}): {5}",
_currentFileName, _currentChunk, _totalChunks, retryAttempt, maxRetries, error.Message);
}
/// <summary>
/// Processes all queued updates from the main thread (must be called from main thread)
/// </summary>
public void ProcessQueuedUpdates()
{
_progressCollector.ProcessQueuedUpdates();
}
/// <summary>
/// Completes all operations and processes final updates (must be called from main thread)
/// </summary>
public void Complete()
{
_progressCollector.Complete();
}
/// <summary>
/// Gets the number of pending updates
/// </summary>
public int PendingUpdates => _progressCollector.PendingProgressUpdates + _progressCollector.PendingVerboseMessages;
}
}
+156
View File
@@ -0,0 +1,156 @@
using System;
using System.Collections.Concurrent;
using System.Management.Automation;
using System.Threading;
namespace PSMinIO.Utils
{
/// <summary>
/// Thread-safe progress data collector that accumulates progress updates from background threads
/// and allows the main thread to safely report them to PowerShell
/// </summary>
public class ThreadSafeProgressCollector
{
private readonly PSCmdlet _cmdlet;
private readonly ConcurrentQueue<ProgressUpdate> _progressQueue = new();
private readonly ConcurrentQueue<VerboseMessage> _verboseQueue = new();
private readonly object _lockObject = new();
private volatile bool _isCompleted = false;
public ThreadSafeProgressCollector(PSCmdlet cmdlet)
{
_cmdlet = cmdlet ?? throw new ArgumentNullException(nameof(cmdlet));
}
/// <summary>
/// Queues a progress update from a background thread
/// </summary>
public void QueueProgressUpdate(int activityId, string activity, string statusDescription, int percentComplete, int parentActivityId = -1)
{
if (_isCompleted) return;
_progressQueue.Enqueue(new ProgressUpdate
{
ActivityId = activityId,
Activity = activity,
StatusDescription = statusDescription,
PercentComplete = percentComplete,
ParentActivityId = parentActivityId,
Timestamp = DateTime.UtcNow
});
}
/// <summary>
/// Queues a progress completion from a background thread
/// </summary>
public void QueueProgressCompletion(int activityId, string activity, int parentActivityId = -1)
{
if (_isCompleted) return;
_progressQueue.Enqueue(new ProgressUpdate
{
ActivityId = activityId,
Activity = activity,
StatusDescription = "Completed",
PercentComplete = 100,
ParentActivityId = parentActivityId,
IsCompleted = true,
Timestamp = DateTime.UtcNow
});
}
/// <summary>
/// Queues a verbose message from a background thread
/// </summary>
public void QueueVerboseMessage(string message, params object[] args)
{
if (_isCompleted) return;
_verboseQueue.Enqueue(new VerboseMessage
{
Message = args.Length > 0 ? string.Format(message, args) : message,
Timestamp = DateTime.UtcNow
});
}
/// <summary>
/// Processes all queued updates from the main thread (safe to call PowerShell methods)
/// </summary>
public void ProcessQueuedUpdates()
{
// Process verbose messages first
while (_verboseQueue.TryDequeue(out var verboseMessage))
{
MinIOLogger.WriteVerbose(_cmdlet, verboseMessage.Message);
}
// Process progress updates
while (_progressQueue.TryDequeue(out var progressUpdate))
{
var progressRecord = new ProgressRecord(
progressUpdate.ActivityId,
progressUpdate.Activity,
progressUpdate.StatusDescription)
{
PercentComplete = progressUpdate.PercentComplete
};
if (progressUpdate.ParentActivityId >= 0)
{
progressRecord.ParentActivityId = progressUpdate.ParentActivityId;
}
if (progressUpdate.IsCompleted)
{
progressRecord.RecordType = ProgressRecordType.Completed;
}
_cmdlet.WriteProgress(progressRecord);
}
}
/// <summary>
/// Marks the collector as completed (no more updates will be accepted)
/// </summary>
public void Complete()
{
_isCompleted = true;
// Process any remaining updates
ProcessQueuedUpdates();
}
/// <summary>
/// Gets the number of pending progress updates
/// </summary>
public int PendingProgressUpdates => _progressQueue.Count;
/// <summary>
/// Gets the number of pending verbose messages
/// </summary>
public int PendingVerboseMessages => _verboseQueue.Count;
/// <summary>
/// Progress update data structure
/// </summary>
private class ProgressUpdate
{
public int ActivityId { get; set; }
public string Activity { get; set; } = string.Empty;
public string StatusDescription { get; set; } = string.Empty;
public int PercentComplete { get; set; }
public int ParentActivityId { get; set; } = -1;
public bool IsCompleted { get; set; }
public DateTime Timestamp { get; set; }
}
/// <summary>
/// Verbose message data structure
/// </summary>
private class VerboseMessage
{
public string Message { get; set; } = string.Empty;
public DateTime Timestamp { get; set; }
}
}
}
+149
View File
@@ -0,0 +1,149 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Management.Automation;
using PSMinIO.Models;
namespace PSMinIO.Utils
{
/// <summary>
/// Thread-safe result collector that accumulates results from background threads
/// and allows the main thread to safely output them via WriteObject
/// </summary>
public class ThreadSafeResultCollector
{
private readonly PSCmdlet _cmdlet;
private readonly ConcurrentQueue<object> _resultQueue = new();
private readonly ConcurrentQueue<ErrorRecord> _errorQueue = new();
private readonly object _lockObject = new();
private volatile bool _isCompleted = false;
public ThreadSafeResultCollector(PSCmdlet cmdlet)
{
_cmdlet = cmdlet ?? throw new ArgumentNullException(nameof(cmdlet));
}
/// <summary>
/// Queues a successful result from a background thread
/// </summary>
public void QueueResult(object result)
{
if (_isCompleted || result == null) return;
_resultQueue.Enqueue(result);
}
/// <summary>
/// Queues an error from a background thread
/// </summary>
public void QueueError(ErrorRecord error)
{
if (_isCompleted || error == null) return;
_errorQueue.Enqueue(error);
}
/// <summary>
/// Queues an error from a background thread using exception details
/// </summary>
public void QueueError(Exception exception, string errorId, ErrorCategory category, object targetObject)
{
if (_isCompleted || exception == null) return;
var errorRecord = new ErrorRecord(exception, errorId, category, targetObject);
_errorQueue.Enqueue(errorRecord);
}
/// <summary>
/// Processes all queued results and errors from the main thread (safe to call PowerShell methods)
/// </summary>
public void ProcessQueuedResults()
{
// Process errors first
while (_errorQueue.TryDequeue(out var error))
{
_cmdlet.WriteError(error);
}
// Process successful results
while (_resultQueue.TryDequeue(out var result))
{
_cmdlet.WriteObject(result);
}
}
/// <summary>
/// Marks the collector as completed and processes all remaining results
/// </summary>
public void Complete()
{
_isCompleted = true;
// Process any remaining results
ProcessQueuedResults();
}
/// <summary>
/// Gets all queued results without processing them (for inspection)
/// </summary>
public List<object> GetQueuedResults()
{
var results = new List<object>();
var tempQueue = new Queue<object>();
// Dequeue all items and re-queue them
while (_resultQueue.TryDequeue(out var result))
{
results.Add(result);
tempQueue.Enqueue(result);
}
// Re-queue the items
while (tempQueue.Count > 0)
{
_resultQueue.Enqueue(tempQueue.Dequeue());
}
return results;
}
/// <summary>
/// Gets all queued errors without processing them (for inspection)
/// </summary>
public List<ErrorRecord> GetQueuedErrors()
{
var errors = new List<ErrorRecord>();
var tempQueue = new Queue<ErrorRecord>();
// Dequeue all items and re-queue them
while (_errorQueue.TryDequeue(out var error))
{
errors.Add(error);
tempQueue.Enqueue(error);
}
// Re-queue the items
while (tempQueue.Count > 0)
{
_errorQueue.Enqueue(tempQueue.Dequeue());
}
return errors;
}
/// <summary>
/// Gets the number of pending results
/// </summary>
public int PendingResults => _resultQueue.Count;
/// <summary>
/// Gets the number of pending errors
/// </summary>
public int PendingErrors => _errorQueue.Count;
/// <summary>
/// Gets whether the collector has any pending items
/// </summary>
public bool HasPendingItems => PendingResults > 0 || PendingErrors > 0;
}
}