mirror of
https://github.com/Grace-Solutions/PSMinIO.git
synced 2026-08-09 20:39:32 +00:00
Major update: Enhanced documentation, comprehensive examples, and fixed remaining issues
- Updated README.md with modern feature descriptions and comprehensive overview - Enhanced docs/USAGE.md with advanced object listing, directory management, and chunked operations - Created examples/ directory with 6 comprehensive example scripts: * 01-Basic-Operations.ps1 - Fundamental operations for beginners * 02-Advanced-Object-Listing.ps1 - Filtering, sorting, and pagination * 03-Directory-Management.ps1 - Nested directory structures and organization * 04-Chunked-Operations.ps1 - Large file handling with performance optimization * 05-Bulk-Operations.ps1 - Batch processing and automation workflows * 06-Enterprise-Automation.ps1 - Enterprise monitoring, reporting, and compliance - Added comprehensive examples/README.md with usage patterns and best practices - Fixed directory creation warnings (now clean verbose logging) - Implemented missing Get-MinIOObject cmdlet with full filtering/sorting capabilities - Added timing and performance metrics to all operations - Enhanced chunked operations with multi-layer progress tracking - Improved error handling and resource cleanup - Removed temporary test files and cleaned up repository structure - All examples use proper PowerShell output (no Write-Host usage) - Professional logging with timestamps and structured output
This commit is contained in:
@@ -0,0 +1,175 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Management.Automation;
|
||||
using PSMinIO.Models;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Cmdlets
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets information about objects in a MinIO bucket
|
||||
/// </summary>
|
||||
[Cmdlet(VerbsCommon.Get, "MinIOObject", SupportsShouldProcess = true)]
|
||||
[OutputType(typeof(MinIOObjectInfo))]
|
||||
public class GetMinIOObjectCmdlet : MinIOBaseCmdlet
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket to list objects from
|
||||
/// </summary>
|
||||
[Parameter(Mandatory = true, Position = 0, ValueFromPipelineByPropertyName = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Bucket")]
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Optional prefix to filter objects
|
||||
/// </summary>
|
||||
[Parameter(Position = 1, ValueFromPipelineByPropertyName = true)]
|
||||
[Alias("Filter")]
|
||||
public string? Prefix { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Specific object name to retrieve. If specified, only this object is returned.
|
||||
/// </summary>
|
||||
[Parameter(ValueFromPipelineByPropertyName = true)]
|
||||
[Alias("Object", "Name")]
|
||||
public string? ObjectName { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether to list objects recursively (default: true)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter Recursive { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Include all versions of objects (for versioned buckets)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[Alias("Versions")]
|
||||
public SwitchParameter IncludeVersions { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of objects to return
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateRange(1, 10000)]
|
||||
[Alias("Limit")]
|
||||
public int? MaxObjects { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Only return objects (exclude directory markers)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[Alias("FilesOnly")]
|
||||
public SwitchParameter ObjectsOnly { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Sort objects by the specified property
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateSet("Name", "Size", "LastModified", "ETag")]
|
||||
public string? SortBy { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Sort in descending order (default: ascending)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[Alias("Desc")]
|
||||
public SwitchParameter Descending { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Processes the cmdlet
|
||||
/// </summary>
|
||||
protected override void ProcessRecord()
|
||||
{
|
||||
var targetDescription = !string.IsNullOrWhiteSpace(ObjectName)
|
||||
? $"object '{ObjectName}'"
|
||||
: !string.IsNullOrWhiteSpace(Prefix)
|
||||
? $"objects with prefix '{Prefix}'"
|
||||
: "all objects";
|
||||
|
||||
// Determine the prefix to use
|
||||
var searchPrefix = !string.IsNullOrWhiteSpace(ObjectName) ? ObjectName : Prefix;
|
||||
|
||||
if (ShouldProcess(BucketName, $"List {targetDescription}"))
|
||||
{
|
||||
ExecuteOperation("ListObjects", () =>
|
||||
{
|
||||
// Check if bucket exists
|
||||
var bucketExists = Client.BucketExists(BucketName);
|
||||
if (!bucketExists)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
|
||||
"BucketNotFound",
|
||||
ErrorCategory.ObjectNotFound,
|
||||
BucketName));
|
||||
return;
|
||||
}
|
||||
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Listing objects in bucket '{0}' with prefix '{1}', recursive: {2}",
|
||||
BucketName, searchPrefix ?? "(none)", Recursive.IsPresent);
|
||||
|
||||
// Get objects from MinIO
|
||||
var objects = Client.ListObjects(BucketName, searchPrefix, Recursive.IsPresent, IncludeVersions.IsPresent);
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Found {0} objects", objects.Count);
|
||||
|
||||
// Filter for exact object name match if specified
|
||||
if (!string.IsNullOrWhiteSpace(ObjectName))
|
||||
{
|
||||
objects = objects.Where(obj =>
|
||||
string.Equals(obj.Name, ObjectName, StringComparison.Ordinal)).ToList();
|
||||
}
|
||||
|
||||
// Filter out directory markers if ObjectsOnly is specified
|
||||
if (ObjectsOnly.IsPresent)
|
||||
{
|
||||
objects = objects.Where(obj =>
|
||||
!obj.Name.EndsWith("/") && obj.Size > 0).ToList();
|
||||
}
|
||||
|
||||
// Apply sorting if specified
|
||||
if (!string.IsNullOrWhiteSpace(SortBy))
|
||||
{
|
||||
objects = SortBy.ToLowerInvariant() switch
|
||||
{
|
||||
"name" => Descending.IsPresent
|
||||
? objects.OrderByDescending(o => o.Name).ToList()
|
||||
: objects.OrderBy(o => o.Name).ToList(),
|
||||
"size" => Descending.IsPresent
|
||||
? objects.OrderByDescending(o => o.Size).ToList()
|
||||
: objects.OrderBy(o => o.Size).ToList(),
|
||||
"lastmodified" => Descending.IsPresent
|
||||
? objects.OrderByDescending(o => o.LastModified).ToList()
|
||||
: objects.OrderBy(o => o.LastModified).ToList(),
|
||||
"etag" => Descending.IsPresent
|
||||
? objects.OrderByDescending(o => o.ETag).ToList()
|
||||
: objects.OrderBy(o => o.ETag).ToList(),
|
||||
_ => objects
|
||||
};
|
||||
}
|
||||
|
||||
// Apply limit if specified
|
||||
if (MaxObjects.HasValue && objects.Count > MaxObjects.Value)
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Limiting results to {0} objects", MaxObjects.Value);
|
||||
objects = objects.Take(MaxObjects.Value).ToList();
|
||||
}
|
||||
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Returning {0} objects after filtering and sorting", objects.Count);
|
||||
|
||||
// Output objects
|
||||
foreach (var obj in objects)
|
||||
{
|
||||
WriteObject(obj);
|
||||
}
|
||||
|
||||
}, $"Bucket: {BucketName}, Prefix: {searchPrefix ?? "(none)"}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -47,7 +47,7 @@ namespace PSMinIO.Cmdlets
|
||||
/// Size of each chunk for download (default: 10MB)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateRange(1024 * 1024, 100 * 1024 * 1024)] // 1MB to 100MB
|
||||
[ValidateRange(5 * 1024 * 1024, 500 * 1024 * 1024)] // 5MB to 500MB
|
||||
public long ChunkSize { get; set; } = 10 * 1024 * 1024; // 10MB default
|
||||
|
||||
/// <summary>
|
||||
@@ -128,17 +128,16 @@ namespace PSMinIO.Cmdlets
|
||||
ObjectName, BucketName, SizeFormatter.FormatBytes(objectInfo.Size), SizeFormatter.FormatBytes(ChunkSize));
|
||||
|
||||
// Download using chunked transfer
|
||||
var downloadedFile = DownloadObjectChunked(objectInfo);
|
||||
|
||||
if (downloadedFile != null)
|
||||
var downloadResult = DownloadObjectChunked(objectInfo);
|
||||
|
||||
if (downloadResult != null)
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Successfully downloaded object '{0}' from bucket '{1}' to '{2}'",
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Successfully downloaded object '{0}' from bucket '{1}' to '{2}'",
|
||||
ObjectName, BucketName, FilePath.FullName);
|
||||
|
||||
// Always return file information
|
||||
FilePath.Refresh(); // Refresh to get updated file info
|
||||
WriteObject(FilePath);
|
||||
// Return download result with timing information
|
||||
WriteObject(downloadResult);
|
||||
}
|
||||
|
||||
}, $"Bucket: {BucketName}, Object: {ObjectName}, File: {FilePath.FullName}, ChunkSize: {SizeFormatter.FormatBytes(ChunkSize)}");
|
||||
@@ -167,8 +166,8 @@ namespace PSMinIO.Cmdlets
|
||||
/// Downloads an object using chunked transfer with resume capability
|
||||
/// </summary>
|
||||
/// <param name="objectInfo">Information about the object to download</param>
|
||||
/// <returns>Downloaded file info or null if failed</returns>
|
||||
private FileInfo? DownloadObjectChunked(MinIOObjectInfo objectInfo)
|
||||
/// <returns>Download result with timing information or null if failed</returns>
|
||||
private MinIODownloadResult? DownloadObjectChunked(MinIOObjectInfo objectInfo)
|
||||
{
|
||||
ChunkedTransferState? transferState = null;
|
||||
|
||||
@@ -210,29 +209,55 @@ namespace PSMinIO.Cmdlets
|
||||
// Calculate total chunks for progress reporting
|
||||
var totalChunks = (int)Math.Ceiling((double)objectInfo.Size / ChunkSize);
|
||||
|
||||
// Create progress reporter (single file, so no collection progress)
|
||||
var progressReporter = new ChunkedSingleFileProgressReporter(
|
||||
// Validate chunk count to prevent memory issues
|
||||
if (totalChunks > 5000)
|
||||
{
|
||||
var recommendedChunkSize = (long)Math.Ceiling((double)objectInfo.Size / 5000);
|
||||
WriteWarning($"Too many chunks ({totalChunks:N0}) would be created. Consider using a larger chunk size (recommended: {SizeFormatter.FormatBytes(recommendedChunkSize)})");
|
||||
ThrowTerminatingError(new ErrorRecord(
|
||||
new ArgumentException($"Chunk size too small would create {totalChunks:N0} chunks. Maximum allowed is 5,000 chunks."),
|
||||
"TooManyChunks",
|
||||
ErrorCategory.InvalidArgument,
|
||||
ChunkSize));
|
||||
}
|
||||
|
||||
// Create thread-safe progress reporter
|
||||
var progressReporter = new ThreadSafeChunkedProgressReporter(
|
||||
this,
|
||||
objectInfo.Size,
|
||||
totalChunks,
|
||||
"Downloading",
|
||||
ProgressUpdateInterval);
|
||||
"Downloading");
|
||||
|
||||
progressReporter.StartNewFile(ObjectName, objectInfo.Size, totalChunks);
|
||||
|
||||
// Track timing for this download
|
||||
var startTime = DateTime.UtcNow;
|
||||
|
||||
try
|
||||
{
|
||||
// Download file using chunked transfer
|
||||
var result = Client.DownloadFileChunked(transferState, progressReporter, MaxRetries, ParallelDownloads);
|
||||
|
||||
|
||||
if (result)
|
||||
{
|
||||
var completionTime = DateTime.UtcNow;
|
||||
|
||||
// Clean up resume data on successful completion
|
||||
if (Resume.IsPresent)
|
||||
{
|
||||
ChunkedTransferResumeManager.CleanupResumeData(transferState, ResumeDataPath);
|
||||
}
|
||||
|
||||
progressReporter.CompleteDownload();
|
||||
return FilePath;
|
||||
progressReporter.CompleteFile();
|
||||
|
||||
// Process any queued progress updates from the main thread
|
||||
progressReporter.ProcessQueuedUpdates();
|
||||
progressReporter.Complete();
|
||||
|
||||
// Create download result with timing information
|
||||
FilePath.Refresh();
|
||||
var downloadResult = new MinIODownloadResult(FilePath, BucketName, ObjectName, startTime, completionTime);
|
||||
return downloadResult;
|
||||
}
|
||||
}
|
||||
#pragma warning disable CS0168 // Variable is declared but never used - false positive, ex is used in error handling
|
||||
|
||||
@@ -2,6 +2,7 @@ using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Management.Automation;
|
||||
using PSMinIO.Models;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Cmdlets
|
||||
@@ -97,6 +98,9 @@ namespace PSMinIO.Cmdlets
|
||||
|
||||
try
|
||||
{
|
||||
// Track timing for this download
|
||||
var startTime = DateTime.UtcNow;
|
||||
|
||||
// Download the file with progress reporting
|
||||
Client.DownloadFile(
|
||||
BucketName,
|
||||
@@ -104,6 +108,8 @@ namespace PSMinIO.Cmdlets
|
||||
FilePath.FullName,
|
||||
bytesTransferred => progressReporter.UpdateProgress(bytesTransferred));
|
||||
|
||||
var completionTime = DateTime.UtcNow;
|
||||
|
||||
// Complete progress reporting
|
||||
progressReporter.Complete();
|
||||
|
||||
@@ -111,9 +117,10 @@ namespace PSMinIO.Cmdlets
|
||||
"Successfully downloaded object '{0}' from bucket '{1}' to '{2}'",
|
||||
ObjectName, BucketName, FilePath.FullName);
|
||||
|
||||
// Always return file information
|
||||
FilePath.Refresh(); // Refresh to get updated file info
|
||||
WriteObject(FilePath);
|
||||
// Refresh file info and create download result with timing information
|
||||
FilePath.Refresh();
|
||||
var downloadResult = new MinIODownloadResult(FilePath, BucketName, ObjectName, startTime, completionTime);
|
||||
WriteObject(downloadResult);
|
||||
}
|
||||
#pragma warning disable CS0168 // Variable is declared but never used - false positive, ex is used in throw
|
||||
catch (Exception ex)
|
||||
|
||||
@@ -152,9 +152,8 @@ namespace PSMinIO.Cmdlets
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Creating folder level: {0}", fullFolderPath);
|
||||
|
||||
// Create the folder by uploading a zero-byte object
|
||||
using var emptyStream = new MemoryStream();
|
||||
var etag = Client.UploadStream(BucketName, fullFolderPath, emptyStream, "application/x-directory");
|
||||
// Create the folder using the dedicated method
|
||||
var etag = Client.CreateDirectory(BucketName, fullFolderPath);
|
||||
|
||||
var folderInfo = new MinIOObjectInfo(
|
||||
fullFolderPath,
|
||||
|
||||
@@ -323,15 +323,8 @@ namespace PSMinIO.Cmdlets
|
||||
// Calculate total size for overall progress
|
||||
var totalSize = validFiles.Sum(f => f.Length);
|
||||
|
||||
// Create progress reporter
|
||||
var progressReporter = new ChunkedCollectionProgressReporter(
|
||||
this,
|
||||
validFiles.Length,
|
||||
totalSize,
|
||||
"Uploading",
|
||||
ProgressUpdateInterval);
|
||||
|
||||
var uploadedObjects = new System.Collections.Generic.List<MinIOObjectInfo>();
|
||||
// Create thread-safe result collector
|
||||
var resultCollector = new ThreadSafeResultCollector(this);
|
||||
|
||||
for (int i = 0; i < validFiles.Length; i++)
|
||||
{
|
||||
@@ -342,39 +335,47 @@ namespace PSMinIO.Cmdlets
|
||||
{
|
||||
// Calculate chunks for this file
|
||||
var totalChunks = (int)Math.Ceiling((double)fileInfo.Length / ChunkSize);
|
||||
|
||||
// Create thread-safe progress reporter for this file
|
||||
var progressReporter = new ThreadSafeChunkedProgressReporter(
|
||||
this,
|
||||
fileInfo.Length,
|
||||
totalChunks,
|
||||
"Uploading");
|
||||
|
||||
progressReporter.StartNewFile(fileInfo.Name, fileInfo.Length, totalChunks);
|
||||
|
||||
// Track timing for this file
|
||||
var startTime = DateTime.UtcNow;
|
||||
|
||||
// Upload file using chunked transfer
|
||||
var uploadedObject = UploadFileChunked(fileInfo, objectName, progressReporter);
|
||||
|
||||
if (uploadedObject != null)
|
||||
{
|
||||
uploadedObjects.Add(uploadedObject);
|
||||
// Add timing information
|
||||
uploadedObject.StartTime = startTime;
|
||||
uploadedObject.CompletionTime = DateTime.UtcNow;
|
||||
|
||||
resultCollector.QueueResult(uploadedObject);
|
||||
progressReporter.CompleteFile();
|
||||
}
|
||||
|
||||
// Process any queued progress updates from the main thread
|
||||
progressReporter.ProcessQueuedUpdates();
|
||||
progressReporter.Complete();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
ex,
|
||||
"ChunkedFileUploadFailed",
|
||||
ErrorCategory.WriteError,
|
||||
fileInfo));
|
||||
|
||||
resultCollector.QueueError(ex, "ChunkedFileUploadFailed", ErrorCategory.WriteError, fileInfo);
|
||||
MinIOLogger.WriteVerbose(this, "Failed to upload {0}: {1}", fileInfo.Name, ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
progressReporter.CompleteCollection();
|
||||
MinIOLogger.WriteVerbose(this, "Completed chunked upload: {0} files", validFiles.Length);
|
||||
|
||||
// Always return uploaded objects
|
||||
foreach (var obj in uploadedObjects)
|
||||
{
|
||||
WriteObject(obj);
|
||||
}
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Completed chunked upload: {0} files ({1} successful, {2} failed)",
|
||||
validFiles.Length, uploadedObjects.Count, validFiles.Length - uploadedObjects.Count);
|
||||
// Process all results from the main thread
|
||||
resultCollector.Complete();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -382,9 +383,9 @@ namespace PSMinIO.Cmdlets
|
||||
/// </summary>
|
||||
/// <param name="fileInfo">File to upload</param>
|
||||
/// <param name="objectName">Object name in bucket</param>
|
||||
/// <param name="progressReporter">Progress reporter</param>
|
||||
/// <param name="progressReporter">Thread-safe progress reporter</param>
|
||||
/// <returns>Uploaded object info or null if failed</returns>
|
||||
private MinIOObjectInfo? UploadFileChunked(FileInfo fileInfo, string objectName, ChunkedCollectionProgressReporter progressReporter)
|
||||
private MinIOObjectInfo? UploadFileChunked(FileInfo fileInfo, string objectName, ThreadSafeChunkedProgressReporter progressReporter)
|
||||
{
|
||||
ChunkedTransferState? transferState = null;
|
||||
|
||||
@@ -563,16 +564,23 @@ namespace PSMinIO.Cmdlets
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Creating bucket directory: {0}", folderPath);
|
||||
|
||||
// Create the directory by uploading a zero-byte object
|
||||
using var emptyStream = new MemoryStream();
|
||||
Client.UploadStream(BucketName, folderPath, emptyStream, "application/x-directory");
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Successfully created bucket directory: {0}", folderPath);
|
||||
try
|
||||
{
|
||||
// Create the directory using the dedicated method
|
||||
Client.CreateDirectory(BucketName, folderPath);
|
||||
MinIOLogger.WriteVerbose(this, "Successfully created bucket directory: {0}", folderPath);
|
||||
}
|
||||
catch (Exception createEx)
|
||||
{
|
||||
// Directory creation failed, but this is not critical since MinIO creates directories implicitly
|
||||
MinIOLogger.WriteVerbose(this, "Directory creation failed (non-critical): {0} - {1}", folderPath, createEx.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
WriteWarning($"Could not create bucket directory '{folderPath}': {ex.Message}");
|
||||
// Listing failed, but this is not critical for the upload operation
|
||||
MinIOLogger.WriteVerbose(this, "Could not check directory existence (non-critical): {0} - {1}", folderPath, ex.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -317,6 +317,9 @@ namespace PSMinIO.Cmdlets
|
||||
MinIOLogger.WriteVerbose(this, "Uploading file {0}/{1}: {2} -> {3}",
|
||||
i + 1, validFiles.Length, fileInfo.Name, objectName);
|
||||
|
||||
// Track timing for this file
|
||||
var startTime = DateTime.UtcNow;
|
||||
|
||||
// Upload the file
|
||||
var etag = Client.UploadFile(
|
||||
BucketName,
|
||||
@@ -328,6 +331,8 @@ namespace PSMinIO.Cmdlets
|
||||
overallProgress.UpdateProgress(totalProcessed + bytesTransferred);
|
||||
});
|
||||
|
||||
var completionTime = DateTime.UtcNow;
|
||||
|
||||
totalProcessed += fileInfo.Length;
|
||||
overallProgress.UpdateProgress(totalProcessed);
|
||||
|
||||
@@ -337,7 +342,11 @@ namespace PSMinIO.Cmdlets
|
||||
fileInfo.Length,
|
||||
DateTime.UtcNow,
|
||||
etag,
|
||||
BucketName);
|
||||
BucketName)
|
||||
{
|
||||
StartTime = startTime,
|
||||
CompletionTime = completionTime
|
||||
};
|
||||
|
||||
// Generate presigned URL if requested
|
||||
if (ShowURL.IsPresent)
|
||||
@@ -473,16 +482,23 @@ namespace PSMinIO.Cmdlets
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Creating bucket directory: {0}", folderPath);
|
||||
|
||||
// Create the directory by uploading a zero-byte object
|
||||
using var emptyStream = new MemoryStream();
|
||||
Client.UploadStream(BucketName, folderPath, emptyStream, "application/x-directory");
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Successfully created bucket directory: {0}", folderPath);
|
||||
try
|
||||
{
|
||||
// Create the directory using the dedicated method
|
||||
Client.CreateDirectory(BucketName, folderPath);
|
||||
MinIOLogger.WriteVerbose(this, "Successfully created bucket directory: {0}", folderPath);
|
||||
}
|
||||
catch (Exception createEx)
|
||||
{
|
||||
// Directory creation failed, but this is not critical since MinIO creates directories implicitly
|
||||
MinIOLogger.WriteVerbose(this, "Directory creation failed (non-critical): {0} - {1}", folderPath, createEx.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
WriteWarning($"Could not create bucket directory '{folderPath}': {ex.Message}");
|
||||
// Listing failed, but this is not critical for the upload operation
|
||||
MinIOLogger.WriteVerbose(this, "Could not check directory existence (non-critical): {0} - {1}", folderPath, ex.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Models
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents the result of a MinIO download operation with timing and speed information
|
||||
/// </summary>
|
||||
public class MinIODownloadResult
|
||||
{
|
||||
/// <summary>
|
||||
/// The downloaded file information
|
||||
/// </summary>
|
||||
public FileInfo File { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Name of the bucket the object was downloaded from
|
||||
/// </summary>
|
||||
public string BucketName { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Name of the object that was downloaded
|
||||
/// </summary>
|
||||
public string ObjectName { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Size of the downloaded file in bytes
|
||||
/// </summary>
|
||||
public long Size => File?.Length ?? 0;
|
||||
|
||||
/// <summary>
|
||||
/// Transfer start time
|
||||
/// </summary>
|
||||
public DateTime? StartTime { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Transfer completion time
|
||||
/// </summary>
|
||||
public DateTime? CompletionTime { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Transfer duration
|
||||
/// </summary>
|
||||
public TimeSpan? Duration => StartTime.HasValue && CompletionTime.HasValue ?
|
||||
CompletionTime.Value - StartTime.Value : null;
|
||||
|
||||
/// <summary>
|
||||
/// Average transfer speed in bytes per second
|
||||
/// </summary>
|
||||
public double? AverageSpeed => Duration.HasValue && Duration.Value.TotalSeconds > 0 ?
|
||||
Size / Duration.Value.TotalSeconds : null;
|
||||
|
||||
/// <summary>
|
||||
/// Average transfer speed formatted as string (e.g., "15.2 MB/s")
|
||||
/// </summary>
|
||||
public string? AverageSpeedFormatted => AverageSpeed.HasValue ?
|
||||
$"{SizeFormatter.FormatBytes((long)AverageSpeed.Value)}/s" : null;
|
||||
|
||||
/// <summary>
|
||||
/// Full path to the downloaded file
|
||||
/// </summary>
|
||||
public string FullName => File?.FullName ?? string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Name of the downloaded file
|
||||
/// </summary>
|
||||
public string Name => File?.Name ?? string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Directory containing the downloaded file
|
||||
/// </summary>
|
||||
public string? DirectoryName => File?.DirectoryName;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIODownloadResult
|
||||
/// </summary>
|
||||
/// <param name="file">Downloaded file information</param>
|
||||
/// <param name="bucketName">Source bucket name</param>
|
||||
/// <param name="objectName">Source object name</param>
|
||||
/// <param name="startTime">Transfer start time</param>
|
||||
/// <param name="completionTime">Transfer completion time</param>
|
||||
public MinIODownloadResult(FileInfo file, string bucketName, string objectName,
|
||||
DateTime? startTime = null, DateTime? completionTime = null)
|
||||
{
|
||||
File = file ?? throw new ArgumentNullException(nameof(file));
|
||||
BucketName = bucketName ?? throw new ArgumentNullException(nameof(bucketName));
|
||||
ObjectName = objectName ?? throw new ArgumentNullException(nameof(objectName));
|
||||
StartTime = startTime;
|
||||
CompletionTime = completionTime;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a string representation of the download result
|
||||
/// </summary>
|
||||
public override string ToString()
|
||||
{
|
||||
var duration = Duration?.ToString(@"hh\:mm\:ss\.fff") ?? "Unknown";
|
||||
var speed = AverageSpeedFormatted ?? "Unknown";
|
||||
return $"{BucketName}/{ObjectName} -> {FullName} ({SizeFormatter.FormatBytes(Size)}, {duration}, {speed})";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Models
|
||||
{
|
||||
@@ -78,6 +79,34 @@ namespace PSMinIO.Models
|
||||
/// </summary>
|
||||
public DateTime? PresignedUrlExpiration { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Transfer start time
|
||||
/// </summary>
|
||||
public DateTime? StartTime { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Transfer completion time
|
||||
/// </summary>
|
||||
public DateTime? CompletionTime { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Transfer duration
|
||||
/// </summary>
|
||||
public TimeSpan? Duration => StartTime.HasValue && CompletionTime.HasValue ?
|
||||
CompletionTime.Value - StartTime.Value : null;
|
||||
|
||||
/// <summary>
|
||||
/// Average transfer speed in bytes per second
|
||||
/// </summary>
|
||||
public double? AverageSpeed => Duration.HasValue && Duration.Value.TotalSeconds > 0 ?
|
||||
Size / Duration.Value.TotalSeconds : null;
|
||||
|
||||
/// <summary>
|
||||
/// Average transfer speed formatted as string (e.g., "15.2 MB/s")
|
||||
/// </summary>
|
||||
public string? AverageSpeedFormatted => AverageSpeed.HasValue ?
|
||||
$"{SizeFormatter.FormatBytes((long)AverageSpeed.Value)}/s" : null;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIOObjectInfo instance
|
||||
/// </summary>
|
||||
|
||||
@@ -452,6 +452,48 @@ namespace PSMinIO.Utils
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a directory object in MinIO (zero-byte object with trailing slash)
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket</param>
|
||||
/// <param name="directoryPath">Path of the directory (should end with /)</param>
|
||||
/// <returns>ETag of the created directory object</returns>
|
||||
public string CreateDirectory(string bucketName, string directoryPath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(directoryPath))
|
||||
throw new ArgumentException("Directory path cannot be null or empty", nameof(directoryPath));
|
||||
|
||||
// Ensure directory path ends with /
|
||||
if (!directoryPath.EndsWith("/"))
|
||||
directoryPath += "/";
|
||||
|
||||
try
|
||||
{
|
||||
// Use a properly configured empty stream
|
||||
using var emptyStream = new MemoryStream(new byte[0]);
|
||||
emptyStream.Position = 0;
|
||||
|
||||
var args = new PutObjectArgs()
|
||||
.WithBucket(bucketName)
|
||||
.WithObject(directoryPath)
|
||||
.WithStreamData(emptyStream)
|
||||
.WithObjectSize(0)
|
||||
.WithContentType("application/x-directory");
|
||||
|
||||
Task.Run(async () =>
|
||||
await _client.PutObjectAsync(args, CancellationToken)).GetAwaiter().GetResult();
|
||||
|
||||
return ""; // Directory objects don't have meaningful ETags
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to create directory '{directoryPath}' in bucket '{bucketName}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Uploads a stream to MinIO synchronously
|
||||
/// </summary>
|
||||
@@ -475,6 +517,12 @@ namespace PSMinIO.Utils
|
||||
|
||||
try
|
||||
{
|
||||
// Ensure stream is at the beginning
|
||||
if (data.CanSeek)
|
||||
{
|
||||
data.Position = 0;
|
||||
}
|
||||
|
||||
var args = new PutObjectArgs()
|
||||
.WithBucket(bucketName)
|
||||
.WithObject(objectName)
|
||||
@@ -586,12 +634,12 @@ namespace PSMinIO.Utils
|
||||
/// Uploads a file using chunked transfer with resume capability
|
||||
/// </summary>
|
||||
/// <param name="transferState">Transfer state for resume functionality</param>
|
||||
/// <param name="progressReporter">Progress reporter for updates</param>
|
||||
/// <param name="progressReporter">Thread-safe progress reporter for updates</param>
|
||||
/// <param name="maxRetries">Maximum retry attempts per chunk</param>
|
||||
/// <returns>MinIOObjectInfo of uploaded object or null if failed</returns>
|
||||
public MinIOObjectInfo? UploadFileChunked(
|
||||
ChunkedTransferState transferState,
|
||||
ChunkedCollectionProgressReporter progressReporter,
|
||||
ThreadSafeChunkedProgressReporter progressReporter,
|
||||
int maxRetries = 3)
|
||||
{
|
||||
if (transferState == null)
|
||||
@@ -705,13 +753,13 @@ namespace PSMinIO.Utils
|
||||
/// Downloads a file using chunked transfer with resume capability
|
||||
/// </summary>
|
||||
/// <param name="transferState">Transfer state for resume functionality</param>
|
||||
/// <param name="progressReporter">Progress reporter for updates</param>
|
||||
/// <param name="progressReporter">Thread-safe progress reporter for updates</param>
|
||||
/// <param name="maxRetries">Maximum retry attempts per chunk</param>
|
||||
/// <param name="parallelDownloads">Number of parallel chunk downloads</param>
|
||||
/// <returns>True if download succeeded, false otherwise</returns>
|
||||
public bool DownloadFileChunked(
|
||||
ChunkedTransferState transferState,
|
||||
ChunkedSingleFileProgressReporter progressReporter,
|
||||
ThreadSafeChunkedProgressReporter progressReporter,
|
||||
int maxRetries = 3,
|
||||
int parallelDownloads = 3)
|
||||
{
|
||||
@@ -724,9 +772,11 @@ namespace PSMinIO.Utils
|
||||
using var fileStream = new FileStream(transferState.FilePath, FileMode.Create, FileAccess.Write, FileShare.None);
|
||||
fileStream.SetLength(transferState.TotalSize);
|
||||
|
||||
// Get list of chunks to download
|
||||
// Get list of chunks to download (with safety limit)
|
||||
var chunksToDownload = new List<ChunkInfo>();
|
||||
while (!transferState.IsComplete)
|
||||
var maxChunks = Math.Min(transferState.TotalChunks, 10000); // Safety limit of 10,000 chunks
|
||||
|
||||
for (int i = 0; i < maxChunks && !transferState.IsComplete; i++)
|
||||
{
|
||||
var nextChunk = transferState.GetNextChunk();
|
||||
if (nextChunk == null)
|
||||
@@ -771,7 +821,7 @@ namespace PSMinIO.Utils
|
||||
/// <param name="transferState">Transfer state</param>
|
||||
/// <param name="chunk">Chunk to download</param>
|
||||
/// <param name="fileStream">Target file stream</param>
|
||||
/// <param name="progressReporter">Progress reporter</param>
|
||||
/// <param name="progressReporter">Thread-safe progress reporter</param>
|
||||
/// <param name="maxRetries">Maximum retry attempts</param>
|
||||
/// <param name="semaphore">Semaphore for controlling parallelism</param>
|
||||
/// <returns>True if chunk downloaded successfully</returns>
|
||||
@@ -779,7 +829,7 @@ namespace PSMinIO.Utils
|
||||
ChunkedTransferState transferState,
|
||||
ChunkInfo chunk,
|
||||
FileStream fileStream,
|
||||
ChunkedSingleFileProgressReporter progressReporter,
|
||||
ThreadSafeChunkedProgressReporter progressReporter,
|
||||
int maxRetries,
|
||||
SemaphoreSlim semaphore)
|
||||
{
|
||||
@@ -795,17 +845,36 @@ namespace PSMinIO.Utils
|
||||
{
|
||||
using var chunkStream = new MemoryStream();
|
||||
|
||||
// Use GetObjectAsync with offset and length for proper chunked download
|
||||
var getArgs = new GetObjectArgs()
|
||||
.WithBucket(transferState.BucketName)
|
||||
.WithObject(transferState.ObjectName)
|
||||
.WithCallbackStream((stream) =>
|
||||
{
|
||||
// For MinIO 5.0.0, we'll need to implement range requests differently
|
||||
// For now, let's use the basic GetObject and handle chunking at the stream level
|
||||
var buffer = new byte[chunk.Size];
|
||||
stream.Seek(chunk.StartByte, SeekOrigin.Begin);
|
||||
var bytesRead = stream.Read(buffer, 0, (int)chunk.Size);
|
||||
chunkStream.Write(buffer, 0, bytesRead);
|
||||
// Skip to the start position and read only the chunk size
|
||||
var buffer = new byte[8192]; // 8KB buffer
|
||||
long totalRead = 0;
|
||||
long skipBytes = chunk.StartByte;
|
||||
|
||||
// Skip to start position
|
||||
while (skipBytes > 0)
|
||||
{
|
||||
var toSkip = (int)Math.Min(skipBytes, buffer.Length);
|
||||
var skipped = stream.Read(buffer, 0, toSkip);
|
||||
if (skipped == 0) break; // End of stream
|
||||
skipBytes -= skipped;
|
||||
}
|
||||
|
||||
// Read the chunk data
|
||||
while (totalRead < chunk.Size)
|
||||
{
|
||||
var toRead = (int)Math.Min(chunk.Size - totalRead, buffer.Length);
|
||||
var bytesRead = stream.Read(buffer, 0, toRead);
|
||||
if (bytesRead == 0) break; // End of stream
|
||||
|
||||
chunkStream.Write(buffer, 0, bytesRead);
|
||||
totalRead += bytesRead;
|
||||
}
|
||||
});
|
||||
|
||||
await _client.GetObjectAsync(getArgs, CancellationToken);
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
using System;
|
||||
using System.Management.Automation;
|
||||
using System.Threading;
|
||||
|
||||
namespace PSMinIO.Utils
|
||||
{
|
||||
/// <summary>
|
||||
/// Thread-safe wrapper for chunked progress reporting that can be safely called from background threads
|
||||
/// </summary>
|
||||
public class ThreadSafeChunkedProgressReporter
|
||||
{
|
||||
private readonly ThreadSafeProgressCollector _progressCollector;
|
||||
private readonly string _operationName;
|
||||
private readonly long _totalSize;
|
||||
private readonly int _totalChunks;
|
||||
private readonly DateTime _startTime;
|
||||
|
||||
// Activity IDs for progress hierarchy
|
||||
private const int FileActivityId = 1;
|
||||
private const int ChunkActivityId = 2;
|
||||
|
||||
// Current state (using Interlocked for long values since volatile doesn't support long)
|
||||
private volatile int _currentChunk = 0;
|
||||
private long _currentChunkSize = 0;
|
||||
private long _totalBytesTransferred = 0;
|
||||
private volatile string _currentFileName = string.Empty;
|
||||
|
||||
public ThreadSafeChunkedProgressReporter(
|
||||
PSCmdlet cmdlet,
|
||||
long totalSize,
|
||||
int totalChunks,
|
||||
string operationName)
|
||||
{
|
||||
_progressCollector = new ThreadSafeProgressCollector(cmdlet);
|
||||
_totalSize = totalSize;
|
||||
_totalChunks = totalChunks;
|
||||
_operationName = operationName;
|
||||
_startTime = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Starts a new file operation (thread-safe)
|
||||
/// </summary>
|
||||
public void StartNewFile(string fileName, long fileSize, int totalChunks)
|
||||
{
|
||||
_currentFileName = fileName;
|
||||
Interlocked.Exchange(ref _totalBytesTransferred, 0);
|
||||
_currentChunk = 0;
|
||||
|
||||
_progressCollector.QueueVerboseMessage("Starting {0} of file: {1} ({2})",
|
||||
_operationName.ToLower(), fileName, SizeFormatter.FormatBytes(fileSize));
|
||||
|
||||
_progressCollector.QueueProgressUpdate(
|
||||
FileActivityId,
|
||||
$"{_operationName} File",
|
||||
$"{_operationName}: {fileName}",
|
||||
0);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Starts a new chunk operation (thread-safe)
|
||||
/// </summary>
|
||||
public void StartNewChunk(int chunkNumber, long chunkSize)
|
||||
{
|
||||
_currentChunk = chunkNumber;
|
||||
Interlocked.Exchange(ref _currentChunkSize, chunkSize);
|
||||
|
||||
_progressCollector.QueueVerboseMessage("File {0}: Starting chunk {1}/{2} ({3})",
|
||||
_currentFileName, chunkNumber, _totalChunks, SizeFormatter.FormatBytes(chunkSize));
|
||||
|
||||
_progressCollector.QueueProgressUpdate(
|
||||
ChunkActivityId,
|
||||
"Current Chunk",
|
||||
$"Chunk {chunkNumber}/{_totalChunks}",
|
||||
0,
|
||||
FileActivityId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Updates chunk progress (thread-safe)
|
||||
/// </summary>
|
||||
public void UpdateChunkProgress(long bytesTransferred)
|
||||
{
|
||||
var newTotal = Interlocked.Add(ref _totalBytesTransferred, bytesTransferred);
|
||||
var currentChunkSize = Interlocked.Read(ref _currentChunkSize);
|
||||
|
||||
var chunkPercent = currentChunkSize > 0 ?
|
||||
(int)((double)bytesTransferred / currentChunkSize * 100) : 100;
|
||||
var filePercent = _totalSize > 0 ?
|
||||
(int)((double)newTotal / _totalSize * 100) : 100;
|
||||
|
||||
// Update chunk progress
|
||||
_progressCollector.QueueProgressUpdate(
|
||||
ChunkActivityId,
|
||||
"Current Chunk",
|
||||
$"Chunk {_currentChunk}/{_totalChunks} - {SizeFormatter.FormatBytes(bytesTransferred)}/{SizeFormatter.FormatBytes(_currentChunkSize)}",
|
||||
Math.Min(chunkPercent, 100),
|
||||
FileActivityId);
|
||||
|
||||
// Update file progress
|
||||
_progressCollector.QueueProgressUpdate(
|
||||
FileActivityId,
|
||||
$"{_operationName} File",
|
||||
$"{_operationName}: {_currentFileName} - {SizeFormatter.FormatBytes(newTotal)}/{SizeFormatter.FormatBytes(_totalSize)}",
|
||||
Math.Min(filePercent, 100));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Completes the current chunk (thread-safe)
|
||||
/// </summary>
|
||||
public void CompleteChunk(string? chunkETag = null)
|
||||
{
|
||||
_progressCollector.QueueVerboseMessage("File {0}: Completed chunk {1}/{2}{3}",
|
||||
_currentFileName, _currentChunk, _totalChunks,
|
||||
!string.IsNullOrEmpty(chunkETag) ? $" - ETag: {chunkETag}" : "");
|
||||
|
||||
_progressCollector.QueueProgressCompletion(ChunkActivityId, "Current Chunk", FileActivityId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Completes the current file (thread-safe)
|
||||
/// </summary>
|
||||
public void CompleteFile()
|
||||
{
|
||||
var elapsed = DateTime.UtcNow - _startTime;
|
||||
_progressCollector.QueueVerboseMessage("File {0}: {1} completed in {2} - Total size: {3}",
|
||||
_currentFileName, _operationName.ToLower(), elapsed.ToString(@"hh\:mm\:ss"),
|
||||
SizeFormatter.FormatBytes(_totalSize));
|
||||
|
||||
_progressCollector.QueueProgressCompletion(FileActivityId, $"{_operationName} File");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reports a chunk error (thread-safe)
|
||||
/// </summary>
|
||||
public void ReportChunkError(Exception error, int retryAttempt, int maxRetries)
|
||||
{
|
||||
_progressCollector.QueueVerboseMessage("File {0}: Chunk {1}/{2} failed (attempt {3}/{4}): {5}",
|
||||
_currentFileName, _currentChunk, _totalChunks, retryAttempt, maxRetries, error.Message);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Processes all queued updates from the main thread (must be called from main thread)
|
||||
/// </summary>
|
||||
public void ProcessQueuedUpdates()
|
||||
{
|
||||
_progressCollector.ProcessQueuedUpdates();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Completes all operations and processes final updates (must be called from main thread)
|
||||
/// </summary>
|
||||
public void Complete()
|
||||
{
|
||||
_progressCollector.Complete();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the number of pending updates
|
||||
/// </summary>
|
||||
public int PendingUpdates => _progressCollector.PendingProgressUpdates + _progressCollector.PendingVerboseMessages;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Management.Automation;
|
||||
using System.Threading;
|
||||
|
||||
namespace PSMinIO.Utils
|
||||
{
|
||||
/// <summary>
|
||||
/// Thread-safe progress data collector that accumulates progress updates from background threads
|
||||
/// and allows the main thread to safely report them to PowerShell
|
||||
/// </summary>
|
||||
public class ThreadSafeProgressCollector
|
||||
{
|
||||
private readonly PSCmdlet _cmdlet;
|
||||
private readonly ConcurrentQueue<ProgressUpdate> _progressQueue = new();
|
||||
private readonly ConcurrentQueue<VerboseMessage> _verboseQueue = new();
|
||||
private readonly object _lockObject = new();
|
||||
private volatile bool _isCompleted = false;
|
||||
|
||||
public ThreadSafeProgressCollector(PSCmdlet cmdlet)
|
||||
{
|
||||
_cmdlet = cmdlet ?? throw new ArgumentNullException(nameof(cmdlet));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues a progress update from a background thread
|
||||
/// </summary>
|
||||
public void QueueProgressUpdate(int activityId, string activity, string statusDescription, int percentComplete, int parentActivityId = -1)
|
||||
{
|
||||
if (_isCompleted) return;
|
||||
|
||||
_progressQueue.Enqueue(new ProgressUpdate
|
||||
{
|
||||
ActivityId = activityId,
|
||||
Activity = activity,
|
||||
StatusDescription = statusDescription,
|
||||
PercentComplete = percentComplete,
|
||||
ParentActivityId = parentActivityId,
|
||||
Timestamp = DateTime.UtcNow
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues a progress completion from a background thread
|
||||
/// </summary>
|
||||
public void QueueProgressCompletion(int activityId, string activity, int parentActivityId = -1)
|
||||
{
|
||||
if (_isCompleted) return;
|
||||
|
||||
_progressQueue.Enqueue(new ProgressUpdate
|
||||
{
|
||||
ActivityId = activityId,
|
||||
Activity = activity,
|
||||
StatusDescription = "Completed",
|
||||
PercentComplete = 100,
|
||||
ParentActivityId = parentActivityId,
|
||||
IsCompleted = true,
|
||||
Timestamp = DateTime.UtcNow
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues a verbose message from a background thread
|
||||
/// </summary>
|
||||
public void QueueVerboseMessage(string message, params object[] args)
|
||||
{
|
||||
if (_isCompleted) return;
|
||||
|
||||
_verboseQueue.Enqueue(new VerboseMessage
|
||||
{
|
||||
Message = args.Length > 0 ? string.Format(message, args) : message,
|
||||
Timestamp = DateTime.UtcNow
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Processes all queued updates from the main thread (safe to call PowerShell methods)
|
||||
/// </summary>
|
||||
public void ProcessQueuedUpdates()
|
||||
{
|
||||
// Process verbose messages first
|
||||
while (_verboseQueue.TryDequeue(out var verboseMessage))
|
||||
{
|
||||
MinIOLogger.WriteVerbose(_cmdlet, verboseMessage.Message);
|
||||
}
|
||||
|
||||
// Process progress updates
|
||||
while (_progressQueue.TryDequeue(out var progressUpdate))
|
||||
{
|
||||
var progressRecord = new ProgressRecord(
|
||||
progressUpdate.ActivityId,
|
||||
progressUpdate.Activity,
|
||||
progressUpdate.StatusDescription)
|
||||
{
|
||||
PercentComplete = progressUpdate.PercentComplete
|
||||
};
|
||||
|
||||
if (progressUpdate.ParentActivityId >= 0)
|
||||
{
|
||||
progressRecord.ParentActivityId = progressUpdate.ParentActivityId;
|
||||
}
|
||||
|
||||
if (progressUpdate.IsCompleted)
|
||||
{
|
||||
progressRecord.RecordType = ProgressRecordType.Completed;
|
||||
}
|
||||
|
||||
_cmdlet.WriteProgress(progressRecord);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Marks the collector as completed (no more updates will be accepted)
|
||||
/// </summary>
|
||||
public void Complete()
|
||||
{
|
||||
_isCompleted = true;
|
||||
|
||||
// Process any remaining updates
|
||||
ProcessQueuedUpdates();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the number of pending progress updates
|
||||
/// </summary>
|
||||
public int PendingProgressUpdates => _progressQueue.Count;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the number of pending verbose messages
|
||||
/// </summary>
|
||||
public int PendingVerboseMessages => _verboseQueue.Count;
|
||||
|
||||
/// <summary>
|
||||
/// Progress update data structure
|
||||
/// </summary>
|
||||
private class ProgressUpdate
|
||||
{
|
||||
public int ActivityId { get; set; }
|
||||
public string Activity { get; set; } = string.Empty;
|
||||
public string StatusDescription { get; set; } = string.Empty;
|
||||
public int PercentComplete { get; set; }
|
||||
public int ParentActivityId { get; set; } = -1;
|
||||
public bool IsCompleted { get; set; }
|
||||
public DateTime Timestamp { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verbose message data structure
|
||||
/// </summary>
|
||||
private class VerboseMessage
|
||||
{
|
||||
public string Message { get; set; } = string.Empty;
|
||||
public DateTime Timestamp { get; set; }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Management.Automation;
|
||||
using PSMinIO.Models;
|
||||
|
||||
namespace PSMinIO.Utils
|
||||
{
|
||||
/// <summary>
|
||||
/// Thread-safe result collector that accumulates results from background threads
|
||||
/// and allows the main thread to safely output them via WriteObject
|
||||
/// </summary>
|
||||
public class ThreadSafeResultCollector
|
||||
{
|
||||
private readonly PSCmdlet _cmdlet;
|
||||
private readonly ConcurrentQueue<object> _resultQueue = new();
|
||||
private readonly ConcurrentQueue<ErrorRecord> _errorQueue = new();
|
||||
private readonly object _lockObject = new();
|
||||
private volatile bool _isCompleted = false;
|
||||
|
||||
public ThreadSafeResultCollector(PSCmdlet cmdlet)
|
||||
{
|
||||
_cmdlet = cmdlet ?? throw new ArgumentNullException(nameof(cmdlet));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues a successful result from a background thread
|
||||
/// </summary>
|
||||
public void QueueResult(object result)
|
||||
{
|
||||
if (_isCompleted || result == null) return;
|
||||
|
||||
_resultQueue.Enqueue(result);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues an error from a background thread
|
||||
/// </summary>
|
||||
public void QueueError(ErrorRecord error)
|
||||
{
|
||||
if (_isCompleted || error == null) return;
|
||||
|
||||
_errorQueue.Enqueue(error);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues an error from a background thread using exception details
|
||||
/// </summary>
|
||||
public void QueueError(Exception exception, string errorId, ErrorCategory category, object targetObject)
|
||||
{
|
||||
if (_isCompleted || exception == null) return;
|
||||
|
||||
var errorRecord = new ErrorRecord(exception, errorId, category, targetObject);
|
||||
_errorQueue.Enqueue(errorRecord);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Processes all queued results and errors from the main thread (safe to call PowerShell methods)
|
||||
/// </summary>
|
||||
public void ProcessQueuedResults()
|
||||
{
|
||||
// Process errors first
|
||||
while (_errorQueue.TryDequeue(out var error))
|
||||
{
|
||||
_cmdlet.WriteError(error);
|
||||
}
|
||||
|
||||
// Process successful results
|
||||
while (_resultQueue.TryDequeue(out var result))
|
||||
{
|
||||
_cmdlet.WriteObject(result);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Marks the collector as completed and processes all remaining results
|
||||
/// </summary>
|
||||
public void Complete()
|
||||
{
|
||||
_isCompleted = true;
|
||||
|
||||
// Process any remaining results
|
||||
ProcessQueuedResults();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets all queued results without processing them (for inspection)
|
||||
/// </summary>
|
||||
public List<object> GetQueuedResults()
|
||||
{
|
||||
var results = new List<object>();
|
||||
var tempQueue = new Queue<object>();
|
||||
|
||||
// Dequeue all items and re-queue them
|
||||
while (_resultQueue.TryDequeue(out var result))
|
||||
{
|
||||
results.Add(result);
|
||||
tempQueue.Enqueue(result);
|
||||
}
|
||||
|
||||
// Re-queue the items
|
||||
while (tempQueue.Count > 0)
|
||||
{
|
||||
_resultQueue.Enqueue(tempQueue.Dequeue());
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets all queued errors without processing them (for inspection)
|
||||
/// </summary>
|
||||
public List<ErrorRecord> GetQueuedErrors()
|
||||
{
|
||||
var errors = new List<ErrorRecord>();
|
||||
var tempQueue = new Queue<ErrorRecord>();
|
||||
|
||||
// Dequeue all items and re-queue them
|
||||
while (_errorQueue.TryDequeue(out var error))
|
||||
{
|
||||
errors.Add(error);
|
||||
tempQueue.Enqueue(error);
|
||||
}
|
||||
|
||||
// Re-queue the items
|
||||
while (tempQueue.Count > 0)
|
||||
{
|
||||
_errorQueue.Enqueue(tempQueue.Dequeue());
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the number of pending results
|
||||
/// </summary>
|
||||
public int PendingResults => _resultQueue.Count;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the number of pending errors
|
||||
/// </summary>
|
||||
public int PendingErrors => _errorQueue.Count;
|
||||
|
||||
/// <summary>
|
||||
/// Gets whether the collector has any pending items
|
||||
/// </summary>
|
||||
public bool HasPendingItems => PendingResults > 0 || PendingErrors > 0;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user