Initial commit: PSMinIO module with chunked transfer support

This commit is contained in:
PSMinIO Developer
2025-07-10 12:58:44 -04:00
parent a306ade1f3
commit 5fdcd31d5e
40 changed files with 9899 additions and 1 deletions
+203
View File
@@ -0,0 +1,203 @@
using System;
using System.Management.Automation;
using PSMinIO.Models;
using PSMinIO.Utils;
namespace PSMinIO.Cmdlets
{
/// <summary>
/// Establishes a connection to a MinIO server
/// </summary>
[Cmdlet(VerbsCommunications.Connect, "MinIO", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Low)]
[OutputType(typeof(MinIOConnection))]
public class ConnectMinIOCmdlet : PSCmdlet
{
/// <summary>
/// MinIO server URI (e.g., https://minio.example.com:9000)
/// </summary>
[Parameter(Position = 0, Mandatory = true)]
[ValidateNotNull]
[Alias("Server", "Url")]
public Uri Endpoint { get; set; } = null!;
/// <summary>
/// Access key for authentication
/// </summary>
[Parameter(Position = 1, Mandatory = true)]
[ValidateNotNullOrEmpty]
[Alias("AccessKeyId")]
public string AccessKey { get; set; } = string.Empty;
/// <summary>
/// Secret key for authentication
/// </summary>
[Parameter(Position = 2, Mandatory = true)]
[ValidateNotNullOrEmpty]
[Alias("SecretAccessKey")]
public string SecretKey { get; set; } = string.Empty;
/// <summary>
/// Region for bucket operations (default: us-east-1)
/// </summary>
[Parameter]
[ValidateNotNullOrEmpty]
public string Region { get; set; } = "us-east-1";
/// <summary>
/// Connection timeout in seconds (default: 30)
/// </summary>
[Parameter]
[ValidateRange(1, 300)]
public int TimeoutSeconds { get; set; } = 30;
/// <summary>
/// Test the connection after establishing it
/// </summary>
[Parameter]
public SwitchParameter TestConnection { get; set; }
/// <summary>
/// Store the connection in a session variable for reuse
/// </summary>
[Parameter]
[ValidateNotNullOrEmpty]
public string? SessionVariable { get; set; }
/// <summary>
/// Skip SSL certificate validation (use with caution)
/// </summary>
[Parameter]
public SwitchParameter SkipCertificateValidation { get; set; }
/// <summary>
/// Skip SSL certificate validation (use with caution)
/// </summary>
[Parameter]
public SwitchParameter SkipCertificateValidation { get; set; }
/// <summary>
/// Accept self-signed certificates
/// </summary>
[Parameter]
public SwitchParameter AcceptSelfSignedCertificates { get; set; }
/// <summary>
/// Accept certificates with hostname mismatches
/// </summary>
[Parameter]
public SwitchParameter AcceptHostnameMismatch { get; set; }
/// <summary>
/// Processes the cmdlet
/// </summary>
protected override void ProcessRecord()
{
// Extract connection details from URI
var useSSL = Endpoint.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase);
var endpointHost = $"{Endpoint.Host}:{Endpoint.Port}";
if (ShouldProcess($"MinIO Server: {Endpoint}", $"Establish connection"))
{
try
{
MinIOLogger.WriteVerbose(this,
"Connecting to MinIO - Endpoint: {0}, UseSSL: {1}, Region: {2}, Timeout: {3}s, SkipCertValidation: {4}",
endpointHost, useSSL, Region, TimeoutSeconds, SkipCertificateValidation.IsPresent);
// Show warning when certificate validation is skipped
if (SkipCertificateValidation.IsPresent)
{
WriteWarning("SSL certificate validation is disabled. This should only be used in development environments with self-signed certificates.");
}
// Create configuration
var configuration = new MinIOConfiguration(
endpointHost,
AccessKey,
SecretKey,
useSSL,
Region,
TimeoutSeconds,
SkipCertificateValidation.IsPresent);
// Create connection
var connection = new MinIOConnection(configuration);
MinIOLogger.WriteVerbose(this, "MinIO connection created successfully");
// Test connection if requested
if (TestConnection.IsPresent)
{
MinIOLogger.WriteVerbose(this, "Testing MinIO connection...");
var testResult = connection.TestConnection();
if (testResult.Success)
{
MinIOLogger.WriteVerbose(this,
"Connection test successful - found {0} buckets in {1}ms",
testResult.BucketCount ?? 0,
testResult.ResponseTime?.TotalMilliseconds ?? 0);
WriteInformation(new InformationRecord(
$"Connection test successful. Found {testResult.BucketCount ?? 0} buckets.",
"ConnectionTest"));
}
else
{
WriteWarning($"Connection test failed: {testResult.Message}");
MinIOLogger.WriteVerbose(this, "Connection test failed: {0}", testResult.Message);
}
}
// Store in session variable if requested
if (!string.IsNullOrWhiteSpace(SessionVariable))
{
SessionState.PSVariable.Set(SessionVariable, connection);
MinIOLogger.WriteVerbose(this, "Connection stored in session variable: {0}", SessionVariable);
}
// Return the connection object
WriteObject(connection);
MinIOLogger.WriteVerbose(this, "Connect-MinIO completed successfully");
}
catch (Exception ex)
{
var errorRecord = new ErrorRecord(
ex,
"ConnectionFailed",
ErrorCategory.ConnectionError,
Endpoint);
WriteError(errorRecord);
}
}
}
/// <summary>
/// Begins processing - validate parameters
/// </summary>
protected override void BeginProcessing()
{
base.BeginProcessing();
// Validate URI scheme
if (!Endpoint.Scheme.Equals("http", StringComparison.OrdinalIgnoreCase) &&
!Endpoint.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase))
{
ThrowTerminatingError(new ErrorRecord(
new ArgumentException($"Invalid URI scheme '{Endpoint.Scheme}'. Only 'http' and 'https' are supported."),
"InvalidUriScheme",
ErrorCategory.InvalidArgument,
Endpoint));
}
// Validate port is specified
if (Endpoint.Port == -1)
{
WriteWarning($"No port specified in URI '{Endpoint}'. MinIO typically uses port 9000.");
}
}
}
}
+180
View File
@@ -0,0 +1,180 @@
using System;
using System.Linq;
using System.Management.Automation;
using PSMinIO.Models;
using PSMinIO.Utils;
namespace PSMinIO.Cmdlets
{
/// <summary>
/// Gets information about MinIO buckets
/// </summary>
[Cmdlet(VerbsCommon.Get, "MinIOBucket", SupportsShouldProcess = true)]
[OutputType(typeof(MinIOBucketInfo))]
public class GetMinIOBucketCmdlet : MinIOBaseCmdlet
{
/// <summary>
/// Name of a specific bucket to retrieve. If not specified, all buckets are returned.
/// </summary>
[Parameter(Position = 0, ValueFromPipeline = true, ValueFromPipelineByPropertyName = true)]
[ValidateNotNullOrEmpty]
[Alias("Bucket")]
public string? BucketName { get; set; }
/// <summary>
/// Include additional statistics like object count and total size for each bucket
/// </summary>
[Parameter]
[Alias("Stats")]
public SwitchParameter IncludeStatistics { get; set; }
/// <summary>
/// Processes the cmdlet
/// </summary>
protected override void ProcessRecord()
{
ValidateConnection();
if (!string.IsNullOrWhiteSpace(BucketName))
{
// Get specific bucket
GetSpecificBucket(BucketName);
}
else
{
// Get all buckets
GetAllBuckets();
}
}
/// <summary>
/// Gets information about a specific bucket
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
private void GetSpecificBucket(string bucketName)
{
ValidateBucketName(bucketName);
if (ShouldProcess(bucketName, "Get bucket information"))
{
ExecuteOperation("GetBucket", () =>
{
// First check if bucket exists
var exists = Client.BucketExists(bucketName);
if (!exists)
{
WriteError(new ErrorRecord(
new InvalidOperationException($"Bucket '{bucketName}' does not exist"),
"BucketNotFound",
ErrorCategory.ObjectNotFound,
bucketName));
return;
}
// Get all buckets and find the specific one
var allBuckets = Client.ListBuckets();
var bucket = allBuckets.FirstOrDefault(b =>
string.Equals(b.Name, bucketName, StringComparison.OrdinalIgnoreCase));
if (bucket != null)
{
if (IncludeStatistics.IsPresent)
{
PopulateBucketStatistics(bucket);
}
WriteObject(bucket);
}
else
{
WriteError(new ErrorRecord(
new InvalidOperationException($"Bucket '{bucketName}' not found in bucket list"),
"BucketNotFound",
ErrorCategory.ObjectNotFound,
bucketName));
}
}, $"Bucket: {bucketName}");
}
}
/// <summary>
/// Gets information about all buckets
/// </summary>
private void GetAllBuckets()
{
if (ShouldProcess("All buckets", "Get bucket information"))
{
ExecuteOperation("ListBuckets", () =>
{
var buckets = Client.ListBuckets();
if (IncludeStatistics.IsPresent)
{
MinIOLogger.WriteVerbose(this, "Gathering statistics for {0} buckets", buckets.Count);
for (int i = 0; i < buckets.Count; i++)
{
var bucket = buckets[i];
// Show progress for statistics gathering
var progressRecord = new ProgressRecord(1, "Gathering Bucket Statistics",
$"Processing bucket: {bucket.Name}")
{
PercentComplete = (int)((double)(i + 1) / buckets.Count * 100)
};
WriteProgress(progressRecord);
PopulateBucketStatistics(bucket);
}
// Complete progress
WriteProgress(new ProgressRecord(1, "Gathering Bucket Statistics", "Completed")
{
PercentComplete = 100,
RecordType = ProgressRecordType.Completed
});
}
// Output all buckets
foreach (var bucket in buckets)
{
WriteObject(bucket);
}
MinIOLogger.WriteVerbose(this, "Retrieved information for {0} buckets", buckets.Count);
});
}
}
/// <summary>
/// Populates additional statistics for a bucket
/// </summary>
/// <param name="bucket">Bucket to populate statistics for</param>
private void PopulateBucketStatistics(MinIOBucketInfo bucket)
{
try
{
MinIOLogger.WriteVerbose(this, "Gathering statistics for bucket: {0}", bucket.Name);
var objects = Client.ListObjects(bucket.Name, recursive: true);
bucket.ObjectCount = objects.Count;
bucket.Size = objects.Sum(obj => obj.Size);
MinIOLogger.WriteVerbose(this,
"Bucket '{0}' statistics: {1} objects, {2} bytes total",
bucket.Name, bucket.ObjectCount, bucket.Size);
}
catch (Exception ex)
{
MinIOLogger.WriteWarning(this,
"Failed to gather statistics for bucket '{0}': {1}",
bucket.Name, ex.Message);
// Set statistics to null to indicate they couldn't be retrieved
bucket.ObjectCount = null;
bucket.Size = null;
}
}
}
}
+279
View File
@@ -0,0 +1,279 @@
using System;
using System.Management.Automation;
using System.Text.Json;
using PSMinIO.Utils;
namespace PSMinIO.Cmdlets
{
/// <summary>
/// Gets the policy for a MinIO bucket
/// </summary>
[Cmdlet(VerbsCommon.Get, "MinIOBucketPolicy", SupportsShouldProcess = false)]
[OutputType(typeof(string), typeof(PSObject))]
public class GetMinIOBucketPolicyCmdlet : MinIOBaseCmdlet
{
/// <summary>
/// Name of the bucket to get the policy for
/// </summary>
[Parameter(Position = 0, Mandatory = true, ValueFromPipeline = true, ValueFromPipelineByPropertyName = true)]
[ValidateNotNullOrEmpty]
[Alias("Bucket")]
public string BucketName { get; set; } = string.Empty;
/// <summary>
/// Return the policy as a formatted object instead of raw JSON
/// </summary>
[Parameter]
public SwitchParameter AsObject { get; set; }
/// <summary>
/// Pretty-print the JSON output
/// </summary>
[Parameter]
public SwitchParameter PrettyPrint { get; set; }
/// <summary>
/// Processes the cmdlet
/// </summary>
protected override void ProcessRecord()
{
ValidateConfiguration();
ValidateBucketName(BucketName);
ExecuteOperation("GetBucketPolicy", () =>
{
// Check if bucket exists
var bucketExists = Client.BucketExists(BucketName);
if (!bucketExists)
{
WriteError(new ErrorRecord(
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
"BucketNotFound",
ErrorCategory.ObjectNotFound,
BucketName));
return;
}
MinIOLogger.WriteVerbose(this, "Retrieving policy for bucket '{0}'", BucketName);
try
{
var policyJson = Client.GetBucketPolicy(BucketName);
if (string.IsNullOrWhiteSpace(policyJson))
{
MinIOLogger.WriteVerbose(this, "No policy is set for bucket '{0}'", BucketName);
if (AsObject.IsPresent)
{
var emptyPolicy = new PSObject();
emptyPolicy.Properties.Add(new PSNoteProperty("BucketName", BucketName));
emptyPolicy.Properties.Add(new PSNoteProperty("HasPolicy", false));
emptyPolicy.Properties.Add(new PSNoteProperty("Policy", null));
emptyPolicy.Properties.Add(new PSNoteProperty("RetrievedAt", DateTime.UtcNow));
WriteObject(emptyPolicy);
}
else
{
WriteObject(string.Empty);
}
return;
}
MinIOLogger.WriteVerbose(this, "Retrieved policy for bucket '{0}' ({1} characters)",
BucketName, policyJson.Length);
if (AsObject.IsPresent)
{
// Parse and return as structured object
var policyObject = CreatePolicyObject(policyJson);
WriteObject(policyObject);
}
else
{
// Return as JSON string
if (PrettyPrint.IsPresent)
{
var formattedJson = FormatJson(policyJson);
WriteObject(formattedJson);
}
else
{
WriteObject(policyJson);
}
}
}
catch (Exception ex) when (ex.Message.Contains("NoSuchBucketPolicy") ||
ex.Message.Contains("policy does not exist"))
{
MinIOLogger.WriteVerbose(this, "No policy is set for bucket '{0}'", BucketName);
if (AsObject.IsPresent)
{
var emptyPolicy = new PSObject();
emptyPolicy.Properties.Add(new PSNoteProperty("BucketName", BucketName));
emptyPolicy.Properties.Add(new PSNoteProperty("HasPolicy", false));
emptyPolicy.Properties.Add(new PSNoteProperty("Policy", null));
emptyPolicy.Properties.Add(new PSNoteProperty("RetrievedAt", DateTime.UtcNow));
WriteObject(emptyPolicy);
}
else
{
WriteObject(string.Empty);
}
}
}, $"Bucket: {BucketName}");
}
/// <summary>
/// Creates a structured policy object from JSON
/// </summary>
/// <param name="policyJson">Policy JSON string</param>
/// <returns>PSObject containing policy information</returns>
private PSObject CreatePolicyObject(string policyJson)
{
var policyObject = new PSObject();
policyObject.Properties.Add(new PSNoteProperty("BucketName", BucketName));
policyObject.Properties.Add(new PSNoteProperty("HasPolicy", true));
policyObject.Properties.Add(new PSNoteProperty("RetrievedAt", DateTime.UtcNow));
try
{
// Parse the JSON to extract key information
using var document = JsonDocument.Parse(policyJson);
var root = document.RootElement;
// Add raw policy
policyObject.Properties.Add(new PSNoteProperty("PolicyJson", policyJson));
// Extract version if present
if (root.TryGetProperty("Version", out var versionElement))
{
policyObject.Properties.Add(new PSNoteProperty("Version", versionElement.GetString()));
}
// Extract statements if present
if (root.TryGetProperty("Statement", out var statementsElement))
{
var statements = new System.Collections.Generic.List<PSObject>();
if (statementsElement.ValueKind == JsonValueKind.Array)
{
foreach (var statement in statementsElement.EnumerateArray())
{
var statementObj = CreateStatementObject(statement);
statements.Add(statementObj);
}
}
else if (statementsElement.ValueKind == JsonValueKind.Object)
{
var statementObj = CreateStatementObject(statementsElement);
statements.Add(statementObj);
}
policyObject.Properties.Add(new PSNoteProperty("Statements", statements.ToArray()));
policyObject.Properties.Add(new PSNoteProperty("StatementCount", statements.Count));
}
// Add formatted JSON
var formattedJson = FormatJson(policyJson);
policyObject.Properties.Add(new PSNoteProperty("FormattedPolicy", formattedJson));
}
catch (Exception ex)
{
MinIOLogger.WriteWarning(this, "Could not parse policy JSON: {0}", ex.Message);
policyObject.Properties.Add(new PSNoteProperty("PolicyJson", policyJson));
policyObject.Properties.Add(new PSNoteProperty("ParseError", ex.Message));
}
return policyObject;
}
/// <summary>
/// Creates a statement object from a JSON element
/// </summary>
/// <param name="statementElement">JSON element representing a statement</param>
/// <returns>PSObject containing statement information</returns>
private PSObject CreateStatementObject(JsonElement statementElement)
{
var statementObj = new PSObject();
try
{
if (statementElement.TryGetProperty("Effect", out var effectElement))
{
statementObj.Properties.Add(new PSNoteProperty("Effect", effectElement.GetString()));
}
if (statementElement.TryGetProperty("Principal", out var principalElement))
{
statementObj.Properties.Add(new PSNoteProperty("Principal", principalElement.ToString()));
}
if (statementElement.TryGetProperty("Action", out var actionElement))
{
if (actionElement.ValueKind == JsonValueKind.Array)
{
var actions = new System.Collections.Generic.List<string>();
foreach (var action in actionElement.EnumerateArray())
{
actions.Add(action.GetString() ?? string.Empty);
}
statementObj.Properties.Add(new PSNoteProperty("Actions", actions.ToArray()));
}
else
{
statementObj.Properties.Add(new PSNoteProperty("Actions", new[] { actionElement.GetString() ?? string.Empty }));
}
}
if (statementElement.TryGetProperty("Resource", out var resourceElement))
{
if (resourceElement.ValueKind == JsonValueKind.Array)
{
var resources = new System.Collections.Generic.List<string>();
foreach (var resource in resourceElement.EnumerateArray())
{
resources.Add(resource.GetString() ?? string.Empty);
}
statementObj.Properties.Add(new PSNoteProperty("Resources", resources.ToArray()));
}
else
{
statementObj.Properties.Add(new PSNoteProperty("Resources", new[] { resourceElement.GetString() ?? string.Empty }));
}
}
}
catch (Exception ex)
{
MinIOLogger.WriteWarning(this, "Could not parse statement: {0}", ex.Message);
statementObj.Properties.Add(new PSNoteProperty("ParseError", ex.Message));
}
return statementObj;
}
/// <summary>
/// Formats JSON string for pretty printing
/// </summary>
/// <param name="json">JSON string to format</param>
/// <returns>Formatted JSON string</returns>
private string FormatJson(string json)
{
try
{
using var document = JsonDocument.Parse(json);
return JsonSerializer.Serialize(document, new JsonSerializerOptions
{
WriteIndented = true
});
}
catch
{
// If formatting fails, return original
return json;
}
}
}
}
@@ -0,0 +1,321 @@
using System;
using System.IO;
using System.Management.Automation;
using PSMinIO.Models;
using PSMinIO.Utils;
namespace PSMinIO.Cmdlets
{
/// <summary>
/// Downloads objects from a MinIO bucket using chunked transfer with resume capability
/// </summary>
[Cmdlet(VerbsCommon.Get, "MinIOObjectContentChunked", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Low)]
[OutputType(typeof(FileInfo))]
public class GetMinIOObjectContentChunkedCmdlet : MinIOBaseCmdlet
{
/// <summary>
/// Name of the bucket to download from
/// </summary>
[Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true)]
[ValidateNotNullOrEmpty]
[Alias("Bucket")]
public string BucketName { get; set; } = string.Empty;
/// <summary>
/// Name of the object to download
/// </summary>
[Parameter(Position = 1, Mandatory = true, ValueFromPipelineByPropertyName = true)]
[ValidateNotNullOrEmpty]
[Alias("Object", "Key")]
public string ObjectName { get; set; } = string.Empty;
/// <summary>
/// FileInfo object representing where the file should be saved
/// </summary>
[Parameter(Position = 2, Mandatory = true)]
[ValidateNotNull]
[Alias("File", "Path")]
public FileInfo FilePath { get; set; } = null!;
/// <summary>
/// Overwrite existing files without prompting
/// </summary>
[Parameter]
public SwitchParameter Force { get; set; }
/// <summary>
/// Size of each chunk for download (default: 10MB)
/// </summary>
[Parameter]
[ValidateRange(1024 * 1024, 100 * 1024 * 1024)] // 1MB to 100MB
public long ChunkSize { get; set; } = 10 * 1024 * 1024; // 10MB default
/// <summary>
/// Enable resume functionality for interrupted downloads
/// </summary>
[Parameter]
public SwitchParameter Resume { get; set; }
/// <summary>
/// Maximum number of retry attempts for failed chunks
/// </summary>
[Parameter]
[ValidateRange(1, 10)]
public int MaxRetries { get; set; } = 3;
/// <summary>
/// Number of parallel chunk downloads (default: 3)
/// </summary>
[Parameter]
[ValidateRange(1, 10)]
public int ParallelDownloads { get; set; } = 3;
/// <summary>
/// Custom path for storing resume data (default: %LOCALAPPDATA%\PSMinIO\Resume)
/// </summary>
[Parameter]
public string? ResumeDataPath { get; set; }
/// <summary>
/// Update progress every N bytes transferred (default: 1MB)
/// </summary>
[Parameter]
[ValidateRange(1024, 10 * 1024 * 1024)] // 1KB to 10MB
public long ProgressUpdateInterval { get; set; } = 1024 * 1024; // 1MB
/// <summary>
/// Processes the cmdlet
/// </summary>
protected override void ProcessRecord()
{
ValidateConnection();
ValidateBucketName(BucketName);
ValidateObjectName(ObjectName);
ValidateAndPrepareFilePath();
if (ShouldProcess($"{BucketName}/{ObjectName}", $"Download using chunked transfer to '{FilePath.FullName}'"))
{
ExecuteOperation("DownloadObjectChunked", () =>
{
// Check if bucket exists
var bucketExists = Client.BucketExists(BucketName);
if (!bucketExists)
{
WriteError(new ErrorRecord(
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
"BucketNotFound",
ErrorCategory.ObjectNotFound,
BucketName));
return;
}
// Get object information
var objectInfo = GetObjectInfo();
if (objectInfo == null)
{
WriteError(new ErrorRecord(
new InvalidOperationException($"Object '{ObjectName}' not found in bucket '{BucketName}'"),
"ObjectNotFound",
ErrorCategory.ObjectNotFound,
ObjectName));
return;
}
MinIOLogger.WriteVerbose(this,
"Starting chunked download of object '{0}' from bucket '{1}' (Size: {2}, ChunkSize: {3})",
ObjectName, BucketName, SizeFormatter.FormatSize(objectInfo.Size), SizeFormatter.FormatSize(ChunkSize));
// Download using chunked transfer
var downloadedFile = DownloadObjectChunked(objectInfo);
if (downloadedFile != null)
{
MinIOLogger.WriteVerbose(this,
"Successfully downloaded object '{0}' from bucket '{1}' to '{2}'",
ObjectName, BucketName, FilePath.FullName);
// Always return file information
FilePath.Refresh(); // Refresh to get updated file info
WriteObject(FilePath);
}
}, $"Bucket: {BucketName}, Object: {ObjectName}, File: {FilePath.FullName}, ChunkSize: {SizeFormatter.FormatSize(ChunkSize)}");
}
}
/// <summary>
/// Gets information about the object to download
/// </summary>
/// <returns>Object information or null if not found</returns>
private MinIOObjectInfo? GetObjectInfo()
{
try
{
var objects = Client.ListObjects(BucketName, ObjectName, false);
return objects.Find(obj => string.Equals(obj.Name, ObjectName, StringComparison.Ordinal));
}
catch (Exception ex)
{
WriteWarning($"Could not get object information: {ex.Message}");
return null;
}
}
/// <summary>
/// Downloads an object using chunked transfer with resume capability
/// </summary>
/// <param name="objectInfo">Information about the object to download</param>
/// <returns>Downloaded file info or null if failed</returns>
private FileInfo? DownloadObjectChunked(MinIOObjectInfo objectInfo)
{
ChunkedTransferState? transferState = null;
// Try to load existing transfer state for resume
if (Resume.IsPresent)
{
transferState = ChunkedTransferResumeManager.LoadTransferState(
BucketName, ObjectName, FilePath.FullName, ChunkedTransferType.Download, ResumeDataPath);
if (transferState != null && !ChunkedTransferResumeManager.IsResumeDataValid(transferState))
{
MinIOLogger.WriteVerbose(this, "Resume data for {0} is invalid, starting fresh download", ObjectName);
transferState = null;
}
}
// Create new transfer state if none exists or resume is disabled
if (transferState == null)
{
transferState = new ChunkedTransferState
{
BucketName = BucketName,
ObjectName = ObjectName,
FilePath = FilePath.FullName,
TotalSize = objectInfo.Size,
ChunkSize = ChunkSize,
ETag = objectInfo.ETag,
TransferType = ChunkedTransferType.Download,
StartTime = DateTime.UtcNow,
LastUpdated = DateTime.UtcNow
};
}
else
{
MinIOLogger.WriteVerbose(this, "Resuming download of {0} from {1:P1} complete",
ObjectName, transferState.ProgressPercentage / 100);
}
// Calculate total chunks for progress reporting
var totalChunks = (int)Math.Ceiling((double)objectInfo.Size / ChunkSize);
// Create progress reporter (single file, so no collection progress)
var progressReporter = new ChunkedSingleFileProgressReporter(
this,
objectInfo.Size,
totalChunks,
"Downloading",
ProgressUpdateInterval);
try
{
// Download file using chunked transfer
var result = Client.DownloadFileChunked(transferState, progressReporter, MaxRetries, ParallelDownloads);
if (result)
{
// Clean up resume data on successful completion
if (Resume.IsPresent)
{
ChunkedTransferResumeManager.CleanupResumeData(transferState, ResumeDataPath);
}
progressReporter.CompleteDownload();
return FilePath;
}
}
catch (Exception ex)
{
// Save resume data on failure if resume is enabled
if (Resume.IsPresent)
{
try
{
ChunkedTransferResumeManager.SaveTransferState(transferState, ResumeDataPath);
MinIOLogger.WriteVerbose(this, "Saved resume data for {0}", ObjectName);
}
catch (Exception saveEx)
{
WriteWarning($"Could not save resume data for {ObjectName}: {saveEx.Message}");
}
}
throw;
}
return null;
}
/// <summary>
/// Validates and prepares the file path for download
/// </summary>
private void ValidateAndPrepareFilePath()
{
if (FilePath == null)
{
ThrowTerminatingError(new ErrorRecord(
new ArgumentException("FilePath cannot be null"),
"InvalidFilePath",
ErrorCategory.InvalidArgument,
FilePath));
}
// Check if file already exists
if (FilePath.Exists && !Force.IsPresent)
{
WriteError(new ErrorRecord(
new InvalidOperationException($"File '{FilePath.FullName}' already exists. Use -Force to overwrite."),
"FileAlreadyExists",
ErrorCategory.ResourceExists,
FilePath));
return;
}
// Ensure the directory exists
var directory = FilePath.Directory;
if (directory != null && !directory.Exists)
{
try
{
directory.Create();
MinIOLogger.WriteVerbose(this, "Created directory: {0}", directory.FullName);
}
catch (Exception ex)
{
ThrowTerminatingError(new ErrorRecord(
new InvalidOperationException($"Cannot create directory '{directory.FullName}': {ex.Message}", ex),
"DirectoryCreationFailed",
ErrorCategory.WriteError,
directory));
}
}
// Check if we can write to the target location
try
{
var testFile = Path.Combine(FilePath.DirectoryName ?? ".", $".psminiotest_{Guid.NewGuid():N}");
File.WriteAllText(testFile, "test");
File.Delete(testFile);
}
catch (Exception ex)
{
ThrowTerminatingError(new ErrorRecord(
new InvalidOperationException($"Cannot write to location '{FilePath.FullName}': {ex.Message}", ex),
"LocationNotWritable",
ErrorCategory.PermissionDenied,
FilePath));
}
}
}
}
+209
View File
@@ -0,0 +1,209 @@
using System;
using System.IO;
using System.Linq;
using System.Management.Automation;
using PSMinIO.Utils;
namespace PSMinIO.Cmdlets
{
/// <summary>
/// Downloads an object from a MinIO bucket to a local file
/// </summary>
[Cmdlet(VerbsCommon.Get, "MinIOObjectContent", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Medium)]
[OutputType(typeof(FileInfo))]
public class GetMinIOObjectContentCmdlet : MinIOBaseCmdlet
{
/// <summary>
/// Name of the bucket containing the object
/// </summary>
[Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true)]
[ValidateNotNullOrEmpty]
[Alias("Bucket")]
public string BucketName { get; set; } = string.Empty;
/// <summary>
/// Name of the object to download
/// </summary>
[Parameter(Position = 1, Mandatory = true, ValueFromPipelineByPropertyName = true)]
[ValidateNotNullOrEmpty]
[Alias("Object", "Key")]
public string ObjectName { get; set; } = string.Empty;
/// <summary>
/// FileInfo object representing where the file should be saved
/// </summary>
[Parameter(Position = 2, Mandatory = true)]
[ValidateNotNull]
[Alias("File", "Path")]
public FileInfo FilePath { get; set; } = null!;
/// <summary>
/// Overwrite the file if it already exists
/// </summary>
[Parameter]
public SwitchParameter Force { get; set; }
/// <summary>
/// Processes the cmdlet
/// </summary>
protected override void ProcessRecord()
{
ValidateConfiguration();
ValidateBucketName(BucketName);
ValidateObjectName(ObjectName);
ValidateAndPrepareFilePath();
if (ShouldProcess($"{BucketName}/{ObjectName}", $"Download to '{FilePath}'"))
{
ExecuteOperation("DownloadObject", () =>
{
// Check if bucket exists
var bucketExists = Client.BucketExists(BucketName);
if (!bucketExists)
{
WriteError(new ErrorRecord(
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
"BucketNotFound",
ErrorCategory.ObjectNotFound,
BucketName));
return;
}
// Check if object exists and get its information
var objectInfo = GetObjectInfo();
if (objectInfo == null)
{
WriteError(new ErrorRecord(
new InvalidOperationException($"Object '{ObjectName}' does not exist in bucket '{BucketName}'"),
"ObjectNotFound",
ErrorCategory.ObjectNotFound,
ObjectName));
return;
}
MinIOLogger.WriteVerbose(this,
"Downloading object '{0}' from bucket '{1}' ({2} bytes) to '{3}'",
ObjectName, BucketName, objectInfo.Size, FilePath);
// Create progress reporter
var progressReporter = new ProgressReporter(
this,
"Downloading Object",
$"Downloading {objectInfo.GetFileName()}",
objectInfo.Size,
1);
try
{
// Download the file with progress reporting
Client.DownloadFile(
BucketName,
ObjectName,
FilePath.FullName,
bytesTransferred => progressReporter.UpdateProgress(bytesTransferred));
// Complete progress reporting
progressReporter.Complete();
MinIOLogger.WriteVerbose(this,
"Successfully downloaded object '{0}' from bucket '{1}' to '{2}'",
ObjectName, BucketName, FilePath.FullName);
// Always return file information
FilePath.Refresh(); // Refresh to get updated file info
WriteObject(FilePath);
}
catch (Exception ex)
{
progressReporter.Complete();
throw;
}
}, $"Bucket: {BucketName}, Object: {ObjectName}, File: {FilePath.FullName}");
}
}
/// <summary>
/// Validates and prepares the file path for download
/// </summary>
private void ValidateAndPrepareFilePath()
{
if (FilePath == null)
{
ThrowTerminatingError(new ErrorRecord(
new ArgumentException("FilePath cannot be null"),
"InvalidFilePath",
ErrorCategory.InvalidArgument,
FilePath));
}
// Check if file already exists
if (FilePath.Exists && !Force.IsPresent)
{
WriteError(new ErrorRecord(
new InvalidOperationException($"File '{FilePath.FullName}' already exists. Use -Force to overwrite."),
"FileAlreadyExists",
ErrorCategory.ResourceExists,
FilePath));
return;
}
// Ensure the directory exists
var directory = FilePath.Directory;
if (directory != null && !directory.Exists)
{
try
{
directory.Create();
MinIOLogger.WriteVerbose(this, "Created directory: {0}", directory.FullName);
}
catch (Exception ex)
{
ThrowTerminatingError(new ErrorRecord(
new InvalidOperationException($"Cannot create directory '{directory.FullName}': {ex.Message}", ex),
"DirectoryCreationFailed",
ErrorCategory.WriteError,
directory));
}
}
// Check if we can write to the target location
try
{
var testFile = Path.Combine(FilePath.DirectoryName ?? ".", $".psminiotest_{Guid.NewGuid():N}");
File.WriteAllText(testFile, "test");
File.Delete(testFile);
}
catch (Exception ex)
{
ThrowTerminatingError(new ErrorRecord(
new InvalidOperationException($"Cannot write to location '{FilePath.FullName}': {ex.Message}", ex),
"LocationNotWritable",
ErrorCategory.PermissionDenied,
FilePath));
}
}
/// <summary>
/// Gets information about the object to be downloaded
/// </summary>
/// <returns>Object information or null if not found</returns>
private Models.MinIOObjectInfo? GetObjectInfo()
{
try
{
var objects = Client.ListObjects(BucketName, ObjectName, false);
return objects.FirstOrDefault(obj =>
string.Equals(obj.Name, ObjectName, StringComparison.Ordinal));
}
catch (Exception ex)
{
MinIOLogger.WriteWarning(this,
"Could not retrieve object information for '{0}': {1}", ObjectName, ex.Message);
return null;
}
}
}
}
+269
View File
@@ -0,0 +1,269 @@
using System;
using System.Linq;
using System.Management.Automation;
using PSMinIO.Models;
using PSMinIO.Utils;
namespace PSMinIO.Cmdlets
{
/// <summary>
/// Gets statistical information about MinIO storage
/// </summary>
[Cmdlet(VerbsCommon.Get, "MinIOStats", SupportsShouldProcess = false)]
[OutputType(typeof(MinIOStats))]
public class GetMinIOStatsCmdlet : MinIOBaseCmdlet
{
/// <summary>
/// Include detailed per-bucket statistics
/// </summary>
[Parameter]
public SwitchParameter IncludeBucketDetails { get; set; }
/// <summary>
/// Include object count and size calculations (may be slow for large buckets)
/// </summary>
[Parameter]
public SwitchParameter IncludeObjectCounts { get; set; }
/// <summary>
/// Maximum number of objects to count per bucket (default: unlimited)
/// </summary>
[Parameter]
[ValidateRange(1, int.MaxValue)]
public int? MaxObjectsToCount { get; set; }
/// <summary>
/// Processes the cmdlet
/// </summary>
protected override void ProcessRecord()
{
ValidateConfiguration();
ExecuteOperation("GetStats", () =>
{
MinIOLogger.WriteVerbose(this, "Gathering MinIO statistics...");
var config = Configuration;
var stats = new MinIOStats
{
Endpoint = config.Endpoint,
UseSSL = config.UseSSL,
ConnectionStatus = "Connected"
};
try
{
// Get basic bucket information
var buckets = Client.ListBuckets();
stats.TotalBuckets = buckets.Count;
MinIOLogger.WriteVerbose(this, "Found {0} buckets", buckets.Count);
if (IncludeObjectCounts.IsPresent && buckets.Count > 0)
{
GatherObjectStatistics(buckets, stats);
}
// Create detailed output if requested
if (IncludeBucketDetails.IsPresent)
{
var detailedStats = CreateDetailedStats(stats, buckets);
WriteObject(detailedStats);
}
else
{
WriteObject(stats);
}
MinIOLogger.WriteVerbose(this, "Statistics gathering completed");
}
catch (Exception ex)
{
stats.ConnectionStatus = $"Error: {ex.Message}";
WriteObject(stats);
throw;
}
}, "Gathering statistics");
}
/// <summary>
/// Gathers object statistics across all buckets
/// </summary>
/// <param name="buckets">List of buckets</param>
/// <param name="stats">Stats object to update</param>
private void GatherObjectStatistics(System.Collections.Generic.List<MinIOBucketInfo> buckets, MinIOStats stats)
{
MinIOLogger.WriteVerbose(this, "Gathering object statistics for {0} buckets...", buckets.Count);
long totalObjects = 0;
long totalSize = 0;
for (int i = 0; i < buckets.Count; i++)
{
var bucket = buckets[i];
// Show progress
var progressRecord = new ProgressRecord(1, "Gathering Statistics",
$"Processing bucket: {bucket.Name}")
{
PercentComplete = (int)((double)(i + 1) / buckets.Count * 100)
};
WriteProgress(progressRecord);
try
{
MinIOLogger.WriteVerbose(this, "Processing bucket: {0}", bucket.Name);
var objects = Client.ListObjects(bucket.Name, recursive: true);
// Apply limit if specified
if (MaxObjectsToCount.HasValue && objects.Count > MaxObjectsToCount.Value)
{
MinIOLogger.WriteVerbose(this,
"Limiting object count for bucket '{0}' to {1} objects",
bucket.Name, MaxObjectsToCount.Value);
objects = objects.Take(MaxObjectsToCount.Value).ToList();
}
var bucketObjectCount = objects.Count;
var bucketSize = objects.Sum(obj => obj.Size);
totalObjects += bucketObjectCount;
totalSize += bucketSize;
MinIOLogger.WriteVerbose(this,
"Bucket '{0}': {1} objects, {2} bytes",
bucket.Name, bucketObjectCount, bucketSize);
}
catch (Exception ex)
{
MinIOLogger.WriteWarning(this,
"Failed to get statistics for bucket '{0}': {1}",
bucket.Name, ex.Message);
}
}
// Complete progress
WriteProgress(new ProgressRecord(1, "Gathering Statistics", "Completed")
{
PercentComplete = 100,
RecordType = ProgressRecordType.Completed
});
stats.TotalObjects = totalObjects;
stats.TotalSize = totalSize;
MinIOLogger.WriteVerbose(this,
"Total statistics: {0} objects, {1} bytes across {2} buckets",
totalObjects, totalSize, buckets.Count);
}
/// <summary>
/// Creates detailed statistics with per-bucket information
/// </summary>
/// <param name="stats">Base statistics</param>
/// <param name="buckets">List of buckets</param>
/// <returns>PSObject with detailed statistics</returns>
private PSObject CreateDetailedStats(MinIOStats stats, System.Collections.Generic.List<MinIOBucketInfo> buckets)
{
var detailedStats = new PSObject();
// Add basic statistics
detailedStats.Properties.Add(new PSNoteProperty("TotalBuckets", stats.TotalBuckets));
detailedStats.Properties.Add(new PSNoteProperty("TotalObjects", stats.TotalObjects));
detailedStats.Properties.Add(new PSNoteProperty("TotalSize", stats.TotalSize));
detailedStats.Properties.Add(new PSNoteProperty("TotalSizeFormatted", FormatBytes(stats.TotalSize)));
detailedStats.Properties.Add(new PSNoteProperty("LastUpdated", stats.LastUpdated));
detailedStats.Properties.Add(new PSNoteProperty("Endpoint", stats.Endpoint));
detailedStats.Properties.Add(new PSNoteProperty("UseSSL", stats.UseSSL));
detailedStats.Properties.Add(new PSNoteProperty("ConnectionStatus", stats.ConnectionStatus));
// Add calculated statistics
detailedStats.Properties.Add(new PSNoteProperty("AverageObjectSize", stats.AverageObjectSize));
detailedStats.Properties.Add(new PSNoteProperty("AverageObjectSizeFormatted", FormatBytes((long)stats.AverageObjectSize)));
detailedStats.Properties.Add(new PSNoteProperty("AverageObjectsPerBucket", stats.AverageObjectsPerBucket));
// Add bucket details if object counts were gathered
if (IncludeObjectCounts.IsPresent)
{
var bucketDetails = new System.Collections.Generic.List<PSObject>();
foreach (var bucket in buckets)
{
try
{
var objects = Client.ListObjects(bucket.Name, recursive: true);
// Apply limit if specified
if (MaxObjectsToCount.HasValue && objects.Count > MaxObjectsToCount.Value)
{
objects = objects.Take(MaxObjectsToCount.Value).ToList();
}
var bucketDetail = new PSObject();
bucketDetail.Properties.Add(new PSNoteProperty("Name", bucket.Name));
bucketDetail.Properties.Add(new PSNoteProperty("Created", bucket.Created));
bucketDetail.Properties.Add(new PSNoteProperty("ObjectCount", objects.Count));
var bucketSize = objects.Sum(obj => obj.Size);
bucketDetail.Properties.Add(new PSNoteProperty("Size", bucketSize));
bucketDetail.Properties.Add(new PSNoteProperty("SizeFormatted", FormatBytes(bucketSize)));
var avgObjectSize = objects.Count > 0 ? (double)bucketSize / objects.Count : 0;
bucketDetail.Properties.Add(new PSNoteProperty("AverageObjectSize", avgObjectSize));
bucketDetail.Properties.Add(new PSNoteProperty("AverageObjectSizeFormatted", FormatBytes((long)avgObjectSize)));
bucketDetails.Add(bucketDetail);
}
catch (Exception ex)
{
var bucketDetail = new PSObject();
bucketDetail.Properties.Add(new PSNoteProperty("Name", bucket.Name));
bucketDetail.Properties.Add(new PSNoteProperty("Created", bucket.Created));
bucketDetail.Properties.Add(new PSNoteProperty("Error", ex.Message));
bucketDetails.Add(bucketDetail);
}
}
detailedStats.Properties.Add(new PSNoteProperty("BucketDetails", bucketDetails.ToArray()));
}
else
{
// Just add basic bucket information
var bucketSummary = buckets.Select(b => new PSObject(new
{
Name = b.Name,
Created = b.Created,
Region = b.Region
})).ToArray();
detailedStats.Properties.Add(new PSNoteProperty("Buckets", bucketSummary));
}
return detailedStats;
}
/// <summary>
/// Formats bytes into a human-readable string
/// </summary>
/// <param name="bytes">Number of bytes</param>
/// <returns>Formatted string</returns>
private static string FormatBytes(long bytes)
{
if (bytes == 0) return "0 B";
string[] sizes = { "B", "KB", "MB", "GB", "TB", "PB" };
int order = 0;
double size = bytes;
while (size >= 1024 && order < sizes.Length - 1)
{
order++;
size = size / 1024;
}
return $"{size:0.##} {sizes[order]}";
}
}
}
+162
View File
@@ -0,0 +1,162 @@
using System;
using System.Management.Automation;
using PSMinIO.Models;
using PSMinIO.Utils;
namespace PSMinIO.Cmdlets
{
/// <summary>
/// Creates a new MinIO bucket
/// </summary>
[Cmdlet(VerbsCommon.New, "MinIOBucket", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Medium)]
[OutputType(typeof(MinIOBucketInfo))]
public class NewMinIOBucketCmdlet : MinIOBaseCmdlet
{
/// <summary>
/// Name of the bucket to create
/// </summary>
[Parameter(Position = 0, Mandatory = true, ValueFromPipeline = true, ValueFromPipelineByPropertyName = true)]
[ValidateNotNullOrEmpty]
[Alias("Bucket")]
public string BucketName { get; set; } = string.Empty;
/// <summary>
/// Region where the bucket should be created
/// </summary>
[Parameter]
[ValidateNotNullOrEmpty]
public string? Region { get; set; }
/// <summary>
/// Force creation even if bucket already exists (no error will be thrown)
/// </summary>
[Parameter]
public SwitchParameter Force { get; set; }
/// <summary>
/// Processes the cmdlet
/// </summary>
protected override void ProcessRecord()
{
ValidateConnection();
ValidateBucketName(BucketName);
// Use region from parameter or configuration
var region = Region ?? Configuration.Region;
if (ShouldProcess(BucketName, $"Create bucket in region '{region}'"))
{
ExecuteOperation("CreateBucket", () =>
{
// Check if bucket already exists
var exists = Client.BucketExists(BucketName);
if (exists)
{
if (Force.IsPresent)
{
MinIOLogger.WriteVerbose(this,
"Bucket '{0}' already exists, but Force parameter specified", BucketName);
}
else
{
WriteError(new ErrorRecord(
new InvalidOperationException($"Bucket '{BucketName}' already exists. Use -Force to suppress this error."),
"BucketAlreadyExists",
ErrorCategory.ResourceExists,
BucketName));
return;
}
}
else
{
// Create the bucket
MinIOLogger.WriteVerbose(this,
"Creating bucket '{0}' in region '{1}'", BucketName, region);
Client.CreateBucket(BucketName, region);
MinIOLogger.WriteVerbose(this,
"Successfully created bucket '{0}'", BucketName);
}
// Always return bucket information
try
{
// Get the created bucket information
var allBuckets = Client.ListBuckets();
var createdBucket = allBuckets.Find(b =>
string.Equals(b.Name, BucketName, StringComparison.OrdinalIgnoreCase));
if (createdBucket != null)
{
createdBucket.Region = region;
WriteObject(createdBucket);
}
else
{
// Fallback: create a basic bucket info object
var bucketInfo = new MinIOBucketInfo(BucketName, DateTime.UtcNow, region);
WriteObject(bucketInfo);
}
}
catch (Exception ex)
{
MinIOLogger.WriteWarning(this,
"Bucket created successfully, but failed to retrieve bucket information: {0}",
ex.Message);
// Still return a basic bucket info object
var bucketInfo = new MinIOBucketInfo(BucketName, DateTime.UtcNow, region);
WriteObject(bucketInfo);
}
}, $"Bucket: {BucketName}, Region: {region}");
}
}
/// <summary>
/// Validates the bucket name according to MinIO/S3 naming conventions
/// </summary>
/// <param name="bucketName">Bucket name to validate</param>
protected override void ValidateBucketName(string bucketName, string parameterName = "BucketName")
{
base.ValidateBucketName(bucketName, parameterName);
// Additional validation for bucket creation
if (bucketName.Contains("..") || bucketName.StartsWith(".") || bucketName.EndsWith("."))
{
ThrowTerminatingError(new ErrorRecord(
new ArgumentException($"Bucket name '{bucketName}' contains invalid characters or patterns"),
"InvalidBucketName",
ErrorCategory.InvalidArgument,
bucketName));
}
// Check for uppercase letters (not allowed in S3/MinIO)
if (bucketName != bucketName.ToLowerInvariant())
{
ThrowTerminatingError(new ErrorRecord(
new ArgumentException($"Bucket name '{bucketName}' must be lowercase"),
"InvalidBucketName",
ErrorCategory.InvalidArgument,
bucketName));
}
// Check for invalid characters
foreach (char c in bucketName)
{
if (!char.IsLetterOrDigit(c) && c != '-' && c != '.')
{
ThrowTerminatingError(new ErrorRecord(
new ArgumentException($"Bucket name '{bucketName}' contains invalid character '{c}'. Only lowercase letters, numbers, hyphens, and periods are allowed."),
"InvalidBucketName",
ErrorCategory.InvalidArgument,
bucketName));
}
}
}
}
}
+185
View File
@@ -0,0 +1,185 @@
using System;
using System.IO;
using System.Management.Automation;
using PSMinIO.Models;
using PSMinIO.Utils;
namespace PSMinIO.Cmdlets
{
/// <summary>
/// Creates a folder (prefix) in a MinIO bucket
/// </summary>
[Cmdlet(VerbsCommon.New, "MinIOFolder", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Low)]
[OutputType(typeof(MinIOObjectInfo))]
public class NewMinIOFolderCmdlet : MinIOBaseCmdlet
{
/// <summary>
/// Name of the bucket to create the folder in
/// </summary>
[Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true)]
[ValidateNotNullOrEmpty]
[Alias("Bucket")]
public string BucketName { get; set; } = string.Empty;
/// <summary>
/// Name of the folder to create (supports nested paths like "folder1/folder2/folder3")
/// </summary>
[Parameter(Position = 1, Mandatory = true, ValueFromPipelineByPropertyName = true)]
[ValidateNotNullOrEmpty]
[Alias("Folder", "Prefix", "Path")]
public string FolderName { get; set; } = string.Empty;
/// <summary>
/// Processes the cmdlet
/// </summary>
protected override void ProcessRecord()
{
ValidateConnection();
ValidateBucketName(BucketName);
// Sanitize the folder name
var sanitizedFolderName = SanitizeFolderPath(FolderName);
if (string.IsNullOrEmpty(sanitizedFolderName))
{
WriteError(new ErrorRecord(
new ArgumentException("Folder name cannot be empty after sanitization"),
"InvalidFolderName",
ErrorCategory.InvalidArgument,
FolderName));
return;
}
if (ShouldProcess($"{BucketName}/{sanitizedFolderName}", "Create folder structure"))
{
ExecuteOperation("CreateFolder", () =>
{
// Check if bucket exists
var bucketExists = Client.BucketExists(BucketName);
if (!bucketExists)
{
WriteError(new ErrorRecord(
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
"BucketNotFound",
ErrorCategory.ObjectNotFound,
BucketName));
return;
}
MinIOLogger.WriteVerbose(this,
"Creating folder structure '{0}' in bucket '{1}'", sanitizedFolderName, BucketName);
// Create the complete folder structure
var createdFolders = CreateFolderStructure(sanitizedFolderName);
MinIOLogger.WriteVerbose(this,
"Successfully created {0} folder level(s) in bucket '{1}'", createdFolders.Count, BucketName);
// Return information about all created folders
foreach (var folder in createdFolders)
{
WriteObject(folder);
}
}, $"Bucket: {BucketName}, Folder: {sanitizedFolderName}");
}
}
/// <summary>
/// Sanitizes folder path and ensures proper format
/// </summary>
/// <param name="folderPath">Raw folder path</param>
/// <returns>Sanitized folder path</returns>
private string SanitizeFolderPath(string folderPath)
{
if (string.IsNullOrWhiteSpace(folderPath))
return string.Empty;
// Remove leading and trailing slashes, then split and clean
var cleanPath = folderPath.Trim().Trim('/', '\\');
// Split by both forward and back slashes, then clean each part
var parts = cleanPath.Split(new char[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries);
// Clean each part and validate
var cleanedParts = new System.Collections.Generic.List<string>();
var invalidChars = new char[] { '<', '>', ':', '"', '|', '?', '*' };
foreach (var part in parts)
{
var cleanPart = part.Trim();
if (string.IsNullOrEmpty(cleanPart))
continue;
// Check for invalid characters
if (cleanPart.IndexOfAny(invalidChars) >= 0)
{
WriteWarning($"Folder part '{cleanPart}' contains invalid characters and will be skipped");
continue;
}
cleanedParts.Add(cleanPart);
}
return string.Join("/", cleanedParts);
}
/// <summary>
/// Creates the complete folder structure, including all intermediate levels
/// </summary>
/// <param name="folderPath">Complete folder path to create</param>
/// <returns>List of created folder objects</returns>
private System.Collections.Generic.List<MinIOObjectInfo> CreateFolderStructure(string folderPath)
{
var createdFolders = new System.Collections.Generic.List<MinIOObjectInfo>();
var parts = folderPath.Split('/');
var currentPath = string.Empty;
foreach (var part in parts)
{
currentPath = string.IsNullOrEmpty(currentPath) ? part : $"{currentPath}/{part}";
var fullFolderPath = $"{currentPath}/";
try
{
// Check if this folder level already exists
var existingObjects = Client.ListObjects(BucketName, fullFolderPath, false);
var folderExists = existingObjects.Any(obj => obj.Name == fullFolderPath);
if (!folderExists)
{
MinIOLogger.WriteVerbose(this, "Creating folder level: {0}", fullFolderPath);
// Create the folder by uploading a zero-byte object
using var emptyStream = new MemoryStream();
var etag = Client.UploadStream(BucketName, fullFolderPath, emptyStream, "application/x-directory");
var folderInfo = new MinIOObjectInfo(
fullFolderPath,
0,
DateTime.UtcNow,
etag,
BucketName)
{
ContentType = "application/x-directory"
};
createdFolders.Add(folderInfo);
MinIOLogger.WriteVerbose(this, "Successfully created folder level: {0}", fullFolderPath);
}
else
{
MinIOLogger.WriteVerbose(this, "Folder level already exists: {0}", fullFolderPath);
}
}
catch (Exception ex)
{
WriteWarning($"Could not create folder level '{fullFolderPath}': {ex.Message}");
}
}
return createdFolders;
}
}
}
+577
View File
@@ -0,0 +1,577 @@
using System;
using System.IO;
using System.Linq;
using System.Management.Automation;
using PSMinIO.Models;
using PSMinIO.Utils;
namespace PSMinIO.Cmdlets
{
/// <summary>
/// Uploads files or directories to a MinIO bucket using chunked transfer with resume capability
/// </summary>
[Cmdlet(VerbsCommon.New, "MinIOObjectChunked", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Low)]
[OutputType(typeof(MinIOObjectInfo))]
public class NewMinIOObjectChunkedCmdlet : MinIOBaseCmdlet
{
/// <summary>
/// Name of the bucket to upload to
/// </summary>
[Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true)]
[ValidateNotNullOrEmpty]
[Alias("Bucket")]
public string BucketName { get; set; } = string.Empty;
/// <summary>
/// Array of FileInfo objects to upload
/// </summary>
[Parameter(Position = 1, Mandatory = true, ValueFromPipeline = true, ParameterSetName = "Files")]
[ValidateNotNull]
[Alias("File", "Files")]
public FileInfo[]? Path { get; set; }
/// <summary>
/// Optional bucket directory path where files should be uploaded (Files parameter set only)
/// </summary>
[Parameter(ParameterSetName = "Files")]
[ValidateNotNullOrEmpty]
[Alias("Folder", "Prefix")]
public string? BucketDirectory { get; set; }
/// <summary>
/// Directory to upload
/// </summary>
[Parameter(Position = 1, Mandatory = true, ValueFromPipelineByPropertyName = true, ParameterSetName = "Directory")]
[ValidateNotNull]
[Alias("Dir", "Folder")]
public DirectoryInfo? Directory { get; set; }
/// <summary>
/// Upload directory contents recursively (only applies to Directory parameter set)
/// </summary>
[Parameter(ParameterSetName = "Directory")]
public SwitchParameter Recursive { get; set; }
/// <summary>
/// Maximum depth for recursive directory upload (0 = unlimited)
/// </summary>
[Parameter(ParameterSetName = "Directory")]
[ValidateRange(0, int.MaxValue)]
public int MaxDepth { get; set; } = 0;
/// <summary>
/// Flatten directory structure (upload all files to bucket root)
/// </summary>
[Parameter(ParameterSetName = "Directory")]
public SwitchParameter Flatten { get; set; }
/// <summary>
/// Script block to filter files for inclusion
/// </summary>
[Parameter(ParameterSetName = "Directory")]
public ScriptBlock? InclusionFilter { get; set; }
/// <summary>
/// Script block to filter files for exclusion
/// </summary>
[Parameter(ParameterSetName = "Directory")]
public ScriptBlock? ExclusionFilter { get; set; }
/// <summary>
/// Overwrite existing objects without prompting
/// </summary>
[Parameter]
public SwitchParameter Force { get; set; }
/// <summary>
/// Generate presigned URLs for uploaded objects
/// </summary>
[Parameter]
public SwitchParameter ShowURL { get; set; }
/// <summary>
/// Expiration time for presigned URLs (default: 1 hour)
/// </summary>
[Parameter]
[ValidateRange("00:01:00", "7.00:00:00")] // 1 minute to 7 days
public TimeSpan Expiration { get; set; } = TimeSpan.FromHours(1);
/// <summary>
/// Size of each chunk for upload (default: 5MB)
/// </summary>
[Parameter]
[ValidateRange(1024 * 1024, 1024 * 1024 * 1024)] // 1MB to 1GB
public long ChunkSize { get; set; } = 5 * 1024 * 1024; // 5MB default
/// <summary>
/// Enable resume functionality for interrupted uploads
/// </summary>
[Parameter]
public SwitchParameter Resume { get; set; }
/// <summary>
/// Maximum number of retry attempts for failed chunks
/// </summary>
[Parameter]
[ValidateRange(1, 10)]
public int MaxRetries { get; set; } = 3;
/// <summary>
/// Custom path for storing resume data (default: %LOCALAPPDATA%\PSMinIO\Resume)
/// </summary>
[Parameter]
public string? ResumeDataPath { get; set; }
/// <summary>
/// Update progress every N bytes transferred (default: 1MB)
/// </summary>
[Parameter]
[ValidateRange(1024, 10 * 1024 * 1024)] // 1KB to 10MB
public long ProgressUpdateInterval { get; set; } = 1024 * 1024; // 1MB
/// <summary>
/// Processes the cmdlet
/// </summary>
protected override void ProcessRecord()
{
ValidateConnection();
ValidateBucketName(BucketName);
if (ParameterSetName == "Files")
{
ProcessFiles();
}
else if (ParameterSetName == "Directory")
{
ProcessDirectory();
}
}
/// <summary>
/// Processes file uploads from FileInfo array
/// </summary>
private void ProcessFiles()
{
if (Path == null || Path.Length == 0)
{
WriteWarning("No files provided for upload");
return;
}
if (ShouldProcess(BucketName, $"Upload {Path.Length} file(s) using chunked transfer"))
{
ExecuteOperation("UploadFilesChunked", () =>
{
// Check if bucket exists first
var bucketExists = Client.BucketExists(BucketName);
if (!bucketExists)
{
WriteError(new ErrorRecord(
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
"BucketNotFound",
ErrorCategory.ObjectNotFound,
BucketName));
return;
}
// Create bucket directory structure if specified
if (!string.IsNullOrWhiteSpace(BucketDirectory))
{
var sanitizedDirectory = SanitizeBucketDirectory(BucketDirectory);
if (!string.IsNullOrEmpty(sanitizedDirectory))
{
MinIOLogger.WriteVerbose(this, "Ensuring bucket directory exists: {0}", sanitizedDirectory);
EnsureBucketDirectoryExists(sanitizedDirectory);
}
}
UploadFileCollectionChunked(Path!);
}, $"Bucket: {BucketName}, Files: {Path.Length}, ChunkSize: {SizeFormatter.FormatSize(ChunkSize)}");
}
}
/// <summary>
/// Processes directory upload
/// </summary>
private void ProcessDirectory()
{
if (Directory == null || !Directory.Exists)
{
WriteError(new ErrorRecord(
new DirectoryNotFoundException($"Directory not found: {Directory?.FullName}"),
"DirectoryNotFound",
ErrorCategory.ObjectNotFound,
Directory));
return;
}
if (ShouldProcess(BucketName, $"Upload directory '{Directory.Name}' using chunked transfer"))
{
ExecuteOperation("UploadDirectoryChunked", () =>
{
// Check if bucket exists first
var bucketExists = Client.BucketExists(BucketName);
if (!bucketExists)
{
WriteError(new ErrorRecord(
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
"BucketNotFound",
ErrorCategory.ObjectNotFound,
BucketName));
return;
}
// Get files from directory
var files = GetDirectoryFiles();
if (files.Length == 0)
{
WriteWarning($"No files found in directory: {Directory.FullName}");
return;
}
MinIOLogger.WriteVerbose(this, "Found {0} files in directory '{1}'", files.Length, Directory.FullName);
UploadFileCollectionChunked(files);
}, $"Bucket: {BucketName}, Directory: {Directory.FullName}, ChunkSize: {SizeFormatter.FormatSize(ChunkSize)}");
}
}
/// <summary>
/// Gets files from directory based on filters and recursion settings
/// </summary>
/// <returns>Array of FileInfo objects</returns>
private FileInfo[] GetDirectoryFiles()
{
var searchOption = Recursive.IsPresent ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly;
var allFiles = Directory!.GetFiles("*", searchOption);
// Apply depth filtering if MaxDepth is specified and we're recursive
if (Recursive.IsPresent && MaxDepth > 0)
{
var basePath = Directory.FullName;
allFiles = allFiles.Where(f =>
{
var relativePath = Path.GetRelativePath(basePath, f.FullName);
var depth = relativePath.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar).Length - 1;
return depth <= MaxDepth;
}).ToArray();
}
// Apply inclusion filter
if (InclusionFilter != null)
{
allFiles = allFiles.Where(f => EvaluateFilter(InclusionFilter, f)).ToArray();
}
// Apply exclusion filter
if (ExclusionFilter != null)
{
allFiles = allFiles.Where(f => !EvaluateFilter(ExclusionFilter, f)).ToArray();
}
return allFiles;
}
/// <summary>
/// Evaluates a script block filter against a file
/// </summary>
/// <param name="filter">Script block filter</param>
/// <param name="file">File to evaluate</param>
/// <returns>True if file matches filter</returns>
private bool EvaluateFilter(ScriptBlock filter, FileInfo file)
{
try
{
var result = filter.InvokeWithContext(null, new[] { new PSVariable("_", file) });
return result.Count > 0 && LanguagePrimitives.IsTrue(result[0]);
}
catch (Exception ex)
{
WriteWarning($"Filter evaluation failed for {file.Name}: {ex.Message}");
return false;
}
}
/// <summary>
/// Uploads a collection of files using chunked transfer
/// </summary>
/// <param name="files">Files to upload</param>
private void UploadFileCollectionChunked(FileInfo[] files)
{
// Filter out files that don't exist
var validFiles = files.Where(f => f.Exists).ToArray();
var skippedCount = files.Length - validFiles.Length;
if (skippedCount > 0)
{
WriteWarning($"Skipped {skippedCount} files that do not exist");
}
if (validFiles.Length == 0)
{
WriteWarning("No valid files found for upload");
return;
}
MinIOLogger.WriteVerbose(this, "Starting chunked upload of {0} files to bucket '{1}' (ChunkSize: {2})",
validFiles.Length, BucketName, SizeFormatter.FormatSize(ChunkSize));
// Calculate total size for overall progress
var totalSize = validFiles.Sum(f => f.Length);
// Create progress reporter
var progressReporter = new ChunkedCollectionProgressReporter(
this,
validFiles.Length,
totalSize,
"Uploading",
ProgressUpdateInterval);
var uploadedObjects = new System.Collections.Generic.List<MinIOObjectInfo>();
for (int i = 0; i < validFiles.Length; i++)
{
var fileInfo = validFiles[i];
var objectName = GetObjectName(fileInfo);
try
{
// Calculate chunks for this file
var totalChunks = (int)Math.Ceiling((double)fileInfo.Length / ChunkSize);
progressReporter.StartNewFile(fileInfo.Name, fileInfo.Length, totalChunks);
// Upload file using chunked transfer
var uploadedObject = UploadFileChunked(fileInfo, objectName, progressReporter);
if (uploadedObject != null)
{
uploadedObjects.Add(uploadedObject);
progressReporter.CompleteFile();
}
}
catch (Exception ex)
{
WriteError(new ErrorRecord(
ex,
"ChunkedFileUploadFailed",
ErrorCategory.WriteError,
fileInfo));
MinIOLogger.WriteVerbose(this, "Failed to upload {0}: {1}", fileInfo.Name, ex.Message);
}
}
progressReporter.CompleteCollection();
// Always return uploaded objects
foreach (var obj in uploadedObjects)
{
WriteObject(obj);
}
MinIOLogger.WriteVerbose(this, "Completed chunked upload: {0} files ({1} successful, {2} failed)",
validFiles.Length, uploadedObjects.Count, validFiles.Length - uploadedObjects.Count);
}
/// <summary>
/// Uploads a single file using chunked transfer with resume capability
/// </summary>
/// <param name="fileInfo">File to upload</param>
/// <param name="objectName">Object name in bucket</param>
/// <param name="progressReporter">Progress reporter</param>
/// <returns>Uploaded object info or null if failed</returns>
private MinIOObjectInfo? UploadFileChunked(FileInfo fileInfo, string objectName, ChunkedCollectionProgressReporter progressReporter)
{
ChunkedTransferState? transferState = null;
// Try to load existing transfer state for resume
if (Resume.IsPresent)
{
transferState = ChunkedTransferResumeManager.LoadTransferState(
BucketName, objectName, fileInfo.FullName, ChunkedTransferType.Upload, ResumeDataPath);
if (transferState != null && !ChunkedTransferResumeManager.IsResumeDataValid(transferState, fileInfo))
{
MinIOLogger.WriteVerbose(this, "Resume data for {0} is invalid, starting fresh upload", fileInfo.Name);
transferState = null;
}
}
// Create new transfer state if none exists or resume is disabled
if (transferState == null)
{
transferState = new ChunkedTransferState
{
BucketName = BucketName,
ObjectName = objectName,
FilePath = fileInfo.FullName,
TotalSize = fileInfo.Length,
ChunkSize = ChunkSize,
LastModified = fileInfo.LastWriteTimeUtc,
TransferType = ChunkedTransferType.Upload,
StartTime = DateTime.UtcNow,
LastUpdated = DateTime.UtcNow
};
}
else
{
MinIOLogger.WriteVerbose(this, "Resuming upload of {0} from {1:P1} complete",
fileInfo.Name, transferState.ProgressPercentage / 100);
}
try
{
// Upload file using chunked transfer
var result = Client.UploadFileChunked(transferState, progressReporter, MaxRetries);
if (result != null)
{
// Generate presigned URL if requested
if (ShowURL.IsPresent)
{
try
{
var presignedUrl = Client.GetPresignedUrl(BucketName, objectName, Expiration);
result.PresignedUrl = presignedUrl;
result.PresignedUrlExpiration = DateTime.UtcNow.Add(Expiration);
}
catch (Exception ex)
{
WriteWarning($"Could not generate presigned URL for {objectName}: {ex.Message}");
}
}
// Clean up resume data on successful completion
if (Resume.IsPresent)
{
ChunkedTransferResumeManager.CleanupResumeData(transferState, ResumeDataPath);
}
return result;
}
}
catch (Exception ex)
{
// Save resume data on failure if resume is enabled
if (Resume.IsPresent)
{
try
{
ChunkedTransferResumeManager.SaveTransferState(transferState, ResumeDataPath);
MinIOLogger.WriteVerbose(this, "Saved resume data for {0}", fileInfo.Name);
}
catch (Exception saveEx)
{
WriteWarning($"Could not save resume data for {fileInfo.Name}: {saveEx.Message}");
}
}
throw;
}
return null;
}
/// <summary>
/// Gets the object name for a file based on the upload context
/// </summary>
/// <param name="file">File to get object name for</param>
/// <returns>Object name to use in MinIO</returns>
private string GetObjectName(FileInfo file)
{
if (ParameterSetName == "Files")
{
// For file uploads, optionally prefix with bucket directory
var objectName = file.Name;
if (!string.IsNullOrWhiteSpace(BucketDirectory))
{
var sanitizedDirectory = SanitizeBucketDirectory(BucketDirectory);
objectName = $"{sanitizedDirectory}/{file.Name}";
}
return objectName;
}
else if (ParameterSetName == "Directory")
{
if (Flatten.IsPresent)
{
// Flatten: use just the filename
return file.Name;
}
else
{
// Maintain directory structure relative to the base directory
var relativePath = Path.GetRelativePath(Directory!.FullName, file.FullName);
return relativePath.Replace('\\', '/'); // Ensure forward slashes for object storage
}
}
return file.Name;
}
/// <summary>
/// Sanitizes bucket directory path and ensures proper format
/// </summary>
/// <param name="directory">Raw directory path</param>
/// <returns>Sanitized directory path</returns>
private string SanitizeBucketDirectory(string directory)
{
if (string.IsNullOrWhiteSpace(directory))
return string.Empty;
// Remove leading and trailing slashes, then split and clean
var cleanDirectory = directory.Trim().Trim('/', '\\');
// Split by both forward and back slashes, then clean each part
var parts = cleanDirectory.Split(new char[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries);
// Clean each part and rejoin with forward slashes
var cleanedParts = parts.Select(part => part.Trim()).Where(part => !string.IsNullOrEmpty(part));
return string.Join("/", cleanedParts);
}
/// <summary>
/// Creates bucket directory structure if it doesn't exist
/// </summary>
/// <param name="directoryPath">Directory path to create</param>
private void EnsureBucketDirectoryExists(string directoryPath)
{
if (string.IsNullOrWhiteSpace(directoryPath))
return;
var parts = directoryPath.Split('/');
var currentPath = string.Empty;
foreach (var part in parts)
{
currentPath = string.IsNullOrEmpty(currentPath) ? part : $"{currentPath}/{part}";
var folderPath = $"{currentPath}/";
try
{
// Check if this directory level already exists by trying to list objects with this prefix
var existingObjects = Client.ListObjects(BucketName, folderPath, false);
var folderExists = existingObjects.Any(obj => obj.Name == folderPath);
if (!folderExists)
{
MinIOLogger.WriteVerbose(this, "Creating bucket directory: {0}", folderPath);
// Create the directory by uploading a zero-byte object
using var emptyStream = new MemoryStream();
Client.UploadStream(BucketName, folderPath, emptyStream, "application/x-directory");
MinIOLogger.WriteVerbose(this, "Successfully created bucket directory: {0}", folderPath);
}
}
catch (Exception ex)
{
WriteWarning($"Could not create bucket directory '{folderPath}': {ex.Message}");
}
}
}
}
}
+489
View File
@@ -0,0 +1,489 @@
using System;
using System.IO;
using System.Linq;
using System.Management.Automation;
using PSMinIO.Models;
using PSMinIO.Utils;
namespace PSMinIO.Cmdlets
{
/// <summary>
/// Uploads files or directories to a MinIO bucket
/// </summary>
[Cmdlet(VerbsCommon.New, "MinIOObject", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Low)]
[OutputType(typeof(MinIOObjectInfo))]
public class NewMinIOObjectCmdlet : MinIOBaseCmdlet
{
/// <summary>
/// Name of the bucket to upload to
/// </summary>
[Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true)]
[ValidateNotNullOrEmpty]
[Alias("Bucket")]
public string BucketName { get; set; } = string.Empty;
/// <summary>
/// Array of FileInfo objects to upload
/// </summary>
[Parameter(Position = 1, Mandatory = true, ValueFromPipeline = true, ParameterSetName = "Files")]
[ValidateNotNull]
[Alias("File", "Files")]
public FileInfo[]? Path { get; set; }
/// <summary>
/// Optional bucket directory path where files should be uploaded (Files parameter set only)
/// </summary>
[Parameter(ParameterSetName = "Files")]
[ValidateNotNullOrEmpty]
[Alias("Folder", "Prefix")]
public string? BucketDirectory { get; set; }
/// <summary>
/// Directory to upload
/// </summary>
[Parameter(Position = 1, Mandatory = true, ValueFromPipelineByPropertyName = true, ParameterSetName = "Directory")]
[ValidateNotNull]
[Alias("Dir", "Folder")]
public DirectoryInfo? Directory { get; set; }
/// <summary>
/// Upload directory contents recursively (only applies to Directory parameter set)
/// </summary>
[Parameter(ParameterSetName = "Directory")]
public SwitchParameter Recursive { get; set; }
/// <summary>
/// Maximum depth for recursive directory upload (0 = unlimited)
/// </summary>
[Parameter(ParameterSetName = "Directory")]
[ValidateRange(0, int.MaxValue)]
public int MaxDepth { get; set; } = 0;
/// <summary>
/// Flatten directory structure (upload all files to bucket root)
/// </summary>
[Parameter(ParameterSetName = "Directory")]
public SwitchParameter Flatten { get; set; }
/// <summary>
/// Script block to filter files for inclusion
/// </summary>
[Parameter(ParameterSetName = "Directory")]
public ScriptBlock? InclusionFilter { get; set; }
/// <summary>
/// Script block to filter files for exclusion
/// </summary>
[Parameter(ParameterSetName = "Directory")]
public ScriptBlock? ExclusionFilter { get; set; }
/// <summary>
/// Overwrite existing objects without prompting
/// </summary>
[Parameter]
public SwitchParameter Force { get; set; }
/// <summary>
/// Generate presigned URLs for uploaded objects
/// </summary>
[Parameter]
public SwitchParameter ShowURL { get; set; }
/// <summary>
/// Expiration time for presigned URLs (default: 1 hour)
/// </summary>
[Parameter]
[ValidateRange("00:01:00", "7.00:00:00")] // 1 minute to 7 days
public TimeSpan Expiration { get; set; } = TimeSpan.FromHours(1);
/// <summary>
/// Processes the cmdlet
/// </summary>
protected override void ProcessRecord()
{
ValidateConnection();
ValidateBucketName(BucketName);
if (ParameterSetName == "Files")
{
ProcessFiles();
}
else if (ParameterSetName == "Directory")
{
ProcessDirectory();
}
}
/// <summary>
/// Processes file uploads from FileInfo array
/// </summary>
private void ProcessFiles()
{
if (Path == null || Path.Length == 0)
{
WriteWarning("No files provided for upload");
return;
}
if (ShouldProcess(BucketName, $"Upload {Path.Length} file(s)"))
{
ExecuteOperation("UploadFiles", () =>
{
// Check if bucket exists first
var bucketExists = Client.BucketExists(BucketName);
if (!bucketExists)
{
WriteError(new ErrorRecord(
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
"BucketNotFound",
ErrorCategory.ObjectNotFound,
BucketName));
return;
}
// Create bucket directory structure if specified
if (!string.IsNullOrWhiteSpace(BucketDirectory))
{
var sanitizedDirectory = SanitizeBucketDirectory(BucketDirectory);
if (!string.IsNullOrEmpty(sanitizedDirectory))
{
MinIOLogger.WriteVerbose(this, "Ensuring bucket directory exists: {0}", sanitizedDirectory);
EnsureBucketDirectoryExists(sanitizedDirectory);
}
}
UploadFileCollection(Path!);
}, $"Bucket: {BucketName}, Files: {Path.Length}");
}
}
/// <summary>
/// Processes directory upload
/// </summary>
private void ProcessDirectory()
{
if (Directory == null || !Directory.Exists)
{
WriteError(new ErrorRecord(
new DirectoryNotFoundException($"Directory not found: {Directory?.FullName}"),
"DirectoryNotFound",
ErrorCategory.ObjectNotFound,
Directory));
return;
}
if (ShouldProcess(BucketName, $"Upload directory '{Directory.Name}'"))
{
ExecuteOperation("UploadDirectory", () =>
{
// Check if bucket exists first
var bucketExists = Client.BucketExists(BucketName);
if (!bucketExists)
{
WriteError(new ErrorRecord(
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
"BucketNotFound",
ErrorCategory.ObjectNotFound,
BucketName));
return;
}
// Get files from directory
var files = GetDirectoryFiles();
if (files.Length == 0)
{
WriteWarning($"No files found in directory: {Directory.FullName}");
return;
}
MinIOLogger.WriteVerbose(this, "Found {0} files in directory '{1}'", files.Length, Directory.FullName);
UploadFileCollection(files);
}, $"Bucket: {BucketName}, Directory: {Directory.FullName}");
}
}
/// <summary>
/// Gets files from directory based on filters and recursion settings
/// </summary>
/// <returns>Array of FileInfo objects</returns>
private FileInfo[] GetDirectoryFiles()
{
var searchOption = Recursive.IsPresent ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly;
var allFiles = Directory!.GetFiles("*", searchOption);
// Apply depth filtering if MaxDepth is specified and we're recursive
if (Recursive.IsPresent && MaxDepth > 0)
{
var basePath = Directory.FullName;
allFiles = allFiles.Where(f =>
{
var relativePath = Path.GetRelativePath(basePath, f.FullName);
var depth = relativePath.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar).Length - 1;
return depth <= MaxDepth;
}).ToArray();
}
// Apply inclusion filter
if (InclusionFilter != null)
{
allFiles = allFiles.Where(f => EvaluateFilter(InclusionFilter, f)).ToArray();
}
// Apply exclusion filter
if (ExclusionFilter != null)
{
allFiles = allFiles.Where(f => !EvaluateFilter(ExclusionFilter, f)).ToArray();
}
return allFiles;
}
/// <summary>
/// Evaluates a script block filter against a file
/// </summary>
/// <param name="filter">Script block filter</param>
/// <param name="file">File to evaluate</param>
/// <returns>True if file matches filter</returns>
private bool EvaluateFilter(ScriptBlock filter, FileInfo file)
{
try
{
var result = filter.InvokeWithContext(null, new[] { new PSVariable("_", file) });
return result.Count > 0 && LanguagePrimitives.IsTrue(result[0]);
}
catch (Exception ex)
{
WriteWarning($"Filter evaluation failed for {file.Name}: {ex.Message}");
return false;
}
}
/// <summary>
/// Uploads a collection of files
/// </summary>
/// <param name="files">Files to upload</param>
private void UploadFileCollection(FileInfo[] files)
{
// Filter out files that don't exist
var validFiles = files.Where(f => f.Exists).ToArray();
var skippedCount = files.Length - validFiles.Length;
if (skippedCount > 0)
{
WriteWarning($"Skipped {skippedCount} files that do not exist");
}
if (validFiles.Length == 0)
{
WriteWarning("No valid files found for upload");
return;
}
MinIOLogger.WriteVerbose(this, "Uploading {0} files to bucket '{1}'", validFiles.Length, BucketName);
// Calculate total size for overall progress
var totalSize = validFiles.Sum(f => f.Length);
var totalProcessed = 0L;
// Create overall progress reporter
var overallProgress = new ProgressReporter(
this,
"Uploading Files",
$"Uploading {validFiles.Length} files",
totalSize,
1);
var uploadedObjects = new System.Collections.Generic.List<MinIOObjectInfo>();
for (int i = 0; i < validFiles.Length; i++)
{
var fileInfo = validFiles[i];
var objectName = GetObjectName(fileInfo);
try
{
// Show file-level progress
var fileProgressRecord = new ProgressRecord(2, "Current File",
$"Uploading: {fileInfo.Name}")
{
PercentComplete = (int)((double)(i + 1) / validFiles.Length * 100),
ParentActivityId = 1
};
WriteProgress(fileProgressRecord);
MinIOLogger.WriteVerbose(this, "Uploading file {0}/{1}: {2} -> {3}",
i + 1, validFiles.Length, fileInfo.Name, objectName);
// Upload the file
var etag = Client.UploadFile(
BucketName,
objectName,
fileInfo.FullName,
null, // Auto-detect content type
bytesTransferred =>
{
overallProgress.UpdateProgress(totalProcessed + bytesTransferred);
});
totalProcessed += fileInfo.Length;
overallProgress.UpdateProgress(totalProcessed);
// Create object info for result
var uploadedObject = new MinIOObjectInfo(
objectName,
fileInfo.Length,
DateTime.UtcNow,
etag,
BucketName);
// Generate presigned URL if requested
if (ShowURL.IsPresent)
{
try
{
var presignedUrl = Client.GetPresignedUrl(BucketName, objectName, Expiration);
uploadedObject.PresignedUrl = presignedUrl;
uploadedObject.PresignedUrlExpiration = DateTime.UtcNow.Add(Expiration);
}
catch (Exception ex)
{
WriteWarning($"Could not generate presigned URL for {objectName}: {ex.Message}");
}
}
uploadedObjects.Add(uploadedObject);
MinIOLogger.WriteVerbose(this, "Successfully uploaded: {0}", fileInfo.Name);
}
catch (Exception ex)
{
WriteError(new ErrorRecord(
ex,
"FileUploadFailed",
ErrorCategory.WriteError,
fileInfo));
MinIOLogger.WriteVerbose(this, "Failed to upload {0}: {1}", fileInfo.Name, ex.Message);
}
}
// Complete progress reporting
overallProgress.Complete();
WriteProgress(new ProgressRecord(2, "Current File", "Completed")
{
PercentComplete = 100,
RecordType = ProgressRecordType.Completed,
ParentActivityId = 1
});
// Always return uploaded objects
foreach (var obj in uploadedObjects)
{
WriteObject(obj);
}
MinIOLogger.WriteVerbose(this, "Completed uploading {0} files ({1} successful, {2} failed)",
validFiles.Length, uploadedObjects.Count, validFiles.Length - uploadedObjects.Count);
}
/// <summary>
/// Gets the object name for a file based on the upload context
/// </summary>
/// <param name="file">File to get object name for</param>
/// <returns>Object name to use in MinIO</returns>
private string GetObjectName(FileInfo file)
{
if (ParameterSetName == "Files")
{
// For file uploads, optionally prefix with bucket directory
var objectName = file.Name;
if (!string.IsNullOrWhiteSpace(BucketDirectory))
{
var sanitizedDirectory = SanitizeBucketDirectory(BucketDirectory);
objectName = $"{sanitizedDirectory}/{file.Name}";
}
return objectName;
}
else if (ParameterSetName == "Directory")
{
if (Flatten.IsPresent)
{
// Flatten: use just the filename
return file.Name;
}
else
{
// Maintain directory structure relative to the base directory
var relativePath = Path.GetRelativePath(Directory!.FullName, file.FullName);
return relativePath.Replace('\\', '/'); // Ensure forward slashes for object storage
}
}
return file.Name;
}
/// <summary>
/// Sanitizes bucket directory path and ensures proper format
/// </summary>
/// <param name="directory">Raw directory path</param>
/// <returns>Sanitized directory path</returns>
private string SanitizeBucketDirectory(string directory)
{
if (string.IsNullOrWhiteSpace(directory))
return string.Empty;
// Remove leading and trailing slashes, then split and clean
var cleanDirectory = directory.Trim().Trim('/', '\\');
// Split by both forward and back slashes, then clean each part
var parts = cleanDirectory.Split(new char[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries);
// Clean each part and rejoin with forward slashes
var cleanedParts = parts.Select(part => part.Trim()).Where(part => !string.IsNullOrEmpty(part));
return string.Join("/", cleanedParts);
}
/// <summary>
/// Creates bucket directory structure if it doesn't exist
/// </summary>
/// <param name="directoryPath">Directory path to create</param>
private void EnsureBucketDirectoryExists(string directoryPath)
{
if (string.IsNullOrWhiteSpace(directoryPath))
return;
var parts = directoryPath.Split('/');
var currentPath = string.Empty;
foreach (var part in parts)
{
currentPath = string.IsNullOrEmpty(currentPath) ? part : $"{currentPath}/{part}";
var folderPath = $"{currentPath}/";
try
{
// Check if this directory level already exists by trying to list objects with this prefix
var existingObjects = Client.ListObjects(BucketName, folderPath, false);
var folderExists = existingObjects.Any(obj => obj.Name == folderPath);
if (!folderExists)
{
MinIOLogger.WriteVerbose(this, "Creating bucket directory: {0}", folderPath);
// Create the directory by uploading a zero-byte object
using var emptyStream = new MemoryStream();
Client.UploadStream(BucketName, folderPath, emptyStream, "application/x-directory");
MinIOLogger.WriteVerbose(this, "Successfully created bucket directory: {0}", folderPath);
}
}
catch (Exception ex)
{
WriteWarning($"Could not create bucket directory '{folderPath}': {ex.Message}");
}
}
}
}
}
+194
View File
@@ -0,0 +1,194 @@
using System;
using System.Management.Automation;
using PSMinIO.Utils;
namespace PSMinIO.Cmdlets
{
/// <summary>
/// Removes a MinIO bucket
/// </summary>
[Cmdlet(VerbsCommon.Remove, "MinIOBucket", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.High)]
public class RemoveMinIOBucketCmdlet : MinIOBaseCmdlet
{
/// <summary>
/// Name of the bucket to remove
/// </summary>
[Parameter(Position = 0, Mandatory = true, ValueFromPipeline = true, ValueFromPipelineByPropertyName = true)]
[ValidateNotNullOrEmpty]
[Alias("Bucket")]
public string BucketName { get; set; } = string.Empty;
/// <summary>
/// Force removal without confirmation prompts
/// </summary>
[Parameter]
public SwitchParameter Force { get; set; }
/// <summary>
/// Remove all objects in the bucket before removing the bucket itself
/// </summary>
[Parameter]
[Alias("Recursive")]
public SwitchParameter RemoveObjects { get; set; }
/// <summary>
/// Processes the cmdlet
/// </summary>
protected override void ProcessRecord()
{
ValidateConfiguration();
ValidateBucketName(BucketName);
// Override confirmation if Force is specified
if (Force.IsPresent)
{
ConfirmPreference = ConfirmImpact.None;
}
var actionDescription = RemoveObjects.IsPresent
? $"Remove bucket '{BucketName}' and all its objects"
: $"Remove bucket '{BucketName}'";
if (ShouldProcess(BucketName, actionDescription))
{
ExecuteOperation("RemoveBucket", () =>
{
// Check if bucket exists
var exists = Client.BucketExists(BucketName);
if (!exists)
{
WriteError(new ErrorRecord(
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
"BucketNotFound",
ErrorCategory.ObjectNotFound,
BucketName));
return;
}
// If RemoveObjects is specified, remove all objects first
if (RemoveObjects.IsPresent)
{
RemoveAllObjectsFromBucket();
}
else
{
// Check if bucket is empty before attempting to remove
CheckBucketEmpty();
}
// Remove the bucket
MinIOLogger.WriteVerbose(this, "Removing bucket '{0}'", BucketName);
Client.DeleteBucket(BucketName);
MinIOLogger.WriteVerbose(this, "Successfully removed bucket '{0}'", BucketName);
}, $"Bucket: {BucketName}");
}
}
/// <summary>
/// Removes all objects from the bucket
/// </summary>
private void RemoveAllObjectsFromBucket()
{
MinIOLogger.WriteVerbose(this, "Removing all objects from bucket '{0}'", BucketName);
try
{
var objects = Client.ListObjects(BucketName, recursive: true);
if (objects.Count == 0)
{
MinIOLogger.WriteVerbose(this, "Bucket '{0}' is already empty", BucketName);
return;
}
MinIOLogger.WriteVerbose(this, "Found {0} objects to remove from bucket '{1}'", objects.Count, BucketName);
// Show progress for object removal
for (int i = 0; i < objects.Count; i++)
{
var obj = objects[i];
var progressRecord = new ProgressRecord(1, "Removing Objects",
$"Removing object: {obj.Name}")
{
PercentComplete = (int)((double)(i + 1) / objects.Count * 100)
};
WriteProgress(progressRecord);
try
{
Client.DeleteObject(BucketName, obj.Name);
MinIOLogger.WriteVerbose(this, "Removed object: {0}", obj.Name);
}
catch (Exception ex)
{
MinIOLogger.WriteWarning(this,
"Failed to remove object '{0}': {1}", obj.Name, ex.Message);
}
}
// Complete progress
WriteProgress(new ProgressRecord(1, "Removing Objects", "Completed")
{
PercentComplete = 100,
RecordType = ProgressRecordType.Completed
});
MinIOLogger.WriteVerbose(this, "Finished removing objects from bucket '{0}'", BucketName);
}
catch (Exception ex)
{
MinIOLogger.WriteWarning(this,
"Failed to list or remove objects from bucket '{0}': {1}", BucketName, ex.Message);
if (!Force.IsPresent)
{
WriteError(new ErrorRecord(
new InvalidOperationException($"Cannot remove objects from bucket '{BucketName}': {ex.Message}"),
"ObjectRemovalFailed",
ErrorCategory.InvalidOperation,
BucketName));
return;
}
}
}
/// <summary>
/// Checks if the bucket is empty and warns if it's not
/// </summary>
private void CheckBucketEmpty()
{
try
{
var objects = Client.ListObjects(BucketName, recursive: true);
if (objects.Count > 0)
{
var message = $"Bucket '{BucketName}' contains {objects.Count} objects. " +
"Use -RemoveObjects parameter to remove all objects first, or remove them manually.";
WriteError(new ErrorRecord(
new InvalidOperationException(message),
"BucketNotEmpty",
ErrorCategory.InvalidOperation,
BucketName));
}
}
catch (Exception ex)
{
MinIOLogger.WriteWarning(this,
"Could not check if bucket '{0}' is empty: {1}", BucketName, ex.Message);
if (!Force.IsPresent)
{
WriteError(new ErrorRecord(
new InvalidOperationException($"Cannot verify bucket '{BucketName}' is empty: {ex.Message}"),
"BucketCheckFailed",
ErrorCategory.InvalidOperation,
BucketName));
}
}
}
}
}
+229
View File
@@ -0,0 +1,229 @@
using System;
using System.Linq;
using System.Management.Automation;
using PSMinIO.Utils;
namespace PSMinIO.Cmdlets
{
/// <summary>
/// Removes an object from a MinIO bucket
/// </summary>
[Cmdlet(VerbsCommon.Remove, "MinIOObject", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.High)]
public class RemoveMinIOObjectCmdlet : MinIOBaseCmdlet
{
/// <summary>
/// Name of the bucket containing the object
/// </summary>
[Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true)]
[ValidateNotNullOrEmpty]
[Alias("Bucket")]
public string BucketName { get; set; } = string.Empty;
/// <summary>
/// Name of the object to remove
/// </summary>
[Parameter(Position = 1, Mandatory = true, ValueFromPipelineByPropertyName = true)]
[ValidateNotNullOrEmpty]
[Alias("Object", "Key")]
public string ObjectName { get; set; } = string.Empty;
/// <summary>
/// Force removal without confirmation prompts
/// </summary>
[Parameter]
public SwitchParameter Force { get; set; }
/// <summary>
/// Remove all objects matching the specified prefix (use with caution)
/// </summary>
[Parameter]
[Alias("Recursive")]
public SwitchParameter RemovePrefix { get; set; }
/// <summary>
/// Processes the cmdlet
/// </summary>
protected override void ProcessRecord()
{
ValidateConfiguration();
ValidateBucketName(BucketName);
ValidateObjectName(ObjectName);
// Override confirmation if Force is specified
if (Force.IsPresent)
{
ConfirmPreference = ConfirmImpact.None;
}
var actionDescription = RemovePrefix.IsPresent
? $"Remove all objects with prefix '{ObjectName}' from bucket '{BucketName}'"
: $"Remove object '{ObjectName}' from bucket '{BucketName}'";
if (ShouldProcess($"{BucketName}/{ObjectName}", actionDescription))
{
ExecuteOperation("RemoveObject", () =>
{
// Check if bucket exists
var bucketExists = Client.BucketExists(BucketName);
if (!bucketExists)
{
WriteError(new ErrorRecord(
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
"BucketNotFound",
ErrorCategory.ObjectNotFound,
BucketName));
return;
}
if (RemovePrefix.IsPresent)
{
RemoveObjectsWithPrefix();
}
else
{
RemoveSingleObject();
}
}, $"Bucket: {BucketName}, Object: {ObjectName}");
}
}
/// <summary>
/// Removes a single object
/// </summary>
private void RemoveSingleObject()
{
// Check if object exists
var objectExists = CheckObjectExists();
if (!objectExists)
{
WriteError(new ErrorRecord(
new InvalidOperationException($"Object '{ObjectName}' does not exist in bucket '{BucketName}'"),
"ObjectNotFound",
ErrorCategory.ObjectNotFound,
ObjectName));
return;
}
MinIOLogger.WriteVerbose(this,
"Removing object '{0}' from bucket '{1}'", ObjectName, BucketName);
Client.DeleteObject(BucketName, ObjectName);
MinIOLogger.WriteVerbose(this,
"Successfully removed object '{0}' from bucket '{1}'", ObjectName, BucketName);
}
/// <summary>
/// Removes all objects with the specified prefix
/// </summary>
private void RemoveObjectsWithPrefix()
{
MinIOLogger.WriteVerbose(this,
"Finding objects with prefix '{0}' in bucket '{1}'", ObjectName, BucketName);
try
{
var objects = Client.ListObjects(BucketName, ObjectName, true);
if (objects.Count == 0)
{
MinIOLogger.WriteVerbose(this,
"No objects found with prefix '{0}' in bucket '{1}'", ObjectName, BucketName);
return;
}
MinIOLogger.WriteVerbose(this,
"Found {0} objects with prefix '{1}' in bucket '{2}'",
objects.Count, ObjectName, BucketName);
// Additional confirmation for prefix removal if not forced
if (!Force.IsPresent && objects.Count > 1)
{
var message = $"This will remove {objects.Count} objects with prefix '{ObjectName}'. Are you sure?";
if (!ShouldContinue(message, "Confirm Multiple Object Removal"))
{
return;
}
}
// Remove objects with progress reporting
for (int i = 0; i < objects.Count; i++)
{
var obj = objects[i];
var progressRecord = new ProgressRecord(1, "Removing Objects",
$"Removing object: {obj.Name}")
{
PercentComplete = (int)((double)(i + 1) / objects.Count * 100)
};
WriteProgress(progressRecord);
try
{
Client.DeleteObject(BucketName, obj.Name);
MinIOLogger.WriteVerbose(this, "Removed object: {0}", obj.Name);
}
catch (Exception ex)
{
MinIOLogger.WriteWarning(this,
"Failed to remove object '{0}': {1}", obj.Name, ex.Message);
if (!Force.IsPresent)
{
WriteError(new ErrorRecord(
new InvalidOperationException($"Failed to remove object '{obj.Name}': {ex.Message}", ex),
"ObjectRemovalFailed",
ErrorCategory.InvalidOperation,
obj.Name));
}
}
}
// Complete progress
WriteProgress(new ProgressRecord(1, "Removing Objects", "Completed")
{
PercentComplete = 100,
RecordType = ProgressRecordType.Completed
});
MinIOLogger.WriteVerbose(this,
"Finished removing objects with prefix '{0}' from bucket '{1}'", ObjectName, BucketName);
}
catch (Exception ex)
{
MinIOLogger.WriteWarning(this,
"Failed to list objects with prefix '{0}' in bucket '{1}': {2}",
ObjectName, BucketName, ex.Message);
WriteError(new ErrorRecord(
new InvalidOperationException($"Cannot list objects with prefix '{ObjectName}': {ex.Message}", ex),
"ObjectListingFailed",
ErrorCategory.InvalidOperation,
ObjectName));
}
}
/// <summary>
/// Checks if the specified object exists
/// </summary>
/// <returns>True if object exists, false otherwise</returns>
private bool CheckObjectExists()
{
try
{
var objects = Client.ListObjects(BucketName, ObjectName, false);
return objects.Any(obj =>
string.Equals(obj.Name, ObjectName, StringComparison.Ordinal));
}
catch (Exception ex)
{
MinIOLogger.WriteWarning(this,
"Could not check if object '{0}' exists: {1}", ObjectName, ex.Message);
// If we can't verify existence, assume it exists and let the delete operation handle it
return true;
}
}
}
}
+336
View File
@@ -0,0 +1,336 @@
using System;
using System.IO;
using System.Management.Automation;
using System.Text.Json;
using PSMinIO.Utils;
namespace PSMinIO.Cmdlets
{
/// <summary>
/// Sets the policy for a MinIO bucket
/// </summary>
[Cmdlet(VerbsCommon.Set, "MinIOBucketPolicy", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Medium)]
public class SetMinIOBucketPolicyCmdlet : MinIOBaseCmdlet
{
/// <summary>
/// Name of the bucket to set the policy for
/// </summary>
[Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true)]
[ValidateNotNullOrEmpty]
[Alias("Bucket")]
public string BucketName { get; set; } = string.Empty;
/// <summary>
/// Policy JSON string
/// </summary>
[Parameter(Position = 1, Mandatory = true, ParameterSetName = "PolicyJson")]
[ValidateNotNullOrEmpty]
[Alias("Json")]
public string? PolicyJson { get; set; }
/// <summary>
/// Path to a file containing the policy JSON
/// </summary>
[Parameter(Mandatory = true, ParameterSetName = "PolicyFile")]
[ValidateNotNullOrEmpty]
[Alias("File")]
public string? PolicyFilePath { get; set; }
/// <summary>
/// Use a predefined canned policy
/// </summary>
[Parameter(Mandatory = true, ParameterSetName = "CannedPolicy")]
[ValidateSet("ReadOnly", "WriteOnly", "ReadWrite", "None")]
public string? CannedPolicy { get; set; }
/// <summary>
/// Prefix for canned policies (default: *)
/// </summary>
[Parameter(ParameterSetName = "CannedPolicy")]
[ValidateNotNullOrEmpty]
public string Prefix { get; set; } = "*";
/// <summary>
/// Validate the policy JSON before setting it
/// </summary>
[Parameter]
public SwitchParameter ValidateOnly { get; set; }
/// <summary>
/// Force setting the policy without confirmation
/// </summary>
[Parameter]
public SwitchParameter Force { get; set; }
/// <summary>
/// Processes the cmdlet
/// </summary>
protected override void ProcessRecord()
{
ValidateConfiguration();
ValidateBucketName(BucketName);
// Get the policy JSON based on the parameter set
var policyJson = GetPolicyJson();
if (string.IsNullOrWhiteSpace(policyJson))
{
return; // Error already written
}
// Validate the policy JSON
if (!ValidatePolicyJson(policyJson))
{
return; // Error already written
}
if (ValidateOnly.IsPresent)
{
WriteObject("Policy JSON is valid");
return;
}
// Override confirmation if Force is specified
if (Force.IsPresent)
{
ConfirmPreference = ConfirmImpact.None;
}
if (ShouldProcess(BucketName, "Set bucket policy"))
{
ExecuteOperation("SetBucketPolicy", () =>
{
// Check if bucket exists
var bucketExists = Client.BucketExists(BucketName);
if (!bucketExists)
{
WriteError(new ErrorRecord(
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
"BucketNotFound",
ErrorCategory.ObjectNotFound,
BucketName));
return;
}
MinIOLogger.WriteVerbose(this, "Setting policy for bucket '{0}'", BucketName);
MinIOLogger.WriteVerbose(this, "Policy JSON ({0} characters): {1}",
policyJson.Length, policyJson.Length > 200 ? policyJson.Substring(0, 200) + "..." : policyJson);
Client.SetBucketPolicy(BucketName, policyJson);
MinIOLogger.WriteVerbose(this, "Successfully set policy for bucket '{0}'", BucketName);
}, $"Bucket: {BucketName}");
}
}
/// <summary>
/// Gets the policy JSON based on the current parameter set
/// </summary>
/// <returns>Policy JSON string or null if error</returns>
private string? GetPolicyJson()
{
switch (ParameterSetName)
{
case "PolicyJson":
return PolicyJson;
case "PolicyFile":
return ReadPolicyFromFile();
case "CannedPolicy":
return GenerateCannedPolicy();
default:
WriteError(new ErrorRecord(
new InvalidOperationException("Unknown parameter set"),
"UnknownParameterSet",
ErrorCategory.InvalidArgument,
null));
return null;
}
}
/// <summary>
/// Reads policy JSON from a file
/// </summary>
/// <returns>Policy JSON string or null if error</returns>
private string? ReadPolicyFromFile()
{
if (string.IsNullOrWhiteSpace(PolicyFilePath))
return null;
try
{
var fullPath = Path.GetFullPath(PolicyFilePath);
if (!File.Exists(fullPath))
{
WriteError(new ErrorRecord(
new FileNotFoundException($"Policy file not found: {fullPath}"),
"PolicyFileNotFound",
ErrorCategory.ObjectNotFound,
fullPath));
return null;
}
var content = File.ReadAllText(fullPath);
MinIOLogger.WriteVerbose(this, "Read policy from file '{0}' ({1} characters)", fullPath, content.Length);
return content;
}
catch (Exception ex)
{
WriteError(new ErrorRecord(
new InvalidOperationException($"Failed to read policy file '{PolicyFilePath}': {ex.Message}", ex),
"PolicyFileReadError",
ErrorCategory.ReadError,
PolicyFilePath));
return null;
}
}
/// <summary>
/// Generates a canned policy based on the specified type
/// </summary>
/// <returns>Policy JSON string</returns>
private string GenerateCannedPolicy()
{
var bucketArn = $"arn:aws:s3:::{BucketName}";
var objectArn = $"arn:aws:s3:::{BucketName}/{Prefix}";
var policy = CannedPolicy?.ToLowerInvariant() switch
{
"readonly" => CreateReadOnlyPolicy(bucketArn, objectArn),
"writeonly" => CreateWriteOnlyPolicy(bucketArn, objectArn),
"readwrite" => CreateReadWritePolicy(bucketArn, objectArn),
"none" => CreateEmptyPolicy(),
_ => throw new ArgumentException($"Unknown canned policy: {CannedPolicy}")
};
MinIOLogger.WriteVerbose(this, "Generated {0} canned policy for bucket '{1}' with prefix '{2}'",
CannedPolicy, BucketName, Prefix);
return JsonSerializer.Serialize(policy, new JsonSerializerOptions { WriteIndented = true });
}
/// <summary>
/// Creates a read-only policy
/// </summary>
private object CreateReadOnlyPolicy(string bucketArn, string objectArn)
{
return new
{
Version = "2012-10-17",
Statement = new[]
{
new
{
Effect = "Allow",
Principal = new { AWS = "*" },
Action = new[] { "s3:GetBucketLocation", "s3:ListBucket" },
Resource = bucketArn
},
new
{
Effect = "Allow",
Principal = new { AWS = "*" },
Action = new[] { "s3:GetObject" },
Resource = objectArn
}
}
};
}
/// <summary>
/// Creates a write-only policy
/// </summary>
private object CreateWriteOnlyPolicy(string bucketArn, string objectArn)
{
return new
{
Version = "2012-10-17",
Statement = new[]
{
new
{
Effect = "Allow",
Principal = new { AWS = "*" },
Action = new[] { "s3:GetBucketLocation", "s3:ListBucketMultipartUploads" },
Resource = bucketArn
},
new
{
Effect = "Allow",
Principal = new { AWS = "*" },
Action = new[] { "s3:PutObject", "s3:AbortMultipartUpload", "s3:DeleteObject", "s3:ListMultipartUploadParts" },
Resource = objectArn
}
}
};
}
/// <summary>
/// Creates a read-write policy
/// </summary>
private object CreateReadWritePolicy(string bucketArn, string objectArn)
{
return new
{
Version = "2012-10-17",
Statement = new[]
{
new
{
Effect = "Allow",
Principal = new { AWS = "*" },
Action = new[] { "s3:GetBucketLocation", "s3:ListBucket", "s3:ListBucketMultipartUploads" },
Resource = bucketArn
},
new
{
Effect = "Allow",
Principal = new { AWS = "*" },
Action = new[] { "s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts" },
Resource = objectArn
}
}
};
}
/// <summary>
/// Creates an empty policy (removes all permissions)
/// </summary>
private object CreateEmptyPolicy()
{
return new
{
Version = "2012-10-17",
Statement = new object[0]
};
}
/// <summary>
/// Validates the policy JSON
/// </summary>
/// <param name="policyJson">Policy JSON to validate</param>
/// <returns>True if valid, false otherwise</returns>
private bool ValidatePolicyJson(string policyJson)
{
try
{
using var document = JsonDocument.Parse(policyJson);
MinIOLogger.WriteVerbose(this, "Policy JSON is valid");
return true;
}
catch (JsonException ex)
{
WriteError(new ErrorRecord(
new ArgumentException($"Invalid policy JSON: {ex.Message}", ex),
"InvalidPolicyJson",
ErrorCategory.InvalidArgument,
policyJson));
return false;
}
}
}
}
+152
View File
@@ -0,0 +1,152 @@
using System.Management.Automation;
using PSMinIO.Utils;
namespace PSMinIO.Cmdlets
{
/// <summary>
/// Tests whether a MinIO bucket exists
/// </summary>
[Cmdlet(VerbsDiagnostic.Test, "MinIOBucketExists", SupportsShouldProcess = false)]
[OutputType(typeof(bool))]
public class TestMinIOBucketExistsCmdlet : MinIOBaseCmdlet
{
/// <summary>
/// Name of the bucket to test
/// </summary>
[Parameter(Position = 0, Mandatory = true, ValueFromPipeline = true, ValueFromPipelineByPropertyName = true)]
[ValidateNotNullOrEmpty]
[Alias("Bucket")]
public string BucketName { get; set; } = string.Empty;
/// <summary>
/// Return detailed information instead of just true/false
/// </summary>
[Parameter]
public SwitchParameter Detailed { get; set; }
/// <summary>
/// Processes the cmdlet
/// </summary>
protected override void ProcessRecord()
{
ValidateConfiguration();
ValidateBucketName(BucketName);
ExecuteOperation("TestBucketExists", () =>
{
MinIOLogger.WriteVerbose(this, "Checking if bucket '{0}' exists", BucketName);
var exists = Client.BucketExists(BucketName);
if (Detailed.IsPresent)
{
// Return detailed information
var result = new PSObject();
result.Properties.Add(new PSNoteProperty("BucketName", BucketName));
result.Properties.Add(new PSNoteProperty("Exists", exists));
result.Properties.Add(new PSNoteProperty("Endpoint", Configuration.Endpoint));
result.Properties.Add(new PSNoteProperty("CheckedAt", System.DateTime.UtcNow));
if (exists)
{
try
{
// Try to get additional bucket information
var allBuckets = Client.ListBuckets();
var bucket = allBuckets.Find(b =>
string.Equals(b.Name, BucketName, System.StringComparison.OrdinalIgnoreCase));
if (bucket != null)
{
result.Properties.Add(new PSNoteProperty("Created", bucket.Created));
result.Properties.Add(new PSNoteProperty("Region", bucket.Region));
}
}
catch (System.Exception ex)
{
MinIOLogger.WriteVerbose(this,
"Could not retrieve additional bucket information: {0}", ex.Message);
}
}
WriteObject(result);
}
else
{
// Return simple boolean result
WriteObject(exists);
}
MinIOLogger.WriteVerbose(this,
"Bucket '{0}' exists: {1}", BucketName, exists);
}, $"Bucket: {BucketName}");
}
}
/// <summary>
/// Custom output type for detailed bucket existence information
/// </summary>
public class BucketExistenceInfo
{
/// <summary>
/// Name of the bucket that was tested
/// </summary>
public string BucketName { get; set; } = string.Empty;
/// <summary>
/// Whether the bucket exists
/// </summary>
public bool Exists { get; set; }
/// <summary>
/// MinIO endpoint that was checked
/// </summary>
public string Endpoint { get; set; } = string.Empty;
/// <summary>
/// When the check was performed
/// </summary>
public System.DateTime CheckedAt { get; set; }
/// <summary>
/// Creation date of the bucket (if it exists and information is available)
/// </summary>
public System.DateTime? Created { get; set; }
/// <summary>
/// Region of the bucket (if it exists and information is available)
/// </summary>
public string? Region { get; set; }
/// <summary>
/// Creates a new BucketExistenceInfo instance
/// </summary>
public BucketExistenceInfo()
{
CheckedAt = System.DateTime.UtcNow;
}
/// <summary>
/// Creates a new BucketExistenceInfo instance with specified values
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <param name="exists">Whether the bucket exists</param>
/// <param name="endpoint">MinIO endpoint</param>
public BucketExistenceInfo(string bucketName, bool exists, string endpoint)
{
BucketName = bucketName ?? string.Empty;
Exists = exists;
Endpoint = endpoint ?? string.Empty;
CheckedAt = System.DateTime.UtcNow;
}
/// <summary>
/// Returns a string representation of the bucket existence info
/// </summary>
public override string ToString()
{
return $"Bucket '{BucketName}' exists: {Exists} (checked at {CheckedAt:yyyy-MM-dd HH:mm:ss} UTC)";
}
}
}
+201
View File
@@ -0,0 +1,201 @@
using System;
using System.Collections.Generic;
namespace PSMinIO.Models
{
/// <summary>
/// Represents the state of a chunked transfer operation for resume functionality
/// </summary>
public class ChunkedTransferState
{
/// <summary>
/// Name of the bucket
/// </summary>
public string BucketName { get; set; } = string.Empty;
/// <summary>
/// Name of the object
/// </summary>
public string ObjectName { get; set; } = string.Empty;
/// <summary>
/// Local file path
/// </summary>
public string FilePath { get; set; } = string.Empty;
/// <summary>
/// Total size of the file/object
/// </summary>
public long TotalSize { get; set; }
/// <summary>
/// Size of each chunk
/// </summary>
public long ChunkSize { get; set; }
/// <summary>
/// List of completed chunks
/// </summary>
public List<ChunkInfo> CompletedChunks { get; set; } = new List<ChunkInfo>();
/// <summary>
/// Last modified time of the source file (for validation)
/// </summary>
public DateTime LastModified { get; set; }
/// <summary>
/// ETag of the object (for validation)
/// </summary>
public string ETag { get; set; } = string.Empty;
/// <summary>
/// Transfer operation type
/// </summary>
public ChunkedTransferType TransferType { get; set; }
/// <summary>
/// When the transfer was started
/// </summary>
public DateTime StartTime { get; set; }
/// <summary>
/// When the transfer state was last updated
/// </summary>
public DateTime LastUpdated { get; set; }
/// <summary>
/// Upload ID for multipart uploads (MinIO specific)
/// </summary>
public string? UploadId { get; set; }
/// <summary>
/// Gets the total number of chunks
/// </summary>
public int TotalChunks => (int)Math.Ceiling((double)TotalSize / ChunkSize);
/// <summary>
/// Gets the number of completed chunks
/// </summary>
public int CompletedChunkCount => CompletedChunks.Count;
/// <summary>
/// Gets the total bytes transferred
/// </summary>
public long BytesTransferred => CompletedChunks.Sum(c => c.Size);
/// <summary>
/// Gets the transfer progress as a percentage
/// </summary>
public double ProgressPercentage => TotalSize > 0 ? (double)BytesTransferred / TotalSize * 100 : 0;
/// <summary>
/// Checks if the transfer is complete
/// </summary>
public bool IsComplete => CompletedChunkCount >= TotalChunks;
/// <summary>
/// Gets the next chunk to transfer
/// </summary>
/// <returns>Next chunk info or null if complete</returns>
public ChunkInfo? GetNextChunk()
{
for (int i = 0; i < TotalChunks; i++)
{
if (!CompletedChunks.Any(c => c.ChunkNumber == i))
{
var startByte = i * ChunkSize;
var endByte = Math.Min(startByte + ChunkSize - 1, TotalSize - 1);
var size = endByte - startByte + 1;
return new ChunkInfo
{
ChunkNumber = i,
StartByte = startByte,
EndByte = endByte,
Size = size,
IsCompleted = false
};
}
}
return null;
}
/// <summary>
/// Marks a chunk as completed
/// </summary>
/// <param name="chunkInfo">Completed chunk information</param>
public void MarkChunkCompleted(ChunkInfo chunkInfo)
{
chunkInfo.IsCompleted = true;
chunkInfo.CompletedTime = DateTime.UtcNow;
// Remove any existing entry for this chunk number
CompletedChunks.RemoveAll(c => c.ChunkNumber == chunkInfo.ChunkNumber);
// Add the completed chunk
CompletedChunks.Add(chunkInfo);
LastUpdated = DateTime.UtcNow;
}
}
/// <summary>
/// Information about a single chunk
/// </summary>
public class ChunkInfo
{
/// <summary>
/// Chunk number (0-based)
/// </summary>
public int ChunkNumber { get; set; }
/// <summary>
/// Starting byte position
/// </summary>
public long StartByte { get; set; }
/// <summary>
/// Ending byte position (inclusive)
/// </summary>
public long EndByte { get; set; }
/// <summary>
/// Size of the chunk in bytes
/// </summary>
public long Size { get; set; }
/// <summary>
/// ETag of the uploaded chunk (for uploads)
/// </summary>
public string ChunkETag { get; set; } = string.Empty;
/// <summary>
/// Whether this chunk has been completed
/// </summary>
public bool IsCompleted { get; set; }
/// <summary>
/// When this chunk was completed
/// </summary>
public DateTime? CompletedTime { get; set; }
/// <summary>
/// Number of retry attempts for this chunk
/// </summary>
public int RetryCount { get; set; }
/// <summary>
/// Last error encountered for this chunk
/// </summary>
public string? LastError { get; set; }
}
/// <summary>
/// Type of chunked transfer operation
/// </summary>
public enum ChunkedTransferType
{
Upload,
Download
}
}
+101
View File
@@ -0,0 +1,101 @@
using System;
namespace PSMinIO.Models
{
/// <summary>
/// Represents information about a MinIO bucket
/// </summary>
public class MinIOBucketInfo
{
/// <summary>
/// Name of the bucket
/// </summary>
public string Name { get; set; } = string.Empty;
/// <summary>
/// Creation date of the bucket
/// </summary>
public DateTime Created { get; set; }
/// <summary>
/// Region where the bucket is located
/// </summary>
public string Region { get; set; } = string.Empty;
/// <summary>
/// Total size of all objects in the bucket (optional, calculated separately)
/// </summary>
public long? Size { get; set; }
/// <summary>
/// Number of objects in the bucket (optional, calculated separately)
/// </summary>
public long? ObjectCount { get; set; }
/// <summary>
/// Creates a new MinIOBucketInfo instance
/// </summary>
public MinIOBucketInfo()
{
}
/// <summary>
/// Creates a new MinIOBucketInfo instance with specified values
/// </summary>
/// <param name="name">Bucket name</param>
/// <param name="created">Creation date</param>
/// <param name="region">Bucket region</param>
public MinIOBucketInfo(string name, DateTime created, string region = "")
{
Name = name ?? string.Empty;
Created = created;
Region = region ?? string.Empty;
}
/// <summary>
/// Creates a MinIOBucketInfo from a Minio.DataModel.Bucket
/// </summary>
/// <param name="bucket">Minio bucket object</param>
/// <returns>MinIOBucketInfo instance</returns>
public static MinIOBucketInfo FromMinioBucket(Minio.DataModel.Bucket bucket)
{
if (bucket == null)
throw new ArgumentNullException(nameof(bucket));
return new MinIOBucketInfo
{
Name = bucket.Name ?? string.Empty,
Created = bucket.CreationDate,
Region = string.Empty // Region is not available in basic bucket info
};
}
/// <summary>
/// Returns a string representation of the bucket info
/// </summary>
public override string ToString()
{
return $"Bucket: {Name} (Created: {Created:yyyy-MM-dd HH:mm:ss})";
}
/// <summary>
/// Determines whether the specified object is equal to the current object
/// </summary>
public override bool Equals(object? obj)
{
if (obj is MinIOBucketInfo other)
{
return string.Equals(Name, other.Name, StringComparison.OrdinalIgnoreCase);
}
return false;
}
/// <summary>
/// Returns a hash code for the current object
/// </summary>
public override int GetHashCode()
{
return Name?.ToLowerInvariant().GetHashCode() ?? 0;
}
}
}
+84
View File
@@ -0,0 +1,84 @@
using System;
namespace PSMinIO.Models
{
/// <summary>
/// Configuration settings for MinIO client connections
/// </summary>
public class MinIOConfiguration
{
/// <summary>
/// MinIO server endpoint (without protocol)
/// </summary>
public string Endpoint { get; set; } = string.Empty;
/// <summary>
/// Access key for authentication
/// </summary>
public string AccessKey { get; set; } = string.Empty;
/// <summary>
/// Secret key for authentication
/// </summary>
public string SecretKey { get; set; } = string.Empty;
/// <summary>
/// Whether to use SSL/TLS for connections
/// </summary>
public bool UseSSL { get; set; } = true;
/// <summary>
/// Optional region for bucket operations
/// </summary>
public string Region { get; set; } = "us-east-1";
/// <summary>
/// Connection timeout in seconds
/// </summary>
public int TimeoutSeconds { get; set; } = 30;
/// <summary>
/// Whether to skip SSL certificate validation
/// </summary>
public bool SkipCertificateValidation { get; set; }
/// <summary>
/// Whether the configuration is valid for creating a client
/// </summary>
public bool IsValid => !string.IsNullOrWhiteSpace(Endpoint) &&
!string.IsNullOrWhiteSpace(AccessKey) &&
!string.IsNullOrWhiteSpace(SecretKey);
/// <summary>
/// Creates a new MinIOConfiguration instance
/// </summary>
public MinIOConfiguration()
{
}
/// <summary>
/// Creates a new MinIOConfiguration instance with specified values
/// </summary>
/// <param name="endpoint">MinIO endpoint</param>
/// <param name="accessKey">Access key</param>
/// <param name="secretKey">Secret key</param>
/// <param name="useSSL">Use SSL flag</param>
/// <param name="region">Optional region</param>
/// <param name="timeoutSeconds">Connection timeout</param>
/// <param name="skipCertificateValidation">Whether to skip SSL certificate validation</param>
public MinIOConfiguration(string endpoint, string accessKey, string secretKey,
bool useSSL = true, string region = "us-east-1", int timeoutSeconds = 30, bool skipCertificateValidation = false)
{
Endpoint = endpoint?.Trim() ?? string.Empty;
AccessKey = accessKey?.Trim() ?? string.Empty;
SecretKey = secretKey?.Trim() ?? string.Empty;
UseSSL = useSSL;
Region = region?.Trim() ?? "us-east-1";
TimeoutSeconds = timeoutSeconds;
SkipCertificateValidation = skipCertificateValidation;
}
}
}
+202
View File
@@ -0,0 +1,202 @@
using System;
using PSMinIO.Utils;
namespace PSMinIO.Models
{
/// <summary>
/// Represents an active MinIO connection with configuration and client wrapper
/// </summary>
public class MinIOConnection : IDisposable
{
private MinIOClientWrapper? _client;
private bool _disposed = false;
/// <summary>
/// Gets the configuration for this connection
/// </summary>
public MinIOConfiguration Configuration { get; }
/// <summary>
/// Gets the MinIO client wrapper instance
/// </summary>
public MinIOClientWrapper Client
{
get
{
if (_disposed)
throw new ObjectDisposedException(nameof(MinIOConnection));
if (_client == null)
{
_client = new MinIOClientWrapper(Configuration);
}
return _client;
}
}
/// <summary>
/// Gets whether this connection is valid and ready to use
/// </summary>
public bool IsValid => Configuration.IsValid && !_disposed;
/// <summary>
/// Gets the connection status
/// </summary>
public string Status
{
get
{
if (_disposed) return "Disposed";
if (!Configuration.IsValid) return "Invalid Configuration";
return "Ready";
}
}
/// <summary>
/// Gets the endpoint URL for display purposes
/// </summary>
public string EndpointUrl
{
get
{
if (string.IsNullOrWhiteSpace(Configuration.Endpoint))
return "Not Set";
var protocol = Configuration.UseSSL ? "https" : "http";
return $"{protocol}://{Configuration.Endpoint}";
}
}
/// <summary>
/// Gets when this connection was created
/// </summary>
public DateTime CreatedAt { get; }
/// <summary>
/// Creates a new MinIOConnection instance
/// </summary>
/// <param name="configuration">MinIO configuration</param>
public MinIOConnection(MinIOConfiguration configuration)
{
Configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
CreatedAt = DateTime.UtcNow;
}
/// <summary>
/// Tests the connection by attempting to list buckets
/// </summary>
/// <returns>Connection test result</returns>
public ConnectionTestResult TestConnection()
{
if (_disposed)
return new ConnectionTestResult(false, "Connection is disposed");
if (!Configuration.IsValid)
return new ConnectionTestResult(false, "Configuration is invalid");
try
{
var startTime = DateTime.UtcNow;
var buckets = Client.ListBuckets();
var duration = DateTime.UtcNow - startTime;
return new ConnectionTestResult(true, "Connection successful")
{
BucketCount = buckets.Count,
ResponseTime = duration,
TestedAt = DateTime.UtcNow
};
}
catch (Exception ex)
{
return new ConnectionTestResult(false, ex.Message)
{
TestedAt = DateTime.UtcNow
};
}
}
/// <summary>
/// Returns a string representation of the connection
/// </summary>
public override string ToString()
{
return $"MinIO Connection: {EndpointUrl} (Status: {Status})";
}
/// <summary>
/// Disposes the connection and underlying client
/// </summary>
public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
/// <summary>
/// Protected dispose method
/// </summary>
/// <param name="disposing">Whether disposing from Dispose method</param>
protected virtual void Dispose(bool disposing)
{
if (!_disposed && disposing)
{
_client?.Dispose();
_client = null;
_disposed = true;
}
}
}
/// <summary>
/// Represents the result of a connection test
/// </summary>
public class ConnectionTestResult
{
/// <summary>
/// Whether the connection test was successful
/// </summary>
public bool Success { get; }
/// <summary>
/// Message describing the test result
/// </summary>
public string Message { get; }
/// <summary>
/// Number of buckets found (if successful)
/// </summary>
public int? BucketCount { get; set; }
/// <summary>
/// Response time for the test
/// </summary>
public TimeSpan? ResponseTime { get; set; }
/// <summary>
/// When the test was performed
/// </summary>
public DateTime? TestedAt { get; set; }
/// <summary>
/// Creates a new ConnectionTestResult
/// </summary>
/// <param name="success">Whether the test was successful</param>
/// <param name="message">Test result message</param>
public ConnectionTestResult(bool success, string message)
{
Success = success;
Message = message ?? string.Empty;
}
/// <summary>
/// Returns a string representation of the test result
/// </summary>
public override string ToString()
{
var result = Success ? "SUCCESS" : "FAILED";
var details = ResponseTime.HasValue ? $" ({ResponseTime.Value.TotalMilliseconds:F0}ms)" : "";
return $"{result}: {Message}{details}";
}
}
}
+249
View File
@@ -0,0 +1,249 @@
using System;
using System.Collections.Generic;
namespace PSMinIO.Models
{
/// <summary>
/// Represents information about a MinIO object
/// </summary>
public class MinIOObjectInfo
{
/// <summary>
/// Name/key of the object
/// </summary>
public string Name { get; set; } = string.Empty;
/// <summary>
/// Size of the object in bytes
/// </summary>
public long Size { get; set; }
/// <summary>
/// Last modified date of the object
/// </summary>
public DateTime LastModified { get; set; }
/// <summary>
/// ETag of the object
/// </summary>
public string ETag { get; set; } = string.Empty;
/// <summary>
/// Content type of the object
/// </summary>
public string ContentType { get; set; } = string.Empty;
/// <summary>
/// Bucket name containing this object
/// </summary>
public string BucketName { get; set; } = string.Empty;
/// <summary>
/// Whether this is a directory/prefix (ends with /)
/// </summary>
public bool IsDirectory => Name.EndsWith("/");
/// <summary>
/// Object metadata/user-defined metadata
/// </summary>
public Dictionary<string, string> Metadata { get; set; } = new Dictionary<string, string>();
/// <summary>
/// Storage class of the object
/// </summary>
public string StorageClass { get; set; } = string.Empty;
/// <summary>
/// Version ID of the object (for versioned buckets)
/// </summary>
public string? VersionId { get; set; }
/// <summary>
/// Whether this object is the latest version
/// </summary>
public bool IsLatestVersion { get; set; } = true;
/// <summary>
/// Whether this object is a delete marker
/// </summary>
public bool IsDeleteMarker { get; set; } = false;
/// <summary>
/// Presigned URL for accessing this object (if generated)
/// </summary>
public string? PresignedUrl { get; set; }
/// <summary>
/// Expiration time for the presigned URL (if generated)
/// </summary>
public DateTime? PresignedUrlExpiration { get; set; }
/// <summary>
/// Creates a new MinIOObjectInfo instance
/// </summary>
public MinIOObjectInfo()
{
}
/// <summary>
/// Creates a new MinIOObjectInfo instance with specified values
/// </summary>
/// <param name="name">Object name</param>
/// <param name="size">Object size</param>
/// <param name="lastModified">Last modified date</param>
/// <param name="etag">ETag</param>
/// <param name="bucketName">Bucket name</param>
public MinIOObjectInfo(string name, long size, DateTime lastModified, string etag, string bucketName)
{
Name = name ?? string.Empty;
Size = size;
LastModified = lastModified;
ETag = etag ?? string.Empty;
BucketName = bucketName ?? string.Empty;
}
/// <summary>
/// Creates a MinIOObjectInfo from a Minio.DataModel.Item
/// </summary>
/// <param name="item">Minio item object</param>
/// <param name="bucketName">Name of the bucket containing the object</param>
/// <returns>MinIOObjectInfo instance</returns>
public static MinIOObjectInfo FromMinioItem(Minio.DataModel.Item item, string bucketName)
{
if (item == null)
throw new ArgumentNullException(nameof(item));
var objectInfo = new MinIOObjectInfo
{
Name = item.Key ?? string.Empty,
Size = (long)(item.Size ?? 0),
LastModified = item.LastModifiedDateTime ?? DateTime.MinValue,
ETag = item.ETag ?? string.Empty,
BucketName = bucketName ?? string.Empty,
StorageClass = item.StorageClass ?? string.Empty
};
// Try to extract version information if available
// Note: The MinIO .NET SDK Item object may have version properties
try
{
// Use reflection to check for version properties that might exist
var itemType = item.GetType();
var versionIdProperty = itemType.GetProperty("VersionId");
if (versionIdProperty != null)
{
objectInfo.VersionId = versionIdProperty.GetValue(item)?.ToString();
}
var isLatestProperty = itemType.GetProperty("IsLatest");
if (isLatestProperty != null && isLatestProperty.GetValue(item) is bool isLatest)
{
objectInfo.IsLatestVersion = isLatest;
}
var isDeleteMarkerProperty = itemType.GetProperty("IsDeleteMarker");
if (isDeleteMarkerProperty != null && isDeleteMarkerProperty.GetValue(item) is bool isDeleteMarker)
{
objectInfo.IsDeleteMarker = isDeleteMarker;
}
}
catch
{
// If reflection fails, just continue without version information
// This ensures compatibility even if the SDK doesn't have these properties
}
// Copy metadata if available
if (item.MetaData != null)
{
foreach (var kvp in item.MetaData)
{
objectInfo.Metadata[kvp.Key] = kvp.Value;
}
}
return objectInfo;
}
/// <summary>
/// Gets the file extension of the object
/// </summary>
public string GetFileExtension()
{
if (IsDirectory || string.IsNullOrEmpty(Name))
return string.Empty;
var lastDotIndex = Name.LastIndexOf('.');
if (lastDotIndex >= 0 && lastDotIndex < Name.Length - 1)
{
return Name.Substring(lastDotIndex);
}
return string.Empty;
}
/// <summary>
/// Gets the directory path of the object (everything before the last /)
/// </summary>
public string GetDirectoryPath()
{
if (string.IsNullOrEmpty(Name))
return string.Empty;
var lastSlashIndex = Name.LastIndexOf('/');
if (lastSlashIndex >= 0)
{
return Name.Substring(0, lastSlashIndex + 1);
}
return string.Empty;
}
/// <summary>
/// Gets the file name without the directory path
/// </summary>
public string GetFileName()
{
if (string.IsNullOrEmpty(Name) || IsDirectory)
return Name;
var lastSlashIndex = Name.LastIndexOf('/');
if (lastSlashIndex >= 0 && lastSlashIndex < Name.Length - 1)
{
return Name.Substring(lastSlashIndex + 1);
}
return Name;
}
/// <summary>
/// Returns a string representation of the object info
/// </summary>
public override string ToString()
{
return $"Object: {Name} ({Size} bytes, Modified: {LastModified:yyyy-MM-dd HH:mm:ss})";
}
/// <summary>
/// Determines whether the specified object is equal to the current object
/// </summary>
public override bool Equals(object? obj)
{
if (obj is MinIOObjectInfo other)
{
return string.Equals(Name, other.Name, StringComparison.Ordinal) &&
string.Equals(BucketName, other.BucketName, StringComparison.OrdinalIgnoreCase);
}
return false;
}
/// <summary>
/// Returns a hash code for the current object
/// </summary>
public override int GetHashCode()
{
return HashCode.Combine(Name, BucketName?.ToLowerInvariant());
}
}
}
+90
View File
@@ -0,0 +1,90 @@
using System;
namespace PSMinIO.Models
{
/// <summary>
/// Represents statistical information about MinIO storage
/// </summary>
public class MinIOStats
{
/// <summary>
/// Total number of buckets
/// </summary>
public int TotalBuckets { get; set; }
/// <summary>
/// Total number of objects across all buckets
/// </summary>
public long TotalObjects { get; set; }
/// <summary>
/// Total size of all objects in bytes
/// </summary>
public long TotalSize { get; set; }
/// <summary>
/// When these statistics were last updated
/// </summary>
public DateTime LastUpdated { get; set; }
/// <summary>
/// MinIO server endpoint
/// </summary>
public string Endpoint { get; set; } = string.Empty;
/// <summary>
/// Whether SSL is being used
/// </summary>
public bool UseSSL { get; set; }
/// <summary>
/// Connection status
/// </summary>
public string ConnectionStatus { get; set; } = "Unknown";
/// <summary>
/// Creates a new MinIOStats instance
/// </summary>
public MinIOStats()
{
LastUpdated = DateTime.UtcNow;
}
/// <summary>
/// Creates a new MinIOStats instance with specified values
/// </summary>
/// <param name="totalBuckets">Total number of buckets</param>
/// <param name="totalObjects">Total number of objects</param>
/// <param name="totalSize">Total size in bytes</param>
/// <param name="endpoint">MinIO endpoint</param>
/// <param name="useSSL">Whether SSL is used</param>
public MinIOStats(int totalBuckets, long totalObjects, long totalSize, string endpoint, bool useSSL)
{
TotalBuckets = totalBuckets;
TotalObjects = totalObjects;
TotalSize = totalSize;
Endpoint = endpoint ?? string.Empty;
UseSSL = useSSL;
LastUpdated = DateTime.UtcNow;
ConnectionStatus = "Connected";
}
/// <summary>
/// Gets the average object size in bytes
/// </summary>
public double AverageObjectSize => TotalObjects > 0 ? (double)TotalSize / TotalObjects : 0;
/// <summary>
/// Gets the average objects per bucket
/// </summary>
public double AverageObjectsPerBucket => TotalBuckets > 0 ? (double)TotalObjects / TotalBuckets : 0;
/// <summary>
/// Returns a string representation of the statistics
/// </summary>
public override string ToString()
{
return $"MinIO Stats: {TotalBuckets} buckets, {TotalObjects} objects, {Utils.SizeFormatter.FormatBytes(TotalSize)} total";
}
}
}
@@ -0,0 +1,267 @@
using System;
using System.Management.Automation;
using PSMinIO.Utils;
namespace PSMinIO.Utils
{
/// <summary>
/// Manages 3-layer progress reporting for chunked file collection operations
/// Layer 1: Collection Progress (overall files)
/// Layer 2: Current File Progress
/// Layer 3: Current Chunk Progress
/// </summary>
public class ChunkedCollectionProgressReporter
{
private readonly PSCmdlet _cmdlet;
private readonly int _totalFiles;
private readonly long _totalSize;
private readonly DateTime _startTime;
private readonly string _operationName;
// Progress tracking
private int _completedFiles = 0;
private long _totalBytesTransferred = 0;
private string _currentFileName = "";
private long _currentFileSize = 0;
private long _currentFileBytesTransferred = 0;
private int _currentChunk = 0;
private int _totalChunks = 0;
private long _currentChunkBytesTransferred = 0;
private long _currentChunkSize = 0;
// Activity IDs for progress hierarchy
private const int CollectionActivityId = 1;
private const int FileActivityId = 2;
private const int ChunkActivityId = 3;
// Progress control
private readonly long _progressUpdateInterval;
private long _lastProgressUpdate = 0;
/// <summary>
/// Creates a new chunked collection progress reporter
/// </summary>
/// <param name="cmdlet">PowerShell cmdlet for progress reporting</param>
/// <param name="totalFiles">Total number of files to process</param>
/// <param name="totalSize">Total size of all files</param>
/// <param name="operationName">Name of the operation (e.g., "Uploading", "Downloading")</param>
/// <param name="progressUpdateInterval">Update progress every N bytes</param>
public ChunkedCollectionProgressReporter(
PSCmdlet cmdlet,
int totalFiles,
long totalSize,
string operationName = "Processing",
long progressUpdateInterval = 1024 * 1024) // 1MB default
{
_cmdlet = cmdlet;
_totalFiles = totalFiles;
_totalSize = totalSize;
_operationName = operationName;
_startTime = DateTime.Now;
_progressUpdateInterval = progressUpdateInterval;
}
/// <summary>
/// Starts processing a new file
/// </summary>
/// <param name="fileName">Name of the file being processed</param>
/// <param name="fileSize">Size of the file</param>
/// <param name="totalChunks">Total number of chunks for this file</param>
public void StartNewFile(string fileName, long fileSize, int totalChunks)
{
_currentFileName = fileName;
_currentFileSize = fileSize;
_currentFileBytesTransferred = 0;
_totalChunks = totalChunks;
_currentChunk = 0;
MinIOLogger.WriteVerbose(_cmdlet, "Starting {0} of file {1}/{2}: {3} ({4})",
_operationName.ToLower(), _completedFiles + 1, _totalFiles, fileName, SizeFormatter.FormatSize(fileSize));
UpdateAllProgress();
}
/// <summary>
/// Starts processing a new chunk
/// </summary>
/// <param name="chunkNumber">Chunk number (1-based for display)</param>
/// <param name="chunkSize">Size of the chunk</param>
public void StartNewChunk(int chunkNumber, long chunkSize)
{
_currentChunk = chunkNumber;
_currentChunkSize = chunkSize;
_currentChunkBytesTransferred = 0;
MinIOLogger.WriteVerbose(_cmdlet, "File {0}: Starting chunk {1}/{2} ({3})",
_currentFileName, chunkNumber, _totalChunks, SizeFormatter.FormatSize(chunkSize));
UpdateAllProgress();
}
/// <summary>
/// Updates chunk progress
/// </summary>
/// <param name="bytesTransferred">Bytes transferred for current chunk</param>
public void UpdateChunkProgress(long bytesTransferred)
{
var chunkDelta = bytesTransferred - _currentChunkBytesTransferred;
_currentChunkBytesTransferred = bytesTransferred;
_currentFileBytesTransferred += chunkDelta;
_totalBytesTransferred += chunkDelta;
// Only update progress if we've transferred enough bytes since last update
if (_totalBytesTransferred - _lastProgressUpdate >= _progressUpdateInterval)
{
UpdateAllProgress();
_lastProgressUpdate = _totalBytesTransferred;
}
}
/// <summary>
/// Completes the current chunk
/// </summary>
/// <param name="chunkETag">ETag of the completed chunk (optional)</param>
public void CompleteChunk(string? chunkETag = null)
{
MinIOLogger.WriteVerbose(_cmdlet, "File {0}: Completed chunk {1}/{2}{3}",
_currentFileName, _currentChunk, _totalChunks,
!string.IsNullOrEmpty(chunkETag) ? $" - ETag: {chunkETag}" : "");
// Mark chunk as completed (only if progress is enabled)
if (_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") &&
_cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue")
return;
var chunkProgress = new ProgressRecord(ChunkActivityId, "Current Chunk", "Completed")
{
PercentComplete = 100,
RecordType = ProgressRecordType.Completed,
ParentActivityId = FileActivityId
};
_cmdlet.WriteProgress(chunkProgress);
}
/// <summary>
/// Completes the current file
/// </summary>
public void CompleteFile()
{
_completedFiles++;
var elapsed = DateTime.Now - _startTime;
MinIOLogger.WriteVerbose(_cmdlet, "File {0}: {1} completed in {2} - Total size: {3}",
_currentFileName, _operationName.ToLower(), elapsed.ToString(@"hh\:mm\:ss"), SizeFormatter.FormatSize(_currentFileSize));
// Mark file as completed (only if progress is enabled)
if (_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") &&
_cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue")
return;
var fileProgress = new ProgressRecord(FileActivityId, "Current File", "Completed")
{
PercentComplete = 100,
RecordType = ProgressRecordType.Completed,
ParentActivityId = CollectionActivityId
};
_cmdlet.WriteProgress(fileProgress);
// Complete chunk progress too
CompleteChunk();
}
/// <summary>
/// Completes the entire collection operation
/// </summary>
public void CompleteCollection()
{
var elapsed = DateTime.Now - _startTime;
MinIOLogger.WriteVerbose(_cmdlet, "Completed {0} {1} files ({2}) in {3}",
_operationName.ToLower(), _totalFiles, SizeFormatter.FormatSize(_totalSize), elapsed.ToString(@"hh\:mm\:ss"));
// Complete all progress records
var collectionProgress = new ProgressRecord(CollectionActivityId, $"{_operationName} Files", "Completed")
{
PercentComplete = 100,
RecordType = ProgressRecordType.Completed
};
_cmdlet.WriteProgress(collectionProgress);
// Complete file progress if enabled
if (!(_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") &&
_cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue"))
{
CompleteFile();
}
}
/// <summary>
/// Reports an error for the current chunk
/// </summary>
/// <param name="error">Error that occurred</param>
/// <param name="retryAttempt">Current retry attempt</param>
/// <param name="maxRetries">Maximum retry attempts</param>
public void ReportChunkError(Exception error, int retryAttempt, int maxRetries)
{
MinIOLogger.WriteVerbose(_cmdlet, "File {0}: Chunk {1}/{2} failed (attempt {3}/{4}): {5}",
_currentFileName, _currentChunk, _totalChunks, retryAttempt, maxRetries, error.Message);
}
/// <summary>
/// Updates all progress layers
/// </summary>
private void UpdateAllProgress()
{
var elapsed = DateTime.Now - _startTime;
var speed = elapsed.TotalSeconds > 0 ? _totalBytesTransferred / elapsed.TotalSeconds : 0;
// Layer 1: Collection Progress (always shown)
var collectionPercent = _totalSize > 0 ? (int)((_totalBytesTransferred * 100) / _totalSize) : 0;
var collectionStatus = $"Files: {_completedFiles}/{_totalFiles} | " +
$"Size: {SizeFormatter.FormatSize(_totalBytesTransferred)}/{SizeFormatter.FormatSize(_totalSize)} | " +
$"Speed: {SizeFormatter.FormatSize((long)speed)}/s | " +
$"Elapsed: {elapsed:hh\\:mm\\:ss}";
var collectionProgress = new ProgressRecord(CollectionActivityId, $"{_operationName} Files", collectionStatus)
{
PercentComplete = collectionPercent
};
_cmdlet.WriteProgress(collectionProgress);
// Check if progress is disabled
if (_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") &&
_cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue")
return;
// Layer 2: Current File Progress
if (!string.IsNullOrEmpty(_currentFileName))
{
var filePercent = _currentFileSize > 0 ? (int)((_currentFileBytesTransferred * 100) / _currentFileSize) : 0;
var fileStatus = $"File: {_currentFileName} | " +
$"Size: {SizeFormatter.FormatSize(_currentFileBytesTransferred)}/{SizeFormatter.FormatSize(_currentFileSize)}";
var fileProgress = new ProgressRecord(FileActivityId, "Current File", fileStatus)
{
PercentComplete = filePercent,
ParentActivityId = CollectionActivityId
};
_cmdlet.WriteProgress(fileProgress);
}
// Layer 3: Current Chunk Progress
if (_currentChunk > 0)
{
var chunkPercent = _currentChunkSize > 0 ? (int)((_currentChunkBytesTransferred * 100) / _currentChunkSize) : 0;
var chunkStatus = $"Chunk: {_currentChunk}/{_totalChunks} | " +
$"Size: {SizeFormatter.FormatSize(_currentChunkBytesTransferred)}/{SizeFormatter.FormatSize(_currentChunkSize)} | " +
$"Speed: {SizeFormatter.FormatSize((long)speed)}/s";
var chunkProgress = new ProgressRecord(ChunkActivityId, "Current Chunk", chunkStatus)
{
PercentComplete = chunkPercent,
ParentActivityId = FileActivityId
};
_cmdlet.WriteProgress(chunkProgress);
}
}
}
}
@@ -0,0 +1,195 @@
using System;
using System.Management.Automation;
using PSMinIO.Utils;
namespace PSMinIO.Utils
{
/// <summary>
/// Manages 2-layer progress reporting for chunked single file operations
/// Layer 1: File Progress
/// Layer 2: Current Chunk Progress
/// </summary>
public class ChunkedSingleFileProgressReporter
{
private readonly PSCmdlet _cmdlet;
private readonly long _totalSize;
private readonly int _totalChunks;
private readonly DateTime _startTime;
private readonly string _operationName;
// Progress tracking
private long _totalBytesTransferred = 0;
private int _currentChunk = 0;
private long _currentChunkBytesTransferred = 0;
private long _currentChunkSize = 0;
// Activity IDs for progress hierarchy
private const int FileActivityId = 1;
private const int ChunkActivityId = 2;
// Progress control
private readonly long _progressUpdateInterval;
private long _lastProgressUpdate = 0;
/// <summary>
/// Creates a new chunked single file progress reporter
/// </summary>
/// <param name="cmdlet">PowerShell cmdlet for progress reporting</param>
/// <param name="totalSize">Total size of the file</param>
/// <param name="totalChunks">Total number of chunks</param>
/// <param name="operationName">Name of the operation (e.g., "Downloading", "Uploading")</param>
/// <param name="progressUpdateInterval">Update progress every N bytes</param>
public ChunkedSingleFileProgressReporter(
PSCmdlet cmdlet,
long totalSize,
int totalChunks,
string operationName = "Processing",
long progressUpdateInterval = 1024 * 1024) // 1MB default
{
_cmdlet = cmdlet;
_totalSize = totalSize;
_totalChunks = totalChunks;
_operationName = operationName;
_startTime = DateTime.Now;
_progressUpdateInterval = progressUpdateInterval;
}
/// <summary>
/// Starts processing a new chunk
/// </summary>
/// <param name="chunkNumber">Chunk number (1-based for display)</param>
/// <param name="chunkSize">Size of the chunk</param>
public void StartNewChunk(int chunkNumber, long chunkSize)
{
_currentChunk = chunkNumber;
_currentChunkSize = chunkSize;
_currentChunkBytesTransferred = 0;
MinIOLogger.WriteVerbose(_cmdlet, "Starting chunk {0}/{1} ({2})",
chunkNumber, _totalChunks, SizeFormatter.FormatSize(chunkSize));
UpdateAllProgress();
}
/// <summary>
/// Updates chunk progress
/// </summary>
/// <param name="bytesTransferred">Bytes transferred for current chunk</param>
public void UpdateChunkProgress(long bytesTransferred)
{
var chunkDelta = bytesTransferred - _currentChunkBytesTransferred;
_currentChunkBytesTransferred = bytesTransferred;
_totalBytesTransferred += chunkDelta;
// Only update progress if we've transferred enough bytes since last update
if (_totalBytesTransferred - _lastProgressUpdate >= _progressUpdateInterval)
{
UpdateAllProgress();
_lastProgressUpdate = _totalBytesTransferred;
}
}
/// <summary>
/// Completes the current chunk
/// </summary>
/// <param name="chunkETag">ETag of the completed chunk (optional)</param>
public void CompleteChunk(string? chunkETag = null)
{
MinIOLogger.WriteVerbose(_cmdlet, "Completed chunk {0}/{1}{2}",
_currentChunk, _totalChunks,
!string.IsNullOrEmpty(chunkETag) ? $" - ETag: {chunkETag}" : "");
// Mark chunk as completed (only if progress is enabled)
if (_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") &&
_cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue")
return;
var chunkProgress = new ProgressRecord(ChunkActivityId, "Current Chunk", "Completed")
{
PercentComplete = 100,
RecordType = ProgressRecordType.Completed,
ParentActivityId = FileActivityId
};
_cmdlet.WriteProgress(chunkProgress);
}
/// <summary>
/// Completes the entire download operation
/// </summary>
public void CompleteDownload()
{
var elapsed = DateTime.Now - _startTime;
MinIOLogger.WriteVerbose(_cmdlet, "Completed {0} ({1}) in {2}",
_operationName.ToLower(), SizeFormatter.FormatSize(_totalSize), elapsed.ToString(@"hh\:mm\:ss"));
// Complete all progress records
var fileProgress = new ProgressRecord(FileActivityId, $"{_operationName} File", "Completed")
{
PercentComplete = 100,
RecordType = ProgressRecordType.Completed
};
_cmdlet.WriteProgress(fileProgress);
// Complete chunk progress if enabled
if (!(_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") &&
_cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue"))
{
CompleteChunk();
}
}
/// <summary>
/// Reports an error for the current chunk
/// </summary>
/// <param name="error">Error that occurred</param>
/// <param name="retryAttempt">Current retry attempt</param>
/// <param name="maxRetries">Maximum retry attempts</param>
public void ReportChunkError(Exception error, int retryAttempt, int maxRetries)
{
MinIOLogger.WriteVerbose(_cmdlet, "Chunk {0}/{1} failed (attempt {2}/{3}): {4}",
_currentChunk, _totalChunks, retryAttempt, maxRetries, error.Message);
}
/// <summary>
/// Updates all progress layers
/// </summary>
private void UpdateAllProgress()
{
var elapsed = DateTime.Now - _startTime;
var speed = elapsed.TotalSeconds > 0 ? _totalBytesTransferred / elapsed.TotalSeconds : 0;
// Layer 1: File Progress (always shown)
var filePercent = _totalSize > 0 ? (int)((_totalBytesTransferred * 100) / _totalSize) : 0;
var fileStatus = $"Size: {SizeFormatter.FormatSize(_totalBytesTransferred)}/{SizeFormatter.FormatSize(_totalSize)} | " +
$"Speed: {SizeFormatter.FormatSize((long)speed)}/s | " +
$"Elapsed: {elapsed:hh\\:mm\\:ss}";
var fileProgress = new ProgressRecord(FileActivityId, $"{_operationName} File", fileStatus)
{
PercentComplete = filePercent
};
_cmdlet.WriteProgress(fileProgress);
// Check if progress is disabled
if (_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") &&
_cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue")
return;
// Layer 2: Current Chunk Progress
if (_currentChunk > 0)
{
var chunkPercent = _currentChunkSize > 0 ? (int)((_currentChunkBytesTransferred * 100) / _currentChunkSize) : 0;
var chunkStatus = $"Chunk: {_currentChunk}/{_totalChunks} | " +
$"Size: {SizeFormatter.FormatSize(_currentChunkBytesTransferred)}/{SizeFormatter.FormatSize(_currentChunkSize)} | " +
$"Speed: {SizeFormatter.FormatSize((long)speed)}/s";
var chunkProgress = new ProgressRecord(ChunkActivityId, "Current Chunk", chunkStatus)
{
PercentComplete = chunkPercent,
ParentActivityId = FileActivityId
};
_cmdlet.WriteProgress(chunkProgress);
}
}
}
}
+250
View File
@@ -0,0 +1,250 @@
using System;
using System.IO;
using System.Text.Json;
using PSMinIO.Models;
namespace PSMinIO.Utils
{
/// <summary>
/// Manages resume data for chunked transfer operations
/// </summary>
public static class ChunkedTransferResumeManager
{
private static readonly string DefaultResumeDirectory = Path.Combine(
Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
"PSMinIO", "Resume");
/// <summary>
/// Saves transfer state for resume functionality
/// </summary>
/// <param name="transferState">Transfer state to save</param>
/// <param name="customPath">Custom path for resume data (optional)</param>
/// <returns>Path where resume data was saved</returns>
public static string SaveTransferState(ChunkedTransferState transferState, string? customPath = null)
{
var resumeDirectory = customPath ?? DefaultResumeDirectory;
// Ensure directory exists
if (!Directory.Exists(resumeDirectory))
{
Directory.CreateDirectory(resumeDirectory);
}
// Generate unique filename based on transfer details
var fileName = GenerateResumeFileName(transferState);
var filePath = Path.Combine(resumeDirectory, fileName);
// Serialize and save
var json = JsonSerializer.Serialize(transferState, new JsonSerializerOptions
{
WriteIndented = true,
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
});
File.WriteAllText(filePath, json);
return filePath;
}
/// <summary>
/// Loads transfer state for resume functionality
/// </summary>
/// <param name="bucketName">Bucket name</param>
/// <param name="objectName">Object name</param>
/// <param name="filePath">Local file path</param>
/// <param name="transferType">Transfer type</param>
/// <param name="customPath">Custom path for resume data (optional)</param>
/// <returns>Transfer state if found, null otherwise</returns>
public static ChunkedTransferState? LoadTransferState(
string bucketName,
string objectName,
string filePath,
ChunkedTransferType transferType,
string? customPath = null)
{
var resumeDirectory = customPath ?? DefaultResumeDirectory;
if (!Directory.Exists(resumeDirectory))
return null;
// Generate expected filename
var tempState = new ChunkedTransferState
{
BucketName = bucketName,
ObjectName = objectName,
FilePath = filePath,
TransferType = transferType
};
var fileName = GenerateResumeFileName(tempState);
var resumeFilePath = Path.Combine(resumeDirectory, fileName);
if (!File.Exists(resumeFilePath))
return null;
try
{
var json = File.ReadAllText(resumeFilePath);
var transferState = JsonSerializer.Deserialize<ChunkedTransferState>(json, new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
});
return transferState;
}
catch (Exception)
{
// If we can't deserialize, treat as no resume data
return null;
}
}
/// <summary>
/// Deletes resume data after successful completion
/// </summary>
/// <param name="transferState">Transfer state to clean up</param>
/// <param name="customPath">Custom path for resume data (optional)</param>
public static void CleanupResumeData(ChunkedTransferState transferState, string? customPath = null)
{
var resumeDirectory = customPath ?? DefaultResumeDirectory;
var fileName = GenerateResumeFileName(transferState);
var filePath = Path.Combine(resumeDirectory, fileName);
try
{
if (File.Exists(filePath))
{
File.Delete(filePath);
}
}
catch (Exception)
{
// Ignore cleanup errors
}
}
/// <summary>
/// Validates if resume data is still valid
/// </summary>
/// <param name="transferState">Transfer state to validate</param>
/// <param name="currentFileInfo">Current file information</param>
/// <returns>True if resume data is valid</returns>
public static bool IsResumeDataValid(ChunkedTransferState transferState, FileInfo? currentFileInfo = null)
{
// Check if transfer state is reasonable
if (transferState == null)
return false;
// For uploads, validate source file hasn't changed
if (transferState.TransferType == ChunkedTransferType.Upload && currentFileInfo != null)
{
if (!currentFileInfo.Exists)
return false;
// Check if file size or last modified time changed
if (currentFileInfo.Length != transferState.TotalSize ||
currentFileInfo.LastWriteTimeUtc != transferState.LastModified)
{
return false;
}
}
// Check if resume data is not too old (e.g., older than 7 days)
if (DateTime.UtcNow - transferState.LastUpdated > TimeSpan.FromDays(7))
return false;
return true;
}
/// <summary>
/// Gets all resume files in the directory
/// </summary>
/// <param name="customPath">Custom path for resume data (optional)</param>
/// <returns>Array of resume file paths</returns>
public static string[] GetResumeFiles(string? customPath = null)
{
var resumeDirectory = customPath ?? DefaultResumeDirectory;
if (!Directory.Exists(resumeDirectory))
return Array.Empty<string>();
return Directory.GetFiles(resumeDirectory, "*.psminioResume");
}
/// <summary>
/// Cleans up old resume files
/// </summary>
/// <param name="olderThanDays">Delete files older than this many days</param>
/// <param name="customPath">Custom path for resume data (optional)</param>
/// <returns>Number of files cleaned up</returns>
public static int CleanupOldResumeFiles(int olderThanDays = 7, string? customPath = null)
{
var resumeFiles = GetResumeFiles(customPath);
var cutoffDate = DateTime.UtcNow.AddDays(-olderThanDays);
var cleanedCount = 0;
foreach (var file in resumeFiles)
{
try
{
var fileInfo = new FileInfo(file);
if (fileInfo.LastWriteTimeUtc < cutoffDate)
{
File.Delete(file);
cleanedCount++;
}
}
catch (Exception)
{
// Ignore cleanup errors
}
}
return cleanedCount;
}
/// <summary>
/// Generates a unique filename for resume data
/// </summary>
/// <param name="transferState">Transfer state</param>
/// <returns>Unique filename</returns>
private static string GenerateResumeFileName(ChunkedTransferState transferState)
{
// Create a hash of the key components to ensure uniqueness
var key = $"{transferState.BucketName}|{transferState.ObjectName}|{transferState.FilePath}|{transferState.TransferType}";
var hash = key.GetHashCode().ToString("X8");
// Include readable components for easier identification
var safeBucketName = MakeSafeFileName(transferState.BucketName);
var safeObjectName = MakeSafeFileName(Path.GetFileName(transferState.ObjectName));
var transferType = transferState.TransferType.ToString().ToLower();
return $"{safeBucketName}_{safeObjectName}_{transferType}_{hash}.psminioResume";
}
/// <summary>
/// Makes a string safe for use as a filename
/// </summary>
/// <param name="input">Input string</param>
/// <returns>Safe filename string</returns>
private static string MakeSafeFileName(string input)
{
if (string.IsNullOrEmpty(input))
return "unknown";
var invalidChars = Path.GetInvalidFileNameChars();
var safe = input;
foreach (var c in invalidChars)
{
safe = safe.Replace(c, '_');
}
// Limit length and remove leading/trailing dots and spaces
safe = safe.Trim(' ', '.');
if (safe.Length > 50)
safe = safe.Substring(0, 50);
return string.IsNullOrEmpty(safe) ? "unknown" : safe;
}
}
}
+285
View File
@@ -0,0 +1,285 @@
using System;
using System.Management.Automation;
using PSMinIO.Models;
namespace PSMinIO.Utils
{
/// <summary>
/// Base class for all MinIO PowerShell cmdlets
/// Provides common functionality including connection management, logging, and client access
/// </summary>
public abstract class MinIOBaseCmdlet : PSCmdlet
{
private MinIOConnection? _connection;
/// <summary>
/// MinIO connection to use for operations. Can be provided via parameter or retrieved from session.
/// </summary>
[Parameter(ValueFromPipeline = true)]
[Alias("Connection")]
public MinIOConnection? MinIOConnection { get; set; }
/// <summary>
/// Name of session variable containing the MinIO connection (default: MinIOConnection)
/// </summary>
[Parameter]
[ValidateNotNullOrEmpty]
public string SessionVariable { get; set; } = "MinIOConnection";
/// <summary>
/// Gets the MinIO connection instance
/// </summary>
protected MinIOConnection Connection
{
get
{
if (_connection == null)
{
_connection = GetConnection();
if (_connection == null)
{
ThrowTerminatingError(new ErrorRecord(
new InvalidOperationException("No MinIO connection available. Use Connect-MinIO to establish a connection, or provide a connection via the -MinIOConnection parameter."),
"NoConnection",
ErrorCategory.ConnectionError,
null));
}
if (!_connection.IsValid)
{
ThrowTerminatingError(new ErrorRecord(
new InvalidOperationException($"MinIO connection is not valid. Status: {_connection.Status}"),
"InvalidConnection",
ErrorCategory.ConnectionError,
_connection));
}
MinIOLogger.LogOperationStart(this, "UsingConnection", $"Endpoint: {_connection.Configuration.Endpoint}");
}
return _connection;
}
}
/// <summary>
/// Gets the MinIO client wrapper instance
/// </summary>
protected MinIOClientWrapper Client => Connection.Client;
/// <summary>
/// Gets the current MinIO configuration
/// </summary>
protected MinIOConfiguration Configuration => Connection.Configuration;
/// <summary>
/// Gets the MinIO connection from parameter or session variable
/// </summary>
/// <returns>MinIO connection or null if not found</returns>
private MinIOConnection? GetConnection()
{
// First, check if connection was provided via parameter
if (MinIOConnection != null)
{
MinIOLogger.WriteVerbose(this, "Using MinIO connection from parameter");
return MinIOConnection;
}
// Next, check session variable
try
{
var sessionConnection = SessionState.PSVariable.GetValue(SessionVariable) as MinIOConnection;
if (sessionConnection != null)
{
MinIOLogger.WriteVerbose(this, "Using MinIO connection from session variable: {0}", SessionVariable);
return sessionConnection;
}
}
catch (Exception ex)
{
MinIOLogger.WriteVerbose(this, "Failed to retrieve connection from session variable '{0}': {1}", SessionVariable, ex.Message);
}
MinIOLogger.WriteVerbose(this, "No MinIO connection found in parameter or session variable '{0}'", SessionVariable);
return null;
}
/// <summary>
/// Validates that the MinIO connection is available and valid
/// </summary>
protected void ValidateConnection()
{
var connection = Connection; // This will throw if invalid
}
/// <summary>
/// Executes an operation with proper error handling and logging
/// </summary>
/// <param name="operationName">Name of the operation for logging</param>
/// <param name="operation">The operation to execute</param>
/// <param name="details">Optional operation details for logging</param>
protected void ExecuteOperation(string operationName, Action operation, string? details = null)
{
if (operation == null)
throw new ArgumentNullException(nameof(operationName));
var startTime = DateTime.UtcNow;
MinIOLogger.LogOperationStart(this, operationName, details);
try
{
operation();
var duration = DateTime.UtcNow - startTime;
MinIOLogger.LogOperationComplete(this, operationName, duration, details);
}
catch (Exception ex)
{
MinIOLogger.LogOperationFailure(this, operationName, ex, details);
// Determine appropriate error category
var category = GetErrorCategory(ex);
WriteError(new ErrorRecord(ex, $"{operationName}Failed", category, null));
}
}
/// <summary>
/// Executes an operation with return value and proper error handling and logging
/// </summary>
/// <typeparam name="T">Return type</typeparam>
/// <param name="operationName">Name of the operation for logging</param>
/// <param name="operation">The operation to execute</param>
/// <param name="details">Optional operation details for logging</param>
/// <returns>Result of the operation</returns>
protected T ExecuteOperation<T>(string operationName, Func<T> operation, string? details = null)
{
if (operation == null)
throw new ArgumentNullException(nameof(operation));
var startTime = DateTime.UtcNow;
MinIOLogger.LogOperationStart(this, operationName, details);
try
{
var result = operation();
var duration = DateTime.UtcNow - startTime;
MinIOLogger.LogOperationComplete(this, operationName, duration, details);
return result;
}
catch (Exception ex)
{
MinIOLogger.LogOperationFailure(this, operationName, ex, details);
// Determine appropriate error category
var category = GetErrorCategory(ex);
ThrowTerminatingError(new ErrorRecord(ex, $"{operationName}Failed", category, null));
// This line will never be reached, but is required for compilation
throw;
}
}
/// <summary>
/// Determines the appropriate PowerShell error category based on the exception type
/// </summary>
/// <param name="exception">The exception to categorize</param>
/// <returns>Appropriate ErrorCategory</returns>
protected virtual ErrorCategory GetErrorCategory(Exception exception)
{
return exception switch
{
ArgumentException => ErrorCategory.InvalidArgument,
ArgumentNullException => ErrorCategory.InvalidArgument,
UnauthorizedAccessException => ErrorCategory.PermissionDenied,
System.Net.WebException => ErrorCategory.ConnectionError,
System.Net.Http.HttpRequestException => ErrorCategory.ConnectionError,
TimeoutException => ErrorCategory.OperationTimeout,
InvalidOperationException => ErrorCategory.InvalidOperation,
NotSupportedException => ErrorCategory.NotImplemented,
FileNotFoundException => ErrorCategory.ObjectNotFound,
DirectoryNotFoundException => ErrorCategory.ObjectNotFound,
_ => ErrorCategory.NotSpecified
};
}
/// <summary>
/// Validates a bucket name according to MinIO/S3 naming rules
/// </summary>
/// <param name="bucketName">Bucket name to validate</param>
/// <param name="parameterName">Parameter name for error reporting</param>
protected void ValidateBucketName(string bucketName, string parameterName = "BucketName")
{
if (string.IsNullOrWhiteSpace(bucketName))
{
ThrowTerminatingError(new ErrorRecord(
new ArgumentException($"{parameterName} cannot be null or empty"),
"InvalidBucketName",
ErrorCategory.InvalidArgument,
bucketName));
}
// Basic bucket name validation (simplified)
if (bucketName.Length < 3 || bucketName.Length > 63)
{
ThrowTerminatingError(new ErrorRecord(
new ArgumentException($"{parameterName} must be between 3 and 63 characters long"),
"InvalidBucketName",
ErrorCategory.InvalidArgument,
bucketName));
}
}
/// <summary>
/// Validates an object name
/// </summary>
/// <param name="objectName">Object name to validate</param>
/// <param name="parameterName">Parameter name for error reporting</param>
protected void ValidateObjectName(string objectName, string parameterName = "ObjectName")
{
if (string.IsNullOrWhiteSpace(objectName))
{
ThrowTerminatingError(new ErrorRecord(
new ArgumentException($"{parameterName} cannot be null or empty"),
"InvalidObjectName",
ErrorCategory.InvalidArgument,
objectName));
}
}
/// <summary>
/// Cleans up resources when the cmdlet is disposed
/// </summary>
protected override void EndProcessing()
{
try
{
// Note: We don't dispose the connection here as it may be shared across cmdlets
// The connection should be disposed by the user when no longer needed
_connection = null;
}
catch (Exception ex)
{
MinIOLogger.WriteVerbose(this, $"Error during cleanup: {ex.Message}");
}
base.EndProcessing();
}
/// <summary>
/// Handles stopping the cmdlet (Ctrl+C)
/// </summary>
protected override void StopProcessing()
{
try
{
_connection?.Client.CancelOperations();
MinIOLogger.WriteVerbose(this, "Operation cancelled by user");
}
catch (Exception ex)
{
MinIOLogger.WriteVerbose(this, $"Error cancelling operations: {ex.Message}");
}
base.StopProcessing();
}
}
}
+930
View File
@@ -0,0 +1,930 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using Minio;
using Minio.DataModel;
using Minio.DataModel.Args;
using PSMinIO.Models;
using PSMinIO.Utils;
namespace PSMinIO.Utils
{
/// <summary>
/// Synchronous wrapper for MinIO client operations
/// Converts async MinIO operations to synchronous calls for PowerShell compatibility
/// </summary>
public class MinIOClientWrapper : IDisposable
{
private readonly IMinioClient _client;
private readonly CancellationTokenSource _cancellationTokenSource;
private bool _disposed = false;
/// <summary>
/// Creates a new MinIOClientWrapper instance
/// </summary>
/// <param name="configuration">MinIO configuration</param>
public MinIOClientWrapper(MinIOConfiguration configuration)
{
if (configuration == null)
throw new ArgumentNullException(nameof(configuration));
if (!configuration.IsValid)
throw new ArgumentException("Invalid MinIO configuration", nameof(configuration));
_cancellationTokenSource = new CancellationTokenSource();
// Create MinIO client with configuration
var clientBuilder = new MinioClient()
.WithEndpoint(configuration.Endpoint)
.WithCredentials(configuration.AccessKey, configuration.SecretKey);
if (configuration.UseSSL)
{
clientBuilder = clientBuilder.WithSSL();
// Configure custom HttpClient for certificate validation if needed
if (configuration.SkipCertificateValidation)
{
var httpClientHandler = new HttpClientHandler()
{
ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => true
};
var httpClient = new HttpClient(httpClientHandler);
clientBuilder = clientBuilder.WithHttpClient(httpClient);
}
}
if (configuration.TimeoutSeconds > 0)
{
clientBuilder = clientBuilder.WithTimeout(configuration.TimeoutSeconds * 1000);
}
_client = clientBuilder.Build();
}
/// <summary>
/// Gets the cancellation token for operations
/// </summary>
public CancellationToken CancellationToken => _cancellationTokenSource.Token;
/// <summary>
/// Lists all buckets synchronously
/// </summary>
/// <returns>List of bucket information</returns>
public List<MinIOBucketInfo> ListBuckets()
{
try
{
var bucketsResult = Task.Run(async () =>
await _client.ListBucketsAsync(CancellationToken)).GetAwaiter().GetResult();
return bucketsResult.Buckets
.Select(MinIOBucketInfo.FromMinioBucket)
.ToList();
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to list buckets: {ex.Message}", ex);
}
}
/// <summary>
/// Checks if a bucket exists synchronously
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <returns>True if bucket exists, false otherwise</returns>
public bool BucketExists(string bucketName)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
try
{
var args = new BucketExistsArgs().WithBucket(bucketName);
return Task.Run(async () =>
await _client.BucketExistsAsync(args, CancellationToken)).GetAwaiter().GetResult();
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to check if bucket '{bucketName}' exists: {ex.Message}", ex);
}
}
/// <summary>
/// Creates a bucket synchronously
/// </summary>
/// <param name="bucketName">Name of the bucket to create</param>
/// <param name="region">Optional region for the bucket</param>
public void CreateBucket(string bucketName, string? region = null)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
try
{
var args = new MakeBucketArgs().WithBucket(bucketName);
if (!string.IsNullOrWhiteSpace(region))
{
args = args.WithLocation(region);
}
Task.Run(async () =>
await _client.MakeBucketAsync(args, CancellationToken)).GetAwaiter().GetResult();
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to create bucket '{bucketName}': {ex.Message}", ex);
}
}
/// <summary>
/// Deletes a bucket synchronously
/// </summary>
/// <param name="bucketName">Name of the bucket to delete</param>
public void DeleteBucket(string bucketName)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
try
{
var args = new RemoveBucketArgs().WithBucket(bucketName);
Task.Run(async () =>
await _client.RemoveBucketAsync(args, CancellationToken)).GetAwaiter().GetResult();
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to delete bucket '{bucketName}': {ex.Message}", ex);
}
}
/// <summary>
/// Lists objects in a bucket synchronously
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <param name="prefix">Optional prefix to filter objects</param>
/// <param name="recursive">Whether to list objects recursively</param>
/// <param name="includeVersions">Whether to include all versions of objects</param>
/// <returns>List of object information</returns>
public List<MinIOObjectInfo> ListObjects(string bucketName, string? prefix = null, bool recursive = true, bool includeVersions = false)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
try
{
var args = new ListObjectsArgs()
.WithBucket(bucketName)
.WithRecursive(recursive);
if (!string.IsNullOrWhiteSpace(prefix))
{
args = args.WithPrefix(prefix);
}
// Add version support if requested
if (includeVersions)
{
args = args.WithVersions(true);
}
var objects = new List<MinIOObjectInfo>();
var observable = _client.ListObjectsAsync(args, CancellationToken);
// Convert async enumerable to synchronous list
var task = Task.Run(async () =>
{
await foreach (var item in observable.WithCancellation(CancellationToken))
{
objects.Add(MinIOObjectInfo.FromMinioItem(item, bucketName));
}
});
task.GetAwaiter().GetResult();
return objects;
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to list objects in bucket '{bucketName}': {ex.Message}", ex);
}
}
/// <summary>
/// Gets bucket policy synchronously
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <returns>Bucket policy as JSON string</returns>
public string GetBucketPolicy(string bucketName)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
try
{
var args = new GetPolicyArgs().WithBucket(bucketName);
return Task.Run(async () =>
await _client.GetPolicyAsync(args, CancellationToken)).GetAwaiter().GetResult();
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to get policy for bucket '{bucketName}': {ex.Message}", ex);
}
}
/// <summary>
/// Sets bucket policy synchronously
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <param name="policy">Policy JSON string</param>
public void SetBucketPolicy(string bucketName, string policy)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
if (string.IsNullOrWhiteSpace(policy))
throw new ArgumentException("Policy cannot be null or empty", nameof(policy));
try
{
var args = new SetPolicyArgs()
.WithBucket(bucketName)
.WithPolicy(policy);
Task.Run(async () =>
await _client.SetPolicyAsync(args, CancellationToken)).GetAwaiter().GetResult();
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to set policy for bucket '{bucketName}': {ex.Message}", ex);
}
}
/// <summary>
/// Deletes an object synchronously
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <param name="objectName">Name of the object to delete</param>
public void DeleteObject(string bucketName, string objectName)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
if (string.IsNullOrWhiteSpace(objectName))
throw new ArgumentException("Object name cannot be null or empty", nameof(objectName));
try
{
var args = new RemoveObjectArgs()
.WithBucket(bucketName)
.WithObject(objectName);
Task.Run(async () =>
await _client.RemoveObjectAsync(args, CancellationToken)).GetAwaiter().GetResult();
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to delete object '{objectName}' from bucket '{bucketName}': {ex.Message}", ex);
}
}
/// <summary>
/// Deletes multiple objects synchronously
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <param name="objectNames">List of object names to delete</param>
public void DeleteObjects(string bucketName, IEnumerable<string> objectNames)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
if (objectNames == null)
throw new ArgumentNullException(nameof(objectNames));
var objectList = objectNames.ToList();
if (objectList.Count == 0)
return;
try
{
var deleteObjectsArgs = new RemoveObjectsArgs()
.WithBucket(bucketName)
.WithObjects(objectList);
var observable = _client.RemoveObjectsAsync(deleteObjectsArgs, CancellationToken);
// Convert async enumerable to synchronous operation
var task = Task.Run(async () =>
{
await foreach (var deleteError in observable.WithCancellation(CancellationToken))
{
if (deleteError.Exception != null)
{
throw new InvalidOperationException($"Failed to delete object '{deleteError.Key}': {deleteError.Exception.Message}", deleteError.Exception);
}
}
});
task.GetAwaiter().GetResult();
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to delete objects from bucket '{bucketName}': {ex.Message}", ex);
}
}
/// <summary>
/// Uploads a file to MinIO synchronously with progress reporting
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <param name="objectName">Name of the object</param>
/// <param name="filePath">Path to the file to upload</param>
/// <param name="contentType">Content type of the file (optional)</param>
/// <param name="progressCallback">Progress callback for reporting upload progress</param>
/// <returns>ETag of the uploaded object</returns>
public string UploadFile(string bucketName, string objectName, string filePath,
string? contentType = null, Action<long>? progressCallback = null)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
if (string.IsNullOrWhiteSpace(objectName))
throw new ArgumentException("Object name cannot be null or empty", nameof(objectName));
if (string.IsNullOrWhiteSpace(filePath))
throw new ArgumentException("File path cannot be null or empty", nameof(filePath));
if (!File.Exists(filePath))
throw new FileNotFoundException($"File not found: {filePath}");
try
{
var fileInfo = new FileInfo(filePath);
var fileSize = fileInfo.Length;
// Determine content type if not provided
if (string.IsNullOrWhiteSpace(contentType))
{
contentType = GetContentType(filePath);
}
var args = new PutObjectArgs()
.WithBucket(bucketName)
.WithObject(objectName)
.WithFileName(filePath)
.WithContentType(contentType);
// Add progress callback if provided
if (progressCallback != null)
{
args = args.WithProgress(new Progress<ProgressReport>(report =>
{
progressCallback(report.TotalBytesTransferred);
}));
}
var result = Task.Run(async () =>
await _client.PutObjectAsync(args, CancellationToken)).GetAwaiter().GetResult();
return result.Etag ?? string.Empty;
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to upload file '{filePath}' to bucket '{bucketName}' as '{objectName}': {ex.Message}", ex);
}
}
/// <summary>
/// Downloads an object from MinIO synchronously with progress reporting
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <param name="objectName">Name of the object</param>
/// <param name="filePath">Path where the file should be saved</param>
/// <param name="progressCallback">Progress callback for reporting download progress</param>
public void DownloadFile(string bucketName, string objectName, string filePath,
Action<long>? progressCallback = null)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
if (string.IsNullOrWhiteSpace(objectName))
throw new ArgumentException("Object name cannot be null or empty", nameof(objectName));
if (string.IsNullOrWhiteSpace(filePath))
throw new ArgumentException("File path cannot be null or empty", nameof(filePath));
try
{
// Ensure the directory exists
var directory = Path.GetDirectoryName(filePath);
if (!string.IsNullOrEmpty(directory) && !Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
var args = new GetObjectArgs()
.WithBucket(bucketName)
.WithObject(objectName)
.WithFile(filePath);
// Add progress callback if provided
if (progressCallback != null)
{
args = args.WithProgress(new Progress<ProgressReport>(report =>
{
progressCallback(report.TotalBytesTransferred);
}));
}
Task.Run(async () =>
await _client.GetObjectAsync(args, CancellationToken)).GetAwaiter().GetResult();
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to download object '{objectName}' from bucket '{bucketName}' to '{filePath}': {ex.Message}", ex);
}
}
/// <summary>
/// Uploads a stream to MinIO synchronously
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <param name="objectName">Name of the object</param>
/// <param name="data">Stream containing the data to upload</param>
/// <param name="contentType">Content type of the data</param>
/// <param name="progressCallback">Progress callback for reporting upload progress</param>
/// <returns>ETag of the uploaded object</returns>
public string UploadStream(string bucketName, string objectName, Stream data,
string contentType = "application/octet-stream", Action<long>? progressCallback = null)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
if (string.IsNullOrWhiteSpace(objectName))
throw new ArgumentException("Object name cannot be null or empty", nameof(objectName));
if (data == null)
throw new ArgumentNullException(nameof(data));
try
{
var args = new PutObjectArgs()
.WithBucket(bucketName)
.WithObject(objectName)
.WithStreamData(data)
.WithObjectSize(data.Length)
.WithContentType(contentType);
// Add progress callback if provided
if (progressCallback != null)
{
args = args.WithProgress(new Progress<ProgressReport>(report =>
{
progressCallback(report.TotalBytesTransferred);
}));
}
var result = Task.Run(async () =>
await _client.PutObjectAsync(args, CancellationToken)).GetAwaiter().GetResult();
return result.Etag ?? string.Empty;
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to upload stream to bucket '{bucketName}' as '{objectName}': {ex.Message}", ex);
}
}
/// <summary>
/// Gets the content type for a file based on its extension
/// </summary>
/// <param name="filePath">Path to the file</param>
/// <returns>Content type string</returns>
private static string GetContentType(string filePath)
{
var extension = Path.GetExtension(filePath).ToLowerInvariant();
return extension switch
{
".txt" => "text/plain",
".html" => "text/html",
".css" => "text/css",
".js" => "application/javascript",
".json" => "application/json",
".xml" => "application/xml",
".pdf" => "application/pdf",
".zip" => "application/zip",
".jpg" or ".jpeg" => "image/jpeg",
".png" => "image/png",
".gif" => "image/gif",
".svg" => "image/svg+xml",
".mp4" => "video/mp4",
".mp3" => "audio/mpeg",
".wav" => "audio/wav",
_ => "application/octet-stream"
};
}
/// <summary>
/// Lists object versions in a bucket synchronously
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <param name="prefix">Optional prefix to filter objects</param>
/// <param name="recursive">Whether to list objects recursively</param>
/// <param name="maxObjects">Maximum number of objects to return (0 = unlimited)</param>
/// <returns>List of object information including versions</returns>
private List<MinIOObjectInfo> ListObjectVersions(string bucketName, string? prefix, bool recursive, int maxObjects)
{
try
{
// For now, fall back to regular object listing since version listing
// may not be available in all MinIO SDK versions
// This can be enhanced when the SDK supports it
return ListObjects(bucketName, prefix, recursive, maxObjects, false);
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to list object versions in bucket '{bucketName}': {ex.Message}", ex);
}
}
/// <summary>
/// Generates a presigned URL for an object
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <param name="objectName">Name of the object</param>
/// <param name="expiry">URL expiry time</param>
/// <returns>Presigned URL</returns>
public string GetPresignedUrl(string bucketName, string objectName, TimeSpan expiry)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
if (string.IsNullOrWhiteSpace(objectName))
throw new ArgumentException("Object name cannot be null or empty", nameof(objectName));
try
{
var args = new PresignedGetObjectArgs()
.WithBucket(bucketName)
.WithObject(objectName)
.WithExpiry((int)expiry.TotalSeconds);
var result = Task.Run(async () =>
await _client.PresignedGetObjectAsync(args)).GetAwaiter().GetResult();
return result ?? string.Empty;
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to generate presigned URL for object '{objectName}' in bucket '{bucketName}': {ex.Message}", ex);
}
}
/// <summary>
/// Uploads a file using chunked transfer with resume capability
/// </summary>
/// <param name="transferState">Transfer state for resume functionality</param>
/// <param name="progressReporter">Progress reporter for updates</param>
/// <param name="maxRetries">Maximum retry attempts per chunk</param>
/// <returns>MinIOObjectInfo of uploaded object or null if failed</returns>
public MinIOObjectInfo? UploadFileChunked(
ChunkedTransferState transferState,
ChunkedCollectionProgressReporter progressReporter,
int maxRetries = 3)
{
if (transferState == null)
throw new ArgumentNullException(nameof(transferState));
try
{
// Start multipart upload if not already started
if (string.IsNullOrEmpty(transferState.UploadId))
{
var initiateArgs = new NewMultipartUploadArgs()
.WithBucket(transferState.BucketName)
.WithObject(transferState.ObjectName);
var initiateResult = Task.Run(async () =>
await _client.NewMultipartUploadAsync(initiateArgs, CancellationToken)).GetAwaiter().GetResult();
transferState.UploadId = initiateResult.UploadId;
}
var completedParts = new List<UploadPartResponse>();
// Process each chunk
while (!transferState.IsComplete)
{
var nextChunk = transferState.GetNextChunk();
if (nextChunk == null)
break;
progressReporter.StartNewChunk(nextChunk.ChunkNumber + 1, nextChunk.Size);
var uploadResult = UploadChunkWithRetry(transferState, nextChunk, progressReporter, maxRetries);
if (uploadResult != null)
{
completedParts.Add(uploadResult);
transferState.MarkChunkCompleted(nextChunk);
progressReporter.CompleteChunk(uploadResult.ETag);
// Save progress for resume
ChunkedTransferResumeManager.SaveTransferState(transferState);
}
else
{
throw new InvalidOperationException($"Failed to upload chunk {nextChunk.ChunkNumber} after {maxRetries} attempts");
}
}
// Complete multipart upload
var completeArgs = new CompleteMultipartUploadArgs()
.WithBucket(transferState.BucketName)
.WithObject(transferState.ObjectName)
.WithUploadId(transferState.UploadId)
.WithETags(completedParts.OrderBy(p => p.PartNumber).Select(p => new Tuple<int, string>(p.PartNumber, p.ETag)));
var completeResult = Task.Run(async () =>
await _client.CompleteMultipartUploadAsync(completeArgs, CancellationToken)).GetAwaiter().GetResult();
// Return object information
return new MinIOObjectInfo(
transferState.ObjectName,
transferState.TotalSize,
DateTime.UtcNow,
completeResult.ETag,
transferState.BucketName);
}
catch (Exception ex)
{
// Abort multipart upload on failure
if (!string.IsNullOrEmpty(transferState.UploadId))
{
try
{
var abortArgs = new AbortMultipartUploadArgs()
.WithBucket(transferState.BucketName)
.WithObject(transferState.ObjectName)
.WithUploadId(transferState.UploadId);
Task.Run(async () =>
await _client.AbortMultipartUploadAsync(abortArgs, CancellationToken)).GetAwaiter().GetResult();
}
catch
{
// Ignore abort errors
}
}
throw new InvalidOperationException($"Chunked upload failed for object '{transferState.ObjectName}': {ex.Message}", ex);
}
}
/// <summary>
/// Uploads a single chunk with retry logic
/// </summary>
/// <param name="transferState">Transfer state</param>
/// <param name="chunk">Chunk to upload</param>
/// <param name="progressReporter">Progress reporter</param>
/// <param name="maxRetries">Maximum retry attempts</param>
/// <returns>Upload part response or null if failed</returns>
private UploadPartResponse? UploadChunkWithRetry(
ChunkedTransferState transferState,
ChunkInfo chunk,
ChunkedCollectionProgressReporter progressReporter,
int maxRetries)
{
for (int attempt = 1; attempt <= maxRetries; attempt++)
{
try
{
using var fileStream = new FileStream(transferState.FilePath, FileMode.Open, FileAccess.Read, FileShare.Read);
fileStream.Seek(chunk.StartByte, SeekOrigin.Begin);
var chunkData = new byte[chunk.Size];
var bytesRead = fileStream.Read(chunkData, 0, (int)chunk.Size);
using var chunkStream = new MemoryStream(chunkData, 0, bytesRead);
var uploadArgs = new UploadPartArgs()
.WithBucket(transferState.BucketName)
.WithObject(transferState.ObjectName)
.WithUploadId(transferState.UploadId)
.WithPartNumber(chunk.ChunkNumber + 1) // MinIO uses 1-based part numbers
.WithPartSize(bytesRead)
.WithStreamData(chunkStream);
// Add progress callback
uploadArgs = uploadArgs.WithProgress(new Progress<ProgressReport>(report =>
{
progressReporter.UpdateChunkProgress(report.TotalBytesTransferred);
}));
var result = Task.Run(async () =>
await _client.UploadPartAsync(uploadArgs, CancellationToken)).GetAwaiter().GetResult();
chunk.ChunkETag = result.ETag;
return result;
}
catch (Exception ex) when (attempt < maxRetries)
{
progressReporter.ReportChunkError(ex, attempt, maxRetries);
// Exponential backoff
var delay = TimeSpan.FromSeconds(Math.Pow(2, attempt));
Task.Delay(delay, CancellationToken).GetAwaiter().GetResult();
}
catch (Exception ex)
{
progressReporter.ReportChunkError(ex, attempt, maxRetries);
chunk.LastError = ex.Message;
chunk.RetryCount = attempt;
return null;
}
}
return null;
}
/// <summary>
/// Downloads a file using chunked transfer with resume capability
/// </summary>
/// <param name="transferState">Transfer state for resume functionality</param>
/// <param name="progressReporter">Progress reporter for updates</param>
/// <param name="maxRetries">Maximum retry attempts per chunk</param>
/// <param name="parallelDownloads">Number of parallel chunk downloads</param>
/// <returns>True if download succeeded, false otherwise</returns>
public bool DownloadFileChunked(
ChunkedTransferState transferState,
ChunkedSingleFileProgressReporter progressReporter,
int maxRetries = 3,
int parallelDownloads = 3)
{
if (transferState == null)
throw new ArgumentNullException(nameof(transferState));
try
{
// Create or open the target file
using var fileStream = new FileStream(transferState.FilePath, FileMode.Create, FileAccess.Write, FileShare.None);
fileStream.SetLength(transferState.TotalSize);
// Get list of chunks to download
var chunksToDownload = new List<ChunkInfo>();
while (!transferState.IsComplete)
{
var nextChunk = transferState.GetNextChunk();
if (nextChunk == null)
break;
chunksToDownload.Add(nextChunk);
}
if (chunksToDownload.Count == 0)
{
return true; // Already complete
}
// Download chunks (with limited parallelism)
var semaphore = new SemaphoreSlim(parallelDownloads, parallelDownloads);
var downloadTasks = chunksToDownload.Select(chunk =>
DownloadChunkAsync(transferState, chunk, fileStream, progressReporter, maxRetries, semaphore)).ToArray();
var results = Task.WhenAll(downloadTasks).GetAwaiter().GetResult();
// Check if all chunks downloaded successfully
var allSucceeded = results.All(r => r);
if (allSucceeded)
{
// Mark all chunks as completed
foreach (var chunk in chunksToDownload)
{
transferState.MarkChunkCompleted(chunk);
}
}
return allSucceeded;
}
catch (Exception ex)
{
throw new InvalidOperationException($"Chunked download failed for object '{transferState.ObjectName}': {ex.Message}", ex);
}
}
/// <summary>
/// Downloads a single chunk asynchronously with retry logic
/// </summary>
/// <param name="transferState">Transfer state</param>
/// <param name="chunk">Chunk to download</param>
/// <param name="fileStream">Target file stream</param>
/// <param name="progressReporter">Progress reporter</param>
/// <param name="maxRetries">Maximum retry attempts</param>
/// <param name="semaphore">Semaphore for controlling parallelism</param>
/// <returns>True if chunk downloaded successfully</returns>
private async Task<bool> DownloadChunkAsync(
ChunkedTransferState transferState,
ChunkInfo chunk,
FileStream fileStream,
ChunkedSingleFileProgressReporter progressReporter,
int maxRetries,
SemaphoreSlim semaphore)
{
await semaphore.WaitAsync(CancellationToken);
try
{
progressReporter.StartNewChunk(chunk.ChunkNumber + 1, chunk.Size);
for (int attempt = 1; attempt <= maxRetries; attempt++)
{
try
{
var getArgs = new GetObjectArgs()
.WithBucket(transferState.BucketName)
.WithObject(transferState.ObjectName)
.WithOffsetAndLength(chunk.StartByte, chunk.Size);
using var chunkStream = new MemoryStream();
await _client.GetObjectAsync(getArgs, (stream) =>
{
stream.CopyTo(chunkStream);
}, CancellationToken);
// Write chunk to file at correct position
lock (fileStream)
{
fileStream.Seek(chunk.StartByte, SeekOrigin.Begin);
chunkStream.Seek(0, SeekOrigin.Begin);
chunkStream.CopyTo(fileStream);
fileStream.Flush();
}
progressReporter.UpdateChunkProgress(chunk.Size);
progressReporter.CompleteChunk();
chunk.IsCompleted = true;
return true;
}
catch (Exception ex) when (attempt < maxRetries)
{
progressReporter.ReportChunkError(ex, attempt, maxRetries);
// Exponential backoff
var delay = TimeSpan.FromSeconds(Math.Pow(2, attempt));
await Task.Delay(delay, CancellationToken);
}
catch (Exception ex)
{
progressReporter.ReportChunkError(ex, attempt, maxRetries);
chunk.LastError = ex.Message;
chunk.RetryCount = attempt;
return false;
}
}
return false;
}
finally
{
semaphore.Release();
}
}
/// <summary>
/// Cancels all ongoing operations
/// </summary>
public void CancelOperations()
{
_cancellationTokenSource.Cancel();
}
/// <summary>
/// Disposes the wrapper and underlying client
/// </summary>
public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
/// <summary>
/// Protected dispose method
/// </summary>
/// <param name="disposing">Whether disposing from Dispose method</param>
protected virtual void Dispose(bool disposing)
{
if (!_disposed && disposing)
{
_cancellationTokenSource?.Cancel();
_cancellationTokenSource?.Dispose();
_client?.Dispose();
_disposed = true;
}
}
}
}
+249
View File
@@ -0,0 +1,249 @@
using System;
using System.Management.Automation;
namespace PSMinIO.Utils
{
/// <summary>
/// Centralized logging utility for PSMinIO module
/// </summary>
public static class MinIOLogger
{
/// <summary>
/// Log levels for different types of messages
/// </summary>
public enum LogLevel
{
Verbose,
Information,
Warning,
Error
}
/// <summary>
/// Writes a verbose message if verbose preference allows it
/// </summary>
/// <param name="cmdlet">The cmdlet instance for context</param>
/// <param name="message">Message to log</param>
/// <param name="args">Optional format arguments</param>
public static void WriteVerbose(PSCmdlet cmdlet, string message, params object[] args)
{
if (cmdlet == null) return;
var formattedMessage = FormatLogMessage(LogLevel.Verbose, message, args);
cmdlet.WriteVerbose(formattedMessage);
}
/// <summary>
/// Writes an information message
/// </summary>
/// <param name="cmdlet">The cmdlet instance for context</param>
/// <param name="message">Message to log</param>
/// <param name="args">Optional format arguments</param>
public static void WriteInformation(PSCmdlet cmdlet, string message, params object[] args)
{
if (cmdlet == null) return;
var formattedMessage = FormatLogMessage(LogLevel.Information, message, args);
// Use WriteInformation if available (PowerShell 5.0+), otherwise WriteVerbose
try
{
var infoRecord = new InformationRecord(formattedMessage, "PSMinIO");
cmdlet.WriteInformation(infoRecord);
}
catch
{
// Fallback to WriteVerbose for older PowerShell versions
cmdlet.WriteVerbose(formattedMessage);
}
}
/// <summary>
/// Writes a warning message
/// </summary>
/// <param name="cmdlet">The cmdlet instance for context</param>
/// <param name="message">Message to log</param>
/// <param name="args">Optional format arguments</param>
public static void WriteWarning(PSCmdlet cmdlet, string message, params object[] args)
{
if (cmdlet == null) return;
var formattedMessage = FormatLogMessage(LogLevel.Warning, message, args);
cmdlet.WriteWarning(formattedMessage);
}
/// <summary>
/// Writes an error message
/// </summary>
/// <param name="cmdlet">The cmdlet instance for context</param>
/// <param name="message">Message to log</param>
/// <param name="args">Optional format arguments</param>
public static void WriteError(PSCmdlet cmdlet, string message, params object[] args)
{
if (cmdlet == null) return;
var formattedMessage = FormatLogMessage(LogLevel.Error, message, args);
var errorRecord = new ErrorRecord(
new InvalidOperationException(formattedMessage),
"PSMinIOError",
ErrorCategory.InvalidOperation,
null);
cmdlet.WriteError(errorRecord);
}
/// <summary>
/// Writes an error from an exception
/// </summary>
/// <param name="cmdlet">The cmdlet instance for context</param>
/// <param name="exception">Exception to log</param>
/// <param name="errorId">Error identifier</param>
/// <param name="category">Error category</param>
/// <param name="targetObject">Target object that caused the error</param>
public static void WriteError(PSCmdlet cmdlet, Exception exception, string errorId,
ErrorCategory category = ErrorCategory.InvalidOperation, object? targetObject = null)
{
if (cmdlet == null) return;
var errorRecord = new ErrorRecord(exception, errorId, category, targetObject);
cmdlet.WriteError(errorRecord);
}
/// <summary>
/// Logs the start of an operation
/// </summary>
/// <param name="cmdlet">The cmdlet instance for context</param>
/// <param name="operation">Name of the operation</param>
/// <param name="details">Optional operation details</param>
public static void LogOperationStart(PSCmdlet cmdlet, string operation, string? details = null)
{
var message = string.IsNullOrEmpty(details)
? $"Starting operation: {operation}"
: $"Starting operation: {operation} - {details}";
WriteVerbose(cmdlet, message);
}
/// <summary>
/// Logs the completion of an operation
/// </summary>
/// <param name="cmdlet">The cmdlet instance for context</param>
/// <param name="operation">Name of the operation</param>
/// <param name="duration">Optional operation duration</param>
/// <param name="details">Optional operation details</param>
public static void LogOperationComplete(PSCmdlet cmdlet, string operation, TimeSpan? duration = null, string? details = null)
{
var message = $"Completed operation: {operation}";
if (duration.HasValue)
{
message += $" (Duration: {duration.Value.TotalMilliseconds:F0}ms)";
}
if (!string.IsNullOrEmpty(details))
{
message += $" - {details}";
}
WriteVerbose(cmdlet, message);
}
/// <summary>
/// Logs an operation failure
/// </summary>
/// <param name="cmdlet">The cmdlet instance for context</param>
/// <param name="operation">Name of the operation</param>
/// <param name="exception">Exception that occurred</param>
/// <param name="details">Optional operation details</param>
public static void LogOperationFailure(PSCmdlet cmdlet, string operation, Exception exception, string? details = null)
{
var message = $"Operation failed: {operation} - {exception.Message}";
if (!string.IsNullOrEmpty(details))
{
message += $" - {details}";
}
WriteError(cmdlet, message);
WriteVerbose(cmdlet, $"Exception details: {exception}");
}
/// <summary>
/// Formats a log message with timestamp and level, automatically formatting byte sizes
/// </summary>
/// <param name="level">Log level</param>
/// <param name="message">Message to format</param>
/// <param name="args">Optional format arguments</param>
/// <returns>Formatted log message</returns>
private static string FormatLogMessage(LogLevel level, string message, params object[] args)
{
var timestamp = DateTime.Now.ToString("yyyy/MM/dd HH:mm:ss.fff");
var levelString = level.ToString().ToUpperInvariant();
// Process arguments to format byte sizes intelligently
var processedArgs = ProcessLogArguments(args);
var formattedMessage = processedArgs.Length > 0 ? string.Format(message, processedArgs) : message;
return $"{timestamp} - [{levelString}] - {formattedMessage}";
}
/// <summary>
/// Processes log arguments to format byte sizes intelligently
/// </summary>
/// <param name="args">Original arguments</param>
/// <returns>Processed arguments with formatted sizes</returns>
private static object[] ProcessLogArguments(object[] args)
{
if (args == null || args.Length == 0)
return args;
var processedArgs = new object[args.Length];
for (int i = 0; i < args.Length; i++)
{
var arg = args[i];
// Check if this looks like a byte size that should be formatted
if (IsLikelyByteSize(arg))
{
if (long.TryParse(arg.ToString(), out var bytes))
{
processedArgs[i] = SizeFormatter.FormatBytes(bytes);
}
else
{
processedArgs[i] = arg;
}
}
else
{
processedArgs[i] = arg;
}
}
return processedArgs;
}
/// <summary>
/// Determines if an argument is likely a byte size that should be formatted
/// </summary>
/// <param name="arg">Argument to check</param>
/// <returns>True if likely a byte size</returns>
private static bool IsLikelyByteSize(object arg)
{
// Only format long integers that are likely byte sizes
// We use a heuristic: values >= 1024 are likely byte sizes
if (arg is long longValue)
{
return longValue >= 1024;
}
if (arg is int intValue)
{
return intValue >= 1024;
}
return false;
}
}
}
+156
View File
@@ -0,0 +1,156 @@
using System;
using System.Diagnostics;
using System.Management.Automation;
namespace PSMinIO.Utils
{
/// <summary>
/// Utility class for reporting progress during file operations
/// </summary>
public class ProgressReporter
{
private readonly PSCmdlet _cmdlet;
private readonly string _activity;
private readonly string _statusDescription;
private readonly long _totalBytes;
private readonly Stopwatch _stopwatch;
private long _bytesProcessed;
private DateTime _lastUpdateTime;
private readonly int _activityId;
/// <summary>
/// Creates a new ProgressReporter instance
/// </summary>
/// <param name="cmdlet">The cmdlet to report progress to</param>
/// <param name="activity">Description of the activity</param>
/// <param name="statusDescription">Status description</param>
/// <param name="totalBytes">Total number of bytes to process</param>
/// <param name="activityId">Unique activity ID for progress reporting</param>
public ProgressReporter(PSCmdlet cmdlet, string activity, string statusDescription, long totalBytes, int activityId = 1)
{
_cmdlet = cmdlet ?? throw new ArgumentNullException(nameof(cmdlet));
_activity = activity ?? throw new ArgumentNullException(nameof(activity));
_statusDescription = statusDescription ?? throw new ArgumentNullException(nameof(statusDescription));
_totalBytes = totalBytes;
_activityId = activityId;
_stopwatch = Stopwatch.StartNew();
_lastUpdateTime = DateTime.UtcNow;
}
/// <summary>
/// Updates the progress with the number of bytes processed
/// </summary>
/// <param name="bytesProcessed">Number of bytes processed so far</param>
public void UpdateProgress(long bytesProcessed)
{
_bytesProcessed = bytesProcessed;
// Only update progress every 100ms to avoid overwhelming the console
var now = DateTime.UtcNow;
if ((now - _lastUpdateTime).TotalMilliseconds < 100 && bytesProcessed < _totalBytes)
{
return;
}
_lastUpdateTime = now;
var percentComplete = _totalBytes > 0 ? (int)((double)bytesProcessed / _totalBytes * 100) : 0;
var currentStatus = FormatCurrentStatus(bytesProcessed);
var progressRecord = new ProgressRecord(_activityId, _activity, currentStatus)
{
PercentComplete = Math.Min(percentComplete, 100)
};
// Add remaining time estimate if we have enough data
if (_stopwatch.ElapsedMilliseconds > 1000 && bytesProcessed > 0 && bytesProcessed < _totalBytes)
{
var remainingTime = EstimateRemainingTime(bytesProcessed);
if (remainingTime.HasValue)
{
progressRecord.SecondsRemaining = (int)remainingTime.Value.TotalSeconds;
}
}
_cmdlet.WriteProgress(progressRecord);
}
/// <summary>
/// Completes the progress reporting
/// </summary>
public void Complete()
{
_stopwatch.Stop();
var progressRecord = new ProgressRecord(_activityId, _activity, "Completed")
{
PercentComplete = 100,
RecordType = ProgressRecordType.Completed
};
_cmdlet.WriteProgress(progressRecord);
// Log completion details
MinIOLogger.WriteVerbose(_cmdlet,
$"Operation completed: {SizeFormatter.FormatBytes(_totalBytes)} processed in {_stopwatch.Elapsed.TotalSeconds:F1} seconds " +
$"(Average speed: {SizeFormatter.FormatBytesPerSecond(CalculateAverageSpeed())})");
}
/// <summary>
/// Formats the current status string
/// </summary>
/// <param name="bytesProcessed">Number of bytes processed</param>
/// <returns>Formatted status string</returns>
private string FormatCurrentStatus(long bytesProcessed)
{
var processedFormatted = SizeFormatter.FormatBytes(bytesProcessed);
var totalFormatted = SizeFormatter.FormatBytes(_totalBytes);
var speed = CalculateCurrentSpeed();
var speedFormatted = SizeFormatter.FormatBytesPerSecond(speed);
return $"{_statusDescription}: {processedFormatted} / {totalFormatted} ({speedFormatted})";
}
/// <summary>
/// Calculates the current transfer speed in bytes per second
/// </summary>
/// <returns>Current speed in bytes per second</returns>
private double CalculateCurrentSpeed()
{
var elapsedSeconds = _stopwatch.Elapsed.TotalSeconds;
return elapsedSeconds > 0 ? _bytesProcessed / elapsedSeconds : 0;
}
/// <summary>
/// Calculates the average transfer speed in bytes per second
/// </summary>
/// <returns>Average speed in bytes per second</returns>
private double CalculateAverageSpeed()
{
var elapsedSeconds = _stopwatch.Elapsed.TotalSeconds;
return elapsedSeconds > 0 ? _totalBytes / elapsedSeconds : 0;
}
/// <summary>
/// Estimates the remaining time for the operation
/// </summary>
/// <param name="bytesProcessed">Number of bytes processed so far</param>
/// <returns>Estimated remaining time, or null if cannot be calculated</returns>
private TimeSpan? EstimateRemainingTime(long bytesProcessed)
{
if (bytesProcessed <= 0 || _stopwatch.ElapsedMilliseconds <= 0)
return null;
var remainingBytes = _totalBytes - bytesProcessed;
var currentSpeed = CalculateCurrentSpeed();
if (currentSpeed <= 0)
return null;
var remainingSeconds = remainingBytes / currentSpeed;
return TimeSpan.FromSeconds(remainingSeconds);
}
}
}
+189
View File
@@ -0,0 +1,189 @@
using System;
namespace PSMinIO.Utils
{
/// <summary>
/// Utility class for formatting byte sizes into human-readable strings
/// </summary>
public static class SizeFormatter
{
/// <summary>
/// Size units in order from smallest to largest
/// </summary>
private static readonly string[] SizeUnits = { "B", "KB", "MB", "GB", "TB", "PB", "EB" };
/// <summary>
/// Formats bytes into a human-readable string with appropriate units
/// </summary>
/// <param name="bytes">Number of bytes</param>
/// <param name="decimalPlaces">Number of decimal places to show (default: 2)</param>
/// <returns>Formatted string with appropriate unit</returns>
public static string FormatBytes(long bytes, int decimalPlaces = 2)
{
if (bytes == 0)
return "0 B";
if (bytes < 0)
return $"-{FormatBytes(-bytes, decimalPlaces)}";
int unitIndex = 0;
double size = bytes;
// Find the appropriate unit
while (size >= 1024 && unitIndex < SizeUnits.Length - 1)
{
size /= 1024;
unitIndex++;
}
// Format with specified decimal places
var formatString = $"{{0:F{decimalPlaces}}} {{1}}";
return string.Format(formatString, size, SizeUnits[unitIndex]);
}
/// <summary>
/// Formats bytes into a human-readable string with appropriate units (double overload)
/// </summary>
/// <param name="bytes">Number of bytes</param>
/// <param name="decimalPlaces">Number of decimal places to show (default: 2)</param>
/// <returns>Formatted string with appropriate unit</returns>
public static string FormatBytes(double bytes, int decimalPlaces = 2)
{
return FormatBytes((long)Math.Round(bytes), decimalPlaces);
}
/// <summary>
/// Formats bytes per second into a human-readable string
/// </summary>
/// <param name="bytesPerSecond">Bytes per second</param>
/// <param name="decimalPlaces">Number of decimal places to show (default: 2)</param>
/// <returns>Formatted string with appropriate unit and "/s" suffix</returns>
public static string FormatBytesPerSecond(double bytesPerSecond, int decimalPlaces = 2)
{
return $"{FormatBytes(bytesPerSecond, decimalPlaces)}/s";
}
/// <summary>
/// Formats a transfer rate with context
/// </summary>
/// <param name="bytesTransferred">Number of bytes transferred</param>
/// <param name="elapsedTime">Time elapsed for the transfer</param>
/// <param name="decimalPlaces">Number of decimal places to show (default: 2)</param>
/// <returns>Formatted transfer rate string</returns>
public static string FormatTransferRate(long bytesTransferred, TimeSpan elapsedTime, int decimalPlaces = 2)
{
if (elapsedTime.TotalSeconds <= 0)
return "0 B/s";
var bytesPerSecond = bytesTransferred / elapsedTime.TotalSeconds;
return FormatBytesPerSecond(bytesPerSecond, decimalPlaces);
}
/// <summary>
/// Formats a progress string showing current/total with percentages
/// </summary>
/// <param name="current">Current bytes processed</param>
/// <param name="total">Total bytes to process</param>
/// <param name="decimalPlaces">Number of decimal places to show (default: 2)</param>
/// <returns>Formatted progress string</returns>
public static string FormatProgress(long current, long total, int decimalPlaces = 2)
{
var currentFormatted = FormatBytes(current, decimalPlaces);
var totalFormatted = FormatBytes(total, decimalPlaces);
if (total > 0)
{
var percentage = (double)current / total * 100;
return $"{currentFormatted} / {totalFormatted} ({percentage:F1}%)";
}
return $"{currentFormatted} / {totalFormatted}";
}
/// <summary>
/// Gets the appropriate unit for a given byte size without formatting
/// </summary>
/// <param name="bytes">Number of bytes</param>
/// <returns>Appropriate unit string</returns>
public static string GetAppropriateUnit(long bytes)
{
if (bytes == 0)
return "B";
int unitIndex = 0;
double size = Math.Abs(bytes);
while (size >= 1024 && unitIndex < SizeUnits.Length - 1)
{
size /= 1024;
unitIndex++;
}
return SizeUnits[unitIndex];
}
/// <summary>
/// Converts bytes to the specified unit
/// </summary>
/// <param name="bytes">Number of bytes</param>
/// <param name="unit">Target unit (B, KB, MB, GB, TB, PB, EB)</param>
/// <returns>Value in the specified unit</returns>
public static double ConvertToUnit(long bytes, string unit)
{
var unitIndex = Array.IndexOf(SizeUnits, unit.ToUpperInvariant());
if (unitIndex == -1)
throw new ArgumentException($"Invalid unit: {unit}. Valid units are: {string.Join(", ", SizeUnits)}");
if (unitIndex == 0) // Bytes
return bytes;
return bytes / Math.Pow(1024, unitIndex);
}
/// <summary>
/// Parses a size string back to bytes (e.g., "1.5 GB" -> bytes)
/// </summary>
/// <param name="sizeString">Size string to parse</param>
/// <returns>Number of bytes</returns>
public static long ParseSizeString(string sizeString)
{
if (string.IsNullOrWhiteSpace(sizeString))
throw new ArgumentException("Size string cannot be null or empty");
var parts = sizeString.Trim().Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (parts.Length != 2)
throw new ArgumentException($"Invalid size string format: {sizeString}. Expected format: '1.5 GB'");
if (!double.TryParse(parts[0], out var value))
throw new ArgumentException($"Invalid numeric value: {parts[0]}");
var unit = parts[1].ToUpperInvariant();
var unitIndex = Array.IndexOf(SizeUnits, unit);
if (unitIndex == -1)
throw new ArgumentException($"Invalid unit: {unit}. Valid units are: {string.Join(", ", SizeUnits)}");
return (long)(value * Math.Pow(1024, unitIndex));
}
/// <summary>
/// Formats a size comparison between two values
/// </summary>
/// <param name="value1">First value in bytes</param>
/// <param name="value2">Second value in bytes</param>
/// <param name="label1">Label for first value</param>
/// <param name="label2">Label for second value</param>
/// <param name="decimalPlaces">Number of decimal places to show (default: 2)</param>
/// <returns>Formatted comparison string</returns>
public static string FormatComparison(long value1, long value2, string label1, string label2, int decimalPlaces = 2)
{
var formatted1 = FormatBytes(value1, decimalPlaces);
var formatted2 = FormatBytes(value2, decimalPlaces);
var difference = value1 - value2;
var diffFormatted = FormatBytes(Math.Abs(difference), decimalPlaces);
var diffDirection = difference >= 0 ? "larger" : "smaller";
return $"{label1}: {formatted1}, {label2}: {formatted2} ({diffFormatted} {diffDirection})";
}
}
}