mirror of
https://github.com/Grace-Solutions/PSMinIO.git
synced 2026-08-09 20:39:32 +00:00
Complete rebuild: Custom REST API implementation
- Removed MinIO SDK dependency completely - Built custom HTTP client with AWS S3 signature v4 support - Created synchronous operations optimized for PowerShell - Added real progress reporting during transfers - Implemented performance metrics in result objects - Fixed null pointer warnings for code quality - Removed Write-Host usage from test scripts - Cleaned up unnecessary DLL dependencies Available cmdlets: - Connect-MinIO: Enhanced connection with certificate options - Get-MinIOBucket: List buckets with sorting and filtering - New-MinIOBucket: Create buckets with region support - Test-MinIOBucketExists: Check bucket existence - Get-MinIOObject: List objects with advanced filtering - New-MinIOObject: Upload files with real progress - Get-MinIOObjectContent: Download files with progress This represents a complete architectural overhaul for better PowerShell compatibility and performance.
This commit is contained in:
PSMinIO Developer
@@ -1,6 +1,6 @@
|
||||
using System;
|
||||
using System.Management.Automation;
|
||||
using PSMinIO.Models;
|
||||
using PSMinIO.Core;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Cmdlets
|
||||
@@ -37,7 +37,7 @@ namespace PSMinIO.Cmdlets
|
||||
public string SecretKey { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Region for bucket operations (default: us-east-1)
|
||||
/// Optional region for bucket operations (default: us-east-1)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateNotNullOrEmpty]
|
||||
@@ -51,40 +51,65 @@ namespace PSMinIO.Cmdlets
|
||||
public int TimeoutSeconds { get; set; } = 30;
|
||||
|
||||
/// <summary>
|
||||
/// Test the connection after establishing it
|
||||
/// Skip SSL certificate validation (for development/self-signed certificates)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter TestConnection { get; set; }
|
||||
public SwitchParameter SkipCertificateValidation { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Store the connection in a session variable for reuse (default: MinIOConnection)
|
||||
/// Maximum number of concurrent connections (default: 10)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateRange(1, 100)]
|
||||
public int MaxConnections { get; set; } = 10;
|
||||
|
||||
/// <summary>
|
||||
/// Default chunk size for multipart uploads in MB (default: 5)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateRange(1, 1024)]
|
||||
public int DefaultChunkSizeMB { get; set; } = 5;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum retry attempts for failed operations (default: 3)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateRange(0, 10)]
|
||||
public int MaxRetries { get; set; } = 3;
|
||||
|
||||
/// <summary>
|
||||
/// Name of session variable to store the connection (default: MinIOConnection)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateNotNullOrEmpty]
|
||||
public string SessionVariable { get; set; } = "MinIOConnection";
|
||||
|
||||
/// <summary>
|
||||
/// Skip SSL certificate validation (use with caution)
|
||||
/// Test the connection after establishing it
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter SkipCertificateValidation { get; set; }
|
||||
public SwitchParameter TestConnection { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Return the connection object instead of storing it in session variable
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter PassThru { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Processes the cmdlet
|
||||
/// </summary>
|
||||
protected override void ProcessRecord()
|
||||
{
|
||||
// Extract connection details from URI
|
||||
var useSSL = Endpoint.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase);
|
||||
// Extract endpoint components
|
||||
var endpointHost = $"{Endpoint.Host}:{Endpoint.Port}";
|
||||
var useSSL = Endpoint.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
if (ShouldProcess($"MinIO Server: {Endpoint}", $"Establish connection"))
|
||||
{
|
||||
try
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Connecting to MinIO - Endpoint: {0}, UseSSL: {1}, Region: {2}, Timeout: {3}s, SkipCertValidation: {4}",
|
||||
endpointHost, useSSL, Region, TimeoutSeconds, SkipCertificateValidation.IsPresent);
|
||||
MinIOLogger.LogConnection(this, endpointHost, useSSL, Region, TimeoutSeconds);
|
||||
|
||||
// Show warning when certificate validation is skipped
|
||||
if (SkipCertificateValidation.IsPresent)
|
||||
@@ -100,50 +125,53 @@ namespace PSMinIO.Cmdlets
|
||||
useSSL,
|
||||
Region,
|
||||
TimeoutSeconds,
|
||||
SkipCertificateValidation.IsPresent);
|
||||
SkipCertificateValidation.IsPresent)
|
||||
{
|
||||
MaxConnections = MaxConnections,
|
||||
DefaultChunkSize = DefaultChunkSizeMB * 1024 * 1024, // Convert MB to bytes
|
||||
MaxRetries = MaxRetries
|
||||
};
|
||||
|
||||
// Validate configuration
|
||||
if (!configuration.IsValid)
|
||||
{
|
||||
var errors = string.Join(", ", configuration.GetValidationErrors());
|
||||
throw new ArgumentException($"Invalid configuration: {errors}");
|
||||
}
|
||||
|
||||
// Create connection
|
||||
var connection = new MinIOConnection(configuration);
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "MinIO connection created successfully");
|
||||
|
||||
// Test connection if requested
|
||||
if (TestConnection.IsPresent)
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Testing MinIO connection...");
|
||||
WriteVerboseMessage("Testing connection by listing buckets...");
|
||||
|
||||
var testResult = connection.TestConnection();
|
||||
|
||||
if (testResult.Success)
|
||||
if (!connection.TestConnection())
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Connection test successful - found {0} buckets in {1}ms",
|
||||
testResult.BucketCount ?? 0,
|
||||
testResult.ResponseTime?.TotalMilliseconds ?? 0);
|
||||
throw new InvalidOperationException("Connection test failed. Please verify your credentials and endpoint.");
|
||||
}
|
||||
|
||||
WriteInformation(new InformationRecord(
|
||||
$"Connection test successful. Found {testResult.BucketCount ?? 0} buckets.",
|
||||
"ConnectionTest"));
|
||||
}
|
||||
else
|
||||
{
|
||||
WriteWarning($"Connection test failed: {testResult.Message}");
|
||||
MinIOLogger.WriteVerbose(this, "Connection test failed: {0}", testResult.Message);
|
||||
}
|
||||
WriteVerboseMessage("Connection test successful");
|
||||
}
|
||||
|
||||
// Store in session variable if requested
|
||||
if (!string.IsNullOrWhiteSpace(SessionVariable))
|
||||
// Store in session variable unless PassThru is specified
|
||||
if (!PassThru.IsPresent)
|
||||
{
|
||||
// Set variable in session state
|
||||
SessionState.PSVariable.Set(SessionVariable, connection);
|
||||
MinIOLogger.WriteVerbose(this, "Connection stored in session variable: {0}", SessionVariable!);
|
||||
WriteVerboseMessage("Connection stored in session variable: {0}", SessionVariable);
|
||||
}
|
||||
|
||||
// Return the connection object
|
||||
WriteObject(connection);
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Connect-MinIO completed successfully");
|
||||
// Output connection object if PassThru is specified or if not storing in session
|
||||
if (PassThru.IsPresent)
|
||||
{
|
||||
WriteObject(connection);
|
||||
}
|
||||
else
|
||||
{
|
||||
WriteVerboseMessage("Connected to MinIO server: {0}", endpointHost);
|
||||
WriteVerboseMessage("Use Get-MinIOBucket or other MinIO cmdlets to interact with the server");
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -152,35 +180,20 @@ namespace PSMinIO.Cmdlets
|
||||
"ConnectionFailed",
|
||||
ErrorCategory.ConnectionError,
|
||||
Endpoint);
|
||||
|
||||
WriteError(errorRecord);
|
||||
|
||||
ThrowTerminatingError(errorRecord);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Begins processing - validate parameters
|
||||
/// Writes verbose output with consistent formatting
|
||||
/// </summary>
|
||||
protected override void BeginProcessing()
|
||||
/// <param name="message">Message to write</param>
|
||||
/// <param name="args">Format arguments</param>
|
||||
private void WriteVerboseMessage(string message, params object[] args)
|
||||
{
|
||||
base.BeginProcessing();
|
||||
|
||||
// Validate URI scheme
|
||||
if (!Endpoint.Scheme.Equals("http", StringComparison.OrdinalIgnoreCase) &&
|
||||
!Endpoint.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
ThrowTerminatingError(new ErrorRecord(
|
||||
new ArgumentException($"Invalid URI scheme '{Endpoint.Scheme}'. Only 'http' and 'https' are supported."),
|
||||
"InvalidUriScheme",
|
||||
ErrorCategory.InvalidArgument,
|
||||
Endpoint));
|
||||
}
|
||||
|
||||
// Validate port is specified
|
||||
if (Endpoint.Port == -1)
|
||||
{
|
||||
WriteWarning($"No port specified in URI '{Endpoint}'. MinIO typically uses port 9000.");
|
||||
}
|
||||
MinIOLogger.WriteVerbose(this, message, args);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,180 +1,143 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Management.Automation;
|
||||
using PSMinIO.Models;
|
||||
using PSMinIO.Utils;
|
||||
using PSMinIO.Core.Models;
|
||||
|
||||
namespace PSMinIO.Cmdlets
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets information about MinIO buckets
|
||||
/// </summary>
|
||||
[Cmdlet(VerbsCommon.Get, "MinIOBucket", SupportsShouldProcess = true)]
|
||||
[Cmdlet(VerbsCommon.Get, "MinIOBucket")]
|
||||
[OutputType(typeof(MinIOBucketInfo))]
|
||||
public class GetMinIOBucketCmdlet : MinIOBaseCmdlet
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of a specific bucket to retrieve. If not specified, all buckets are returned.
|
||||
/// Name of a specific bucket to retrieve (supports wildcards)
|
||||
/// </summary>
|
||||
[Parameter(Position = 0, ValueFromPipeline = true, ValueFromPipelineByPropertyName = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[SupportsWildcards]
|
||||
[Alias("Bucket")]
|
||||
public string? BucketName { get; set; }
|
||||
public string? Name { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Include additional statistics like object count and total size for each bucket
|
||||
/// Include bucket statistics (object count and total size)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[Alias("Stats")]
|
||||
public SwitchParameter IncludeStatistics { get; set; }
|
||||
public SwitchParameter IncludeStats { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Include bucket policy information
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter IncludePolicy { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Sort buckets by the specified property
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateSet("Name", "CreationDate", "ObjectCount", "TotalSize")]
|
||||
public string SortBy { get; set; } = "Name";
|
||||
|
||||
/// <summary>
|
||||
/// Sort in descending order
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter Descending { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Processes the cmdlet
|
||||
/// </summary>
|
||||
protected override void ProcessRecord()
|
||||
{
|
||||
ValidateConnection();
|
||||
ExecuteOperation("ListBuckets", () =>
|
||||
{
|
||||
WriteVerboseMessage("Retrieving bucket list from MinIO server");
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(BucketName))
|
||||
{
|
||||
// Get specific bucket
|
||||
GetSpecificBucket(BucketName!);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Get all buckets
|
||||
GetAllBuckets();
|
||||
}
|
||||
}
|
||||
var buckets = S3Client.ListBuckets();
|
||||
|
||||
/// <summary>
|
||||
/// Gets information about a specific bucket
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket</param>
|
||||
private void GetSpecificBucket(string bucketName)
|
||||
{
|
||||
ValidateBucketName(bucketName);
|
||||
WriteVerboseMessage("Retrieved {0} buckets", buckets.Count);
|
||||
|
||||
if (ShouldProcess(bucketName, "Get bucket information"))
|
||||
{
|
||||
ExecuteOperation("GetBucket", () =>
|
||||
// Filter by name if specified
|
||||
if (!string.IsNullOrEmpty(Name))
|
||||
{
|
||||
// First check if bucket exists
|
||||
var exists = Client.BucketExists(bucketName);
|
||||
if (!exists)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Bucket '{bucketName}' does not exist"),
|
||||
"BucketNotFound",
|
||||
ErrorCategory.ObjectNotFound,
|
||||
bucketName));
|
||||
return;
|
||||
}
|
||||
var wildcardPattern = new WildcardPattern(Name, WildcardOptions.IgnoreCase);
|
||||
buckets = buckets.Where(b => b.Name != null && wildcardPattern.IsMatch(b.Name)).ToList();
|
||||
WriteVerboseMessage("Filtered to {0} buckets matching pattern '{1}'", buckets.Count, Name);
|
||||
}
|
||||
|
||||
// Get all buckets and find the specific one
|
||||
var allBuckets = Client.ListBuckets();
|
||||
var bucket = allBuckets.FirstOrDefault(b =>
|
||||
string.Equals(b.Name, bucketName, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
if (bucket != null)
|
||||
{
|
||||
if (IncludeStatistics.IsPresent)
|
||||
{
|
||||
PopulateBucketStatistics(bucket);
|
||||
}
|
||||
|
||||
WriteObject(bucket);
|
||||
}
|
||||
else
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Bucket '{bucketName}' not found in bucket list"),
|
||||
"BucketNotFound",
|
||||
ErrorCategory.ObjectNotFound,
|
||||
bucketName));
|
||||
}
|
||||
}, $"Bucket: {bucketName}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets information about all buckets
|
||||
/// </summary>
|
||||
private void GetAllBuckets()
|
||||
{
|
||||
if (ShouldProcess("All buckets", "Get bucket information"))
|
||||
{
|
||||
ExecuteOperation("ListBuckets", () =>
|
||||
// Enhance bucket information if requested
|
||||
if (IncludeStats.IsPresent || IncludePolicy.IsPresent)
|
||||
{
|
||||
var buckets = Client.ListBuckets();
|
||||
|
||||
if (IncludeStatistics.IsPresent)
|
||||
for (int i = 0; i < buckets.Count; i++)
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Gathering statistics for {0} buckets", buckets.Count);
|
||||
|
||||
for (int i = 0; i < buckets.Count; i++)
|
||||
var bucket = buckets[i];
|
||||
WriteVerboseMessage("Enhancing information for bucket '{0}' ({1}/{2})", bucket.Name, i + 1, buckets.Count);
|
||||
|
||||
try
|
||||
{
|
||||
var bucket = buckets[i];
|
||||
|
||||
// Show progress for statistics gathering
|
||||
var progressRecord = new ProgressRecord(1, "Gathering Bucket Statistics",
|
||||
$"Processing bucket: {bucket.Name}")
|
||||
// Get bucket statistics
|
||||
if (IncludeStats.IsPresent)
|
||||
{
|
||||
PercentComplete = (int)((double)(i + 1) / buckets.Count * 100)
|
||||
};
|
||||
WriteProgress(progressRecord);
|
||||
WriteVerboseMessage("Getting statistics for bucket '{0}'", bucket.Name);
|
||||
var objects = S3Client.ListObjects(bucket.Name, recursive: true, maxObjects: int.MaxValue);
|
||||
bucket.ObjectCount = objects.Count;
|
||||
bucket.TotalSize = objects.Sum(o => o.Size);
|
||||
WriteVerboseMessage("Bucket '{0}' contains {1} objects ({2})",
|
||||
bucket.Name, bucket.ObjectCount, Utils.SizeFormatter.FormatBytes(bucket.TotalSize ?? 0));
|
||||
}
|
||||
|
||||
PopulateBucketStatistics(bucket);
|
||||
// Get bucket policy
|
||||
if (IncludePolicy.IsPresent)
|
||||
{
|
||||
WriteVerboseMessage("Getting policy for bucket '{0}'", bucket.Name);
|
||||
try
|
||||
{
|
||||
bucket.Policy = S3Client.GetBucketPolicy(bucket.Name);
|
||||
WriteVerboseMessage("Retrieved policy for bucket '{0}'", bucket.Name);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
WriteVerboseMessage("No policy found for bucket '{0}': {1}", bucket.Name, ex.Message);
|
||||
bucket.Policy = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Complete progress
|
||||
WriteProgress(new ProgressRecord(1, "Gathering Bucket Statistics", "Completed")
|
||||
catch (Exception ex)
|
||||
{
|
||||
PercentComplete = 100,
|
||||
RecordType = ProgressRecordType.Completed
|
||||
});
|
||||
WriteWarningMessage("Failed to enhance information for bucket '{0}': {1}", bucket.Name, ex.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Output all buckets
|
||||
foreach (var bucket in buckets)
|
||||
{
|
||||
WriteObject(bucket);
|
||||
}
|
||||
// Sort buckets
|
||||
buckets = SortBy switch
|
||||
{
|
||||
"Name" => Descending.IsPresent
|
||||
? buckets.OrderByDescending(b => b.Name).ToList()
|
||||
: buckets.OrderBy(b => b.Name).ToList(),
|
||||
"CreationDate" => Descending.IsPresent
|
||||
? buckets.OrderByDescending(b => b.CreationDate).ToList()
|
||||
: buckets.OrderBy(b => b.CreationDate).ToList(),
|
||||
"ObjectCount" => Descending.IsPresent
|
||||
? buckets.OrderByDescending(b => b.ObjectCount ?? 0).ToList()
|
||||
: buckets.OrderBy(b => b.ObjectCount ?? 0).ToList(),
|
||||
"TotalSize" => Descending.IsPresent
|
||||
? buckets.OrderByDescending(b => b.TotalSize ?? 0).ToList()
|
||||
: buckets.OrderBy(b => b.TotalSize ?? 0).ToList(),
|
||||
_ => buckets
|
||||
};
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Retrieved information for {0} buckets", buckets.Count);
|
||||
});
|
||||
}
|
||||
}
|
||||
WriteVerboseMessage("Returning {0} buckets sorted by {1} ({2})",
|
||||
buckets.Count, SortBy, Descending.IsPresent ? "descending" : "ascending");
|
||||
|
||||
/// <summary>
|
||||
/// Populates additional statistics for a bucket
|
||||
/// </summary>
|
||||
/// <param name="bucket">Bucket to populate statistics for</param>
|
||||
private void PopulateBucketStatistics(MinIOBucketInfo bucket)
|
||||
{
|
||||
try
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Gathering statistics for bucket: {0}", bucket.Name);
|
||||
|
||||
var objects = Client.ListObjects(bucket.Name, recursive: true);
|
||||
|
||||
bucket.ObjectCount = objects.Count;
|
||||
bucket.Size = objects.Sum(obj => obj.Size);
|
||||
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Bucket '{0}' statistics: {1} objects, {2} bytes total",
|
||||
bucket.Name, bucket.ObjectCount, bucket.Size);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
MinIOLogger.WriteWarning(this,
|
||||
"Failed to gather statistics for bucket '{0}': {1}",
|
||||
bucket.Name, ex.Message);
|
||||
|
||||
// Set statistics to null to indicate they couldn't be retrieved
|
||||
bucket.ObjectCount = null;
|
||||
bucket.Size = null;
|
||||
}
|
||||
// Output results
|
||||
foreach (var bucket in buckets)
|
||||
{
|
||||
WriteObject(bucket);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,279 +0,0 @@
|
||||
using System;
|
||||
using System.Management.Automation;
|
||||
using System.Text.Json;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Cmdlets
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the policy for a MinIO bucket
|
||||
/// </summary>
|
||||
[Cmdlet(VerbsCommon.Get, "MinIOBucketPolicy", SupportsShouldProcess = false)]
|
||||
[OutputType(typeof(string), typeof(PSObject))]
|
||||
public class GetMinIOBucketPolicyCmdlet : MinIOBaseCmdlet
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket to get the policy for
|
||||
/// </summary>
|
||||
[Parameter(Position = 0, Mandatory = true, ValueFromPipeline = true, ValueFromPipelineByPropertyName = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Bucket")]
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Return the policy as a formatted object instead of raw JSON
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter AsObject { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Pretty-print the JSON output
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter PrettyPrint { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Processes the cmdlet
|
||||
/// </summary>
|
||||
protected override void ProcessRecord()
|
||||
{
|
||||
ValidateConnection();
|
||||
ValidateBucketName(BucketName);
|
||||
|
||||
ExecuteOperation("GetBucketPolicy", () =>
|
||||
{
|
||||
// Check if bucket exists
|
||||
var bucketExists = Client.BucketExists(BucketName);
|
||||
if (!bucketExists)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
|
||||
"BucketNotFound",
|
||||
ErrorCategory.ObjectNotFound,
|
||||
BucketName));
|
||||
return;
|
||||
}
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Retrieving policy for bucket '{0}'", BucketName);
|
||||
|
||||
try
|
||||
{
|
||||
var policyJson = Client.GetBucketPolicy(BucketName);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(policyJson))
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "No policy is set for bucket '{0}'", BucketName);
|
||||
|
||||
if (AsObject.IsPresent)
|
||||
{
|
||||
var emptyPolicy = new PSObject();
|
||||
emptyPolicy.Properties.Add(new PSNoteProperty("BucketName", BucketName));
|
||||
emptyPolicy.Properties.Add(new PSNoteProperty("HasPolicy", false));
|
||||
emptyPolicy.Properties.Add(new PSNoteProperty("Policy", null));
|
||||
emptyPolicy.Properties.Add(new PSNoteProperty("RetrievedAt", DateTime.UtcNow));
|
||||
WriteObject(emptyPolicy);
|
||||
}
|
||||
else
|
||||
{
|
||||
WriteObject(string.Empty);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Retrieved policy for bucket '{0}' ({1} characters)",
|
||||
BucketName, policyJson.Length);
|
||||
|
||||
if (AsObject.IsPresent)
|
||||
{
|
||||
// Parse and return as structured object
|
||||
var policyObject = CreatePolicyObject(policyJson);
|
||||
WriteObject(policyObject);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Return as JSON string
|
||||
if (PrettyPrint.IsPresent)
|
||||
{
|
||||
var formattedJson = FormatJson(policyJson);
|
||||
WriteObject(formattedJson);
|
||||
}
|
||||
else
|
||||
{
|
||||
WriteObject(policyJson);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex.Message.Contains("NoSuchBucketPolicy") ||
|
||||
ex.Message.Contains("policy does not exist"))
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "No policy is set for bucket '{0}'", BucketName);
|
||||
|
||||
if (AsObject.IsPresent)
|
||||
{
|
||||
var emptyPolicy = new PSObject();
|
||||
emptyPolicy.Properties.Add(new PSNoteProperty("BucketName", BucketName));
|
||||
emptyPolicy.Properties.Add(new PSNoteProperty("HasPolicy", false));
|
||||
emptyPolicy.Properties.Add(new PSNoteProperty("Policy", null));
|
||||
emptyPolicy.Properties.Add(new PSNoteProperty("RetrievedAt", DateTime.UtcNow));
|
||||
WriteObject(emptyPolicy);
|
||||
}
|
||||
else
|
||||
{
|
||||
WriteObject(string.Empty);
|
||||
}
|
||||
}
|
||||
|
||||
}, $"Bucket: {BucketName}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a structured policy object from JSON
|
||||
/// </summary>
|
||||
/// <param name="policyJson">Policy JSON string</param>
|
||||
/// <returns>PSObject containing policy information</returns>
|
||||
private PSObject CreatePolicyObject(string policyJson)
|
||||
{
|
||||
var policyObject = new PSObject();
|
||||
policyObject.Properties.Add(new PSNoteProperty("BucketName", BucketName));
|
||||
policyObject.Properties.Add(new PSNoteProperty("HasPolicy", true));
|
||||
policyObject.Properties.Add(new PSNoteProperty("RetrievedAt", DateTime.UtcNow));
|
||||
|
||||
try
|
||||
{
|
||||
// Parse the JSON to extract key information
|
||||
using var document = JsonDocument.Parse(policyJson);
|
||||
var root = document.RootElement;
|
||||
|
||||
// Add raw policy
|
||||
policyObject.Properties.Add(new PSNoteProperty("PolicyJson", policyJson));
|
||||
|
||||
// Extract version if present
|
||||
if (root.TryGetProperty("Version", out var versionElement))
|
||||
{
|
||||
policyObject.Properties.Add(new PSNoteProperty("Version", versionElement.GetString()));
|
||||
}
|
||||
|
||||
// Extract statements if present
|
||||
if (root.TryGetProperty("Statement", out var statementsElement))
|
||||
{
|
||||
var statements = new System.Collections.Generic.List<PSObject>();
|
||||
|
||||
if (statementsElement.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
foreach (var statement in statementsElement.EnumerateArray())
|
||||
{
|
||||
var statementObj = CreateStatementObject(statement);
|
||||
statements.Add(statementObj);
|
||||
}
|
||||
}
|
||||
else if (statementsElement.ValueKind == JsonValueKind.Object)
|
||||
{
|
||||
var statementObj = CreateStatementObject(statementsElement);
|
||||
statements.Add(statementObj);
|
||||
}
|
||||
|
||||
policyObject.Properties.Add(new PSNoteProperty("Statements", statements.ToArray()));
|
||||
policyObject.Properties.Add(new PSNoteProperty("StatementCount", statements.Count));
|
||||
}
|
||||
|
||||
// Add formatted JSON
|
||||
var formattedJson = FormatJson(policyJson);
|
||||
policyObject.Properties.Add(new PSNoteProperty("FormattedPolicy", formattedJson));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
MinIOLogger.WriteWarning(this, "Could not parse policy JSON: {0}", ex.Message);
|
||||
policyObject.Properties.Add(new PSNoteProperty("PolicyJson", policyJson));
|
||||
policyObject.Properties.Add(new PSNoteProperty("ParseError", ex.Message));
|
||||
}
|
||||
|
||||
return policyObject;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a statement object from a JSON element
|
||||
/// </summary>
|
||||
/// <param name="statementElement">JSON element representing a statement</param>
|
||||
/// <returns>PSObject containing statement information</returns>
|
||||
private PSObject CreateStatementObject(JsonElement statementElement)
|
||||
{
|
||||
var statementObj = new PSObject();
|
||||
|
||||
try
|
||||
{
|
||||
if (statementElement.TryGetProperty("Effect", out var effectElement))
|
||||
{
|
||||
statementObj.Properties.Add(new PSNoteProperty("Effect", effectElement.GetString()));
|
||||
}
|
||||
|
||||
if (statementElement.TryGetProperty("Principal", out var principalElement))
|
||||
{
|
||||
statementObj.Properties.Add(new PSNoteProperty("Principal", principalElement.ToString()));
|
||||
}
|
||||
|
||||
if (statementElement.TryGetProperty("Action", out var actionElement))
|
||||
{
|
||||
if (actionElement.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
var actions = new System.Collections.Generic.List<string>();
|
||||
foreach (var action in actionElement.EnumerateArray())
|
||||
{
|
||||
actions.Add(action.GetString() ?? string.Empty);
|
||||
}
|
||||
statementObj.Properties.Add(new PSNoteProperty("Actions", actions.ToArray()));
|
||||
}
|
||||
else
|
||||
{
|
||||
statementObj.Properties.Add(new PSNoteProperty("Actions", new[] { actionElement.GetString() ?? string.Empty }));
|
||||
}
|
||||
}
|
||||
|
||||
if (statementElement.TryGetProperty("Resource", out var resourceElement))
|
||||
{
|
||||
if (resourceElement.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
var resources = new System.Collections.Generic.List<string>();
|
||||
foreach (var resource in resourceElement.EnumerateArray())
|
||||
{
|
||||
resources.Add(resource.GetString() ?? string.Empty);
|
||||
}
|
||||
statementObj.Properties.Add(new PSNoteProperty("Resources", resources.ToArray()));
|
||||
}
|
||||
else
|
||||
{
|
||||
statementObj.Properties.Add(new PSNoteProperty("Resources", new[] { resourceElement.GetString() ?? string.Empty }));
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
MinIOLogger.WriteWarning(this, "Could not parse statement: {0}", ex.Message);
|
||||
statementObj.Properties.Add(new PSNoteProperty("ParseError", ex.Message));
|
||||
}
|
||||
|
||||
return statementObj;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Formats JSON string for pretty printing
|
||||
/// </summary>
|
||||
/// <param name="json">JSON string to format</param>
|
||||
/// <returns>Formatted JSON string</returns>
|
||||
private string FormatJson(string json)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var document = JsonDocument.Parse(json);
|
||||
return JsonSerializer.Serialize(document, new JsonSerializerOptions
|
||||
{
|
||||
WriteIndented = true
|
||||
});
|
||||
}
|
||||
catch
|
||||
{
|
||||
// If formatting fails, return original
|
||||
return json;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,175 +1,152 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Management.Automation;
|
||||
using PSMinIO.Models;
|
||||
using PSMinIO.Utils;
|
||||
using PSMinIO.Core.Models;
|
||||
|
||||
namespace PSMinIO.Cmdlets
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets information about objects in a MinIO bucket
|
||||
/// Gets objects from MinIO buckets
|
||||
/// </summary>
|
||||
[Cmdlet(VerbsCommon.Get, "MinIOObject", SupportsShouldProcess = true)]
|
||||
[Cmdlet(VerbsCommon.Get, "MinIOObject")]
|
||||
[OutputType(typeof(MinIOObjectInfo))]
|
||||
public class GetMinIOObjectCmdlet : MinIOBaseCmdlet
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket to list objects from
|
||||
/// Name of the bucket
|
||||
/// </summary>
|
||||
[Parameter(Mandatory = true, Position = 0, ValueFromPipelineByPropertyName = true)]
|
||||
[Parameter(Position = 0, Mandatory = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Bucket")]
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Optional prefix to filter objects
|
||||
/// Object name or prefix to filter objects (supports wildcards)
|
||||
/// </summary>
|
||||
[Parameter(Position = 1, ValueFromPipelineByPropertyName = true)]
|
||||
[Alias("Filter")]
|
||||
public string? Prefix { get; set; }
|
||||
[Parameter(Position = 1, ValueFromPipeline = true, ValueFromPipelineByPropertyName = true)]
|
||||
[SupportsWildcards]
|
||||
[Alias("Key", "Prefix")]
|
||||
public string? Name { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Specific object name to retrieve. If specified, only this object is returned.
|
||||
/// </summary>
|
||||
[Parameter(ValueFromPipelineByPropertyName = true)]
|
||||
[Alias("Object", "Name")]
|
||||
public string? ObjectName { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether to list objects recursively (default: true)
|
||||
/// List objects recursively (default: true)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter Recursive { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Include all versions of objects (for versioned buckets)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[Alias("Versions")]
|
||||
public SwitchParameter IncludeVersions { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of objects to return
|
||||
/// Maximum number of objects to return (default: 1000)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateRange(1, 10000)]
|
||||
[Alias("Limit")]
|
||||
public int? MaxObjects { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Only return objects (exclude directory markers)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[Alias("FilesOnly")]
|
||||
public SwitchParameter ObjectsOnly { get; set; }
|
||||
public int MaxObjects { get; set; } = 1000;
|
||||
|
||||
/// <summary>
|
||||
/// Sort objects by the specified property
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateSet("Name", "Size", "LastModified", "ETag")]
|
||||
public string? SortBy { get; set; }
|
||||
[ValidateSet("Name", "Size", "LastModified")]
|
||||
public string SortBy { get; set; } = "Name";
|
||||
|
||||
/// <summary>
|
||||
/// Sort in descending order (default: ascending)
|
||||
/// Sort in descending order
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[Alias("Desc")]
|
||||
public SwitchParameter Descending { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Include only directories (objects ending with /)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter DirectoriesOnly { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Exclude directories (objects ending with /)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter ExcludeDirectories { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Processes the cmdlet
|
||||
/// </summary>
|
||||
protected override void ProcessRecord()
|
||||
{
|
||||
var targetDescription = !string.IsNullOrWhiteSpace(ObjectName)
|
||||
? $"object '{ObjectName}'"
|
||||
: !string.IsNullOrWhiteSpace(Prefix)
|
||||
? $"objects with prefix '{Prefix}'"
|
||||
: "all objects";
|
||||
ValidateBucketName(BucketName);
|
||||
|
||||
// Determine the prefix to use
|
||||
var searchPrefix = !string.IsNullOrWhiteSpace(ObjectName) ? ObjectName : Prefix;
|
||||
|
||||
if (ShouldProcess(BucketName, $"List {targetDescription}"))
|
||||
ExecuteOperation("ListObjects", () =>
|
||||
{
|
||||
ExecuteOperation("ListObjects", () =>
|
||||
WriteVerboseMessage("Listing objects in bucket '{0}'", BucketName);
|
||||
|
||||
// Check if bucket exists
|
||||
if (!S3Client.BucketExists(BucketName))
|
||||
{
|
||||
// Check if bucket exists
|
||||
var bucketExists = Client.BucketExists(BucketName);
|
||||
if (!bucketExists)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
|
||||
"BucketNotFound",
|
||||
ErrorCategory.ObjectNotFound,
|
||||
BucketName));
|
||||
return;
|
||||
}
|
||||
var errorRecord = new ErrorRecord(
|
||||
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
|
||||
"BucketNotFound",
|
||||
ErrorCategory.ObjectNotFound,
|
||||
BucketName);
|
||||
ThrowTerminatingError(errorRecord);
|
||||
}
|
||||
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Listing objects in bucket '{0}' with prefix '{1}', recursive: {2}",
|
||||
BucketName, searchPrefix ?? "(none)", Recursive.IsPresent);
|
||||
// Determine prefix for API call
|
||||
string? apiPrefix = null;
|
||||
if (!string.IsNullOrEmpty(Name) && !WildcardPattern.ContainsWildcardCharacters(Name))
|
||||
{
|
||||
// If no wildcards, use as prefix for efficient API filtering
|
||||
apiPrefix = Name;
|
||||
}
|
||||
|
||||
// Get objects from MinIO
|
||||
var objects = Client.ListObjects(BucketName, searchPrefix, Recursive.IsPresent, IncludeVersions.IsPresent);
|
||||
WriteVerboseMessage("Retrieving objects with prefix '{0}', recursive: {1}, max: {2}",
|
||||
apiPrefix ?? "(none)", Recursive.IsPresent, MaxObjects);
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Found {0} objects", objects.Count);
|
||||
// Get objects from MinIO
|
||||
var objects = S3Client.ListObjects(BucketName, apiPrefix, Recursive.IsPresent, MaxObjects);
|
||||
|
||||
// Filter for exact object name match if specified
|
||||
if (!string.IsNullOrWhiteSpace(ObjectName))
|
||||
{
|
||||
objects = objects.Where(obj =>
|
||||
string.Equals(obj.Name, ObjectName, StringComparison.Ordinal)).ToList();
|
||||
}
|
||||
WriteVerboseMessage("Retrieved {0} objects from MinIO", objects.Count);
|
||||
|
||||
// Filter out directory markers if ObjectsOnly is specified
|
||||
if (ObjectsOnly.IsPresent)
|
||||
{
|
||||
objects = objects.Where(obj =>
|
||||
!obj.Name.EndsWith("/") && obj.Size > 0).ToList();
|
||||
}
|
||||
// Apply client-side filtering if wildcards are used
|
||||
if (!string.IsNullOrEmpty(Name) && WildcardPattern.ContainsWildcardCharacters(Name))
|
||||
{
|
||||
var wildcardPattern = new WildcardPattern(Name, WildcardOptions.IgnoreCase);
|
||||
objects = objects.Where(o => o.Name != null && wildcardPattern.IsMatch(o.Name)).ToList();
|
||||
WriteVerboseMessage("Filtered to {0} objects matching pattern '{1}'", objects.Count, Name);
|
||||
}
|
||||
|
||||
// Apply sorting if specified
|
||||
if (!string.IsNullOrWhiteSpace(SortBy))
|
||||
{
|
||||
objects = SortBy.ToLowerInvariant() switch
|
||||
{
|
||||
"name" => Descending.IsPresent
|
||||
? objects.OrderByDescending(o => o.Name).ToList()
|
||||
: objects.OrderBy(o => o.Name).ToList(),
|
||||
"size" => Descending.IsPresent
|
||||
? objects.OrderByDescending(o => o.Size).ToList()
|
||||
: objects.OrderBy(o => o.Size).ToList(),
|
||||
"lastmodified" => Descending.IsPresent
|
||||
? objects.OrderByDescending(o => o.LastModified).ToList()
|
||||
: objects.OrderBy(o => o.LastModified).ToList(),
|
||||
"etag" => Descending.IsPresent
|
||||
? objects.OrderByDescending(o => o.ETag).ToList()
|
||||
: objects.OrderBy(o => o.ETag).ToList(),
|
||||
_ => objects
|
||||
};
|
||||
}
|
||||
// Apply directory filters
|
||||
if (DirectoriesOnly.IsPresent)
|
||||
{
|
||||
objects = objects.Where(o => o.IsDirectory).ToList();
|
||||
WriteVerboseMessage("Filtered to {0} directories only", objects.Count);
|
||||
}
|
||||
else if (ExcludeDirectories.IsPresent)
|
||||
{
|
||||
objects = objects.Where(o => !o.IsDirectory).ToList();
|
||||
WriteVerboseMessage("Filtered to {0} objects excluding directories", objects.Count);
|
||||
}
|
||||
|
||||
// Apply limit if specified
|
||||
if (MaxObjects.HasValue && objects.Count > MaxObjects.Value)
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Limiting results to {0} objects", MaxObjects.Value);
|
||||
objects = objects.Take(MaxObjects.Value).ToList();
|
||||
}
|
||||
// Sort objects
|
||||
objects = SortBy switch
|
||||
{
|
||||
"Name" => Descending.IsPresent
|
||||
? objects.OrderByDescending(o => o.Name).ToList()
|
||||
: objects.OrderBy(o => o.Name).ToList(),
|
||||
"Size" => Descending.IsPresent
|
||||
? objects.OrderByDescending(o => o.Size).ToList()
|
||||
: objects.OrderBy(o => o.Size).ToList(),
|
||||
"LastModified" => Descending.IsPresent
|
||||
? objects.OrderByDescending(o => o.LastModified).ToList()
|
||||
: objects.OrderBy(o => o.LastModified).ToList(),
|
||||
_ => objects
|
||||
};
|
||||
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Returning {0} objects after filtering and sorting", objects.Count);
|
||||
WriteVerboseMessage("Returning {0} objects sorted by {1} ({2})",
|
||||
objects.Count, SortBy, Descending.IsPresent ? "descending" : "ascending");
|
||||
|
||||
// Output objects
|
||||
foreach (var obj in objects)
|
||||
{
|
||||
WriteObject(obj);
|
||||
}
|
||||
|
||||
}, $"Bucket: {BucketName}, Prefix: {searchPrefix ?? "(none)"}");
|
||||
}
|
||||
// Output results
|
||||
foreach (var obj in objects)
|
||||
{
|
||||
WriteObject(obj);
|
||||
}
|
||||
}, $"Bucket: {BucketName}, Name: {Name ?? "(all)"}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,348 +0,0 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Management.Automation;
|
||||
using PSMinIO.Models;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Cmdlets
|
||||
{
|
||||
/// <summary>
|
||||
/// Downloads objects from a MinIO bucket using chunked transfer with resume capability
|
||||
/// </summary>
|
||||
[Cmdlet(VerbsCommon.Get, "MinIOObjectContentChunked", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Low)]
|
||||
[OutputType(typeof(FileInfo))]
|
||||
public class GetMinIOObjectContentChunkedCmdlet : MinIOBaseCmdlet
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket to download from
|
||||
/// </summary>
|
||||
[Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Bucket")]
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Name of the object to download
|
||||
/// </summary>
|
||||
[Parameter(Position = 1, Mandatory = true, ValueFromPipelineByPropertyName = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Object", "Key")]
|
||||
public string ObjectName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// FileInfo object representing where the file should be saved
|
||||
/// </summary>
|
||||
[Parameter(Position = 2, Mandatory = true)]
|
||||
[ValidateNotNull]
|
||||
[Alias("File", "Path")]
|
||||
public FileInfo FilePath { get; set; } = null!;
|
||||
|
||||
/// <summary>
|
||||
/// Overwrite existing files without prompting
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter Force { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Size of each chunk for download (default: 10MB)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateRange(5 * 1024 * 1024, 500 * 1024 * 1024)] // 5MB to 500MB
|
||||
public long ChunkSize { get; set; } = 10 * 1024 * 1024; // 10MB default
|
||||
|
||||
/// <summary>
|
||||
/// Enable resume functionality for interrupted downloads
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter Resume { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of retry attempts for failed chunks
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateRange(1, 10)]
|
||||
public int MaxRetries { get; set; } = 3;
|
||||
|
||||
/// <summary>
|
||||
/// Number of parallel chunk downloads (default: 3)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateRange(1, 10)]
|
||||
public int ParallelDownloads { get; set; } = 3;
|
||||
|
||||
/// <summary>
|
||||
/// Custom path for storing resume data (default: %LOCALAPPDATA%\PSMinIO\Resume)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public string? ResumeDataPath { get; set; }
|
||||
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Update progress every N bytes transferred (default: 1MB)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateRange(1024, 10 * 1024 * 1024)] // 1KB to 10MB
|
||||
public long ProgressUpdateInterval { get; set; } = 1024 * 1024; // 1MB
|
||||
|
||||
/// <summary>
|
||||
/// Processes the cmdlet
|
||||
/// </summary>
|
||||
protected override void ProcessRecord()
|
||||
{
|
||||
ValidateConnection();
|
||||
ValidateBucketName(BucketName);
|
||||
ValidateObjectName(ObjectName);
|
||||
ValidateAndPrepareFilePath();
|
||||
|
||||
if (ShouldProcess($"{BucketName}/{ObjectName}", $"Download using chunked transfer to '{FilePath.FullName}'"))
|
||||
{
|
||||
ExecuteOperation("DownloadObjectChunked", () =>
|
||||
{
|
||||
// Check if bucket exists
|
||||
var bucketExists = Client.BucketExists(BucketName);
|
||||
if (!bucketExists)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
|
||||
"BucketNotFound",
|
||||
ErrorCategory.ObjectNotFound,
|
||||
BucketName));
|
||||
return;
|
||||
}
|
||||
|
||||
// Get object information
|
||||
var objectInfo = GetObjectInfo();
|
||||
if (objectInfo == null)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Object '{ObjectName}' not found in bucket '{BucketName}'"),
|
||||
"ObjectNotFound",
|
||||
ErrorCategory.ObjectNotFound,
|
||||
ObjectName));
|
||||
return;
|
||||
}
|
||||
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Starting chunked download of object '{0}' from bucket '{1}' (Size: {2}, ChunkSize: {3})",
|
||||
ObjectName, BucketName, SizeFormatter.FormatBytes(objectInfo.Size), SizeFormatter.FormatBytes(ChunkSize));
|
||||
|
||||
// Download using chunked transfer
|
||||
var downloadResult = DownloadObjectChunked(objectInfo);
|
||||
|
||||
if (downloadResult != null)
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Successfully downloaded object '{0}' from bucket '{1}' to '{2}'",
|
||||
ObjectName, BucketName, FilePath.FullName);
|
||||
|
||||
// Return download result with timing information
|
||||
WriteObject(downloadResult);
|
||||
}
|
||||
|
||||
}, $"Bucket: {BucketName}, Object: {ObjectName}, File: {FilePath.FullName}, ChunkSize: {SizeFormatter.FormatBytes(ChunkSize)}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets information about the object to download
|
||||
/// </summary>
|
||||
/// <returns>Object information or null if not found</returns>
|
||||
private MinIOObjectInfo? GetObjectInfo()
|
||||
{
|
||||
try
|
||||
{
|
||||
var objects = Client.ListObjects(BucketName, ObjectName, false);
|
||||
return objects.Find(obj => string.Equals(obj.Name, ObjectName, StringComparison.Ordinal));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
WriteWarning($"Could not get object information: {ex.Message}");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Downloads an object using chunked transfer with resume capability
|
||||
/// </summary>
|
||||
/// <param name="objectInfo">Information about the object to download</param>
|
||||
/// <returns>Download result with timing information or null if failed</returns>
|
||||
private MinIODownloadResult? DownloadObjectChunked(MinIOObjectInfo objectInfo)
|
||||
{
|
||||
ChunkedTransferState? transferState = null;
|
||||
|
||||
// Try to load existing transfer state for resume
|
||||
if (Resume.IsPresent)
|
||||
{
|
||||
transferState = ChunkedTransferResumeManager.LoadTransferState(
|
||||
BucketName, ObjectName, FilePath.FullName, ChunkedTransferType.Download, ResumeDataPath);
|
||||
|
||||
if (transferState != null && !ChunkedTransferResumeManager.IsResumeDataValid(transferState))
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Resume data for {0} is invalid, starting fresh download", ObjectName);
|
||||
transferState = null;
|
||||
}
|
||||
}
|
||||
|
||||
// Create new transfer state if none exists or resume is disabled
|
||||
if (transferState == null)
|
||||
{
|
||||
transferState = new ChunkedTransferState
|
||||
{
|
||||
BucketName = BucketName,
|
||||
ObjectName = ObjectName,
|
||||
FilePath = FilePath.FullName,
|
||||
TotalSize = objectInfo.Size,
|
||||
ChunkSize = ChunkSize,
|
||||
ETag = objectInfo.ETag,
|
||||
TransferType = ChunkedTransferType.Download,
|
||||
StartTime = DateTime.UtcNow,
|
||||
LastUpdated = DateTime.UtcNow
|
||||
};
|
||||
}
|
||||
else
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Resuming download of {0} from {1:P1} complete",
|
||||
ObjectName, transferState.ProgressPercentage / 100);
|
||||
}
|
||||
|
||||
// Calculate total chunks for progress reporting
|
||||
var totalChunks = (int)Math.Ceiling((double)objectInfo.Size / ChunkSize);
|
||||
|
||||
// Validate chunk count to prevent memory issues
|
||||
if (totalChunks > 5000)
|
||||
{
|
||||
var recommendedChunkSize = (long)Math.Ceiling((double)objectInfo.Size / 5000);
|
||||
WriteWarning($"Too many chunks ({totalChunks:N0}) would be created. Consider using a larger chunk size (recommended: {SizeFormatter.FormatBytes(recommendedChunkSize)})");
|
||||
ThrowTerminatingError(new ErrorRecord(
|
||||
new ArgumentException($"Chunk size too small would create {totalChunks:N0} chunks. Maximum allowed is 5,000 chunks."),
|
||||
"TooManyChunks",
|
||||
ErrorCategory.InvalidArgument,
|
||||
ChunkSize));
|
||||
}
|
||||
|
||||
// Create thread-safe progress reporter
|
||||
var progressReporter = new ThreadSafeChunkedProgressReporter(
|
||||
this,
|
||||
objectInfo.Size,
|
||||
totalChunks,
|
||||
"Downloading");
|
||||
|
||||
progressReporter.StartNewFile(ObjectName, objectInfo.Size, totalChunks);
|
||||
|
||||
// Track timing for this download
|
||||
var startTime = DateTime.UtcNow;
|
||||
|
||||
try
|
||||
{
|
||||
// Download file using chunked transfer
|
||||
var result = Client.DownloadFileChunked(transferState, progressReporter, MaxRetries, ParallelDownloads);
|
||||
|
||||
if (result)
|
||||
{
|
||||
var completionTime = DateTime.UtcNow;
|
||||
|
||||
// Clean up resume data on successful completion
|
||||
if (Resume.IsPresent)
|
||||
{
|
||||
ChunkedTransferResumeManager.CleanupResumeData(transferState, ResumeDataPath);
|
||||
}
|
||||
|
||||
progressReporter.CompleteFile();
|
||||
|
||||
// Process any queued progress updates from the main thread
|
||||
progressReporter.ProcessQueuedUpdates();
|
||||
progressReporter.Complete();
|
||||
|
||||
// Create download result with timing information
|
||||
FilePath.Refresh();
|
||||
var downloadResult = new MinIODownloadResult(FilePath, BucketName, ObjectName, startTime, completionTime);
|
||||
return downloadResult;
|
||||
}
|
||||
}
|
||||
#pragma warning disable CS0168 // Variable is declared but never used - false positive, ex is used in error handling
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Save resume data on failure if resume is enabled
|
||||
if (Resume.IsPresent)
|
||||
{
|
||||
try
|
||||
{
|
||||
ChunkedTransferResumeManager.SaveTransferState(transferState, ResumeDataPath);
|
||||
MinIOLogger.WriteVerbose(this, "Saved resume data for {0}", ObjectName);
|
||||
}
|
||||
catch (Exception saveEx)
|
||||
{
|
||||
WriteWarning($"Could not save resume data for {ObjectName}: {saveEx.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
throw;
|
||||
}
|
||||
#pragma warning restore CS0168
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates and prepares the file path for download
|
||||
/// </summary>
|
||||
private void ValidateAndPrepareFilePath()
|
||||
{
|
||||
if (FilePath == null)
|
||||
{
|
||||
ThrowTerminatingError(new ErrorRecord(
|
||||
new ArgumentException("FilePath cannot be null"),
|
||||
"InvalidFilePath",
|
||||
ErrorCategory.InvalidArgument,
|
||||
FilePath));
|
||||
}
|
||||
|
||||
// Check if file already exists
|
||||
if (FilePath!.Exists && !Force.IsPresent)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"File '{FilePath.FullName}' already exists. Use -Force to overwrite."),
|
||||
"FileAlreadyExists",
|
||||
ErrorCategory.ResourceExists,
|
||||
FilePath));
|
||||
return;
|
||||
}
|
||||
|
||||
// Ensure the directory exists
|
||||
var directory = FilePath.Directory;
|
||||
if (directory != null && !directory.Exists)
|
||||
{
|
||||
try
|
||||
{
|
||||
directory.Create();
|
||||
MinIOLogger.WriteVerbose(this, "Created directory: {0}", directory.FullName);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ThrowTerminatingError(new ErrorRecord(
|
||||
new InvalidOperationException($"Cannot create directory '{directory.FullName}': {ex.Message}", ex),
|
||||
"DirectoryCreationFailed",
|
||||
ErrorCategory.WriteError,
|
||||
directory));
|
||||
}
|
||||
}
|
||||
|
||||
// Check if we can write to the target location
|
||||
try
|
||||
{
|
||||
var testFile = Path.Combine(FilePath.DirectoryName ?? ".", $".psminiotest_{Guid.NewGuid():N}");
|
||||
File.WriteAllText(testFile, "test");
|
||||
File.Delete(testFile);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ThrowTerminatingError(new ErrorRecord(
|
||||
new InvalidOperationException($"Cannot write to location '{FilePath.FullName}': {ex.Message}", ex),
|
||||
"LocationNotWritable",
|
||||
ErrorCategory.PermissionDenied,
|
||||
FilePath));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,220 +1,205 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Management.Automation;
|
||||
using PSMinIO.Models;
|
||||
using PSMinIO.Core.Models;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Cmdlets
|
||||
{
|
||||
/// <summary>
|
||||
/// Downloads an object from a MinIO bucket to a local file
|
||||
/// Downloads objects from MinIO
|
||||
/// </summary>
|
||||
[Cmdlet(VerbsCommon.Get, "MinIOObjectContent", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Medium)]
|
||||
[OutputType(typeof(FileInfo))]
|
||||
[OutputType(typeof(MinIODownloadResult))]
|
||||
public class GetMinIOObjectContentCmdlet : MinIOBaseCmdlet
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket containing the object
|
||||
/// Name of the bucket
|
||||
/// </summary>
|
||||
[Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true)]
|
||||
[Parameter(Position = 0, Mandatory = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Bucket")]
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Name of the object to download
|
||||
/// </summary>
|
||||
[Parameter(Position = 1, Mandatory = true, ValueFromPipelineByPropertyName = true)]
|
||||
[Parameter(Position = 1, Mandatory = true, ValueFromPipeline = true, ValueFromPipelineByPropertyName = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Object", "Key")]
|
||||
[Alias("Key", "Name")]
|
||||
public string ObjectName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// FileInfo object representing where the file should be saved
|
||||
/// Local file path where the object should be saved
|
||||
/// </summary>
|
||||
[Parameter(Position = 2, Mandatory = true)]
|
||||
[ValidateNotNull]
|
||||
[Alias("File", "Path")]
|
||||
public FileInfo FilePath { get; set; } = null!;
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Path", "FilePath")]
|
||||
public string LocalPath { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Overwrite the file if it already exists
|
||||
/// Force overwrite if local file already exists
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter Force { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Create directory structure if it doesn't exist
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter CreateDirectories { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Return the download result information
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter PassThru { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Processes the cmdlet
|
||||
/// </summary>
|
||||
protected override void ProcessRecord()
|
||||
{
|
||||
ValidateConnection();
|
||||
ValidateBucketName(BucketName);
|
||||
ValidateObjectName(ObjectName);
|
||||
ValidateAndPrepareFilePath();
|
||||
|
||||
if (ShouldProcess($"{BucketName}/{ObjectName}", $"Download to '{FilePath}'"))
|
||||
{
|
||||
ExecuteOperation("DownloadObject", () =>
|
||||
{
|
||||
// Check if bucket exists
|
||||
var bucketExists = Client.BucketExists(BucketName);
|
||||
if (!bucketExists)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
|
||||
"BucketNotFound",
|
||||
ErrorCategory.ObjectNotFound,
|
||||
BucketName));
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if object exists and get its information
|
||||
var objectInfo = GetObjectInfo();
|
||||
if (objectInfo == null)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Object '{ObjectName}' does not exist in bucket '{BucketName}'"),
|
||||
"ObjectNotFound",
|
||||
ErrorCategory.ObjectNotFound,
|
||||
ObjectName));
|
||||
return;
|
||||
}
|
||||
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Downloading object '{0}' from bucket '{1}' ({2} bytes) to '{3}'",
|
||||
ObjectName, BucketName, objectInfo.Size, FilePath);
|
||||
|
||||
// Create progress reporter
|
||||
var progressReporter = new ProgressReporter(
|
||||
this,
|
||||
"Downloading Object",
|
||||
$"Downloading {objectInfo.GetFileName()}",
|
||||
objectInfo.Size,
|
||||
1);
|
||||
|
||||
try
|
||||
{
|
||||
// Track timing for this download
|
||||
var startTime = DateTime.UtcNow;
|
||||
|
||||
// Download the file with progress reporting
|
||||
Client.DownloadFile(
|
||||
BucketName,
|
||||
ObjectName,
|
||||
FilePath.FullName,
|
||||
bytesTransferred => progressReporter.UpdateProgress(bytesTransferred));
|
||||
|
||||
var completionTime = DateTime.UtcNow;
|
||||
|
||||
// Complete progress reporting
|
||||
progressReporter.Complete();
|
||||
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Successfully downloaded object '{0}' from bucket '{1}' to '{2}'",
|
||||
ObjectName, BucketName, FilePath.FullName);
|
||||
|
||||
// Refresh file info and create download result with timing information
|
||||
FilePath.Refresh();
|
||||
var downloadResult = new MinIODownloadResult(FilePath, BucketName, ObjectName, startTime, completionTime);
|
||||
WriteObject(downloadResult);
|
||||
}
|
||||
#pragma warning disable CS0168 // Variable is declared but never used - false positive, ex is used in throw
|
||||
catch (Exception ex)
|
||||
{
|
||||
progressReporter.Complete();
|
||||
throw;
|
||||
}
|
||||
#pragma warning restore CS0168
|
||||
|
||||
}, $"Bucket: {BucketName}, Object: {ObjectName}, File: {FilePath.FullName}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates and prepares the file path for download
|
||||
/// </summary>
|
||||
private void ValidateAndPrepareFilePath()
|
||||
{
|
||||
if (FilePath == null)
|
||||
// Validate local path
|
||||
if (string.IsNullOrWhiteSpace(LocalPath))
|
||||
{
|
||||
ThrowTerminatingError(new ErrorRecord(
|
||||
new ArgumentException("FilePath cannot be null"),
|
||||
"InvalidFilePath",
|
||||
new ArgumentException("Local path cannot be null or empty"),
|
||||
"InvalidLocalPath",
|
||||
ErrorCategory.InvalidArgument,
|
||||
FilePath));
|
||||
LocalPath));
|
||||
}
|
||||
|
||||
// Check if file already exists
|
||||
if (FilePath!.Exists && !Force.IsPresent)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"File '{FilePath.FullName}' already exists. Use -Force to overwrite."),
|
||||
"FileAlreadyExists",
|
||||
ErrorCategory.ResourceExists,
|
||||
FilePath));
|
||||
return;
|
||||
}
|
||||
|
||||
// Ensure the directory exists
|
||||
var directory = FilePath.Directory;
|
||||
if (directory != null && !directory.Exists)
|
||||
{
|
||||
try
|
||||
{
|
||||
directory.Create();
|
||||
MinIOLogger.WriteVerbose(this, "Created directory: {0}", directory.FullName);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ThrowTerminatingError(new ErrorRecord(
|
||||
new InvalidOperationException($"Cannot create directory '{directory.FullName}': {ex.Message}", ex),
|
||||
"DirectoryCreationFailed",
|
||||
ErrorCategory.WriteError,
|
||||
directory));
|
||||
}
|
||||
}
|
||||
|
||||
// Check if we can write to the target location
|
||||
try
|
||||
{
|
||||
var testFile = Path.Combine(FilePath.DirectoryName ?? ".", $".psminiotest_{Guid.NewGuid():N}");
|
||||
using (var testStream = File.Create(testFile))
|
||||
{
|
||||
testStream.WriteByte(0);
|
||||
}
|
||||
File.Delete(testFile);
|
||||
}
|
||||
catch (Exception ex)
|
||||
if (File.Exists(LocalPath) && !Force.IsPresent)
|
||||
{
|
||||
ThrowTerminatingError(new ErrorRecord(
|
||||
new InvalidOperationException($"Cannot write to location '{FilePath.FullName}': {ex.Message}", ex),
|
||||
"LocationNotWritable",
|
||||
ErrorCategory.PermissionDenied,
|
||||
FilePath));
|
||||
new InvalidOperationException($"File already exists: {LocalPath}. Use -Force to overwrite."),
|
||||
"FileAlreadyExists",
|
||||
ErrorCategory.ResourceExists,
|
||||
LocalPath));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets information about the object to be downloaded
|
||||
/// </summary>
|
||||
/// <returns>Object information or null if not found</returns>
|
||||
private Models.MinIOObjectInfo? GetObjectInfo()
|
||||
{
|
||||
try
|
||||
if (ShouldProcess($"{BucketName}/{ObjectName}", "Download object"))
|
||||
{
|
||||
var objects = Client.ListObjects(BucketName, ObjectName, false);
|
||||
return objects.FirstOrDefault(obj =>
|
||||
string.Equals(obj.Name, ObjectName, StringComparison.Ordinal));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
MinIOLogger.WriteWarning(this,
|
||||
"Could not retrieve object information for '{0}': {1}", ObjectName, ex.Message);
|
||||
return null;
|
||||
var result = ExecuteOperation("DownloadObject", () =>
|
||||
{
|
||||
WriteVerboseMessage("Downloading object '{0}' from bucket '{1}' to '{2}'",
|
||||
ObjectName, BucketName, LocalPath);
|
||||
|
||||
// Create download result
|
||||
var downloadResult = new MinIODownloadResult(BucketName, ObjectName, 0)
|
||||
{
|
||||
TargetFilePath = LocalPath
|
||||
};
|
||||
downloadResult.MarkStarted();
|
||||
|
||||
try
|
||||
{
|
||||
// Check if bucket exists
|
||||
if (!S3Client.BucketExists(BucketName))
|
||||
{
|
||||
throw new InvalidOperationException($"Bucket '{BucketName}' does not exist");
|
||||
}
|
||||
|
||||
// Create directory if needed
|
||||
var directory = Path.GetDirectoryName(LocalPath);
|
||||
if (!string.IsNullOrEmpty(directory) && !Directory.Exists(directory))
|
||||
{
|
||||
if (CreateDirectories.IsPresent)
|
||||
{
|
||||
WriteVerboseMessage("Creating directory: {0}", directory);
|
||||
Directory.CreateDirectory(directory);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new DirectoryNotFoundException($"Directory does not exist: {directory}. Use -CreateDirectories to create it.");
|
||||
}
|
||||
}
|
||||
|
||||
// Track progress
|
||||
long lastReportedBytes = 0;
|
||||
var startTime = DateTime.UtcNow;
|
||||
|
||||
// Download the object
|
||||
using (var fileStream = new FileStream(LocalPath, FileMode.Create, FileAccess.Write))
|
||||
{
|
||||
var bytesDownloaded = S3Client.GetObject(
|
||||
BucketName,
|
||||
ObjectName,
|
||||
fileStream,
|
||||
bytesTransferred =>
|
||||
{
|
||||
downloadResult.BytesTransferred = bytesTransferred;
|
||||
|
||||
// Report progress every 1MB or at completion
|
||||
if (bytesTransferred - lastReportedBytes >= 1024 * 1024 ||
|
||||
(downloadResult.TotalSize > 0 && bytesTransferred >= downloadResult.TotalSize))
|
||||
{
|
||||
var elapsed = DateTime.UtcNow - startTime;
|
||||
var speed = elapsed.TotalSeconds > 0 ? bytesTransferred / elapsed.TotalSeconds : 0;
|
||||
var percentage = downloadResult.TotalSize > 0 ?
|
||||
(double)bytesTransferred / downloadResult.TotalSize * 100 : 0;
|
||||
|
||||
WriteVerboseMessage("Download progress: {0:F1}% ({1}/{2}) at {3}",
|
||||
percentage,
|
||||
SizeFormatter.FormatBytes(bytesTransferred),
|
||||
downloadResult.TotalSize > 0 ? SizeFormatter.FormatBytes(downloadResult.TotalSize) : "Unknown",
|
||||
SizeFormatter.FormatSpeed(speed));
|
||||
|
||||
lastReportedBytes = bytesTransferred;
|
||||
}
|
||||
});
|
||||
|
||||
downloadResult.TotalSize = bytesDownloaded;
|
||||
downloadResult.BytesTransferred = bytesDownloaded;
|
||||
}
|
||||
|
||||
downloadResult.MarkCompleted();
|
||||
|
||||
var duration = downloadResult.Duration ?? TimeSpan.Zero;
|
||||
var averageSpeed = downloadResult.AverageSpeed ?? 0;
|
||||
|
||||
WriteVerboseMessage("Download completed: {0} in {1} at average speed of {2}",
|
||||
SizeFormatter.FormatBytes(downloadResult.BytesTransferred),
|
||||
SizeFormatter.FormatDuration(duration),
|
||||
SizeFormatter.FormatSpeed(averageSpeed));
|
||||
|
||||
// Set file timestamps if available
|
||||
try
|
||||
{
|
||||
var fileInfo = new FileInfo(LocalPath);
|
||||
if (downloadResult.LastModified.HasValue)
|
||||
{
|
||||
fileInfo.LastWriteTimeUtc = downloadResult.LastModified.Value;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
WriteVerboseMessage("Could not set file timestamps: {0}", ex.Message);
|
||||
}
|
||||
|
||||
return downloadResult;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
downloadResult.MarkFailed(ex.Message);
|
||||
throw;
|
||||
}
|
||||
}, $"Bucket: {BucketName}, Object: {ObjectName}, LocalPath: {LocalPath}");
|
||||
|
||||
// Return result if requested
|
||||
if (PassThru.IsPresent)
|
||||
{
|
||||
WriteObject(result);
|
||||
}
|
||||
else
|
||||
{
|
||||
WriteVerboseMessage("Successfully downloaded object '{0}' from bucket '{1}' to '{2}'",
|
||||
ObjectName, BucketName, LocalPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,269 +0,0 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Management.Automation;
|
||||
using PSMinIO.Models;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Cmdlets
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets statistical information about MinIO storage
|
||||
/// </summary>
|
||||
[Cmdlet(VerbsCommon.Get, "MinIOStats", SupportsShouldProcess = false)]
|
||||
[OutputType(typeof(MinIOStats))]
|
||||
public class GetMinIOStatsCmdlet : MinIOBaseCmdlet
|
||||
{
|
||||
/// <summary>
|
||||
/// Include detailed per-bucket statistics
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter IncludeBucketDetails { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Include object count and size calculations (may be slow for large buckets)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter IncludeObjectCounts { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of objects to count per bucket (default: unlimited)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateRange(1, int.MaxValue)]
|
||||
public int? MaxObjectsToCount { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Processes the cmdlet
|
||||
/// </summary>
|
||||
protected override void ProcessRecord()
|
||||
{
|
||||
ValidateConnection();
|
||||
|
||||
ExecuteOperation("GetStats", () =>
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Gathering MinIO statistics...");
|
||||
|
||||
var config = Configuration;
|
||||
var stats = new MinIOStats
|
||||
{
|
||||
Endpoint = config.Endpoint,
|
||||
UseSSL = config.UseSSL,
|
||||
ConnectionStatus = "Connected"
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
// Get basic bucket information
|
||||
var buckets = Client.ListBuckets();
|
||||
stats.TotalBuckets = buckets.Count;
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Found {0} buckets", buckets.Count);
|
||||
|
||||
if (IncludeObjectCounts.IsPresent && buckets.Count > 0)
|
||||
{
|
||||
GatherObjectStatistics(buckets, stats);
|
||||
}
|
||||
|
||||
// Create detailed output if requested
|
||||
if (IncludeBucketDetails.IsPresent)
|
||||
{
|
||||
var detailedStats = CreateDetailedStats(stats, buckets);
|
||||
WriteObject(detailedStats);
|
||||
}
|
||||
else
|
||||
{
|
||||
WriteObject(stats);
|
||||
}
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Statistics gathering completed");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
stats.ConnectionStatus = $"Error: {ex.Message}";
|
||||
WriteObject(stats);
|
||||
throw;
|
||||
}
|
||||
|
||||
}, "Gathering statistics");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gathers object statistics across all buckets
|
||||
/// </summary>
|
||||
/// <param name="buckets">List of buckets</param>
|
||||
/// <param name="stats">Stats object to update</param>
|
||||
private void GatherObjectStatistics(System.Collections.Generic.List<MinIOBucketInfo> buckets, MinIOStats stats)
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Gathering object statistics for {0} buckets...", buckets.Count);
|
||||
|
||||
long totalObjects = 0;
|
||||
long totalSize = 0;
|
||||
|
||||
for (int i = 0; i < buckets.Count; i++)
|
||||
{
|
||||
var bucket = buckets[i];
|
||||
|
||||
// Show progress
|
||||
var progressRecord = new ProgressRecord(1, "Gathering Statistics",
|
||||
$"Processing bucket: {bucket.Name}")
|
||||
{
|
||||
PercentComplete = (int)((double)(i + 1) / buckets.Count * 100)
|
||||
};
|
||||
WriteProgress(progressRecord);
|
||||
|
||||
try
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Processing bucket: {0}", bucket.Name);
|
||||
|
||||
var objects = Client.ListObjects(bucket.Name, recursive: true);
|
||||
|
||||
// Apply limit if specified
|
||||
if (MaxObjectsToCount.HasValue && objects.Count > MaxObjectsToCount.Value)
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Limiting object count for bucket '{0}' to {1} objects",
|
||||
bucket.Name, MaxObjectsToCount.Value);
|
||||
objects = objects.Take(MaxObjectsToCount.Value).ToList();
|
||||
}
|
||||
|
||||
var bucketObjectCount = objects.Count;
|
||||
var bucketSize = objects.Sum(obj => obj.Size);
|
||||
|
||||
totalObjects += bucketObjectCount;
|
||||
totalSize += bucketSize;
|
||||
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Bucket '{0}': {1} objects, {2} bytes",
|
||||
bucket.Name, bucketObjectCount, bucketSize);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
MinIOLogger.WriteWarning(this,
|
||||
"Failed to get statistics for bucket '{0}': {1}",
|
||||
bucket.Name, ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
// Complete progress
|
||||
WriteProgress(new ProgressRecord(1, "Gathering Statistics", "Completed")
|
||||
{
|
||||
PercentComplete = 100,
|
||||
RecordType = ProgressRecordType.Completed
|
||||
});
|
||||
|
||||
stats.TotalObjects = totalObjects;
|
||||
stats.TotalSize = totalSize;
|
||||
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Total statistics: {0} objects, {1} bytes across {2} buckets",
|
||||
totalObjects, totalSize, buckets.Count);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates detailed statistics with per-bucket information
|
||||
/// </summary>
|
||||
/// <param name="stats">Base statistics</param>
|
||||
/// <param name="buckets">List of buckets</param>
|
||||
/// <returns>PSObject with detailed statistics</returns>
|
||||
private PSObject CreateDetailedStats(MinIOStats stats, System.Collections.Generic.List<MinIOBucketInfo> buckets)
|
||||
{
|
||||
var detailedStats = new PSObject();
|
||||
|
||||
// Add basic statistics
|
||||
detailedStats.Properties.Add(new PSNoteProperty("TotalBuckets", stats.TotalBuckets));
|
||||
detailedStats.Properties.Add(new PSNoteProperty("TotalObjects", stats.TotalObjects));
|
||||
detailedStats.Properties.Add(new PSNoteProperty("TotalSize", stats.TotalSize));
|
||||
detailedStats.Properties.Add(new PSNoteProperty("TotalSizeFormatted", FormatBytes(stats.TotalSize)));
|
||||
detailedStats.Properties.Add(new PSNoteProperty("LastUpdated", stats.LastUpdated));
|
||||
detailedStats.Properties.Add(new PSNoteProperty("Endpoint", stats.Endpoint));
|
||||
detailedStats.Properties.Add(new PSNoteProperty("UseSSL", stats.UseSSL));
|
||||
detailedStats.Properties.Add(new PSNoteProperty("ConnectionStatus", stats.ConnectionStatus));
|
||||
|
||||
// Add calculated statistics
|
||||
detailedStats.Properties.Add(new PSNoteProperty("AverageObjectSize", stats.AverageObjectSize));
|
||||
detailedStats.Properties.Add(new PSNoteProperty("AverageObjectSizeFormatted", FormatBytes((long)stats.AverageObjectSize)));
|
||||
detailedStats.Properties.Add(new PSNoteProperty("AverageObjectsPerBucket", stats.AverageObjectsPerBucket));
|
||||
|
||||
// Add bucket details if object counts were gathered
|
||||
if (IncludeObjectCounts.IsPresent)
|
||||
{
|
||||
var bucketDetails = new System.Collections.Generic.List<PSObject>();
|
||||
|
||||
foreach (var bucket in buckets)
|
||||
{
|
||||
try
|
||||
{
|
||||
var objects = Client.ListObjects(bucket.Name, recursive: true);
|
||||
|
||||
// Apply limit if specified
|
||||
if (MaxObjectsToCount.HasValue && objects.Count > MaxObjectsToCount.Value)
|
||||
{
|
||||
objects = objects.Take(MaxObjectsToCount.Value).ToList();
|
||||
}
|
||||
|
||||
var bucketDetail = new PSObject();
|
||||
bucketDetail.Properties.Add(new PSNoteProperty("Name", bucket.Name));
|
||||
bucketDetail.Properties.Add(new PSNoteProperty("Created", bucket.Created));
|
||||
bucketDetail.Properties.Add(new PSNoteProperty("ObjectCount", objects.Count));
|
||||
|
||||
var bucketSize = objects.Sum(obj => obj.Size);
|
||||
bucketDetail.Properties.Add(new PSNoteProperty("Size", bucketSize));
|
||||
bucketDetail.Properties.Add(new PSNoteProperty("SizeFormatted", FormatBytes(bucketSize)));
|
||||
|
||||
var avgObjectSize = objects.Count > 0 ? (double)bucketSize / objects.Count : 0;
|
||||
bucketDetail.Properties.Add(new PSNoteProperty("AverageObjectSize", avgObjectSize));
|
||||
bucketDetail.Properties.Add(new PSNoteProperty("AverageObjectSizeFormatted", FormatBytes((long)avgObjectSize)));
|
||||
|
||||
bucketDetails.Add(bucketDetail);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
var bucketDetail = new PSObject();
|
||||
bucketDetail.Properties.Add(new PSNoteProperty("Name", bucket.Name));
|
||||
bucketDetail.Properties.Add(new PSNoteProperty("Created", bucket.Created));
|
||||
bucketDetail.Properties.Add(new PSNoteProperty("Error", ex.Message));
|
||||
bucketDetails.Add(bucketDetail);
|
||||
}
|
||||
}
|
||||
|
||||
detailedStats.Properties.Add(new PSNoteProperty("BucketDetails", bucketDetails.ToArray()));
|
||||
}
|
||||
else
|
||||
{
|
||||
// Just add basic bucket information
|
||||
var bucketSummary = buckets.Select(b => new PSObject(new
|
||||
{
|
||||
Name = b.Name,
|
||||
Created = b.Created,
|
||||
Region = b.Region
|
||||
})).ToArray();
|
||||
|
||||
detailedStats.Properties.Add(new PSNoteProperty("Buckets", bucketSummary));
|
||||
}
|
||||
|
||||
return detailedStats;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Formats bytes into a human-readable string
|
||||
/// </summary>
|
||||
/// <param name="bytes">Number of bytes</param>
|
||||
/// <returns>Formatted string</returns>
|
||||
private static string FormatBytes(long bytes)
|
||||
{
|
||||
if (bytes == 0) return "0 B";
|
||||
|
||||
string[] sizes = { "B", "KB", "MB", "GB", "TB", "PB" };
|
||||
int order = 0;
|
||||
double size = bytes;
|
||||
|
||||
while (size >= 1024 && order < sizes.Length - 1)
|
||||
{
|
||||
order++;
|
||||
size = size / 1024;
|
||||
}
|
||||
|
||||
return $"{size:0.##} {sizes[order]}";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,10 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Management.Automation;
|
||||
using PSMinIO.Models;
|
||||
using PSMinIO.Core;
|
||||
using PSMinIO.Core.S3;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Utils
|
||||
namespace PSMinIO.Cmdlets
|
||||
{
|
||||
/// <summary>
|
||||
/// Base class for all MinIO PowerShell cmdlets
|
||||
@@ -54,74 +55,54 @@ namespace PSMinIO.Utils
|
||||
ErrorCategory.ConnectionError,
|
||||
_connection));
|
||||
}
|
||||
|
||||
MinIOLogger.LogOperationStart(this, "UsingConnection", $"Endpoint: {_connection.Configuration.Endpoint}");
|
||||
}
|
||||
return _connection;
|
||||
|
||||
return _connection!;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the MinIO client wrapper instance
|
||||
/// Gets the MinIO S3 client instance
|
||||
/// </summary>
|
||||
protected MinIOClientWrapper Client => Connection.Client;
|
||||
protected MinIOS3Client S3Client => Connection.S3Client;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the current MinIO configuration
|
||||
/// </summary>
|
||||
protected MinIOConfiguration Configuration => Connection.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the MinIO connection from parameter or session variable
|
||||
/// Gets the connection from parameter or session variable
|
||||
/// </summary>
|
||||
/// <returns>MinIO connection or null if not found</returns>
|
||||
private MinIOConnection? GetConnection()
|
||||
{
|
||||
// First, check if connection was provided via parameter
|
||||
// First, try the parameter
|
||||
if (MinIOConnection != null)
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Using MinIO connection from parameter");
|
||||
return MinIOConnection;
|
||||
}
|
||||
|
||||
// Next, check session variable
|
||||
// Then try the session variable
|
||||
try
|
||||
{
|
||||
var sessionConnection = SessionState.PSVariable.GetValue(SessionVariable) as MinIOConnection;
|
||||
if (sessionConnection != null)
|
||||
var sessionConnection = SessionState.PSVariable.GetValue(SessionVariable);
|
||||
if (sessionConnection is MinIOConnection connection)
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Using MinIO connection from session variable: {0}", SessionVariable);
|
||||
return sessionConnection;
|
||||
return connection;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
catch
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Failed to retrieve connection from session variable '{0}': {1}", SessionVariable, ex.Message);
|
||||
// Ignore errors when accessing session variables
|
||||
}
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "No MinIO connection found in parameter or session variable '{0}'", SessionVariable);
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates that the MinIO connection is available and valid
|
||||
/// </summary>
|
||||
protected void ValidateConnection()
|
||||
{
|
||||
var connection = Connection; // This will throw if invalid
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes an operation with proper error handling and logging
|
||||
/// Executes an operation with consistent error handling and logging
|
||||
/// </summary>
|
||||
/// <param name="operationName">Name of the operation for logging</param>
|
||||
/// <param name="operation">The operation to execute</param>
|
||||
/// <param name="operation">Operation to execute</param>
|
||||
/// <param name="details">Optional operation details for logging</param>
|
||||
protected void ExecuteOperation(string operationName, Action operation, string? details = null)
|
||||
{
|
||||
if (operation == null)
|
||||
throw new ArgumentNullException(nameof(operationName));
|
||||
|
||||
var startTime = DateTime.UtcNow;
|
||||
MinIOLogger.LogOperationStart(this, operationName, details);
|
||||
|
||||
@@ -138,23 +119,20 @@ namespace PSMinIO.Utils
|
||||
// Determine appropriate error category
|
||||
var category = GetErrorCategory(ex);
|
||||
|
||||
WriteError(new ErrorRecord(ex, $"{operationName}Failed", category, null));
|
||||
ThrowTerminatingError(new ErrorRecord(ex, $"{operationName}Failed", category, null));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes an operation with return value and proper error handling and logging
|
||||
/// Executes an operation with return value and consistent error handling and logging
|
||||
/// </summary>
|
||||
/// <typeparam name="T">Return type</typeparam>
|
||||
/// <param name="operationName">Name of the operation for logging</param>
|
||||
/// <param name="operation">The operation to execute</param>
|
||||
/// <param name="operation">Operation to execute</param>
|
||||
/// <param name="details">Optional operation details for logging</param>
|
||||
/// <returns>Result of the operation</returns>
|
||||
/// <returns>Operation result</returns>
|
||||
protected T ExecuteOperation<T>(string operationName, Func<T> operation, string? details = null)
|
||||
{
|
||||
if (operation == null)
|
||||
throw new ArgumentNullException(nameof(operation));
|
||||
|
||||
var startTime = DateTime.UtcNow;
|
||||
MinIOLogger.LogOperationStart(this, operationName, details);
|
||||
|
||||
@@ -180,30 +158,29 @@ namespace PSMinIO.Utils
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines the appropriate PowerShell error category based on the exception type
|
||||
/// Determines the appropriate PowerShell error category for an exception
|
||||
/// </summary>
|
||||
/// <param name="exception">The exception to categorize</param>
|
||||
/// <returns>Appropriate ErrorCategory</returns>
|
||||
protected virtual ErrorCategory GetErrorCategory(Exception exception)
|
||||
/// <param name="exception">Exception to categorize</param>
|
||||
/// <returns>Appropriate error category</returns>
|
||||
protected ErrorCategory GetErrorCategory(Exception exception)
|
||||
{
|
||||
return exception switch
|
||||
{
|
||||
ArgumentNullException => ErrorCategory.InvalidArgument,
|
||||
ArgumentException => ErrorCategory.InvalidArgument,
|
||||
ArgumentException or ArgumentNullException => ErrorCategory.InvalidArgument,
|
||||
UnauthorizedAccessException => ErrorCategory.PermissionDenied,
|
||||
System.Net.WebException => ErrorCategory.ConnectionError,
|
||||
System.Net.Http.HttpRequestException => ErrorCategory.ConnectionError,
|
||||
TimeoutException => ErrorCategory.OperationTimeout,
|
||||
InvalidOperationException => ErrorCategory.InvalidOperation,
|
||||
System.IO.FileNotFoundException => ErrorCategory.ObjectNotFound,
|
||||
System.IO.DirectoryNotFoundException => ErrorCategory.ObjectNotFound,
|
||||
System.IO.IOException => ErrorCategory.WriteError,
|
||||
NotSupportedException => ErrorCategory.NotImplemented,
|
||||
FileNotFoundException => ErrorCategory.ObjectNotFound,
|
||||
DirectoryNotFoundException => ErrorCategory.ObjectNotFound,
|
||||
_ => ErrorCategory.NotSpecified
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates a bucket name according to MinIO/S3 naming rules
|
||||
/// Validates that a bucket name is valid according to S3 naming rules
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Bucket name to validate</param>
|
||||
/// <param name="parameterName">Parameter name for error reporting</param>
|
||||
@@ -212,17 +189,26 @@ namespace PSMinIO.Utils
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
{
|
||||
ThrowTerminatingError(new ErrorRecord(
|
||||
new ArgumentException($"{parameterName} cannot be null or empty"),
|
||||
new ArgumentException($"Bucket name cannot be null or empty", parameterName),
|
||||
"InvalidBucketName",
|
||||
ErrorCategory.InvalidArgument,
|
||||
bucketName));
|
||||
}
|
||||
|
||||
// Basic bucket name validation (simplified)
|
||||
// Basic S3 bucket name validation
|
||||
if (bucketName.Length < 3 || bucketName.Length > 63)
|
||||
{
|
||||
ThrowTerminatingError(new ErrorRecord(
|
||||
new ArgumentException($"{parameterName} must be between 3 and 63 characters long"),
|
||||
new ArgumentException($"Bucket name must be between 3 and 63 characters long", parameterName),
|
||||
"InvalidBucketName",
|
||||
ErrorCategory.InvalidArgument,
|
||||
bucketName));
|
||||
}
|
||||
|
||||
if (!System.Text.RegularExpressions.Regex.IsMatch(bucketName, @"^[a-z0-9][a-z0-9\-]*[a-z0-9]$"))
|
||||
{
|
||||
ThrowTerminatingError(new ErrorRecord(
|
||||
new ArgumentException($"Bucket name contains invalid characters. Must contain only lowercase letters, numbers, and hyphens", parameterName),
|
||||
"InvalidBucketName",
|
||||
ErrorCategory.InvalidArgument,
|
||||
bucketName));
|
||||
@@ -230,7 +216,7 @@ namespace PSMinIO.Utils
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates an object name
|
||||
/// Validates that an object name is valid
|
||||
/// </summary>
|
||||
/// <param name="objectName">Object name to validate</param>
|
||||
/// <param name="parameterName">Parameter name for error reporting</param>
|
||||
@@ -239,7 +225,16 @@ namespace PSMinIO.Utils
|
||||
if (string.IsNullOrWhiteSpace(objectName))
|
||||
{
|
||||
ThrowTerminatingError(new ErrorRecord(
|
||||
new ArgumentException($"{parameterName} cannot be null or empty"),
|
||||
new ArgumentException($"Object name cannot be null or empty", parameterName),
|
||||
"InvalidObjectName",
|
||||
ErrorCategory.InvalidArgument,
|
||||
objectName));
|
||||
}
|
||||
|
||||
if (objectName.Length > 1024)
|
||||
{
|
||||
ThrowTerminatingError(new ErrorRecord(
|
||||
new ArgumentException($"Object name cannot be longer than 1024 characters", parameterName),
|
||||
"InvalidObjectName",
|
||||
ErrorCategory.InvalidArgument,
|
||||
objectName));
|
||||
@@ -247,40 +242,33 @@ namespace PSMinIO.Utils
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cleans up resources when the cmdlet is disposed
|
||||
/// Writes verbose output with consistent formatting
|
||||
/// </summary>
|
||||
protected override void EndProcessing()
|
||||
/// <param name="message">Message to write</param>
|
||||
/// <param name="args">Format arguments</param>
|
||||
protected void WriteVerboseMessage(string message, params object[] args)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Note: We don't dispose the connection here as it may be shared across cmdlets
|
||||
// The connection should be disposed by the user when no longer needed
|
||||
_connection = null;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, $"Error during cleanup: {ex.Message}");
|
||||
}
|
||||
|
||||
base.EndProcessing();
|
||||
MinIOLogger.WriteVerbose(this, message, args);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Handles stopping the cmdlet (Ctrl+C)
|
||||
/// Writes warning output with consistent formatting
|
||||
/// </summary>
|
||||
protected override void StopProcessing()
|
||||
/// <param name="message">Warning message to write</param>
|
||||
/// <param name="args">Format arguments</param>
|
||||
protected void WriteWarningMessage(string message, params object[] args)
|
||||
{
|
||||
try
|
||||
{
|
||||
_connection?.Client.CancelOperations();
|
||||
MinIOLogger.WriteVerbose(this, "Operation cancelled by user");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, $"Error cancelling operations: {ex.Message}");
|
||||
}
|
||||
MinIOLogger.WriteWarning(this, message, args);
|
||||
}
|
||||
|
||||
base.StopProcessing();
|
||||
/// <summary>
|
||||
/// Writes debug output with consistent formatting
|
||||
/// </summary>
|
||||
/// <param name="message">Debug message to write</param>
|
||||
/// <param name="args">Format arguments</param>
|
||||
protected void WriteDebugMessage(string message, params object[] args)
|
||||
{
|
||||
MinIOLogger.WriteDebug(this, message, args);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
using System;
|
||||
using System.Management.Automation;
|
||||
using PSMinIO.Models;
|
||||
using PSMinIO.Utils;
|
||||
using PSMinIO.Core.Models;
|
||||
|
||||
namespace PSMinIO.Cmdlets
|
||||
{
|
||||
@@ -21,141 +20,119 @@ namespace PSMinIO.Cmdlets
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Region where the bucket should be created
|
||||
/// Region where the bucket should be created (default: us-east-1)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateNotNullOrEmpty]
|
||||
public string? Region { get; set; }
|
||||
public string Region { get; set; } = "us-east-1";
|
||||
|
||||
/// <summary>
|
||||
/// Force creation even if bucket already exists (no error will be thrown)
|
||||
/// Force creation even if bucket already exists
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter Force { get; set; }
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Return the created bucket information
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter PassThru { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Processes the cmdlet
|
||||
/// </summary>
|
||||
protected override void ProcessRecord()
|
||||
{
|
||||
ValidateConnection();
|
||||
ValidateBucketNameForCreation(BucketName);
|
||||
ValidateBucketName(BucketName);
|
||||
|
||||
// Use region from parameter or configuration
|
||||
var region = Region ?? Configuration.Region;
|
||||
|
||||
if (ShouldProcess(BucketName, $"Create bucket in region '{region}'"))
|
||||
if (ShouldProcess(BucketName, "Create MinIO bucket"))
|
||||
{
|
||||
ExecuteOperation("CreateBucket", () =>
|
||||
{
|
||||
WriteVerboseMessage("Creating bucket '{0}' in region '{1}'", BucketName, Region);
|
||||
|
||||
// Check if bucket already exists
|
||||
var exists = Client.BucketExists(BucketName);
|
||||
|
||||
if (exists)
|
||||
bool bucketExists = false;
|
||||
try
|
||||
{
|
||||
bucketExists = S3Client.BucketExists(BucketName);
|
||||
WriteVerboseMessage("Bucket existence check: {0}", bucketExists ? "exists" : "does not exist");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
WriteVerboseMessage("Could not check bucket existence: {0}", ex.Message);
|
||||
}
|
||||
|
||||
if (bucketExists)
|
||||
{
|
||||
if (Force.IsPresent)
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Bucket '{0}' already exists, but Force parameter specified", BucketName);
|
||||
WriteVerboseMessage("Bucket '{0}' already exists, but Force parameter specified", BucketName);
|
||||
}
|
||||
else
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Bucket '{BucketName}' already exists. Use -Force to suppress this error."),
|
||||
var errorMessage = $"Bucket '{BucketName}' already exists. Use -Force to suppress this error.";
|
||||
var errorRecord = new ErrorRecord(
|
||||
new InvalidOperationException(errorMessage),
|
||||
"BucketAlreadyExists",
|
||||
ErrorCategory.ResourceExists,
|
||||
BucketName));
|
||||
return;
|
||||
BucketName);
|
||||
|
||||
ThrowTerminatingError(errorRecord);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Create the bucket
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Creating bucket '{0}' in region '{1}'", BucketName, region);
|
||||
|
||||
Client.CreateBucket(BucketName, region);
|
||||
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Successfully created bucket '{0}'", BucketName);
|
||||
try
|
||||
{
|
||||
S3Client.CreateBucket(BucketName, Region);
|
||||
WriteVerboseMessage("Successfully created bucket '{0}'", BucketName);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
WriteVerboseMessage("Failed to create bucket '{0}': {1}", BucketName, ex.Message);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
// Always return bucket information
|
||||
try
|
||||
// Return bucket information if requested
|
||||
if (PassThru.IsPresent)
|
||||
{
|
||||
// Get the created bucket information
|
||||
var allBuckets = Client.ListBuckets();
|
||||
var createdBucket = allBuckets.Find(b =>
|
||||
string.Equals(b.Name, BucketName, StringComparison.OrdinalIgnoreCase));
|
||||
WriteVerboseMessage("Retrieving information for created bucket '{0}'", BucketName);
|
||||
|
||||
if (createdBucket != null)
|
||||
try
|
||||
{
|
||||
createdBucket.Region = region;
|
||||
WriteObject(createdBucket);
|
||||
// Get updated bucket list to find our bucket
|
||||
var buckets = S3Client.ListBuckets();
|
||||
var createdBucket = buckets.Find(b => string.Equals(b.Name, BucketName, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
if (createdBucket != null)
|
||||
{
|
||||
createdBucket.Region = Region;
|
||||
WriteObject(createdBucket);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Create a basic bucket info object if we can't find it in the list
|
||||
var bucketInfo = new MinIOBucketInfo(BucketName, DateTime.UtcNow, Region);
|
||||
WriteObject(bucketInfo);
|
||||
}
|
||||
}
|
||||
else
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Fallback: create a basic bucket info object
|
||||
var bucketInfo = new MinIOBucketInfo(BucketName, DateTime.UtcNow, region);
|
||||
WriteWarningMessage("Bucket created successfully but could not retrieve bucket information: {0}", ex.Message);
|
||||
|
||||
// Create a basic bucket info object
|
||||
var bucketInfo = new MinIOBucketInfo(BucketName, DateTime.UtcNow, Region);
|
||||
WriteObject(bucketInfo);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
else
|
||||
{
|
||||
MinIOLogger.WriteWarning(this,
|
||||
"Bucket created successfully, but failed to retrieve bucket information: {0}",
|
||||
ex.Message);
|
||||
|
||||
// Still return a basic bucket info object
|
||||
var bucketInfo = new MinIOBucketInfo(BucketName, DateTime.UtcNow, region);
|
||||
WriteObject(bucketInfo);
|
||||
WriteVerboseMessage("Bucket '{0}' created successfully", BucketName);
|
||||
}
|
||||
|
||||
}, $"Bucket: {BucketName}, Region: {region}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates the bucket name according to MinIO/S3 naming conventions for bucket creation
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Bucket name to validate</param>
|
||||
private void ValidateBucketNameForCreation(string bucketName, string parameterName = "BucketName")
|
||||
{
|
||||
base.ValidateBucketName(bucketName, parameterName);
|
||||
|
||||
// Additional validation for bucket creation
|
||||
if (bucketName.Contains("..") || bucketName.StartsWith(".") || bucketName.EndsWith("."))
|
||||
{
|
||||
ThrowTerminatingError(new ErrorRecord(
|
||||
new ArgumentException($"Bucket name '{bucketName}' contains invalid characters or patterns"),
|
||||
"InvalidBucketName",
|
||||
ErrorCategory.InvalidArgument,
|
||||
bucketName));
|
||||
}
|
||||
|
||||
// Check for uppercase letters (not allowed in S3/MinIO)
|
||||
if (bucketName != bucketName.ToLowerInvariant())
|
||||
{
|
||||
ThrowTerminatingError(new ErrorRecord(
|
||||
new ArgumentException($"Bucket name '{bucketName}' must be lowercase"),
|
||||
"InvalidBucketName",
|
||||
ErrorCategory.InvalidArgument,
|
||||
bucketName));
|
||||
}
|
||||
|
||||
// Check for invalid characters
|
||||
foreach (char c in bucketName)
|
||||
{
|
||||
if (!char.IsLetterOrDigit(c) && c != '-' && c != '.')
|
||||
{
|
||||
ThrowTerminatingError(new ErrorRecord(
|
||||
new ArgumentException($"Bucket name '{bucketName}' contains invalid character '{c}'. Only lowercase letters, numbers, hyphens, and periods are allowed."),
|
||||
"InvalidBucketName",
|
||||
ErrorCategory.InvalidArgument,
|
||||
bucketName));
|
||||
}
|
||||
}, $"Bucket: {BucketName}, Region: {Region}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,185 +0,0 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Management.Automation;
|
||||
using PSMinIO.Models;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Cmdlets
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates a folder (prefix) in a MinIO bucket
|
||||
/// </summary>
|
||||
[Cmdlet(VerbsCommon.New, "MinIOFolder", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Low)]
|
||||
[OutputType(typeof(MinIOObjectInfo))]
|
||||
public class NewMinIOFolderCmdlet : MinIOBaseCmdlet
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket to create the folder in
|
||||
/// </summary>
|
||||
[Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Bucket")]
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Name of the folder to create (supports nested paths like "folder1/folder2/folder3")
|
||||
/// </summary>
|
||||
[Parameter(Position = 1, Mandatory = true, ValueFromPipelineByPropertyName = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Folder", "Prefix", "Path")]
|
||||
public string FolderName { get; set; } = string.Empty;
|
||||
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Processes the cmdlet
|
||||
/// </summary>
|
||||
protected override void ProcessRecord()
|
||||
{
|
||||
ValidateConnection();
|
||||
ValidateBucketName(BucketName);
|
||||
|
||||
// Sanitize the folder name
|
||||
var sanitizedFolderName = SanitizeFolderPath(FolderName);
|
||||
if (string.IsNullOrEmpty(sanitizedFolderName))
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new ArgumentException("Folder name cannot be empty after sanitization"),
|
||||
"InvalidFolderName",
|
||||
ErrorCategory.InvalidArgument,
|
||||
FolderName));
|
||||
return;
|
||||
}
|
||||
|
||||
if (ShouldProcess($"{BucketName}/{sanitizedFolderName}", "Create folder structure"))
|
||||
{
|
||||
ExecuteOperation("CreateFolder", () =>
|
||||
{
|
||||
// Check if bucket exists
|
||||
var bucketExists = Client.BucketExists(BucketName);
|
||||
if (!bucketExists)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
|
||||
"BucketNotFound",
|
||||
ErrorCategory.ObjectNotFound,
|
||||
BucketName));
|
||||
return;
|
||||
}
|
||||
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Creating folder structure '{0}' in bucket '{1}'", sanitizedFolderName, BucketName);
|
||||
|
||||
// Create the complete folder structure
|
||||
var createdFolders = CreateFolderStructure(sanitizedFolderName);
|
||||
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Successfully created {0} folder level(s) in bucket '{1}'", createdFolders.Count, BucketName);
|
||||
|
||||
// Return information about all created folders
|
||||
foreach (var folder in createdFolders)
|
||||
{
|
||||
WriteObject(folder);
|
||||
}
|
||||
|
||||
}, $"Bucket: {BucketName}, Folder: {sanitizedFolderName}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sanitizes folder path and ensures proper format
|
||||
/// </summary>
|
||||
/// <param name="folderPath">Raw folder path</param>
|
||||
/// <returns>Sanitized folder path</returns>
|
||||
private string SanitizeFolderPath(string folderPath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(folderPath))
|
||||
return string.Empty;
|
||||
|
||||
// Remove leading and trailing slashes, then split and clean
|
||||
var cleanPath = folderPath.Trim().Trim('/', '\\');
|
||||
|
||||
// Split by both forward and back slashes, then clean each part
|
||||
var parts = cleanPath.Split(new char[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
|
||||
// Clean each part and validate
|
||||
var cleanedParts = new System.Collections.Generic.List<string>();
|
||||
var invalidChars = new char[] { '<', '>', ':', '"', '|', '?', '*' };
|
||||
|
||||
foreach (var part in parts)
|
||||
{
|
||||
var cleanPart = part.Trim();
|
||||
if (string.IsNullOrEmpty(cleanPart))
|
||||
continue;
|
||||
|
||||
// Check for invalid characters
|
||||
if (cleanPart.IndexOfAny(invalidChars) >= 0)
|
||||
{
|
||||
WriteWarning($"Folder part '{cleanPart}' contains invalid characters and will be skipped");
|
||||
continue;
|
||||
}
|
||||
|
||||
cleanedParts.Add(cleanPart);
|
||||
}
|
||||
|
||||
return string.Join("/", cleanedParts);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates the complete folder structure, including all intermediate levels
|
||||
/// </summary>
|
||||
/// <param name="folderPath">Complete folder path to create</param>
|
||||
/// <returns>List of created folder objects</returns>
|
||||
private System.Collections.Generic.List<MinIOObjectInfo> CreateFolderStructure(string folderPath)
|
||||
{
|
||||
var createdFolders = new System.Collections.Generic.List<MinIOObjectInfo>();
|
||||
var parts = folderPath.Split('/');
|
||||
var currentPath = string.Empty;
|
||||
|
||||
foreach (var part in parts)
|
||||
{
|
||||
currentPath = string.IsNullOrEmpty(currentPath) ? part : $"{currentPath}/{part}";
|
||||
var fullFolderPath = $"{currentPath}/";
|
||||
|
||||
try
|
||||
{
|
||||
// Check if this folder level already exists
|
||||
var existingObjects = Client.ListObjects(BucketName, fullFolderPath, false);
|
||||
var folderExists = existingObjects.Any(obj => obj.Name == fullFolderPath);
|
||||
|
||||
if (!folderExists)
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Creating folder level: {0}", fullFolderPath);
|
||||
|
||||
// Create the folder using the dedicated method
|
||||
var etag = Client.CreateDirectory(BucketName, fullFolderPath);
|
||||
|
||||
var folderInfo = new MinIOObjectInfo(
|
||||
fullFolderPath,
|
||||
0,
|
||||
DateTime.UtcNow,
|
||||
etag,
|
||||
BucketName)
|
||||
{
|
||||
ContentType = "application/x-directory"
|
||||
};
|
||||
|
||||
createdFolders.Add(folderInfo);
|
||||
MinIOLogger.WriteVerbose(this, "Successfully created folder level: {0}", fullFolderPath);
|
||||
}
|
||||
else
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Folder level already exists: {0}", fullFolderPath);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
WriteWarning($"Could not create folder level '{fullFolderPath}': {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
return createdFolders;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,588 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Management.Automation;
|
||||
using PSMinIO.Models;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Cmdlets
|
||||
{
|
||||
/// <summary>
|
||||
/// Uploads files or directories to a MinIO bucket using chunked transfer with resume capability
|
||||
/// </summary>
|
||||
[Cmdlet(VerbsCommon.New, "MinIOObjectChunked", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Low)]
|
||||
[OutputType(typeof(MinIOObjectInfo))]
|
||||
public class NewMinIOObjectChunkedCmdlet : MinIOBaseCmdlet
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket to upload to
|
||||
/// </summary>
|
||||
[Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Bucket")]
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Array of FileInfo objects to upload
|
||||
/// </summary>
|
||||
[Parameter(Position = 1, Mandatory = true, ValueFromPipeline = true, ParameterSetName = "Files")]
|
||||
[ValidateNotNull]
|
||||
[Alias("File", "Files")]
|
||||
public FileInfo[]? Path { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional bucket directory path where files should be uploaded (Files parameter set only)
|
||||
/// </summary>
|
||||
[Parameter(ParameterSetName = "Files")]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Prefix")]
|
||||
public string? BucketDirectory { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Directory to upload
|
||||
/// </summary>
|
||||
[Parameter(Position = 1, Mandatory = true, ValueFromPipelineByPropertyName = true, ParameterSetName = "Directory")]
|
||||
[ValidateNotNull]
|
||||
[Alias("Dir")]
|
||||
public DirectoryInfo? Directory { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Upload directory contents recursively (only applies to Directory parameter set)
|
||||
/// </summary>
|
||||
[Parameter(ParameterSetName = "Directory")]
|
||||
public SwitchParameter Recursive { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Maximum depth for recursive directory upload (0 = unlimited)
|
||||
/// </summary>
|
||||
[Parameter(ParameterSetName = "Directory")]
|
||||
[ValidateRange(0, int.MaxValue)]
|
||||
public int MaxDepth { get; set; } = 0;
|
||||
|
||||
/// <summary>
|
||||
/// Flatten directory structure (upload all files to bucket root)
|
||||
/// </summary>
|
||||
[Parameter(ParameterSetName = "Directory")]
|
||||
public SwitchParameter Flatten { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Script block to filter files for inclusion
|
||||
/// </summary>
|
||||
[Parameter(ParameterSetName = "Directory")]
|
||||
public ScriptBlock? InclusionFilter { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Script block to filter files for exclusion
|
||||
/// </summary>
|
||||
[Parameter(ParameterSetName = "Directory")]
|
||||
public ScriptBlock? ExclusionFilter { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Overwrite existing objects without prompting
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter Force { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Generate presigned URLs for uploaded objects
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter ShowURL { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Expiration time for presigned URLs (default: 1 hour)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateRange("00:01:00", "7.00:00:00")] // 1 minute to 7 days
|
||||
public TimeSpan Expiration { get; set; } = TimeSpan.FromHours(1);
|
||||
|
||||
/// <summary>
|
||||
/// Size of each chunk for upload (default: 5MB)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateRange(1024 * 1024, 1024 * 1024 * 1024)] // 1MB to 1GB
|
||||
public long ChunkSize { get; set; } = 5 * 1024 * 1024; // 5MB default
|
||||
|
||||
/// <summary>
|
||||
/// Enable resume functionality for interrupted uploads
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter Resume { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of retry attempts for failed chunks
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateRange(1, 10)]
|
||||
public int MaxRetries { get; set; } = 3;
|
||||
|
||||
/// <summary>
|
||||
/// Custom path for storing resume data (default: %LOCALAPPDATA%\PSMinIO\Resume)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public string? ResumeDataPath { get; set; }
|
||||
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Update progress every N bytes transferred (default: 1MB)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateRange(1024, 10 * 1024 * 1024)] // 1KB to 10MB
|
||||
public long ProgressUpdateInterval { get; set; } = 1024 * 1024; // 1MB
|
||||
|
||||
/// <summary>
|
||||
/// Processes the cmdlet
|
||||
/// </summary>
|
||||
protected override void ProcessRecord()
|
||||
{
|
||||
ValidateConnection();
|
||||
ValidateBucketName(BucketName);
|
||||
|
||||
if (ParameterSetName == "Files")
|
||||
{
|
||||
ProcessFiles();
|
||||
}
|
||||
else if (ParameterSetName == "Directory")
|
||||
{
|
||||
ProcessDirectory();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Processes file uploads from FileInfo array
|
||||
/// </summary>
|
||||
private void ProcessFiles()
|
||||
{
|
||||
if (Path == null || Path.Length == 0)
|
||||
{
|
||||
WriteWarning("No files provided for upload");
|
||||
return;
|
||||
}
|
||||
|
||||
if (ShouldProcess(BucketName, $"Upload {Path.Length} file(s) using chunked transfer"))
|
||||
{
|
||||
ExecuteOperation("UploadFilesChunked", () =>
|
||||
{
|
||||
// Check if bucket exists first
|
||||
var bucketExists = Client.BucketExists(BucketName);
|
||||
if (!bucketExists)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
|
||||
"BucketNotFound",
|
||||
ErrorCategory.ObjectNotFound,
|
||||
BucketName));
|
||||
return;
|
||||
}
|
||||
|
||||
// Create bucket directory structure if specified
|
||||
if (!string.IsNullOrWhiteSpace(BucketDirectory))
|
||||
{
|
||||
var sanitizedDirectory = SanitizeBucketDirectory(BucketDirectory!);
|
||||
if (!string.IsNullOrEmpty(sanitizedDirectory))
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Ensuring bucket directory exists: {0}", sanitizedDirectory);
|
||||
EnsureBucketDirectoryExists(sanitizedDirectory);
|
||||
}
|
||||
}
|
||||
|
||||
UploadFileCollectionChunked(Path!);
|
||||
|
||||
}, $"Bucket: {BucketName}, Files: {Path.Length}, ChunkSize: {SizeFormatter.FormatBytes(ChunkSize)}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Processes directory upload
|
||||
/// </summary>
|
||||
private void ProcessDirectory()
|
||||
{
|
||||
if (Directory == null || !Directory.Exists)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new DirectoryNotFoundException($"Directory not found: {Directory?.FullName}"),
|
||||
"DirectoryNotFound",
|
||||
ErrorCategory.ObjectNotFound,
|
||||
Directory));
|
||||
return;
|
||||
}
|
||||
|
||||
if (ShouldProcess(BucketName, $"Upload directory '{Directory.Name}' using chunked transfer"))
|
||||
{
|
||||
ExecuteOperation("UploadDirectoryChunked", () =>
|
||||
{
|
||||
// Check if bucket exists first
|
||||
var bucketExists = Client.BucketExists(BucketName);
|
||||
if (!bucketExists)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
|
||||
"BucketNotFound",
|
||||
ErrorCategory.ObjectNotFound,
|
||||
BucketName));
|
||||
return;
|
||||
}
|
||||
|
||||
// Get files from directory
|
||||
var files = GetDirectoryFiles();
|
||||
if (files.Length == 0)
|
||||
{
|
||||
WriteWarning($"No files found in directory: {Directory.FullName}");
|
||||
return;
|
||||
}
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Found {0} files in directory '{1}'", files.Length, Directory.FullName);
|
||||
UploadFileCollectionChunked(files);
|
||||
|
||||
}, $"Bucket: {BucketName}, Directory: {Directory.FullName}, ChunkSize: {SizeFormatter.FormatBytes(ChunkSize)}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets files from directory based on filters and recursion settings
|
||||
/// </summary>
|
||||
/// <returns>Array of FileInfo objects</returns>
|
||||
private FileInfo[] GetDirectoryFiles()
|
||||
{
|
||||
var searchOption = Recursive.IsPresent ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly;
|
||||
var allFiles = Directory!.GetFiles("*", searchOption);
|
||||
|
||||
// Apply depth filtering if MaxDepth is specified and we're recursive
|
||||
if (Recursive.IsPresent && MaxDepth > 0)
|
||||
{
|
||||
var basePath = Directory.FullName;
|
||||
allFiles = allFiles.Where(f =>
|
||||
{
|
||||
var relativePath = f.FullName.Substring(basePath.Length).TrimStart('\\', '/');
|
||||
var depth = relativePath.Split(new char[] { '\\', '/' }, StringSplitOptions.RemoveEmptyEntries).Length - 1;
|
||||
return depth <= MaxDepth;
|
||||
}).ToArray();
|
||||
}
|
||||
|
||||
// Apply inclusion filter
|
||||
if (InclusionFilter != null)
|
||||
{
|
||||
allFiles = allFiles.Where(f => EvaluateFilter(InclusionFilter, f)).ToArray();
|
||||
}
|
||||
|
||||
// Apply exclusion filter
|
||||
if (ExclusionFilter != null)
|
||||
{
|
||||
allFiles = allFiles.Where(f => !EvaluateFilter(ExclusionFilter, f)).ToArray();
|
||||
}
|
||||
|
||||
return allFiles;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates a script block filter against a file
|
||||
/// </summary>
|
||||
/// <param name="filter">Script block filter</param>
|
||||
/// <param name="file">File to evaluate</param>
|
||||
/// <returns>True if file matches filter</returns>
|
||||
private bool EvaluateFilter(ScriptBlock filter, FileInfo file)
|
||||
{
|
||||
try
|
||||
{
|
||||
var result = filter.InvokeWithContext(null, new List<PSVariable> { new PSVariable("_", file) });
|
||||
return result.Count > 0 && LanguagePrimitives.IsTrue(result[0]);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
WriteWarning($"Filter evaluation failed for {file.Name}: {ex.Message}");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Uploads a collection of files using chunked transfer
|
||||
/// </summary>
|
||||
/// <param name="files">Files to upload</param>
|
||||
private void UploadFileCollectionChunked(FileInfo[] files)
|
||||
{
|
||||
// Filter out files that don't exist
|
||||
var validFiles = files.Where(f => f.Exists).ToArray();
|
||||
var skippedCount = files.Length - validFiles.Length;
|
||||
|
||||
if (skippedCount > 0)
|
||||
{
|
||||
WriteWarning($"Skipped {skippedCount} files that do not exist");
|
||||
}
|
||||
|
||||
if (validFiles.Length == 0)
|
||||
{
|
||||
WriteWarning("No valid files found for upload");
|
||||
return;
|
||||
}
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Starting chunked upload of {0} files to bucket '{1}' (ChunkSize: {2})",
|
||||
validFiles.Length, BucketName, SizeFormatter.FormatBytes(ChunkSize));
|
||||
|
||||
// Calculate total size for overall progress
|
||||
var totalSize = validFiles.Sum(f => f.Length);
|
||||
|
||||
// Create thread-safe result collector
|
||||
var resultCollector = new ThreadSafeResultCollector(this);
|
||||
|
||||
for (int i = 0; i < validFiles.Length; i++)
|
||||
{
|
||||
var fileInfo = validFiles[i];
|
||||
var objectName = GetObjectName(fileInfo);
|
||||
|
||||
try
|
||||
{
|
||||
// Calculate chunks for this file
|
||||
var totalChunks = (int)Math.Ceiling((double)fileInfo.Length / ChunkSize);
|
||||
|
||||
// Create thread-safe progress reporter for this file
|
||||
var progressReporter = new ThreadSafeChunkedProgressReporter(
|
||||
this,
|
||||
fileInfo.Length,
|
||||
totalChunks,
|
||||
"Uploading");
|
||||
|
||||
progressReporter.StartNewFile(fileInfo.Name, fileInfo.Length, totalChunks);
|
||||
|
||||
// Track timing for this file
|
||||
var startTime = DateTime.UtcNow;
|
||||
|
||||
// Upload file using chunked transfer
|
||||
var uploadedObject = UploadFileChunked(fileInfo, objectName, progressReporter);
|
||||
|
||||
if (uploadedObject != null)
|
||||
{
|
||||
// Add timing information
|
||||
uploadedObject.StartTime = startTime;
|
||||
uploadedObject.CompletionTime = DateTime.UtcNow;
|
||||
|
||||
resultCollector.QueueResult(uploadedObject);
|
||||
progressReporter.CompleteFile();
|
||||
}
|
||||
|
||||
// Process any queued progress updates from the main thread
|
||||
progressReporter.ProcessQueuedUpdates();
|
||||
progressReporter.Complete();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
resultCollector.QueueError(ex, "ChunkedFileUploadFailed", ErrorCategory.WriteError, fileInfo);
|
||||
MinIOLogger.WriteVerbose(this, "Failed to upload {0}: {1}", fileInfo.Name, ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Completed chunked upload: {0} files", validFiles.Length);
|
||||
|
||||
// Process all results from the main thread
|
||||
resultCollector.Complete();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Uploads a single file using chunked transfer with resume capability
|
||||
/// </summary>
|
||||
/// <param name="fileInfo">File to upload</param>
|
||||
/// <param name="objectName">Object name in bucket</param>
|
||||
/// <param name="progressReporter">Thread-safe progress reporter</param>
|
||||
/// <returns>Uploaded object info or null if failed</returns>
|
||||
private MinIOObjectInfo? UploadFileChunked(FileInfo fileInfo, string objectName, ThreadSafeChunkedProgressReporter progressReporter)
|
||||
{
|
||||
ChunkedTransferState? transferState = null;
|
||||
|
||||
// Try to load existing transfer state for resume
|
||||
if (Resume.IsPresent)
|
||||
{
|
||||
transferState = ChunkedTransferResumeManager.LoadTransferState(
|
||||
BucketName, objectName, fileInfo.FullName, ChunkedTransferType.Upload, ResumeDataPath);
|
||||
|
||||
if (transferState != null && !ChunkedTransferResumeManager.IsResumeDataValid(transferState, fileInfo))
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Resume data for {0} is invalid, starting fresh upload", fileInfo.Name);
|
||||
transferState = null;
|
||||
}
|
||||
}
|
||||
|
||||
// Create new transfer state if none exists or resume is disabled
|
||||
if (transferState == null)
|
||||
{
|
||||
transferState = new ChunkedTransferState
|
||||
{
|
||||
BucketName = BucketName,
|
||||
ObjectName = objectName,
|
||||
FilePath = fileInfo.FullName,
|
||||
TotalSize = fileInfo.Length,
|
||||
ChunkSize = ChunkSize,
|
||||
LastModified = fileInfo.LastWriteTimeUtc,
|
||||
TransferType = ChunkedTransferType.Upload,
|
||||
StartTime = DateTime.UtcNow,
|
||||
LastUpdated = DateTime.UtcNow
|
||||
};
|
||||
}
|
||||
else
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Resuming upload of {0} from {1:P1} complete",
|
||||
fileInfo.Name, transferState.ProgressPercentage / 100);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// Upload file using chunked transfer
|
||||
var result = Client.UploadFileChunked(transferState, progressReporter, MaxRetries);
|
||||
|
||||
if (result != null)
|
||||
{
|
||||
// Generate presigned URL if requested
|
||||
if (ShowURL.IsPresent)
|
||||
{
|
||||
try
|
||||
{
|
||||
var presignedUrl = Client.GetPresignedUrl(BucketName, objectName, Expiration);
|
||||
result.PresignedUrl = presignedUrl;
|
||||
result.PresignedUrlExpiration = DateTime.UtcNow.Add(Expiration);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
WriteWarning($"Could not generate presigned URL for {objectName}: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up resume data on successful completion
|
||||
if (Resume.IsPresent)
|
||||
{
|
||||
ChunkedTransferResumeManager.CleanupResumeData(transferState, ResumeDataPath);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
#pragma warning disable CS0168 // Variable is declared but never used - false positive, ex is used in throw
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Save resume data on failure if resume is enabled
|
||||
if (Resume.IsPresent)
|
||||
{
|
||||
try
|
||||
{
|
||||
ChunkedTransferResumeManager.SaveTransferState(transferState, ResumeDataPath);
|
||||
MinIOLogger.WriteVerbose(this, "Saved resume data for {0}", fileInfo.Name);
|
||||
}
|
||||
catch (Exception saveEx)
|
||||
{
|
||||
WriteWarning($"Could not save resume data for {fileInfo.Name}: {saveEx.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
throw;
|
||||
}
|
||||
#pragma warning restore CS0168
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the object name for a file based on the upload context
|
||||
/// </summary>
|
||||
/// <param name="file">File to get object name for</param>
|
||||
/// <returns>Object name to use in MinIO</returns>
|
||||
private string GetObjectName(FileInfo file)
|
||||
{
|
||||
if (ParameterSetName == "Files")
|
||||
{
|
||||
// For file uploads, optionally prefix with bucket directory
|
||||
var objectName = file.Name;
|
||||
if (!string.IsNullOrWhiteSpace(BucketDirectory))
|
||||
{
|
||||
var sanitizedDirectory = SanitizeBucketDirectory(BucketDirectory!);
|
||||
objectName = $"{sanitizedDirectory}/{file.Name}";
|
||||
}
|
||||
return objectName;
|
||||
}
|
||||
else if (ParameterSetName == "Directory")
|
||||
{
|
||||
if (Flatten.IsPresent)
|
||||
{
|
||||
// Flatten: use just the filename
|
||||
return file.Name;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Maintain directory structure relative to the base directory
|
||||
var relativePath = file.FullName.Substring(Directory!.FullName.Length).TrimStart('\\', '/');
|
||||
return relativePath.Replace('\\', '/'); // Ensure forward slashes for object storage
|
||||
}
|
||||
}
|
||||
|
||||
return file.Name;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sanitizes bucket directory path and ensures proper format
|
||||
/// </summary>
|
||||
/// <param name="directory">Raw directory path</param>
|
||||
/// <returns>Sanitized directory path</returns>
|
||||
private string SanitizeBucketDirectory(string directory)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(directory))
|
||||
return string.Empty;
|
||||
|
||||
// Remove leading and trailing slashes, then split and clean
|
||||
var cleanDirectory = directory.Trim().Trim('/', '\\');
|
||||
|
||||
// Split by both forward and back slashes, then clean each part
|
||||
var parts = cleanDirectory.Split(new char[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
|
||||
// Clean each part and rejoin with forward slashes
|
||||
var cleanedParts = parts.Select(part => part.Trim()).Where(part => !string.IsNullOrEmpty(part));
|
||||
|
||||
return string.Join("/", cleanedParts);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates bucket directory structure if it doesn't exist
|
||||
/// </summary>
|
||||
/// <param name="directoryPath">Directory path to create</param>
|
||||
private void EnsureBucketDirectoryExists(string directoryPath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(directoryPath))
|
||||
return;
|
||||
|
||||
var parts = directoryPath.Split('/');
|
||||
var currentPath = string.Empty;
|
||||
|
||||
foreach (var part in parts)
|
||||
{
|
||||
currentPath = string.IsNullOrEmpty(currentPath) ? part : $"{currentPath}/{part}";
|
||||
var folderPath = $"{currentPath}/";
|
||||
|
||||
try
|
||||
{
|
||||
// Check if this directory level already exists by trying to list objects with this prefix
|
||||
var existingObjects = Client.ListObjects(BucketName, folderPath, false);
|
||||
var folderExists = existingObjects.Any(obj => obj.Name == folderPath);
|
||||
|
||||
if (!folderExists)
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Creating bucket directory: {0}", folderPath);
|
||||
|
||||
try
|
||||
{
|
||||
// Create the directory using the dedicated method
|
||||
Client.CreateDirectory(BucketName, folderPath);
|
||||
MinIOLogger.WriteVerbose(this, "Successfully created bucket directory: {0}", folderPath);
|
||||
}
|
||||
catch (Exception createEx)
|
||||
{
|
||||
// Directory creation failed, but this is not critical since MinIO creates directories implicitly
|
||||
MinIOLogger.WriteVerbose(this, "Directory creation failed (non-critical): {0} - {1}", folderPath, createEx.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Listing failed, but this is not critical for the upload operation
|
||||
MinIOLogger.WriteVerbose(this, "Could not check directory existence (non-critical): {0} - {1}", folderPath, ex.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+175
-430
@@ -1,506 +1,251 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Management.Automation;
|
||||
using PSMinIO.Models;
|
||||
using PSMinIO.Core.Models;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Cmdlets
|
||||
{
|
||||
/// <summary>
|
||||
/// Uploads files or directories to a MinIO bucket
|
||||
/// Uploads objects to MinIO
|
||||
/// </summary>
|
||||
[Cmdlet(VerbsCommon.New, "MinIOObject", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Low)]
|
||||
[OutputType(typeof(MinIOObjectInfo))]
|
||||
[Cmdlet(VerbsCommon.New, "MinIOObject", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Medium)]
|
||||
[OutputType(typeof(MinIOUploadResult))]
|
||||
public class NewMinIOObjectCmdlet : MinIOBaseCmdlet
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket to upload to
|
||||
/// Name of the bucket
|
||||
/// </summary>
|
||||
[Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true)]
|
||||
[Parameter(Position = 0, Mandatory = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Bucket")]
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Array of FileInfo objects to upload
|
||||
/// Name of the object (if not specified, uses filename)
|
||||
/// </summary>
|
||||
[Parameter(Position = 1, Mandatory = true, ValueFromPipeline = true, ParameterSetName = "Files")]
|
||||
[ValidateNotNull]
|
||||
[Alias("File", "Files")]
|
||||
public FileInfo[]? Path { get; set; }
|
||||
[Parameter(Position = 1)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
public string? ObjectName { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional bucket directory path where files should be uploaded (Files parameter set only)
|
||||
/// File to upload
|
||||
/// </summary>
|
||||
[Parameter(ParameterSetName = "Files")]
|
||||
[Parameter(Position = 2, Mandatory = true, ValueFromPipeline = true)]
|
||||
[ValidateNotNull]
|
||||
public FileInfo File { get; set; } = null!;
|
||||
|
||||
/// <summary>
|
||||
/// Content type of the file (auto-detected if not specified)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Prefix")]
|
||||
public string? ContentType { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Custom metadata for the object
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public Hashtable? Metadata { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Directory prefix in the bucket (creates nested structure)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Folder", "Directory")]
|
||||
public string? BucketDirectory { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Directory to upload
|
||||
/// </summary>
|
||||
[Parameter(Position = 1, Mandatory = true, ValueFromPipelineByPropertyName = true, ParameterSetName = "Directory")]
|
||||
[ValidateNotNull]
|
||||
[Alias("Dir")]
|
||||
public DirectoryInfo? Directory { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Upload directory contents recursively (only applies to Directory parameter set)
|
||||
/// </summary>
|
||||
[Parameter(ParameterSetName = "Directory")]
|
||||
public SwitchParameter Recursive { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Maximum depth for recursive directory upload (0 = unlimited)
|
||||
/// </summary>
|
||||
[Parameter(ParameterSetName = "Directory")]
|
||||
[ValidateRange(0, int.MaxValue)]
|
||||
public int MaxDepth { get; set; } = 0;
|
||||
|
||||
/// <summary>
|
||||
/// Flatten directory structure (upload all files to bucket root)
|
||||
/// </summary>
|
||||
[Parameter(ParameterSetName = "Directory")]
|
||||
public SwitchParameter Flatten { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Script block to filter files for inclusion
|
||||
/// </summary>
|
||||
[Parameter(ParameterSetName = "Directory")]
|
||||
public ScriptBlock? InclusionFilter { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Script block to filter files for exclusion
|
||||
/// </summary>
|
||||
[Parameter(ParameterSetName = "Directory")]
|
||||
public ScriptBlock? ExclusionFilter { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Overwrite existing objects without prompting
|
||||
/// Force overwrite if object already exists
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter Force { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Generate presigned URLs for uploaded objects
|
||||
/// Return the upload result information
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter ShowURL { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Expiration time for presigned URLs (default: 1 hour)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateRange("00:01:00", "7.00:00:00")] // 1 minute to 7 days
|
||||
public TimeSpan Expiration { get; set; } = TimeSpan.FromHours(1);
|
||||
public SwitchParameter PassThru { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Processes the cmdlet
|
||||
/// </summary>
|
||||
protected override void ProcessRecord()
|
||||
{
|
||||
ValidateConnection();
|
||||
ValidateBucketName(BucketName);
|
||||
|
||||
if (ParameterSetName == "Files")
|
||||
// Validate file exists
|
||||
if (!File.Exists)
|
||||
{
|
||||
ProcessFiles();
|
||||
}
|
||||
else if (ParameterSetName == "Directory")
|
||||
{
|
||||
ProcessDirectory();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Processes file uploads from FileInfo array
|
||||
/// </summary>
|
||||
private void ProcessFiles()
|
||||
{
|
||||
if (Path == null || Path.Length == 0)
|
||||
{
|
||||
WriteWarning("No files provided for upload");
|
||||
return;
|
||||
}
|
||||
|
||||
if (ShouldProcess(BucketName, $"Upload {Path.Length} file(s)"))
|
||||
{
|
||||
ExecuteOperation("UploadFiles", () =>
|
||||
{
|
||||
// Check if bucket exists first
|
||||
var bucketExists = Client.BucketExists(BucketName);
|
||||
if (!bucketExists)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
|
||||
"BucketNotFound",
|
||||
ErrorCategory.ObjectNotFound,
|
||||
BucketName));
|
||||
return;
|
||||
}
|
||||
|
||||
// Create bucket directory structure if specified
|
||||
if (!string.IsNullOrWhiteSpace(BucketDirectory))
|
||||
{
|
||||
var sanitizedDirectory = SanitizeBucketDirectory(BucketDirectory!);
|
||||
if (!string.IsNullOrEmpty(sanitizedDirectory))
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Ensuring bucket directory exists: {0}", sanitizedDirectory);
|
||||
EnsureBucketDirectoryExists(sanitizedDirectory);
|
||||
}
|
||||
}
|
||||
|
||||
UploadFileCollection(Path!);
|
||||
|
||||
}, $"Bucket: {BucketName}, Files: {Path.Length}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Processes directory upload
|
||||
/// </summary>
|
||||
private void ProcessDirectory()
|
||||
{
|
||||
if (Directory == null || !Directory.Exists)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new DirectoryNotFoundException($"Directory not found: {Directory?.FullName}"),
|
||||
"DirectoryNotFound",
|
||||
var errorRecord = new ErrorRecord(
|
||||
new FileNotFoundException($"File not found: {File.FullName}"),
|
||||
"FileNotFound",
|
||||
ErrorCategory.ObjectNotFound,
|
||||
Directory));
|
||||
return;
|
||||
File);
|
||||
ThrowTerminatingError(errorRecord);
|
||||
}
|
||||
|
||||
if (ShouldProcess(BucketName, $"Upload directory '{Directory.Name}'"))
|
||||
// Determine object name
|
||||
var finalObjectName = ObjectName ?? File?.Name ?? "unknown";
|
||||
|
||||
// Add bucket directory prefix if specified
|
||||
if (!string.IsNullOrEmpty(BucketDirectory))
|
||||
{
|
||||
ExecuteOperation("UploadDirectory", () =>
|
||||
var cleanDirectory = BucketDirectory.Trim('/');
|
||||
if (!string.IsNullOrEmpty(cleanDirectory))
|
||||
{
|
||||
// Check if bucket exists first
|
||||
var bucketExists = Client.BucketExists(BucketName);
|
||||
if (!bucketExists)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
|
||||
"BucketNotFound",
|
||||
ErrorCategory.ObjectNotFound,
|
||||
BucketName));
|
||||
return;
|
||||
}
|
||||
|
||||
// Get files from directory
|
||||
var files = GetDirectoryFiles();
|
||||
if (files.Length == 0)
|
||||
{
|
||||
WriteWarning($"No files found in directory: {Directory.FullName}");
|
||||
return;
|
||||
}
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Found {0} files in directory '{1}'", files.Length, Directory.FullName);
|
||||
UploadFileCollection(files);
|
||||
|
||||
}, $"Bucket: {BucketName}, Directory: {Directory.FullName}");
|
||||
finalObjectName = $"{cleanDirectory}/{finalObjectName}";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets files from directory based on filters and recursion settings
|
||||
/// </summary>
|
||||
/// <returns>Array of FileInfo objects</returns>
|
||||
private FileInfo[] GetDirectoryFiles()
|
||||
{
|
||||
var searchOption = Recursive.IsPresent ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly;
|
||||
var allFiles = Directory!.GetFiles("*", searchOption);
|
||||
ValidateObjectName(finalObjectName);
|
||||
|
||||
// Apply depth filtering if MaxDepth is specified and we're recursive
|
||||
if (Recursive.IsPresent && MaxDepth > 0)
|
||||
if (ShouldProcess($"{BucketName}/{finalObjectName}", "Upload file"))
|
||||
{
|
||||
var basePath = Directory.FullName;
|
||||
allFiles = allFiles.Where(f =>
|
||||
var result = ExecuteOperation("UploadObject", () =>
|
||||
{
|
||||
var relativePath = f.FullName.Substring(basePath.Length).TrimStart('\\', '/');
|
||||
var depth = relativePath.Split(new char[] { '\\', '/' }, StringSplitOptions.RemoveEmptyEntries).Length - 1;
|
||||
return depth <= MaxDepth;
|
||||
}).ToArray();
|
||||
}
|
||||
WriteVerboseMessage("Uploading file '{0}' to bucket '{1}' as object '{2}'",
|
||||
File.FullName, BucketName, finalObjectName);
|
||||
|
||||
// Apply inclusion filter
|
||||
if (InclusionFilter != null)
|
||||
{
|
||||
allFiles = allFiles.Where(f => EvaluateFilter(InclusionFilter, f)).ToArray();
|
||||
}
|
||||
|
||||
// Apply exclusion filter
|
||||
if (ExclusionFilter != null)
|
||||
{
|
||||
allFiles = allFiles.Where(f => !EvaluateFilter(ExclusionFilter, f)).ToArray();
|
||||
}
|
||||
|
||||
return allFiles;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates a script block filter against a file
|
||||
/// </summary>
|
||||
/// <param name="filter">Script block filter</param>
|
||||
/// <param name="file">File to evaluate</param>
|
||||
/// <returns>True if file matches filter</returns>
|
||||
private bool EvaluateFilter(ScriptBlock filter, FileInfo file)
|
||||
{
|
||||
try
|
||||
{
|
||||
var result = filter.InvokeWithContext(null, new List<PSVariable> { new PSVariable("_", file) });
|
||||
return result.Count > 0 && LanguagePrimitives.IsTrue(result[0]);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
WriteWarning($"Filter evaluation failed for {file.Name}: {ex.Message}");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Uploads a collection of files
|
||||
/// </summary>
|
||||
/// <param name="files">Files to upload</param>
|
||||
private void UploadFileCollection(FileInfo[] files)
|
||||
{
|
||||
// Filter out files that don't exist
|
||||
var validFiles = files.Where(f => f.Exists).ToArray();
|
||||
var skippedCount = files.Length - validFiles.Length;
|
||||
|
||||
if (skippedCount > 0)
|
||||
{
|
||||
WriteWarning($"Skipped {skippedCount} files that do not exist");
|
||||
}
|
||||
|
||||
if (validFiles.Length == 0)
|
||||
{
|
||||
WriteWarning("No valid files found for upload");
|
||||
return;
|
||||
}
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Uploading {0} files to bucket '{1}'", validFiles.Length, BucketName);
|
||||
|
||||
// Calculate total size for overall progress
|
||||
var totalSize = validFiles.Sum(f => f.Length);
|
||||
var totalProcessed = 0L;
|
||||
|
||||
// Create overall progress reporter
|
||||
var overallProgress = new ProgressReporter(
|
||||
this,
|
||||
"Uploading Files",
|
||||
$"Uploading {validFiles.Length} files",
|
||||
totalSize,
|
||||
1);
|
||||
|
||||
var uploadedObjects = new System.Collections.Generic.List<MinIOObjectInfo>();
|
||||
|
||||
for (int i = 0; i < validFiles.Length; i++)
|
||||
{
|
||||
var fileInfo = validFiles[i];
|
||||
var objectName = GetObjectName(fileInfo);
|
||||
|
||||
try
|
||||
{
|
||||
// Show file-level progress
|
||||
var fileProgressRecord = new ProgressRecord(2, "Current File",
|
||||
$"Uploading: {fileInfo.Name}")
|
||||
// Create upload result
|
||||
var uploadResult = new MinIOUploadResult(BucketName, finalObjectName, string.Empty, File.Length)
|
||||
{
|
||||
PercentComplete = (int)((double)(i + 1) / validFiles.Length * 100),
|
||||
ParentActivityId = 1
|
||||
SourceFilePath = File.FullName
|
||||
};
|
||||
WriteProgress(fileProgressRecord);
|
||||
uploadResult.MarkStarted();
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Uploading file {0}/{1}: {2} -> {3}",
|
||||
i + 1, validFiles.Length, fileInfo.Name, objectName);
|
||||
|
||||
// Track timing for this file
|
||||
var startTime = DateTime.UtcNow;
|
||||
|
||||
// Upload the file
|
||||
var etag = Client.UploadFile(
|
||||
BucketName,
|
||||
objectName,
|
||||
fileInfo.FullName,
|
||||
null, // Auto-detect content type
|
||||
bytesTransferred =>
|
||||
{
|
||||
overallProgress.UpdateProgress(totalProcessed + bytesTransferred);
|
||||
});
|
||||
|
||||
var completionTime = DateTime.UtcNow;
|
||||
|
||||
totalProcessed += fileInfo.Length;
|
||||
overallProgress.UpdateProgress(totalProcessed);
|
||||
|
||||
// Create object info for result
|
||||
var uploadedObject = new MinIOObjectInfo(
|
||||
objectName,
|
||||
fileInfo.Length,
|
||||
DateTime.UtcNow,
|
||||
etag,
|
||||
BucketName)
|
||||
try
|
||||
{
|
||||
StartTime = startTime,
|
||||
CompletionTime = completionTime
|
||||
};
|
||||
// Check if bucket exists
|
||||
if (!S3Client.BucketExists(BucketName))
|
||||
{
|
||||
throw new InvalidOperationException($"Bucket '{BucketName}' does not exist");
|
||||
}
|
||||
|
||||
// Generate presigned URL if requested
|
||||
if (ShowURL.IsPresent)
|
||||
// Determine content type
|
||||
var fileContentType = ContentType ?? GetContentType(File.Extension);
|
||||
uploadResult.ContentType = fileContentType;
|
||||
WriteVerboseMessage("Using content type: {0}", fileContentType);
|
||||
|
||||
// Convert metadata
|
||||
Dictionary<string, string>? metadata = null;
|
||||
if (Metadata != null && Metadata.Count > 0)
|
||||
{
|
||||
metadata = new Dictionary<string, string>();
|
||||
foreach (var key in Metadata.Keys)
|
||||
{
|
||||
if (key != null && Metadata[key] != null)
|
||||
{
|
||||
metadata[key.ToString()!] = Metadata[key]!.ToString()!;
|
||||
}
|
||||
}
|
||||
WriteVerboseMessage("Added {0} metadata entries", metadata.Count);
|
||||
}
|
||||
|
||||
// Track progress
|
||||
var fileSize = File.Length;
|
||||
var startTime = DateTime.UtcNow;
|
||||
long lastReportedBytes = 0;
|
||||
|
||||
WriteVerboseMessage("Starting upload of {0}", SizeFormatter.FormatBytes(fileSize));
|
||||
|
||||
// Upload the file
|
||||
string etag;
|
||||
using (var fileStream = File.OpenRead())
|
||||
{
|
||||
etag = S3Client.PutObject(
|
||||
BucketName,
|
||||
finalObjectName,
|
||||
fileStream,
|
||||
fileContentType,
|
||||
metadata,
|
||||
bytesTransferred =>
|
||||
{
|
||||
uploadResult.BytesTransferred = bytesTransferred;
|
||||
|
||||
// Report progress every 1MB or at completion
|
||||
if (bytesTransferred - lastReportedBytes >= 1024 * 1024 || bytesTransferred == fileSize)
|
||||
{
|
||||
var elapsed = DateTime.UtcNow - startTime;
|
||||
var speed = elapsed.TotalSeconds > 0 ? bytesTransferred / elapsed.TotalSeconds : 0;
|
||||
var percentage = fileSize > 0 ? (double)bytesTransferred / fileSize * 100 : 100;
|
||||
|
||||
WriteVerboseMessage("Upload progress: {0:F1}% ({1}/{2}) at {3}",
|
||||
percentage,
|
||||
SizeFormatter.FormatBytes(bytesTransferred),
|
||||
SizeFormatter.FormatBytes(fileSize),
|
||||
SizeFormatter.FormatSpeed(speed));
|
||||
|
||||
lastReportedBytes = bytesTransferred;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
uploadResult.ETag = etag;
|
||||
uploadResult.MarkCompleted();
|
||||
|
||||
var duration = uploadResult.Duration ?? TimeSpan.Zero;
|
||||
var averageSpeed = uploadResult.AverageSpeed ?? 0;
|
||||
|
||||
WriteVerboseMessage("Upload completed in {0} at average speed of {1}",
|
||||
SizeFormatter.FormatDuration(duration),
|
||||
SizeFormatter.FormatSpeed(averageSpeed));
|
||||
|
||||
return uploadResult;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
try
|
||||
{
|
||||
var presignedUrl = Client.GetPresignedUrl(BucketName, objectName, Expiration);
|
||||
uploadedObject.PresignedUrl = presignedUrl;
|
||||
uploadedObject.PresignedUrlExpiration = DateTime.UtcNow.Add(Expiration);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
WriteWarning($"Could not generate presigned URL for {objectName}: {ex.Message}");
|
||||
}
|
||||
uploadResult.MarkFailed(ex.Message);
|
||||
throw;
|
||||
}
|
||||
|
||||
uploadedObjects.Add(uploadedObject);
|
||||
MinIOLogger.WriteVerbose(this, "Successfully uploaded: {0}", fileInfo.Name);
|
||||
}
|
||||
catch (Exception ex)
|
||||
}, $"File: {File.FullName}, Bucket: {BucketName}, Object: {finalObjectName}");
|
||||
|
||||
// Return result if requested
|
||||
if (PassThru.IsPresent)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
ex,
|
||||
"FileUploadFailed",
|
||||
ErrorCategory.WriteError,
|
||||
fileInfo));
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Failed to upload {0}: {1}", fileInfo.Name, ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
// Complete progress reporting
|
||||
overallProgress.Complete();
|
||||
WriteProgress(new ProgressRecord(2, "Current File", "Completed")
|
||||
{
|
||||
PercentComplete = 100,
|
||||
RecordType = ProgressRecordType.Completed,
|
||||
ParentActivityId = 1
|
||||
});
|
||||
|
||||
// Always return uploaded objects
|
||||
foreach (var obj in uploadedObjects)
|
||||
{
|
||||
WriteObject(obj);
|
||||
}
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Completed uploading {0} files ({1} successful, {2} failed)",
|
||||
validFiles.Length, uploadedObjects.Count, validFiles.Length - uploadedObjects.Count);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the object name for a file based on the upload context
|
||||
/// </summary>
|
||||
/// <param name="file">File to get object name for</param>
|
||||
/// <returns>Object name to use in MinIO</returns>
|
||||
private string GetObjectName(FileInfo file)
|
||||
{
|
||||
if (ParameterSetName == "Files")
|
||||
{
|
||||
// For file uploads, optionally prefix with bucket directory
|
||||
var objectName = file.Name;
|
||||
if (!string.IsNullOrWhiteSpace(BucketDirectory))
|
||||
{
|
||||
var sanitizedDirectory = SanitizeBucketDirectory(BucketDirectory!);
|
||||
objectName = $"{sanitizedDirectory}/{file.Name}";
|
||||
}
|
||||
return objectName;
|
||||
}
|
||||
else if (ParameterSetName == "Directory")
|
||||
{
|
||||
if (Flatten.IsPresent)
|
||||
{
|
||||
// Flatten: use just the filename
|
||||
return file.Name;
|
||||
WriteObject(result);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Maintain directory structure relative to the base directory
|
||||
var relativePath = file.FullName.Substring(Directory!.FullName.Length).TrimStart('\\', '/');
|
||||
return relativePath.Replace('\\', '/'); // Ensure forward slashes for object storage
|
||||
WriteVerboseMessage("Successfully uploaded object '{0}' to bucket '{1}'", finalObjectName, BucketName);
|
||||
}
|
||||
}
|
||||
|
||||
return file.Name;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sanitizes bucket directory path and ensures proper format
|
||||
/// Gets the content type for a file based on its extension
|
||||
/// </summary>
|
||||
/// <param name="directory">Raw directory path</param>
|
||||
/// <returns>Sanitized directory path</returns>
|
||||
private string SanitizeBucketDirectory(string directory)
|
||||
/// <param name="extension">File extension</param>
|
||||
/// <returns>Content type string</returns>
|
||||
private static string GetContentType(string extension)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(directory))
|
||||
return string.Empty;
|
||||
|
||||
// Remove leading and trailing slashes, then split and clean
|
||||
var cleanDirectory = directory.Trim().Trim('/', '\\');
|
||||
|
||||
// Split by both forward and back slashes, then clean each part
|
||||
var parts = cleanDirectory.Split(new char[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
|
||||
// Clean each part and rejoin with forward slashes
|
||||
var cleanedParts = parts.Select(part => part.Trim()).Where(part => !string.IsNullOrEmpty(part));
|
||||
|
||||
return string.Join("/", cleanedParts);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates bucket directory structure if it doesn't exist
|
||||
/// </summary>
|
||||
/// <param name="directoryPath">Directory path to create</param>
|
||||
private void EnsureBucketDirectoryExists(string directoryPath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(directoryPath))
|
||||
return;
|
||||
|
||||
var parts = directoryPath.Split('/');
|
||||
var currentPath = string.Empty;
|
||||
|
||||
foreach (var part in parts)
|
||||
return extension.ToLowerInvariant() switch
|
||||
{
|
||||
currentPath = string.IsNullOrEmpty(currentPath) ? part : $"{currentPath}/{part}";
|
||||
var folderPath = $"{currentPath}/";
|
||||
|
||||
try
|
||||
{
|
||||
// Check if this directory level already exists by trying to list objects with this prefix
|
||||
var existingObjects = Client.ListObjects(BucketName, folderPath, false);
|
||||
var folderExists = existingObjects.Any(obj => obj.Name == folderPath);
|
||||
|
||||
if (!folderExists)
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Creating bucket directory: {0}", folderPath);
|
||||
|
||||
try
|
||||
{
|
||||
// Create the directory using the dedicated method
|
||||
Client.CreateDirectory(BucketName, folderPath);
|
||||
MinIOLogger.WriteVerbose(this, "Successfully created bucket directory: {0}", folderPath);
|
||||
}
|
||||
catch (Exception createEx)
|
||||
{
|
||||
// Directory creation failed, but this is not critical since MinIO creates directories implicitly
|
||||
MinIOLogger.WriteVerbose(this, "Directory creation failed (non-critical): {0} - {1}", folderPath, createEx.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Listing failed, but this is not critical for the upload operation
|
||||
MinIOLogger.WriteVerbose(this, "Could not check directory existence (non-critical): {0} - {1}", folderPath, ex.Message);
|
||||
}
|
||||
}
|
||||
".txt" => "text/plain",
|
||||
".html" or ".htm" => "text/html",
|
||||
".css" => "text/css",
|
||||
".js" => "application/javascript",
|
||||
".json" => "application/json",
|
||||
".xml" => "application/xml",
|
||||
".pdf" => "application/pdf",
|
||||
".zip" => "application/zip",
|
||||
".jpg" or ".jpeg" => "image/jpeg",
|
||||
".png" => "image/png",
|
||||
".gif" => "image/gif",
|
||||
".svg" => "image/svg+xml",
|
||||
".mp4" => "video/mp4",
|
||||
".mp3" => "audio/mpeg",
|
||||
".wav" => "audio/wav",
|
||||
".csv" => "text/csv",
|
||||
".doc" => "application/msword",
|
||||
".docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
".xls" => "application/vnd.ms-excel",
|
||||
".xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
_ => "application/octet-stream"
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,190 +0,0 @@
|
||||
using System;
|
||||
using System.Management.Automation;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Cmdlets
|
||||
{
|
||||
/// <summary>
|
||||
/// Removes a MinIO bucket
|
||||
/// </summary>
|
||||
[Cmdlet(VerbsCommon.Remove, "MinIOBucket", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.High)]
|
||||
public class RemoveMinIOBucketCmdlet : MinIOBaseCmdlet
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket to remove
|
||||
/// </summary>
|
||||
[Parameter(Position = 0, Mandatory = true, ValueFromPipeline = true, ValueFromPipelineByPropertyName = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Bucket")]
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Force removal without confirmation prompts
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter Force { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Remove all objects in the bucket before removing the bucket itself
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[Alias("Recursive")]
|
||||
public SwitchParameter RemoveObjects { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Processes the cmdlet
|
||||
/// </summary>
|
||||
protected override void ProcessRecord()
|
||||
{
|
||||
ValidateConnection();
|
||||
ValidateBucketName(BucketName);
|
||||
|
||||
// Force parameter is handled by ShouldProcess automatically
|
||||
|
||||
var actionDescription = RemoveObjects.IsPresent
|
||||
? $"Remove bucket '{BucketName}' and all its objects"
|
||||
: $"Remove bucket '{BucketName}'";
|
||||
|
||||
if (ShouldProcess(BucketName, actionDescription))
|
||||
{
|
||||
ExecuteOperation("RemoveBucket", () =>
|
||||
{
|
||||
// Check if bucket exists
|
||||
var exists = Client.BucketExists(BucketName);
|
||||
if (!exists)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
|
||||
"BucketNotFound",
|
||||
ErrorCategory.ObjectNotFound,
|
||||
BucketName));
|
||||
return;
|
||||
}
|
||||
|
||||
// If RemoveObjects is specified, remove all objects first
|
||||
if (RemoveObjects.IsPresent)
|
||||
{
|
||||
RemoveAllObjectsFromBucket();
|
||||
}
|
||||
else
|
||||
{
|
||||
// Check if bucket is empty before attempting to remove
|
||||
CheckBucketEmpty();
|
||||
}
|
||||
|
||||
// Remove the bucket
|
||||
MinIOLogger.WriteVerbose(this, "Removing bucket '{0}'", BucketName);
|
||||
Client.DeleteBucket(BucketName);
|
||||
MinIOLogger.WriteVerbose(this, "Successfully removed bucket '{0}'", BucketName);
|
||||
|
||||
}, $"Bucket: {BucketName}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes all objects from the bucket
|
||||
/// </summary>
|
||||
private void RemoveAllObjectsFromBucket()
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Removing all objects from bucket '{0}'", BucketName);
|
||||
|
||||
try
|
||||
{
|
||||
var objects = Client.ListObjects(BucketName, recursive: true);
|
||||
|
||||
if (objects.Count == 0)
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Bucket '{0}' is already empty", BucketName);
|
||||
return;
|
||||
}
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Found {0} objects to remove from bucket '{1}'", objects.Count, BucketName);
|
||||
|
||||
// Show progress for object removal
|
||||
for (int i = 0; i < objects.Count; i++)
|
||||
{
|
||||
var obj = objects[i];
|
||||
|
||||
var progressRecord = new ProgressRecord(1, "Removing Objects",
|
||||
$"Removing object: {obj.Name}")
|
||||
{
|
||||
PercentComplete = (int)((double)(i + 1) / objects.Count * 100)
|
||||
};
|
||||
WriteProgress(progressRecord);
|
||||
|
||||
try
|
||||
{
|
||||
Client.DeleteObject(BucketName, obj.Name);
|
||||
MinIOLogger.WriteVerbose(this, "Removed object: {0}", obj.Name);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
MinIOLogger.WriteWarning(this,
|
||||
"Failed to remove object '{0}': {1}", obj.Name, ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
// Complete progress
|
||||
WriteProgress(new ProgressRecord(1, "Removing Objects", "Completed")
|
||||
{
|
||||
PercentComplete = 100,
|
||||
RecordType = ProgressRecordType.Completed
|
||||
});
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Finished removing objects from bucket '{0}'", BucketName);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
MinIOLogger.WriteWarning(this,
|
||||
"Failed to list or remove objects from bucket '{0}': {1}", BucketName, ex.Message);
|
||||
|
||||
if (!Force.IsPresent)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Cannot remove objects from bucket '{BucketName}': {ex.Message}"),
|
||||
"ObjectRemovalFailed",
|
||||
ErrorCategory.InvalidOperation,
|
||||
BucketName));
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if the bucket is empty and warns if it's not
|
||||
/// </summary>
|
||||
private void CheckBucketEmpty()
|
||||
{
|
||||
try
|
||||
{
|
||||
var objects = Client.ListObjects(BucketName, recursive: true);
|
||||
|
||||
if (objects.Count > 0)
|
||||
{
|
||||
var message = $"Bucket '{BucketName}' contains {objects.Count} objects. " +
|
||||
"Use -RemoveObjects parameter to remove all objects first, or remove them manually.";
|
||||
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException(message),
|
||||
"BucketNotEmpty",
|
||||
ErrorCategory.InvalidOperation,
|
||||
BucketName));
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
MinIOLogger.WriteWarning(this,
|
||||
"Could not check if bucket '{0}' is empty: {1}", BucketName, ex.Message);
|
||||
|
||||
if (!Force.IsPresent)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Cannot verify bucket '{BucketName}' is empty: {ex.Message}"),
|
||||
"BucketCheckFailed",
|
||||
ErrorCategory.InvalidOperation,
|
||||
BucketName));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,225 +0,0 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Management.Automation;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Cmdlets
|
||||
{
|
||||
/// <summary>
|
||||
/// Removes an object from a MinIO bucket
|
||||
/// </summary>
|
||||
[Cmdlet(VerbsCommon.Remove, "MinIOObject", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.High)]
|
||||
public class RemoveMinIOObjectCmdlet : MinIOBaseCmdlet
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket containing the object
|
||||
/// </summary>
|
||||
[Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Bucket")]
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Name of the object to remove
|
||||
/// </summary>
|
||||
[Parameter(Position = 1, Mandatory = true, ValueFromPipelineByPropertyName = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Object", "Key")]
|
||||
public string ObjectName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Force removal without confirmation prompts
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter Force { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Remove all objects matching the specified prefix (use with caution)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[Alias("Recursive")]
|
||||
public SwitchParameter RemovePrefix { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Processes the cmdlet
|
||||
/// </summary>
|
||||
protected override void ProcessRecord()
|
||||
{
|
||||
ValidateConnection();
|
||||
ValidateBucketName(BucketName);
|
||||
ValidateObjectName(ObjectName);
|
||||
|
||||
// Force parameter is handled by ShouldProcess automatically
|
||||
|
||||
var actionDescription = RemovePrefix.IsPresent
|
||||
? $"Remove all objects with prefix '{ObjectName}' from bucket '{BucketName}'"
|
||||
: $"Remove object '{ObjectName}' from bucket '{BucketName}'";
|
||||
|
||||
if (ShouldProcess($"{BucketName}/{ObjectName}", actionDescription))
|
||||
{
|
||||
ExecuteOperation("RemoveObject", () =>
|
||||
{
|
||||
// Check if bucket exists
|
||||
var bucketExists = Client.BucketExists(BucketName);
|
||||
if (!bucketExists)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
|
||||
"BucketNotFound",
|
||||
ErrorCategory.ObjectNotFound,
|
||||
BucketName));
|
||||
return;
|
||||
}
|
||||
|
||||
if (RemovePrefix.IsPresent)
|
||||
{
|
||||
RemoveObjectsWithPrefix();
|
||||
}
|
||||
else
|
||||
{
|
||||
RemoveSingleObject();
|
||||
}
|
||||
|
||||
}, $"Bucket: {BucketName}, Object: {ObjectName}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes a single object
|
||||
/// </summary>
|
||||
private void RemoveSingleObject()
|
||||
{
|
||||
// Check if object exists
|
||||
var objectExists = CheckObjectExists();
|
||||
if (!objectExists)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Object '{ObjectName}' does not exist in bucket '{BucketName}'"),
|
||||
"ObjectNotFound",
|
||||
ErrorCategory.ObjectNotFound,
|
||||
ObjectName));
|
||||
return;
|
||||
}
|
||||
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Removing object '{0}' from bucket '{1}'", ObjectName, BucketName);
|
||||
|
||||
Client.DeleteObject(BucketName, ObjectName);
|
||||
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Successfully removed object '{0}' from bucket '{1}'", ObjectName, BucketName);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes all objects with the specified prefix
|
||||
/// </summary>
|
||||
private void RemoveObjectsWithPrefix()
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Finding objects with prefix '{0}' in bucket '{1}'", ObjectName, BucketName);
|
||||
|
||||
try
|
||||
{
|
||||
var objects = Client.ListObjects(BucketName, ObjectName, true);
|
||||
|
||||
if (objects.Count == 0)
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"No objects found with prefix '{0}' in bucket '{1}'", ObjectName, BucketName);
|
||||
return;
|
||||
}
|
||||
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Found {0} objects with prefix '{1}' in bucket '{2}'",
|
||||
objects.Count, ObjectName, BucketName);
|
||||
|
||||
// Additional confirmation for prefix removal if not forced
|
||||
if (!Force.IsPresent && objects.Count > 1)
|
||||
{
|
||||
var message = $"This will remove {objects.Count} objects with prefix '{ObjectName}'. Are you sure?";
|
||||
if (!ShouldContinue(message, "Confirm Multiple Object Removal"))
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Remove objects with progress reporting
|
||||
for (int i = 0; i < objects.Count; i++)
|
||||
{
|
||||
var obj = objects[i];
|
||||
|
||||
var progressRecord = new ProgressRecord(1, "Removing Objects",
|
||||
$"Removing object: {obj.Name}")
|
||||
{
|
||||
PercentComplete = (int)((double)(i + 1) / objects.Count * 100)
|
||||
};
|
||||
WriteProgress(progressRecord);
|
||||
|
||||
try
|
||||
{
|
||||
Client.DeleteObject(BucketName, obj.Name);
|
||||
MinIOLogger.WriteVerbose(this, "Removed object: {0}", obj.Name);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
MinIOLogger.WriteWarning(this,
|
||||
"Failed to remove object '{0}': {1}", obj.Name, ex.Message);
|
||||
|
||||
if (!Force.IsPresent)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Failed to remove object '{obj.Name}': {ex.Message}", ex),
|
||||
"ObjectRemovalFailed",
|
||||
ErrorCategory.InvalidOperation,
|
||||
obj.Name));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Complete progress
|
||||
WriteProgress(new ProgressRecord(1, "Removing Objects", "Completed")
|
||||
{
|
||||
PercentComplete = 100,
|
||||
RecordType = ProgressRecordType.Completed
|
||||
});
|
||||
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Finished removing objects with prefix '{0}' from bucket '{1}'", ObjectName, BucketName);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
MinIOLogger.WriteWarning(this,
|
||||
"Failed to list objects with prefix '{0}' in bucket '{1}': {2}",
|
||||
ObjectName, BucketName, ex.Message);
|
||||
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Cannot list objects with prefix '{ObjectName}': {ex.Message}", ex),
|
||||
"ObjectListingFailed",
|
||||
ErrorCategory.InvalidOperation,
|
||||
ObjectName));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if the specified object exists
|
||||
/// </summary>
|
||||
/// <returns>True if object exists, false otherwise</returns>
|
||||
private bool CheckObjectExists()
|
||||
{
|
||||
try
|
||||
{
|
||||
var objects = Client.ListObjects(BucketName, ObjectName, false);
|
||||
return objects.Any(obj =>
|
||||
string.Equals(obj.Name, ObjectName, StringComparison.Ordinal));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
MinIOLogger.WriteWarning(this,
|
||||
"Could not check if object '{0}' exists: {1}", ObjectName, ex.Message);
|
||||
|
||||
// If we can't verify existence, assume it exists and let the delete operation handle it
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,332 +0,0 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Management.Automation;
|
||||
using System.Text.Json;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Cmdlets
|
||||
{
|
||||
/// <summary>
|
||||
/// Sets the policy for a MinIO bucket
|
||||
/// </summary>
|
||||
[Cmdlet(VerbsCommon.Set, "MinIOBucketPolicy", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Medium)]
|
||||
public class SetMinIOBucketPolicyCmdlet : MinIOBaseCmdlet
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket to set the policy for
|
||||
/// </summary>
|
||||
[Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Bucket")]
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Policy JSON string
|
||||
/// </summary>
|
||||
[Parameter(Position = 1, Mandatory = true, ParameterSetName = "PolicyJson")]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Json")]
|
||||
public string? PolicyJson { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Path to a file containing the policy JSON
|
||||
/// </summary>
|
||||
[Parameter(Mandatory = true, ParameterSetName = "PolicyFile")]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("File")]
|
||||
public string? PolicyFilePath { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Use a predefined canned policy
|
||||
/// </summary>
|
||||
[Parameter(Mandatory = true, ParameterSetName = "CannedPolicy")]
|
||||
[ValidateSet("ReadOnly", "WriteOnly", "ReadWrite", "None")]
|
||||
public string? CannedPolicy { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Prefix for canned policies (default: *)
|
||||
/// </summary>
|
||||
[Parameter(ParameterSetName = "CannedPolicy")]
|
||||
[ValidateNotNullOrEmpty]
|
||||
public string Prefix { get; set; } = "*";
|
||||
|
||||
/// <summary>
|
||||
/// Validate the policy JSON before setting it
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter ValidateOnly { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Force setting the policy without confirmation
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter Force { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Processes the cmdlet
|
||||
/// </summary>
|
||||
protected override void ProcessRecord()
|
||||
{
|
||||
ValidateConnection();
|
||||
ValidateBucketName(BucketName);
|
||||
|
||||
// Get the policy JSON based on the parameter set
|
||||
var policyJson = GetPolicyJson();
|
||||
if (string.IsNullOrWhiteSpace(policyJson))
|
||||
{
|
||||
return; // Error already written
|
||||
}
|
||||
|
||||
// Validate the policy JSON
|
||||
if (!ValidatePolicyJson(policyJson!))
|
||||
{
|
||||
return; // Error already written
|
||||
}
|
||||
|
||||
if (ValidateOnly.IsPresent)
|
||||
{
|
||||
WriteObject("Policy JSON is valid");
|
||||
return;
|
||||
}
|
||||
|
||||
// Force parameter is handled by ShouldProcess automatically
|
||||
|
||||
if (ShouldProcess(BucketName, "Set bucket policy"))
|
||||
{
|
||||
ExecuteOperation("SetBucketPolicy", () =>
|
||||
{
|
||||
// Check if bucket exists
|
||||
var bucketExists = Client.BucketExists(BucketName);
|
||||
if (!bucketExists)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Bucket '{BucketName}' does not exist"),
|
||||
"BucketNotFound",
|
||||
ErrorCategory.ObjectNotFound,
|
||||
BucketName));
|
||||
return;
|
||||
}
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Setting policy for bucket '{0}'", BucketName);
|
||||
MinIOLogger.WriteVerbose(this, "Policy JSON ({0} characters): {1}",
|
||||
policyJson!.Length, policyJson.Length > 200 ? policyJson.Substring(0, 200) + "..." : policyJson);
|
||||
|
||||
Client.SetBucketPolicy(BucketName, policyJson);
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Successfully set policy for bucket '{0}'", BucketName);
|
||||
|
||||
}, $"Bucket: {BucketName}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the policy JSON based on the current parameter set
|
||||
/// </summary>
|
||||
/// <returns>Policy JSON string or null if error</returns>
|
||||
private string? GetPolicyJson()
|
||||
{
|
||||
switch (ParameterSetName)
|
||||
{
|
||||
case "PolicyJson":
|
||||
return PolicyJson;
|
||||
|
||||
case "PolicyFile":
|
||||
return ReadPolicyFromFile();
|
||||
|
||||
case "CannedPolicy":
|
||||
return GenerateCannedPolicy();
|
||||
|
||||
default:
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException("Unknown parameter set"),
|
||||
"UnknownParameterSet",
|
||||
ErrorCategory.InvalidArgument,
|
||||
null));
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads policy JSON from a file
|
||||
/// </summary>
|
||||
/// <returns>Policy JSON string or null if error</returns>
|
||||
private string? ReadPolicyFromFile()
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(PolicyFilePath))
|
||||
return null;
|
||||
|
||||
try
|
||||
{
|
||||
var fullPath = Path.GetFullPath(PolicyFilePath);
|
||||
|
||||
if (!File.Exists(fullPath))
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new FileNotFoundException($"Policy file not found: {fullPath}"),
|
||||
"PolicyFileNotFound",
|
||||
ErrorCategory.ObjectNotFound,
|
||||
fullPath));
|
||||
return null;
|
||||
}
|
||||
|
||||
var content = File.ReadAllText(fullPath);
|
||||
MinIOLogger.WriteVerbose(this, "Read policy from file '{0}' ({1} characters)", fullPath, content.Length);
|
||||
|
||||
return content;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new InvalidOperationException($"Failed to read policy file '{PolicyFilePath}': {ex.Message}", ex),
|
||||
"PolicyFileReadError",
|
||||
ErrorCategory.ReadError,
|
||||
PolicyFilePath));
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates a canned policy based on the specified type
|
||||
/// </summary>
|
||||
/// <returns>Policy JSON string</returns>
|
||||
private string GenerateCannedPolicy()
|
||||
{
|
||||
var bucketArn = $"arn:aws:s3:::{BucketName}";
|
||||
var objectArn = $"arn:aws:s3:::{BucketName}/{Prefix}";
|
||||
|
||||
var policy = CannedPolicy?.ToLowerInvariant() switch
|
||||
{
|
||||
"readonly" => CreateReadOnlyPolicy(bucketArn, objectArn),
|
||||
"writeonly" => CreateWriteOnlyPolicy(bucketArn, objectArn),
|
||||
"readwrite" => CreateReadWritePolicy(bucketArn, objectArn),
|
||||
"none" => CreateEmptyPolicy(),
|
||||
_ => throw new ArgumentException($"Unknown canned policy: {CannedPolicy}")
|
||||
};
|
||||
|
||||
MinIOLogger.WriteVerbose(this, "Generated {0} canned policy for bucket '{1}' with prefix '{2}'",
|
||||
CannedPolicy, BucketName, Prefix);
|
||||
|
||||
return JsonSerializer.Serialize(policy, new JsonSerializerOptions { WriteIndented = true });
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a read-only policy
|
||||
/// </summary>
|
||||
private object CreateReadOnlyPolicy(string bucketArn, string objectArn)
|
||||
{
|
||||
return new
|
||||
{
|
||||
Version = "2012-10-17",
|
||||
Statement = new[]
|
||||
{
|
||||
new
|
||||
{
|
||||
Effect = "Allow",
|
||||
Principal = new { AWS = "*" },
|
||||
Action = new[] { "s3:GetBucketLocation", "s3:ListBucket" },
|
||||
Resource = bucketArn
|
||||
},
|
||||
new
|
||||
{
|
||||
Effect = "Allow",
|
||||
Principal = new { AWS = "*" },
|
||||
Action = new[] { "s3:GetObject" },
|
||||
Resource = objectArn
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a write-only policy
|
||||
/// </summary>
|
||||
private object CreateWriteOnlyPolicy(string bucketArn, string objectArn)
|
||||
{
|
||||
return new
|
||||
{
|
||||
Version = "2012-10-17",
|
||||
Statement = new[]
|
||||
{
|
||||
new
|
||||
{
|
||||
Effect = "Allow",
|
||||
Principal = new { AWS = "*" },
|
||||
Action = new[] { "s3:GetBucketLocation", "s3:ListBucketMultipartUploads" },
|
||||
Resource = bucketArn
|
||||
},
|
||||
new
|
||||
{
|
||||
Effect = "Allow",
|
||||
Principal = new { AWS = "*" },
|
||||
Action = new[] { "s3:PutObject", "s3:AbortMultipartUpload", "s3:DeleteObject", "s3:ListMultipartUploadParts" },
|
||||
Resource = objectArn
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a read-write policy
|
||||
/// </summary>
|
||||
private object CreateReadWritePolicy(string bucketArn, string objectArn)
|
||||
{
|
||||
return new
|
||||
{
|
||||
Version = "2012-10-17",
|
||||
Statement = new[]
|
||||
{
|
||||
new
|
||||
{
|
||||
Effect = "Allow",
|
||||
Principal = new { AWS = "*" },
|
||||
Action = new[] { "s3:GetBucketLocation", "s3:ListBucket", "s3:ListBucketMultipartUploads" },
|
||||
Resource = bucketArn
|
||||
},
|
||||
new
|
||||
{
|
||||
Effect = "Allow",
|
||||
Principal = new { AWS = "*" },
|
||||
Action = new[] { "s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts" },
|
||||
Resource = objectArn
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates an empty policy (removes all permissions)
|
||||
/// </summary>
|
||||
private object CreateEmptyPolicy()
|
||||
{
|
||||
return new
|
||||
{
|
||||
Version = "2012-10-17",
|
||||
Statement = new object[0]
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates the policy JSON
|
||||
/// </summary>
|
||||
/// <param name="policyJson">Policy JSON to validate</param>
|
||||
/// <returns>True if valid, false otherwise</returns>
|
||||
private bool ValidatePolicyJson(string policyJson)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var document = JsonDocument.Parse(policyJson);
|
||||
MinIOLogger.WriteVerbose(this, "Policy JSON is valid");
|
||||
return true;
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
WriteError(new ErrorRecord(
|
||||
new ArgumentException($"Invalid policy JSON: {ex.Message}", ex),
|
||||
"InvalidPolicyJson",
|
||||
ErrorCategory.InvalidArgument,
|
||||
policyJson));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,11 @@
|
||||
using System.Management.Automation;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Cmdlets
|
||||
{
|
||||
/// <summary>
|
||||
/// Tests whether a MinIO bucket exists
|
||||
/// </summary>
|
||||
[Cmdlet(VerbsDiagnostic.Test, "MinIOBucketExists", SupportsShouldProcess = false)]
|
||||
[Cmdlet(VerbsDiagnostic.Test, "MinIOBucketExists")]
|
||||
[OutputType(typeof(bool))]
|
||||
public class TestMinIOBucketExistsCmdlet : MinIOBaseCmdlet
|
||||
{
|
||||
@@ -16,137 +15,27 @@ namespace PSMinIO.Cmdlets
|
||||
[Parameter(Position = 0, Mandatory = true, ValueFromPipeline = true, ValueFromPipelineByPropertyName = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Bucket")]
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Return detailed information instead of just true/false
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter Detailed { get; set; }
|
||||
public string Name { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Processes the cmdlet
|
||||
/// </summary>
|
||||
protected override void ProcessRecord()
|
||||
{
|
||||
ValidateConnection();
|
||||
ValidateBucketName(BucketName);
|
||||
ValidateBucketName(Name);
|
||||
|
||||
ExecuteOperation("TestBucketExists", () =>
|
||||
var exists = ExecuteOperation("CheckBucketExists", () =>
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this, "Checking if bucket '{0}' exists", BucketName);
|
||||
WriteVerboseMessage("Checking if bucket '{0}' exists", Name);
|
||||
|
||||
var exists = Client.BucketExists(BucketName);
|
||||
var bucketExists = S3Client.BucketExists(Name);
|
||||
|
||||
WriteVerboseMessage("Bucket '{0}' {1}", Name, bucketExists ? "exists" : "does not exist");
|
||||
|
||||
return bucketExists;
|
||||
}, $"Bucket: {Name}");
|
||||
|
||||
if (Detailed.IsPresent)
|
||||
{
|
||||
// Return detailed information
|
||||
var result = new PSObject();
|
||||
result.Properties.Add(new PSNoteProperty("BucketName", BucketName));
|
||||
result.Properties.Add(new PSNoteProperty("Exists", exists));
|
||||
result.Properties.Add(new PSNoteProperty("Endpoint", Configuration.Endpoint));
|
||||
result.Properties.Add(new PSNoteProperty("CheckedAt", System.DateTime.UtcNow));
|
||||
|
||||
if (exists)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Try to get additional bucket information
|
||||
var allBuckets = Client.ListBuckets();
|
||||
var bucket = allBuckets.Find(b =>
|
||||
string.Equals(b.Name, BucketName, System.StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
if (bucket != null)
|
||||
{
|
||||
result.Properties.Add(new PSNoteProperty("Created", bucket.Created));
|
||||
result.Properties.Add(new PSNoteProperty("Region", bucket.Region));
|
||||
}
|
||||
}
|
||||
catch (System.Exception ex)
|
||||
{
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Could not retrieve additional bucket information: {0}", ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
WriteObject(result);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Return simple boolean result
|
||||
WriteObject(exists);
|
||||
}
|
||||
|
||||
MinIOLogger.WriteVerbose(this,
|
||||
"Bucket '{0}' exists: {1}", BucketName, exists);
|
||||
|
||||
}, $"Bucket: {BucketName}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Custom output type for detailed bucket existence information
|
||||
/// </summary>
|
||||
public class BucketExistenceInfo
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket that was tested
|
||||
/// </summary>
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Whether the bucket exists
|
||||
/// </summary>
|
||||
public bool Exists { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// MinIO endpoint that was checked
|
||||
/// </summary>
|
||||
public string Endpoint { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// When the check was performed
|
||||
/// </summary>
|
||||
public System.DateTime CheckedAt { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Creation date of the bucket (if it exists and information is available)
|
||||
/// </summary>
|
||||
public System.DateTime? Created { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Region of the bucket (if it exists and information is available)
|
||||
/// </summary>
|
||||
public string? Region { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new BucketExistenceInfo instance
|
||||
/// </summary>
|
||||
public BucketExistenceInfo()
|
||||
{
|
||||
CheckedAt = System.DateTime.UtcNow;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new BucketExistenceInfo instance with specified values
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket</param>
|
||||
/// <param name="exists">Whether the bucket exists</param>
|
||||
/// <param name="endpoint">MinIO endpoint</param>
|
||||
public BucketExistenceInfo(string bucketName, bool exists, string endpoint)
|
||||
{
|
||||
BucketName = bucketName ?? string.Empty;
|
||||
Exists = exists;
|
||||
Endpoint = endpoint ?? string.Empty;
|
||||
CheckedAt = System.DateTime.UtcNow;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a string representation of the bucket existence info
|
||||
/// </summary>
|
||||
public override string ToString()
|
||||
{
|
||||
return $"Bucket '{BucketName}' exists: {Exists} (checked at {CheckedAt:yyyy-MM-dd HH:mm:ss} UTC)";
|
||||
WriteObject(exists);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,344 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Core.Http
|
||||
{
|
||||
/// <summary>
|
||||
/// Custom HTTP client for MinIO REST API operations with AWS S3 signature support
|
||||
/// Provides synchronous operations optimized for PowerShell with progress reporting
|
||||
/// </summary>
|
||||
public class MinIOHttpClient : IDisposable
|
||||
{
|
||||
private readonly HttpClient _httpClient;
|
||||
private readonly MinIOConfiguration _configuration;
|
||||
private bool _disposed = false;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIOHttpClient instance
|
||||
/// </summary>
|
||||
/// <param name="configuration">MinIO configuration</param>
|
||||
public MinIOHttpClient(MinIOConfiguration configuration)
|
||||
{
|
||||
_configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
|
||||
|
||||
if (!_configuration.IsValid)
|
||||
{
|
||||
var errors = string.Join(", ", _configuration.GetValidationErrors());
|
||||
throw new ArgumentException($"Invalid MinIO configuration: {errors}", nameof(configuration));
|
||||
}
|
||||
|
||||
// Create HTTP client with custom configuration
|
||||
var handler = new HttpClientHandler();
|
||||
|
||||
// Configure certificate validation if needed
|
||||
if (_configuration.SkipCertificateValidation)
|
||||
{
|
||||
handler.ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => true;
|
||||
}
|
||||
|
||||
_httpClient = new HttpClient(handler)
|
||||
{
|
||||
Timeout = TimeSpan.FromSeconds(_configuration.TimeoutSeconds),
|
||||
BaseAddress = new Uri(_configuration.BaseUrl)
|
||||
};
|
||||
|
||||
// Set default headers
|
||||
_httpClient.DefaultRequestHeaders.Add("User-Agent", "PSMinIO/2.0.0");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes a synchronous HTTP request with AWS S3 signature
|
||||
/// </summary>
|
||||
/// <param name="method">HTTP method</param>
|
||||
/// <param name="path">Request path</param>
|
||||
/// <param name="queryParameters">Query parameters</param>
|
||||
/// <param name="headers">Additional headers</param>
|
||||
/// <param name="content">Request content</param>
|
||||
/// <param name="progressCallback">Progress callback for uploads/downloads</param>
|
||||
/// <returns>HTTP response</returns>
|
||||
public HttpResponseMessage ExecuteRequest(
|
||||
HttpMethod method,
|
||||
string path,
|
||||
Dictionary<string, string>? queryParameters = null,
|
||||
Dictionary<string, string>? headers = null,
|
||||
HttpContent? content = null,
|
||||
Action<long>? progressCallback = null)
|
||||
{
|
||||
var request = CreateRequest(method, path, queryParameters, headers, content);
|
||||
|
||||
// Sign the request with AWS S3 signature
|
||||
SignRequest(request);
|
||||
|
||||
// Execute the request synchronously
|
||||
var response = _httpClient.SendAsync(request).GetAwaiter().GetResult();
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes a synchronous HTTP request and returns the response content as string
|
||||
/// </summary>
|
||||
/// <param name="method">HTTP method</param>
|
||||
/// <param name="path">Request path</param>
|
||||
/// <param name="queryParameters">Query parameters</param>
|
||||
/// <param name="headers">Additional headers</param>
|
||||
/// <param name="content">Request content</param>
|
||||
/// <returns>Response content as string</returns>
|
||||
public string ExecuteRequestForString(
|
||||
HttpMethod method,
|
||||
string path,
|
||||
Dictionary<string, string>? queryParameters = null,
|
||||
Dictionary<string, string>? headers = null,
|
||||
HttpContent? content = null)
|
||||
{
|
||||
using var response = ExecuteRequest(method, path, queryParameters, headers, content);
|
||||
response.EnsureSuccessStatusCode();
|
||||
return response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes a synchronous HTTP request and returns the response content as byte array
|
||||
/// </summary>
|
||||
/// <param name="method">HTTP method</param>
|
||||
/// <param name="path">Request path</param>
|
||||
/// <param name="queryParameters">Query parameters</param>
|
||||
/// <param name="headers">Additional headers</param>
|
||||
/// <param name="content">Request content</param>
|
||||
/// <returns>Response content as byte array</returns>
|
||||
public byte[] ExecuteRequestForBytes(
|
||||
HttpMethod method,
|
||||
string path,
|
||||
Dictionary<string, string>? queryParameters = null,
|
||||
Dictionary<string, string>? headers = null,
|
||||
HttpContent? content = null)
|
||||
{
|
||||
using var response = ExecuteRequest(method, path, queryParameters, headers, content);
|
||||
response.EnsureSuccessStatusCode();
|
||||
return response.Content.ReadAsByteArrayAsync().GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Downloads content to a stream with progress reporting
|
||||
/// </summary>
|
||||
/// <param name="path">Request path</param>
|
||||
/// <param name="outputStream">Output stream</param>
|
||||
/// <param name="queryParameters">Query parameters</param>
|
||||
/// <param name="headers">Additional headers</param>
|
||||
/// <param name="progressCallback">Progress callback</param>
|
||||
/// <returns>Total bytes downloaded</returns>
|
||||
public long DownloadToStream(
|
||||
string path,
|
||||
Stream outputStream,
|
||||
Dictionary<string, string>? queryParameters = null,
|
||||
Dictionary<string, string>? headers = null,
|
||||
Action<long>? progressCallback = null)
|
||||
{
|
||||
using var response = ExecuteRequest(HttpMethod.Get, path, queryParameters, headers);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
using var contentStream = response.Content.ReadAsStreamAsync().GetAwaiter().GetResult();
|
||||
return CopyStreamWithProgress(contentStream, outputStream, progressCallback);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Uploads content from a stream with progress reporting
|
||||
/// </summary>
|
||||
/// <param name="method">HTTP method (PUT or POST)</param>
|
||||
/// <param name="path">Request path</param>
|
||||
/// <param name="inputStream">Input stream</param>
|
||||
/// <param name="contentType">Content type</param>
|
||||
/// <param name="queryParameters">Query parameters</param>
|
||||
/// <param name="headers">Additional headers</param>
|
||||
/// <param name="progressCallback">Progress callback</param>
|
||||
/// <returns>HTTP response</returns>
|
||||
public HttpResponseMessage UploadFromStream(
|
||||
HttpMethod method,
|
||||
string path,
|
||||
Stream inputStream,
|
||||
string contentType = "application/octet-stream",
|
||||
Dictionary<string, string>? queryParameters = null,
|
||||
Dictionary<string, string>? headers = null,
|
||||
Action<long>? progressCallback = null)
|
||||
{
|
||||
var content = new ProgressStreamContent(inputStream, contentType, progressCallback);
|
||||
return ExecuteRequest(method, path, queryParameters, headers, content);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates an HTTP request message
|
||||
/// </summary>
|
||||
private HttpRequestMessage CreateRequest(
|
||||
HttpMethod method,
|
||||
string path,
|
||||
Dictionary<string, string>? queryParameters,
|
||||
Dictionary<string, string>? headers,
|
||||
HttpContent? content)
|
||||
{
|
||||
// Build the request URI
|
||||
var uriBuilder = new UriBuilder(_configuration.BaseUrl)
|
||||
{
|
||||
Path = path
|
||||
};
|
||||
|
||||
if (queryParameters != null && queryParameters.Count > 0)
|
||||
{
|
||||
var query = new StringBuilder();
|
||||
foreach (var kvp in queryParameters)
|
||||
{
|
||||
if (query.Length > 0) query.Append("&");
|
||||
query.Append($"{Uri.EscapeDataString(kvp.Key)}={Uri.EscapeDataString(kvp.Value)}");
|
||||
}
|
||||
uriBuilder.Query = query.ToString();
|
||||
}
|
||||
|
||||
var request = new HttpRequestMessage(method, uriBuilder.Uri)
|
||||
{
|
||||
Content = content
|
||||
};
|
||||
|
||||
// Add custom headers
|
||||
if (headers != null)
|
||||
{
|
||||
foreach (var kvp in headers)
|
||||
{
|
||||
request.Headers.TryAddWithoutValidation(kvp.Key, kvp.Value);
|
||||
}
|
||||
}
|
||||
|
||||
return request;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Signs the HTTP request with AWS S3 signature v4
|
||||
/// </summary>
|
||||
private void SignRequest(HttpRequestMessage request)
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
var dateStamp = now.ToString("yyyyMMdd");
|
||||
var timeStamp = now.ToString("yyyyMMddTHHmmssZ");
|
||||
|
||||
// Add required headers
|
||||
request.Headers.TryAddWithoutValidation("x-amz-date", timeStamp);
|
||||
request.Headers.TryAddWithoutValidation("x-amz-content-sha256", "UNSIGNED-PAYLOAD");
|
||||
|
||||
// Create authorization header
|
||||
var authHeader = CreateAuthorizationHeader(request, dateStamp, timeStamp);
|
||||
request.Headers.TryAddWithoutValidation("Authorization", authHeader);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates the AWS S3 authorization header
|
||||
/// </summary>
|
||||
private string CreateAuthorizationHeader(HttpRequestMessage request, string dateStamp, string timeStamp)
|
||||
{
|
||||
var algorithm = "AWS4-HMAC-SHA256";
|
||||
var credentialScope = $"{dateStamp}/{_configuration.Region}/s3/aws4_request";
|
||||
var credential = $"{_configuration.AccessKey}/{credentialScope}";
|
||||
|
||||
// Create canonical request
|
||||
var canonicalRequest = CreateCanonicalRequest(request);
|
||||
var canonicalRequestHash = ComputeSHA256Hash(canonicalRequest);
|
||||
|
||||
// Create string to sign
|
||||
var stringToSign = $"{algorithm}\n{timeStamp}\n{credentialScope}\n{canonicalRequestHash}";
|
||||
|
||||
// Calculate signature
|
||||
var signature = CalculateSignature(stringToSign, dateStamp);
|
||||
|
||||
return $"{algorithm} Credential={credential}, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature={signature}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates the canonical request for AWS S3 signature
|
||||
/// </summary>
|
||||
private string CreateCanonicalRequest(HttpRequestMessage request)
|
||||
{
|
||||
var method = request.Method.Method;
|
||||
var path = request.RequestUri?.AbsolutePath ?? "/";
|
||||
var query = request.RequestUri?.Query?.TrimStart('?') ?? "";
|
||||
|
||||
// Canonical headers (must be sorted)
|
||||
var canonicalHeaders = "host:" + request.RequestUri?.Host + "\n" +
|
||||
"x-amz-content-sha256:UNSIGNED-PAYLOAD\n" +
|
||||
"x-amz-date:" + request.Headers.GetValues("x-amz-date").FirstOrDefault() + "\n";
|
||||
|
||||
var signedHeaders = "host;x-amz-content-sha256;x-amz-date";
|
||||
var payloadHash = "UNSIGNED-PAYLOAD";
|
||||
|
||||
return $"{method}\n{path}\n{query}\n{canonicalHeaders}\n{signedHeaders}\n{payloadHash}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Calculates the AWS S3 signature
|
||||
/// </summary>
|
||||
private string CalculateSignature(string stringToSign, string dateStamp)
|
||||
{
|
||||
var kSecret = Encoding.UTF8.GetBytes($"AWS4{_configuration.SecretKey}");
|
||||
var kDate = ComputeHMACSHA256(kSecret, dateStamp);
|
||||
var kRegion = ComputeHMACSHA256(kDate, _configuration.Region);
|
||||
var kService = ComputeHMACSHA256(kRegion, "s3");
|
||||
var kSigning = ComputeHMACSHA256(kService, "aws4_request");
|
||||
var signature = ComputeHMACSHA256(kSigning, stringToSign);
|
||||
|
||||
return BitConverter.ToString(signature).Replace("-", "").ToLowerInvariant();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes SHA256 hash
|
||||
/// </summary>
|
||||
private string ComputeSHA256Hash(string input)
|
||||
{
|
||||
using var sha256 = SHA256.Create();
|
||||
var hash = sha256.ComputeHash(Encoding.UTF8.GetBytes(input));
|
||||
return BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes HMAC-SHA256
|
||||
/// </summary>
|
||||
private byte[] ComputeHMACSHA256(byte[] key, string data)
|
||||
{
|
||||
using var hmac = new HMACSHA256(key);
|
||||
return hmac.ComputeHash(Encoding.UTF8.GetBytes(data));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Copies stream with progress reporting
|
||||
/// </summary>
|
||||
private long CopyStreamWithProgress(Stream source, Stream destination, Action<long>? progressCallback)
|
||||
{
|
||||
var buffer = new byte[8192];
|
||||
long totalBytes = 0;
|
||||
int bytesRead;
|
||||
|
||||
while ((bytesRead = source.Read(buffer, 0, buffer.Length)) > 0)
|
||||
{
|
||||
destination.Write(buffer, 0, bytesRead);
|
||||
totalBytes += bytesRead;
|
||||
progressCallback?.Invoke(totalBytes);
|
||||
}
|
||||
|
||||
return totalBytes;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Disposes the HTTP client
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
if (!_disposed)
|
||||
{
|
||||
_httpClient?.Dispose();
|
||||
_disposed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace PSMinIO.Core.Http
|
||||
{
|
||||
/// <summary>
|
||||
/// HTTP content that wraps a stream and provides progress reporting during upload
|
||||
/// </summary>
|
||||
public class ProgressStreamContent : HttpContent
|
||||
{
|
||||
private readonly Stream _stream;
|
||||
private readonly Action<long>? _progressCallback;
|
||||
private readonly long _totalBytes;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new ProgressStreamContent instance
|
||||
/// </summary>
|
||||
/// <param name="stream">Source stream</param>
|
||||
/// <param name="contentType">Content type</param>
|
||||
/// <param name="progressCallback">Progress callback</param>
|
||||
public ProgressStreamContent(Stream stream, string contentType, Action<long>? progressCallback = null)
|
||||
{
|
||||
_stream = stream ?? throw new ArgumentNullException(nameof(stream));
|
||||
_progressCallback = progressCallback;
|
||||
_totalBytes = stream.CanSeek ? stream.Length : -1;
|
||||
|
||||
Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue(contentType);
|
||||
if (_totalBytes >= 0)
|
||||
{
|
||||
Headers.ContentLength = _totalBytes;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Serializes the HTTP content to a stream
|
||||
/// </summary>
|
||||
/// <param name="stream">Target stream</param>
|
||||
/// <param name="context">Transport context</param>
|
||||
/// <returns>Task representing the async operation</returns>
|
||||
protected override async Task SerializeToStreamAsync(Stream stream, TransportContext? context)
|
||||
{
|
||||
const int bufferSize = 8192;
|
||||
var buffer = new byte[bufferSize];
|
||||
long totalBytesRead = 0;
|
||||
int bytesRead;
|
||||
|
||||
// Reset source stream position if possible
|
||||
if (_stream.CanSeek)
|
||||
{
|
||||
_stream.Position = 0;
|
||||
}
|
||||
|
||||
while ((bytesRead = await _stream.ReadAsync(buffer, 0, buffer.Length)) > 0)
|
||||
{
|
||||
await stream.WriteAsync(buffer, 0, bytesRead);
|
||||
totalBytesRead += bytesRead;
|
||||
_progressCallback?.Invoke(totalBytesRead);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether the HTTP content has a valid length in bytes
|
||||
/// </summary>
|
||||
/// <param name="length">Content length</param>
|
||||
/// <returns>True if length is valid</returns>
|
||||
protected override bool TryComputeLength(out long length)
|
||||
{
|
||||
length = _totalBytes;
|
||||
return _totalBytes >= 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Disposes the content
|
||||
/// </summary>
|
||||
/// <param name="disposing">Whether disposing</param>
|
||||
protected override void Dispose(bool disposing)
|
||||
{
|
||||
if (disposing)
|
||||
{
|
||||
_stream?.Dispose();
|
||||
}
|
||||
base.Dispose(disposing);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace PSMinIO.Core
|
||||
{
|
||||
/// <summary>
|
||||
/// Configuration settings for MinIO client connections
|
||||
/// </summary>
|
||||
public class MinIOConfiguration
|
||||
{
|
||||
/// <summary>
|
||||
/// MinIO server endpoint (e.g., "minio.example.com:9000")
|
||||
/// </summary>
|
||||
public string Endpoint { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Access key for authentication
|
||||
/// </summary>
|
||||
public string AccessKey { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Secret key for authentication
|
||||
/// </summary>
|
||||
public string SecretKey { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Whether to use SSL/TLS for connections
|
||||
/// </summary>
|
||||
public bool UseSSL { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Optional region for bucket operations
|
||||
/// </summary>
|
||||
public string Region { get; set; } = "us-east-1";
|
||||
|
||||
/// <summary>
|
||||
/// Connection timeout in seconds
|
||||
/// </summary>
|
||||
public int TimeoutSeconds { get; set; } = 30;
|
||||
|
||||
/// <summary>
|
||||
/// Whether to skip SSL certificate validation (for development/self-signed certificates)
|
||||
/// </summary>
|
||||
public bool SkipCertificateValidation { get; set; } = false;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of concurrent connections
|
||||
/// </summary>
|
||||
public int MaxConnections { get; set; } = 10;
|
||||
|
||||
/// <summary>
|
||||
/// Default chunk size for multipart uploads (in bytes)
|
||||
/// </summary>
|
||||
public long DefaultChunkSize { get; set; } = 5 * 1024 * 1024; // 5MB
|
||||
|
||||
/// <summary>
|
||||
/// Maximum retry attempts for failed operations
|
||||
/// </summary>
|
||||
public int MaxRetries { get; set; } = 3;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIOConfiguration instance
|
||||
/// </summary>
|
||||
public MinIOConfiguration()
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIOConfiguration instance with specified values
|
||||
/// </summary>
|
||||
/// <param name="endpoint">MinIO server endpoint</param>
|
||||
/// <param name="accessKey">Access key</param>
|
||||
/// <param name="secretKey">Secret key</param>
|
||||
/// <param name="useSSL">Whether to use SSL</param>
|
||||
/// <param name="region">Optional region</param>
|
||||
/// <param name="timeoutSeconds">Connection timeout</param>
|
||||
/// <param name="skipCertificateValidation">Whether to skip certificate validation</param>
|
||||
public MinIOConfiguration(
|
||||
string endpoint,
|
||||
string accessKey,
|
||||
string secretKey,
|
||||
bool useSSL = true,
|
||||
string region = "us-east-1",
|
||||
int timeoutSeconds = 30,
|
||||
bool skipCertificateValidation = false)
|
||||
{
|
||||
Endpoint = endpoint ?? throw new ArgumentNullException(nameof(endpoint));
|
||||
AccessKey = accessKey ?? throw new ArgumentNullException(nameof(accessKey));
|
||||
SecretKey = secretKey ?? throw new ArgumentNullException(nameof(secretKey));
|
||||
UseSSL = useSSL;
|
||||
Region = region ?? "us-east-1";
|
||||
TimeoutSeconds = timeoutSeconds;
|
||||
SkipCertificateValidation = skipCertificateValidation;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the base URL for the MinIO server
|
||||
/// </summary>
|
||||
public string BaseUrl => $"{(UseSSL ? "https" : "http")}://{Endpoint}";
|
||||
|
||||
/// <summary>
|
||||
/// Validates the configuration
|
||||
/// </summary>
|
||||
/// <returns>True if configuration is valid</returns>
|
||||
public bool IsValid =>
|
||||
!string.IsNullOrWhiteSpace(Endpoint) &&
|
||||
!string.IsNullOrWhiteSpace(AccessKey) &&
|
||||
!string.IsNullOrWhiteSpace(SecretKey) &&
|
||||
TimeoutSeconds > 0 &&
|
||||
MaxConnections > 0 &&
|
||||
DefaultChunkSize > 0 &&
|
||||
MaxRetries >= 0;
|
||||
|
||||
/// <summary>
|
||||
/// Gets validation errors for the configuration
|
||||
/// </summary>
|
||||
/// <returns>Array of validation error messages</returns>
|
||||
public string[] GetValidationErrors()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (string.IsNullOrWhiteSpace(Endpoint))
|
||||
errors.Add("Endpoint is required");
|
||||
|
||||
if (string.IsNullOrWhiteSpace(AccessKey))
|
||||
errors.Add("AccessKey is required");
|
||||
|
||||
if (string.IsNullOrWhiteSpace(SecretKey))
|
||||
errors.Add("SecretKey is required");
|
||||
|
||||
if (TimeoutSeconds <= 0)
|
||||
errors.Add("TimeoutSeconds must be greater than 0");
|
||||
|
||||
if (MaxConnections <= 0)
|
||||
errors.Add("MaxConnections must be greater than 0");
|
||||
|
||||
if (DefaultChunkSize <= 0)
|
||||
errors.Add("DefaultChunkSize must be greater than 0");
|
||||
|
||||
if (MaxRetries < 0)
|
||||
errors.Add("MaxRetries must be 0 or greater");
|
||||
|
||||
return errors.ToArray();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a string representation of the configuration (without sensitive data)
|
||||
/// </summary>
|
||||
public override string ToString()
|
||||
{
|
||||
return $"MinIO Configuration - Endpoint: {Endpoint}, UseSSL: {UseSSL}, Region: {Region}, Timeout: {TimeoutSeconds}s";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
using System;
|
||||
using PSMinIO.Core.S3;
|
||||
|
||||
namespace PSMinIO.Core
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents an active MinIO connection with configuration and S3 client
|
||||
/// </summary>
|
||||
public class MinIOConnection : IDisposable
|
||||
{
|
||||
private MinIOS3Client? _s3Client;
|
||||
private bool _disposed = false;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the configuration for this connection
|
||||
/// </summary>
|
||||
public MinIOConfiguration Configuration { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the connection status
|
||||
/// </summary>
|
||||
public ConnectionStatus Status { get; private set; } = ConnectionStatus.Disconnected;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the time when the connection was established
|
||||
/// </summary>
|
||||
public DateTime? ConnectedAt { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the last activity time
|
||||
/// </summary>
|
||||
public DateTime? LastActivity { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the MinIO S3 client instance
|
||||
/// </summary>
|
||||
public MinIOS3Client S3Client
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(MinIOConnection));
|
||||
|
||||
if (_s3Client == null)
|
||||
{
|
||||
_s3Client = new MinIOS3Client(Configuration);
|
||||
Status = ConnectionStatus.Connected;
|
||||
ConnectedAt = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
LastActivity = DateTime.UtcNow;
|
||||
return _s3Client;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets whether the connection is valid and ready to use
|
||||
/// </summary>
|
||||
public bool IsValid => !_disposed && Configuration.IsValid && Status != ConnectionStatus.Failed;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIOConnection instance
|
||||
/// </summary>
|
||||
/// <param name="configuration">MinIO configuration</param>
|
||||
public MinIOConnection(MinIOConfiguration configuration)
|
||||
{
|
||||
Configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
|
||||
|
||||
if (!Configuration.IsValid)
|
||||
{
|
||||
var errors = string.Join(", ", Configuration.GetValidationErrors());
|
||||
throw new ArgumentException($"Invalid MinIO configuration: {errors}", nameof(configuration));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tests the connection by attempting to list buckets
|
||||
/// </summary>
|
||||
/// <returns>True if connection test succeeds</returns>
|
||||
public bool TestConnection()
|
||||
{
|
||||
if (_disposed)
|
||||
return false;
|
||||
|
||||
try
|
||||
{
|
||||
// Attempt to list buckets as a connection test
|
||||
var buckets = S3Client.ListBuckets();
|
||||
Status = ConnectionStatus.Connected;
|
||||
return true;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
Status = ConnectionStatus.Failed;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets connection statistics
|
||||
/// </summary>
|
||||
/// <returns>Connection statistics</returns>
|
||||
public ConnectionStats GetStats()
|
||||
{
|
||||
return new ConnectionStats
|
||||
{
|
||||
Status = Status,
|
||||
ConnectedAt = ConnectedAt,
|
||||
LastActivity = LastActivity,
|
||||
Uptime = ConnectedAt.HasValue ? DateTime.UtcNow - ConnectedAt.Value : null,
|
||||
Configuration = Configuration
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a string representation of the connection
|
||||
/// </summary>
|
||||
public override string ToString()
|
||||
{
|
||||
var statusStr = Status switch
|
||||
{
|
||||
ConnectionStatus.Connected => "Connected",
|
||||
ConnectionStatus.Disconnected => "Disconnected",
|
||||
ConnectionStatus.Failed => "Failed",
|
||||
_ => "Unknown"
|
||||
};
|
||||
|
||||
return $"MinIO Connection - {Configuration.Endpoint} ({statusStr})";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Disposes the connection and underlying resources
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
if (!_disposed)
|
||||
{
|
||||
_s3Client?.Dispose();
|
||||
Status = ConnectionStatus.Disconnected;
|
||||
_disposed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Connection status enumeration
|
||||
/// </summary>
|
||||
public enum ConnectionStatus
|
||||
{
|
||||
/// <summary>
|
||||
/// Connection is not established
|
||||
/// </summary>
|
||||
Disconnected,
|
||||
|
||||
/// <summary>
|
||||
/// Connection is active and working
|
||||
/// </summary>
|
||||
Connected,
|
||||
|
||||
/// <summary>
|
||||
/// Connection failed or encountered an error
|
||||
/// </summary>
|
||||
Failed
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Connection statistics
|
||||
/// </summary>
|
||||
public class ConnectionStats
|
||||
{
|
||||
/// <summary>
|
||||
/// Current connection status
|
||||
/// </summary>
|
||||
public ConnectionStatus Status { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When the connection was established
|
||||
/// </summary>
|
||||
public DateTime? ConnectedAt { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Last activity time
|
||||
/// </summary>
|
||||
public DateTime? LastActivity { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Connection uptime
|
||||
/// </summary>
|
||||
public TimeSpan? Uptime { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Connection configuration (without sensitive data)
|
||||
/// </summary>
|
||||
public MinIOConfiguration Configuration { get; set; } = null!;
|
||||
|
||||
/// <summary>
|
||||
/// Returns a string representation of the stats
|
||||
/// </summary>
|
||||
public override string ToString()
|
||||
{
|
||||
var uptimeStr = Uptime.HasValue ? $", Uptime: {Uptime.Value:hh\\:mm\\:ss}" : "";
|
||||
return $"Status: {Status}, Endpoint: {Configuration.Endpoint}{uptimeStr}";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace PSMinIO.Core.Models
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents information about a MinIO bucket
|
||||
/// </summary>
|
||||
public class MinIOBucketInfo
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket
|
||||
/// </summary>
|
||||
public string Name { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Creation date of the bucket
|
||||
/// </summary>
|
||||
public DateTime CreationDate { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Region where the bucket is located
|
||||
/// </summary>
|
||||
public string Region { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Bucket policy (if retrieved)
|
||||
/// </summary>
|
||||
public string? Policy { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Bucket versioning status
|
||||
/// </summary>
|
||||
public BucketVersioningStatus VersioningStatus { get; set; } = BucketVersioningStatus.Unversioned;
|
||||
|
||||
/// <summary>
|
||||
/// Bucket encryption configuration
|
||||
/// </summary>
|
||||
public BucketEncryptionInfo? Encryption { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Bucket tags
|
||||
/// </summary>
|
||||
public Dictionary<string, string> Tags { get; set; } = new Dictionary<string, string>();
|
||||
|
||||
/// <summary>
|
||||
/// Number of objects in the bucket (if counted)
|
||||
/// </summary>
|
||||
public long? ObjectCount { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Total size of all objects in the bucket (if calculated)
|
||||
/// </summary>
|
||||
public long? TotalSize { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIOBucketInfo instance
|
||||
/// </summary>
|
||||
public MinIOBucketInfo()
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIOBucketInfo instance with specified values
|
||||
/// </summary>
|
||||
/// <param name="name">Bucket name</param>
|
||||
/// <param name="creationDate">Creation date</param>
|
||||
/// <param name="region">Bucket region</param>
|
||||
public MinIOBucketInfo(string name, DateTime creationDate, string region = "")
|
||||
{
|
||||
Name = name ?? throw new ArgumentNullException(nameof(name));
|
||||
CreationDate = creationDate;
|
||||
Region = region ?? string.Empty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a string representation of the bucket info
|
||||
/// </summary>
|
||||
public override string ToString()
|
||||
{
|
||||
return $"Bucket: {Name} (Created: {CreationDate:yyyy-MM-dd HH:mm:ss}, Region: {Region})";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether the specified object is equal to the current object
|
||||
/// </summary>
|
||||
public override bool Equals(object? obj)
|
||||
{
|
||||
if (obj is MinIOBucketInfo other)
|
||||
{
|
||||
return string.Equals(Name, other.Name, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a hash code for the current object
|
||||
/// </summary>
|
||||
public override int GetHashCode()
|
||||
{
|
||||
return Name?.ToLowerInvariant()?.GetHashCode() ?? 0;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Bucket versioning status
|
||||
/// </summary>
|
||||
public enum BucketVersioningStatus
|
||||
{
|
||||
/// <summary>
|
||||
/// Versioning is not enabled
|
||||
/// </summary>
|
||||
Unversioned,
|
||||
|
||||
/// <summary>
|
||||
/// Versioning is enabled
|
||||
/// </summary>
|
||||
Enabled,
|
||||
|
||||
/// <summary>
|
||||
/// Versioning is suspended
|
||||
/// </summary>
|
||||
Suspended
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Bucket encryption information
|
||||
/// </summary>
|
||||
public class BucketEncryptionInfo
|
||||
{
|
||||
/// <summary>
|
||||
/// Encryption algorithm (e.g., "AES256", "aws:kms")
|
||||
/// </summary>
|
||||
public string Algorithm { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// KMS key ID (for KMS encryption)
|
||||
/// </summary>
|
||||
public string? KmsKeyId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether encryption is enabled
|
||||
/// </summary>
|
||||
public bool IsEnabled { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new BucketEncryptionInfo instance
|
||||
/// </summary>
|
||||
public BucketEncryptionInfo()
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new BucketEncryptionInfo instance with specified values
|
||||
/// </summary>
|
||||
/// <param name="algorithm">Encryption algorithm</param>
|
||||
/// <param name="isEnabled">Whether encryption is enabled</param>
|
||||
/// <param name="kmsKeyId">Optional KMS key ID</param>
|
||||
public BucketEncryptionInfo(string algorithm, bool isEnabled, string? kmsKeyId = null)
|
||||
{
|
||||
Algorithm = algorithm ?? throw new ArgumentNullException(nameof(algorithm));
|
||||
IsEnabled = isEnabled;
|
||||
KmsKeyId = kmsKeyId;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a string representation of the encryption info
|
||||
/// </summary>
|
||||
public override string ToString()
|
||||
{
|
||||
return $"Encryption: {Algorithm} (Enabled: {IsEnabled})";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,7 @@ using System;
|
||||
using System.Collections.Generic;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Models
|
||||
namespace PSMinIO.Core.Models
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents information about a MinIO object
|
||||
@@ -131,64 +131,6 @@ namespace PSMinIO.Models
|
||||
BucketName = bucketName ?? string.Empty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a MinIOObjectInfo from a Minio.DataModel.Item
|
||||
/// </summary>
|
||||
/// <param name="item">Minio item object</param>
|
||||
/// <param name="bucketName">Name of the bucket containing the object</param>
|
||||
/// <returns>MinIOObjectInfo instance</returns>
|
||||
public static MinIOObjectInfo FromMinioItem(Minio.DataModel.Item item, string bucketName)
|
||||
{
|
||||
if (item == null)
|
||||
throw new ArgumentNullException(nameof(item));
|
||||
|
||||
var objectInfo = new MinIOObjectInfo
|
||||
{
|
||||
Name = item.Key ?? string.Empty,
|
||||
Size = (long)item.Size,
|
||||
LastModified = item.LastModifiedDateTime ?? DateTime.MinValue,
|
||||
ETag = item.ETag ?? string.Empty,
|
||||
BucketName = bucketName ?? string.Empty,
|
||||
StorageClass = string.Empty // StorageClass not available in MinIO 5.0.0 Item
|
||||
};
|
||||
|
||||
// Try to extract version information if available
|
||||
// Note: The MinIO .NET SDK Item object may have version properties
|
||||
try
|
||||
{
|
||||
// Use reflection to check for version properties that might exist
|
||||
var itemType = item.GetType();
|
||||
|
||||
var versionIdProperty = itemType.GetProperty("VersionId");
|
||||
if (versionIdProperty != null)
|
||||
{
|
||||
objectInfo.VersionId = versionIdProperty.GetValue(item)?.ToString();
|
||||
}
|
||||
|
||||
var isLatestProperty = itemType.GetProperty("IsLatest");
|
||||
if (isLatestProperty != null && isLatestProperty.GetValue(item) is bool isLatest)
|
||||
{
|
||||
objectInfo.IsLatestVersion = isLatest;
|
||||
}
|
||||
|
||||
var isDeleteMarkerProperty = itemType.GetProperty("IsDeleteMarker");
|
||||
if (isDeleteMarkerProperty != null && isDeleteMarkerProperty.GetValue(item) is bool isDeleteMarker)
|
||||
{
|
||||
objectInfo.IsDeleteMarker = isDeleteMarker;
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// If reflection fails, just continue without version information
|
||||
// This ensures compatibility even if the SDK doesn't have these properties
|
||||
}
|
||||
|
||||
// Metadata not available in MinIO 5.0.0 Item class
|
||||
// objectInfo.Metadata remains empty
|
||||
|
||||
return objectInfo;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the file extension of the object
|
||||
/// </summary>
|
||||
@@ -245,7 +187,7 @@ namespace PSMinIO.Models
|
||||
/// </summary>
|
||||
public override string ToString()
|
||||
{
|
||||
return $"Object: {Name} ({Size} bytes, Modified: {LastModified:yyyy-MM-dd HH:mm:ss})";
|
||||
return $"Object: {Name} ({SizeFormatter.FormatBytes(Size)}, Modified: {LastModified:yyyy-MM-dd HH:mm:ss})";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -0,0 +1,292 @@
|
||||
using System;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Core.Models
|
||||
{
|
||||
/// <summary>
|
||||
/// Base class for MinIO operation results with performance metrics
|
||||
/// </summary>
|
||||
public abstract class MinIOOperationResult
|
||||
{
|
||||
/// <summary>
|
||||
/// When the operation started
|
||||
/// </summary>
|
||||
public DateTime? StartTime { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When the operation completed
|
||||
/// </summary>
|
||||
public DateTime? EndTime { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Duration of the operation
|
||||
/// </summary>
|
||||
public TimeSpan? Duration => StartTime.HasValue && EndTime.HasValue
|
||||
? EndTime.Value - StartTime.Value
|
||||
: null;
|
||||
|
||||
/// <summary>
|
||||
/// Duration formatted as a human-readable string
|
||||
/// </summary>
|
||||
public string? DurationFormatted => Duration.HasValue
|
||||
? SizeFormatter.FormatDuration(Duration.Value)
|
||||
: null;
|
||||
|
||||
/// <summary>
|
||||
/// Whether the operation was successful
|
||||
/// </summary>
|
||||
public bool Success { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Error message if the operation failed
|
||||
/// </summary>
|
||||
public string? ErrorMessage { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Sets the start time to the current UTC time
|
||||
/// </summary>
|
||||
public void MarkStarted()
|
||||
{
|
||||
StartTime = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets the end time to the current UTC time
|
||||
/// </summary>
|
||||
public void MarkCompleted()
|
||||
{
|
||||
EndTime = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Marks the operation as failed with an error message
|
||||
/// </summary>
|
||||
/// <param name="errorMessage">Error message</param>
|
||||
public void MarkFailed(string errorMessage)
|
||||
{
|
||||
EndTime = DateTime.UtcNow;
|
||||
Success = false;
|
||||
ErrorMessage = errorMessage;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result for transfer operations (upload/download) with speed metrics
|
||||
/// </summary>
|
||||
public abstract class MinIOTransferResult : MinIOOperationResult
|
||||
{
|
||||
/// <summary>
|
||||
/// Number of bytes transferred
|
||||
/// </summary>
|
||||
public long BytesTransferred { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Total size of the transfer
|
||||
/// </summary>
|
||||
public long TotalSize { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Average transfer speed in bytes per second
|
||||
/// </summary>
|
||||
public double? AverageSpeed => Duration.HasValue && Duration.Value.TotalSeconds > 0
|
||||
? BytesTransferred / Duration.Value.TotalSeconds
|
||||
: null;
|
||||
|
||||
/// <summary>
|
||||
/// Average transfer speed formatted as a human-readable string
|
||||
/// </summary>
|
||||
public string? AverageSpeedFormatted => AverageSpeed.HasValue
|
||||
? SizeFormatter.FormatSpeed(AverageSpeed.Value)
|
||||
: null;
|
||||
|
||||
/// <summary>
|
||||
/// Bytes transferred formatted as a human-readable string
|
||||
/// </summary>
|
||||
public string BytesTransferredFormatted => SizeFormatter.FormatBytes(BytesTransferred);
|
||||
|
||||
/// <summary>
|
||||
/// Total size formatted as a human-readable string
|
||||
/// </summary>
|
||||
public string TotalSizeFormatted => SizeFormatter.FormatBytes(TotalSize);
|
||||
|
||||
/// <summary>
|
||||
/// Transfer completion percentage
|
||||
/// </summary>
|
||||
public double PercentComplete => TotalSize > 0 ? (double)BytesTransferred / TotalSize * 100 : 100;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result for upload operations
|
||||
/// </summary>
|
||||
public class MinIOUploadResult : MinIOTransferResult
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket
|
||||
/// </summary>
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Name of the uploaded object
|
||||
/// </summary>
|
||||
public string ObjectName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// ETag of the uploaded object
|
||||
/// </summary>
|
||||
public string ETag { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Content type of the uploaded object
|
||||
/// </summary>
|
||||
public string ContentType { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Source file path (if uploaded from file)
|
||||
/// </summary>
|
||||
public string? SourceFilePath { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIOUploadResult
|
||||
/// </summary>
|
||||
public MinIOUploadResult()
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIOUploadResult with specified values
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Bucket name</param>
|
||||
/// <param name="objectName">Object name</param>
|
||||
/// <param name="etag">ETag</param>
|
||||
/// <param name="totalSize">Total size</param>
|
||||
public MinIOUploadResult(string bucketName, string objectName, string etag, long totalSize)
|
||||
{
|
||||
BucketName = bucketName ?? string.Empty;
|
||||
ObjectName = objectName ?? string.Empty;
|
||||
ETag = etag ?? string.Empty;
|
||||
TotalSize = totalSize;
|
||||
BytesTransferred = totalSize; // Assume complete transfer
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a string representation of the upload result
|
||||
/// </summary>
|
||||
public override string ToString()
|
||||
{
|
||||
var status = Success ? "Success" : "Failed";
|
||||
var speed = AverageSpeedFormatted ?? "Unknown";
|
||||
var duration = DurationFormatted ?? "Unknown";
|
||||
|
||||
return $"Upload {status}: {ObjectName} ({TotalSizeFormatted}) in {duration} at {speed}";
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result for download operations
|
||||
/// </summary>
|
||||
public class MinIODownloadResult : MinIOTransferResult
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket
|
||||
/// </summary>
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Name of the downloaded object
|
||||
/// </summary>
|
||||
public string ObjectName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Target file path (if downloaded to file)
|
||||
/// </summary>
|
||||
public string? TargetFilePath { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Content type of the downloaded object
|
||||
/// </summary>
|
||||
public string ContentType { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Last modified date of the object
|
||||
/// </summary>
|
||||
public DateTime? LastModified { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIODownloadResult
|
||||
/// </summary>
|
||||
public MinIODownloadResult()
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIODownloadResult with specified values
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Bucket name</param>
|
||||
/// <param name="objectName">Object name</param>
|
||||
/// <param name="totalSize">Total size</param>
|
||||
public MinIODownloadResult(string bucketName, string objectName, long totalSize)
|
||||
{
|
||||
BucketName = bucketName ?? string.Empty;
|
||||
ObjectName = objectName ?? string.Empty;
|
||||
TotalSize = totalSize;
|
||||
BytesTransferred = totalSize; // Assume complete transfer
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a string representation of the download result
|
||||
/// </summary>
|
||||
public override string ToString()
|
||||
{
|
||||
var status = Success ? "Success" : "Failed";
|
||||
var speed = AverageSpeedFormatted ?? "Unknown";
|
||||
var duration = DurationFormatted ?? "Unknown";
|
||||
|
||||
return $"Download {status}: {ObjectName} ({TotalSizeFormatted}) in {duration} at {speed}";
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result for bucket operations
|
||||
/// </summary>
|
||||
public class MinIOBucketResult : MinIOOperationResult
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket
|
||||
/// </summary>
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Operation that was performed
|
||||
/// </summary>
|
||||
public string Operation { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIOBucketResult
|
||||
/// </summary>
|
||||
public MinIOBucketResult()
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIOBucketResult with specified values
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Bucket name</param>
|
||||
/// <param name="operation">Operation performed</param>
|
||||
public MinIOBucketResult(string bucketName, string operation)
|
||||
{
|
||||
BucketName = bucketName ?? string.Empty;
|
||||
Operation = operation ?? string.Empty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a string representation of the bucket result
|
||||
/// </summary>
|
||||
public override string ToString()
|
||||
{
|
||||
var status = Success ? "Success" : "Failed";
|
||||
var duration = DurationFormatted ?? "Unknown";
|
||||
|
||||
return $"{Operation} {status}: {BucketName} in {duration}";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,435 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Net.Http;
|
||||
using System.Text.Json;
|
||||
using System.Xml.Linq;
|
||||
using PSMinIO.Core.Http;
|
||||
using PSMinIO.Core.Models;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Core.S3
|
||||
{
|
||||
/// <summary>
|
||||
/// MinIO S3 API client implementation using custom REST API calls
|
||||
/// Provides synchronous operations optimized for PowerShell with real progress reporting
|
||||
/// </summary>
|
||||
public class MinIOS3Client : IDisposable
|
||||
{
|
||||
private readonly MinIOHttpClient _httpClient;
|
||||
private readonly MinIOConfiguration _configuration;
|
||||
private bool _disposed = false;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIOS3Client instance
|
||||
/// </summary>
|
||||
/// <param name="configuration">MinIO configuration</param>
|
||||
public MinIOS3Client(MinIOConfiguration configuration)
|
||||
{
|
||||
_configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
|
||||
_httpClient = new MinIOHttpClient(configuration);
|
||||
}
|
||||
|
||||
#region Bucket Operations
|
||||
|
||||
/// <summary>
|
||||
/// Lists all buckets
|
||||
/// </summary>
|
||||
/// <returns>List of bucket information</returns>
|
||||
public List<MinIOBucketInfo> ListBuckets()
|
||||
{
|
||||
try
|
||||
{
|
||||
var response = _httpClient.ExecuteRequestForString(HttpMethod.Get, "/");
|
||||
return ParseListBucketsResponse(response);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to list buckets: {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if a bucket exists
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket</param>
|
||||
/// <returns>True if bucket exists</returns>
|
||||
public bool BucketExists(string bucketName)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
try
|
||||
{
|
||||
using var response = _httpClient.ExecuteRequest(HttpMethod.Head, $"/{bucketName}");
|
||||
return response.IsSuccessStatusCode;
|
||||
}
|
||||
catch (HttpRequestException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to check if bucket '{bucketName}' exists: {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new bucket
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket to create</param>
|
||||
/// <param name="region">Optional region for the bucket</param>
|
||||
public void CreateBucket(string bucketName, string? region = null)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
try
|
||||
{
|
||||
HttpContent? content = null;
|
||||
|
||||
// If region is specified and different from default, create location constraint
|
||||
if (!string.IsNullOrEmpty(region) && region != "us-east-1")
|
||||
{
|
||||
var locationConstraint = $@"
|
||||
<CreateBucketConfiguration>
|
||||
<LocationConstraint>{region}</LocationConstraint>
|
||||
</CreateBucketConfiguration>";
|
||||
content = new StringContent(locationConstraint, System.Text.Encoding.UTF8, "application/xml");
|
||||
}
|
||||
|
||||
using var response = _httpClient.ExecuteRequest(HttpMethod.Put, $"/{bucketName}", content: content);
|
||||
response.EnsureSuccessStatusCode();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to create bucket '{bucketName}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes a bucket
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket to delete</param>
|
||||
public void DeleteBucket(string bucketName)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
try
|
||||
{
|
||||
using var response = _httpClient.ExecuteRequest(HttpMethod.Delete, $"/{bucketName}");
|
||||
response.EnsureSuccessStatusCode();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to delete bucket '{bucketName}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets bucket policy
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket</param>
|
||||
/// <returns>Bucket policy as JSON string</returns>
|
||||
public string GetBucketPolicy(string bucketName)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
try
|
||||
{
|
||||
var queryParams = new Dictionary<string, string> { { "policy", "" } };
|
||||
return _httpClient.ExecuteRequestForString(HttpMethod.Get, $"/{bucketName}", queryParams);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to get policy for bucket '{bucketName}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets bucket policy
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket</param>
|
||||
/// <param name="policy">Policy JSON string</param>
|
||||
public void SetBucketPolicy(string bucketName, string policy)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(policy))
|
||||
throw new ArgumentException("Policy cannot be null or empty", nameof(policy));
|
||||
|
||||
try
|
||||
{
|
||||
var queryParams = new Dictionary<string, string> { { "policy", "" } };
|
||||
var content = new StringContent(policy, System.Text.Encoding.UTF8, "application/json");
|
||||
|
||||
using var response = _httpClient.ExecuteRequest(HttpMethod.Put, $"/{bucketName}", queryParams, content: content);
|
||||
response.EnsureSuccessStatusCode();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to set policy for bucket '{bucketName}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Object Operations
|
||||
|
||||
/// <summary>
|
||||
/// Lists objects in a bucket
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket</param>
|
||||
/// <param name="prefix">Optional prefix to filter objects</param>
|
||||
/// <param name="recursive">Whether to list objects recursively</param>
|
||||
/// <param name="maxObjects">Maximum number of objects to return</param>
|
||||
/// <returns>List of object information</returns>
|
||||
public List<MinIOObjectInfo> ListObjects(string bucketName, string? prefix = null, bool recursive = true, int maxObjects = 1000)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
try
|
||||
{
|
||||
var queryParams = new Dictionary<string, string>
|
||||
{
|
||||
{ "list-type", "2" },
|
||||
{ "max-keys", maxObjects.ToString() }
|
||||
};
|
||||
|
||||
if (!string.IsNullOrEmpty(prefix))
|
||||
queryParams["prefix"] = prefix;
|
||||
|
||||
if (!recursive)
|
||||
queryParams["delimiter"] = "/";
|
||||
|
||||
var response = _httpClient.ExecuteRequestForString(HttpMethod.Get, $"/{bucketName}", queryParams);
|
||||
return ParseListObjectsResponse(response, bucketName);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to list objects in bucket '{bucketName}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Uploads an object from a stream
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket</param>
|
||||
/// <param name="objectName">Name of the object</param>
|
||||
/// <param name="stream">Source stream</param>
|
||||
/// <param name="contentType">Content type</param>
|
||||
/// <param name="metadata">Optional metadata</param>
|
||||
/// <param name="progressCallback">Progress callback</param>
|
||||
/// <returns>ETag of the uploaded object</returns>
|
||||
public string PutObject(
|
||||
string bucketName,
|
||||
string objectName,
|
||||
Stream stream,
|
||||
string contentType = "application/octet-stream",
|
||||
Dictionary<string, string>? metadata = null,
|
||||
Action<long>? progressCallback = null)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(objectName))
|
||||
throw new ArgumentException("Object name cannot be null or empty", nameof(objectName));
|
||||
|
||||
if (stream == null)
|
||||
throw new ArgumentNullException(nameof(stream));
|
||||
|
||||
try
|
||||
{
|
||||
var headers = new Dictionary<string, string>();
|
||||
|
||||
// Add metadata headers
|
||||
if (metadata != null)
|
||||
{
|
||||
foreach (var kvp in metadata)
|
||||
{
|
||||
headers[$"x-amz-meta-{kvp.Key}"] = kvp.Value;
|
||||
}
|
||||
}
|
||||
|
||||
using var response = _httpClient.UploadFromStream(
|
||||
HttpMethod.Put,
|
||||
$"/{bucketName}/{objectName}",
|
||||
stream,
|
||||
contentType,
|
||||
headers: headers,
|
||||
progressCallback: progressCallback);
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
// Extract ETag from response headers
|
||||
if (response.Headers.ETag != null)
|
||||
{
|
||||
return response.Headers.ETag.Tag.Trim('"');
|
||||
}
|
||||
|
||||
return string.Empty;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to upload object '{objectName}' to bucket '{bucketName}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Downloads an object to a stream
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket</param>
|
||||
/// <param name="objectName">Name of the object</param>
|
||||
/// <param name="stream">Target stream</param>
|
||||
/// <param name="progressCallback">Progress callback</param>
|
||||
/// <returns>Number of bytes downloaded</returns>
|
||||
public long GetObject(
|
||||
string bucketName,
|
||||
string objectName,
|
||||
Stream stream,
|
||||
Action<long>? progressCallback = null)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(objectName))
|
||||
throw new ArgumentException("Object name cannot be null or empty", nameof(objectName));
|
||||
|
||||
if (stream == null)
|
||||
throw new ArgumentNullException(nameof(stream));
|
||||
|
||||
try
|
||||
{
|
||||
return _httpClient.DownloadToStream($"/{bucketName}/{objectName}", stream, progressCallback: progressCallback);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to download object '{objectName}' from bucket '{bucketName}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes an object
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket</param>
|
||||
/// <param name="objectName">Name of the object</param>
|
||||
public void DeleteObject(string bucketName, string objectName)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(objectName))
|
||||
throw new ArgumentException("Object name cannot be null or empty", nameof(objectName));
|
||||
|
||||
try
|
||||
{
|
||||
using var response = _httpClient.ExecuteRequest(HttpMethod.Delete, $"/{bucketName}/{objectName}");
|
||||
response.EnsureSuccessStatusCode();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to delete object '{objectName}' from bucket '{bucketName}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
/// <summary>
|
||||
/// Parses the list buckets XML response
|
||||
/// </summary>
|
||||
private List<MinIOBucketInfo> ParseListBucketsResponse(string xmlResponse)
|
||||
{
|
||||
var buckets = new List<MinIOBucketInfo>();
|
||||
|
||||
try
|
||||
{
|
||||
var doc = XDocument.Parse(xmlResponse);
|
||||
var ns = doc.Root?.GetDefaultNamespace();
|
||||
|
||||
var bucketElements = doc.Descendants(ns + "Bucket");
|
||||
foreach (var bucketElement in bucketElements)
|
||||
{
|
||||
var name = bucketElement.Element(ns + "Name")?.Value ?? string.Empty;
|
||||
var creationDateStr = bucketElement.Element(ns + "CreationDate")?.Value ?? string.Empty;
|
||||
|
||||
if (DateTime.TryParse(creationDateStr, out var creationDate))
|
||||
{
|
||||
buckets.Add(new MinIOBucketInfo(name, creationDate));
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to parse list buckets response: {ex.Message}", ex);
|
||||
}
|
||||
|
||||
return buckets;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parses the list objects XML response
|
||||
/// </summary>
|
||||
private List<MinIOObjectInfo> ParseListObjectsResponse(string xmlResponse, string bucketName)
|
||||
{
|
||||
var objects = new List<MinIOObjectInfo>();
|
||||
|
||||
try
|
||||
{
|
||||
var doc = XDocument.Parse(xmlResponse);
|
||||
var ns = doc.Root?.GetDefaultNamespace();
|
||||
|
||||
var contentElements = doc.Descendants(ns + "Contents");
|
||||
foreach (var contentElement in contentElements)
|
||||
{
|
||||
var keyElement = contentElement.Element(ns + "Key");
|
||||
var lastModifiedElement = contentElement.Element(ns + "LastModified");
|
||||
var etagElement = contentElement.Element(ns + "ETag");
|
||||
var sizeElement = contentElement.Element(ns + "Size");
|
||||
var storageClassElement = contentElement.Element(ns + "StorageClass");
|
||||
|
||||
var key = keyElement?.Value ?? string.Empty;
|
||||
var lastModifiedStr = lastModifiedElement?.Value ?? string.Empty;
|
||||
var etag = etagElement?.Value?.Trim('"') ?? string.Empty;
|
||||
var sizeStr = sizeElement?.Value ?? "0";
|
||||
var storageClass = storageClassElement?.Value ?? string.Empty;
|
||||
|
||||
if (DateTime.TryParse(lastModifiedStr, out var lastModified) &&
|
||||
long.TryParse(sizeStr, out var size))
|
||||
{
|
||||
var objectInfo = new MinIOObjectInfo(key, size, lastModified, etag, bucketName)
|
||||
{
|
||||
StorageClass = storageClass
|
||||
};
|
||||
objects.Add(objectInfo);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to parse list objects response: {ex.Message}", ex);
|
||||
}
|
||||
|
||||
return objects;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
/// <summary>
|
||||
/// Disposes the S3 client
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
if (!_disposed)
|
||||
{
|
||||
_httpClient?.Dispose();
|
||||
_disposed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,203 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
|
||||
namespace PSMinIO.Models
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents the state of a chunked transfer operation for resume functionality
|
||||
/// </summary>
|
||||
public class ChunkedTransferState
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket
|
||||
/// </summary>
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Name of the object
|
||||
/// </summary>
|
||||
public string ObjectName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Local file path
|
||||
/// </summary>
|
||||
public string FilePath { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Total size of the file/object
|
||||
/// </summary>
|
||||
public long TotalSize { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Size of each chunk
|
||||
/// </summary>
|
||||
public long ChunkSize { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// List of completed chunks
|
||||
/// </summary>
|
||||
public List<ChunkInfo> CompletedChunks { get; set; } = new List<ChunkInfo>();
|
||||
|
||||
/// <summary>
|
||||
/// Last modified time of the source file (for validation)
|
||||
/// </summary>
|
||||
public DateTime LastModified { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// ETag of the object (for validation)
|
||||
/// </summary>
|
||||
public string ETag { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Transfer operation type
|
||||
/// </summary>
|
||||
public ChunkedTransferType TransferType { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When the transfer was started
|
||||
/// </summary>
|
||||
public DateTime StartTime { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When the transfer state was last updated
|
||||
/// </summary>
|
||||
public DateTime LastUpdated { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Upload ID for multipart uploads (MinIO specific)
|
||||
/// </summary>
|
||||
public string? UploadId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the total number of chunks
|
||||
/// </summary>
|
||||
public int TotalChunks => (int)Math.Ceiling((double)TotalSize / ChunkSize);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the number of completed chunks
|
||||
/// </summary>
|
||||
public int CompletedChunkCount => CompletedChunks.Count;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the total bytes transferred
|
||||
/// </summary>
|
||||
public long BytesTransferred => CompletedChunks.Sum(c => c.Size);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the transfer progress as a percentage
|
||||
/// </summary>
|
||||
public double ProgressPercentage => TotalSize > 0 ? (double)BytesTransferred / TotalSize * 100 : 0;
|
||||
|
||||
/// <summary>
|
||||
/// Checks if the transfer is complete
|
||||
/// </summary>
|
||||
public bool IsComplete => CompletedChunkCount >= TotalChunks;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the next chunk to transfer
|
||||
/// </summary>
|
||||
/// <returns>Next chunk info or null if complete</returns>
|
||||
public ChunkInfo? GetNextChunk()
|
||||
{
|
||||
for (int i = 0; i < TotalChunks; i++)
|
||||
{
|
||||
if (!CompletedChunks.Any(c => c.ChunkNumber == i))
|
||||
{
|
||||
var startByte = i * ChunkSize;
|
||||
var endByte = Math.Min(startByte + ChunkSize - 1, TotalSize - 1);
|
||||
var size = endByte - startByte + 1;
|
||||
|
||||
return new ChunkInfo
|
||||
{
|
||||
ChunkNumber = i,
|
||||
StartByte = startByte,
|
||||
EndByte = endByte,
|
||||
Size = size,
|
||||
IsCompleted = false
|
||||
};
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Marks a chunk as completed
|
||||
/// </summary>
|
||||
/// <param name="chunkInfo">Completed chunk information</param>
|
||||
public void MarkChunkCompleted(ChunkInfo chunkInfo)
|
||||
{
|
||||
chunkInfo.IsCompleted = true;
|
||||
chunkInfo.CompletedTime = DateTime.UtcNow;
|
||||
|
||||
// Remove any existing entry for this chunk number
|
||||
CompletedChunks.RemoveAll(c => c.ChunkNumber == chunkInfo.ChunkNumber);
|
||||
|
||||
// Add the completed chunk
|
||||
CompletedChunks.Add(chunkInfo);
|
||||
|
||||
LastUpdated = DateTime.UtcNow;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Information about a single chunk
|
||||
/// </summary>
|
||||
public class ChunkInfo
|
||||
{
|
||||
/// <summary>
|
||||
/// Chunk number (0-based)
|
||||
/// </summary>
|
||||
public int ChunkNumber { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Starting byte position
|
||||
/// </summary>
|
||||
public long StartByte { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Ending byte position (inclusive)
|
||||
/// </summary>
|
||||
public long EndByte { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Size of the chunk in bytes
|
||||
/// </summary>
|
||||
public long Size { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// ETag of the uploaded chunk (for uploads)
|
||||
/// </summary>
|
||||
public string ChunkETag { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Whether this chunk has been completed
|
||||
/// </summary>
|
||||
public bool IsCompleted { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When this chunk was completed
|
||||
/// </summary>
|
||||
public DateTime? CompletedTime { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of retry attempts for this chunk
|
||||
/// </summary>
|
||||
public int RetryCount { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Last error encountered for this chunk
|
||||
/// </summary>
|
||||
public string? LastError { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Type of chunked transfer operation
|
||||
/// </summary>
|
||||
public enum ChunkedTransferType
|
||||
{
|
||||
Upload,
|
||||
Download
|
||||
}
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
using System;
|
||||
|
||||
namespace PSMinIO.Models
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents information about a MinIO bucket
|
||||
/// </summary>
|
||||
public class MinIOBucketInfo
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket
|
||||
/// </summary>
|
||||
public string Name { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Creation date of the bucket
|
||||
/// </summary>
|
||||
public DateTime Created { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Region where the bucket is located
|
||||
/// </summary>
|
||||
public string Region { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Total size of all objects in the bucket (optional, calculated separately)
|
||||
/// </summary>
|
||||
public long? Size { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of objects in the bucket (optional, calculated separately)
|
||||
/// </summary>
|
||||
public long? ObjectCount { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIOBucketInfo instance
|
||||
/// </summary>
|
||||
public MinIOBucketInfo()
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIOBucketInfo instance with specified values
|
||||
/// </summary>
|
||||
/// <param name="name">Bucket name</param>
|
||||
/// <param name="created">Creation date</param>
|
||||
/// <param name="region">Bucket region</param>
|
||||
public MinIOBucketInfo(string name, DateTime created, string region = "")
|
||||
{
|
||||
Name = name ?? string.Empty;
|
||||
Created = created;
|
||||
Region = region ?? string.Empty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a MinIOBucketInfo from a Minio.DataModel.Bucket
|
||||
/// </summary>
|
||||
/// <param name="bucket">Minio bucket object</param>
|
||||
/// <returns>MinIOBucketInfo instance</returns>
|
||||
public static MinIOBucketInfo FromMinioBucket(Minio.DataModel.Bucket bucket)
|
||||
{
|
||||
if (bucket == null)
|
||||
throw new ArgumentNullException(nameof(bucket));
|
||||
|
||||
return new MinIOBucketInfo
|
||||
{
|
||||
Name = bucket.Name ?? string.Empty,
|
||||
Created = DateTime.TryParse(bucket.CreationDate, out var createdDate) ? createdDate : DateTime.MinValue,
|
||||
Region = string.Empty // Region is not available in basic bucket info
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a string representation of the bucket info
|
||||
/// </summary>
|
||||
public override string ToString()
|
||||
{
|
||||
return $"Bucket: {Name} (Created: {Created:yyyy-MM-dd HH:mm:ss})";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether the specified object is equal to the current object
|
||||
/// </summary>
|
||||
public override bool Equals(object? obj)
|
||||
{
|
||||
if (obj is MinIOBucketInfo other)
|
||||
{
|
||||
return string.Equals(Name, other.Name, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a hash code for the current object
|
||||
/// </summary>
|
||||
public override int GetHashCode()
|
||||
{
|
||||
return Name?.ToLowerInvariant().GetHashCode() ?? 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,84 +0,0 @@
|
||||
using System;
|
||||
|
||||
namespace PSMinIO.Models
|
||||
{
|
||||
/// <summary>
|
||||
/// Configuration settings for MinIO client connections
|
||||
/// </summary>
|
||||
public class MinIOConfiguration
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// MinIO server endpoint (without protocol)
|
||||
/// </summary>
|
||||
public string Endpoint { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Access key for authentication
|
||||
/// </summary>
|
||||
public string AccessKey { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Secret key for authentication
|
||||
/// </summary>
|
||||
public string SecretKey { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Whether to use SSL/TLS for connections
|
||||
/// </summary>
|
||||
public bool UseSSL { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Optional region for bucket operations
|
||||
/// </summary>
|
||||
public string Region { get; set; } = "us-east-1";
|
||||
|
||||
/// <summary>
|
||||
/// Connection timeout in seconds
|
||||
/// </summary>
|
||||
public int TimeoutSeconds { get; set; } = 30;
|
||||
|
||||
/// <summary>
|
||||
/// Whether to skip SSL certificate validation
|
||||
/// </summary>
|
||||
public bool SkipCertificateValidation { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether the configuration is valid for creating a client
|
||||
/// </summary>
|
||||
public bool IsValid => !string.IsNullOrWhiteSpace(Endpoint) &&
|
||||
!string.IsNullOrWhiteSpace(AccessKey) &&
|
||||
!string.IsNullOrWhiteSpace(SecretKey);
|
||||
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIOConfiguration instance
|
||||
/// </summary>
|
||||
public MinIOConfiguration()
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIOConfiguration instance with specified values
|
||||
/// </summary>
|
||||
/// <param name="endpoint">MinIO endpoint</param>
|
||||
/// <param name="accessKey">Access key</param>
|
||||
/// <param name="secretKey">Secret key</param>
|
||||
/// <param name="useSSL">Use SSL flag</param>
|
||||
/// <param name="region">Optional region</param>
|
||||
/// <param name="timeoutSeconds">Connection timeout</param>
|
||||
/// <param name="skipCertificateValidation">Whether to skip SSL certificate validation</param>
|
||||
public MinIOConfiguration(string endpoint, string accessKey, string secretKey,
|
||||
bool useSSL = true, string region = "us-east-1", int timeoutSeconds = 30, bool skipCertificateValidation = false)
|
||||
{
|
||||
Endpoint = endpoint?.Trim() ?? string.Empty;
|
||||
AccessKey = accessKey?.Trim() ?? string.Empty;
|
||||
SecretKey = secretKey?.Trim() ?? string.Empty;
|
||||
UseSSL = useSSL;
|
||||
Region = region?.Trim() ?? "us-east-1";
|
||||
TimeoutSeconds = timeoutSeconds;
|
||||
SkipCertificateValidation = skipCertificateValidation;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,202 +0,0 @@
|
||||
using System;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Models
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents an active MinIO connection with configuration and client wrapper
|
||||
/// </summary>
|
||||
public class MinIOConnection : IDisposable
|
||||
{
|
||||
private MinIOClientWrapper? _client;
|
||||
private bool _disposed = false;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the configuration for this connection
|
||||
/// </summary>
|
||||
public MinIOConfiguration Configuration { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the MinIO client wrapper instance
|
||||
/// </summary>
|
||||
public MinIOClientWrapper Client
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(MinIOConnection));
|
||||
|
||||
if (_client == null)
|
||||
{
|
||||
_client = new MinIOClientWrapper(Configuration);
|
||||
}
|
||||
return _client;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets whether this connection is valid and ready to use
|
||||
/// </summary>
|
||||
public bool IsValid => Configuration.IsValid && !_disposed;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the connection status
|
||||
/// </summary>
|
||||
public string Status
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_disposed) return "Disposed";
|
||||
if (!Configuration.IsValid) return "Invalid Configuration";
|
||||
return "Ready";
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the endpoint URL for display purposes
|
||||
/// </summary>
|
||||
public string EndpointUrl
|
||||
{
|
||||
get
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(Configuration.Endpoint))
|
||||
return "Not Set";
|
||||
|
||||
var protocol = Configuration.UseSSL ? "https" : "http";
|
||||
return $"{protocol}://{Configuration.Endpoint}";
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets when this connection was created
|
||||
/// </summary>
|
||||
public DateTime CreatedAt { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIOConnection instance
|
||||
/// </summary>
|
||||
/// <param name="configuration">MinIO configuration</param>
|
||||
public MinIOConnection(MinIOConfiguration configuration)
|
||||
{
|
||||
Configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
|
||||
CreatedAt = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tests the connection by attempting to list buckets
|
||||
/// </summary>
|
||||
/// <returns>Connection test result</returns>
|
||||
public ConnectionTestResult TestConnection()
|
||||
{
|
||||
if (_disposed)
|
||||
return new ConnectionTestResult(false, "Connection is disposed");
|
||||
|
||||
if (!Configuration.IsValid)
|
||||
return new ConnectionTestResult(false, "Configuration is invalid");
|
||||
|
||||
try
|
||||
{
|
||||
var startTime = DateTime.UtcNow;
|
||||
var buckets = Client.ListBuckets();
|
||||
var duration = DateTime.UtcNow - startTime;
|
||||
|
||||
return new ConnectionTestResult(true, "Connection successful")
|
||||
{
|
||||
BucketCount = buckets.Count,
|
||||
ResponseTime = duration,
|
||||
TestedAt = DateTime.UtcNow
|
||||
};
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return new ConnectionTestResult(false, ex.Message)
|
||||
{
|
||||
TestedAt = DateTime.UtcNow
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a string representation of the connection
|
||||
/// </summary>
|
||||
public override string ToString()
|
||||
{
|
||||
return $"MinIO Connection: {EndpointUrl} (Status: {Status})";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Disposes the connection and underlying client
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(true);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Protected dispose method
|
||||
/// </summary>
|
||||
/// <param name="disposing">Whether disposing from Dispose method</param>
|
||||
protected virtual void Dispose(bool disposing)
|
||||
{
|
||||
if (!_disposed && disposing)
|
||||
{
|
||||
_client?.Dispose();
|
||||
_client = null;
|
||||
_disposed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents the result of a connection test
|
||||
/// </summary>
|
||||
public class ConnectionTestResult
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether the connection test was successful
|
||||
/// </summary>
|
||||
public bool Success { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Message describing the test result
|
||||
/// </summary>
|
||||
public string Message { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of buckets found (if successful)
|
||||
/// </summary>
|
||||
public int? BucketCount { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Response time for the test
|
||||
/// </summary>
|
||||
public TimeSpan? ResponseTime { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When the test was performed
|
||||
/// </summary>
|
||||
public DateTime? TestedAt { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new ConnectionTestResult
|
||||
/// </summary>
|
||||
/// <param name="success">Whether the test was successful</param>
|
||||
/// <param name="message">Test result message</param>
|
||||
public ConnectionTestResult(bool success, string message)
|
||||
{
|
||||
Success = success;
|
||||
Message = message ?? string.Empty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a string representation of the test result
|
||||
/// </summary>
|
||||
public override string ToString()
|
||||
{
|
||||
var result = Success ? "SUCCESS" : "FAILED";
|
||||
var details = ResponseTime.HasValue ? $" ({ResponseTime.Value.TotalMilliseconds:F0}ms)" : "";
|
||||
return $"{result}: {Message}{details}";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Models
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents the result of a MinIO download operation with timing and speed information
|
||||
/// </summary>
|
||||
public class MinIODownloadResult
|
||||
{
|
||||
/// <summary>
|
||||
/// The downloaded file information
|
||||
/// </summary>
|
||||
public FileInfo File { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Name of the bucket the object was downloaded from
|
||||
/// </summary>
|
||||
public string BucketName { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Name of the object that was downloaded
|
||||
/// </summary>
|
||||
public string ObjectName { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Size of the downloaded file in bytes
|
||||
/// </summary>
|
||||
public long Size => File?.Length ?? 0;
|
||||
|
||||
/// <summary>
|
||||
/// Transfer start time
|
||||
/// </summary>
|
||||
public DateTime? StartTime { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Transfer completion time
|
||||
/// </summary>
|
||||
public DateTime? CompletionTime { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Transfer duration
|
||||
/// </summary>
|
||||
public TimeSpan? Duration => StartTime.HasValue && CompletionTime.HasValue ?
|
||||
CompletionTime.Value - StartTime.Value : null;
|
||||
|
||||
/// <summary>
|
||||
/// Average transfer speed in bytes per second
|
||||
/// </summary>
|
||||
public double? AverageSpeed => Duration.HasValue && Duration.Value.TotalSeconds > 0 ?
|
||||
Size / Duration.Value.TotalSeconds : null;
|
||||
|
||||
/// <summary>
|
||||
/// Average transfer speed formatted as string (e.g., "15.2 MB/s")
|
||||
/// </summary>
|
||||
public string? AverageSpeedFormatted => AverageSpeed.HasValue ?
|
||||
$"{SizeFormatter.FormatBytes((long)AverageSpeed.Value)}/s" : null;
|
||||
|
||||
/// <summary>
|
||||
/// Full path to the downloaded file
|
||||
/// </summary>
|
||||
public string FullName => File?.FullName ?? string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Name of the downloaded file
|
||||
/// </summary>
|
||||
public string Name => File?.Name ?? string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Directory containing the downloaded file
|
||||
/// </summary>
|
||||
public string? DirectoryName => File?.DirectoryName;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIODownloadResult
|
||||
/// </summary>
|
||||
/// <param name="file">Downloaded file information</param>
|
||||
/// <param name="bucketName">Source bucket name</param>
|
||||
/// <param name="objectName">Source object name</param>
|
||||
/// <param name="startTime">Transfer start time</param>
|
||||
/// <param name="completionTime">Transfer completion time</param>
|
||||
public MinIODownloadResult(FileInfo file, string bucketName, string objectName,
|
||||
DateTime? startTime = null, DateTime? completionTime = null)
|
||||
{
|
||||
File = file ?? throw new ArgumentNullException(nameof(file));
|
||||
BucketName = bucketName ?? throw new ArgumentNullException(nameof(bucketName));
|
||||
ObjectName = objectName ?? throw new ArgumentNullException(nameof(objectName));
|
||||
StartTime = startTime;
|
||||
CompletionTime = completionTime;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a string representation of the download result
|
||||
/// </summary>
|
||||
public override string ToString()
|
||||
{
|
||||
var duration = Duration?.ToString(@"hh\:mm\:ss\.fff") ?? "Unknown";
|
||||
var speed = AverageSpeedFormatted ?? "Unknown";
|
||||
return $"{BucketName}/{ObjectName} -> {FullName} ({SizeFormatter.FormatBytes(Size)}, {duration}, {speed})";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
using System;
|
||||
|
||||
namespace PSMinIO.Models
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents statistical information about MinIO storage
|
||||
/// </summary>
|
||||
public class MinIOStats
|
||||
{
|
||||
/// <summary>
|
||||
/// Total number of buckets
|
||||
/// </summary>
|
||||
public int TotalBuckets { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Total number of objects across all buckets
|
||||
/// </summary>
|
||||
public long TotalObjects { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Total size of all objects in bytes
|
||||
/// </summary>
|
||||
public long TotalSize { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When these statistics were last updated
|
||||
/// </summary>
|
||||
public DateTime LastUpdated { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// MinIO server endpoint
|
||||
/// </summary>
|
||||
public string Endpoint { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Whether SSL is being used
|
||||
/// </summary>
|
||||
public bool UseSSL { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Connection status
|
||||
/// </summary>
|
||||
public string ConnectionStatus { get; set; } = "Unknown";
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIOStats instance
|
||||
/// </summary>
|
||||
public MinIOStats()
|
||||
{
|
||||
LastUpdated = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIOStats instance with specified values
|
||||
/// </summary>
|
||||
/// <param name="totalBuckets">Total number of buckets</param>
|
||||
/// <param name="totalObjects">Total number of objects</param>
|
||||
/// <param name="totalSize">Total size in bytes</param>
|
||||
/// <param name="endpoint">MinIO endpoint</param>
|
||||
/// <param name="useSSL">Whether SSL is used</param>
|
||||
public MinIOStats(int totalBuckets, long totalObjects, long totalSize, string endpoint, bool useSSL)
|
||||
{
|
||||
TotalBuckets = totalBuckets;
|
||||
TotalObjects = totalObjects;
|
||||
TotalSize = totalSize;
|
||||
Endpoint = endpoint ?? string.Empty;
|
||||
UseSSL = useSSL;
|
||||
LastUpdated = DateTime.UtcNow;
|
||||
ConnectionStatus = "Connected";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the average object size in bytes
|
||||
/// </summary>
|
||||
public double AverageObjectSize => TotalObjects > 0 ? (double)TotalSize / TotalObjects : 0;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the average objects per bucket
|
||||
/// </summary>
|
||||
public double AverageObjectsPerBucket => TotalBuckets > 0 ? (double)TotalObjects / TotalBuckets : 0;
|
||||
|
||||
/// <summary>
|
||||
/// Returns a string representation of the statistics
|
||||
/// </summary>
|
||||
public override string ToString()
|
||||
{
|
||||
return $"MinIO Stats: {TotalBuckets} buckets, {TotalObjects} objects, {Utils.SizeFormatter.FormatBytes(TotalSize)} total";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -31,12 +31,9 @@ using System.Runtime.InteropServices;
|
||||
//
|
||||
// You can specify all the values or you can default the Build and Revision Numbers
|
||||
// by using the '*' as shown below:
|
||||
// [assembly: AssemblyVersion("2025.07.11.1151")]
|
||||
[assembly: AssemblyVersion("2025.07.11.1151")]
|
||||
[assembly: AssemblyFileVersion("2025.07.11.1151")]
|
||||
[assembly: AssemblyInformationalVersion("2025.07.11.1151")]
|
||||
|
||||
|
||||
|
||||
// [assembly: AssemblyVersion("2025.07.11.1421")]
|
||||
[assembly: AssemblyVersion("2025.07.11.1421")]
|
||||
[assembly: AssemblyFileVersion("2025.07.11.1421")]
|
||||
[assembly: AssemblyInformationalVersion("2025.07.11.1421")]
|
||||
|
||||
|
||||
|
||||
@@ -1,267 +0,0 @@
|
||||
using System;
|
||||
using System.Management.Automation;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Utils
|
||||
{
|
||||
/// <summary>
|
||||
/// Manages 3-layer progress reporting for chunked file collection operations
|
||||
/// Layer 1: Collection Progress (overall files)
|
||||
/// Layer 2: Current File Progress
|
||||
/// Layer 3: Current Chunk Progress
|
||||
/// </summary>
|
||||
public class ChunkedCollectionProgressReporter
|
||||
{
|
||||
private readonly PSCmdlet _cmdlet;
|
||||
private readonly int _totalFiles;
|
||||
private readonly long _totalSize;
|
||||
private readonly DateTime _startTime;
|
||||
private readonly string _operationName;
|
||||
|
||||
// Progress tracking
|
||||
private int _completedFiles = 0;
|
||||
private long _totalBytesTransferred = 0;
|
||||
private string _currentFileName = "";
|
||||
private long _currentFileSize = 0;
|
||||
private long _currentFileBytesTransferred = 0;
|
||||
private int _currentChunk = 0;
|
||||
private int _totalChunks = 0;
|
||||
private long _currentChunkBytesTransferred = 0;
|
||||
private long _currentChunkSize = 0;
|
||||
|
||||
// Activity IDs for progress hierarchy
|
||||
private const int CollectionActivityId = 1;
|
||||
private const int FileActivityId = 2;
|
||||
private const int ChunkActivityId = 3;
|
||||
|
||||
// Progress control
|
||||
private readonly long _progressUpdateInterval;
|
||||
private long _lastProgressUpdate = 0;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new chunked collection progress reporter
|
||||
/// </summary>
|
||||
/// <param name="cmdlet">PowerShell cmdlet for progress reporting</param>
|
||||
/// <param name="totalFiles">Total number of files to process</param>
|
||||
/// <param name="totalSize">Total size of all files</param>
|
||||
/// <param name="operationName">Name of the operation (e.g., "Uploading", "Downloading")</param>
|
||||
/// <param name="progressUpdateInterval">Update progress every N bytes</param>
|
||||
public ChunkedCollectionProgressReporter(
|
||||
PSCmdlet cmdlet,
|
||||
int totalFiles,
|
||||
long totalSize,
|
||||
string operationName = "Processing",
|
||||
long progressUpdateInterval = 1024 * 1024) // 1MB default
|
||||
{
|
||||
_cmdlet = cmdlet;
|
||||
_totalFiles = totalFiles;
|
||||
_totalSize = totalSize;
|
||||
_operationName = operationName;
|
||||
_startTime = DateTime.Now;
|
||||
_progressUpdateInterval = progressUpdateInterval;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Starts processing a new file
|
||||
/// </summary>
|
||||
/// <param name="fileName">Name of the file being processed</param>
|
||||
/// <param name="fileSize">Size of the file</param>
|
||||
/// <param name="totalChunks">Total number of chunks for this file</param>
|
||||
public void StartNewFile(string fileName, long fileSize, int totalChunks)
|
||||
{
|
||||
_currentFileName = fileName;
|
||||
_currentFileSize = fileSize;
|
||||
_currentFileBytesTransferred = 0;
|
||||
_totalChunks = totalChunks;
|
||||
_currentChunk = 0;
|
||||
|
||||
MinIOLogger.WriteVerbose(_cmdlet, "Starting {0} of file {1}/{2}: {3} ({4})",
|
||||
_operationName.ToLower(), _completedFiles + 1, _totalFiles, fileName, SizeFormatter.FormatBytes(fileSize));
|
||||
|
||||
UpdateAllProgress();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Starts processing a new chunk
|
||||
/// </summary>
|
||||
/// <param name="chunkNumber">Chunk number (1-based for display)</param>
|
||||
/// <param name="chunkSize">Size of the chunk</param>
|
||||
public void StartNewChunk(int chunkNumber, long chunkSize)
|
||||
{
|
||||
_currentChunk = chunkNumber;
|
||||
_currentChunkSize = chunkSize;
|
||||
_currentChunkBytesTransferred = 0;
|
||||
|
||||
MinIOLogger.WriteVerbose(_cmdlet, "File {0}: Starting chunk {1}/{2} ({3})",
|
||||
_currentFileName, chunkNumber, _totalChunks, SizeFormatter.FormatBytes(chunkSize));
|
||||
|
||||
UpdateAllProgress();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Updates chunk progress
|
||||
/// </summary>
|
||||
/// <param name="bytesTransferred">Bytes transferred for current chunk</param>
|
||||
public void UpdateChunkProgress(long bytesTransferred)
|
||||
{
|
||||
var chunkDelta = bytesTransferred - _currentChunkBytesTransferred;
|
||||
_currentChunkBytesTransferred = bytesTransferred;
|
||||
_currentFileBytesTransferred += chunkDelta;
|
||||
_totalBytesTransferred += chunkDelta;
|
||||
|
||||
// Only update progress if we've transferred enough bytes since last update
|
||||
if (_totalBytesTransferred - _lastProgressUpdate >= _progressUpdateInterval)
|
||||
{
|
||||
UpdateAllProgress();
|
||||
_lastProgressUpdate = _totalBytesTransferred;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Completes the current chunk
|
||||
/// </summary>
|
||||
/// <param name="chunkETag">ETag of the completed chunk (optional)</param>
|
||||
public void CompleteChunk(string? chunkETag = null)
|
||||
{
|
||||
MinIOLogger.WriteVerbose(_cmdlet, "File {0}: Completed chunk {1}/{2}{3}",
|
||||
_currentFileName, _currentChunk, _totalChunks,
|
||||
!string.IsNullOrEmpty(chunkETag) ? $" - ETag: {chunkETag}" : "");
|
||||
|
||||
// Mark chunk as completed (only if progress is enabled)
|
||||
if (_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") &&
|
||||
_cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue")
|
||||
return;
|
||||
|
||||
var chunkProgress = new ProgressRecord(ChunkActivityId, "Current Chunk", "Completed")
|
||||
{
|
||||
PercentComplete = 100,
|
||||
RecordType = ProgressRecordType.Completed,
|
||||
ParentActivityId = FileActivityId
|
||||
};
|
||||
_cmdlet.WriteProgress(chunkProgress);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Completes the current file
|
||||
/// </summary>
|
||||
public void CompleteFile()
|
||||
{
|
||||
_completedFiles++;
|
||||
|
||||
var elapsed = DateTime.Now - _startTime;
|
||||
MinIOLogger.WriteVerbose(_cmdlet, "File {0}: {1} completed in {2} - Total size: {3}",
|
||||
_currentFileName, _operationName.ToLower(), elapsed.ToString(@"hh\:mm\:ss"), SizeFormatter.FormatBytes(_currentFileSize));
|
||||
|
||||
// Mark file as completed (only if progress is enabled)
|
||||
if (_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") &&
|
||||
_cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue")
|
||||
return;
|
||||
|
||||
var fileProgress = new ProgressRecord(FileActivityId, "Current File", "Completed")
|
||||
{
|
||||
PercentComplete = 100,
|
||||
RecordType = ProgressRecordType.Completed,
|
||||
ParentActivityId = CollectionActivityId
|
||||
};
|
||||
_cmdlet.WriteProgress(fileProgress);
|
||||
|
||||
// Complete chunk progress too
|
||||
CompleteChunk();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Completes the entire collection operation
|
||||
/// </summary>
|
||||
public void CompleteCollection()
|
||||
{
|
||||
var elapsed = DateTime.Now - _startTime;
|
||||
MinIOLogger.WriteVerbose(_cmdlet, "Completed {0} {1} files ({2}) in {3}",
|
||||
_operationName.ToLower(), _totalFiles, SizeFormatter.FormatBytes(_totalSize), elapsed.ToString(@"hh\:mm\:ss"));
|
||||
|
||||
// Complete all progress records
|
||||
var collectionProgress = new ProgressRecord(CollectionActivityId, $"{_operationName} Files", "Completed")
|
||||
{
|
||||
PercentComplete = 100,
|
||||
RecordType = ProgressRecordType.Completed
|
||||
};
|
||||
_cmdlet.WriteProgress(collectionProgress);
|
||||
|
||||
// Complete file progress if enabled
|
||||
if (!(_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") &&
|
||||
_cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue"))
|
||||
{
|
||||
CompleteFile();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reports an error for the current chunk
|
||||
/// </summary>
|
||||
/// <param name="error">Error that occurred</param>
|
||||
/// <param name="retryAttempt">Current retry attempt</param>
|
||||
/// <param name="maxRetries">Maximum retry attempts</param>
|
||||
public void ReportChunkError(Exception error, int retryAttempt, int maxRetries)
|
||||
{
|
||||
MinIOLogger.WriteVerbose(_cmdlet, "File {0}: Chunk {1}/{2} failed (attempt {3}/{4}): {5}",
|
||||
_currentFileName, _currentChunk, _totalChunks, retryAttempt, maxRetries, error.Message);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Updates all progress layers
|
||||
/// </summary>
|
||||
private void UpdateAllProgress()
|
||||
{
|
||||
var elapsed = DateTime.Now - _startTime;
|
||||
var speed = elapsed.TotalSeconds > 0 ? _totalBytesTransferred / elapsed.TotalSeconds : 0;
|
||||
|
||||
// Layer 1: Collection Progress (always shown)
|
||||
var collectionPercent = _totalSize > 0 ? (int)((_totalBytesTransferred * 100) / _totalSize) : 0;
|
||||
var collectionStatus = $"Files: {_completedFiles}/{_totalFiles} | " +
|
||||
$"Size: {SizeFormatter.FormatBytes(_totalBytesTransferred)}/{SizeFormatter.FormatBytes(_totalSize)} | " +
|
||||
$"Speed: {SizeFormatter.FormatBytes((long)speed)}/s | " +
|
||||
$"Elapsed: {elapsed:hh\\:mm\\:ss}";
|
||||
|
||||
var collectionProgress = new ProgressRecord(CollectionActivityId, $"{_operationName} Files", collectionStatus)
|
||||
{
|
||||
PercentComplete = collectionPercent
|
||||
};
|
||||
_cmdlet.WriteProgress(collectionProgress);
|
||||
|
||||
// Check if progress is disabled
|
||||
if (_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") &&
|
||||
_cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue")
|
||||
return;
|
||||
|
||||
// Layer 2: Current File Progress
|
||||
if (!string.IsNullOrEmpty(_currentFileName))
|
||||
{
|
||||
var filePercent = _currentFileSize > 0 ? (int)((_currentFileBytesTransferred * 100) / _currentFileSize) : 0;
|
||||
var fileStatus = $"File: {_currentFileName} | " +
|
||||
$"Size: {SizeFormatter.FormatBytes(_currentFileBytesTransferred)}/{SizeFormatter.FormatBytes(_currentFileSize)}";
|
||||
|
||||
var fileProgress = new ProgressRecord(FileActivityId, "Current File", fileStatus)
|
||||
{
|
||||
PercentComplete = filePercent,
|
||||
ParentActivityId = CollectionActivityId
|
||||
};
|
||||
_cmdlet.WriteProgress(fileProgress);
|
||||
}
|
||||
|
||||
// Layer 3: Current Chunk Progress
|
||||
if (_currentChunk > 0)
|
||||
{
|
||||
var chunkPercent = _currentChunkSize > 0 ? (int)((_currentChunkBytesTransferred * 100) / _currentChunkSize) : 0;
|
||||
var chunkStatus = $"Chunk: {_currentChunk}/{_totalChunks} | " +
|
||||
$"Size: {SizeFormatter.FormatBytes(_currentChunkBytesTransferred)}/{SizeFormatter.FormatBytes(_currentChunkSize)} | " +
|
||||
$"Speed: {SizeFormatter.FormatBytes((long)speed)}/s";
|
||||
|
||||
var chunkProgress = new ProgressRecord(ChunkActivityId, "Current Chunk", chunkStatus)
|
||||
{
|
||||
PercentComplete = chunkPercent,
|
||||
ParentActivityId = FileActivityId
|
||||
};
|
||||
_cmdlet.WriteProgress(chunkProgress);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,195 +0,0 @@
|
||||
using System;
|
||||
using System.Management.Automation;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Utils
|
||||
{
|
||||
/// <summary>
|
||||
/// Manages 2-layer progress reporting for chunked single file operations
|
||||
/// Layer 1: File Progress
|
||||
/// Layer 2: Current Chunk Progress
|
||||
/// </summary>
|
||||
public class ChunkedSingleFileProgressReporter
|
||||
{
|
||||
private readonly PSCmdlet _cmdlet;
|
||||
private readonly long _totalSize;
|
||||
private readonly int _totalChunks;
|
||||
private readonly DateTime _startTime;
|
||||
private readonly string _operationName;
|
||||
|
||||
// Progress tracking
|
||||
private long _totalBytesTransferred = 0;
|
||||
private int _currentChunk = 0;
|
||||
private long _currentChunkBytesTransferred = 0;
|
||||
private long _currentChunkSize = 0;
|
||||
|
||||
// Activity IDs for progress hierarchy
|
||||
private const int FileActivityId = 1;
|
||||
private const int ChunkActivityId = 2;
|
||||
|
||||
// Progress control
|
||||
private readonly long _progressUpdateInterval;
|
||||
private long _lastProgressUpdate = 0;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new chunked single file progress reporter
|
||||
/// </summary>
|
||||
/// <param name="cmdlet">PowerShell cmdlet for progress reporting</param>
|
||||
/// <param name="totalSize">Total size of the file</param>
|
||||
/// <param name="totalChunks">Total number of chunks</param>
|
||||
/// <param name="operationName">Name of the operation (e.g., "Downloading", "Uploading")</param>
|
||||
/// <param name="progressUpdateInterval">Update progress every N bytes</param>
|
||||
public ChunkedSingleFileProgressReporter(
|
||||
PSCmdlet cmdlet,
|
||||
long totalSize,
|
||||
int totalChunks,
|
||||
string operationName = "Processing",
|
||||
long progressUpdateInterval = 1024 * 1024) // 1MB default
|
||||
{
|
||||
_cmdlet = cmdlet;
|
||||
_totalSize = totalSize;
|
||||
_totalChunks = totalChunks;
|
||||
_operationName = operationName;
|
||||
_startTime = DateTime.Now;
|
||||
_progressUpdateInterval = progressUpdateInterval;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Starts processing a new chunk
|
||||
/// </summary>
|
||||
/// <param name="chunkNumber">Chunk number (1-based for display)</param>
|
||||
/// <param name="chunkSize">Size of the chunk</param>
|
||||
public void StartNewChunk(int chunkNumber, long chunkSize)
|
||||
{
|
||||
_currentChunk = chunkNumber;
|
||||
_currentChunkSize = chunkSize;
|
||||
_currentChunkBytesTransferred = 0;
|
||||
|
||||
MinIOLogger.WriteVerbose(_cmdlet, "Starting chunk {0}/{1} ({2})",
|
||||
chunkNumber, _totalChunks, SizeFormatter.FormatBytes(chunkSize));
|
||||
|
||||
UpdateAllProgress();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Updates chunk progress
|
||||
/// </summary>
|
||||
/// <param name="bytesTransferred">Bytes transferred for current chunk</param>
|
||||
public void UpdateChunkProgress(long bytesTransferred)
|
||||
{
|
||||
var chunkDelta = bytesTransferred - _currentChunkBytesTransferred;
|
||||
_currentChunkBytesTransferred = bytesTransferred;
|
||||
_totalBytesTransferred += chunkDelta;
|
||||
|
||||
// Only update progress if we've transferred enough bytes since last update
|
||||
if (_totalBytesTransferred - _lastProgressUpdate >= _progressUpdateInterval)
|
||||
{
|
||||
UpdateAllProgress();
|
||||
_lastProgressUpdate = _totalBytesTransferred;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Completes the current chunk
|
||||
/// </summary>
|
||||
/// <param name="chunkETag">ETag of the completed chunk (optional)</param>
|
||||
public void CompleteChunk(string? chunkETag = null)
|
||||
{
|
||||
MinIOLogger.WriteVerbose(_cmdlet, "Completed chunk {0}/{1}{2}",
|
||||
_currentChunk, _totalChunks,
|
||||
!string.IsNullOrEmpty(chunkETag) ? $" - ETag: {chunkETag}" : "");
|
||||
|
||||
// Mark chunk as completed (only if progress is enabled)
|
||||
if (_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") &&
|
||||
_cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue")
|
||||
return;
|
||||
|
||||
var chunkProgress = new ProgressRecord(ChunkActivityId, "Current Chunk", "Completed")
|
||||
{
|
||||
PercentComplete = 100,
|
||||
RecordType = ProgressRecordType.Completed,
|
||||
ParentActivityId = FileActivityId
|
||||
};
|
||||
_cmdlet.WriteProgress(chunkProgress);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Completes the entire download operation
|
||||
/// </summary>
|
||||
public void CompleteDownload()
|
||||
{
|
||||
var elapsed = DateTime.Now - _startTime;
|
||||
MinIOLogger.WriteVerbose(_cmdlet, "Completed {0} ({1}) in {2}",
|
||||
_operationName.ToLower(), SizeFormatter.FormatBytes(_totalSize), elapsed.ToString(@"hh\:mm\:ss"));
|
||||
|
||||
// Complete all progress records
|
||||
var fileProgress = new ProgressRecord(FileActivityId, $"{_operationName} File", "Completed")
|
||||
{
|
||||
PercentComplete = 100,
|
||||
RecordType = ProgressRecordType.Completed
|
||||
};
|
||||
_cmdlet.WriteProgress(fileProgress);
|
||||
|
||||
// Complete chunk progress if enabled
|
||||
if (!(_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") &&
|
||||
_cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue"))
|
||||
{
|
||||
CompleteChunk();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reports an error for the current chunk
|
||||
/// </summary>
|
||||
/// <param name="error">Error that occurred</param>
|
||||
/// <param name="retryAttempt">Current retry attempt</param>
|
||||
/// <param name="maxRetries">Maximum retry attempts</param>
|
||||
public void ReportChunkError(Exception error, int retryAttempt, int maxRetries)
|
||||
{
|
||||
MinIOLogger.WriteVerbose(_cmdlet, "Chunk {0}/{1} failed (attempt {2}/{3}): {4}",
|
||||
_currentChunk, _totalChunks, retryAttempt, maxRetries, error.Message);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Updates all progress layers
|
||||
/// </summary>
|
||||
private void UpdateAllProgress()
|
||||
{
|
||||
var elapsed = DateTime.Now - _startTime;
|
||||
var speed = elapsed.TotalSeconds > 0 ? _totalBytesTransferred / elapsed.TotalSeconds : 0;
|
||||
|
||||
// Layer 1: File Progress (always shown)
|
||||
var filePercent = _totalSize > 0 ? (int)((_totalBytesTransferred * 100) / _totalSize) : 0;
|
||||
var fileStatus = $"Size: {SizeFormatter.FormatBytes(_totalBytesTransferred)}/{SizeFormatter.FormatBytes(_totalSize)} | " +
|
||||
$"Speed: {SizeFormatter.FormatBytes((long)speed)}/s | " +
|
||||
$"Elapsed: {elapsed:hh\\:mm\\:ss}";
|
||||
|
||||
var fileProgress = new ProgressRecord(FileActivityId, $"{_operationName} File", fileStatus)
|
||||
{
|
||||
PercentComplete = filePercent
|
||||
};
|
||||
_cmdlet.WriteProgress(fileProgress);
|
||||
|
||||
// Check if progress is disabled
|
||||
if (_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") &&
|
||||
_cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue")
|
||||
return;
|
||||
|
||||
// Layer 2: Current Chunk Progress
|
||||
if (_currentChunk > 0)
|
||||
{
|
||||
var chunkPercent = _currentChunkSize > 0 ? (int)((_currentChunkBytesTransferred * 100) / _currentChunkSize) : 0;
|
||||
var chunkStatus = $"Chunk: {_currentChunk}/{_totalChunks} | " +
|
||||
$"Size: {SizeFormatter.FormatBytes(_currentChunkBytesTransferred)}/{SizeFormatter.FormatBytes(_currentChunkSize)} | " +
|
||||
$"Speed: {SizeFormatter.FormatBytes((long)speed)}/s";
|
||||
|
||||
var chunkProgress = new ProgressRecord(ChunkActivityId, "Current Chunk", chunkStatus)
|
||||
{
|
||||
PercentComplete = chunkPercent,
|
||||
ParentActivityId = FileActivityId
|
||||
};
|
||||
_cmdlet.WriteProgress(chunkProgress);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,250 +0,0 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Text.Json;
|
||||
using PSMinIO.Models;
|
||||
|
||||
namespace PSMinIO.Utils
|
||||
{
|
||||
/// <summary>
|
||||
/// Manages resume data for chunked transfer operations
|
||||
/// </summary>
|
||||
public static class ChunkedTransferResumeManager
|
||||
{
|
||||
private static readonly string DefaultResumeDirectory = Path.Combine(
|
||||
Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
|
||||
"PSMinIO", "Resume");
|
||||
|
||||
/// <summary>
|
||||
/// Saves transfer state for resume functionality
|
||||
/// </summary>
|
||||
/// <param name="transferState">Transfer state to save</param>
|
||||
/// <param name="customPath">Custom path for resume data (optional)</param>
|
||||
/// <returns>Path where resume data was saved</returns>
|
||||
public static string SaveTransferState(ChunkedTransferState transferState, string? customPath = null)
|
||||
{
|
||||
var resumeDirectory = customPath ?? DefaultResumeDirectory;
|
||||
|
||||
// Ensure directory exists
|
||||
if (!Directory.Exists(resumeDirectory))
|
||||
{
|
||||
Directory.CreateDirectory(resumeDirectory);
|
||||
}
|
||||
|
||||
// Generate unique filename based on transfer details
|
||||
var fileName = GenerateResumeFileName(transferState);
|
||||
var filePath = Path.Combine(resumeDirectory, fileName);
|
||||
|
||||
// Serialize and save
|
||||
var json = JsonSerializer.Serialize(transferState, new JsonSerializerOptions
|
||||
{
|
||||
WriteIndented = true,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
||||
});
|
||||
|
||||
File.WriteAllText(filePath, json);
|
||||
return filePath;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads transfer state for resume functionality
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Bucket name</param>
|
||||
/// <param name="objectName">Object name</param>
|
||||
/// <param name="filePath">Local file path</param>
|
||||
/// <param name="transferType">Transfer type</param>
|
||||
/// <param name="customPath">Custom path for resume data (optional)</param>
|
||||
/// <returns>Transfer state if found, null otherwise</returns>
|
||||
public static ChunkedTransferState? LoadTransferState(
|
||||
string bucketName,
|
||||
string objectName,
|
||||
string filePath,
|
||||
ChunkedTransferType transferType,
|
||||
string? customPath = null)
|
||||
{
|
||||
var resumeDirectory = customPath ?? DefaultResumeDirectory;
|
||||
|
||||
if (!Directory.Exists(resumeDirectory))
|
||||
return null;
|
||||
|
||||
// Generate expected filename
|
||||
var tempState = new ChunkedTransferState
|
||||
{
|
||||
BucketName = bucketName,
|
||||
ObjectName = objectName,
|
||||
FilePath = filePath,
|
||||
TransferType = transferType
|
||||
};
|
||||
|
||||
var fileName = GenerateResumeFileName(tempState);
|
||||
var resumeFilePath = Path.Combine(resumeDirectory, fileName);
|
||||
|
||||
if (!File.Exists(resumeFilePath))
|
||||
return null;
|
||||
|
||||
try
|
||||
{
|
||||
var json = File.ReadAllText(resumeFilePath);
|
||||
var transferState = JsonSerializer.Deserialize<ChunkedTransferState>(json, new JsonSerializerOptions
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
||||
});
|
||||
|
||||
return transferState;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
// If we can't deserialize, treat as no resume data
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes resume data after successful completion
|
||||
/// </summary>
|
||||
/// <param name="transferState">Transfer state to clean up</param>
|
||||
/// <param name="customPath">Custom path for resume data (optional)</param>
|
||||
public static void CleanupResumeData(ChunkedTransferState transferState, string? customPath = null)
|
||||
{
|
||||
var resumeDirectory = customPath ?? DefaultResumeDirectory;
|
||||
var fileName = GenerateResumeFileName(transferState);
|
||||
var filePath = Path.Combine(resumeDirectory, fileName);
|
||||
|
||||
try
|
||||
{
|
||||
if (File.Exists(filePath))
|
||||
{
|
||||
File.Delete(filePath);
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates if resume data is still valid
|
||||
/// </summary>
|
||||
/// <param name="transferState">Transfer state to validate</param>
|
||||
/// <param name="currentFileInfo">Current file information</param>
|
||||
/// <returns>True if resume data is valid</returns>
|
||||
public static bool IsResumeDataValid(ChunkedTransferState transferState, FileInfo? currentFileInfo = null)
|
||||
{
|
||||
// Check if transfer state is reasonable
|
||||
if (transferState == null)
|
||||
return false;
|
||||
|
||||
// For uploads, validate source file hasn't changed
|
||||
if (transferState.TransferType == ChunkedTransferType.Upload && currentFileInfo != null)
|
||||
{
|
||||
if (!currentFileInfo.Exists)
|
||||
return false;
|
||||
|
||||
// Check if file size or last modified time changed
|
||||
if (currentFileInfo.Length != transferState.TotalSize ||
|
||||
currentFileInfo.LastWriteTimeUtc != transferState.LastModified)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Check if resume data is not too old (e.g., older than 7 days)
|
||||
if (DateTime.UtcNow - transferState.LastUpdated > TimeSpan.FromDays(7))
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets all resume files in the directory
|
||||
/// </summary>
|
||||
/// <param name="customPath">Custom path for resume data (optional)</param>
|
||||
/// <returns>Array of resume file paths</returns>
|
||||
public static string[] GetResumeFiles(string? customPath = null)
|
||||
{
|
||||
var resumeDirectory = customPath ?? DefaultResumeDirectory;
|
||||
|
||||
if (!Directory.Exists(resumeDirectory))
|
||||
return Array.Empty<string>();
|
||||
|
||||
return Directory.GetFiles(resumeDirectory, "*.psminioResume");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cleans up old resume files
|
||||
/// </summary>
|
||||
/// <param name="olderThanDays">Delete files older than this many days</param>
|
||||
/// <param name="customPath">Custom path for resume data (optional)</param>
|
||||
/// <returns>Number of files cleaned up</returns>
|
||||
public static int CleanupOldResumeFiles(int olderThanDays = 7, string? customPath = null)
|
||||
{
|
||||
var resumeFiles = GetResumeFiles(customPath);
|
||||
var cutoffDate = DateTime.UtcNow.AddDays(-olderThanDays);
|
||||
var cleanedCount = 0;
|
||||
|
||||
foreach (var file in resumeFiles)
|
||||
{
|
||||
try
|
||||
{
|
||||
var fileInfo = new FileInfo(file);
|
||||
if (fileInfo.LastWriteTimeUtc < cutoffDate)
|
||||
{
|
||||
File.Delete(file);
|
||||
cleanedCount++;
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
}
|
||||
|
||||
return cleanedCount;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates a unique filename for resume data
|
||||
/// </summary>
|
||||
/// <param name="transferState">Transfer state</param>
|
||||
/// <returns>Unique filename</returns>
|
||||
private static string GenerateResumeFileName(ChunkedTransferState transferState)
|
||||
{
|
||||
// Create a hash of the key components to ensure uniqueness
|
||||
var key = $"{transferState.BucketName}|{transferState.ObjectName}|{transferState.FilePath}|{transferState.TransferType}";
|
||||
var hash = key.GetHashCode().ToString("X8");
|
||||
|
||||
// Include readable components for easier identification
|
||||
var safeBucketName = MakeSafeFileName(transferState.BucketName);
|
||||
var safeObjectName = MakeSafeFileName(Path.GetFileName(transferState.ObjectName));
|
||||
var transferType = transferState.TransferType.ToString().ToLower();
|
||||
|
||||
return $"{safeBucketName}_{safeObjectName}_{transferType}_{hash}.psminioResume";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Makes a string safe for use as a filename
|
||||
/// </summary>
|
||||
/// <param name="input">Input string</param>
|
||||
/// <returns>Safe filename string</returns>
|
||||
private static string MakeSafeFileName(string input)
|
||||
{
|
||||
if (string.IsNullOrEmpty(input))
|
||||
return "unknown";
|
||||
|
||||
var invalidChars = Path.GetInvalidFileNameChars();
|
||||
var safe = input;
|
||||
|
||||
foreach (var c in invalidChars)
|
||||
{
|
||||
safe = safe.Replace(c, '_');
|
||||
}
|
||||
|
||||
// Limit length and remove leading/trailing dots and spaces
|
||||
safe = safe.Trim(' ', '.');
|
||||
if (safe.Length > 50)
|
||||
safe = safe.Substring(0, 50);
|
||||
|
||||
return string.IsNullOrEmpty(safe) ? "unknown" : safe;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,954 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Minio;
|
||||
using Minio.DataModel;
|
||||
using PSMinIO.Models;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Utils
|
||||
{
|
||||
/// <summary>
|
||||
/// Synchronous wrapper for MinIO client operations
|
||||
/// Converts async MinIO operations to synchronous calls for PowerShell compatibility
|
||||
/// </summary>
|
||||
public class MinIOClientWrapper : IDisposable
|
||||
{
|
||||
private readonly IMinioClient _client;
|
||||
private readonly CancellationTokenSource _cancellationTokenSource;
|
||||
private bool _disposed = false;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new MinIOClientWrapper instance
|
||||
/// </summary>
|
||||
/// <param name="configuration">MinIO configuration</param>
|
||||
public MinIOClientWrapper(MinIOConfiguration configuration)
|
||||
{
|
||||
if (configuration == null)
|
||||
throw new ArgumentNullException(nameof(configuration));
|
||||
|
||||
if (!configuration.IsValid)
|
||||
throw new ArgumentException("Invalid MinIO configuration", nameof(configuration));
|
||||
|
||||
_cancellationTokenSource = new CancellationTokenSource();
|
||||
|
||||
// Create MinIO client with configuration
|
||||
var clientBuilder = new MinioClient()
|
||||
.WithEndpoint(configuration.Endpoint)
|
||||
.WithCredentials(configuration.AccessKey, configuration.SecretKey);
|
||||
|
||||
if (configuration.UseSSL)
|
||||
{
|
||||
clientBuilder = clientBuilder.WithSSL();
|
||||
|
||||
// Configure custom HttpClient for certificate validation if needed
|
||||
if (configuration.SkipCertificateValidation)
|
||||
{
|
||||
var httpClientHandler = new HttpClientHandler()
|
||||
{
|
||||
ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => true
|
||||
};
|
||||
var httpClient = new HttpClient(httpClientHandler);
|
||||
clientBuilder = clientBuilder.WithHttpClient(httpClient);
|
||||
}
|
||||
}
|
||||
|
||||
if (configuration.TimeoutSeconds > 0)
|
||||
{
|
||||
clientBuilder = clientBuilder.WithTimeout(configuration.TimeoutSeconds * 1000);
|
||||
}
|
||||
|
||||
_client = clientBuilder.Build();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the cancellation token for operations
|
||||
/// </summary>
|
||||
public CancellationToken CancellationToken => _cancellationTokenSource.Token;
|
||||
|
||||
/// <summary>
|
||||
/// Lists all buckets synchronously
|
||||
/// </summary>
|
||||
/// <returns>List of bucket information</returns>
|
||||
public List<MinIOBucketInfo> ListBuckets()
|
||||
{
|
||||
try
|
||||
{
|
||||
var bucketsResult = Task.Run(async () =>
|
||||
await _client.ListBucketsAsync(CancellationToken)).GetAwaiter().GetResult();
|
||||
|
||||
return bucketsResult.Buckets
|
||||
.Select(MinIOBucketInfo.FromMinioBucket)
|
||||
.ToList();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to list buckets: {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if a bucket exists synchronously
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket</param>
|
||||
/// <returns>True if bucket exists, false otherwise</returns>
|
||||
public bool BucketExists(string bucketName)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
try
|
||||
{
|
||||
var args = new BucketExistsArgs().WithBucket(bucketName);
|
||||
return Task.Run(async () =>
|
||||
await _client.BucketExistsAsync(args, CancellationToken)).GetAwaiter().GetResult();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to check if bucket '{bucketName}' exists: {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a bucket synchronously
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket to create</param>
|
||||
/// <param name="region">Optional region for the bucket</param>
|
||||
public void CreateBucket(string bucketName, string? region = null)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
try
|
||||
{
|
||||
var args = new MakeBucketArgs().WithBucket(bucketName);
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(region))
|
||||
{
|
||||
args = args.WithLocation(region);
|
||||
}
|
||||
|
||||
Task.Run(async () =>
|
||||
await _client.MakeBucketAsync(args, CancellationToken)).GetAwaiter().GetResult();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to create bucket '{bucketName}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes a bucket synchronously
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket to delete</param>
|
||||
public void DeleteBucket(string bucketName)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
try
|
||||
{
|
||||
var args = new RemoveBucketArgs().WithBucket(bucketName);
|
||||
Task.Run(async () =>
|
||||
await _client.RemoveBucketAsync(args, CancellationToken)).GetAwaiter().GetResult();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to delete bucket '{bucketName}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Lists objects in a bucket synchronously
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket</param>
|
||||
/// <param name="prefix">Optional prefix to filter objects</param>
|
||||
/// <param name="recursive">Whether to list objects recursively</param>
|
||||
/// <param name="includeVersions">Whether to include all versions of objects</param>
|
||||
/// <returns>List of object information</returns>
|
||||
public List<MinIOObjectInfo> ListObjects(string bucketName, string? prefix = null, bool recursive = true, bool includeVersions = false)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
try
|
||||
{
|
||||
var args = new ListObjectsArgs()
|
||||
.WithBucket(bucketName)
|
||||
.WithRecursive(recursive);
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(prefix))
|
||||
{
|
||||
args = args.WithPrefix(prefix);
|
||||
}
|
||||
|
||||
// Add version support if requested
|
||||
if (includeVersions)
|
||||
{
|
||||
args = args.WithVersions(true);
|
||||
}
|
||||
|
||||
var objects = new List<MinIOObjectInfo>();
|
||||
var observable = _client.ListObjectsAsync(args, CancellationToken);
|
||||
|
||||
// Convert observable to synchronous list
|
||||
var task = Task.Run(() =>
|
||||
{
|
||||
var tcs = new TaskCompletionSource<bool>();
|
||||
observable.Subscribe(
|
||||
onNext: item => objects.Add(MinIOObjectInfo.FromMinioItem(item, bucketName)),
|
||||
onError: ex => tcs.SetException(ex),
|
||||
onCompleted: () => tcs.SetResult(true)
|
||||
);
|
||||
return tcs.Task;
|
||||
});
|
||||
|
||||
task.GetAwaiter().GetResult();
|
||||
return objects;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to list objects in bucket '{bucketName}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets bucket policy synchronously
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket</param>
|
||||
/// <returns>Bucket policy as JSON string</returns>
|
||||
public string GetBucketPolicy(string bucketName)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
try
|
||||
{
|
||||
var args = new GetPolicyArgs().WithBucket(bucketName);
|
||||
return Task.Run(async () =>
|
||||
await _client.GetPolicyAsync(args, CancellationToken)).GetAwaiter().GetResult();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to get policy for bucket '{bucketName}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets bucket policy synchronously
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket</param>
|
||||
/// <param name="policy">Policy JSON string</param>
|
||||
public void SetBucketPolicy(string bucketName, string policy)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(policy))
|
||||
throw new ArgumentException("Policy cannot be null or empty", nameof(policy));
|
||||
|
||||
try
|
||||
{
|
||||
var args = new SetPolicyArgs()
|
||||
.WithBucket(bucketName)
|
||||
.WithPolicy(policy);
|
||||
|
||||
Task.Run(async () =>
|
||||
await _client.SetPolicyAsync(args, CancellationToken)).GetAwaiter().GetResult();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to set policy for bucket '{bucketName}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes an object synchronously
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket</param>
|
||||
/// <param name="objectName">Name of the object to delete</param>
|
||||
public void DeleteObject(string bucketName, string objectName)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(objectName))
|
||||
throw new ArgumentException("Object name cannot be null or empty", nameof(objectName));
|
||||
|
||||
try
|
||||
{
|
||||
var args = new RemoveObjectArgs()
|
||||
.WithBucket(bucketName)
|
||||
.WithObject(objectName);
|
||||
|
||||
Task.Run(async () =>
|
||||
await _client.RemoveObjectAsync(args, CancellationToken)).GetAwaiter().GetResult();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to delete object '{objectName}' from bucket '{bucketName}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes multiple objects synchronously
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket</param>
|
||||
/// <param name="objectNames">List of object names to delete</param>
|
||||
public void DeleteObjects(string bucketName, IEnumerable<string> objectNames)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
if (objectNames == null)
|
||||
throw new ArgumentNullException(nameof(objectNames));
|
||||
|
||||
var objectList = objectNames.ToList();
|
||||
if (objectList.Count == 0)
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
var deleteObjectsArgs = new RemoveObjectsArgs()
|
||||
.WithBucket(bucketName)
|
||||
.WithObjects(objectList);
|
||||
|
||||
var observableTask = _client.RemoveObjectsAsync(deleteObjectsArgs, CancellationToken);
|
||||
|
||||
// Convert observable to synchronous operation
|
||||
var task = Task.Run(async () =>
|
||||
{
|
||||
var observable = await observableTask;
|
||||
var tcs = new TaskCompletionSource<bool>();
|
||||
observable.Subscribe(
|
||||
onNext: deleteError =>
|
||||
{
|
||||
// In MinIO 5.0.0, DeleteError might have different properties
|
||||
// For now, just check if there's an error and report it
|
||||
if (!string.IsNullOrEmpty(deleteError.Message))
|
||||
{
|
||||
tcs.SetException(new InvalidOperationException($"Failed to delete object '{deleteError.Key}': {deleteError.Message}"));
|
||||
}
|
||||
},
|
||||
onError: ex => tcs.SetException(ex),
|
||||
onCompleted: () => tcs.SetResult(true)
|
||||
);
|
||||
return await tcs.Task;
|
||||
});
|
||||
|
||||
task.GetAwaiter().GetResult();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to delete objects from bucket '{bucketName}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Uploads a file to MinIO synchronously with progress reporting
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket</param>
|
||||
/// <param name="objectName">Name of the object</param>
|
||||
/// <param name="filePath">Path to the file to upload</param>
|
||||
/// <param name="contentType">Content type of the file (optional)</param>
|
||||
/// <param name="progressCallback">Progress callback for reporting upload progress</param>
|
||||
/// <returns>ETag of the uploaded object</returns>
|
||||
public string UploadFile(string bucketName, string objectName, string filePath,
|
||||
string? contentType = null, Action<long>? progressCallback = null)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(objectName))
|
||||
throw new ArgumentException("Object name cannot be null or empty", nameof(objectName));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(filePath))
|
||||
throw new ArgumentException("File path cannot be null or empty", nameof(filePath));
|
||||
|
||||
if (!File.Exists(filePath))
|
||||
throw new FileNotFoundException($"File not found: {filePath}");
|
||||
|
||||
try
|
||||
{
|
||||
var fileInfo = new FileInfo(filePath);
|
||||
var fileSize = fileInfo.Length;
|
||||
|
||||
// Determine content type if not provided
|
||||
if (string.IsNullOrWhiteSpace(contentType))
|
||||
{
|
||||
contentType = GetContentType(filePath);
|
||||
}
|
||||
|
||||
// Use explicit file stream management to ensure proper handle release
|
||||
using var fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
|
||||
var args = new PutObjectArgs()
|
||||
.WithBucket(bucketName)
|
||||
.WithObject(objectName)
|
||||
.WithStreamData(fileStream)
|
||||
.WithObjectSize(fileSize)
|
||||
.WithContentType(contentType);
|
||||
|
||||
// Progress tracking not available in MinIO 4.0.7
|
||||
// progressCallback is ignored for now
|
||||
|
||||
Task.Run(async () =>
|
||||
await _client.PutObjectAsync(args, CancellationToken)).GetAwaiter().GetResult();
|
||||
|
||||
return string.Empty; // MinIO 4.0.7 PutObjectAsync returns void
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to upload file '{filePath}' to bucket '{bucketName}' as '{objectName}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Downloads an object from MinIO synchronously with progress reporting
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket</param>
|
||||
/// <param name="objectName">Name of the object</param>
|
||||
/// <param name="filePath">Path where the file should be saved</param>
|
||||
/// <param name="progressCallback">Progress callback for reporting download progress</param>
|
||||
public void DownloadFile(string bucketName, string objectName, string filePath,
|
||||
Action<long>? progressCallback = null)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(objectName))
|
||||
throw new ArgumentException("Object name cannot be null or empty", nameof(objectName));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(filePath))
|
||||
throw new ArgumentException("File path cannot be null or empty", nameof(filePath));
|
||||
|
||||
try
|
||||
{
|
||||
// Ensure the directory exists
|
||||
var directory = Path.GetDirectoryName(filePath);
|
||||
if (!string.IsNullOrEmpty(directory) && !Directory.Exists(directory))
|
||||
{
|
||||
Directory.CreateDirectory(directory);
|
||||
}
|
||||
|
||||
var args = new GetObjectArgs()
|
||||
.WithBucket(bucketName)
|
||||
.WithObject(objectName)
|
||||
.WithFile(filePath);
|
||||
|
||||
// Progress tracking not available in MinIO 5.0.0
|
||||
// progressCallback is ignored for now
|
||||
|
||||
Task.Run(async () =>
|
||||
await _client.GetObjectAsync(args, CancellationToken)).GetAwaiter().GetResult();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to download object '{objectName}' from bucket '{bucketName}' to '{filePath}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a directory object in MinIO (zero-byte object with trailing slash)
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket</param>
|
||||
/// <param name="directoryPath">Path of the directory (should end with /)</param>
|
||||
/// <returns>ETag of the created directory object</returns>
|
||||
public string CreateDirectory(string bucketName, string directoryPath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(directoryPath))
|
||||
throw new ArgumentException("Directory path cannot be null or empty", nameof(directoryPath));
|
||||
|
||||
// Ensure directory path ends with /
|
||||
if (!directoryPath.EndsWith("/"))
|
||||
directoryPath += "/";
|
||||
|
||||
try
|
||||
{
|
||||
// Use a properly configured empty stream
|
||||
using var emptyStream = new MemoryStream(new byte[0]);
|
||||
emptyStream.Position = 0;
|
||||
|
||||
var args = new PutObjectArgs()
|
||||
.WithBucket(bucketName)
|
||||
.WithObject(directoryPath)
|
||||
.WithStreamData(emptyStream)
|
||||
.WithObjectSize(0)
|
||||
.WithContentType("application/x-directory");
|
||||
|
||||
Task.Run(async () =>
|
||||
await _client.PutObjectAsync(args, CancellationToken)).GetAwaiter().GetResult();
|
||||
|
||||
return ""; // Directory objects don't have meaningful ETags
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to create directory '{directoryPath}' in bucket '{bucketName}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Uploads a stream to MinIO synchronously
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket</param>
|
||||
/// <param name="objectName">Name of the object</param>
|
||||
/// <param name="data">Stream containing the data to upload</param>
|
||||
/// <param name="contentType">Content type of the data</param>
|
||||
/// <param name="progressCallback">Progress callback for reporting upload progress</param>
|
||||
/// <returns>ETag of the uploaded object</returns>
|
||||
public string UploadStream(string bucketName, string objectName, Stream data,
|
||||
string contentType = "application/octet-stream", Action<long>? progressCallback = null)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(objectName))
|
||||
throw new ArgumentException("Object name cannot be null or empty", nameof(objectName));
|
||||
|
||||
if (data == null)
|
||||
throw new ArgumentNullException(nameof(data));
|
||||
|
||||
try
|
||||
{
|
||||
// Ensure stream is at the beginning
|
||||
if (data.CanSeek)
|
||||
{
|
||||
data.Position = 0;
|
||||
}
|
||||
|
||||
var args = new PutObjectArgs()
|
||||
.WithBucket(bucketName)
|
||||
.WithObject(objectName)
|
||||
.WithStreamData(data)
|
||||
.WithObjectSize(data.Length)
|
||||
.WithContentType(contentType);
|
||||
|
||||
// Progress tracking not available in MinIO 5.0.0
|
||||
// progressCallback is ignored for now
|
||||
|
||||
Task.Run(async () =>
|
||||
await _client.PutObjectAsync(args, CancellationToken)).GetAwaiter().GetResult();
|
||||
|
||||
return string.Empty; // MinIO 4.0.7 PutObjectAsync returns void
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to upload stream to bucket '{bucketName}' as '{objectName}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the content type for a file based on its extension
|
||||
/// </summary>
|
||||
/// <param name="filePath">Path to the file</param>
|
||||
/// <returns>Content type string</returns>
|
||||
private static string GetContentType(string filePath)
|
||||
{
|
||||
var extension = Path.GetExtension(filePath).ToLowerInvariant();
|
||||
|
||||
return extension switch
|
||||
{
|
||||
".txt" => "text/plain",
|
||||
".html" => "text/html",
|
||||
".css" => "text/css",
|
||||
".js" => "application/javascript",
|
||||
".json" => "application/json",
|
||||
".xml" => "application/xml",
|
||||
".pdf" => "application/pdf",
|
||||
".zip" => "application/zip",
|
||||
".jpg" or ".jpeg" => "image/jpeg",
|
||||
".png" => "image/png",
|
||||
".gif" => "image/gif",
|
||||
".svg" => "image/svg+xml",
|
||||
".mp4" => "video/mp4",
|
||||
".mp3" => "audio/mpeg",
|
||||
".wav" => "audio/wav",
|
||||
_ => "application/octet-stream"
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Lists object versions in a bucket synchronously
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket</param>
|
||||
/// <param name="prefix">Optional prefix to filter objects</param>
|
||||
/// <param name="recursive">Whether to list objects recursively</param>
|
||||
/// <param name="maxObjects">Maximum number of objects to return (0 = unlimited)</param>
|
||||
/// <returns>List of object information including versions</returns>
|
||||
private List<MinIOObjectInfo> ListObjectVersions(string bucketName, string? prefix, bool recursive, int maxObjects)
|
||||
{
|
||||
try
|
||||
{
|
||||
// For now, fall back to regular object listing since version listing
|
||||
// may not be available in all MinIO SDK versions
|
||||
// This can be enhanced when the SDK supports it
|
||||
return ListObjects(bucketName, prefix, recursive, false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to list object versions in bucket '{bucketName}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates a presigned URL for an object
|
||||
/// </summary>
|
||||
/// <param name="bucketName">Name of the bucket</param>
|
||||
/// <param name="objectName">Name of the object</param>
|
||||
/// <param name="expiry">URL expiry time</param>
|
||||
/// <returns>Presigned URL</returns>
|
||||
public string GetPresignedUrl(string bucketName, string objectName, TimeSpan expiry)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bucketName))
|
||||
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(objectName))
|
||||
throw new ArgumentException("Object name cannot be null or empty", nameof(objectName));
|
||||
|
||||
try
|
||||
{
|
||||
var args = new PresignedGetObjectArgs()
|
||||
.WithBucket(bucketName)
|
||||
.WithObject(objectName)
|
||||
.WithExpiry((int)expiry.TotalSeconds);
|
||||
|
||||
var result = Task.Run(async () =>
|
||||
await _client.PresignedGetObjectAsync(args)).GetAwaiter().GetResult();
|
||||
|
||||
return result ?? string.Empty;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to generate presigned URL for object '{objectName}' in bucket '{bucketName}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Uploads a file using chunked transfer with resume capability
|
||||
/// </summary>
|
||||
/// <param name="transferState">Transfer state for resume functionality</param>
|
||||
/// <param name="progressReporter">Thread-safe progress reporter for updates</param>
|
||||
/// <param name="maxRetries">Maximum retry attempts per chunk</param>
|
||||
/// <returns>MinIOObjectInfo of uploaded object or null if failed</returns>
|
||||
public MinIOObjectInfo? UploadFileChunked(
|
||||
ChunkedTransferState transferState,
|
||||
ThreadSafeChunkedProgressReporter progressReporter,
|
||||
int maxRetries = 3)
|
||||
{
|
||||
if (transferState == null)
|
||||
throw new ArgumentNullException(nameof(transferState));
|
||||
|
||||
try
|
||||
{
|
||||
// For now, implement chunked upload using regular PutObject with progress tracking
|
||||
// This simulates chunked behavior by reading the file in chunks and reporting progress
|
||||
using var fileStream = new FileStream(transferState.FilePath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
|
||||
var totalChunks = (int)Math.Ceiling((double)transferState.TotalSize / transferState.ChunkSize);
|
||||
var buffer = new byte[transferState.ChunkSize];
|
||||
long totalBytesRead = 0;
|
||||
|
||||
// Create a progress-tracking stream wrapper
|
||||
var progressStream = new ProgressTrackingStream(fileStream, (bytesRead) =>
|
||||
{
|
||||
var currentChunk = (int)(totalBytesRead / transferState.ChunkSize) + 1;
|
||||
var chunkProgress = bytesRead % transferState.ChunkSize;
|
||||
|
||||
if (currentChunk <= totalChunks)
|
||||
{
|
||||
progressReporter.StartNewChunk(currentChunk, Math.Min(transferState.ChunkSize, transferState.TotalSize - totalBytesRead));
|
||||
progressReporter.UpdateChunkProgress(chunkProgress);
|
||||
|
||||
if (chunkProgress == 0 && bytesRead > 0) // Chunk completed
|
||||
{
|
||||
progressReporter.CompleteChunk();
|
||||
}
|
||||
}
|
||||
|
||||
totalBytesRead = bytesRead;
|
||||
});
|
||||
|
||||
// Upload the file
|
||||
var putArgs = new PutObjectArgs()
|
||||
.WithBucket(transferState.BucketName)
|
||||
.WithObject(transferState.ObjectName)
|
||||
.WithStreamData(progressStream)
|
||||
.WithObjectSize(transferState.TotalSize);
|
||||
|
||||
Task.Run(async () =>
|
||||
{
|
||||
await _client.PutObjectAsync(putArgs, CancellationToken);
|
||||
}).GetAwaiter().GetResult();
|
||||
|
||||
// Return object information
|
||||
return new MinIOObjectInfo(
|
||||
transferState.ObjectName,
|
||||
transferState.TotalSize,
|
||||
DateTime.UtcNow,
|
||||
"simulated-etag", // MinIO 5.0.0 PutObject doesn't return ETag directly
|
||||
transferState.BucketName);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Chunked upload failed for object '{transferState.ObjectName}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Progress tracking stream wrapper
|
||||
/// </summary>
|
||||
private class ProgressTrackingStream : Stream
|
||||
{
|
||||
private readonly Stream _baseStream;
|
||||
private readonly Action<long> _progressCallback;
|
||||
private long _totalBytesRead = 0;
|
||||
|
||||
public ProgressTrackingStream(Stream baseStream, Action<long> progressCallback)
|
||||
{
|
||||
_baseStream = baseStream;
|
||||
_progressCallback = progressCallback;
|
||||
}
|
||||
|
||||
public override bool CanRead => _baseStream.CanRead;
|
||||
public override bool CanSeek => _baseStream.CanSeek;
|
||||
public override bool CanWrite => _baseStream.CanWrite;
|
||||
public override long Length => _baseStream.Length;
|
||||
public override long Position
|
||||
{
|
||||
get => _baseStream.Position;
|
||||
set => _baseStream.Position = value;
|
||||
}
|
||||
|
||||
public override int Read(byte[] buffer, int offset, int count)
|
||||
{
|
||||
var bytesRead = _baseStream.Read(buffer, offset, count);
|
||||
_totalBytesRead += bytesRead;
|
||||
_progressCallback(_totalBytesRead);
|
||||
return bytesRead;
|
||||
}
|
||||
|
||||
public override void Flush() => _baseStream.Flush();
|
||||
public override long Seek(long offset, SeekOrigin origin) => _baseStream.Seek(offset, origin);
|
||||
public override void SetLength(long value) => _baseStream.SetLength(value);
|
||||
public override void Write(byte[] buffer, int offset, int count) => _baseStream.Write(buffer, offset, count);
|
||||
|
||||
protected override void Dispose(bool disposing)
|
||||
{
|
||||
if (disposing)
|
||||
{
|
||||
_baseStream?.Dispose();
|
||||
}
|
||||
base.Dispose(disposing);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Downloads a file using chunked transfer with resume capability
|
||||
/// </summary>
|
||||
/// <param name="transferState">Transfer state for resume functionality</param>
|
||||
/// <param name="progressReporter">Thread-safe progress reporter for updates</param>
|
||||
/// <param name="maxRetries">Maximum retry attempts per chunk</param>
|
||||
/// <param name="parallelDownloads">Number of parallel chunk downloads</param>
|
||||
/// <returns>True if download succeeded, false otherwise</returns>
|
||||
public bool DownloadFileChunked(
|
||||
ChunkedTransferState transferState,
|
||||
ThreadSafeChunkedProgressReporter progressReporter,
|
||||
int maxRetries = 3,
|
||||
int parallelDownloads = 3)
|
||||
{
|
||||
if (transferState == null)
|
||||
throw new ArgumentNullException(nameof(transferState));
|
||||
|
||||
try
|
||||
{
|
||||
// Create or open the target file
|
||||
using var fileStream = new FileStream(transferState.FilePath, FileMode.Create, FileAccess.Write, FileShare.None);
|
||||
fileStream.SetLength(transferState.TotalSize);
|
||||
|
||||
// Get list of chunks to download (with safety limit)
|
||||
var chunksToDownload = new List<ChunkInfo>();
|
||||
var maxChunks = Math.Min(transferState.TotalChunks, 10000); // Safety limit of 10,000 chunks
|
||||
|
||||
for (int i = 0; i < maxChunks && !transferState.IsComplete; i++)
|
||||
{
|
||||
var nextChunk = transferState.GetNextChunk();
|
||||
if (nextChunk == null)
|
||||
break;
|
||||
chunksToDownload.Add(nextChunk);
|
||||
}
|
||||
|
||||
if (chunksToDownload.Count == 0)
|
||||
{
|
||||
return true; // Already complete
|
||||
}
|
||||
|
||||
// Download chunks (with limited parallelism)
|
||||
var semaphore = new SemaphoreSlim(parallelDownloads, parallelDownloads);
|
||||
var downloadTasks = chunksToDownload.Select(chunk =>
|
||||
DownloadChunkAsync(transferState, chunk, fileStream, progressReporter, maxRetries, semaphore)).ToArray();
|
||||
|
||||
var results = Task.WhenAll(downloadTasks).GetAwaiter().GetResult();
|
||||
|
||||
// Check if all chunks downloaded successfully
|
||||
var allSucceeded = results.All(r => r);
|
||||
if (allSucceeded)
|
||||
{
|
||||
// Mark all chunks as completed
|
||||
foreach (var chunk in chunksToDownload)
|
||||
{
|
||||
transferState.MarkChunkCompleted(chunk);
|
||||
}
|
||||
}
|
||||
|
||||
return allSucceeded;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Chunked download failed for object '{transferState.ObjectName}': {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Downloads a single chunk asynchronously with retry logic
|
||||
/// </summary>
|
||||
/// <param name="transferState">Transfer state</param>
|
||||
/// <param name="chunk">Chunk to download</param>
|
||||
/// <param name="fileStream">Target file stream</param>
|
||||
/// <param name="progressReporter">Thread-safe progress reporter</param>
|
||||
/// <param name="maxRetries">Maximum retry attempts</param>
|
||||
/// <param name="semaphore">Semaphore for controlling parallelism</param>
|
||||
/// <returns>True if chunk downloaded successfully</returns>
|
||||
private async Task<bool> DownloadChunkAsync(
|
||||
ChunkedTransferState transferState,
|
||||
ChunkInfo chunk,
|
||||
FileStream fileStream,
|
||||
ThreadSafeChunkedProgressReporter progressReporter,
|
||||
int maxRetries,
|
||||
SemaphoreSlim semaphore)
|
||||
{
|
||||
await semaphore.WaitAsync(CancellationToken);
|
||||
|
||||
try
|
||||
{
|
||||
progressReporter.StartNewChunk(chunk.ChunkNumber + 1, chunk.Size);
|
||||
|
||||
for (int attempt = 1; attempt <= maxRetries; attempt++)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var chunkStream = new MemoryStream();
|
||||
|
||||
// Use GetObjectAsync with offset and length for proper chunked download
|
||||
var getArgs = new GetObjectArgs()
|
||||
.WithBucket(transferState.BucketName)
|
||||
.WithObject(transferState.ObjectName)
|
||||
.WithCallbackStream((stream) =>
|
||||
{
|
||||
// Skip to the start position and read only the chunk size
|
||||
var buffer = new byte[8192]; // 8KB buffer
|
||||
long totalRead = 0;
|
||||
long skipBytes = chunk.StartByte;
|
||||
|
||||
// Skip to start position
|
||||
while (skipBytes > 0)
|
||||
{
|
||||
var toSkip = (int)Math.Min(skipBytes, buffer.Length);
|
||||
var skipped = stream.Read(buffer, 0, toSkip);
|
||||
if (skipped == 0) break; // End of stream
|
||||
skipBytes -= skipped;
|
||||
}
|
||||
|
||||
// Read the chunk data
|
||||
while (totalRead < chunk.Size)
|
||||
{
|
||||
var toRead = (int)Math.Min(chunk.Size - totalRead, buffer.Length);
|
||||
var bytesRead = stream.Read(buffer, 0, toRead);
|
||||
if (bytesRead == 0) break; // End of stream
|
||||
|
||||
chunkStream.Write(buffer, 0, bytesRead);
|
||||
totalRead += bytesRead;
|
||||
}
|
||||
});
|
||||
|
||||
await _client.GetObjectAsync(getArgs, CancellationToken);
|
||||
|
||||
// Write chunk to file at correct position
|
||||
lock (fileStream)
|
||||
{
|
||||
fileStream.Seek(chunk.StartByte, SeekOrigin.Begin);
|
||||
chunkStream.Seek(0, SeekOrigin.Begin);
|
||||
chunkStream.CopyTo(fileStream);
|
||||
fileStream.Flush();
|
||||
}
|
||||
|
||||
progressReporter.UpdateChunkProgress(chunk.Size);
|
||||
progressReporter.CompleteChunk();
|
||||
|
||||
chunk.IsCompleted = true;
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex) when (attempt < maxRetries)
|
||||
{
|
||||
progressReporter.ReportChunkError(ex, attempt, maxRetries);
|
||||
|
||||
// Exponential backoff
|
||||
var delay = TimeSpan.FromSeconds(Math.Pow(2, attempt));
|
||||
await Task.Delay(delay, CancellationToken);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
progressReporter.ReportChunkError(ex, attempt, maxRetries);
|
||||
chunk.LastError = ex.Message;
|
||||
chunk.RetryCount = attempt;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
finally
|
||||
{
|
||||
semaphore.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cancels all ongoing operations
|
||||
/// </summary>
|
||||
public void CancelOperations()
|
||||
{
|
||||
_cancellationTokenSource.Cancel();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Disposes the wrapper and underlying client
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(true);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Protected dispose method
|
||||
/// </summary>
|
||||
/// <param name="disposing">Whether disposing from Dispose method</param>
|
||||
protected virtual void Dispose(bool disposing)
|
||||
{
|
||||
if (!_disposed && disposing)
|
||||
{
|
||||
_cancellationTokenSource?.Cancel();
|
||||
_cancellationTokenSource?.Dispose();
|
||||
_client?.Dispose();
|
||||
_disposed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+111
-173
@@ -4,23 +4,13 @@ using System.Management.Automation;
|
||||
namespace PSMinIO.Utils
|
||||
{
|
||||
/// <summary>
|
||||
/// Centralized logging utility for PSMinIO module
|
||||
/// Centralized logging utility for PSMinIO operations
|
||||
/// Provides consistent logging with timestamps and proper PowerShell integration
|
||||
/// </summary>
|
||||
public static class MinIOLogger
|
||||
{
|
||||
/// <summary>
|
||||
/// Log levels for different types of messages
|
||||
/// </summary>
|
||||
public enum LogLevel
|
||||
{
|
||||
Verbose,
|
||||
Information,
|
||||
Warning,
|
||||
Error
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a verbose message if verbose preference allows it
|
||||
/// Writes a verbose message with timestamp
|
||||
/// </summary>
|
||||
/// <param name="cmdlet">The cmdlet instance for context</param>
|
||||
/// <param name="message">Message to log</param>
|
||||
@@ -29,220 +19,168 @@ namespace PSMinIO.Utils
|
||||
{
|
||||
if (cmdlet == null) return;
|
||||
|
||||
var formattedMessage = FormatLogMessage(LogLevel.Verbose, message, args);
|
||||
cmdlet.WriteVerbose(formattedMessage);
|
||||
var timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss");
|
||||
var formattedMessage = args.Length > 0 ? string.Format(message, args) : message;
|
||||
cmdlet.WriteVerbose($"[{timestamp}] {formattedMessage}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes an information message
|
||||
/// Writes a warning message with timestamp
|
||||
/// </summary>
|
||||
/// <param name="cmdlet">The cmdlet instance for context</param>
|
||||
/// <param name="message">Message to log</param>
|
||||
/// <param name="args">Optional format arguments</param>
|
||||
public static void WriteInformation(PSCmdlet cmdlet, string message, params object[] args)
|
||||
{
|
||||
if (cmdlet == null) return;
|
||||
|
||||
var formattedMessage = FormatLogMessage(LogLevel.Information, message, args);
|
||||
|
||||
// Use WriteInformation if available (PowerShell 5.0+), otherwise WriteVerbose
|
||||
try
|
||||
{
|
||||
var infoRecord = new InformationRecord(formattedMessage, "PSMinIO");
|
||||
cmdlet.WriteInformation(infoRecord);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Fallback to WriteVerbose for older PowerShell versions
|
||||
cmdlet.WriteVerbose(formattedMessage);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a warning message
|
||||
/// </summary>
|
||||
/// <param name="cmdlet">The cmdlet instance for context</param>
|
||||
/// <param name="message">Message to log</param>
|
||||
/// <param name="message">Warning message to log</param>
|
||||
/// <param name="args">Optional format arguments</param>
|
||||
public static void WriteWarning(PSCmdlet cmdlet, string message, params object[] args)
|
||||
{
|
||||
if (cmdlet == null) return;
|
||||
|
||||
var formattedMessage = FormatLogMessage(LogLevel.Warning, message, args);
|
||||
cmdlet.WriteWarning(formattedMessage);
|
||||
var timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss");
|
||||
var formattedMessage = args.Length > 0 ? string.Format(message, args) : message;
|
||||
cmdlet.WriteWarning($"[{timestamp}] {formattedMessage}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes an error message
|
||||
/// Writes a debug message with timestamp
|
||||
/// </summary>
|
||||
/// <param name="cmdlet">The cmdlet instance for context</param>
|
||||
/// <param name="message">Message to log</param>
|
||||
/// <param name="message">Debug message to log</param>
|
||||
/// <param name="args">Optional format arguments</param>
|
||||
public static void WriteError(PSCmdlet cmdlet, string message, params object[] args)
|
||||
public static void WriteDebug(PSCmdlet cmdlet, string message, params object[] args)
|
||||
{
|
||||
if (cmdlet == null) return;
|
||||
|
||||
var formattedMessage = FormatLogMessage(LogLevel.Error, message, args);
|
||||
var errorRecord = new ErrorRecord(
|
||||
new InvalidOperationException(formattedMessage),
|
||||
"PSMinIOError",
|
||||
ErrorCategory.InvalidOperation,
|
||||
null);
|
||||
|
||||
cmdlet.WriteError(errorRecord);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes an error from an exception
|
||||
/// </summary>
|
||||
/// <param name="cmdlet">The cmdlet instance for context</param>
|
||||
/// <param name="exception">Exception to log</param>
|
||||
/// <param name="errorId">Error identifier</param>
|
||||
/// <param name="category">Error category</param>
|
||||
/// <param name="targetObject">Target object that caused the error</param>
|
||||
public static void WriteError(PSCmdlet cmdlet, Exception exception, string errorId,
|
||||
ErrorCategory category = ErrorCategory.InvalidOperation, object? targetObject = null)
|
||||
{
|
||||
if (cmdlet == null) return;
|
||||
|
||||
var errorRecord = new ErrorRecord(exception, errorId, category, targetObject);
|
||||
cmdlet.WriteError(errorRecord);
|
||||
var timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss");
|
||||
var formattedMessage = args.Length > 0 ? string.Format(message, args) : message;
|
||||
cmdlet.WriteDebug($"[{timestamp}] {formattedMessage}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Logs the start of an operation
|
||||
/// </summary>
|
||||
/// <param name="cmdlet">The cmdlet instance for context</param>
|
||||
/// <param name="operation">Name of the operation</param>
|
||||
/// <param name="operationName">Name of the operation</param>
|
||||
/// <param name="details">Optional operation details</param>
|
||||
public static void LogOperationStart(PSCmdlet cmdlet, string operation, string? details = null)
|
||||
public static void LogOperationStart(PSCmdlet cmdlet, string operationName, string? details = null)
|
||||
{
|
||||
var message = string.IsNullOrEmpty(details)
|
||||
? $"Starting operation: {operation}"
|
||||
: $"Starting operation: {operation} - {details}";
|
||||
? $"Starting {operationName}"
|
||||
: $"Starting {operationName} - {details}";
|
||||
|
||||
WriteVerbose(cmdlet, message);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Logs the completion of an operation
|
||||
/// Logs the successful completion of an operation
|
||||
/// </summary>
|
||||
/// <param name="cmdlet">The cmdlet instance for context</param>
|
||||
/// <param name="operation">Name of the operation</param>
|
||||
/// <param name="duration">Optional operation duration</param>
|
||||
/// <param name="operationName">Name of the operation</param>
|
||||
/// <param name="duration">Operation duration</param>
|
||||
/// <param name="details">Optional operation details</param>
|
||||
public static void LogOperationComplete(PSCmdlet cmdlet, string operation, TimeSpan? duration = null, string? details = null)
|
||||
public static void LogOperationComplete(PSCmdlet cmdlet, string operationName, TimeSpan duration, string? details = null)
|
||||
{
|
||||
var message = $"Completed operation: {operation}";
|
||||
|
||||
if (duration.HasValue)
|
||||
{
|
||||
message += $" (Duration: {duration.Value.TotalMilliseconds:F0}ms)";
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(details))
|
||||
{
|
||||
message += $" - {details}";
|
||||
}
|
||||
var durationStr = SizeFormatter.FormatDuration(duration);
|
||||
var message = string.IsNullOrEmpty(details)
|
||||
? $"Completed {operationName} in {durationStr}"
|
||||
: $"Completed {operationName} in {durationStr} - {details}";
|
||||
|
||||
WriteVerbose(cmdlet, message);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Logs an operation failure
|
||||
/// Logs the failure of an operation
|
||||
/// </summary>
|
||||
/// <param name="cmdlet">The cmdlet instance for context</param>
|
||||
/// <param name="operation">Name of the operation</param>
|
||||
/// <param name="operationName">Name of the operation</param>
|
||||
/// <param name="exception">Exception that occurred</param>
|
||||
/// <param name="details">Optional operation details</param>
|
||||
public static void LogOperationFailure(PSCmdlet cmdlet, string operation, Exception exception, string? details = null)
|
||||
public static void LogOperationFailure(PSCmdlet cmdlet, string operationName, Exception exception, string? details = null)
|
||||
{
|
||||
var message = $"Operation failed: {operation} - {exception.Message}";
|
||||
var message = string.IsNullOrEmpty(details)
|
||||
? $"Failed {operationName}: {exception.Message}"
|
||||
: $"Failed {operationName} - {details}: {exception.Message}";
|
||||
|
||||
if (!string.IsNullOrEmpty(details))
|
||||
{
|
||||
message += $" - {details}";
|
||||
}
|
||||
WriteWarning(cmdlet, message);
|
||||
WriteDebug(cmdlet, "Exception details: {0}", exception.ToString());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Logs transfer progress information
|
||||
/// </summary>
|
||||
/// <param name="cmdlet">The cmdlet instance for context</param>
|
||||
/// <param name="operationName">Name of the transfer operation</param>
|
||||
/// <param name="bytesTransferred">Bytes transferred so far</param>
|
||||
/// <param name="totalBytes">Total bytes to transfer</param>
|
||||
/// <param name="speed">Current transfer speed in bytes per second</param>
|
||||
public static void LogTransferProgress(PSCmdlet cmdlet, string operationName, long bytesTransferred, long totalBytes, double speed)
|
||||
{
|
||||
var percentage = totalBytes > 0 ? (double)bytesTransferred / totalBytes * 100 : 0;
|
||||
var speedStr = SizeFormatter.FormatSpeed(speed);
|
||||
var transferredStr = SizeFormatter.FormatBytes(bytesTransferred);
|
||||
var totalStr = SizeFormatter.FormatBytes(totalBytes);
|
||||
|
||||
WriteVerbose(cmdlet, "{0} progress: {1:F1}% ({2}/{3}) at {4}",
|
||||
operationName, percentage, transferredStr, totalStr, speedStr);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Logs connection information
|
||||
/// </summary>
|
||||
/// <param name="cmdlet">The cmdlet instance for context</param>
|
||||
/// <param name="endpoint">MinIO endpoint</param>
|
||||
/// <param name="useSSL">Whether SSL is being used</param>
|
||||
/// <param name="region">Region</param>
|
||||
/// <param name="timeout">Timeout in seconds</param>
|
||||
public static void LogConnection(PSCmdlet cmdlet, string endpoint, bool useSSL, string region, int timeout)
|
||||
{
|
||||
WriteVerbose(cmdlet, "Connecting to MinIO - Endpoint: {0}, UseSSL: {1}, Region: {2}, Timeout: {3}s",
|
||||
endpoint, useSSL, region, timeout);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Logs retry attempt information
|
||||
/// </summary>
|
||||
/// <param name="cmdlet">The cmdlet instance for context</param>
|
||||
/// <param name="operationName">Name of the operation being retried</param>
|
||||
/// <param name="attempt">Current attempt number</param>
|
||||
/// <param name="maxAttempts">Maximum number of attempts</param>
|
||||
/// <param name="delay">Delay before retry</param>
|
||||
/// <param name="reason">Reason for retry</param>
|
||||
public static void LogRetryAttempt(PSCmdlet cmdlet, string operationName, int attempt, int maxAttempts, TimeSpan delay, string reason)
|
||||
{
|
||||
WriteVerbose(cmdlet, "Retrying {0} (attempt {1}/{2}) after {3} - {4}",
|
||||
operationName, attempt, maxAttempts, SizeFormatter.FormatDuration(delay), reason);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Logs HTTP request information for debugging
|
||||
/// </summary>
|
||||
/// <param name="cmdlet">The cmdlet instance for context</param>
|
||||
/// <param name="method">HTTP method</param>
|
||||
/// <param name="url">Request URL</param>
|
||||
/// <param name="contentLength">Content length (if applicable)</param>
|
||||
public static void LogHttpRequest(PSCmdlet cmdlet, string method, string url, long? contentLength = null)
|
||||
{
|
||||
var message = contentLength.HasValue
|
||||
? $"HTTP {method} {url} (Content-Length: {SizeFormatter.FormatBytes(contentLength.Value)})"
|
||||
: $"HTTP {method} {url}";
|
||||
|
||||
WriteError(cmdlet, message);
|
||||
WriteVerbose(cmdlet, $"Exception details: {exception}");
|
||||
WriteDebug(cmdlet, message);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Formats a log message with timestamp and level, automatically formatting byte sizes
|
||||
/// Logs HTTP response information for debugging
|
||||
/// </summary>
|
||||
/// <param name="level">Log level</param>
|
||||
/// <param name="message">Message to format</param>
|
||||
/// <param name="args">Optional format arguments</param>
|
||||
/// <returns>Formatted log message</returns>
|
||||
private static string FormatLogMessage(LogLevel level, string message, params object[] args)
|
||||
/// <param name="cmdlet">The cmdlet instance for context</param>
|
||||
/// <param name="statusCode">HTTP status code</param>
|
||||
/// <param name="contentLength">Response content length (if known)</param>
|
||||
/// <param name="duration">Request duration</param>
|
||||
public static void LogHttpResponse(PSCmdlet cmdlet, int statusCode, long? contentLength, TimeSpan duration)
|
||||
{
|
||||
var timestamp = DateTime.Now.ToString("yyyy/MM/dd HH:mm:ss.fff");
|
||||
|
||||
// Process arguments to format byte sizes intelligently
|
||||
var processedArgs = ProcessLogArguments(args);
|
||||
var formattedMessage = processedArgs.Length > 0 ? string.Format(message, processedArgs) : message;
|
||||
|
||||
return $"{timestamp} - {formattedMessage}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Processes log arguments to format byte sizes intelligently
|
||||
/// </summary>
|
||||
/// <param name="args">Original arguments</param>
|
||||
/// <returns>Processed arguments with formatted sizes</returns>
|
||||
private static object[] ProcessLogArguments(object[] args)
|
||||
{
|
||||
if (args == null || args.Length == 0)
|
||||
return args ?? new object[0];
|
||||
|
||||
var processedArgs = new object[args.Length];
|
||||
|
||||
for (int i = 0; i < args.Length; i++)
|
||||
{
|
||||
var arg = args[i];
|
||||
|
||||
// Check if this looks like a byte size that should be formatted
|
||||
if (IsLikelyByteSize(arg))
|
||||
{
|
||||
if (long.TryParse(arg.ToString(), out var bytes))
|
||||
{
|
||||
processedArgs[i] = SizeFormatter.FormatBytes(bytes);
|
||||
}
|
||||
else
|
||||
{
|
||||
processedArgs[i] = arg;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
processedArgs[i] = arg;
|
||||
}
|
||||
}
|
||||
|
||||
return processedArgs;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines if an argument is likely a byte size that should be formatted
|
||||
/// </summary>
|
||||
/// <param name="arg">Argument to check</param>
|
||||
/// <returns>True if likely a byte size</returns>
|
||||
private static bool IsLikelyByteSize(object arg)
|
||||
{
|
||||
// Only format long integers that are likely byte sizes
|
||||
// We use a heuristic: values >= 1024 are likely byte sizes
|
||||
if (arg is long longValue)
|
||||
{
|
||||
return longValue >= 1024;
|
||||
}
|
||||
|
||||
if (arg is int intValue)
|
||||
{
|
||||
return intValue >= 1024;
|
||||
}
|
||||
|
||||
return false;
|
||||
var message = contentLength.HasValue
|
||||
? $"HTTP {statusCode} (Content-Length: {SizeFormatter.FormatBytes(contentLength.Value)}) in {SizeFormatter.FormatDuration(duration)}"
|
||||
: $"HTTP {statusCode} in {SizeFormatter.FormatDuration(duration)}";
|
||||
|
||||
WriteDebug(cmdlet, message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,156 +0,0 @@
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.Management.Automation;
|
||||
|
||||
namespace PSMinIO.Utils
|
||||
{
|
||||
/// <summary>
|
||||
/// Utility class for reporting progress during file operations
|
||||
/// </summary>
|
||||
public class ProgressReporter
|
||||
{
|
||||
private readonly PSCmdlet _cmdlet;
|
||||
private readonly string _activity;
|
||||
private readonly string _statusDescription;
|
||||
private readonly long _totalBytes;
|
||||
private readonly Stopwatch _stopwatch;
|
||||
private long _bytesProcessed;
|
||||
private DateTime _lastUpdateTime;
|
||||
private readonly int _activityId;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new ProgressReporter instance
|
||||
/// </summary>
|
||||
/// <param name="cmdlet">The cmdlet to report progress to</param>
|
||||
/// <param name="activity">Description of the activity</param>
|
||||
/// <param name="statusDescription">Status description</param>
|
||||
/// <param name="totalBytes">Total number of bytes to process</param>
|
||||
/// <param name="activityId">Unique activity ID for progress reporting</param>
|
||||
public ProgressReporter(PSCmdlet cmdlet, string activity, string statusDescription, long totalBytes, int activityId = 1)
|
||||
{
|
||||
_cmdlet = cmdlet ?? throw new ArgumentNullException(nameof(cmdlet));
|
||||
_activity = activity ?? throw new ArgumentNullException(nameof(activity));
|
||||
_statusDescription = statusDescription ?? throw new ArgumentNullException(nameof(statusDescription));
|
||||
_totalBytes = totalBytes;
|
||||
_activityId = activityId;
|
||||
_stopwatch = Stopwatch.StartNew();
|
||||
_lastUpdateTime = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Updates the progress with the number of bytes processed
|
||||
/// </summary>
|
||||
/// <param name="bytesProcessed">Number of bytes processed so far</param>
|
||||
public void UpdateProgress(long bytesProcessed)
|
||||
{
|
||||
_bytesProcessed = bytesProcessed;
|
||||
|
||||
// Only update progress every 100ms to avoid overwhelming the console
|
||||
var now = DateTime.UtcNow;
|
||||
if ((now - _lastUpdateTime).TotalMilliseconds < 100 && bytesProcessed < _totalBytes)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_lastUpdateTime = now;
|
||||
|
||||
var percentComplete = _totalBytes > 0 ? (int)((double)bytesProcessed / _totalBytes * 100) : 0;
|
||||
var currentStatus = FormatCurrentStatus(bytesProcessed);
|
||||
|
||||
var progressRecord = new ProgressRecord(_activityId, _activity, currentStatus)
|
||||
{
|
||||
PercentComplete = Math.Min(percentComplete, 100)
|
||||
};
|
||||
|
||||
// Add remaining time estimate if we have enough data
|
||||
if (_stopwatch.ElapsedMilliseconds > 1000 && bytesProcessed > 0 && bytesProcessed < _totalBytes)
|
||||
{
|
||||
var remainingTime = EstimateRemainingTime(bytesProcessed);
|
||||
if (remainingTime.HasValue)
|
||||
{
|
||||
progressRecord.SecondsRemaining = (int)remainingTime.Value.TotalSeconds;
|
||||
}
|
||||
}
|
||||
|
||||
_cmdlet.WriteProgress(progressRecord);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Completes the progress reporting
|
||||
/// </summary>
|
||||
public void Complete()
|
||||
{
|
||||
_stopwatch.Stop();
|
||||
|
||||
var progressRecord = new ProgressRecord(_activityId, _activity, "Completed")
|
||||
{
|
||||
PercentComplete = 100,
|
||||
RecordType = ProgressRecordType.Completed
|
||||
};
|
||||
|
||||
_cmdlet.WriteProgress(progressRecord);
|
||||
|
||||
// Log completion details
|
||||
MinIOLogger.WriteVerbose(_cmdlet,
|
||||
$"Operation completed: {SizeFormatter.FormatBytes(_totalBytes)} processed in {_stopwatch.Elapsed.TotalSeconds:F1} seconds " +
|
||||
$"(Average speed: {SizeFormatter.FormatBytesPerSecond(CalculateAverageSpeed())})");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Formats the current status string
|
||||
/// </summary>
|
||||
/// <param name="bytesProcessed">Number of bytes processed</param>
|
||||
/// <returns>Formatted status string</returns>
|
||||
private string FormatCurrentStatus(long bytesProcessed)
|
||||
{
|
||||
var processedFormatted = SizeFormatter.FormatBytes(bytesProcessed);
|
||||
var totalFormatted = SizeFormatter.FormatBytes(_totalBytes);
|
||||
var speed = CalculateCurrentSpeed();
|
||||
var speedFormatted = SizeFormatter.FormatBytesPerSecond(speed);
|
||||
|
||||
return $"{_statusDescription}: {processedFormatted} / {totalFormatted} ({speedFormatted})";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Calculates the current transfer speed in bytes per second
|
||||
/// </summary>
|
||||
/// <returns>Current speed in bytes per second</returns>
|
||||
private double CalculateCurrentSpeed()
|
||||
{
|
||||
var elapsedSeconds = _stopwatch.Elapsed.TotalSeconds;
|
||||
return elapsedSeconds > 0 ? _bytesProcessed / elapsedSeconds : 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Calculates the average transfer speed in bytes per second
|
||||
/// </summary>
|
||||
/// <returns>Average speed in bytes per second</returns>
|
||||
private double CalculateAverageSpeed()
|
||||
{
|
||||
var elapsedSeconds = _stopwatch.Elapsed.TotalSeconds;
|
||||
return elapsedSeconds > 0 ? _totalBytes / elapsedSeconds : 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Estimates the remaining time for the operation
|
||||
/// </summary>
|
||||
/// <param name="bytesProcessed">Number of bytes processed so far</param>
|
||||
/// <returns>Estimated remaining time, or null if cannot be calculated</returns>
|
||||
private TimeSpan? EstimateRemainingTime(long bytesProcessed)
|
||||
{
|
||||
if (bytesProcessed <= 0 || _stopwatch.ElapsedMilliseconds <= 0)
|
||||
return null;
|
||||
|
||||
var remainingBytes = _totalBytes - bytesProcessed;
|
||||
var currentSpeed = CalculateCurrentSpeed();
|
||||
|
||||
if (currentSpeed <= 0)
|
||||
return null;
|
||||
|
||||
var remainingSeconds = remainingBytes / currentSpeed;
|
||||
return TimeSpan.FromSeconds(remainingSeconds);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
+101
-126
@@ -3,21 +3,18 @@ using System;
|
||||
namespace PSMinIO.Utils
|
||||
{
|
||||
/// <summary>
|
||||
/// Utility class for formatting byte sizes into human-readable strings
|
||||
/// Utility class for formatting byte sizes in human-readable format
|
||||
/// </summary>
|
||||
public static class SizeFormatter
|
||||
{
|
||||
/// <summary>
|
||||
/// Size units in order from smallest to largest
|
||||
/// </summary>
|
||||
private static readonly string[] SizeUnits = { "B", "KB", "MB", "GB", "TB", "PB", "EB" };
|
||||
private static readonly string[] SizeUnits = { "B", "KB", "MB", "GB", "TB", "PB" };
|
||||
|
||||
/// <summary>
|
||||
/// Formats bytes into a human-readable string with appropriate units
|
||||
/// Formats a byte count as a human-readable string with appropriate units
|
||||
/// </summary>
|
||||
/// <param name="bytes">Number of bytes</param>
|
||||
/// <param name="decimalPlaces">Number of decimal places to show (default: 2)</param>
|
||||
/// <returns>Formatted string with appropriate unit</returns>
|
||||
/// <returns>Formatted size string (e.g., "1.23 MB")</returns>
|
||||
public static string FormatBytes(long bytes, int decimalPlaces = 2)
|
||||
{
|
||||
if (bytes == 0)
|
||||
@@ -29,161 +26,139 @@ namespace PSMinIO.Utils
|
||||
int unitIndex = 0;
|
||||
double size = bytes;
|
||||
|
||||
// Find the appropriate unit
|
||||
while (size >= 1024 && unitIndex < SizeUnits.Length - 1)
|
||||
{
|
||||
size /= 1024;
|
||||
unitIndex++;
|
||||
}
|
||||
|
||||
// Format with specified decimal places
|
||||
var formatString = $"{{0:F{decimalPlaces}}} {{1}}";
|
||||
return string.Format(formatString, size, SizeUnits[unitIndex]);
|
||||
return $"{size.ToString($"F{decimalPlaces}")} {SizeUnits[unitIndex]}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Formats bytes into a human-readable string with appropriate units (double overload)
|
||||
/// Formats a byte count as a human-readable string with appropriate units (double overload)
|
||||
/// </summary>
|
||||
/// <param name="bytes">Number of bytes</param>
|
||||
/// <param name="decimalPlaces">Number of decimal places to show (default: 2)</param>
|
||||
/// <returns>Formatted string with appropriate unit</returns>
|
||||
/// <returns>Formatted size string (e.g., "1.23 MB")</returns>
|
||||
public static string FormatBytes(double bytes, int decimalPlaces = 2)
|
||||
{
|
||||
return FormatBytes((long)Math.Round(bytes), decimalPlaces);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Formats bytes per second into a human-readable string
|
||||
/// Parses a human-readable size string back to bytes
|
||||
/// </summary>
|
||||
/// <param name="bytesPerSecond">Bytes per second</param>
|
||||
/// <param name="sizeString">Size string (e.g., "1.5 MB")</param>
|
||||
/// <returns>Number of bytes</returns>
|
||||
/// <exception cref="ArgumentException">Thrown when the size string is invalid</exception>
|
||||
public static long ParseBytes(string sizeString)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(sizeString))
|
||||
throw new ArgumentException("Size string cannot be null or empty", nameof(sizeString));
|
||||
|
||||
sizeString = sizeString.Trim().ToUpperInvariant();
|
||||
|
||||
// Handle just numbers (assume bytes)
|
||||
if (double.TryParse(sizeString, out double justNumber))
|
||||
return (long)justNumber;
|
||||
|
||||
// Find the last space or digit-to-letter boundary
|
||||
int unitStartIndex = -1;
|
||||
for (int i = sizeString.Length - 1; i >= 0; i--)
|
||||
{
|
||||
if (char.IsDigit(sizeString[i]) || sizeString[i] == '.')
|
||||
{
|
||||
unitStartIndex = i + 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (unitStartIndex == -1 || unitStartIndex >= sizeString.Length)
|
||||
throw new ArgumentException($"Invalid size string format: {sizeString}", nameof(sizeString));
|
||||
|
||||
string numberPart = sizeString.Substring(0, unitStartIndex).Trim();
|
||||
string unitPart = sizeString.Substring(unitStartIndex).Trim();
|
||||
|
||||
if (!double.TryParse(numberPart, out double value))
|
||||
throw new ArgumentException($"Invalid numeric value: {numberPart}", nameof(sizeString));
|
||||
|
||||
long multiplier = GetMultiplierForUnit(unitPart);
|
||||
return (long)(value * multiplier);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the multiplier for a given unit
|
||||
/// </summary>
|
||||
/// <param name="unit">Unit string (e.g., "MB", "GB")</param>
|
||||
/// <returns>Multiplier value</returns>
|
||||
private static long GetMultiplierForUnit(string unit)
|
||||
{
|
||||
return unit switch
|
||||
{
|
||||
"B" or "BYTE" or "BYTES" => 1L,
|
||||
"KB" or "KILOBYTE" or "KILOBYTES" => 1024L,
|
||||
"MB" or "MEGABYTE" or "MEGABYTES" => 1024L * 1024L,
|
||||
"GB" or "GIGABYTE" or "GIGABYTES" => 1024L * 1024L * 1024L,
|
||||
"TB" or "TERABYTE" or "TERABYTES" => 1024L * 1024L * 1024L * 1024L,
|
||||
"PB" or "PETABYTE" or "PETABYTES" => 1024L * 1024L * 1024L * 1024L * 1024L,
|
||||
_ => throw new ArgumentException($"Unknown unit: {unit}")
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Formats transfer speed in bytes per second
|
||||
/// </summary>
|
||||
/// <param name="bytesPerSecond">Transfer speed in bytes per second</param>
|
||||
/// <param name="decimalPlaces">Number of decimal places to show (default: 2)</param>
|
||||
/// <returns>Formatted string with appropriate unit and "/s" suffix</returns>
|
||||
public static string FormatBytesPerSecond(double bytesPerSecond, int decimalPlaces = 2)
|
||||
/// <returns>Formatted speed string (e.g., "1.23 MB/s")</returns>
|
||||
public static string FormatSpeed(double bytesPerSecond, int decimalPlaces = 2)
|
||||
{
|
||||
return $"{FormatBytes(bytesPerSecond, decimalPlaces)}/s";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Formats a transfer rate with context
|
||||
/// Formats a time duration in a human-readable format
|
||||
/// </summary>
|
||||
/// <param name="bytesTransferred">Number of bytes transferred</param>
|
||||
/// <param name="elapsedTime">Time elapsed for the transfer</param>
|
||||
/// <param name="decimalPlaces">Number of decimal places to show (default: 2)</param>
|
||||
/// <returns>Formatted transfer rate string</returns>
|
||||
public static string FormatTransferRate(long bytesTransferred, TimeSpan elapsedTime, int decimalPlaces = 2)
|
||||
/// <param name="duration">Duration to format</param>
|
||||
/// <returns>Formatted duration string (e.g., "1h 23m 45s")</returns>
|
||||
public static string FormatDuration(TimeSpan duration)
|
||||
{
|
||||
if (elapsedTime.TotalSeconds <= 0)
|
||||
return "0 B/s";
|
||||
if (duration.TotalSeconds < 1)
|
||||
return $"{duration.TotalMilliseconds:F0}ms";
|
||||
|
||||
var bytesPerSecond = bytesTransferred / elapsedTime.TotalSeconds;
|
||||
return FormatBytesPerSecond(bytesPerSecond, decimalPlaces);
|
||||
if (duration.TotalMinutes < 1)
|
||||
return $"{duration.TotalSeconds:F1}s";
|
||||
|
||||
if (duration.TotalHours < 1)
|
||||
return $"{duration.Minutes}m {duration.Seconds}s";
|
||||
|
||||
if (duration.TotalDays < 1)
|
||||
return $"{duration.Hours}h {duration.Minutes}m {duration.Seconds}s";
|
||||
|
||||
return $"{duration.Days}d {duration.Hours}h {duration.Minutes}m";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Formats a progress string showing current/total with percentages
|
||||
/// Calculates and formats estimated time remaining
|
||||
/// </summary>
|
||||
/// <param name="current">Current bytes processed</param>
|
||||
/// <param name="total">Total bytes to process</param>
|
||||
/// <param name="decimalPlaces">Number of decimal places to show (default: 2)</param>
|
||||
/// <returns>Formatted progress string</returns>
|
||||
public static string FormatProgress(long current, long total, int decimalPlaces = 2)
|
||||
/// <param name="bytesTransferred">Bytes already transferred</param>
|
||||
/// <param name="totalBytes">Total bytes to transfer</param>
|
||||
/// <param name="elapsedTime">Time elapsed so far</param>
|
||||
/// <returns>Formatted ETA string</returns>
|
||||
public static string FormatETA(long bytesTransferred, long totalBytes, TimeSpan elapsedTime)
|
||||
{
|
||||
var currentFormatted = FormatBytes(current, decimalPlaces);
|
||||
var totalFormatted = FormatBytes(total, decimalPlaces);
|
||||
|
||||
if (total > 0)
|
||||
{
|
||||
var percentage = (double)current / total * 100;
|
||||
return $"{currentFormatted} / {totalFormatted} ({percentage:F1}%)";
|
||||
}
|
||||
|
||||
return $"{currentFormatted} / {totalFormatted}";
|
||||
}
|
||||
if (bytesTransferred <= 0 || totalBytes <= 0 || elapsedTime.TotalSeconds <= 0)
|
||||
return "Unknown";
|
||||
|
||||
/// <summary>
|
||||
/// Gets the appropriate unit for a given byte size without formatting
|
||||
/// </summary>
|
||||
/// <param name="bytes">Number of bytes</param>
|
||||
/// <returns>Appropriate unit string</returns>
|
||||
public static string GetAppropriateUnit(long bytes)
|
||||
{
|
||||
if (bytes == 0)
|
||||
return "B";
|
||||
if (bytesTransferred >= totalBytes)
|
||||
return "Complete";
|
||||
|
||||
int unitIndex = 0;
|
||||
double size = Math.Abs(bytes);
|
||||
double bytesPerSecond = bytesTransferred / elapsedTime.TotalSeconds;
|
||||
long remainingBytes = totalBytes - bytesTransferred;
|
||||
double remainingSeconds = remainingBytes / bytesPerSecond;
|
||||
|
||||
while (size >= 1024 && unitIndex < SizeUnits.Length - 1)
|
||||
{
|
||||
size /= 1024;
|
||||
unitIndex++;
|
||||
}
|
||||
|
||||
return SizeUnits[unitIndex];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Converts bytes to the specified unit
|
||||
/// </summary>
|
||||
/// <param name="bytes">Number of bytes</param>
|
||||
/// <param name="unit">Target unit (B, KB, MB, GB, TB, PB, EB)</param>
|
||||
/// <returns>Value in the specified unit</returns>
|
||||
public static double ConvertToUnit(long bytes, string unit)
|
||||
{
|
||||
var unitIndex = Array.IndexOf(SizeUnits, unit.ToUpperInvariant());
|
||||
if (unitIndex == -1)
|
||||
throw new ArgumentException($"Invalid unit: {unit}. Valid units are: {string.Join(", ", SizeUnits)}");
|
||||
|
||||
if (unitIndex == 0) // Bytes
|
||||
return bytes;
|
||||
|
||||
return bytes / Math.Pow(1024, unitIndex);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parses a size string back to bytes (e.g., "1.5 GB" -> bytes)
|
||||
/// </summary>
|
||||
/// <param name="sizeString">Size string to parse</param>
|
||||
/// <returns>Number of bytes</returns>
|
||||
public static long ParseSizeString(string sizeString)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(sizeString))
|
||||
throw new ArgumentException("Size string cannot be null or empty");
|
||||
|
||||
var parts = sizeString.Trim().Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
if (parts.Length != 2)
|
||||
throw new ArgumentException($"Invalid size string format: {sizeString}. Expected format: '1.5 GB'");
|
||||
|
||||
if (!double.TryParse(parts[0], out var value))
|
||||
throw new ArgumentException($"Invalid numeric value: {parts[0]}");
|
||||
|
||||
var unit = parts[1].ToUpperInvariant();
|
||||
var unitIndex = Array.IndexOf(SizeUnits, unit);
|
||||
if (unitIndex == -1)
|
||||
throw new ArgumentException($"Invalid unit: {unit}. Valid units are: {string.Join(", ", SizeUnits)}");
|
||||
|
||||
return (long)(value * Math.Pow(1024, unitIndex));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Formats a size comparison between two values
|
||||
/// </summary>
|
||||
/// <param name="value1">First value in bytes</param>
|
||||
/// <param name="value2">Second value in bytes</param>
|
||||
/// <param name="label1">Label for first value</param>
|
||||
/// <param name="label2">Label for second value</param>
|
||||
/// <param name="decimalPlaces">Number of decimal places to show (default: 2)</param>
|
||||
/// <returns>Formatted comparison string</returns>
|
||||
public static string FormatComparison(long value1, long value2, string label1, string label2, int decimalPlaces = 2)
|
||||
{
|
||||
var formatted1 = FormatBytes(value1, decimalPlaces);
|
||||
var formatted2 = FormatBytes(value2, decimalPlaces);
|
||||
|
||||
var difference = value1 - value2;
|
||||
var diffFormatted = FormatBytes(Math.Abs(difference), decimalPlaces);
|
||||
var diffDirection = difference >= 0 ? "larger" : "smaller";
|
||||
|
||||
return $"{label1}: {formatted1}, {label2}: {formatted2} ({diffFormatted} {diffDirection})";
|
||||
return FormatDuration(TimeSpan.FromSeconds(remainingSeconds));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,163 +0,0 @@
|
||||
using System;
|
||||
using System.Management.Automation;
|
||||
using System.Threading;
|
||||
|
||||
namespace PSMinIO.Utils
|
||||
{
|
||||
/// <summary>
|
||||
/// Thread-safe wrapper for chunked progress reporting that can be safely called from background threads
|
||||
/// </summary>
|
||||
public class ThreadSafeChunkedProgressReporter
|
||||
{
|
||||
private readonly ThreadSafeProgressCollector _progressCollector;
|
||||
private readonly string _operationName;
|
||||
private readonly long _totalSize;
|
||||
private readonly int _totalChunks;
|
||||
private readonly DateTime _startTime;
|
||||
|
||||
// Activity IDs for progress hierarchy
|
||||
private const int FileActivityId = 1;
|
||||
private const int ChunkActivityId = 2;
|
||||
|
||||
// Current state (using Interlocked for long values since volatile doesn't support long)
|
||||
private volatile int _currentChunk = 0;
|
||||
private long _currentChunkSize = 0;
|
||||
private long _totalBytesTransferred = 0;
|
||||
private volatile string _currentFileName = string.Empty;
|
||||
|
||||
public ThreadSafeChunkedProgressReporter(
|
||||
PSCmdlet cmdlet,
|
||||
long totalSize,
|
||||
int totalChunks,
|
||||
string operationName)
|
||||
{
|
||||
_progressCollector = new ThreadSafeProgressCollector(cmdlet);
|
||||
_totalSize = totalSize;
|
||||
_totalChunks = totalChunks;
|
||||
_operationName = operationName;
|
||||
_startTime = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Starts a new file operation (thread-safe)
|
||||
/// </summary>
|
||||
public void StartNewFile(string fileName, long fileSize, int totalChunks)
|
||||
{
|
||||
_currentFileName = fileName;
|
||||
Interlocked.Exchange(ref _totalBytesTransferred, 0);
|
||||
_currentChunk = 0;
|
||||
|
||||
_progressCollector.QueueVerboseMessage("Starting {0} of file: {1} ({2})",
|
||||
_operationName.ToLower(), fileName, SizeFormatter.FormatBytes(fileSize));
|
||||
|
||||
_progressCollector.QueueProgressUpdate(
|
||||
FileActivityId,
|
||||
$"{_operationName} File",
|
||||
$"{_operationName}: {fileName}",
|
||||
0);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Starts a new chunk operation (thread-safe)
|
||||
/// </summary>
|
||||
public void StartNewChunk(int chunkNumber, long chunkSize)
|
||||
{
|
||||
_currentChunk = chunkNumber;
|
||||
Interlocked.Exchange(ref _currentChunkSize, chunkSize);
|
||||
|
||||
_progressCollector.QueueVerboseMessage("File {0}: Starting chunk {1}/{2} ({3})",
|
||||
_currentFileName, chunkNumber, _totalChunks, SizeFormatter.FormatBytes(chunkSize));
|
||||
|
||||
_progressCollector.QueueProgressUpdate(
|
||||
ChunkActivityId,
|
||||
"Current Chunk",
|
||||
$"Chunk {chunkNumber}/{_totalChunks}",
|
||||
0,
|
||||
FileActivityId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Updates chunk progress (thread-safe)
|
||||
/// </summary>
|
||||
public void UpdateChunkProgress(long bytesTransferred)
|
||||
{
|
||||
var newTotal = Interlocked.Add(ref _totalBytesTransferred, bytesTransferred);
|
||||
var currentChunkSize = Interlocked.Read(ref _currentChunkSize);
|
||||
|
||||
var chunkPercent = currentChunkSize > 0 ?
|
||||
(int)((double)bytesTransferred / currentChunkSize * 100) : 100;
|
||||
var filePercent = _totalSize > 0 ?
|
||||
(int)((double)newTotal / _totalSize * 100) : 100;
|
||||
|
||||
// Update chunk progress
|
||||
_progressCollector.QueueProgressUpdate(
|
||||
ChunkActivityId,
|
||||
"Current Chunk",
|
||||
$"Chunk {_currentChunk}/{_totalChunks} - {SizeFormatter.FormatBytes(bytesTransferred)}/{SizeFormatter.FormatBytes(_currentChunkSize)}",
|
||||
Math.Min(chunkPercent, 100),
|
||||
FileActivityId);
|
||||
|
||||
// Update file progress
|
||||
_progressCollector.QueueProgressUpdate(
|
||||
FileActivityId,
|
||||
$"{_operationName} File",
|
||||
$"{_operationName}: {_currentFileName} - {SizeFormatter.FormatBytes(newTotal)}/{SizeFormatter.FormatBytes(_totalSize)}",
|
||||
Math.Min(filePercent, 100));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Completes the current chunk (thread-safe)
|
||||
/// </summary>
|
||||
public void CompleteChunk(string? chunkETag = null)
|
||||
{
|
||||
_progressCollector.QueueVerboseMessage("File {0}: Completed chunk {1}/{2}{3}",
|
||||
_currentFileName, _currentChunk, _totalChunks,
|
||||
!string.IsNullOrEmpty(chunkETag) ? $" - ETag: {chunkETag}" : "");
|
||||
|
||||
_progressCollector.QueueProgressCompletion(ChunkActivityId, "Current Chunk", FileActivityId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Completes the current file (thread-safe)
|
||||
/// </summary>
|
||||
public void CompleteFile()
|
||||
{
|
||||
var elapsed = DateTime.UtcNow - _startTime;
|
||||
_progressCollector.QueueVerboseMessage("File {0}: {1} completed in {2} - Total size: {3}",
|
||||
_currentFileName, _operationName.ToLower(), elapsed.ToString(@"hh\:mm\:ss"),
|
||||
SizeFormatter.FormatBytes(_totalSize));
|
||||
|
||||
_progressCollector.QueueProgressCompletion(FileActivityId, $"{_operationName} File");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reports a chunk error (thread-safe)
|
||||
/// </summary>
|
||||
public void ReportChunkError(Exception error, int retryAttempt, int maxRetries)
|
||||
{
|
||||
_progressCollector.QueueVerboseMessage("File {0}: Chunk {1}/{2} failed (attempt {3}/{4}): {5}",
|
||||
_currentFileName, _currentChunk, _totalChunks, retryAttempt, maxRetries, error.Message);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Processes all queued updates from the main thread (must be called from main thread)
|
||||
/// </summary>
|
||||
public void ProcessQueuedUpdates()
|
||||
{
|
||||
_progressCollector.ProcessQueuedUpdates();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Completes all operations and processes final updates (must be called from main thread)
|
||||
/// </summary>
|
||||
public void Complete()
|
||||
{
|
||||
_progressCollector.Complete();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the number of pending updates
|
||||
/// </summary>
|
||||
public int PendingUpdates => _progressCollector.PendingProgressUpdates + _progressCollector.PendingVerboseMessages;
|
||||
}
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Management.Automation;
|
||||
using System.Threading;
|
||||
|
||||
namespace PSMinIO.Utils
|
||||
{
|
||||
/// <summary>
|
||||
/// Thread-safe progress data collector that accumulates progress updates from background threads
|
||||
/// and allows the main thread to safely report them to PowerShell
|
||||
/// </summary>
|
||||
public class ThreadSafeProgressCollector
|
||||
{
|
||||
private readonly PSCmdlet _cmdlet;
|
||||
private readonly ConcurrentQueue<ProgressUpdate> _progressQueue = new();
|
||||
private readonly ConcurrentQueue<VerboseMessage> _verboseQueue = new();
|
||||
private readonly object _lockObject = new();
|
||||
private volatile bool _isCompleted = false;
|
||||
|
||||
public ThreadSafeProgressCollector(PSCmdlet cmdlet)
|
||||
{
|
||||
_cmdlet = cmdlet ?? throw new ArgumentNullException(nameof(cmdlet));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues a progress update from a background thread
|
||||
/// </summary>
|
||||
public void QueueProgressUpdate(int activityId, string activity, string statusDescription, int percentComplete, int parentActivityId = -1)
|
||||
{
|
||||
if (_isCompleted) return;
|
||||
|
||||
_progressQueue.Enqueue(new ProgressUpdate
|
||||
{
|
||||
ActivityId = activityId,
|
||||
Activity = activity,
|
||||
StatusDescription = statusDescription,
|
||||
PercentComplete = percentComplete,
|
||||
ParentActivityId = parentActivityId,
|
||||
Timestamp = DateTime.UtcNow
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues a progress completion from a background thread
|
||||
/// </summary>
|
||||
public void QueueProgressCompletion(int activityId, string activity, int parentActivityId = -1)
|
||||
{
|
||||
if (_isCompleted) return;
|
||||
|
||||
_progressQueue.Enqueue(new ProgressUpdate
|
||||
{
|
||||
ActivityId = activityId,
|
||||
Activity = activity,
|
||||
StatusDescription = "Completed",
|
||||
PercentComplete = 100,
|
||||
ParentActivityId = parentActivityId,
|
||||
IsCompleted = true,
|
||||
Timestamp = DateTime.UtcNow
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues a verbose message from a background thread
|
||||
/// </summary>
|
||||
public void QueueVerboseMessage(string message, params object[] args)
|
||||
{
|
||||
if (_isCompleted) return;
|
||||
|
||||
_verboseQueue.Enqueue(new VerboseMessage
|
||||
{
|
||||
Message = args.Length > 0 ? string.Format(message, args) : message,
|
||||
Timestamp = DateTime.UtcNow
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Processes all queued updates from the main thread (safe to call PowerShell methods)
|
||||
/// </summary>
|
||||
public void ProcessQueuedUpdates()
|
||||
{
|
||||
// Process verbose messages first
|
||||
while (_verboseQueue.TryDequeue(out var verboseMessage))
|
||||
{
|
||||
MinIOLogger.WriteVerbose(_cmdlet, verboseMessage.Message);
|
||||
}
|
||||
|
||||
// Process progress updates
|
||||
while (_progressQueue.TryDequeue(out var progressUpdate))
|
||||
{
|
||||
var progressRecord = new ProgressRecord(
|
||||
progressUpdate.ActivityId,
|
||||
progressUpdate.Activity,
|
||||
progressUpdate.StatusDescription)
|
||||
{
|
||||
PercentComplete = progressUpdate.PercentComplete
|
||||
};
|
||||
|
||||
if (progressUpdate.ParentActivityId >= 0)
|
||||
{
|
||||
progressRecord.ParentActivityId = progressUpdate.ParentActivityId;
|
||||
}
|
||||
|
||||
if (progressUpdate.IsCompleted)
|
||||
{
|
||||
progressRecord.RecordType = ProgressRecordType.Completed;
|
||||
}
|
||||
|
||||
_cmdlet.WriteProgress(progressRecord);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Marks the collector as completed (no more updates will be accepted)
|
||||
/// </summary>
|
||||
public void Complete()
|
||||
{
|
||||
_isCompleted = true;
|
||||
|
||||
// Process any remaining updates
|
||||
ProcessQueuedUpdates();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the number of pending progress updates
|
||||
/// </summary>
|
||||
public int PendingProgressUpdates => _progressQueue.Count;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the number of pending verbose messages
|
||||
/// </summary>
|
||||
public int PendingVerboseMessages => _verboseQueue.Count;
|
||||
|
||||
/// <summary>
|
||||
/// Progress update data structure
|
||||
/// </summary>
|
||||
private class ProgressUpdate
|
||||
{
|
||||
public int ActivityId { get; set; }
|
||||
public string Activity { get; set; } = string.Empty;
|
||||
public string StatusDescription { get; set; } = string.Empty;
|
||||
public int PercentComplete { get; set; }
|
||||
public int ParentActivityId { get; set; } = -1;
|
||||
public bool IsCompleted { get; set; }
|
||||
public DateTime Timestamp { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verbose message data structure
|
||||
/// </summary>
|
||||
private class VerboseMessage
|
||||
{
|
||||
public string Message { get; set; } = string.Empty;
|
||||
public DateTime Timestamp { get; set; }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,149 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Management.Automation;
|
||||
using PSMinIO.Models;
|
||||
|
||||
namespace PSMinIO.Utils
|
||||
{
|
||||
/// <summary>
|
||||
/// Thread-safe result collector that accumulates results from background threads
|
||||
/// and allows the main thread to safely output them via WriteObject
|
||||
/// </summary>
|
||||
public class ThreadSafeResultCollector
|
||||
{
|
||||
private readonly PSCmdlet _cmdlet;
|
||||
private readonly ConcurrentQueue<object> _resultQueue = new();
|
||||
private readonly ConcurrentQueue<ErrorRecord> _errorQueue = new();
|
||||
private readonly object _lockObject = new();
|
||||
private volatile bool _isCompleted = false;
|
||||
|
||||
public ThreadSafeResultCollector(PSCmdlet cmdlet)
|
||||
{
|
||||
_cmdlet = cmdlet ?? throw new ArgumentNullException(nameof(cmdlet));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues a successful result from a background thread
|
||||
/// </summary>
|
||||
public void QueueResult(object result)
|
||||
{
|
||||
if (_isCompleted || result == null) return;
|
||||
|
||||
_resultQueue.Enqueue(result);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues an error from a background thread
|
||||
/// </summary>
|
||||
public void QueueError(ErrorRecord error)
|
||||
{
|
||||
if (_isCompleted || error == null) return;
|
||||
|
||||
_errorQueue.Enqueue(error);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues an error from a background thread using exception details
|
||||
/// </summary>
|
||||
public void QueueError(Exception exception, string errorId, ErrorCategory category, object targetObject)
|
||||
{
|
||||
if (_isCompleted || exception == null) return;
|
||||
|
||||
var errorRecord = new ErrorRecord(exception, errorId, category, targetObject);
|
||||
_errorQueue.Enqueue(errorRecord);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Processes all queued results and errors from the main thread (safe to call PowerShell methods)
|
||||
/// </summary>
|
||||
public void ProcessQueuedResults()
|
||||
{
|
||||
// Process errors first
|
||||
while (_errorQueue.TryDequeue(out var error))
|
||||
{
|
||||
_cmdlet.WriteError(error);
|
||||
}
|
||||
|
||||
// Process successful results
|
||||
while (_resultQueue.TryDequeue(out var result))
|
||||
{
|
||||
_cmdlet.WriteObject(result);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Marks the collector as completed and processes all remaining results
|
||||
/// </summary>
|
||||
public void Complete()
|
||||
{
|
||||
_isCompleted = true;
|
||||
|
||||
// Process any remaining results
|
||||
ProcessQueuedResults();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets all queued results without processing them (for inspection)
|
||||
/// </summary>
|
||||
public List<object> GetQueuedResults()
|
||||
{
|
||||
var results = new List<object>();
|
||||
var tempQueue = new Queue<object>();
|
||||
|
||||
// Dequeue all items and re-queue them
|
||||
while (_resultQueue.TryDequeue(out var result))
|
||||
{
|
||||
results.Add(result);
|
||||
tempQueue.Enqueue(result);
|
||||
}
|
||||
|
||||
// Re-queue the items
|
||||
while (tempQueue.Count > 0)
|
||||
{
|
||||
_resultQueue.Enqueue(tempQueue.Dequeue());
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets all queued errors without processing them (for inspection)
|
||||
/// </summary>
|
||||
public List<ErrorRecord> GetQueuedErrors()
|
||||
{
|
||||
var errors = new List<ErrorRecord>();
|
||||
var tempQueue = new Queue<ErrorRecord>();
|
||||
|
||||
// Dequeue all items and re-queue them
|
||||
while (_errorQueue.TryDequeue(out var error))
|
||||
{
|
||||
errors.Add(error);
|
||||
tempQueue.Enqueue(error);
|
||||
}
|
||||
|
||||
// Re-queue the items
|
||||
while (tempQueue.Count > 0)
|
||||
{
|
||||
_errorQueue.Enqueue(tempQueue.Dequeue());
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the number of pending results
|
||||
/// </summary>
|
||||
public int PendingResults => _resultQueue.Count;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the number of pending errors
|
||||
/// </summary>
|
||||
public int PendingErrors => _errorQueue.Count;
|
||||
|
||||
/// <summary>
|
||||
/// Gets whether the collector has any pending items
|
||||
/// </summary>
|
||||
public bool HasPendingItems => PendingResults > 0 || PendingErrors > 0;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user