mirror of
https://github.com/Grace-Solutions/PSMinIO.git
synced 2026-07-26 06:48:13 +00:00
Implement comprehensive MinIO feature set with enterprise-grade functionality
✅ MULTIPART UPLOAD SUPPORT: • MultipartUploadManager with parallel processing and resume capability • Configurable chunk sizes (minimum 5MB for S3 compatibility) • Parallel uploads with semaphore-controlled concurrency • Resume functionality with upload ID and completed parts tracking • Comprehensive progress tracking with speed calculations • New-MinIOObjectMultipart cmdlet with FileInfo support ✅ MULTIPART DOWNLOAD WITH RESUME: • MultipartDownloadManager with parallel processing • Resume capability for interrupted downloads • Range request support for efficient chunked downloads • Enhanced progress tracking for large files • Get-MinIOObjectContentMultipart cmdlet with resume support ✅ PRESIGNED URL GENERATION: • PresignedUrlGenerator with AWS Signature Version 4 • Configurable expiration times (1 second to 7 days) • Permission-based access control (GET, PUT, DELETE, HEAD) • Additional headers support for custom requirements • New-MinIOPresignedUrl and Get-MinIOPresignedUrl cmdlets ✅ BUCKET POLICY MANAGEMENT: • BucketPolicyManager with comprehensive validation • Get, set, delete policy operations with JSON support • Policy validation and formatting capabilities • Built-in policy templates (read-only, read-write) • Get/Set/Remove-MinIOBucketPolicy cmdlets with ShouldProcess ✅ ADVANCED OBJECT METADATA HANDLING: • AdvancedMetadataHandler for S3-compatible metadata • Custom headers and content encoding support • Cache control and expiration settings • Server-side encryption configuration • Object tagging and user metadata support ✅ ARCHITECTURE ENHANCEMENTS: • Thread-safe operations with proper semaphore management • Comprehensive error handling and validation • Progress tracking integration with existing PSMinIO patterns • Consistent parameter naming and PowerShell best practices • Enterprise-grade logging and metrics collection ✅ CMDLET FEATURES: • FileInfo parameter support for type safety • ShouldProcess support for destructive operations • Comprehensive parameter validation and error handling • Always-return object pattern (no PassThru needed) • Pipeline compatibility and verbose logging ✅ PERFORMANCE OPTIMIZATIONS: • Configurable parallel processing (1-10 threads) • Intelligent chunk sizing with S3 compatibility • Memory-efficient streaming operations • Resume capability to minimize data transfer • Real-time speed and ETA calculations Features: True multipart uploads/downloads, presigned URLs, bucket policies, advanced metadata, parallel processing, resume capability, comprehensive progress tracking, and enterprise-grade reliability.
This commit is contained in:
@@ -71,8 +71,15 @@
|
||||
'Get-MinIOObject',
|
||||
'New-MinIOObject',
|
||||
'Get-MinIOObjectContent',
|
||||
'Get-MinIOObjectContentMultipart',
|
||||
'New-MinIOObjectMultipart',
|
||||
'Get-MinIOZipArchive',
|
||||
'New-MinIOZipArchive'
|
||||
'New-MinIOZipArchive',
|
||||
'Get-MinIOPresignedUrl',
|
||||
'New-MinIOPresignedUrl',
|
||||
'Get-MinIOBucketPolicy',
|
||||
'Set-MinIOBucketPolicy',
|
||||
'Remove-MinIOBucketPolicy'
|
||||
)
|
||||
|
||||
# Variables to export from this module
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Management.Automation;
|
||||
using PSMinIO.Core.Http;
|
||||
using PSMinIO.Core.Models;
|
||||
using PSMinIO.Core.S3;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Cmdlets
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets bucket policy
|
||||
/// </summary>
|
||||
[Cmdlet(VerbsCommon.Get, "MinIOBucketPolicy")]
|
||||
[OutputType(typeof(BucketPolicyResult))]
|
||||
public class GetMinIOBucketPolicyCmdlet : MinIOBaseCmdlet
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket
|
||||
/// </summary>
|
||||
[Parameter(Position = 0, Mandatory = true, ValueFromPipeline = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Bucket")]
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Processes the cmdlet
|
||||
/// </summary>
|
||||
protected override void ProcessRecord()
|
||||
{
|
||||
var result = ExecuteOperation("GetBucketPolicy", () =>
|
||||
{
|
||||
WriteVerboseMessage("Getting bucket policy for: {0}", BucketName);
|
||||
|
||||
// Get connection and create policy manager
|
||||
var connection = Connection;
|
||||
var httpClient = new MinIOHttpClient(connection.Configuration);
|
||||
var progressCollector = new ThreadSafeProgressCollector(this);
|
||||
var policyManager = new BucketPolicyManager(httpClient, progressCollector);
|
||||
|
||||
// Get bucket policy
|
||||
var policyResult = policyManager.GetBucketPolicy(BucketName);
|
||||
|
||||
if (policyResult.HasPolicy)
|
||||
{
|
||||
WriteVerboseMessage("Retrieved bucket policy successfully");
|
||||
}
|
||||
else
|
||||
{
|
||||
WriteVerboseMessage("No policy found for bucket: {0}", BucketName);
|
||||
}
|
||||
|
||||
return policyResult;
|
||||
|
||||
}, $"Bucket: {BucketName}");
|
||||
|
||||
// Always return the result object
|
||||
WriteObject(result);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets bucket policy
|
||||
/// </summary>
|
||||
[Cmdlet(VerbsCommon.Set, "MinIOBucketPolicy", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.High)]
|
||||
[OutputType(typeof(BucketPolicyResult))]
|
||||
public class SetMinIOBucketPolicyCmdlet : MinIOBaseCmdlet
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket
|
||||
/// </summary>
|
||||
[Parameter(Position = 0, Mandatory = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Bucket")]
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Bucket policy object
|
||||
/// </summary>
|
||||
[Parameter(Position = 1, Mandatory = true, ParameterSetName = "PolicyObject")]
|
||||
[ValidateNotNull]
|
||||
public BucketPolicy Policy { get; set; } = null!;
|
||||
|
||||
/// <summary>
|
||||
/// Bucket policy as JSON string
|
||||
/// </summary>
|
||||
[Parameter(Position = 1, Mandatory = true, ParameterSetName = "PolicyJson")]
|
||||
[ValidateNotNullOrEmpty]
|
||||
public string PolicyJson { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Create a simple read-only policy for public access
|
||||
/// </summary>
|
||||
[Parameter(Mandatory = true, ParameterSetName = "ReadOnly")]
|
||||
public SwitchParameter ReadOnly { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Object prefix for the policy (optional)
|
||||
/// </summary>
|
||||
[Parameter(ParameterSetName = "ReadOnly")]
|
||||
[Parameter(ParameterSetName = "ReadWrite")]
|
||||
public string? ObjectPrefix { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Create a read-write policy for specific principals
|
||||
/// </summary>
|
||||
[Parameter(Mandatory = true, ParameterSetName = "ReadWrite")]
|
||||
public SwitchParameter ReadWrite { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// List of principals for read-write policy
|
||||
/// </summary>
|
||||
[Parameter(Mandatory = true, ParameterSetName = "ReadWrite")]
|
||||
[ValidateNotNull]
|
||||
public string[] Principals { get; set; } = Array.Empty<string>();
|
||||
|
||||
/// <summary>
|
||||
/// Processes the cmdlet
|
||||
/// </summary>
|
||||
protected override void ProcessRecord()
|
||||
{
|
||||
if (ShouldProcess(BucketName, "Set bucket policy"))
|
||||
{
|
||||
var result = ExecuteOperation("SetBucketPolicy", () =>
|
||||
{
|
||||
WriteVerboseMessage("Setting bucket policy for: {0}", BucketName);
|
||||
|
||||
// Get connection and create policy manager
|
||||
var connection = Connection;
|
||||
var httpClient = new MinIOHttpClient(connection.Configuration);
|
||||
var progressCollector = new ThreadSafeProgressCollector(this);
|
||||
var policyManager = new BucketPolicyManager(httpClient, progressCollector);
|
||||
|
||||
BucketPolicyResult policyResult;
|
||||
|
||||
switch (ParameterSetName)
|
||||
{
|
||||
case "PolicyObject":
|
||||
policyResult = policyManager.SetBucketPolicy(BucketName, Policy);
|
||||
break;
|
||||
case "PolicyJson":
|
||||
policyResult = policyManager.SetBucketPolicyFromJson(BucketName, PolicyJson);
|
||||
break;
|
||||
case "ReadOnly":
|
||||
var readOnlyPolicy = policyManager.CreateReadOnlyPolicy(BucketName, ObjectPrefix);
|
||||
policyResult = policyManager.SetBucketPolicy(BucketName, readOnlyPolicy);
|
||||
WriteVerboseMessage("Created read-only policy for public access");
|
||||
break;
|
||||
case "ReadWrite":
|
||||
var readWritePolicy = policyManager.CreateReadWritePolicy(BucketName, new List<string>(Principals), ObjectPrefix);
|
||||
policyResult = policyManager.SetBucketPolicy(BucketName, readWritePolicy);
|
||||
WriteVerboseMessage("Created read-write policy for {0} principals", Principals.Length);
|
||||
break;
|
||||
default:
|
||||
throw new InvalidOperationException($"Unknown parameter set: {ParameterSetName}");
|
||||
}
|
||||
|
||||
if (policyResult.IsValid)
|
||||
{
|
||||
WriteVerboseMessage("Set bucket policy successfully");
|
||||
}
|
||||
else
|
||||
{
|
||||
WriteWarning($"Failed to set bucket policy: {policyResult.Error}");
|
||||
}
|
||||
|
||||
return policyResult;
|
||||
|
||||
}, $"Bucket: {BucketName}, ParameterSet: {ParameterSetName}");
|
||||
|
||||
// Always return the result object
|
||||
WriteObject(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes bucket policy
|
||||
/// </summary>
|
||||
[Cmdlet(VerbsCommon.Remove, "MinIOBucketPolicy", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.High)]
|
||||
[OutputType(typeof(BucketPolicyResult))]
|
||||
public class RemoveMinIOBucketPolicyCmdlet : MinIOBaseCmdlet
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket
|
||||
/// </summary>
|
||||
[Parameter(Position = 0, Mandatory = true, ValueFromPipeline = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Bucket")]
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Processes the cmdlet
|
||||
/// </summary>
|
||||
protected override void ProcessRecord()
|
||||
{
|
||||
if (ShouldProcess(BucketName, "Remove bucket policy"))
|
||||
{
|
||||
var result = ExecuteOperation("RemoveBucketPolicy", () =>
|
||||
{
|
||||
WriteVerboseMessage("Removing bucket policy for: {0}", BucketName);
|
||||
|
||||
// Get connection and create policy manager
|
||||
var connection = Connection;
|
||||
var httpClient = new MinIOHttpClient(connection.Configuration);
|
||||
var progressCollector = new ThreadSafeProgressCollector(this);
|
||||
var policyManager = new BucketPolicyManager(httpClient, progressCollector);
|
||||
|
||||
// Remove bucket policy
|
||||
var policyResult = policyManager.DeleteBucketPolicy(BucketName);
|
||||
|
||||
if (policyResult.IsValid)
|
||||
{
|
||||
WriteVerboseMessage("Removed bucket policy successfully");
|
||||
}
|
||||
else
|
||||
{
|
||||
WriteWarning($"Failed to remove bucket policy: {policyResult.Error}");
|
||||
}
|
||||
|
||||
return policyResult;
|
||||
|
||||
}, $"Bucket: {BucketName}");
|
||||
|
||||
// Always return the result object
|
||||
WriteObject(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -64,7 +64,7 @@ namespace PSMinIO.Cmdlets
|
||||
{
|
||||
var wildcardPattern = new WildcardPattern(BucketName, WildcardOptions.IgnoreCase);
|
||||
buckets = buckets.Where(b => b.Name != null && wildcardPattern.IsMatch(b.Name)).ToList();
|
||||
WriteVerboseMessage("Filtered to {0} buckets matching pattern '{1}'", buckets.Count, BucketName);
|
||||
WriteVerboseMessage("Filtered to {0} buckets matching pattern '{1}'", buckets.Count, BucketName!);
|
||||
}
|
||||
|
||||
// Enhance bucket information if requested
|
||||
|
||||
@@ -108,7 +108,7 @@ namespace PSMinIO.Cmdlets
|
||||
{
|
||||
var wildcardPattern = new WildcardPattern(Name, WildcardOptions.IgnoreCase);
|
||||
objects = objects.Where(o => o.Name != null && wildcardPattern.IsMatch(o.Name)).ToList();
|
||||
WriteVerboseMessage("Filtered to {0} objects matching pattern '{1}'", objects.Count, Name);
|
||||
WriteVerboseMessage("Filtered to {0} objects matching pattern '{1}'", objects.Count, Name!);
|
||||
}
|
||||
|
||||
// Apply directory filters
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Management.Automation;
|
||||
using PSMinIO.Core.Http;
|
||||
using PSMinIO.Core.Models;
|
||||
using PSMinIO.Core.S3;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Cmdlets
|
||||
{
|
||||
/// <summary>
|
||||
/// Downloads objects using multipart download with resume capability
|
||||
/// </summary>
|
||||
[Cmdlet(VerbsCommon.Get, "MinIOObjectContentMultipart", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Medium)]
|
||||
[OutputType(typeof(MultipartDownloadResult))]
|
||||
public class GetMinIOObjectContentMultipartCmdlet : MinIOBaseCmdlet
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket containing the object
|
||||
/// </summary>
|
||||
[Parameter(Position = 0, Mandatory = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Bucket")]
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Name of the object to download
|
||||
/// </summary>
|
||||
[Parameter(Position = 1, Mandatory = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Object", "Key")]
|
||||
public string ObjectName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Local file path where the object will be saved
|
||||
/// </summary>
|
||||
[Parameter(Position = 2, Mandatory = true)]
|
||||
[ValidateNotNull]
|
||||
[Alias("File", "Path")]
|
||||
public FileInfo DestinationPath { get; set; } = null!;
|
||||
|
||||
/// <summary>
|
||||
/// Size of each download chunk in bytes (default: 32MB)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateRange(1024 * 1024, long.MaxValue)] // Minimum 1MB
|
||||
public long ChunkSize { get; set; } = 32 * 1024 * 1024; // 32MB default
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of parallel download threads (default: 4)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateRange(1, 8)]
|
||||
public int MaxParallelDownloads { get; set; } = 4;
|
||||
|
||||
/// <summary>
|
||||
/// Resume download if destination file already exists
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter Resume { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Overwrite existing file without prompting
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter Force { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Processes the cmdlet
|
||||
/// </summary>
|
||||
protected override void ProcessRecord()
|
||||
{
|
||||
// Validate destination path
|
||||
if (DestinationPath.Exists && !Resume.IsPresent && !Force.IsPresent)
|
||||
{
|
||||
var errorRecord = new ErrorRecord(
|
||||
new InvalidOperationException($"Destination file already exists: {DestinationPath.FullName}. Use -Resume to continue or -Force to overwrite."),
|
||||
"DestinationFileExists",
|
||||
ErrorCategory.ResourceExists,
|
||||
DestinationPath);
|
||||
ThrowTerminatingError(errorRecord);
|
||||
}
|
||||
|
||||
// Ensure destination directory exists
|
||||
if (DestinationPath.Directory != null && !DestinationPath.Directory.Exists)
|
||||
{
|
||||
DestinationPath.Directory.Create();
|
||||
WriteVerboseMessage("Created destination directory: {0}", DestinationPath.Directory.FullName);
|
||||
}
|
||||
|
||||
var operationDescription = Resume.IsPresent && DestinationPath.Exists
|
||||
? $"Resume multipart download of '{ObjectName}' from bucket '{BucketName}'"
|
||||
: $"Multipart download '{ObjectName}' from bucket '{BucketName}'";
|
||||
|
||||
if (ShouldProcess(DestinationPath.FullName, operationDescription))
|
||||
{
|
||||
var result = ExecuteOperation("MultipartDownload", () =>
|
||||
{
|
||||
WriteVerboseMessage("Starting multipart download: {0}/{1} -> {2}",
|
||||
BucketName, ObjectName, DestinationPath.FullName);
|
||||
WriteVerboseMessage("Chunk size: {0}, Max parallel downloads: {1}, Resume: {2}",
|
||||
SizeFormatter.FormatBytes(ChunkSize), MaxParallelDownloads, Resume.IsPresent);
|
||||
|
||||
// Get connection and create download manager
|
||||
var connection = Connection;
|
||||
var httpClient = new MinIOHttpClient(connection.Configuration);
|
||||
var progressCollector = new ThreadSafeProgressCollector(this);
|
||||
var downloadManager = new MultipartDownloadManager(httpClient, progressCollector,
|
||||
MaxParallelDownloads, ChunkSize);
|
||||
|
||||
// Perform multipart download
|
||||
var downloadResult = downloadManager.DownloadFile(BucketName, ObjectName, DestinationPath,
|
||||
ChunkSize, Resume.IsPresent);
|
||||
|
||||
if (downloadResult.IsCompleted)
|
||||
{
|
||||
WriteVerboseMessage("Multipart download completed successfully");
|
||||
WriteVerboseMessage("Downloaded: {0} in {1} at {2}",
|
||||
SizeFormatter.FormatBytes(downloadResult.DownloadedSize),
|
||||
SizeFormatter.FormatDuration(downloadResult.Duration),
|
||||
SizeFormatter.FormatSpeed(downloadResult.AverageSpeed));
|
||||
}
|
||||
else
|
||||
{
|
||||
WriteWarning($"Multipart download failed: {downloadResult.Error}");
|
||||
}
|
||||
|
||||
return downloadResult;
|
||||
|
||||
}, $"Bucket: {BucketName}, Object: {ObjectName}, Destination: {DestinationPath.FullName}");
|
||||
|
||||
// Always return the result object
|
||||
WriteObject(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Management.Automation;
|
||||
using PSMinIO.Core.Http;
|
||||
using PSMinIO.Core.Models;
|
||||
using PSMinIO.Core.S3;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Cmdlets
|
||||
{
|
||||
/// <summary>
|
||||
/// Uploads objects using multipart upload with parallel processing and resume capability
|
||||
/// </summary>
|
||||
[Cmdlet(VerbsCommon.New, "MinIOObjectMultipart", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Medium)]
|
||||
[OutputType(typeof(MultipartUploadResult))]
|
||||
public class NewMinIOObjectMultipartCmdlet : MinIOBaseCmdlet
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket to upload to
|
||||
/// </summary>
|
||||
[Parameter(Position = 0, Mandatory = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Bucket")]
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Local file to upload
|
||||
/// </summary>
|
||||
[Parameter(Position = 1, Mandatory = true, ValueFromPipeline = true)]
|
||||
[ValidateNotNull]
|
||||
[Alias("File", "Path")]
|
||||
public FileInfo FilePath { get; set; } = null!;
|
||||
|
||||
/// <summary>
|
||||
/// Name of the object in the bucket (optional, uses filename if not specified)
|
||||
/// </summary>
|
||||
[Parameter(Position = 2)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Object", "Key")]
|
||||
public string? ObjectName { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Size of each upload chunk in bytes (default: 64MB, minimum: 5MB for S3 compatibility)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateRange(5 * 1024 * 1024, long.MaxValue)] // Minimum 5MB for S3 compatibility
|
||||
public long ChunkSize { get; set; } = 64 * 1024 * 1024; // 64MB default
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of parallel upload threads (default: 4)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateRange(1, 10)]
|
||||
public int MaxParallelUploads { get; set; } = 4;
|
||||
|
||||
/// <summary>
|
||||
/// Content type of the object (auto-detected if not specified)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateNotNullOrEmpty]
|
||||
public string? ContentType { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Custom metadata for the object
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public Dictionary<string, string>? Metadata { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Resume upload using existing upload ID
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
[ValidateNotNullOrEmpty]
|
||||
public string? ResumeUploadId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Completed parts for resume (used with ResumeUploadId)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public PartInfo[]? CompletedParts { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Overwrite existing object without prompting
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter Force { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Processes the cmdlet
|
||||
/// </summary>
|
||||
protected override void ProcessRecord()
|
||||
{
|
||||
// Validate file exists
|
||||
if (!FilePath.Exists)
|
||||
{
|
||||
var errorRecord = new ErrorRecord(
|
||||
new FileNotFoundException($"File not found: {FilePath.FullName}"),
|
||||
"FileNotFound",
|
||||
ErrorCategory.ObjectNotFound,
|
||||
FilePath);
|
||||
ThrowTerminatingError(errorRecord);
|
||||
}
|
||||
|
||||
// Determine object name
|
||||
var effectiveObjectName = ObjectName ?? FilePath.Name;
|
||||
|
||||
// Auto-detect content type if not specified
|
||||
var effectiveContentType = ContentType ?? GetContentType(FilePath.Extension);
|
||||
|
||||
// Add content type to metadata
|
||||
var effectiveMetadata = Metadata ?? new Dictionary<string, string>();
|
||||
if (!string.IsNullOrEmpty(effectiveContentType))
|
||||
{
|
||||
effectiveMetadata["Content-Type"] = effectiveContentType;
|
||||
}
|
||||
|
||||
var operationDescription = !string.IsNullOrEmpty(ResumeUploadId)
|
||||
? $"Resume multipart upload of '{FilePath.Name}' to bucket '{BucketName}'"
|
||||
: $"Multipart upload '{FilePath.Name}' to bucket '{BucketName}'";
|
||||
|
||||
if (ShouldProcess(effectiveObjectName, operationDescription))
|
||||
{
|
||||
var result = ExecuteOperation("MultipartUpload", () =>
|
||||
{
|
||||
WriteVerboseMessage("Starting multipart upload: {0} -> {1}/{2}",
|
||||
FilePath.FullName, BucketName, effectiveObjectName);
|
||||
WriteVerboseMessage("File size: {0}, Chunk size: {1}, Max parallel uploads: {2}",
|
||||
SizeFormatter.FormatBytes(FilePath.Length), SizeFormatter.FormatBytes(ChunkSize), MaxParallelUploads);
|
||||
|
||||
if (!string.IsNullOrEmpty(ResumeUploadId))
|
||||
{
|
||||
WriteVerboseMessage("Resuming upload with ID: {0}", ResumeUploadId);
|
||||
if (CompletedParts != null && CompletedParts.Length > 0)
|
||||
{
|
||||
WriteVerboseMessage("Found {0} completed parts", CompletedParts.Length);
|
||||
}
|
||||
}
|
||||
|
||||
// Get connection and create upload manager
|
||||
var connection = Connection;
|
||||
var httpClient = new MinIOHttpClient(connection.Configuration);
|
||||
var progressCollector = new ThreadSafeProgressCollector(this);
|
||||
var uploadManager = new MultipartUploadManager(httpClient, progressCollector,
|
||||
MaxParallelUploads, ChunkSize);
|
||||
|
||||
// Perform multipart upload
|
||||
var uploadResult = uploadManager.UploadFile(BucketName, effectiveObjectName, FilePath,
|
||||
effectiveMetadata, ChunkSize, ResumeUploadId, CompletedParts?.ToList());
|
||||
|
||||
if (uploadResult.IsCompleted)
|
||||
{
|
||||
WriteVerboseMessage("Multipart upload completed successfully");
|
||||
WriteVerboseMessage("Uploaded: {0} in {1} at {2}",
|
||||
SizeFormatter.FormatBytes(uploadResult.TotalSize),
|
||||
SizeFormatter.FormatDuration(uploadResult.Duration),
|
||||
SizeFormatter.FormatSpeed(uploadResult.AverageSpeed));
|
||||
WriteVerboseMessage("ETag: {0}", uploadResult.ETag);
|
||||
}
|
||||
else
|
||||
{
|
||||
WriteWarning($"Multipart upload failed: {uploadResult.Error}");
|
||||
if (uploadResult.CompletedParts.Count > 0)
|
||||
{
|
||||
WriteWarning($"Upload can be resumed using UploadId: {uploadResult.UploadId}");
|
||||
WriteWarning($"Completed parts: {uploadResult.CompletedParts.Count}/{uploadResult.TotalParts}");
|
||||
}
|
||||
}
|
||||
|
||||
return uploadResult;
|
||||
|
||||
}, $"File: {FilePath.FullName}, Bucket: {BucketName}, Object: {effectiveObjectName}");
|
||||
|
||||
// Always return the result object
|
||||
WriteObject(result);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets content type based on file extension
|
||||
/// </summary>
|
||||
private string GetContentType(string extension)
|
||||
{
|
||||
return extension.ToLowerInvariant() switch
|
||||
{
|
||||
".txt" => "text/plain",
|
||||
".html" => "text/html",
|
||||
".css" => "text/css",
|
||||
".js" => "application/javascript",
|
||||
".json" => "application/json",
|
||||
".xml" => "application/xml",
|
||||
".pdf" => "application/pdf",
|
||||
".zip" => "application/zip",
|
||||
".jpg" or ".jpeg" => "image/jpeg",
|
||||
".png" => "image/png",
|
||||
".gif" => "image/gif",
|
||||
".svg" => "image/svg+xml",
|
||||
".mp4" => "video/mp4",
|
||||
".mp3" => "audio/mpeg",
|
||||
".wav" => "audio/wav",
|
||||
".doc" => "application/msword",
|
||||
".docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
".xls" => "application/vnd.ms-excel",
|
||||
".xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
".ppt" => "application/vnd.ms-powerpoint",
|
||||
".pptx" => "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
_ => "application/octet-stream"
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Management.Automation;
|
||||
using System.Net.Http;
|
||||
using PSMinIO.Core.Models;
|
||||
using PSMinIO.Core.S3;
|
||||
|
||||
namespace PSMinIO.Cmdlets
|
||||
{
|
||||
/// <summary>
|
||||
/// Generates presigned URLs for temporary access to MinIO objects
|
||||
/// </summary>
|
||||
[Cmdlet(VerbsCommon.New, "MinIOPresignedUrl")]
|
||||
[OutputType(typeof(PresignedUrlResult))]
|
||||
public class NewMinIOPresignedUrlCmdlet : MinIOBaseCmdlet
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket
|
||||
/// </summary>
|
||||
[Parameter(Position = 0, Mandatory = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Bucket")]
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Name of the object
|
||||
/// </summary>
|
||||
[Parameter(Position = 1, Mandatory = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Object", "Key")]
|
||||
public string ObjectName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// HTTP method for the presigned URL
|
||||
/// </summary>
|
||||
[Parameter(Position = 2)]
|
||||
[ValidateSet("GET", "PUT", "DELETE", "HEAD")]
|
||||
public string Method { get; set; } = "GET";
|
||||
|
||||
/// <summary>
|
||||
/// Expiration time for the URL (default: 1 hour, maximum: 7 days)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public TimeSpan Expiration { get; set; } = TimeSpan.FromHours(1);
|
||||
|
||||
/// <summary>
|
||||
/// Additional headers to include in the presigned URL
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public Dictionary<string, string>? Headers { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Show the URL in the console (for easy copying)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter ShowUrl { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Processes the cmdlet
|
||||
/// </summary>
|
||||
protected override void ProcessRecord()
|
||||
{
|
||||
// Validate expiration time
|
||||
if (Expiration.TotalSeconds < 1)
|
||||
{
|
||||
ThrowTerminatingError(new ErrorRecord(
|
||||
new ArgumentException("Expiration must be at least 1 second"),
|
||||
"InvalidExpiration",
|
||||
ErrorCategory.InvalidArgument,
|
||||
Expiration));
|
||||
}
|
||||
if (Expiration.TotalDays > 7)
|
||||
{
|
||||
ThrowTerminatingError(new ErrorRecord(
|
||||
new ArgumentException("Expiration cannot exceed 7 days"),
|
||||
"InvalidExpiration",
|
||||
ErrorCategory.InvalidArgument,
|
||||
Expiration));
|
||||
}
|
||||
|
||||
var result = ExecuteOperation("GeneratePresignedUrl", () =>
|
||||
{
|
||||
WriteVerboseMessage("Generating presigned URL for: {0}/{1}", BucketName, ObjectName);
|
||||
WriteVerboseMessage("Method: {0}, Expiration: {1}", Method, Expiration);
|
||||
|
||||
// Get connection details
|
||||
var connection = Connection;
|
||||
|
||||
// Create presigned URL generator
|
||||
var generator = new PresignedUrlGenerator(
|
||||
connection.Configuration.Endpoint,
|
||||
connection.Configuration.AccessKey,
|
||||
connection.Configuration.SecretKey,
|
||||
connection.Configuration.Region,
|
||||
connection.Configuration.UseSSL);
|
||||
|
||||
// Parse HTTP method
|
||||
var httpMethod = Method.ToUpperInvariant() switch
|
||||
{
|
||||
"GET" => HttpMethod.Get,
|
||||
"PUT" => HttpMethod.Put,
|
||||
"DELETE" => HttpMethod.Delete,
|
||||
"HEAD" => HttpMethod.Head,
|
||||
_ => HttpMethod.Get
|
||||
};
|
||||
|
||||
// Generate presigned URL
|
||||
var urlResult = generator.GeneratePresignedUrl(httpMethod, BucketName, ObjectName, Expiration, Headers);
|
||||
|
||||
WriteVerboseMessage("Generated presigned URL successfully");
|
||||
WriteVerboseMessage("URL expires: {0}", urlResult.FormattedExpiration);
|
||||
|
||||
if (ShowUrl.IsPresent)
|
||||
{
|
||||
WriteInformation($"Presigned URL: {urlResult.Url}", new string[] { "PresignedUrl" });
|
||||
}
|
||||
|
||||
return urlResult;
|
||||
|
||||
}, $"Bucket: {BucketName}, Object: {ObjectName}, Method: {Method}");
|
||||
|
||||
// Always return the result object
|
||||
WriteObject(result);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cmdlet for generating GET presigned URLs (download)
|
||||
/// </summary>
|
||||
[Cmdlet(VerbsCommon.Get, "MinIOPresignedUrl")]
|
||||
[OutputType(typeof(PresignedUrlResult))]
|
||||
public class GetMinIOPresignedUrlCmdlet : MinIOBaseCmdlet
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the bucket
|
||||
/// </summary>
|
||||
[Parameter(Position = 0, Mandatory = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Bucket")]
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Name of the object
|
||||
/// </summary>
|
||||
[Parameter(Position = 1, Mandatory = true)]
|
||||
[ValidateNotNullOrEmpty]
|
||||
[Alias("Object", "Key")]
|
||||
public string ObjectName { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Expiration time for the URL (default: 1 hour, maximum: 7 days)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public TimeSpan Expiration { get; set; } = TimeSpan.FromHours(1);
|
||||
|
||||
/// <summary>
|
||||
/// Show the URL in the console (for easy copying)
|
||||
/// </summary>
|
||||
[Parameter]
|
||||
public SwitchParameter ShowUrl { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Processes the cmdlet
|
||||
/// </summary>
|
||||
protected override void ProcessRecord()
|
||||
{
|
||||
// Validate expiration time
|
||||
if (Expiration.TotalSeconds < 1)
|
||||
{
|
||||
ThrowTerminatingError(new ErrorRecord(
|
||||
new ArgumentException("Expiration must be at least 1 second"),
|
||||
"InvalidExpiration",
|
||||
ErrorCategory.InvalidArgument,
|
||||
Expiration));
|
||||
}
|
||||
if (Expiration.TotalDays > 7)
|
||||
{
|
||||
ThrowTerminatingError(new ErrorRecord(
|
||||
new ArgumentException("Expiration cannot exceed 7 days"),
|
||||
"InvalidExpiration",
|
||||
ErrorCategory.InvalidArgument,
|
||||
Expiration));
|
||||
}
|
||||
|
||||
var result = ExecuteOperation("GeneratePresignedGetUrl", () =>
|
||||
{
|
||||
WriteVerboseMessage("Generating presigned GET URL for: {0}/{1}", BucketName, ObjectName);
|
||||
WriteVerboseMessage("Expiration: {0}", Expiration);
|
||||
|
||||
// Get connection details
|
||||
var connection = Connection;
|
||||
|
||||
// Create presigned URL generator
|
||||
var generator = new PresignedUrlGenerator(
|
||||
connection.Configuration.Endpoint,
|
||||
connection.Configuration.AccessKey,
|
||||
connection.Configuration.SecretKey,
|
||||
connection.Configuration.Region,
|
||||
connection.Configuration.UseSSL);
|
||||
|
||||
// Generate presigned GET URL
|
||||
var urlResult = generator.GeneratePresignedGetUrl(BucketName, ObjectName, Expiration);
|
||||
|
||||
WriteVerboseMessage("Generated presigned GET URL successfully");
|
||||
WriteVerboseMessage("URL expires: {0}", urlResult.FormattedExpiration);
|
||||
|
||||
if (ShowUrl.IsPresent)
|
||||
{
|
||||
WriteInformation($"Presigned URL: {urlResult.Url}", new string[] { "PresignedUrl" });
|
||||
}
|
||||
|
||||
return urlResult;
|
||||
|
||||
}, $"Bucket: {BucketName}, Object: {ObjectName}");
|
||||
|
||||
// Always return the result object
|
||||
WriteObject(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,358 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
|
||||
namespace PSMinIO.Core.S3
|
||||
{
|
||||
/// <summary>
|
||||
/// Handles advanced object metadata with S3-compatible headers and custom metadata
|
||||
/// </summary>
|
||||
public class AdvancedMetadataHandler
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates headers from metadata configuration
|
||||
/// </summary>
|
||||
public Dictionary<string, string> CreateHeaders(ObjectMetadata metadata)
|
||||
{
|
||||
if (metadata == null) throw new ArgumentNullException(nameof(metadata));
|
||||
|
||||
var headers = new Dictionary<string, string>();
|
||||
|
||||
// Content-Type
|
||||
if (!string.IsNullOrEmpty(metadata.ContentType))
|
||||
{
|
||||
headers["Content-Type"] = metadata.ContentType;
|
||||
}
|
||||
|
||||
// Content-Encoding
|
||||
if (!string.IsNullOrEmpty(metadata.ContentEncoding))
|
||||
{
|
||||
headers["Content-Encoding"] = metadata.ContentEncoding;
|
||||
}
|
||||
|
||||
// Content-Language
|
||||
if (!string.IsNullOrEmpty(metadata.ContentLanguage))
|
||||
{
|
||||
headers["Content-Language"] = metadata.ContentLanguage;
|
||||
}
|
||||
|
||||
// Content-Disposition
|
||||
if (!string.IsNullOrEmpty(metadata.ContentDisposition))
|
||||
{
|
||||
headers["Content-Disposition"] = metadata.ContentDisposition;
|
||||
}
|
||||
|
||||
// Cache-Control
|
||||
if (!string.IsNullOrEmpty(metadata.CacheControl))
|
||||
{
|
||||
headers["Cache-Control"] = metadata.CacheControl;
|
||||
}
|
||||
|
||||
// Expires
|
||||
if (metadata.Expires.HasValue)
|
||||
{
|
||||
headers["Expires"] = metadata.Expires.Value.ToString("R", CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
// Server-Side Encryption
|
||||
if (!string.IsNullOrEmpty(metadata.ServerSideEncryption))
|
||||
{
|
||||
headers["x-amz-server-side-encryption"] = metadata.ServerSideEncryption;
|
||||
}
|
||||
|
||||
// SSE-KMS Key ID
|
||||
if (!string.IsNullOrEmpty(metadata.SSEKMSKeyId))
|
||||
{
|
||||
headers["x-amz-server-side-encryption-aws-kms-key-id"] = metadata.SSEKMSKeyId;
|
||||
}
|
||||
|
||||
// SSE-C Algorithm
|
||||
if (!string.IsNullOrEmpty(metadata.SSECustomerAlgorithm))
|
||||
{
|
||||
headers["x-amz-server-side-encryption-customer-algorithm"] = metadata.SSECustomerAlgorithm;
|
||||
}
|
||||
|
||||
// SSE-C Key
|
||||
if (!string.IsNullOrEmpty(metadata.SSECustomerKey))
|
||||
{
|
||||
headers["x-amz-server-side-encryption-customer-key"] = metadata.SSECustomerKey;
|
||||
}
|
||||
|
||||
// SSE-C Key MD5
|
||||
if (!string.IsNullOrEmpty(metadata.SSECustomerKeyMD5))
|
||||
{
|
||||
headers["x-amz-server-side-encryption-customer-key-MD5"] = metadata.SSECustomerKeyMD5;
|
||||
}
|
||||
|
||||
// Storage Class
|
||||
if (!string.IsNullOrEmpty(metadata.StorageClass))
|
||||
{
|
||||
headers["x-amz-storage-class"] = metadata.StorageClass;
|
||||
}
|
||||
|
||||
// Website Redirect Location
|
||||
if (!string.IsNullOrEmpty(metadata.WebsiteRedirectLocation))
|
||||
{
|
||||
headers["x-amz-website-redirect-location"] = metadata.WebsiteRedirectLocation;
|
||||
}
|
||||
|
||||
// Object Lock Mode
|
||||
if (!string.IsNullOrEmpty(metadata.ObjectLockMode))
|
||||
{
|
||||
headers["x-amz-object-lock-mode"] = metadata.ObjectLockMode;
|
||||
}
|
||||
|
||||
// Object Lock Retain Until Date
|
||||
if (metadata.ObjectLockRetainUntilDate.HasValue)
|
||||
{
|
||||
headers["x-amz-object-lock-retain-until-date"] =
|
||||
metadata.ObjectLockRetainUntilDate.Value.ToString("yyyy-MM-ddTHH:mm:ss.fffZ", CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
// Object Lock Legal Hold
|
||||
if (metadata.ObjectLockLegalHoldStatus.HasValue)
|
||||
{
|
||||
headers["x-amz-object-lock-legal-hold"] = metadata.ObjectLockLegalHoldStatus.Value ? "ON" : "OFF";
|
||||
}
|
||||
|
||||
// Tagging
|
||||
if (metadata.Tags != null && metadata.Tags.Count > 0)
|
||||
{
|
||||
var tagString = string.Join("&", metadata.Tags.Select(kvp => $"{Uri.EscapeDataString(kvp.Key)}={Uri.EscapeDataString(kvp.Value)}"));
|
||||
headers["x-amz-tagging"] = tagString;
|
||||
}
|
||||
|
||||
// Custom metadata (x-amz-meta-*)
|
||||
if (metadata.UserMetadata != null && metadata.UserMetadata.Count > 0)
|
||||
{
|
||||
foreach (var kvp in metadata.UserMetadata)
|
||||
{
|
||||
var key = kvp.Key.StartsWith("x-amz-meta-") ? kvp.Key : $"x-amz-meta-{kvp.Key}";
|
||||
headers[key] = kvp.Value;
|
||||
}
|
||||
}
|
||||
|
||||
// Custom headers
|
||||
if (metadata.CustomHeaders != null && metadata.CustomHeaders.Count > 0)
|
||||
{
|
||||
foreach (var kvp in metadata.CustomHeaders)
|
||||
{
|
||||
headers[kvp.Key] = kvp.Value;
|
||||
}
|
||||
}
|
||||
|
||||
return headers;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parses metadata from response headers
|
||||
/// </summary>
|
||||
public ObjectMetadata ParseMetadata(HttpResponseMessage response)
|
||||
{
|
||||
if (response == null) throw new ArgumentNullException(nameof(response));
|
||||
|
||||
var metadata = new ObjectMetadata();
|
||||
|
||||
// Process response headers
|
||||
foreach (var header in response.Headers)
|
||||
{
|
||||
var key = header.Key.ToLowerInvariant();
|
||||
var value = string.Join(", ", header.Value);
|
||||
|
||||
ProcessHeader(metadata, key, value, header.Key);
|
||||
}
|
||||
|
||||
// Process content headers
|
||||
foreach (var header in response.Content.Headers)
|
||||
{
|
||||
var key = header.Key.ToLowerInvariant();
|
||||
var value = string.Join(", ", header.Value);
|
||||
ProcessHeader(metadata, key, value, header.Key);
|
||||
}
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Processes a single header into metadata
|
||||
/// </summary>
|
||||
private void ProcessHeader(ObjectMetadata metadata, string key, string value, string originalKey)
|
||||
{
|
||||
switch (key)
|
||||
{
|
||||
case "content-type":
|
||||
metadata.ContentType = value;
|
||||
break;
|
||||
case "content-encoding":
|
||||
metadata.ContentEncoding = value;
|
||||
break;
|
||||
case "content-language":
|
||||
metadata.ContentLanguage = value;
|
||||
break;
|
||||
case "content-disposition":
|
||||
metadata.ContentDisposition = value;
|
||||
break;
|
||||
case "cache-control":
|
||||
metadata.CacheControl = value;
|
||||
break;
|
||||
case "expires":
|
||||
if (DateTime.TryParse(value, out var expires))
|
||||
metadata.Expires = expires;
|
||||
break;
|
||||
case "x-amz-server-side-encryption":
|
||||
metadata.ServerSideEncryption = value;
|
||||
break;
|
||||
case "x-amz-server-side-encryption-aws-kms-key-id":
|
||||
metadata.SSEKMSKeyId = value;
|
||||
break;
|
||||
case "x-amz-server-side-encryption-customer-algorithm":
|
||||
metadata.SSECustomerAlgorithm = value;
|
||||
break;
|
||||
case "x-amz-storage-class":
|
||||
metadata.StorageClass = value;
|
||||
break;
|
||||
case "x-amz-website-redirect-location":
|
||||
metadata.WebsiteRedirectLocation = value;
|
||||
break;
|
||||
case "x-amz-object-lock-mode":
|
||||
metadata.ObjectLockMode = value;
|
||||
break;
|
||||
case "x-amz-object-lock-retain-until-date":
|
||||
if (DateTime.TryParse(value, out var retainUntil))
|
||||
metadata.ObjectLockRetainUntilDate = retainUntil;
|
||||
break;
|
||||
case "x-amz-object-lock-legal-hold":
|
||||
metadata.ObjectLockLegalHoldStatus = value.Equals("ON", StringComparison.OrdinalIgnoreCase);
|
||||
break;
|
||||
case "x-amz-tagging":
|
||||
metadata.Tags = ParseTagString(value);
|
||||
break;
|
||||
default:
|
||||
if (key.StartsWith("x-amz-meta-"))
|
||||
{
|
||||
metadata.UserMetadata ??= new Dictionary<string, string>();
|
||||
metadata.UserMetadata[key] = value;
|
||||
}
|
||||
else
|
||||
{
|
||||
metadata.CustomHeaders ??= new Dictionary<string, string>();
|
||||
metadata.CustomHeaders[originalKey] = value;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates cache control header value
|
||||
/// </summary>
|
||||
public string CreateCacheControlHeader(CacheControlSettings settings)
|
||||
{
|
||||
if (settings == null) throw new ArgumentNullException(nameof(settings));
|
||||
|
||||
var directives = new List<string>();
|
||||
|
||||
if (settings.NoCache) directives.Add("no-cache");
|
||||
if (settings.NoStore) directives.Add("no-store");
|
||||
if (settings.MustRevalidate) directives.Add("must-revalidate");
|
||||
if (settings.Public) directives.Add("public");
|
||||
if (settings.Private) directives.Add("private");
|
||||
|
||||
if (settings.MaxAge.HasValue)
|
||||
directives.Add($"max-age={settings.MaxAge.Value.TotalSeconds:F0}");
|
||||
|
||||
if (settings.SMaxAge.HasValue)
|
||||
directives.Add($"s-maxage={settings.SMaxAge.Value.TotalSeconds:F0}");
|
||||
|
||||
return string.Join(", ", directives);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates content disposition header value
|
||||
/// </summary>
|
||||
public string CreateContentDispositionHeader(string disposition, string? filename = null)
|
||||
{
|
||||
if (string.IsNullOrEmpty(disposition)) throw new ArgumentException("Disposition cannot be null or empty", nameof(disposition));
|
||||
|
||||
var result = disposition;
|
||||
if (!string.IsNullOrEmpty(filename))
|
||||
{
|
||||
result += $"; filename=\"{filename}\"";
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parses tag string into dictionary
|
||||
/// </summary>
|
||||
private Dictionary<string, string> ParseTagString(string tagString)
|
||||
{
|
||||
var tags = new Dictionary<string, string>();
|
||||
if (string.IsNullOrEmpty(tagString)) return tags;
|
||||
|
||||
var pairs = tagString.Split(new char[] { '&' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
foreach (var pair in pairs)
|
||||
{
|
||||
var parts = pair.Split(new char[] { '=' }, 2);
|
||||
if (parts.Length == 2)
|
||||
{
|
||||
var key = Uri.UnescapeDataString(parts[0]);
|
||||
var value = Uri.UnescapeDataString(parts[1]);
|
||||
tags[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
return tags;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Comprehensive object metadata configuration
|
||||
/// </summary>
|
||||
public class ObjectMetadata
|
||||
{
|
||||
// Standard HTTP headers
|
||||
public string? ContentType { get; set; }
|
||||
public string? ContentEncoding { get; set; }
|
||||
public string? ContentLanguage { get; set; }
|
||||
public string? ContentDisposition { get; set; }
|
||||
public string? CacheControl { get; set; }
|
||||
public DateTime? Expires { get; set; }
|
||||
|
||||
// S3-specific headers
|
||||
public string? ServerSideEncryption { get; set; }
|
||||
public string? SSEKMSKeyId { get; set; }
|
||||
public string? SSECustomerAlgorithm { get; set; }
|
||||
public string? SSECustomerKey { get; set; }
|
||||
public string? SSECustomerKeyMD5 { get; set; }
|
||||
public string? StorageClass { get; set; }
|
||||
public string? WebsiteRedirectLocation { get; set; }
|
||||
|
||||
// Object Lock
|
||||
public string? ObjectLockMode { get; set; }
|
||||
public DateTime? ObjectLockRetainUntilDate { get; set; }
|
||||
public bool? ObjectLockLegalHoldStatus { get; set; }
|
||||
|
||||
// Tags and custom metadata
|
||||
public Dictionary<string, string>? Tags { get; set; }
|
||||
public Dictionary<string, string>? UserMetadata { get; set; }
|
||||
public Dictionary<string, string>? CustomHeaders { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cache control settings
|
||||
/// </summary>
|
||||
public class CacheControlSettings
|
||||
{
|
||||
public bool NoCache { get; set; }
|
||||
public bool NoStore { get; set; }
|
||||
public bool MustRevalidate { get; set; }
|
||||
public bool Public { get; set; }
|
||||
public bool Private { get; set; }
|
||||
public TimeSpan? MaxAge { get; set; }
|
||||
public TimeSpan? SMaxAge { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using PSMinIO.Core.Http;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Core.S3
|
||||
{
|
||||
/// <summary>
|
||||
/// Manages bucket policies with comprehensive validation and formatting
|
||||
/// </summary>
|
||||
public class BucketPolicyManager
|
||||
{
|
||||
private readonly MinIOHttpClient _httpClient;
|
||||
private readonly ThreadSafeProgressCollector _progressCollector;
|
||||
|
||||
public BucketPolicyManager(MinIOHttpClient httpClient, ThreadSafeProgressCollector progressCollector)
|
||||
{
|
||||
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
|
||||
_progressCollector = progressCollector ?? throw new ArgumentNullException(nameof(progressCollector));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the bucket policy
|
||||
/// </summary>
|
||||
public BucketPolicyResult GetBucketPolicy(string bucketName)
|
||||
{
|
||||
if (string.IsNullOrEmpty(bucketName)) throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
_progressCollector.QueueVerboseMessage("Getting bucket policy for: {0}", bucketName);
|
||||
|
||||
try
|
||||
{
|
||||
var queryParams = new Dictionary<string, string> { { "policy", "" } };
|
||||
var response = _httpClient.ExecuteRequestForString(HttpMethod.Get, $"/{bucketName}", queryParams);
|
||||
|
||||
var policy = JsonSerializer.Deserialize<BucketPolicy>(response, new JsonSerializerOptions
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
PropertyNameCaseInsensitive = true
|
||||
});
|
||||
|
||||
_progressCollector.QueueVerboseMessage("Successfully retrieved bucket policy");
|
||||
|
||||
return new BucketPolicyResult
|
||||
{
|
||||
BucketName = bucketName,
|
||||
Policy = policy,
|
||||
PolicyJson = response,
|
||||
IsValid = ValidatePolicy(policy!),
|
||||
HasPolicy = true
|
||||
};
|
||||
}
|
||||
catch (HttpRequestException ex) when (ex.Message.Contains("404") || ex.Message.Contains("NoSuchBucketPolicy"))
|
||||
{
|
||||
_progressCollector.QueueVerboseMessage("No policy found for bucket: {0}", bucketName);
|
||||
return new BucketPolicyResult
|
||||
{
|
||||
BucketName = bucketName,
|
||||
HasPolicy = false,
|
||||
IsValid = true
|
||||
};
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_progressCollector.QueueVerboseMessage("Failed to get bucket policy: {0}", ex.Message);
|
||||
return new BucketPolicyResult
|
||||
{
|
||||
BucketName = bucketName,
|
||||
HasPolicy = false,
|
||||
IsValid = false,
|
||||
Error = ex.Message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets the bucket policy
|
||||
/// </summary>
|
||||
public BucketPolicyResult SetBucketPolicy(string bucketName, BucketPolicy policy)
|
||||
{
|
||||
if (string.IsNullOrEmpty(bucketName)) throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
if (policy == null) throw new ArgumentNullException(nameof(policy));
|
||||
|
||||
_progressCollector.QueueVerboseMessage("Setting bucket policy for: {0}", bucketName);
|
||||
|
||||
// Validate policy before setting
|
||||
if (!ValidatePolicy(policy))
|
||||
{
|
||||
return new BucketPolicyResult
|
||||
{
|
||||
BucketName = bucketName,
|
||||
Policy = policy,
|
||||
IsValid = false,
|
||||
Error = "Policy validation failed"
|
||||
};
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var policyJson = JsonSerializer.Serialize(policy, new JsonSerializerOptions
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = true
|
||||
});
|
||||
|
||||
var queryParams = new Dictionary<string, string> { { "policy", "" } };
|
||||
using var content = new StringContent(policyJson, Encoding.UTF8, "application/json");
|
||||
|
||||
_httpClient.ExecuteRequest(HttpMethod.Put, $"/{bucketName}", queryParams, content: content);
|
||||
|
||||
_progressCollector.QueueVerboseMessage("Successfully set bucket policy");
|
||||
|
||||
return new BucketPolicyResult
|
||||
{
|
||||
BucketName = bucketName,
|
||||
Policy = policy,
|
||||
PolicyJson = policyJson,
|
||||
IsValid = true,
|
||||
HasPolicy = true
|
||||
};
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_progressCollector.QueueVerboseMessage("Failed to set bucket policy: {0}", ex.Message);
|
||||
return new BucketPolicyResult
|
||||
{
|
||||
BucketName = bucketName,
|
||||
Policy = policy,
|
||||
IsValid = false,
|
||||
Error = ex.Message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets bucket policy from JSON string
|
||||
/// </summary>
|
||||
public BucketPolicyResult SetBucketPolicyFromJson(string bucketName, string policyJson)
|
||||
{
|
||||
if (string.IsNullOrEmpty(bucketName)) throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
if (string.IsNullOrEmpty(policyJson)) throw new ArgumentException("Policy JSON cannot be null or empty", nameof(policyJson));
|
||||
|
||||
try
|
||||
{
|
||||
var policy = JsonSerializer.Deserialize<BucketPolicy>(policyJson, new JsonSerializerOptions
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
PropertyNameCaseInsensitive = true
|
||||
});
|
||||
|
||||
return SetBucketPolicy(bucketName, policy!);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
return new BucketPolicyResult
|
||||
{
|
||||
BucketName = bucketName,
|
||||
PolicyJson = policyJson,
|
||||
IsValid = false,
|
||||
Error = $"Invalid JSON: {ex.Message}"
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes the bucket policy
|
||||
/// </summary>
|
||||
public BucketPolicyResult DeleteBucketPolicy(string bucketName)
|
||||
{
|
||||
if (string.IsNullOrEmpty(bucketName)) throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
|
||||
_progressCollector.QueueVerboseMessage("Deleting bucket policy for: {0}", bucketName);
|
||||
|
||||
try
|
||||
{
|
||||
var queryParams = new Dictionary<string, string> { { "policy", "" } };
|
||||
_httpClient.ExecuteRequest(HttpMethod.Delete, $"/{bucketName}", queryParams);
|
||||
|
||||
_progressCollector.QueueVerboseMessage("Successfully deleted bucket policy");
|
||||
|
||||
return new BucketPolicyResult
|
||||
{
|
||||
BucketName = bucketName,
|
||||
HasPolicy = false,
|
||||
IsValid = true
|
||||
};
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_progressCollector.QueueVerboseMessage("Failed to delete bucket policy: {0}", ex.Message);
|
||||
return new BucketPolicyResult
|
||||
{
|
||||
BucketName = bucketName,
|
||||
IsValid = false,
|
||||
Error = ex.Message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a simple read-only policy for public access
|
||||
/// </summary>
|
||||
public BucketPolicy CreateReadOnlyPolicy(string bucketName, string? objectPrefix = null)
|
||||
{
|
||||
var resource = string.IsNullOrEmpty(objectPrefix)
|
||||
? $"arn:aws:s3:::{bucketName}/*"
|
||||
: $"arn:aws:s3:::{bucketName}/{objectPrefix}*";
|
||||
|
||||
return new BucketPolicy
|
||||
{
|
||||
Version = "2012-10-17",
|
||||
Statement = new List<PolicyStatement>
|
||||
{
|
||||
new PolicyStatement
|
||||
{
|
||||
Effect = "Allow",
|
||||
Principal = new { AWS = "*" },
|
||||
Action = new List<string> { "s3:GetObject" },
|
||||
Resource = new List<string> { resource }
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a read-write policy for specific principals
|
||||
/// </summary>
|
||||
public BucketPolicy CreateReadWritePolicy(string bucketName, List<string> principals, string? objectPrefix = null)
|
||||
{
|
||||
var resource = string.IsNullOrEmpty(objectPrefix)
|
||||
? $"arn:aws:s3:::{bucketName}/*"
|
||||
: $"arn:aws:s3:::{bucketName}/{objectPrefix}*";
|
||||
|
||||
return new BucketPolicy
|
||||
{
|
||||
Version = "2012-10-17",
|
||||
Statement = new List<PolicyStatement>
|
||||
{
|
||||
new PolicyStatement
|
||||
{
|
||||
Effect = "Allow",
|
||||
Principal = new { AWS = principals },
|
||||
Action = new List<string> { "s3:GetObject", "s3:PutObject", "s3:DeleteObject" },
|
||||
Resource = new List<string> { resource }
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates a bucket policy
|
||||
/// </summary>
|
||||
private bool ValidatePolicy(BucketPolicy policy)
|
||||
{
|
||||
if (policy == null) return false;
|
||||
if (string.IsNullOrEmpty(policy.Version)) return false;
|
||||
if (policy.Statement == null || policy.Statement.Count == 0) return false;
|
||||
|
||||
foreach (var statement in policy.Statement)
|
||||
{
|
||||
if (string.IsNullOrEmpty(statement.Effect)) return false;
|
||||
if (statement.Effect != "Allow" && statement.Effect != "Deny") return false;
|
||||
if (statement.Action == null || statement.Action.Count == 0) return false;
|
||||
if (statement.Resource == null || statement.Resource.Count == 0) return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Bucket policy structure
|
||||
/// </summary>
|
||||
public class BucketPolicy
|
||||
{
|
||||
public string Version { get; set; } = "2012-10-17";
|
||||
public List<PolicyStatement> Statement { get; set; } = new List<PolicyStatement>();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Policy statement structure
|
||||
/// </summary>
|
||||
public class PolicyStatement
|
||||
{
|
||||
public string Effect { get; set; } = string.Empty;
|
||||
public object? Principal { get; set; }
|
||||
public List<string> Action { get; set; } = new List<string>();
|
||||
public List<string> Resource { get; set; } = new List<string>();
|
||||
public Dictionary<string, object>? Condition { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of bucket policy operations
|
||||
/// </summary>
|
||||
public class BucketPolicyResult
|
||||
{
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
public BucketPolicy? Policy { get; set; }
|
||||
public string? PolicyJson { get; set; }
|
||||
public bool HasPolicy { get; set; }
|
||||
public bool IsValid { get; set; }
|
||||
public string? Error { get; set; }
|
||||
public DateTime OperationTime { get; set; } = DateTime.UtcNow;
|
||||
}
|
||||
}
|
||||
@@ -201,7 +201,7 @@ namespace PSMinIO.Core.S3
|
||||
};
|
||||
|
||||
if (!string.IsNullOrEmpty(prefix))
|
||||
queryParams["prefix"] = prefix;
|
||||
queryParams["prefix"] = prefix!;
|
||||
|
||||
if (!recursive)
|
||||
queryParams["delimiter"] = "/";
|
||||
|
||||
@@ -0,0 +1,284 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using PSMinIO.Core.Http;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Core.S3
|
||||
{
|
||||
/// <summary>
|
||||
/// Manages multipart downloads with parallel processing and resume capability
|
||||
/// </summary>
|
||||
public class MultipartDownloadManager
|
||||
{
|
||||
private readonly MinIOHttpClient _httpClient;
|
||||
private readonly ThreadSafeProgressCollector _progressCollector;
|
||||
private readonly int _maxParallelDownloads;
|
||||
private readonly long _defaultChunkSize;
|
||||
|
||||
public MultipartDownloadManager(MinIOHttpClient httpClient, ThreadSafeProgressCollector progressCollector,
|
||||
int maxParallelDownloads = 4, long defaultChunkSize = 32 * 1024 * 1024) // 32MB default
|
||||
{
|
||||
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
|
||||
_progressCollector = progressCollector ?? throw new ArgumentNullException(nameof(progressCollector));
|
||||
_maxParallelDownloads = Math.Max(1, Math.Min(maxParallelDownloads, 8)); // Limit to 1-8
|
||||
_defaultChunkSize = Math.Max(1024 * 1024, defaultChunkSize); // Minimum 1MB
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Downloads a file using multipart download with parallel processing
|
||||
/// </summary>
|
||||
public MultipartDownloadResult DownloadFile(string bucketName, string objectName, FileInfo destinationFile,
|
||||
long? chunkSize = null, bool resumeIfExists = true)
|
||||
{
|
||||
if (string.IsNullOrEmpty(bucketName)) throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
if (string.IsNullOrEmpty(objectName)) throw new ArgumentException("Object name cannot be null or empty", nameof(objectName));
|
||||
if (destinationFile == null) throw new ArgumentNullException(nameof(destinationFile));
|
||||
|
||||
var effectiveChunkSize = chunkSize ?? _defaultChunkSize;
|
||||
var startTime = DateTime.UtcNow;
|
||||
|
||||
_progressCollector.QueueVerboseMessage("Starting multipart download: {0}/{1} -> {2}",
|
||||
bucketName, objectName, destinationFile.FullName);
|
||||
|
||||
try
|
||||
{
|
||||
// Get object metadata to determine size
|
||||
var objectInfo = GetObjectInfo(bucketName, objectName);
|
||||
var totalSize = objectInfo.Size;
|
||||
var totalChunks = (int)Math.Ceiling((double)totalSize / effectiveChunkSize);
|
||||
|
||||
_progressCollector.QueueVerboseMessage("Object size: {0}, Chunk size: {1}, Total chunks: {2}",
|
||||
SizeFormatter.FormatBytes(totalSize), SizeFormatter.FormatBytes(effectiveChunkSize), totalChunks);
|
||||
|
||||
// Check for resume capability
|
||||
var resumeOffset = 0L;
|
||||
if (resumeIfExists && destinationFile.Exists)
|
||||
{
|
||||
resumeOffset = destinationFile.Length;
|
||||
if (resumeOffset >= totalSize)
|
||||
{
|
||||
_progressCollector.QueueVerboseMessage("File already completely downloaded");
|
||||
return new MultipartDownloadResult
|
||||
{
|
||||
BucketName = bucketName,
|
||||
ObjectName = objectName,
|
||||
DestinationPath = destinationFile.FullName,
|
||||
TotalSize = totalSize,
|
||||
DownloadedSize = totalSize,
|
||||
ChunkSize = effectiveChunkSize,
|
||||
TotalChunks = totalChunks,
|
||||
Duration = TimeSpan.Zero,
|
||||
IsCompleted = true,
|
||||
WasResumed = true
|
||||
};
|
||||
}
|
||||
_progressCollector.QueueVerboseMessage("Resuming download from offset: {0}",
|
||||
SizeFormatter.FormatBytes(resumeOffset));
|
||||
}
|
||||
|
||||
// Ensure destination directory exists
|
||||
if (destinationFile.Directory != null && !destinationFile.Directory.Exists)
|
||||
{
|
||||
destinationFile.Directory.Create();
|
||||
}
|
||||
|
||||
// Create or open file for writing
|
||||
using var fileStream = resumeIfExists && destinationFile.Exists
|
||||
? new FileStream(destinationFile.FullName, FileMode.Append, FileAccess.Write)
|
||||
: new FileStream(destinationFile.FullName, FileMode.Create, FileAccess.Write);
|
||||
|
||||
// Calculate chunks to download
|
||||
var remainingSize = totalSize - resumeOffset;
|
||||
var startChunk = (int)(resumeOffset / effectiveChunkSize);
|
||||
var chunksToDownload = (int)Math.Ceiling((double)remainingSize / effectiveChunkSize);
|
||||
|
||||
var downloadedBytes = resumeOffset;
|
||||
var chunks = new ConcurrentDictionary<int, DownloadPartInfo>();
|
||||
var downloadTasks = new List<Task>();
|
||||
var semaphore = new SemaphoreSlim(_maxParallelDownloads, _maxParallelDownloads);
|
||||
|
||||
// Download chunks in parallel
|
||||
for (int chunkIndex = startChunk; chunkIndex < startChunk + chunksToDownload; chunkIndex++)
|
||||
{
|
||||
var chunkNum = chunkIndex;
|
||||
var chunkOffset = chunkNum * effectiveChunkSize;
|
||||
var chunkEnd = Math.Min(chunkOffset + effectiveChunkSize - 1, totalSize - 1);
|
||||
var chunkActualSize = chunkEnd - chunkOffset + 1;
|
||||
|
||||
var downloadTask = Task.Run(() =>
|
||||
{
|
||||
semaphore.Wait();
|
||||
try
|
||||
{
|
||||
var chunkData = DownloadChunk(bucketName, objectName, chunkOffset, chunkEnd);
|
||||
|
||||
chunks.TryAdd(chunkNum, new DownloadPartInfo
|
||||
{
|
||||
PartNumber = chunkNum,
|
||||
Offset = chunkOffset,
|
||||
Size = chunkData.Length,
|
||||
Data = chunkData
|
||||
});
|
||||
|
||||
// Update progress
|
||||
var currentDownloaded = Interlocked.Add(ref downloadedBytes, chunkData.Length);
|
||||
var progress = (double)currentDownloaded / totalSize * 100;
|
||||
var elapsed = DateTime.UtcNow - startTime;
|
||||
var speed = elapsed.TotalSeconds > 0 ? (currentDownloaded - resumeOffset) / elapsed.TotalSeconds : 0;
|
||||
|
||||
_progressCollector.QueueProgressUpdate(2, "Multipart Download",
|
||||
$"Part {chunkNum - startChunk + 1}/{chunksToDownload} - {SizeFormatter.FormatBytes(currentDownloaded)}/{SizeFormatter.FormatBytes(totalSize)} at {SizeFormatter.FormatSpeed(speed)}",
|
||||
(int)progress);
|
||||
|
||||
_progressCollector.QueueVerboseMessage("Downloaded chunk {0}/{1} ({2})",
|
||||
chunkNum - startChunk + 1, chunksToDownload, SizeFormatter.FormatBytes(chunkData.Length));
|
||||
}
|
||||
finally
|
||||
{
|
||||
semaphore.Release();
|
||||
}
|
||||
});
|
||||
|
||||
downloadTasks.Add(downloadTask);
|
||||
}
|
||||
|
||||
// Wait for all downloads to complete
|
||||
Task.WaitAll(downloadTasks.ToArray());
|
||||
|
||||
// Write parts to file in order
|
||||
var sortedParts = chunks.Values.OrderBy(c => c.PartNumber).ToList();
|
||||
foreach (var part in sortedParts)
|
||||
{
|
||||
fileStream.Write(part.Data, 0, part.Data.Length);
|
||||
}
|
||||
|
||||
var totalDuration = DateTime.UtcNow - startTime;
|
||||
var averageSpeed = totalDuration.TotalSeconds > 0 ? remainingSize / totalDuration.TotalSeconds : 0;
|
||||
|
||||
_progressCollector.QueueVerboseMessage("Multipart download completed successfully");
|
||||
_progressCollector.QueueVerboseMessage("Total time: {0}, Average speed: {1}",
|
||||
SizeFormatter.FormatDuration(totalDuration), SizeFormatter.FormatSpeed(averageSpeed));
|
||||
|
||||
_progressCollector.QueueProgressCompletion(2, "Multipart Download");
|
||||
|
||||
return new MultipartDownloadResult
|
||||
{
|
||||
BucketName = bucketName,
|
||||
ObjectName = objectName,
|
||||
DestinationPath = destinationFile.FullName,
|
||||
TotalSize = totalSize,
|
||||
DownloadedSize = totalSize,
|
||||
ChunkSize = effectiveChunkSize,
|
||||
TotalChunks = totalChunks,
|
||||
CompletedParts = sortedParts,
|
||||
Duration = totalDuration,
|
||||
AverageSpeed = averageSpeed,
|
||||
IsCompleted = true,
|
||||
WasResumed = resumeOffset > 0
|
||||
};
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_progressCollector.QueueVerboseMessage("Multipart download failed: {0}", ex.Message);
|
||||
|
||||
var partialDuration = DateTime.UtcNow - startTime;
|
||||
|
||||
return new MultipartDownloadResult
|
||||
{
|
||||
BucketName = bucketName,
|
||||
ObjectName = objectName,
|
||||
DestinationPath = destinationFile.FullName,
|
||||
ChunkSize = effectiveChunkSize,
|
||||
Duration = partialDuration,
|
||||
IsCompleted = false,
|
||||
Error = ex.Message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets object information including size
|
||||
/// </summary>
|
||||
private ObjectInfo GetObjectInfo(string bucketName, string objectName)
|
||||
{
|
||||
var response = _httpClient.ExecuteRequest(HttpMethod.Head, $"/{bucketName}/{objectName}");
|
||||
|
||||
var contentLength = response.Content.Headers.ContentLength ?? 0;
|
||||
var lastModified = response.Content.Headers.LastModified?.DateTime ?? DateTime.MinValue;
|
||||
var etag = response.Headers.ETag?.Tag?.Trim('"') ?? "";
|
||||
|
||||
return new ObjectInfo
|
||||
{
|
||||
Name = objectName,
|
||||
Size = contentLength,
|
||||
LastModified = lastModified,
|
||||
ETag = etag
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Downloads a specific part using range request
|
||||
/// </summary>
|
||||
private byte[] DownloadChunk(string bucketName, string objectName, long startByte, long endByte)
|
||||
{
|
||||
var headers = new Dictionary<string, string>
|
||||
{
|
||||
{ "Range", $"bytes={startByte}-{endByte}" }
|
||||
};
|
||||
|
||||
var response = _httpClient.ExecuteRequest(HttpMethod.Get, $"/{bucketName}/{objectName}",
|
||||
headers: headers);
|
||||
|
||||
return response.Content.ReadAsByteArrayAsync().GetAwaiter().GetResult();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Information about a downloaded part
|
||||
/// </summary>
|
||||
public class DownloadPartInfo
|
||||
{
|
||||
public int PartNumber { get; set; }
|
||||
public long Offset { get; set; }
|
||||
public long Size { get; set; }
|
||||
public byte[] Data { get; set; } = Array.Empty<byte>();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of multipart download operation
|
||||
/// </summary>
|
||||
public class MultipartDownloadResult
|
||||
{
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
public string ObjectName { get; set; } = string.Empty;
|
||||
public string DestinationPath { get; set; } = string.Empty;
|
||||
public long TotalSize { get; set; }
|
||||
public long DownloadedSize { get; set; }
|
||||
public long ChunkSize { get; set; }
|
||||
public int TotalChunks { get; set; }
|
||||
public List<DownloadPartInfo> CompletedParts { get; set; } = new List<DownloadPartInfo>();
|
||||
public TimeSpan Duration { get; set; }
|
||||
public double AverageSpeed { get; set; }
|
||||
public bool IsCompleted { get; set; }
|
||||
public bool WasResumed { get; set; }
|
||||
public string? Error { get; set; }
|
||||
public double CompletionPercentage => TotalSize > 0 ? (double)DownloadedSize / TotalSize * 100 : 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Basic object information
|
||||
/// </summary>
|
||||
public class ObjectInfo
|
||||
{
|
||||
public string Name { get; set; } = string.Empty;
|
||||
public long Size { get; set; }
|
||||
public DateTime LastModified { get; set; }
|
||||
public string ETag { get; set; } = string.Empty;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Xml.Linq;
|
||||
using PSMinIO.Core.Http;
|
||||
using PSMinIO.Utils;
|
||||
|
||||
namespace PSMinIO.Core.S3
|
||||
{
|
||||
/// <summary>
|
||||
/// Manages multipart uploads with parallel processing and resume capability
|
||||
/// </summary>
|
||||
public class MultipartUploadManager
|
||||
{
|
||||
private readonly MinIOHttpClient _httpClient;
|
||||
private readonly ThreadSafeProgressCollector _progressCollector;
|
||||
private readonly int _maxParallelUploads;
|
||||
private readonly long _defaultChunkSize;
|
||||
|
||||
public MultipartUploadManager(MinIOHttpClient httpClient, ThreadSafeProgressCollector progressCollector,
|
||||
int maxParallelUploads = 4, long defaultChunkSize = 64 * 1024 * 1024) // 64MB default
|
||||
{
|
||||
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
|
||||
_progressCollector = progressCollector ?? throw new ArgumentNullException(nameof(progressCollector));
|
||||
_maxParallelUploads = Math.Max(1, Math.Min(maxParallelUploads, 10)); // Limit to 1-10
|
||||
_defaultChunkSize = Math.Max(5 * 1024 * 1024, defaultChunkSize); // Minimum 5MB for S3 compatibility
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Uploads a file using multipart upload with parallel processing
|
||||
/// </summary>
|
||||
public MultipartUploadResult UploadFile(string bucketName, string objectName, FileInfo fileInfo,
|
||||
Dictionary<string, string>? metadata = null, long? chunkSize = null,
|
||||
string? resumeUploadId = null, List<PartInfo>? completedParts = null)
|
||||
{
|
||||
if (string.IsNullOrEmpty(bucketName)) throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
if (string.IsNullOrEmpty(objectName)) throw new ArgumentException("Object name cannot be null or empty", nameof(objectName));
|
||||
if (fileInfo == null || !fileInfo.Exists) throw new ArgumentException("File must exist", nameof(fileInfo));
|
||||
|
||||
var effectiveChunkSize = chunkSize ?? _defaultChunkSize;
|
||||
var totalSize = fileInfo.Length;
|
||||
var totalParts = (int)Math.Ceiling((double)totalSize / effectiveChunkSize);
|
||||
|
||||
_progressCollector.QueueVerboseMessage("Starting multipart upload: {0} -> {1}/{2}",
|
||||
fileInfo.Name, bucketName, objectName);
|
||||
_progressCollector.QueueVerboseMessage("File size: {0}, Chunk size: {1}, Total parts: {2}",
|
||||
SizeFormatter.FormatBytes(totalSize), SizeFormatter.FormatBytes(effectiveChunkSize), totalParts);
|
||||
|
||||
var uploadId = resumeUploadId;
|
||||
var parts = new ConcurrentDictionary<int, PartInfo>();
|
||||
var operationStartTime = DateTime.UtcNow;
|
||||
|
||||
// Add completed parts if resuming
|
||||
if (completedParts != null)
|
||||
{
|
||||
foreach (var part in completedParts)
|
||||
{
|
||||
parts.TryAdd(part.PartNumber, part);
|
||||
}
|
||||
_progressCollector.QueueVerboseMessage("Resuming upload with {0} completed parts", completedParts.Count);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// Initialize multipart upload if not resuming
|
||||
if (string.IsNullOrEmpty(uploadId))
|
||||
{
|
||||
uploadId = InitiateMultipartUpload(bucketName, objectName, metadata);
|
||||
_progressCollector.QueueVerboseMessage("Initiated multipart upload with ID: {0}", uploadId);
|
||||
}
|
||||
|
||||
// Upload parts in parallel
|
||||
var uploadTasks = new List<Task>();
|
||||
var semaphore = new SemaphoreSlim(_maxParallelUploads, _maxParallelUploads);
|
||||
var uploadedBytes = completedParts?.Sum(p => p.Size) ?? 0;
|
||||
|
||||
for (int partNumber = 1; partNumber <= totalParts; partNumber++)
|
||||
{
|
||||
// Skip already completed parts
|
||||
if (parts.ContainsKey(partNumber))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var partNum = partNumber;
|
||||
var partOffset = (partNum - 1) * effectiveChunkSize;
|
||||
var partSize = Math.Min(effectiveChunkSize, totalSize - partOffset);
|
||||
|
||||
var uploadTask = Task.Run(() =>
|
||||
{
|
||||
semaphore.Wait();
|
||||
try
|
||||
{
|
||||
var partInfo = UploadPart(bucketName, objectName, uploadId!, fileInfo,
|
||||
partNum, partOffset, partSize);
|
||||
|
||||
parts.TryAdd(partNum, partInfo);
|
||||
|
||||
// Update progress
|
||||
var currentUploaded = Interlocked.Add(ref uploadedBytes, partSize);
|
||||
var progress = (double)currentUploaded / totalSize * 100;
|
||||
var elapsed = DateTime.UtcNow - operationStartTime;
|
||||
var speed = elapsed.TotalSeconds > 0 ? currentUploaded / elapsed.TotalSeconds : 0;
|
||||
|
||||
_progressCollector.QueueProgressUpdate(1, "Multipart Upload",
|
||||
$"Part {partNum}/{totalParts} - {SizeFormatter.FormatBytes(currentUploaded)}/{SizeFormatter.FormatBytes(totalSize)} at {SizeFormatter.FormatSpeed(speed)}",
|
||||
(int)progress);
|
||||
|
||||
_progressCollector.QueueVerboseMessage("Completed part {0}/{1} ({2})",
|
||||
partNum, totalParts, SizeFormatter.FormatBytes(partSize));
|
||||
}
|
||||
finally
|
||||
{
|
||||
semaphore.Release();
|
||||
}
|
||||
});
|
||||
|
||||
uploadTasks.Add(uploadTask);
|
||||
}
|
||||
|
||||
// Wait for all uploads to complete
|
||||
Task.WaitAll(uploadTasks.ToArray());
|
||||
|
||||
// Complete multipart upload
|
||||
var sortedParts = parts.Values.OrderBy(p => p.PartNumber).ToList();
|
||||
var result = CompleteMultipartUpload(bucketName, objectName, uploadId!, sortedParts);
|
||||
|
||||
var totalDuration = DateTime.UtcNow - operationStartTime;
|
||||
var averageSpeed = totalDuration.TotalSeconds > 0 ? totalSize / totalDuration.TotalSeconds : 0;
|
||||
|
||||
_progressCollector.QueueVerboseMessage("Multipart upload completed successfully");
|
||||
_progressCollector.QueueVerboseMessage("Total time: {0}, Average speed: {1}",
|
||||
SizeFormatter.FormatDuration(totalDuration), SizeFormatter.FormatSpeed(averageSpeed));
|
||||
|
||||
_progressCollector.QueueProgressCompletion(1, "Multipart Upload");
|
||||
|
||||
return new MultipartUploadResult
|
||||
{
|
||||
BucketName = bucketName,
|
||||
ObjectName = objectName,
|
||||
UploadId = uploadId!,
|
||||
ETag = result.ETag,
|
||||
TotalSize = totalSize,
|
||||
ChunkSize = effectiveChunkSize,
|
||||
TotalParts = totalParts,
|
||||
CompletedParts = sortedParts,
|
||||
Duration = totalDuration,
|
||||
AverageSpeed = averageSpeed,
|
||||
IsCompleted = true
|
||||
};
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_progressCollector.QueueVerboseMessage("Multipart upload failed: {0}", ex.Message);
|
||||
|
||||
// Return partial result for resume capability
|
||||
var sortedParts = parts.Values.OrderBy(p => p.PartNumber).ToList();
|
||||
var partialDuration = DateTime.UtcNow - operationStartTime;
|
||||
|
||||
return new MultipartUploadResult
|
||||
{
|
||||
BucketName = bucketName,
|
||||
ObjectName = objectName,
|
||||
UploadId = uploadId!,
|
||||
TotalSize = totalSize,
|
||||
ChunkSize = effectiveChunkSize,
|
||||
TotalParts = totalParts,
|
||||
CompletedParts = sortedParts,
|
||||
Duration = partialDuration,
|
||||
IsCompleted = false,
|
||||
Error = ex.Message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initiates a multipart upload
|
||||
/// </summary>
|
||||
private string InitiateMultipartUpload(string bucketName, string objectName, Dictionary<string, string>? metadata)
|
||||
{
|
||||
var queryParams = new Dictionary<string, string> { { "uploads", "" } };
|
||||
var headers = new Dictionary<string, string>();
|
||||
|
||||
// Add metadata as headers
|
||||
if (metadata != null)
|
||||
{
|
||||
foreach (var kvp in metadata)
|
||||
{
|
||||
headers[$"x-amz-meta-{kvp.Key}"] = kvp.Value;
|
||||
}
|
||||
}
|
||||
|
||||
var response = _httpClient.ExecuteRequestForString(HttpMethod.Post, $"/{bucketName}/{objectName}",
|
||||
queryParams, headers);
|
||||
|
||||
var doc = XDocument.Parse(response);
|
||||
var uploadId = doc.Descendants("UploadId").FirstOrDefault()?.Value;
|
||||
|
||||
if (string.IsNullOrEmpty(uploadId))
|
||||
{
|
||||
throw new InvalidOperationException("Failed to initiate multipart upload - no upload ID returned");
|
||||
}
|
||||
|
||||
return uploadId;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Uploads a single part
|
||||
/// </summary>
|
||||
private PartInfo UploadPart(string bucketName, string objectName, string uploadId,
|
||||
FileInfo fileInfo, int partNumber, long offset, long size)
|
||||
{
|
||||
var queryParams = new Dictionary<string, string>
|
||||
{
|
||||
{ "partNumber", partNumber.ToString() },
|
||||
{ "uploadId", uploadId }
|
||||
};
|
||||
|
||||
using var fileStream = fileInfo.OpenRead();
|
||||
fileStream.Seek(offset, SeekOrigin.Begin);
|
||||
|
||||
var buffer = new byte[size];
|
||||
var totalRead = 0;
|
||||
while (totalRead < size)
|
||||
{
|
||||
var bytesRead = fileStream.Read(buffer, totalRead, (int)(size - totalRead));
|
||||
if (bytesRead == 0) break;
|
||||
totalRead += bytesRead;
|
||||
}
|
||||
|
||||
using var content = new ByteArrayContent(buffer, 0, totalRead);
|
||||
content.Headers.ContentLength = totalRead;
|
||||
|
||||
// Calculate MD5 for integrity
|
||||
using var md5 = MD5.Create();
|
||||
var md5Hash = md5.ComputeHash(buffer, 0, totalRead);
|
||||
var md5String = Convert.ToBase64String(md5Hash);
|
||||
content.Headers.Add("Content-MD5", md5String);
|
||||
|
||||
var response = _httpClient.ExecuteRequest(HttpMethod.Put, $"/{bucketName}/{objectName}",
|
||||
queryParams, content: content);
|
||||
|
||||
var etag = response.Headers.ETag?.Tag?.Trim('"') ?? "";
|
||||
|
||||
return new PartInfo
|
||||
{
|
||||
PartNumber = partNumber,
|
||||
ETag = etag,
|
||||
Size = totalRead,
|
||||
MD5Hash = md5String
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Completes the multipart upload
|
||||
/// </summary>
|
||||
private CompleteMultipartUploadResult CompleteMultipartUpload(string bucketName, string objectName,
|
||||
string uploadId, List<PartInfo> parts)
|
||||
{
|
||||
var queryParams = new Dictionary<string, string> { { "uploadId", uploadId } };
|
||||
|
||||
var xml = new XElement("CompleteMultipartUpload",
|
||||
parts.Select(p => new XElement("Part",
|
||||
new XElement("PartNumber", p.PartNumber),
|
||||
new XElement("ETag", p.ETag)
|
||||
))
|
||||
);
|
||||
|
||||
var xmlContent = xml.ToString();
|
||||
using var content = new StringContent(xmlContent, Encoding.UTF8, "application/xml");
|
||||
|
||||
var response = _httpClient.ExecuteRequestForString(HttpMethod.Post, $"/{bucketName}/{objectName}",
|
||||
queryParams, content: content);
|
||||
|
||||
var doc = XDocument.Parse(response);
|
||||
var etag = doc.Descendants("ETag").FirstOrDefault()?.Value?.Trim('"') ?? "";
|
||||
|
||||
return new CompleteMultipartUploadResult
|
||||
{
|
||||
ETag = etag,
|
||||
Location = doc.Descendants("Location").FirstOrDefault()?.Value ?? ""
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Information about an uploaded part
|
||||
/// </summary>
|
||||
public class PartInfo
|
||||
{
|
||||
public int PartNumber { get; set; }
|
||||
public string ETag { get; set; } = string.Empty;
|
||||
public long Size { get; set; }
|
||||
public string MD5Hash { get; set; } = string.Empty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of multipart upload operation
|
||||
/// </summary>
|
||||
public class MultipartUploadResult
|
||||
{
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
public string ObjectName { get; set; } = string.Empty;
|
||||
public string UploadId { get; set; } = string.Empty;
|
||||
public string ETag { get; set; } = string.Empty;
|
||||
public long TotalSize { get; set; }
|
||||
public long ChunkSize { get; set; }
|
||||
public int TotalParts { get; set; }
|
||||
public List<PartInfo> CompletedParts { get; set; } = new List<PartInfo>();
|
||||
public TimeSpan Duration { get; set; }
|
||||
public double AverageSpeed { get; set; }
|
||||
public bool IsCompleted { get; set; }
|
||||
public string? Error { get; set; }
|
||||
public double CompletionPercentage => TotalParts > 0 ? (double)CompletedParts.Count / TotalParts * 100 : 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of completing multipart upload
|
||||
/// </summary>
|
||||
public class CompleteMultipartUploadResult
|
||||
{
|
||||
public string ETag { get; set; } = string.Empty;
|
||||
public string Location { get; set; } = string.Empty;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Web;
|
||||
|
||||
namespace PSMinIO.Core.S3
|
||||
{
|
||||
/// <summary>
|
||||
/// Generates presigned URLs for temporary access to MinIO objects
|
||||
/// </summary>
|
||||
public class PresignedUrlGenerator
|
||||
{
|
||||
private readonly string _endpoint;
|
||||
private readonly string _accessKey;
|
||||
private readonly string _secretKey;
|
||||
private readonly string _region;
|
||||
private readonly bool _useSSL;
|
||||
|
||||
public PresignedUrlGenerator(string endpoint, string accessKey, string secretKey, string region = "us-east-1", bool useSSL = true)
|
||||
{
|
||||
_endpoint = endpoint?.TrimEnd('/') ?? throw new ArgumentNullException(nameof(endpoint));
|
||||
_accessKey = accessKey ?? throw new ArgumentNullException(nameof(accessKey));
|
||||
_secretKey = secretKey ?? throw new ArgumentNullException(nameof(secretKey));
|
||||
_region = region ?? "us-east-1";
|
||||
_useSSL = useSSL;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates a presigned URL for GET operations (download)
|
||||
/// </summary>
|
||||
public PresignedUrlResult GeneratePresignedGetUrl(string bucketName, string objectName,
|
||||
TimeSpan expiration, Dictionary<string, string>? additionalHeaders = null)
|
||||
{
|
||||
return GeneratePresignedUrl(HttpMethod.Get, bucketName, objectName, expiration, additionalHeaders);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates a presigned URL for PUT operations (upload)
|
||||
/// </summary>
|
||||
public PresignedUrlResult GeneratePresignedPutUrl(string bucketName, string objectName,
|
||||
TimeSpan expiration, Dictionary<string, string>? additionalHeaders = null)
|
||||
{
|
||||
return GeneratePresignedUrl(HttpMethod.Put, bucketName, objectName, expiration, additionalHeaders);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates a presigned URL for DELETE operations
|
||||
/// </summary>
|
||||
public PresignedUrlResult GeneratePresignedDeleteUrl(string bucketName, string objectName,
|
||||
TimeSpan expiration, Dictionary<string, string>? additionalHeaders = null)
|
||||
{
|
||||
return GeneratePresignedUrl(HttpMethod.Delete, bucketName, objectName, expiration, additionalHeaders);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates a presigned URL for HEAD operations (metadata)
|
||||
/// </summary>
|
||||
public PresignedUrlResult GeneratePresignedHeadUrl(string bucketName, string objectName,
|
||||
TimeSpan expiration, Dictionary<string, string>? additionalHeaders = null)
|
||||
{
|
||||
return GeneratePresignedUrl(HttpMethod.Head, bucketName, objectName, expiration, additionalHeaders);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates a presigned URL with custom HTTP method
|
||||
/// </summary>
|
||||
public PresignedUrlResult GeneratePresignedUrl(HttpMethod method, string bucketName, string objectName,
|
||||
TimeSpan expiration, Dictionary<string, string>? additionalHeaders = null)
|
||||
{
|
||||
if (string.IsNullOrEmpty(bucketName)) throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
|
||||
if (string.IsNullOrEmpty(objectName)) throw new ArgumentException("Object name cannot be null or empty", nameof(objectName));
|
||||
if (expiration.TotalSeconds < 1) throw new ArgumentException("Expiration must be at least 1 second", nameof(expiration));
|
||||
if (expiration.TotalDays > 7) throw new ArgumentException("Expiration cannot exceed 7 days", nameof(expiration));
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var expirationTime = now.Add(expiration);
|
||||
var expirationUnix = ((DateTimeOffset)expirationTime).ToUnixTimeSeconds();
|
||||
|
||||
// Build the canonical request
|
||||
var httpMethod = method.Method.ToUpperInvariant();
|
||||
var canonicalUri = $"/{bucketName}/{Uri.EscapeDataString(objectName)}";
|
||||
var timestamp = now.ToString("yyyyMMddTHHmmssZ", CultureInfo.InvariantCulture);
|
||||
var dateStamp = now.ToString("yyyyMMdd", CultureInfo.InvariantCulture);
|
||||
|
||||
// Query parameters for AWS Signature Version 4
|
||||
var queryParams = new Dictionary<string, string>
|
||||
{
|
||||
{ "X-Amz-Algorithm", "AWS4-HMAC-SHA256" },
|
||||
{ "X-Amz-Credential", $"{_accessKey}/{dateStamp}/{_region}/s3/aws4_request" },
|
||||
{ "X-Amz-Date", timestamp },
|
||||
{ "X-Amz-Expires", expiration.TotalSeconds.ToString("F0", CultureInfo.InvariantCulture) },
|
||||
{ "X-Amz-SignedHeaders", "host" }
|
||||
};
|
||||
|
||||
// Add additional headers to signed headers if provided
|
||||
var signedHeaders = new List<string> { "host" };
|
||||
var canonicalHeaders = $"host:{GetHostFromEndpoint()}\n";
|
||||
|
||||
if (additionalHeaders != null && additionalHeaders.Count > 0)
|
||||
{
|
||||
foreach (var header in additionalHeaders.OrderBy(h => h.Key.ToLowerInvariant()))
|
||||
{
|
||||
var headerName = header.Key.ToLowerInvariant();
|
||||
signedHeaders.Add(headerName);
|
||||
canonicalHeaders += $"{headerName}:{header.Value}\n";
|
||||
}
|
||||
queryParams["X-Amz-SignedHeaders"] = string.Join(";", signedHeaders.OrderBy(h => h));
|
||||
}
|
||||
|
||||
// Build canonical query string
|
||||
var canonicalQueryString = string.Join("&",
|
||||
queryParams.OrderBy(p => p.Key).Select(p => $"{Uri.EscapeDataString(p.Key)}={Uri.EscapeDataString(p.Value)}"));
|
||||
|
||||
// Create canonical request
|
||||
var canonicalRequest = $"{httpMethod}\n{canonicalUri}\n{canonicalQueryString}\n{canonicalHeaders}\n{string.Join(";", signedHeaders.OrderBy(h => h))}\nUNSIGNED-PAYLOAD";
|
||||
|
||||
// Create string to sign
|
||||
var credentialScope = $"{dateStamp}/{_region}/s3/aws4_request";
|
||||
var stringToSign = $"AWS4-HMAC-SHA256\n{timestamp}\n{credentialScope}\n{ComputeSHA256Hash(canonicalRequest)}";
|
||||
|
||||
// Calculate signature
|
||||
var signature = ComputeSignature(stringToSign, dateStamp);
|
||||
queryParams["X-Amz-Signature"] = signature;
|
||||
|
||||
// Build final URL
|
||||
var scheme = _useSSL ? "https" : "http";
|
||||
var finalQueryString = string.Join("&",
|
||||
queryParams.OrderBy(p => p.Key).Select(p => $"{Uri.EscapeDataString(p.Key)}={Uri.EscapeDataString(p.Value)}"));
|
||||
var presignedUrl = $"{scheme}://{GetHostFromEndpoint()}{canonicalUri}?{finalQueryString}";
|
||||
|
||||
return new PresignedUrlResult
|
||||
{
|
||||
Url = presignedUrl,
|
||||
Method = httpMethod,
|
||||
BucketName = bucketName,
|
||||
ObjectName = objectName,
|
||||
ExpirationTime = expirationTime,
|
||||
CreatedTime = now,
|
||||
Duration = expiration,
|
||||
AdditionalHeaders = additionalHeaders?.ToDictionary(kvp => kvp.Key, kvp => kvp.Value)
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extracts host from endpoint
|
||||
/// </summary>
|
||||
private string GetHostFromEndpoint()
|
||||
{
|
||||
var uri = new Uri(_useSSL ? $"https://{_endpoint}" : $"http://{_endpoint}");
|
||||
return uri.Host + (uri.IsDefaultPort ? "" : $":{uri.Port}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes SHA256 hash
|
||||
/// </summary>
|
||||
private static string ComputeSHA256Hash(string input)
|
||||
{
|
||||
using var sha256 = SHA256.Create();
|
||||
var bytes = sha256.ComputeHash(Encoding.UTF8.GetBytes(input));
|
||||
return BitConverter.ToString(bytes).Replace("-", "").ToLowerInvariant();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes AWS Signature Version 4 signature
|
||||
/// </summary>
|
||||
private string ComputeSignature(string stringToSign, string dateStamp)
|
||||
{
|
||||
var kDate = ComputeHMACSHA256($"AWS4{_secretKey}", dateStamp);
|
||||
var kRegion = ComputeHMACSHA256(kDate, _region);
|
||||
var kService = ComputeHMACSHA256(kRegion, "s3");
|
||||
var kSigning = ComputeHMACSHA256(kService, "aws4_request");
|
||||
var signature = ComputeHMACSHA256(kSigning, stringToSign);
|
||||
|
||||
return BitConverter.ToString(signature).Replace("-", "").ToLowerInvariant();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes HMAC-SHA256
|
||||
/// </summary>
|
||||
private static byte[] ComputeHMACSHA256(string key, string data)
|
||||
{
|
||||
return ComputeHMACSHA256(Encoding.UTF8.GetBytes(key), data);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes HMAC-SHA256 with byte array key
|
||||
/// </summary>
|
||||
private static byte[] ComputeHMACSHA256(byte[] key, string data)
|
||||
{
|
||||
using var hmac = new HMACSHA256(key);
|
||||
return hmac.ComputeHash(Encoding.UTF8.GetBytes(data));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of presigned URL generation
|
||||
/// </summary>
|
||||
public class PresignedUrlResult
|
||||
{
|
||||
public string Url { get; set; } = string.Empty;
|
||||
public string Method { get; set; } = string.Empty;
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
public string ObjectName { get; set; } = string.Empty;
|
||||
public DateTime ExpirationTime { get; set; }
|
||||
public DateTime CreatedTime { get; set; }
|
||||
public TimeSpan Duration { get; set; }
|
||||
public Dictionary<string, string>? AdditionalHeaders { get; set; }
|
||||
public bool IsExpired => DateTime.UtcNow > ExpirationTime;
|
||||
public TimeSpan TimeUntilExpiration => ExpirationTime > DateTime.UtcNow ? ExpirationTime - DateTime.UtcNow : TimeSpan.Zero;
|
||||
public string FormattedExpiration => ExpirationTime.ToString("yyyy-MM-dd HH:mm:ss UTC");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Presigned URL request configuration
|
||||
/// </summary>
|
||||
public class PresignedUrlRequest
|
||||
{
|
||||
public string BucketName { get; set; } = string.Empty;
|
||||
public string ObjectName { get; set; } = string.Empty;
|
||||
public HttpMethod Method { get; set; } = HttpMethod.Get;
|
||||
public TimeSpan Expiration { get; set; } = TimeSpan.FromHours(1);
|
||||
public Dictionary<string, string>? Headers { get; set; }
|
||||
public Dictionary<string, string>? QueryParameters { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -194,7 +194,7 @@ namespace PSMinIO.Utils
|
||||
var fullPath = fileInfo.FullName;
|
||||
if (fullPath.StartsWith(basePath, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var relativePath = fullPath.Substring(basePath.Length).TrimStart('\\', '/');
|
||||
var relativePath = fullPath.Substring(basePath!.Length).TrimStart('\\', '/');
|
||||
return relativePath.Replace('\\', '/'); // Use forward slashes for zip entries
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user