diff --git a/Module/PSMinIO/PSMinIO.psd1 b/Module/PSMinIO/PSMinIO.psd1 index 0c2d207..78a0b98 100644 --- a/Module/PSMinIO/PSMinIO.psd1 +++ b/Module/PSMinIO/PSMinIO.psd1 @@ -71,8 +71,15 @@ 'Get-MinIOObject', 'New-MinIOObject', 'Get-MinIOObjectContent', + 'Get-MinIOObjectContentMultipart', + 'New-MinIOObjectMultipart', 'Get-MinIOZipArchive', - 'New-MinIOZipArchive' + 'New-MinIOZipArchive', + 'Get-MinIOPresignedUrl', + 'New-MinIOPresignedUrl', + 'Get-MinIOBucketPolicy', + 'Set-MinIOBucketPolicy', + 'Remove-MinIOBucketPolicy' ) # Variables to export from this module diff --git a/src/Cmdlets/BucketPolicyCmdlets.cs b/src/Cmdlets/BucketPolicyCmdlets.cs new file mode 100644 index 0000000..baa2af1 --- /dev/null +++ b/src/Cmdlets/BucketPolicyCmdlets.cs @@ -0,0 +1,230 @@ +using System; +using System.Collections.Generic; +using System.Management.Automation; +using PSMinIO.Core.Http; +using PSMinIO.Core.Models; +using PSMinIO.Core.S3; +using PSMinIO.Utils; + +namespace PSMinIO.Cmdlets +{ + /// + /// Gets bucket policy + /// + [Cmdlet(VerbsCommon.Get, "MinIOBucketPolicy")] + [OutputType(typeof(BucketPolicyResult))] + public class GetMinIOBucketPolicyCmdlet : MinIOBaseCmdlet + { + /// + /// Name of the bucket + /// + [Parameter(Position = 0, Mandatory = true, ValueFromPipeline = true)] + [ValidateNotNullOrEmpty] + [Alias("Bucket")] + public string BucketName { get; set; } = string.Empty; + + /// + /// Processes the cmdlet + /// + protected override void ProcessRecord() + { + var result = ExecuteOperation("GetBucketPolicy", () => + { + WriteVerboseMessage("Getting bucket policy for: {0}", BucketName); + + // Get connection and create policy manager + var connection = Connection; + var httpClient = new MinIOHttpClient(connection.Configuration); + var progressCollector = new ThreadSafeProgressCollector(this); + var policyManager = new BucketPolicyManager(httpClient, progressCollector); + + // Get bucket policy + var policyResult = policyManager.GetBucketPolicy(BucketName); + + if (policyResult.HasPolicy) + { + WriteVerboseMessage("Retrieved bucket policy successfully"); + } + else + { + WriteVerboseMessage("No policy found for bucket: {0}", BucketName); + } + + return policyResult; + + }, $"Bucket: {BucketName}"); + + // Always return the result object + WriteObject(result); + } + } + + /// + /// Sets bucket policy + /// + [Cmdlet(VerbsCommon.Set, "MinIOBucketPolicy", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.High)] + [OutputType(typeof(BucketPolicyResult))] + public class SetMinIOBucketPolicyCmdlet : MinIOBaseCmdlet + { + /// + /// Name of the bucket + /// + [Parameter(Position = 0, Mandatory = true)] + [ValidateNotNullOrEmpty] + [Alias("Bucket")] + public string BucketName { get; set; } = string.Empty; + + /// + /// Bucket policy object + /// + [Parameter(Position = 1, Mandatory = true, ParameterSetName = "PolicyObject")] + [ValidateNotNull] + public BucketPolicy Policy { get; set; } = null!; + + /// + /// Bucket policy as JSON string + /// + [Parameter(Position = 1, Mandatory = true, ParameterSetName = "PolicyJson")] + [ValidateNotNullOrEmpty] + public string PolicyJson { get; set; } = string.Empty; + + /// + /// Create a simple read-only policy for public access + /// + [Parameter(Mandatory = true, ParameterSetName = "ReadOnly")] + public SwitchParameter ReadOnly { get; set; } + + /// + /// Object prefix for the policy (optional) + /// + [Parameter(ParameterSetName = "ReadOnly")] + [Parameter(ParameterSetName = "ReadWrite")] + public string? ObjectPrefix { get; set; } + + /// + /// Create a read-write policy for specific principals + /// + [Parameter(Mandatory = true, ParameterSetName = "ReadWrite")] + public SwitchParameter ReadWrite { get; set; } + + /// + /// List of principals for read-write policy + /// + [Parameter(Mandatory = true, ParameterSetName = "ReadWrite")] + [ValidateNotNull] + public string[] Principals { get; set; } = Array.Empty(); + + /// + /// Processes the cmdlet + /// + protected override void ProcessRecord() + { + if (ShouldProcess(BucketName, "Set bucket policy")) + { + var result = ExecuteOperation("SetBucketPolicy", () => + { + WriteVerboseMessage("Setting bucket policy for: {0}", BucketName); + + // Get connection and create policy manager + var connection = Connection; + var httpClient = new MinIOHttpClient(connection.Configuration); + var progressCollector = new ThreadSafeProgressCollector(this); + var policyManager = new BucketPolicyManager(httpClient, progressCollector); + + BucketPolicyResult policyResult; + + switch (ParameterSetName) + { + case "PolicyObject": + policyResult = policyManager.SetBucketPolicy(BucketName, Policy); + break; + case "PolicyJson": + policyResult = policyManager.SetBucketPolicyFromJson(BucketName, PolicyJson); + break; + case "ReadOnly": + var readOnlyPolicy = policyManager.CreateReadOnlyPolicy(BucketName, ObjectPrefix); + policyResult = policyManager.SetBucketPolicy(BucketName, readOnlyPolicy); + WriteVerboseMessage("Created read-only policy for public access"); + break; + case "ReadWrite": + var readWritePolicy = policyManager.CreateReadWritePolicy(BucketName, new List(Principals), ObjectPrefix); + policyResult = policyManager.SetBucketPolicy(BucketName, readWritePolicy); + WriteVerboseMessage("Created read-write policy for {0} principals", Principals.Length); + break; + default: + throw new InvalidOperationException($"Unknown parameter set: {ParameterSetName}"); + } + + if (policyResult.IsValid) + { + WriteVerboseMessage("Set bucket policy successfully"); + } + else + { + WriteWarning($"Failed to set bucket policy: {policyResult.Error}"); + } + + return policyResult; + + }, $"Bucket: {BucketName}, ParameterSet: {ParameterSetName}"); + + // Always return the result object + WriteObject(result); + } + } + } + + /// + /// Removes bucket policy + /// + [Cmdlet(VerbsCommon.Remove, "MinIOBucketPolicy", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.High)] + [OutputType(typeof(BucketPolicyResult))] + public class RemoveMinIOBucketPolicyCmdlet : MinIOBaseCmdlet + { + /// + /// Name of the bucket + /// + [Parameter(Position = 0, Mandatory = true, ValueFromPipeline = true)] + [ValidateNotNullOrEmpty] + [Alias("Bucket")] + public string BucketName { get; set; } = string.Empty; + + /// + /// Processes the cmdlet + /// + protected override void ProcessRecord() + { + if (ShouldProcess(BucketName, "Remove bucket policy")) + { + var result = ExecuteOperation("RemoveBucketPolicy", () => + { + WriteVerboseMessage("Removing bucket policy for: {0}", BucketName); + + // Get connection and create policy manager + var connection = Connection; + var httpClient = new MinIOHttpClient(connection.Configuration); + var progressCollector = new ThreadSafeProgressCollector(this); + var policyManager = new BucketPolicyManager(httpClient, progressCollector); + + // Remove bucket policy + var policyResult = policyManager.DeleteBucketPolicy(BucketName); + + if (policyResult.IsValid) + { + WriteVerboseMessage("Removed bucket policy successfully"); + } + else + { + WriteWarning($"Failed to remove bucket policy: {policyResult.Error}"); + } + + return policyResult; + + }, $"Bucket: {BucketName}"); + + // Always return the result object + WriteObject(result); + } + } + } +} diff --git a/src/Cmdlets/GetMinIOBucketCmdlet.cs b/src/Cmdlets/GetMinIOBucketCmdlet.cs index 95e392b..8214276 100644 --- a/src/Cmdlets/GetMinIOBucketCmdlet.cs +++ b/src/Cmdlets/GetMinIOBucketCmdlet.cs @@ -64,7 +64,7 @@ namespace PSMinIO.Cmdlets { var wildcardPattern = new WildcardPattern(BucketName, WildcardOptions.IgnoreCase); buckets = buckets.Where(b => b.Name != null && wildcardPattern.IsMatch(b.Name)).ToList(); - WriteVerboseMessage("Filtered to {0} buckets matching pattern '{1}'", buckets.Count, BucketName); + WriteVerboseMessage("Filtered to {0} buckets matching pattern '{1}'", buckets.Count, BucketName!); } // Enhance bucket information if requested diff --git a/src/Cmdlets/GetMinIOObjectCmdlet.cs b/src/Cmdlets/GetMinIOObjectCmdlet.cs index 961e46f..e92f337 100644 --- a/src/Cmdlets/GetMinIOObjectCmdlet.cs +++ b/src/Cmdlets/GetMinIOObjectCmdlet.cs @@ -108,7 +108,7 @@ namespace PSMinIO.Cmdlets { var wildcardPattern = new WildcardPattern(Name, WildcardOptions.IgnoreCase); objects = objects.Where(o => o.Name != null && wildcardPattern.IsMatch(o.Name)).ToList(); - WriteVerboseMessage("Filtered to {0} objects matching pattern '{1}'", objects.Count, Name); + WriteVerboseMessage("Filtered to {0} objects matching pattern '{1}'", objects.Count, Name!); } // Apply directory filters diff --git a/src/Cmdlets/GetMinIOObjectContentMultipartCmdlet.cs b/src/Cmdlets/GetMinIOObjectContentMultipartCmdlet.cs new file mode 100644 index 0000000..d6c89bb --- /dev/null +++ b/src/Cmdlets/GetMinIOObjectContentMultipartCmdlet.cs @@ -0,0 +1,137 @@ +using System; +using System.IO; +using System.Management.Automation; +using PSMinIO.Core.Http; +using PSMinIO.Core.Models; +using PSMinIO.Core.S3; +using PSMinIO.Utils; + +namespace PSMinIO.Cmdlets +{ + /// + /// Downloads objects using multipart download with resume capability + /// + [Cmdlet(VerbsCommon.Get, "MinIOObjectContentMultipart", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Medium)] + [OutputType(typeof(MultipartDownloadResult))] + public class GetMinIOObjectContentMultipartCmdlet : MinIOBaseCmdlet + { + /// + /// Name of the bucket containing the object + /// + [Parameter(Position = 0, Mandatory = true)] + [ValidateNotNullOrEmpty] + [Alias("Bucket")] + public string BucketName { get; set; } = string.Empty; + + /// + /// Name of the object to download + /// + [Parameter(Position = 1, Mandatory = true)] + [ValidateNotNullOrEmpty] + [Alias("Object", "Key")] + public string ObjectName { get; set; } = string.Empty; + + /// + /// Local file path where the object will be saved + /// + [Parameter(Position = 2, Mandatory = true)] + [ValidateNotNull] + [Alias("File", "Path")] + public FileInfo DestinationPath { get; set; } = null!; + + /// + /// Size of each download chunk in bytes (default: 32MB) + /// + [Parameter] + [ValidateRange(1024 * 1024, long.MaxValue)] // Minimum 1MB + public long ChunkSize { get; set; } = 32 * 1024 * 1024; // 32MB default + + /// + /// Maximum number of parallel download threads (default: 4) + /// + [Parameter] + [ValidateRange(1, 8)] + public int MaxParallelDownloads { get; set; } = 4; + + /// + /// Resume download if destination file already exists + /// + [Parameter] + public SwitchParameter Resume { get; set; } + + /// + /// Overwrite existing file without prompting + /// + [Parameter] + public SwitchParameter Force { get; set; } + + /// + /// Processes the cmdlet + /// + protected override void ProcessRecord() + { + // Validate destination path + if (DestinationPath.Exists && !Resume.IsPresent && !Force.IsPresent) + { + var errorRecord = new ErrorRecord( + new InvalidOperationException($"Destination file already exists: {DestinationPath.FullName}. Use -Resume to continue or -Force to overwrite."), + "DestinationFileExists", + ErrorCategory.ResourceExists, + DestinationPath); + ThrowTerminatingError(errorRecord); + } + + // Ensure destination directory exists + if (DestinationPath.Directory != null && !DestinationPath.Directory.Exists) + { + DestinationPath.Directory.Create(); + WriteVerboseMessage("Created destination directory: {0}", DestinationPath.Directory.FullName); + } + + var operationDescription = Resume.IsPresent && DestinationPath.Exists + ? $"Resume multipart download of '{ObjectName}' from bucket '{BucketName}'" + : $"Multipart download '{ObjectName}' from bucket '{BucketName}'"; + + if (ShouldProcess(DestinationPath.FullName, operationDescription)) + { + var result = ExecuteOperation("MultipartDownload", () => + { + WriteVerboseMessage("Starting multipart download: {0}/{1} -> {2}", + BucketName, ObjectName, DestinationPath.FullName); + WriteVerboseMessage("Chunk size: {0}, Max parallel downloads: {1}, Resume: {2}", + SizeFormatter.FormatBytes(ChunkSize), MaxParallelDownloads, Resume.IsPresent); + + // Get connection and create download manager + var connection = Connection; + var httpClient = new MinIOHttpClient(connection.Configuration); + var progressCollector = new ThreadSafeProgressCollector(this); + var downloadManager = new MultipartDownloadManager(httpClient, progressCollector, + MaxParallelDownloads, ChunkSize); + + // Perform multipart download + var downloadResult = downloadManager.DownloadFile(BucketName, ObjectName, DestinationPath, + ChunkSize, Resume.IsPresent); + + if (downloadResult.IsCompleted) + { + WriteVerboseMessage("Multipart download completed successfully"); + WriteVerboseMessage("Downloaded: {0} in {1} at {2}", + SizeFormatter.FormatBytes(downloadResult.DownloadedSize), + SizeFormatter.FormatDuration(downloadResult.Duration), + SizeFormatter.FormatSpeed(downloadResult.AverageSpeed)); + } + else + { + WriteWarning($"Multipart download failed: {downloadResult.Error}"); + } + + return downloadResult; + + }, $"Bucket: {BucketName}, Object: {ObjectName}, Destination: {DestinationPath.FullName}"); + + // Always return the result object + WriteObject(result); + } + } + } +} diff --git a/src/Cmdlets/NewMinIOObjectMultipartCmdlet.cs b/src/Cmdlets/NewMinIOObjectMultipartCmdlet.cs new file mode 100644 index 0000000..a7bf295 --- /dev/null +++ b/src/Cmdlets/NewMinIOObjectMultipartCmdlet.cs @@ -0,0 +1,212 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Management.Automation; +using PSMinIO.Core.Http; +using PSMinIO.Core.Models; +using PSMinIO.Core.S3; +using PSMinIO.Utils; + +namespace PSMinIO.Cmdlets +{ + /// + /// Uploads objects using multipart upload with parallel processing and resume capability + /// + [Cmdlet(VerbsCommon.New, "MinIOObjectMultipart", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Medium)] + [OutputType(typeof(MultipartUploadResult))] + public class NewMinIOObjectMultipartCmdlet : MinIOBaseCmdlet + { + /// + /// Name of the bucket to upload to + /// + [Parameter(Position = 0, Mandatory = true)] + [ValidateNotNullOrEmpty] + [Alias("Bucket")] + public string BucketName { get; set; } = string.Empty; + + /// + /// Local file to upload + /// + [Parameter(Position = 1, Mandatory = true, ValueFromPipeline = true)] + [ValidateNotNull] + [Alias("File", "Path")] + public FileInfo FilePath { get; set; } = null!; + + /// + /// Name of the object in the bucket (optional, uses filename if not specified) + /// + [Parameter(Position = 2)] + [ValidateNotNullOrEmpty] + [Alias("Object", "Key")] + public string? ObjectName { get; set; } + + /// + /// Size of each upload chunk in bytes (default: 64MB, minimum: 5MB for S3 compatibility) + /// + [Parameter] + [ValidateRange(5 * 1024 * 1024, long.MaxValue)] // Minimum 5MB for S3 compatibility + public long ChunkSize { get; set; } = 64 * 1024 * 1024; // 64MB default + + /// + /// Maximum number of parallel upload threads (default: 4) + /// + [Parameter] + [ValidateRange(1, 10)] + public int MaxParallelUploads { get; set; } = 4; + + /// + /// Content type of the object (auto-detected if not specified) + /// + [Parameter] + [ValidateNotNullOrEmpty] + public string? ContentType { get; set; } + + /// + /// Custom metadata for the object + /// + [Parameter] + public Dictionary? Metadata { get; set; } + + /// + /// Resume upload using existing upload ID + /// + [Parameter] + [ValidateNotNullOrEmpty] + public string? ResumeUploadId { get; set; } + + /// + /// Completed parts for resume (used with ResumeUploadId) + /// + [Parameter] + public PartInfo[]? CompletedParts { get; set; } + + /// + /// Overwrite existing object without prompting + /// + [Parameter] + public SwitchParameter Force { get; set; } + + /// + /// Processes the cmdlet + /// + protected override void ProcessRecord() + { + // Validate file exists + if (!FilePath.Exists) + { + var errorRecord = new ErrorRecord( + new FileNotFoundException($"File not found: {FilePath.FullName}"), + "FileNotFound", + ErrorCategory.ObjectNotFound, + FilePath); + ThrowTerminatingError(errorRecord); + } + + // Determine object name + var effectiveObjectName = ObjectName ?? FilePath.Name; + + // Auto-detect content type if not specified + var effectiveContentType = ContentType ?? GetContentType(FilePath.Extension); + + // Add content type to metadata + var effectiveMetadata = Metadata ?? new Dictionary(); + if (!string.IsNullOrEmpty(effectiveContentType)) + { + effectiveMetadata["Content-Type"] = effectiveContentType; + } + + var operationDescription = !string.IsNullOrEmpty(ResumeUploadId) + ? $"Resume multipart upload of '{FilePath.Name}' to bucket '{BucketName}'" + : $"Multipart upload '{FilePath.Name}' to bucket '{BucketName}'"; + + if (ShouldProcess(effectiveObjectName, operationDescription)) + { + var result = ExecuteOperation("MultipartUpload", () => + { + WriteVerboseMessage("Starting multipart upload: {0} -> {1}/{2}", + FilePath.FullName, BucketName, effectiveObjectName); + WriteVerboseMessage("File size: {0}, Chunk size: {1}, Max parallel uploads: {2}", + SizeFormatter.FormatBytes(FilePath.Length), SizeFormatter.FormatBytes(ChunkSize), MaxParallelUploads); + + if (!string.IsNullOrEmpty(ResumeUploadId)) + { + WriteVerboseMessage("Resuming upload with ID: {0}", ResumeUploadId); + if (CompletedParts != null && CompletedParts.Length > 0) + { + WriteVerboseMessage("Found {0} completed parts", CompletedParts.Length); + } + } + + // Get connection and create upload manager + var connection = Connection; + var httpClient = new MinIOHttpClient(connection.Configuration); + var progressCollector = new ThreadSafeProgressCollector(this); + var uploadManager = new MultipartUploadManager(httpClient, progressCollector, + MaxParallelUploads, ChunkSize); + + // Perform multipart upload + var uploadResult = uploadManager.UploadFile(BucketName, effectiveObjectName, FilePath, + effectiveMetadata, ChunkSize, ResumeUploadId, CompletedParts?.ToList()); + + if (uploadResult.IsCompleted) + { + WriteVerboseMessage("Multipart upload completed successfully"); + WriteVerboseMessage("Uploaded: {0} in {1} at {2}", + SizeFormatter.FormatBytes(uploadResult.TotalSize), + SizeFormatter.FormatDuration(uploadResult.Duration), + SizeFormatter.FormatSpeed(uploadResult.AverageSpeed)); + WriteVerboseMessage("ETag: {0}", uploadResult.ETag); + } + else + { + WriteWarning($"Multipart upload failed: {uploadResult.Error}"); + if (uploadResult.CompletedParts.Count > 0) + { + WriteWarning($"Upload can be resumed using UploadId: {uploadResult.UploadId}"); + WriteWarning($"Completed parts: {uploadResult.CompletedParts.Count}/{uploadResult.TotalParts}"); + } + } + + return uploadResult; + + }, $"File: {FilePath.FullName}, Bucket: {BucketName}, Object: {effectiveObjectName}"); + + // Always return the result object + WriteObject(result); + } + } + + /// + /// Gets content type based on file extension + /// + private string GetContentType(string extension) + { + return extension.ToLowerInvariant() switch + { + ".txt" => "text/plain", + ".html" => "text/html", + ".css" => "text/css", + ".js" => "application/javascript", + ".json" => "application/json", + ".xml" => "application/xml", + ".pdf" => "application/pdf", + ".zip" => "application/zip", + ".jpg" or ".jpeg" => "image/jpeg", + ".png" => "image/png", + ".gif" => "image/gif", + ".svg" => "image/svg+xml", + ".mp4" => "video/mp4", + ".mp3" => "audio/mpeg", + ".wav" => "audio/wav", + ".doc" => "application/msword", + ".docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ".xls" => "application/vnd.ms-excel", + ".xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ".ppt" => "application/vnd.ms-powerpoint", + ".pptx" => "application/vnd.openxmlformats-officedocument.presentationml.presentation", + _ => "application/octet-stream" + }; + } + } +} diff --git a/src/Cmdlets/NewMinIOPresignedUrlCmdlet.cs b/src/Cmdlets/NewMinIOPresignedUrlCmdlet.cs new file mode 100644 index 0000000..0059c5e --- /dev/null +++ b/src/Cmdlets/NewMinIOPresignedUrlCmdlet.cs @@ -0,0 +1,220 @@ +using System; +using System.Collections.Generic; +using System.Management.Automation; +using System.Net.Http; +using PSMinIO.Core.Models; +using PSMinIO.Core.S3; + +namespace PSMinIO.Cmdlets +{ + /// + /// Generates presigned URLs for temporary access to MinIO objects + /// + [Cmdlet(VerbsCommon.New, "MinIOPresignedUrl")] + [OutputType(typeof(PresignedUrlResult))] + public class NewMinIOPresignedUrlCmdlet : MinIOBaseCmdlet + { + /// + /// Name of the bucket + /// + [Parameter(Position = 0, Mandatory = true)] + [ValidateNotNullOrEmpty] + [Alias("Bucket")] + public string BucketName { get; set; } = string.Empty; + + /// + /// Name of the object + /// + [Parameter(Position = 1, Mandatory = true)] + [ValidateNotNullOrEmpty] + [Alias("Object", "Key")] + public string ObjectName { get; set; } = string.Empty; + + /// + /// HTTP method for the presigned URL + /// + [Parameter(Position = 2)] + [ValidateSet("GET", "PUT", "DELETE", "HEAD")] + public string Method { get; set; } = "GET"; + + /// + /// Expiration time for the URL (default: 1 hour, maximum: 7 days) + /// + [Parameter] + public TimeSpan Expiration { get; set; } = TimeSpan.FromHours(1); + + /// + /// Additional headers to include in the presigned URL + /// + [Parameter] + public Dictionary? Headers { get; set; } + + /// + /// Show the URL in the console (for easy copying) + /// + [Parameter] + public SwitchParameter ShowUrl { get; set; } + + /// + /// Processes the cmdlet + /// + protected override void ProcessRecord() + { + // Validate expiration time + if (Expiration.TotalSeconds < 1) + { + ThrowTerminatingError(new ErrorRecord( + new ArgumentException("Expiration must be at least 1 second"), + "InvalidExpiration", + ErrorCategory.InvalidArgument, + Expiration)); + } + if (Expiration.TotalDays > 7) + { + ThrowTerminatingError(new ErrorRecord( + new ArgumentException("Expiration cannot exceed 7 days"), + "InvalidExpiration", + ErrorCategory.InvalidArgument, + Expiration)); + } + + var result = ExecuteOperation("GeneratePresignedUrl", () => + { + WriteVerboseMessage("Generating presigned URL for: {0}/{1}", BucketName, ObjectName); + WriteVerboseMessage("Method: {0}, Expiration: {1}", Method, Expiration); + + // Get connection details + var connection = Connection; + + // Create presigned URL generator + var generator = new PresignedUrlGenerator( + connection.Configuration.Endpoint, + connection.Configuration.AccessKey, + connection.Configuration.SecretKey, + connection.Configuration.Region, + connection.Configuration.UseSSL); + + // Parse HTTP method + var httpMethod = Method.ToUpperInvariant() switch + { + "GET" => HttpMethod.Get, + "PUT" => HttpMethod.Put, + "DELETE" => HttpMethod.Delete, + "HEAD" => HttpMethod.Head, + _ => HttpMethod.Get + }; + + // Generate presigned URL + var urlResult = generator.GeneratePresignedUrl(httpMethod, BucketName, ObjectName, Expiration, Headers); + + WriteVerboseMessage("Generated presigned URL successfully"); + WriteVerboseMessage("URL expires: {0}", urlResult.FormattedExpiration); + + if (ShowUrl.IsPresent) + { + WriteInformation($"Presigned URL: {urlResult.Url}", new string[] { "PresignedUrl" }); + } + + return urlResult; + + }, $"Bucket: {BucketName}, Object: {ObjectName}, Method: {Method}"); + + // Always return the result object + WriteObject(result); + } + } + + /// + /// Cmdlet for generating GET presigned URLs (download) + /// + [Cmdlet(VerbsCommon.Get, "MinIOPresignedUrl")] + [OutputType(typeof(PresignedUrlResult))] + public class GetMinIOPresignedUrlCmdlet : MinIOBaseCmdlet + { + /// + /// Name of the bucket + /// + [Parameter(Position = 0, Mandatory = true)] + [ValidateNotNullOrEmpty] + [Alias("Bucket")] + public string BucketName { get; set; } = string.Empty; + + /// + /// Name of the object + /// + [Parameter(Position = 1, Mandatory = true)] + [ValidateNotNullOrEmpty] + [Alias("Object", "Key")] + public string ObjectName { get; set; } = string.Empty; + + /// + /// Expiration time for the URL (default: 1 hour, maximum: 7 days) + /// + [Parameter] + public TimeSpan Expiration { get; set; } = TimeSpan.FromHours(1); + + /// + /// Show the URL in the console (for easy copying) + /// + [Parameter] + public SwitchParameter ShowUrl { get; set; } + + /// + /// Processes the cmdlet + /// + protected override void ProcessRecord() + { + // Validate expiration time + if (Expiration.TotalSeconds < 1) + { + ThrowTerminatingError(new ErrorRecord( + new ArgumentException("Expiration must be at least 1 second"), + "InvalidExpiration", + ErrorCategory.InvalidArgument, + Expiration)); + } + if (Expiration.TotalDays > 7) + { + ThrowTerminatingError(new ErrorRecord( + new ArgumentException("Expiration cannot exceed 7 days"), + "InvalidExpiration", + ErrorCategory.InvalidArgument, + Expiration)); + } + + var result = ExecuteOperation("GeneratePresignedGetUrl", () => + { + WriteVerboseMessage("Generating presigned GET URL for: {0}/{1}", BucketName, ObjectName); + WriteVerboseMessage("Expiration: {0}", Expiration); + + // Get connection details + var connection = Connection; + + // Create presigned URL generator + var generator = new PresignedUrlGenerator( + connection.Configuration.Endpoint, + connection.Configuration.AccessKey, + connection.Configuration.SecretKey, + connection.Configuration.Region, + connection.Configuration.UseSSL); + + // Generate presigned GET URL + var urlResult = generator.GeneratePresignedGetUrl(BucketName, ObjectName, Expiration); + + WriteVerboseMessage("Generated presigned GET URL successfully"); + WriteVerboseMessage("URL expires: {0}", urlResult.FormattedExpiration); + + if (ShowUrl.IsPresent) + { + WriteInformation($"Presigned URL: {urlResult.Url}", new string[] { "PresignedUrl" }); + } + + return urlResult; + + }, $"Bucket: {BucketName}, Object: {ObjectName}"); + + // Always return the result object + WriteObject(result); + } + } +} diff --git a/src/Core/S3/AdvancedMetadataHandler.cs b/src/Core/S3/AdvancedMetadataHandler.cs new file mode 100644 index 0000000..df7b0c4 --- /dev/null +++ b/src/Core/S3/AdvancedMetadataHandler.cs @@ -0,0 +1,358 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net.Http; +using System.Text; + +namespace PSMinIO.Core.S3 +{ + /// + /// Handles advanced object metadata with S3-compatible headers and custom metadata + /// + public class AdvancedMetadataHandler + { + /// + /// Creates headers from metadata configuration + /// + public Dictionary CreateHeaders(ObjectMetadata metadata) + { + if (metadata == null) throw new ArgumentNullException(nameof(metadata)); + + var headers = new Dictionary(); + + // Content-Type + if (!string.IsNullOrEmpty(metadata.ContentType)) + { + headers["Content-Type"] = metadata.ContentType; + } + + // Content-Encoding + if (!string.IsNullOrEmpty(metadata.ContentEncoding)) + { + headers["Content-Encoding"] = metadata.ContentEncoding; + } + + // Content-Language + if (!string.IsNullOrEmpty(metadata.ContentLanguage)) + { + headers["Content-Language"] = metadata.ContentLanguage; + } + + // Content-Disposition + if (!string.IsNullOrEmpty(metadata.ContentDisposition)) + { + headers["Content-Disposition"] = metadata.ContentDisposition; + } + + // Cache-Control + if (!string.IsNullOrEmpty(metadata.CacheControl)) + { + headers["Cache-Control"] = metadata.CacheControl; + } + + // Expires + if (metadata.Expires.HasValue) + { + headers["Expires"] = metadata.Expires.Value.ToString("R", CultureInfo.InvariantCulture); + } + + // Server-Side Encryption + if (!string.IsNullOrEmpty(metadata.ServerSideEncryption)) + { + headers["x-amz-server-side-encryption"] = metadata.ServerSideEncryption; + } + + // SSE-KMS Key ID + if (!string.IsNullOrEmpty(metadata.SSEKMSKeyId)) + { + headers["x-amz-server-side-encryption-aws-kms-key-id"] = metadata.SSEKMSKeyId; + } + + // SSE-C Algorithm + if (!string.IsNullOrEmpty(metadata.SSECustomerAlgorithm)) + { + headers["x-amz-server-side-encryption-customer-algorithm"] = metadata.SSECustomerAlgorithm; + } + + // SSE-C Key + if (!string.IsNullOrEmpty(metadata.SSECustomerKey)) + { + headers["x-amz-server-side-encryption-customer-key"] = metadata.SSECustomerKey; + } + + // SSE-C Key MD5 + if (!string.IsNullOrEmpty(metadata.SSECustomerKeyMD5)) + { + headers["x-amz-server-side-encryption-customer-key-MD5"] = metadata.SSECustomerKeyMD5; + } + + // Storage Class + if (!string.IsNullOrEmpty(metadata.StorageClass)) + { + headers["x-amz-storage-class"] = metadata.StorageClass; + } + + // Website Redirect Location + if (!string.IsNullOrEmpty(metadata.WebsiteRedirectLocation)) + { + headers["x-amz-website-redirect-location"] = metadata.WebsiteRedirectLocation; + } + + // Object Lock Mode + if (!string.IsNullOrEmpty(metadata.ObjectLockMode)) + { + headers["x-amz-object-lock-mode"] = metadata.ObjectLockMode; + } + + // Object Lock Retain Until Date + if (metadata.ObjectLockRetainUntilDate.HasValue) + { + headers["x-amz-object-lock-retain-until-date"] = + metadata.ObjectLockRetainUntilDate.Value.ToString("yyyy-MM-ddTHH:mm:ss.fffZ", CultureInfo.InvariantCulture); + } + + // Object Lock Legal Hold + if (metadata.ObjectLockLegalHoldStatus.HasValue) + { + headers["x-amz-object-lock-legal-hold"] = metadata.ObjectLockLegalHoldStatus.Value ? "ON" : "OFF"; + } + + // Tagging + if (metadata.Tags != null && metadata.Tags.Count > 0) + { + var tagString = string.Join("&", metadata.Tags.Select(kvp => $"{Uri.EscapeDataString(kvp.Key)}={Uri.EscapeDataString(kvp.Value)}")); + headers["x-amz-tagging"] = tagString; + } + + // Custom metadata (x-amz-meta-*) + if (metadata.UserMetadata != null && metadata.UserMetadata.Count > 0) + { + foreach (var kvp in metadata.UserMetadata) + { + var key = kvp.Key.StartsWith("x-amz-meta-") ? kvp.Key : $"x-amz-meta-{kvp.Key}"; + headers[key] = kvp.Value; + } + } + + // Custom headers + if (metadata.CustomHeaders != null && metadata.CustomHeaders.Count > 0) + { + foreach (var kvp in metadata.CustomHeaders) + { + headers[kvp.Key] = kvp.Value; + } + } + + return headers; + } + + /// + /// Parses metadata from response headers + /// + public ObjectMetadata ParseMetadata(HttpResponseMessage response) + { + if (response == null) throw new ArgumentNullException(nameof(response)); + + var metadata = new ObjectMetadata(); + + // Process response headers + foreach (var header in response.Headers) + { + var key = header.Key.ToLowerInvariant(); + var value = string.Join(", ", header.Value); + + ProcessHeader(metadata, key, value, header.Key); + } + + // Process content headers + foreach (var header in response.Content.Headers) + { + var key = header.Key.ToLowerInvariant(); + var value = string.Join(", ", header.Value); + ProcessHeader(metadata, key, value, header.Key); + } + + return metadata; + } + + /// + /// Processes a single header into metadata + /// + private void ProcessHeader(ObjectMetadata metadata, string key, string value, string originalKey) + { + switch (key) + { + case "content-type": + metadata.ContentType = value; + break; + case "content-encoding": + metadata.ContentEncoding = value; + break; + case "content-language": + metadata.ContentLanguage = value; + break; + case "content-disposition": + metadata.ContentDisposition = value; + break; + case "cache-control": + metadata.CacheControl = value; + break; + case "expires": + if (DateTime.TryParse(value, out var expires)) + metadata.Expires = expires; + break; + case "x-amz-server-side-encryption": + metadata.ServerSideEncryption = value; + break; + case "x-amz-server-side-encryption-aws-kms-key-id": + metadata.SSEKMSKeyId = value; + break; + case "x-amz-server-side-encryption-customer-algorithm": + metadata.SSECustomerAlgorithm = value; + break; + case "x-amz-storage-class": + metadata.StorageClass = value; + break; + case "x-amz-website-redirect-location": + metadata.WebsiteRedirectLocation = value; + break; + case "x-amz-object-lock-mode": + metadata.ObjectLockMode = value; + break; + case "x-amz-object-lock-retain-until-date": + if (DateTime.TryParse(value, out var retainUntil)) + metadata.ObjectLockRetainUntilDate = retainUntil; + break; + case "x-amz-object-lock-legal-hold": + metadata.ObjectLockLegalHoldStatus = value.Equals("ON", StringComparison.OrdinalIgnoreCase); + break; + case "x-amz-tagging": + metadata.Tags = ParseTagString(value); + break; + default: + if (key.StartsWith("x-amz-meta-")) + { + metadata.UserMetadata ??= new Dictionary(); + metadata.UserMetadata[key] = value; + } + else + { + metadata.CustomHeaders ??= new Dictionary(); + metadata.CustomHeaders[originalKey] = value; + } + break; + } + } + + /// + /// Creates cache control header value + /// + public string CreateCacheControlHeader(CacheControlSettings settings) + { + if (settings == null) throw new ArgumentNullException(nameof(settings)); + + var directives = new List(); + + if (settings.NoCache) directives.Add("no-cache"); + if (settings.NoStore) directives.Add("no-store"); + if (settings.MustRevalidate) directives.Add("must-revalidate"); + if (settings.Public) directives.Add("public"); + if (settings.Private) directives.Add("private"); + + if (settings.MaxAge.HasValue) + directives.Add($"max-age={settings.MaxAge.Value.TotalSeconds:F0}"); + + if (settings.SMaxAge.HasValue) + directives.Add($"s-maxage={settings.SMaxAge.Value.TotalSeconds:F0}"); + + return string.Join(", ", directives); + } + + /// + /// Creates content disposition header value + /// + public string CreateContentDispositionHeader(string disposition, string? filename = null) + { + if (string.IsNullOrEmpty(disposition)) throw new ArgumentException("Disposition cannot be null or empty", nameof(disposition)); + + var result = disposition; + if (!string.IsNullOrEmpty(filename)) + { + result += $"; filename=\"{filename}\""; + } + + return result; + } + + /// + /// Parses tag string into dictionary + /// + private Dictionary ParseTagString(string tagString) + { + var tags = new Dictionary(); + if (string.IsNullOrEmpty(tagString)) return tags; + + var pairs = tagString.Split(new char[] { '&' }, StringSplitOptions.RemoveEmptyEntries); + foreach (var pair in pairs) + { + var parts = pair.Split(new char[] { '=' }, 2); + if (parts.Length == 2) + { + var key = Uri.UnescapeDataString(parts[0]); + var value = Uri.UnescapeDataString(parts[1]); + tags[key] = value; + } + } + + return tags; + } + } + + /// + /// Comprehensive object metadata configuration + /// + public class ObjectMetadata + { + // Standard HTTP headers + public string? ContentType { get; set; } + public string? ContentEncoding { get; set; } + public string? ContentLanguage { get; set; } + public string? ContentDisposition { get; set; } + public string? CacheControl { get; set; } + public DateTime? Expires { get; set; } + + // S3-specific headers + public string? ServerSideEncryption { get; set; } + public string? SSEKMSKeyId { get; set; } + public string? SSECustomerAlgorithm { get; set; } + public string? SSECustomerKey { get; set; } + public string? SSECustomerKeyMD5 { get; set; } + public string? StorageClass { get; set; } + public string? WebsiteRedirectLocation { get; set; } + + // Object Lock + public string? ObjectLockMode { get; set; } + public DateTime? ObjectLockRetainUntilDate { get; set; } + public bool? ObjectLockLegalHoldStatus { get; set; } + + // Tags and custom metadata + public Dictionary? Tags { get; set; } + public Dictionary? UserMetadata { get; set; } + public Dictionary? CustomHeaders { get; set; } + } + + /// + /// Cache control settings + /// + public class CacheControlSettings + { + public bool NoCache { get; set; } + public bool NoStore { get; set; } + public bool MustRevalidate { get; set; } + public bool Public { get; set; } + public bool Private { get; set; } + public TimeSpan? MaxAge { get; set; } + public TimeSpan? SMaxAge { get; set; } + } +} diff --git a/src/Core/S3/BucketPolicyManager.cs b/src/Core/S3/BucketPolicyManager.cs new file mode 100644 index 0000000..ebe293b --- /dev/null +++ b/src/Core/S3/BucketPolicyManager.cs @@ -0,0 +1,309 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using PSMinIO.Core.Http; +using PSMinIO.Utils; + +namespace PSMinIO.Core.S3 +{ + /// + /// Manages bucket policies with comprehensive validation and formatting + /// + public class BucketPolicyManager + { + private readonly MinIOHttpClient _httpClient; + private readonly ThreadSafeProgressCollector _progressCollector; + + public BucketPolicyManager(MinIOHttpClient httpClient, ThreadSafeProgressCollector progressCollector) + { + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + _progressCollector = progressCollector ?? throw new ArgumentNullException(nameof(progressCollector)); + } + + /// + /// Gets the bucket policy + /// + public BucketPolicyResult GetBucketPolicy(string bucketName) + { + if (string.IsNullOrEmpty(bucketName)) throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); + + _progressCollector.QueueVerboseMessage("Getting bucket policy for: {0}", bucketName); + + try + { + var queryParams = new Dictionary { { "policy", "" } }; + var response = _httpClient.ExecuteRequestForString(HttpMethod.Get, $"/{bucketName}", queryParams); + + var policy = JsonSerializer.Deserialize(response, new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + PropertyNameCaseInsensitive = true + }); + + _progressCollector.QueueVerboseMessage("Successfully retrieved bucket policy"); + + return new BucketPolicyResult + { + BucketName = bucketName, + Policy = policy, + PolicyJson = response, + IsValid = ValidatePolicy(policy!), + HasPolicy = true + }; + } + catch (HttpRequestException ex) when (ex.Message.Contains("404") || ex.Message.Contains("NoSuchBucketPolicy")) + { + _progressCollector.QueueVerboseMessage("No policy found for bucket: {0}", bucketName); + return new BucketPolicyResult + { + BucketName = bucketName, + HasPolicy = false, + IsValid = true + }; + } + catch (Exception ex) + { + _progressCollector.QueueVerboseMessage("Failed to get bucket policy: {0}", ex.Message); + return new BucketPolicyResult + { + BucketName = bucketName, + HasPolicy = false, + IsValid = false, + Error = ex.Message + }; + } + } + + /// + /// Sets the bucket policy + /// + public BucketPolicyResult SetBucketPolicy(string bucketName, BucketPolicy policy) + { + if (string.IsNullOrEmpty(bucketName)) throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); + if (policy == null) throw new ArgumentNullException(nameof(policy)); + + _progressCollector.QueueVerboseMessage("Setting bucket policy for: {0}", bucketName); + + // Validate policy before setting + if (!ValidatePolicy(policy)) + { + return new BucketPolicyResult + { + BucketName = bucketName, + Policy = policy, + IsValid = false, + Error = "Policy validation failed" + }; + } + + try + { + var policyJson = JsonSerializer.Serialize(policy, new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true + }); + + var queryParams = new Dictionary { { "policy", "" } }; + using var content = new StringContent(policyJson, Encoding.UTF8, "application/json"); + + _httpClient.ExecuteRequest(HttpMethod.Put, $"/{bucketName}", queryParams, content: content); + + _progressCollector.QueueVerboseMessage("Successfully set bucket policy"); + + return new BucketPolicyResult + { + BucketName = bucketName, + Policy = policy, + PolicyJson = policyJson, + IsValid = true, + HasPolicy = true + }; + } + catch (Exception ex) + { + _progressCollector.QueueVerboseMessage("Failed to set bucket policy: {0}", ex.Message); + return new BucketPolicyResult + { + BucketName = bucketName, + Policy = policy, + IsValid = false, + Error = ex.Message + }; + } + } + + /// + /// Sets bucket policy from JSON string + /// + public BucketPolicyResult SetBucketPolicyFromJson(string bucketName, string policyJson) + { + if (string.IsNullOrEmpty(bucketName)) throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); + if (string.IsNullOrEmpty(policyJson)) throw new ArgumentException("Policy JSON cannot be null or empty", nameof(policyJson)); + + try + { + var policy = JsonSerializer.Deserialize(policyJson, new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + PropertyNameCaseInsensitive = true + }); + + return SetBucketPolicy(bucketName, policy!); + } + catch (JsonException ex) + { + return new BucketPolicyResult + { + BucketName = bucketName, + PolicyJson = policyJson, + IsValid = false, + Error = $"Invalid JSON: {ex.Message}" + }; + } + } + + /// + /// Deletes the bucket policy + /// + public BucketPolicyResult DeleteBucketPolicy(string bucketName) + { + if (string.IsNullOrEmpty(bucketName)) throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); + + _progressCollector.QueueVerboseMessage("Deleting bucket policy for: {0}", bucketName); + + try + { + var queryParams = new Dictionary { { "policy", "" } }; + _httpClient.ExecuteRequest(HttpMethod.Delete, $"/{bucketName}", queryParams); + + _progressCollector.QueueVerboseMessage("Successfully deleted bucket policy"); + + return new BucketPolicyResult + { + BucketName = bucketName, + HasPolicy = false, + IsValid = true + }; + } + catch (Exception ex) + { + _progressCollector.QueueVerboseMessage("Failed to delete bucket policy: {0}", ex.Message); + return new BucketPolicyResult + { + BucketName = bucketName, + IsValid = false, + Error = ex.Message + }; + } + } + + /// + /// Creates a simple read-only policy for public access + /// + public BucketPolicy CreateReadOnlyPolicy(string bucketName, string? objectPrefix = null) + { + var resource = string.IsNullOrEmpty(objectPrefix) + ? $"arn:aws:s3:::{bucketName}/*" + : $"arn:aws:s3:::{bucketName}/{objectPrefix}*"; + + return new BucketPolicy + { + Version = "2012-10-17", + Statement = new List + { + new PolicyStatement + { + Effect = "Allow", + Principal = new { AWS = "*" }, + Action = new List { "s3:GetObject" }, + Resource = new List { resource } + } + } + }; + } + + /// + /// Creates a read-write policy for specific principals + /// + public BucketPolicy CreateReadWritePolicy(string bucketName, List principals, string? objectPrefix = null) + { + var resource = string.IsNullOrEmpty(objectPrefix) + ? $"arn:aws:s3:::{bucketName}/*" + : $"arn:aws:s3:::{bucketName}/{objectPrefix}*"; + + return new BucketPolicy + { + Version = "2012-10-17", + Statement = new List + { + new PolicyStatement + { + Effect = "Allow", + Principal = new { AWS = principals }, + Action = new List { "s3:GetObject", "s3:PutObject", "s3:DeleteObject" }, + Resource = new List { resource } + } + } + }; + } + + /// + /// Validates a bucket policy + /// + private bool ValidatePolicy(BucketPolicy policy) + { + if (policy == null) return false; + if (string.IsNullOrEmpty(policy.Version)) return false; + if (policy.Statement == null || policy.Statement.Count == 0) return false; + + foreach (var statement in policy.Statement) + { + if (string.IsNullOrEmpty(statement.Effect)) return false; + if (statement.Effect != "Allow" && statement.Effect != "Deny") return false; + if (statement.Action == null || statement.Action.Count == 0) return false; + if (statement.Resource == null || statement.Resource.Count == 0) return false; + } + + return true; + } + } + + /// + /// Bucket policy structure + /// + public class BucketPolicy + { + public string Version { get; set; } = "2012-10-17"; + public List Statement { get; set; } = new List(); + } + + /// + /// Policy statement structure + /// + public class PolicyStatement + { + public string Effect { get; set; } = string.Empty; + public object? Principal { get; set; } + public List Action { get; set; } = new List(); + public List Resource { get; set; } = new List(); + public Dictionary? Condition { get; set; } + } + + /// + /// Result of bucket policy operations + /// + public class BucketPolicyResult + { + public string BucketName { get; set; } = string.Empty; + public BucketPolicy? Policy { get; set; } + public string? PolicyJson { get; set; } + public bool HasPolicy { get; set; } + public bool IsValid { get; set; } + public string? Error { get; set; } + public DateTime OperationTime { get; set; } = DateTime.UtcNow; + } +} diff --git a/src/Core/S3/MinIOS3Client.cs b/src/Core/S3/MinIOS3Client.cs index ba68174..aaa3bd5 100644 --- a/src/Core/S3/MinIOS3Client.cs +++ b/src/Core/S3/MinIOS3Client.cs @@ -201,7 +201,7 @@ namespace PSMinIO.Core.S3 }; if (!string.IsNullOrEmpty(prefix)) - queryParams["prefix"] = prefix; + queryParams["prefix"] = prefix!; if (!recursive) queryParams["delimiter"] = "/"; diff --git a/src/Core/S3/MultipartDownloadManager.cs b/src/Core/S3/MultipartDownloadManager.cs new file mode 100644 index 0000000..4aac587 --- /dev/null +++ b/src/Core/S3/MultipartDownloadManager.cs @@ -0,0 +1,284 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using PSMinIO.Core.Http; +using PSMinIO.Utils; + +namespace PSMinIO.Core.S3 +{ + /// + /// Manages multipart downloads with parallel processing and resume capability + /// + public class MultipartDownloadManager + { + private readonly MinIOHttpClient _httpClient; + private readonly ThreadSafeProgressCollector _progressCollector; + private readonly int _maxParallelDownloads; + private readonly long _defaultChunkSize; + + public MultipartDownloadManager(MinIOHttpClient httpClient, ThreadSafeProgressCollector progressCollector, + int maxParallelDownloads = 4, long defaultChunkSize = 32 * 1024 * 1024) // 32MB default + { + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + _progressCollector = progressCollector ?? throw new ArgumentNullException(nameof(progressCollector)); + _maxParallelDownloads = Math.Max(1, Math.Min(maxParallelDownloads, 8)); // Limit to 1-8 + _defaultChunkSize = Math.Max(1024 * 1024, defaultChunkSize); // Minimum 1MB + } + + /// + /// Downloads a file using multipart download with parallel processing + /// + public MultipartDownloadResult DownloadFile(string bucketName, string objectName, FileInfo destinationFile, + long? chunkSize = null, bool resumeIfExists = true) + { + if (string.IsNullOrEmpty(bucketName)) throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); + if (string.IsNullOrEmpty(objectName)) throw new ArgumentException("Object name cannot be null or empty", nameof(objectName)); + if (destinationFile == null) throw new ArgumentNullException(nameof(destinationFile)); + + var effectiveChunkSize = chunkSize ?? _defaultChunkSize; + var startTime = DateTime.UtcNow; + + _progressCollector.QueueVerboseMessage("Starting multipart download: {0}/{1} -> {2}", + bucketName, objectName, destinationFile.FullName); + + try + { + // Get object metadata to determine size + var objectInfo = GetObjectInfo(bucketName, objectName); + var totalSize = objectInfo.Size; + var totalChunks = (int)Math.Ceiling((double)totalSize / effectiveChunkSize); + + _progressCollector.QueueVerboseMessage("Object size: {0}, Chunk size: {1}, Total chunks: {2}", + SizeFormatter.FormatBytes(totalSize), SizeFormatter.FormatBytes(effectiveChunkSize), totalChunks); + + // Check for resume capability + var resumeOffset = 0L; + if (resumeIfExists && destinationFile.Exists) + { + resumeOffset = destinationFile.Length; + if (resumeOffset >= totalSize) + { + _progressCollector.QueueVerboseMessage("File already completely downloaded"); + return new MultipartDownloadResult + { + BucketName = bucketName, + ObjectName = objectName, + DestinationPath = destinationFile.FullName, + TotalSize = totalSize, + DownloadedSize = totalSize, + ChunkSize = effectiveChunkSize, + TotalChunks = totalChunks, + Duration = TimeSpan.Zero, + IsCompleted = true, + WasResumed = true + }; + } + _progressCollector.QueueVerboseMessage("Resuming download from offset: {0}", + SizeFormatter.FormatBytes(resumeOffset)); + } + + // Ensure destination directory exists + if (destinationFile.Directory != null && !destinationFile.Directory.Exists) + { + destinationFile.Directory.Create(); + } + + // Create or open file for writing + using var fileStream = resumeIfExists && destinationFile.Exists + ? new FileStream(destinationFile.FullName, FileMode.Append, FileAccess.Write) + : new FileStream(destinationFile.FullName, FileMode.Create, FileAccess.Write); + + // Calculate chunks to download + var remainingSize = totalSize - resumeOffset; + var startChunk = (int)(resumeOffset / effectiveChunkSize); + var chunksToDownload = (int)Math.Ceiling((double)remainingSize / effectiveChunkSize); + + var downloadedBytes = resumeOffset; + var chunks = new ConcurrentDictionary(); + var downloadTasks = new List(); + var semaphore = new SemaphoreSlim(_maxParallelDownloads, _maxParallelDownloads); + + // Download chunks in parallel + for (int chunkIndex = startChunk; chunkIndex < startChunk + chunksToDownload; chunkIndex++) + { + var chunkNum = chunkIndex; + var chunkOffset = chunkNum * effectiveChunkSize; + var chunkEnd = Math.Min(chunkOffset + effectiveChunkSize - 1, totalSize - 1); + var chunkActualSize = chunkEnd - chunkOffset + 1; + + var downloadTask = Task.Run(() => + { + semaphore.Wait(); + try + { + var chunkData = DownloadChunk(bucketName, objectName, chunkOffset, chunkEnd); + + chunks.TryAdd(chunkNum, new DownloadPartInfo + { + PartNumber = chunkNum, + Offset = chunkOffset, + Size = chunkData.Length, + Data = chunkData + }); + + // Update progress + var currentDownloaded = Interlocked.Add(ref downloadedBytes, chunkData.Length); + var progress = (double)currentDownloaded / totalSize * 100; + var elapsed = DateTime.UtcNow - startTime; + var speed = elapsed.TotalSeconds > 0 ? (currentDownloaded - resumeOffset) / elapsed.TotalSeconds : 0; + + _progressCollector.QueueProgressUpdate(2, "Multipart Download", + $"Part {chunkNum - startChunk + 1}/{chunksToDownload} - {SizeFormatter.FormatBytes(currentDownloaded)}/{SizeFormatter.FormatBytes(totalSize)} at {SizeFormatter.FormatSpeed(speed)}", + (int)progress); + + _progressCollector.QueueVerboseMessage("Downloaded chunk {0}/{1} ({2})", + chunkNum - startChunk + 1, chunksToDownload, SizeFormatter.FormatBytes(chunkData.Length)); + } + finally + { + semaphore.Release(); + } + }); + + downloadTasks.Add(downloadTask); + } + + // Wait for all downloads to complete + Task.WaitAll(downloadTasks.ToArray()); + + // Write parts to file in order + var sortedParts = chunks.Values.OrderBy(c => c.PartNumber).ToList(); + foreach (var part in sortedParts) + { + fileStream.Write(part.Data, 0, part.Data.Length); + } + + var totalDuration = DateTime.UtcNow - startTime; + var averageSpeed = totalDuration.TotalSeconds > 0 ? remainingSize / totalDuration.TotalSeconds : 0; + + _progressCollector.QueueVerboseMessage("Multipart download completed successfully"); + _progressCollector.QueueVerboseMessage("Total time: {0}, Average speed: {1}", + SizeFormatter.FormatDuration(totalDuration), SizeFormatter.FormatSpeed(averageSpeed)); + + _progressCollector.QueueProgressCompletion(2, "Multipart Download"); + + return new MultipartDownloadResult + { + BucketName = bucketName, + ObjectName = objectName, + DestinationPath = destinationFile.FullName, + TotalSize = totalSize, + DownloadedSize = totalSize, + ChunkSize = effectiveChunkSize, + TotalChunks = totalChunks, + CompletedParts = sortedParts, + Duration = totalDuration, + AverageSpeed = averageSpeed, + IsCompleted = true, + WasResumed = resumeOffset > 0 + }; + } + catch (Exception ex) + { + _progressCollector.QueueVerboseMessage("Multipart download failed: {0}", ex.Message); + + var partialDuration = DateTime.UtcNow - startTime; + + return new MultipartDownloadResult + { + BucketName = bucketName, + ObjectName = objectName, + DestinationPath = destinationFile.FullName, + ChunkSize = effectiveChunkSize, + Duration = partialDuration, + IsCompleted = false, + Error = ex.Message + }; + } + } + + /// + /// Gets object information including size + /// + private ObjectInfo GetObjectInfo(string bucketName, string objectName) + { + var response = _httpClient.ExecuteRequest(HttpMethod.Head, $"/{bucketName}/{objectName}"); + + var contentLength = response.Content.Headers.ContentLength ?? 0; + var lastModified = response.Content.Headers.LastModified?.DateTime ?? DateTime.MinValue; + var etag = response.Headers.ETag?.Tag?.Trim('"') ?? ""; + + return new ObjectInfo + { + Name = objectName, + Size = contentLength, + LastModified = lastModified, + ETag = etag + }; + } + + /// + /// Downloads a specific part using range request + /// + private byte[] DownloadChunk(string bucketName, string objectName, long startByte, long endByte) + { + var headers = new Dictionary + { + { "Range", $"bytes={startByte}-{endByte}" } + }; + + var response = _httpClient.ExecuteRequest(HttpMethod.Get, $"/{bucketName}/{objectName}", + headers: headers); + + return response.Content.ReadAsByteArrayAsync().GetAwaiter().GetResult(); + } + } + + /// + /// Information about a downloaded part + /// + public class DownloadPartInfo + { + public int PartNumber { get; set; } + public long Offset { get; set; } + public long Size { get; set; } + public byte[] Data { get; set; } = Array.Empty(); + } + + /// + /// Result of multipart download operation + /// + public class MultipartDownloadResult + { + public string BucketName { get; set; } = string.Empty; + public string ObjectName { get; set; } = string.Empty; + public string DestinationPath { get; set; } = string.Empty; + public long TotalSize { get; set; } + public long DownloadedSize { get; set; } + public long ChunkSize { get; set; } + public int TotalChunks { get; set; } + public List CompletedParts { get; set; } = new List(); + public TimeSpan Duration { get; set; } + public double AverageSpeed { get; set; } + public bool IsCompleted { get; set; } + public bool WasResumed { get; set; } + public string? Error { get; set; } + public double CompletionPercentage => TotalSize > 0 ? (double)DownloadedSize / TotalSize * 100 : 0; + } + + /// + /// Basic object information + /// + public class ObjectInfo + { + public string Name { get; set; } = string.Empty; + public long Size { get; set; } + public DateTime LastModified { get; set; } + public string ETag { get; set; } = string.Empty; + } +} diff --git a/src/Core/S3/MultipartUploadManager.cs b/src/Core/S3/MultipartUploadManager.cs new file mode 100644 index 0000000..5a96ff7 --- /dev/null +++ b/src/Core/S3/MultipartUploadManager.cs @@ -0,0 +1,332 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; +using PSMinIO.Core.Http; +using PSMinIO.Utils; + +namespace PSMinIO.Core.S3 +{ + /// + /// Manages multipart uploads with parallel processing and resume capability + /// + public class MultipartUploadManager + { + private readonly MinIOHttpClient _httpClient; + private readonly ThreadSafeProgressCollector _progressCollector; + private readonly int _maxParallelUploads; + private readonly long _defaultChunkSize; + + public MultipartUploadManager(MinIOHttpClient httpClient, ThreadSafeProgressCollector progressCollector, + int maxParallelUploads = 4, long defaultChunkSize = 64 * 1024 * 1024) // 64MB default + { + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + _progressCollector = progressCollector ?? throw new ArgumentNullException(nameof(progressCollector)); + _maxParallelUploads = Math.Max(1, Math.Min(maxParallelUploads, 10)); // Limit to 1-10 + _defaultChunkSize = Math.Max(5 * 1024 * 1024, defaultChunkSize); // Minimum 5MB for S3 compatibility + } + + /// + /// Uploads a file using multipart upload with parallel processing + /// + public MultipartUploadResult UploadFile(string bucketName, string objectName, FileInfo fileInfo, + Dictionary? metadata = null, long? chunkSize = null, + string? resumeUploadId = null, List? completedParts = null) + { + if (string.IsNullOrEmpty(bucketName)) throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); + if (string.IsNullOrEmpty(objectName)) throw new ArgumentException("Object name cannot be null or empty", nameof(objectName)); + if (fileInfo == null || !fileInfo.Exists) throw new ArgumentException("File must exist", nameof(fileInfo)); + + var effectiveChunkSize = chunkSize ?? _defaultChunkSize; + var totalSize = fileInfo.Length; + var totalParts = (int)Math.Ceiling((double)totalSize / effectiveChunkSize); + + _progressCollector.QueueVerboseMessage("Starting multipart upload: {0} -> {1}/{2}", + fileInfo.Name, bucketName, objectName); + _progressCollector.QueueVerboseMessage("File size: {0}, Chunk size: {1}, Total parts: {2}", + SizeFormatter.FormatBytes(totalSize), SizeFormatter.FormatBytes(effectiveChunkSize), totalParts); + + var uploadId = resumeUploadId; + var parts = new ConcurrentDictionary(); + var operationStartTime = DateTime.UtcNow; + + // Add completed parts if resuming + if (completedParts != null) + { + foreach (var part in completedParts) + { + parts.TryAdd(part.PartNumber, part); + } + _progressCollector.QueueVerboseMessage("Resuming upload with {0} completed parts", completedParts.Count); + } + + try + { + // Initialize multipart upload if not resuming + if (string.IsNullOrEmpty(uploadId)) + { + uploadId = InitiateMultipartUpload(bucketName, objectName, metadata); + _progressCollector.QueueVerboseMessage("Initiated multipart upload with ID: {0}", uploadId); + } + + // Upload parts in parallel + var uploadTasks = new List(); + var semaphore = new SemaphoreSlim(_maxParallelUploads, _maxParallelUploads); + var uploadedBytes = completedParts?.Sum(p => p.Size) ?? 0; + + for (int partNumber = 1; partNumber <= totalParts; partNumber++) + { + // Skip already completed parts + if (parts.ContainsKey(partNumber)) + { + continue; + } + + var partNum = partNumber; + var partOffset = (partNum - 1) * effectiveChunkSize; + var partSize = Math.Min(effectiveChunkSize, totalSize - partOffset); + + var uploadTask = Task.Run(() => + { + semaphore.Wait(); + try + { + var partInfo = UploadPart(bucketName, objectName, uploadId!, fileInfo, + partNum, partOffset, partSize); + + parts.TryAdd(partNum, partInfo); + + // Update progress + var currentUploaded = Interlocked.Add(ref uploadedBytes, partSize); + var progress = (double)currentUploaded / totalSize * 100; + var elapsed = DateTime.UtcNow - operationStartTime; + var speed = elapsed.TotalSeconds > 0 ? currentUploaded / elapsed.TotalSeconds : 0; + + _progressCollector.QueueProgressUpdate(1, "Multipart Upload", + $"Part {partNum}/{totalParts} - {SizeFormatter.FormatBytes(currentUploaded)}/{SizeFormatter.FormatBytes(totalSize)} at {SizeFormatter.FormatSpeed(speed)}", + (int)progress); + + _progressCollector.QueueVerboseMessage("Completed part {0}/{1} ({2})", + partNum, totalParts, SizeFormatter.FormatBytes(partSize)); + } + finally + { + semaphore.Release(); + } + }); + + uploadTasks.Add(uploadTask); + } + + // Wait for all uploads to complete + Task.WaitAll(uploadTasks.ToArray()); + + // Complete multipart upload + var sortedParts = parts.Values.OrderBy(p => p.PartNumber).ToList(); + var result = CompleteMultipartUpload(bucketName, objectName, uploadId!, sortedParts); + + var totalDuration = DateTime.UtcNow - operationStartTime; + var averageSpeed = totalDuration.TotalSeconds > 0 ? totalSize / totalDuration.TotalSeconds : 0; + + _progressCollector.QueueVerboseMessage("Multipart upload completed successfully"); + _progressCollector.QueueVerboseMessage("Total time: {0}, Average speed: {1}", + SizeFormatter.FormatDuration(totalDuration), SizeFormatter.FormatSpeed(averageSpeed)); + + _progressCollector.QueueProgressCompletion(1, "Multipart Upload"); + + return new MultipartUploadResult + { + BucketName = bucketName, + ObjectName = objectName, + UploadId = uploadId!, + ETag = result.ETag, + TotalSize = totalSize, + ChunkSize = effectiveChunkSize, + TotalParts = totalParts, + CompletedParts = sortedParts, + Duration = totalDuration, + AverageSpeed = averageSpeed, + IsCompleted = true + }; + } + catch (Exception ex) + { + _progressCollector.QueueVerboseMessage("Multipart upload failed: {0}", ex.Message); + + // Return partial result for resume capability + var sortedParts = parts.Values.OrderBy(p => p.PartNumber).ToList(); + var partialDuration = DateTime.UtcNow - operationStartTime; + + return new MultipartUploadResult + { + BucketName = bucketName, + ObjectName = objectName, + UploadId = uploadId!, + TotalSize = totalSize, + ChunkSize = effectiveChunkSize, + TotalParts = totalParts, + CompletedParts = sortedParts, + Duration = partialDuration, + IsCompleted = false, + Error = ex.Message + }; + } + } + + /// + /// Initiates a multipart upload + /// + private string InitiateMultipartUpload(string bucketName, string objectName, Dictionary? metadata) + { + var queryParams = new Dictionary { { "uploads", "" } }; + var headers = new Dictionary(); + + // Add metadata as headers + if (metadata != null) + { + foreach (var kvp in metadata) + { + headers[$"x-amz-meta-{kvp.Key}"] = kvp.Value; + } + } + + var response = _httpClient.ExecuteRequestForString(HttpMethod.Post, $"/{bucketName}/{objectName}", + queryParams, headers); + + var doc = XDocument.Parse(response); + var uploadId = doc.Descendants("UploadId").FirstOrDefault()?.Value; + + if (string.IsNullOrEmpty(uploadId)) + { + throw new InvalidOperationException("Failed to initiate multipart upload - no upload ID returned"); + } + + return uploadId; + } + + /// + /// Uploads a single part + /// + private PartInfo UploadPart(string bucketName, string objectName, string uploadId, + FileInfo fileInfo, int partNumber, long offset, long size) + { + var queryParams = new Dictionary + { + { "partNumber", partNumber.ToString() }, + { "uploadId", uploadId } + }; + + using var fileStream = fileInfo.OpenRead(); + fileStream.Seek(offset, SeekOrigin.Begin); + + var buffer = new byte[size]; + var totalRead = 0; + while (totalRead < size) + { + var bytesRead = fileStream.Read(buffer, totalRead, (int)(size - totalRead)); + if (bytesRead == 0) break; + totalRead += bytesRead; + } + + using var content = new ByteArrayContent(buffer, 0, totalRead); + content.Headers.ContentLength = totalRead; + + // Calculate MD5 for integrity + using var md5 = MD5.Create(); + var md5Hash = md5.ComputeHash(buffer, 0, totalRead); + var md5String = Convert.ToBase64String(md5Hash); + content.Headers.Add("Content-MD5", md5String); + + var response = _httpClient.ExecuteRequest(HttpMethod.Put, $"/{bucketName}/{objectName}", + queryParams, content: content); + + var etag = response.Headers.ETag?.Tag?.Trim('"') ?? ""; + + return new PartInfo + { + PartNumber = partNumber, + ETag = etag, + Size = totalRead, + MD5Hash = md5String + }; + } + + /// + /// Completes the multipart upload + /// + private CompleteMultipartUploadResult CompleteMultipartUpload(string bucketName, string objectName, + string uploadId, List parts) + { + var queryParams = new Dictionary { { "uploadId", uploadId } }; + + var xml = new XElement("CompleteMultipartUpload", + parts.Select(p => new XElement("Part", + new XElement("PartNumber", p.PartNumber), + new XElement("ETag", p.ETag) + )) + ); + + var xmlContent = xml.ToString(); + using var content = new StringContent(xmlContent, Encoding.UTF8, "application/xml"); + + var response = _httpClient.ExecuteRequestForString(HttpMethod.Post, $"/{bucketName}/{objectName}", + queryParams, content: content); + + var doc = XDocument.Parse(response); + var etag = doc.Descendants("ETag").FirstOrDefault()?.Value?.Trim('"') ?? ""; + + return new CompleteMultipartUploadResult + { + ETag = etag, + Location = doc.Descendants("Location").FirstOrDefault()?.Value ?? "" + }; + } + } + + /// + /// Information about an uploaded part + /// + public class PartInfo + { + public int PartNumber { get; set; } + public string ETag { get; set; } = string.Empty; + public long Size { get; set; } + public string MD5Hash { get; set; } = string.Empty; + } + + /// + /// Result of multipart upload operation + /// + public class MultipartUploadResult + { + public string BucketName { get; set; } = string.Empty; + public string ObjectName { get; set; } = string.Empty; + public string UploadId { get; set; } = string.Empty; + public string ETag { get; set; } = string.Empty; + public long TotalSize { get; set; } + public long ChunkSize { get; set; } + public int TotalParts { get; set; } + public List CompletedParts { get; set; } = new List(); + public TimeSpan Duration { get; set; } + public double AverageSpeed { get; set; } + public bool IsCompleted { get; set; } + public string? Error { get; set; } + public double CompletionPercentage => TotalParts > 0 ? (double)CompletedParts.Count / TotalParts * 100 : 0; + } + + /// + /// Result of completing multipart upload + /// + public class CompleteMultipartUploadResult + { + public string ETag { get; set; } = string.Empty; + public string Location { get; set; } = string.Empty; + } +} diff --git a/src/Core/S3/PresignedUrlGenerator.cs b/src/Core/S3/PresignedUrlGenerator.cs new file mode 100644 index 0000000..3e28868 --- /dev/null +++ b/src/Core/S3/PresignedUrlGenerator.cs @@ -0,0 +1,229 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text; +using System.Web; + +namespace PSMinIO.Core.S3 +{ + /// + /// Generates presigned URLs for temporary access to MinIO objects + /// + public class PresignedUrlGenerator + { + private readonly string _endpoint; + private readonly string _accessKey; + private readonly string _secretKey; + private readonly string _region; + private readonly bool _useSSL; + + public PresignedUrlGenerator(string endpoint, string accessKey, string secretKey, string region = "us-east-1", bool useSSL = true) + { + _endpoint = endpoint?.TrimEnd('/') ?? throw new ArgumentNullException(nameof(endpoint)); + _accessKey = accessKey ?? throw new ArgumentNullException(nameof(accessKey)); + _secretKey = secretKey ?? throw new ArgumentNullException(nameof(secretKey)); + _region = region ?? "us-east-1"; + _useSSL = useSSL; + } + + /// + /// Generates a presigned URL for GET operations (download) + /// + public PresignedUrlResult GeneratePresignedGetUrl(string bucketName, string objectName, + TimeSpan expiration, Dictionary? additionalHeaders = null) + { + return GeneratePresignedUrl(HttpMethod.Get, bucketName, objectName, expiration, additionalHeaders); + } + + /// + /// Generates a presigned URL for PUT operations (upload) + /// + public PresignedUrlResult GeneratePresignedPutUrl(string bucketName, string objectName, + TimeSpan expiration, Dictionary? additionalHeaders = null) + { + return GeneratePresignedUrl(HttpMethod.Put, bucketName, objectName, expiration, additionalHeaders); + } + + /// + /// Generates a presigned URL for DELETE operations + /// + public PresignedUrlResult GeneratePresignedDeleteUrl(string bucketName, string objectName, + TimeSpan expiration, Dictionary? additionalHeaders = null) + { + return GeneratePresignedUrl(HttpMethod.Delete, bucketName, objectName, expiration, additionalHeaders); + } + + /// + /// Generates a presigned URL for HEAD operations (metadata) + /// + public PresignedUrlResult GeneratePresignedHeadUrl(string bucketName, string objectName, + TimeSpan expiration, Dictionary? additionalHeaders = null) + { + return GeneratePresignedUrl(HttpMethod.Head, bucketName, objectName, expiration, additionalHeaders); + } + + /// + /// Generates a presigned URL with custom HTTP method + /// + public PresignedUrlResult GeneratePresignedUrl(HttpMethod method, string bucketName, string objectName, + TimeSpan expiration, Dictionary? additionalHeaders = null) + { + if (string.IsNullOrEmpty(bucketName)) throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); + if (string.IsNullOrEmpty(objectName)) throw new ArgumentException("Object name cannot be null or empty", nameof(objectName)); + if (expiration.TotalSeconds < 1) throw new ArgumentException("Expiration must be at least 1 second", nameof(expiration)); + if (expiration.TotalDays > 7) throw new ArgumentException("Expiration cannot exceed 7 days", nameof(expiration)); + + var now = DateTime.UtcNow; + var expirationTime = now.Add(expiration); + var expirationUnix = ((DateTimeOffset)expirationTime).ToUnixTimeSeconds(); + + // Build the canonical request + var httpMethod = method.Method.ToUpperInvariant(); + var canonicalUri = $"/{bucketName}/{Uri.EscapeDataString(objectName)}"; + var timestamp = now.ToString("yyyyMMddTHHmmssZ", CultureInfo.InvariantCulture); + var dateStamp = now.ToString("yyyyMMdd", CultureInfo.InvariantCulture); + + // Query parameters for AWS Signature Version 4 + var queryParams = new Dictionary + { + { "X-Amz-Algorithm", "AWS4-HMAC-SHA256" }, + { "X-Amz-Credential", $"{_accessKey}/{dateStamp}/{_region}/s3/aws4_request" }, + { "X-Amz-Date", timestamp }, + { "X-Amz-Expires", expiration.TotalSeconds.ToString("F0", CultureInfo.InvariantCulture) }, + { "X-Amz-SignedHeaders", "host" } + }; + + // Add additional headers to signed headers if provided + var signedHeaders = new List { "host" }; + var canonicalHeaders = $"host:{GetHostFromEndpoint()}\n"; + + if (additionalHeaders != null && additionalHeaders.Count > 0) + { + foreach (var header in additionalHeaders.OrderBy(h => h.Key.ToLowerInvariant())) + { + var headerName = header.Key.ToLowerInvariant(); + signedHeaders.Add(headerName); + canonicalHeaders += $"{headerName}:{header.Value}\n"; + } + queryParams["X-Amz-SignedHeaders"] = string.Join(";", signedHeaders.OrderBy(h => h)); + } + + // Build canonical query string + var canonicalQueryString = string.Join("&", + queryParams.OrderBy(p => p.Key).Select(p => $"{Uri.EscapeDataString(p.Key)}={Uri.EscapeDataString(p.Value)}")); + + // Create canonical request + var canonicalRequest = $"{httpMethod}\n{canonicalUri}\n{canonicalQueryString}\n{canonicalHeaders}\n{string.Join(";", signedHeaders.OrderBy(h => h))}\nUNSIGNED-PAYLOAD"; + + // Create string to sign + var credentialScope = $"{dateStamp}/{_region}/s3/aws4_request"; + var stringToSign = $"AWS4-HMAC-SHA256\n{timestamp}\n{credentialScope}\n{ComputeSHA256Hash(canonicalRequest)}"; + + // Calculate signature + var signature = ComputeSignature(stringToSign, dateStamp); + queryParams["X-Amz-Signature"] = signature; + + // Build final URL + var scheme = _useSSL ? "https" : "http"; + var finalQueryString = string.Join("&", + queryParams.OrderBy(p => p.Key).Select(p => $"{Uri.EscapeDataString(p.Key)}={Uri.EscapeDataString(p.Value)}")); + var presignedUrl = $"{scheme}://{GetHostFromEndpoint()}{canonicalUri}?{finalQueryString}"; + + return new PresignedUrlResult + { + Url = presignedUrl, + Method = httpMethod, + BucketName = bucketName, + ObjectName = objectName, + ExpirationTime = expirationTime, + CreatedTime = now, + Duration = expiration, + AdditionalHeaders = additionalHeaders?.ToDictionary(kvp => kvp.Key, kvp => kvp.Value) + }; + } + + /// + /// Extracts host from endpoint + /// + private string GetHostFromEndpoint() + { + var uri = new Uri(_useSSL ? $"https://{_endpoint}" : $"http://{_endpoint}"); + return uri.Host + (uri.IsDefaultPort ? "" : $":{uri.Port}"); + } + + /// + /// Computes SHA256 hash + /// + private static string ComputeSHA256Hash(string input) + { + using var sha256 = SHA256.Create(); + var bytes = sha256.ComputeHash(Encoding.UTF8.GetBytes(input)); + return BitConverter.ToString(bytes).Replace("-", "").ToLowerInvariant(); + } + + /// + /// Computes AWS Signature Version 4 signature + /// + private string ComputeSignature(string stringToSign, string dateStamp) + { + var kDate = ComputeHMACSHA256($"AWS4{_secretKey}", dateStamp); + var kRegion = ComputeHMACSHA256(kDate, _region); + var kService = ComputeHMACSHA256(kRegion, "s3"); + var kSigning = ComputeHMACSHA256(kService, "aws4_request"); + var signature = ComputeHMACSHA256(kSigning, stringToSign); + + return BitConverter.ToString(signature).Replace("-", "").ToLowerInvariant(); + } + + /// + /// Computes HMAC-SHA256 + /// + private static byte[] ComputeHMACSHA256(string key, string data) + { + return ComputeHMACSHA256(Encoding.UTF8.GetBytes(key), data); + } + + /// + /// Computes HMAC-SHA256 with byte array key + /// + private static byte[] ComputeHMACSHA256(byte[] key, string data) + { + using var hmac = new HMACSHA256(key); + return hmac.ComputeHash(Encoding.UTF8.GetBytes(data)); + } + } + + /// + /// Result of presigned URL generation + /// + public class PresignedUrlResult + { + public string Url { get; set; } = string.Empty; + public string Method { get; set; } = string.Empty; + public string BucketName { get; set; } = string.Empty; + public string ObjectName { get; set; } = string.Empty; + public DateTime ExpirationTime { get; set; } + public DateTime CreatedTime { get; set; } + public TimeSpan Duration { get; set; } + public Dictionary? AdditionalHeaders { get; set; } + public bool IsExpired => DateTime.UtcNow > ExpirationTime; + public TimeSpan TimeUntilExpiration => ExpirationTime > DateTime.UtcNow ? ExpirationTime - DateTime.UtcNow : TimeSpan.Zero; + public string FormattedExpiration => ExpirationTime.ToString("yyyy-MM-dd HH:mm:ss UTC"); + } + + /// + /// Presigned URL request configuration + /// + public class PresignedUrlRequest + { + public string BucketName { get; set; } = string.Empty; + public string ObjectName { get; set; } = string.Empty; + public HttpMethod Method { get; set; } = HttpMethod.Get; + public TimeSpan Expiration { get; set; } = TimeSpan.FromHours(1); + public Dictionary? Headers { get; set; } + public Dictionary? QueryParameters { get; set; } + } +} diff --git a/src/Utils/ZipArchiveBuilder.cs b/src/Utils/ZipArchiveBuilder.cs index 2eb45e7..0132e1d 100644 --- a/src/Utils/ZipArchiveBuilder.cs +++ b/src/Utils/ZipArchiveBuilder.cs @@ -194,7 +194,7 @@ namespace PSMinIO.Utils var fullPath = fileInfo.FullName; if (fullPath.StartsWith(basePath, StringComparison.OrdinalIgnoreCase)) { - var relativePath = fullPath.Substring(basePath.Length).TrimStart('\\', '/'); + var relativePath = fullPath.Substring(basePath!.Length).TrimStart('\\', '/'); return relativePath.Replace('\\', '/'); // Use forward slashes for zip entries }