diff --git a/Module/PSMinIO/PSMinIO.psd1 b/Module/PSMinIO/PSMinIO.psd1 index 6ef24a9..f14394e 100644 --- a/Module/PSMinIO/PSMinIO.psd1 +++ b/Module/PSMinIO/PSMinIO.psd1 @@ -3,7 +3,7 @@ RootModule = 'bin\PSMinIO.dll' # Version number of this module. - ModuleVersion = '2025.07.11.1151' + ModuleVersion = '2025.07.11.1421' # Supported PSEditions CompatiblePSEditions = @('Desktop', 'Core') @@ -45,7 +45,7 @@ # RequiredModules = @() # Assemblies that must be loaded prior to importing this module - RequiredAssemblies = @('bin\PSMinIO.dll', 'bin\Minio.dll') + RequiredAssemblies = @('bin\PSMinIO.dll') # Script files (.ps1) that are run in the caller's environment prior to importing this module. # ScriptsToProcess = @() @@ -67,18 +67,10 @@ 'Connect-MinIO', 'Get-MinIOBucket', 'New-MinIOBucket', - 'Remove-MinIOBucket', 'Test-MinIOBucketExists', 'Get-MinIOObject', 'New-MinIOObject', - 'New-MinIOObjectChunked', - 'New-MinIOFolder', - 'Get-MinIOObjectContent', - 'Get-MinIOObjectContentChunked', - 'Remove-MinIOObject', - 'Get-MinIOBucketPolicy', - 'Set-MinIOBucketPolicy', - 'Get-MinIOStats' + 'Get-MinIOObjectContent' ) # Variables to export from this module @@ -97,7 +89,6 @@ FileList = @( 'PSMinIO.psd1', 'bin\PSMinIO.dll', - 'bin\Minio.dll', 'types\PSMinIO.Types.ps1xml', 'types\PSMinIO.Format.ps1xml' ) @@ -119,7 +110,7 @@ # ReleaseNotes of this module ReleaseNotes = @' -## Version 2025.07.11.1151 - Enhanced Release +## Version 2025.07.11.1421 - Enhanced Release ### Major Features - Complete Get-MinIOObject cmdlet with advanced filtering, sorting, and pagination @@ -170,3 +161,7 @@ This release provides enterprise-grade functionality with professional documenta + + + + diff --git a/Module/PSMinIO/bin/CommunityToolkit.HighPerformance.dll b/Module/PSMinIO/bin/CommunityToolkit.HighPerformance.dll deleted file mode 100644 index a8e240e..0000000 Binary files a/Module/PSMinIO/bin/CommunityToolkit.HighPerformance.dll and /dev/null differ diff --git a/Module/PSMinIO/bin/Microsoft.Bcl.HashCode.dll b/Module/PSMinIO/bin/Microsoft.Bcl.HashCode.dll deleted file mode 100644 index 0de0f25..0000000 Binary files a/Module/PSMinIO/bin/Microsoft.Bcl.HashCode.dll and /dev/null differ diff --git a/Module/PSMinIO/bin/Minio.dll b/Module/PSMinIO/bin/Minio.dll deleted file mode 100644 index 5ae010a..0000000 Binary files a/Module/PSMinIO/bin/Minio.dll and /dev/null differ diff --git a/Module/PSMinIO/bin/Newtonsoft.Json.dll b/Module/PSMinIO/bin/Newtonsoft.Json.dll deleted file mode 100644 index 1ffeabe..0000000 Binary files a/Module/PSMinIO/bin/Newtonsoft.Json.dll and /dev/null differ diff --git a/Module/PSMinIO/bin/PSMinIO.dll b/Module/PSMinIO/bin/PSMinIO.dll index f4fe4f4..5823803 100644 Binary files a/Module/PSMinIO/bin/PSMinIO.dll and b/Module/PSMinIO/bin/PSMinIO.dll differ diff --git a/Module/PSMinIO/bin/PSMinIO.pdb b/Module/PSMinIO/bin/PSMinIO.pdb index 04b5665..ff34238 100644 Binary files a/Module/PSMinIO/bin/PSMinIO.pdb and b/Module/PSMinIO/bin/PSMinIO.pdb differ diff --git a/Module/PSMinIO/bin/PSMinIO.psd1 b/Module/PSMinIO/bin/PSMinIO.psd1 deleted file mode 100644 index 419796d..0000000 --- a/Module/PSMinIO/bin/PSMinIO.psd1 +++ /dev/null @@ -1,171 +0,0 @@ -@{ - # Script module or binary module file associated with this manifest. - RootModule = 'bin\PSMinIO.dll' - - # Version number of this module. - ModuleVersion = '2025.07.11.1151' - - # Supported PSEditions - CompatiblePSEditions = @('Desktop', 'Core') - - # ID used to uniquely identify this module - GUID = 'a1b2c3d4-e5f6-7890-abcd-ef1234567890' - - # Author of this module - Author = 'Grace Solutions' - - # Company or vendor of this module - CompanyName = 'Grace Solutions' - - # Copyright statement for this module - Copyright = '(c) 2025 Grace Solutions. All rights reserved.' - - # Description of the functionality provided by this module - Description = 'A comprehensive PowerShell module for MinIO object storage operations with enterprise-grade features including chunked transfers, advanced object listing, directory management, and performance monitoring. Built on the official Minio .NET SDK.' - - # Minimum version of the PowerShell engine required by this module - PowerShellVersion = '5.1' - - # Name of the PowerShell host required by this module - # PowerShellHostName = '' - - # Minimum version of the PowerShell host required by this module - # PowerShellHostVersion = '' - - # Minimum version of Microsoft .NET Framework required by this module. This prerequisite is valid for the PowerShell Desktop edition only. - DotNetFrameworkVersion = '4.7.2' - - # Minimum version of the common language runtime (CLR) required by this module. This prerequisite is valid for the PowerShell Desktop edition only. - CLRVersion = '4.0' - - # Processor architecture (None, X86, Amd64) required by this module - # ProcessorArchitecture = '' - - # Modules that must be imported into the global environment prior to importing this module - # RequiredModules = @() - - # Assemblies that must be loaded prior to importing this module - RequiredAssemblies = @('bin\PSMinIO.dll', 'bin\Minio.dll') - - # Script files (.ps1) that are run in the caller's environment prior to importing this module. - # ScriptsToProcess = @() - - # Type files (.ps1xml) to be loaded when importing this module - TypesToProcess = @('types\PSMinIO.Types.ps1xml') - - # Format files (.ps1xml) to be loaded when importing this module - FormatsToProcess = @('types\PSMinIO.Format.ps1xml') - - # Modules to import as nested modules of the module specified in RootModule/ModuleToProcess - # NestedModules = @() - - # Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export. - FunctionsToExport = @() - - # Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export. - CmdletsToExport = @( - 'Connect-MinIO', - 'Get-MinIOBucket', - 'New-MinIOBucket', - 'Remove-MinIOBucket', - 'Test-MinIOBucketExists', - 'Get-MinIOObject', - 'New-MinIOObject', - 'New-MinIOObjectChunked', - 'New-MinIOFolder', - 'Get-MinIOObjectContent', - 'Get-MinIOObjectContentChunked', - 'Remove-MinIOObject', - 'Get-MinIOBucketPolicy', - 'Set-MinIOBucketPolicy', - 'Get-MinIOStats' - ) - - # Variables to export from this module - VariablesToExport = @() - - # Aliases to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no aliases to export. - AliasesToExport = @() - - # DSC resources to export from this module - # DscResourcesToExport = @() - - # List of all modules packaged with this module - # ModuleList = @() - - # List of all files packaged with this module - FileList = @( - 'PSMinIO.psd1', - 'bin\PSMinIO.dll', - 'bin\Minio.dll', - 'types\PSMinIO.Types.ps1xml', - 'types\PSMinIO.Format.ps1xml' - ) - - # Private data to pass to the module specified in RootModule/ModuleToProcess. This may also contain a PSData hashtable with additional module metadata used by PowerShell. - PrivateData = @{ - PSData = @{ - # Tags applied to this module. These help with module discovery in online galleries. - Tags = @('MinIO', 'ObjectStorage', 'S3', 'Cloud', 'Storage', 'Bucket', 'Object', 'Enterprise', 'Chunked', 'Performance', 'Monitoring', 'Automation', 'Backup', 'AWS', 'Compatible') - - # A URL to the license for this module. - LicenseUri = 'https://github.com/Grace-Solutions/PSMinIO/blob/main/LICENSE' - - # A URL to the main website for this project. - ProjectUri = 'https://github.com/Grace-Solutions/PSMinIO' - - # A URL to an icon representing this module. - # IconUri = '' - - # ReleaseNotes of this module - ReleaseNotes = @' -## Version 2025.07.11.1151 - Enhanced Release - -### Major Features -- Complete Get-MinIOObject cmdlet with advanced filtering, sorting, and pagination -- Enhanced directory management with automatic nested structure creation -- Advanced chunked operations with configurable chunk sizes and multi-layer progress tracking -- Comprehensive timing and performance metrics for all operations -- Enterprise-grade automation examples and monitoring capabilities - -### Issues Fixed -- Fixed directory creation warnings (now clean verbose logging) -- Implemented missing Get-MinIOObject cmdlet with full functionality -- Resolved threading and progress reporting issues - -### Documentation and Examples -- Updated README.md and comprehensive USAGE.md documentation -- Created comprehensive example scripts in scripts/examples/ directory -- Added enterprise automation patterns and best practices -- Professional logging with no Write-Host usage - -### Technical Improvements -- Thread-safe operations for chunked transfers -- Enhanced error handling and resource management -- Performance optimization with intelligent defaults -- Centralized version management system - -This release provides enterprise-grade functionality with professional documentation and comprehensive examples. -'@ - - # Prerelease string of this module - # Prerelease = '' - - # Flag to indicate whether the module requires explicit user acceptance for install/update/save - # RequireLicenseAcceptance = $false - - # External dependent modules of this module - # ExternalModuleDependencies = @() - } - } - - # HelpInfo URI of this module - # HelpInfoURI = '' - - # Default prefix for commands exported from this module. Override the default prefix using Import-Module -Prefix. - # DefaultCommandPrefix = '' -} - - - - diff --git a/Module/PSMinIO/bin/System.Memory.dll b/Module/PSMinIO/bin/System.Memory.dll deleted file mode 100644 index 1e6aef8..0000000 Binary files a/Module/PSMinIO/bin/System.Memory.dll and /dev/null differ diff --git a/Module/PSMinIO/bin/System.Reactive.dll b/Module/PSMinIO/bin/System.Reactive.dll deleted file mode 100644 index ec69a7f..0000000 Binary files a/Module/PSMinIO/bin/System.Reactive.dll and /dev/null differ diff --git a/Module/PSMinIO/bin/System.Runtime.CompilerServices.Unsafe.dll b/Module/PSMinIO/bin/System.Runtime.CompilerServices.Unsafe.dll deleted file mode 100644 index b17135b..0000000 Binary files a/Module/PSMinIO/bin/System.Runtime.CompilerServices.Unsafe.dll and /dev/null differ diff --git a/Module/PSMinIO/bin/System.Threading.Tasks.Extensions.dll b/Module/PSMinIO/bin/System.Threading.Tasks.Extensions.dll deleted file mode 100644 index dfab234..0000000 Binary files a/Module/PSMinIO/bin/System.Threading.Tasks.Extensions.dll and /dev/null differ diff --git a/PSMinIO.csproj b/PSMinIO.csproj index 38422f3..ca164de 100644 --- a/PSMinIO.csproj +++ b/PSMinIO.csproj @@ -27,13 +27,12 @@ - + + - - PreserveNewest diff --git a/Publish/PSMinIO/PSMinIO.psd1 b/Publish/PSMinIO/PSMinIO.psd1 index 8ba618b..6ef24a9 100644 --- a/Publish/PSMinIO/PSMinIO.psd1 +++ b/Publish/PSMinIO/PSMinIO.psd1 @@ -3,7 +3,7 @@ RootModule = 'bin\PSMinIO.dll' # Version number of this module. - ModuleVersion = '2.0.0' + ModuleVersion = '2025.07.11.1151' # Supported PSEditions CompatiblePSEditions = @('Desktop', 'Core') @@ -119,9 +119,9 @@ # ReleaseNotes of this module ReleaseNotes = @' -## Version 2.0.0 - Major Enhancement Release +## Version 2025.07.11.1151 - Enhanced Release -### Major Features Added +### Major Features - Complete Get-MinIOObject cmdlet with advanced filtering, sorting, and pagination - Enhanced directory management with automatic nested structure creation - Advanced chunked operations with configurable chunk sizes and multi-layer progress tracking @@ -135,7 +135,7 @@ ### Documentation and Examples - Updated README.md and comprehensive USAGE.md documentation -- Created 6 detailed example scripts covering all use cases +- Created comprehensive example scripts in scripts/examples/ directory - Added enterprise automation patterns and best practices - Professional logging with no Write-Host usage @@ -143,7 +143,7 @@ - Thread-safe operations for chunked transfers - Enhanced error handling and resource management - Performance optimization with intelligent defaults -- Repository cleanup and organization +- Centralized version management system This release provides enterprise-grade functionality with professional documentation and comprehensive examples. '@ @@ -165,3 +165,8 @@ This release provides enterprise-grade functionality with professional documenta # Default prefix for commands exported from this module. Override the default prefix using Import-Module -Prefix. # DefaultCommandPrefix = '' } + + + + + diff --git a/scripts/Build.ps1 b/scripts/Build.ps1 index 743b3a0..ab9395c 100644 --- a/scripts/Build.ps1 +++ b/scripts/Build.ps1 @@ -152,7 +152,6 @@ try { $filesToCopy = @( 'PSMinIO.dll', 'PSMinIO.pdb', - 'Minio.dll', 'System.Text.Json.dll' ) diff --git a/scripts/Quick-Build.ps1 b/scripts/Quick-Build.ps1 index 0e384ff..eacf98e 100644 --- a/scripts/Quick-Build.ps1 +++ b/scripts/Quick-Build.ps1 @@ -44,7 +44,7 @@ if (!(Test-Path $moduleDir)) { $filesToCopy = @( @{ Source = "$buildOutput\PSMinIO.dll"; Dest = "$moduleDir\PSMinIO.dll" } @{ Source = "$buildOutput\PSMinIO.pdb"; Dest = "$moduleDir\PSMinIO.pdb" } - @{ Source = "$buildOutput\Minio.dll"; Dest = "$moduleDir\Minio.dll" } + @{ Source = "$buildOutput\System.Text.Json.dll"; Dest = "$moduleDir\System.Text.Json.dll" } ) foreach ($file in $filesToCopy) { diff --git a/scripts/Test-NewImplementation.ps1 b/scripts/Test-NewImplementation.ps1 new file mode 100644 index 0000000..40a3d19 --- /dev/null +++ b/scripts/Test-NewImplementation.ps1 @@ -0,0 +1,183 @@ +#!/usr/bin/env pwsh +<# +.SYNOPSIS + Test script for the new PSMinIO implementation with custom REST API + +.DESCRIPTION + This script demonstrates the new PSMinIO module capabilities including: + - Connection management with custom REST API + - Real progress reporting during transfers + - Performance metrics in result objects + - Enhanced error handling and logging + +.PARAMETER Endpoint + MinIO server endpoint (e.g., https://minio.example.com:9000) + +.PARAMETER AccessKey + MinIO access key + +.PARAMETER SecretKey + MinIO secret key + +.PARAMETER TestBucket + Name of test bucket to create (default: psminiotest) + +.EXAMPLE + .\Test-NewImplementation.ps1 -Endpoint "https://play.min.io" -AccessKey "Q3AM3UQ867SPQQA43P2F" -SecretKey "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$Endpoint, + + [Parameter(Mandatory = $true)] + [string]$AccessKey, + + [Parameter(Mandatory = $true)] + [string]$SecretKey, + + [Parameter()] + [string]$TestBucket = "psminiotest-$(Get-Date -Format 'yyyyMMdd-HHmmss')" +) + +# Import the module +Write-Host "Importing PSMinIO module..." -ForegroundColor Green +Import-Module "$PSScriptRoot\..\Module\PSMinIO\PSMinIO.psd1" -Force + +try { + # Test 1: Connection + Write-Host "`n=== Test 1: Connection ===" -ForegroundColor Yellow + Write-Host "Connecting to MinIO server: $Endpoint" -ForegroundColor Cyan + + $connection = Connect-MinIO -Endpoint $Endpoint -AccessKey $AccessKey -SecretKey $SecretKey -TestConnection -PassThru -Verbose + Write-Host "✅ Connection successful!" -ForegroundColor Green + Write-Host "Connection Status: $($connection.Status)" -ForegroundColor White + + # Test 2: List Buckets + Write-Host "`n=== Test 2: List Buckets ===" -ForegroundColor Yellow + Write-Host "Listing existing buckets..." -ForegroundColor Cyan + + $buckets = Get-MinIOBucket -Verbose + Write-Host "✅ Found $($buckets.Count) buckets" -ForegroundColor Green + + if ($buckets.Count -gt 0) { + Write-Host "First few buckets:" -ForegroundColor White + $buckets | Select-Object -First 3 | Format-Table Name, CreationDate, @{Name="Age"; Expression={(Get-Date) - $_.CreationDate}} + } + + # Test 3: Create Test Bucket + Write-Host "`n=== Test 3: Create Test Bucket ===" -ForegroundColor Yellow + Write-Host "Creating test bucket: $TestBucket" -ForegroundColor Cyan + + $bucketResult = New-MinIOBucket -Name $TestBucket -PassThru -Verbose + Write-Host "✅ Bucket created successfully!" -ForegroundColor Green + Write-Host "Bucket: $($bucketResult.Name)" -ForegroundColor White + + # Test 4: Test Bucket Exists + Write-Host "`n=== Test 4: Test Bucket Exists ===" -ForegroundColor Yellow + Write-Host "Testing if bucket exists..." -ForegroundColor Cyan + + $exists = Test-MinIOBucketExists -Name $TestBucket -Verbose + Write-Host "✅ Bucket exists: $exists" -ForegroundColor Green + + # Test 5: Create Test File + Write-Host "`n=== Test 5: Create and Upload Test File ===" -ForegroundColor Yellow + $testFile = "$env:TEMP\psminiotest-$(Get-Date -Format 'yyyyMMddHHmmss').txt" + $testContent = @" +PSMinIO Test File +================ +Created: $(Get-Date) +Endpoint: $Endpoint +Bucket: $TestBucket + +This is a test file created by the PSMinIO test script. +The new implementation uses a custom REST API client for better PowerShell compatibility. + +Features demonstrated: +- Real progress reporting +- Performance metrics +- Enhanced error handling +- Synchronous operations optimized for PowerShell + +Test data: $('A' * 100) +"@ + + Write-Host "Creating test file: $testFile" -ForegroundColor Cyan + $testContent | Out-File -FilePath $testFile -Encoding UTF8 + $fileInfo = Get-Item $testFile + Write-Host "✅ Test file created: $($fileInfo.Length) bytes" -ForegroundColor Green + + # Test 6: Upload File + Write-Host "`n=== Test 6: Upload File with Progress ===" -ForegroundColor Yellow + Write-Host "Uploading test file..." -ForegroundColor Cyan + + $uploadResult = New-MinIOObject -BucketName $TestBucket -File $fileInfo -PassThru -Verbose + Write-Host "✅ Upload completed!" -ForegroundColor Green + Write-Host "Upload Result:" -ForegroundColor White + $uploadResult | Format-List ObjectName, TotalSizeFormatted, DurationFormatted, AverageSpeedFormatted, Success + + # Test 7: List Objects + Write-Host "`n=== Test 7: List Objects ===" -ForegroundColor Yellow + Write-Host "Listing objects in test bucket..." -ForegroundColor Cyan + + $objects = Get-MinIOObject -BucketName $TestBucket -Verbose + Write-Host "✅ Found $($objects.Count) objects" -ForegroundColor Green + + if ($objects.Count -gt 0) { + Write-Host "Objects in bucket:" -ForegroundColor White + $objects | Format-Table Name, @{Name="Size"; Expression={$_.Size}}, LastModified + } + + # Test 8: Download File + Write-Host "`n=== Test 8: Download File with Progress ===" -ForegroundColor Yellow + $downloadFile = "$env:TEMP\psminiotest-download-$(Get-Date -Format 'yyyyMMddHHmmss').txt" + Write-Host "Downloading file to: $downloadFile" -ForegroundColor Cyan + + $downloadResult = Get-MinIOObjectContent -BucketName $TestBucket -ObjectName $fileInfo.Name -LocalPath $downloadFile -PassThru -Verbose + Write-Host "✅ Download completed!" -ForegroundColor Green + Write-Host "Download Result:" -ForegroundColor White + $downloadResult | Format-List ObjectName, TotalSizeFormatted, DurationFormatted, AverageSpeedFormatted, Success + + # Verify download + $downloadedContent = Get-Content $downloadFile -Raw + $originalContent = Get-Content $testFile -Raw + if ($downloadedContent -eq $originalContent) { + Write-Host "✅ File content verification passed!" -ForegroundColor Green + } else { + Write-Host "❌ File content verification failed!" -ForegroundColor Red + } + + Write-Host "`n=== Test Summary ===" -ForegroundColor Yellow + Write-Host "✅ All tests completed successfully!" -ForegroundColor Green + Write-Host "New PSMinIO implementation is working correctly with:" -ForegroundColor White + Write-Host " - Custom REST API client" -ForegroundColor White + Write-Host " - Real progress reporting" -ForegroundColor White + Write-Host " - Performance metrics" -ForegroundColor White + Write-Host " - Enhanced error handling" -ForegroundColor White + Write-Host " - PowerShell-optimized synchronous operations" -ForegroundColor White + +} catch { + Write-Host "❌ Test failed: $($_.Exception.Message)" -ForegroundColor Red + Write-Host "Stack trace:" -ForegroundColor Red + Write-Host $_.ScriptStackTrace -ForegroundColor Red +} finally { + # Cleanup + Write-Host "`n=== Cleanup ===" -ForegroundColor Yellow + + # Remove test files + if (Test-Path $testFile) { + Remove-Item $testFile -Force + Write-Host "Removed test file: $testFile" -ForegroundColor Gray + } + + if (Test-Path $downloadFile) { + Remove-Item $downloadFile -Force + Write-Host "Removed download file: $downloadFile" -ForegroundColor Gray + } + + # Note: We're not removing the test bucket to avoid issues with the test environment + Write-Host "Note: Test bucket '$TestBucket' was left for manual cleanup" -ForegroundColor Gray + + Write-Host "Cleanup completed." -ForegroundColor Gray +} diff --git a/scripts/Test-Upload-Download.ps1 b/scripts/Test-Upload-Download.ps1 new file mode 100644 index 0000000..d7ce0b6 --- /dev/null +++ b/scripts/Test-Upload-Download.ps1 @@ -0,0 +1,70 @@ +#!/usr/bin/env pwsh +<# +.SYNOPSIS + Quick test of PSMinIO upload and download functionality + +.DESCRIPTION + Tests the new PSMinIO implementation with file upload and download +#> + +# Import the module +Import-Module "$PSScriptRoot\..\Module\PSMinIO\PSMinIO.psd1" -Force + +# Connect to MinIO +Write-Verbose "Connecting to MinIO..." -Verbose +Connect-MinIO -Endpoint "https://api.s3.gracesolution.info" -AccessKey "T34Wg85SAwezUa3sk3m4" -SecretKey "PxEmnbQoQJTJsDocSEV6mSSscDpJMiJCayPv93xe" -Verbose + +# Create test file +$testFile = Join-Path $env:TEMP "psminiotest.txt" +$testContent = @" +PSMinIO Test File +================ +Created: $(Get-Date) +Test data: $('A' * 1000) +"@ + +Write-Verbose "Creating test file: $testFile" -Verbose +$testContent | Out-File -FilePath $testFile -Encoding UTF8 +$fileInfo = Get-Item $testFile +Write-Verbose "Test file created: $($fileInfo.Length) bytes" -Verbose + +# Upload file +$testBucket = "psminiotest-20250711-142415" +Write-Verbose "Uploading file to bucket: $testBucket" -Verbose +$uploadResult = New-MinIOObject -BucketName $testBucket -File $fileInfo -PassThru -Verbose + +Write-Output "Upload Result:" +if ($uploadResult) { + $uploadResult | Format-List ObjectName, TotalSizeFormatted, DurationFormatted, AverageSpeedFormatted, Success +} + +# List objects +Write-Verbose "Listing objects in bucket..." -Verbose +$objects = Get-MinIOObject -BucketName $testBucket -Verbose +$objects | Format-Table Name, Size, LastModified + +# Download file +$downloadFile = Join-Path $env:TEMP "psminiotest-download.txt" +Write-Verbose "Downloading file to: $downloadFile" -Verbose +$downloadResult = Get-MinIOObjectContent -BucketName $testBucket -ObjectName $fileInfo.Name -LocalPath $downloadFile -PassThru -Verbose + +Write-Output "Download Result:" +if ($downloadResult) { + $downloadResult | Format-List ObjectName, TotalSizeFormatted, DurationFormatted, AverageSpeedFormatted, Success +} + +# Verify content +$originalContent = Get-Content $testFile -Raw -ErrorAction SilentlyContinue +$downloadedContent = Get-Content $downloadFile -Raw -ErrorAction SilentlyContinue + +if ($originalContent -and $downloadedContent -and ($originalContent -eq $downloadedContent)) { + Write-Output "✅ File content verification PASSED!" +} else { + Write-Output "❌ File content verification FAILED!" +} + +# Cleanup +Remove-Item $testFile -Force -ErrorAction SilentlyContinue +Remove-Item $downloadFile -Force -ErrorAction SilentlyContinue + +Write-Output "Test completed!" diff --git a/src/Cmdlets/ConnectMinIOCmdlet.cs b/src/Cmdlets/ConnectMinIOCmdlet.cs index cc43e37..3d08706 100644 --- a/src/Cmdlets/ConnectMinIOCmdlet.cs +++ b/src/Cmdlets/ConnectMinIOCmdlet.cs @@ -1,6 +1,6 @@ using System; using System.Management.Automation; -using PSMinIO.Models; +using PSMinIO.Core; using PSMinIO.Utils; namespace PSMinIO.Cmdlets @@ -37,7 +37,7 @@ namespace PSMinIO.Cmdlets public string SecretKey { get; set; } = string.Empty; /// - /// Region for bucket operations (default: us-east-1) + /// Optional region for bucket operations (default: us-east-1) /// [Parameter] [ValidateNotNullOrEmpty] @@ -51,40 +51,65 @@ namespace PSMinIO.Cmdlets public int TimeoutSeconds { get; set; } = 30; /// - /// Test the connection after establishing it + /// Skip SSL certificate validation (for development/self-signed certificates) /// [Parameter] - public SwitchParameter TestConnection { get; set; } + public SwitchParameter SkipCertificateValidation { get; set; } /// - /// Store the connection in a session variable for reuse (default: MinIOConnection) + /// Maximum number of concurrent connections (default: 10) + /// + [Parameter] + [ValidateRange(1, 100)] + public int MaxConnections { get; set; } = 10; + + /// + /// Default chunk size for multipart uploads in MB (default: 5) + /// + [Parameter] + [ValidateRange(1, 1024)] + public int DefaultChunkSizeMB { get; set; } = 5; + + /// + /// Maximum retry attempts for failed operations (default: 3) + /// + [Parameter] + [ValidateRange(0, 10)] + public int MaxRetries { get; set; } = 3; + + /// + /// Name of session variable to store the connection (default: MinIOConnection) /// [Parameter] [ValidateNotNullOrEmpty] public string SessionVariable { get; set; } = "MinIOConnection"; /// - /// Skip SSL certificate validation (use with caution) + /// Test the connection after establishing it /// [Parameter] - public SwitchParameter SkipCertificateValidation { get; set; } + public SwitchParameter TestConnection { get; set; } + + /// + /// Return the connection object instead of storing it in session variable + /// + [Parameter] + public SwitchParameter PassThru { get; set; } /// /// Processes the cmdlet /// protected override void ProcessRecord() { - // Extract connection details from URI - var useSSL = Endpoint.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase); + // Extract endpoint components var endpointHost = $"{Endpoint.Host}:{Endpoint.Port}"; + var useSSL = Endpoint.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase); if (ShouldProcess($"MinIO Server: {Endpoint}", $"Establish connection")) { try { - MinIOLogger.WriteVerbose(this, - "Connecting to MinIO - Endpoint: {0}, UseSSL: {1}, Region: {2}, Timeout: {3}s, SkipCertValidation: {4}", - endpointHost, useSSL, Region, TimeoutSeconds, SkipCertificateValidation.IsPresent); + MinIOLogger.LogConnection(this, endpointHost, useSSL, Region, TimeoutSeconds); // Show warning when certificate validation is skipped if (SkipCertificateValidation.IsPresent) @@ -100,50 +125,53 @@ namespace PSMinIO.Cmdlets useSSL, Region, TimeoutSeconds, - SkipCertificateValidation.IsPresent); + SkipCertificateValidation.IsPresent) + { + MaxConnections = MaxConnections, + DefaultChunkSize = DefaultChunkSizeMB * 1024 * 1024, // Convert MB to bytes + MaxRetries = MaxRetries + }; + + // Validate configuration + if (!configuration.IsValid) + { + var errors = string.Join(", ", configuration.GetValidationErrors()); + throw new ArgumentException($"Invalid configuration: {errors}"); + } // Create connection var connection = new MinIOConnection(configuration); - MinIOLogger.WriteVerbose(this, "MinIO connection created successfully"); - // Test connection if requested if (TestConnection.IsPresent) { - MinIOLogger.WriteVerbose(this, "Testing MinIO connection..."); + WriteVerboseMessage("Testing connection by listing buckets..."); - var testResult = connection.TestConnection(); - - if (testResult.Success) + if (!connection.TestConnection()) { - MinIOLogger.WriteVerbose(this, - "Connection test successful - found {0} buckets in {1}ms", - testResult.BucketCount ?? 0, - testResult.ResponseTime?.TotalMilliseconds ?? 0); + throw new InvalidOperationException("Connection test failed. Please verify your credentials and endpoint."); + } - WriteInformation(new InformationRecord( - $"Connection test successful. Found {testResult.BucketCount ?? 0} buckets.", - "ConnectionTest")); - } - else - { - WriteWarning($"Connection test failed: {testResult.Message}"); - MinIOLogger.WriteVerbose(this, "Connection test failed: {0}", testResult.Message); - } + WriteVerboseMessage("Connection test successful"); } - // Store in session variable if requested - if (!string.IsNullOrWhiteSpace(SessionVariable)) + // Store in session variable unless PassThru is specified + if (!PassThru.IsPresent) { - // Set variable in session state SessionState.PSVariable.Set(SessionVariable, connection); - MinIOLogger.WriteVerbose(this, "Connection stored in session variable: {0}", SessionVariable!); + WriteVerboseMessage("Connection stored in session variable: {0}", SessionVariable); } - // Return the connection object - WriteObject(connection); - - MinIOLogger.WriteVerbose(this, "Connect-MinIO completed successfully"); + // Output connection object if PassThru is specified or if not storing in session + if (PassThru.IsPresent) + { + WriteObject(connection); + } + else + { + WriteVerboseMessage("Connected to MinIO server: {0}", endpointHost); + WriteVerboseMessage("Use Get-MinIOBucket or other MinIO cmdlets to interact with the server"); + } } catch (Exception ex) { @@ -152,35 +180,20 @@ namespace PSMinIO.Cmdlets "ConnectionFailed", ErrorCategory.ConnectionError, Endpoint); - - WriteError(errorRecord); + + ThrowTerminatingError(errorRecord); } } } /// - /// Begins processing - validate parameters + /// Writes verbose output with consistent formatting /// - protected override void BeginProcessing() + /// Message to write + /// Format arguments + private void WriteVerboseMessage(string message, params object[] args) { - base.BeginProcessing(); - - // Validate URI scheme - if (!Endpoint.Scheme.Equals("http", StringComparison.OrdinalIgnoreCase) && - !Endpoint.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase)) - { - ThrowTerminatingError(new ErrorRecord( - new ArgumentException($"Invalid URI scheme '{Endpoint.Scheme}'. Only 'http' and 'https' are supported."), - "InvalidUriScheme", - ErrorCategory.InvalidArgument, - Endpoint)); - } - - // Validate port is specified - if (Endpoint.Port == -1) - { - WriteWarning($"No port specified in URI '{Endpoint}'. MinIO typically uses port 9000."); - } + MinIOLogger.WriteVerbose(this, message, args); } } } diff --git a/src/Cmdlets/GetMinIOBucketCmdlet.cs b/src/Cmdlets/GetMinIOBucketCmdlet.cs index f7764b4..e04cb13 100644 --- a/src/Cmdlets/GetMinIOBucketCmdlet.cs +++ b/src/Cmdlets/GetMinIOBucketCmdlet.cs @@ -1,180 +1,143 @@ using System; using System.Linq; using System.Management.Automation; -using PSMinIO.Models; -using PSMinIO.Utils; +using PSMinIO.Core.Models; namespace PSMinIO.Cmdlets { /// /// Gets information about MinIO buckets /// - [Cmdlet(VerbsCommon.Get, "MinIOBucket", SupportsShouldProcess = true)] + [Cmdlet(VerbsCommon.Get, "MinIOBucket")] [OutputType(typeof(MinIOBucketInfo))] public class GetMinIOBucketCmdlet : MinIOBaseCmdlet { /// - /// Name of a specific bucket to retrieve. If not specified, all buckets are returned. + /// Name of a specific bucket to retrieve (supports wildcards) /// [Parameter(Position = 0, ValueFromPipeline = true, ValueFromPipelineByPropertyName = true)] - [ValidateNotNullOrEmpty] + [SupportsWildcards] [Alias("Bucket")] - public string? BucketName { get; set; } + public string? Name { get; set; } /// - /// Include additional statistics like object count and total size for each bucket + /// Include bucket statistics (object count and total size) /// [Parameter] - [Alias("Stats")] - public SwitchParameter IncludeStatistics { get; set; } + public SwitchParameter IncludeStats { get; set; } + + /// + /// Include bucket policy information + /// + [Parameter] + public SwitchParameter IncludePolicy { get; set; } + + /// + /// Sort buckets by the specified property + /// + [Parameter] + [ValidateSet("Name", "CreationDate", "ObjectCount", "TotalSize")] + public string SortBy { get; set; } = "Name"; + + /// + /// Sort in descending order + /// + [Parameter] + public SwitchParameter Descending { get; set; } /// /// Processes the cmdlet /// protected override void ProcessRecord() { - ValidateConnection(); + ExecuteOperation("ListBuckets", () => + { + WriteVerboseMessage("Retrieving bucket list from MinIO server"); - if (!string.IsNullOrWhiteSpace(BucketName)) - { - // Get specific bucket - GetSpecificBucket(BucketName!); - } - else - { // Get all buckets - GetAllBuckets(); - } - } + var buckets = S3Client.ListBuckets(); - /// - /// Gets information about a specific bucket - /// - /// Name of the bucket - private void GetSpecificBucket(string bucketName) - { - ValidateBucketName(bucketName); + WriteVerboseMessage("Retrieved {0} buckets", buckets.Count); - if (ShouldProcess(bucketName, "Get bucket information")) - { - ExecuteOperation("GetBucket", () => + // Filter by name if specified + if (!string.IsNullOrEmpty(Name)) { - // First check if bucket exists - var exists = Client.BucketExists(bucketName); - if (!exists) - { - WriteError(new ErrorRecord( - new InvalidOperationException($"Bucket '{bucketName}' does not exist"), - "BucketNotFound", - ErrorCategory.ObjectNotFound, - bucketName)); - return; - } + var wildcardPattern = new WildcardPattern(Name, WildcardOptions.IgnoreCase); + buckets = buckets.Where(b => b.Name != null && wildcardPattern.IsMatch(b.Name)).ToList(); + WriteVerboseMessage("Filtered to {0} buckets matching pattern '{1}'", buckets.Count, Name); + } - // Get all buckets and find the specific one - var allBuckets = Client.ListBuckets(); - var bucket = allBuckets.FirstOrDefault(b => - string.Equals(b.Name, bucketName, StringComparison.OrdinalIgnoreCase)); - - if (bucket != null) - { - if (IncludeStatistics.IsPresent) - { - PopulateBucketStatistics(bucket); - } - - WriteObject(bucket); - } - else - { - WriteError(new ErrorRecord( - new InvalidOperationException($"Bucket '{bucketName}' not found in bucket list"), - "BucketNotFound", - ErrorCategory.ObjectNotFound, - bucketName)); - } - }, $"Bucket: {bucketName}"); - } - } - - /// - /// Gets information about all buckets - /// - private void GetAllBuckets() - { - if (ShouldProcess("All buckets", "Get bucket information")) - { - ExecuteOperation("ListBuckets", () => + // Enhance bucket information if requested + if (IncludeStats.IsPresent || IncludePolicy.IsPresent) { - var buckets = Client.ListBuckets(); - - if (IncludeStatistics.IsPresent) + for (int i = 0; i < buckets.Count; i++) { - MinIOLogger.WriteVerbose(this, "Gathering statistics for {0} buckets", buckets.Count); - - for (int i = 0; i < buckets.Count; i++) + var bucket = buckets[i]; + WriteVerboseMessage("Enhancing information for bucket '{0}' ({1}/{2})", bucket.Name, i + 1, buckets.Count); + + try { - var bucket = buckets[i]; - - // Show progress for statistics gathering - var progressRecord = new ProgressRecord(1, "Gathering Bucket Statistics", - $"Processing bucket: {bucket.Name}") + // Get bucket statistics + if (IncludeStats.IsPresent) { - PercentComplete = (int)((double)(i + 1) / buckets.Count * 100) - }; - WriteProgress(progressRecord); + WriteVerboseMessage("Getting statistics for bucket '{0}'", bucket.Name); + var objects = S3Client.ListObjects(bucket.Name, recursive: true, maxObjects: int.MaxValue); + bucket.ObjectCount = objects.Count; + bucket.TotalSize = objects.Sum(o => o.Size); + WriteVerboseMessage("Bucket '{0}' contains {1} objects ({2})", + bucket.Name, bucket.ObjectCount, Utils.SizeFormatter.FormatBytes(bucket.TotalSize ?? 0)); + } - PopulateBucketStatistics(bucket); + // Get bucket policy + if (IncludePolicy.IsPresent) + { + WriteVerboseMessage("Getting policy for bucket '{0}'", bucket.Name); + try + { + bucket.Policy = S3Client.GetBucketPolicy(bucket.Name); + WriteVerboseMessage("Retrieved policy for bucket '{0}'", bucket.Name); + } + catch (Exception ex) + { + WriteVerboseMessage("No policy found for bucket '{0}': {1}", bucket.Name, ex.Message); + bucket.Policy = null; + } + } } - - // Complete progress - WriteProgress(new ProgressRecord(1, "Gathering Bucket Statistics", "Completed") + catch (Exception ex) { - PercentComplete = 100, - RecordType = ProgressRecordType.Completed - }); + WriteWarningMessage("Failed to enhance information for bucket '{0}': {1}", bucket.Name, ex.Message); + } } + } - // Output all buckets - foreach (var bucket in buckets) - { - WriteObject(bucket); - } + // Sort buckets + buckets = SortBy switch + { + "Name" => Descending.IsPresent + ? buckets.OrderByDescending(b => b.Name).ToList() + : buckets.OrderBy(b => b.Name).ToList(), + "CreationDate" => Descending.IsPresent + ? buckets.OrderByDescending(b => b.CreationDate).ToList() + : buckets.OrderBy(b => b.CreationDate).ToList(), + "ObjectCount" => Descending.IsPresent + ? buckets.OrderByDescending(b => b.ObjectCount ?? 0).ToList() + : buckets.OrderBy(b => b.ObjectCount ?? 0).ToList(), + "TotalSize" => Descending.IsPresent + ? buckets.OrderByDescending(b => b.TotalSize ?? 0).ToList() + : buckets.OrderBy(b => b.TotalSize ?? 0).ToList(), + _ => buckets + }; - MinIOLogger.WriteVerbose(this, "Retrieved information for {0} buckets", buckets.Count); - }); - } - } + WriteVerboseMessage("Returning {0} buckets sorted by {1} ({2})", + buckets.Count, SortBy, Descending.IsPresent ? "descending" : "ascending"); - /// - /// Populates additional statistics for a bucket - /// - /// Bucket to populate statistics for - private void PopulateBucketStatistics(MinIOBucketInfo bucket) - { - try - { - MinIOLogger.WriteVerbose(this, "Gathering statistics for bucket: {0}", bucket.Name); - - var objects = Client.ListObjects(bucket.Name, recursive: true); - - bucket.ObjectCount = objects.Count; - bucket.Size = objects.Sum(obj => obj.Size); - - MinIOLogger.WriteVerbose(this, - "Bucket '{0}' statistics: {1} objects, {2} bytes total", - bucket.Name, bucket.ObjectCount, bucket.Size); - } - catch (Exception ex) - { - MinIOLogger.WriteWarning(this, - "Failed to gather statistics for bucket '{0}': {1}", - bucket.Name, ex.Message); - - // Set statistics to null to indicate they couldn't be retrieved - bucket.ObjectCount = null; - bucket.Size = null; - } + // Output results + foreach (var bucket in buckets) + { + WriteObject(bucket); + } + }); } } } diff --git a/src/Cmdlets/GetMinIOBucketPolicyCmdlet.cs b/src/Cmdlets/GetMinIOBucketPolicyCmdlet.cs deleted file mode 100644 index 5e68023..0000000 --- a/src/Cmdlets/GetMinIOBucketPolicyCmdlet.cs +++ /dev/null @@ -1,279 +0,0 @@ -using System; -using System.Management.Automation; -using System.Text.Json; -using PSMinIO.Utils; - -namespace PSMinIO.Cmdlets -{ - /// - /// Gets the policy for a MinIO bucket - /// - [Cmdlet(VerbsCommon.Get, "MinIOBucketPolicy", SupportsShouldProcess = false)] - [OutputType(typeof(string), typeof(PSObject))] - public class GetMinIOBucketPolicyCmdlet : MinIOBaseCmdlet - { - /// - /// Name of the bucket to get the policy for - /// - [Parameter(Position = 0, Mandatory = true, ValueFromPipeline = true, ValueFromPipelineByPropertyName = true)] - [ValidateNotNullOrEmpty] - [Alias("Bucket")] - public string BucketName { get; set; } = string.Empty; - - /// - /// Return the policy as a formatted object instead of raw JSON - /// - [Parameter] - public SwitchParameter AsObject { get; set; } - - /// - /// Pretty-print the JSON output - /// - [Parameter] - public SwitchParameter PrettyPrint { get; set; } - - /// - /// Processes the cmdlet - /// - protected override void ProcessRecord() - { - ValidateConnection(); - ValidateBucketName(BucketName); - - ExecuteOperation("GetBucketPolicy", () => - { - // Check if bucket exists - var bucketExists = Client.BucketExists(BucketName); - if (!bucketExists) - { - WriteError(new ErrorRecord( - new InvalidOperationException($"Bucket '{BucketName}' does not exist"), - "BucketNotFound", - ErrorCategory.ObjectNotFound, - BucketName)); - return; - } - - MinIOLogger.WriteVerbose(this, "Retrieving policy for bucket '{0}'", BucketName); - - try - { - var policyJson = Client.GetBucketPolicy(BucketName); - - if (string.IsNullOrWhiteSpace(policyJson)) - { - MinIOLogger.WriteVerbose(this, "No policy is set for bucket '{0}'", BucketName); - - if (AsObject.IsPresent) - { - var emptyPolicy = new PSObject(); - emptyPolicy.Properties.Add(new PSNoteProperty("BucketName", BucketName)); - emptyPolicy.Properties.Add(new PSNoteProperty("HasPolicy", false)); - emptyPolicy.Properties.Add(new PSNoteProperty("Policy", null)); - emptyPolicy.Properties.Add(new PSNoteProperty("RetrievedAt", DateTime.UtcNow)); - WriteObject(emptyPolicy); - } - else - { - WriteObject(string.Empty); - } - return; - } - - MinIOLogger.WriteVerbose(this, "Retrieved policy for bucket '{0}' ({1} characters)", - BucketName, policyJson.Length); - - if (AsObject.IsPresent) - { - // Parse and return as structured object - var policyObject = CreatePolicyObject(policyJson); - WriteObject(policyObject); - } - else - { - // Return as JSON string - if (PrettyPrint.IsPresent) - { - var formattedJson = FormatJson(policyJson); - WriteObject(formattedJson); - } - else - { - WriteObject(policyJson); - } - } - } - catch (Exception ex) when (ex.Message.Contains("NoSuchBucketPolicy") || - ex.Message.Contains("policy does not exist")) - { - MinIOLogger.WriteVerbose(this, "No policy is set for bucket '{0}'", BucketName); - - if (AsObject.IsPresent) - { - var emptyPolicy = new PSObject(); - emptyPolicy.Properties.Add(new PSNoteProperty("BucketName", BucketName)); - emptyPolicy.Properties.Add(new PSNoteProperty("HasPolicy", false)); - emptyPolicy.Properties.Add(new PSNoteProperty("Policy", null)); - emptyPolicy.Properties.Add(new PSNoteProperty("RetrievedAt", DateTime.UtcNow)); - WriteObject(emptyPolicy); - } - else - { - WriteObject(string.Empty); - } - } - - }, $"Bucket: {BucketName}"); - } - - /// - /// Creates a structured policy object from JSON - /// - /// Policy JSON string - /// PSObject containing policy information - private PSObject CreatePolicyObject(string policyJson) - { - var policyObject = new PSObject(); - policyObject.Properties.Add(new PSNoteProperty("BucketName", BucketName)); - policyObject.Properties.Add(new PSNoteProperty("HasPolicy", true)); - policyObject.Properties.Add(new PSNoteProperty("RetrievedAt", DateTime.UtcNow)); - - try - { - // Parse the JSON to extract key information - using var document = JsonDocument.Parse(policyJson); - var root = document.RootElement; - - // Add raw policy - policyObject.Properties.Add(new PSNoteProperty("PolicyJson", policyJson)); - - // Extract version if present - if (root.TryGetProperty("Version", out var versionElement)) - { - policyObject.Properties.Add(new PSNoteProperty("Version", versionElement.GetString())); - } - - // Extract statements if present - if (root.TryGetProperty("Statement", out var statementsElement)) - { - var statements = new System.Collections.Generic.List(); - - if (statementsElement.ValueKind == JsonValueKind.Array) - { - foreach (var statement in statementsElement.EnumerateArray()) - { - var statementObj = CreateStatementObject(statement); - statements.Add(statementObj); - } - } - else if (statementsElement.ValueKind == JsonValueKind.Object) - { - var statementObj = CreateStatementObject(statementsElement); - statements.Add(statementObj); - } - - policyObject.Properties.Add(new PSNoteProperty("Statements", statements.ToArray())); - policyObject.Properties.Add(new PSNoteProperty("StatementCount", statements.Count)); - } - - // Add formatted JSON - var formattedJson = FormatJson(policyJson); - policyObject.Properties.Add(new PSNoteProperty("FormattedPolicy", formattedJson)); - } - catch (Exception ex) - { - MinIOLogger.WriteWarning(this, "Could not parse policy JSON: {0}", ex.Message); - policyObject.Properties.Add(new PSNoteProperty("PolicyJson", policyJson)); - policyObject.Properties.Add(new PSNoteProperty("ParseError", ex.Message)); - } - - return policyObject; - } - - /// - /// Creates a statement object from a JSON element - /// - /// JSON element representing a statement - /// PSObject containing statement information - private PSObject CreateStatementObject(JsonElement statementElement) - { - var statementObj = new PSObject(); - - try - { - if (statementElement.TryGetProperty("Effect", out var effectElement)) - { - statementObj.Properties.Add(new PSNoteProperty("Effect", effectElement.GetString())); - } - - if (statementElement.TryGetProperty("Principal", out var principalElement)) - { - statementObj.Properties.Add(new PSNoteProperty("Principal", principalElement.ToString())); - } - - if (statementElement.TryGetProperty("Action", out var actionElement)) - { - if (actionElement.ValueKind == JsonValueKind.Array) - { - var actions = new System.Collections.Generic.List(); - foreach (var action in actionElement.EnumerateArray()) - { - actions.Add(action.GetString() ?? string.Empty); - } - statementObj.Properties.Add(new PSNoteProperty("Actions", actions.ToArray())); - } - else - { - statementObj.Properties.Add(new PSNoteProperty("Actions", new[] { actionElement.GetString() ?? string.Empty })); - } - } - - if (statementElement.TryGetProperty("Resource", out var resourceElement)) - { - if (resourceElement.ValueKind == JsonValueKind.Array) - { - var resources = new System.Collections.Generic.List(); - foreach (var resource in resourceElement.EnumerateArray()) - { - resources.Add(resource.GetString() ?? string.Empty); - } - statementObj.Properties.Add(new PSNoteProperty("Resources", resources.ToArray())); - } - else - { - statementObj.Properties.Add(new PSNoteProperty("Resources", new[] { resourceElement.GetString() ?? string.Empty })); - } - } - } - catch (Exception ex) - { - MinIOLogger.WriteWarning(this, "Could not parse statement: {0}", ex.Message); - statementObj.Properties.Add(new PSNoteProperty("ParseError", ex.Message)); - } - - return statementObj; - } - - /// - /// Formats JSON string for pretty printing - /// - /// JSON string to format - /// Formatted JSON string - private string FormatJson(string json) - { - try - { - using var document = JsonDocument.Parse(json); - return JsonSerializer.Serialize(document, new JsonSerializerOptions - { - WriteIndented = true - }); - } - catch - { - // If formatting fails, return original - return json; - } - } - } -} diff --git a/src/Cmdlets/GetMinIOObjectCmdlet.cs b/src/Cmdlets/GetMinIOObjectCmdlet.cs index 5a2090b..961e46f 100644 --- a/src/Cmdlets/GetMinIOObjectCmdlet.cs +++ b/src/Cmdlets/GetMinIOObjectCmdlet.cs @@ -1,175 +1,152 @@ using System; using System.Linq; using System.Management.Automation; -using PSMinIO.Models; -using PSMinIO.Utils; +using PSMinIO.Core.Models; namespace PSMinIO.Cmdlets { /// - /// Gets information about objects in a MinIO bucket + /// Gets objects from MinIO buckets /// - [Cmdlet(VerbsCommon.Get, "MinIOObject", SupportsShouldProcess = true)] + [Cmdlet(VerbsCommon.Get, "MinIOObject")] [OutputType(typeof(MinIOObjectInfo))] public class GetMinIOObjectCmdlet : MinIOBaseCmdlet { /// - /// Name of the bucket to list objects from + /// Name of the bucket /// - [Parameter(Mandatory = true, Position = 0, ValueFromPipelineByPropertyName = true)] + [Parameter(Position = 0, Mandatory = true)] [ValidateNotNullOrEmpty] - [Alias("Bucket")] public string BucketName { get; set; } = string.Empty; /// - /// Optional prefix to filter objects + /// Object name or prefix to filter objects (supports wildcards) /// - [Parameter(Position = 1, ValueFromPipelineByPropertyName = true)] - [Alias("Filter")] - public string? Prefix { get; set; } + [Parameter(Position = 1, ValueFromPipeline = true, ValueFromPipelineByPropertyName = true)] + [SupportsWildcards] + [Alias("Key", "Prefix")] + public string? Name { get; set; } /// - /// Specific object name to retrieve. If specified, only this object is returned. - /// - [Parameter(ValueFromPipelineByPropertyName = true)] - [Alias("Object", "Name")] - public string? ObjectName { get; set; } - - /// - /// Whether to list objects recursively (default: true) + /// List objects recursively (default: true) /// [Parameter] public SwitchParameter Recursive { get; set; } = true; /// - /// Include all versions of objects (for versioned buckets) - /// - [Parameter] - [Alias("Versions")] - public SwitchParameter IncludeVersions { get; set; } - - /// - /// Maximum number of objects to return + /// Maximum number of objects to return (default: 1000) /// [Parameter] [ValidateRange(1, 10000)] - [Alias("Limit")] - public int? MaxObjects { get; set; } - - /// - /// Only return objects (exclude directory markers) - /// - [Parameter] - [Alias("FilesOnly")] - public SwitchParameter ObjectsOnly { get; set; } + public int MaxObjects { get; set; } = 1000; /// /// Sort objects by the specified property /// [Parameter] - [ValidateSet("Name", "Size", "LastModified", "ETag")] - public string? SortBy { get; set; } + [ValidateSet("Name", "Size", "LastModified")] + public string SortBy { get; set; } = "Name"; /// - /// Sort in descending order (default: ascending) + /// Sort in descending order /// [Parameter] - [Alias("Desc")] public SwitchParameter Descending { get; set; } + /// + /// Include only directories (objects ending with /) + /// + [Parameter] + public SwitchParameter DirectoriesOnly { get; set; } + + /// + /// Exclude directories (objects ending with /) + /// + [Parameter] + public SwitchParameter ExcludeDirectories { get; set; } + /// /// Processes the cmdlet /// protected override void ProcessRecord() { - var targetDescription = !string.IsNullOrWhiteSpace(ObjectName) - ? $"object '{ObjectName}'" - : !string.IsNullOrWhiteSpace(Prefix) - ? $"objects with prefix '{Prefix}'" - : "all objects"; + ValidateBucketName(BucketName); - // Determine the prefix to use - var searchPrefix = !string.IsNullOrWhiteSpace(ObjectName) ? ObjectName : Prefix; - - if (ShouldProcess(BucketName, $"List {targetDescription}")) + ExecuteOperation("ListObjects", () => { - ExecuteOperation("ListObjects", () => + WriteVerboseMessage("Listing objects in bucket '{0}'", BucketName); + + // Check if bucket exists + if (!S3Client.BucketExists(BucketName)) { - // Check if bucket exists - var bucketExists = Client.BucketExists(BucketName); - if (!bucketExists) - { - WriteError(new ErrorRecord( - new InvalidOperationException($"Bucket '{BucketName}' does not exist"), - "BucketNotFound", - ErrorCategory.ObjectNotFound, - BucketName)); - return; - } + var errorRecord = new ErrorRecord( + new InvalidOperationException($"Bucket '{BucketName}' does not exist"), + "BucketNotFound", + ErrorCategory.ObjectNotFound, + BucketName); + ThrowTerminatingError(errorRecord); + } - MinIOLogger.WriteVerbose(this, - "Listing objects in bucket '{0}' with prefix '{1}', recursive: {2}", - BucketName, searchPrefix ?? "(none)", Recursive.IsPresent); + // Determine prefix for API call + string? apiPrefix = null; + if (!string.IsNullOrEmpty(Name) && !WildcardPattern.ContainsWildcardCharacters(Name)) + { + // If no wildcards, use as prefix for efficient API filtering + apiPrefix = Name; + } - // Get objects from MinIO - var objects = Client.ListObjects(BucketName, searchPrefix, Recursive.IsPresent, IncludeVersions.IsPresent); + WriteVerboseMessage("Retrieving objects with prefix '{0}', recursive: {1}, max: {2}", + apiPrefix ?? "(none)", Recursive.IsPresent, MaxObjects); - MinIOLogger.WriteVerbose(this, "Found {0} objects", objects.Count); + // Get objects from MinIO + var objects = S3Client.ListObjects(BucketName, apiPrefix, Recursive.IsPresent, MaxObjects); - // Filter for exact object name match if specified - if (!string.IsNullOrWhiteSpace(ObjectName)) - { - objects = objects.Where(obj => - string.Equals(obj.Name, ObjectName, StringComparison.Ordinal)).ToList(); - } + WriteVerboseMessage("Retrieved {0} objects from MinIO", objects.Count); - // Filter out directory markers if ObjectsOnly is specified - if (ObjectsOnly.IsPresent) - { - objects = objects.Where(obj => - !obj.Name.EndsWith("/") && obj.Size > 0).ToList(); - } + // Apply client-side filtering if wildcards are used + if (!string.IsNullOrEmpty(Name) && WildcardPattern.ContainsWildcardCharacters(Name)) + { + var wildcardPattern = new WildcardPattern(Name, WildcardOptions.IgnoreCase); + objects = objects.Where(o => o.Name != null && wildcardPattern.IsMatch(o.Name)).ToList(); + WriteVerboseMessage("Filtered to {0} objects matching pattern '{1}'", objects.Count, Name); + } - // Apply sorting if specified - if (!string.IsNullOrWhiteSpace(SortBy)) - { - objects = SortBy.ToLowerInvariant() switch - { - "name" => Descending.IsPresent - ? objects.OrderByDescending(o => o.Name).ToList() - : objects.OrderBy(o => o.Name).ToList(), - "size" => Descending.IsPresent - ? objects.OrderByDescending(o => o.Size).ToList() - : objects.OrderBy(o => o.Size).ToList(), - "lastmodified" => Descending.IsPresent - ? objects.OrderByDescending(o => o.LastModified).ToList() - : objects.OrderBy(o => o.LastModified).ToList(), - "etag" => Descending.IsPresent - ? objects.OrderByDescending(o => o.ETag).ToList() - : objects.OrderBy(o => o.ETag).ToList(), - _ => objects - }; - } + // Apply directory filters + if (DirectoriesOnly.IsPresent) + { + objects = objects.Where(o => o.IsDirectory).ToList(); + WriteVerboseMessage("Filtered to {0} directories only", objects.Count); + } + else if (ExcludeDirectories.IsPresent) + { + objects = objects.Where(o => !o.IsDirectory).ToList(); + WriteVerboseMessage("Filtered to {0} objects excluding directories", objects.Count); + } - // Apply limit if specified - if (MaxObjects.HasValue && objects.Count > MaxObjects.Value) - { - MinIOLogger.WriteVerbose(this, - "Limiting results to {0} objects", MaxObjects.Value); - objects = objects.Take(MaxObjects.Value).ToList(); - } + // Sort objects + objects = SortBy switch + { + "Name" => Descending.IsPresent + ? objects.OrderByDescending(o => o.Name).ToList() + : objects.OrderBy(o => o.Name).ToList(), + "Size" => Descending.IsPresent + ? objects.OrderByDescending(o => o.Size).ToList() + : objects.OrderBy(o => o.Size).ToList(), + "LastModified" => Descending.IsPresent + ? objects.OrderByDescending(o => o.LastModified).ToList() + : objects.OrderBy(o => o.LastModified).ToList(), + _ => objects + }; - MinIOLogger.WriteVerbose(this, - "Returning {0} objects after filtering and sorting", objects.Count); + WriteVerboseMessage("Returning {0} objects sorted by {1} ({2})", + objects.Count, SortBy, Descending.IsPresent ? "descending" : "ascending"); - // Output objects - foreach (var obj in objects) - { - WriteObject(obj); - } - - }, $"Bucket: {BucketName}, Prefix: {searchPrefix ?? "(none)"}"); - } + // Output results + foreach (var obj in objects) + { + WriteObject(obj); + } + }, $"Bucket: {BucketName}, Name: {Name ?? "(all)"}"); } } } diff --git a/src/Cmdlets/GetMinIOObjectContentChunkedCmdlet.cs b/src/Cmdlets/GetMinIOObjectContentChunkedCmdlet.cs deleted file mode 100644 index cee5a98..0000000 --- a/src/Cmdlets/GetMinIOObjectContentChunkedCmdlet.cs +++ /dev/null @@ -1,348 +0,0 @@ -using System; -using System.IO; -using System.Management.Automation; -using PSMinIO.Models; -using PSMinIO.Utils; - -namespace PSMinIO.Cmdlets -{ - /// - /// Downloads objects from a MinIO bucket using chunked transfer with resume capability - /// - [Cmdlet(VerbsCommon.Get, "MinIOObjectContentChunked", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Low)] - [OutputType(typeof(FileInfo))] - public class GetMinIOObjectContentChunkedCmdlet : MinIOBaseCmdlet - { - /// - /// Name of the bucket to download from - /// - [Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true)] - [ValidateNotNullOrEmpty] - [Alias("Bucket")] - public string BucketName { get; set; } = string.Empty; - - /// - /// Name of the object to download - /// - [Parameter(Position = 1, Mandatory = true, ValueFromPipelineByPropertyName = true)] - [ValidateNotNullOrEmpty] - [Alias("Object", "Key")] - public string ObjectName { get; set; } = string.Empty; - - /// - /// FileInfo object representing where the file should be saved - /// - [Parameter(Position = 2, Mandatory = true)] - [ValidateNotNull] - [Alias("File", "Path")] - public FileInfo FilePath { get; set; } = null!; - - /// - /// Overwrite existing files without prompting - /// - [Parameter] - public SwitchParameter Force { get; set; } - - /// - /// Size of each chunk for download (default: 10MB) - /// - [Parameter] - [ValidateRange(5 * 1024 * 1024, 500 * 1024 * 1024)] // 5MB to 500MB - public long ChunkSize { get; set; } = 10 * 1024 * 1024; // 10MB default - - /// - /// Enable resume functionality for interrupted downloads - /// - [Parameter] - public SwitchParameter Resume { get; set; } - - /// - /// Maximum number of retry attempts for failed chunks - /// - [Parameter] - [ValidateRange(1, 10)] - public int MaxRetries { get; set; } = 3; - - /// - /// Number of parallel chunk downloads (default: 3) - /// - [Parameter] - [ValidateRange(1, 10)] - public int ParallelDownloads { get; set; } = 3; - - /// - /// Custom path for storing resume data (default: %LOCALAPPDATA%\PSMinIO\Resume) - /// - [Parameter] - public string? ResumeDataPath { get; set; } - - - - /// - /// Update progress every N bytes transferred (default: 1MB) - /// - [Parameter] - [ValidateRange(1024, 10 * 1024 * 1024)] // 1KB to 10MB - public long ProgressUpdateInterval { get; set; } = 1024 * 1024; // 1MB - - /// - /// Processes the cmdlet - /// - protected override void ProcessRecord() - { - ValidateConnection(); - ValidateBucketName(BucketName); - ValidateObjectName(ObjectName); - ValidateAndPrepareFilePath(); - - if (ShouldProcess($"{BucketName}/{ObjectName}", $"Download using chunked transfer to '{FilePath.FullName}'")) - { - ExecuteOperation("DownloadObjectChunked", () => - { - // Check if bucket exists - var bucketExists = Client.BucketExists(BucketName); - if (!bucketExists) - { - WriteError(new ErrorRecord( - new InvalidOperationException($"Bucket '{BucketName}' does not exist"), - "BucketNotFound", - ErrorCategory.ObjectNotFound, - BucketName)); - return; - } - - // Get object information - var objectInfo = GetObjectInfo(); - if (objectInfo == null) - { - WriteError(new ErrorRecord( - new InvalidOperationException($"Object '{ObjectName}' not found in bucket '{BucketName}'"), - "ObjectNotFound", - ErrorCategory.ObjectNotFound, - ObjectName)); - return; - } - - MinIOLogger.WriteVerbose(this, - "Starting chunked download of object '{0}' from bucket '{1}' (Size: {2}, ChunkSize: {3})", - ObjectName, BucketName, SizeFormatter.FormatBytes(objectInfo.Size), SizeFormatter.FormatBytes(ChunkSize)); - - // Download using chunked transfer - var downloadResult = DownloadObjectChunked(objectInfo); - - if (downloadResult != null) - { - MinIOLogger.WriteVerbose(this, - "Successfully downloaded object '{0}' from bucket '{1}' to '{2}'", - ObjectName, BucketName, FilePath.FullName); - - // Return download result with timing information - WriteObject(downloadResult); - } - - }, $"Bucket: {BucketName}, Object: {ObjectName}, File: {FilePath.FullName}, ChunkSize: {SizeFormatter.FormatBytes(ChunkSize)}"); - } - } - - /// - /// Gets information about the object to download - /// - /// Object information or null if not found - private MinIOObjectInfo? GetObjectInfo() - { - try - { - var objects = Client.ListObjects(BucketName, ObjectName, false); - return objects.Find(obj => string.Equals(obj.Name, ObjectName, StringComparison.Ordinal)); - } - catch (Exception ex) - { - WriteWarning($"Could not get object information: {ex.Message}"); - return null; - } - } - - /// - /// Downloads an object using chunked transfer with resume capability - /// - /// Information about the object to download - /// Download result with timing information or null if failed - private MinIODownloadResult? DownloadObjectChunked(MinIOObjectInfo objectInfo) - { - ChunkedTransferState? transferState = null; - - // Try to load existing transfer state for resume - if (Resume.IsPresent) - { - transferState = ChunkedTransferResumeManager.LoadTransferState( - BucketName, ObjectName, FilePath.FullName, ChunkedTransferType.Download, ResumeDataPath); - - if (transferState != null && !ChunkedTransferResumeManager.IsResumeDataValid(transferState)) - { - MinIOLogger.WriteVerbose(this, "Resume data for {0} is invalid, starting fresh download", ObjectName); - transferState = null; - } - } - - // Create new transfer state if none exists or resume is disabled - if (transferState == null) - { - transferState = new ChunkedTransferState - { - BucketName = BucketName, - ObjectName = ObjectName, - FilePath = FilePath.FullName, - TotalSize = objectInfo.Size, - ChunkSize = ChunkSize, - ETag = objectInfo.ETag, - TransferType = ChunkedTransferType.Download, - StartTime = DateTime.UtcNow, - LastUpdated = DateTime.UtcNow - }; - } - else - { - MinIOLogger.WriteVerbose(this, "Resuming download of {0} from {1:P1} complete", - ObjectName, transferState.ProgressPercentage / 100); - } - - // Calculate total chunks for progress reporting - var totalChunks = (int)Math.Ceiling((double)objectInfo.Size / ChunkSize); - - // Validate chunk count to prevent memory issues - if (totalChunks > 5000) - { - var recommendedChunkSize = (long)Math.Ceiling((double)objectInfo.Size / 5000); - WriteWarning($"Too many chunks ({totalChunks:N0}) would be created. Consider using a larger chunk size (recommended: {SizeFormatter.FormatBytes(recommendedChunkSize)})"); - ThrowTerminatingError(new ErrorRecord( - new ArgumentException($"Chunk size too small would create {totalChunks:N0} chunks. Maximum allowed is 5,000 chunks."), - "TooManyChunks", - ErrorCategory.InvalidArgument, - ChunkSize)); - } - - // Create thread-safe progress reporter - var progressReporter = new ThreadSafeChunkedProgressReporter( - this, - objectInfo.Size, - totalChunks, - "Downloading"); - - progressReporter.StartNewFile(ObjectName, objectInfo.Size, totalChunks); - - // Track timing for this download - var startTime = DateTime.UtcNow; - - try - { - // Download file using chunked transfer - var result = Client.DownloadFileChunked(transferState, progressReporter, MaxRetries, ParallelDownloads); - - if (result) - { - var completionTime = DateTime.UtcNow; - - // Clean up resume data on successful completion - if (Resume.IsPresent) - { - ChunkedTransferResumeManager.CleanupResumeData(transferState, ResumeDataPath); - } - - progressReporter.CompleteFile(); - - // Process any queued progress updates from the main thread - progressReporter.ProcessQueuedUpdates(); - progressReporter.Complete(); - - // Create download result with timing information - FilePath.Refresh(); - var downloadResult = new MinIODownloadResult(FilePath, BucketName, ObjectName, startTime, completionTime); - return downloadResult; - } - } -#pragma warning disable CS0168 // Variable is declared but never used - false positive, ex is used in error handling - catch (Exception ex) - { - // Save resume data on failure if resume is enabled - if (Resume.IsPresent) - { - try - { - ChunkedTransferResumeManager.SaveTransferState(transferState, ResumeDataPath); - MinIOLogger.WriteVerbose(this, "Saved resume data for {0}", ObjectName); - } - catch (Exception saveEx) - { - WriteWarning($"Could not save resume data for {ObjectName}: {saveEx.Message}"); - } - } - - throw; - } -#pragma warning restore CS0168 - - return null; - } - - /// - /// Validates and prepares the file path for download - /// - private void ValidateAndPrepareFilePath() - { - if (FilePath == null) - { - ThrowTerminatingError(new ErrorRecord( - new ArgumentException("FilePath cannot be null"), - "InvalidFilePath", - ErrorCategory.InvalidArgument, - FilePath)); - } - - // Check if file already exists - if (FilePath!.Exists && !Force.IsPresent) - { - WriteError(new ErrorRecord( - new InvalidOperationException($"File '{FilePath.FullName}' already exists. Use -Force to overwrite."), - "FileAlreadyExists", - ErrorCategory.ResourceExists, - FilePath)); - return; - } - - // Ensure the directory exists - var directory = FilePath.Directory; - if (directory != null && !directory.Exists) - { - try - { - directory.Create(); - MinIOLogger.WriteVerbose(this, "Created directory: {0}", directory.FullName); - } - catch (Exception ex) - { - ThrowTerminatingError(new ErrorRecord( - new InvalidOperationException($"Cannot create directory '{directory.FullName}': {ex.Message}", ex), - "DirectoryCreationFailed", - ErrorCategory.WriteError, - directory)); - } - } - - // Check if we can write to the target location - try - { - var testFile = Path.Combine(FilePath.DirectoryName ?? ".", $".psminiotest_{Guid.NewGuid():N}"); - File.WriteAllText(testFile, "test"); - File.Delete(testFile); - } - catch (Exception ex) - { - ThrowTerminatingError(new ErrorRecord( - new InvalidOperationException($"Cannot write to location '{FilePath.FullName}': {ex.Message}", ex), - "LocationNotWritable", - ErrorCategory.PermissionDenied, - FilePath)); - } - } - } -} diff --git a/src/Cmdlets/GetMinIOObjectContentCmdlet.cs b/src/Cmdlets/GetMinIOObjectContentCmdlet.cs index 41a3a8f..813db28 100644 --- a/src/Cmdlets/GetMinIOObjectContentCmdlet.cs +++ b/src/Cmdlets/GetMinIOObjectContentCmdlet.cs @@ -1,220 +1,205 @@ using System; using System.IO; -using System.Linq; using System.Management.Automation; -using PSMinIO.Models; +using PSMinIO.Core.Models; using PSMinIO.Utils; namespace PSMinIO.Cmdlets { /// - /// Downloads an object from a MinIO bucket to a local file + /// Downloads objects from MinIO /// [Cmdlet(VerbsCommon.Get, "MinIOObjectContent", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Medium)] - [OutputType(typeof(FileInfo))] + [OutputType(typeof(MinIODownloadResult))] public class GetMinIOObjectContentCmdlet : MinIOBaseCmdlet { /// - /// Name of the bucket containing the object + /// Name of the bucket /// - [Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true)] + [Parameter(Position = 0, Mandatory = true)] [ValidateNotNullOrEmpty] - [Alias("Bucket")] public string BucketName { get; set; } = string.Empty; /// /// Name of the object to download /// - [Parameter(Position = 1, Mandatory = true, ValueFromPipelineByPropertyName = true)] + [Parameter(Position = 1, Mandatory = true, ValueFromPipeline = true, ValueFromPipelineByPropertyName = true)] [ValidateNotNullOrEmpty] - [Alias("Object", "Key")] + [Alias("Key", "Name")] public string ObjectName { get; set; } = string.Empty; /// - /// FileInfo object representing where the file should be saved + /// Local file path where the object should be saved /// [Parameter(Position = 2, Mandatory = true)] - [ValidateNotNull] - [Alias("File", "Path")] - public FileInfo FilePath { get; set; } = null!; + [ValidateNotNullOrEmpty] + [Alias("Path", "FilePath")] + public string LocalPath { get; set; } = string.Empty; /// - /// Overwrite the file if it already exists + /// Force overwrite if local file already exists /// [Parameter] public SwitchParameter Force { get; set; } + /// + /// Create directory structure if it doesn't exist + /// + [Parameter] + public SwitchParameter CreateDirectories { get; set; } + /// + /// Return the download result information + /// + [Parameter] + public SwitchParameter PassThru { get; set; } /// /// Processes the cmdlet /// protected override void ProcessRecord() { - ValidateConnection(); ValidateBucketName(BucketName); ValidateObjectName(ObjectName); - ValidateAndPrepareFilePath(); - if (ShouldProcess($"{BucketName}/{ObjectName}", $"Download to '{FilePath}'")) - { - ExecuteOperation("DownloadObject", () => - { - // Check if bucket exists - var bucketExists = Client.BucketExists(BucketName); - if (!bucketExists) - { - WriteError(new ErrorRecord( - new InvalidOperationException($"Bucket '{BucketName}' does not exist"), - "BucketNotFound", - ErrorCategory.ObjectNotFound, - BucketName)); - return; - } - - // Check if object exists and get its information - var objectInfo = GetObjectInfo(); - if (objectInfo == null) - { - WriteError(new ErrorRecord( - new InvalidOperationException($"Object '{ObjectName}' does not exist in bucket '{BucketName}'"), - "ObjectNotFound", - ErrorCategory.ObjectNotFound, - ObjectName)); - return; - } - - MinIOLogger.WriteVerbose(this, - "Downloading object '{0}' from bucket '{1}' ({2} bytes) to '{3}'", - ObjectName, BucketName, objectInfo.Size, FilePath); - - // Create progress reporter - var progressReporter = new ProgressReporter( - this, - "Downloading Object", - $"Downloading {objectInfo.GetFileName()}", - objectInfo.Size, - 1); - - try - { - // Track timing for this download - var startTime = DateTime.UtcNow; - - // Download the file with progress reporting - Client.DownloadFile( - BucketName, - ObjectName, - FilePath.FullName, - bytesTransferred => progressReporter.UpdateProgress(bytesTransferred)); - - var completionTime = DateTime.UtcNow; - - // Complete progress reporting - progressReporter.Complete(); - - MinIOLogger.WriteVerbose(this, - "Successfully downloaded object '{0}' from bucket '{1}' to '{2}'", - ObjectName, BucketName, FilePath.FullName); - - // Refresh file info and create download result with timing information - FilePath.Refresh(); - var downloadResult = new MinIODownloadResult(FilePath, BucketName, ObjectName, startTime, completionTime); - WriteObject(downloadResult); - } -#pragma warning disable CS0168 // Variable is declared but never used - false positive, ex is used in throw - catch (Exception ex) - { - progressReporter.Complete(); - throw; - } -#pragma warning restore CS0168 - - }, $"Bucket: {BucketName}, Object: {ObjectName}, File: {FilePath.FullName}"); - } - } - - /// - /// Validates and prepares the file path for download - /// - private void ValidateAndPrepareFilePath() - { - if (FilePath == null) + // Validate local path + if (string.IsNullOrWhiteSpace(LocalPath)) { ThrowTerminatingError(new ErrorRecord( - new ArgumentException("FilePath cannot be null"), - "InvalidFilePath", + new ArgumentException("Local path cannot be null or empty"), + "InvalidLocalPath", ErrorCategory.InvalidArgument, - FilePath)); + LocalPath)); } // Check if file already exists - if (FilePath!.Exists && !Force.IsPresent) - { - WriteError(new ErrorRecord( - new InvalidOperationException($"File '{FilePath.FullName}' already exists. Use -Force to overwrite."), - "FileAlreadyExists", - ErrorCategory.ResourceExists, - FilePath)); - return; - } - - // Ensure the directory exists - var directory = FilePath.Directory; - if (directory != null && !directory.Exists) - { - try - { - directory.Create(); - MinIOLogger.WriteVerbose(this, "Created directory: {0}", directory.FullName); - } - catch (Exception ex) - { - ThrowTerminatingError(new ErrorRecord( - new InvalidOperationException($"Cannot create directory '{directory.FullName}': {ex.Message}", ex), - "DirectoryCreationFailed", - ErrorCategory.WriteError, - directory)); - } - } - - // Check if we can write to the target location - try - { - var testFile = Path.Combine(FilePath.DirectoryName ?? ".", $".psminiotest_{Guid.NewGuid():N}"); - using (var testStream = File.Create(testFile)) - { - testStream.WriteByte(0); - } - File.Delete(testFile); - } - catch (Exception ex) + if (File.Exists(LocalPath) && !Force.IsPresent) { ThrowTerminatingError(new ErrorRecord( - new InvalidOperationException($"Cannot write to location '{FilePath.FullName}': {ex.Message}", ex), - "LocationNotWritable", - ErrorCategory.PermissionDenied, - FilePath)); + new InvalidOperationException($"File already exists: {LocalPath}. Use -Force to overwrite."), + "FileAlreadyExists", + ErrorCategory.ResourceExists, + LocalPath)); } - } - /// - /// Gets information about the object to be downloaded - /// - /// Object information or null if not found - private Models.MinIOObjectInfo? GetObjectInfo() - { - try + if (ShouldProcess($"{BucketName}/{ObjectName}", "Download object")) { - var objects = Client.ListObjects(BucketName, ObjectName, false); - return objects.FirstOrDefault(obj => - string.Equals(obj.Name, ObjectName, StringComparison.Ordinal)); - } - catch (Exception ex) - { - MinIOLogger.WriteWarning(this, - "Could not retrieve object information for '{0}': {1}", ObjectName, ex.Message); - return null; + var result = ExecuteOperation("DownloadObject", () => + { + WriteVerboseMessage("Downloading object '{0}' from bucket '{1}' to '{2}'", + ObjectName, BucketName, LocalPath); + + // Create download result + var downloadResult = new MinIODownloadResult(BucketName, ObjectName, 0) + { + TargetFilePath = LocalPath + }; + downloadResult.MarkStarted(); + + try + { + // Check if bucket exists + if (!S3Client.BucketExists(BucketName)) + { + throw new InvalidOperationException($"Bucket '{BucketName}' does not exist"); + } + + // Create directory if needed + var directory = Path.GetDirectoryName(LocalPath); + if (!string.IsNullOrEmpty(directory) && !Directory.Exists(directory)) + { + if (CreateDirectories.IsPresent) + { + WriteVerboseMessage("Creating directory: {0}", directory); + Directory.CreateDirectory(directory); + } + else + { + throw new DirectoryNotFoundException($"Directory does not exist: {directory}. Use -CreateDirectories to create it."); + } + } + + // Track progress + long lastReportedBytes = 0; + var startTime = DateTime.UtcNow; + + // Download the object + using (var fileStream = new FileStream(LocalPath, FileMode.Create, FileAccess.Write)) + { + var bytesDownloaded = S3Client.GetObject( + BucketName, + ObjectName, + fileStream, + bytesTransferred => + { + downloadResult.BytesTransferred = bytesTransferred; + + // Report progress every 1MB or at completion + if (bytesTransferred - lastReportedBytes >= 1024 * 1024 || + (downloadResult.TotalSize > 0 && bytesTransferred >= downloadResult.TotalSize)) + { + var elapsed = DateTime.UtcNow - startTime; + var speed = elapsed.TotalSeconds > 0 ? bytesTransferred / elapsed.TotalSeconds : 0; + var percentage = downloadResult.TotalSize > 0 ? + (double)bytesTransferred / downloadResult.TotalSize * 100 : 0; + + WriteVerboseMessage("Download progress: {0:F1}% ({1}/{2}) at {3}", + percentage, + SizeFormatter.FormatBytes(bytesTransferred), + downloadResult.TotalSize > 0 ? SizeFormatter.FormatBytes(downloadResult.TotalSize) : "Unknown", + SizeFormatter.FormatSpeed(speed)); + + lastReportedBytes = bytesTransferred; + } + }); + + downloadResult.TotalSize = bytesDownloaded; + downloadResult.BytesTransferred = bytesDownloaded; + } + + downloadResult.MarkCompleted(); + + var duration = downloadResult.Duration ?? TimeSpan.Zero; + var averageSpeed = downloadResult.AverageSpeed ?? 0; + + WriteVerboseMessage("Download completed: {0} in {1} at average speed of {2}", + SizeFormatter.FormatBytes(downloadResult.BytesTransferred), + SizeFormatter.FormatDuration(duration), + SizeFormatter.FormatSpeed(averageSpeed)); + + // Set file timestamps if available + try + { + var fileInfo = new FileInfo(LocalPath); + if (downloadResult.LastModified.HasValue) + { + fileInfo.LastWriteTimeUtc = downloadResult.LastModified.Value; + } + } + catch (Exception ex) + { + WriteVerboseMessage("Could not set file timestamps: {0}", ex.Message); + } + + return downloadResult; + } + catch (Exception ex) + { + downloadResult.MarkFailed(ex.Message); + throw; + } + }, $"Bucket: {BucketName}, Object: {ObjectName}, LocalPath: {LocalPath}"); + + // Return result if requested + if (PassThru.IsPresent) + { + WriteObject(result); + } + else + { + WriteVerboseMessage("Successfully downloaded object '{0}' from bucket '{1}' to '{2}'", + ObjectName, BucketName, LocalPath); + } } } } diff --git a/src/Cmdlets/GetMinIOStatsCmdlet.cs b/src/Cmdlets/GetMinIOStatsCmdlet.cs deleted file mode 100644 index 575207f..0000000 --- a/src/Cmdlets/GetMinIOStatsCmdlet.cs +++ /dev/null @@ -1,269 +0,0 @@ -using System; -using System.Linq; -using System.Management.Automation; -using PSMinIO.Models; -using PSMinIO.Utils; - -namespace PSMinIO.Cmdlets -{ - /// - /// Gets statistical information about MinIO storage - /// - [Cmdlet(VerbsCommon.Get, "MinIOStats", SupportsShouldProcess = false)] - [OutputType(typeof(MinIOStats))] - public class GetMinIOStatsCmdlet : MinIOBaseCmdlet - { - /// - /// Include detailed per-bucket statistics - /// - [Parameter] - public SwitchParameter IncludeBucketDetails { get; set; } - - /// - /// Include object count and size calculations (may be slow for large buckets) - /// - [Parameter] - public SwitchParameter IncludeObjectCounts { get; set; } - - /// - /// Maximum number of objects to count per bucket (default: unlimited) - /// - [Parameter] - [ValidateRange(1, int.MaxValue)] - public int? MaxObjectsToCount { get; set; } - - /// - /// Processes the cmdlet - /// - protected override void ProcessRecord() - { - ValidateConnection(); - - ExecuteOperation("GetStats", () => - { - MinIOLogger.WriteVerbose(this, "Gathering MinIO statistics..."); - - var config = Configuration; - var stats = new MinIOStats - { - Endpoint = config.Endpoint, - UseSSL = config.UseSSL, - ConnectionStatus = "Connected" - }; - - try - { - // Get basic bucket information - var buckets = Client.ListBuckets(); - stats.TotalBuckets = buckets.Count; - - MinIOLogger.WriteVerbose(this, "Found {0} buckets", buckets.Count); - - if (IncludeObjectCounts.IsPresent && buckets.Count > 0) - { - GatherObjectStatistics(buckets, stats); - } - - // Create detailed output if requested - if (IncludeBucketDetails.IsPresent) - { - var detailedStats = CreateDetailedStats(stats, buckets); - WriteObject(detailedStats); - } - else - { - WriteObject(stats); - } - - MinIOLogger.WriteVerbose(this, "Statistics gathering completed"); - } - catch (Exception ex) - { - stats.ConnectionStatus = $"Error: {ex.Message}"; - WriteObject(stats); - throw; - } - - }, "Gathering statistics"); - } - - /// - /// Gathers object statistics across all buckets - /// - /// List of buckets - /// Stats object to update - private void GatherObjectStatistics(System.Collections.Generic.List buckets, MinIOStats stats) - { - MinIOLogger.WriteVerbose(this, "Gathering object statistics for {0} buckets...", buckets.Count); - - long totalObjects = 0; - long totalSize = 0; - - for (int i = 0; i < buckets.Count; i++) - { - var bucket = buckets[i]; - - // Show progress - var progressRecord = new ProgressRecord(1, "Gathering Statistics", - $"Processing bucket: {bucket.Name}") - { - PercentComplete = (int)((double)(i + 1) / buckets.Count * 100) - }; - WriteProgress(progressRecord); - - try - { - MinIOLogger.WriteVerbose(this, "Processing bucket: {0}", bucket.Name); - - var objects = Client.ListObjects(bucket.Name, recursive: true); - - // Apply limit if specified - if (MaxObjectsToCount.HasValue && objects.Count > MaxObjectsToCount.Value) - { - MinIOLogger.WriteVerbose(this, - "Limiting object count for bucket '{0}' to {1} objects", - bucket.Name, MaxObjectsToCount.Value); - objects = objects.Take(MaxObjectsToCount.Value).ToList(); - } - - var bucketObjectCount = objects.Count; - var bucketSize = objects.Sum(obj => obj.Size); - - totalObjects += bucketObjectCount; - totalSize += bucketSize; - - MinIOLogger.WriteVerbose(this, - "Bucket '{0}': {1} objects, {2} bytes", - bucket.Name, bucketObjectCount, bucketSize); - } - catch (Exception ex) - { - MinIOLogger.WriteWarning(this, - "Failed to get statistics for bucket '{0}': {1}", - bucket.Name, ex.Message); - } - } - - // Complete progress - WriteProgress(new ProgressRecord(1, "Gathering Statistics", "Completed") - { - PercentComplete = 100, - RecordType = ProgressRecordType.Completed - }); - - stats.TotalObjects = totalObjects; - stats.TotalSize = totalSize; - - MinIOLogger.WriteVerbose(this, - "Total statistics: {0} objects, {1} bytes across {2} buckets", - totalObjects, totalSize, buckets.Count); - } - - /// - /// Creates detailed statistics with per-bucket information - /// - /// Base statistics - /// List of buckets - /// PSObject with detailed statistics - private PSObject CreateDetailedStats(MinIOStats stats, System.Collections.Generic.List buckets) - { - var detailedStats = new PSObject(); - - // Add basic statistics - detailedStats.Properties.Add(new PSNoteProperty("TotalBuckets", stats.TotalBuckets)); - detailedStats.Properties.Add(new PSNoteProperty("TotalObjects", stats.TotalObjects)); - detailedStats.Properties.Add(new PSNoteProperty("TotalSize", stats.TotalSize)); - detailedStats.Properties.Add(new PSNoteProperty("TotalSizeFormatted", FormatBytes(stats.TotalSize))); - detailedStats.Properties.Add(new PSNoteProperty("LastUpdated", stats.LastUpdated)); - detailedStats.Properties.Add(new PSNoteProperty("Endpoint", stats.Endpoint)); - detailedStats.Properties.Add(new PSNoteProperty("UseSSL", stats.UseSSL)); - detailedStats.Properties.Add(new PSNoteProperty("ConnectionStatus", stats.ConnectionStatus)); - - // Add calculated statistics - detailedStats.Properties.Add(new PSNoteProperty("AverageObjectSize", stats.AverageObjectSize)); - detailedStats.Properties.Add(new PSNoteProperty("AverageObjectSizeFormatted", FormatBytes((long)stats.AverageObjectSize))); - detailedStats.Properties.Add(new PSNoteProperty("AverageObjectsPerBucket", stats.AverageObjectsPerBucket)); - - // Add bucket details if object counts were gathered - if (IncludeObjectCounts.IsPresent) - { - var bucketDetails = new System.Collections.Generic.List(); - - foreach (var bucket in buckets) - { - try - { - var objects = Client.ListObjects(bucket.Name, recursive: true); - - // Apply limit if specified - if (MaxObjectsToCount.HasValue && objects.Count > MaxObjectsToCount.Value) - { - objects = objects.Take(MaxObjectsToCount.Value).ToList(); - } - - var bucketDetail = new PSObject(); - bucketDetail.Properties.Add(new PSNoteProperty("Name", bucket.Name)); - bucketDetail.Properties.Add(new PSNoteProperty("Created", bucket.Created)); - bucketDetail.Properties.Add(new PSNoteProperty("ObjectCount", objects.Count)); - - var bucketSize = objects.Sum(obj => obj.Size); - bucketDetail.Properties.Add(new PSNoteProperty("Size", bucketSize)); - bucketDetail.Properties.Add(new PSNoteProperty("SizeFormatted", FormatBytes(bucketSize))); - - var avgObjectSize = objects.Count > 0 ? (double)bucketSize / objects.Count : 0; - bucketDetail.Properties.Add(new PSNoteProperty("AverageObjectSize", avgObjectSize)); - bucketDetail.Properties.Add(new PSNoteProperty("AverageObjectSizeFormatted", FormatBytes((long)avgObjectSize))); - - bucketDetails.Add(bucketDetail); - } - catch (Exception ex) - { - var bucketDetail = new PSObject(); - bucketDetail.Properties.Add(new PSNoteProperty("Name", bucket.Name)); - bucketDetail.Properties.Add(new PSNoteProperty("Created", bucket.Created)); - bucketDetail.Properties.Add(new PSNoteProperty("Error", ex.Message)); - bucketDetails.Add(bucketDetail); - } - } - - detailedStats.Properties.Add(new PSNoteProperty("BucketDetails", bucketDetails.ToArray())); - } - else - { - // Just add basic bucket information - var bucketSummary = buckets.Select(b => new PSObject(new - { - Name = b.Name, - Created = b.Created, - Region = b.Region - })).ToArray(); - - detailedStats.Properties.Add(new PSNoteProperty("Buckets", bucketSummary)); - } - - return detailedStats; - } - - /// - /// Formats bytes into a human-readable string - /// - /// Number of bytes - /// Formatted string - private static string FormatBytes(long bytes) - { - if (bytes == 0) return "0 B"; - - string[] sizes = { "B", "KB", "MB", "GB", "TB", "PB" }; - int order = 0; - double size = bytes; - - while (size >= 1024 && order < sizes.Length - 1) - { - order++; - size = size / 1024; - } - - return $"{size:0.##} {sizes[order]}"; - } - } -} diff --git a/src/Utils/MinIOBaseCmdlet.cs b/src/Cmdlets/MinIOBaseCmdlet.cs similarity index 63% rename from src/Utils/MinIOBaseCmdlet.cs rename to src/Cmdlets/MinIOBaseCmdlet.cs index ab8b276..ee6a33c 100644 --- a/src/Utils/MinIOBaseCmdlet.cs +++ b/src/Cmdlets/MinIOBaseCmdlet.cs @@ -1,9 +1,10 @@ using System; -using System.IO; using System.Management.Automation; -using PSMinIO.Models; +using PSMinIO.Core; +using PSMinIO.Core.S3; +using PSMinIO.Utils; -namespace PSMinIO.Utils +namespace PSMinIO.Cmdlets { /// /// Base class for all MinIO PowerShell cmdlets @@ -54,74 +55,54 @@ namespace PSMinIO.Utils ErrorCategory.ConnectionError, _connection)); } - - MinIOLogger.LogOperationStart(this, "UsingConnection", $"Endpoint: {_connection.Configuration.Endpoint}"); } - return _connection; + + return _connection!; } } /// - /// Gets the MinIO client wrapper instance + /// Gets the MinIO S3 client instance /// - protected MinIOClientWrapper Client => Connection.Client; + protected MinIOS3Client S3Client => Connection.S3Client; /// - /// Gets the current MinIO configuration - /// - protected MinIOConfiguration Configuration => Connection.Configuration; - - /// - /// Gets the MinIO connection from parameter or session variable + /// Gets the connection from parameter or session variable /// /// MinIO connection or null if not found private MinIOConnection? GetConnection() { - // First, check if connection was provided via parameter + // First, try the parameter if (MinIOConnection != null) { - MinIOLogger.WriteVerbose(this, "Using MinIO connection from parameter"); return MinIOConnection; } - // Next, check session variable + // Then try the session variable try { - var sessionConnection = SessionState.PSVariable.GetValue(SessionVariable) as MinIOConnection; - if (sessionConnection != null) + var sessionConnection = SessionState.PSVariable.GetValue(SessionVariable); + if (sessionConnection is MinIOConnection connection) { - MinIOLogger.WriteVerbose(this, "Using MinIO connection from session variable: {0}", SessionVariable); - return sessionConnection; + return connection; } } - catch (Exception ex) + catch { - MinIOLogger.WriteVerbose(this, "Failed to retrieve connection from session variable '{0}': {1}", SessionVariable, ex.Message); + // Ignore errors when accessing session variables } - MinIOLogger.WriteVerbose(this, "No MinIO connection found in parameter or session variable '{0}'", SessionVariable); return null; } /// - /// Validates that the MinIO connection is available and valid - /// - protected void ValidateConnection() - { - var connection = Connection; // This will throw if invalid - } - - /// - /// Executes an operation with proper error handling and logging + /// Executes an operation with consistent error handling and logging /// /// Name of the operation for logging - /// The operation to execute + /// Operation to execute /// Optional operation details for logging protected void ExecuteOperation(string operationName, Action operation, string? details = null) { - if (operation == null) - throw new ArgumentNullException(nameof(operationName)); - var startTime = DateTime.UtcNow; MinIOLogger.LogOperationStart(this, operationName, details); @@ -138,23 +119,20 @@ namespace PSMinIO.Utils // Determine appropriate error category var category = GetErrorCategory(ex); - WriteError(new ErrorRecord(ex, $"{operationName}Failed", category, null)); + ThrowTerminatingError(new ErrorRecord(ex, $"{operationName}Failed", category, null)); } } /// - /// Executes an operation with return value and proper error handling and logging + /// Executes an operation with return value and consistent error handling and logging /// /// Return type /// Name of the operation for logging - /// The operation to execute + /// Operation to execute /// Optional operation details for logging - /// Result of the operation + /// Operation result protected T ExecuteOperation(string operationName, Func operation, string? details = null) { - if (operation == null) - throw new ArgumentNullException(nameof(operation)); - var startTime = DateTime.UtcNow; MinIOLogger.LogOperationStart(this, operationName, details); @@ -180,30 +158,29 @@ namespace PSMinIO.Utils } /// - /// Determines the appropriate PowerShell error category based on the exception type + /// Determines the appropriate PowerShell error category for an exception /// - /// The exception to categorize - /// Appropriate ErrorCategory - protected virtual ErrorCategory GetErrorCategory(Exception exception) + /// Exception to categorize + /// Appropriate error category + protected ErrorCategory GetErrorCategory(Exception exception) { return exception switch { - ArgumentNullException => ErrorCategory.InvalidArgument, - ArgumentException => ErrorCategory.InvalidArgument, + ArgumentException or ArgumentNullException => ErrorCategory.InvalidArgument, UnauthorizedAccessException => ErrorCategory.PermissionDenied, - System.Net.WebException => ErrorCategory.ConnectionError, System.Net.Http.HttpRequestException => ErrorCategory.ConnectionError, TimeoutException => ErrorCategory.OperationTimeout, InvalidOperationException => ErrorCategory.InvalidOperation, + System.IO.FileNotFoundException => ErrorCategory.ObjectNotFound, + System.IO.DirectoryNotFoundException => ErrorCategory.ObjectNotFound, + System.IO.IOException => ErrorCategory.WriteError, NotSupportedException => ErrorCategory.NotImplemented, - FileNotFoundException => ErrorCategory.ObjectNotFound, - DirectoryNotFoundException => ErrorCategory.ObjectNotFound, _ => ErrorCategory.NotSpecified }; } /// - /// Validates a bucket name according to MinIO/S3 naming rules + /// Validates that a bucket name is valid according to S3 naming rules /// /// Bucket name to validate /// Parameter name for error reporting @@ -212,17 +189,26 @@ namespace PSMinIO.Utils if (string.IsNullOrWhiteSpace(bucketName)) { ThrowTerminatingError(new ErrorRecord( - new ArgumentException($"{parameterName} cannot be null or empty"), + new ArgumentException($"Bucket name cannot be null or empty", parameterName), "InvalidBucketName", ErrorCategory.InvalidArgument, bucketName)); } - // Basic bucket name validation (simplified) + // Basic S3 bucket name validation if (bucketName.Length < 3 || bucketName.Length > 63) { ThrowTerminatingError(new ErrorRecord( - new ArgumentException($"{parameterName} must be between 3 and 63 characters long"), + new ArgumentException($"Bucket name must be between 3 and 63 characters long", parameterName), + "InvalidBucketName", + ErrorCategory.InvalidArgument, + bucketName)); + } + + if (!System.Text.RegularExpressions.Regex.IsMatch(bucketName, @"^[a-z0-9][a-z0-9\-]*[a-z0-9]$")) + { + ThrowTerminatingError(new ErrorRecord( + new ArgumentException($"Bucket name contains invalid characters. Must contain only lowercase letters, numbers, and hyphens", parameterName), "InvalidBucketName", ErrorCategory.InvalidArgument, bucketName)); @@ -230,7 +216,7 @@ namespace PSMinIO.Utils } /// - /// Validates an object name + /// Validates that an object name is valid /// /// Object name to validate /// Parameter name for error reporting @@ -239,7 +225,16 @@ namespace PSMinIO.Utils if (string.IsNullOrWhiteSpace(objectName)) { ThrowTerminatingError(new ErrorRecord( - new ArgumentException($"{parameterName} cannot be null or empty"), + new ArgumentException($"Object name cannot be null or empty", parameterName), + "InvalidObjectName", + ErrorCategory.InvalidArgument, + objectName)); + } + + if (objectName.Length > 1024) + { + ThrowTerminatingError(new ErrorRecord( + new ArgumentException($"Object name cannot be longer than 1024 characters", parameterName), "InvalidObjectName", ErrorCategory.InvalidArgument, objectName)); @@ -247,40 +242,33 @@ namespace PSMinIO.Utils } /// - /// Cleans up resources when the cmdlet is disposed + /// Writes verbose output with consistent formatting /// - protected override void EndProcessing() + /// Message to write + /// Format arguments + protected void WriteVerboseMessage(string message, params object[] args) { - try - { - // Note: We don't dispose the connection here as it may be shared across cmdlets - // The connection should be disposed by the user when no longer needed - _connection = null; - } - catch (Exception ex) - { - MinIOLogger.WriteVerbose(this, $"Error during cleanup: {ex.Message}"); - } - - base.EndProcessing(); + MinIOLogger.WriteVerbose(this, message, args); } /// - /// Handles stopping the cmdlet (Ctrl+C) + /// Writes warning output with consistent formatting /// - protected override void StopProcessing() + /// Warning message to write + /// Format arguments + protected void WriteWarningMessage(string message, params object[] args) { - try - { - _connection?.Client.CancelOperations(); - MinIOLogger.WriteVerbose(this, "Operation cancelled by user"); - } - catch (Exception ex) - { - MinIOLogger.WriteVerbose(this, $"Error cancelling operations: {ex.Message}"); - } + MinIOLogger.WriteWarning(this, message, args); + } - base.StopProcessing(); + /// + /// Writes debug output with consistent formatting + /// + /// Debug message to write + /// Format arguments + protected void WriteDebugMessage(string message, params object[] args) + { + MinIOLogger.WriteDebug(this, message, args); } } } diff --git a/src/Cmdlets/NewMinIOBucketCmdlet.cs b/src/Cmdlets/NewMinIOBucketCmdlet.cs index 44489dc..4892b85 100644 --- a/src/Cmdlets/NewMinIOBucketCmdlet.cs +++ b/src/Cmdlets/NewMinIOBucketCmdlet.cs @@ -1,7 +1,6 @@ using System; using System.Management.Automation; -using PSMinIO.Models; -using PSMinIO.Utils; +using PSMinIO.Core.Models; namespace PSMinIO.Cmdlets { @@ -21,141 +20,119 @@ namespace PSMinIO.Cmdlets public string BucketName { get; set; } = string.Empty; /// - /// Region where the bucket should be created + /// Region where the bucket should be created (default: us-east-1) /// [Parameter] [ValidateNotNullOrEmpty] - public string? Region { get; set; } + public string Region { get; set; } = "us-east-1"; /// - /// Force creation even if bucket already exists (no error will be thrown) + /// Force creation even if bucket already exists /// [Parameter] public SwitchParameter Force { get; set; } - + /// + /// Return the created bucket information + /// + [Parameter] + public SwitchParameter PassThru { get; set; } /// /// Processes the cmdlet /// protected override void ProcessRecord() { - ValidateConnection(); - ValidateBucketNameForCreation(BucketName); + ValidateBucketName(BucketName); - // Use region from parameter or configuration - var region = Region ?? Configuration.Region; - - if (ShouldProcess(BucketName, $"Create bucket in region '{region}'")) + if (ShouldProcess(BucketName, "Create MinIO bucket")) { ExecuteOperation("CreateBucket", () => { + WriteVerboseMessage("Creating bucket '{0}' in region '{1}'", BucketName, Region); + // Check if bucket already exists - var exists = Client.BucketExists(BucketName); - - if (exists) + bool bucketExists = false; + try + { + bucketExists = S3Client.BucketExists(BucketName); + WriteVerboseMessage("Bucket existence check: {0}", bucketExists ? "exists" : "does not exist"); + } + catch (Exception ex) + { + WriteVerboseMessage("Could not check bucket existence: {0}", ex.Message); + } + + if (bucketExists) { if (Force.IsPresent) { - MinIOLogger.WriteVerbose(this, - "Bucket '{0}' already exists, but Force parameter specified", BucketName); + WriteVerboseMessage("Bucket '{0}' already exists, but Force parameter specified", BucketName); } else { - WriteError(new ErrorRecord( - new InvalidOperationException($"Bucket '{BucketName}' already exists. Use -Force to suppress this error."), + var errorMessage = $"Bucket '{BucketName}' already exists. Use -Force to suppress this error."; + var errorRecord = new ErrorRecord( + new InvalidOperationException(errorMessage), "BucketAlreadyExists", ErrorCategory.ResourceExists, - BucketName)); - return; + BucketName); + + ThrowTerminatingError(errorRecord); } } else { // Create the bucket - MinIOLogger.WriteVerbose(this, - "Creating bucket '{0}' in region '{1}'", BucketName, region); - - Client.CreateBucket(BucketName, region); - - MinIOLogger.WriteVerbose(this, - "Successfully created bucket '{0}'", BucketName); + try + { + S3Client.CreateBucket(BucketName, Region); + WriteVerboseMessage("Successfully created bucket '{0}'", BucketName); + } + catch (Exception ex) + { + WriteVerboseMessage("Failed to create bucket '{0}': {1}", BucketName, ex.Message); + throw; + } } - // Always return bucket information - try + // Return bucket information if requested + if (PassThru.IsPresent) { - // Get the created bucket information - var allBuckets = Client.ListBuckets(); - var createdBucket = allBuckets.Find(b => - string.Equals(b.Name, BucketName, StringComparison.OrdinalIgnoreCase)); + WriteVerboseMessage("Retrieving information for created bucket '{0}'", BucketName); - if (createdBucket != null) + try { - createdBucket.Region = region; - WriteObject(createdBucket); + // Get updated bucket list to find our bucket + var buckets = S3Client.ListBuckets(); + var createdBucket = buckets.Find(b => string.Equals(b.Name, BucketName, StringComparison.OrdinalIgnoreCase)); + + if (createdBucket != null) + { + createdBucket.Region = Region; + WriteObject(createdBucket); + } + else + { + // Create a basic bucket info object if we can't find it in the list + var bucketInfo = new MinIOBucketInfo(BucketName, DateTime.UtcNow, Region); + WriteObject(bucketInfo); + } } - else + catch (Exception ex) { - // Fallback: create a basic bucket info object - var bucketInfo = new MinIOBucketInfo(BucketName, DateTime.UtcNow, region); + WriteWarningMessage("Bucket created successfully but could not retrieve bucket information: {0}", ex.Message); + + // Create a basic bucket info object + var bucketInfo = new MinIOBucketInfo(BucketName, DateTime.UtcNow, Region); WriteObject(bucketInfo); } } - catch (Exception ex) + else { - MinIOLogger.WriteWarning(this, - "Bucket created successfully, but failed to retrieve bucket information: {0}", - ex.Message); - - // Still return a basic bucket info object - var bucketInfo = new MinIOBucketInfo(BucketName, DateTime.UtcNow, region); - WriteObject(bucketInfo); + WriteVerboseMessage("Bucket '{0}' created successfully", BucketName); } - - }, $"Bucket: {BucketName}, Region: {region}"); - } - } - - /// - /// Validates the bucket name according to MinIO/S3 naming conventions for bucket creation - /// - /// Bucket name to validate - private void ValidateBucketNameForCreation(string bucketName, string parameterName = "BucketName") - { - base.ValidateBucketName(bucketName, parameterName); - - // Additional validation for bucket creation - if (bucketName.Contains("..") || bucketName.StartsWith(".") || bucketName.EndsWith(".")) - { - ThrowTerminatingError(new ErrorRecord( - new ArgumentException($"Bucket name '{bucketName}' contains invalid characters or patterns"), - "InvalidBucketName", - ErrorCategory.InvalidArgument, - bucketName)); - } - - // Check for uppercase letters (not allowed in S3/MinIO) - if (bucketName != bucketName.ToLowerInvariant()) - { - ThrowTerminatingError(new ErrorRecord( - new ArgumentException($"Bucket name '{bucketName}' must be lowercase"), - "InvalidBucketName", - ErrorCategory.InvalidArgument, - bucketName)); - } - - // Check for invalid characters - foreach (char c in bucketName) - { - if (!char.IsLetterOrDigit(c) && c != '-' && c != '.') - { - ThrowTerminatingError(new ErrorRecord( - new ArgumentException($"Bucket name '{bucketName}' contains invalid character '{c}'. Only lowercase letters, numbers, hyphens, and periods are allowed."), - "InvalidBucketName", - ErrorCategory.InvalidArgument, - bucketName)); - } + }, $"Bucket: {BucketName}, Region: {Region}"); } } } diff --git a/src/Cmdlets/NewMinIOFolderCmdlet.cs b/src/Cmdlets/NewMinIOFolderCmdlet.cs deleted file mode 100644 index 1f5a7e5..0000000 --- a/src/Cmdlets/NewMinIOFolderCmdlet.cs +++ /dev/null @@ -1,185 +0,0 @@ -using System; -using System.IO; -using System.Linq; -using System.Management.Automation; -using PSMinIO.Models; -using PSMinIO.Utils; - -namespace PSMinIO.Cmdlets -{ - /// - /// Creates a folder (prefix) in a MinIO bucket - /// - [Cmdlet(VerbsCommon.New, "MinIOFolder", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Low)] - [OutputType(typeof(MinIOObjectInfo))] - public class NewMinIOFolderCmdlet : MinIOBaseCmdlet - { - /// - /// Name of the bucket to create the folder in - /// - [Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true)] - [ValidateNotNullOrEmpty] - [Alias("Bucket")] - public string BucketName { get; set; } = string.Empty; - - /// - /// Name of the folder to create (supports nested paths like "folder1/folder2/folder3") - /// - [Parameter(Position = 1, Mandatory = true, ValueFromPipelineByPropertyName = true)] - [ValidateNotNullOrEmpty] - [Alias("Folder", "Prefix", "Path")] - public string FolderName { get; set; } = string.Empty; - - - - /// - /// Processes the cmdlet - /// - protected override void ProcessRecord() - { - ValidateConnection(); - ValidateBucketName(BucketName); - - // Sanitize the folder name - var sanitizedFolderName = SanitizeFolderPath(FolderName); - if (string.IsNullOrEmpty(sanitizedFolderName)) - { - WriteError(new ErrorRecord( - new ArgumentException("Folder name cannot be empty after sanitization"), - "InvalidFolderName", - ErrorCategory.InvalidArgument, - FolderName)); - return; - } - - if (ShouldProcess($"{BucketName}/{sanitizedFolderName}", "Create folder structure")) - { - ExecuteOperation("CreateFolder", () => - { - // Check if bucket exists - var bucketExists = Client.BucketExists(BucketName); - if (!bucketExists) - { - WriteError(new ErrorRecord( - new InvalidOperationException($"Bucket '{BucketName}' does not exist"), - "BucketNotFound", - ErrorCategory.ObjectNotFound, - BucketName)); - return; - } - - MinIOLogger.WriteVerbose(this, - "Creating folder structure '{0}' in bucket '{1}'", sanitizedFolderName, BucketName); - - // Create the complete folder structure - var createdFolders = CreateFolderStructure(sanitizedFolderName); - - MinIOLogger.WriteVerbose(this, - "Successfully created {0} folder level(s) in bucket '{1}'", createdFolders.Count, BucketName); - - // Return information about all created folders - foreach (var folder in createdFolders) - { - WriteObject(folder); - } - - }, $"Bucket: {BucketName}, Folder: {sanitizedFolderName}"); - } - } - - /// - /// Sanitizes folder path and ensures proper format - /// - /// Raw folder path - /// Sanitized folder path - private string SanitizeFolderPath(string folderPath) - { - if (string.IsNullOrWhiteSpace(folderPath)) - return string.Empty; - - // Remove leading and trailing slashes, then split and clean - var cleanPath = folderPath.Trim().Trim('/', '\\'); - - // Split by both forward and back slashes, then clean each part - var parts = cleanPath.Split(new char[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries); - - // Clean each part and validate - var cleanedParts = new System.Collections.Generic.List(); - var invalidChars = new char[] { '<', '>', ':', '"', '|', '?', '*' }; - - foreach (var part in parts) - { - var cleanPart = part.Trim(); - if (string.IsNullOrEmpty(cleanPart)) - continue; - - // Check for invalid characters - if (cleanPart.IndexOfAny(invalidChars) >= 0) - { - WriteWarning($"Folder part '{cleanPart}' contains invalid characters and will be skipped"); - continue; - } - - cleanedParts.Add(cleanPart); - } - - return string.Join("/", cleanedParts); - } - - /// - /// Creates the complete folder structure, including all intermediate levels - /// - /// Complete folder path to create - /// List of created folder objects - private System.Collections.Generic.List CreateFolderStructure(string folderPath) - { - var createdFolders = new System.Collections.Generic.List(); - var parts = folderPath.Split('/'); - var currentPath = string.Empty; - - foreach (var part in parts) - { - currentPath = string.IsNullOrEmpty(currentPath) ? part : $"{currentPath}/{part}"; - var fullFolderPath = $"{currentPath}/"; - - try - { - // Check if this folder level already exists - var existingObjects = Client.ListObjects(BucketName, fullFolderPath, false); - var folderExists = existingObjects.Any(obj => obj.Name == fullFolderPath); - - if (!folderExists) - { - MinIOLogger.WriteVerbose(this, "Creating folder level: {0}", fullFolderPath); - - // Create the folder using the dedicated method - var etag = Client.CreateDirectory(BucketName, fullFolderPath); - - var folderInfo = new MinIOObjectInfo( - fullFolderPath, - 0, - DateTime.UtcNow, - etag, - BucketName) - { - ContentType = "application/x-directory" - }; - - createdFolders.Add(folderInfo); - MinIOLogger.WriteVerbose(this, "Successfully created folder level: {0}", fullFolderPath); - } - else - { - MinIOLogger.WriteVerbose(this, "Folder level already exists: {0}", fullFolderPath); - } - } - catch (Exception ex) - { - WriteWarning($"Could not create folder level '{fullFolderPath}': {ex.Message}"); - } - } - - return createdFolders; - } - } -} diff --git a/src/Cmdlets/NewMinIOObjectChunkedCmdlet.cs b/src/Cmdlets/NewMinIOObjectChunkedCmdlet.cs deleted file mode 100644 index 8d4ade2..0000000 --- a/src/Cmdlets/NewMinIOObjectChunkedCmdlet.cs +++ /dev/null @@ -1,588 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Management.Automation; -using PSMinIO.Models; -using PSMinIO.Utils; - -namespace PSMinIO.Cmdlets -{ - /// - /// Uploads files or directories to a MinIO bucket using chunked transfer with resume capability - /// - [Cmdlet(VerbsCommon.New, "MinIOObjectChunked", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Low)] - [OutputType(typeof(MinIOObjectInfo))] - public class NewMinIOObjectChunkedCmdlet : MinIOBaseCmdlet - { - /// - /// Name of the bucket to upload to - /// - [Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true)] - [ValidateNotNullOrEmpty] - [Alias("Bucket")] - public string BucketName { get; set; } = string.Empty; - - /// - /// Array of FileInfo objects to upload - /// - [Parameter(Position = 1, Mandatory = true, ValueFromPipeline = true, ParameterSetName = "Files")] - [ValidateNotNull] - [Alias("File", "Files")] - public FileInfo[]? Path { get; set; } - - /// - /// Optional bucket directory path where files should be uploaded (Files parameter set only) - /// - [Parameter(ParameterSetName = "Files")] - [ValidateNotNullOrEmpty] - [Alias("Prefix")] - public string? BucketDirectory { get; set; } - - /// - /// Directory to upload - /// - [Parameter(Position = 1, Mandatory = true, ValueFromPipelineByPropertyName = true, ParameterSetName = "Directory")] - [ValidateNotNull] - [Alias("Dir")] - public DirectoryInfo? Directory { get; set; } - - /// - /// Upload directory contents recursively (only applies to Directory parameter set) - /// - [Parameter(ParameterSetName = "Directory")] - public SwitchParameter Recursive { get; set; } - - /// - /// Maximum depth for recursive directory upload (0 = unlimited) - /// - [Parameter(ParameterSetName = "Directory")] - [ValidateRange(0, int.MaxValue)] - public int MaxDepth { get; set; } = 0; - - /// - /// Flatten directory structure (upload all files to bucket root) - /// - [Parameter(ParameterSetName = "Directory")] - public SwitchParameter Flatten { get; set; } - - /// - /// Script block to filter files for inclusion - /// - [Parameter(ParameterSetName = "Directory")] - public ScriptBlock? InclusionFilter { get; set; } - - /// - /// Script block to filter files for exclusion - /// - [Parameter(ParameterSetName = "Directory")] - public ScriptBlock? ExclusionFilter { get; set; } - - /// - /// Overwrite existing objects without prompting - /// - [Parameter] - public SwitchParameter Force { get; set; } - - /// - /// Generate presigned URLs for uploaded objects - /// - [Parameter] - public SwitchParameter ShowURL { get; set; } - - /// - /// Expiration time for presigned URLs (default: 1 hour) - /// - [Parameter] - [ValidateRange("00:01:00", "7.00:00:00")] // 1 minute to 7 days - public TimeSpan Expiration { get; set; } = TimeSpan.FromHours(1); - - /// - /// Size of each chunk for upload (default: 5MB) - /// - [Parameter] - [ValidateRange(1024 * 1024, 1024 * 1024 * 1024)] // 1MB to 1GB - public long ChunkSize { get; set; } = 5 * 1024 * 1024; // 5MB default - - /// - /// Enable resume functionality for interrupted uploads - /// - [Parameter] - public SwitchParameter Resume { get; set; } - - /// - /// Maximum number of retry attempts for failed chunks - /// - [Parameter] - [ValidateRange(1, 10)] - public int MaxRetries { get; set; } = 3; - - /// - /// Custom path for storing resume data (default: %LOCALAPPDATA%\PSMinIO\Resume) - /// - [Parameter] - public string? ResumeDataPath { get; set; } - - - - /// - /// Update progress every N bytes transferred (default: 1MB) - /// - [Parameter] - [ValidateRange(1024, 10 * 1024 * 1024)] // 1KB to 10MB - public long ProgressUpdateInterval { get; set; } = 1024 * 1024; // 1MB - - /// - /// Processes the cmdlet - /// - protected override void ProcessRecord() - { - ValidateConnection(); - ValidateBucketName(BucketName); - - if (ParameterSetName == "Files") - { - ProcessFiles(); - } - else if (ParameterSetName == "Directory") - { - ProcessDirectory(); - } - } - - /// - /// Processes file uploads from FileInfo array - /// - private void ProcessFiles() - { - if (Path == null || Path.Length == 0) - { - WriteWarning("No files provided for upload"); - return; - } - - if (ShouldProcess(BucketName, $"Upload {Path.Length} file(s) using chunked transfer")) - { - ExecuteOperation("UploadFilesChunked", () => - { - // Check if bucket exists first - var bucketExists = Client.BucketExists(BucketName); - if (!bucketExists) - { - WriteError(new ErrorRecord( - new InvalidOperationException($"Bucket '{BucketName}' does not exist"), - "BucketNotFound", - ErrorCategory.ObjectNotFound, - BucketName)); - return; - } - - // Create bucket directory structure if specified - if (!string.IsNullOrWhiteSpace(BucketDirectory)) - { - var sanitizedDirectory = SanitizeBucketDirectory(BucketDirectory!); - if (!string.IsNullOrEmpty(sanitizedDirectory)) - { - MinIOLogger.WriteVerbose(this, "Ensuring bucket directory exists: {0}", sanitizedDirectory); - EnsureBucketDirectoryExists(sanitizedDirectory); - } - } - - UploadFileCollectionChunked(Path!); - - }, $"Bucket: {BucketName}, Files: {Path.Length}, ChunkSize: {SizeFormatter.FormatBytes(ChunkSize)}"); - } - } - - /// - /// Processes directory upload - /// - private void ProcessDirectory() - { - if (Directory == null || !Directory.Exists) - { - WriteError(new ErrorRecord( - new DirectoryNotFoundException($"Directory not found: {Directory?.FullName}"), - "DirectoryNotFound", - ErrorCategory.ObjectNotFound, - Directory)); - return; - } - - if (ShouldProcess(BucketName, $"Upload directory '{Directory.Name}' using chunked transfer")) - { - ExecuteOperation("UploadDirectoryChunked", () => - { - // Check if bucket exists first - var bucketExists = Client.BucketExists(BucketName); - if (!bucketExists) - { - WriteError(new ErrorRecord( - new InvalidOperationException($"Bucket '{BucketName}' does not exist"), - "BucketNotFound", - ErrorCategory.ObjectNotFound, - BucketName)); - return; - } - - // Get files from directory - var files = GetDirectoryFiles(); - if (files.Length == 0) - { - WriteWarning($"No files found in directory: {Directory.FullName}"); - return; - } - - MinIOLogger.WriteVerbose(this, "Found {0} files in directory '{1}'", files.Length, Directory.FullName); - UploadFileCollectionChunked(files); - - }, $"Bucket: {BucketName}, Directory: {Directory.FullName}, ChunkSize: {SizeFormatter.FormatBytes(ChunkSize)}"); - } - } - - /// - /// Gets files from directory based on filters and recursion settings - /// - /// Array of FileInfo objects - private FileInfo[] GetDirectoryFiles() - { - var searchOption = Recursive.IsPresent ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly; - var allFiles = Directory!.GetFiles("*", searchOption); - - // Apply depth filtering if MaxDepth is specified and we're recursive - if (Recursive.IsPresent && MaxDepth > 0) - { - var basePath = Directory.FullName; - allFiles = allFiles.Where(f => - { - var relativePath = f.FullName.Substring(basePath.Length).TrimStart('\\', '/'); - var depth = relativePath.Split(new char[] { '\\', '/' }, StringSplitOptions.RemoveEmptyEntries).Length - 1; - return depth <= MaxDepth; - }).ToArray(); - } - - // Apply inclusion filter - if (InclusionFilter != null) - { - allFiles = allFiles.Where(f => EvaluateFilter(InclusionFilter, f)).ToArray(); - } - - // Apply exclusion filter - if (ExclusionFilter != null) - { - allFiles = allFiles.Where(f => !EvaluateFilter(ExclusionFilter, f)).ToArray(); - } - - return allFiles; - } - - /// - /// Evaluates a script block filter against a file - /// - /// Script block filter - /// File to evaluate - /// True if file matches filter - private bool EvaluateFilter(ScriptBlock filter, FileInfo file) - { - try - { - var result = filter.InvokeWithContext(null, new List { new PSVariable("_", file) }); - return result.Count > 0 && LanguagePrimitives.IsTrue(result[0]); - } - catch (Exception ex) - { - WriteWarning($"Filter evaluation failed for {file.Name}: {ex.Message}"); - return false; - } - } - - /// - /// Uploads a collection of files using chunked transfer - /// - /// Files to upload - private void UploadFileCollectionChunked(FileInfo[] files) - { - // Filter out files that don't exist - var validFiles = files.Where(f => f.Exists).ToArray(); - var skippedCount = files.Length - validFiles.Length; - - if (skippedCount > 0) - { - WriteWarning($"Skipped {skippedCount} files that do not exist"); - } - - if (validFiles.Length == 0) - { - WriteWarning("No valid files found for upload"); - return; - } - - MinIOLogger.WriteVerbose(this, "Starting chunked upload of {0} files to bucket '{1}' (ChunkSize: {2})", - validFiles.Length, BucketName, SizeFormatter.FormatBytes(ChunkSize)); - - // Calculate total size for overall progress - var totalSize = validFiles.Sum(f => f.Length); - - // Create thread-safe result collector - var resultCollector = new ThreadSafeResultCollector(this); - - for (int i = 0; i < validFiles.Length; i++) - { - var fileInfo = validFiles[i]; - var objectName = GetObjectName(fileInfo); - - try - { - // Calculate chunks for this file - var totalChunks = (int)Math.Ceiling((double)fileInfo.Length / ChunkSize); - - // Create thread-safe progress reporter for this file - var progressReporter = new ThreadSafeChunkedProgressReporter( - this, - fileInfo.Length, - totalChunks, - "Uploading"); - - progressReporter.StartNewFile(fileInfo.Name, fileInfo.Length, totalChunks); - - // Track timing for this file - var startTime = DateTime.UtcNow; - - // Upload file using chunked transfer - var uploadedObject = UploadFileChunked(fileInfo, objectName, progressReporter); - - if (uploadedObject != null) - { - // Add timing information - uploadedObject.StartTime = startTime; - uploadedObject.CompletionTime = DateTime.UtcNow; - - resultCollector.QueueResult(uploadedObject); - progressReporter.CompleteFile(); - } - - // Process any queued progress updates from the main thread - progressReporter.ProcessQueuedUpdates(); - progressReporter.Complete(); - } - catch (Exception ex) - { - resultCollector.QueueError(ex, "ChunkedFileUploadFailed", ErrorCategory.WriteError, fileInfo); - MinIOLogger.WriteVerbose(this, "Failed to upload {0}: {1}", fileInfo.Name, ex.Message); - } - } - - MinIOLogger.WriteVerbose(this, "Completed chunked upload: {0} files", validFiles.Length); - - // Process all results from the main thread - resultCollector.Complete(); - } - - /// - /// Uploads a single file using chunked transfer with resume capability - /// - /// File to upload - /// Object name in bucket - /// Thread-safe progress reporter - /// Uploaded object info or null if failed - private MinIOObjectInfo? UploadFileChunked(FileInfo fileInfo, string objectName, ThreadSafeChunkedProgressReporter progressReporter) - { - ChunkedTransferState? transferState = null; - - // Try to load existing transfer state for resume - if (Resume.IsPresent) - { - transferState = ChunkedTransferResumeManager.LoadTransferState( - BucketName, objectName, fileInfo.FullName, ChunkedTransferType.Upload, ResumeDataPath); - - if (transferState != null && !ChunkedTransferResumeManager.IsResumeDataValid(transferState, fileInfo)) - { - MinIOLogger.WriteVerbose(this, "Resume data for {0} is invalid, starting fresh upload", fileInfo.Name); - transferState = null; - } - } - - // Create new transfer state if none exists or resume is disabled - if (transferState == null) - { - transferState = new ChunkedTransferState - { - BucketName = BucketName, - ObjectName = objectName, - FilePath = fileInfo.FullName, - TotalSize = fileInfo.Length, - ChunkSize = ChunkSize, - LastModified = fileInfo.LastWriteTimeUtc, - TransferType = ChunkedTransferType.Upload, - StartTime = DateTime.UtcNow, - LastUpdated = DateTime.UtcNow - }; - } - else - { - MinIOLogger.WriteVerbose(this, "Resuming upload of {0} from {1:P1} complete", - fileInfo.Name, transferState.ProgressPercentage / 100); - } - - try - { - // Upload file using chunked transfer - var result = Client.UploadFileChunked(transferState, progressReporter, MaxRetries); - - if (result != null) - { - // Generate presigned URL if requested - if (ShowURL.IsPresent) - { - try - { - var presignedUrl = Client.GetPresignedUrl(BucketName, objectName, Expiration); - result.PresignedUrl = presignedUrl; - result.PresignedUrlExpiration = DateTime.UtcNow.Add(Expiration); - } - catch (Exception ex) - { - WriteWarning($"Could not generate presigned URL for {objectName}: {ex.Message}"); - } - } - - // Clean up resume data on successful completion - if (Resume.IsPresent) - { - ChunkedTransferResumeManager.CleanupResumeData(transferState, ResumeDataPath); - } - - return result; - } - } -#pragma warning disable CS0168 // Variable is declared but never used - false positive, ex is used in throw - catch (Exception ex) - { - // Save resume data on failure if resume is enabled - if (Resume.IsPresent) - { - try - { - ChunkedTransferResumeManager.SaveTransferState(transferState, ResumeDataPath); - MinIOLogger.WriteVerbose(this, "Saved resume data for {0}", fileInfo.Name); - } - catch (Exception saveEx) - { - WriteWarning($"Could not save resume data for {fileInfo.Name}: {saveEx.Message}"); - } - } - - throw; - } -#pragma warning restore CS0168 - - return null; - } - - /// - /// Gets the object name for a file based on the upload context - /// - /// File to get object name for - /// Object name to use in MinIO - private string GetObjectName(FileInfo file) - { - if (ParameterSetName == "Files") - { - // For file uploads, optionally prefix with bucket directory - var objectName = file.Name; - if (!string.IsNullOrWhiteSpace(BucketDirectory)) - { - var sanitizedDirectory = SanitizeBucketDirectory(BucketDirectory!); - objectName = $"{sanitizedDirectory}/{file.Name}"; - } - return objectName; - } - else if (ParameterSetName == "Directory") - { - if (Flatten.IsPresent) - { - // Flatten: use just the filename - return file.Name; - } - else - { - // Maintain directory structure relative to the base directory - var relativePath = file.FullName.Substring(Directory!.FullName.Length).TrimStart('\\', '/'); - return relativePath.Replace('\\', '/'); // Ensure forward slashes for object storage - } - } - - return file.Name; - } - - /// - /// Sanitizes bucket directory path and ensures proper format - /// - /// Raw directory path - /// Sanitized directory path - private string SanitizeBucketDirectory(string directory) - { - if (string.IsNullOrWhiteSpace(directory)) - return string.Empty; - - // Remove leading and trailing slashes, then split and clean - var cleanDirectory = directory.Trim().Trim('/', '\\'); - - // Split by both forward and back slashes, then clean each part - var parts = cleanDirectory.Split(new char[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries); - - // Clean each part and rejoin with forward slashes - var cleanedParts = parts.Select(part => part.Trim()).Where(part => !string.IsNullOrEmpty(part)); - - return string.Join("/", cleanedParts); - } - - /// - /// Creates bucket directory structure if it doesn't exist - /// - /// Directory path to create - private void EnsureBucketDirectoryExists(string directoryPath) - { - if (string.IsNullOrWhiteSpace(directoryPath)) - return; - - var parts = directoryPath.Split('/'); - var currentPath = string.Empty; - - foreach (var part in parts) - { - currentPath = string.IsNullOrEmpty(currentPath) ? part : $"{currentPath}/{part}"; - var folderPath = $"{currentPath}/"; - - try - { - // Check if this directory level already exists by trying to list objects with this prefix - var existingObjects = Client.ListObjects(BucketName, folderPath, false); - var folderExists = existingObjects.Any(obj => obj.Name == folderPath); - - if (!folderExists) - { - MinIOLogger.WriteVerbose(this, "Creating bucket directory: {0}", folderPath); - - try - { - // Create the directory using the dedicated method - Client.CreateDirectory(BucketName, folderPath); - MinIOLogger.WriteVerbose(this, "Successfully created bucket directory: {0}", folderPath); - } - catch (Exception createEx) - { - // Directory creation failed, but this is not critical since MinIO creates directories implicitly - MinIOLogger.WriteVerbose(this, "Directory creation failed (non-critical): {0} - {1}", folderPath, createEx.Message); - } - } - } - catch (Exception ex) - { - // Listing failed, but this is not critical for the upload operation - MinIOLogger.WriteVerbose(this, "Could not check directory existence (non-critical): {0} - {1}", folderPath, ex.Message); - } - } - } - } -} diff --git a/src/Cmdlets/NewMinIOObjectCmdlet.cs b/src/Cmdlets/NewMinIOObjectCmdlet.cs index eab19dc..b7e33e3 100644 --- a/src/Cmdlets/NewMinIOObjectCmdlet.cs +++ b/src/Cmdlets/NewMinIOObjectCmdlet.cs @@ -1,506 +1,251 @@ using System; +using System.Collections; using System.Collections.Generic; using System.IO; -using System.Linq; using System.Management.Automation; -using PSMinIO.Models; +using PSMinIO.Core.Models; using PSMinIO.Utils; namespace PSMinIO.Cmdlets { /// - /// Uploads files or directories to a MinIO bucket + /// Uploads objects to MinIO /// - [Cmdlet(VerbsCommon.New, "MinIOObject", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Low)] - [OutputType(typeof(MinIOObjectInfo))] + [Cmdlet(VerbsCommon.New, "MinIOObject", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Medium)] + [OutputType(typeof(MinIOUploadResult))] public class NewMinIOObjectCmdlet : MinIOBaseCmdlet { /// - /// Name of the bucket to upload to + /// Name of the bucket /// - [Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true)] + [Parameter(Position = 0, Mandatory = true)] [ValidateNotNullOrEmpty] - [Alias("Bucket")] public string BucketName { get; set; } = string.Empty; /// - /// Array of FileInfo objects to upload + /// Name of the object (if not specified, uses filename) /// - [Parameter(Position = 1, Mandatory = true, ValueFromPipeline = true, ParameterSetName = "Files")] - [ValidateNotNull] - [Alias("File", "Files")] - public FileInfo[]? Path { get; set; } + [Parameter(Position = 1)] + [ValidateNotNullOrEmpty] + public string? ObjectName { get; set; } /// - /// Optional bucket directory path where files should be uploaded (Files parameter set only) + /// File to upload /// - [Parameter(ParameterSetName = "Files")] + [Parameter(Position = 2, Mandatory = true, ValueFromPipeline = true)] + [ValidateNotNull] + public FileInfo File { get; set; } = null!; + + /// + /// Content type of the file (auto-detected if not specified) + /// + [Parameter] [ValidateNotNullOrEmpty] - [Alias("Prefix")] + public string? ContentType { get; set; } + + /// + /// Custom metadata for the object + /// + [Parameter] + public Hashtable? Metadata { get; set; } + + /// + /// Directory prefix in the bucket (creates nested structure) + /// + [Parameter] + [ValidateNotNullOrEmpty] + [Alias("Folder", "Directory")] public string? BucketDirectory { get; set; } /// - /// Directory to upload - /// - [Parameter(Position = 1, Mandatory = true, ValueFromPipelineByPropertyName = true, ParameterSetName = "Directory")] - [ValidateNotNull] - [Alias("Dir")] - public DirectoryInfo? Directory { get; set; } - - /// - /// Upload directory contents recursively (only applies to Directory parameter set) - /// - [Parameter(ParameterSetName = "Directory")] - public SwitchParameter Recursive { get; set; } - - /// - /// Maximum depth for recursive directory upload (0 = unlimited) - /// - [Parameter(ParameterSetName = "Directory")] - [ValidateRange(0, int.MaxValue)] - public int MaxDepth { get; set; } = 0; - - /// - /// Flatten directory structure (upload all files to bucket root) - /// - [Parameter(ParameterSetName = "Directory")] - public SwitchParameter Flatten { get; set; } - - /// - /// Script block to filter files for inclusion - /// - [Parameter(ParameterSetName = "Directory")] - public ScriptBlock? InclusionFilter { get; set; } - - /// - /// Script block to filter files for exclusion - /// - [Parameter(ParameterSetName = "Directory")] - public ScriptBlock? ExclusionFilter { get; set; } - - /// - /// Overwrite existing objects without prompting + /// Force overwrite if object already exists /// [Parameter] public SwitchParameter Force { get; set; } /// - /// Generate presigned URLs for uploaded objects + /// Return the upload result information /// [Parameter] - public SwitchParameter ShowURL { get; set; } - - /// - /// Expiration time for presigned URLs (default: 1 hour) - /// - [Parameter] - [ValidateRange("00:01:00", "7.00:00:00")] // 1 minute to 7 days - public TimeSpan Expiration { get; set; } = TimeSpan.FromHours(1); + public SwitchParameter PassThru { get; set; } /// /// Processes the cmdlet /// protected override void ProcessRecord() { - ValidateConnection(); ValidateBucketName(BucketName); - if (ParameterSetName == "Files") + // Validate file exists + if (!File.Exists) { - ProcessFiles(); - } - else if (ParameterSetName == "Directory") - { - ProcessDirectory(); - } - } - - /// - /// Processes file uploads from FileInfo array - /// - private void ProcessFiles() - { - if (Path == null || Path.Length == 0) - { - WriteWarning("No files provided for upload"); - return; - } - - if (ShouldProcess(BucketName, $"Upload {Path.Length} file(s)")) - { - ExecuteOperation("UploadFiles", () => - { - // Check if bucket exists first - var bucketExists = Client.BucketExists(BucketName); - if (!bucketExists) - { - WriteError(new ErrorRecord( - new InvalidOperationException($"Bucket '{BucketName}' does not exist"), - "BucketNotFound", - ErrorCategory.ObjectNotFound, - BucketName)); - return; - } - - // Create bucket directory structure if specified - if (!string.IsNullOrWhiteSpace(BucketDirectory)) - { - var sanitizedDirectory = SanitizeBucketDirectory(BucketDirectory!); - if (!string.IsNullOrEmpty(sanitizedDirectory)) - { - MinIOLogger.WriteVerbose(this, "Ensuring bucket directory exists: {0}", sanitizedDirectory); - EnsureBucketDirectoryExists(sanitizedDirectory); - } - } - - UploadFileCollection(Path!); - - }, $"Bucket: {BucketName}, Files: {Path.Length}"); - } - } - - /// - /// Processes directory upload - /// - private void ProcessDirectory() - { - if (Directory == null || !Directory.Exists) - { - WriteError(new ErrorRecord( - new DirectoryNotFoundException($"Directory not found: {Directory?.FullName}"), - "DirectoryNotFound", + var errorRecord = new ErrorRecord( + new FileNotFoundException($"File not found: {File.FullName}"), + "FileNotFound", ErrorCategory.ObjectNotFound, - Directory)); - return; + File); + ThrowTerminatingError(errorRecord); } - if (ShouldProcess(BucketName, $"Upload directory '{Directory.Name}'")) + // Determine object name + var finalObjectName = ObjectName ?? File?.Name ?? "unknown"; + + // Add bucket directory prefix if specified + if (!string.IsNullOrEmpty(BucketDirectory)) { - ExecuteOperation("UploadDirectory", () => + var cleanDirectory = BucketDirectory.Trim('/'); + if (!string.IsNullOrEmpty(cleanDirectory)) { - // Check if bucket exists first - var bucketExists = Client.BucketExists(BucketName); - if (!bucketExists) - { - WriteError(new ErrorRecord( - new InvalidOperationException($"Bucket '{BucketName}' does not exist"), - "BucketNotFound", - ErrorCategory.ObjectNotFound, - BucketName)); - return; - } - - // Get files from directory - var files = GetDirectoryFiles(); - if (files.Length == 0) - { - WriteWarning($"No files found in directory: {Directory.FullName}"); - return; - } - - MinIOLogger.WriteVerbose(this, "Found {0} files in directory '{1}'", files.Length, Directory.FullName); - UploadFileCollection(files); - - }, $"Bucket: {BucketName}, Directory: {Directory.FullName}"); + finalObjectName = $"{cleanDirectory}/{finalObjectName}"; + } } - } - /// - /// Gets files from directory based on filters and recursion settings - /// - /// Array of FileInfo objects - private FileInfo[] GetDirectoryFiles() - { - var searchOption = Recursive.IsPresent ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly; - var allFiles = Directory!.GetFiles("*", searchOption); + ValidateObjectName(finalObjectName); - // Apply depth filtering if MaxDepth is specified and we're recursive - if (Recursive.IsPresent && MaxDepth > 0) + if (ShouldProcess($"{BucketName}/{finalObjectName}", "Upload file")) { - var basePath = Directory.FullName; - allFiles = allFiles.Where(f => + var result = ExecuteOperation("UploadObject", () => { - var relativePath = f.FullName.Substring(basePath.Length).TrimStart('\\', '/'); - var depth = relativePath.Split(new char[] { '\\', '/' }, StringSplitOptions.RemoveEmptyEntries).Length - 1; - return depth <= MaxDepth; - }).ToArray(); - } + WriteVerboseMessage("Uploading file '{0}' to bucket '{1}' as object '{2}'", + File.FullName, BucketName, finalObjectName); - // Apply inclusion filter - if (InclusionFilter != null) - { - allFiles = allFiles.Where(f => EvaluateFilter(InclusionFilter, f)).ToArray(); - } - - // Apply exclusion filter - if (ExclusionFilter != null) - { - allFiles = allFiles.Where(f => !EvaluateFilter(ExclusionFilter, f)).ToArray(); - } - - return allFiles; - } - - /// - /// Evaluates a script block filter against a file - /// - /// Script block filter - /// File to evaluate - /// True if file matches filter - private bool EvaluateFilter(ScriptBlock filter, FileInfo file) - { - try - { - var result = filter.InvokeWithContext(null, new List { new PSVariable("_", file) }); - return result.Count > 0 && LanguagePrimitives.IsTrue(result[0]); - } - catch (Exception ex) - { - WriteWarning($"Filter evaluation failed for {file.Name}: {ex.Message}"); - return false; - } - } - - /// - /// Uploads a collection of files - /// - /// Files to upload - private void UploadFileCollection(FileInfo[] files) - { - // Filter out files that don't exist - var validFiles = files.Where(f => f.Exists).ToArray(); - var skippedCount = files.Length - validFiles.Length; - - if (skippedCount > 0) - { - WriteWarning($"Skipped {skippedCount} files that do not exist"); - } - - if (validFiles.Length == 0) - { - WriteWarning("No valid files found for upload"); - return; - } - - MinIOLogger.WriteVerbose(this, "Uploading {0} files to bucket '{1}'", validFiles.Length, BucketName); - - // Calculate total size for overall progress - var totalSize = validFiles.Sum(f => f.Length); - var totalProcessed = 0L; - - // Create overall progress reporter - var overallProgress = new ProgressReporter( - this, - "Uploading Files", - $"Uploading {validFiles.Length} files", - totalSize, - 1); - - var uploadedObjects = new System.Collections.Generic.List(); - - for (int i = 0; i < validFiles.Length; i++) - { - var fileInfo = validFiles[i]; - var objectName = GetObjectName(fileInfo); - - try - { - // Show file-level progress - var fileProgressRecord = new ProgressRecord(2, "Current File", - $"Uploading: {fileInfo.Name}") + // Create upload result + var uploadResult = new MinIOUploadResult(BucketName, finalObjectName, string.Empty, File.Length) { - PercentComplete = (int)((double)(i + 1) / validFiles.Length * 100), - ParentActivityId = 1 + SourceFilePath = File.FullName }; - WriteProgress(fileProgressRecord); + uploadResult.MarkStarted(); - MinIOLogger.WriteVerbose(this, "Uploading file {0}/{1}: {2} -> {3}", - i + 1, validFiles.Length, fileInfo.Name, objectName); - - // Track timing for this file - var startTime = DateTime.UtcNow; - - // Upload the file - var etag = Client.UploadFile( - BucketName, - objectName, - fileInfo.FullName, - null, // Auto-detect content type - bytesTransferred => - { - overallProgress.UpdateProgress(totalProcessed + bytesTransferred); - }); - - var completionTime = DateTime.UtcNow; - - totalProcessed += fileInfo.Length; - overallProgress.UpdateProgress(totalProcessed); - - // Create object info for result - var uploadedObject = new MinIOObjectInfo( - objectName, - fileInfo.Length, - DateTime.UtcNow, - etag, - BucketName) + try { - StartTime = startTime, - CompletionTime = completionTime - }; + // Check if bucket exists + if (!S3Client.BucketExists(BucketName)) + { + throw new InvalidOperationException($"Bucket '{BucketName}' does not exist"); + } - // Generate presigned URL if requested - if (ShowURL.IsPresent) + // Determine content type + var fileContentType = ContentType ?? GetContentType(File.Extension); + uploadResult.ContentType = fileContentType; + WriteVerboseMessage("Using content type: {0}", fileContentType); + + // Convert metadata + Dictionary? metadata = null; + if (Metadata != null && Metadata.Count > 0) + { + metadata = new Dictionary(); + foreach (var key in Metadata.Keys) + { + if (key != null && Metadata[key] != null) + { + metadata[key.ToString()!] = Metadata[key]!.ToString()!; + } + } + WriteVerboseMessage("Added {0} metadata entries", metadata.Count); + } + + // Track progress + var fileSize = File.Length; + var startTime = DateTime.UtcNow; + long lastReportedBytes = 0; + + WriteVerboseMessage("Starting upload of {0}", SizeFormatter.FormatBytes(fileSize)); + + // Upload the file + string etag; + using (var fileStream = File.OpenRead()) + { + etag = S3Client.PutObject( + BucketName, + finalObjectName, + fileStream, + fileContentType, + metadata, + bytesTransferred => + { + uploadResult.BytesTransferred = bytesTransferred; + + // Report progress every 1MB or at completion + if (bytesTransferred - lastReportedBytes >= 1024 * 1024 || bytesTransferred == fileSize) + { + var elapsed = DateTime.UtcNow - startTime; + var speed = elapsed.TotalSeconds > 0 ? bytesTransferred / elapsed.TotalSeconds : 0; + var percentage = fileSize > 0 ? (double)bytesTransferred / fileSize * 100 : 100; + + WriteVerboseMessage("Upload progress: {0:F1}% ({1}/{2}) at {3}", + percentage, + SizeFormatter.FormatBytes(bytesTransferred), + SizeFormatter.FormatBytes(fileSize), + SizeFormatter.FormatSpeed(speed)); + + lastReportedBytes = bytesTransferred; + } + }); + } + + uploadResult.ETag = etag; + uploadResult.MarkCompleted(); + + var duration = uploadResult.Duration ?? TimeSpan.Zero; + var averageSpeed = uploadResult.AverageSpeed ?? 0; + + WriteVerboseMessage("Upload completed in {0} at average speed of {1}", + SizeFormatter.FormatDuration(duration), + SizeFormatter.FormatSpeed(averageSpeed)); + + return uploadResult; + } + catch (Exception ex) { - try - { - var presignedUrl = Client.GetPresignedUrl(BucketName, objectName, Expiration); - uploadedObject.PresignedUrl = presignedUrl; - uploadedObject.PresignedUrlExpiration = DateTime.UtcNow.Add(Expiration); - } - catch (Exception ex) - { - WriteWarning($"Could not generate presigned URL for {objectName}: {ex.Message}"); - } + uploadResult.MarkFailed(ex.Message); + throw; } - uploadedObjects.Add(uploadedObject); - MinIOLogger.WriteVerbose(this, "Successfully uploaded: {0}", fileInfo.Name); - } - catch (Exception ex) + }, $"File: {File.FullName}, Bucket: {BucketName}, Object: {finalObjectName}"); + + // Return result if requested + if (PassThru.IsPresent) { - WriteError(new ErrorRecord( - ex, - "FileUploadFailed", - ErrorCategory.WriteError, - fileInfo)); - - MinIOLogger.WriteVerbose(this, "Failed to upload {0}: {1}", fileInfo.Name, ex.Message); - } - } - - // Complete progress reporting - overallProgress.Complete(); - WriteProgress(new ProgressRecord(2, "Current File", "Completed") - { - PercentComplete = 100, - RecordType = ProgressRecordType.Completed, - ParentActivityId = 1 - }); - - // Always return uploaded objects - foreach (var obj in uploadedObjects) - { - WriteObject(obj); - } - - MinIOLogger.WriteVerbose(this, "Completed uploading {0} files ({1} successful, {2} failed)", - validFiles.Length, uploadedObjects.Count, validFiles.Length - uploadedObjects.Count); - } - - /// - /// Gets the object name for a file based on the upload context - /// - /// File to get object name for - /// Object name to use in MinIO - private string GetObjectName(FileInfo file) - { - if (ParameterSetName == "Files") - { - // For file uploads, optionally prefix with bucket directory - var objectName = file.Name; - if (!string.IsNullOrWhiteSpace(BucketDirectory)) - { - var sanitizedDirectory = SanitizeBucketDirectory(BucketDirectory!); - objectName = $"{sanitizedDirectory}/{file.Name}"; - } - return objectName; - } - else if (ParameterSetName == "Directory") - { - if (Flatten.IsPresent) - { - // Flatten: use just the filename - return file.Name; + WriteObject(result); } else { - // Maintain directory structure relative to the base directory - var relativePath = file.FullName.Substring(Directory!.FullName.Length).TrimStart('\\', '/'); - return relativePath.Replace('\\', '/'); // Ensure forward slashes for object storage + WriteVerboseMessage("Successfully uploaded object '{0}' to bucket '{1}'", finalObjectName, BucketName); } } - - return file.Name; } /// - /// Sanitizes bucket directory path and ensures proper format + /// Gets the content type for a file based on its extension /// - /// Raw directory path - /// Sanitized directory path - private string SanitizeBucketDirectory(string directory) + /// File extension + /// Content type string + private static string GetContentType(string extension) { - if (string.IsNullOrWhiteSpace(directory)) - return string.Empty; - - // Remove leading and trailing slashes, then split and clean - var cleanDirectory = directory.Trim().Trim('/', '\\'); - - // Split by both forward and back slashes, then clean each part - var parts = cleanDirectory.Split(new char[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries); - - // Clean each part and rejoin with forward slashes - var cleanedParts = parts.Select(part => part.Trim()).Where(part => !string.IsNullOrEmpty(part)); - - return string.Join("/", cleanedParts); - } - - /// - /// Creates bucket directory structure if it doesn't exist - /// - /// Directory path to create - private void EnsureBucketDirectoryExists(string directoryPath) - { - if (string.IsNullOrWhiteSpace(directoryPath)) - return; - - var parts = directoryPath.Split('/'); - var currentPath = string.Empty; - - foreach (var part in parts) + return extension.ToLowerInvariant() switch { - currentPath = string.IsNullOrEmpty(currentPath) ? part : $"{currentPath}/{part}"; - var folderPath = $"{currentPath}/"; - - try - { - // Check if this directory level already exists by trying to list objects with this prefix - var existingObjects = Client.ListObjects(BucketName, folderPath, false); - var folderExists = existingObjects.Any(obj => obj.Name == folderPath); - - if (!folderExists) - { - MinIOLogger.WriteVerbose(this, "Creating bucket directory: {0}", folderPath); - - try - { - // Create the directory using the dedicated method - Client.CreateDirectory(BucketName, folderPath); - MinIOLogger.WriteVerbose(this, "Successfully created bucket directory: {0}", folderPath); - } - catch (Exception createEx) - { - // Directory creation failed, but this is not critical since MinIO creates directories implicitly - MinIOLogger.WriteVerbose(this, "Directory creation failed (non-critical): {0} - {1}", folderPath, createEx.Message); - } - } - } - catch (Exception ex) - { - // Listing failed, but this is not critical for the upload operation - MinIOLogger.WriteVerbose(this, "Could not check directory existence (non-critical): {0} - {1}", folderPath, ex.Message); - } - } + ".txt" => "text/plain", + ".html" or ".htm" => "text/html", + ".css" => "text/css", + ".js" => "application/javascript", + ".json" => "application/json", + ".xml" => "application/xml", + ".pdf" => "application/pdf", + ".zip" => "application/zip", + ".jpg" or ".jpeg" => "image/jpeg", + ".png" => "image/png", + ".gif" => "image/gif", + ".svg" => "image/svg+xml", + ".mp4" => "video/mp4", + ".mp3" => "audio/mpeg", + ".wav" => "audio/wav", + ".csv" => "text/csv", + ".doc" => "application/msword", + ".docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ".xls" => "application/vnd.ms-excel", + ".xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + _ => "application/octet-stream" + }; } } } diff --git a/src/Cmdlets/RemoveMinIOBucketCmdlet.cs b/src/Cmdlets/RemoveMinIOBucketCmdlet.cs deleted file mode 100644 index b17731e..0000000 --- a/src/Cmdlets/RemoveMinIOBucketCmdlet.cs +++ /dev/null @@ -1,190 +0,0 @@ -using System; -using System.Management.Automation; -using PSMinIO.Utils; - -namespace PSMinIO.Cmdlets -{ - /// - /// Removes a MinIO bucket - /// - [Cmdlet(VerbsCommon.Remove, "MinIOBucket", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.High)] - public class RemoveMinIOBucketCmdlet : MinIOBaseCmdlet - { - /// - /// Name of the bucket to remove - /// - [Parameter(Position = 0, Mandatory = true, ValueFromPipeline = true, ValueFromPipelineByPropertyName = true)] - [ValidateNotNullOrEmpty] - [Alias("Bucket")] - public string BucketName { get; set; } = string.Empty; - - /// - /// Force removal without confirmation prompts - /// - [Parameter] - public SwitchParameter Force { get; set; } - - /// - /// Remove all objects in the bucket before removing the bucket itself - /// - [Parameter] - [Alias("Recursive")] - public SwitchParameter RemoveObjects { get; set; } - - /// - /// Processes the cmdlet - /// - protected override void ProcessRecord() - { - ValidateConnection(); - ValidateBucketName(BucketName); - - // Force parameter is handled by ShouldProcess automatically - - var actionDescription = RemoveObjects.IsPresent - ? $"Remove bucket '{BucketName}' and all its objects" - : $"Remove bucket '{BucketName}'"; - - if (ShouldProcess(BucketName, actionDescription)) - { - ExecuteOperation("RemoveBucket", () => - { - // Check if bucket exists - var exists = Client.BucketExists(BucketName); - if (!exists) - { - WriteError(new ErrorRecord( - new InvalidOperationException($"Bucket '{BucketName}' does not exist"), - "BucketNotFound", - ErrorCategory.ObjectNotFound, - BucketName)); - return; - } - - // If RemoveObjects is specified, remove all objects first - if (RemoveObjects.IsPresent) - { - RemoveAllObjectsFromBucket(); - } - else - { - // Check if bucket is empty before attempting to remove - CheckBucketEmpty(); - } - - // Remove the bucket - MinIOLogger.WriteVerbose(this, "Removing bucket '{0}'", BucketName); - Client.DeleteBucket(BucketName); - MinIOLogger.WriteVerbose(this, "Successfully removed bucket '{0}'", BucketName); - - }, $"Bucket: {BucketName}"); - } - } - - /// - /// Removes all objects from the bucket - /// - private void RemoveAllObjectsFromBucket() - { - MinIOLogger.WriteVerbose(this, "Removing all objects from bucket '{0}'", BucketName); - - try - { - var objects = Client.ListObjects(BucketName, recursive: true); - - if (objects.Count == 0) - { - MinIOLogger.WriteVerbose(this, "Bucket '{0}' is already empty", BucketName); - return; - } - - MinIOLogger.WriteVerbose(this, "Found {0} objects to remove from bucket '{1}'", objects.Count, BucketName); - - // Show progress for object removal - for (int i = 0; i < objects.Count; i++) - { - var obj = objects[i]; - - var progressRecord = new ProgressRecord(1, "Removing Objects", - $"Removing object: {obj.Name}") - { - PercentComplete = (int)((double)(i + 1) / objects.Count * 100) - }; - WriteProgress(progressRecord); - - try - { - Client.DeleteObject(BucketName, obj.Name); - MinIOLogger.WriteVerbose(this, "Removed object: {0}", obj.Name); - } - catch (Exception ex) - { - MinIOLogger.WriteWarning(this, - "Failed to remove object '{0}': {1}", obj.Name, ex.Message); - } - } - - // Complete progress - WriteProgress(new ProgressRecord(1, "Removing Objects", "Completed") - { - PercentComplete = 100, - RecordType = ProgressRecordType.Completed - }); - - MinIOLogger.WriteVerbose(this, "Finished removing objects from bucket '{0}'", BucketName); - } - catch (Exception ex) - { - MinIOLogger.WriteWarning(this, - "Failed to list or remove objects from bucket '{0}': {1}", BucketName, ex.Message); - - if (!Force.IsPresent) - { - WriteError(new ErrorRecord( - new InvalidOperationException($"Cannot remove objects from bucket '{BucketName}': {ex.Message}"), - "ObjectRemovalFailed", - ErrorCategory.InvalidOperation, - BucketName)); - return; - } - } - } - - /// - /// Checks if the bucket is empty and warns if it's not - /// - private void CheckBucketEmpty() - { - try - { - var objects = Client.ListObjects(BucketName, recursive: true); - - if (objects.Count > 0) - { - var message = $"Bucket '{BucketName}' contains {objects.Count} objects. " + - "Use -RemoveObjects parameter to remove all objects first, or remove them manually."; - - WriteError(new ErrorRecord( - new InvalidOperationException(message), - "BucketNotEmpty", - ErrorCategory.InvalidOperation, - BucketName)); - } - } - catch (Exception ex) - { - MinIOLogger.WriteWarning(this, - "Could not check if bucket '{0}' is empty: {1}", BucketName, ex.Message); - - if (!Force.IsPresent) - { - WriteError(new ErrorRecord( - new InvalidOperationException($"Cannot verify bucket '{BucketName}' is empty: {ex.Message}"), - "BucketCheckFailed", - ErrorCategory.InvalidOperation, - BucketName)); - } - } - } - } -} diff --git a/src/Cmdlets/RemoveMinIOObjectCmdlet.cs b/src/Cmdlets/RemoveMinIOObjectCmdlet.cs deleted file mode 100644 index b42b957..0000000 --- a/src/Cmdlets/RemoveMinIOObjectCmdlet.cs +++ /dev/null @@ -1,225 +0,0 @@ -using System; -using System.Linq; -using System.Management.Automation; -using PSMinIO.Utils; - -namespace PSMinIO.Cmdlets -{ - /// - /// Removes an object from a MinIO bucket - /// - [Cmdlet(VerbsCommon.Remove, "MinIOObject", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.High)] - public class RemoveMinIOObjectCmdlet : MinIOBaseCmdlet - { - /// - /// Name of the bucket containing the object - /// - [Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true)] - [ValidateNotNullOrEmpty] - [Alias("Bucket")] - public string BucketName { get; set; } = string.Empty; - - /// - /// Name of the object to remove - /// - [Parameter(Position = 1, Mandatory = true, ValueFromPipelineByPropertyName = true)] - [ValidateNotNullOrEmpty] - [Alias("Object", "Key")] - public string ObjectName { get; set; } = string.Empty; - - /// - /// Force removal without confirmation prompts - /// - [Parameter] - public SwitchParameter Force { get; set; } - - /// - /// Remove all objects matching the specified prefix (use with caution) - /// - [Parameter] - [Alias("Recursive")] - public SwitchParameter RemovePrefix { get; set; } - - /// - /// Processes the cmdlet - /// - protected override void ProcessRecord() - { - ValidateConnection(); - ValidateBucketName(BucketName); - ValidateObjectName(ObjectName); - - // Force parameter is handled by ShouldProcess automatically - - var actionDescription = RemovePrefix.IsPresent - ? $"Remove all objects with prefix '{ObjectName}' from bucket '{BucketName}'" - : $"Remove object '{ObjectName}' from bucket '{BucketName}'"; - - if (ShouldProcess($"{BucketName}/{ObjectName}", actionDescription)) - { - ExecuteOperation("RemoveObject", () => - { - // Check if bucket exists - var bucketExists = Client.BucketExists(BucketName); - if (!bucketExists) - { - WriteError(new ErrorRecord( - new InvalidOperationException($"Bucket '{BucketName}' does not exist"), - "BucketNotFound", - ErrorCategory.ObjectNotFound, - BucketName)); - return; - } - - if (RemovePrefix.IsPresent) - { - RemoveObjectsWithPrefix(); - } - else - { - RemoveSingleObject(); - } - - }, $"Bucket: {BucketName}, Object: {ObjectName}"); - } - } - - /// - /// Removes a single object - /// - private void RemoveSingleObject() - { - // Check if object exists - var objectExists = CheckObjectExists(); - if (!objectExists) - { - WriteError(new ErrorRecord( - new InvalidOperationException($"Object '{ObjectName}' does not exist in bucket '{BucketName}'"), - "ObjectNotFound", - ErrorCategory.ObjectNotFound, - ObjectName)); - return; - } - - MinIOLogger.WriteVerbose(this, - "Removing object '{0}' from bucket '{1}'", ObjectName, BucketName); - - Client.DeleteObject(BucketName, ObjectName); - - MinIOLogger.WriteVerbose(this, - "Successfully removed object '{0}' from bucket '{1}'", ObjectName, BucketName); - } - - /// - /// Removes all objects with the specified prefix - /// - private void RemoveObjectsWithPrefix() - { - MinIOLogger.WriteVerbose(this, - "Finding objects with prefix '{0}' in bucket '{1}'", ObjectName, BucketName); - - try - { - var objects = Client.ListObjects(BucketName, ObjectName, true); - - if (objects.Count == 0) - { - MinIOLogger.WriteVerbose(this, - "No objects found with prefix '{0}' in bucket '{1}'", ObjectName, BucketName); - return; - } - - MinIOLogger.WriteVerbose(this, - "Found {0} objects with prefix '{1}' in bucket '{2}'", - objects.Count, ObjectName, BucketName); - - // Additional confirmation for prefix removal if not forced - if (!Force.IsPresent && objects.Count > 1) - { - var message = $"This will remove {objects.Count} objects with prefix '{ObjectName}'. Are you sure?"; - if (!ShouldContinue(message, "Confirm Multiple Object Removal")) - { - return; - } - } - - // Remove objects with progress reporting - for (int i = 0; i < objects.Count; i++) - { - var obj = objects[i]; - - var progressRecord = new ProgressRecord(1, "Removing Objects", - $"Removing object: {obj.Name}") - { - PercentComplete = (int)((double)(i + 1) / objects.Count * 100) - }; - WriteProgress(progressRecord); - - try - { - Client.DeleteObject(BucketName, obj.Name); - MinIOLogger.WriteVerbose(this, "Removed object: {0}", obj.Name); - } - catch (Exception ex) - { - MinIOLogger.WriteWarning(this, - "Failed to remove object '{0}': {1}", obj.Name, ex.Message); - - if (!Force.IsPresent) - { - WriteError(new ErrorRecord( - new InvalidOperationException($"Failed to remove object '{obj.Name}': {ex.Message}", ex), - "ObjectRemovalFailed", - ErrorCategory.InvalidOperation, - obj.Name)); - } - } - } - - // Complete progress - WriteProgress(new ProgressRecord(1, "Removing Objects", "Completed") - { - PercentComplete = 100, - RecordType = ProgressRecordType.Completed - }); - - MinIOLogger.WriteVerbose(this, - "Finished removing objects with prefix '{0}' from bucket '{1}'", ObjectName, BucketName); - } - catch (Exception ex) - { - MinIOLogger.WriteWarning(this, - "Failed to list objects with prefix '{0}' in bucket '{1}': {2}", - ObjectName, BucketName, ex.Message); - - WriteError(new ErrorRecord( - new InvalidOperationException($"Cannot list objects with prefix '{ObjectName}': {ex.Message}", ex), - "ObjectListingFailed", - ErrorCategory.InvalidOperation, - ObjectName)); - } - } - - /// - /// Checks if the specified object exists - /// - /// True if object exists, false otherwise - private bool CheckObjectExists() - { - try - { - var objects = Client.ListObjects(BucketName, ObjectName, false); - return objects.Any(obj => - string.Equals(obj.Name, ObjectName, StringComparison.Ordinal)); - } - catch (Exception ex) - { - MinIOLogger.WriteWarning(this, - "Could not check if object '{0}' exists: {1}", ObjectName, ex.Message); - - // If we can't verify existence, assume it exists and let the delete operation handle it - return true; - } - } - } -} diff --git a/src/Cmdlets/SetMinIOBucketPolicyCmdlet.cs b/src/Cmdlets/SetMinIOBucketPolicyCmdlet.cs deleted file mode 100644 index 8d4e2f5..0000000 --- a/src/Cmdlets/SetMinIOBucketPolicyCmdlet.cs +++ /dev/null @@ -1,332 +0,0 @@ -using System; -using System.IO; -using System.Management.Automation; -using System.Text.Json; -using PSMinIO.Utils; - -namespace PSMinIO.Cmdlets -{ - /// - /// Sets the policy for a MinIO bucket - /// - [Cmdlet(VerbsCommon.Set, "MinIOBucketPolicy", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.Medium)] - public class SetMinIOBucketPolicyCmdlet : MinIOBaseCmdlet - { - /// - /// Name of the bucket to set the policy for - /// - [Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true)] - [ValidateNotNullOrEmpty] - [Alias("Bucket")] - public string BucketName { get; set; } = string.Empty; - - /// - /// Policy JSON string - /// - [Parameter(Position = 1, Mandatory = true, ParameterSetName = "PolicyJson")] - [ValidateNotNullOrEmpty] - [Alias("Json")] - public string? PolicyJson { get; set; } - - /// - /// Path to a file containing the policy JSON - /// - [Parameter(Mandatory = true, ParameterSetName = "PolicyFile")] - [ValidateNotNullOrEmpty] - [Alias("File")] - public string? PolicyFilePath { get; set; } - - /// - /// Use a predefined canned policy - /// - [Parameter(Mandatory = true, ParameterSetName = "CannedPolicy")] - [ValidateSet("ReadOnly", "WriteOnly", "ReadWrite", "None")] - public string? CannedPolicy { get; set; } - - /// - /// Prefix for canned policies (default: *) - /// - [Parameter(ParameterSetName = "CannedPolicy")] - [ValidateNotNullOrEmpty] - public string Prefix { get; set; } = "*"; - - /// - /// Validate the policy JSON before setting it - /// - [Parameter] - public SwitchParameter ValidateOnly { get; set; } - - /// - /// Force setting the policy without confirmation - /// - [Parameter] - public SwitchParameter Force { get; set; } - - /// - /// Processes the cmdlet - /// - protected override void ProcessRecord() - { - ValidateConnection(); - ValidateBucketName(BucketName); - - // Get the policy JSON based on the parameter set - var policyJson = GetPolicyJson(); - if (string.IsNullOrWhiteSpace(policyJson)) - { - return; // Error already written - } - - // Validate the policy JSON - if (!ValidatePolicyJson(policyJson!)) - { - return; // Error already written - } - - if (ValidateOnly.IsPresent) - { - WriteObject("Policy JSON is valid"); - return; - } - - // Force parameter is handled by ShouldProcess automatically - - if (ShouldProcess(BucketName, "Set bucket policy")) - { - ExecuteOperation("SetBucketPolicy", () => - { - // Check if bucket exists - var bucketExists = Client.BucketExists(BucketName); - if (!bucketExists) - { - WriteError(new ErrorRecord( - new InvalidOperationException($"Bucket '{BucketName}' does not exist"), - "BucketNotFound", - ErrorCategory.ObjectNotFound, - BucketName)); - return; - } - - MinIOLogger.WriteVerbose(this, "Setting policy for bucket '{0}'", BucketName); - MinIOLogger.WriteVerbose(this, "Policy JSON ({0} characters): {1}", - policyJson!.Length, policyJson.Length > 200 ? policyJson.Substring(0, 200) + "..." : policyJson); - - Client.SetBucketPolicy(BucketName, policyJson); - - MinIOLogger.WriteVerbose(this, "Successfully set policy for bucket '{0}'", BucketName); - - }, $"Bucket: {BucketName}"); - } - } - - /// - /// Gets the policy JSON based on the current parameter set - /// - /// Policy JSON string or null if error - private string? GetPolicyJson() - { - switch (ParameterSetName) - { - case "PolicyJson": - return PolicyJson; - - case "PolicyFile": - return ReadPolicyFromFile(); - - case "CannedPolicy": - return GenerateCannedPolicy(); - - default: - WriteError(new ErrorRecord( - new InvalidOperationException("Unknown parameter set"), - "UnknownParameterSet", - ErrorCategory.InvalidArgument, - null)); - return null; - } - } - - /// - /// Reads policy JSON from a file - /// - /// Policy JSON string or null if error - private string? ReadPolicyFromFile() - { - if (string.IsNullOrWhiteSpace(PolicyFilePath)) - return null; - - try - { - var fullPath = Path.GetFullPath(PolicyFilePath); - - if (!File.Exists(fullPath)) - { - WriteError(new ErrorRecord( - new FileNotFoundException($"Policy file not found: {fullPath}"), - "PolicyFileNotFound", - ErrorCategory.ObjectNotFound, - fullPath)); - return null; - } - - var content = File.ReadAllText(fullPath); - MinIOLogger.WriteVerbose(this, "Read policy from file '{0}' ({1} characters)", fullPath, content.Length); - - return content; - } - catch (Exception ex) - { - WriteError(new ErrorRecord( - new InvalidOperationException($"Failed to read policy file '{PolicyFilePath}': {ex.Message}", ex), - "PolicyFileReadError", - ErrorCategory.ReadError, - PolicyFilePath)); - return null; - } - } - - /// - /// Generates a canned policy based on the specified type - /// - /// Policy JSON string - private string GenerateCannedPolicy() - { - var bucketArn = $"arn:aws:s3:::{BucketName}"; - var objectArn = $"arn:aws:s3:::{BucketName}/{Prefix}"; - - var policy = CannedPolicy?.ToLowerInvariant() switch - { - "readonly" => CreateReadOnlyPolicy(bucketArn, objectArn), - "writeonly" => CreateWriteOnlyPolicy(bucketArn, objectArn), - "readwrite" => CreateReadWritePolicy(bucketArn, objectArn), - "none" => CreateEmptyPolicy(), - _ => throw new ArgumentException($"Unknown canned policy: {CannedPolicy}") - }; - - MinIOLogger.WriteVerbose(this, "Generated {0} canned policy for bucket '{1}' with prefix '{2}'", - CannedPolicy, BucketName, Prefix); - - return JsonSerializer.Serialize(policy, new JsonSerializerOptions { WriteIndented = true }); - } - - /// - /// Creates a read-only policy - /// - private object CreateReadOnlyPolicy(string bucketArn, string objectArn) - { - return new - { - Version = "2012-10-17", - Statement = new[] - { - new - { - Effect = "Allow", - Principal = new { AWS = "*" }, - Action = new[] { "s3:GetBucketLocation", "s3:ListBucket" }, - Resource = bucketArn - }, - new - { - Effect = "Allow", - Principal = new { AWS = "*" }, - Action = new[] { "s3:GetObject" }, - Resource = objectArn - } - } - }; - } - - /// - /// Creates a write-only policy - /// - private object CreateWriteOnlyPolicy(string bucketArn, string objectArn) - { - return new - { - Version = "2012-10-17", - Statement = new[] - { - new - { - Effect = "Allow", - Principal = new { AWS = "*" }, - Action = new[] { "s3:GetBucketLocation", "s3:ListBucketMultipartUploads" }, - Resource = bucketArn - }, - new - { - Effect = "Allow", - Principal = new { AWS = "*" }, - Action = new[] { "s3:PutObject", "s3:AbortMultipartUpload", "s3:DeleteObject", "s3:ListMultipartUploadParts" }, - Resource = objectArn - } - } - }; - } - - /// - /// Creates a read-write policy - /// - private object CreateReadWritePolicy(string bucketArn, string objectArn) - { - return new - { - Version = "2012-10-17", - Statement = new[] - { - new - { - Effect = "Allow", - Principal = new { AWS = "*" }, - Action = new[] { "s3:GetBucketLocation", "s3:ListBucket", "s3:ListBucketMultipartUploads" }, - Resource = bucketArn - }, - new - { - Effect = "Allow", - Principal = new { AWS = "*" }, - Action = new[] { "s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts" }, - Resource = objectArn - } - } - }; - } - - /// - /// Creates an empty policy (removes all permissions) - /// - private object CreateEmptyPolicy() - { - return new - { - Version = "2012-10-17", - Statement = new object[0] - }; - } - - /// - /// Validates the policy JSON - /// - /// Policy JSON to validate - /// True if valid, false otherwise - private bool ValidatePolicyJson(string policyJson) - { - try - { - using var document = JsonDocument.Parse(policyJson); - MinIOLogger.WriteVerbose(this, "Policy JSON is valid"); - return true; - } - catch (JsonException ex) - { - WriteError(new ErrorRecord( - new ArgumentException($"Invalid policy JSON: {ex.Message}", ex), - "InvalidPolicyJson", - ErrorCategory.InvalidArgument, - policyJson)); - return false; - } - } - } -} diff --git a/src/Cmdlets/TestMinIOBucketExistsCmdlet.cs b/src/Cmdlets/TestMinIOBucketExistsCmdlet.cs index 55ed548..0ec9aac 100644 --- a/src/Cmdlets/TestMinIOBucketExistsCmdlet.cs +++ b/src/Cmdlets/TestMinIOBucketExistsCmdlet.cs @@ -1,12 +1,11 @@ using System.Management.Automation; -using PSMinIO.Utils; namespace PSMinIO.Cmdlets { /// /// Tests whether a MinIO bucket exists /// - [Cmdlet(VerbsDiagnostic.Test, "MinIOBucketExists", SupportsShouldProcess = false)] + [Cmdlet(VerbsDiagnostic.Test, "MinIOBucketExists")] [OutputType(typeof(bool))] public class TestMinIOBucketExistsCmdlet : MinIOBaseCmdlet { @@ -16,137 +15,27 @@ namespace PSMinIO.Cmdlets [Parameter(Position = 0, Mandatory = true, ValueFromPipeline = true, ValueFromPipelineByPropertyName = true)] [ValidateNotNullOrEmpty] [Alias("Bucket")] - public string BucketName { get; set; } = string.Empty; - - /// - /// Return detailed information instead of just true/false - /// - [Parameter] - public SwitchParameter Detailed { get; set; } + public string Name { get; set; } = string.Empty; /// /// Processes the cmdlet /// protected override void ProcessRecord() { - ValidateConnection(); - ValidateBucketName(BucketName); + ValidateBucketName(Name); - ExecuteOperation("TestBucketExists", () => + var exists = ExecuteOperation("CheckBucketExists", () => { - MinIOLogger.WriteVerbose(this, "Checking if bucket '{0}' exists", BucketName); + WriteVerboseMessage("Checking if bucket '{0}' exists", Name); - var exists = Client.BucketExists(BucketName); + var bucketExists = S3Client.BucketExists(Name); + + WriteVerboseMessage("Bucket '{0}' {1}", Name, bucketExists ? "exists" : "does not exist"); + + return bucketExists; + }, $"Bucket: {Name}"); - if (Detailed.IsPresent) - { - // Return detailed information - var result = new PSObject(); - result.Properties.Add(new PSNoteProperty("BucketName", BucketName)); - result.Properties.Add(new PSNoteProperty("Exists", exists)); - result.Properties.Add(new PSNoteProperty("Endpoint", Configuration.Endpoint)); - result.Properties.Add(new PSNoteProperty("CheckedAt", System.DateTime.UtcNow)); - - if (exists) - { - try - { - // Try to get additional bucket information - var allBuckets = Client.ListBuckets(); - var bucket = allBuckets.Find(b => - string.Equals(b.Name, BucketName, System.StringComparison.OrdinalIgnoreCase)); - - if (bucket != null) - { - result.Properties.Add(new PSNoteProperty("Created", bucket.Created)); - result.Properties.Add(new PSNoteProperty("Region", bucket.Region)); - } - } - catch (System.Exception ex) - { - MinIOLogger.WriteVerbose(this, - "Could not retrieve additional bucket information: {0}", ex.Message); - } - } - - WriteObject(result); - } - else - { - // Return simple boolean result - WriteObject(exists); - } - - MinIOLogger.WriteVerbose(this, - "Bucket '{0}' exists: {1}", BucketName, exists); - - }, $"Bucket: {BucketName}"); - } - } - - /// - /// Custom output type for detailed bucket existence information - /// - public class BucketExistenceInfo - { - /// - /// Name of the bucket that was tested - /// - public string BucketName { get; set; } = string.Empty; - - /// - /// Whether the bucket exists - /// - public bool Exists { get; set; } - - /// - /// MinIO endpoint that was checked - /// - public string Endpoint { get; set; } = string.Empty; - - /// - /// When the check was performed - /// - public System.DateTime CheckedAt { get; set; } - - /// - /// Creation date of the bucket (if it exists and information is available) - /// - public System.DateTime? Created { get; set; } - - /// - /// Region of the bucket (if it exists and information is available) - /// - public string? Region { get; set; } - - /// - /// Creates a new BucketExistenceInfo instance - /// - public BucketExistenceInfo() - { - CheckedAt = System.DateTime.UtcNow; - } - - /// - /// Creates a new BucketExistenceInfo instance with specified values - /// - /// Name of the bucket - /// Whether the bucket exists - /// MinIO endpoint - public BucketExistenceInfo(string bucketName, bool exists, string endpoint) - { - BucketName = bucketName ?? string.Empty; - Exists = exists; - Endpoint = endpoint ?? string.Empty; - CheckedAt = System.DateTime.UtcNow; - } - - /// - /// Returns a string representation of the bucket existence info - /// - public override string ToString() - { - return $"Bucket '{BucketName}' exists: {Exists} (checked at {CheckedAt:yyyy-MM-dd HH:mm:ss} UTC)"; + WriteObject(exists); } } } diff --git a/src/Core/Http/MinIOHttpClient.cs b/src/Core/Http/MinIOHttpClient.cs new file mode 100644 index 0000000..ae57ce7 --- /dev/null +++ b/src/Core/Http/MinIOHttpClient.cs @@ -0,0 +1,344 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using PSMinIO.Utils; + +namespace PSMinIO.Core.Http +{ + /// + /// Custom HTTP client for MinIO REST API operations with AWS S3 signature support + /// Provides synchronous operations optimized for PowerShell with progress reporting + /// + public class MinIOHttpClient : IDisposable + { + private readonly HttpClient _httpClient; + private readonly MinIOConfiguration _configuration; + private bool _disposed = false; + + /// + /// Creates a new MinIOHttpClient instance + /// + /// MinIO configuration + public MinIOHttpClient(MinIOConfiguration configuration) + { + _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + + if (!_configuration.IsValid) + { + var errors = string.Join(", ", _configuration.GetValidationErrors()); + throw new ArgumentException($"Invalid MinIO configuration: {errors}", nameof(configuration)); + } + + // Create HTTP client with custom configuration + var handler = new HttpClientHandler(); + + // Configure certificate validation if needed + if (_configuration.SkipCertificateValidation) + { + handler.ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => true; + } + + _httpClient = new HttpClient(handler) + { + Timeout = TimeSpan.FromSeconds(_configuration.TimeoutSeconds), + BaseAddress = new Uri(_configuration.BaseUrl) + }; + + // Set default headers + _httpClient.DefaultRequestHeaders.Add("User-Agent", "PSMinIO/2.0.0"); + } + + /// + /// Executes a synchronous HTTP request with AWS S3 signature + /// + /// HTTP method + /// Request path + /// Query parameters + /// Additional headers + /// Request content + /// Progress callback for uploads/downloads + /// HTTP response + public HttpResponseMessage ExecuteRequest( + HttpMethod method, + string path, + Dictionary? queryParameters = null, + Dictionary? headers = null, + HttpContent? content = null, + Action? progressCallback = null) + { + var request = CreateRequest(method, path, queryParameters, headers, content); + + // Sign the request with AWS S3 signature + SignRequest(request); + + // Execute the request synchronously + var response = _httpClient.SendAsync(request).GetAwaiter().GetResult(); + + return response; + } + + /// + /// Executes a synchronous HTTP request and returns the response content as string + /// + /// HTTP method + /// Request path + /// Query parameters + /// Additional headers + /// Request content + /// Response content as string + public string ExecuteRequestForString( + HttpMethod method, + string path, + Dictionary? queryParameters = null, + Dictionary? headers = null, + HttpContent? content = null) + { + using var response = ExecuteRequest(method, path, queryParameters, headers, content); + response.EnsureSuccessStatusCode(); + return response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); + } + + /// + /// Executes a synchronous HTTP request and returns the response content as byte array + /// + /// HTTP method + /// Request path + /// Query parameters + /// Additional headers + /// Request content + /// Response content as byte array + public byte[] ExecuteRequestForBytes( + HttpMethod method, + string path, + Dictionary? queryParameters = null, + Dictionary? headers = null, + HttpContent? content = null) + { + using var response = ExecuteRequest(method, path, queryParameters, headers, content); + response.EnsureSuccessStatusCode(); + return response.Content.ReadAsByteArrayAsync().GetAwaiter().GetResult(); + } + + /// + /// Downloads content to a stream with progress reporting + /// + /// Request path + /// Output stream + /// Query parameters + /// Additional headers + /// Progress callback + /// Total bytes downloaded + public long DownloadToStream( + string path, + Stream outputStream, + Dictionary? queryParameters = null, + Dictionary? headers = null, + Action? progressCallback = null) + { + using var response = ExecuteRequest(HttpMethod.Get, path, queryParameters, headers); + response.EnsureSuccessStatusCode(); + + using var contentStream = response.Content.ReadAsStreamAsync().GetAwaiter().GetResult(); + return CopyStreamWithProgress(contentStream, outputStream, progressCallback); + } + + /// + /// Uploads content from a stream with progress reporting + /// + /// HTTP method (PUT or POST) + /// Request path + /// Input stream + /// Content type + /// Query parameters + /// Additional headers + /// Progress callback + /// HTTP response + public HttpResponseMessage UploadFromStream( + HttpMethod method, + string path, + Stream inputStream, + string contentType = "application/octet-stream", + Dictionary? queryParameters = null, + Dictionary? headers = null, + Action? progressCallback = null) + { + var content = new ProgressStreamContent(inputStream, contentType, progressCallback); + return ExecuteRequest(method, path, queryParameters, headers, content); + } + + /// + /// Creates an HTTP request message + /// + private HttpRequestMessage CreateRequest( + HttpMethod method, + string path, + Dictionary? queryParameters, + Dictionary? headers, + HttpContent? content) + { + // Build the request URI + var uriBuilder = new UriBuilder(_configuration.BaseUrl) + { + Path = path + }; + + if (queryParameters != null && queryParameters.Count > 0) + { + var query = new StringBuilder(); + foreach (var kvp in queryParameters) + { + if (query.Length > 0) query.Append("&"); + query.Append($"{Uri.EscapeDataString(kvp.Key)}={Uri.EscapeDataString(kvp.Value)}"); + } + uriBuilder.Query = query.ToString(); + } + + var request = new HttpRequestMessage(method, uriBuilder.Uri) + { + Content = content + }; + + // Add custom headers + if (headers != null) + { + foreach (var kvp in headers) + { + request.Headers.TryAddWithoutValidation(kvp.Key, kvp.Value); + } + } + + return request; + } + + /// + /// Signs the HTTP request with AWS S3 signature v4 + /// + private void SignRequest(HttpRequestMessage request) + { + var now = DateTime.UtcNow; + var dateStamp = now.ToString("yyyyMMdd"); + var timeStamp = now.ToString("yyyyMMddTHHmmssZ"); + + // Add required headers + request.Headers.TryAddWithoutValidation("x-amz-date", timeStamp); + request.Headers.TryAddWithoutValidation("x-amz-content-sha256", "UNSIGNED-PAYLOAD"); + + // Create authorization header + var authHeader = CreateAuthorizationHeader(request, dateStamp, timeStamp); + request.Headers.TryAddWithoutValidation("Authorization", authHeader); + } + + /// + /// Creates the AWS S3 authorization header + /// + private string CreateAuthorizationHeader(HttpRequestMessage request, string dateStamp, string timeStamp) + { + var algorithm = "AWS4-HMAC-SHA256"; + var credentialScope = $"{dateStamp}/{_configuration.Region}/s3/aws4_request"; + var credential = $"{_configuration.AccessKey}/{credentialScope}"; + + // Create canonical request + var canonicalRequest = CreateCanonicalRequest(request); + var canonicalRequestHash = ComputeSHA256Hash(canonicalRequest); + + // Create string to sign + var stringToSign = $"{algorithm}\n{timeStamp}\n{credentialScope}\n{canonicalRequestHash}"; + + // Calculate signature + var signature = CalculateSignature(stringToSign, dateStamp); + + return $"{algorithm} Credential={credential}, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature={signature}"; + } + + /// + /// Creates the canonical request for AWS S3 signature + /// + private string CreateCanonicalRequest(HttpRequestMessage request) + { + var method = request.Method.Method; + var path = request.RequestUri?.AbsolutePath ?? "/"; + var query = request.RequestUri?.Query?.TrimStart('?') ?? ""; + + // Canonical headers (must be sorted) + var canonicalHeaders = "host:" + request.RequestUri?.Host + "\n" + + "x-amz-content-sha256:UNSIGNED-PAYLOAD\n" + + "x-amz-date:" + request.Headers.GetValues("x-amz-date").FirstOrDefault() + "\n"; + + var signedHeaders = "host;x-amz-content-sha256;x-amz-date"; + var payloadHash = "UNSIGNED-PAYLOAD"; + + return $"{method}\n{path}\n{query}\n{canonicalHeaders}\n{signedHeaders}\n{payloadHash}"; + } + + /// + /// Calculates the AWS S3 signature + /// + private string CalculateSignature(string stringToSign, string dateStamp) + { + var kSecret = Encoding.UTF8.GetBytes($"AWS4{_configuration.SecretKey}"); + var kDate = ComputeHMACSHA256(kSecret, dateStamp); + var kRegion = ComputeHMACSHA256(kDate, _configuration.Region); + var kService = ComputeHMACSHA256(kRegion, "s3"); + var kSigning = ComputeHMACSHA256(kService, "aws4_request"); + var signature = ComputeHMACSHA256(kSigning, stringToSign); + + return BitConverter.ToString(signature).Replace("-", "").ToLowerInvariant(); + } + + /// + /// Computes SHA256 hash + /// + private string ComputeSHA256Hash(string input) + { + using var sha256 = SHA256.Create(); + var hash = sha256.ComputeHash(Encoding.UTF8.GetBytes(input)); + return BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant(); + } + + /// + /// Computes HMAC-SHA256 + /// + private byte[] ComputeHMACSHA256(byte[] key, string data) + { + using var hmac = new HMACSHA256(key); + return hmac.ComputeHash(Encoding.UTF8.GetBytes(data)); + } + + /// + /// Copies stream with progress reporting + /// + private long CopyStreamWithProgress(Stream source, Stream destination, Action? progressCallback) + { + var buffer = new byte[8192]; + long totalBytes = 0; + int bytesRead; + + while ((bytesRead = source.Read(buffer, 0, buffer.Length)) > 0) + { + destination.Write(buffer, 0, bytesRead); + totalBytes += bytesRead; + progressCallback?.Invoke(totalBytes); + } + + return totalBytes; + } + + /// + /// Disposes the HTTP client + /// + public void Dispose() + { + if (!_disposed) + { + _httpClient?.Dispose(); + _disposed = true; + } + } + } +} diff --git a/src/Core/Http/ProgressStreamContent.cs b/src/Core/Http/ProgressStreamContent.cs new file mode 100644 index 0000000..24f5e8b --- /dev/null +++ b/src/Core/Http/ProgressStreamContent.cs @@ -0,0 +1,88 @@ +using System; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; + +namespace PSMinIO.Core.Http +{ + /// + /// HTTP content that wraps a stream and provides progress reporting during upload + /// + public class ProgressStreamContent : HttpContent + { + private readonly Stream _stream; + private readonly Action? _progressCallback; + private readonly long _totalBytes; + + /// + /// Creates a new ProgressStreamContent instance + /// + /// Source stream + /// Content type + /// Progress callback + public ProgressStreamContent(Stream stream, string contentType, Action? progressCallback = null) + { + _stream = stream ?? throw new ArgumentNullException(nameof(stream)); + _progressCallback = progressCallback; + _totalBytes = stream.CanSeek ? stream.Length : -1; + + Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue(contentType); + if (_totalBytes >= 0) + { + Headers.ContentLength = _totalBytes; + } + } + + /// + /// Serializes the HTTP content to a stream + /// + /// Target stream + /// Transport context + /// Task representing the async operation + protected override async Task SerializeToStreamAsync(Stream stream, TransportContext? context) + { + const int bufferSize = 8192; + var buffer = new byte[bufferSize]; + long totalBytesRead = 0; + int bytesRead; + + // Reset source stream position if possible + if (_stream.CanSeek) + { + _stream.Position = 0; + } + + while ((bytesRead = await _stream.ReadAsync(buffer, 0, buffer.Length)) > 0) + { + await stream.WriteAsync(buffer, 0, bytesRead); + totalBytesRead += bytesRead; + _progressCallback?.Invoke(totalBytesRead); + } + } + + /// + /// Determines whether the HTTP content has a valid length in bytes + /// + /// Content length + /// True if length is valid + protected override bool TryComputeLength(out long length) + { + length = _totalBytes; + return _totalBytes >= 0; + } + + /// + /// Disposes the content + /// + /// Whether disposing + protected override void Dispose(bool disposing) + { + if (disposing) + { + _stream?.Dispose(); + } + base.Dispose(disposing); + } + } +} diff --git a/src/Core/MinIOConfiguration.cs b/src/Core/MinIOConfiguration.cs new file mode 100644 index 0000000..2a7142d --- /dev/null +++ b/src/Core/MinIOConfiguration.cs @@ -0,0 +1,154 @@ +using System; +using System.Collections.Generic; + +namespace PSMinIO.Core +{ + /// + /// Configuration settings for MinIO client connections + /// + public class MinIOConfiguration + { + /// + /// MinIO server endpoint (e.g., "minio.example.com:9000") + /// + public string Endpoint { get; set; } = string.Empty; + + /// + /// Access key for authentication + /// + public string AccessKey { get; set; } = string.Empty; + + /// + /// Secret key for authentication + /// + public string SecretKey { get; set; } = string.Empty; + + /// + /// Whether to use SSL/TLS for connections + /// + public bool UseSSL { get; set; } = true; + + /// + /// Optional region for bucket operations + /// + public string Region { get; set; } = "us-east-1"; + + /// + /// Connection timeout in seconds + /// + public int TimeoutSeconds { get; set; } = 30; + + /// + /// Whether to skip SSL certificate validation (for development/self-signed certificates) + /// + public bool SkipCertificateValidation { get; set; } = false; + + /// + /// Maximum number of concurrent connections + /// + public int MaxConnections { get; set; } = 10; + + /// + /// Default chunk size for multipart uploads (in bytes) + /// + public long DefaultChunkSize { get; set; } = 5 * 1024 * 1024; // 5MB + + /// + /// Maximum retry attempts for failed operations + /// + public int MaxRetries { get; set; } = 3; + + /// + /// Creates a new MinIOConfiguration instance + /// + public MinIOConfiguration() + { + } + + /// + /// Creates a new MinIOConfiguration instance with specified values + /// + /// MinIO server endpoint + /// Access key + /// Secret key + /// Whether to use SSL + /// Optional region + /// Connection timeout + /// Whether to skip certificate validation + public MinIOConfiguration( + string endpoint, + string accessKey, + string secretKey, + bool useSSL = true, + string region = "us-east-1", + int timeoutSeconds = 30, + bool skipCertificateValidation = false) + { + Endpoint = endpoint ?? throw new ArgumentNullException(nameof(endpoint)); + AccessKey = accessKey ?? throw new ArgumentNullException(nameof(accessKey)); + SecretKey = secretKey ?? throw new ArgumentNullException(nameof(secretKey)); + UseSSL = useSSL; + Region = region ?? "us-east-1"; + TimeoutSeconds = timeoutSeconds; + SkipCertificateValidation = skipCertificateValidation; + } + + /// + /// Gets the base URL for the MinIO server + /// + public string BaseUrl => $"{(UseSSL ? "https" : "http")}://{Endpoint}"; + + /// + /// Validates the configuration + /// + /// True if configuration is valid + public bool IsValid => + !string.IsNullOrWhiteSpace(Endpoint) && + !string.IsNullOrWhiteSpace(AccessKey) && + !string.IsNullOrWhiteSpace(SecretKey) && + TimeoutSeconds > 0 && + MaxConnections > 0 && + DefaultChunkSize > 0 && + MaxRetries >= 0; + + /// + /// Gets validation errors for the configuration + /// + /// Array of validation error messages + public string[] GetValidationErrors() + { + var errors = new List(); + + if (string.IsNullOrWhiteSpace(Endpoint)) + errors.Add("Endpoint is required"); + + if (string.IsNullOrWhiteSpace(AccessKey)) + errors.Add("AccessKey is required"); + + if (string.IsNullOrWhiteSpace(SecretKey)) + errors.Add("SecretKey is required"); + + if (TimeoutSeconds <= 0) + errors.Add("TimeoutSeconds must be greater than 0"); + + if (MaxConnections <= 0) + errors.Add("MaxConnections must be greater than 0"); + + if (DefaultChunkSize <= 0) + errors.Add("DefaultChunkSize must be greater than 0"); + + if (MaxRetries < 0) + errors.Add("MaxRetries must be 0 or greater"); + + return errors.ToArray(); + } + + /// + /// Returns a string representation of the configuration (without sensitive data) + /// + public override string ToString() + { + return $"MinIO Configuration - Endpoint: {Endpoint}, UseSSL: {UseSSL}, Region: {Region}, Timeout: {TimeoutSeconds}s"; + } + } +} diff --git a/src/Core/MinIOConnection.cs b/src/Core/MinIOConnection.cs new file mode 100644 index 0000000..68b3e76 --- /dev/null +++ b/src/Core/MinIOConnection.cs @@ -0,0 +1,205 @@ +using System; +using PSMinIO.Core.S3; + +namespace PSMinIO.Core +{ + /// + /// Represents an active MinIO connection with configuration and S3 client + /// + public class MinIOConnection : IDisposable + { + private MinIOS3Client? _s3Client; + private bool _disposed = false; + + /// + /// Gets the configuration for this connection + /// + public MinIOConfiguration Configuration { get; } + + /// + /// Gets the connection status + /// + public ConnectionStatus Status { get; private set; } = ConnectionStatus.Disconnected; + + /// + /// Gets the time when the connection was established + /// + public DateTime? ConnectedAt { get; private set; } + + /// + /// Gets the last activity time + /// + public DateTime? LastActivity { get; private set; } + + /// + /// Gets the MinIO S3 client instance + /// + public MinIOS3Client S3Client + { + get + { + if (_disposed) + throw new ObjectDisposedException(nameof(MinIOConnection)); + + if (_s3Client == null) + { + _s3Client = new MinIOS3Client(Configuration); + Status = ConnectionStatus.Connected; + ConnectedAt = DateTime.UtcNow; + } + + LastActivity = DateTime.UtcNow; + return _s3Client; + } + } + + /// + /// Gets whether the connection is valid and ready to use + /// + public bool IsValid => !_disposed && Configuration.IsValid && Status != ConnectionStatus.Failed; + + /// + /// Creates a new MinIOConnection instance + /// + /// MinIO configuration + public MinIOConnection(MinIOConfiguration configuration) + { + Configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + + if (!Configuration.IsValid) + { + var errors = string.Join(", ", Configuration.GetValidationErrors()); + throw new ArgumentException($"Invalid MinIO configuration: {errors}", nameof(configuration)); + } + } + + /// + /// Tests the connection by attempting to list buckets + /// + /// True if connection test succeeds + public bool TestConnection() + { + if (_disposed) + return false; + + try + { + // Attempt to list buckets as a connection test + var buckets = S3Client.ListBuckets(); + Status = ConnectionStatus.Connected; + return true; + } + catch (Exception) + { + Status = ConnectionStatus.Failed; + return false; + } + } + + /// + /// Gets connection statistics + /// + /// Connection statistics + public ConnectionStats GetStats() + { + return new ConnectionStats + { + Status = Status, + ConnectedAt = ConnectedAt, + LastActivity = LastActivity, + Uptime = ConnectedAt.HasValue ? DateTime.UtcNow - ConnectedAt.Value : null, + Configuration = Configuration + }; + } + + /// + /// Returns a string representation of the connection + /// + public override string ToString() + { + var statusStr = Status switch + { + ConnectionStatus.Connected => "Connected", + ConnectionStatus.Disconnected => "Disconnected", + ConnectionStatus.Failed => "Failed", + _ => "Unknown" + }; + + return $"MinIO Connection - {Configuration.Endpoint} ({statusStr})"; + } + + /// + /// Disposes the connection and underlying resources + /// + public void Dispose() + { + if (!_disposed) + { + _s3Client?.Dispose(); + Status = ConnectionStatus.Disconnected; + _disposed = true; + } + } + } + + /// + /// Connection status enumeration + /// + public enum ConnectionStatus + { + /// + /// Connection is not established + /// + Disconnected, + + /// + /// Connection is active and working + /// + Connected, + + /// + /// Connection failed or encountered an error + /// + Failed + } + + /// + /// Connection statistics + /// + public class ConnectionStats + { + /// + /// Current connection status + /// + public ConnectionStatus Status { get; set; } + + /// + /// When the connection was established + /// + public DateTime? ConnectedAt { get; set; } + + /// + /// Last activity time + /// + public DateTime? LastActivity { get; set; } + + /// + /// Connection uptime + /// + public TimeSpan? Uptime { get; set; } + + /// + /// Connection configuration (without sensitive data) + /// + public MinIOConfiguration Configuration { get; set; } = null!; + + /// + /// Returns a string representation of the stats + /// + public override string ToString() + { + var uptimeStr = Uptime.HasValue ? $", Uptime: {Uptime.Value:hh\\:mm\\:ss}" : ""; + return $"Status: {Status}, Endpoint: {Configuration.Endpoint}{uptimeStr}"; + } + } +} diff --git a/src/Core/Models/MinIOBucketInfo.cs b/src/Core/Models/MinIOBucketInfo.cs new file mode 100644 index 0000000..e88f03f --- /dev/null +++ b/src/Core/Models/MinIOBucketInfo.cs @@ -0,0 +1,174 @@ +using System; +using System.Collections.Generic; + +namespace PSMinIO.Core.Models +{ + /// + /// Represents information about a MinIO bucket + /// + public class MinIOBucketInfo + { + /// + /// Name of the bucket + /// + public string Name { get; set; } = string.Empty; + + /// + /// Creation date of the bucket + /// + public DateTime CreationDate { get; set; } + + /// + /// Region where the bucket is located + /// + public string Region { get; set; } = string.Empty; + + /// + /// Bucket policy (if retrieved) + /// + public string? Policy { get; set; } + + /// + /// Bucket versioning status + /// + public BucketVersioningStatus VersioningStatus { get; set; } = BucketVersioningStatus.Unversioned; + + /// + /// Bucket encryption configuration + /// + public BucketEncryptionInfo? Encryption { get; set; } + + /// + /// Bucket tags + /// + public Dictionary Tags { get; set; } = new Dictionary(); + + /// + /// Number of objects in the bucket (if counted) + /// + public long? ObjectCount { get; set; } + + /// + /// Total size of all objects in the bucket (if calculated) + /// + public long? TotalSize { get; set; } + + /// + /// Creates a new MinIOBucketInfo instance + /// + public MinIOBucketInfo() + { + } + + /// + /// Creates a new MinIOBucketInfo instance with specified values + /// + /// Bucket name + /// Creation date + /// Bucket region + public MinIOBucketInfo(string name, DateTime creationDate, string region = "") + { + Name = name ?? throw new ArgumentNullException(nameof(name)); + CreationDate = creationDate; + Region = region ?? string.Empty; + } + + /// + /// Returns a string representation of the bucket info + /// + public override string ToString() + { + return $"Bucket: {Name} (Created: {CreationDate:yyyy-MM-dd HH:mm:ss}, Region: {Region})"; + } + + /// + /// Determines whether the specified object is equal to the current object + /// + public override bool Equals(object? obj) + { + if (obj is MinIOBucketInfo other) + { + return string.Equals(Name, other.Name, StringComparison.OrdinalIgnoreCase); + } + return false; + } + + /// + /// Returns a hash code for the current object + /// + public override int GetHashCode() + { + return Name?.ToLowerInvariant()?.GetHashCode() ?? 0; + } + } + + /// + /// Bucket versioning status + /// + public enum BucketVersioningStatus + { + /// + /// Versioning is not enabled + /// + Unversioned, + + /// + /// Versioning is enabled + /// + Enabled, + + /// + /// Versioning is suspended + /// + Suspended + } + + /// + /// Bucket encryption information + /// + public class BucketEncryptionInfo + { + /// + /// Encryption algorithm (e.g., "AES256", "aws:kms") + /// + public string Algorithm { get; set; } = string.Empty; + + /// + /// KMS key ID (for KMS encryption) + /// + public string? KmsKeyId { get; set; } + + /// + /// Whether encryption is enabled + /// + public bool IsEnabled { get; set; } + + /// + /// Creates a new BucketEncryptionInfo instance + /// + public BucketEncryptionInfo() + { + } + + /// + /// Creates a new BucketEncryptionInfo instance with specified values + /// + /// Encryption algorithm + /// Whether encryption is enabled + /// Optional KMS key ID + public BucketEncryptionInfo(string algorithm, bool isEnabled, string? kmsKeyId = null) + { + Algorithm = algorithm ?? throw new ArgumentNullException(nameof(algorithm)); + IsEnabled = isEnabled; + KmsKeyId = kmsKeyId; + } + + /// + /// Returns a string representation of the encryption info + /// + public override string ToString() + { + return $"Encryption: {Algorithm} (Enabled: {IsEnabled})"; + } + } +} diff --git a/src/Models/MinIOObjectInfo.cs b/src/Core/Models/MinIOObjectInfo.cs similarity index 72% rename from src/Models/MinIOObjectInfo.cs rename to src/Core/Models/MinIOObjectInfo.cs index 39faead..71e2391 100644 --- a/src/Models/MinIOObjectInfo.cs +++ b/src/Core/Models/MinIOObjectInfo.cs @@ -2,7 +2,7 @@ using System; using System.Collections.Generic; using PSMinIO.Utils; -namespace PSMinIO.Models +namespace PSMinIO.Core.Models { /// /// Represents information about a MinIO object @@ -131,64 +131,6 @@ namespace PSMinIO.Models BucketName = bucketName ?? string.Empty; } - /// - /// Creates a MinIOObjectInfo from a Minio.DataModel.Item - /// - /// Minio item object - /// Name of the bucket containing the object - /// MinIOObjectInfo instance - public static MinIOObjectInfo FromMinioItem(Minio.DataModel.Item item, string bucketName) - { - if (item == null) - throw new ArgumentNullException(nameof(item)); - - var objectInfo = new MinIOObjectInfo - { - Name = item.Key ?? string.Empty, - Size = (long)item.Size, - LastModified = item.LastModifiedDateTime ?? DateTime.MinValue, - ETag = item.ETag ?? string.Empty, - BucketName = bucketName ?? string.Empty, - StorageClass = string.Empty // StorageClass not available in MinIO 5.0.0 Item - }; - - // Try to extract version information if available - // Note: The MinIO .NET SDK Item object may have version properties - try - { - // Use reflection to check for version properties that might exist - var itemType = item.GetType(); - - var versionIdProperty = itemType.GetProperty("VersionId"); - if (versionIdProperty != null) - { - objectInfo.VersionId = versionIdProperty.GetValue(item)?.ToString(); - } - - var isLatestProperty = itemType.GetProperty("IsLatest"); - if (isLatestProperty != null && isLatestProperty.GetValue(item) is bool isLatest) - { - objectInfo.IsLatestVersion = isLatest; - } - - var isDeleteMarkerProperty = itemType.GetProperty("IsDeleteMarker"); - if (isDeleteMarkerProperty != null && isDeleteMarkerProperty.GetValue(item) is bool isDeleteMarker) - { - objectInfo.IsDeleteMarker = isDeleteMarker; - } - } - catch - { - // If reflection fails, just continue without version information - // This ensures compatibility even if the SDK doesn't have these properties - } - - // Metadata not available in MinIO 5.0.0 Item class - // objectInfo.Metadata remains empty - - return objectInfo; - } - /// /// Gets the file extension of the object /// @@ -245,7 +187,7 @@ namespace PSMinIO.Models /// public override string ToString() { - return $"Object: {Name} ({Size} bytes, Modified: {LastModified:yyyy-MM-dd HH:mm:ss})"; + return $"Object: {Name} ({SizeFormatter.FormatBytes(Size)}, Modified: {LastModified:yyyy-MM-dd HH:mm:ss})"; } /// diff --git a/src/Core/Models/MinIOOperationResult.cs b/src/Core/Models/MinIOOperationResult.cs new file mode 100644 index 0000000..29d43c3 --- /dev/null +++ b/src/Core/Models/MinIOOperationResult.cs @@ -0,0 +1,292 @@ +using System; +using PSMinIO.Utils; + +namespace PSMinIO.Core.Models +{ + /// + /// Base class for MinIO operation results with performance metrics + /// + public abstract class MinIOOperationResult + { + /// + /// When the operation started + /// + public DateTime? StartTime { get; set; } + + /// + /// When the operation completed + /// + public DateTime? EndTime { get; set; } + + /// + /// Duration of the operation + /// + public TimeSpan? Duration => StartTime.HasValue && EndTime.HasValue + ? EndTime.Value - StartTime.Value + : null; + + /// + /// Duration formatted as a human-readable string + /// + public string? DurationFormatted => Duration.HasValue + ? SizeFormatter.FormatDuration(Duration.Value) + : null; + + /// + /// Whether the operation was successful + /// + public bool Success { get; set; } = true; + + /// + /// Error message if the operation failed + /// + public string? ErrorMessage { get; set; } + + /// + /// Sets the start time to the current UTC time + /// + public void MarkStarted() + { + StartTime = DateTime.UtcNow; + } + + /// + /// Sets the end time to the current UTC time + /// + public void MarkCompleted() + { + EndTime = DateTime.UtcNow; + } + + /// + /// Marks the operation as failed with an error message + /// + /// Error message + public void MarkFailed(string errorMessage) + { + EndTime = DateTime.UtcNow; + Success = false; + ErrorMessage = errorMessage; + } + } + + /// + /// Result for transfer operations (upload/download) with speed metrics + /// + public abstract class MinIOTransferResult : MinIOOperationResult + { + /// + /// Number of bytes transferred + /// + public long BytesTransferred { get; set; } + + /// + /// Total size of the transfer + /// + public long TotalSize { get; set; } + + /// + /// Average transfer speed in bytes per second + /// + public double? AverageSpeed => Duration.HasValue && Duration.Value.TotalSeconds > 0 + ? BytesTransferred / Duration.Value.TotalSeconds + : null; + + /// + /// Average transfer speed formatted as a human-readable string + /// + public string? AverageSpeedFormatted => AverageSpeed.HasValue + ? SizeFormatter.FormatSpeed(AverageSpeed.Value) + : null; + + /// + /// Bytes transferred formatted as a human-readable string + /// + public string BytesTransferredFormatted => SizeFormatter.FormatBytes(BytesTransferred); + + /// + /// Total size formatted as a human-readable string + /// + public string TotalSizeFormatted => SizeFormatter.FormatBytes(TotalSize); + + /// + /// Transfer completion percentage + /// + public double PercentComplete => TotalSize > 0 ? (double)BytesTransferred / TotalSize * 100 : 100; + } + + /// + /// Result for upload operations + /// + public class MinIOUploadResult : MinIOTransferResult + { + /// + /// Name of the bucket + /// + public string BucketName { get; set; } = string.Empty; + + /// + /// Name of the uploaded object + /// + public string ObjectName { get; set; } = string.Empty; + + /// + /// ETag of the uploaded object + /// + public string ETag { get; set; } = string.Empty; + + /// + /// Content type of the uploaded object + /// + public string ContentType { get; set; } = string.Empty; + + /// + /// Source file path (if uploaded from file) + /// + public string? SourceFilePath { get; set; } + + /// + /// Creates a new MinIOUploadResult + /// + public MinIOUploadResult() + { + } + + /// + /// Creates a new MinIOUploadResult with specified values + /// + /// Bucket name + /// Object name + /// ETag + /// Total size + public MinIOUploadResult(string bucketName, string objectName, string etag, long totalSize) + { + BucketName = bucketName ?? string.Empty; + ObjectName = objectName ?? string.Empty; + ETag = etag ?? string.Empty; + TotalSize = totalSize; + BytesTransferred = totalSize; // Assume complete transfer + } + + /// + /// Returns a string representation of the upload result + /// + public override string ToString() + { + var status = Success ? "Success" : "Failed"; + var speed = AverageSpeedFormatted ?? "Unknown"; + var duration = DurationFormatted ?? "Unknown"; + + return $"Upload {status}: {ObjectName} ({TotalSizeFormatted}) in {duration} at {speed}"; + } + } + + /// + /// Result for download operations + /// + public class MinIODownloadResult : MinIOTransferResult + { + /// + /// Name of the bucket + /// + public string BucketName { get; set; } = string.Empty; + + /// + /// Name of the downloaded object + /// + public string ObjectName { get; set; } = string.Empty; + + /// + /// Target file path (if downloaded to file) + /// + public string? TargetFilePath { get; set; } + + /// + /// Content type of the downloaded object + /// + public string ContentType { get; set; } = string.Empty; + + /// + /// Last modified date of the object + /// + public DateTime? LastModified { get; set; } + + /// + /// Creates a new MinIODownloadResult + /// + public MinIODownloadResult() + { + } + + /// + /// Creates a new MinIODownloadResult with specified values + /// + /// Bucket name + /// Object name + /// Total size + public MinIODownloadResult(string bucketName, string objectName, long totalSize) + { + BucketName = bucketName ?? string.Empty; + ObjectName = objectName ?? string.Empty; + TotalSize = totalSize; + BytesTransferred = totalSize; // Assume complete transfer + } + + /// + /// Returns a string representation of the download result + /// + public override string ToString() + { + var status = Success ? "Success" : "Failed"; + var speed = AverageSpeedFormatted ?? "Unknown"; + var duration = DurationFormatted ?? "Unknown"; + + return $"Download {status}: {ObjectName} ({TotalSizeFormatted}) in {duration} at {speed}"; + } + } + + /// + /// Result for bucket operations + /// + public class MinIOBucketResult : MinIOOperationResult + { + /// + /// Name of the bucket + /// + public string BucketName { get; set; } = string.Empty; + + /// + /// Operation that was performed + /// + public string Operation { get; set; } = string.Empty; + + /// + /// Creates a new MinIOBucketResult + /// + public MinIOBucketResult() + { + } + + /// + /// Creates a new MinIOBucketResult with specified values + /// + /// Bucket name + /// Operation performed + public MinIOBucketResult(string bucketName, string operation) + { + BucketName = bucketName ?? string.Empty; + Operation = operation ?? string.Empty; + } + + /// + /// Returns a string representation of the bucket result + /// + public override string ToString() + { + var status = Success ? "Success" : "Failed"; + var duration = DurationFormatted ?? "Unknown"; + + return $"{Operation} {status}: {BucketName} in {duration}"; + } + } +} diff --git a/src/Core/S3/MinIOS3Client.cs b/src/Core/S3/MinIOS3Client.cs new file mode 100644 index 0000000..ba68174 --- /dev/null +++ b/src/Core/S3/MinIOS3Client.cs @@ -0,0 +1,435 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net.Http; +using System.Text.Json; +using System.Xml.Linq; +using PSMinIO.Core.Http; +using PSMinIO.Core.Models; +using PSMinIO.Utils; + +namespace PSMinIO.Core.S3 +{ + /// + /// MinIO S3 API client implementation using custom REST API calls + /// Provides synchronous operations optimized for PowerShell with real progress reporting + /// + public class MinIOS3Client : IDisposable + { + private readonly MinIOHttpClient _httpClient; + private readonly MinIOConfiguration _configuration; + private bool _disposed = false; + + /// + /// Creates a new MinIOS3Client instance + /// + /// MinIO configuration + public MinIOS3Client(MinIOConfiguration configuration) + { + _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + _httpClient = new MinIOHttpClient(configuration); + } + + #region Bucket Operations + + /// + /// Lists all buckets + /// + /// List of bucket information + public List ListBuckets() + { + try + { + var response = _httpClient.ExecuteRequestForString(HttpMethod.Get, "/"); + return ParseListBucketsResponse(response); + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to list buckets: {ex.Message}", ex); + } + } + + /// + /// Checks if a bucket exists + /// + /// Name of the bucket + /// True if bucket exists + public bool BucketExists(string bucketName) + { + if (string.IsNullOrWhiteSpace(bucketName)) + throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); + + try + { + using var response = _httpClient.ExecuteRequest(HttpMethod.Head, $"/{bucketName}"); + return response.IsSuccessStatusCode; + } + catch (HttpRequestException) + { + return false; + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to check if bucket '{bucketName}' exists: {ex.Message}", ex); + } + } + + /// + /// Creates a new bucket + /// + /// Name of the bucket to create + /// Optional region for the bucket + public void CreateBucket(string bucketName, string? region = null) + { + if (string.IsNullOrWhiteSpace(bucketName)) + throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); + + try + { + HttpContent? content = null; + + // If region is specified and different from default, create location constraint + if (!string.IsNullOrEmpty(region) && region != "us-east-1") + { + var locationConstraint = $@" + + {region} + "; + content = new StringContent(locationConstraint, System.Text.Encoding.UTF8, "application/xml"); + } + + using var response = _httpClient.ExecuteRequest(HttpMethod.Put, $"/{bucketName}", content: content); + response.EnsureSuccessStatusCode(); + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to create bucket '{bucketName}': {ex.Message}", ex); + } + } + + /// + /// Deletes a bucket + /// + /// Name of the bucket to delete + public void DeleteBucket(string bucketName) + { + if (string.IsNullOrWhiteSpace(bucketName)) + throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); + + try + { + using var response = _httpClient.ExecuteRequest(HttpMethod.Delete, $"/{bucketName}"); + response.EnsureSuccessStatusCode(); + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to delete bucket '{bucketName}': {ex.Message}", ex); + } + } + + /// + /// Gets bucket policy + /// + /// Name of the bucket + /// Bucket policy as JSON string + public string GetBucketPolicy(string bucketName) + { + if (string.IsNullOrWhiteSpace(bucketName)) + throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); + + try + { + var queryParams = new Dictionary { { "policy", "" } }; + return _httpClient.ExecuteRequestForString(HttpMethod.Get, $"/{bucketName}", queryParams); + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to get policy for bucket '{bucketName}': {ex.Message}", ex); + } + } + + /// + /// Sets bucket policy + /// + /// Name of the bucket + /// Policy JSON string + public void SetBucketPolicy(string bucketName, string policy) + { + if (string.IsNullOrWhiteSpace(bucketName)) + throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); + + if (string.IsNullOrWhiteSpace(policy)) + throw new ArgumentException("Policy cannot be null or empty", nameof(policy)); + + try + { + var queryParams = new Dictionary { { "policy", "" } }; + var content = new StringContent(policy, System.Text.Encoding.UTF8, "application/json"); + + using var response = _httpClient.ExecuteRequest(HttpMethod.Put, $"/{bucketName}", queryParams, content: content); + response.EnsureSuccessStatusCode(); + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to set policy for bucket '{bucketName}': {ex.Message}", ex); + } + } + + #endregion + + #region Object Operations + + /// + /// Lists objects in a bucket + /// + /// Name of the bucket + /// Optional prefix to filter objects + /// Whether to list objects recursively + /// Maximum number of objects to return + /// List of object information + public List ListObjects(string bucketName, string? prefix = null, bool recursive = true, int maxObjects = 1000) + { + if (string.IsNullOrWhiteSpace(bucketName)) + throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); + + try + { + var queryParams = new Dictionary + { + { "list-type", "2" }, + { "max-keys", maxObjects.ToString() } + }; + + if (!string.IsNullOrEmpty(prefix)) + queryParams["prefix"] = prefix; + + if (!recursive) + queryParams["delimiter"] = "/"; + + var response = _httpClient.ExecuteRequestForString(HttpMethod.Get, $"/{bucketName}", queryParams); + return ParseListObjectsResponse(response, bucketName); + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to list objects in bucket '{bucketName}': {ex.Message}", ex); + } + } + + /// + /// Uploads an object from a stream + /// + /// Name of the bucket + /// Name of the object + /// Source stream + /// Content type + /// Optional metadata + /// Progress callback + /// ETag of the uploaded object + public string PutObject( + string bucketName, + string objectName, + Stream stream, + string contentType = "application/octet-stream", + Dictionary? metadata = null, + Action? progressCallback = null) + { + if (string.IsNullOrWhiteSpace(bucketName)) + throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); + + if (string.IsNullOrWhiteSpace(objectName)) + throw new ArgumentException("Object name cannot be null or empty", nameof(objectName)); + + if (stream == null) + throw new ArgumentNullException(nameof(stream)); + + try + { + var headers = new Dictionary(); + + // Add metadata headers + if (metadata != null) + { + foreach (var kvp in metadata) + { + headers[$"x-amz-meta-{kvp.Key}"] = kvp.Value; + } + } + + using var response = _httpClient.UploadFromStream( + HttpMethod.Put, + $"/{bucketName}/{objectName}", + stream, + contentType, + headers: headers, + progressCallback: progressCallback); + + response.EnsureSuccessStatusCode(); + + // Extract ETag from response headers + if (response.Headers.ETag != null) + { + return response.Headers.ETag.Tag.Trim('"'); + } + + return string.Empty; + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to upload object '{objectName}' to bucket '{bucketName}': {ex.Message}", ex); + } + } + + /// + /// Downloads an object to a stream + /// + /// Name of the bucket + /// Name of the object + /// Target stream + /// Progress callback + /// Number of bytes downloaded + public long GetObject( + string bucketName, + string objectName, + Stream stream, + Action? progressCallback = null) + { + if (string.IsNullOrWhiteSpace(bucketName)) + throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); + + if (string.IsNullOrWhiteSpace(objectName)) + throw new ArgumentException("Object name cannot be null or empty", nameof(objectName)); + + if (stream == null) + throw new ArgumentNullException(nameof(stream)); + + try + { + return _httpClient.DownloadToStream($"/{bucketName}/{objectName}", stream, progressCallback: progressCallback); + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to download object '{objectName}' from bucket '{bucketName}': {ex.Message}", ex); + } + } + + /// + /// Deletes an object + /// + /// Name of the bucket + /// Name of the object + public void DeleteObject(string bucketName, string objectName) + { + if (string.IsNullOrWhiteSpace(bucketName)) + throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); + + if (string.IsNullOrWhiteSpace(objectName)) + throw new ArgumentException("Object name cannot be null or empty", nameof(objectName)); + + try + { + using var response = _httpClient.ExecuteRequest(HttpMethod.Delete, $"/{bucketName}/{objectName}"); + response.EnsureSuccessStatusCode(); + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to delete object '{objectName}' from bucket '{bucketName}': {ex.Message}", ex); + } + } + + #endregion + + #region Helper Methods + + /// + /// Parses the list buckets XML response + /// + private List ParseListBucketsResponse(string xmlResponse) + { + var buckets = new List(); + + try + { + var doc = XDocument.Parse(xmlResponse); + var ns = doc.Root?.GetDefaultNamespace(); + + var bucketElements = doc.Descendants(ns + "Bucket"); + foreach (var bucketElement in bucketElements) + { + var name = bucketElement.Element(ns + "Name")?.Value ?? string.Empty; + var creationDateStr = bucketElement.Element(ns + "CreationDate")?.Value ?? string.Empty; + + if (DateTime.TryParse(creationDateStr, out var creationDate)) + { + buckets.Add(new MinIOBucketInfo(name, creationDate)); + } + } + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to parse list buckets response: {ex.Message}", ex); + } + + return buckets; + } + + /// + /// Parses the list objects XML response + /// + private List ParseListObjectsResponse(string xmlResponse, string bucketName) + { + var objects = new List(); + + try + { + var doc = XDocument.Parse(xmlResponse); + var ns = doc.Root?.GetDefaultNamespace(); + + var contentElements = doc.Descendants(ns + "Contents"); + foreach (var contentElement in contentElements) + { + var keyElement = contentElement.Element(ns + "Key"); + var lastModifiedElement = contentElement.Element(ns + "LastModified"); + var etagElement = contentElement.Element(ns + "ETag"); + var sizeElement = contentElement.Element(ns + "Size"); + var storageClassElement = contentElement.Element(ns + "StorageClass"); + + var key = keyElement?.Value ?? string.Empty; + var lastModifiedStr = lastModifiedElement?.Value ?? string.Empty; + var etag = etagElement?.Value?.Trim('"') ?? string.Empty; + var sizeStr = sizeElement?.Value ?? "0"; + var storageClass = storageClassElement?.Value ?? string.Empty; + + if (DateTime.TryParse(lastModifiedStr, out var lastModified) && + long.TryParse(sizeStr, out var size)) + { + var objectInfo = new MinIOObjectInfo(key, size, lastModified, etag, bucketName) + { + StorageClass = storageClass + }; + objects.Add(objectInfo); + } + } + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to parse list objects response: {ex.Message}", ex); + } + + return objects; + } + + #endregion + + /// + /// Disposes the S3 client + /// + public void Dispose() + { + if (!_disposed) + { + _httpClient?.Dispose(); + _disposed = true; + } + } + } +} diff --git a/src/Models/ChunkedTransferState.cs b/src/Models/ChunkedTransferState.cs deleted file mode 100644 index ca0f628..0000000 --- a/src/Models/ChunkedTransferState.cs +++ /dev/null @@ -1,203 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; - -namespace PSMinIO.Models -{ - /// - /// Represents the state of a chunked transfer operation for resume functionality - /// - public class ChunkedTransferState - { - /// - /// Name of the bucket - /// - public string BucketName { get; set; } = string.Empty; - - /// - /// Name of the object - /// - public string ObjectName { get; set; } = string.Empty; - - /// - /// Local file path - /// - public string FilePath { get; set; } = string.Empty; - - /// - /// Total size of the file/object - /// - public long TotalSize { get; set; } - - /// - /// Size of each chunk - /// - public long ChunkSize { get; set; } - - /// - /// List of completed chunks - /// - public List CompletedChunks { get; set; } = new List(); - - /// - /// Last modified time of the source file (for validation) - /// - public DateTime LastModified { get; set; } - - /// - /// ETag of the object (for validation) - /// - public string ETag { get; set; } = string.Empty; - - /// - /// Transfer operation type - /// - public ChunkedTransferType TransferType { get; set; } - - /// - /// When the transfer was started - /// - public DateTime StartTime { get; set; } - - /// - /// When the transfer state was last updated - /// - public DateTime LastUpdated { get; set; } - - /// - /// Upload ID for multipart uploads (MinIO specific) - /// - public string? UploadId { get; set; } - - /// - /// Gets the total number of chunks - /// - public int TotalChunks => (int)Math.Ceiling((double)TotalSize / ChunkSize); - - /// - /// Gets the number of completed chunks - /// - public int CompletedChunkCount => CompletedChunks.Count; - - /// - /// Gets the total bytes transferred - /// - public long BytesTransferred => CompletedChunks.Sum(c => c.Size); - - /// - /// Gets the transfer progress as a percentage - /// - public double ProgressPercentage => TotalSize > 0 ? (double)BytesTransferred / TotalSize * 100 : 0; - - /// - /// Checks if the transfer is complete - /// - public bool IsComplete => CompletedChunkCount >= TotalChunks; - - /// - /// Gets the next chunk to transfer - /// - /// Next chunk info or null if complete - public ChunkInfo? GetNextChunk() - { - for (int i = 0; i < TotalChunks; i++) - { - if (!CompletedChunks.Any(c => c.ChunkNumber == i)) - { - var startByte = i * ChunkSize; - var endByte = Math.Min(startByte + ChunkSize - 1, TotalSize - 1); - var size = endByte - startByte + 1; - - return new ChunkInfo - { - ChunkNumber = i, - StartByte = startByte, - EndByte = endByte, - Size = size, - IsCompleted = false - }; - } - } - return null; - } - - /// - /// Marks a chunk as completed - /// - /// Completed chunk information - public void MarkChunkCompleted(ChunkInfo chunkInfo) - { - chunkInfo.IsCompleted = true; - chunkInfo.CompletedTime = DateTime.UtcNow; - - // Remove any existing entry for this chunk number - CompletedChunks.RemoveAll(c => c.ChunkNumber == chunkInfo.ChunkNumber); - - // Add the completed chunk - CompletedChunks.Add(chunkInfo); - - LastUpdated = DateTime.UtcNow; - } - } - - /// - /// Information about a single chunk - /// - public class ChunkInfo - { - /// - /// Chunk number (0-based) - /// - public int ChunkNumber { get; set; } - - /// - /// Starting byte position - /// - public long StartByte { get; set; } - - /// - /// Ending byte position (inclusive) - /// - public long EndByte { get; set; } - - /// - /// Size of the chunk in bytes - /// - public long Size { get; set; } - - /// - /// ETag of the uploaded chunk (for uploads) - /// - public string ChunkETag { get; set; } = string.Empty; - - /// - /// Whether this chunk has been completed - /// - public bool IsCompleted { get; set; } - - /// - /// When this chunk was completed - /// - public DateTime? CompletedTime { get; set; } - - /// - /// Number of retry attempts for this chunk - /// - public int RetryCount { get; set; } - - /// - /// Last error encountered for this chunk - /// - public string? LastError { get; set; } - } - - /// - /// Type of chunked transfer operation - /// - public enum ChunkedTransferType - { - Upload, - Download - } -} diff --git a/src/Models/MinIOBucketInfo.cs b/src/Models/MinIOBucketInfo.cs deleted file mode 100644 index 5f99a74..0000000 --- a/src/Models/MinIOBucketInfo.cs +++ /dev/null @@ -1,101 +0,0 @@ -using System; - -namespace PSMinIO.Models -{ - /// - /// Represents information about a MinIO bucket - /// - public class MinIOBucketInfo - { - /// - /// Name of the bucket - /// - public string Name { get; set; } = string.Empty; - - /// - /// Creation date of the bucket - /// - public DateTime Created { get; set; } - - /// - /// Region where the bucket is located - /// - public string Region { get; set; } = string.Empty; - - /// - /// Total size of all objects in the bucket (optional, calculated separately) - /// - public long? Size { get; set; } - - /// - /// Number of objects in the bucket (optional, calculated separately) - /// - public long? ObjectCount { get; set; } - - /// - /// Creates a new MinIOBucketInfo instance - /// - public MinIOBucketInfo() - { - } - - /// - /// Creates a new MinIOBucketInfo instance with specified values - /// - /// Bucket name - /// Creation date - /// Bucket region - public MinIOBucketInfo(string name, DateTime created, string region = "") - { - Name = name ?? string.Empty; - Created = created; - Region = region ?? string.Empty; - } - - /// - /// Creates a MinIOBucketInfo from a Minio.DataModel.Bucket - /// - /// Minio bucket object - /// MinIOBucketInfo instance - public static MinIOBucketInfo FromMinioBucket(Minio.DataModel.Bucket bucket) - { - if (bucket == null) - throw new ArgumentNullException(nameof(bucket)); - - return new MinIOBucketInfo - { - Name = bucket.Name ?? string.Empty, - Created = DateTime.TryParse(bucket.CreationDate, out var createdDate) ? createdDate : DateTime.MinValue, - Region = string.Empty // Region is not available in basic bucket info - }; - } - - /// - /// Returns a string representation of the bucket info - /// - public override string ToString() - { - return $"Bucket: {Name} (Created: {Created:yyyy-MM-dd HH:mm:ss})"; - } - - /// - /// Determines whether the specified object is equal to the current object - /// - public override bool Equals(object? obj) - { - if (obj is MinIOBucketInfo other) - { - return string.Equals(Name, other.Name, StringComparison.OrdinalIgnoreCase); - } - return false; - } - - /// - /// Returns a hash code for the current object - /// - public override int GetHashCode() - { - return Name?.ToLowerInvariant().GetHashCode() ?? 0; - } - } -} diff --git a/src/Models/MinIOConfiguration.cs b/src/Models/MinIOConfiguration.cs deleted file mode 100644 index cad492a..0000000 --- a/src/Models/MinIOConfiguration.cs +++ /dev/null @@ -1,84 +0,0 @@ -using System; - -namespace PSMinIO.Models -{ - /// - /// Configuration settings for MinIO client connections - /// - public class MinIOConfiguration - { - - /// - /// MinIO server endpoint (without protocol) - /// - public string Endpoint { get; set; } = string.Empty; - - /// - /// Access key for authentication - /// - public string AccessKey { get; set; } = string.Empty; - - /// - /// Secret key for authentication - /// - public string SecretKey { get; set; } = string.Empty; - - /// - /// Whether to use SSL/TLS for connections - /// - public bool UseSSL { get; set; } = true; - - /// - /// Optional region for bucket operations - /// - public string Region { get; set; } = "us-east-1"; - - /// - /// Connection timeout in seconds - /// - public int TimeoutSeconds { get; set; } = 30; - - /// - /// Whether to skip SSL certificate validation - /// - public bool SkipCertificateValidation { get; set; } - - /// - /// Whether the configuration is valid for creating a client - /// - public bool IsValid => !string.IsNullOrWhiteSpace(Endpoint) && - !string.IsNullOrWhiteSpace(AccessKey) && - !string.IsNullOrWhiteSpace(SecretKey); - - - - /// - /// Creates a new MinIOConfiguration instance - /// - public MinIOConfiguration() - { - } - - /// - /// Creates a new MinIOConfiguration instance with specified values - /// - /// MinIO endpoint - /// Access key - /// Secret key - /// Use SSL flag - /// Optional region - /// Connection timeout - /// Whether to skip SSL certificate validation - public MinIOConfiguration(string endpoint, string accessKey, string secretKey, - bool useSSL = true, string region = "us-east-1", int timeoutSeconds = 30, bool skipCertificateValidation = false) - { - Endpoint = endpoint?.Trim() ?? string.Empty; - AccessKey = accessKey?.Trim() ?? string.Empty; - SecretKey = secretKey?.Trim() ?? string.Empty; - UseSSL = useSSL; - Region = region?.Trim() ?? "us-east-1"; - TimeoutSeconds = timeoutSeconds; - SkipCertificateValidation = skipCertificateValidation; - } - } -} diff --git a/src/Models/MinIOConnection.cs b/src/Models/MinIOConnection.cs deleted file mode 100644 index 18c6d69..0000000 --- a/src/Models/MinIOConnection.cs +++ /dev/null @@ -1,202 +0,0 @@ -using System; -using PSMinIO.Utils; - -namespace PSMinIO.Models -{ - /// - /// Represents an active MinIO connection with configuration and client wrapper - /// - public class MinIOConnection : IDisposable - { - private MinIOClientWrapper? _client; - private bool _disposed = false; - - /// - /// Gets the configuration for this connection - /// - public MinIOConfiguration Configuration { get; } - - /// - /// Gets the MinIO client wrapper instance - /// - public MinIOClientWrapper Client - { - get - { - if (_disposed) - throw new ObjectDisposedException(nameof(MinIOConnection)); - - if (_client == null) - { - _client = new MinIOClientWrapper(Configuration); - } - return _client; - } - } - - /// - /// Gets whether this connection is valid and ready to use - /// - public bool IsValid => Configuration.IsValid && !_disposed; - - /// - /// Gets the connection status - /// - public string Status - { - get - { - if (_disposed) return "Disposed"; - if (!Configuration.IsValid) return "Invalid Configuration"; - return "Ready"; - } - } - - /// - /// Gets the endpoint URL for display purposes - /// - public string EndpointUrl - { - get - { - if (string.IsNullOrWhiteSpace(Configuration.Endpoint)) - return "Not Set"; - - var protocol = Configuration.UseSSL ? "https" : "http"; - return $"{protocol}://{Configuration.Endpoint}"; - } - } - - /// - /// Gets when this connection was created - /// - public DateTime CreatedAt { get; } - - /// - /// Creates a new MinIOConnection instance - /// - /// MinIO configuration - public MinIOConnection(MinIOConfiguration configuration) - { - Configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); - CreatedAt = DateTime.UtcNow; - } - - /// - /// Tests the connection by attempting to list buckets - /// - /// Connection test result - public ConnectionTestResult TestConnection() - { - if (_disposed) - return new ConnectionTestResult(false, "Connection is disposed"); - - if (!Configuration.IsValid) - return new ConnectionTestResult(false, "Configuration is invalid"); - - try - { - var startTime = DateTime.UtcNow; - var buckets = Client.ListBuckets(); - var duration = DateTime.UtcNow - startTime; - - return new ConnectionTestResult(true, "Connection successful") - { - BucketCount = buckets.Count, - ResponseTime = duration, - TestedAt = DateTime.UtcNow - }; - } - catch (Exception ex) - { - return new ConnectionTestResult(false, ex.Message) - { - TestedAt = DateTime.UtcNow - }; - } - } - - /// - /// Returns a string representation of the connection - /// - public override string ToString() - { - return $"MinIO Connection: {EndpointUrl} (Status: {Status})"; - } - - /// - /// Disposes the connection and underlying client - /// - public void Dispose() - { - Dispose(true); - GC.SuppressFinalize(this); - } - - /// - /// Protected dispose method - /// - /// Whether disposing from Dispose method - protected virtual void Dispose(bool disposing) - { - if (!_disposed && disposing) - { - _client?.Dispose(); - _client = null; - _disposed = true; - } - } - } - - /// - /// Represents the result of a connection test - /// - public class ConnectionTestResult - { - /// - /// Whether the connection test was successful - /// - public bool Success { get; } - - /// - /// Message describing the test result - /// - public string Message { get; } - - /// - /// Number of buckets found (if successful) - /// - public int? BucketCount { get; set; } - - /// - /// Response time for the test - /// - public TimeSpan? ResponseTime { get; set; } - - /// - /// When the test was performed - /// - public DateTime? TestedAt { get; set; } - - /// - /// Creates a new ConnectionTestResult - /// - /// Whether the test was successful - /// Test result message - public ConnectionTestResult(bool success, string message) - { - Success = success; - Message = message ?? string.Empty; - } - - /// - /// Returns a string representation of the test result - /// - public override string ToString() - { - var result = Success ? "SUCCESS" : "FAILED"; - var details = ResponseTime.HasValue ? $" ({ResponseTime.Value.TotalMilliseconds:F0}ms)" : ""; - return $"{result}: {Message}{details}"; - } - } -} diff --git a/src/Models/MinIODownloadResult.cs b/src/Models/MinIODownloadResult.cs deleted file mode 100644 index 1bba766..0000000 --- a/src/Models/MinIODownloadResult.cs +++ /dev/null @@ -1,103 +0,0 @@ -using System; -using System.IO; -using PSMinIO.Utils; - -namespace PSMinIO.Models -{ - /// - /// Represents the result of a MinIO download operation with timing and speed information - /// - public class MinIODownloadResult - { - /// - /// The downloaded file information - /// - public FileInfo File { get; set; } - - /// - /// Name of the bucket the object was downloaded from - /// - public string BucketName { get; set; } - - /// - /// Name of the object that was downloaded - /// - public string ObjectName { get; set; } - - /// - /// Size of the downloaded file in bytes - /// - public long Size => File?.Length ?? 0; - - /// - /// Transfer start time - /// - public DateTime? StartTime { get; set; } - - /// - /// Transfer completion time - /// - public DateTime? CompletionTime { get; set; } - - /// - /// Transfer duration - /// - public TimeSpan? Duration => StartTime.HasValue && CompletionTime.HasValue ? - CompletionTime.Value - StartTime.Value : null; - - /// - /// Average transfer speed in bytes per second - /// - public double? AverageSpeed => Duration.HasValue && Duration.Value.TotalSeconds > 0 ? - Size / Duration.Value.TotalSeconds : null; - - /// - /// Average transfer speed formatted as string (e.g., "15.2 MB/s") - /// - public string? AverageSpeedFormatted => AverageSpeed.HasValue ? - $"{SizeFormatter.FormatBytes((long)AverageSpeed.Value)}/s" : null; - - /// - /// Full path to the downloaded file - /// - public string FullName => File?.FullName ?? string.Empty; - - /// - /// Name of the downloaded file - /// - public string Name => File?.Name ?? string.Empty; - - /// - /// Directory containing the downloaded file - /// - public string? DirectoryName => File?.DirectoryName; - - /// - /// Creates a new MinIODownloadResult - /// - /// Downloaded file information - /// Source bucket name - /// Source object name - /// Transfer start time - /// Transfer completion time - public MinIODownloadResult(FileInfo file, string bucketName, string objectName, - DateTime? startTime = null, DateTime? completionTime = null) - { - File = file ?? throw new ArgumentNullException(nameof(file)); - BucketName = bucketName ?? throw new ArgumentNullException(nameof(bucketName)); - ObjectName = objectName ?? throw new ArgumentNullException(nameof(objectName)); - StartTime = startTime; - CompletionTime = completionTime; - } - - /// - /// Returns a string representation of the download result - /// - public override string ToString() - { - var duration = Duration?.ToString(@"hh\:mm\:ss\.fff") ?? "Unknown"; - var speed = AverageSpeedFormatted ?? "Unknown"; - return $"{BucketName}/{ObjectName} -> {FullName} ({SizeFormatter.FormatBytes(Size)}, {duration}, {speed})"; - } - } -} diff --git a/src/Models/MinIOStats.cs b/src/Models/MinIOStats.cs deleted file mode 100644 index 4c0ccc0..0000000 --- a/src/Models/MinIOStats.cs +++ /dev/null @@ -1,90 +0,0 @@ -using System; - -namespace PSMinIO.Models -{ - /// - /// Represents statistical information about MinIO storage - /// - public class MinIOStats - { - /// - /// Total number of buckets - /// - public int TotalBuckets { get; set; } - - /// - /// Total number of objects across all buckets - /// - public long TotalObjects { get; set; } - - /// - /// Total size of all objects in bytes - /// - public long TotalSize { get; set; } - - /// - /// When these statistics were last updated - /// - public DateTime LastUpdated { get; set; } - - /// - /// MinIO server endpoint - /// - public string Endpoint { get; set; } = string.Empty; - - /// - /// Whether SSL is being used - /// - public bool UseSSL { get; set; } - - /// - /// Connection status - /// - public string ConnectionStatus { get; set; } = "Unknown"; - - /// - /// Creates a new MinIOStats instance - /// - public MinIOStats() - { - LastUpdated = DateTime.UtcNow; - } - - /// - /// Creates a new MinIOStats instance with specified values - /// - /// Total number of buckets - /// Total number of objects - /// Total size in bytes - /// MinIO endpoint - /// Whether SSL is used - public MinIOStats(int totalBuckets, long totalObjects, long totalSize, string endpoint, bool useSSL) - { - TotalBuckets = totalBuckets; - TotalObjects = totalObjects; - TotalSize = totalSize; - Endpoint = endpoint ?? string.Empty; - UseSSL = useSSL; - LastUpdated = DateTime.UtcNow; - ConnectionStatus = "Connected"; - } - - /// - /// Gets the average object size in bytes - /// - public double AverageObjectSize => TotalObjects > 0 ? (double)TotalSize / TotalObjects : 0; - - /// - /// Gets the average objects per bucket - /// - public double AverageObjectsPerBucket => TotalBuckets > 0 ? (double)TotalObjects / TotalBuckets : 0; - - /// - /// Returns a string representation of the statistics - /// - public override string ToString() - { - return $"MinIO Stats: {TotalBuckets} buckets, {TotalObjects} objects, {Utils.SizeFormatter.FormatBytes(TotalSize)} total"; - } - } -} diff --git a/src/Properties/AssemblyInfo.cs b/src/Properties/AssemblyInfo.cs index dd047c4..465b4dc 100644 --- a/src/Properties/AssemblyInfo.cs +++ b/src/Properties/AssemblyInfo.cs @@ -31,12 +31,9 @@ using System.Runtime.InteropServices; // // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: -// [assembly: AssemblyVersion("2025.07.11.1151")] -[assembly: AssemblyVersion("2025.07.11.1151")] -[assembly: AssemblyFileVersion("2025.07.11.1151")] -[assembly: AssemblyInformationalVersion("2025.07.11.1151")] - - - +// [assembly: AssemblyVersion("2025.07.11.1421")] +[assembly: AssemblyVersion("2025.07.11.1421")] +[assembly: AssemblyFileVersion("2025.07.11.1421")] +[assembly: AssemblyInformationalVersion("2025.07.11.1421")] diff --git a/src/Utils/ChunkedCollectionProgressReporter.cs b/src/Utils/ChunkedCollectionProgressReporter.cs deleted file mode 100644 index c0979ed..0000000 --- a/src/Utils/ChunkedCollectionProgressReporter.cs +++ /dev/null @@ -1,267 +0,0 @@ -using System; -using System.Management.Automation; -using PSMinIO.Utils; - -namespace PSMinIO.Utils -{ - /// - /// Manages 3-layer progress reporting for chunked file collection operations - /// Layer 1: Collection Progress (overall files) - /// Layer 2: Current File Progress - /// Layer 3: Current Chunk Progress - /// - public class ChunkedCollectionProgressReporter - { - private readonly PSCmdlet _cmdlet; - private readonly int _totalFiles; - private readonly long _totalSize; - private readonly DateTime _startTime; - private readonly string _operationName; - - // Progress tracking - private int _completedFiles = 0; - private long _totalBytesTransferred = 0; - private string _currentFileName = ""; - private long _currentFileSize = 0; - private long _currentFileBytesTransferred = 0; - private int _currentChunk = 0; - private int _totalChunks = 0; - private long _currentChunkBytesTransferred = 0; - private long _currentChunkSize = 0; - - // Activity IDs for progress hierarchy - private const int CollectionActivityId = 1; - private const int FileActivityId = 2; - private const int ChunkActivityId = 3; - - // Progress control - private readonly long _progressUpdateInterval; - private long _lastProgressUpdate = 0; - - /// - /// Creates a new chunked collection progress reporter - /// - /// PowerShell cmdlet for progress reporting - /// Total number of files to process - /// Total size of all files - /// Name of the operation (e.g., "Uploading", "Downloading") - /// Update progress every N bytes - public ChunkedCollectionProgressReporter( - PSCmdlet cmdlet, - int totalFiles, - long totalSize, - string operationName = "Processing", - long progressUpdateInterval = 1024 * 1024) // 1MB default - { - _cmdlet = cmdlet; - _totalFiles = totalFiles; - _totalSize = totalSize; - _operationName = operationName; - _startTime = DateTime.Now; - _progressUpdateInterval = progressUpdateInterval; - } - - /// - /// Starts processing a new file - /// - /// Name of the file being processed - /// Size of the file - /// Total number of chunks for this file - public void StartNewFile(string fileName, long fileSize, int totalChunks) - { - _currentFileName = fileName; - _currentFileSize = fileSize; - _currentFileBytesTransferred = 0; - _totalChunks = totalChunks; - _currentChunk = 0; - - MinIOLogger.WriteVerbose(_cmdlet, "Starting {0} of file {1}/{2}: {3} ({4})", - _operationName.ToLower(), _completedFiles + 1, _totalFiles, fileName, SizeFormatter.FormatBytes(fileSize)); - - UpdateAllProgress(); - } - - /// - /// Starts processing a new chunk - /// - /// Chunk number (1-based for display) - /// Size of the chunk - public void StartNewChunk(int chunkNumber, long chunkSize) - { - _currentChunk = chunkNumber; - _currentChunkSize = chunkSize; - _currentChunkBytesTransferred = 0; - - MinIOLogger.WriteVerbose(_cmdlet, "File {0}: Starting chunk {1}/{2} ({3})", - _currentFileName, chunkNumber, _totalChunks, SizeFormatter.FormatBytes(chunkSize)); - - UpdateAllProgress(); - } - - /// - /// Updates chunk progress - /// - /// Bytes transferred for current chunk - public void UpdateChunkProgress(long bytesTransferred) - { - var chunkDelta = bytesTransferred - _currentChunkBytesTransferred; - _currentChunkBytesTransferred = bytesTransferred; - _currentFileBytesTransferred += chunkDelta; - _totalBytesTransferred += chunkDelta; - - // Only update progress if we've transferred enough bytes since last update - if (_totalBytesTransferred - _lastProgressUpdate >= _progressUpdateInterval) - { - UpdateAllProgress(); - _lastProgressUpdate = _totalBytesTransferred; - } - } - - /// - /// Completes the current chunk - /// - /// ETag of the completed chunk (optional) - public void CompleteChunk(string? chunkETag = null) - { - MinIOLogger.WriteVerbose(_cmdlet, "File {0}: Completed chunk {1}/{2}{3}", - _currentFileName, _currentChunk, _totalChunks, - !string.IsNullOrEmpty(chunkETag) ? $" - ETag: {chunkETag}" : ""); - - // Mark chunk as completed (only if progress is enabled) - if (_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") && - _cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue") - return; - - var chunkProgress = new ProgressRecord(ChunkActivityId, "Current Chunk", "Completed") - { - PercentComplete = 100, - RecordType = ProgressRecordType.Completed, - ParentActivityId = FileActivityId - }; - _cmdlet.WriteProgress(chunkProgress); - } - - /// - /// Completes the current file - /// - public void CompleteFile() - { - _completedFiles++; - - var elapsed = DateTime.Now - _startTime; - MinIOLogger.WriteVerbose(_cmdlet, "File {0}: {1} completed in {2} - Total size: {3}", - _currentFileName, _operationName.ToLower(), elapsed.ToString(@"hh\:mm\:ss"), SizeFormatter.FormatBytes(_currentFileSize)); - - // Mark file as completed (only if progress is enabled) - if (_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") && - _cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue") - return; - - var fileProgress = new ProgressRecord(FileActivityId, "Current File", "Completed") - { - PercentComplete = 100, - RecordType = ProgressRecordType.Completed, - ParentActivityId = CollectionActivityId - }; - _cmdlet.WriteProgress(fileProgress); - - // Complete chunk progress too - CompleteChunk(); - } - - /// - /// Completes the entire collection operation - /// - public void CompleteCollection() - { - var elapsed = DateTime.Now - _startTime; - MinIOLogger.WriteVerbose(_cmdlet, "Completed {0} {1} files ({2}) in {3}", - _operationName.ToLower(), _totalFiles, SizeFormatter.FormatBytes(_totalSize), elapsed.ToString(@"hh\:mm\:ss")); - - // Complete all progress records - var collectionProgress = new ProgressRecord(CollectionActivityId, $"{_operationName} Files", "Completed") - { - PercentComplete = 100, - RecordType = ProgressRecordType.Completed - }; - _cmdlet.WriteProgress(collectionProgress); - - // Complete file progress if enabled - if (!(_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") && - _cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue")) - { - CompleteFile(); - } - } - - /// - /// Reports an error for the current chunk - /// - /// Error that occurred - /// Current retry attempt - /// Maximum retry attempts - public void ReportChunkError(Exception error, int retryAttempt, int maxRetries) - { - MinIOLogger.WriteVerbose(_cmdlet, "File {0}: Chunk {1}/{2} failed (attempt {3}/{4}): {5}", - _currentFileName, _currentChunk, _totalChunks, retryAttempt, maxRetries, error.Message); - } - - /// - /// Updates all progress layers - /// - private void UpdateAllProgress() - { - var elapsed = DateTime.Now - _startTime; - var speed = elapsed.TotalSeconds > 0 ? _totalBytesTransferred / elapsed.TotalSeconds : 0; - - // Layer 1: Collection Progress (always shown) - var collectionPercent = _totalSize > 0 ? (int)((_totalBytesTransferred * 100) / _totalSize) : 0; - var collectionStatus = $"Files: {_completedFiles}/{_totalFiles} | " + - $"Size: {SizeFormatter.FormatBytes(_totalBytesTransferred)}/{SizeFormatter.FormatBytes(_totalSize)} | " + - $"Speed: {SizeFormatter.FormatBytes((long)speed)}/s | " + - $"Elapsed: {elapsed:hh\\:mm\\:ss}"; - - var collectionProgress = new ProgressRecord(CollectionActivityId, $"{_operationName} Files", collectionStatus) - { - PercentComplete = collectionPercent - }; - _cmdlet.WriteProgress(collectionProgress); - - // Check if progress is disabled - if (_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") && - _cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue") - return; - - // Layer 2: Current File Progress - if (!string.IsNullOrEmpty(_currentFileName)) - { - var filePercent = _currentFileSize > 0 ? (int)((_currentFileBytesTransferred * 100) / _currentFileSize) : 0; - var fileStatus = $"File: {_currentFileName} | " + - $"Size: {SizeFormatter.FormatBytes(_currentFileBytesTransferred)}/{SizeFormatter.FormatBytes(_currentFileSize)}"; - - var fileProgress = new ProgressRecord(FileActivityId, "Current File", fileStatus) - { - PercentComplete = filePercent, - ParentActivityId = CollectionActivityId - }; - _cmdlet.WriteProgress(fileProgress); - } - - // Layer 3: Current Chunk Progress - if (_currentChunk > 0) - { - var chunkPercent = _currentChunkSize > 0 ? (int)((_currentChunkBytesTransferred * 100) / _currentChunkSize) : 0; - var chunkStatus = $"Chunk: {_currentChunk}/{_totalChunks} | " + - $"Size: {SizeFormatter.FormatBytes(_currentChunkBytesTransferred)}/{SizeFormatter.FormatBytes(_currentChunkSize)} | " + - $"Speed: {SizeFormatter.FormatBytes((long)speed)}/s"; - - var chunkProgress = new ProgressRecord(ChunkActivityId, "Current Chunk", chunkStatus) - { - PercentComplete = chunkPercent, - ParentActivityId = FileActivityId - }; - _cmdlet.WriteProgress(chunkProgress); - } - } - } -} diff --git a/src/Utils/ChunkedSingleFileProgressReporter.cs b/src/Utils/ChunkedSingleFileProgressReporter.cs deleted file mode 100644 index 9c999fb..0000000 --- a/src/Utils/ChunkedSingleFileProgressReporter.cs +++ /dev/null @@ -1,195 +0,0 @@ -using System; -using System.Management.Automation; -using PSMinIO.Utils; - -namespace PSMinIO.Utils -{ - /// - /// Manages 2-layer progress reporting for chunked single file operations - /// Layer 1: File Progress - /// Layer 2: Current Chunk Progress - /// - public class ChunkedSingleFileProgressReporter - { - private readonly PSCmdlet _cmdlet; - private readonly long _totalSize; - private readonly int _totalChunks; - private readonly DateTime _startTime; - private readonly string _operationName; - - // Progress tracking - private long _totalBytesTransferred = 0; - private int _currentChunk = 0; - private long _currentChunkBytesTransferred = 0; - private long _currentChunkSize = 0; - - // Activity IDs for progress hierarchy - private const int FileActivityId = 1; - private const int ChunkActivityId = 2; - - // Progress control - private readonly long _progressUpdateInterval; - private long _lastProgressUpdate = 0; - - /// - /// Creates a new chunked single file progress reporter - /// - /// PowerShell cmdlet for progress reporting - /// Total size of the file - /// Total number of chunks - /// Name of the operation (e.g., "Downloading", "Uploading") - /// Update progress every N bytes - public ChunkedSingleFileProgressReporter( - PSCmdlet cmdlet, - long totalSize, - int totalChunks, - string operationName = "Processing", - long progressUpdateInterval = 1024 * 1024) // 1MB default - { - _cmdlet = cmdlet; - _totalSize = totalSize; - _totalChunks = totalChunks; - _operationName = operationName; - _startTime = DateTime.Now; - _progressUpdateInterval = progressUpdateInterval; - } - - /// - /// Starts processing a new chunk - /// - /// Chunk number (1-based for display) - /// Size of the chunk - public void StartNewChunk(int chunkNumber, long chunkSize) - { - _currentChunk = chunkNumber; - _currentChunkSize = chunkSize; - _currentChunkBytesTransferred = 0; - - MinIOLogger.WriteVerbose(_cmdlet, "Starting chunk {0}/{1} ({2})", - chunkNumber, _totalChunks, SizeFormatter.FormatBytes(chunkSize)); - - UpdateAllProgress(); - } - - /// - /// Updates chunk progress - /// - /// Bytes transferred for current chunk - public void UpdateChunkProgress(long bytesTransferred) - { - var chunkDelta = bytesTransferred - _currentChunkBytesTransferred; - _currentChunkBytesTransferred = bytesTransferred; - _totalBytesTransferred += chunkDelta; - - // Only update progress if we've transferred enough bytes since last update - if (_totalBytesTransferred - _lastProgressUpdate >= _progressUpdateInterval) - { - UpdateAllProgress(); - _lastProgressUpdate = _totalBytesTransferred; - } - } - - /// - /// Completes the current chunk - /// - /// ETag of the completed chunk (optional) - public void CompleteChunk(string? chunkETag = null) - { - MinIOLogger.WriteVerbose(_cmdlet, "Completed chunk {0}/{1}{2}", - _currentChunk, _totalChunks, - !string.IsNullOrEmpty(chunkETag) ? $" - ETag: {chunkETag}" : ""); - - // Mark chunk as completed (only if progress is enabled) - if (_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") && - _cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue") - return; - - var chunkProgress = new ProgressRecord(ChunkActivityId, "Current Chunk", "Completed") - { - PercentComplete = 100, - RecordType = ProgressRecordType.Completed, - ParentActivityId = FileActivityId - }; - _cmdlet.WriteProgress(chunkProgress); - } - - /// - /// Completes the entire download operation - /// - public void CompleteDownload() - { - var elapsed = DateTime.Now - _startTime; - MinIOLogger.WriteVerbose(_cmdlet, "Completed {0} ({1}) in {2}", - _operationName.ToLower(), SizeFormatter.FormatBytes(_totalSize), elapsed.ToString(@"hh\:mm\:ss")); - - // Complete all progress records - var fileProgress = new ProgressRecord(FileActivityId, $"{_operationName} File", "Completed") - { - PercentComplete = 100, - RecordType = ProgressRecordType.Completed - }; - _cmdlet.WriteProgress(fileProgress); - - // Complete chunk progress if enabled - if (!(_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") && - _cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue")) - { - CompleteChunk(); - } - } - - /// - /// Reports an error for the current chunk - /// - /// Error that occurred - /// Current retry attempt - /// Maximum retry attempts - public void ReportChunkError(Exception error, int retryAttempt, int maxRetries) - { - MinIOLogger.WriteVerbose(_cmdlet, "Chunk {0}/{1} failed (attempt {2}/{3}): {4}", - _currentChunk, _totalChunks, retryAttempt, maxRetries, error.Message); - } - - /// - /// Updates all progress layers - /// - private void UpdateAllProgress() - { - var elapsed = DateTime.Now - _startTime; - var speed = elapsed.TotalSeconds > 0 ? _totalBytesTransferred / elapsed.TotalSeconds : 0; - - // Layer 1: File Progress (always shown) - var filePercent = _totalSize > 0 ? (int)((_totalBytesTransferred * 100) / _totalSize) : 0; - var fileStatus = $"Size: {SizeFormatter.FormatBytes(_totalBytesTransferred)}/{SizeFormatter.FormatBytes(_totalSize)} | " + - $"Speed: {SizeFormatter.FormatBytes((long)speed)}/s | " + - $"Elapsed: {elapsed:hh\\:mm\\:ss}"; - - var fileProgress = new ProgressRecord(FileActivityId, $"{_operationName} File", fileStatus) - { - PercentComplete = filePercent - }; - _cmdlet.WriteProgress(fileProgress); - - // Check if progress is disabled - if (_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") && - _cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue") - return; - - // Layer 2: Current Chunk Progress - if (_currentChunk > 0) - { - var chunkPercent = _currentChunkSize > 0 ? (int)((_currentChunkBytesTransferred * 100) / _currentChunkSize) : 0; - var chunkStatus = $"Chunk: {_currentChunk}/{_totalChunks} | " + - $"Size: {SizeFormatter.FormatBytes(_currentChunkBytesTransferred)}/{SizeFormatter.FormatBytes(_currentChunkSize)} | " + - $"Speed: {SizeFormatter.FormatBytes((long)speed)}/s"; - - var chunkProgress = new ProgressRecord(ChunkActivityId, "Current Chunk", chunkStatus) - { - PercentComplete = chunkPercent, - ParentActivityId = FileActivityId - }; - _cmdlet.WriteProgress(chunkProgress); - } - } - } -} diff --git a/src/Utils/ChunkedTransferResumeManager.cs b/src/Utils/ChunkedTransferResumeManager.cs deleted file mode 100644 index 5ba9440..0000000 --- a/src/Utils/ChunkedTransferResumeManager.cs +++ /dev/null @@ -1,250 +0,0 @@ -using System; -using System.IO; -using System.Text.Json; -using PSMinIO.Models; - -namespace PSMinIO.Utils -{ - /// - /// Manages resume data for chunked transfer operations - /// - public static class ChunkedTransferResumeManager - { - private static readonly string DefaultResumeDirectory = Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), - "PSMinIO", "Resume"); - - /// - /// Saves transfer state for resume functionality - /// - /// Transfer state to save - /// Custom path for resume data (optional) - /// Path where resume data was saved - public static string SaveTransferState(ChunkedTransferState transferState, string? customPath = null) - { - var resumeDirectory = customPath ?? DefaultResumeDirectory; - - // Ensure directory exists - if (!Directory.Exists(resumeDirectory)) - { - Directory.CreateDirectory(resumeDirectory); - } - - // Generate unique filename based on transfer details - var fileName = GenerateResumeFileName(transferState); - var filePath = Path.Combine(resumeDirectory, fileName); - - // Serialize and save - var json = JsonSerializer.Serialize(transferState, new JsonSerializerOptions - { - WriteIndented = true, - PropertyNamingPolicy = JsonNamingPolicy.CamelCase - }); - - File.WriteAllText(filePath, json); - return filePath; - } - - /// - /// Loads transfer state for resume functionality - /// - /// Bucket name - /// Object name - /// Local file path - /// Transfer type - /// Custom path for resume data (optional) - /// Transfer state if found, null otherwise - public static ChunkedTransferState? LoadTransferState( - string bucketName, - string objectName, - string filePath, - ChunkedTransferType transferType, - string? customPath = null) - { - var resumeDirectory = customPath ?? DefaultResumeDirectory; - - if (!Directory.Exists(resumeDirectory)) - return null; - - // Generate expected filename - var tempState = new ChunkedTransferState - { - BucketName = bucketName, - ObjectName = objectName, - FilePath = filePath, - TransferType = transferType - }; - - var fileName = GenerateResumeFileName(tempState); - var resumeFilePath = Path.Combine(resumeDirectory, fileName); - - if (!File.Exists(resumeFilePath)) - return null; - - try - { - var json = File.ReadAllText(resumeFilePath); - var transferState = JsonSerializer.Deserialize(json, new JsonSerializerOptions - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase - }); - - return transferState; - } - catch (Exception) - { - // If we can't deserialize, treat as no resume data - return null; - } - } - - /// - /// Deletes resume data after successful completion - /// - /// Transfer state to clean up - /// Custom path for resume data (optional) - public static void CleanupResumeData(ChunkedTransferState transferState, string? customPath = null) - { - var resumeDirectory = customPath ?? DefaultResumeDirectory; - var fileName = GenerateResumeFileName(transferState); - var filePath = Path.Combine(resumeDirectory, fileName); - - try - { - if (File.Exists(filePath)) - { - File.Delete(filePath); - } - } - catch (Exception) - { - // Ignore cleanup errors - } - } - - /// - /// Validates if resume data is still valid - /// - /// Transfer state to validate - /// Current file information - /// True if resume data is valid - public static bool IsResumeDataValid(ChunkedTransferState transferState, FileInfo? currentFileInfo = null) - { - // Check if transfer state is reasonable - if (transferState == null) - return false; - - // For uploads, validate source file hasn't changed - if (transferState.TransferType == ChunkedTransferType.Upload && currentFileInfo != null) - { - if (!currentFileInfo.Exists) - return false; - - // Check if file size or last modified time changed - if (currentFileInfo.Length != transferState.TotalSize || - currentFileInfo.LastWriteTimeUtc != transferState.LastModified) - { - return false; - } - } - - // Check if resume data is not too old (e.g., older than 7 days) - if (DateTime.UtcNow - transferState.LastUpdated > TimeSpan.FromDays(7)) - return false; - - return true; - } - - /// - /// Gets all resume files in the directory - /// - /// Custom path for resume data (optional) - /// Array of resume file paths - public static string[] GetResumeFiles(string? customPath = null) - { - var resumeDirectory = customPath ?? DefaultResumeDirectory; - - if (!Directory.Exists(resumeDirectory)) - return Array.Empty(); - - return Directory.GetFiles(resumeDirectory, "*.psminioResume"); - } - - /// - /// Cleans up old resume files - /// - /// Delete files older than this many days - /// Custom path for resume data (optional) - /// Number of files cleaned up - public static int CleanupOldResumeFiles(int olderThanDays = 7, string? customPath = null) - { - var resumeFiles = GetResumeFiles(customPath); - var cutoffDate = DateTime.UtcNow.AddDays(-olderThanDays); - var cleanedCount = 0; - - foreach (var file in resumeFiles) - { - try - { - var fileInfo = new FileInfo(file); - if (fileInfo.LastWriteTimeUtc < cutoffDate) - { - File.Delete(file); - cleanedCount++; - } - } - catch (Exception) - { - // Ignore cleanup errors - } - } - - return cleanedCount; - } - - /// - /// Generates a unique filename for resume data - /// - /// Transfer state - /// Unique filename - private static string GenerateResumeFileName(ChunkedTransferState transferState) - { - // Create a hash of the key components to ensure uniqueness - var key = $"{transferState.BucketName}|{transferState.ObjectName}|{transferState.FilePath}|{transferState.TransferType}"; - var hash = key.GetHashCode().ToString("X8"); - - // Include readable components for easier identification - var safeBucketName = MakeSafeFileName(transferState.BucketName); - var safeObjectName = MakeSafeFileName(Path.GetFileName(transferState.ObjectName)); - var transferType = transferState.TransferType.ToString().ToLower(); - - return $"{safeBucketName}_{safeObjectName}_{transferType}_{hash}.psminioResume"; - } - - /// - /// Makes a string safe for use as a filename - /// - /// Input string - /// Safe filename string - private static string MakeSafeFileName(string input) - { - if (string.IsNullOrEmpty(input)) - return "unknown"; - - var invalidChars = Path.GetInvalidFileNameChars(); - var safe = input; - - foreach (var c in invalidChars) - { - safe = safe.Replace(c, '_'); - } - - // Limit length and remove leading/trailing dots and spaces - safe = safe.Trim(' ', '.'); - if (safe.Length > 50) - safe = safe.Substring(0, 50); - - return string.IsNullOrEmpty(safe) ? "unknown" : safe; - } - } -} diff --git a/src/Utils/MinIOClientWrapper.cs b/src/Utils/MinIOClientWrapper.cs deleted file mode 100644 index 03de0df..0000000 --- a/src/Utils/MinIOClientWrapper.cs +++ /dev/null @@ -1,954 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using Minio; -using Minio.DataModel; -using PSMinIO.Models; -using PSMinIO.Utils; - -namespace PSMinIO.Utils -{ - /// - /// Synchronous wrapper for MinIO client operations - /// Converts async MinIO operations to synchronous calls for PowerShell compatibility - /// - public class MinIOClientWrapper : IDisposable - { - private readonly IMinioClient _client; - private readonly CancellationTokenSource _cancellationTokenSource; - private bool _disposed = false; - - /// - /// Creates a new MinIOClientWrapper instance - /// - /// MinIO configuration - public MinIOClientWrapper(MinIOConfiguration configuration) - { - if (configuration == null) - throw new ArgumentNullException(nameof(configuration)); - - if (!configuration.IsValid) - throw new ArgumentException("Invalid MinIO configuration", nameof(configuration)); - - _cancellationTokenSource = new CancellationTokenSource(); - - // Create MinIO client with configuration - var clientBuilder = new MinioClient() - .WithEndpoint(configuration.Endpoint) - .WithCredentials(configuration.AccessKey, configuration.SecretKey); - - if (configuration.UseSSL) - { - clientBuilder = clientBuilder.WithSSL(); - - // Configure custom HttpClient for certificate validation if needed - if (configuration.SkipCertificateValidation) - { - var httpClientHandler = new HttpClientHandler() - { - ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => true - }; - var httpClient = new HttpClient(httpClientHandler); - clientBuilder = clientBuilder.WithHttpClient(httpClient); - } - } - - if (configuration.TimeoutSeconds > 0) - { - clientBuilder = clientBuilder.WithTimeout(configuration.TimeoutSeconds * 1000); - } - - _client = clientBuilder.Build(); - } - - /// - /// Gets the cancellation token for operations - /// - public CancellationToken CancellationToken => _cancellationTokenSource.Token; - - /// - /// Lists all buckets synchronously - /// - /// List of bucket information - public List ListBuckets() - { - try - { - var bucketsResult = Task.Run(async () => - await _client.ListBucketsAsync(CancellationToken)).GetAwaiter().GetResult(); - - return bucketsResult.Buckets - .Select(MinIOBucketInfo.FromMinioBucket) - .ToList(); - } - catch (Exception ex) - { - throw new InvalidOperationException($"Failed to list buckets: {ex.Message}", ex); - } - } - - /// - /// Checks if a bucket exists synchronously - /// - /// Name of the bucket - /// True if bucket exists, false otherwise - public bool BucketExists(string bucketName) - { - if (string.IsNullOrWhiteSpace(bucketName)) - throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); - - try - { - var args = new BucketExistsArgs().WithBucket(bucketName); - return Task.Run(async () => - await _client.BucketExistsAsync(args, CancellationToken)).GetAwaiter().GetResult(); - } - catch (Exception ex) - { - throw new InvalidOperationException($"Failed to check if bucket '{bucketName}' exists: {ex.Message}", ex); - } - } - - /// - /// Creates a bucket synchronously - /// - /// Name of the bucket to create - /// Optional region for the bucket - public void CreateBucket(string bucketName, string? region = null) - { - if (string.IsNullOrWhiteSpace(bucketName)) - throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); - - try - { - var args = new MakeBucketArgs().WithBucket(bucketName); - - if (!string.IsNullOrWhiteSpace(region)) - { - args = args.WithLocation(region); - } - - Task.Run(async () => - await _client.MakeBucketAsync(args, CancellationToken)).GetAwaiter().GetResult(); - } - catch (Exception ex) - { - throw new InvalidOperationException($"Failed to create bucket '{bucketName}': {ex.Message}", ex); - } - } - - /// - /// Deletes a bucket synchronously - /// - /// Name of the bucket to delete - public void DeleteBucket(string bucketName) - { - if (string.IsNullOrWhiteSpace(bucketName)) - throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); - - try - { - var args = new RemoveBucketArgs().WithBucket(bucketName); - Task.Run(async () => - await _client.RemoveBucketAsync(args, CancellationToken)).GetAwaiter().GetResult(); - } - catch (Exception ex) - { - throw new InvalidOperationException($"Failed to delete bucket '{bucketName}': {ex.Message}", ex); - } - } - - /// - /// Lists objects in a bucket synchronously - /// - /// Name of the bucket - /// Optional prefix to filter objects - /// Whether to list objects recursively - /// Whether to include all versions of objects - /// List of object information - public List ListObjects(string bucketName, string? prefix = null, bool recursive = true, bool includeVersions = false) - { - if (string.IsNullOrWhiteSpace(bucketName)) - throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); - - try - { - var args = new ListObjectsArgs() - .WithBucket(bucketName) - .WithRecursive(recursive); - - if (!string.IsNullOrWhiteSpace(prefix)) - { - args = args.WithPrefix(prefix); - } - - // Add version support if requested - if (includeVersions) - { - args = args.WithVersions(true); - } - - var objects = new List(); - var observable = _client.ListObjectsAsync(args, CancellationToken); - - // Convert observable to synchronous list - var task = Task.Run(() => - { - var tcs = new TaskCompletionSource(); - observable.Subscribe( - onNext: item => objects.Add(MinIOObjectInfo.FromMinioItem(item, bucketName)), - onError: ex => tcs.SetException(ex), - onCompleted: () => tcs.SetResult(true) - ); - return tcs.Task; - }); - - task.GetAwaiter().GetResult(); - return objects; - } - catch (Exception ex) - { - throw new InvalidOperationException($"Failed to list objects in bucket '{bucketName}': {ex.Message}", ex); - } - } - - /// - /// Gets bucket policy synchronously - /// - /// Name of the bucket - /// Bucket policy as JSON string - public string GetBucketPolicy(string bucketName) - { - if (string.IsNullOrWhiteSpace(bucketName)) - throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); - - try - { - var args = new GetPolicyArgs().WithBucket(bucketName); - return Task.Run(async () => - await _client.GetPolicyAsync(args, CancellationToken)).GetAwaiter().GetResult(); - } - catch (Exception ex) - { - throw new InvalidOperationException($"Failed to get policy for bucket '{bucketName}': {ex.Message}", ex); - } - } - - /// - /// Sets bucket policy synchronously - /// - /// Name of the bucket - /// Policy JSON string - public void SetBucketPolicy(string bucketName, string policy) - { - if (string.IsNullOrWhiteSpace(bucketName)) - throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); - - if (string.IsNullOrWhiteSpace(policy)) - throw new ArgumentException("Policy cannot be null or empty", nameof(policy)); - - try - { - var args = new SetPolicyArgs() - .WithBucket(bucketName) - .WithPolicy(policy); - - Task.Run(async () => - await _client.SetPolicyAsync(args, CancellationToken)).GetAwaiter().GetResult(); - } - catch (Exception ex) - { - throw new InvalidOperationException($"Failed to set policy for bucket '{bucketName}': {ex.Message}", ex); - } - } - - /// - /// Deletes an object synchronously - /// - /// Name of the bucket - /// Name of the object to delete - public void DeleteObject(string bucketName, string objectName) - { - if (string.IsNullOrWhiteSpace(bucketName)) - throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); - - if (string.IsNullOrWhiteSpace(objectName)) - throw new ArgumentException("Object name cannot be null or empty", nameof(objectName)); - - try - { - var args = new RemoveObjectArgs() - .WithBucket(bucketName) - .WithObject(objectName); - - Task.Run(async () => - await _client.RemoveObjectAsync(args, CancellationToken)).GetAwaiter().GetResult(); - } - catch (Exception ex) - { - throw new InvalidOperationException($"Failed to delete object '{objectName}' from bucket '{bucketName}': {ex.Message}", ex); - } - } - - /// - /// Deletes multiple objects synchronously - /// - /// Name of the bucket - /// List of object names to delete - public void DeleteObjects(string bucketName, IEnumerable objectNames) - { - if (string.IsNullOrWhiteSpace(bucketName)) - throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); - - if (objectNames == null) - throw new ArgumentNullException(nameof(objectNames)); - - var objectList = objectNames.ToList(); - if (objectList.Count == 0) - return; - - try - { - var deleteObjectsArgs = new RemoveObjectsArgs() - .WithBucket(bucketName) - .WithObjects(objectList); - - var observableTask = _client.RemoveObjectsAsync(deleteObjectsArgs, CancellationToken); - - // Convert observable to synchronous operation - var task = Task.Run(async () => - { - var observable = await observableTask; - var tcs = new TaskCompletionSource(); - observable.Subscribe( - onNext: deleteError => - { - // In MinIO 5.0.0, DeleteError might have different properties - // For now, just check if there's an error and report it - if (!string.IsNullOrEmpty(deleteError.Message)) - { - tcs.SetException(new InvalidOperationException($"Failed to delete object '{deleteError.Key}': {deleteError.Message}")); - } - }, - onError: ex => tcs.SetException(ex), - onCompleted: () => tcs.SetResult(true) - ); - return await tcs.Task; - }); - - task.GetAwaiter().GetResult(); - } - catch (Exception ex) - { - throw new InvalidOperationException($"Failed to delete objects from bucket '{bucketName}': {ex.Message}", ex); - } - } - - /// - /// Uploads a file to MinIO synchronously with progress reporting - /// - /// Name of the bucket - /// Name of the object - /// Path to the file to upload - /// Content type of the file (optional) - /// Progress callback for reporting upload progress - /// ETag of the uploaded object - public string UploadFile(string bucketName, string objectName, string filePath, - string? contentType = null, Action? progressCallback = null) - { - if (string.IsNullOrWhiteSpace(bucketName)) - throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); - - if (string.IsNullOrWhiteSpace(objectName)) - throw new ArgumentException("Object name cannot be null or empty", nameof(objectName)); - - if (string.IsNullOrWhiteSpace(filePath)) - throw new ArgumentException("File path cannot be null or empty", nameof(filePath)); - - if (!File.Exists(filePath)) - throw new FileNotFoundException($"File not found: {filePath}"); - - try - { - var fileInfo = new FileInfo(filePath); - var fileSize = fileInfo.Length; - - // Determine content type if not provided - if (string.IsNullOrWhiteSpace(contentType)) - { - contentType = GetContentType(filePath); - } - - // Use explicit file stream management to ensure proper handle release - using var fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read); - - var args = new PutObjectArgs() - .WithBucket(bucketName) - .WithObject(objectName) - .WithStreamData(fileStream) - .WithObjectSize(fileSize) - .WithContentType(contentType); - - // Progress tracking not available in MinIO 4.0.7 - // progressCallback is ignored for now - - Task.Run(async () => - await _client.PutObjectAsync(args, CancellationToken)).GetAwaiter().GetResult(); - - return string.Empty; // MinIO 4.0.7 PutObjectAsync returns void - } - catch (Exception ex) - { - throw new InvalidOperationException($"Failed to upload file '{filePath}' to bucket '{bucketName}' as '{objectName}': {ex.Message}", ex); - } - } - - /// - /// Downloads an object from MinIO synchronously with progress reporting - /// - /// Name of the bucket - /// Name of the object - /// Path where the file should be saved - /// Progress callback for reporting download progress - public void DownloadFile(string bucketName, string objectName, string filePath, - Action? progressCallback = null) - { - if (string.IsNullOrWhiteSpace(bucketName)) - throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); - - if (string.IsNullOrWhiteSpace(objectName)) - throw new ArgumentException("Object name cannot be null or empty", nameof(objectName)); - - if (string.IsNullOrWhiteSpace(filePath)) - throw new ArgumentException("File path cannot be null or empty", nameof(filePath)); - - try - { - // Ensure the directory exists - var directory = Path.GetDirectoryName(filePath); - if (!string.IsNullOrEmpty(directory) && !Directory.Exists(directory)) - { - Directory.CreateDirectory(directory); - } - - var args = new GetObjectArgs() - .WithBucket(bucketName) - .WithObject(objectName) - .WithFile(filePath); - - // Progress tracking not available in MinIO 5.0.0 - // progressCallback is ignored for now - - Task.Run(async () => - await _client.GetObjectAsync(args, CancellationToken)).GetAwaiter().GetResult(); - } - catch (Exception ex) - { - throw new InvalidOperationException($"Failed to download object '{objectName}' from bucket '{bucketName}' to '{filePath}': {ex.Message}", ex); - } - } - - /// - /// Creates a directory object in MinIO (zero-byte object with trailing slash) - /// - /// Name of the bucket - /// Path of the directory (should end with /) - /// ETag of the created directory object - public string CreateDirectory(string bucketName, string directoryPath) - { - if (string.IsNullOrWhiteSpace(bucketName)) - throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); - - if (string.IsNullOrWhiteSpace(directoryPath)) - throw new ArgumentException("Directory path cannot be null or empty", nameof(directoryPath)); - - // Ensure directory path ends with / - if (!directoryPath.EndsWith("/")) - directoryPath += "/"; - - try - { - // Use a properly configured empty stream - using var emptyStream = new MemoryStream(new byte[0]); - emptyStream.Position = 0; - - var args = new PutObjectArgs() - .WithBucket(bucketName) - .WithObject(directoryPath) - .WithStreamData(emptyStream) - .WithObjectSize(0) - .WithContentType("application/x-directory"); - - Task.Run(async () => - await _client.PutObjectAsync(args, CancellationToken)).GetAwaiter().GetResult(); - - return ""; // Directory objects don't have meaningful ETags - } - catch (Exception ex) - { - throw new InvalidOperationException($"Failed to create directory '{directoryPath}' in bucket '{bucketName}': {ex.Message}", ex); - } - } - - /// - /// Uploads a stream to MinIO synchronously - /// - /// Name of the bucket - /// Name of the object - /// Stream containing the data to upload - /// Content type of the data - /// Progress callback for reporting upload progress - /// ETag of the uploaded object - public string UploadStream(string bucketName, string objectName, Stream data, - string contentType = "application/octet-stream", Action? progressCallback = null) - { - if (string.IsNullOrWhiteSpace(bucketName)) - throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); - - if (string.IsNullOrWhiteSpace(objectName)) - throw new ArgumentException("Object name cannot be null or empty", nameof(objectName)); - - if (data == null) - throw new ArgumentNullException(nameof(data)); - - try - { - // Ensure stream is at the beginning - if (data.CanSeek) - { - data.Position = 0; - } - - var args = new PutObjectArgs() - .WithBucket(bucketName) - .WithObject(objectName) - .WithStreamData(data) - .WithObjectSize(data.Length) - .WithContentType(contentType); - - // Progress tracking not available in MinIO 5.0.0 - // progressCallback is ignored for now - - Task.Run(async () => - await _client.PutObjectAsync(args, CancellationToken)).GetAwaiter().GetResult(); - - return string.Empty; // MinIO 4.0.7 PutObjectAsync returns void - } - catch (Exception ex) - { - throw new InvalidOperationException($"Failed to upload stream to bucket '{bucketName}' as '{objectName}': {ex.Message}", ex); - } - } - - /// - /// Gets the content type for a file based on its extension - /// - /// Path to the file - /// Content type string - private static string GetContentType(string filePath) - { - var extension = Path.GetExtension(filePath).ToLowerInvariant(); - - return extension switch - { - ".txt" => "text/plain", - ".html" => "text/html", - ".css" => "text/css", - ".js" => "application/javascript", - ".json" => "application/json", - ".xml" => "application/xml", - ".pdf" => "application/pdf", - ".zip" => "application/zip", - ".jpg" or ".jpeg" => "image/jpeg", - ".png" => "image/png", - ".gif" => "image/gif", - ".svg" => "image/svg+xml", - ".mp4" => "video/mp4", - ".mp3" => "audio/mpeg", - ".wav" => "audio/wav", - _ => "application/octet-stream" - }; - } - - /// - /// Lists object versions in a bucket synchronously - /// - /// Name of the bucket - /// Optional prefix to filter objects - /// Whether to list objects recursively - /// Maximum number of objects to return (0 = unlimited) - /// List of object information including versions - private List ListObjectVersions(string bucketName, string? prefix, bool recursive, int maxObjects) - { - try - { - // For now, fall back to regular object listing since version listing - // may not be available in all MinIO SDK versions - // This can be enhanced when the SDK supports it - return ListObjects(bucketName, prefix, recursive, false); - } - catch (Exception ex) - { - throw new InvalidOperationException($"Failed to list object versions in bucket '{bucketName}': {ex.Message}", ex); - } - } - - /// - /// Generates a presigned URL for an object - /// - /// Name of the bucket - /// Name of the object - /// URL expiry time - /// Presigned URL - public string GetPresignedUrl(string bucketName, string objectName, TimeSpan expiry) - { - if (string.IsNullOrWhiteSpace(bucketName)) - throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName)); - - if (string.IsNullOrWhiteSpace(objectName)) - throw new ArgumentException("Object name cannot be null or empty", nameof(objectName)); - - try - { - var args = new PresignedGetObjectArgs() - .WithBucket(bucketName) - .WithObject(objectName) - .WithExpiry((int)expiry.TotalSeconds); - - var result = Task.Run(async () => - await _client.PresignedGetObjectAsync(args)).GetAwaiter().GetResult(); - - return result ?? string.Empty; - } - catch (Exception ex) - { - throw new InvalidOperationException($"Failed to generate presigned URL for object '{objectName}' in bucket '{bucketName}': {ex.Message}", ex); - } - } - - /// - /// Uploads a file using chunked transfer with resume capability - /// - /// Transfer state for resume functionality - /// Thread-safe progress reporter for updates - /// Maximum retry attempts per chunk - /// MinIOObjectInfo of uploaded object or null if failed - public MinIOObjectInfo? UploadFileChunked( - ChunkedTransferState transferState, - ThreadSafeChunkedProgressReporter progressReporter, - int maxRetries = 3) - { - if (transferState == null) - throw new ArgumentNullException(nameof(transferState)); - - try - { - // For now, implement chunked upload using regular PutObject with progress tracking - // This simulates chunked behavior by reading the file in chunks and reporting progress - using var fileStream = new FileStream(transferState.FilePath, FileMode.Open, FileAccess.Read, FileShare.Read); - - var totalChunks = (int)Math.Ceiling((double)transferState.TotalSize / transferState.ChunkSize); - var buffer = new byte[transferState.ChunkSize]; - long totalBytesRead = 0; - - // Create a progress-tracking stream wrapper - var progressStream = new ProgressTrackingStream(fileStream, (bytesRead) => - { - var currentChunk = (int)(totalBytesRead / transferState.ChunkSize) + 1; - var chunkProgress = bytesRead % transferState.ChunkSize; - - if (currentChunk <= totalChunks) - { - progressReporter.StartNewChunk(currentChunk, Math.Min(transferState.ChunkSize, transferState.TotalSize - totalBytesRead)); - progressReporter.UpdateChunkProgress(chunkProgress); - - if (chunkProgress == 0 && bytesRead > 0) // Chunk completed - { - progressReporter.CompleteChunk(); - } - } - - totalBytesRead = bytesRead; - }); - - // Upload the file - var putArgs = new PutObjectArgs() - .WithBucket(transferState.BucketName) - .WithObject(transferState.ObjectName) - .WithStreamData(progressStream) - .WithObjectSize(transferState.TotalSize); - - Task.Run(async () => - { - await _client.PutObjectAsync(putArgs, CancellationToken); - }).GetAwaiter().GetResult(); - - // Return object information - return new MinIOObjectInfo( - transferState.ObjectName, - transferState.TotalSize, - DateTime.UtcNow, - "simulated-etag", // MinIO 5.0.0 PutObject doesn't return ETag directly - transferState.BucketName); - } - catch (Exception ex) - { - throw new InvalidOperationException($"Chunked upload failed for object '{transferState.ObjectName}': {ex.Message}", ex); - } - } - - /// - /// Progress tracking stream wrapper - /// - private class ProgressTrackingStream : Stream - { - private readonly Stream _baseStream; - private readonly Action _progressCallback; - private long _totalBytesRead = 0; - - public ProgressTrackingStream(Stream baseStream, Action progressCallback) - { - _baseStream = baseStream; - _progressCallback = progressCallback; - } - - public override bool CanRead => _baseStream.CanRead; - public override bool CanSeek => _baseStream.CanSeek; - public override bool CanWrite => _baseStream.CanWrite; - public override long Length => _baseStream.Length; - public override long Position - { - get => _baseStream.Position; - set => _baseStream.Position = value; - } - - public override int Read(byte[] buffer, int offset, int count) - { - var bytesRead = _baseStream.Read(buffer, offset, count); - _totalBytesRead += bytesRead; - _progressCallback(_totalBytesRead); - return bytesRead; - } - - public override void Flush() => _baseStream.Flush(); - public override long Seek(long offset, SeekOrigin origin) => _baseStream.Seek(offset, origin); - public override void SetLength(long value) => _baseStream.SetLength(value); - public override void Write(byte[] buffer, int offset, int count) => _baseStream.Write(buffer, offset, count); - - protected override void Dispose(bool disposing) - { - if (disposing) - { - _baseStream?.Dispose(); - } - base.Dispose(disposing); - } - } - - /// - /// Downloads a file using chunked transfer with resume capability - /// - /// Transfer state for resume functionality - /// Thread-safe progress reporter for updates - /// Maximum retry attempts per chunk - /// Number of parallel chunk downloads - /// True if download succeeded, false otherwise - public bool DownloadFileChunked( - ChunkedTransferState transferState, - ThreadSafeChunkedProgressReporter progressReporter, - int maxRetries = 3, - int parallelDownloads = 3) - { - if (transferState == null) - throw new ArgumentNullException(nameof(transferState)); - - try - { - // Create or open the target file - using var fileStream = new FileStream(transferState.FilePath, FileMode.Create, FileAccess.Write, FileShare.None); - fileStream.SetLength(transferState.TotalSize); - - // Get list of chunks to download (with safety limit) - var chunksToDownload = new List(); - var maxChunks = Math.Min(transferState.TotalChunks, 10000); // Safety limit of 10,000 chunks - - for (int i = 0; i < maxChunks && !transferState.IsComplete; i++) - { - var nextChunk = transferState.GetNextChunk(); - if (nextChunk == null) - break; - chunksToDownload.Add(nextChunk); - } - - if (chunksToDownload.Count == 0) - { - return true; // Already complete - } - - // Download chunks (with limited parallelism) - var semaphore = new SemaphoreSlim(parallelDownloads, parallelDownloads); - var downloadTasks = chunksToDownload.Select(chunk => - DownloadChunkAsync(transferState, chunk, fileStream, progressReporter, maxRetries, semaphore)).ToArray(); - - var results = Task.WhenAll(downloadTasks).GetAwaiter().GetResult(); - - // Check if all chunks downloaded successfully - var allSucceeded = results.All(r => r); - if (allSucceeded) - { - // Mark all chunks as completed - foreach (var chunk in chunksToDownload) - { - transferState.MarkChunkCompleted(chunk); - } - } - - return allSucceeded; - } - catch (Exception ex) - { - throw new InvalidOperationException($"Chunked download failed for object '{transferState.ObjectName}': {ex.Message}", ex); - } - } - - /// - /// Downloads a single chunk asynchronously with retry logic - /// - /// Transfer state - /// Chunk to download - /// Target file stream - /// Thread-safe progress reporter - /// Maximum retry attempts - /// Semaphore for controlling parallelism - /// True if chunk downloaded successfully - private async Task DownloadChunkAsync( - ChunkedTransferState transferState, - ChunkInfo chunk, - FileStream fileStream, - ThreadSafeChunkedProgressReporter progressReporter, - int maxRetries, - SemaphoreSlim semaphore) - { - await semaphore.WaitAsync(CancellationToken); - - try - { - progressReporter.StartNewChunk(chunk.ChunkNumber + 1, chunk.Size); - - for (int attempt = 1; attempt <= maxRetries; attempt++) - { - try - { - using var chunkStream = new MemoryStream(); - - // Use GetObjectAsync with offset and length for proper chunked download - var getArgs = new GetObjectArgs() - .WithBucket(transferState.BucketName) - .WithObject(transferState.ObjectName) - .WithCallbackStream((stream) => - { - // Skip to the start position and read only the chunk size - var buffer = new byte[8192]; // 8KB buffer - long totalRead = 0; - long skipBytes = chunk.StartByte; - - // Skip to start position - while (skipBytes > 0) - { - var toSkip = (int)Math.Min(skipBytes, buffer.Length); - var skipped = stream.Read(buffer, 0, toSkip); - if (skipped == 0) break; // End of stream - skipBytes -= skipped; - } - - // Read the chunk data - while (totalRead < chunk.Size) - { - var toRead = (int)Math.Min(chunk.Size - totalRead, buffer.Length); - var bytesRead = stream.Read(buffer, 0, toRead); - if (bytesRead == 0) break; // End of stream - - chunkStream.Write(buffer, 0, bytesRead); - totalRead += bytesRead; - } - }); - - await _client.GetObjectAsync(getArgs, CancellationToken); - - // Write chunk to file at correct position - lock (fileStream) - { - fileStream.Seek(chunk.StartByte, SeekOrigin.Begin); - chunkStream.Seek(0, SeekOrigin.Begin); - chunkStream.CopyTo(fileStream); - fileStream.Flush(); - } - - progressReporter.UpdateChunkProgress(chunk.Size); - progressReporter.CompleteChunk(); - - chunk.IsCompleted = true; - return true; - } - catch (Exception ex) when (attempt < maxRetries) - { - progressReporter.ReportChunkError(ex, attempt, maxRetries); - - // Exponential backoff - var delay = TimeSpan.FromSeconds(Math.Pow(2, attempt)); - await Task.Delay(delay, CancellationToken); - } - catch (Exception ex) - { - progressReporter.ReportChunkError(ex, attempt, maxRetries); - chunk.LastError = ex.Message; - chunk.RetryCount = attempt; - return false; - } - } - - return false; - } - finally - { - semaphore.Release(); - } - } - - /// - /// Cancels all ongoing operations - /// - public void CancelOperations() - { - _cancellationTokenSource.Cancel(); - } - - /// - /// Disposes the wrapper and underlying client - /// - public void Dispose() - { - Dispose(true); - GC.SuppressFinalize(this); - } - - /// - /// Protected dispose method - /// - /// Whether disposing from Dispose method - protected virtual void Dispose(bool disposing) - { - if (!_disposed && disposing) - { - _cancellationTokenSource?.Cancel(); - _cancellationTokenSource?.Dispose(); - _client?.Dispose(); - _disposed = true; - } - } - } -} diff --git a/src/Utils/MinIOLogger.cs b/src/Utils/MinIOLogger.cs index 89e1c7b..24da4f6 100644 --- a/src/Utils/MinIOLogger.cs +++ b/src/Utils/MinIOLogger.cs @@ -4,23 +4,13 @@ using System.Management.Automation; namespace PSMinIO.Utils { /// - /// Centralized logging utility for PSMinIO module + /// Centralized logging utility for PSMinIO operations + /// Provides consistent logging with timestamps and proper PowerShell integration /// public static class MinIOLogger { /// - /// Log levels for different types of messages - /// - public enum LogLevel - { - Verbose, - Information, - Warning, - Error - } - - /// - /// Writes a verbose message if verbose preference allows it + /// Writes a verbose message with timestamp /// /// The cmdlet instance for context /// Message to log @@ -29,220 +19,168 @@ namespace PSMinIO.Utils { if (cmdlet == null) return; - var formattedMessage = FormatLogMessage(LogLevel.Verbose, message, args); - cmdlet.WriteVerbose(formattedMessage); + var timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"); + var formattedMessage = args.Length > 0 ? string.Format(message, args) : message; + cmdlet.WriteVerbose($"[{timestamp}] {formattedMessage}"); } /// - /// Writes an information message + /// Writes a warning message with timestamp /// /// The cmdlet instance for context - /// Message to log - /// Optional format arguments - public static void WriteInformation(PSCmdlet cmdlet, string message, params object[] args) - { - if (cmdlet == null) return; - - var formattedMessage = FormatLogMessage(LogLevel.Information, message, args); - - // Use WriteInformation if available (PowerShell 5.0+), otherwise WriteVerbose - try - { - var infoRecord = new InformationRecord(formattedMessage, "PSMinIO"); - cmdlet.WriteInformation(infoRecord); - } - catch - { - // Fallback to WriteVerbose for older PowerShell versions - cmdlet.WriteVerbose(formattedMessage); - } - } - - /// - /// Writes a warning message - /// - /// The cmdlet instance for context - /// Message to log + /// Warning message to log /// Optional format arguments public static void WriteWarning(PSCmdlet cmdlet, string message, params object[] args) { if (cmdlet == null) return; - var formattedMessage = FormatLogMessage(LogLevel.Warning, message, args); - cmdlet.WriteWarning(formattedMessage); + var timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"); + var formattedMessage = args.Length > 0 ? string.Format(message, args) : message; + cmdlet.WriteWarning($"[{timestamp}] {formattedMessage}"); } /// - /// Writes an error message + /// Writes a debug message with timestamp /// /// The cmdlet instance for context - /// Message to log + /// Debug message to log /// Optional format arguments - public static void WriteError(PSCmdlet cmdlet, string message, params object[] args) + public static void WriteDebug(PSCmdlet cmdlet, string message, params object[] args) { if (cmdlet == null) return; - var formattedMessage = FormatLogMessage(LogLevel.Error, message, args); - var errorRecord = new ErrorRecord( - new InvalidOperationException(formattedMessage), - "PSMinIOError", - ErrorCategory.InvalidOperation, - null); - - cmdlet.WriteError(errorRecord); - } - - /// - /// Writes an error from an exception - /// - /// The cmdlet instance for context - /// Exception to log - /// Error identifier - /// Error category - /// Target object that caused the error - public static void WriteError(PSCmdlet cmdlet, Exception exception, string errorId, - ErrorCategory category = ErrorCategory.InvalidOperation, object? targetObject = null) - { - if (cmdlet == null) return; - - var errorRecord = new ErrorRecord(exception, errorId, category, targetObject); - cmdlet.WriteError(errorRecord); + var timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"); + var formattedMessage = args.Length > 0 ? string.Format(message, args) : message; + cmdlet.WriteDebug($"[{timestamp}] {formattedMessage}"); } /// /// Logs the start of an operation /// /// The cmdlet instance for context - /// Name of the operation + /// Name of the operation /// Optional operation details - public static void LogOperationStart(PSCmdlet cmdlet, string operation, string? details = null) + public static void LogOperationStart(PSCmdlet cmdlet, string operationName, string? details = null) { var message = string.IsNullOrEmpty(details) - ? $"Starting operation: {operation}" - : $"Starting operation: {operation} - {details}"; + ? $"Starting {operationName}" + : $"Starting {operationName} - {details}"; WriteVerbose(cmdlet, message); } /// - /// Logs the completion of an operation + /// Logs the successful completion of an operation /// /// The cmdlet instance for context - /// Name of the operation - /// Optional operation duration + /// Name of the operation + /// Operation duration /// Optional operation details - public static void LogOperationComplete(PSCmdlet cmdlet, string operation, TimeSpan? duration = null, string? details = null) + public static void LogOperationComplete(PSCmdlet cmdlet, string operationName, TimeSpan duration, string? details = null) { - var message = $"Completed operation: {operation}"; - - if (duration.HasValue) - { - message += $" (Duration: {duration.Value.TotalMilliseconds:F0}ms)"; - } - - if (!string.IsNullOrEmpty(details)) - { - message += $" - {details}"; - } + var durationStr = SizeFormatter.FormatDuration(duration); + var message = string.IsNullOrEmpty(details) + ? $"Completed {operationName} in {durationStr}" + : $"Completed {operationName} in {durationStr} - {details}"; WriteVerbose(cmdlet, message); } /// - /// Logs an operation failure + /// Logs the failure of an operation /// /// The cmdlet instance for context - /// Name of the operation + /// Name of the operation /// Exception that occurred /// Optional operation details - public static void LogOperationFailure(PSCmdlet cmdlet, string operation, Exception exception, string? details = null) + public static void LogOperationFailure(PSCmdlet cmdlet, string operationName, Exception exception, string? details = null) { - var message = $"Operation failed: {operation} - {exception.Message}"; + var message = string.IsNullOrEmpty(details) + ? $"Failed {operationName}: {exception.Message}" + : $"Failed {operationName} - {details}: {exception.Message}"; - if (!string.IsNullOrEmpty(details)) - { - message += $" - {details}"; - } + WriteWarning(cmdlet, message); + WriteDebug(cmdlet, "Exception details: {0}", exception.ToString()); + } + + /// + /// Logs transfer progress information + /// + /// The cmdlet instance for context + /// Name of the transfer operation + /// Bytes transferred so far + /// Total bytes to transfer + /// Current transfer speed in bytes per second + public static void LogTransferProgress(PSCmdlet cmdlet, string operationName, long bytesTransferred, long totalBytes, double speed) + { + var percentage = totalBytes > 0 ? (double)bytesTransferred / totalBytes * 100 : 0; + var speedStr = SizeFormatter.FormatSpeed(speed); + var transferredStr = SizeFormatter.FormatBytes(bytesTransferred); + var totalStr = SizeFormatter.FormatBytes(totalBytes); + + WriteVerbose(cmdlet, "{0} progress: {1:F1}% ({2}/{3}) at {4}", + operationName, percentage, transferredStr, totalStr, speedStr); + } + + /// + /// Logs connection information + /// + /// The cmdlet instance for context + /// MinIO endpoint + /// Whether SSL is being used + /// Region + /// Timeout in seconds + public static void LogConnection(PSCmdlet cmdlet, string endpoint, bool useSSL, string region, int timeout) + { + WriteVerbose(cmdlet, "Connecting to MinIO - Endpoint: {0}, UseSSL: {1}, Region: {2}, Timeout: {3}s", + endpoint, useSSL, region, timeout); + } + + /// + /// Logs retry attempt information + /// + /// The cmdlet instance for context + /// Name of the operation being retried + /// Current attempt number + /// Maximum number of attempts + /// Delay before retry + /// Reason for retry + public static void LogRetryAttempt(PSCmdlet cmdlet, string operationName, int attempt, int maxAttempts, TimeSpan delay, string reason) + { + WriteVerbose(cmdlet, "Retrying {0} (attempt {1}/{2}) after {3} - {4}", + operationName, attempt, maxAttempts, SizeFormatter.FormatDuration(delay), reason); + } + + /// + /// Logs HTTP request information for debugging + /// + /// The cmdlet instance for context + /// HTTP method + /// Request URL + /// Content length (if applicable) + public static void LogHttpRequest(PSCmdlet cmdlet, string method, string url, long? contentLength = null) + { + var message = contentLength.HasValue + ? $"HTTP {method} {url} (Content-Length: {SizeFormatter.FormatBytes(contentLength.Value)})" + : $"HTTP {method} {url}"; - WriteError(cmdlet, message); - WriteVerbose(cmdlet, $"Exception details: {exception}"); + WriteDebug(cmdlet, message); } /// - /// Formats a log message with timestamp and level, automatically formatting byte sizes + /// Logs HTTP response information for debugging /// - /// Log level - /// Message to format - /// Optional format arguments - /// Formatted log message - private static string FormatLogMessage(LogLevel level, string message, params object[] args) + /// The cmdlet instance for context + /// HTTP status code + /// Response content length (if known) + /// Request duration + public static void LogHttpResponse(PSCmdlet cmdlet, int statusCode, long? contentLength, TimeSpan duration) { - var timestamp = DateTime.Now.ToString("yyyy/MM/dd HH:mm:ss.fff"); - - // Process arguments to format byte sizes intelligently - var processedArgs = ProcessLogArguments(args); - var formattedMessage = processedArgs.Length > 0 ? string.Format(message, processedArgs) : message; - - return $"{timestamp} - {formattedMessage}"; - } - - /// - /// Processes log arguments to format byte sizes intelligently - /// - /// Original arguments - /// Processed arguments with formatted sizes - private static object[] ProcessLogArguments(object[] args) - { - if (args == null || args.Length == 0) - return args ?? new object[0]; - - var processedArgs = new object[args.Length]; - - for (int i = 0; i < args.Length; i++) - { - var arg = args[i]; - - // Check if this looks like a byte size that should be formatted - if (IsLikelyByteSize(arg)) - { - if (long.TryParse(arg.ToString(), out var bytes)) - { - processedArgs[i] = SizeFormatter.FormatBytes(bytes); - } - else - { - processedArgs[i] = arg; - } - } - else - { - processedArgs[i] = arg; - } - } - - return processedArgs; - } - - /// - /// Determines if an argument is likely a byte size that should be formatted - /// - /// Argument to check - /// True if likely a byte size - private static bool IsLikelyByteSize(object arg) - { - // Only format long integers that are likely byte sizes - // We use a heuristic: values >= 1024 are likely byte sizes - if (arg is long longValue) - { - return longValue >= 1024; - } - - if (arg is int intValue) - { - return intValue >= 1024; - } - - return false; + var message = contentLength.HasValue + ? $"HTTP {statusCode} (Content-Length: {SizeFormatter.FormatBytes(contentLength.Value)}) in {SizeFormatter.FormatDuration(duration)}" + : $"HTTP {statusCode} in {SizeFormatter.FormatDuration(duration)}"; + + WriteDebug(cmdlet, message); } } } diff --git a/src/Utils/ProgressReporter.cs b/src/Utils/ProgressReporter.cs deleted file mode 100644 index 8cb54f4..0000000 --- a/src/Utils/ProgressReporter.cs +++ /dev/null @@ -1,156 +0,0 @@ -using System; -using System.Diagnostics; -using System.Management.Automation; - -namespace PSMinIO.Utils -{ - /// - /// Utility class for reporting progress during file operations - /// - public class ProgressReporter - { - private readonly PSCmdlet _cmdlet; - private readonly string _activity; - private readonly string _statusDescription; - private readonly long _totalBytes; - private readonly Stopwatch _stopwatch; - private long _bytesProcessed; - private DateTime _lastUpdateTime; - private readonly int _activityId; - - /// - /// Creates a new ProgressReporter instance - /// - /// The cmdlet to report progress to - /// Description of the activity - /// Status description - /// Total number of bytes to process - /// Unique activity ID for progress reporting - public ProgressReporter(PSCmdlet cmdlet, string activity, string statusDescription, long totalBytes, int activityId = 1) - { - _cmdlet = cmdlet ?? throw new ArgumentNullException(nameof(cmdlet)); - _activity = activity ?? throw new ArgumentNullException(nameof(activity)); - _statusDescription = statusDescription ?? throw new ArgumentNullException(nameof(statusDescription)); - _totalBytes = totalBytes; - _activityId = activityId; - _stopwatch = Stopwatch.StartNew(); - _lastUpdateTime = DateTime.UtcNow; - } - - /// - /// Updates the progress with the number of bytes processed - /// - /// Number of bytes processed so far - public void UpdateProgress(long bytesProcessed) - { - _bytesProcessed = bytesProcessed; - - // Only update progress every 100ms to avoid overwhelming the console - var now = DateTime.UtcNow; - if ((now - _lastUpdateTime).TotalMilliseconds < 100 && bytesProcessed < _totalBytes) - { - return; - } - - _lastUpdateTime = now; - - var percentComplete = _totalBytes > 0 ? (int)((double)bytesProcessed / _totalBytes * 100) : 0; - var currentStatus = FormatCurrentStatus(bytesProcessed); - - var progressRecord = new ProgressRecord(_activityId, _activity, currentStatus) - { - PercentComplete = Math.Min(percentComplete, 100) - }; - - // Add remaining time estimate if we have enough data - if (_stopwatch.ElapsedMilliseconds > 1000 && bytesProcessed > 0 && bytesProcessed < _totalBytes) - { - var remainingTime = EstimateRemainingTime(bytesProcessed); - if (remainingTime.HasValue) - { - progressRecord.SecondsRemaining = (int)remainingTime.Value.TotalSeconds; - } - } - - _cmdlet.WriteProgress(progressRecord); - } - - /// - /// Completes the progress reporting - /// - public void Complete() - { - _stopwatch.Stop(); - - var progressRecord = new ProgressRecord(_activityId, _activity, "Completed") - { - PercentComplete = 100, - RecordType = ProgressRecordType.Completed - }; - - _cmdlet.WriteProgress(progressRecord); - - // Log completion details - MinIOLogger.WriteVerbose(_cmdlet, - $"Operation completed: {SizeFormatter.FormatBytes(_totalBytes)} processed in {_stopwatch.Elapsed.TotalSeconds:F1} seconds " + - $"(Average speed: {SizeFormatter.FormatBytesPerSecond(CalculateAverageSpeed())})"); - } - - /// - /// Formats the current status string - /// - /// Number of bytes processed - /// Formatted status string - private string FormatCurrentStatus(long bytesProcessed) - { - var processedFormatted = SizeFormatter.FormatBytes(bytesProcessed); - var totalFormatted = SizeFormatter.FormatBytes(_totalBytes); - var speed = CalculateCurrentSpeed(); - var speedFormatted = SizeFormatter.FormatBytesPerSecond(speed); - - return $"{_statusDescription}: {processedFormatted} / {totalFormatted} ({speedFormatted})"; - } - - /// - /// Calculates the current transfer speed in bytes per second - /// - /// Current speed in bytes per second - private double CalculateCurrentSpeed() - { - var elapsedSeconds = _stopwatch.Elapsed.TotalSeconds; - return elapsedSeconds > 0 ? _bytesProcessed / elapsedSeconds : 0; - } - - /// - /// Calculates the average transfer speed in bytes per second - /// - /// Average speed in bytes per second - private double CalculateAverageSpeed() - { - var elapsedSeconds = _stopwatch.Elapsed.TotalSeconds; - return elapsedSeconds > 0 ? _totalBytes / elapsedSeconds : 0; - } - - /// - /// Estimates the remaining time for the operation - /// - /// Number of bytes processed so far - /// Estimated remaining time, or null if cannot be calculated - private TimeSpan? EstimateRemainingTime(long bytesProcessed) - { - if (bytesProcessed <= 0 || _stopwatch.ElapsedMilliseconds <= 0) - return null; - - var remainingBytes = _totalBytes - bytesProcessed; - var currentSpeed = CalculateCurrentSpeed(); - - if (currentSpeed <= 0) - return null; - - var remainingSeconds = remainingBytes / currentSpeed; - return TimeSpan.FromSeconds(remainingSeconds); - } - - - } -} diff --git a/src/Utils/SizeFormatter.cs b/src/Utils/SizeFormatter.cs index 5ebe431..4228f2e 100644 --- a/src/Utils/SizeFormatter.cs +++ b/src/Utils/SizeFormatter.cs @@ -3,21 +3,18 @@ using System; namespace PSMinIO.Utils { /// - /// Utility class for formatting byte sizes into human-readable strings + /// Utility class for formatting byte sizes in human-readable format /// public static class SizeFormatter { - /// - /// Size units in order from smallest to largest - /// - private static readonly string[] SizeUnits = { "B", "KB", "MB", "GB", "TB", "PB", "EB" }; + private static readonly string[] SizeUnits = { "B", "KB", "MB", "GB", "TB", "PB" }; /// - /// Formats bytes into a human-readable string with appropriate units + /// Formats a byte count as a human-readable string with appropriate units /// /// Number of bytes /// Number of decimal places to show (default: 2) - /// Formatted string with appropriate unit + /// Formatted size string (e.g., "1.23 MB") public static string FormatBytes(long bytes, int decimalPlaces = 2) { if (bytes == 0) @@ -29,161 +26,139 @@ namespace PSMinIO.Utils int unitIndex = 0; double size = bytes; - // Find the appropriate unit while (size >= 1024 && unitIndex < SizeUnits.Length - 1) { size /= 1024; unitIndex++; } - // Format with specified decimal places - var formatString = $"{{0:F{decimalPlaces}}} {{1}}"; - return string.Format(formatString, size, SizeUnits[unitIndex]); + return $"{size.ToString($"F{decimalPlaces}")} {SizeUnits[unitIndex]}"; } /// - /// Formats bytes into a human-readable string with appropriate units (double overload) + /// Formats a byte count as a human-readable string with appropriate units (double overload) /// /// Number of bytes /// Number of decimal places to show (default: 2) - /// Formatted string with appropriate unit + /// Formatted size string (e.g., "1.23 MB") public static string FormatBytes(double bytes, int decimalPlaces = 2) { return FormatBytes((long)Math.Round(bytes), decimalPlaces); } /// - /// Formats bytes per second into a human-readable string + /// Parses a human-readable size string back to bytes /// - /// Bytes per second + /// Size string (e.g., "1.5 MB") + /// Number of bytes + /// Thrown when the size string is invalid + public static long ParseBytes(string sizeString) + { + if (string.IsNullOrWhiteSpace(sizeString)) + throw new ArgumentException("Size string cannot be null or empty", nameof(sizeString)); + + sizeString = sizeString.Trim().ToUpperInvariant(); + + // Handle just numbers (assume bytes) + if (double.TryParse(sizeString, out double justNumber)) + return (long)justNumber; + + // Find the last space or digit-to-letter boundary + int unitStartIndex = -1; + for (int i = sizeString.Length - 1; i >= 0; i--) + { + if (char.IsDigit(sizeString[i]) || sizeString[i] == '.') + { + unitStartIndex = i + 1; + break; + } + } + + if (unitStartIndex == -1 || unitStartIndex >= sizeString.Length) + throw new ArgumentException($"Invalid size string format: {sizeString}", nameof(sizeString)); + + string numberPart = sizeString.Substring(0, unitStartIndex).Trim(); + string unitPart = sizeString.Substring(unitStartIndex).Trim(); + + if (!double.TryParse(numberPart, out double value)) + throw new ArgumentException($"Invalid numeric value: {numberPart}", nameof(sizeString)); + + long multiplier = GetMultiplierForUnit(unitPart); + return (long)(value * multiplier); + } + + /// + /// Gets the multiplier for a given unit + /// + /// Unit string (e.g., "MB", "GB") + /// Multiplier value + private static long GetMultiplierForUnit(string unit) + { + return unit switch + { + "B" or "BYTE" or "BYTES" => 1L, + "KB" or "KILOBYTE" or "KILOBYTES" => 1024L, + "MB" or "MEGABYTE" or "MEGABYTES" => 1024L * 1024L, + "GB" or "GIGABYTE" or "GIGABYTES" => 1024L * 1024L * 1024L, + "TB" or "TERABYTE" or "TERABYTES" => 1024L * 1024L * 1024L * 1024L, + "PB" or "PETABYTE" or "PETABYTES" => 1024L * 1024L * 1024L * 1024L * 1024L, + _ => throw new ArgumentException($"Unknown unit: {unit}") + }; + } + + /// + /// Formats transfer speed in bytes per second + /// + /// Transfer speed in bytes per second /// Number of decimal places to show (default: 2) - /// Formatted string with appropriate unit and "/s" suffix - public static string FormatBytesPerSecond(double bytesPerSecond, int decimalPlaces = 2) + /// Formatted speed string (e.g., "1.23 MB/s") + public static string FormatSpeed(double bytesPerSecond, int decimalPlaces = 2) { return $"{FormatBytes(bytesPerSecond, decimalPlaces)}/s"; } /// - /// Formats a transfer rate with context + /// Formats a time duration in a human-readable format /// - /// Number of bytes transferred - /// Time elapsed for the transfer - /// Number of decimal places to show (default: 2) - /// Formatted transfer rate string - public static string FormatTransferRate(long bytesTransferred, TimeSpan elapsedTime, int decimalPlaces = 2) + /// Duration to format + /// Formatted duration string (e.g., "1h 23m 45s") + public static string FormatDuration(TimeSpan duration) { - if (elapsedTime.TotalSeconds <= 0) - return "0 B/s"; + if (duration.TotalSeconds < 1) + return $"{duration.TotalMilliseconds:F0}ms"; - var bytesPerSecond = bytesTransferred / elapsedTime.TotalSeconds; - return FormatBytesPerSecond(bytesPerSecond, decimalPlaces); + if (duration.TotalMinutes < 1) + return $"{duration.TotalSeconds:F1}s"; + + if (duration.TotalHours < 1) + return $"{duration.Minutes}m {duration.Seconds}s"; + + if (duration.TotalDays < 1) + return $"{duration.Hours}h {duration.Minutes}m {duration.Seconds}s"; + + return $"{duration.Days}d {duration.Hours}h {duration.Minutes}m"; } /// - /// Formats a progress string showing current/total with percentages + /// Calculates and formats estimated time remaining /// - /// Current bytes processed - /// Total bytes to process - /// Number of decimal places to show (default: 2) - /// Formatted progress string - public static string FormatProgress(long current, long total, int decimalPlaces = 2) + /// Bytes already transferred + /// Total bytes to transfer + /// Time elapsed so far + /// Formatted ETA string + public static string FormatETA(long bytesTransferred, long totalBytes, TimeSpan elapsedTime) { - var currentFormatted = FormatBytes(current, decimalPlaces); - var totalFormatted = FormatBytes(total, decimalPlaces); - - if (total > 0) - { - var percentage = (double)current / total * 100; - return $"{currentFormatted} / {totalFormatted} ({percentage:F1}%)"; - } - - return $"{currentFormatted} / {totalFormatted}"; - } + if (bytesTransferred <= 0 || totalBytes <= 0 || elapsedTime.TotalSeconds <= 0) + return "Unknown"; - /// - /// Gets the appropriate unit for a given byte size without formatting - /// - /// Number of bytes - /// Appropriate unit string - public static string GetAppropriateUnit(long bytes) - { - if (bytes == 0) - return "B"; + if (bytesTransferred >= totalBytes) + return "Complete"; - int unitIndex = 0; - double size = Math.Abs(bytes); + double bytesPerSecond = bytesTransferred / elapsedTime.TotalSeconds; + long remainingBytes = totalBytes - bytesTransferred; + double remainingSeconds = remainingBytes / bytesPerSecond; - while (size >= 1024 && unitIndex < SizeUnits.Length - 1) - { - size /= 1024; - unitIndex++; - } - - return SizeUnits[unitIndex]; - } - - /// - /// Converts bytes to the specified unit - /// - /// Number of bytes - /// Target unit (B, KB, MB, GB, TB, PB, EB) - /// Value in the specified unit - public static double ConvertToUnit(long bytes, string unit) - { - var unitIndex = Array.IndexOf(SizeUnits, unit.ToUpperInvariant()); - if (unitIndex == -1) - throw new ArgumentException($"Invalid unit: {unit}. Valid units are: {string.Join(", ", SizeUnits)}"); - - if (unitIndex == 0) // Bytes - return bytes; - - return bytes / Math.Pow(1024, unitIndex); - } - - /// - /// Parses a size string back to bytes (e.g., "1.5 GB" -> bytes) - /// - /// Size string to parse - /// Number of bytes - public static long ParseSizeString(string sizeString) - { - if (string.IsNullOrWhiteSpace(sizeString)) - throw new ArgumentException("Size string cannot be null or empty"); - - var parts = sizeString.Trim().Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); - if (parts.Length != 2) - throw new ArgumentException($"Invalid size string format: {sizeString}. Expected format: '1.5 GB'"); - - if (!double.TryParse(parts[0], out var value)) - throw new ArgumentException($"Invalid numeric value: {parts[0]}"); - - var unit = parts[1].ToUpperInvariant(); - var unitIndex = Array.IndexOf(SizeUnits, unit); - if (unitIndex == -1) - throw new ArgumentException($"Invalid unit: {unit}. Valid units are: {string.Join(", ", SizeUnits)}"); - - return (long)(value * Math.Pow(1024, unitIndex)); - } - - /// - /// Formats a size comparison between two values - /// - /// First value in bytes - /// Second value in bytes - /// Label for first value - /// Label for second value - /// Number of decimal places to show (default: 2) - /// Formatted comparison string - public static string FormatComparison(long value1, long value2, string label1, string label2, int decimalPlaces = 2) - { - var formatted1 = FormatBytes(value1, decimalPlaces); - var formatted2 = FormatBytes(value2, decimalPlaces); - - var difference = value1 - value2; - var diffFormatted = FormatBytes(Math.Abs(difference), decimalPlaces); - var diffDirection = difference >= 0 ? "larger" : "smaller"; - - return $"{label1}: {formatted1}, {label2}: {formatted2} ({diffFormatted} {diffDirection})"; + return FormatDuration(TimeSpan.FromSeconds(remainingSeconds)); } } } diff --git a/src/Utils/ThreadSafeChunkedProgressReporter.cs b/src/Utils/ThreadSafeChunkedProgressReporter.cs deleted file mode 100644 index ae26f98..0000000 --- a/src/Utils/ThreadSafeChunkedProgressReporter.cs +++ /dev/null @@ -1,163 +0,0 @@ -using System; -using System.Management.Automation; -using System.Threading; - -namespace PSMinIO.Utils -{ - /// - /// Thread-safe wrapper for chunked progress reporting that can be safely called from background threads - /// - public class ThreadSafeChunkedProgressReporter - { - private readonly ThreadSafeProgressCollector _progressCollector; - private readonly string _operationName; - private readonly long _totalSize; - private readonly int _totalChunks; - private readonly DateTime _startTime; - - // Activity IDs for progress hierarchy - private const int FileActivityId = 1; - private const int ChunkActivityId = 2; - - // Current state (using Interlocked for long values since volatile doesn't support long) - private volatile int _currentChunk = 0; - private long _currentChunkSize = 0; - private long _totalBytesTransferred = 0; - private volatile string _currentFileName = string.Empty; - - public ThreadSafeChunkedProgressReporter( - PSCmdlet cmdlet, - long totalSize, - int totalChunks, - string operationName) - { - _progressCollector = new ThreadSafeProgressCollector(cmdlet); - _totalSize = totalSize; - _totalChunks = totalChunks; - _operationName = operationName; - _startTime = DateTime.UtcNow; - } - - /// - /// Starts a new file operation (thread-safe) - /// - public void StartNewFile(string fileName, long fileSize, int totalChunks) - { - _currentFileName = fileName; - Interlocked.Exchange(ref _totalBytesTransferred, 0); - _currentChunk = 0; - - _progressCollector.QueueVerboseMessage("Starting {0} of file: {1} ({2})", - _operationName.ToLower(), fileName, SizeFormatter.FormatBytes(fileSize)); - - _progressCollector.QueueProgressUpdate( - FileActivityId, - $"{_operationName} File", - $"{_operationName}: {fileName}", - 0); - } - - /// - /// Starts a new chunk operation (thread-safe) - /// - public void StartNewChunk(int chunkNumber, long chunkSize) - { - _currentChunk = chunkNumber; - Interlocked.Exchange(ref _currentChunkSize, chunkSize); - - _progressCollector.QueueVerboseMessage("File {0}: Starting chunk {1}/{2} ({3})", - _currentFileName, chunkNumber, _totalChunks, SizeFormatter.FormatBytes(chunkSize)); - - _progressCollector.QueueProgressUpdate( - ChunkActivityId, - "Current Chunk", - $"Chunk {chunkNumber}/{_totalChunks}", - 0, - FileActivityId); - } - - /// - /// Updates chunk progress (thread-safe) - /// - public void UpdateChunkProgress(long bytesTransferred) - { - var newTotal = Interlocked.Add(ref _totalBytesTransferred, bytesTransferred); - var currentChunkSize = Interlocked.Read(ref _currentChunkSize); - - var chunkPercent = currentChunkSize > 0 ? - (int)((double)bytesTransferred / currentChunkSize * 100) : 100; - var filePercent = _totalSize > 0 ? - (int)((double)newTotal / _totalSize * 100) : 100; - - // Update chunk progress - _progressCollector.QueueProgressUpdate( - ChunkActivityId, - "Current Chunk", - $"Chunk {_currentChunk}/{_totalChunks} - {SizeFormatter.FormatBytes(bytesTransferred)}/{SizeFormatter.FormatBytes(_currentChunkSize)}", - Math.Min(chunkPercent, 100), - FileActivityId); - - // Update file progress - _progressCollector.QueueProgressUpdate( - FileActivityId, - $"{_operationName} File", - $"{_operationName}: {_currentFileName} - {SizeFormatter.FormatBytes(newTotal)}/{SizeFormatter.FormatBytes(_totalSize)}", - Math.Min(filePercent, 100)); - } - - /// - /// Completes the current chunk (thread-safe) - /// - public void CompleteChunk(string? chunkETag = null) - { - _progressCollector.QueueVerboseMessage("File {0}: Completed chunk {1}/{2}{3}", - _currentFileName, _currentChunk, _totalChunks, - !string.IsNullOrEmpty(chunkETag) ? $" - ETag: {chunkETag}" : ""); - - _progressCollector.QueueProgressCompletion(ChunkActivityId, "Current Chunk", FileActivityId); - } - - /// - /// Completes the current file (thread-safe) - /// - public void CompleteFile() - { - var elapsed = DateTime.UtcNow - _startTime; - _progressCollector.QueueVerboseMessage("File {0}: {1} completed in {2} - Total size: {3}", - _currentFileName, _operationName.ToLower(), elapsed.ToString(@"hh\:mm\:ss"), - SizeFormatter.FormatBytes(_totalSize)); - - _progressCollector.QueueProgressCompletion(FileActivityId, $"{_operationName} File"); - } - - /// - /// Reports a chunk error (thread-safe) - /// - public void ReportChunkError(Exception error, int retryAttempt, int maxRetries) - { - _progressCollector.QueueVerboseMessage("File {0}: Chunk {1}/{2} failed (attempt {3}/{4}): {5}", - _currentFileName, _currentChunk, _totalChunks, retryAttempt, maxRetries, error.Message); - } - - /// - /// Processes all queued updates from the main thread (must be called from main thread) - /// - public void ProcessQueuedUpdates() - { - _progressCollector.ProcessQueuedUpdates(); - } - - /// - /// Completes all operations and processes final updates (must be called from main thread) - /// - public void Complete() - { - _progressCollector.Complete(); - } - - /// - /// Gets the number of pending updates - /// - public int PendingUpdates => _progressCollector.PendingProgressUpdates + _progressCollector.PendingVerboseMessages; - } -} diff --git a/src/Utils/ThreadSafeProgressCollector.cs b/src/Utils/ThreadSafeProgressCollector.cs deleted file mode 100644 index b34af0a..0000000 --- a/src/Utils/ThreadSafeProgressCollector.cs +++ /dev/null @@ -1,156 +0,0 @@ -using System; -using System.Collections.Concurrent; -using System.Management.Automation; -using System.Threading; - -namespace PSMinIO.Utils -{ - /// - /// Thread-safe progress data collector that accumulates progress updates from background threads - /// and allows the main thread to safely report them to PowerShell - /// - public class ThreadSafeProgressCollector - { - private readonly PSCmdlet _cmdlet; - private readonly ConcurrentQueue _progressQueue = new(); - private readonly ConcurrentQueue _verboseQueue = new(); - private readonly object _lockObject = new(); - private volatile bool _isCompleted = false; - - public ThreadSafeProgressCollector(PSCmdlet cmdlet) - { - _cmdlet = cmdlet ?? throw new ArgumentNullException(nameof(cmdlet)); - } - - /// - /// Queues a progress update from a background thread - /// - public void QueueProgressUpdate(int activityId, string activity, string statusDescription, int percentComplete, int parentActivityId = -1) - { - if (_isCompleted) return; - - _progressQueue.Enqueue(new ProgressUpdate - { - ActivityId = activityId, - Activity = activity, - StatusDescription = statusDescription, - PercentComplete = percentComplete, - ParentActivityId = parentActivityId, - Timestamp = DateTime.UtcNow - }); - } - - /// - /// Queues a progress completion from a background thread - /// - public void QueueProgressCompletion(int activityId, string activity, int parentActivityId = -1) - { - if (_isCompleted) return; - - _progressQueue.Enqueue(new ProgressUpdate - { - ActivityId = activityId, - Activity = activity, - StatusDescription = "Completed", - PercentComplete = 100, - ParentActivityId = parentActivityId, - IsCompleted = true, - Timestamp = DateTime.UtcNow - }); - } - - /// - /// Queues a verbose message from a background thread - /// - public void QueueVerboseMessage(string message, params object[] args) - { - if (_isCompleted) return; - - _verboseQueue.Enqueue(new VerboseMessage - { - Message = args.Length > 0 ? string.Format(message, args) : message, - Timestamp = DateTime.UtcNow - }); - } - - /// - /// Processes all queued updates from the main thread (safe to call PowerShell methods) - /// - public void ProcessQueuedUpdates() - { - // Process verbose messages first - while (_verboseQueue.TryDequeue(out var verboseMessage)) - { - MinIOLogger.WriteVerbose(_cmdlet, verboseMessage.Message); - } - - // Process progress updates - while (_progressQueue.TryDequeue(out var progressUpdate)) - { - var progressRecord = new ProgressRecord( - progressUpdate.ActivityId, - progressUpdate.Activity, - progressUpdate.StatusDescription) - { - PercentComplete = progressUpdate.PercentComplete - }; - - if (progressUpdate.ParentActivityId >= 0) - { - progressRecord.ParentActivityId = progressUpdate.ParentActivityId; - } - - if (progressUpdate.IsCompleted) - { - progressRecord.RecordType = ProgressRecordType.Completed; - } - - _cmdlet.WriteProgress(progressRecord); - } - } - - /// - /// Marks the collector as completed (no more updates will be accepted) - /// - public void Complete() - { - _isCompleted = true; - - // Process any remaining updates - ProcessQueuedUpdates(); - } - - /// - /// Gets the number of pending progress updates - /// - public int PendingProgressUpdates => _progressQueue.Count; - - /// - /// Gets the number of pending verbose messages - /// - public int PendingVerboseMessages => _verboseQueue.Count; - - /// - /// Progress update data structure - /// - private class ProgressUpdate - { - public int ActivityId { get; set; } - public string Activity { get; set; } = string.Empty; - public string StatusDescription { get; set; } = string.Empty; - public int PercentComplete { get; set; } - public int ParentActivityId { get; set; } = -1; - public bool IsCompleted { get; set; } - public DateTime Timestamp { get; set; } - } - - /// - /// Verbose message data structure - /// - private class VerboseMessage - { - public string Message { get; set; } = string.Empty; - public DateTime Timestamp { get; set; } - } - } -} diff --git a/src/Utils/ThreadSafeResultCollector.cs b/src/Utils/ThreadSafeResultCollector.cs deleted file mode 100644 index 7d76005..0000000 --- a/src/Utils/ThreadSafeResultCollector.cs +++ /dev/null @@ -1,149 +0,0 @@ -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Management.Automation; -using PSMinIO.Models; - -namespace PSMinIO.Utils -{ - /// - /// Thread-safe result collector that accumulates results from background threads - /// and allows the main thread to safely output them via WriteObject - /// - public class ThreadSafeResultCollector - { - private readonly PSCmdlet _cmdlet; - private readonly ConcurrentQueue _resultQueue = new(); - private readonly ConcurrentQueue _errorQueue = new(); - private readonly object _lockObject = new(); - private volatile bool _isCompleted = false; - - public ThreadSafeResultCollector(PSCmdlet cmdlet) - { - _cmdlet = cmdlet ?? throw new ArgumentNullException(nameof(cmdlet)); - } - - /// - /// Queues a successful result from a background thread - /// - public void QueueResult(object result) - { - if (_isCompleted || result == null) return; - - _resultQueue.Enqueue(result); - } - - /// - /// Queues an error from a background thread - /// - public void QueueError(ErrorRecord error) - { - if (_isCompleted || error == null) return; - - _errorQueue.Enqueue(error); - } - - /// - /// Queues an error from a background thread using exception details - /// - public void QueueError(Exception exception, string errorId, ErrorCategory category, object targetObject) - { - if (_isCompleted || exception == null) return; - - var errorRecord = new ErrorRecord(exception, errorId, category, targetObject); - _errorQueue.Enqueue(errorRecord); - } - - /// - /// Processes all queued results and errors from the main thread (safe to call PowerShell methods) - /// - public void ProcessQueuedResults() - { - // Process errors first - while (_errorQueue.TryDequeue(out var error)) - { - _cmdlet.WriteError(error); - } - - // Process successful results - while (_resultQueue.TryDequeue(out var result)) - { - _cmdlet.WriteObject(result); - } - } - - /// - /// Marks the collector as completed and processes all remaining results - /// - public void Complete() - { - _isCompleted = true; - - // Process any remaining results - ProcessQueuedResults(); - } - - /// - /// Gets all queued results without processing them (for inspection) - /// - public List GetQueuedResults() - { - var results = new List(); - var tempQueue = new Queue(); - - // Dequeue all items and re-queue them - while (_resultQueue.TryDequeue(out var result)) - { - results.Add(result); - tempQueue.Enqueue(result); - } - - // Re-queue the items - while (tempQueue.Count > 0) - { - _resultQueue.Enqueue(tempQueue.Dequeue()); - } - - return results; - } - - /// - /// Gets all queued errors without processing them (for inspection) - /// - public List GetQueuedErrors() - { - var errors = new List(); - var tempQueue = new Queue(); - - // Dequeue all items and re-queue them - while (_errorQueue.TryDequeue(out var error)) - { - errors.Add(error); - tempQueue.Enqueue(error); - } - - // Re-queue the items - while (tempQueue.Count > 0) - { - _errorQueue.Enqueue(tempQueue.Dequeue()); - } - - return errors; - } - - /// - /// Gets the number of pending results - /// - public int PendingResults => _resultQueue.Count; - - /// - /// Gets the number of pending errors - /// - public int PendingErrors => _errorQueue.Count; - - /// - /// Gets whether the collector has any pending items - /// - public bool HasPendingItems => PendingResults > 0 || PendingErrors > 0; - } -}