Initial commit: PSMinIO module with chunked transfer support

This commit is contained in:
PSMinIO Developer
2025-07-10 12:58:44 -04:00
parent a306ade1f3
commit 5fdcd31d5e
40 changed files with 9899 additions and 1 deletions
@@ -0,0 +1,267 @@
using System;
using System.Management.Automation;
using PSMinIO.Utils;
namespace PSMinIO.Utils
{
/// <summary>
/// Manages 3-layer progress reporting for chunked file collection operations
/// Layer 1: Collection Progress (overall files)
/// Layer 2: Current File Progress
/// Layer 3: Current Chunk Progress
/// </summary>
public class ChunkedCollectionProgressReporter
{
private readonly PSCmdlet _cmdlet;
private readonly int _totalFiles;
private readonly long _totalSize;
private readonly DateTime _startTime;
private readonly string _operationName;
// Progress tracking
private int _completedFiles = 0;
private long _totalBytesTransferred = 0;
private string _currentFileName = "";
private long _currentFileSize = 0;
private long _currentFileBytesTransferred = 0;
private int _currentChunk = 0;
private int _totalChunks = 0;
private long _currentChunkBytesTransferred = 0;
private long _currentChunkSize = 0;
// Activity IDs for progress hierarchy
private const int CollectionActivityId = 1;
private const int FileActivityId = 2;
private const int ChunkActivityId = 3;
// Progress control
private readonly long _progressUpdateInterval;
private long _lastProgressUpdate = 0;
/// <summary>
/// Creates a new chunked collection progress reporter
/// </summary>
/// <param name="cmdlet">PowerShell cmdlet for progress reporting</param>
/// <param name="totalFiles">Total number of files to process</param>
/// <param name="totalSize">Total size of all files</param>
/// <param name="operationName">Name of the operation (e.g., "Uploading", "Downloading")</param>
/// <param name="progressUpdateInterval">Update progress every N bytes</param>
public ChunkedCollectionProgressReporter(
PSCmdlet cmdlet,
int totalFiles,
long totalSize,
string operationName = "Processing",
long progressUpdateInterval = 1024 * 1024) // 1MB default
{
_cmdlet = cmdlet;
_totalFiles = totalFiles;
_totalSize = totalSize;
_operationName = operationName;
_startTime = DateTime.Now;
_progressUpdateInterval = progressUpdateInterval;
}
/// <summary>
/// Starts processing a new file
/// </summary>
/// <param name="fileName">Name of the file being processed</param>
/// <param name="fileSize">Size of the file</param>
/// <param name="totalChunks">Total number of chunks for this file</param>
public void StartNewFile(string fileName, long fileSize, int totalChunks)
{
_currentFileName = fileName;
_currentFileSize = fileSize;
_currentFileBytesTransferred = 0;
_totalChunks = totalChunks;
_currentChunk = 0;
MinIOLogger.WriteVerbose(_cmdlet, "Starting {0} of file {1}/{2}: {3} ({4})",
_operationName.ToLower(), _completedFiles + 1, _totalFiles, fileName, SizeFormatter.FormatSize(fileSize));
UpdateAllProgress();
}
/// <summary>
/// Starts processing a new chunk
/// </summary>
/// <param name="chunkNumber">Chunk number (1-based for display)</param>
/// <param name="chunkSize">Size of the chunk</param>
public void StartNewChunk(int chunkNumber, long chunkSize)
{
_currentChunk = chunkNumber;
_currentChunkSize = chunkSize;
_currentChunkBytesTransferred = 0;
MinIOLogger.WriteVerbose(_cmdlet, "File {0}: Starting chunk {1}/{2} ({3})",
_currentFileName, chunkNumber, _totalChunks, SizeFormatter.FormatSize(chunkSize));
UpdateAllProgress();
}
/// <summary>
/// Updates chunk progress
/// </summary>
/// <param name="bytesTransferred">Bytes transferred for current chunk</param>
public void UpdateChunkProgress(long bytesTransferred)
{
var chunkDelta = bytesTransferred - _currentChunkBytesTransferred;
_currentChunkBytesTransferred = bytesTransferred;
_currentFileBytesTransferred += chunkDelta;
_totalBytesTransferred += chunkDelta;
// Only update progress if we've transferred enough bytes since last update
if (_totalBytesTransferred - _lastProgressUpdate >= _progressUpdateInterval)
{
UpdateAllProgress();
_lastProgressUpdate = _totalBytesTransferred;
}
}
/// <summary>
/// Completes the current chunk
/// </summary>
/// <param name="chunkETag">ETag of the completed chunk (optional)</param>
public void CompleteChunk(string? chunkETag = null)
{
MinIOLogger.WriteVerbose(_cmdlet, "File {0}: Completed chunk {1}/{2}{3}",
_currentFileName, _currentChunk, _totalChunks,
!string.IsNullOrEmpty(chunkETag) ? $" - ETag: {chunkETag}" : "");
// Mark chunk as completed (only if progress is enabled)
if (_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") &&
_cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue")
return;
var chunkProgress = new ProgressRecord(ChunkActivityId, "Current Chunk", "Completed")
{
PercentComplete = 100,
RecordType = ProgressRecordType.Completed,
ParentActivityId = FileActivityId
};
_cmdlet.WriteProgress(chunkProgress);
}
/// <summary>
/// Completes the current file
/// </summary>
public void CompleteFile()
{
_completedFiles++;
var elapsed = DateTime.Now - _startTime;
MinIOLogger.WriteVerbose(_cmdlet, "File {0}: {1} completed in {2} - Total size: {3}",
_currentFileName, _operationName.ToLower(), elapsed.ToString(@"hh\:mm\:ss"), SizeFormatter.FormatSize(_currentFileSize));
// Mark file as completed (only if progress is enabled)
if (_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") &&
_cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue")
return;
var fileProgress = new ProgressRecord(FileActivityId, "Current File", "Completed")
{
PercentComplete = 100,
RecordType = ProgressRecordType.Completed,
ParentActivityId = CollectionActivityId
};
_cmdlet.WriteProgress(fileProgress);
// Complete chunk progress too
CompleteChunk();
}
/// <summary>
/// Completes the entire collection operation
/// </summary>
public void CompleteCollection()
{
var elapsed = DateTime.Now - _startTime;
MinIOLogger.WriteVerbose(_cmdlet, "Completed {0} {1} files ({2}) in {3}",
_operationName.ToLower(), _totalFiles, SizeFormatter.FormatSize(_totalSize), elapsed.ToString(@"hh\:mm\:ss"));
// Complete all progress records
var collectionProgress = new ProgressRecord(CollectionActivityId, $"{_operationName} Files", "Completed")
{
PercentComplete = 100,
RecordType = ProgressRecordType.Completed
};
_cmdlet.WriteProgress(collectionProgress);
// Complete file progress if enabled
if (!(_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") &&
_cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue"))
{
CompleteFile();
}
}
/// <summary>
/// Reports an error for the current chunk
/// </summary>
/// <param name="error">Error that occurred</param>
/// <param name="retryAttempt">Current retry attempt</param>
/// <param name="maxRetries">Maximum retry attempts</param>
public void ReportChunkError(Exception error, int retryAttempt, int maxRetries)
{
MinIOLogger.WriteVerbose(_cmdlet, "File {0}: Chunk {1}/{2} failed (attempt {3}/{4}): {5}",
_currentFileName, _currentChunk, _totalChunks, retryAttempt, maxRetries, error.Message);
}
/// <summary>
/// Updates all progress layers
/// </summary>
private void UpdateAllProgress()
{
var elapsed = DateTime.Now - _startTime;
var speed = elapsed.TotalSeconds > 0 ? _totalBytesTransferred / elapsed.TotalSeconds : 0;
// Layer 1: Collection Progress (always shown)
var collectionPercent = _totalSize > 0 ? (int)((_totalBytesTransferred * 100) / _totalSize) : 0;
var collectionStatus = $"Files: {_completedFiles}/{_totalFiles} | " +
$"Size: {SizeFormatter.FormatSize(_totalBytesTransferred)}/{SizeFormatter.FormatSize(_totalSize)} | " +
$"Speed: {SizeFormatter.FormatSize((long)speed)}/s | " +
$"Elapsed: {elapsed:hh\\:mm\\:ss}";
var collectionProgress = new ProgressRecord(CollectionActivityId, $"{_operationName} Files", collectionStatus)
{
PercentComplete = collectionPercent
};
_cmdlet.WriteProgress(collectionProgress);
// Check if progress is disabled
if (_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") &&
_cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue")
return;
// Layer 2: Current File Progress
if (!string.IsNullOrEmpty(_currentFileName))
{
var filePercent = _currentFileSize > 0 ? (int)((_currentFileBytesTransferred * 100) / _currentFileSize) : 0;
var fileStatus = $"File: {_currentFileName} | " +
$"Size: {SizeFormatter.FormatSize(_currentFileBytesTransferred)}/{SizeFormatter.FormatSize(_currentFileSize)}";
var fileProgress = new ProgressRecord(FileActivityId, "Current File", fileStatus)
{
PercentComplete = filePercent,
ParentActivityId = CollectionActivityId
};
_cmdlet.WriteProgress(fileProgress);
}
// Layer 3: Current Chunk Progress
if (_currentChunk > 0)
{
var chunkPercent = _currentChunkSize > 0 ? (int)((_currentChunkBytesTransferred * 100) / _currentChunkSize) : 0;
var chunkStatus = $"Chunk: {_currentChunk}/{_totalChunks} | " +
$"Size: {SizeFormatter.FormatSize(_currentChunkBytesTransferred)}/{SizeFormatter.FormatSize(_currentChunkSize)} | " +
$"Speed: {SizeFormatter.FormatSize((long)speed)}/s";
var chunkProgress = new ProgressRecord(ChunkActivityId, "Current Chunk", chunkStatus)
{
PercentComplete = chunkPercent,
ParentActivityId = FileActivityId
};
_cmdlet.WriteProgress(chunkProgress);
}
}
}
}
@@ -0,0 +1,195 @@
using System;
using System.Management.Automation;
using PSMinIO.Utils;
namespace PSMinIO.Utils
{
/// <summary>
/// Manages 2-layer progress reporting for chunked single file operations
/// Layer 1: File Progress
/// Layer 2: Current Chunk Progress
/// </summary>
public class ChunkedSingleFileProgressReporter
{
private readonly PSCmdlet _cmdlet;
private readonly long _totalSize;
private readonly int _totalChunks;
private readonly DateTime _startTime;
private readonly string _operationName;
// Progress tracking
private long _totalBytesTransferred = 0;
private int _currentChunk = 0;
private long _currentChunkBytesTransferred = 0;
private long _currentChunkSize = 0;
// Activity IDs for progress hierarchy
private const int FileActivityId = 1;
private const int ChunkActivityId = 2;
// Progress control
private readonly long _progressUpdateInterval;
private long _lastProgressUpdate = 0;
/// <summary>
/// Creates a new chunked single file progress reporter
/// </summary>
/// <param name="cmdlet">PowerShell cmdlet for progress reporting</param>
/// <param name="totalSize">Total size of the file</param>
/// <param name="totalChunks">Total number of chunks</param>
/// <param name="operationName">Name of the operation (e.g., "Downloading", "Uploading")</param>
/// <param name="progressUpdateInterval">Update progress every N bytes</param>
public ChunkedSingleFileProgressReporter(
PSCmdlet cmdlet,
long totalSize,
int totalChunks,
string operationName = "Processing",
long progressUpdateInterval = 1024 * 1024) // 1MB default
{
_cmdlet = cmdlet;
_totalSize = totalSize;
_totalChunks = totalChunks;
_operationName = operationName;
_startTime = DateTime.Now;
_progressUpdateInterval = progressUpdateInterval;
}
/// <summary>
/// Starts processing a new chunk
/// </summary>
/// <param name="chunkNumber">Chunk number (1-based for display)</param>
/// <param name="chunkSize">Size of the chunk</param>
public void StartNewChunk(int chunkNumber, long chunkSize)
{
_currentChunk = chunkNumber;
_currentChunkSize = chunkSize;
_currentChunkBytesTransferred = 0;
MinIOLogger.WriteVerbose(_cmdlet, "Starting chunk {0}/{1} ({2})",
chunkNumber, _totalChunks, SizeFormatter.FormatSize(chunkSize));
UpdateAllProgress();
}
/// <summary>
/// Updates chunk progress
/// </summary>
/// <param name="bytesTransferred">Bytes transferred for current chunk</param>
public void UpdateChunkProgress(long bytesTransferred)
{
var chunkDelta = bytesTransferred - _currentChunkBytesTransferred;
_currentChunkBytesTransferred = bytesTransferred;
_totalBytesTransferred += chunkDelta;
// Only update progress if we've transferred enough bytes since last update
if (_totalBytesTransferred - _lastProgressUpdate >= _progressUpdateInterval)
{
UpdateAllProgress();
_lastProgressUpdate = _totalBytesTransferred;
}
}
/// <summary>
/// Completes the current chunk
/// </summary>
/// <param name="chunkETag">ETag of the completed chunk (optional)</param>
public void CompleteChunk(string? chunkETag = null)
{
MinIOLogger.WriteVerbose(_cmdlet, "Completed chunk {0}/{1}{2}",
_currentChunk, _totalChunks,
!string.IsNullOrEmpty(chunkETag) ? $" - ETag: {chunkETag}" : "");
// Mark chunk as completed (only if progress is enabled)
if (_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") &&
_cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue")
return;
var chunkProgress = new ProgressRecord(ChunkActivityId, "Current Chunk", "Completed")
{
PercentComplete = 100,
RecordType = ProgressRecordType.Completed,
ParentActivityId = FileActivityId
};
_cmdlet.WriteProgress(chunkProgress);
}
/// <summary>
/// Completes the entire download operation
/// </summary>
public void CompleteDownload()
{
var elapsed = DateTime.Now - _startTime;
MinIOLogger.WriteVerbose(_cmdlet, "Completed {0} ({1}) in {2}",
_operationName.ToLower(), SizeFormatter.FormatSize(_totalSize), elapsed.ToString(@"hh\:mm\:ss"));
// Complete all progress records
var fileProgress = new ProgressRecord(FileActivityId, $"{_operationName} File", "Completed")
{
PercentComplete = 100,
RecordType = ProgressRecordType.Completed
};
_cmdlet.WriteProgress(fileProgress);
// Complete chunk progress if enabled
if (!(_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") &&
_cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue"))
{
CompleteChunk();
}
}
/// <summary>
/// Reports an error for the current chunk
/// </summary>
/// <param name="error">Error that occurred</param>
/// <param name="retryAttempt">Current retry attempt</param>
/// <param name="maxRetries">Maximum retry attempts</param>
public void ReportChunkError(Exception error, int retryAttempt, int maxRetries)
{
MinIOLogger.WriteVerbose(_cmdlet, "Chunk {0}/{1} failed (attempt {2}/{3}): {4}",
_currentChunk, _totalChunks, retryAttempt, maxRetries, error.Message);
}
/// <summary>
/// Updates all progress layers
/// </summary>
private void UpdateAllProgress()
{
var elapsed = DateTime.Now - _startTime;
var speed = elapsed.TotalSeconds > 0 ? _totalBytesTransferred / elapsed.TotalSeconds : 0;
// Layer 1: File Progress (always shown)
var filePercent = _totalSize > 0 ? (int)((_totalBytesTransferred * 100) / _totalSize) : 0;
var fileStatus = $"Size: {SizeFormatter.FormatSize(_totalBytesTransferred)}/{SizeFormatter.FormatSize(_totalSize)} | " +
$"Speed: {SizeFormatter.FormatSize((long)speed)}/s | " +
$"Elapsed: {elapsed:hh\\:mm\\:ss}";
var fileProgress = new ProgressRecord(FileActivityId, $"{_operationName} File", fileStatus)
{
PercentComplete = filePercent
};
_cmdlet.WriteProgress(fileProgress);
// Check if progress is disabled
if (_cmdlet.MyInvocation.BoundParameters.ContainsKey("ProgressAction") &&
_cmdlet.MyInvocation.BoundParameters["ProgressAction"].ToString() == "SilentlyContinue")
return;
// Layer 2: Current Chunk Progress
if (_currentChunk > 0)
{
var chunkPercent = _currentChunkSize > 0 ? (int)((_currentChunkBytesTransferred * 100) / _currentChunkSize) : 0;
var chunkStatus = $"Chunk: {_currentChunk}/{_totalChunks} | " +
$"Size: {SizeFormatter.FormatSize(_currentChunkBytesTransferred)}/{SizeFormatter.FormatSize(_currentChunkSize)} | " +
$"Speed: {SizeFormatter.FormatSize((long)speed)}/s";
var chunkProgress = new ProgressRecord(ChunkActivityId, "Current Chunk", chunkStatus)
{
PercentComplete = chunkPercent,
ParentActivityId = FileActivityId
};
_cmdlet.WriteProgress(chunkProgress);
}
}
}
}
+250
View File
@@ -0,0 +1,250 @@
using System;
using System.IO;
using System.Text.Json;
using PSMinIO.Models;
namespace PSMinIO.Utils
{
/// <summary>
/// Manages resume data for chunked transfer operations
/// </summary>
public static class ChunkedTransferResumeManager
{
private static readonly string DefaultResumeDirectory = Path.Combine(
Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
"PSMinIO", "Resume");
/// <summary>
/// Saves transfer state for resume functionality
/// </summary>
/// <param name="transferState">Transfer state to save</param>
/// <param name="customPath">Custom path for resume data (optional)</param>
/// <returns>Path where resume data was saved</returns>
public static string SaveTransferState(ChunkedTransferState transferState, string? customPath = null)
{
var resumeDirectory = customPath ?? DefaultResumeDirectory;
// Ensure directory exists
if (!Directory.Exists(resumeDirectory))
{
Directory.CreateDirectory(resumeDirectory);
}
// Generate unique filename based on transfer details
var fileName = GenerateResumeFileName(transferState);
var filePath = Path.Combine(resumeDirectory, fileName);
// Serialize and save
var json = JsonSerializer.Serialize(transferState, new JsonSerializerOptions
{
WriteIndented = true,
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
});
File.WriteAllText(filePath, json);
return filePath;
}
/// <summary>
/// Loads transfer state for resume functionality
/// </summary>
/// <param name="bucketName">Bucket name</param>
/// <param name="objectName">Object name</param>
/// <param name="filePath">Local file path</param>
/// <param name="transferType">Transfer type</param>
/// <param name="customPath">Custom path for resume data (optional)</param>
/// <returns>Transfer state if found, null otherwise</returns>
public static ChunkedTransferState? LoadTransferState(
string bucketName,
string objectName,
string filePath,
ChunkedTransferType transferType,
string? customPath = null)
{
var resumeDirectory = customPath ?? DefaultResumeDirectory;
if (!Directory.Exists(resumeDirectory))
return null;
// Generate expected filename
var tempState = new ChunkedTransferState
{
BucketName = bucketName,
ObjectName = objectName,
FilePath = filePath,
TransferType = transferType
};
var fileName = GenerateResumeFileName(tempState);
var resumeFilePath = Path.Combine(resumeDirectory, fileName);
if (!File.Exists(resumeFilePath))
return null;
try
{
var json = File.ReadAllText(resumeFilePath);
var transferState = JsonSerializer.Deserialize<ChunkedTransferState>(json, new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
});
return transferState;
}
catch (Exception)
{
// If we can't deserialize, treat as no resume data
return null;
}
}
/// <summary>
/// Deletes resume data after successful completion
/// </summary>
/// <param name="transferState">Transfer state to clean up</param>
/// <param name="customPath">Custom path for resume data (optional)</param>
public static void CleanupResumeData(ChunkedTransferState transferState, string? customPath = null)
{
var resumeDirectory = customPath ?? DefaultResumeDirectory;
var fileName = GenerateResumeFileName(transferState);
var filePath = Path.Combine(resumeDirectory, fileName);
try
{
if (File.Exists(filePath))
{
File.Delete(filePath);
}
}
catch (Exception)
{
// Ignore cleanup errors
}
}
/// <summary>
/// Validates if resume data is still valid
/// </summary>
/// <param name="transferState">Transfer state to validate</param>
/// <param name="currentFileInfo">Current file information</param>
/// <returns>True if resume data is valid</returns>
public static bool IsResumeDataValid(ChunkedTransferState transferState, FileInfo? currentFileInfo = null)
{
// Check if transfer state is reasonable
if (transferState == null)
return false;
// For uploads, validate source file hasn't changed
if (transferState.TransferType == ChunkedTransferType.Upload && currentFileInfo != null)
{
if (!currentFileInfo.Exists)
return false;
// Check if file size or last modified time changed
if (currentFileInfo.Length != transferState.TotalSize ||
currentFileInfo.LastWriteTimeUtc != transferState.LastModified)
{
return false;
}
}
// Check if resume data is not too old (e.g., older than 7 days)
if (DateTime.UtcNow - transferState.LastUpdated > TimeSpan.FromDays(7))
return false;
return true;
}
/// <summary>
/// Gets all resume files in the directory
/// </summary>
/// <param name="customPath">Custom path for resume data (optional)</param>
/// <returns>Array of resume file paths</returns>
public static string[] GetResumeFiles(string? customPath = null)
{
var resumeDirectory = customPath ?? DefaultResumeDirectory;
if (!Directory.Exists(resumeDirectory))
return Array.Empty<string>();
return Directory.GetFiles(resumeDirectory, "*.psminioResume");
}
/// <summary>
/// Cleans up old resume files
/// </summary>
/// <param name="olderThanDays">Delete files older than this many days</param>
/// <param name="customPath">Custom path for resume data (optional)</param>
/// <returns>Number of files cleaned up</returns>
public static int CleanupOldResumeFiles(int olderThanDays = 7, string? customPath = null)
{
var resumeFiles = GetResumeFiles(customPath);
var cutoffDate = DateTime.UtcNow.AddDays(-olderThanDays);
var cleanedCount = 0;
foreach (var file in resumeFiles)
{
try
{
var fileInfo = new FileInfo(file);
if (fileInfo.LastWriteTimeUtc < cutoffDate)
{
File.Delete(file);
cleanedCount++;
}
}
catch (Exception)
{
// Ignore cleanup errors
}
}
return cleanedCount;
}
/// <summary>
/// Generates a unique filename for resume data
/// </summary>
/// <param name="transferState">Transfer state</param>
/// <returns>Unique filename</returns>
private static string GenerateResumeFileName(ChunkedTransferState transferState)
{
// Create a hash of the key components to ensure uniqueness
var key = $"{transferState.BucketName}|{transferState.ObjectName}|{transferState.FilePath}|{transferState.TransferType}";
var hash = key.GetHashCode().ToString("X8");
// Include readable components for easier identification
var safeBucketName = MakeSafeFileName(transferState.BucketName);
var safeObjectName = MakeSafeFileName(Path.GetFileName(transferState.ObjectName));
var transferType = transferState.TransferType.ToString().ToLower();
return $"{safeBucketName}_{safeObjectName}_{transferType}_{hash}.psminioResume";
}
/// <summary>
/// Makes a string safe for use as a filename
/// </summary>
/// <param name="input">Input string</param>
/// <returns>Safe filename string</returns>
private static string MakeSafeFileName(string input)
{
if (string.IsNullOrEmpty(input))
return "unknown";
var invalidChars = Path.GetInvalidFileNameChars();
var safe = input;
foreach (var c in invalidChars)
{
safe = safe.Replace(c, '_');
}
// Limit length and remove leading/trailing dots and spaces
safe = safe.Trim(' ', '.');
if (safe.Length > 50)
safe = safe.Substring(0, 50);
return string.IsNullOrEmpty(safe) ? "unknown" : safe;
}
}
}
+285
View File
@@ -0,0 +1,285 @@
using System;
using System.Management.Automation;
using PSMinIO.Models;
namespace PSMinIO.Utils
{
/// <summary>
/// Base class for all MinIO PowerShell cmdlets
/// Provides common functionality including connection management, logging, and client access
/// </summary>
public abstract class MinIOBaseCmdlet : PSCmdlet
{
private MinIOConnection? _connection;
/// <summary>
/// MinIO connection to use for operations. Can be provided via parameter or retrieved from session.
/// </summary>
[Parameter(ValueFromPipeline = true)]
[Alias("Connection")]
public MinIOConnection? MinIOConnection { get; set; }
/// <summary>
/// Name of session variable containing the MinIO connection (default: MinIOConnection)
/// </summary>
[Parameter]
[ValidateNotNullOrEmpty]
public string SessionVariable { get; set; } = "MinIOConnection";
/// <summary>
/// Gets the MinIO connection instance
/// </summary>
protected MinIOConnection Connection
{
get
{
if (_connection == null)
{
_connection = GetConnection();
if (_connection == null)
{
ThrowTerminatingError(new ErrorRecord(
new InvalidOperationException("No MinIO connection available. Use Connect-MinIO to establish a connection, or provide a connection via the -MinIOConnection parameter."),
"NoConnection",
ErrorCategory.ConnectionError,
null));
}
if (!_connection.IsValid)
{
ThrowTerminatingError(new ErrorRecord(
new InvalidOperationException($"MinIO connection is not valid. Status: {_connection.Status}"),
"InvalidConnection",
ErrorCategory.ConnectionError,
_connection));
}
MinIOLogger.LogOperationStart(this, "UsingConnection", $"Endpoint: {_connection.Configuration.Endpoint}");
}
return _connection;
}
}
/// <summary>
/// Gets the MinIO client wrapper instance
/// </summary>
protected MinIOClientWrapper Client => Connection.Client;
/// <summary>
/// Gets the current MinIO configuration
/// </summary>
protected MinIOConfiguration Configuration => Connection.Configuration;
/// <summary>
/// Gets the MinIO connection from parameter or session variable
/// </summary>
/// <returns>MinIO connection or null if not found</returns>
private MinIOConnection? GetConnection()
{
// First, check if connection was provided via parameter
if (MinIOConnection != null)
{
MinIOLogger.WriteVerbose(this, "Using MinIO connection from parameter");
return MinIOConnection;
}
// Next, check session variable
try
{
var sessionConnection = SessionState.PSVariable.GetValue(SessionVariable) as MinIOConnection;
if (sessionConnection != null)
{
MinIOLogger.WriteVerbose(this, "Using MinIO connection from session variable: {0}", SessionVariable);
return sessionConnection;
}
}
catch (Exception ex)
{
MinIOLogger.WriteVerbose(this, "Failed to retrieve connection from session variable '{0}': {1}", SessionVariable, ex.Message);
}
MinIOLogger.WriteVerbose(this, "No MinIO connection found in parameter or session variable '{0}'", SessionVariable);
return null;
}
/// <summary>
/// Validates that the MinIO connection is available and valid
/// </summary>
protected void ValidateConnection()
{
var connection = Connection; // This will throw if invalid
}
/// <summary>
/// Executes an operation with proper error handling and logging
/// </summary>
/// <param name="operationName">Name of the operation for logging</param>
/// <param name="operation">The operation to execute</param>
/// <param name="details">Optional operation details for logging</param>
protected void ExecuteOperation(string operationName, Action operation, string? details = null)
{
if (operation == null)
throw new ArgumentNullException(nameof(operationName));
var startTime = DateTime.UtcNow;
MinIOLogger.LogOperationStart(this, operationName, details);
try
{
operation();
var duration = DateTime.UtcNow - startTime;
MinIOLogger.LogOperationComplete(this, operationName, duration, details);
}
catch (Exception ex)
{
MinIOLogger.LogOperationFailure(this, operationName, ex, details);
// Determine appropriate error category
var category = GetErrorCategory(ex);
WriteError(new ErrorRecord(ex, $"{operationName}Failed", category, null));
}
}
/// <summary>
/// Executes an operation with return value and proper error handling and logging
/// </summary>
/// <typeparam name="T">Return type</typeparam>
/// <param name="operationName">Name of the operation for logging</param>
/// <param name="operation">The operation to execute</param>
/// <param name="details">Optional operation details for logging</param>
/// <returns>Result of the operation</returns>
protected T ExecuteOperation<T>(string operationName, Func<T> operation, string? details = null)
{
if (operation == null)
throw new ArgumentNullException(nameof(operation));
var startTime = DateTime.UtcNow;
MinIOLogger.LogOperationStart(this, operationName, details);
try
{
var result = operation();
var duration = DateTime.UtcNow - startTime;
MinIOLogger.LogOperationComplete(this, operationName, duration, details);
return result;
}
catch (Exception ex)
{
MinIOLogger.LogOperationFailure(this, operationName, ex, details);
// Determine appropriate error category
var category = GetErrorCategory(ex);
ThrowTerminatingError(new ErrorRecord(ex, $"{operationName}Failed", category, null));
// This line will never be reached, but is required for compilation
throw;
}
}
/// <summary>
/// Determines the appropriate PowerShell error category based on the exception type
/// </summary>
/// <param name="exception">The exception to categorize</param>
/// <returns>Appropriate ErrorCategory</returns>
protected virtual ErrorCategory GetErrorCategory(Exception exception)
{
return exception switch
{
ArgumentException => ErrorCategory.InvalidArgument,
ArgumentNullException => ErrorCategory.InvalidArgument,
UnauthorizedAccessException => ErrorCategory.PermissionDenied,
System.Net.WebException => ErrorCategory.ConnectionError,
System.Net.Http.HttpRequestException => ErrorCategory.ConnectionError,
TimeoutException => ErrorCategory.OperationTimeout,
InvalidOperationException => ErrorCategory.InvalidOperation,
NotSupportedException => ErrorCategory.NotImplemented,
FileNotFoundException => ErrorCategory.ObjectNotFound,
DirectoryNotFoundException => ErrorCategory.ObjectNotFound,
_ => ErrorCategory.NotSpecified
};
}
/// <summary>
/// Validates a bucket name according to MinIO/S3 naming rules
/// </summary>
/// <param name="bucketName">Bucket name to validate</param>
/// <param name="parameterName">Parameter name for error reporting</param>
protected void ValidateBucketName(string bucketName, string parameterName = "BucketName")
{
if (string.IsNullOrWhiteSpace(bucketName))
{
ThrowTerminatingError(new ErrorRecord(
new ArgumentException($"{parameterName} cannot be null or empty"),
"InvalidBucketName",
ErrorCategory.InvalidArgument,
bucketName));
}
// Basic bucket name validation (simplified)
if (bucketName.Length < 3 || bucketName.Length > 63)
{
ThrowTerminatingError(new ErrorRecord(
new ArgumentException($"{parameterName} must be between 3 and 63 characters long"),
"InvalidBucketName",
ErrorCategory.InvalidArgument,
bucketName));
}
}
/// <summary>
/// Validates an object name
/// </summary>
/// <param name="objectName">Object name to validate</param>
/// <param name="parameterName">Parameter name for error reporting</param>
protected void ValidateObjectName(string objectName, string parameterName = "ObjectName")
{
if (string.IsNullOrWhiteSpace(objectName))
{
ThrowTerminatingError(new ErrorRecord(
new ArgumentException($"{parameterName} cannot be null or empty"),
"InvalidObjectName",
ErrorCategory.InvalidArgument,
objectName));
}
}
/// <summary>
/// Cleans up resources when the cmdlet is disposed
/// </summary>
protected override void EndProcessing()
{
try
{
// Note: We don't dispose the connection here as it may be shared across cmdlets
// The connection should be disposed by the user when no longer needed
_connection = null;
}
catch (Exception ex)
{
MinIOLogger.WriteVerbose(this, $"Error during cleanup: {ex.Message}");
}
base.EndProcessing();
}
/// <summary>
/// Handles stopping the cmdlet (Ctrl+C)
/// </summary>
protected override void StopProcessing()
{
try
{
_connection?.Client.CancelOperations();
MinIOLogger.WriteVerbose(this, "Operation cancelled by user");
}
catch (Exception ex)
{
MinIOLogger.WriteVerbose(this, $"Error cancelling operations: {ex.Message}");
}
base.StopProcessing();
}
}
}
+930
View File
@@ -0,0 +1,930 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using Minio;
using Minio.DataModel;
using Minio.DataModel.Args;
using PSMinIO.Models;
using PSMinIO.Utils;
namespace PSMinIO.Utils
{
/// <summary>
/// Synchronous wrapper for MinIO client operations
/// Converts async MinIO operations to synchronous calls for PowerShell compatibility
/// </summary>
public class MinIOClientWrapper : IDisposable
{
private readonly IMinioClient _client;
private readonly CancellationTokenSource _cancellationTokenSource;
private bool _disposed = false;
/// <summary>
/// Creates a new MinIOClientWrapper instance
/// </summary>
/// <param name="configuration">MinIO configuration</param>
public MinIOClientWrapper(MinIOConfiguration configuration)
{
if (configuration == null)
throw new ArgumentNullException(nameof(configuration));
if (!configuration.IsValid)
throw new ArgumentException("Invalid MinIO configuration", nameof(configuration));
_cancellationTokenSource = new CancellationTokenSource();
// Create MinIO client with configuration
var clientBuilder = new MinioClient()
.WithEndpoint(configuration.Endpoint)
.WithCredentials(configuration.AccessKey, configuration.SecretKey);
if (configuration.UseSSL)
{
clientBuilder = clientBuilder.WithSSL();
// Configure custom HttpClient for certificate validation if needed
if (configuration.SkipCertificateValidation)
{
var httpClientHandler = new HttpClientHandler()
{
ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => true
};
var httpClient = new HttpClient(httpClientHandler);
clientBuilder = clientBuilder.WithHttpClient(httpClient);
}
}
if (configuration.TimeoutSeconds > 0)
{
clientBuilder = clientBuilder.WithTimeout(configuration.TimeoutSeconds * 1000);
}
_client = clientBuilder.Build();
}
/// <summary>
/// Gets the cancellation token for operations
/// </summary>
public CancellationToken CancellationToken => _cancellationTokenSource.Token;
/// <summary>
/// Lists all buckets synchronously
/// </summary>
/// <returns>List of bucket information</returns>
public List<MinIOBucketInfo> ListBuckets()
{
try
{
var bucketsResult = Task.Run(async () =>
await _client.ListBucketsAsync(CancellationToken)).GetAwaiter().GetResult();
return bucketsResult.Buckets
.Select(MinIOBucketInfo.FromMinioBucket)
.ToList();
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to list buckets: {ex.Message}", ex);
}
}
/// <summary>
/// Checks if a bucket exists synchronously
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <returns>True if bucket exists, false otherwise</returns>
public bool BucketExists(string bucketName)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
try
{
var args = new BucketExistsArgs().WithBucket(bucketName);
return Task.Run(async () =>
await _client.BucketExistsAsync(args, CancellationToken)).GetAwaiter().GetResult();
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to check if bucket '{bucketName}' exists: {ex.Message}", ex);
}
}
/// <summary>
/// Creates a bucket synchronously
/// </summary>
/// <param name="bucketName">Name of the bucket to create</param>
/// <param name="region">Optional region for the bucket</param>
public void CreateBucket(string bucketName, string? region = null)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
try
{
var args = new MakeBucketArgs().WithBucket(bucketName);
if (!string.IsNullOrWhiteSpace(region))
{
args = args.WithLocation(region);
}
Task.Run(async () =>
await _client.MakeBucketAsync(args, CancellationToken)).GetAwaiter().GetResult();
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to create bucket '{bucketName}': {ex.Message}", ex);
}
}
/// <summary>
/// Deletes a bucket synchronously
/// </summary>
/// <param name="bucketName">Name of the bucket to delete</param>
public void DeleteBucket(string bucketName)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
try
{
var args = new RemoveBucketArgs().WithBucket(bucketName);
Task.Run(async () =>
await _client.RemoveBucketAsync(args, CancellationToken)).GetAwaiter().GetResult();
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to delete bucket '{bucketName}': {ex.Message}", ex);
}
}
/// <summary>
/// Lists objects in a bucket synchronously
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <param name="prefix">Optional prefix to filter objects</param>
/// <param name="recursive">Whether to list objects recursively</param>
/// <param name="includeVersions">Whether to include all versions of objects</param>
/// <returns>List of object information</returns>
public List<MinIOObjectInfo> ListObjects(string bucketName, string? prefix = null, bool recursive = true, bool includeVersions = false)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
try
{
var args = new ListObjectsArgs()
.WithBucket(bucketName)
.WithRecursive(recursive);
if (!string.IsNullOrWhiteSpace(prefix))
{
args = args.WithPrefix(prefix);
}
// Add version support if requested
if (includeVersions)
{
args = args.WithVersions(true);
}
var objects = new List<MinIOObjectInfo>();
var observable = _client.ListObjectsAsync(args, CancellationToken);
// Convert async enumerable to synchronous list
var task = Task.Run(async () =>
{
await foreach (var item in observable.WithCancellation(CancellationToken))
{
objects.Add(MinIOObjectInfo.FromMinioItem(item, bucketName));
}
});
task.GetAwaiter().GetResult();
return objects;
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to list objects in bucket '{bucketName}': {ex.Message}", ex);
}
}
/// <summary>
/// Gets bucket policy synchronously
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <returns>Bucket policy as JSON string</returns>
public string GetBucketPolicy(string bucketName)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
try
{
var args = new GetPolicyArgs().WithBucket(bucketName);
return Task.Run(async () =>
await _client.GetPolicyAsync(args, CancellationToken)).GetAwaiter().GetResult();
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to get policy for bucket '{bucketName}': {ex.Message}", ex);
}
}
/// <summary>
/// Sets bucket policy synchronously
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <param name="policy">Policy JSON string</param>
public void SetBucketPolicy(string bucketName, string policy)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
if (string.IsNullOrWhiteSpace(policy))
throw new ArgumentException("Policy cannot be null or empty", nameof(policy));
try
{
var args = new SetPolicyArgs()
.WithBucket(bucketName)
.WithPolicy(policy);
Task.Run(async () =>
await _client.SetPolicyAsync(args, CancellationToken)).GetAwaiter().GetResult();
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to set policy for bucket '{bucketName}': {ex.Message}", ex);
}
}
/// <summary>
/// Deletes an object synchronously
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <param name="objectName">Name of the object to delete</param>
public void DeleteObject(string bucketName, string objectName)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
if (string.IsNullOrWhiteSpace(objectName))
throw new ArgumentException("Object name cannot be null or empty", nameof(objectName));
try
{
var args = new RemoveObjectArgs()
.WithBucket(bucketName)
.WithObject(objectName);
Task.Run(async () =>
await _client.RemoveObjectAsync(args, CancellationToken)).GetAwaiter().GetResult();
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to delete object '{objectName}' from bucket '{bucketName}': {ex.Message}", ex);
}
}
/// <summary>
/// Deletes multiple objects synchronously
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <param name="objectNames">List of object names to delete</param>
public void DeleteObjects(string bucketName, IEnumerable<string> objectNames)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
if (objectNames == null)
throw new ArgumentNullException(nameof(objectNames));
var objectList = objectNames.ToList();
if (objectList.Count == 0)
return;
try
{
var deleteObjectsArgs = new RemoveObjectsArgs()
.WithBucket(bucketName)
.WithObjects(objectList);
var observable = _client.RemoveObjectsAsync(deleteObjectsArgs, CancellationToken);
// Convert async enumerable to synchronous operation
var task = Task.Run(async () =>
{
await foreach (var deleteError in observable.WithCancellation(CancellationToken))
{
if (deleteError.Exception != null)
{
throw new InvalidOperationException($"Failed to delete object '{deleteError.Key}': {deleteError.Exception.Message}", deleteError.Exception);
}
}
});
task.GetAwaiter().GetResult();
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to delete objects from bucket '{bucketName}': {ex.Message}", ex);
}
}
/// <summary>
/// Uploads a file to MinIO synchronously with progress reporting
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <param name="objectName">Name of the object</param>
/// <param name="filePath">Path to the file to upload</param>
/// <param name="contentType">Content type of the file (optional)</param>
/// <param name="progressCallback">Progress callback for reporting upload progress</param>
/// <returns>ETag of the uploaded object</returns>
public string UploadFile(string bucketName, string objectName, string filePath,
string? contentType = null, Action<long>? progressCallback = null)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
if (string.IsNullOrWhiteSpace(objectName))
throw new ArgumentException("Object name cannot be null or empty", nameof(objectName));
if (string.IsNullOrWhiteSpace(filePath))
throw new ArgumentException("File path cannot be null or empty", nameof(filePath));
if (!File.Exists(filePath))
throw new FileNotFoundException($"File not found: {filePath}");
try
{
var fileInfo = new FileInfo(filePath);
var fileSize = fileInfo.Length;
// Determine content type if not provided
if (string.IsNullOrWhiteSpace(contentType))
{
contentType = GetContentType(filePath);
}
var args = new PutObjectArgs()
.WithBucket(bucketName)
.WithObject(objectName)
.WithFileName(filePath)
.WithContentType(contentType);
// Add progress callback if provided
if (progressCallback != null)
{
args = args.WithProgress(new Progress<ProgressReport>(report =>
{
progressCallback(report.TotalBytesTransferred);
}));
}
var result = Task.Run(async () =>
await _client.PutObjectAsync(args, CancellationToken)).GetAwaiter().GetResult();
return result.Etag ?? string.Empty;
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to upload file '{filePath}' to bucket '{bucketName}' as '{objectName}': {ex.Message}", ex);
}
}
/// <summary>
/// Downloads an object from MinIO synchronously with progress reporting
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <param name="objectName">Name of the object</param>
/// <param name="filePath">Path where the file should be saved</param>
/// <param name="progressCallback">Progress callback for reporting download progress</param>
public void DownloadFile(string bucketName, string objectName, string filePath,
Action<long>? progressCallback = null)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
if (string.IsNullOrWhiteSpace(objectName))
throw new ArgumentException("Object name cannot be null or empty", nameof(objectName));
if (string.IsNullOrWhiteSpace(filePath))
throw new ArgumentException("File path cannot be null or empty", nameof(filePath));
try
{
// Ensure the directory exists
var directory = Path.GetDirectoryName(filePath);
if (!string.IsNullOrEmpty(directory) && !Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
var args = new GetObjectArgs()
.WithBucket(bucketName)
.WithObject(objectName)
.WithFile(filePath);
// Add progress callback if provided
if (progressCallback != null)
{
args = args.WithProgress(new Progress<ProgressReport>(report =>
{
progressCallback(report.TotalBytesTransferred);
}));
}
Task.Run(async () =>
await _client.GetObjectAsync(args, CancellationToken)).GetAwaiter().GetResult();
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to download object '{objectName}' from bucket '{bucketName}' to '{filePath}': {ex.Message}", ex);
}
}
/// <summary>
/// Uploads a stream to MinIO synchronously
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <param name="objectName">Name of the object</param>
/// <param name="data">Stream containing the data to upload</param>
/// <param name="contentType">Content type of the data</param>
/// <param name="progressCallback">Progress callback for reporting upload progress</param>
/// <returns>ETag of the uploaded object</returns>
public string UploadStream(string bucketName, string objectName, Stream data,
string contentType = "application/octet-stream", Action<long>? progressCallback = null)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
if (string.IsNullOrWhiteSpace(objectName))
throw new ArgumentException("Object name cannot be null or empty", nameof(objectName));
if (data == null)
throw new ArgumentNullException(nameof(data));
try
{
var args = new PutObjectArgs()
.WithBucket(bucketName)
.WithObject(objectName)
.WithStreamData(data)
.WithObjectSize(data.Length)
.WithContentType(contentType);
// Add progress callback if provided
if (progressCallback != null)
{
args = args.WithProgress(new Progress<ProgressReport>(report =>
{
progressCallback(report.TotalBytesTransferred);
}));
}
var result = Task.Run(async () =>
await _client.PutObjectAsync(args, CancellationToken)).GetAwaiter().GetResult();
return result.Etag ?? string.Empty;
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to upload stream to bucket '{bucketName}' as '{objectName}': {ex.Message}", ex);
}
}
/// <summary>
/// Gets the content type for a file based on its extension
/// </summary>
/// <param name="filePath">Path to the file</param>
/// <returns>Content type string</returns>
private static string GetContentType(string filePath)
{
var extension = Path.GetExtension(filePath).ToLowerInvariant();
return extension switch
{
".txt" => "text/plain",
".html" => "text/html",
".css" => "text/css",
".js" => "application/javascript",
".json" => "application/json",
".xml" => "application/xml",
".pdf" => "application/pdf",
".zip" => "application/zip",
".jpg" or ".jpeg" => "image/jpeg",
".png" => "image/png",
".gif" => "image/gif",
".svg" => "image/svg+xml",
".mp4" => "video/mp4",
".mp3" => "audio/mpeg",
".wav" => "audio/wav",
_ => "application/octet-stream"
};
}
/// <summary>
/// Lists object versions in a bucket synchronously
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <param name="prefix">Optional prefix to filter objects</param>
/// <param name="recursive">Whether to list objects recursively</param>
/// <param name="maxObjects">Maximum number of objects to return (0 = unlimited)</param>
/// <returns>List of object information including versions</returns>
private List<MinIOObjectInfo> ListObjectVersions(string bucketName, string? prefix, bool recursive, int maxObjects)
{
try
{
// For now, fall back to regular object listing since version listing
// may not be available in all MinIO SDK versions
// This can be enhanced when the SDK supports it
return ListObjects(bucketName, prefix, recursive, maxObjects, false);
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to list object versions in bucket '{bucketName}': {ex.Message}", ex);
}
}
/// <summary>
/// Generates a presigned URL for an object
/// </summary>
/// <param name="bucketName">Name of the bucket</param>
/// <param name="objectName">Name of the object</param>
/// <param name="expiry">URL expiry time</param>
/// <returns>Presigned URL</returns>
public string GetPresignedUrl(string bucketName, string objectName, TimeSpan expiry)
{
if (string.IsNullOrWhiteSpace(bucketName))
throw new ArgumentException("Bucket name cannot be null or empty", nameof(bucketName));
if (string.IsNullOrWhiteSpace(objectName))
throw new ArgumentException("Object name cannot be null or empty", nameof(objectName));
try
{
var args = new PresignedGetObjectArgs()
.WithBucket(bucketName)
.WithObject(objectName)
.WithExpiry((int)expiry.TotalSeconds);
var result = Task.Run(async () =>
await _client.PresignedGetObjectAsync(args)).GetAwaiter().GetResult();
return result ?? string.Empty;
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to generate presigned URL for object '{objectName}' in bucket '{bucketName}': {ex.Message}", ex);
}
}
/// <summary>
/// Uploads a file using chunked transfer with resume capability
/// </summary>
/// <param name="transferState">Transfer state for resume functionality</param>
/// <param name="progressReporter">Progress reporter for updates</param>
/// <param name="maxRetries">Maximum retry attempts per chunk</param>
/// <returns>MinIOObjectInfo of uploaded object or null if failed</returns>
public MinIOObjectInfo? UploadFileChunked(
ChunkedTransferState transferState,
ChunkedCollectionProgressReporter progressReporter,
int maxRetries = 3)
{
if (transferState == null)
throw new ArgumentNullException(nameof(transferState));
try
{
// Start multipart upload if not already started
if (string.IsNullOrEmpty(transferState.UploadId))
{
var initiateArgs = new NewMultipartUploadArgs()
.WithBucket(transferState.BucketName)
.WithObject(transferState.ObjectName);
var initiateResult = Task.Run(async () =>
await _client.NewMultipartUploadAsync(initiateArgs, CancellationToken)).GetAwaiter().GetResult();
transferState.UploadId = initiateResult.UploadId;
}
var completedParts = new List<UploadPartResponse>();
// Process each chunk
while (!transferState.IsComplete)
{
var nextChunk = transferState.GetNextChunk();
if (nextChunk == null)
break;
progressReporter.StartNewChunk(nextChunk.ChunkNumber + 1, nextChunk.Size);
var uploadResult = UploadChunkWithRetry(transferState, nextChunk, progressReporter, maxRetries);
if (uploadResult != null)
{
completedParts.Add(uploadResult);
transferState.MarkChunkCompleted(nextChunk);
progressReporter.CompleteChunk(uploadResult.ETag);
// Save progress for resume
ChunkedTransferResumeManager.SaveTransferState(transferState);
}
else
{
throw new InvalidOperationException($"Failed to upload chunk {nextChunk.ChunkNumber} after {maxRetries} attempts");
}
}
// Complete multipart upload
var completeArgs = new CompleteMultipartUploadArgs()
.WithBucket(transferState.BucketName)
.WithObject(transferState.ObjectName)
.WithUploadId(transferState.UploadId)
.WithETags(completedParts.OrderBy(p => p.PartNumber).Select(p => new Tuple<int, string>(p.PartNumber, p.ETag)));
var completeResult = Task.Run(async () =>
await _client.CompleteMultipartUploadAsync(completeArgs, CancellationToken)).GetAwaiter().GetResult();
// Return object information
return new MinIOObjectInfo(
transferState.ObjectName,
transferState.TotalSize,
DateTime.UtcNow,
completeResult.ETag,
transferState.BucketName);
}
catch (Exception ex)
{
// Abort multipart upload on failure
if (!string.IsNullOrEmpty(transferState.UploadId))
{
try
{
var abortArgs = new AbortMultipartUploadArgs()
.WithBucket(transferState.BucketName)
.WithObject(transferState.ObjectName)
.WithUploadId(transferState.UploadId);
Task.Run(async () =>
await _client.AbortMultipartUploadAsync(abortArgs, CancellationToken)).GetAwaiter().GetResult();
}
catch
{
// Ignore abort errors
}
}
throw new InvalidOperationException($"Chunked upload failed for object '{transferState.ObjectName}': {ex.Message}", ex);
}
}
/// <summary>
/// Uploads a single chunk with retry logic
/// </summary>
/// <param name="transferState">Transfer state</param>
/// <param name="chunk">Chunk to upload</param>
/// <param name="progressReporter">Progress reporter</param>
/// <param name="maxRetries">Maximum retry attempts</param>
/// <returns>Upload part response or null if failed</returns>
private UploadPartResponse? UploadChunkWithRetry(
ChunkedTransferState transferState,
ChunkInfo chunk,
ChunkedCollectionProgressReporter progressReporter,
int maxRetries)
{
for (int attempt = 1; attempt <= maxRetries; attempt++)
{
try
{
using var fileStream = new FileStream(transferState.FilePath, FileMode.Open, FileAccess.Read, FileShare.Read);
fileStream.Seek(chunk.StartByte, SeekOrigin.Begin);
var chunkData = new byte[chunk.Size];
var bytesRead = fileStream.Read(chunkData, 0, (int)chunk.Size);
using var chunkStream = new MemoryStream(chunkData, 0, bytesRead);
var uploadArgs = new UploadPartArgs()
.WithBucket(transferState.BucketName)
.WithObject(transferState.ObjectName)
.WithUploadId(transferState.UploadId)
.WithPartNumber(chunk.ChunkNumber + 1) // MinIO uses 1-based part numbers
.WithPartSize(bytesRead)
.WithStreamData(chunkStream);
// Add progress callback
uploadArgs = uploadArgs.WithProgress(new Progress<ProgressReport>(report =>
{
progressReporter.UpdateChunkProgress(report.TotalBytesTransferred);
}));
var result = Task.Run(async () =>
await _client.UploadPartAsync(uploadArgs, CancellationToken)).GetAwaiter().GetResult();
chunk.ChunkETag = result.ETag;
return result;
}
catch (Exception ex) when (attempt < maxRetries)
{
progressReporter.ReportChunkError(ex, attempt, maxRetries);
// Exponential backoff
var delay = TimeSpan.FromSeconds(Math.Pow(2, attempt));
Task.Delay(delay, CancellationToken).GetAwaiter().GetResult();
}
catch (Exception ex)
{
progressReporter.ReportChunkError(ex, attempt, maxRetries);
chunk.LastError = ex.Message;
chunk.RetryCount = attempt;
return null;
}
}
return null;
}
/// <summary>
/// Downloads a file using chunked transfer with resume capability
/// </summary>
/// <param name="transferState">Transfer state for resume functionality</param>
/// <param name="progressReporter">Progress reporter for updates</param>
/// <param name="maxRetries">Maximum retry attempts per chunk</param>
/// <param name="parallelDownloads">Number of parallel chunk downloads</param>
/// <returns>True if download succeeded, false otherwise</returns>
public bool DownloadFileChunked(
ChunkedTransferState transferState,
ChunkedSingleFileProgressReporter progressReporter,
int maxRetries = 3,
int parallelDownloads = 3)
{
if (transferState == null)
throw new ArgumentNullException(nameof(transferState));
try
{
// Create or open the target file
using var fileStream = new FileStream(transferState.FilePath, FileMode.Create, FileAccess.Write, FileShare.None);
fileStream.SetLength(transferState.TotalSize);
// Get list of chunks to download
var chunksToDownload = new List<ChunkInfo>();
while (!transferState.IsComplete)
{
var nextChunk = transferState.GetNextChunk();
if (nextChunk == null)
break;
chunksToDownload.Add(nextChunk);
}
if (chunksToDownload.Count == 0)
{
return true; // Already complete
}
// Download chunks (with limited parallelism)
var semaphore = new SemaphoreSlim(parallelDownloads, parallelDownloads);
var downloadTasks = chunksToDownload.Select(chunk =>
DownloadChunkAsync(transferState, chunk, fileStream, progressReporter, maxRetries, semaphore)).ToArray();
var results = Task.WhenAll(downloadTasks).GetAwaiter().GetResult();
// Check if all chunks downloaded successfully
var allSucceeded = results.All(r => r);
if (allSucceeded)
{
// Mark all chunks as completed
foreach (var chunk in chunksToDownload)
{
transferState.MarkChunkCompleted(chunk);
}
}
return allSucceeded;
}
catch (Exception ex)
{
throw new InvalidOperationException($"Chunked download failed for object '{transferState.ObjectName}': {ex.Message}", ex);
}
}
/// <summary>
/// Downloads a single chunk asynchronously with retry logic
/// </summary>
/// <param name="transferState">Transfer state</param>
/// <param name="chunk">Chunk to download</param>
/// <param name="fileStream">Target file stream</param>
/// <param name="progressReporter">Progress reporter</param>
/// <param name="maxRetries">Maximum retry attempts</param>
/// <param name="semaphore">Semaphore for controlling parallelism</param>
/// <returns>True if chunk downloaded successfully</returns>
private async Task<bool> DownloadChunkAsync(
ChunkedTransferState transferState,
ChunkInfo chunk,
FileStream fileStream,
ChunkedSingleFileProgressReporter progressReporter,
int maxRetries,
SemaphoreSlim semaphore)
{
await semaphore.WaitAsync(CancellationToken);
try
{
progressReporter.StartNewChunk(chunk.ChunkNumber + 1, chunk.Size);
for (int attempt = 1; attempt <= maxRetries; attempt++)
{
try
{
var getArgs = new GetObjectArgs()
.WithBucket(transferState.BucketName)
.WithObject(transferState.ObjectName)
.WithOffsetAndLength(chunk.StartByte, chunk.Size);
using var chunkStream = new MemoryStream();
await _client.GetObjectAsync(getArgs, (stream) =>
{
stream.CopyTo(chunkStream);
}, CancellationToken);
// Write chunk to file at correct position
lock (fileStream)
{
fileStream.Seek(chunk.StartByte, SeekOrigin.Begin);
chunkStream.Seek(0, SeekOrigin.Begin);
chunkStream.CopyTo(fileStream);
fileStream.Flush();
}
progressReporter.UpdateChunkProgress(chunk.Size);
progressReporter.CompleteChunk();
chunk.IsCompleted = true;
return true;
}
catch (Exception ex) when (attempt < maxRetries)
{
progressReporter.ReportChunkError(ex, attempt, maxRetries);
// Exponential backoff
var delay = TimeSpan.FromSeconds(Math.Pow(2, attempt));
await Task.Delay(delay, CancellationToken);
}
catch (Exception ex)
{
progressReporter.ReportChunkError(ex, attempt, maxRetries);
chunk.LastError = ex.Message;
chunk.RetryCount = attempt;
return false;
}
}
return false;
}
finally
{
semaphore.Release();
}
}
/// <summary>
/// Cancels all ongoing operations
/// </summary>
public void CancelOperations()
{
_cancellationTokenSource.Cancel();
}
/// <summary>
/// Disposes the wrapper and underlying client
/// </summary>
public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
/// <summary>
/// Protected dispose method
/// </summary>
/// <param name="disposing">Whether disposing from Dispose method</param>
protected virtual void Dispose(bool disposing)
{
if (!_disposed && disposing)
{
_cancellationTokenSource?.Cancel();
_cancellationTokenSource?.Dispose();
_client?.Dispose();
_disposed = true;
}
}
}
}
+249
View File
@@ -0,0 +1,249 @@
using System;
using System.Management.Automation;
namespace PSMinIO.Utils
{
/// <summary>
/// Centralized logging utility for PSMinIO module
/// </summary>
public static class MinIOLogger
{
/// <summary>
/// Log levels for different types of messages
/// </summary>
public enum LogLevel
{
Verbose,
Information,
Warning,
Error
}
/// <summary>
/// Writes a verbose message if verbose preference allows it
/// </summary>
/// <param name="cmdlet">The cmdlet instance for context</param>
/// <param name="message">Message to log</param>
/// <param name="args">Optional format arguments</param>
public static void WriteVerbose(PSCmdlet cmdlet, string message, params object[] args)
{
if (cmdlet == null) return;
var formattedMessage = FormatLogMessage(LogLevel.Verbose, message, args);
cmdlet.WriteVerbose(formattedMessage);
}
/// <summary>
/// Writes an information message
/// </summary>
/// <param name="cmdlet">The cmdlet instance for context</param>
/// <param name="message">Message to log</param>
/// <param name="args">Optional format arguments</param>
public static void WriteInformation(PSCmdlet cmdlet, string message, params object[] args)
{
if (cmdlet == null) return;
var formattedMessage = FormatLogMessage(LogLevel.Information, message, args);
// Use WriteInformation if available (PowerShell 5.0+), otherwise WriteVerbose
try
{
var infoRecord = new InformationRecord(formattedMessage, "PSMinIO");
cmdlet.WriteInformation(infoRecord);
}
catch
{
// Fallback to WriteVerbose for older PowerShell versions
cmdlet.WriteVerbose(formattedMessage);
}
}
/// <summary>
/// Writes a warning message
/// </summary>
/// <param name="cmdlet">The cmdlet instance for context</param>
/// <param name="message">Message to log</param>
/// <param name="args">Optional format arguments</param>
public static void WriteWarning(PSCmdlet cmdlet, string message, params object[] args)
{
if (cmdlet == null) return;
var formattedMessage = FormatLogMessage(LogLevel.Warning, message, args);
cmdlet.WriteWarning(formattedMessage);
}
/// <summary>
/// Writes an error message
/// </summary>
/// <param name="cmdlet">The cmdlet instance for context</param>
/// <param name="message">Message to log</param>
/// <param name="args">Optional format arguments</param>
public static void WriteError(PSCmdlet cmdlet, string message, params object[] args)
{
if (cmdlet == null) return;
var formattedMessage = FormatLogMessage(LogLevel.Error, message, args);
var errorRecord = new ErrorRecord(
new InvalidOperationException(formattedMessage),
"PSMinIOError",
ErrorCategory.InvalidOperation,
null);
cmdlet.WriteError(errorRecord);
}
/// <summary>
/// Writes an error from an exception
/// </summary>
/// <param name="cmdlet">The cmdlet instance for context</param>
/// <param name="exception">Exception to log</param>
/// <param name="errorId">Error identifier</param>
/// <param name="category">Error category</param>
/// <param name="targetObject">Target object that caused the error</param>
public static void WriteError(PSCmdlet cmdlet, Exception exception, string errorId,
ErrorCategory category = ErrorCategory.InvalidOperation, object? targetObject = null)
{
if (cmdlet == null) return;
var errorRecord = new ErrorRecord(exception, errorId, category, targetObject);
cmdlet.WriteError(errorRecord);
}
/// <summary>
/// Logs the start of an operation
/// </summary>
/// <param name="cmdlet">The cmdlet instance for context</param>
/// <param name="operation">Name of the operation</param>
/// <param name="details">Optional operation details</param>
public static void LogOperationStart(PSCmdlet cmdlet, string operation, string? details = null)
{
var message = string.IsNullOrEmpty(details)
? $"Starting operation: {operation}"
: $"Starting operation: {operation} - {details}";
WriteVerbose(cmdlet, message);
}
/// <summary>
/// Logs the completion of an operation
/// </summary>
/// <param name="cmdlet">The cmdlet instance for context</param>
/// <param name="operation">Name of the operation</param>
/// <param name="duration">Optional operation duration</param>
/// <param name="details">Optional operation details</param>
public static void LogOperationComplete(PSCmdlet cmdlet, string operation, TimeSpan? duration = null, string? details = null)
{
var message = $"Completed operation: {operation}";
if (duration.HasValue)
{
message += $" (Duration: {duration.Value.TotalMilliseconds:F0}ms)";
}
if (!string.IsNullOrEmpty(details))
{
message += $" - {details}";
}
WriteVerbose(cmdlet, message);
}
/// <summary>
/// Logs an operation failure
/// </summary>
/// <param name="cmdlet">The cmdlet instance for context</param>
/// <param name="operation">Name of the operation</param>
/// <param name="exception">Exception that occurred</param>
/// <param name="details">Optional operation details</param>
public static void LogOperationFailure(PSCmdlet cmdlet, string operation, Exception exception, string? details = null)
{
var message = $"Operation failed: {operation} - {exception.Message}";
if (!string.IsNullOrEmpty(details))
{
message += $" - {details}";
}
WriteError(cmdlet, message);
WriteVerbose(cmdlet, $"Exception details: {exception}");
}
/// <summary>
/// Formats a log message with timestamp and level, automatically formatting byte sizes
/// </summary>
/// <param name="level">Log level</param>
/// <param name="message">Message to format</param>
/// <param name="args">Optional format arguments</param>
/// <returns>Formatted log message</returns>
private static string FormatLogMessage(LogLevel level, string message, params object[] args)
{
var timestamp = DateTime.Now.ToString("yyyy/MM/dd HH:mm:ss.fff");
var levelString = level.ToString().ToUpperInvariant();
// Process arguments to format byte sizes intelligently
var processedArgs = ProcessLogArguments(args);
var formattedMessage = processedArgs.Length > 0 ? string.Format(message, processedArgs) : message;
return $"{timestamp} - [{levelString}] - {formattedMessage}";
}
/// <summary>
/// Processes log arguments to format byte sizes intelligently
/// </summary>
/// <param name="args">Original arguments</param>
/// <returns>Processed arguments with formatted sizes</returns>
private static object[] ProcessLogArguments(object[] args)
{
if (args == null || args.Length == 0)
return args;
var processedArgs = new object[args.Length];
for (int i = 0; i < args.Length; i++)
{
var arg = args[i];
// Check if this looks like a byte size that should be formatted
if (IsLikelyByteSize(arg))
{
if (long.TryParse(arg.ToString(), out var bytes))
{
processedArgs[i] = SizeFormatter.FormatBytes(bytes);
}
else
{
processedArgs[i] = arg;
}
}
else
{
processedArgs[i] = arg;
}
}
return processedArgs;
}
/// <summary>
/// Determines if an argument is likely a byte size that should be formatted
/// </summary>
/// <param name="arg">Argument to check</param>
/// <returns>True if likely a byte size</returns>
private static bool IsLikelyByteSize(object arg)
{
// Only format long integers that are likely byte sizes
// We use a heuristic: values >= 1024 are likely byte sizes
if (arg is long longValue)
{
return longValue >= 1024;
}
if (arg is int intValue)
{
return intValue >= 1024;
}
return false;
}
}
}
+156
View File
@@ -0,0 +1,156 @@
using System;
using System.Diagnostics;
using System.Management.Automation;
namespace PSMinIO.Utils
{
/// <summary>
/// Utility class for reporting progress during file operations
/// </summary>
public class ProgressReporter
{
private readonly PSCmdlet _cmdlet;
private readonly string _activity;
private readonly string _statusDescription;
private readonly long _totalBytes;
private readonly Stopwatch _stopwatch;
private long _bytesProcessed;
private DateTime _lastUpdateTime;
private readonly int _activityId;
/// <summary>
/// Creates a new ProgressReporter instance
/// </summary>
/// <param name="cmdlet">The cmdlet to report progress to</param>
/// <param name="activity">Description of the activity</param>
/// <param name="statusDescription">Status description</param>
/// <param name="totalBytes">Total number of bytes to process</param>
/// <param name="activityId">Unique activity ID for progress reporting</param>
public ProgressReporter(PSCmdlet cmdlet, string activity, string statusDescription, long totalBytes, int activityId = 1)
{
_cmdlet = cmdlet ?? throw new ArgumentNullException(nameof(cmdlet));
_activity = activity ?? throw new ArgumentNullException(nameof(activity));
_statusDescription = statusDescription ?? throw new ArgumentNullException(nameof(statusDescription));
_totalBytes = totalBytes;
_activityId = activityId;
_stopwatch = Stopwatch.StartNew();
_lastUpdateTime = DateTime.UtcNow;
}
/// <summary>
/// Updates the progress with the number of bytes processed
/// </summary>
/// <param name="bytesProcessed">Number of bytes processed so far</param>
public void UpdateProgress(long bytesProcessed)
{
_bytesProcessed = bytesProcessed;
// Only update progress every 100ms to avoid overwhelming the console
var now = DateTime.UtcNow;
if ((now - _lastUpdateTime).TotalMilliseconds < 100 && bytesProcessed < _totalBytes)
{
return;
}
_lastUpdateTime = now;
var percentComplete = _totalBytes > 0 ? (int)((double)bytesProcessed / _totalBytes * 100) : 0;
var currentStatus = FormatCurrentStatus(bytesProcessed);
var progressRecord = new ProgressRecord(_activityId, _activity, currentStatus)
{
PercentComplete = Math.Min(percentComplete, 100)
};
// Add remaining time estimate if we have enough data
if (_stopwatch.ElapsedMilliseconds > 1000 && bytesProcessed > 0 && bytesProcessed < _totalBytes)
{
var remainingTime = EstimateRemainingTime(bytesProcessed);
if (remainingTime.HasValue)
{
progressRecord.SecondsRemaining = (int)remainingTime.Value.TotalSeconds;
}
}
_cmdlet.WriteProgress(progressRecord);
}
/// <summary>
/// Completes the progress reporting
/// </summary>
public void Complete()
{
_stopwatch.Stop();
var progressRecord = new ProgressRecord(_activityId, _activity, "Completed")
{
PercentComplete = 100,
RecordType = ProgressRecordType.Completed
};
_cmdlet.WriteProgress(progressRecord);
// Log completion details
MinIOLogger.WriteVerbose(_cmdlet,
$"Operation completed: {SizeFormatter.FormatBytes(_totalBytes)} processed in {_stopwatch.Elapsed.TotalSeconds:F1} seconds " +
$"(Average speed: {SizeFormatter.FormatBytesPerSecond(CalculateAverageSpeed())})");
}
/// <summary>
/// Formats the current status string
/// </summary>
/// <param name="bytesProcessed">Number of bytes processed</param>
/// <returns>Formatted status string</returns>
private string FormatCurrentStatus(long bytesProcessed)
{
var processedFormatted = SizeFormatter.FormatBytes(bytesProcessed);
var totalFormatted = SizeFormatter.FormatBytes(_totalBytes);
var speed = CalculateCurrentSpeed();
var speedFormatted = SizeFormatter.FormatBytesPerSecond(speed);
return $"{_statusDescription}: {processedFormatted} / {totalFormatted} ({speedFormatted})";
}
/// <summary>
/// Calculates the current transfer speed in bytes per second
/// </summary>
/// <returns>Current speed in bytes per second</returns>
private double CalculateCurrentSpeed()
{
var elapsedSeconds = _stopwatch.Elapsed.TotalSeconds;
return elapsedSeconds > 0 ? _bytesProcessed / elapsedSeconds : 0;
}
/// <summary>
/// Calculates the average transfer speed in bytes per second
/// </summary>
/// <returns>Average speed in bytes per second</returns>
private double CalculateAverageSpeed()
{
var elapsedSeconds = _stopwatch.Elapsed.TotalSeconds;
return elapsedSeconds > 0 ? _totalBytes / elapsedSeconds : 0;
}
/// <summary>
/// Estimates the remaining time for the operation
/// </summary>
/// <param name="bytesProcessed">Number of bytes processed so far</param>
/// <returns>Estimated remaining time, or null if cannot be calculated</returns>
private TimeSpan? EstimateRemainingTime(long bytesProcessed)
{
if (bytesProcessed <= 0 || _stopwatch.ElapsedMilliseconds <= 0)
return null;
var remainingBytes = _totalBytes - bytesProcessed;
var currentSpeed = CalculateCurrentSpeed();
if (currentSpeed <= 0)
return null;
var remainingSeconds = remainingBytes / currentSpeed;
return TimeSpan.FromSeconds(remainingSeconds);
}
}
}
+189
View File
@@ -0,0 +1,189 @@
using System;
namespace PSMinIO.Utils
{
/// <summary>
/// Utility class for formatting byte sizes into human-readable strings
/// </summary>
public static class SizeFormatter
{
/// <summary>
/// Size units in order from smallest to largest
/// </summary>
private static readonly string[] SizeUnits = { "B", "KB", "MB", "GB", "TB", "PB", "EB" };
/// <summary>
/// Formats bytes into a human-readable string with appropriate units
/// </summary>
/// <param name="bytes">Number of bytes</param>
/// <param name="decimalPlaces">Number of decimal places to show (default: 2)</param>
/// <returns>Formatted string with appropriate unit</returns>
public static string FormatBytes(long bytes, int decimalPlaces = 2)
{
if (bytes == 0)
return "0 B";
if (bytes < 0)
return $"-{FormatBytes(-bytes, decimalPlaces)}";
int unitIndex = 0;
double size = bytes;
// Find the appropriate unit
while (size >= 1024 && unitIndex < SizeUnits.Length - 1)
{
size /= 1024;
unitIndex++;
}
// Format with specified decimal places
var formatString = $"{{0:F{decimalPlaces}}} {{1}}";
return string.Format(formatString, size, SizeUnits[unitIndex]);
}
/// <summary>
/// Formats bytes into a human-readable string with appropriate units (double overload)
/// </summary>
/// <param name="bytes">Number of bytes</param>
/// <param name="decimalPlaces">Number of decimal places to show (default: 2)</param>
/// <returns>Formatted string with appropriate unit</returns>
public static string FormatBytes(double bytes, int decimalPlaces = 2)
{
return FormatBytes((long)Math.Round(bytes), decimalPlaces);
}
/// <summary>
/// Formats bytes per second into a human-readable string
/// </summary>
/// <param name="bytesPerSecond">Bytes per second</param>
/// <param name="decimalPlaces">Number of decimal places to show (default: 2)</param>
/// <returns>Formatted string with appropriate unit and "/s" suffix</returns>
public static string FormatBytesPerSecond(double bytesPerSecond, int decimalPlaces = 2)
{
return $"{FormatBytes(bytesPerSecond, decimalPlaces)}/s";
}
/// <summary>
/// Formats a transfer rate with context
/// </summary>
/// <param name="bytesTransferred">Number of bytes transferred</param>
/// <param name="elapsedTime">Time elapsed for the transfer</param>
/// <param name="decimalPlaces">Number of decimal places to show (default: 2)</param>
/// <returns>Formatted transfer rate string</returns>
public static string FormatTransferRate(long bytesTransferred, TimeSpan elapsedTime, int decimalPlaces = 2)
{
if (elapsedTime.TotalSeconds <= 0)
return "0 B/s";
var bytesPerSecond = bytesTransferred / elapsedTime.TotalSeconds;
return FormatBytesPerSecond(bytesPerSecond, decimalPlaces);
}
/// <summary>
/// Formats a progress string showing current/total with percentages
/// </summary>
/// <param name="current">Current bytes processed</param>
/// <param name="total">Total bytes to process</param>
/// <param name="decimalPlaces">Number of decimal places to show (default: 2)</param>
/// <returns>Formatted progress string</returns>
public static string FormatProgress(long current, long total, int decimalPlaces = 2)
{
var currentFormatted = FormatBytes(current, decimalPlaces);
var totalFormatted = FormatBytes(total, decimalPlaces);
if (total > 0)
{
var percentage = (double)current / total * 100;
return $"{currentFormatted} / {totalFormatted} ({percentage:F1}%)";
}
return $"{currentFormatted} / {totalFormatted}";
}
/// <summary>
/// Gets the appropriate unit for a given byte size without formatting
/// </summary>
/// <param name="bytes">Number of bytes</param>
/// <returns>Appropriate unit string</returns>
public static string GetAppropriateUnit(long bytes)
{
if (bytes == 0)
return "B";
int unitIndex = 0;
double size = Math.Abs(bytes);
while (size >= 1024 && unitIndex < SizeUnits.Length - 1)
{
size /= 1024;
unitIndex++;
}
return SizeUnits[unitIndex];
}
/// <summary>
/// Converts bytes to the specified unit
/// </summary>
/// <param name="bytes">Number of bytes</param>
/// <param name="unit">Target unit (B, KB, MB, GB, TB, PB, EB)</param>
/// <returns>Value in the specified unit</returns>
public static double ConvertToUnit(long bytes, string unit)
{
var unitIndex = Array.IndexOf(SizeUnits, unit.ToUpperInvariant());
if (unitIndex == -1)
throw new ArgumentException($"Invalid unit: {unit}. Valid units are: {string.Join(", ", SizeUnits)}");
if (unitIndex == 0) // Bytes
return bytes;
return bytes / Math.Pow(1024, unitIndex);
}
/// <summary>
/// Parses a size string back to bytes (e.g., "1.5 GB" -> bytes)
/// </summary>
/// <param name="sizeString">Size string to parse</param>
/// <returns>Number of bytes</returns>
public static long ParseSizeString(string sizeString)
{
if (string.IsNullOrWhiteSpace(sizeString))
throw new ArgumentException("Size string cannot be null or empty");
var parts = sizeString.Trim().Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (parts.Length != 2)
throw new ArgumentException($"Invalid size string format: {sizeString}. Expected format: '1.5 GB'");
if (!double.TryParse(parts[0], out var value))
throw new ArgumentException($"Invalid numeric value: {parts[0]}");
var unit = parts[1].ToUpperInvariant();
var unitIndex = Array.IndexOf(SizeUnits, unit);
if (unitIndex == -1)
throw new ArgumentException($"Invalid unit: {unit}. Valid units are: {string.Join(", ", SizeUnits)}");
return (long)(value * Math.Pow(1024, unitIndex));
}
/// <summary>
/// Formats a size comparison between two values
/// </summary>
/// <param name="value1">First value in bytes</param>
/// <param name="value2">Second value in bytes</param>
/// <param name="label1">Label for first value</param>
/// <param name="label2">Label for second value</param>
/// <param name="decimalPlaces">Number of decimal places to show (default: 2)</param>
/// <returns>Formatted comparison string</returns>
public static string FormatComparison(long value1, long value2, string label1, string label2, int decimalPlaces = 2)
{
var formatted1 = FormatBytes(value1, decimalPlaces);
var formatted2 = FormatBytes(value2, decimalPlaces);
var difference = value1 - value2;
var diffFormatted = FormatBytes(Math.Abs(difference), decimalPlaces);
var diffDirection = difference >= 0 ? "larger" : "smaller";
return $"{label1}: {formatted1}, {label2}: {formatted2} ({diffFormatted} {diffDirection})";
}
}
}