diff --git a/Module/PSMinIO/bin/PSMinIO.dll b/Module/PSMinIO/bin/PSMinIO.dll index a5de2f6..c8312c2 100644 Binary files a/Module/PSMinIO/bin/PSMinIO.dll and b/Module/PSMinIO/bin/PSMinIO.dll differ diff --git a/Module/PSMinIO/bin/PSMinIO.pdb b/Module/PSMinIO/bin/PSMinIO.pdb index fedba98..41a77d1 100644 Binary files a/Module/PSMinIO/bin/PSMinIO.pdb and b/Module/PSMinIO/bin/PSMinIO.pdb differ diff --git a/src/Cmdlets/NewMinIOZipArchiveCmdlet.cs b/src/Cmdlets/NewMinIOZipArchiveCmdlet.cs index ec9f517..742acdf 100644 --- a/src/Cmdlets/NewMinIOZipArchiveCmdlet.cs +++ b/src/Cmdlets/NewMinIOZipArchiveCmdlet.cs @@ -80,11 +80,11 @@ namespace PSMinIO.Cmdlets public ScriptBlock? ExclusionFilter { get; set; } /// - /// Compression level to use + /// Compression level to use. Adaptive automatically selects optimal compression based on file type and size. /// [Parameter] - [ValidateSet("Optimal", "Fastest", "NoCompression")] - public string CompressionLevel { get; set; } = "Optimal"; + [ValidateSet("Optimal", "Fastest", "NoCompression", "Adaptive")] + public string CompressionLevel { get; set; } = "Adaptive"; /// /// Archive mode (Create, Update for appending) @@ -198,10 +198,12 @@ namespace PSMinIO.Cmdlets return zipBuilder.CreateResult(DestinationPath.FullName); } - WriteVerboseMessage("Adding {0} files to zip archive", validFiles.Length); + WriteVerboseMessage("Adding {0} files to zip archive with {0} compression", + validFiles.Length, CompressionLevel == "Adaptive" ? "adaptive" : CompressionLevel.ToLowerInvariant()); - // Add files to zip - zipBuilder.AddFiles(validFiles.Cast(), BasePath, compressionLevel); + // Add files to zip - pass null for adaptive compression + var effectiveCompressionLevel = CompressionLevel == "Adaptive" ? null : (System.IO.Compression.CompressionLevel?)compressionLevel; + zipBuilder.AddFiles(validFiles.Cast(), BasePath, effectiveCompressionLevel); return zipBuilder.CreateResult(DestinationPath.FullName); } @@ -230,13 +232,15 @@ namespace PSMinIO.Cmdlets return zipBuilder.CreateResult(DestinationPath.FullName); } - WriteVerboseMessage("Adding directory to zip: {0} ({1} files)", Directory.Name, files.Length); + WriteVerboseMessage("Adding directory to zip: {0} ({1} files) with {2} compression", + Directory.Name, files.Length, CompressionLevel == "Adaptive" ? "adaptive" : CompressionLevel.ToLowerInvariant()); // Determine base path for entries var basePath = BasePath ?? (IncludeBaseDirectory.IsPresent ? Directory.Parent?.FullName : Directory.FullName); - // Add files to zip - zipBuilder.AddFiles(files.Cast(), basePath, compressionLevel); + // Add files to zip - pass null for adaptive compression + var effectiveCompressionLevel = CompressionLevel == "Adaptive" ? null : (System.IO.Compression.CompressionLevel?)compressionLevel; + zipBuilder.AddFiles(files.Cast(), basePath, effectiveCompressionLevel); return zipBuilder.CreateResult(DestinationPath.FullName); } diff --git a/src/Utils/ZipArchiveBuilder.cs b/src/Utils/ZipArchiveBuilder.cs index 875db99..a072177 100644 --- a/src/Utils/ZipArchiveBuilder.cs +++ b/src/Utils/ZipArchiveBuilder.cs @@ -17,6 +17,10 @@ namespace PSMinIO.Utils private readonly bool _leaveOpen; private bool _disposed = false; + // Performance optimization fields + private const long FlushThreshold = 10 * 1024 * 1024; // 10MB + private long _bytesWrittenSinceFlush = 0; + // Progress tracking public event EventHandler? ProgressChanged; public event EventHandler? FileAdded; @@ -56,13 +60,91 @@ namespace PSMinIO.Utils return new ZipArchiveBuilder(fileStream, mode, false); } + /// + /// Gets optimal buffer size based on file size + /// + private static int GetOptimalBufferSize(long fileSize) + { + return fileSize switch + { + < 64 * 1024 => 8 * 1024, // < 64KB: 8KB buffer (small files) + < 1024 * 1024 => 64 * 1024, // < 1MB: 64KB buffer + < 10 * 1024 * 1024 => 256 * 1024, // < 10MB: 256KB buffer + < 100 * 1024 * 1024 => 1024 * 1024, // < 100MB: 1MB buffer + _ => 4 * 1024 * 1024 // >= 100MB: 4MB buffer (large files) + }; + } + + /// + /// Gets optimal compression level based on file characteristics + /// + private static CompressionLevel GetOptimalCompressionLevel(FileInfo fileInfo, CompressionLevel? userSpecified = null) + { + // If user specified a compression level, always use it + if (userSpecified.HasValue) + return userSpecified.Value; + + var extension = fileInfo.Extension.ToLowerInvariant(); + + // Already compressed formats - use fastest to avoid double compression overhead + if (IsAlreadyCompressed(extension)) + return CompressionLevel.Fastest; + + // Small files (< 1MB) - use optimal for better compression ratio + if (fileInfo.Length < 1024 * 1024) + return CompressionLevel.Optimal; + + // Large files (> 100MB) - prioritize speed + if (fileInfo.Length > 100 * 1024 * 1024) + return CompressionLevel.Fastest; + + // Medium files - balance speed vs compression + return CompressionLevel.Optimal; + } + + /// + /// Checks if file extension indicates already compressed content + /// + private static bool IsAlreadyCompressed(string extension) + { + var compressedExtensions = new HashSet + { + // Archives + ".zip", ".rar", ".7z", ".gz", ".bz2", ".xz", ".tar", + // Images + ".jpg", ".jpeg", ".png", ".gif", ".webp", ".avif", + // Audio + ".mp3", ".aac", ".ogg", ".m4a", ".flac", + // Video + ".mp4", ".mkv", ".avi", ".mov", ".webm", ".m4v", + // Documents + ".pdf", ".docx", ".xlsx", ".pptx" + }; + + return compressedExtensions.Contains(extension); + } + + /// + /// Optimizes file processing order for better performance + /// + private static IEnumerable OptimizeFileProcessingOrder(IEnumerable files) + { + var fileInfos = files.OfType().ToList(); + + // Sort by directory first (improves disk access patterns) + // Then by size (small files first for quick progress feedback) + return fileInfos.OrderBy(f => f.DirectoryName) + .ThenBy(f => f.Length) + .ThenBy(f => f.Name); // Consistent ordering for same-size files + } + /// /// Adds a single file to the zip archive /// /// File to add /// Name of the entry in the zip (optional, uses file name if null) - /// Compression level to use - public void AddFile(FileInfo fileInfo, string? entryName = null, CompressionLevel compressionLevel = CompressionLevel.Optimal) + /// Compression level to use (null for adaptive compression) + public void AddFile(FileInfo fileInfo, string? entryName = null, CompressionLevel? compressionLevel = null) { if (fileInfo == null) throw new ArgumentNullException(nameof(fileInfo)); if (!fileInfo.Exists) throw new FileNotFoundException($"File not found: {fileInfo.FullName}"); @@ -70,21 +152,33 @@ namespace PSMinIO.Utils entryName ??= fileInfo.Name; var fileStartTime = DateTime.UtcNow; + // Get optimal compression level and buffer size + var effectiveCompressionLevel = GetOptimalCompressionLevel(fileInfo, compressionLevel); + var bufferSize = GetOptimalBufferSize(fileInfo.Length); + // Create entry in zip - var entry = _archive.CreateEntry(entryName, compressionLevel); + var entry = _archive.CreateEntry(entryName, effectiveCompressionLevel); entry.LastWriteTime = fileInfo.LastWriteTime; long bytesProcessed = 0; using (var fileStream = fileInfo.OpenRead()) using (var entryStream = entry.Open()) { - var buffer = new byte[81920]; // 80KB buffer for good performance + var buffer = new byte[bufferSize]; // Adaptive buffer size int bytesRead; while ((bytesRead = fileStream.Read(buffer, 0, buffer.Length)) > 0) { entryStream.Write(buffer, 0, bytesRead); bytesProcessed += bytesRead; + _bytesWrittenSinceFlush += bytesRead; + + // Periodic flushing based on size threshold + if (_bytesWrittenSinceFlush >= FlushThreshold) + { + entryStream.Flush(); + _bytesWrittenSinceFlush = 0; + } // Report progress OnProgressChanged(new ZipProgressEventArgs @@ -98,6 +192,9 @@ namespace PSMinIO.Utils ElapsedTime = DateTime.UtcNow - StartTime }); } + + // Final flush for this file + entryStream.Flush(); } // Update metrics (access CompressedLength after the entry stream is closed) @@ -132,30 +229,47 @@ namespace PSMinIO.Utils } /// - /// Adds multiple files to the zip archive + /// Adds multiple files to the zip archive with optimized processing order /// /// Files to add /// Base path to remove from entry names (optional) - /// Compression level to use - public void AddFiles(IEnumerable files, string? basePath = null, CompressionLevel compressionLevel = CompressionLevel.Optimal) + /// Compression level to use (null for adaptive compression) + public void AddFiles(IEnumerable files, string? basePath = null, CompressionLevel? compressionLevel = null) { if (files == null) throw new ArgumentNullException(nameof(files)); + // Separate files and directories for optimized processing + var fileList = new List(); + var directories = new List(); + foreach (var file in files) { if (file is FileInfo fileInfo) { - var entryName = GetEntryName(fileInfo, basePath); - AddFile(fileInfo, entryName, compressionLevel); + fileList.Add(fileInfo); } else if (file is DirectoryInfo dirInfo) { - // Add directory files recursively - var dirFiles = dirInfo.GetFiles("*", SearchOption.AllDirectories); - var dirBasePath = basePath ?? dirInfo.FullName; - AddFiles(dirFiles, dirBasePath, compressionLevel); + directories.Add(dirInfo); } } + + // Process directories first to collect all files + foreach (var dirInfo in directories) + { + var dirFiles = dirInfo.GetFiles("*", SearchOption.AllDirectories); + fileList.AddRange(dirFiles); + } + + // Optimize file processing order for better performance + var optimizedFiles = OptimizeFileProcessingOrder(fileList); + + // Process files in optimized order + foreach (var fileInfo in optimizedFiles) + { + var entryName = GetEntryName(fileInfo, basePath); + AddFile(fileInfo, entryName, compressionLevel); + } } /// @@ -163,15 +277,15 @@ namespace PSMinIO.Utils /// /// Directory to add /// Whether to include the base directory in entry names - /// Compression level to use - public void AddDirectory(DirectoryInfo directoryInfo, bool includeBaseDirectory = true, CompressionLevel compressionLevel = CompressionLevel.Optimal) + /// Compression level to use (null for adaptive compression) + public void AddDirectory(DirectoryInfo directoryInfo, bool includeBaseDirectory = true, CompressionLevel? compressionLevel = null) { if (directoryInfo == null) throw new ArgumentNullException(nameof(directoryInfo)); if (!directoryInfo.Exists) throw new DirectoryNotFoundException($"Directory not found: {directoryInfo.FullName}"); var files = directoryInfo.GetFiles("*", SearchOption.AllDirectories); var basePath = includeBaseDirectory ? directoryInfo.Parent?.FullName : directoryInfo.FullName; - + AddFiles(files, basePath, compressionLevel); } diff --git a/src/Utils/ZipBuilder.cs b/src/Utils/ZipBuilder.cs index 6daa5c3..886c1bb 100644 --- a/src/Utils/ZipBuilder.cs +++ b/src/Utils/ZipBuilder.cs @@ -80,8 +80,8 @@ namespace PSMinIO.Utils /// /// Files to add /// Base path for entry names (optional) - /// Compression level - public void AddFiles(IEnumerable files, string? basePath = null, CompressionLevel compressionLevel = CompressionLevel.Optimal) + /// Compression level (null for adaptive compression) + public void AddFiles(IEnumerable files, string? basePath = null, CompressionLevel? compressionLevel = null) { if (files == null) throw new ArgumentNullException(nameof(files)); @@ -104,8 +104,8 @@ namespace PSMinIO.Utils /// /// Directory to add /// Include base directory in entry names - /// Compression level - public void AddDirectory(DirectoryInfo directoryInfo, bool includeBaseDirectory = true, CompressionLevel compressionLevel = CompressionLevel.Optimal) + /// Compression level (null for adaptive compression) + public void AddDirectory(DirectoryInfo directoryInfo, bool includeBaseDirectory = true, CompressionLevel? compressionLevel = null) { if (directoryInfo == null) throw new ArgumentNullException(nameof(directoryInfo));