Implement enhanced upload cmdlet with FileInfo[] and universal BucketDirectory support

Enhanced New-MinIOObject cmdlet with advanced parameter sets: Files parameter set: Support for FileInfo[] (single or multiple files) Directory parameter set: DirectoryInfo with recursive options Removed redundant SingleFile parameter set (FileInfo[] handles both cases) Universal BucketDirectory support: Available for both Files and Directory parameter sets Handles paths like '/Folder1/Folder2/Folder3' or 'Folder1\\Folder2\\Folder3' Automatic multilevel bucket directory creation Enhanced path sanitization with proper forward slash formatting Advanced directory upload features: Recursive parameter with MaxDepth control Flatten parameter to upload all files to bucket root InclusionFilter and ExclusionFilter ScriptBlock parameters Force parameter for overwriting existing objects Comprehensive file collection processing: Multi-file upload with progress tracking Automatic object name inference from filenames Directory structure preservation or flattening Error handling for individual file failures PassThru parameter for returning upload results Enhanced path processing: Cross-platform path handling (Windows/Unix) Automatic directory structure creation in buckets Verbose logging for directory operations Non-critical error handling for directory creation Architecture: Built on custom REST API with thread-safe operations and comprehensive error handling.
2026-07-26 06:48:13 +00:00 · 2025-07-11 17:32:27 -04:00
parent 2c3cd7fb41
commit e94cb35c0c
1 changed files with 423 additions and 27 deletions
@@ -2,6 +2,7 @@ using System;
 using System.Collections;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Management.Automation;
 using PSMinIO.Core.Models;
 using PSMinIO.Utils;
@@ -24,18 +25,11 @@ namespace PSMinIO.Cmdlets
        public string BucketName { get; set; } = string.Empty;

        /// <summary>
-        /// Single file to upload
-        /// </summary>
-        [Parameter(Position = 1, Mandatory = true, ValueFromPipeline = true, ParameterSetName = "SingleFile")]
-        [ValidateNotNull]
-        public FileInfo? File { get; set; }
-
-        /// <summary>
-        /// Array of FileInfo objects to upload
+        /// Array of FileInfo objects to upload (supports single or multiple files)
        /// </summary>
        [Parameter(Position = 1, Mandatory = true, ValueFromPipeline = true, ParameterSetName = "Files")]
        [ValidateNotNull]
-        [Alias("Files")]
+        [Alias("File", "Files")]
        public FileInfo[]? Path { get; set; }

        /// <summary>
@@ -47,16 +41,9 @@ namespace PSMinIO.Cmdlets
        public DirectoryInfo? Directory { get; set; }

        /// <summary>
-        /// Object name in the bucket (defaults to filename) - SingleFile parameter set only
+        /// Optional bucket directory path where files should be uploaded (available for both Files and Directory parameter sets)
        /// </summary>
-        [Parameter(Position = 2, ParameterSetName = "SingleFile")]
-        [ValidateNotNullOrEmpty]
-        public string? ObjectName { get; set; }
-
-        /// <summary>
-        /// Optional bucket directory path where files should be uploaded (Files parameter set only)
-        /// </summary>
-        [Parameter(ParameterSetName = "Files")]
+        [Parameter]
        [ValidateNotNullOrEmpty]
        [Alias("Prefix", "Folder")]
        public string? BucketDirectory { get; set; }
@@ -111,12 +98,6 @@ namespace PSMinIO.Cmdlets
        [Parameter]
        public SwitchParameter Force { get; set; }

-        /// <summary>
-        /// Force overwrite if object already exists
-        /// </summary>
-        [Parameter]
-        public SwitchParameter Force { get; set; }
-
        /// <summary>
        /// Return the upload result information
        /// </summary>
@@ -132,9 +113,6 @@ namespace PSMinIO.Cmdlets

            switch (ParameterSetName)
            {
-                case "SingleFile":
-                    ProcessSingleFile();
-                    break;
                case "Files":
                    ProcessFiles();
                    break;
@@ -146,6 +124,94 @@ namespace PSMinIO.Cmdlets
            }
        }

+        /// <summary>
+        /// Processes file uploads from FileInfo array
+        /// </summary>
+        private void ProcessFiles()
+        {
+            if (Path == null || Path.Length == 0)
+            {
+                WriteWarning("No files provided for upload");
+                return;
+            }
+
+            if (ShouldProcess(BucketName, $"Upload {Path.Length} file(s)"))
+            {
+                ExecuteOperation("UploadFiles", () =>
+                {
+                    // Check if bucket exists first
+                    if (!S3Client.BucketExists(BucketName))
+                    {
+                        throw new InvalidOperationException($"Bucket '{BucketName}' does not exist");
+                    }
+
+                    // Create bucket directory structure if specified
+                    if (!string.IsNullOrWhiteSpace(BucketDirectory))
+                    {
+                        var sanitizedDirectory = SanitizeBucketDirectory(BucketDirectory!);
+                        if (!string.IsNullOrEmpty(sanitizedDirectory))
+                        {
+                            WriteVerboseMessage("Ensuring bucket directory exists: {0}", sanitizedDirectory);
+                            EnsureBucketDirectoryExists(sanitizedDirectory);
+                        }
+                    }
+
+                    UploadFileCollection(Path!);
+                }, $"Bucket: {BucketName}, Files: {Path.Length}");
+            }
+        }
+
+        /// <summary>
+        /// Processes directory upload
+        /// </summary>
+        private void ProcessDirectory()
+        {
+            if (Directory == null || !Directory.Exists)
+            {
+                var errorRecord = new ErrorRecord(
+                    new DirectoryNotFoundException($"Directory not found: {Directory?.FullName}"),
+                    "DirectoryNotFound",
+                    ErrorCategory.ObjectNotFound,
+                    Directory);
+                ThrowTerminatingError(errorRecord);
+                return;
+            }
+
+            if (ShouldProcess(BucketName, $"Upload directory '{Directory.Name}'"))
+            {
+                ExecuteOperation("UploadDirectory", () =>
+                {
+                    // Check if bucket exists first
+                    if (!S3Client.BucketExists(BucketName))
+                    {
+                        throw new InvalidOperationException($"Bucket '{BucketName}' does not exist");
+                    }
+
+                    // Create bucket directory structure if specified
+                    if (!string.IsNullOrWhiteSpace(BucketDirectory))
+                    {
+                        var sanitizedDirectory = SanitizeBucketDirectory(BucketDirectory!);
+                        if (!string.IsNullOrEmpty(sanitizedDirectory))
+                        {
+                            WriteVerboseMessage("Ensuring bucket directory exists: {0}", sanitizedDirectory);
+                            EnsureBucketDirectoryExists(sanitizedDirectory);
+                        }
+                    }
+
+                    // Get files from directory
+                    var files = GetDirectoryFiles();
+                    if (files.Length == 0)
+                    {
+                        WriteWarning($"No files found in directory: {Directory.FullName}");
+                        return;
+                    }
+
+                    WriteVerboseMessage("Found {0} files in directory '{1}'", files.Length, Directory.FullName);
+                    UploadFileCollection(files);
+                }, $"Bucket: {BucketName}, Directory: {Directory.FullName}");
+            }
+        }
+
        /// <summary>
        /// Gets the content type for a file based on its extension
        /// </summary>
@@ -178,5 +244,335 @@ namespace PSMinIO.Cmdlets
                _ => "application/octet-stream"
            };
        }
+
+        /// <summary>
+        /// Gets files from directory based on filters and recursion settings
+        /// </summary>
+        /// <returns>Array of FileInfo objects</returns>
+        private FileInfo[] GetDirectoryFiles()
+        {
+            var searchOption = Recursive.IsPresent ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly;
+            var allFiles = Directory!.GetFiles("*", searchOption);
+
+            // Apply depth filtering if MaxDepth is specified and we're recursive
+            if (Recursive.IsPresent && MaxDepth > 0)
+            {
+                var basePath = Directory.FullName;
+                allFiles = allFiles.Where(f =>
+                {
+                    var relativePath = f.FullName.Substring(basePath.Length).TrimStart('\\', '/');
+                    var depth = relativePath.Split(new char[] { '\\', '/' }, StringSplitOptions.RemoveEmptyEntries).Length - 1;
+                    return depth <= MaxDepth;
+                }).ToArray();
+            }
+
+            // Apply inclusion filter
+            if (InclusionFilter != null)
+            {
+                allFiles = allFiles.Where(f => EvaluateFilter(InclusionFilter, f)).ToArray();
+            }
+
+            // Apply exclusion filter
+            if (ExclusionFilter != null)
+            {
+                allFiles = allFiles.Where(f => !EvaluateFilter(ExclusionFilter, f)).ToArray();
+            }
+
+            return allFiles;
+        }
+
+        /// <summary>
+        /// Evaluates a script block filter against a file
+        /// </summary>
+        /// <param name="filter">Script block filter</param>
+        /// <param name="file">File to evaluate</param>
+        /// <returns>True if file matches filter</returns>
+        private bool EvaluateFilter(ScriptBlock filter, FileInfo file)
+        {
+            try
+            {
+                var result = filter.InvokeWithContext(null, new List<PSVariable> { new PSVariable("_", file) });
+                return result.Count > 0 && LanguagePrimitives.IsTrue(result[0]);
+            }
+            catch (Exception ex)
+            {
+                WriteWarning($"Filter evaluation failed for {file.Name}: {ex.Message}");
+                return false;
+            }
+        }
+
+        /// <summary>
+        /// Uploads a collection of files
+        /// </summary>
+        /// <param name="files">Files to upload</param>
+        private void UploadFileCollection(FileInfo[] files)
+        {
+            // Filter out files that don't exist
+            var validFiles = files.Where(f => f.Exists).ToArray();
+            var skippedCount = files.Length - validFiles.Length;
+
+            if (skippedCount > 0)
+            {
+                WriteWarning($"Skipped {skippedCount} files that do not exist");
+            }
+
+            if (validFiles.Length == 0)
+            {
+                WriteWarning("No valid files found for upload");
+                return;
+            }
+
+            WriteVerboseMessage("Uploading {0} files to bucket '{1}'", validFiles.Length, BucketName);
+
+            // Calculate total size for overall progress
+            var totalSize = validFiles.Sum(f => f.Length);
+            var totalProcessed = 0L;
+
+            var uploadedObjects = new List<MinIOUploadResult>();
+
+            for (int i = 0; i < validFiles.Length; i++)
+            {
+                var fileInfo = validFiles[i];
+                var objectName = GetObjectName(fileInfo);
+
+                try
+                {
+                    WriteVerboseMessage("Uploading file {0}/{1}: {2} -> {3}",
+                        i + 1, validFiles.Length, fileInfo.Name, objectName);
+
+                    var uploadResult = UploadSingleFile(fileInfo, objectName);
+                    if (uploadResult != null)
+                    {
+                        uploadedObjects.Add(uploadResult);
+                        totalProcessed += fileInfo.Length;
+                    }
+
+                    WriteVerboseMessage("Successfully uploaded: {0}", fileInfo.Name);
+                }
+                catch (Exception ex)
+                {
+                    var errorRecord = new ErrorRecord(ex, "FileUploadFailed", ErrorCategory.WriteError, fileInfo);
+                    WriteError(errorRecord);
+                    WriteVerboseMessage("Failed to upload {0}: {1}", fileInfo.Name, ex.Message);
+                }
+            }
+
+            // Always return uploaded objects if PassThru is specified
+            if (PassThru.IsPresent)
+            {
+                foreach (var obj in uploadedObjects)
+                {
+                    WriteObject(obj);
+                }
+            }
+
+            WriteVerboseMessage("Completed uploading {0} files ({1} successful, {2} failed)",
+                validFiles.Length, uploadedObjects.Count, validFiles.Length - uploadedObjects.Count);
+        }
+
+        /// <summary>
+        /// Gets the object name for a file based on the upload context
+        /// </summary>
+        /// <param name="file">File to get object name for</param>
+        /// <returns>Object name to use in MinIO</returns>
+        private string GetObjectName(FileInfo file)
+        {
+            string objectName;
+
+            if (ParameterSetName == "Files")
+            {
+                // For file uploads, use just the filename
+                objectName = file.Name;
+            }
+            else if (ParameterSetName == "Directory")
+            {
+                if (Flatten.IsPresent)
+                {
+                    // Flatten: use just the filename
+                    objectName = file.Name;
+                }
+                else
+                {
+                    // Maintain directory structure relative to the base directory
+                    var relativePath = file.FullName.Substring(Directory!.FullName.Length).TrimStart('\\', '/');
+                    objectName = relativePath.Replace('\\', '/'); // Ensure forward slashes for object storage
+                }
+            }
+            else
+            {
+                objectName = file.Name;
+            }
+
+            // Apply bucket directory prefix if specified (for both parameter sets)
+            if (!string.IsNullOrWhiteSpace(BucketDirectory))
+            {
+                var sanitizedDirectory = SanitizeBucketDirectory(BucketDirectory!);
+                if (!string.IsNullOrEmpty(sanitizedDirectory))
+                {
+                    objectName = $"{sanitizedDirectory}/{objectName}";
+                }
+            }
+
+            return objectName;
+        }
+
+        /// <summary>
+        /// Sanitizes bucket directory path and ensures proper format
+        /// Handles paths like "/Folder1/Folder2/Folder3" or "Folder1\Folder2\Folder3"
+        /// </summary>
+        /// <param name="directory">Raw directory path</param>
+        /// <returns>Sanitized directory path with forward slashes</returns>
+        private string SanitizeBucketDirectory(string directory)
+        {
+            if (string.IsNullOrWhiteSpace(directory))
+                return string.Empty;
+
+            // Remove leading and trailing whitespace and slashes
+            var cleanDirectory = directory.Trim().Trim('/', '\\');
+
+            if (string.IsNullOrEmpty(cleanDirectory))
+                return string.Empty;
+
+            // Split by both forward and back slashes, then clean each part
+            var parts = cleanDirectory.Split(new char[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries);
+
+            // Clean each part: trim whitespace, remove invalid characters, ensure not empty
+            var cleanedParts = parts
+                .Select(part => part.Trim())
+                .Where(part => !string.IsNullOrEmpty(part))
+                .Select(part => part.Replace("\\", "").Replace("/", "")) // Remove any remaining slashes
+                .Where(part => !string.IsNullOrEmpty(part)); // Filter again after cleaning
+
+            // Join with forward slashes (S3/MinIO standard)
+            var result = string.Join("/", cleanedParts);
+
+            WriteVerboseMessage("Sanitized bucket directory: '{0}' -> '{1}'", directory, result);
+            return result;
+        }
+
+        /// <summary>
+        /// Creates bucket directory structure if it doesn't exist
+        /// </summary>
+        /// <param name="directoryPath">Directory path to create</param>
+        private void EnsureBucketDirectoryExists(string directoryPath)
+        {
+            if (string.IsNullOrWhiteSpace(directoryPath))
+                return;
+
+            var parts = directoryPath.Split('/');
+            var currentPath = string.Empty;
+
+            foreach (var part in parts)
+            {
+                currentPath = string.IsNullOrEmpty(currentPath) ? part : $"{currentPath}/{part}";
+                var folderPath = $"{currentPath}/";
+
+                try
+                {
+                    // Check if this directory level already exists by trying to list objects with this prefix
+                    var existingObjects = S3Client.ListObjects(BucketName, folderPath, false);
+                    var folderExists = existingObjects.Any(obj => obj.Name == folderPath);
+
+                    if (!folderExists)
+                    {
+                        WriteVerboseMessage("Creating bucket directory: {0}", folderPath);
+                        try
+                        {
+                            // Create the directory by uploading an empty object with trailing slash
+                            using (var emptyStream = new MemoryStream())
+                            {
+                                S3Client.PutObject(BucketName, folderPath, emptyStream, "application/x-directory", null, null);
+                            }
+                            WriteVerboseMessage("Successfully created bucket directory: {0}", folderPath);
+                        }
+                        catch (Exception createEx)
+                        {
+                            // Directory creation failed, but this is not critical since MinIO creates directories implicitly
+                            WriteVerboseMessage("Directory creation failed (non-critical): {0} - {1}", folderPath, createEx.Message);
+                        }
+                    }
+                }
+                catch (Exception ex)
+                {
+                    // Listing failed, but this is not critical for the upload operation
+                    WriteVerboseMessage("Could not check directory existence (non-critical): {0} - {1}", folderPath, ex.Message);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Uploads a single file and returns the result
+        /// </summary>
+        /// <param name="fileInfo">File to upload</param>
+        /// <param name="objectName">Object name in bucket</param>
+        /// <returns>Upload result</returns>
+        private MinIOUploadResult? UploadSingleFile(FileInfo fileInfo, string objectName)
+        {
+            // Create upload result
+            var uploadResult = new MinIOUploadResult(BucketName, objectName, string.Empty, fileInfo.Length)
+            {
+                SourceFilePath = fileInfo.FullName
+            };
+            uploadResult.MarkStarted();
+
+            try
+            {
+                // Determine content type
+                var fileContentType = ContentType ?? GetContentType(fileInfo.Extension);
+                uploadResult.ContentType = fileContentType;
+
+                // Convert metadata
+                Dictionary<string, string>? metadata = null;
+                if (Metadata != null && Metadata.Count > 0)
+                {
+                    metadata = new Dictionary<string, string>();
+                    foreach (var key in Metadata.Keys)
+                    {
+                        if (key != null && Metadata[key] != null)
+                        {
+                            metadata[key.ToString()!] = Metadata[key]!.ToString()!;
+                        }
+                    }
+                }
+
+                // Upload the file
+                string etag;
+                using (var fileStream = fileInfo.OpenRead())
+                {
+                    etag = S3Client.PutObject(
+                        BucketName,
+                        objectName,
+                        fileStream,
+                        fileContentType,
+                        metadata,
+                        bytesTransferred =>
+                        {
+                            // Only update the result object - no PowerShell calls from background thread
+                            uploadResult.BytesTransferred = bytesTransferred;
+                        });
+                }
+
+                uploadResult.ETag = etag;
+                uploadResult.MarkCompleted();
+
+                var duration = uploadResult.Duration ?? TimeSpan.Zero;
+                var averageSpeed = uploadResult.AverageSpeed ?? 0;
+
+                // Report final progress from main thread
+                var percentage = fileInfo.Length > 0 ? (double)uploadResult.BytesTransferred / fileInfo.Length * 100 : 100;
+                WriteVerboseMessage("Upload progress: {0:F1}% ({1}/{2}) at {3}",
+                    percentage,
+                    SizeFormatter.FormatBytes(uploadResult.BytesTransferred),
+                    SizeFormatter.FormatBytes(fileInfo.Length),
+                    SizeFormatter.FormatSpeed(averageSpeed));
+
+                return uploadResult;
+            }
+            catch (Exception ex)
+            {
+                uploadResult.MarkFailed(ex.Message);
+                throw;
+            }
+        }
    }
 }