Implement comprehensive bucket directory functionality across all upload cmdlets

COMPREHENSIVE BUCKET DIRECTORY SUPPORT:

 ADDED BUCKETDIRECTORY TO MULTIPART UPLOAD:
   Added BucketDirectory parameter to New-MinIOObjectMultipart
   Supports same path formats as regular upload cmdlet
   Automatic directory creation before multipart upload
   Consistent parameter aliases: Prefix, Folder

 NEW DEDICATED FOLDER MANAGEMENT CMDLETS:

 NEW-MINIOBUCKETFOLDER:
   Creates bucket folders with multi-level support
   Handles all path formats: '/Folder', 'Folder1/Folder2/Folder3', '\Folder\Folder1\Folder3'
   Recursive creation by default (-Recursive parameter)
   Force overwrite option (-Force parameter)
   Returns MinIOObjectInfo object for created folder

 REMOVE-MINIOBUCKETFOLDER:
   Removes bucket folders with safety checks
   Recursive deletion option (-Recursive parameter)
   Prevents accidental deletion of non-empty folders
   Force option to bypass safety checks
   Returns MinIOObjectInfo object for removed folder

 SHARED CORE FUNCTIONALITY:

 SANITIZEBUCKETDIRECTORY METHOD:
   Handles multiple input formats consistently
   Supports: '/Folder', 'Folder1/Folder2/Folder3', 'Folder', '\Folder\Folder1\Folder3'
   Removes invalid characters and normalizes to forward slashes
   Comprehensive path cleaning and validation

 ENSUREBUCKETDIRECTORYEXISTS METHOD:
   Recursive directory creation
   Checks existence before creation (efficiency)
   Creates directory markers as empty objects with trailing slash
   Non-critical error handling (MinIO creates implicitly)

 SUPPORTED PATH FORMATS:
   '/Folder'  'Folder'
   'Folder1/Folder2/Folder3'  'Folder1/Folder2/Folder3'
   'Folder'  'Folder'
   '\Folder\Folder1\Folder3'  'Folder/Folder1/Folder3'
   '  /Folder1//Folder2/  '  'Folder1/Folder2'

 USAGE EXAMPLES:
  # Create nested folder structure
  New-MinIOBucketFolder -BucketName 'mybucket' -FolderPath 'Documents/Projects/2024'

  # Upload with bucket directory
  New-MinIOObjectMultipart -BucketName 'mybucket' -FilePath 'file.zip' -BucketDirectory 'Archives/2024'

  # Remove folder recursively
  Remove-MinIOBucketFolder -BucketName 'mybucket' -FolderPath 'OldData' -Recursive

Complete bucket directory ecosystem with consistent functionality!
This commit is contained in:
PSMinIO Developer
2025-07-14 23:09:31 -04:00
parent da378e5d9b
commit dd8b63cfb5
4 changed files with 540 additions and 2 deletions
Binary file not shown.
+212
View File
@@ -0,0 +1,212 @@
using System;
using System.IO;
using System.Linq;
using System.Management.Automation;
using PSMinIO.Core.Models;
using PSMinIO.Utils;
namespace PSMinIO.Cmdlets
{
/// <summary>
/// Creates bucket folder structure with support for multi-level paths
/// </summary>
[Cmdlet(VerbsCommon.New, "MinIOBucketFolder", SupportsShouldProcess = true)]
[OutputType(typeof(MinIOObjectInfo))]
public class NewMinIOBucketFolderCmdlet : MinIOBaseCmdlet
{
/// <summary>
/// Name of the bucket where the folder should be created
/// </summary>
[Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true)]
[ValidateNotNullOrEmpty]
public string BucketName { get; set; } = string.Empty;
/// <summary>
/// Folder path to create (supports multiple formats: "/Folder", "Folder1/Folder2/Folder3", "Folder", "\Folder\Folder1\Folder3")
/// </summary>
[Parameter(Position = 1, Mandatory = true, ValueFromPipeline = true, ValueFromPipelineByPropertyName = true)]
[ValidateNotNullOrEmpty]
[Alias("Path", "Directory", "Prefix")]
public string FolderPath { get; set; } = string.Empty;
/// <summary>
/// Create parent folders if they don't exist (recursive creation)
/// </summary>
[Parameter]
public SwitchParameter Recursive { get; set; } = true;
/// <summary>
/// Overwrite existing folder without prompting
/// </summary>
[Parameter]
public SwitchParameter Force { get; set; }
/// <summary>
/// Processes the cmdlet
/// </summary>
protected override void ProcessRecord()
{
ValidateBucketName(BucketName);
// Sanitize the folder path
var sanitizedPath = SanitizeBucketDirectory(FolderPath);
if (string.IsNullOrEmpty(sanitizedPath))
{
var errorRecord = new ErrorRecord(
new ArgumentException("Invalid folder path after sanitization"),
"InvalidFolderPath",
ErrorCategory.InvalidArgument,
FolderPath);
ThrowTerminatingError(errorRecord);
}
var folderObjectName = $"{sanitizedPath}/";
if (ShouldProcess(folderObjectName, $"Create folder in bucket '{BucketName}'"))
{
var result = ExecuteOperation("CreateBucketFolder", () =>
{
// Check if bucket exists first
if (!S3Client.BucketExists(BucketName))
{
throw new InvalidOperationException($"Bucket '{BucketName}' does not exist");
}
WriteVerboseMessage("Creating bucket folder: {0} in bucket: {1}", sanitizedPath, BucketName);
// Check if folder already exists
var existingObjects = S3Client.ListObjects(BucketName, folderObjectName, false);
var folderExists = existingObjects.Any(obj => obj.Name == folderObjectName);
if (folderExists && !Force.IsPresent)
{
throw new InvalidOperationException($"Folder '{sanitizedPath}' already exists in bucket '{BucketName}'. Use -Force to overwrite.");
}
// Create folder structure recursively if requested
if (Recursive.IsPresent)
{
EnsureBucketDirectoryExists(sanitizedPath);
}
else
{
// Create only the final folder
CreateSingleFolder(folderObjectName);
}
// Return the created folder as MinIOObjectInfo
var folderInfo = new MinIOObjectInfo
{
Name = folderObjectName,
BucketName = BucketName,
Size = 0,
LastModified = DateTime.UtcNow,
ContentType = "application/x-directory",
ETag = "",
StartTime = DateTime.UtcNow,
CompletionTime = DateTime.UtcNow
};
WriteVerboseMessage("Successfully created bucket folder: {0}", sanitizedPath);
return folderInfo;
}, $"Bucket: {BucketName}, Folder: {sanitizedPath}");
WriteObject(result);
}
}
/// <summary>
/// Sanitizes bucket directory path and ensures proper format
/// Handles paths like "/Folder1/Folder2/Folder3" or "Folder1\Folder2\Folder3"
/// </summary>
/// <param name="directory">Raw directory path</param>
/// <returns>Sanitized directory path with forward slashes</returns>
private string SanitizeBucketDirectory(string directory)
{
if (string.IsNullOrWhiteSpace(directory))
return string.Empty;
// Remove leading and trailing whitespace and slashes
var cleanDirectory = directory.Trim().Trim('/', '\\');
if (string.IsNullOrEmpty(cleanDirectory))
return string.Empty;
// Split by both forward and back slashes, then clean each part
var parts = cleanDirectory.Split(new char[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries);
// Clean each part: trim whitespace, remove invalid characters, ensure not empty
var cleanedParts = parts
.Select(part => part.Trim())
.Where(part => !string.IsNullOrEmpty(part))
.Select(part => part.Replace("\\", "").Replace("/", "")) // Remove any remaining slashes
.Where(part => !string.IsNullOrEmpty(part)); // Filter again after cleaning
// Join with forward slashes (S3/MinIO standard)
var result = string.Join("/", cleanedParts);
WriteVerboseMessage("Sanitized bucket directory: '{0}' -> '{1}'", directory, result);
return result;
}
/// <summary>
/// Creates bucket directory structure recursively if it doesn't exist
/// </summary>
/// <param name="directoryPath">Directory path to create</param>
private void EnsureBucketDirectoryExists(string directoryPath)
{
if (string.IsNullOrWhiteSpace(directoryPath))
return;
var parts = directoryPath.Split('/');
var currentPath = string.Empty;
foreach (var part in parts)
{
currentPath = string.IsNullOrEmpty(currentPath) ? part : $"{currentPath}/{part}";
var folderPath = $"{currentPath}/";
try
{
// Check if this directory level already exists
var existingObjects = S3Client.ListObjects(BucketName, folderPath, false);
var folderExists = existingObjects.Any(obj => obj.Name == folderPath);
if (!folderExists)
{
CreateSingleFolder(folderPath);
}
}
catch (Exception ex)
{
WriteVerboseMessage("Could not check directory existence: {0} - {1}", folderPath, ex.Message);
// Try to create anyway
CreateSingleFolder(folderPath);
}
}
}
/// <summary>
/// Creates a single folder by uploading an empty object with trailing slash
/// </summary>
/// <param name="folderPath">Folder path with trailing slash</param>
private void CreateSingleFolder(string folderPath)
{
try
{
WriteVerboseMessage("Creating bucket directory: {0}", folderPath);
using (var emptyStream = new MemoryStream())
{
S3Client.PutObject(BucketName, folderPath, emptyStream, "application/x-directory", null, null);
}
WriteVerboseMessage("Successfully created bucket directory: {0}", folderPath);
}
catch (Exception createEx)
{
WriteVerboseMessage("Directory creation failed: {0} - {1}", folderPath, createEx.Message);
throw new InvalidOperationException($"Failed to create folder '{folderPath}': {createEx.Message}", createEx);
}
}
}
}
+121 -2
View File
@@ -55,6 +55,14 @@ namespace PSMinIO.Cmdlets
[ValidateRange(1, 10)]
public int MaxParallelUploads { get; set; } = 4;
/// <summary>
/// Optional bucket directory path where files should be uploaded
/// </summary>
[Parameter]
[ValidateNotNullOrEmpty]
[Alias("Prefix", "Folder")]
public string? BucketDirectory { get; set; }
/// <summary>
/// Content type of the object (auto-detected if not specified)
/// </summary>
@@ -106,6 +114,16 @@ namespace PSMinIO.Cmdlets
// Determine object name
var effectiveObjectName = ObjectName ?? FilePath.Name;
// Apply bucket directory prefix if specified
if (!string.IsNullOrWhiteSpace(BucketDirectory))
{
var sanitizedDirectory = SanitizeBucketDirectory(BucketDirectory!);
if (!string.IsNullOrEmpty(sanitizedDirectory))
{
effectiveObjectName = $"{sanitizedDirectory}/{effectiveObjectName}";
}
}
// Auto-detect content type if not specified
var effectiveContentType = ContentType ?? GetContentType(FilePath.Extension);
@@ -124,9 +142,26 @@ namespace PSMinIO.Cmdlets
{
var result = ExecuteOperation("MultipartUpload", () =>
{
WriteVerboseMessage("Starting multipart upload: {0} -> {1}/{2}",
// Check if bucket exists first
if (!S3Client.BucketExists(BucketName))
{
throw new InvalidOperationException($"Bucket '{BucketName}' does not exist");
}
// Create bucket directory structure if specified
if (!string.IsNullOrWhiteSpace(BucketDirectory))
{
var sanitizedDirectory = SanitizeBucketDirectory(BucketDirectory!);
if (!string.IsNullOrEmpty(sanitizedDirectory))
{
WriteVerboseMessage("Ensuring bucket directory exists: {0}", sanitizedDirectory);
EnsureBucketDirectoryExists(sanitizedDirectory);
}
}
WriteVerboseMessage("Starting multipart upload: {0} -> {1}/{2}",
FilePath.FullName, BucketName, effectiveObjectName);
WriteVerboseMessage("File size: {0}, Chunk size: {1}, Max parallel uploads: {2}",
WriteVerboseMessage("File size: {0}, Chunk size: {1}, Max parallel uploads: {2}",
SizeFormatter.FormatBytes(FilePath.Length), SizeFormatter.FormatBytes(ChunkSize), MaxParallelUploads);
if (!string.IsNullOrEmpty(ResumeUploadId))
@@ -208,5 +243,89 @@ namespace PSMinIO.Cmdlets
_ => "application/octet-stream"
};
}
/// <summary>
/// Sanitizes bucket directory path and ensures proper format
/// Handles paths like "/Folder1/Folder2/Folder3" or "Folder1\Folder2\Folder3"
/// </summary>
/// <param name="directory">Raw directory path</param>
/// <returns>Sanitized directory path with forward slashes</returns>
private string SanitizeBucketDirectory(string directory)
{
if (string.IsNullOrWhiteSpace(directory))
return string.Empty;
// Remove leading and trailing whitespace and slashes
var cleanDirectory = directory.Trim().Trim('/', '\\');
if (string.IsNullOrEmpty(cleanDirectory))
return string.Empty;
// Split by both forward and back slashes, then clean each part
var parts = cleanDirectory.Split(new char[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries);
// Clean each part: trim whitespace, remove invalid characters, ensure not empty
var cleanedParts = parts
.Select(part => part.Trim())
.Where(part => !string.IsNullOrEmpty(part))
.Select(part => part.Replace("\\", "").Replace("/", "")) // Remove any remaining slashes
.Where(part => !string.IsNullOrEmpty(part)); // Filter again after cleaning
// Join with forward slashes (S3/MinIO standard)
var result = string.Join("/", cleanedParts);
WriteVerboseMessage("Sanitized bucket directory: '{0}' -> '{1}'", directory, result);
return result;
}
/// <summary>
/// Creates bucket directory structure if it doesn't exist
/// </summary>
/// <param name="directoryPath">Directory path to create</param>
private void EnsureBucketDirectoryExists(string directoryPath)
{
if (string.IsNullOrWhiteSpace(directoryPath))
return;
var parts = directoryPath.Split('/');
var currentPath = string.Empty;
foreach (var part in parts)
{
currentPath = string.IsNullOrEmpty(currentPath) ? part : $"{currentPath}/{part}";
var folderPath = $"{currentPath}/";
try
{
// Check if this directory level already exists by trying to list objects with this prefix
var existingObjects = S3Client.ListObjects(BucketName, folderPath, false);
var folderExists = existingObjects.Any(obj => obj.Name == folderPath);
if (!folderExists)
{
WriteVerboseMessage("Creating bucket directory: {0}", folderPath);
try
{
// Create the directory by uploading an empty object with trailing slash
using (var emptyStream = new MemoryStream())
{
S3Client.PutObject(BucketName, folderPath, emptyStream, "application/x-directory", null, null);
}
WriteVerboseMessage("Successfully created bucket directory: {0}", folderPath);
}
catch (Exception createEx)
{
// Directory creation failed, but this is not critical since MinIO creates directories implicitly
WriteVerboseMessage("Directory creation failed (non-critical): {0} - {1}", folderPath, createEx.Message);
}
}
}
catch (Exception ex)
{
// Listing failed, but this is not critical for the upload operation
WriteVerboseMessage("Could not check directory existence (non-critical): {0} - {1}", folderPath, ex.Message);
}
}
}
}
}
@@ -0,0 +1,207 @@
using System;
using System.Linq;
using System.Management.Automation;
using PSMinIO.Core.Models;
using PSMinIO.Utils;
namespace PSMinIO.Cmdlets
{
/// <summary>
/// Removes bucket folder structure with support for recursive deletion
/// </summary>
[Cmdlet(VerbsCommon.Remove, "MinIOBucketFolder", SupportsShouldProcess = true, ConfirmImpact = ConfirmImpact.High)]
[OutputType(typeof(MinIOObjectInfo))]
public class RemoveMinIOBucketFolderCmdlet : MinIOBaseCmdlet
{
/// <summary>
/// Name of the bucket containing the folder to remove
/// </summary>
[Parameter(Position = 0, Mandatory = true, ValueFromPipelineByPropertyName = true)]
[ValidateNotNullOrEmpty]
public string BucketName { get; set; } = string.Empty;
/// <summary>
/// Folder path to remove (supports multiple formats: "/Folder", "Folder1/Folder2/Folder3", "Folder", "\Folder\Folder1\Folder3")
/// </summary>
[Parameter(Position = 1, Mandatory = true, ValueFromPipeline = true, ValueFromPipelineByPropertyName = true)]
[ValidateNotNullOrEmpty]
[Alias("Path", "Directory", "Prefix")]
public string FolderPath { get; set; } = string.Empty;
/// <summary>
/// Remove folder and all its contents recursively
/// </summary>
[Parameter]
public SwitchParameter Recursive { get; set; }
/// <summary>
/// Remove folder without prompting for confirmation
/// </summary>
[Parameter]
public SwitchParameter Force { get; set; }
/// <summary>
/// Processes the cmdlet
/// </summary>
protected override void ProcessRecord()
{
ValidateBucketName(BucketName);
// Sanitize the folder path
var sanitizedPath = SanitizeBucketDirectory(FolderPath);
if (string.IsNullOrEmpty(sanitizedPath))
{
var errorRecord = new ErrorRecord(
new ArgumentException("Invalid folder path after sanitization"),
"InvalidFolderPath",
ErrorCategory.InvalidArgument,
FolderPath);
ThrowTerminatingError(errorRecord);
}
var folderObjectName = $"{sanitizedPath}/";
if (ShouldProcess(folderObjectName, $"Remove folder from bucket '{BucketName}'"))
{
var result = ExecuteOperation("RemoveBucketFolder", () =>
{
// Check if bucket exists first
if (!S3Client.BucketExists(BucketName))
{
throw new InvalidOperationException($"Bucket '{BucketName}' does not exist");
}
WriteVerboseMessage("Removing bucket folder: {0} from bucket: {1}", sanitizedPath, BucketName);
// Check if folder exists
var existingObjects = S3Client.ListObjects(BucketName, folderObjectName, false);
var folderExists = existingObjects.Any(obj => obj.Name == folderObjectName);
if (!folderExists)
{
WriteVerboseMessage("Folder '{0}' does not exist in bucket '{1}'", sanitizedPath, BucketName);
return null;
}
// Get folder info before deletion
var folderInfo = new MinIOObjectInfo
{
Name = folderObjectName,
BucketName = BucketName,
Size = 0,
LastModified = DateTime.UtcNow,
ContentType = "application/x-directory",
ETag = "",
StartTime = DateTime.UtcNow
};
if (Recursive.IsPresent)
{
// Remove folder and all contents recursively
RemoveFolderRecursively(sanitizedPath);
}
else
{
// Check if folder has contents
var folderContents = S3Client.ListObjects(BucketName, sanitizedPath + "/", true);
var hasContents = folderContents.Any(obj => obj.Name != folderObjectName);
if (hasContents && !Force.IsPresent)
{
throw new InvalidOperationException($"Folder '{sanitizedPath}' is not empty. Use -Recursive to remove contents or -Force to ignore this check.");
}
// Remove only the folder marker
RemoveSingleFolder(folderObjectName);
}
folderInfo.CompletionTime = DateTime.UtcNow;
WriteVerboseMessage("Successfully removed bucket folder: {0}", sanitizedPath);
return folderInfo;
}, $"Bucket: {BucketName}, Folder: {sanitizedPath}");
if (result != null)
{
WriteObject(result);
}
}
}
/// <summary>
/// Sanitizes bucket directory path and ensures proper format
/// </summary>
private string SanitizeBucketDirectory(string directory)
{
if (string.IsNullOrWhiteSpace(directory))
return string.Empty;
// Remove leading and trailing whitespace and slashes
var cleanDirectory = directory.Trim().Trim('/', '\\');
if (string.IsNullOrEmpty(cleanDirectory))
return string.Empty;
// Split by both forward and back slashes, then clean each part
var parts = cleanDirectory.Split(new char[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries);
// Clean each part: trim whitespace, remove invalid characters, ensure not empty
var cleanedParts = parts
.Select(part => part.Trim())
.Where(part => !string.IsNullOrEmpty(part))
.Select(part => part.Replace("\\", "").Replace("/", "")) // Remove any remaining slashes
.Where(part => !string.IsNullOrEmpty(part)); // Filter again after cleaning
// Join with forward slashes (S3/MinIO standard)
var result = string.Join("/", cleanedParts);
WriteVerboseMessage("Sanitized bucket directory: '{0}' -> '{1}'", directory, result);
return result;
}
/// <summary>
/// Removes folder and all its contents recursively
/// </summary>
private void RemoveFolderRecursively(string folderPath)
{
WriteVerboseMessage("Removing folder recursively: {0}", folderPath);
// List all objects with the folder prefix
var objectsToDelete = S3Client.ListObjects(BucketName, folderPath + "/", true);
foreach (var obj in objectsToDelete)
{
try
{
WriteVerboseMessage("Removing object: {0}", obj.Name);
S3Client.DeleteObject(BucketName, obj.Name);
}
catch (Exception ex)
{
WriteVerboseMessage("Failed to remove object '{0}': {1}", obj.Name, ex.Message);
throw new InvalidOperationException($"Failed to remove object '{obj.Name}': {ex.Message}", ex);
}
}
WriteVerboseMessage("Successfully removed folder recursively: {0}", folderPath);
}
/// <summary>
/// Removes a single folder marker object
/// </summary>
private void RemoveSingleFolder(string folderObjectName)
{
try
{
WriteVerboseMessage("Removing folder marker: {0}", folderObjectName);
S3Client.DeleteObject(BucketName, folderObjectName);
WriteVerboseMessage("Successfully removed folder marker: {0}", folderObjectName);
}
catch (Exception ex)
{
WriteVerboseMessage("Failed to remove folder marker '{0}': {1}", folderObjectName, ex.Message);
throw new InvalidOperationException($"Failed to remove folder '{folderObjectName}': {ex.Message}", ex);
}
}
}
}