gsadmin/PSMinIO
1
0
Fork 0
mirror of https://github.com/Grace-Solutions/PSMinIO.git synced 2026-08-07 03:33:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Clean up redundant logging and add intelligent chunk logging

Browse Source 
� LOGGING IMPROVEMENTS:

✅ REMOVED REDUNDANT UPLOAD ID LOGGING:
  • Eliminated duplicate 'Beginning multipart upload - Upload ID' message
  • Now shows single 'Initiated multipart upload - Upload ID' message
  • Cleaner, less verbose output while maintaining visibility

� INTELLIGENT CHUNK LOGGING:
  • Added selective chunk start logging (only for chunks >= 32MB)
  • Added milestone-based completion logging:
    - Every 10th part completed
    - Large chunks (>= 32MB)
    - Final part completion
  • Reduces log noise while maintaining progress visibility

� LOGGING STRATEGY:
  • Start: Log for significant chunks (32MB+) to show activity
  • Progress: Log every 10th part + large chunks + final part
  • Avoids flooding logs with small chunk messages
  • Maintains visibility into upload progress

✅ BENEFITS:
  • Cleaner verbose output without redundancy
  • Appropriate level of detail for chunk processing
  • Better signal-to-noise ratio in logs
  • Still shows progress for long-running operations

Perfect balance of visibility and conciseness!
This commit is contained in:
PSMinIO Developer
2025-07-14 21:58:44 -04:00
parent 33cefbafe9
commit 75925fcf14
2 changed files with 17 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Module/PSMinIO/bin/PSMinIO.dll
BIN
View File
Binary file not shown.
src/Core/S3/MultipartUploadManager.cs
+17 -7
View File
@@ -85,15 +85,14 @@ namespace PSMinIO.Core.S3
if (string.IsNullOrEmpty(uploadId))
{
uploadId = InitiateMultipartUpload(bucketName, objectName, metadata);
_progressCollector.QueueVerboseMessage("Initiated multipart upload with ID: {0}", uploadId);
_progressCollector.QueueVerboseMessage("Initiated multipart upload - Upload ID: {0}", uploadId!);
}
else
{
_progressCollector.QueueVerboseMessage("Resuming multipart upload with ID: {0}", uploadId);
_progressCollector.QueueVerboseMessage("Resuming multipart upload - Upload ID: {0}", uploadId!);
}
// Log upload details before starting
_progressCollector.QueueVerboseMessage("Beginning multipart upload - Upload ID: {0}", uploadId!);
// Log upload configuration
_progressCollector.QueueVerboseMessage("Upload configuration - Chunk size: {0}, Total parts: {1}, Max parallel: {2}",
SizeFormatter.FormatBytes(effectiveChunkSize), totalParts, _maxParallelUploads);
@@ -122,9 +121,16 @@ namespace PSMinIO.Core.S3
semaphore.Wait();
try
{
// Log chunk start (not too verbose - only for larger chunks)
if (partSize >= 32 * 1024 * 1024) // Log for chunks >= 32MB
{
_progressCollector.QueueVerboseMessage("Starting upload of part {0}/{1} ({2})",
partNum, totalParts, SizeFormatter.FormatBytes(partSize));
}
var partInfo = UploadPart(bucketName, objectName, uploadId!, fileInfo,
partNum, partOffset, partSize);
parts.TryAdd(partNum, partInfo);
// Update progress
@@ -138,8 +144,12 @@ namespace PSMinIO.Core.S3
$"Part {partNum}/{totalParts} - {SizeFormatter.FormatBytes(currentUploaded)}/{SizeFormatter.FormatBytes(totalSize)} at {SizeFormatter.FormatSpeed(speed)}",
(int)fileProgress, 1);
_progressCollector.QueueVerboseMessage("Completed part {0}/{1} ({2})",
partNum, totalParts, SizeFormatter.FormatBytes(partSize));
// Log completion for larger chunks or milestone parts
if (partSize >= 32 * 1024 * 1024 || partNum % 10 == 0 || partNum == totalParts)
{
_progressCollector.QueueVerboseMessage("Completed part {0}/{1} ({2})",
partNum, totalParts, SizeFormatter.FormatBytes(partSize));
}
// Process progress updates immediately
_progressCollector.ProcessQueuedUpdates();