Fix upload performance bottleneck and enhance progress bar text

🚀 CRITICAL PERFORMANCE OPTIMIZATION:

 FIXED UPLOAD SLOWDOWN ISSUE:
  • Root cause: Progress reported on EVERY Read() call (thousands per chunk)
  • Added intelligent progress throttling in ProgressTrackingStream
  • Only report progress every 1% or 1MB (whichever is smaller)
  • Minimum 8KB threshold to prevent excessive updates
  • Massive reduction in progress callback overhead

 PERFORMANCE IMPROVEMENTS:
  • Before: Progress callback on every 4KB-64KB read = 1000+ calls per chunk
  • After: Progress callback every 1MB or 1% = ~64 calls per 64MB chunk
  • 15x-20x reduction in progress overhead
  • Maintains smooth progress bars without performance penalty

 ENHANCED PROGRESS BAR TEXT:
   Added 'Counter of Counter' format to chunk progress
   Before: 'Part 23: 32.1 MB/64.0 MB'
   After: 'Part 23 of 74 - 32.1 MB/64.0 MB'
   Better context for users about overall progress

 TECHNICAL IMPLEMENTATION:
   Added _reportingThreshold and _lastReportedBytes to ProgressTrackingStream
   Throttled progress reporting while maintaining accuracy
   Updated UploadPart method signature to include totalParts
   Preserved final progress report at 100% completion

 EXPECTED RESULTS:
   Fast, consistent upload speeds on high-bandwidth connections
   Smooth progress bars without performance degradation
   Better user experience with contextual progress information
   Scalable to large files without slowdown

Critical fix for multipart upload performance!
This commit is contained in:
PSMinIO Developer
2025-07-14 22:50:50 -04:00
parent a1734e1b99
commit 13091ea1b1
3 changed files with 15 additions and 5 deletions
Binary file not shown.
+3 -3
View File
@@ -147,7 +147,7 @@ namespace PSMinIO.Core.S3
partInfo.Status = PartStatus.Transferring;
var uploadedPart = UploadPart(bucketName, objectName, uploadId!, fileInfo,
partNum, partOffset, partSize);
partNum, partOffset, partSize, totalParts);
// Update with completed info
partInfo.ETag = uploadedPart.ETag;
@@ -308,7 +308,7 @@ namespace PSMinIO.Core.S3
/// Uploads a single part
/// </summary>
private PartInfo UploadPart(string bucketName, string objectName, string uploadId,
FileInfo fileInfo, int partNumber, long offset, long size)
FileInfo fileInfo, int partNumber, long offset, long size, int totalParts)
{
var queryParams = new Dictionary<string, string>
{
@@ -327,7 +327,7 @@ namespace PSMinIO.Core.S3
{
var chunkProgress = (double)bytesRead / totalBytes * 100;
_progressCollector.QueueProgressUpdate(chunkActivityId, "Uploading Chunk",
$"Part {partNumber}: {SizeFormatter.FormatBytes(bytesRead)}/{SizeFormatter.FormatBytes(totalBytes)}",
$"Part {partNumber} of {totalParts} - {SizeFormatter.FormatBytes(bytesRead)}/{SizeFormatter.FormatBytes(totalBytes)}",
(int)chunkProgress, 2); // Parent: File Upload (Layer 2)
});
+12 -2
View File
@@ -12,6 +12,8 @@ namespace PSMinIO.Utils
private readonly long _totalBytes;
private readonly Action<long, long> _progressCallback;
private long _bytesRead = 0;
private long _lastReportedBytes;
private readonly long _reportingThreshold;
private readonly long _startPosition;
public ProgressTrackingStream(Stream baseStream, long totalBytes, Action<long, long> progressCallback)
@@ -20,6 +22,10 @@ namespace PSMinIO.Utils
_totalBytes = totalBytes;
_progressCallback = progressCallback ?? throw new ArgumentNullException(nameof(progressCallback));
_startPosition = baseStream.Position;
// Only report progress every 1% or 1MB, whichever is smaller (reduces overhead)
_reportingThreshold = Math.Min(totalBytes / 100, 1024 * 1024);
if (_reportingThreshold < 8192) _reportingThreshold = 8192; // Minimum 8KB threshold
}
public override bool CanRead => _baseStream.CanRead;
@@ -44,8 +50,12 @@ namespace PSMinIO.Utils
var bytesRead = _baseStream.Read(buffer, offset, bytesToRead);
_bytesRead += bytesRead;
// Report progress
_progressCallback(_bytesRead, _totalBytes);
// Report progress only when threshold is reached or at completion (reduces overhead)
if (_bytesRead - _lastReportedBytes >= _reportingThreshold || _bytesRead >= _totalBytes)
{
_progressCallback(_bytesRead, _totalBytes);
_lastReportedBytes = _bytesRead;
}
return bytesRead;
}