fix(ecstore): harden rebalance data movement (#3234)

* fix(ecstore): harden rebalance data movement

* fix(ecstore): preserve failed rebalance status

* fix(ecstore): avoid rebalance walkdir total timeout

* fix(ecstore): retry rebalance listing timeouts

* fix(ecstore): restrict data movement resume target

* refactor(ecstore): simplify multipart movement target

* fix(ecstore): restore resume target checks

* perf(ecstore): speed up rebalance bucket merges

* fix: keep rebalance listing alive on transient failures

---------

Co-authored-by: houseme <housemecn@gmail.com>

crates/scanner/src/scanner_folder.rs

@@ -147,6 +147,13 @@ fn should_yield_after_object(object_count: u64, yield_every: u64) -> bool {
     yield_every > 0 && object_count.is_multiple_of(yield_every)
 }
 
+const SCANNER_FAILED_OBJECT_LOG_INITIAL_LIMIT: usize = 16;
+const SCANNER_FAILED_OBJECT_LOG_EVERY: usize = 1024;
+
+fn should_log_failed_object(failed_objects: usize) -> bool {
+    failed_objects <= SCANNER_FAILED_OBJECT_LOG_INITIAL_LIMIT || failed_objects.is_multiple_of(SCANNER_FAILED_OBJECT_LOG_EVERY)
+}
+
 fn record_scanner_ilm_action_if_queued(metrics: &Metrics, count: u64, queued: bool) -> bool {
     if queued {
         metrics.record_scanner_ilm_action(count);
@@ -1286,8 +1293,12 @@ impl FolderScanner {
                             into.failed_objects += 1;
                             self.record_failed(&item.path);
 
-                            // Only log non-skip errors to avoid noise
-                            warn!("scan_folder: failed to get size for item {}: {}", item.path, e);
+                            if should_log_failed_object(into.failed_objects) {
+                                warn!(
+                                    failed_objects = into.failed_objects,
+                                    "scan_folder: failed to get size for item {}: {}", item.path, e
+                                );
+                            }
                         }
 
                         timer.sleep().await;
@@ -2975,4 +2986,14 @@ mod tests {
         assert!(result.is_ok(), "expected symlinked child directory to be ignored");
         assert_eq!(into.failed_objects, 0, "expected ignored symlink not to count as a failed object");
     }
+
+    #[test]
+    fn test_should_log_failed_object_samples_after_initial_limit() {
+        assert!(should_log_failed_object(1));
+        assert!(should_log_failed_object(SCANNER_FAILED_OBJECT_LOG_INITIAL_LIMIT));
+        assert!(!should_log_failed_object(SCANNER_FAILED_OBJECT_LOG_INITIAL_LIMIT + 1));
+        assert!(should_log_failed_object(SCANNER_FAILED_OBJECT_LOG_EVERY));
+        assert!(!should_log_failed_object(SCANNER_FAILED_OBJECT_LOG_EVERY + 1));
+        assert!(should_log_failed_object(SCANNER_FAILED_OBJECT_LOG_EVERY * 2));
+    }
 }

crates/scanner/src/scanner_io.rs

@@ -945,13 +945,11 @@ impl ScannerIODisk for Disk {
         let data = match self.read_metadata(&item.bucket, &item.object_path()).await {
             Ok(data) => data,
             Err(e) => {
-                warn!(
-                    "Failed to read metadata: {e}, bucket={}, object_path={}",
+                return Err(StorageError::other(format!(
+                    "failed to read metadata: {e}, bucket={}, object_path={}",
                     &item.bucket,
                     &item.object_path()
-                );
-
-                return Err(StorageError::other("failed to read metadata".to_string()));
+                )));
             }
         };