fix(ecstore): handle stalled recovery reads and listings (#3790)

* fix(ecstore): handle stalled recovery reads and listings

* fix(rio): start HTTP stall timeout on read

* fix(ecstore): handle stalled reads and partial lists

* fix(ecstore): retire stalled shards and list errors

* fix(ecstore): preserve list merge lookahead entries

* fix(ecstore): bound zero-copy shard reads

* fix(ecstore): hedge stalled shard reads

* fix(ecstore): retire abandoned shard reads

* fix(ecstore): include part identity in metadata quorum

* fix(ecstore): validate heal shard sources

* fix(ecstore): verify reconstructed read shards

* chore(ecstore): log slow object read stages

* fix(heal): throttle auto heal during recovery

* fix(scanner): yield to foreground reads

* fix(scanner): track streaming object reads

* fix(ecstore): avoid false read heal fanout

* fix(ecstore): verify codec streaming reconstruction sources

* fix(ecstore): preserve quorum progress on slow shards

* fix(storage): restore read timeout facade

* fix(ecstore): retain fallback readers after quorum

* chore: allow decode helper argument lists

---------

Co-authored-by: overtrue <anzhengchao@gmail.com>

crates/scanner/src/scanner_io.rs

@@ -366,6 +366,10 @@ fn scanner_concurrency_limit(configured: usize, available: usize) -> usize {
         return 0;
     }
 
+    if crate::current_foreground_read_activity() > 0 {
+        return 1;
+    }
+
     if configured == 0 {
         available
     } else {
@@ -1656,25 +1660,53 @@ mod tests {
     }
 
     #[test]
+    #[serial]
     fn scanner_concurrency_limit_preserves_available_when_unconfigured() {
+        crate::reset_foreground_read_activity_for_test();
         assert_eq!(scanner_concurrency_limit(0, 4), 4);
     }
 
     #[test]
+    #[serial]
     fn scanner_concurrency_limit_caps_to_configured_value() {
+        crate::reset_foreground_read_activity_for_test();
         assert_eq!(scanner_concurrency_limit(2, 4), 2);
     }
 
     #[test]
+    #[serial]
     fn scanner_concurrency_limit_never_exceeds_available_work() {
+        crate::reset_foreground_read_activity_for_test();
         assert_eq!(scanner_concurrency_limit(8, 4), 4);
     }
 
     #[test]
+    #[serial]
     fn scanner_concurrency_limit_handles_no_available_work() {
+        crate::reset_foreground_read_activity_for_test();
         assert_eq!(scanner_concurrency_limit(2, 0), 0);
     }
 
+    #[test]
+    #[serial]
+    fn scanner_concurrency_limit_yields_to_foreground_reads() {
+        crate::reset_foreground_read_activity_for_test();
+        crate::set_foreground_read_activity(8);
+        assert_eq!(scanner_concurrency_limit(0, 4), 1);
+        assert_eq!(scanner_concurrency_limit(3, 4), 1);
+        crate::reset_foreground_read_activity_for_test();
+    }
+
+    #[test]
+    #[serial]
+    fn scanner_concurrency_limit_yields_to_streaming_reads() {
+        crate::reset_foreground_read_activity_for_test();
+        let _guard = crate::ForegroundReadGuard::new();
+
+        assert_eq!(scanner_concurrency_limit(0, 4), 1);
+        assert_eq!(scanner_concurrency_limit(3, 4), 1);
+    }
+
     #[test]
     fn decrement_atomic_usize_saturates_at_zero() {
         let counter = AtomicUsize::new(1);