fix(ecstore): handle stalled recovery reads and listings (#3790)

* fix(ecstore): handle stalled recovery reads and listings

* fix(rio): start HTTP stall timeout on read

* fix(ecstore): handle stalled reads and partial lists

* fix(ecstore): retire stalled shards and list errors

* fix(ecstore): preserve list merge lookahead entries

* fix(ecstore): bound zero-copy shard reads

* fix(ecstore): hedge stalled shard reads

* fix(ecstore): retire abandoned shard reads

* fix(ecstore): include part identity in metadata quorum

* fix(ecstore): validate heal shard sources

* fix(ecstore): verify reconstructed read shards

* chore(ecstore): log slow object read stages

* fix(heal): throttle auto heal during recovery

* fix(scanner): yield to foreground reads

* fix(scanner): track streaming object reads

* fix(ecstore): avoid false read heal fanout

* fix(ecstore): verify codec streaming reconstruction sources

* fix(ecstore): preserve quorum progress on slow shards

* fix(storage): restore read timeout facade

* fix(ecstore): retain fallback readers after quorum

* chore: allow decode helper argument lists

---------

Co-authored-by: overtrue <anzhengchao@gmail.com>

crates/scanner/src/lib.rs

@@ -63,11 +63,53 @@ pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
 use std::sync::atomic::{AtomicU64, Ordering};
 
 static SCANNER_ACTIVE_WORK_UNITS: AtomicU64 = AtomicU64::new(0);
+static SCANNER_FOREGROUND_READ_ACTIVITY: AtomicU64 = AtomicU64::new(0);
+static SCANNER_FOREGROUND_STREAM_READS: AtomicU64 = AtomicU64::new(0);
 
 pub fn current_scanner_activity() -> u64 {
     SCANNER_ACTIVE_WORK_UNITS.load(Ordering::Relaxed)
 }
 
+pub fn set_foreground_read_activity(active: usize) {
+    let active = u64::try_from(active).unwrap_or(u64::MAX);
+    SCANNER_FOREGROUND_READ_ACTIVITY.store(active, Ordering::Relaxed);
+}
+
+pub fn current_foreground_read_activity() -> u64 {
+    SCANNER_FOREGROUND_READ_ACTIVITY
+        .load(Ordering::Relaxed)
+        .max(SCANNER_FOREGROUND_STREAM_READS.load(Ordering::Relaxed))
+}
+
+#[derive(Debug)]
+pub struct ForegroundReadGuard;
+
+impl ForegroundReadGuard {
+    pub fn new() -> Self {
+        SCANNER_FOREGROUND_STREAM_READS.fetch_add(1, Ordering::Relaxed);
+        Self
+    }
+}
+
+impl Default for ForegroundReadGuard {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Drop for ForegroundReadGuard {
+    fn drop(&mut self) {
+        let _ =
+            SCANNER_FOREGROUND_STREAM_READS.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| current.checked_sub(1));
+    }
+}
+
+#[cfg(test)]
+pub(crate) fn reset_foreground_read_activity_for_test() {
+    SCANNER_FOREGROUND_READ_ACTIVITY.store(0, Ordering::Relaxed);
+    SCANNER_FOREGROUND_STREAM_READS.store(0, Ordering::Relaxed);
+}
+
 pub(crate) struct ScannerActivityGuard;
 
 impl ScannerActivityGuard {
@@ -342,3 +384,36 @@ impl<T> ScannerObjectIO for T where
         >
 {
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serial_test::serial;
+
+    #[test]
+    #[serial]
+    fn foreground_read_guard_tracks_stream_lifetime() {
+        reset_foreground_read_activity_for_test();
+        assert_eq!(current_foreground_read_activity(), 0);
+
+        {
+            let _guard = ForegroundReadGuard::new();
+            assert_eq!(current_foreground_read_activity(), 1);
+        }
+
+        assert_eq!(current_foreground_read_activity(), 0);
+    }
+
+    #[test]
+    #[serial]
+    fn foreground_read_activity_keeps_larger_signal() {
+        reset_foreground_read_activity_for_test();
+        let _guard = ForegroundReadGuard::new();
+
+        set_foreground_read_activity(3);
+        assert_eq!(current_foreground_read_activity(), 3);
+
+        set_foreground_read_activity(0);
+        assert_eq!(current_foreground_read_activity(), 1);
+    }
+}