fix(ecstore): handle stalled recovery reads and listings (#3790)

* fix(ecstore): handle stalled recovery reads and listings

* fix(rio): start HTTP stall timeout on read

* fix(ecstore): handle stalled reads and partial lists

* fix(ecstore): retire stalled shards and list errors

* fix(ecstore): preserve list merge lookahead entries

* fix(ecstore): bound zero-copy shard reads

* fix(ecstore): hedge stalled shard reads

* fix(ecstore): retire abandoned shard reads

* fix(ecstore): include part identity in metadata quorum

* fix(ecstore): validate heal shard sources

* fix(ecstore): verify reconstructed read shards

* chore(ecstore): log slow object read stages

* fix(heal): throttle auto heal during recovery

* fix(scanner): yield to foreground reads

* fix(scanner): track streaming object reads

* fix(ecstore): avoid false read heal fanout

* fix(ecstore): verify codec streaming reconstruction sources

* fix(ecstore): preserve quorum progress on slow shards

* fix(storage): restore read timeout facade

* fix(ecstore): retain fallback readers after quorum

* chore: allow decode helper argument lists

---------

Co-authored-by: overtrue <anzhengchao@gmail.com>

crates/ecstore/src/erasure/coding/decode_reader.rs

@@ -479,9 +479,12 @@ mod tests {
     use crate::erasure::codec::bridge::{
         CodecStreamingDecodeEngine, ErasureDecodeEngine, LegacyEcDecodeEngine, RustfsCodecDecodeEngine,
     };
-    use crate::erasure::coding::Erasure;
+    use crate::erasure::coding::decode::ParallelReader;
+    use crate::erasure::coding::{BitrotReader, Erasure};
     use crate::set_disk::shard_source::{ShardSlot, StripeReadState};
+    use rustfs_utils::HashAlgorithm;
     use std::collections::VecDeque;
+    use std::io::Cursor;
     use std::sync::Arc;
     use std::sync::atomic::{AtomicUsize, Ordering};
     use tokio::io::AsyncReadExt;
@@ -611,6 +614,46 @@ mod tests {
         assert_eq!(decoded, data);
     }
 
+    #[tokio::test]
+    async fn erasure_decode_reader_rejects_inconsistent_reconstruction_sources() {
+        const DATA_SHARDS: usize = 2;
+        const PARITY_SHARDS: usize = 2;
+        const BLOCK_SIZE: usize = 64;
+
+        let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
+        let data = (0u8..64u8).collect::<Vec<_>>();
+        let encoded = erasure.encode_data(&data).expect("test stripe should encode");
+        let mut corrupt_parity = encoded[DATA_SHARDS].to_vec();
+        corrupt_parity[0] ^= 0x80;
+
+        let source = VecStripeSource {
+            stripes: VecDeque::from([StripeReadState::from_parts(
+                vec![
+                    None,
+                    Some(encoded[1].to_vec()),
+                    Some(corrupt_parity),
+                    Some(encoded[DATA_SHARDS + 1].to_vec()),
+                ],
+                Vec::new(),
+                DATA_SHARDS,
+            )]),
+            read_quorum: DATA_SHARDS,
+            read_count: None,
+        };
+        let engine = LegacyEcDecodeEngine::new(erasure);
+        let mut reader = ErasureDecodeReader::new(source, engine, data.len()).expect("reader should be constructed");
+        let mut decoded = Vec::new();
+
+        let err = reader
+            .read_to_end(&mut decoded)
+            .await
+            .expect_err("streaming reader must reject inconsistent reconstruction sources");
+
+        assert_eq!(err.kind(), ErrorKind::InvalidData);
+        assert!(err.to_string().contains("inconsistent read source shards"));
+        assert!(decoded.is_empty());
+    }
+
     #[tokio::test]
     async fn erasure_decode_reader_rustfs_engine_matches_legacy_with_missing_data() {
         let erasure = Erasure::new(4, 2, 32);
@@ -641,6 +684,55 @@ mod tests {
         assert!(decoded.is_empty());
     }
 
+    #[tokio::test]
+    async fn erasure_decode_reader_verifying_parallel_source_rejects_inconsistent_reconstruction_sources() {
+        const DATA_SHARDS: usize = 2;
+        const PARITY_SHARDS: usize = 2;
+        const BLOCK_SIZE: usize = 64;
+
+        let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
+        let data = (0u8..64u8).collect::<Vec<_>>();
+        let shard_size = erasure.shard_size();
+        let encoded = erasure.encode_data(&data).expect("test stripe should encode");
+        let mut corrupt_parity = encoded[DATA_SHARDS].to_vec();
+        corrupt_parity[0] ^= 0x80;
+        let readers = vec![
+            None,
+            Some(BitrotReader::new(
+                Cursor::new(encoded[1].to_vec()),
+                shard_size,
+                HashAlgorithm::None,
+                false,
+            )),
+            Some(BitrotReader::new(Cursor::new(corrupt_parity), shard_size, HashAlgorithm::None, false)),
+            Some(BitrotReader::new(
+                Cursor::new(encoded[DATA_SHARDS + 1].to_vec()),
+                shard_size,
+                HashAlgorithm::None,
+                false,
+            )),
+        ];
+        let source = ParallelReader::new_with_metrics_path_and_reconstruction_verification(
+            readers,
+            erasure.clone(),
+            0,
+            data.len(),
+            Some(GET_OBJECT_PATH_CODEC_STREAMING),
+        );
+        let engine = LegacyEcDecodeEngine::new(erasure);
+        let mut reader = ErasureDecodeReader::new(source, engine, data.len()).expect("reader should be constructed");
+        let mut decoded = Vec::new();
+
+        let err = reader
+            .read_to_end(&mut decoded)
+            .await
+            .expect_err("streaming reader must reject inconsistent reconstruction sources");
+
+        assert_eq!(err.kind(), ErrorKind::InvalidData);
+        assert!(err.to_string().contains("inconsistent read source shards"));
+        assert!(decoded.is_empty());
+    }
+
     #[tokio::test]
     async fn erasure_decode_reader_codec_streaming_engine_enum_matches_legacy() {
         let erasure = Erasure::new(4, 2, 32);