fix(ecstore): handle stalled recovery reads and listings (#3790)

* fix(ecstore): handle stalled recovery reads and listings

* fix(rio): start HTTP stall timeout on read

* fix(ecstore): handle stalled reads and partial lists

* fix(ecstore): retire stalled shards and list errors

* fix(ecstore): preserve list merge lookahead entries

* fix(ecstore): bound zero-copy shard reads

* fix(ecstore): hedge stalled shard reads

* fix(ecstore): retire abandoned shard reads

* fix(ecstore): include part identity in metadata quorum

* fix(ecstore): validate heal shard sources

* fix(ecstore): verify reconstructed read shards

* chore(ecstore): log slow object read stages

* fix(heal): throttle auto heal during recovery

* fix(scanner): yield to foreground reads

* fix(scanner): track streaming object reads

* fix(ecstore): avoid false read heal fanout

* fix(ecstore): verify codec streaming reconstruction sources

* fix(ecstore): preserve quorum progress on slow shards

* fix(storage): restore read timeout facade

* fix(ecstore): retain fallback readers after quorum

* chore: allow decode helper argument lists

---------

Co-authored-by: overtrue <anzhengchao@gmail.com>

crates/ecstore/src/erasure/coding/erasure.rs

@@ -182,6 +182,34 @@ impl LegacyReedSolomonEncoder {
         self.encode_parity(shards)
     }
 
+    fn verify(&self, shards: &[&[u8]]) -> io::Result<bool> {
+        let expected_shards = self.data_shards + self.parity_shards;
+        if shards.len() != expected_shards {
+            return Err(io::Error::other(format!(
+                "invalid shard count: got {}, expected {}",
+                shards.len(),
+                expected_shards
+            )));
+        }
+        if shards.iter().all(|shard| shard.is_empty()) {
+            return Ok(true);
+        }
+
+        let mut expected = shards.iter().map(|shard| Some(shard.to_vec())).collect::<Vec<_>>();
+        self.encode_parity(&mut expected)?;
+
+        for index in self.data_shards..expected_shards {
+            let Some(expected_parity) = expected[index].as_ref() else {
+                return Err(io::Error::other(format!("missing parity shard {index} after verification encode")));
+            };
+            if expected_parity.as_slice() != shards[index] {
+                return Ok(false);
+            }
+        }
+
+        Ok(true)
+    }
+
     fn encode_parity(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
         encode_parity_shards(shards, self.data_shards, self.parity_shards, |shards| self.encode(shards))
     }
@@ -257,6 +285,19 @@ impl ReedSolomonEncoder {
         self.encode_parity(shards)
     }
 
+    pub fn verify(&self, shards: &[&[u8]]) -> io::Result<bool> {
+        if shards.iter().all(|shard| shard.is_empty()) {
+            return Ok(true);
+        }
+
+        if let Some(ref rs) = self.encoder {
+            rs.verify(shards)
+                .map_err(|e| io::Error::other(format!("Reed-Solomon verify failed: {e:?}")))
+        } else {
+            Ok(true)
+        }
+    }
+
     fn encode_parity(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
         encode_parity_shards(shards, self.data_shards, self.parity_shards, |shards| self.encode(shards))
     }
@@ -656,6 +697,92 @@ impl Erasure {
         Ok(())
     }
 
+    pub(crate) fn decode_data_with_reconstruction_verification(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
+        let missing_data_source = shards.iter().take(self.data_shards).any(|shard| shard.is_none());
+        let available_shards = shards.iter().filter(|shard| shard.is_some()).count();
+        let source_parity = if missing_data_source && available_shards > self.data_shards {
+            shards
+                .iter()
+                .enumerate()
+                .skip(self.data_shards)
+                .filter_map(|(index, shard)| shard.as_ref().map(|shard| (index, shard.clone())))
+                .collect::<Vec<_>>()
+        } else {
+            Vec::new()
+        };
+
+        if source_parity.is_empty() {
+            return self.decode_data(shards);
+        }
+
+        self.decode_data_and_parity(shards)?;
+        for (index, source) in source_parity {
+            let Some(rebuilt) = shards[index].as_ref() else {
+                return Err(io::Error::new(
+                    io::ErrorKind::InvalidData,
+                    "missing rebuilt parity shard after read verification",
+                ));
+            };
+            if rebuilt != &source {
+                warn!(
+                    shard_index = index,
+                    data_shards = self.data_shards,
+                    parity_shards = self.parity_shards,
+                    "erasure decode rejected inconsistent read source shards"
+                );
+                return Err(io::Error::new(io::ErrorKind::InvalidData, "inconsistent read source shards"));
+            }
+        }
+
+        Ok(())
+    }
+
+    pub(crate) fn verify_data_and_parity(&self, shards: &[Option<Vec<u8>>]) -> io::Result<bool> {
+        let expected_shards = self.total_shard_count();
+        if shards.len() != expected_shards {
+            return Err(io::Error::other(format!(
+                "invalid shard count: got {}, expected {}",
+                shards.len(),
+                expected_shards
+            )));
+        }
+        if self.parity_shards == 0 {
+            return Ok(true);
+        }
+
+        let mut shard_refs = Vec::with_capacity(expected_shards);
+        let mut shard_len = None;
+        for (index, shard) in shards.iter().enumerate() {
+            let shard = shard
+                .as_deref()
+                .ok_or_else(|| io::Error::other(format!("missing shard {index} for data/parity verification")))?;
+            if let Some(expected_len) = shard_len {
+                if shard.len() != expected_len {
+                    return Err(io::Error::other(format!(
+                        "inconsistent shard length at index {index}: got {}, expected {}",
+                        shard.len(),
+                        expected_len
+                    )));
+                }
+            } else {
+                shard_len = Some(shard.len());
+            }
+            shard_refs.push(shard);
+        }
+
+        if self.uses_legacy {
+            if let Some(encoder) = self.legacy_encoder.as_ref() {
+                encoder.verify(&shard_refs)
+            } else {
+                Err(io::Error::other("parity_shards > 0, uses_legacy but legacy_encoder is None"))
+            }
+        } else if let Some(encoder) = self.encoder.as_ref() {
+            encoder.verify(&shard_refs)
+        } else {
+            Err(io::Error::other("parity_shards > 0, but encoder is None"))
+        }
+    }
+
     /// Get the total number of shards (data + parity).
     pub fn total_shard_count(&self) -> usize {
         self.data_shards + self.parity_shards