test(ecstore): add validation suite coverage gates (#4378)

crates/ecstore/src/erasure/coding/encode.rs

@@ -709,15 +709,109 @@ mod tests {
         }
     }
 
+    #[derive(Clone, Default)]
+    struct FailingWriteWriter;
+
+    impl AsyncWrite for FailingWriteWriter {
+        fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll<std::io::Result<usize>> {
+            Poll::Ready(Err(std::io::Error::other("injected write failure")))
+        }
+
+        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+    }
+
+    #[derive(Clone, Default)]
+    struct ShortWriteWriter;
+
+    impl AsyncWrite for ShortWriteWriter {
+        fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
+            Poll::Ready(Ok(buf.len().saturating_sub(1)))
+        }
+
+        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+    }
+
+    #[derive(Clone, Default)]
+    struct ShutdownFailWriter {
+        buffered: Vec<u8>,
+    }
+
+    impl AsyncWrite for ShutdownFailWriter {
+        fn poll_write(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
+            self.buffered.extend_from_slice(buf);
+            Poll::Ready(Ok(buf.len()))
+        }
+
+        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            Poll::Ready(Err(std::io::Error::other("injected shutdown failure")))
+        }
+    }
+
+    fn bitrot_writer<W>(writer: W, shard_size: usize) -> BitrotWriterWrapper
+    where
+        W: AsyncWrite + Send + Sync + Unpin + 'static,
+    {
+        BitrotWriterWrapper::new(CustomWriter::new_tokio_writer(writer), shard_size, HashAlgorithm::HighwayHash256S)
+    }
+
+    #[tokio::test]
+    async fn multi_writer_short_write_fails_before_shutdown() {
+        let mut writers = vec![Some(bitrot_writer(ShortWriteWriter, 16))];
+        let err = {
+            let mut writer = MultiWriter::new(&mut writers, 1);
+            writer
+                .write(vec![Bytes::from_static(b"short-write payload")])
+                .await
+                .expect_err("short writes must fail the shard writer")
+        };
+
+        assert!(err.to_string().contains("Failed to write data"));
+        assert!(writers[0].is_none(), "short-write shard must be removed before commit");
+    }
+
+    #[tokio::test]
+    async fn drain_queued_inflight_bytes_consumes_pending_blocks() {
+        let (tx, mut rx) = mpsc::channel(2);
+        tx.send(vec![Bytes::from_static(b"queued")]).await.unwrap();
+        drop(tx);
+
+        drain_queued_inflight_bytes(&mut rx).await;
+
+        assert!(rx.recv().await.is_none());
+    }
+
+    #[tokio::test]
+    async fn drain_queued_batched_inflight_bytes_consumes_pending_batches() {
+        let (tx, mut rx) = mpsc::channel(2);
+        tx.send(vec![vec![Bytes::from_static(b"queued")]]).await.unwrap();
+        drop(tx);
+
+        drain_queued_batched_inflight_bytes(&mut rx).await;
+
+        assert!(rx.recv().await.is_none());
+    }
+
     #[tokio::test]
     async fn encode_shutdowns_writers_after_small_shards() {
         let committed = Arc::new(Mutex::new(Vec::new()));
         let writer = DeferredCommitWriter::new(committed.clone());
-        let mut writers = vec![Some(BitrotWriterWrapper::new(
-            CustomWriter::new_tokio_writer(writer),
-            16,
-            HashAlgorithm::HighwayHash256S,
-        ))];
+        let mut writers = vec![Some(bitrot_writer(writer, 16))];
 
         let erasure = Arc::new(Erasure::new(1, 0, 16));
         let reader = tokio::io::BufReader::new(Cursor::new(b"small payload".to_vec()));
@@ -727,6 +821,87 @@ mod tests {
         assert!(!committed.lock().unwrap().is_empty());
     }
 
+    #[tokio::test]
+    async fn encode_streaming_write_quorum_failure_aborts_and_reports_error() {
+        const DATA_SHARDS: usize = 2;
+        const PARITY_SHARDS: usize = 2;
+        const BLOCK_SIZE: usize = 64;
+
+        let committed = Arc::new(Mutex::new(Vec::new()));
+        let mut writers = vec![
+            Some(bitrot_writer(DeferredCommitWriter::new(committed.clone()), BLOCK_SIZE / DATA_SHARDS)),
+            Some(bitrot_writer(FailingWriteWriter, BLOCK_SIZE / DATA_SHARDS)),
+            None,
+            None,
+        ];
+
+        let payload = vec![3u8; BLOCK_SIZE * 8];
+        let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
+        let reader = tokio::io::BufReader::new(Cursor::new(payload));
+        let err = erasure
+            .encode(reader, &mut writers, DATA_SHARDS)
+            .await
+            .expect_err("streaming encode must fail when write quorum is unavailable");
+
+        assert!(err.to_string().contains("Failed to write data"));
+    }
+
+    #[tokio::test]
+    async fn encode_inline_small_write_quorum_failure_does_not_commit_partial_data() {
+        const DATA_SHARDS: usize = 2;
+        const PARITY_SHARDS: usize = 2;
+        const BLOCK_SIZE: usize = 64;
+
+        let committed = Arc::new(Mutex::new(Vec::new()));
+        let mut writers = vec![
+            Some(bitrot_writer(DeferredCommitWriter::new(committed.clone()), BLOCK_SIZE / DATA_SHARDS)),
+            Some(bitrot_writer(FailingWriteWriter, BLOCK_SIZE / DATA_SHARDS)),
+            None,
+            None,
+        ];
+
+        let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
+        let reader = tokio::io::BufReader::new(Cursor::new(b"write quorum failure payload".to_vec()));
+        let err = erasure
+            .encode_inline_small(reader, &mut writers, DATA_SHARDS)
+            .await
+            .expect_err("write quorum failure must fail the inline encode");
+
+        assert!(err.to_string().contains("Failed to write data"));
+        assert!(
+            committed.lock().expect("committed buffer should be lockable").is_empty(),
+            "successful writer must not be committed when write quorum fails before shutdown"
+        );
+    }
+
+    #[tokio::test]
+    async fn encode_inline_small_shutdown_quorum_failure_is_reported() {
+        const DATA_SHARDS: usize = 2;
+        const PARITY_SHARDS: usize = 2;
+        const BLOCK_SIZE: usize = 64;
+
+        let committed = Arc::new(Mutex::new(Vec::new()));
+        let mut writers = vec![
+            Some(bitrot_writer(DeferredCommitWriter::new(committed.clone()), BLOCK_SIZE / DATA_SHARDS)),
+            Some(bitrot_writer(ShutdownFailWriter::default(), BLOCK_SIZE / DATA_SHARDS)),
+            Some(bitrot_writer(ShutdownFailWriter::default(), BLOCK_SIZE / DATA_SHARDS)),
+            Some(bitrot_writer(ShutdownFailWriter::default(), BLOCK_SIZE / DATA_SHARDS)),
+        ];
+
+        let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
+        let reader = tokio::io::BufReader::new(Cursor::new(b"shutdown quorum failure payload".to_vec()));
+        let err = erasure
+            .encode_inline_small(reader, &mut writers, DATA_SHARDS)
+            .await
+            .expect_err("shutdown quorum failure must fail the inline encode");
+
+        assert!(err.to_string().contains("Failed to shutdown writers"));
+        assert!(
+            !committed.lock().expect("committed buffer should be lockable").is_empty(),
+            "the successful writer should have committed before shutdown quorum failure was reported"
+        );
+    }
+
     #[tokio::test]
     async fn encode_returns_unexpected_eof_for_truncated_limited_reader() {
         let committed = Arc::new(Mutex::new(Vec::new()));
@@ -773,11 +948,7 @@ mod tests {
     async fn encode_works_on_current_thread_runtime() {
         let committed = Arc::new(Mutex::new(Vec::new()));
         let writer = DeferredCommitWriter::new(committed);
-        let mut writers = vec![Some(BitrotWriterWrapper::new(
-            CustomWriter::new_tokio_writer(writer),
-            16,
-            HashAlgorithm::HighwayHash256S,
-        ))];
+        let mut writers = vec![Some(bitrot_writer(writer, 16))];
 
         let erasure = Arc::new(Erasure::new(1, 0, 16));
         let reader = tokio::io::BufReader::new(Cursor::new(b"current-thread payload".to_vec()));
@@ -786,6 +957,78 @@ mod tests {
         assert_eq!(written, b"current-thread payload".len());
     }
 
+    #[tokio::test]
+    async fn encode_batched_writes_full_and_tail_batches() {
+        const DATA_SHARDS: usize = 2;
+        const PARITY_SHARDS: usize = 2;
+        const TOTAL_SHARDS: usize = DATA_SHARDS + PARITY_SHARDS;
+        const BLOCK_SIZE: usize = 32;
+
+        let committed: Vec<Arc<Mutex<Vec<u8>>>> = (0..TOTAL_SHARDS).map(|_| Arc::new(Mutex::new(Vec::new()))).collect();
+        let mut writers: Vec<Option<BitrotWriterWrapper>> = committed
+            .iter()
+            .map(|c| Some(bitrot_writer(DeferredCommitWriter::new(c.clone()), BLOCK_SIZE / DATA_SHARDS)))
+            .collect();
+
+        let payload = vec![7u8; BLOCK_SIZE * 5 + 3];
+        let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
+        let reader = tokio::io::BufReader::new(Cursor::new(payload.clone()));
+        let (_reader, total) = erasure
+            .encode_batched(reader, &mut writers, DATA_SHARDS)
+            .await
+            .expect("batched encode should write full and tail batches");
+
+        assert_eq!(total, payload.len());
+        for (index, committed) in committed.iter().enumerate() {
+            assert!(
+                !committed.lock().expect("committed buffer should be lockable").is_empty(),
+                "shard {index} should receive committed batched data"
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn encode_batched_write_quorum_failure_aborts_and_reports_error() {
+        const DATA_SHARDS: usize = 2;
+        const PARITY_SHARDS: usize = 2;
+        const BLOCK_SIZE: usize = 32;
+
+        let committed = Arc::new(Mutex::new(Vec::new()));
+        let mut writers = vec![
+            Some(bitrot_writer(DeferredCommitWriter::new(committed.clone()), BLOCK_SIZE / DATA_SHARDS)),
+            Some(bitrot_writer(FailingWriteWriter, BLOCK_SIZE / DATA_SHARDS)),
+            None,
+            None,
+        ];
+
+        let payload = vec![9u8; BLOCK_SIZE * 8];
+        let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
+        let reader = tokio::io::BufReader::new(Cursor::new(payload));
+        let err = erasure
+            .encode_batched(reader, &mut writers, DATA_SHARDS)
+            .await
+            .expect_err("batched encode must fail when write quorum is unavailable");
+
+        assert!(err.to_string().contains("Failed to write data"));
+    }
+
+    #[tokio::test]
+    async fn encode_batched_rejects_zero_block_size() {
+        let committed = Arc::new(Mutex::new(Vec::new()));
+        let writer = DeferredCommitWriter::new(committed);
+        let mut writers = vec![Some(bitrot_writer(writer, 16))];
+
+        let erasure = Arc::new(Erasure::new(1, 0, 0));
+        let reader = tokio::io::BufReader::new(Cursor::new(b"payload".to_vec()));
+        let err = erasure
+            .encode_batched(reader, &mut writers, 1)
+            .await
+            .expect_err("zero block size must be rejected");
+
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
+        assert!(err.to_string().contains("block_size"));
+    }
+
     /// encode_inline_small: empty reader returns (reader, 0) without writing to any shard.
     #[tokio::test]
     async fn encode_inline_small_empty_stream_returns_zero() {

crates/ecstore/src/erasure/coding/erasure.rs

@@ -1056,6 +1056,80 @@ mod tests {
         }
     }
 
+    #[test]
+    fn decode_data_rejects_missing_data_above_parity_budget() {
+        for uses_legacy in [false, true] {
+            let erasure = Erasure::new_with_options(3, 2, 64, uses_legacy);
+            let data = b"read restore must fail when too many data shards are gone";
+            let encoded = erasure.encode_data(data).expect("encode should succeed");
+            let mut shards = optional_shards(&encoded);
+            shards[0] = None;
+            shards[1] = None;
+            shards[2] = None;
+
+            let err = erasure
+                .decode_data(&mut shards)
+                .expect_err("decode_data must fail when missing data shards exceed parity");
+
+            assert!(!err.to_string().is_empty());
+            assert!(shards.iter().take(erasure.data_shards).any(Option::is_none));
+        }
+    }
+
+    #[test]
+    fn decode_data_and_parity_rejects_wrong_shard_count() {
+        for uses_legacy in [false, true] {
+            let erasure = Erasure::new_with_options(4, 2, 128, uses_legacy);
+            let data = b"wrong shard count should not be accepted as a restored object";
+            let encoded = erasure.encode_data(data).expect("encode should succeed");
+            let mut shards = optional_shards(&encoded);
+            let _ = shards.pop();
+
+            let err = erasure
+                .decode_data_and_parity(&mut shards)
+                .expect_err("decode_data_and_parity must reject wrong shard count");
+
+            assert!(!err.to_string().is_empty());
+        }
+    }
+
+    #[test]
+    fn decode_data_with_verification_rejects_corrupt_surplus_parity() {
+        let erasure = Erasure::new(3, 2, 128);
+        let data = b"verified reads must fail closed on stale surplus parity";
+        let encoded = erasure.encode_data(data).expect("encode should succeed");
+        let mut shards = optional_shards(&encoded);
+        shards[1] = None;
+        shards[erasure.data_shards].as_mut().expect("parity shard should be present")[0] ^= 0x7d;
+
+        let err = erasure
+            .decode_data_with_reconstruction_verification(&mut shards)
+            .expect_err("verified decode must reject inconsistent parity");
+
+        assert_eq!(err.kind(), io::ErrorKind::InvalidData);
+    }
+
+    #[test]
+    fn verify_data_and_parity_rejects_missing_and_mismatched_shards() {
+        let erasure = Erasure::new(4, 2, 128);
+        let data = b"verification must reject incomplete or malformed shard sets";
+        let encoded = erasure.encode_data(data).expect("encode should succeed");
+        let mut shards = optional_shards(&encoded);
+
+        shards[0] = None;
+        let missing = erasure
+            .verify_data_and_parity(&shards)
+            .expect_err("verification must reject missing shards");
+        assert!(missing.to_string().contains("missing shard"));
+
+        let mut mismatched = optional_shards(&encoded);
+        let _ = mismatched[0].as_mut().expect("data shard should be present").pop();
+        let mismatch = erasure
+            .verify_data_and_parity(&mismatched)
+            .expect_err("verification must reject mismatched shard lengths");
+        assert!(mismatch.to_string().contains("inconsistent shard length"));
+    }
+
     #[test]
     fn decode_data_and_parity_leaves_complete_shards_unchanged() {
         let erasure = Erasure::new(4, 2, 128);

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -3427,6 +3427,43 @@ impl SetDisks {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::io::Cursor;
+    use tokio::io::AsyncReadExt;
+
+    fn metadata_test_fileinfo(object: &str) -> FileInfo {
+        let mut fi = FileInfo::new(object, 2, 2);
+        fi.volume = "bucket".to_string();
+        fi.name = object.to_string();
+        fi.size = 1;
+        fi.erasure.index = 1;
+        fi.metadata.insert("etag".to_string(), "etag-1".to_string());
+        fi.add_object_part(1, "part-etag-1".to_string(), 1, None, 1, None, None);
+        fi
+    }
+
+    fn read_part_test_part(number: usize, etag: &str) -> ObjectPartInfo {
+        ObjectPartInfo {
+            number,
+            etag: etag.to_string(),
+            ..Default::default()
+        }
+    }
+
+    fn read_part_test_error(number: usize, error: &str) -> ObjectPartInfo {
+        ObjectPartInfo {
+            number,
+            error: Some(error.to_string()),
+            ..Default::default()
+        }
+    }
+
+    fn failed_read_repair_submitter(_request: rustfs_common::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture {
+        Box::pin(async { ReadRepairAdmissionOutcome::Failed("injected submit failure".to_string()) })
+    }
+
+    fn accepted_read_repair_submitter(_request: rustfs_common::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture {
+        Box::pin(async { ReadRepairAdmissionOutcome::Response(HealAdmissionResult::Accepted) })
+    }
 
     #[test]
     fn dangling_delete_grace_defaults_to_one_hour() {
@@ -3444,4 +3481,224 @@ mod tests {
             assert!(dangling_delete_grace().is_zero());
         });
     }
+
+    #[tokio::test]
+    async fn multipart_codec_streaming_reader_zero_buffer_is_noop() {
+        let reader = tokio::io::BufReader::new(Cursor::new(b"payload".to_vec()));
+        let mut reader = MultipartCodecStreamingReader::new(vec![Box::new(reader)]);
+        let mut out = [];
+
+        let read = reader
+            .read(&mut out)
+            .await
+            .expect("zero-length read should not poll inner readers");
+
+        assert_eq!(read, 0);
+        assert_eq!(reader.readers.len(), 1);
+    }
+
+    #[test]
+    fn metadata_fanout_observation_classifies_invalid_and_ignored_results() {
+        let invalid = MetadataFanoutObservation::from_file_info(&FileInfo::default(), Duration::from_millis(7));
+        assert_eq!(invalid.outcome, GET_METADATA_RESPONSE_ERROR);
+        assert!(!invalid.valid);
+        assert!(!invalid.ignored);
+
+        let ignored = MetadataFanoutObservation::from_error(&DiskError::DiskNotFound, Duration::from_millis(9));
+        assert_eq!(ignored.outcome, GET_METADATA_RESPONSE_DISK_NOT_FOUND);
+        assert!(!ignored.valid);
+        assert!(ignored.ignored);
+
+        let corrupt = MetadataFanoutObservation::from_error(&DiskError::FileCorrupt, Duration::from_millis(11));
+        assert_eq!(corrupt.outcome, GET_METADATA_RESPONSE_CORRUPT);
+        assert!(!corrupt.ignored);
+    }
+
+    #[test]
+    fn metadata_fanout_diagnostics_reports_counts_and_latency_edges() {
+        let diagnostics = MetadataFanoutDiagnostics::new(
+            Duration::from_millis(40),
+            vec![
+                MetadataFanoutObservation {
+                    outcome: GET_METADATA_RESPONSE_VALID,
+                    elapsed: Duration::from_millis(30),
+                    valid: true,
+                    ignored: false,
+                },
+                MetadataFanoutObservation {
+                    outcome: GET_METADATA_RESPONSE_IGNORED,
+                    elapsed: Duration::from_millis(10),
+                    valid: false,
+                    ignored: true,
+                },
+                MetadataFanoutObservation {
+                    outcome: GET_METADATA_RESPONSE_ERROR,
+                    elapsed: Duration::from_millis(20),
+                    valid: false,
+                    ignored: false,
+                },
+            ],
+        );
+
+        assert_eq!(diagnostics.total_responses(), 3);
+        assert_eq!(diagnostics.valid_responses(), 1);
+        assert_eq!(diagnostics.ignored_responses(), 1);
+        assert_eq!(diagnostics.error_responses(), 2);
+        assert_eq!(diagnostics.first_response_latency(), Some(Duration::from_millis(10)));
+        assert_eq!(diagnostics.first_valid_response_latency(), Some(Duration::from_millis(30)));
+        assert_eq!(diagnostics.slowest_response_latency(), Some(Duration::from_millis(30)));
+        assert_eq!(diagnostics.quorum_candidate_latency(0), Some(Duration::ZERO));
+        assert_eq!(diagnostics.quorum_candidate_latency(1), Some(Duration::from_millis(30)));
+        assert_eq!(diagnostics.quorum_candidate_latency(2), None);
+    }
+
+    #[test]
+    fn metadata_quorum_accumulator_counts_invalid_metadata_and_ignored_errors() {
+        let mut accumulator = MetadataQuorumAccumulator::new(4, 2, true);
+
+        accumulator.observe_file_info(&FileInfo::default());
+        accumulator.observe_error(&DiskError::DiskNotFound);
+
+        assert_eq!(accumulator.hard_errors, 1);
+        assert_eq!(accumulator.ignored_errors, 1);
+        assert_eq!(accumulator.final_miss_reason(), GET_METADATA_EARLY_STOP_REASON_ERROR);
+    }
+
+    #[test]
+    fn metadata_quorum_accumulator_candidate_quorum_handles_zero_parity_and_invalid_candidates() {
+        let accumulator = MetadataQuorumAccumulator::new(4, 0, true);
+        let candidate = metadata_test_fileinfo("object");
+        assert_eq!(accumulator.candidate_read_quorum(&candidate), Some(4));
+        assert_eq!(accumulator.missing_response_quorum(), 4);
+
+        let accumulator = MetadataQuorumAccumulator::new(4, 2, true);
+        let mut deleted = candidate.clone();
+        deleted.deleted = true;
+        assert_eq!(accumulator.candidate_read_quorum(&deleted), None);
+
+        let mut empty = candidate.clone();
+        empty.size = 0;
+        assert_eq!(accumulator.candidate_read_quorum(&empty), None);
+
+        let mut impossible_parity = candidate;
+        impossible_parity.erasure.parity_blocks = 4;
+        assert_eq!(accumulator.candidate_read_quorum(&impossible_parity), None);
+    }
+
+    #[test]
+    fn confirmed_missing_part_error_recognizes_legacy_and_s3_markers() {
+        assert!(!is_confirmed_missing_part_error(None));
+        assert!(is_confirmed_missing_part_error(Some("file not found")));
+        assert!(is_confirmed_missing_part_error(Some("No such file or directory")));
+        assert!(is_confirmed_missing_part_error(Some("Specified part could not be found")));
+        assert!(is_confirmed_missing_part_error(Some("part.7 not found")));
+        assert!(!is_confirmed_missing_part_error(Some("part.7 missing")));
+        assert!(!is_confirmed_missing_part_error(Some("permission denied")));
+    }
+
+    #[test]
+    fn resolve_read_part_handles_mismatched_and_transient_responses_without_false_missing() {
+        let responses = vec![
+            Some(Vec::new()),
+            Some(vec![read_part_test_error(1, "permission denied")]),
+            None,
+        ];
+
+        let err = resolve_read_part_from_responses("bucket", "upload/part.1.meta", 1, 0, 1, &responses, 2)
+            .expect_err("mismatched and transient responses must not be treated as confirmed missing");
+
+        assert_eq!(err, DiskError::ErasureReadQuorum);
+    }
+
+    #[test]
+    fn resolve_read_part_accepts_alternate_missing_error_markers() {
+        let responses = vec![
+            Some(vec![read_part_test_error(1, "Specified part could not be found")]),
+            Some(vec![read_part_test_error(1, "part.1 not found")]),
+            Some(vec![read_part_test_part(1, "stale-etag")]),
+        ];
+
+        let part = resolve_read_part_from_responses("bucket", "upload/part.1.meta", 1, 0, 1, &responses, 2)
+            .expect("alternate missing markers should satisfy missing quorum");
+
+        assert_eq!(part.number, 1);
+        assert_eq!(part.error.as_deref(), Some("part.1 not found"));
+        assert!(part.etag.is_empty());
+    }
+
+    #[test]
+    fn shard_read_costs_for_empty_disk_set_are_empty() {
+        assert!(shard_read_costs_for_disks(&[]).is_empty());
+    }
+
+    #[tokio::test]
+    async fn reserve_read_repair_heal_dedupes_by_object_version_and_set() {
+        let object = format!("object-{}", Uuid::new_v4());
+        let key = reserve_read_repair_heal("bucket", &object, Some("version-1"), 1, 2)
+            .await
+            .expect("first read-repair reservation should be accepted");
+
+        assert_eq!(key.version_id.as_deref(), Some("version-1"));
+        assert!(
+            reserve_read_repair_heal("bucket", &object, Some("version-1"), 1, 2)
+                .await
+                .is_none()
+        );
+
+        release_read_repair_heal_reservation(&key).await;
+        let retry_key = reserve_read_repair_heal("bucket", &object, Some("version-1"), 1, 2)
+            .await
+            .expect("released read-repair reservation should allow retry");
+        release_read_repair_heal_reservation(&retry_key).await;
+    }
+
+    #[tokio::test]
+    async fn submit_read_repair_heal_releases_reservation_after_submitter_failure() {
+        let object = format!("object-{}", Uuid::new_v4());
+        submit_read_repair_heal_with_submitter(
+            ReadRepairHealSubmission {
+                bucket: "bucket",
+                object: &object,
+                version_id: None,
+                pool_index: 1,
+                set_index: 2,
+                part_number: Some(1),
+                reason: "test",
+            },
+            failed_read_repair_submitter,
+        )
+        .await;
+
+        for _ in 0..20 {
+            if let Some(key) = reserve_read_repair_heal("bucket", &object, None, 1, 2).await {
+                release_read_repair_heal_reservation(&key).await;
+                return;
+            }
+            tokio::time::sleep(Duration::from_millis(10)).await;
+        }
+
+        panic!("failed read-repair submission should release its dedup reservation");
+    }
+
+    #[tokio::test]
+    async fn submit_read_repair_heal_keeps_admitted_reservation_deduped() {
+        let object = format!("object-{}", Uuid::new_v4());
+        submit_read_repair_heal_with_submitter(
+            ReadRepairHealSubmission {
+                bucket: "bucket",
+                object: &object,
+                version_id: None,
+                pool_index: 3,
+                set_index: 4,
+                part_number: None,
+                reason: "test",
+            },
+            accepted_read_repair_submitter,
+        )
+        .await;
+
+        assert!(reserve_read_repair_heal("bucket", &object, None, 3, 4).await.is_none());
+        let key = ReadRepairHealCacheKey::new("bucket", &object, None, 3, 4);
+        release_read_repair_heal_reservation(&key).await;
+    }
 }