diff --git a/crates/ecstore/src/erasure/coding/encode.rs b/crates/ecstore/src/erasure/coding/encode.rs index da1b31274..71c1abe05 100644 --- a/crates/ecstore/src/erasure/coding/encode.rs +++ b/crates/ecstore/src/erasure/coding/encode.rs @@ -709,15 +709,109 @@ mod tests { } } + #[derive(Clone, Default)] + struct FailingWriteWriter; + + impl AsyncWrite for FailingWriteWriter { + fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll> { + Poll::Ready(Err(std::io::Error::other("injected write failure"))) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + } + + #[derive(Clone, Default)] + struct ShortWriteWriter; + + impl AsyncWrite for ShortWriteWriter { + fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll> { + Poll::Ready(Ok(buf.len().saturating_sub(1))) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + } + + #[derive(Clone, Default)] + struct ShutdownFailWriter { + buffered: Vec, + } + + impl AsyncWrite for ShutdownFailWriter { + fn poll_write(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll> { + self.buffered.extend_from_slice(buf); + Poll::Ready(Ok(buf.len())) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Err(std::io::Error::other("injected shutdown failure"))) + } + } + + fn bitrot_writer(writer: W, shard_size: usize) -> BitrotWriterWrapper + where + W: AsyncWrite + Send + Sync + Unpin + 'static, + { + BitrotWriterWrapper::new(CustomWriter::new_tokio_writer(writer), shard_size, HashAlgorithm::HighwayHash256S) + } + + #[tokio::test] + async fn multi_writer_short_write_fails_before_shutdown() { + let mut writers = vec![Some(bitrot_writer(ShortWriteWriter, 16))]; + let err = { + let mut writer = MultiWriter::new(&mut writers, 1); + writer + .write(vec![Bytes::from_static(b"short-write payload")]) + .await + .expect_err("short writes must fail the shard writer") + }; + + assert!(err.to_string().contains("Failed to write data")); + assert!(writers[0].is_none(), "short-write shard must be removed before commit"); + } + + #[tokio::test] + async fn drain_queued_inflight_bytes_consumes_pending_blocks() { + let (tx, mut rx) = mpsc::channel(2); + tx.send(vec![Bytes::from_static(b"queued")]).await.unwrap(); + drop(tx); + + drain_queued_inflight_bytes(&mut rx).await; + + assert!(rx.recv().await.is_none()); + } + + #[tokio::test] + async fn drain_queued_batched_inflight_bytes_consumes_pending_batches() { + let (tx, mut rx) = mpsc::channel(2); + tx.send(vec![vec![Bytes::from_static(b"queued")]]).await.unwrap(); + drop(tx); + + drain_queued_batched_inflight_bytes(&mut rx).await; + + assert!(rx.recv().await.is_none()); + } + #[tokio::test] async fn encode_shutdowns_writers_after_small_shards() { let committed = Arc::new(Mutex::new(Vec::new())); let writer = DeferredCommitWriter::new(committed.clone()); - let mut writers = vec![Some(BitrotWriterWrapper::new( - CustomWriter::new_tokio_writer(writer), - 16, - HashAlgorithm::HighwayHash256S, - ))]; + let mut writers = vec![Some(bitrot_writer(writer, 16))]; let erasure = Arc::new(Erasure::new(1, 0, 16)); let reader = tokio::io::BufReader::new(Cursor::new(b"small payload".to_vec())); @@ -727,6 +821,87 @@ mod tests { assert!(!committed.lock().unwrap().is_empty()); } + #[tokio::test] + async fn encode_streaming_write_quorum_failure_aborts_and_reports_error() { + const DATA_SHARDS: usize = 2; + const PARITY_SHARDS: usize = 2; + const BLOCK_SIZE: usize = 64; + + let committed = Arc::new(Mutex::new(Vec::new())); + let mut writers = vec![ + Some(bitrot_writer(DeferredCommitWriter::new(committed.clone()), BLOCK_SIZE / DATA_SHARDS)), + Some(bitrot_writer(FailingWriteWriter, BLOCK_SIZE / DATA_SHARDS)), + None, + None, + ]; + + let payload = vec![3u8; BLOCK_SIZE * 8]; + let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE)); + let reader = tokio::io::BufReader::new(Cursor::new(payload)); + let err = erasure + .encode(reader, &mut writers, DATA_SHARDS) + .await + .expect_err("streaming encode must fail when write quorum is unavailable"); + + assert!(err.to_string().contains("Failed to write data")); + } + + #[tokio::test] + async fn encode_inline_small_write_quorum_failure_does_not_commit_partial_data() { + const DATA_SHARDS: usize = 2; + const PARITY_SHARDS: usize = 2; + const BLOCK_SIZE: usize = 64; + + let committed = Arc::new(Mutex::new(Vec::new())); + let mut writers = vec![ + Some(bitrot_writer(DeferredCommitWriter::new(committed.clone()), BLOCK_SIZE / DATA_SHARDS)), + Some(bitrot_writer(FailingWriteWriter, BLOCK_SIZE / DATA_SHARDS)), + None, + None, + ]; + + let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE)); + let reader = tokio::io::BufReader::new(Cursor::new(b"write quorum failure payload".to_vec())); + let err = erasure + .encode_inline_small(reader, &mut writers, DATA_SHARDS) + .await + .expect_err("write quorum failure must fail the inline encode"); + + assert!(err.to_string().contains("Failed to write data")); + assert!( + committed.lock().expect("committed buffer should be lockable").is_empty(), + "successful writer must not be committed when write quorum fails before shutdown" + ); + } + + #[tokio::test] + async fn encode_inline_small_shutdown_quorum_failure_is_reported() { + const DATA_SHARDS: usize = 2; + const PARITY_SHARDS: usize = 2; + const BLOCK_SIZE: usize = 64; + + let committed = Arc::new(Mutex::new(Vec::new())); + let mut writers = vec![ + Some(bitrot_writer(DeferredCommitWriter::new(committed.clone()), BLOCK_SIZE / DATA_SHARDS)), + Some(bitrot_writer(ShutdownFailWriter::default(), BLOCK_SIZE / DATA_SHARDS)), + Some(bitrot_writer(ShutdownFailWriter::default(), BLOCK_SIZE / DATA_SHARDS)), + Some(bitrot_writer(ShutdownFailWriter::default(), BLOCK_SIZE / DATA_SHARDS)), + ]; + + let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE)); + let reader = tokio::io::BufReader::new(Cursor::new(b"shutdown quorum failure payload".to_vec())); + let err = erasure + .encode_inline_small(reader, &mut writers, DATA_SHARDS) + .await + .expect_err("shutdown quorum failure must fail the inline encode"); + + assert!(err.to_string().contains("Failed to shutdown writers")); + assert!( + !committed.lock().expect("committed buffer should be lockable").is_empty(), + "the successful writer should have committed before shutdown quorum failure was reported" + ); + } + #[tokio::test] async fn encode_returns_unexpected_eof_for_truncated_limited_reader() { let committed = Arc::new(Mutex::new(Vec::new())); @@ -773,11 +948,7 @@ mod tests { async fn encode_works_on_current_thread_runtime() { let committed = Arc::new(Mutex::new(Vec::new())); let writer = DeferredCommitWriter::new(committed); - let mut writers = vec![Some(BitrotWriterWrapper::new( - CustomWriter::new_tokio_writer(writer), - 16, - HashAlgorithm::HighwayHash256S, - ))]; + let mut writers = vec![Some(bitrot_writer(writer, 16))]; let erasure = Arc::new(Erasure::new(1, 0, 16)); let reader = tokio::io::BufReader::new(Cursor::new(b"current-thread payload".to_vec())); @@ -786,6 +957,78 @@ mod tests { assert_eq!(written, b"current-thread payload".len()); } + #[tokio::test] + async fn encode_batched_writes_full_and_tail_batches() { + const DATA_SHARDS: usize = 2; + const PARITY_SHARDS: usize = 2; + const TOTAL_SHARDS: usize = DATA_SHARDS + PARITY_SHARDS; + const BLOCK_SIZE: usize = 32; + + let committed: Vec>>> = (0..TOTAL_SHARDS).map(|_| Arc::new(Mutex::new(Vec::new()))).collect(); + let mut writers: Vec> = committed + .iter() + .map(|c| Some(bitrot_writer(DeferredCommitWriter::new(c.clone()), BLOCK_SIZE / DATA_SHARDS))) + .collect(); + + let payload = vec![7u8; BLOCK_SIZE * 5 + 3]; + let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE)); + let reader = tokio::io::BufReader::new(Cursor::new(payload.clone())); + let (_reader, total) = erasure + .encode_batched(reader, &mut writers, DATA_SHARDS) + .await + .expect("batched encode should write full and tail batches"); + + assert_eq!(total, payload.len()); + for (index, committed) in committed.iter().enumerate() { + assert!( + !committed.lock().expect("committed buffer should be lockable").is_empty(), + "shard {index} should receive committed batched data" + ); + } + } + + #[tokio::test] + async fn encode_batched_write_quorum_failure_aborts_and_reports_error() { + const DATA_SHARDS: usize = 2; + const PARITY_SHARDS: usize = 2; + const BLOCK_SIZE: usize = 32; + + let committed = Arc::new(Mutex::new(Vec::new())); + let mut writers = vec![ + Some(bitrot_writer(DeferredCommitWriter::new(committed.clone()), BLOCK_SIZE / DATA_SHARDS)), + Some(bitrot_writer(FailingWriteWriter, BLOCK_SIZE / DATA_SHARDS)), + None, + None, + ]; + + let payload = vec![9u8; BLOCK_SIZE * 8]; + let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE)); + let reader = tokio::io::BufReader::new(Cursor::new(payload)); + let err = erasure + .encode_batched(reader, &mut writers, DATA_SHARDS) + .await + .expect_err("batched encode must fail when write quorum is unavailable"); + + assert!(err.to_string().contains("Failed to write data")); + } + + #[tokio::test] + async fn encode_batched_rejects_zero_block_size() { + let committed = Arc::new(Mutex::new(Vec::new())); + let writer = DeferredCommitWriter::new(committed); + let mut writers = vec![Some(bitrot_writer(writer, 16))]; + + let erasure = Arc::new(Erasure::new(1, 0, 0)); + let reader = tokio::io::BufReader::new(Cursor::new(b"payload".to_vec())); + let err = erasure + .encode_batched(reader, &mut writers, 1) + .await + .expect_err("zero block size must be rejected"); + + assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); + assert!(err.to_string().contains("block_size")); + } + /// encode_inline_small: empty reader returns (reader, 0) without writing to any shard. #[tokio::test] async fn encode_inline_small_empty_stream_returns_zero() { diff --git a/crates/ecstore/src/erasure/coding/erasure.rs b/crates/ecstore/src/erasure/coding/erasure.rs index 3cf21f843..023bfab55 100644 --- a/crates/ecstore/src/erasure/coding/erasure.rs +++ b/crates/ecstore/src/erasure/coding/erasure.rs @@ -1056,6 +1056,80 @@ mod tests { } } + #[test] + fn decode_data_rejects_missing_data_above_parity_budget() { + for uses_legacy in [false, true] { + let erasure = Erasure::new_with_options(3, 2, 64, uses_legacy); + let data = b"read restore must fail when too many data shards are gone"; + let encoded = erasure.encode_data(data).expect("encode should succeed"); + let mut shards = optional_shards(&encoded); + shards[0] = None; + shards[1] = None; + shards[2] = None; + + let err = erasure + .decode_data(&mut shards) + .expect_err("decode_data must fail when missing data shards exceed parity"); + + assert!(!err.to_string().is_empty()); + assert!(shards.iter().take(erasure.data_shards).any(Option::is_none)); + } + } + + #[test] + fn decode_data_and_parity_rejects_wrong_shard_count() { + for uses_legacy in [false, true] { + let erasure = Erasure::new_with_options(4, 2, 128, uses_legacy); + let data = b"wrong shard count should not be accepted as a restored object"; + let encoded = erasure.encode_data(data).expect("encode should succeed"); + let mut shards = optional_shards(&encoded); + let _ = shards.pop(); + + let err = erasure + .decode_data_and_parity(&mut shards) + .expect_err("decode_data_and_parity must reject wrong shard count"); + + assert!(!err.to_string().is_empty()); + } + } + + #[test] + fn decode_data_with_verification_rejects_corrupt_surplus_parity() { + let erasure = Erasure::new(3, 2, 128); + let data = b"verified reads must fail closed on stale surplus parity"; + let encoded = erasure.encode_data(data).expect("encode should succeed"); + let mut shards = optional_shards(&encoded); + shards[1] = None; + shards[erasure.data_shards].as_mut().expect("parity shard should be present")[0] ^= 0x7d; + + let err = erasure + .decode_data_with_reconstruction_verification(&mut shards) + .expect_err("verified decode must reject inconsistent parity"); + + assert_eq!(err.kind(), io::ErrorKind::InvalidData); + } + + #[test] + fn verify_data_and_parity_rejects_missing_and_mismatched_shards() { + let erasure = Erasure::new(4, 2, 128); + let data = b"verification must reject incomplete or malformed shard sets"; + let encoded = erasure.encode_data(data).expect("encode should succeed"); + let mut shards = optional_shards(&encoded); + + shards[0] = None; + let missing = erasure + .verify_data_and_parity(&shards) + .expect_err("verification must reject missing shards"); + assert!(missing.to_string().contains("missing shard")); + + let mut mismatched = optional_shards(&encoded); + let _ = mismatched[0].as_mut().expect("data shard should be present").pop(); + let mismatch = erasure + .verify_data_and_parity(&mismatched) + .expect_err("verification must reject mismatched shard lengths"); + assert!(mismatch.to_string().contains("inconsistent shard length")); + } + #[test] fn decode_data_and_parity_leaves_complete_shards_unchanged() { let erasure = Erasure::new(4, 2, 128); diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index e54f8bb75..da34e1690 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -3427,6 +3427,43 @@ impl SetDisks { #[cfg(test)] mod tests { use super::*; + use std::io::Cursor; + use tokio::io::AsyncReadExt; + + fn metadata_test_fileinfo(object: &str) -> FileInfo { + let mut fi = FileInfo::new(object, 2, 2); + fi.volume = "bucket".to_string(); + fi.name = object.to_string(); + fi.size = 1; + fi.erasure.index = 1; + fi.metadata.insert("etag".to_string(), "etag-1".to_string()); + fi.add_object_part(1, "part-etag-1".to_string(), 1, None, 1, None, None); + fi + } + + fn read_part_test_part(number: usize, etag: &str) -> ObjectPartInfo { + ObjectPartInfo { + number, + etag: etag.to_string(), + ..Default::default() + } + } + + fn read_part_test_error(number: usize, error: &str) -> ObjectPartInfo { + ObjectPartInfo { + number, + error: Some(error.to_string()), + ..Default::default() + } + } + + fn failed_read_repair_submitter(_request: rustfs_common::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture { + Box::pin(async { ReadRepairAdmissionOutcome::Failed("injected submit failure".to_string()) }) + } + + fn accepted_read_repair_submitter(_request: rustfs_common::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture { + Box::pin(async { ReadRepairAdmissionOutcome::Response(HealAdmissionResult::Accepted) }) + } #[test] fn dangling_delete_grace_defaults_to_one_hour() { @@ -3444,4 +3481,224 @@ mod tests { assert!(dangling_delete_grace().is_zero()); }); } + + #[tokio::test] + async fn multipart_codec_streaming_reader_zero_buffer_is_noop() { + let reader = tokio::io::BufReader::new(Cursor::new(b"payload".to_vec())); + let mut reader = MultipartCodecStreamingReader::new(vec![Box::new(reader)]); + let mut out = []; + + let read = reader + .read(&mut out) + .await + .expect("zero-length read should not poll inner readers"); + + assert_eq!(read, 0); + assert_eq!(reader.readers.len(), 1); + } + + #[test] + fn metadata_fanout_observation_classifies_invalid_and_ignored_results() { + let invalid = MetadataFanoutObservation::from_file_info(&FileInfo::default(), Duration::from_millis(7)); + assert_eq!(invalid.outcome, GET_METADATA_RESPONSE_ERROR); + assert!(!invalid.valid); + assert!(!invalid.ignored); + + let ignored = MetadataFanoutObservation::from_error(&DiskError::DiskNotFound, Duration::from_millis(9)); + assert_eq!(ignored.outcome, GET_METADATA_RESPONSE_DISK_NOT_FOUND); + assert!(!ignored.valid); + assert!(ignored.ignored); + + let corrupt = MetadataFanoutObservation::from_error(&DiskError::FileCorrupt, Duration::from_millis(11)); + assert_eq!(corrupt.outcome, GET_METADATA_RESPONSE_CORRUPT); + assert!(!corrupt.ignored); + } + + #[test] + fn metadata_fanout_diagnostics_reports_counts_and_latency_edges() { + let diagnostics = MetadataFanoutDiagnostics::new( + Duration::from_millis(40), + vec![ + MetadataFanoutObservation { + outcome: GET_METADATA_RESPONSE_VALID, + elapsed: Duration::from_millis(30), + valid: true, + ignored: false, + }, + MetadataFanoutObservation { + outcome: GET_METADATA_RESPONSE_IGNORED, + elapsed: Duration::from_millis(10), + valid: false, + ignored: true, + }, + MetadataFanoutObservation { + outcome: GET_METADATA_RESPONSE_ERROR, + elapsed: Duration::from_millis(20), + valid: false, + ignored: false, + }, + ], + ); + + assert_eq!(diagnostics.total_responses(), 3); + assert_eq!(diagnostics.valid_responses(), 1); + assert_eq!(diagnostics.ignored_responses(), 1); + assert_eq!(diagnostics.error_responses(), 2); + assert_eq!(diagnostics.first_response_latency(), Some(Duration::from_millis(10))); + assert_eq!(diagnostics.first_valid_response_latency(), Some(Duration::from_millis(30))); + assert_eq!(diagnostics.slowest_response_latency(), Some(Duration::from_millis(30))); + assert_eq!(diagnostics.quorum_candidate_latency(0), Some(Duration::ZERO)); + assert_eq!(diagnostics.quorum_candidate_latency(1), Some(Duration::from_millis(30))); + assert_eq!(diagnostics.quorum_candidate_latency(2), None); + } + + #[test] + fn metadata_quorum_accumulator_counts_invalid_metadata_and_ignored_errors() { + let mut accumulator = MetadataQuorumAccumulator::new(4, 2, true); + + accumulator.observe_file_info(&FileInfo::default()); + accumulator.observe_error(&DiskError::DiskNotFound); + + assert_eq!(accumulator.hard_errors, 1); + assert_eq!(accumulator.ignored_errors, 1); + assert_eq!(accumulator.final_miss_reason(), GET_METADATA_EARLY_STOP_REASON_ERROR); + } + + #[test] + fn metadata_quorum_accumulator_candidate_quorum_handles_zero_parity_and_invalid_candidates() { + let accumulator = MetadataQuorumAccumulator::new(4, 0, true); + let candidate = metadata_test_fileinfo("object"); + assert_eq!(accumulator.candidate_read_quorum(&candidate), Some(4)); + assert_eq!(accumulator.missing_response_quorum(), 4); + + let accumulator = MetadataQuorumAccumulator::new(4, 2, true); + let mut deleted = candidate.clone(); + deleted.deleted = true; + assert_eq!(accumulator.candidate_read_quorum(&deleted), None); + + let mut empty = candidate.clone(); + empty.size = 0; + assert_eq!(accumulator.candidate_read_quorum(&empty), None); + + let mut impossible_parity = candidate; + impossible_parity.erasure.parity_blocks = 4; + assert_eq!(accumulator.candidate_read_quorum(&impossible_parity), None); + } + + #[test] + fn confirmed_missing_part_error_recognizes_legacy_and_s3_markers() { + assert!(!is_confirmed_missing_part_error(None)); + assert!(is_confirmed_missing_part_error(Some("file not found"))); + assert!(is_confirmed_missing_part_error(Some("No such file or directory"))); + assert!(is_confirmed_missing_part_error(Some("Specified part could not be found"))); + assert!(is_confirmed_missing_part_error(Some("part.7 not found"))); + assert!(!is_confirmed_missing_part_error(Some("part.7 missing"))); + assert!(!is_confirmed_missing_part_error(Some("permission denied"))); + } + + #[test] + fn resolve_read_part_handles_mismatched_and_transient_responses_without_false_missing() { + let responses = vec![ + Some(Vec::new()), + Some(vec![read_part_test_error(1, "permission denied")]), + None, + ]; + + let err = resolve_read_part_from_responses("bucket", "upload/part.1.meta", 1, 0, 1, &responses, 2) + .expect_err("mismatched and transient responses must not be treated as confirmed missing"); + + assert_eq!(err, DiskError::ErasureReadQuorum); + } + + #[test] + fn resolve_read_part_accepts_alternate_missing_error_markers() { + let responses = vec![ + Some(vec![read_part_test_error(1, "Specified part could not be found")]), + Some(vec![read_part_test_error(1, "part.1 not found")]), + Some(vec![read_part_test_part(1, "stale-etag")]), + ]; + + let part = resolve_read_part_from_responses("bucket", "upload/part.1.meta", 1, 0, 1, &responses, 2) + .expect("alternate missing markers should satisfy missing quorum"); + + assert_eq!(part.number, 1); + assert_eq!(part.error.as_deref(), Some("part.1 not found")); + assert!(part.etag.is_empty()); + } + + #[test] + fn shard_read_costs_for_empty_disk_set_are_empty() { + assert!(shard_read_costs_for_disks(&[]).is_empty()); + } + + #[tokio::test] + async fn reserve_read_repair_heal_dedupes_by_object_version_and_set() { + let object = format!("object-{}", Uuid::new_v4()); + let key = reserve_read_repair_heal("bucket", &object, Some("version-1"), 1, 2) + .await + .expect("first read-repair reservation should be accepted"); + + assert_eq!(key.version_id.as_deref(), Some("version-1")); + assert!( + reserve_read_repair_heal("bucket", &object, Some("version-1"), 1, 2) + .await + .is_none() + ); + + release_read_repair_heal_reservation(&key).await; + let retry_key = reserve_read_repair_heal("bucket", &object, Some("version-1"), 1, 2) + .await + .expect("released read-repair reservation should allow retry"); + release_read_repair_heal_reservation(&retry_key).await; + } + + #[tokio::test] + async fn submit_read_repair_heal_releases_reservation_after_submitter_failure() { + let object = format!("object-{}", Uuid::new_v4()); + submit_read_repair_heal_with_submitter( + ReadRepairHealSubmission { + bucket: "bucket", + object: &object, + version_id: None, + pool_index: 1, + set_index: 2, + part_number: Some(1), + reason: "test", + }, + failed_read_repair_submitter, + ) + .await; + + for _ in 0..20 { + if let Some(key) = reserve_read_repair_heal("bucket", &object, None, 1, 2).await { + release_read_repair_heal_reservation(&key).await; + return; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + + panic!("failed read-repair submission should release its dedup reservation"); + } + + #[tokio::test] + async fn submit_read_repair_heal_keeps_admitted_reservation_deduped() { + let object = format!("object-{}", Uuid::new_v4()); + submit_read_repair_heal_with_submitter( + ReadRepairHealSubmission { + bucket: "bucket", + object: &object, + version_id: None, + pool_index: 3, + set_index: 4, + part_number: None, + reason: "test", + }, + accepted_read_repair_submitter, + ) + .await; + + assert!(reserve_read_repair_heal("bucket", &object, None, 3, 4).await.is_none()); + let key = ReadRepairHealCacheKey::new("bucket", &object, None, 3, 4); + release_read_repair_heal_reservation(&key).await; + } } diff --git a/crates/ecstore/tests/legacy_bitrot_read_test.rs b/crates/ecstore/tests/legacy_bitrot_read_test.rs index 3ba8d6785..7b8df7815 100644 --- a/crates/ecstore/tests/legacy_bitrot_read_test.rs +++ b/crates/ecstore/tests/legacy_bitrot_read_test.rs @@ -31,7 +31,7 @@ mod storage_api; use rustfs_filemeta::{FileInfoOpts, get_file_info}; use rustfs_utils::HashAlgorithm; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use storage_api::legacy_bitrot_read::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, create_bitrot_reader, new_disk}; use tokio::fs; @@ -44,15 +44,46 @@ fn workspace_root() -> PathBuf { .to_path_buf() } +fn legacy_test_root() -> PathBuf { + std::env::var_os("RUSTFS_LEGACY_TEST_ROOT") + .map(PathBuf::from) + .unwrap_or_else(workspace_root) +} + +fn legacy_test_disk(root: &Path) -> String { + if let Some(disk) = std::env::var_os("RUSTFS_LEGACY_TEST_DISK") { + return disk.to_string_lossy().into_owned(); + } + if root.join("test/.minio.sys/format.json").exists() { + "test".to_string() + } else { + "test1".to_string() + } +} + +fn legacy_format_exists(root: &Path, disk_name: &str) -> bool { + let disk_root = root.join(disk_name); + disk_root.join(".rustfs.sys/format.json").exists() || disk_root.join(".minio.sys/format.json").exists() +} + +fn legacy_object_meta_exists(root: &Path, disk_name: &str, bucket: &str, object: &str) -> bool { + root.join(disk_name) + .join(bucket) + .join(object) + .join(STORAGE_FORMAT_FILE) + .exists() +} + fn legacy_test_data_exists() -> bool { if std::env::var("RUSTFS_SKIP_LEGACY_TEST").unwrap_or_default() == "1" { return false; } - let root = workspace_root(); - let format_path = root.join("test1/.rustfs.sys/format.json"); - let ktvzip_meta = root.join("test1/vvvv/ktvzip.tar.gz").join(STORAGE_FORMAT_FILE); - let path_traversal_meta = root.join("test1/vvvv/path_traversal.md").join(STORAGE_FORMAT_FILE); - format_path.exists() && (ktvzip_meta.exists() || path_traversal_meta.exists()) + let root = legacy_test_root(); + let disk_name = legacy_test_disk(&root); + let bucket = "vvvv"; + legacy_format_exists(&root, &disk_name) + && (legacy_object_meta_exists(&root, &disk_name, bucket, "ktvzip.tar.gz") + || legacy_object_meta_exists(&root, &disk_name, bucket, "path_traversal.md")) } async fn run_legacy_bitrot_test_for_object(root: &std::path::Path, disk_name: &str, bucket: &str, object: &str) -> bool { @@ -227,23 +258,25 @@ async fn run_legacy_bitrot_test_for_object(root: &std::path::Path, disk_name: &s #[tokio::test] async fn test_legacy_bitrot_read() { if !legacy_test_data_exists() { - eprintln!("Skipping legacy bitrot test: test1/vvvv xl.meta or RUSTFS_SKIP_LEGACY_TEST"); + eprintln!("Skipping legacy bitrot test: legacy fixture xl.meta not found or RUSTFS_SKIP_LEGACY_TEST=1"); return; } - // let root = workspace_root(); - let root = PathBuf::from("/Users/weisd/project/minio"); - - let disk_name = "test"; + let root = legacy_test_root(); + let disk_name = legacy_test_disk(&root); let bucket = "vvvv"; + let mut checked = 0usize; - // Try both EC (part files) and inline objects - let ok = run_legacy_bitrot_test_for_object(&root, disk_name, bucket, "ktvzip.tar.gz").await; + for object in ["ktvzip.tar.gz", "path_traversal.md"] { + if !legacy_object_meta_exists(&root, &disk_name, bucket, object) { + eprintln!("Skipping missing legacy object fixture: {object}"); + continue; + } - eprintln!("ok: {:?}", ok); + checked += 1; + let ok = run_legacy_bitrot_test_for_object(&root, &disk_name, bucket, object).await; + assert!(ok, "create_bitrot_reader failed for {object}"); + } - assert!(ok, "create_bitrot_reader failed for both ktvzip.tar.gz and path_traversal.md"); - - let ok = run_legacy_bitrot_test_for_object(&root, disk_name, bucket, "path_traversal.md").await; - assert!(ok, "create_bitrot_reader failed for path_traversal.md"); + assert!(checked > 0, "legacy bitrot fixture preflight found no readable objects"); } diff --git a/crates/ecstore/tests/minio_generated_read_test.rs b/crates/ecstore/tests/minio_generated_read_test.rs index aa6d29d46..e2d11bcc2 100644 --- a/crates/ecstore/tests/minio_generated_read_test.rs +++ b/crates/ecstore/tests/minio_generated_read_test.rs @@ -117,6 +117,52 @@ fn sha256_hex(bytes: &[u8]) -> String { hex_simd::encode_to_string(Sha256::digest(bytes), hex_simd::AsciiCase::Lower) } +async fn load_fixture_reader_input(case_id: &str) -> (ObjectInfo, Vec, String) { + let case_dir = require_fixture_case(case_id); + let manifest: ManifestRecord = read_json(&case_dir.join("manifest.json")); + let expected_sha256 = read_plaintext_sha256(&case_dir); + let file_info = load_file_info(&case_dir, &manifest); + let encrypted = encrypted_fixture_bytes(&case_dir, &manifest, &file_info).await; + let object_info = load_object_info(&file_info, &manifest); + + (object_info, encrypted, expected_sha256) +} + +async fn read_fixture_plaintext(encrypted: Vec, object_info: ObjectInfo, kms_key_b64: String) -> Result, String> { + let object_size = object_info.size; + + async_with_vars( + [ + ("__RUSTFS_SSE_SIMPLE_CMK", Some(kms_key_b64)), + ("RUSTFS_SSE_S3_MASTER_KEY", None::), + ], + async move { + let (mut reader, offset, length) = GetObjectReader::new( + Box::new(Cursor::new(encrypted)), + None, + &object_info, + &ObjectOptions::default(), + &http::HeaderMap::new(), + ) + .await + .map_err(|err| format!("construct GetObjectReader from MinIO raw fixture: {err:?}"))?; + + if offset != 0 || length != object_size { + return Err(format!("unexpected fixture range offset={offset} length={length} size={object_size}")); + } + + let mut plaintext = Vec::new(); + reader + .read_to_end(&mut plaintext) + .await + .map_err(|err| format!("read plaintext from MinIO raw fixture: {err}"))?; + + Ok(plaintext) + }, + ) + .await +} + async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, file_info: &FileInfo) -> Vec { let mut disks = Vec::with_capacity(file_info.erasure.distribution.len()); for disk_number in 1..=file_info.erasure.distribution.len() { @@ -204,43 +250,44 @@ async fn reads_minio_generated_sse_kms_multipart_fixture() { assert_fixture_round_trip("sse-kms-multipart-8m", 8 * 1024 * 1024).await; } +#[tokio::test] +#[ignore = "requires generated MinIO fixture data and a local static KMS key"] +async fn rejects_minio_generated_sse_s3_fixture_with_wrong_kms_key() { + let (object_info, encrypted, _) = load_fixture_reader_input("sse-s3-multipart-8m").await; + let wrong_key_b64 = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE=".to_string(); + + let result = read_fixture_plaintext(encrypted, object_info, wrong_key_b64).await; + + assert!(result.is_err(), "wrong KMS key must fail closed"); +} + +#[tokio::test] +#[ignore = "requires generated MinIO fixture data and a local static KMS key"] +async fn rejects_minio_generated_sse_s3_fixture_with_truncated_ciphertext() { + let (object_info, mut encrypted, expected_sha256) = load_fixture_reader_input("sse-s3-multipart-8m").await; + encrypted.truncate(encrypted.len() / 2); + + let result = read_fixture_plaintext(encrypted, object_info, minio_static_kms_key_b64()).await; + + if let Ok(plaintext) = result { + assert_ne!( + sha256_hex(&plaintext), + expected_sha256, + "truncated ciphertext must not restore the original plaintext" + ); + } +} + async fn assert_fixture_round_trip(case_id: &str, expected_size: i64) { - let case_dir = require_fixture_case(case_id); - let manifest: ManifestRecord = read_json(&case_dir.join("manifest.json")); - let expected_sha256 = read_plaintext_sha256(&case_dir); - let file_info = load_file_info(&case_dir, &manifest); - let encrypted = encrypted_fixture_bytes(&case_dir, &manifest, &file_info).await; - let object_info = load_object_info(&file_info, &manifest); + let (object_info, encrypted, expected_sha256) = load_fixture_reader_input(case_id).await; + let object_size = object_info.size; let kms_key_b64 = minio_static_kms_key_b64(); - async_with_vars( - [ - ("__RUSTFS_SSE_SIMPLE_CMK", Some(kms_key_b64)), - ("RUSTFS_SSE_S3_MASTER_KEY", None::), - ], - async { - let (mut reader, offset, length) = GetObjectReader::new( - Box::new(Cursor::new(encrypted)), - None, - &object_info, - &ObjectOptions::default(), - &http::HeaderMap::new(), - ) - .await - .expect("construct GetObjectReader from MinIO raw fixture"); + let plaintext = read_fixture_plaintext(encrypted, object_info, kms_key_b64) + .await + .expect("fixture must restore with the configured KMS key"); - let mut plaintext = Vec::new(); - reader - .read_to_end(&mut plaintext) - .await - .expect("read plaintext from MinIO raw fixture"); - - assert_eq!(offset, 0); - assert_eq!(length, object_info.size); - assert_eq!(reader.object_info.size, expected_size); - assert_eq!(plaintext.len(), expected_size as usize); - assert_eq!(sha256_hex(&plaintext), expected_sha256); - }, - ) - .await; + assert_eq!(object_size, expected_size); + assert_eq!(plaintext.len(), expected_size as usize); + assert_eq!(sha256_hex(&plaintext), expected_sha256); } diff --git a/docs/testing/ecstore-validation-suite-design.md b/docs/testing/ecstore-validation-suite-design.md new file mode 100644 index 000000000..4eb3b41d5 --- /dev/null +++ b/docs/testing/ecstore-validation-suite-design.md @@ -0,0 +1,376 @@ +# ECStore Validation Suite Design + +This document defines the validation suite RustFS should use before claiming +ECStore erasure-coding correctness, durability, and fault-resilience coverage. +It is a suite design, not a claim that all tests already exist. + +## Goal + +Provide one command that runs the full ECStore confidence suite and produces a +single pass/fail result plus artifacts. The command should exercise both: + +- white-box invariants in `rustfs-ecstore`, `rustfs-filemeta`, and related + helpers; +- black-box S3 and admin behavior against real single-node and distributed + erasure deployments. + +The suite can reduce release risk; it cannot prove the absence of all defects. +It must therefore combine deterministic matrices, negative tests, fuzz/corpus +tests, chaos tests, and explicit artifact review. + +## Proposed Entry Point + +Use the top-level runner for repeatable local, CI, and release checks: + +```bash +scripts/run_ecstore_validation_suite.sh --profile full +``` + +Profiles: + +| Profile | Purpose | Expected cost | +| --- | --- | --- | +| `quick` | PR smoke for EC logic and existing single-node reliability tests. | minutes | +| `full` | Release gate: white-box, e2e, chaos, S3 compatibility subset, coverage. | hours | +| `destructive` | Manual/nightly gate: distributed 4-node/16-disk, crash/restart, rebalance/decommission fault injection. | hours+ | +| `fuzz` | Malformed metadata/RPC/corpus fuzzing with fixed seed output. | bounded by budget | + +The current runner wires the existing high-signal checks. The matrix below +still tracks required follow-up coverage before `full` can be treated as a +complete release gate. + +The runner writes artifacts under +`target/ecstore-validation//`: + +- command transcript and environment; +- randomized seeds; +- object manifests with SHA256; +- per-disk layout snapshots before and after mutation; +- server logs; +- junit/nextest output; +- `blackbox-matrix.tsv` with selected black-box and fixture gates; +- coverage report; +- failure reproducer instructions. + +## Acceptance Rules + +The suite passes only when all selected profile commands pass and every +scenario asserts both API-visible behavior and on-disk state where applicable. + +Required fail-closed rules: + +- never return corrupted object bytes; +- never silently accept forged or split-brain metadata; +- never downgrade write quorum to read quorum; +- never leave a mixed old/new object after partial commit; +- never panic on malformed EC metadata; +- return typed errors or quorum failures for invalid states. + +Unit-test coverage is a hard release-gate input: `rustfs-ecstore` unit line +coverage for the EC-critical scope must be at least 95%, with 100% as the +target for EC read, write, decode, heal, metadata quorum, and rollback paths. +A lower threshold is only acceptable for a temporary, explicitly documented +exception tied to missing testability or unreachable code. Full-crate coverage +is still reported as an observation metric, but it must not hide EC regressions +behind unrelated modules. + +Tests must not pass by only checking constants, helper calls, deleted branches, +or implementation details. Every test needs a reader-facing or storage-state +assertion. + +Fixture-backed checks are optional for local smoke runs, but explicit in the +artifact stream. `--require-fixtures` turns missing legacy or MinIO generated +fixtures into an early `ecstore-fixture-gate` failure before expensive black-box +steps run. The MinIO generated fixture gate requires both +`RUSTFS_MINIO_FIXTURE_ROOT` and `RUSTFS_MINIO_STATIC_KMS_KEY_B64`. + +## White-Box Matrix + +### Erasure Algorithm + +Target files: + +- `crates/ecstore/src/erasure/coding/erasure.rs` +- `crates/ecstore/src/erasure/coding/encode.rs` +- `crates/ecstore/src/erasure/coding/decode.rs` +- `crates/ecstore/src/erasure/coding/decode_reader.rs` +- `crates/ecstore/src/erasure/codec/bridge.rs` + +Coverage: + +| Area | Scenarios | Assertions | +| --- | --- | --- | +| Shard geometry | legacy/current shard-size formulas; lengths `0`, `1`, `block-1`, `block`, `block+1`, multi-block tail | no divide-by-zero; shard/file/range offsets match expected | +| Encode/decode | `(data, parity)` sets `2+2`, `4+2`, `8+8`; random payloads; missing shards up to parity | reconstructed data equals original | +| Negative reconstruction | missing shards above parity; inconsistent shard lengths; corrupt surplus parity | typed error, no partial success | +| Source verification | missing data shard plus extra parity source | rebuilt parity must match source parity | +| Legacy compatibility | old shard formula and legacy checksum data | legacy files decode and heal correctly | +| Streaming decode | legacy engine vs RustFS codec engine on same stripe stream | bytes and errors are equivalent | +| Range output | head/middle/tail/suffix; cross-block and final-short-stripe ranges | exact byte range, no over-read/under-read | + +Add property tests with fixed replay seeds for payload, range, and missing-shard +selection. + +### Bitrot and Reader Alignment + +Target files: + +- `crates/ecstore/src/erasure/coding/bitrot.rs` +- `crates/ecstore/src/erasure/coding/decode.rs` +- `crates/ecstore/src/set_disk/core/io_primitives.rs` +- `crates/ecstore/src/set_disk/shard_source.rs` + +Coverage: + +| Area | Scenarios | Assertions | +| --- | --- | --- | +| Hash framing | valid hash+data; wrong hash; truncated hash; truncated data | invalid data never succeeds | +| Short shard | short read under normal hash, `skip_verify`, and hash-none | `UnexpectedEof` or equivalent typed error | +| Lockstep reads | mid-stream data shard failure; pending/timeout reader; final short stripe | each live reader advances exactly one stripe; failed reader retires | +| Adaptive reads | hedged parity fallback and timeout retirement | no shard desync; reconstructed bytes match original | +| Shard source order | out-of-order read completion and missing slots | slots resolve by shard index | +| Deferred readers | data-blocks-first setup opens deferred parity at correct offset | parity fallback uses aligned data | + +Use instrumented readers that record shard index, stripe index, read count, +offset, and retirement reason. + +### Metadata, Quorum, and Commit Atomicity + +Target files: + +- `crates/filemeta/src/filemeta/version.rs` +- `crates/ecstore/src/set_disk/read.rs` +- `crates/ecstore/src/set_disk/metadata.rs` +- `crates/ecstore/src/set_disk/ops/object.rs` +- `crates/ecstore/src/set_disk/core/io_primitives.rs` +- `crates/ecstore/src/disk/local.rs` + +Coverage: + +| Area | Scenarios | Assertions | +| --- | --- | --- | +| Metadata tamper | same `version_id`/`mod_time`, divergent data dir, parts, ETag, size, checksum, inline flag, erasure distribution | previous committed version or read quorum error; no arbitrary latest | +| Early stop | valid quorum, stale quorum, corrupt trailing disks, slow trailing disks | early-stop only on safe identity | +| Quorum downgrade | read quorum vs write quorum; delete marker quorum; version-not-found quorum | no mutation below write quorum | +| Rename atomicity | failure before data rename, after data rename, after metadata rename, cleanup failure | object is old or new; never mixed | +| Rollback | failed commit quorum and stale temp data | rollback preserves old metadata/data | +| Malformed metadata | oversized lengths, bad CRC, invalid versions, invalid UUID/timestamp/enum, huge parts | bounded memory; typed error; no panic | + +Fault injection should be explicit and deterministic, preferably through local +disk mocks for unit tests and process-level disk manipulation for e2e tests. + +## Black-Box Matrix + +Use `crates/e2e_test` for real S3/admin behavior and extend +`crates/e2e_test/src/chaos.rs` rather than duplicating ad hoc helpers. + +The current runner emits `blackbox-matrix.tsv` for every invocation. It is a +machine-readable manifest of selected black-box scenarios, their commands, +required fixture environment, and whether each row is enabled, disabled, or +missing an optional/required fixture. + +Current runner rows: + +| Profile | Scenario | Gate | Fixture env | +| --- | --- | --- | --- | +| `quick` | single-node disk fault read/write | e2e black box | none | +| `quick` | degraded erasure disk rebuild | e2e black box | none | +| `quick` | namespace lock quorum under EC ops | e2e black box | none | +| `full` | legacy bitrot read fixture restore | fixture black box | `RUSTFS_LEGACY_TEST_ROOT`, `RUSTFS_LEGACY_TEST_DISK` | +| `full` | MinIO generated encrypted read and negative restore fixture | fixture black box | `RUSTFS_MINIO_FIXTURE_ROOT`, `RUSTFS_MINIO_STATIC_KMS_KEY_B64` | +| `full` | S3 multipart/range/versioning/delete subset | S3 black box | none | +| `destructive` | distributed cluster concurrency | e2e black box | none | +| `destructive` | stale multipart cleanup cluster | e2e black box | none | +| `destructive` | delete marker migration semantics | e2e black box | none | + +### Single-Node 4-Disk EC + +| Scenario | Required assertions | +| --- | --- | +| baseline PUT/GET/HEAD/List for tiny, inline, block-boundary, multi-block, multipart objects | SHA256 manifest matches; metadata is consistent on all disks | +| one disk offline during read | existing objects readable; no corrupted bytes | +| one disk offline during write | write succeeds only when write quorum holds; restored disk is healed | +| above-parity disk loss | GET/PUT fails with quorum error; no partial bytes accepted | +| corrupt data shard and parity shard | GET returns original bytes or fails closed; read-repair/heal restores | +| corrupt inline `xl.meta` | fail closed or heal; no forged inline data | +| range read with offline/corrupt shard | exact range bytes; invalid ranges produce expected S3 errors | +| multipart part resend and concurrent same-part writes | final complete object matches chosen committed parts | +| crash during multipart complete/put/delete | after restart only old or new full version is visible | + +Existing anchors: + +- `crates/e2e_test/src/reliability_disk_fault_test.rs` +- `crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs` +- `crates/e2e_test/src/chaos.rs` + +### Distributed 4-Node / 16-Disk EC + +| Scenario | Required assertions | +| --- | --- | +| node/disk outage while reading large objects | no EOF/truncation; SHA256 manifest matches | +| write while one remote node is down | write follows quorum; later heal reconstructs remote disk | +| remote shard bitrot | degraded read uses clean shards; no bad bytes | +| concurrent GET/PUT/DELETE/List on same key | no 500 for expected conflicts; no dirty reads | +| range GET matrix for large objects | sequential and parallel ranges match full-object hash | +| internode timeout/slow disk | typed error or fallback; no desync | + +This layer is mandatory because single-process unit tests cannot prove RPC, +HTTP/2, timeout, and distributed quorum behavior. + +### Versioning, Delete Markers, and Migration + +| Scenario | Required assertions | +| --- | --- | +| latest delete marker | GET/HEAD/ListObjectVersions match S3 semantics | +| explicit `versionId` for old versions | exact old bytes and metadata | +| suspended/null version | no version ordering regression | +| delete marker during heal/rebalance/decommission | marker visibility and history are preserved | +| orphan directory cleanup | real objects are not purged; tombstones behave correctly | + +### Heal, Rebalance, and Decommission + +| Scenario | Required assertions | +| --- | --- | +| auto heal and admin deep heal | data hash unchanged; `xl.meta` and format data rebuilt | +| heal interruption/restart | idempotent recovery; no dangling temp objects | +| two-pool rebalance with versioned/multipart objects | source and target pools have consistent versions | +| decommission cancel/restart/finalize | target readable; source cleanup safe | +| rebalance/decommission with node outage | progress resumes; no duplicate or missing versions | + +Existing scripts under `scripts/test/decommission_*.sh` should be wrapped into +the `destructive` profile only after they emit machine-readable pass/fail +artifacts. + +## S3 Compatibility and Large Object Gates + +The full profile should include a targeted S3 compatibility subset, not the +entire compatibility suite by default: + +```bash +TESTEXPR="multipart or range or versioning or delete" \ +DEPLOY_MODE=build \ +MAXFAIL=0 \ +./scripts/s3-tests/run.sh +``` + +Large object gates: + +- `scripts/run_get_codec_streaming_smoke.sh` for legacy/codec GET parity; +- `scripts/run_gt1g_get_http_matrix.sh` for sequential and parallel range GET; +- `scripts/run_gt1g_multipart_put_matrix.sh` for multipart PUT paths. + +These should be artifact-producing optional stages in `full` or `destructive` +profiles, not hidden local-only commands. + +## Fuzz and Corpus Gates + +Add bounded fuzz/corpus tests for: + +- `xl.meta` MessagePack and legacy filemeta versions; +- protobuf/RPC payload decoding; +- checksum and bitrot headers; +- range offset/length overflow; +- object names with path traversal, encoded separators, and symlink components; +- huge inline metadata and huge part counts. + +Each corpus failure must save the input bytes and the minimized reproducer under +the suite artifact directory. + +## Coverage Snapshot Target + +The runner enforces unit line coverage for `rustfs-ecstore` in `full` and +`destructive` profiles. The default threshold is 95%, and the target remains +100% for EC-critical paths: + +```bash +scripts/run_ecstore_validation_suite.sh --profile full --unit-coverage-min 95 +scripts/run_ecstore_validation_suite.sh --profile full --unit-coverage-min 100 +scripts/run_ecstore_validation_suite.sh --profile full --unit-coverage-scope crate +``` + +The default hard gate scope is `ec-critical`, covering: + +- `crates/ecstore/src/erasure/**` +- `crates/ecstore/src/set_disk/read.rs` +- `crates/ecstore/src/set_disk/shard_source.rs` +- `crates/ecstore/src/set_disk/metadata.rs` +- `crates/ecstore/src/set_disk/ops/object.rs` +- `crates/ecstore/src/set_disk/core/io_primitives.rs` +- `crates/ecstore/src/disk/local.rs` + +Minimum release-gate target: + +- `cargo-llvm-cov` must be installed unless `--skip-coverage` is explicitly set; +- `rustfs-ecstore` EC-critical unit line coverage is at least 95%; +- EC read/write/decode/heal/quorum/rollback code should trend toward 100%; +- full-crate unit line coverage is recorded for visibility but is not the EC + hard gate; +- all HIGH rows in this document have positive and negative tests; +- changed EC/read/write/heal lines are covered; +- branch coverage is reviewed for reconstruction, quorum, and error paths; +- uncovered branches are either intentionally unreachable or tracked. + +The `quick` profile also runs +`cargo test -p rustfs-ecstore --lib -- --test-threads=1` so the white-box smoke +includes all current `rustfs-ecstore` library unit tests without parallel test +context cross-talk, not only focused EC filters. + +Coverage artifacts: + +- `target/ecstore-validation//coverage/ecstore/lcov.info` +- `target/ecstore-validation//coverage/ecstore/summary.tsv` +- `target/ecstore-validation//coverage/ecstore/files.tsv` + +Local validation on 2026-07-07 found that the current suite is not yet at the +target: full-crate unit line coverage was 69.32%; the EC-critical scope was +84.32% after the first negative read/write/recovery additions. The lowest +EC-critical files were `set_disk/ops/object.rs`, `io_primitives.rs`, +`disk/local.rs`, `bitrot.rs`, and `encode.rs`. These are gaps to close before +the default 95% gate can pass. + +## Initial Command Set + +Use the runner first: + +```bash +scripts/run_ecstore_validation_suite.sh --profile quick +scripts/run_ecstore_validation_suite.sh --profile full +scripts/run_ecstore_validation_suite.sh --profile destructive +scripts/run_ecstore_validation_suite.sh --profile fuzz +``` + +The split-run equivalent for the quick profile is: + +```bash +cargo test -p rustfs-filemeta --lib +cargo test -p rustfs-ecstore --lib erasure +cargo test -p rustfs-ecstore --lib set_disk::read +cargo test -p rustfs-ecstore --lib set_disk::core::io_primitives +cargo test -p rustfs-ecstore --lib set_disk::tests::test_rename_data_quorum_failure_rolls_back_destination_object +cargo test -p rustfs-ecstore --lib disk::local +cargo test -p rustfs-ecstore --lib -- --test-threads=1 +cargo test --package e2e_test reliability_disk_fault_test -- --nocapture +cargo test --package e2e_test heal_erasure_disk_rebuild_test -- --nocapture +cargo test --package e2e_test namespace_lock_quorum_test -- --nocapture +``` + +Fixture-backed tests should run when the fixture path is present: + +```bash +cargo test -p rustfs-ecstore --test legacy_bitrot_read_test -- --nocapture +cargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored --nocapture +``` + +## Multi-Expert Adversarial Review Summary + +Three independent read-only reviews were run for this design: + +| Reviewer | Main challenge | Resulting requirement | +| --- | --- | --- | +| Algorithm correctness | A single run only samples one timing/layout/hash combination. | matrix/property tests plus instrumented reader alignment checks | +| Black-box reliability | Existing tests miss distributed, shard-desync, range+fault, and migration-fault combinations. | add 4-node/16-disk and destructive profiles | +| Security and fault review | Metadata tamper, quorum downgrade, bitrot bypass, and rename atomicity must be end-to-end. | HIGH matrix rows block completion claims | + +Current verdict: design accepted as a target. The runner exists and captures +artifacts; implementation remains incomplete until the missing matrix rows have +artifact-backed evidence. diff --git a/scripts/run_ecstore_validation_suite.sh b/scripts/run_ecstore_validation_suite.sh new file mode 100755 index 000000000..23a9c218a --- /dev/null +++ b/scripts/run_ecstore_validation_suite.sh @@ -0,0 +1,541 @@ +#!/usr/bin/env bash +set -euo pipefail + +PROFILE="quick" +OUT_DIR="" +DRY_RUN=false +SKIP_E2E=false +SKIP_S3_TESTS=false +SKIP_COVERAGE=false +REQUIRE_FIXTURES=false +UNIT_COVERAGE_MIN="95" +UNIT_COVERAGE_TARGET="100" +UNIT_COVERAGE_SCOPE="ec-critical" + +usage() { + cat <<'USAGE' +Usage: + scripts/run_ecstore_validation_suite.sh [options] + +Profiles: + quick Core EC read/write/recovery checks for PR smoke (default) + full quick + s3-tests subset + fixture tests + coverage + destructive full + cluster/concurrency/destructive recovery checks + fuzz bounded fuzz smoke for malformed storage inputs + +Options: + --profile + --out-dir Default: target/ecstore-validation/ + --skip-e2e Skip e2e_test commands + --skip-s3-tests Skip scripts/s3-tests/run.sh in full/destructive + --skip-coverage Explicitly skip the mandatory unit coverage gate + --unit-coverage-min + Minimum rustfs-ecstore unit line coverage (default: 95) + --unit-coverage-scope + Coverage scope for the hard gate (default: ec-critical) + --require-fixtures Fail when optional EC fixture tests cannot run + --dry-run Print commands and write metadata without executing + -h, --help + +Artifacts: + /run-metadata.env + /summary.tsv + /blackbox-matrix.tsv + /logs/*.log +USAGE +} + +arg_value() { + local flag="$1" + local value="${2:-}" + if [[ -z "$value" || "$value" == --* ]]; then + echo "ERROR: missing value for $flag" >&2 + exit 1 + fi + printf '%s\n' "$value" +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --profile) PROFILE="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --skip-e2e) SKIP_E2E=true; shift ;; + --skip-s3-tests) SKIP_S3_TESTS=true; shift ;; + --skip-coverage) SKIP_COVERAGE=true; shift ;; + --unit-coverage-min) UNIT_COVERAGE_MIN="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --unit-coverage-scope) UNIT_COVERAGE_SCOPE="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --require-fixtures) REQUIRE_FIXTURES=true; shift ;; + --dry-run) DRY_RUN=true; shift ;; + -h|--help) usage; exit 0 ;; + *) + echo "ERROR: unknown arg: $1" >&2 + usage + exit 1 + ;; + esac + done +} + +validate_args() { + case "$PROFILE" in + quick|full|destructive|fuzz) ;; + *) + echo "ERROR: --profile must be quick, full, destructive, or fuzz" >&2 + exit 1 + ;; + esac + + if ! [[ "$UNIT_COVERAGE_MIN" =~ ^([0-9]|[1-9][0-9]|100)(\.[0-9]+)?$ ]]; then + echo "ERROR: --unit-coverage-min must be between 0 and 100" >&2 + exit 1 + fi + case "$UNIT_COVERAGE_SCOPE" in + ec-critical|crate) ;; + *) + echo "ERROR: --unit-coverage-scope must be ec-critical or crate" >&2 + exit 1 + ;; + esac +} + +setup_workspace() { + local root + root="$(git rev-parse --show-toplevel)" + cd "$root" + + if [[ -z "$OUT_DIR" ]]; then + OUT_DIR="target/ecstore-validation/$(date -u +%Y%m%dT%H%M%SZ)" + fi + mkdir -p "$OUT_DIR/logs" + + SUMMARY="$OUT_DIR/summary.tsv" + BLACKBOX_MATRIX="$OUT_DIR/blackbox-matrix.tsv" + printf 'status\tlabel\tlog\n' >"$SUMMARY" +} + +write_metadata() { + { + printf 'profile=%s\n' "$PROFILE" + printf 'started_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf 'dry_run=%s\n' "$DRY_RUN" + printf 'skip_e2e=%s\n' "$SKIP_E2E" + printf 'skip_s3_tests=%s\n' "$SKIP_S3_TESTS" + printf 'skip_coverage=%s\n' "$SKIP_COVERAGE" + printf 'unit_coverage_min=%s\n' "$UNIT_COVERAGE_MIN" + printf 'unit_coverage_target=%s\n' "$UNIT_COVERAGE_TARGET" + printf 'unit_coverage_scope=%s\n' "$UNIT_COVERAGE_SCOPE" + printf 'require_fixtures=%s\n' "$REQUIRE_FIXTURES" + printf 'git_commit=%s\n' "$(git rev-parse HEAD 2>/dev/null || true)" + printf 'git_branch=%s\n' "$(git branch --show-current 2>/dev/null || true)" + printf 'rustc=%s\n' "$(rustc --version 2>/dev/null || true)" + printf 'cargo=%s\n' "$(cargo --version 2>/dev/null || true)" + } >"$OUT_DIR/run-metadata.env" +} + +safe_label() { + printf '%s\n' "$1" | tr -c 'A-Za-z0-9_.-' '_' +} + +quote_command() { + local quoted=() + local arg + for arg in "$@"; do + quoted+=("$(printf '%q' "$arg")") + done + printf '%s ' "${quoted[@]}" +} + +record_skip() { + local label="$1" + local reason="$2" + local log="$OUT_DIR/logs/$(safe_label "$label").log" + printf 'SKIP: %s\n' "$reason" | tee "$log" + printf 'SKIP\t%s\t%s\n' "$label" "$log" >>"$SUMMARY" +} + +run_step() { + local label="$1" + shift + local log="$OUT_DIR/logs/$(safe_label "$label").log" + + { + printf 'label=%s\n' "$label" + printf 'command=' + quote_command "$@" + printf '\n' + } >"$log" + + if [[ "$DRY_RUN" == "true" ]]; then + printf 'DRY-RUN: ' + quote_command "$@" + printf '\n' + printf 'DRY_RUN\t%s\t%s\n' "$label" "$log" >>"$SUMMARY" + return 0 + fi + + printf 'RUN: %s\n' "$label" + if "$@" 2>&1 | tee -a "$log"; then + printf 'PASS\t%s\t%s\n' "$label" "$log" >>"$SUMMARY" + else + local status=$? + printf 'FAIL\t%s\t%s\n' "$label" "$log" >>"$SUMMARY" + return "$status" + fi +} + +run_core_unit_steps() { + run_step "filemeta-lib" cargo test -p rustfs-filemeta --lib + run_step "ecstore-erasure-lib" cargo test -p rustfs-ecstore --lib erasure + run_step "ecstore-set-disk-read-lib" cargo test -p rustfs-ecstore --lib set_disk::read + run_step "ecstore-io-primitives-lib" cargo test -p rustfs-ecstore --lib set_disk::core::io_primitives + run_step "ecstore-rename-rollback" \ + cargo test -p rustfs-ecstore --lib set_disk::tests::test_rename_data_quorum_failure_rolls_back_destination_object + run_step "ecstore-disk-local-lib" cargo test -p rustfs-ecstore --lib disk::local + run_step "ecstore-lib-all" cargo test -p rustfs-ecstore --lib -- --test-threads=1 +} + +run_quick_e2e_steps() { + if [[ "$SKIP_E2E" == "true" ]]; then + record_skip "e2e-quick" "--skip-e2e was set" + return + fi + + run_step "e2e-reliability-disk-fault" cargo test --package e2e_test reliability_disk_fault_test -- --nocapture + run_step "e2e-heal-erasure-disk-rebuild" cargo test --package e2e_test heal_erasure_disk_rebuild_test -- --nocapture + run_step "e2e-namespace-lock-quorum" cargo test --package e2e_test namespace_lock_quorum_test -- --nocapture +} + +run_quick_profile() { + run_core_unit_steps + run_quick_e2e_steps +} + +fixture_available() { + [[ -n "${RUSTFS_MINIO_FIXTURE_ROOT:-}" && -d "${RUSTFS_MINIO_FIXTURE_ROOT}" && -n "${RUSTFS_MINIO_STATIC_KMS_KEY_B64:-}" ]] +} + +legacy_fixture_available() { + local root="${RUSTFS_LEGACY_TEST_ROOT:-$(pwd)}" + local disk="${RUSTFS_LEGACY_TEST_DISK:-}" + if [[ -z "$disk" ]]; then + if [[ -f "$root/test/.minio.sys/format.json" ]]; then + disk="test" + else + disk="test1" + fi + fi + + if [[ ! -f "$root/$disk/.rustfs.sys/format.json" && ! -f "$root/$disk/.minio.sys/format.json" ]]; then + return 1 + fi + + [[ -f "$root/$disk/vvvv/ktvzip.tar.gz/xl.meta" || -f "$root/$disk/vvvv/path_traversal.md/xl.meta" ]] +} + +minio_fixture_missing_reason() { + if [[ -z "${RUSTFS_MINIO_FIXTURE_ROOT:-}" ]]; then + printf 'RUSTFS_MINIO_FIXTURE_ROOT is not set\n' + elif [[ ! -d "${RUSTFS_MINIO_FIXTURE_ROOT}" ]]; then + printf 'RUSTFS_MINIO_FIXTURE_ROOT does not exist: %s\n' "$RUSTFS_MINIO_FIXTURE_ROOT" + elif [[ -z "${RUSTFS_MINIO_STATIC_KMS_KEY_B64:-}" ]]; then + printf 'RUSTFS_MINIO_STATIC_KMS_KEY_B64 is not set\n' + else + printf 'unknown MinIO fixture gate failure\n' + fi +} + +write_blackbox_matrix() { + local e2e_status="enabled" + local s3_status="enabled" + local destructive_status="enabled" + local legacy_status="missing-optional" + local minio_status="missing-optional" + if [[ "$SKIP_E2E" == "true" ]]; then + e2e_status="disabled" + destructive_status="disabled" + fi + if [[ "$SKIP_S3_TESTS" == "true" ]]; then + s3_status="disabled" + fi + if legacy_fixture_available; then + legacy_status="enabled" + elif [[ "$REQUIRE_FIXTURES" == "true" ]]; then + legacy_status="missing-required" + fi + if fixture_available; then + minio_status="enabled" + elif [[ "$REQUIRE_FIXTURES" == "true" ]]; then + minio_status="missing-required" + fi + + { + printf 'profile\tscenario\tgate\tcommand\tfixture_env\tstatus\n' + printf 'quick\tsingle-node disk fault read/write\tblack-box\tcargo test --package e2e_test reliability_disk_fault_test -- --nocapture\tnone\t%s\n' "$e2e_status" + printf 'quick\theal degraded erasure disk rebuild\tblack-box\tcargo test --package e2e_test heal_erasure_disk_rebuild_test -- --nocapture\tnone\t%s\n' "$e2e_status" + printf 'quick\tnamespace lock quorum under EC ops\tblack-box\tcargo test --package e2e_test namespace_lock_quorum_test -- --nocapture\tnone\t%s\n' "$e2e_status" + printf 'full\tlegacy bitrot read fixture restore\tfixture\tcargo test -p rustfs-ecstore --test legacy_bitrot_read_test -- --nocapture\tRUSTFS_LEGACY_TEST_ROOT,RUSTFS_LEGACY_TEST_DISK\t%s\n' "$legacy_status" + printf 'full\tMinIO generated encrypted read and negative restore fixture\tfixture\tcargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored --nocapture\tRUSTFS_MINIO_FIXTURE_ROOT,RUSTFS_MINIO_STATIC_KMS_KEY_B64\t%s\n' "$minio_status" + printf 'full\tS3 multipart range versioning delete subset\tblack-box\tenv TESTEXPR=\"multipart or range or versioning or delete\" DEPLOY_MODE=build MAXFAIL=0 ./scripts/s3-tests/run.sh\tnone\t%s\n' "$s3_status" + printf 'destructive\tdistributed cluster concurrency\tblack-box\tcargo test --package e2e_test cluster_concurrency_test -- --nocapture\tnone\t%s\n' "$destructive_status" + printf 'destructive\tstale multipart cleanup cluster\tblack-box\tcargo test --package e2e_test stale_multipart_cleanup_cluster_test -- --nocapture\tnone\t%s\n' "$destructive_status" + printf 'destructive\tdelete marker migration semantics\tblack-box\tcargo test --package e2e_test delete_marker_migration_semantics_test -- --nocapture\tnone\t%s\n' "$destructive_status" + } >"$BLACKBOX_MATRIX" +} + +run_fixture_gate() { + local label="ecstore-fixture-gate" + local log="$OUT_DIR/logs/$(safe_label "$label").log" + local legacy_ok=false + local minio_ok=false + if legacy_fixture_available; then + legacy_ok=true + fi + if fixture_available; then + minio_ok=true + fi + + { + printf 'label=%s\n' "$label" + printf 'legacy_fixture_available=%s\n' "$legacy_ok" + printf 'minio_fixture_available=%s\n' "$minio_ok" + printf 'require_fixtures=%s\n' "$REQUIRE_FIXTURES" + if [[ "$minio_ok" != "true" ]]; then + printf 'minio_fixture_reason=%s\n' "$(minio_fixture_missing_reason)" + fi + } | tee "$log" + + if [[ "$DRY_RUN" == "true" ]]; then + printf 'DRY_RUN\t%s\t%s\n' "$label" "$log" >>"$SUMMARY" + return 0 + fi + + if [[ "$legacy_ok" == "true" && "$minio_ok" == "true" ]]; then + printf 'PASS\t%s\t%s\n' "$label" "$log" >>"$SUMMARY" + return 0 + fi + + if [[ "$REQUIRE_FIXTURES" == "true" ]]; then + printf 'FAIL\t%s\t%s\n' "$label" "$log" >>"$SUMMARY" + return 1 + fi + + printf 'SKIP\t%s\t%s\n' "$label" "$log" >>"$SUMMARY" +} + +run_fixture_steps() { + if legacy_fixture_available; then + run_step "ecstore-legacy-bitrot-read-fixture" cargo test -p rustfs-ecstore --test legacy_bitrot_read_test -- --nocapture + elif [[ "$REQUIRE_FIXTURES" == "true" ]]; then + echo "ERROR: RUSTFS_LEGACY_TEST_ROOT/RUSTFS_LEGACY_TEST_DISK is required for legacy bitrot fixture tests" >&2 + exit 1 + else + record_skip "ecstore-legacy-bitrot-read-fixture" "legacy bitrot fixture is not present" + fi + + if fixture_available; then + run_step "ecstore-minio-generated-read-fixture" \ + cargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored --nocapture + elif [[ "$REQUIRE_FIXTURES" == "true" ]]; then + echo "ERROR: $(minio_fixture_missing_reason)" >&2 + exit 1 + else + record_skip "ecstore-minio-generated-read-fixture" "$(minio_fixture_missing_reason)" + fi +} + +run_s3_subset() { + if [[ "$SKIP_S3_TESTS" == "true" ]]; then + record_skip "s3-tests-ec-subset" "--skip-s3-tests was set" + return + fi + + run_step "s3-tests-ec-subset" \ + env TESTEXPR="multipart or range or versioning or delete" DEPLOY_MODE=build MAXFAIL=0 ./scripts/s3-tests/run.sh +} + +write_coverage_reports() { + local lcov_path="$1" + local coverage_summary="$2" + local coverage_files="$3" + local root="$4" + + awk -v root="$root" -v scope="$UNIT_COVERAGE_SCOPE" -v summary="$coverage_summary" -v files="$coverage_files" ' + function is_crate_file(f) { + return index(f, root "/crates/ecstore/src/") == 1 + } + function is_ec_critical_file(f) { + return index(f, root "/crates/ecstore/src/erasure/") == 1 || + f == root "/crates/ecstore/src/set_disk/read.rs" || + f == root "/crates/ecstore/src/set_disk/shard_source.rs" || + f == root "/crates/ecstore/src/set_disk/metadata.rs" || + f == root "/crates/ecstore/src/set_disk/ops/object.rs" || + f == root "/crates/ecstore/src/set_disk/core/io_primitives.rs" || + f == root "/crates/ecstore/src/disk/local.rs" + } + function is_gate_file(f) { + return scope == "crate" ? is_crate_file(f) : is_ec_critical_file(f) + } + function pct(hit, found) { + return found == 0 ? 0 : (hit / found) * 100 + } + function flush() { + if (file == "") { + return + } + if (is_crate_file(file)) { + crate_hit += hit + crate_found += found + } + if (is_ec_critical_file(file)) { + ec_hit += hit + ec_found += found + } + if (is_gate_file(file)) { + gate_hit += hit + gate_found += found + } + if (is_crate_file(file)) { + printf "%s\t%.2f\t%d\t%d\n", file, pct(hit, found), hit, found >> files + } + } + BEGIN { + print "file\tline_coverage\thit\tfound" > files + } + /^SF:/ { + flush() + file = substr($0, 4) + hit = 0 + found = 0 + next + } + /^LH:/ { hit += substr($0, 4); next } + /^LF:/ { found += substr($0, 4); next } + END { + flush() + if (gate_found == 0) { + exit 2 + } + print "metric\tactual\tmin\ttarget\tscope" > summary + printf "gate_line_coverage\t%.2f\t%s\t%s\t%s\n", pct(gate_hit, gate_found), min, target, scope >> summary + printf "ec_critical_line_coverage\t%.2f\t%s\t%s\tec-critical\n", pct(ec_hit, ec_found), min, target >> summary + printf "crate_line_coverage\t%.2f\t%s\t%s\tcrate\n", pct(crate_hit, crate_found), min, target >> summary + } + ' min="$UNIT_COVERAGE_MIN" target="$UNIT_COVERAGE_TARGET" "$lcov_path" +} + +run_coverage() { + if [[ "$SKIP_COVERAGE" == "true" ]]; then + record_skip "ecstore-coverage" "--skip-coverage was set" + return + fi + + local label="ecstore-unit-coverage" + local log="$OUT_DIR/logs/$(safe_label "$label").log" + local coverage_dir="$OUT_DIR/coverage/ecstore" + local lcov_path="$coverage_dir/lcov.info" + local coverage_summary="$coverage_dir/summary.tsv" + local coverage_files="$coverage_dir/files.tsv" + local cmd=(cargo llvm-cov -p rustfs-ecstore --lib --lcov --output-path "$lcov_path" -- --test-threads=1) + local root + root="$(pwd)" + mkdir -p "$coverage_dir" + + { + printf 'label=%s\n' "$label" + printf 'unit_coverage_min=%s\n' "$UNIT_COVERAGE_MIN" + printf 'unit_coverage_target=%s\n' "$UNIT_COVERAGE_TARGET" + printf 'unit_coverage_scope=%s\n' "$UNIT_COVERAGE_SCOPE" + printf 'command=' + quote_command "${cmd[@]}" + printf '\n' + } >"$log" + + if [[ "$DRY_RUN" == "true" ]]; then + printf 'DRY-RUN: ' + quote_command "${cmd[@]}" + printf '\n' + printf 'DRY_RUN\t%s\t%s\n' "$label" "$log" >>"$SUMMARY" + return + fi + + if ! command -v cargo-llvm-cov >/dev/null 2>&1; then + printf 'ERROR: cargo-llvm-cov is required for the ECStore unit coverage gate\n' | tee -a "$log" + printf 'FAIL\t%s\t%s\n' "$label" "$log" >>"$SUMMARY" + return 127 + fi + + printf 'RUN: %s\n' "$label" + if "${cmd[@]}" 2>&1 | tee -a "$log"; then + : + else + local status=$? + printf 'FAIL\t%s\t%s\n' "$label" "$log" >>"$SUMMARY" + return "$status" + fi + + if ! write_coverage_reports "$lcov_path" "$coverage_summary" "$coverage_files" "$root"; then + printf 'ERROR: unable to compute scoped coverage from %s\n' "$lcov_path" | tee -a "$log" + printf 'FAIL\t%s\t%s\n' "$label" "$log" >>"$SUMMARY" + return 1 + fi + + local line_coverage + line_coverage="$(awk -F '\t' '$1 == "gate_line_coverage" { print $2 }' "$coverage_summary")" + printf 'gate_line_coverage=%s%% scope=%s min=%s%% target=%s%%\n' \ + "$line_coverage" "$UNIT_COVERAGE_SCOPE" "$UNIT_COVERAGE_MIN" "$UNIT_COVERAGE_TARGET" | tee -a "$log" + printf 'coverage_summary=%s\ncoverage_files=%s\n' "$coverage_summary" "$coverage_files" | tee -a "$log" + + if awk -v actual="$line_coverage" -v min="$UNIT_COVERAGE_MIN" 'BEGIN { exit !(actual + 0 >= min + 0) }'; then + printf 'PASS\t%s\t%s\n' "$label" "$log" >>"$SUMMARY" + else + printf 'FAIL: %s coverage %s%% is below %s%%\n' "$UNIT_COVERAGE_SCOPE" "$line_coverage" "$UNIT_COVERAGE_MIN" | tee -a "$log" + printf 'FAIL\t%s\t%s\n' "$label" "$log" >>"$SUMMARY" + return 1 + fi +} + +run_full_profile() { + run_quick_profile + run_fixture_gate + run_fixture_steps + run_s3_subset + run_coverage +} + +run_destructive_profile() { + run_full_profile + if [[ "$SKIP_E2E" == "true" ]]; then + record_skip "e2e-destructive" "--skip-e2e was set" + return + fi + + run_step "e2e-cluster-concurrency" cargo test --package e2e_test cluster_concurrency_test -- --nocapture + run_step "e2e-stale-multipart-cleanup-cluster" cargo test --package e2e_test stale_multipart_cleanup_cluster_test -- --nocapture + run_step "e2e-delete-marker-migration-semantics" cargo test --package e2e_test delete_marker_migration_semantics_test -- --nocapture +} + +run_fuzz_profile() { + local max_total_time="${MAX_TOTAL_TIME:-60}" + run_step "fuzz-storage-inputs" env MAX_TOTAL_TIME="$max_total_time" ./scripts/fuzz/run.sh +} + +main() { + parse_args "$@" + validate_args + setup_workspace + write_metadata + write_blackbox_matrix + + case "$PROFILE" in + quick) run_quick_profile ;; + full) run_full_profile ;; + destructive) run_destructive_profile ;; + fuzz) run_fuzz_profile ;; + esac + + printf 'summary=%s\n' "$SUMMARY" + printf 'artifacts=%s\n' "$OUT_DIR" +} + +main "$@"