mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
test(ecstore): add validation suite coverage gates (#4378)
This commit is contained in:
Zhengchao An
@@ -709,15 +709,109 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct FailingWriteWriter;
|
||||
|
||||
impl AsyncWrite for FailingWriteWriter {
|
||||
fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll<std::io::Result<usize>> {
|
||||
Poll::Ready(Err(std::io::Error::other("injected write failure")))
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct ShortWriteWriter;
|
||||
|
||||
impl AsyncWrite for ShortWriteWriter {
|
||||
fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
|
||||
Poll::Ready(Ok(buf.len().saturating_sub(1)))
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct ShutdownFailWriter {
|
||||
buffered: Vec<u8>,
|
||||
}
|
||||
|
||||
impl AsyncWrite for ShutdownFailWriter {
|
||||
fn poll_write(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
|
||||
self.buffered.extend_from_slice(buf);
|
||||
Poll::Ready(Ok(buf.len()))
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Poll::Ready(Err(std::io::Error::other("injected shutdown failure")))
|
||||
}
|
||||
}
|
||||
|
||||
fn bitrot_writer<W>(writer: W, shard_size: usize) -> BitrotWriterWrapper
|
||||
where
|
||||
W: AsyncWrite + Send + Sync + Unpin + 'static,
|
||||
{
|
||||
BitrotWriterWrapper::new(CustomWriter::new_tokio_writer(writer), shard_size, HashAlgorithm::HighwayHash256S)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multi_writer_short_write_fails_before_shutdown() {
|
||||
let mut writers = vec![Some(bitrot_writer(ShortWriteWriter, 16))];
|
||||
let err = {
|
||||
let mut writer = MultiWriter::new(&mut writers, 1);
|
||||
writer
|
||||
.write(vec![Bytes::from_static(b"short-write payload")])
|
||||
.await
|
||||
.expect_err("short writes must fail the shard writer")
|
||||
};
|
||||
|
||||
assert!(err.to_string().contains("Failed to write data"));
|
||||
assert!(writers[0].is_none(), "short-write shard must be removed before commit");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn drain_queued_inflight_bytes_consumes_pending_blocks() {
|
||||
let (tx, mut rx) = mpsc::channel(2);
|
||||
tx.send(vec![Bytes::from_static(b"queued")]).await.unwrap();
|
||||
drop(tx);
|
||||
|
||||
drain_queued_inflight_bytes(&mut rx).await;
|
||||
|
||||
assert!(rx.recv().await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn drain_queued_batched_inflight_bytes_consumes_pending_batches() {
|
||||
let (tx, mut rx) = mpsc::channel(2);
|
||||
tx.send(vec![vec![Bytes::from_static(b"queued")]]).await.unwrap();
|
||||
drop(tx);
|
||||
|
||||
drain_queued_batched_inflight_bytes(&mut rx).await;
|
||||
|
||||
assert!(rx.recv().await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn encode_shutdowns_writers_after_small_shards() {
|
||||
let committed = Arc::new(Mutex::new(Vec::new()));
|
||||
let writer = DeferredCommitWriter::new(committed.clone());
|
||||
let mut writers = vec![Some(BitrotWriterWrapper::new(
|
||||
CustomWriter::new_tokio_writer(writer),
|
||||
16,
|
||||
HashAlgorithm::HighwayHash256S,
|
||||
))];
|
||||
let mut writers = vec![Some(bitrot_writer(writer, 16))];
|
||||
|
||||
let erasure = Arc::new(Erasure::new(1, 0, 16));
|
||||
let reader = tokio::io::BufReader::new(Cursor::new(b"small payload".to_vec()));
|
||||
@@ -727,6 +821,87 @@ mod tests {
|
||||
assert!(!committed.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn encode_streaming_write_quorum_failure_aborts_and_reports_error() {
|
||||
const DATA_SHARDS: usize = 2;
|
||||
const PARITY_SHARDS: usize = 2;
|
||||
const BLOCK_SIZE: usize = 64;
|
||||
|
||||
let committed = Arc::new(Mutex::new(Vec::new()));
|
||||
let mut writers = vec![
|
||||
Some(bitrot_writer(DeferredCommitWriter::new(committed.clone()), BLOCK_SIZE / DATA_SHARDS)),
|
||||
Some(bitrot_writer(FailingWriteWriter, BLOCK_SIZE / DATA_SHARDS)),
|
||||
None,
|
||||
None,
|
||||
];
|
||||
|
||||
let payload = vec![3u8; BLOCK_SIZE * 8];
|
||||
let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
|
||||
let reader = tokio::io::BufReader::new(Cursor::new(payload));
|
||||
let err = erasure
|
||||
.encode(reader, &mut writers, DATA_SHARDS)
|
||||
.await
|
||||
.expect_err("streaming encode must fail when write quorum is unavailable");
|
||||
|
||||
assert!(err.to_string().contains("Failed to write data"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn encode_inline_small_write_quorum_failure_does_not_commit_partial_data() {
|
||||
const DATA_SHARDS: usize = 2;
|
||||
const PARITY_SHARDS: usize = 2;
|
||||
const BLOCK_SIZE: usize = 64;
|
||||
|
||||
let committed = Arc::new(Mutex::new(Vec::new()));
|
||||
let mut writers = vec![
|
||||
Some(bitrot_writer(DeferredCommitWriter::new(committed.clone()), BLOCK_SIZE / DATA_SHARDS)),
|
||||
Some(bitrot_writer(FailingWriteWriter, BLOCK_SIZE / DATA_SHARDS)),
|
||||
None,
|
||||
None,
|
||||
];
|
||||
|
||||
let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
|
||||
let reader = tokio::io::BufReader::new(Cursor::new(b"write quorum failure payload".to_vec()));
|
||||
let err = erasure
|
||||
.encode_inline_small(reader, &mut writers, DATA_SHARDS)
|
||||
.await
|
||||
.expect_err("write quorum failure must fail the inline encode");
|
||||
|
||||
assert!(err.to_string().contains("Failed to write data"));
|
||||
assert!(
|
||||
committed.lock().expect("committed buffer should be lockable").is_empty(),
|
||||
"successful writer must not be committed when write quorum fails before shutdown"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn encode_inline_small_shutdown_quorum_failure_is_reported() {
|
||||
const DATA_SHARDS: usize = 2;
|
||||
const PARITY_SHARDS: usize = 2;
|
||||
const BLOCK_SIZE: usize = 64;
|
||||
|
||||
let committed = Arc::new(Mutex::new(Vec::new()));
|
||||
let mut writers = vec![
|
||||
Some(bitrot_writer(DeferredCommitWriter::new(committed.clone()), BLOCK_SIZE / DATA_SHARDS)),
|
||||
Some(bitrot_writer(ShutdownFailWriter::default(), BLOCK_SIZE / DATA_SHARDS)),
|
||||
Some(bitrot_writer(ShutdownFailWriter::default(), BLOCK_SIZE / DATA_SHARDS)),
|
||||
Some(bitrot_writer(ShutdownFailWriter::default(), BLOCK_SIZE / DATA_SHARDS)),
|
||||
];
|
||||
|
||||
let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
|
||||
let reader = tokio::io::BufReader::new(Cursor::new(b"shutdown quorum failure payload".to_vec()));
|
||||
let err = erasure
|
||||
.encode_inline_small(reader, &mut writers, DATA_SHARDS)
|
||||
.await
|
||||
.expect_err("shutdown quorum failure must fail the inline encode");
|
||||
|
||||
assert!(err.to_string().contains("Failed to shutdown writers"));
|
||||
assert!(
|
||||
!committed.lock().expect("committed buffer should be lockable").is_empty(),
|
||||
"the successful writer should have committed before shutdown quorum failure was reported"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn encode_returns_unexpected_eof_for_truncated_limited_reader() {
|
||||
let committed = Arc::new(Mutex::new(Vec::new()));
|
||||
@@ -773,11 +948,7 @@ mod tests {
|
||||
async fn encode_works_on_current_thread_runtime() {
|
||||
let committed = Arc::new(Mutex::new(Vec::new()));
|
||||
let writer = DeferredCommitWriter::new(committed);
|
||||
let mut writers = vec![Some(BitrotWriterWrapper::new(
|
||||
CustomWriter::new_tokio_writer(writer),
|
||||
16,
|
||||
HashAlgorithm::HighwayHash256S,
|
||||
))];
|
||||
let mut writers = vec![Some(bitrot_writer(writer, 16))];
|
||||
|
||||
let erasure = Arc::new(Erasure::new(1, 0, 16));
|
||||
let reader = tokio::io::BufReader::new(Cursor::new(b"current-thread payload".to_vec()));
|
||||
@@ -786,6 +957,78 @@ mod tests {
|
||||
assert_eq!(written, b"current-thread payload".len());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn encode_batched_writes_full_and_tail_batches() {
|
||||
const DATA_SHARDS: usize = 2;
|
||||
const PARITY_SHARDS: usize = 2;
|
||||
const TOTAL_SHARDS: usize = DATA_SHARDS + PARITY_SHARDS;
|
||||
const BLOCK_SIZE: usize = 32;
|
||||
|
||||
let committed: Vec<Arc<Mutex<Vec<u8>>>> = (0..TOTAL_SHARDS).map(|_| Arc::new(Mutex::new(Vec::new()))).collect();
|
||||
let mut writers: Vec<Option<BitrotWriterWrapper>> = committed
|
||||
.iter()
|
||||
.map(|c| Some(bitrot_writer(DeferredCommitWriter::new(c.clone()), BLOCK_SIZE / DATA_SHARDS)))
|
||||
.collect();
|
||||
|
||||
let payload = vec![7u8; BLOCK_SIZE * 5 + 3];
|
||||
let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
|
||||
let reader = tokio::io::BufReader::new(Cursor::new(payload.clone()));
|
||||
let (_reader, total) = erasure
|
||||
.encode_batched(reader, &mut writers, DATA_SHARDS)
|
||||
.await
|
||||
.expect("batched encode should write full and tail batches");
|
||||
|
||||
assert_eq!(total, payload.len());
|
||||
for (index, committed) in committed.iter().enumerate() {
|
||||
assert!(
|
||||
!committed.lock().expect("committed buffer should be lockable").is_empty(),
|
||||
"shard {index} should receive committed batched data"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn encode_batched_write_quorum_failure_aborts_and_reports_error() {
|
||||
const DATA_SHARDS: usize = 2;
|
||||
const PARITY_SHARDS: usize = 2;
|
||||
const BLOCK_SIZE: usize = 32;
|
||||
|
||||
let committed = Arc::new(Mutex::new(Vec::new()));
|
||||
let mut writers = vec![
|
||||
Some(bitrot_writer(DeferredCommitWriter::new(committed.clone()), BLOCK_SIZE / DATA_SHARDS)),
|
||||
Some(bitrot_writer(FailingWriteWriter, BLOCK_SIZE / DATA_SHARDS)),
|
||||
None,
|
||||
None,
|
||||
];
|
||||
|
||||
let payload = vec![9u8; BLOCK_SIZE * 8];
|
||||
let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
|
||||
let reader = tokio::io::BufReader::new(Cursor::new(payload));
|
||||
let err = erasure
|
||||
.encode_batched(reader, &mut writers, DATA_SHARDS)
|
||||
.await
|
||||
.expect_err("batched encode must fail when write quorum is unavailable");
|
||||
|
||||
assert!(err.to_string().contains("Failed to write data"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn encode_batched_rejects_zero_block_size() {
|
||||
let committed = Arc::new(Mutex::new(Vec::new()));
|
||||
let writer = DeferredCommitWriter::new(committed);
|
||||
let mut writers = vec![Some(bitrot_writer(writer, 16))];
|
||||
|
||||
let erasure = Arc::new(Erasure::new(1, 0, 0));
|
||||
let reader = tokio::io::BufReader::new(Cursor::new(b"payload".to_vec()));
|
||||
let err = erasure
|
||||
.encode_batched(reader, &mut writers, 1)
|
||||
.await
|
||||
.expect_err("zero block size must be rejected");
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
|
||||
assert!(err.to_string().contains("block_size"));
|
||||
}
|
||||
|
||||
/// encode_inline_small: empty reader returns (reader, 0) without writing to any shard.
|
||||
#[tokio::test]
|
||||
async fn encode_inline_small_empty_stream_returns_zero() {
|
||||
|
||||
@@ -1056,6 +1056,80 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_data_rejects_missing_data_above_parity_budget() {
|
||||
for uses_legacy in [false, true] {
|
||||
let erasure = Erasure::new_with_options(3, 2, 64, uses_legacy);
|
||||
let data = b"read restore must fail when too many data shards are gone";
|
||||
let encoded = erasure.encode_data(data).expect("encode should succeed");
|
||||
let mut shards = optional_shards(&encoded);
|
||||
shards[0] = None;
|
||||
shards[1] = None;
|
||||
shards[2] = None;
|
||||
|
||||
let err = erasure
|
||||
.decode_data(&mut shards)
|
||||
.expect_err("decode_data must fail when missing data shards exceed parity");
|
||||
|
||||
assert!(!err.to_string().is_empty());
|
||||
assert!(shards.iter().take(erasure.data_shards).any(Option::is_none));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_data_and_parity_rejects_wrong_shard_count() {
|
||||
for uses_legacy in [false, true] {
|
||||
let erasure = Erasure::new_with_options(4, 2, 128, uses_legacy);
|
||||
let data = b"wrong shard count should not be accepted as a restored object";
|
||||
let encoded = erasure.encode_data(data).expect("encode should succeed");
|
||||
let mut shards = optional_shards(&encoded);
|
||||
let _ = shards.pop();
|
||||
|
||||
let err = erasure
|
||||
.decode_data_and_parity(&mut shards)
|
||||
.expect_err("decode_data_and_parity must reject wrong shard count");
|
||||
|
||||
assert!(!err.to_string().is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_data_with_verification_rejects_corrupt_surplus_parity() {
|
||||
let erasure = Erasure::new(3, 2, 128);
|
||||
let data = b"verified reads must fail closed on stale surplus parity";
|
||||
let encoded = erasure.encode_data(data).expect("encode should succeed");
|
||||
let mut shards = optional_shards(&encoded);
|
||||
shards[1] = None;
|
||||
shards[erasure.data_shards].as_mut().expect("parity shard should be present")[0] ^= 0x7d;
|
||||
|
||||
let err = erasure
|
||||
.decode_data_with_reconstruction_verification(&mut shards)
|
||||
.expect_err("verified decode must reject inconsistent parity");
|
||||
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_data_and_parity_rejects_missing_and_mismatched_shards() {
|
||||
let erasure = Erasure::new(4, 2, 128);
|
||||
let data = b"verification must reject incomplete or malformed shard sets";
|
||||
let encoded = erasure.encode_data(data).expect("encode should succeed");
|
||||
let mut shards = optional_shards(&encoded);
|
||||
|
||||
shards[0] = None;
|
||||
let missing = erasure
|
||||
.verify_data_and_parity(&shards)
|
||||
.expect_err("verification must reject missing shards");
|
||||
assert!(missing.to_string().contains("missing shard"));
|
||||
|
||||
let mut mismatched = optional_shards(&encoded);
|
||||
let _ = mismatched[0].as_mut().expect("data shard should be present").pop();
|
||||
let mismatch = erasure
|
||||
.verify_data_and_parity(&mismatched)
|
||||
.expect_err("verification must reject mismatched shard lengths");
|
||||
assert!(mismatch.to_string().contains("inconsistent shard length"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_data_and_parity_leaves_complete_shards_unchanged() {
|
||||
let erasure = Erasure::new(4, 2, 128);
|
||||
|
||||
@@ -3427,6 +3427,43 @@ impl SetDisks {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Cursor;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
fn metadata_test_fileinfo(object: &str) -> FileInfo {
|
||||
let mut fi = FileInfo::new(object, 2, 2);
|
||||
fi.volume = "bucket".to_string();
|
||||
fi.name = object.to_string();
|
||||
fi.size = 1;
|
||||
fi.erasure.index = 1;
|
||||
fi.metadata.insert("etag".to_string(), "etag-1".to_string());
|
||||
fi.add_object_part(1, "part-etag-1".to_string(), 1, None, 1, None, None);
|
||||
fi
|
||||
}
|
||||
|
||||
fn read_part_test_part(number: usize, etag: &str) -> ObjectPartInfo {
|
||||
ObjectPartInfo {
|
||||
number,
|
||||
etag: etag.to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn read_part_test_error(number: usize, error: &str) -> ObjectPartInfo {
|
||||
ObjectPartInfo {
|
||||
number,
|
||||
error: Some(error.to_string()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn failed_read_repair_submitter(_request: rustfs_common::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture {
|
||||
Box::pin(async { ReadRepairAdmissionOutcome::Failed("injected submit failure".to_string()) })
|
||||
}
|
||||
|
||||
fn accepted_read_repair_submitter(_request: rustfs_common::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture {
|
||||
Box::pin(async { ReadRepairAdmissionOutcome::Response(HealAdmissionResult::Accepted) })
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dangling_delete_grace_defaults_to_one_hour() {
|
||||
@@ -3444,4 +3481,224 @@ mod tests {
|
||||
assert!(dangling_delete_grace().is_zero());
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_codec_streaming_reader_zero_buffer_is_noop() {
|
||||
let reader = tokio::io::BufReader::new(Cursor::new(b"payload".to_vec()));
|
||||
let mut reader = MultipartCodecStreamingReader::new(vec![Box::new(reader)]);
|
||||
let mut out = [];
|
||||
|
||||
let read = reader
|
||||
.read(&mut out)
|
||||
.await
|
||||
.expect("zero-length read should not poll inner readers");
|
||||
|
||||
assert_eq!(read, 0);
|
||||
assert_eq!(reader.readers.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_fanout_observation_classifies_invalid_and_ignored_results() {
|
||||
let invalid = MetadataFanoutObservation::from_file_info(&FileInfo::default(), Duration::from_millis(7));
|
||||
assert_eq!(invalid.outcome, GET_METADATA_RESPONSE_ERROR);
|
||||
assert!(!invalid.valid);
|
||||
assert!(!invalid.ignored);
|
||||
|
||||
let ignored = MetadataFanoutObservation::from_error(&DiskError::DiskNotFound, Duration::from_millis(9));
|
||||
assert_eq!(ignored.outcome, GET_METADATA_RESPONSE_DISK_NOT_FOUND);
|
||||
assert!(!ignored.valid);
|
||||
assert!(ignored.ignored);
|
||||
|
||||
let corrupt = MetadataFanoutObservation::from_error(&DiskError::FileCorrupt, Duration::from_millis(11));
|
||||
assert_eq!(corrupt.outcome, GET_METADATA_RESPONSE_CORRUPT);
|
||||
assert!(!corrupt.ignored);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_fanout_diagnostics_reports_counts_and_latency_edges() {
|
||||
let diagnostics = MetadataFanoutDiagnostics::new(
|
||||
Duration::from_millis(40),
|
||||
vec![
|
||||
MetadataFanoutObservation {
|
||||
outcome: GET_METADATA_RESPONSE_VALID,
|
||||
elapsed: Duration::from_millis(30),
|
||||
valid: true,
|
||||
ignored: false,
|
||||
},
|
||||
MetadataFanoutObservation {
|
||||
outcome: GET_METADATA_RESPONSE_IGNORED,
|
||||
elapsed: Duration::from_millis(10),
|
||||
valid: false,
|
||||
ignored: true,
|
||||
},
|
||||
MetadataFanoutObservation {
|
||||
outcome: GET_METADATA_RESPONSE_ERROR,
|
||||
elapsed: Duration::from_millis(20),
|
||||
valid: false,
|
||||
ignored: false,
|
||||
},
|
||||
],
|
||||
);
|
||||
|
||||
assert_eq!(diagnostics.total_responses(), 3);
|
||||
assert_eq!(diagnostics.valid_responses(), 1);
|
||||
assert_eq!(diagnostics.ignored_responses(), 1);
|
||||
assert_eq!(diagnostics.error_responses(), 2);
|
||||
assert_eq!(diagnostics.first_response_latency(), Some(Duration::from_millis(10)));
|
||||
assert_eq!(diagnostics.first_valid_response_latency(), Some(Duration::from_millis(30)));
|
||||
assert_eq!(diagnostics.slowest_response_latency(), Some(Duration::from_millis(30)));
|
||||
assert_eq!(diagnostics.quorum_candidate_latency(0), Some(Duration::ZERO));
|
||||
assert_eq!(diagnostics.quorum_candidate_latency(1), Some(Duration::from_millis(30)));
|
||||
assert_eq!(diagnostics.quorum_candidate_latency(2), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_quorum_accumulator_counts_invalid_metadata_and_ignored_errors() {
|
||||
let mut accumulator = MetadataQuorumAccumulator::new(4, 2, true);
|
||||
|
||||
accumulator.observe_file_info(&FileInfo::default());
|
||||
accumulator.observe_error(&DiskError::DiskNotFound);
|
||||
|
||||
assert_eq!(accumulator.hard_errors, 1);
|
||||
assert_eq!(accumulator.ignored_errors, 1);
|
||||
assert_eq!(accumulator.final_miss_reason(), GET_METADATA_EARLY_STOP_REASON_ERROR);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_quorum_accumulator_candidate_quorum_handles_zero_parity_and_invalid_candidates() {
|
||||
let accumulator = MetadataQuorumAccumulator::new(4, 0, true);
|
||||
let candidate = metadata_test_fileinfo("object");
|
||||
assert_eq!(accumulator.candidate_read_quorum(&candidate), Some(4));
|
||||
assert_eq!(accumulator.missing_response_quorum(), 4);
|
||||
|
||||
let accumulator = MetadataQuorumAccumulator::new(4, 2, true);
|
||||
let mut deleted = candidate.clone();
|
||||
deleted.deleted = true;
|
||||
assert_eq!(accumulator.candidate_read_quorum(&deleted), None);
|
||||
|
||||
let mut empty = candidate.clone();
|
||||
empty.size = 0;
|
||||
assert_eq!(accumulator.candidate_read_quorum(&empty), None);
|
||||
|
||||
let mut impossible_parity = candidate;
|
||||
impossible_parity.erasure.parity_blocks = 4;
|
||||
assert_eq!(accumulator.candidate_read_quorum(&impossible_parity), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn confirmed_missing_part_error_recognizes_legacy_and_s3_markers() {
|
||||
assert!(!is_confirmed_missing_part_error(None));
|
||||
assert!(is_confirmed_missing_part_error(Some("file not found")));
|
||||
assert!(is_confirmed_missing_part_error(Some("No such file or directory")));
|
||||
assert!(is_confirmed_missing_part_error(Some("Specified part could not be found")));
|
||||
assert!(is_confirmed_missing_part_error(Some("part.7 not found")));
|
||||
assert!(!is_confirmed_missing_part_error(Some("part.7 missing")));
|
||||
assert!(!is_confirmed_missing_part_error(Some("permission denied")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_read_part_handles_mismatched_and_transient_responses_without_false_missing() {
|
||||
let responses = vec![
|
||||
Some(Vec::new()),
|
||||
Some(vec![read_part_test_error(1, "permission denied")]),
|
||||
None,
|
||||
];
|
||||
|
||||
let err = resolve_read_part_from_responses("bucket", "upload/part.1.meta", 1, 0, 1, &responses, 2)
|
||||
.expect_err("mismatched and transient responses must not be treated as confirmed missing");
|
||||
|
||||
assert_eq!(err, DiskError::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_read_part_accepts_alternate_missing_error_markers() {
|
||||
let responses = vec![
|
||||
Some(vec![read_part_test_error(1, "Specified part could not be found")]),
|
||||
Some(vec![read_part_test_error(1, "part.1 not found")]),
|
||||
Some(vec![read_part_test_part(1, "stale-etag")]),
|
||||
];
|
||||
|
||||
let part = resolve_read_part_from_responses("bucket", "upload/part.1.meta", 1, 0, 1, &responses, 2)
|
||||
.expect("alternate missing markers should satisfy missing quorum");
|
||||
|
||||
assert_eq!(part.number, 1);
|
||||
assert_eq!(part.error.as_deref(), Some("part.1 not found"));
|
||||
assert!(part.etag.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shard_read_costs_for_empty_disk_set_are_empty() {
|
||||
assert!(shard_read_costs_for_disks(&[]).is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reserve_read_repair_heal_dedupes_by_object_version_and_set() {
|
||||
let object = format!("object-{}", Uuid::new_v4());
|
||||
let key = reserve_read_repair_heal("bucket", &object, Some("version-1"), 1, 2)
|
||||
.await
|
||||
.expect("first read-repair reservation should be accepted");
|
||||
|
||||
assert_eq!(key.version_id.as_deref(), Some("version-1"));
|
||||
assert!(
|
||||
reserve_read_repair_heal("bucket", &object, Some("version-1"), 1, 2)
|
||||
.await
|
||||
.is_none()
|
||||
);
|
||||
|
||||
release_read_repair_heal_reservation(&key).await;
|
||||
let retry_key = reserve_read_repair_heal("bucket", &object, Some("version-1"), 1, 2)
|
||||
.await
|
||||
.expect("released read-repair reservation should allow retry");
|
||||
release_read_repair_heal_reservation(&retry_key).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn submit_read_repair_heal_releases_reservation_after_submitter_failure() {
|
||||
let object = format!("object-{}", Uuid::new_v4());
|
||||
submit_read_repair_heal_with_submitter(
|
||||
ReadRepairHealSubmission {
|
||||
bucket: "bucket",
|
||||
object: &object,
|
||||
version_id: None,
|
||||
pool_index: 1,
|
||||
set_index: 2,
|
||||
part_number: Some(1),
|
||||
reason: "test",
|
||||
},
|
||||
failed_read_repair_submitter,
|
||||
)
|
||||
.await;
|
||||
|
||||
for _ in 0..20 {
|
||||
if let Some(key) = reserve_read_repair_heal("bucket", &object, None, 1, 2).await {
|
||||
release_read_repair_heal_reservation(&key).await;
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
|
||||
panic!("failed read-repair submission should release its dedup reservation");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn submit_read_repair_heal_keeps_admitted_reservation_deduped() {
|
||||
let object = format!("object-{}", Uuid::new_v4());
|
||||
submit_read_repair_heal_with_submitter(
|
||||
ReadRepairHealSubmission {
|
||||
bucket: "bucket",
|
||||
object: &object,
|
||||
version_id: None,
|
||||
pool_index: 3,
|
||||
set_index: 4,
|
||||
part_number: None,
|
||||
reason: "test",
|
||||
},
|
||||
accepted_read_repair_submitter,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(reserve_read_repair_heal("bucket", &object, None, 3, 4).await.is_none());
|
||||
let key = ReadRepairHealCacheKey::new("bucket", &object, None, 3, 4);
|
||||
release_read_repair_heal_reservation(&key).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ mod storage_api;
|
||||
|
||||
use rustfs_filemeta::{FileInfoOpts, get_file_info};
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
use std::path::PathBuf;
|
||||
use std::path::{Path, PathBuf};
|
||||
use storage_api::legacy_bitrot_read::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, create_bitrot_reader, new_disk};
|
||||
use tokio::fs;
|
||||
|
||||
@@ -44,15 +44,46 @@ fn workspace_root() -> PathBuf {
|
||||
.to_path_buf()
|
||||
}
|
||||
|
||||
fn legacy_test_root() -> PathBuf {
|
||||
std::env::var_os("RUSTFS_LEGACY_TEST_ROOT")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(workspace_root)
|
||||
}
|
||||
|
||||
fn legacy_test_disk(root: &Path) -> String {
|
||||
if let Some(disk) = std::env::var_os("RUSTFS_LEGACY_TEST_DISK") {
|
||||
return disk.to_string_lossy().into_owned();
|
||||
}
|
||||
if root.join("test/.minio.sys/format.json").exists() {
|
||||
"test".to_string()
|
||||
} else {
|
||||
"test1".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn legacy_format_exists(root: &Path, disk_name: &str) -> bool {
|
||||
let disk_root = root.join(disk_name);
|
||||
disk_root.join(".rustfs.sys/format.json").exists() || disk_root.join(".minio.sys/format.json").exists()
|
||||
}
|
||||
|
||||
fn legacy_object_meta_exists(root: &Path, disk_name: &str, bucket: &str, object: &str) -> bool {
|
||||
root.join(disk_name)
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(STORAGE_FORMAT_FILE)
|
||||
.exists()
|
||||
}
|
||||
|
||||
fn legacy_test_data_exists() -> bool {
|
||||
if std::env::var("RUSTFS_SKIP_LEGACY_TEST").unwrap_or_default() == "1" {
|
||||
return false;
|
||||
}
|
||||
let root = workspace_root();
|
||||
let format_path = root.join("test1/.rustfs.sys/format.json");
|
||||
let ktvzip_meta = root.join("test1/vvvv/ktvzip.tar.gz").join(STORAGE_FORMAT_FILE);
|
||||
let path_traversal_meta = root.join("test1/vvvv/path_traversal.md").join(STORAGE_FORMAT_FILE);
|
||||
format_path.exists() && (ktvzip_meta.exists() || path_traversal_meta.exists())
|
||||
let root = legacy_test_root();
|
||||
let disk_name = legacy_test_disk(&root);
|
||||
let bucket = "vvvv";
|
||||
legacy_format_exists(&root, &disk_name)
|
||||
&& (legacy_object_meta_exists(&root, &disk_name, bucket, "ktvzip.tar.gz")
|
||||
|| legacy_object_meta_exists(&root, &disk_name, bucket, "path_traversal.md"))
|
||||
}
|
||||
|
||||
async fn run_legacy_bitrot_test_for_object(root: &std::path::Path, disk_name: &str, bucket: &str, object: &str) -> bool {
|
||||
@@ -227,23 +258,25 @@ async fn run_legacy_bitrot_test_for_object(root: &std::path::Path, disk_name: &s
|
||||
#[tokio::test]
|
||||
async fn test_legacy_bitrot_read() {
|
||||
if !legacy_test_data_exists() {
|
||||
eprintln!("Skipping legacy bitrot test: test1/vvvv xl.meta or RUSTFS_SKIP_LEGACY_TEST");
|
||||
eprintln!("Skipping legacy bitrot test: legacy fixture xl.meta not found or RUSTFS_SKIP_LEGACY_TEST=1");
|
||||
return;
|
||||
}
|
||||
|
||||
// let root = workspace_root();
|
||||
let root = PathBuf::from("/Users/weisd/project/minio");
|
||||
|
||||
let disk_name = "test";
|
||||
let root = legacy_test_root();
|
||||
let disk_name = legacy_test_disk(&root);
|
||||
let bucket = "vvvv";
|
||||
let mut checked = 0usize;
|
||||
|
||||
// Try both EC (part files) and inline objects
|
||||
let ok = run_legacy_bitrot_test_for_object(&root, disk_name, bucket, "ktvzip.tar.gz").await;
|
||||
for object in ["ktvzip.tar.gz", "path_traversal.md"] {
|
||||
if !legacy_object_meta_exists(&root, &disk_name, bucket, object) {
|
||||
eprintln!("Skipping missing legacy object fixture: {object}");
|
||||
continue;
|
||||
}
|
||||
|
||||
eprintln!("ok: {:?}", ok);
|
||||
checked += 1;
|
||||
let ok = run_legacy_bitrot_test_for_object(&root, &disk_name, bucket, object).await;
|
||||
assert!(ok, "create_bitrot_reader failed for {object}");
|
||||
}
|
||||
|
||||
assert!(ok, "create_bitrot_reader failed for both ktvzip.tar.gz and path_traversal.md");
|
||||
|
||||
let ok = run_legacy_bitrot_test_for_object(&root, disk_name, bucket, "path_traversal.md").await;
|
||||
assert!(ok, "create_bitrot_reader failed for path_traversal.md");
|
||||
assert!(checked > 0, "legacy bitrot fixture preflight found no readable objects");
|
||||
}
|
||||
|
||||
@@ -117,6 +117,52 @@ fn sha256_hex(bytes: &[u8]) -> String {
|
||||
hex_simd::encode_to_string(Sha256::digest(bytes), hex_simd::AsciiCase::Lower)
|
||||
}
|
||||
|
||||
async fn load_fixture_reader_input(case_id: &str) -> (ObjectInfo, Vec<u8>, String) {
|
||||
let case_dir = require_fixture_case(case_id);
|
||||
let manifest: ManifestRecord = read_json(&case_dir.join("manifest.json"));
|
||||
let expected_sha256 = read_plaintext_sha256(&case_dir);
|
||||
let file_info = load_file_info(&case_dir, &manifest);
|
||||
let encrypted = encrypted_fixture_bytes(&case_dir, &manifest, &file_info).await;
|
||||
let object_info = load_object_info(&file_info, &manifest);
|
||||
|
||||
(object_info, encrypted, expected_sha256)
|
||||
}
|
||||
|
||||
async fn read_fixture_plaintext(encrypted: Vec<u8>, object_info: ObjectInfo, kms_key_b64: String) -> Result<Vec<u8>, String> {
|
||||
let object_size = object_info.size;
|
||||
|
||||
async_with_vars(
|
||||
[
|
||||
("__RUSTFS_SSE_SIMPLE_CMK", Some(kms_key_b64)),
|
||||
("RUSTFS_SSE_S3_MASTER_KEY", None::<String>),
|
||||
],
|
||||
async move {
|
||||
let (mut reader, offset, length) = GetObjectReader::new(
|
||||
Box::new(Cursor::new(encrypted)),
|
||||
None,
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&http::HeaderMap::new(),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| format!("construct GetObjectReader from MinIO raw fixture: {err:?}"))?;
|
||||
|
||||
if offset != 0 || length != object_size {
|
||||
return Err(format!("unexpected fixture range offset={offset} length={length} size={object_size}"));
|
||||
}
|
||||
|
||||
let mut plaintext = Vec::new();
|
||||
reader
|
||||
.read_to_end(&mut plaintext)
|
||||
.await
|
||||
.map_err(|err| format!("read plaintext from MinIO raw fixture: {err}"))?;
|
||||
|
||||
Ok(plaintext)
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, file_info: &FileInfo) -> Vec<u8> {
|
||||
let mut disks = Vec::with_capacity(file_info.erasure.distribution.len());
|
||||
for disk_number in 1..=file_info.erasure.distribution.len() {
|
||||
@@ -204,43 +250,44 @@ async fn reads_minio_generated_sse_kms_multipart_fixture() {
|
||||
assert_fixture_round_trip("sse-kms-multipart-8m", 8 * 1024 * 1024).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
|
||||
async fn rejects_minio_generated_sse_s3_fixture_with_wrong_kms_key() {
|
||||
let (object_info, encrypted, _) = load_fixture_reader_input("sse-s3-multipart-8m").await;
|
||||
let wrong_key_b64 = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE=".to_string();
|
||||
|
||||
let result = read_fixture_plaintext(encrypted, object_info, wrong_key_b64).await;
|
||||
|
||||
assert!(result.is_err(), "wrong KMS key must fail closed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
|
||||
async fn rejects_minio_generated_sse_s3_fixture_with_truncated_ciphertext() {
|
||||
let (object_info, mut encrypted, expected_sha256) = load_fixture_reader_input("sse-s3-multipart-8m").await;
|
||||
encrypted.truncate(encrypted.len() / 2);
|
||||
|
||||
let result = read_fixture_plaintext(encrypted, object_info, minio_static_kms_key_b64()).await;
|
||||
|
||||
if let Ok(plaintext) = result {
|
||||
assert_ne!(
|
||||
sha256_hex(&plaintext),
|
||||
expected_sha256,
|
||||
"truncated ciphertext must not restore the original plaintext"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_fixture_round_trip(case_id: &str, expected_size: i64) {
|
||||
let case_dir = require_fixture_case(case_id);
|
||||
let manifest: ManifestRecord = read_json(&case_dir.join("manifest.json"));
|
||||
let expected_sha256 = read_plaintext_sha256(&case_dir);
|
||||
let file_info = load_file_info(&case_dir, &manifest);
|
||||
let encrypted = encrypted_fixture_bytes(&case_dir, &manifest, &file_info).await;
|
||||
let object_info = load_object_info(&file_info, &manifest);
|
||||
let (object_info, encrypted, expected_sha256) = load_fixture_reader_input(case_id).await;
|
||||
let object_size = object_info.size;
|
||||
let kms_key_b64 = minio_static_kms_key_b64();
|
||||
|
||||
async_with_vars(
|
||||
[
|
||||
("__RUSTFS_SSE_SIMPLE_CMK", Some(kms_key_b64)),
|
||||
("RUSTFS_SSE_S3_MASTER_KEY", None::<String>),
|
||||
],
|
||||
async {
|
||||
let (mut reader, offset, length) = GetObjectReader::new(
|
||||
Box::new(Cursor::new(encrypted)),
|
||||
None,
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&http::HeaderMap::new(),
|
||||
)
|
||||
.await
|
||||
.expect("construct GetObjectReader from MinIO raw fixture");
|
||||
let plaintext = read_fixture_plaintext(encrypted, object_info, kms_key_b64)
|
||||
.await
|
||||
.expect("fixture must restore with the configured KMS key");
|
||||
|
||||
let mut plaintext = Vec::new();
|
||||
reader
|
||||
.read_to_end(&mut plaintext)
|
||||
.await
|
||||
.expect("read plaintext from MinIO raw fixture");
|
||||
|
||||
assert_eq!(offset, 0);
|
||||
assert_eq!(length, object_info.size);
|
||||
assert_eq!(reader.object_info.size, expected_size);
|
||||
assert_eq!(plaintext.len(), expected_size as usize);
|
||||
assert_eq!(sha256_hex(&plaintext), expected_sha256);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(object_size, expected_size);
|
||||
assert_eq!(plaintext.len(), expected_size as usize);
|
||||
assert_eq!(sha256_hex(&plaintext), expected_sha256);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,376 @@
|
||||
# ECStore Validation Suite Design
|
||||
|
||||
This document defines the validation suite RustFS should use before claiming
|
||||
ECStore erasure-coding correctness, durability, and fault-resilience coverage.
|
||||
It is a suite design, not a claim that all tests already exist.
|
||||
|
||||
## Goal
|
||||
|
||||
Provide one command that runs the full ECStore confidence suite and produces a
|
||||
single pass/fail result plus artifacts. The command should exercise both:
|
||||
|
||||
- white-box invariants in `rustfs-ecstore`, `rustfs-filemeta`, and related
|
||||
helpers;
|
||||
- black-box S3 and admin behavior against real single-node and distributed
|
||||
erasure deployments.
|
||||
|
||||
The suite can reduce release risk; it cannot prove the absence of all defects.
|
||||
It must therefore combine deterministic matrices, negative tests, fuzz/corpus
|
||||
tests, chaos tests, and explicit artifact review.
|
||||
|
||||
## Proposed Entry Point
|
||||
|
||||
Use the top-level runner for repeatable local, CI, and release checks:
|
||||
|
||||
```bash
|
||||
scripts/run_ecstore_validation_suite.sh --profile full
|
||||
```
|
||||
|
||||
Profiles:
|
||||
|
||||
| Profile | Purpose | Expected cost |
|
||||
| --- | --- | --- |
|
||||
| `quick` | PR smoke for EC logic and existing single-node reliability tests. | minutes |
|
||||
| `full` | Release gate: white-box, e2e, chaos, S3 compatibility subset, coverage. | hours |
|
||||
| `destructive` | Manual/nightly gate: distributed 4-node/16-disk, crash/restart, rebalance/decommission fault injection. | hours+ |
|
||||
| `fuzz` | Malformed metadata/RPC/corpus fuzzing with fixed seed output. | bounded by budget |
|
||||
|
||||
The current runner wires the existing high-signal checks. The matrix below
|
||||
still tracks required follow-up coverage before `full` can be treated as a
|
||||
complete release gate.
|
||||
|
||||
The runner writes artifacts under
|
||||
`target/ecstore-validation/<timestamp>/`:
|
||||
|
||||
- command transcript and environment;
|
||||
- randomized seeds;
|
||||
- object manifests with SHA256;
|
||||
- per-disk layout snapshots before and after mutation;
|
||||
- server logs;
|
||||
- junit/nextest output;
|
||||
- `blackbox-matrix.tsv` with selected black-box and fixture gates;
|
||||
- coverage report;
|
||||
- failure reproducer instructions.
|
||||
|
||||
## Acceptance Rules
|
||||
|
||||
The suite passes only when all selected profile commands pass and every
|
||||
scenario asserts both API-visible behavior and on-disk state where applicable.
|
||||
|
||||
Required fail-closed rules:
|
||||
|
||||
- never return corrupted object bytes;
|
||||
- never silently accept forged or split-brain metadata;
|
||||
- never downgrade write quorum to read quorum;
|
||||
- never leave a mixed old/new object after partial commit;
|
||||
- never panic on malformed EC metadata;
|
||||
- return typed errors or quorum failures for invalid states.
|
||||
|
||||
Unit-test coverage is a hard release-gate input: `rustfs-ecstore` unit line
|
||||
coverage for the EC-critical scope must be at least 95%, with 100% as the
|
||||
target for EC read, write, decode, heal, metadata quorum, and rollback paths.
|
||||
A lower threshold is only acceptable for a temporary, explicitly documented
|
||||
exception tied to missing testability or unreachable code. Full-crate coverage
|
||||
is still reported as an observation metric, but it must not hide EC regressions
|
||||
behind unrelated modules.
|
||||
|
||||
Tests must not pass by only checking constants, helper calls, deleted branches,
|
||||
or implementation details. Every test needs a reader-facing or storage-state
|
||||
assertion.
|
||||
|
||||
Fixture-backed checks are optional for local smoke runs, but explicit in the
|
||||
artifact stream. `--require-fixtures` turns missing legacy or MinIO generated
|
||||
fixtures into an early `ecstore-fixture-gate` failure before expensive black-box
|
||||
steps run. The MinIO generated fixture gate requires both
|
||||
`RUSTFS_MINIO_FIXTURE_ROOT` and `RUSTFS_MINIO_STATIC_KMS_KEY_B64`.
|
||||
|
||||
## White-Box Matrix
|
||||
|
||||
### Erasure Algorithm
|
||||
|
||||
Target files:
|
||||
|
||||
- `crates/ecstore/src/erasure/coding/erasure.rs`
|
||||
- `crates/ecstore/src/erasure/coding/encode.rs`
|
||||
- `crates/ecstore/src/erasure/coding/decode.rs`
|
||||
- `crates/ecstore/src/erasure/coding/decode_reader.rs`
|
||||
- `crates/ecstore/src/erasure/codec/bridge.rs`
|
||||
|
||||
Coverage:
|
||||
|
||||
| Area | Scenarios | Assertions |
|
||||
| --- | --- | --- |
|
||||
| Shard geometry | legacy/current shard-size formulas; lengths `0`, `1`, `block-1`, `block`, `block+1`, multi-block tail | no divide-by-zero; shard/file/range offsets match expected |
|
||||
| Encode/decode | `(data, parity)` sets `2+2`, `4+2`, `8+8`; random payloads; missing shards up to parity | reconstructed data equals original |
|
||||
| Negative reconstruction | missing shards above parity; inconsistent shard lengths; corrupt surplus parity | typed error, no partial success |
|
||||
| Source verification | missing data shard plus extra parity source | rebuilt parity must match source parity |
|
||||
| Legacy compatibility | old shard formula and legacy checksum data | legacy files decode and heal correctly |
|
||||
| Streaming decode | legacy engine vs RustFS codec engine on same stripe stream | bytes and errors are equivalent |
|
||||
| Range output | head/middle/tail/suffix; cross-block and final-short-stripe ranges | exact byte range, no over-read/under-read |
|
||||
|
||||
Add property tests with fixed replay seeds for payload, range, and missing-shard
|
||||
selection.
|
||||
|
||||
### Bitrot and Reader Alignment
|
||||
|
||||
Target files:
|
||||
|
||||
- `crates/ecstore/src/erasure/coding/bitrot.rs`
|
||||
- `crates/ecstore/src/erasure/coding/decode.rs`
|
||||
- `crates/ecstore/src/set_disk/core/io_primitives.rs`
|
||||
- `crates/ecstore/src/set_disk/shard_source.rs`
|
||||
|
||||
Coverage:
|
||||
|
||||
| Area | Scenarios | Assertions |
|
||||
| --- | --- | --- |
|
||||
| Hash framing | valid hash+data; wrong hash; truncated hash; truncated data | invalid data never succeeds |
|
||||
| Short shard | short read under normal hash, `skip_verify`, and hash-none | `UnexpectedEof` or equivalent typed error |
|
||||
| Lockstep reads | mid-stream data shard failure; pending/timeout reader; final short stripe | each live reader advances exactly one stripe; failed reader retires |
|
||||
| Adaptive reads | hedged parity fallback and timeout retirement | no shard desync; reconstructed bytes match original |
|
||||
| Shard source order | out-of-order read completion and missing slots | slots resolve by shard index |
|
||||
| Deferred readers | data-blocks-first setup opens deferred parity at correct offset | parity fallback uses aligned data |
|
||||
|
||||
Use instrumented readers that record shard index, stripe index, read count,
|
||||
offset, and retirement reason.
|
||||
|
||||
### Metadata, Quorum, and Commit Atomicity
|
||||
|
||||
Target files:
|
||||
|
||||
- `crates/filemeta/src/filemeta/version.rs`
|
||||
- `crates/ecstore/src/set_disk/read.rs`
|
||||
- `crates/ecstore/src/set_disk/metadata.rs`
|
||||
- `crates/ecstore/src/set_disk/ops/object.rs`
|
||||
- `crates/ecstore/src/set_disk/core/io_primitives.rs`
|
||||
- `crates/ecstore/src/disk/local.rs`
|
||||
|
||||
Coverage:
|
||||
|
||||
| Area | Scenarios | Assertions |
|
||||
| --- | --- | --- |
|
||||
| Metadata tamper | same `version_id`/`mod_time`, divergent data dir, parts, ETag, size, checksum, inline flag, erasure distribution | previous committed version or read quorum error; no arbitrary latest |
|
||||
| Early stop | valid quorum, stale quorum, corrupt trailing disks, slow trailing disks | early-stop only on safe identity |
|
||||
| Quorum downgrade | read quorum vs write quorum; delete marker quorum; version-not-found quorum | no mutation below write quorum |
|
||||
| Rename atomicity | failure before data rename, after data rename, after metadata rename, cleanup failure | object is old or new; never mixed |
|
||||
| Rollback | failed commit quorum and stale temp data | rollback preserves old metadata/data |
|
||||
| Malformed metadata | oversized lengths, bad CRC, invalid versions, invalid UUID/timestamp/enum, huge parts | bounded memory; typed error; no panic |
|
||||
|
||||
Fault injection should be explicit and deterministic, preferably through local
|
||||
disk mocks for unit tests and process-level disk manipulation for e2e tests.
|
||||
|
||||
## Black-Box Matrix
|
||||
|
||||
Use `crates/e2e_test` for real S3/admin behavior and extend
|
||||
`crates/e2e_test/src/chaos.rs` rather than duplicating ad hoc helpers.
|
||||
|
||||
The current runner emits `blackbox-matrix.tsv` for every invocation. It is a
|
||||
machine-readable manifest of selected black-box scenarios, their commands,
|
||||
required fixture environment, and whether each row is enabled, disabled, or
|
||||
missing an optional/required fixture.
|
||||
|
||||
Current runner rows:
|
||||
|
||||
| Profile | Scenario | Gate | Fixture env |
|
||||
| --- | --- | --- | --- |
|
||||
| `quick` | single-node disk fault read/write | e2e black box | none |
|
||||
| `quick` | degraded erasure disk rebuild | e2e black box | none |
|
||||
| `quick` | namespace lock quorum under EC ops | e2e black box | none |
|
||||
| `full` | legacy bitrot read fixture restore | fixture black box | `RUSTFS_LEGACY_TEST_ROOT`, `RUSTFS_LEGACY_TEST_DISK` |
|
||||
| `full` | MinIO generated encrypted read and negative restore fixture | fixture black box | `RUSTFS_MINIO_FIXTURE_ROOT`, `RUSTFS_MINIO_STATIC_KMS_KEY_B64` |
|
||||
| `full` | S3 multipart/range/versioning/delete subset | S3 black box | none |
|
||||
| `destructive` | distributed cluster concurrency | e2e black box | none |
|
||||
| `destructive` | stale multipart cleanup cluster | e2e black box | none |
|
||||
| `destructive` | delete marker migration semantics | e2e black box | none |
|
||||
|
||||
### Single-Node 4-Disk EC
|
||||
|
||||
| Scenario | Required assertions |
|
||||
| --- | --- |
|
||||
| baseline PUT/GET/HEAD/List for tiny, inline, block-boundary, multi-block, multipart objects | SHA256 manifest matches; metadata is consistent on all disks |
|
||||
| one disk offline during read | existing objects readable; no corrupted bytes |
|
||||
| one disk offline during write | write succeeds only when write quorum holds; restored disk is healed |
|
||||
| above-parity disk loss | GET/PUT fails with quorum error; no partial bytes accepted |
|
||||
| corrupt data shard and parity shard | GET returns original bytes or fails closed; read-repair/heal restores |
|
||||
| corrupt inline `xl.meta` | fail closed or heal; no forged inline data |
|
||||
| range read with offline/corrupt shard | exact range bytes; invalid ranges produce expected S3 errors |
|
||||
| multipart part resend and concurrent same-part writes | final complete object matches chosen committed parts |
|
||||
| crash during multipart complete/put/delete | after restart only old or new full version is visible |
|
||||
|
||||
Existing anchors:
|
||||
|
||||
- `crates/e2e_test/src/reliability_disk_fault_test.rs`
|
||||
- `crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs`
|
||||
- `crates/e2e_test/src/chaos.rs`
|
||||
|
||||
### Distributed 4-Node / 16-Disk EC
|
||||
|
||||
| Scenario | Required assertions |
|
||||
| --- | --- |
|
||||
| node/disk outage while reading large objects | no EOF/truncation; SHA256 manifest matches |
|
||||
| write while one remote node is down | write follows quorum; later heal reconstructs remote disk |
|
||||
| remote shard bitrot | degraded read uses clean shards; no bad bytes |
|
||||
| concurrent GET/PUT/DELETE/List on same key | no 500 for expected conflicts; no dirty reads |
|
||||
| range GET matrix for large objects | sequential and parallel ranges match full-object hash |
|
||||
| internode timeout/slow disk | typed error or fallback; no desync |
|
||||
|
||||
This layer is mandatory because single-process unit tests cannot prove RPC,
|
||||
HTTP/2, timeout, and distributed quorum behavior.
|
||||
|
||||
### Versioning, Delete Markers, and Migration
|
||||
|
||||
| Scenario | Required assertions |
|
||||
| --- | --- |
|
||||
| latest delete marker | GET/HEAD/ListObjectVersions match S3 semantics |
|
||||
| explicit `versionId` for old versions | exact old bytes and metadata |
|
||||
| suspended/null version | no version ordering regression |
|
||||
| delete marker during heal/rebalance/decommission | marker visibility and history are preserved |
|
||||
| orphan directory cleanup | real objects are not purged; tombstones behave correctly |
|
||||
|
||||
### Heal, Rebalance, and Decommission
|
||||
|
||||
| Scenario | Required assertions |
|
||||
| --- | --- |
|
||||
| auto heal and admin deep heal | data hash unchanged; `xl.meta` and format data rebuilt |
|
||||
| heal interruption/restart | idempotent recovery; no dangling temp objects |
|
||||
| two-pool rebalance with versioned/multipart objects | source and target pools have consistent versions |
|
||||
| decommission cancel/restart/finalize | target readable; source cleanup safe |
|
||||
| rebalance/decommission with node outage | progress resumes; no duplicate or missing versions |
|
||||
|
||||
Existing scripts under `scripts/test/decommission_*.sh` should be wrapped into
|
||||
the `destructive` profile only after they emit machine-readable pass/fail
|
||||
artifacts.
|
||||
|
||||
## S3 Compatibility and Large Object Gates
|
||||
|
||||
The full profile should include a targeted S3 compatibility subset, not the
|
||||
entire compatibility suite by default:
|
||||
|
||||
```bash
|
||||
TESTEXPR="multipart or range or versioning or delete" \
|
||||
DEPLOY_MODE=build \
|
||||
MAXFAIL=0 \
|
||||
./scripts/s3-tests/run.sh
|
||||
```
|
||||
|
||||
Large object gates:
|
||||
|
||||
- `scripts/run_get_codec_streaming_smoke.sh` for legacy/codec GET parity;
|
||||
- `scripts/run_gt1g_get_http_matrix.sh` for sequential and parallel range GET;
|
||||
- `scripts/run_gt1g_multipart_put_matrix.sh` for multipart PUT paths.
|
||||
|
||||
These should be artifact-producing optional stages in `full` or `destructive`
|
||||
profiles, not hidden local-only commands.
|
||||
|
||||
## Fuzz and Corpus Gates
|
||||
|
||||
Add bounded fuzz/corpus tests for:
|
||||
|
||||
- `xl.meta` MessagePack and legacy filemeta versions;
|
||||
- protobuf/RPC payload decoding;
|
||||
- checksum and bitrot headers;
|
||||
- range offset/length overflow;
|
||||
- object names with path traversal, encoded separators, and symlink components;
|
||||
- huge inline metadata and huge part counts.
|
||||
|
||||
Each corpus failure must save the input bytes and the minimized reproducer under
|
||||
the suite artifact directory.
|
||||
|
||||
## Coverage Snapshot Target
|
||||
|
||||
The runner enforces unit line coverage for `rustfs-ecstore` in `full` and
|
||||
`destructive` profiles. The default threshold is 95%, and the target remains
|
||||
100% for EC-critical paths:
|
||||
|
||||
```bash
|
||||
scripts/run_ecstore_validation_suite.sh --profile full --unit-coverage-min 95
|
||||
scripts/run_ecstore_validation_suite.sh --profile full --unit-coverage-min 100
|
||||
scripts/run_ecstore_validation_suite.sh --profile full --unit-coverage-scope crate
|
||||
```
|
||||
|
||||
The default hard gate scope is `ec-critical`, covering:
|
||||
|
||||
- `crates/ecstore/src/erasure/**`
|
||||
- `crates/ecstore/src/set_disk/read.rs`
|
||||
- `crates/ecstore/src/set_disk/shard_source.rs`
|
||||
- `crates/ecstore/src/set_disk/metadata.rs`
|
||||
- `crates/ecstore/src/set_disk/ops/object.rs`
|
||||
- `crates/ecstore/src/set_disk/core/io_primitives.rs`
|
||||
- `crates/ecstore/src/disk/local.rs`
|
||||
|
||||
Minimum release-gate target:
|
||||
|
||||
- `cargo-llvm-cov` must be installed unless `--skip-coverage` is explicitly set;
|
||||
- `rustfs-ecstore` EC-critical unit line coverage is at least 95%;
|
||||
- EC read/write/decode/heal/quorum/rollback code should trend toward 100%;
|
||||
- full-crate unit line coverage is recorded for visibility but is not the EC
|
||||
hard gate;
|
||||
- all HIGH rows in this document have positive and negative tests;
|
||||
- changed EC/read/write/heal lines are covered;
|
||||
- branch coverage is reviewed for reconstruction, quorum, and error paths;
|
||||
- uncovered branches are either intentionally unreachable or tracked.
|
||||
|
||||
The `quick` profile also runs
|
||||
`cargo test -p rustfs-ecstore --lib -- --test-threads=1` so the white-box smoke
|
||||
includes all current `rustfs-ecstore` library unit tests without parallel test
|
||||
context cross-talk, not only focused EC filters.
|
||||
|
||||
Coverage artifacts:
|
||||
|
||||
- `target/ecstore-validation/<timestamp>/coverage/ecstore/lcov.info`
|
||||
- `target/ecstore-validation/<timestamp>/coverage/ecstore/summary.tsv`
|
||||
- `target/ecstore-validation/<timestamp>/coverage/ecstore/files.tsv`
|
||||
|
||||
Local validation on 2026-07-07 found that the current suite is not yet at the
|
||||
target: full-crate unit line coverage was 69.32%; the EC-critical scope was
|
||||
84.32% after the first negative read/write/recovery additions. The lowest
|
||||
EC-critical files were `set_disk/ops/object.rs`, `io_primitives.rs`,
|
||||
`disk/local.rs`, `bitrot.rs`, and `encode.rs`. These are gaps to close before
|
||||
the default 95% gate can pass.
|
||||
|
||||
## Initial Command Set
|
||||
|
||||
Use the runner first:
|
||||
|
||||
```bash
|
||||
scripts/run_ecstore_validation_suite.sh --profile quick
|
||||
scripts/run_ecstore_validation_suite.sh --profile full
|
||||
scripts/run_ecstore_validation_suite.sh --profile destructive
|
||||
scripts/run_ecstore_validation_suite.sh --profile fuzz
|
||||
```
|
||||
|
||||
The split-run equivalent for the quick profile is:
|
||||
|
||||
```bash
|
||||
cargo test -p rustfs-filemeta --lib
|
||||
cargo test -p rustfs-ecstore --lib erasure
|
||||
cargo test -p rustfs-ecstore --lib set_disk::read
|
||||
cargo test -p rustfs-ecstore --lib set_disk::core::io_primitives
|
||||
cargo test -p rustfs-ecstore --lib set_disk::tests::test_rename_data_quorum_failure_rolls_back_destination_object
|
||||
cargo test -p rustfs-ecstore --lib disk::local
|
||||
cargo test -p rustfs-ecstore --lib -- --test-threads=1
|
||||
cargo test --package e2e_test reliability_disk_fault_test -- --nocapture
|
||||
cargo test --package e2e_test heal_erasure_disk_rebuild_test -- --nocapture
|
||||
cargo test --package e2e_test namespace_lock_quorum_test -- --nocapture
|
||||
```
|
||||
|
||||
Fixture-backed tests should run when the fixture path is present:
|
||||
|
||||
```bash
|
||||
cargo test -p rustfs-ecstore --test legacy_bitrot_read_test -- --nocapture
|
||||
cargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored --nocapture
|
||||
```
|
||||
|
||||
## Multi-Expert Adversarial Review Summary
|
||||
|
||||
Three independent read-only reviews were run for this design:
|
||||
|
||||
| Reviewer | Main challenge | Resulting requirement |
|
||||
| --- | --- | --- |
|
||||
| Algorithm correctness | A single run only samples one timing/layout/hash combination. | matrix/property tests plus instrumented reader alignment checks |
|
||||
| Black-box reliability | Existing tests miss distributed, shard-desync, range+fault, and migration-fault combinations. | add 4-node/16-disk and destructive profiles |
|
||||
| Security and fault review | Metadata tamper, quorum downgrade, bitrot bypass, and rename atomicity must be end-to-end. | HIGH matrix rows block completion claims |
|
||||
|
||||
Current verdict: design accepted as a target. The runner exists and captures
|
||||
artifacts; implementation remains incomplete until the missing matrix rows have
|
||||
artifact-backed evidence.
|
||||
Executable
+541
@@ -0,0 +1,541 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
PROFILE="quick"
|
||||
OUT_DIR=""
|
||||
DRY_RUN=false
|
||||
SKIP_E2E=false
|
||||
SKIP_S3_TESTS=false
|
||||
SKIP_COVERAGE=false
|
||||
REQUIRE_FIXTURES=false
|
||||
UNIT_COVERAGE_MIN="95"
|
||||
UNIT_COVERAGE_TARGET="100"
|
||||
UNIT_COVERAGE_SCOPE="ec-critical"
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/run_ecstore_validation_suite.sh [options]
|
||||
|
||||
Profiles:
|
||||
quick Core EC read/write/recovery checks for PR smoke (default)
|
||||
full quick + s3-tests subset + fixture tests + coverage
|
||||
destructive full + cluster/concurrency/destructive recovery checks
|
||||
fuzz bounded fuzz smoke for malformed storage inputs
|
||||
|
||||
Options:
|
||||
--profile <quick|full|destructive|fuzz>
|
||||
--out-dir <path> Default: target/ecstore-validation/<timestamp>
|
||||
--skip-e2e Skip e2e_test commands
|
||||
--skip-s3-tests Skip scripts/s3-tests/run.sh in full/destructive
|
||||
--skip-coverage Explicitly skip the mandatory unit coverage gate
|
||||
--unit-coverage-min <percent>
|
||||
Minimum rustfs-ecstore unit line coverage (default: 95)
|
||||
--unit-coverage-scope <ec-critical|crate>
|
||||
Coverage scope for the hard gate (default: ec-critical)
|
||||
--require-fixtures Fail when optional EC fixture tests cannot run
|
||||
--dry-run Print commands and write metadata without executing
|
||||
-h, --help
|
||||
|
||||
Artifacts:
|
||||
<out-dir>/run-metadata.env
|
||||
<out-dir>/summary.tsv
|
||||
<out-dir>/blackbox-matrix.tsv
|
||||
<out-dir>/logs/*.log
|
||||
USAGE
|
||||
}
|
||||
|
||||
arg_value() {
|
||||
local flag="$1"
|
||||
local value="${2:-}"
|
||||
if [[ -z "$value" || "$value" == --* ]]; then
|
||||
echo "ERROR: missing value for $flag" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s\n' "$value"
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--profile) PROFILE="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--skip-e2e) SKIP_E2E=true; shift ;;
|
||||
--skip-s3-tests) SKIP_S3_TESTS=true; shift ;;
|
||||
--skip-coverage) SKIP_COVERAGE=true; shift ;;
|
||||
--unit-coverage-min) UNIT_COVERAGE_MIN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--unit-coverage-scope) UNIT_COVERAGE_SCOPE="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--require-fixtures) REQUIRE_FIXTURES=true; shift ;;
|
||||
--dry-run) DRY_RUN=true; shift ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*)
|
||||
echo "ERROR: unknown arg: $1" >&2
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
validate_args() {
|
||||
case "$PROFILE" in
|
||||
quick|full|destructive|fuzz) ;;
|
||||
*)
|
||||
echo "ERROR: --profile must be quick, full, destructive, or fuzz" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
if ! [[ "$UNIT_COVERAGE_MIN" =~ ^([0-9]|[1-9][0-9]|100)(\.[0-9]+)?$ ]]; then
|
||||
echo "ERROR: --unit-coverage-min must be between 0 and 100" >&2
|
||||
exit 1
|
||||
fi
|
||||
case "$UNIT_COVERAGE_SCOPE" in
|
||||
ec-critical|crate) ;;
|
||||
*)
|
||||
echo "ERROR: --unit-coverage-scope must be ec-critical or crate" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
setup_workspace() {
|
||||
local root
|
||||
root="$(git rev-parse --show-toplevel)"
|
||||
cd "$root"
|
||||
|
||||
if [[ -z "$OUT_DIR" ]]; then
|
||||
OUT_DIR="target/ecstore-validation/$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
fi
|
||||
mkdir -p "$OUT_DIR/logs"
|
||||
|
||||
SUMMARY="$OUT_DIR/summary.tsv"
|
||||
BLACKBOX_MATRIX="$OUT_DIR/blackbox-matrix.tsv"
|
||||
printf 'status\tlabel\tlog\n' >"$SUMMARY"
|
||||
}
|
||||
|
||||
write_metadata() {
|
||||
{
|
||||
printf 'profile=%s\n' "$PROFILE"
|
||||
printf 'started_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
printf 'dry_run=%s\n' "$DRY_RUN"
|
||||
printf 'skip_e2e=%s\n' "$SKIP_E2E"
|
||||
printf 'skip_s3_tests=%s\n' "$SKIP_S3_TESTS"
|
||||
printf 'skip_coverage=%s\n' "$SKIP_COVERAGE"
|
||||
printf 'unit_coverage_min=%s\n' "$UNIT_COVERAGE_MIN"
|
||||
printf 'unit_coverage_target=%s\n' "$UNIT_COVERAGE_TARGET"
|
||||
printf 'unit_coverage_scope=%s\n' "$UNIT_COVERAGE_SCOPE"
|
||||
printf 'require_fixtures=%s\n' "$REQUIRE_FIXTURES"
|
||||
printf 'git_commit=%s\n' "$(git rev-parse HEAD 2>/dev/null || true)"
|
||||
printf 'git_branch=%s\n' "$(git branch --show-current 2>/dev/null || true)"
|
||||
printf 'rustc=%s\n' "$(rustc --version 2>/dev/null || true)"
|
||||
printf 'cargo=%s\n' "$(cargo --version 2>/dev/null || true)"
|
||||
} >"$OUT_DIR/run-metadata.env"
|
||||
}
|
||||
|
||||
safe_label() {
|
||||
printf '%s\n' "$1" | tr -c 'A-Za-z0-9_.-' '_'
|
||||
}
|
||||
|
||||
quote_command() {
|
||||
local quoted=()
|
||||
local arg
|
||||
for arg in "$@"; do
|
||||
quoted+=("$(printf '%q' "$arg")")
|
||||
done
|
||||
printf '%s ' "${quoted[@]}"
|
||||
}
|
||||
|
||||
record_skip() {
|
||||
local label="$1"
|
||||
local reason="$2"
|
||||
local log="$OUT_DIR/logs/$(safe_label "$label").log"
|
||||
printf 'SKIP: %s\n' "$reason" | tee "$log"
|
||||
printf 'SKIP\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
}
|
||||
|
||||
run_step() {
|
||||
local label="$1"
|
||||
shift
|
||||
local log="$OUT_DIR/logs/$(safe_label "$label").log"
|
||||
|
||||
{
|
||||
printf 'label=%s\n' "$label"
|
||||
printf 'command='
|
||||
quote_command "$@"
|
||||
printf '\n'
|
||||
} >"$log"
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
printf 'DRY-RUN: '
|
||||
quote_command "$@"
|
||||
printf '\n'
|
||||
printf 'DRY_RUN\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
return 0
|
||||
fi
|
||||
|
||||
printf 'RUN: %s\n' "$label"
|
||||
if "$@" 2>&1 | tee -a "$log"; then
|
||||
printf 'PASS\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
else
|
||||
local status=$?
|
||||
printf 'FAIL\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
return "$status"
|
||||
fi
|
||||
}
|
||||
|
||||
run_core_unit_steps() {
|
||||
run_step "filemeta-lib" cargo test -p rustfs-filemeta --lib
|
||||
run_step "ecstore-erasure-lib" cargo test -p rustfs-ecstore --lib erasure
|
||||
run_step "ecstore-set-disk-read-lib" cargo test -p rustfs-ecstore --lib set_disk::read
|
||||
run_step "ecstore-io-primitives-lib" cargo test -p rustfs-ecstore --lib set_disk::core::io_primitives
|
||||
run_step "ecstore-rename-rollback" \
|
||||
cargo test -p rustfs-ecstore --lib set_disk::tests::test_rename_data_quorum_failure_rolls_back_destination_object
|
||||
run_step "ecstore-disk-local-lib" cargo test -p rustfs-ecstore --lib disk::local
|
||||
run_step "ecstore-lib-all" cargo test -p rustfs-ecstore --lib -- --test-threads=1
|
||||
}
|
||||
|
||||
run_quick_e2e_steps() {
|
||||
if [[ "$SKIP_E2E" == "true" ]]; then
|
||||
record_skip "e2e-quick" "--skip-e2e was set"
|
||||
return
|
||||
fi
|
||||
|
||||
run_step "e2e-reliability-disk-fault" cargo test --package e2e_test reliability_disk_fault_test -- --nocapture
|
||||
run_step "e2e-heal-erasure-disk-rebuild" cargo test --package e2e_test heal_erasure_disk_rebuild_test -- --nocapture
|
||||
run_step "e2e-namespace-lock-quorum" cargo test --package e2e_test namespace_lock_quorum_test -- --nocapture
|
||||
}
|
||||
|
||||
run_quick_profile() {
|
||||
run_core_unit_steps
|
||||
run_quick_e2e_steps
|
||||
}
|
||||
|
||||
fixture_available() {
|
||||
[[ -n "${RUSTFS_MINIO_FIXTURE_ROOT:-}" && -d "${RUSTFS_MINIO_FIXTURE_ROOT}" && -n "${RUSTFS_MINIO_STATIC_KMS_KEY_B64:-}" ]]
|
||||
}
|
||||
|
||||
legacy_fixture_available() {
|
||||
local root="${RUSTFS_LEGACY_TEST_ROOT:-$(pwd)}"
|
||||
local disk="${RUSTFS_LEGACY_TEST_DISK:-}"
|
||||
if [[ -z "$disk" ]]; then
|
||||
if [[ -f "$root/test/.minio.sys/format.json" ]]; then
|
||||
disk="test"
|
||||
else
|
||||
disk="test1"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ ! -f "$root/$disk/.rustfs.sys/format.json" && ! -f "$root/$disk/.minio.sys/format.json" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
[[ -f "$root/$disk/vvvv/ktvzip.tar.gz/xl.meta" || -f "$root/$disk/vvvv/path_traversal.md/xl.meta" ]]
|
||||
}
|
||||
|
||||
minio_fixture_missing_reason() {
|
||||
if [[ -z "${RUSTFS_MINIO_FIXTURE_ROOT:-}" ]]; then
|
||||
printf 'RUSTFS_MINIO_FIXTURE_ROOT is not set\n'
|
||||
elif [[ ! -d "${RUSTFS_MINIO_FIXTURE_ROOT}" ]]; then
|
||||
printf 'RUSTFS_MINIO_FIXTURE_ROOT does not exist: %s\n' "$RUSTFS_MINIO_FIXTURE_ROOT"
|
||||
elif [[ -z "${RUSTFS_MINIO_STATIC_KMS_KEY_B64:-}" ]]; then
|
||||
printf 'RUSTFS_MINIO_STATIC_KMS_KEY_B64 is not set\n'
|
||||
else
|
||||
printf 'unknown MinIO fixture gate failure\n'
|
||||
fi
|
||||
}
|
||||
|
||||
write_blackbox_matrix() {
|
||||
local e2e_status="enabled"
|
||||
local s3_status="enabled"
|
||||
local destructive_status="enabled"
|
||||
local legacy_status="missing-optional"
|
||||
local minio_status="missing-optional"
|
||||
if [[ "$SKIP_E2E" == "true" ]]; then
|
||||
e2e_status="disabled"
|
||||
destructive_status="disabled"
|
||||
fi
|
||||
if [[ "$SKIP_S3_TESTS" == "true" ]]; then
|
||||
s3_status="disabled"
|
||||
fi
|
||||
if legacy_fixture_available; then
|
||||
legacy_status="enabled"
|
||||
elif [[ "$REQUIRE_FIXTURES" == "true" ]]; then
|
||||
legacy_status="missing-required"
|
||||
fi
|
||||
if fixture_available; then
|
||||
minio_status="enabled"
|
||||
elif [[ "$REQUIRE_FIXTURES" == "true" ]]; then
|
||||
minio_status="missing-required"
|
||||
fi
|
||||
|
||||
{
|
||||
printf 'profile\tscenario\tgate\tcommand\tfixture_env\tstatus\n'
|
||||
printf 'quick\tsingle-node disk fault read/write\tblack-box\tcargo test --package e2e_test reliability_disk_fault_test -- --nocapture\tnone\t%s\n' "$e2e_status"
|
||||
printf 'quick\theal degraded erasure disk rebuild\tblack-box\tcargo test --package e2e_test heal_erasure_disk_rebuild_test -- --nocapture\tnone\t%s\n' "$e2e_status"
|
||||
printf 'quick\tnamespace lock quorum under EC ops\tblack-box\tcargo test --package e2e_test namespace_lock_quorum_test -- --nocapture\tnone\t%s\n' "$e2e_status"
|
||||
printf 'full\tlegacy bitrot read fixture restore\tfixture\tcargo test -p rustfs-ecstore --test legacy_bitrot_read_test -- --nocapture\tRUSTFS_LEGACY_TEST_ROOT,RUSTFS_LEGACY_TEST_DISK\t%s\n' "$legacy_status"
|
||||
printf 'full\tMinIO generated encrypted read and negative restore fixture\tfixture\tcargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored --nocapture\tRUSTFS_MINIO_FIXTURE_ROOT,RUSTFS_MINIO_STATIC_KMS_KEY_B64\t%s\n' "$minio_status"
|
||||
printf 'full\tS3 multipart range versioning delete subset\tblack-box\tenv TESTEXPR=\"multipart or range or versioning or delete\" DEPLOY_MODE=build MAXFAIL=0 ./scripts/s3-tests/run.sh\tnone\t%s\n' "$s3_status"
|
||||
printf 'destructive\tdistributed cluster concurrency\tblack-box\tcargo test --package e2e_test cluster_concurrency_test -- --nocapture\tnone\t%s\n' "$destructive_status"
|
||||
printf 'destructive\tstale multipart cleanup cluster\tblack-box\tcargo test --package e2e_test stale_multipart_cleanup_cluster_test -- --nocapture\tnone\t%s\n' "$destructive_status"
|
||||
printf 'destructive\tdelete marker migration semantics\tblack-box\tcargo test --package e2e_test delete_marker_migration_semantics_test -- --nocapture\tnone\t%s\n' "$destructive_status"
|
||||
} >"$BLACKBOX_MATRIX"
|
||||
}
|
||||
|
||||
run_fixture_gate() {
|
||||
local label="ecstore-fixture-gate"
|
||||
local log="$OUT_DIR/logs/$(safe_label "$label").log"
|
||||
local legacy_ok=false
|
||||
local minio_ok=false
|
||||
if legacy_fixture_available; then
|
||||
legacy_ok=true
|
||||
fi
|
||||
if fixture_available; then
|
||||
minio_ok=true
|
||||
fi
|
||||
|
||||
{
|
||||
printf 'label=%s\n' "$label"
|
||||
printf 'legacy_fixture_available=%s\n' "$legacy_ok"
|
||||
printf 'minio_fixture_available=%s\n' "$minio_ok"
|
||||
printf 'require_fixtures=%s\n' "$REQUIRE_FIXTURES"
|
||||
if [[ "$minio_ok" != "true" ]]; then
|
||||
printf 'minio_fixture_reason=%s\n' "$(minio_fixture_missing_reason)"
|
||||
fi
|
||||
} | tee "$log"
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
printf 'DRY_RUN\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ "$legacy_ok" == "true" && "$minio_ok" == "true" ]]; then
|
||||
printf 'PASS\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ "$REQUIRE_FIXTURES" == "true" ]]; then
|
||||
printf 'FAIL\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
return 1
|
||||
fi
|
||||
|
||||
printf 'SKIP\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
}
|
||||
|
||||
run_fixture_steps() {
|
||||
if legacy_fixture_available; then
|
||||
run_step "ecstore-legacy-bitrot-read-fixture" cargo test -p rustfs-ecstore --test legacy_bitrot_read_test -- --nocapture
|
||||
elif [[ "$REQUIRE_FIXTURES" == "true" ]]; then
|
||||
echo "ERROR: RUSTFS_LEGACY_TEST_ROOT/RUSTFS_LEGACY_TEST_DISK is required for legacy bitrot fixture tests" >&2
|
||||
exit 1
|
||||
else
|
||||
record_skip "ecstore-legacy-bitrot-read-fixture" "legacy bitrot fixture is not present"
|
||||
fi
|
||||
|
||||
if fixture_available; then
|
||||
run_step "ecstore-minio-generated-read-fixture" \
|
||||
cargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored --nocapture
|
||||
elif [[ "$REQUIRE_FIXTURES" == "true" ]]; then
|
||||
echo "ERROR: $(minio_fixture_missing_reason)" >&2
|
||||
exit 1
|
||||
else
|
||||
record_skip "ecstore-minio-generated-read-fixture" "$(minio_fixture_missing_reason)"
|
||||
fi
|
||||
}
|
||||
|
||||
run_s3_subset() {
|
||||
if [[ "$SKIP_S3_TESTS" == "true" ]]; then
|
||||
record_skip "s3-tests-ec-subset" "--skip-s3-tests was set"
|
||||
return
|
||||
fi
|
||||
|
||||
run_step "s3-tests-ec-subset" \
|
||||
env TESTEXPR="multipart or range or versioning or delete" DEPLOY_MODE=build MAXFAIL=0 ./scripts/s3-tests/run.sh
|
||||
}
|
||||
|
||||
write_coverage_reports() {
|
||||
local lcov_path="$1"
|
||||
local coverage_summary="$2"
|
||||
local coverage_files="$3"
|
||||
local root="$4"
|
||||
|
||||
awk -v root="$root" -v scope="$UNIT_COVERAGE_SCOPE" -v summary="$coverage_summary" -v files="$coverage_files" '
|
||||
function is_crate_file(f) {
|
||||
return index(f, root "/crates/ecstore/src/") == 1
|
||||
}
|
||||
function is_ec_critical_file(f) {
|
||||
return index(f, root "/crates/ecstore/src/erasure/") == 1 ||
|
||||
f == root "/crates/ecstore/src/set_disk/read.rs" ||
|
||||
f == root "/crates/ecstore/src/set_disk/shard_source.rs" ||
|
||||
f == root "/crates/ecstore/src/set_disk/metadata.rs" ||
|
||||
f == root "/crates/ecstore/src/set_disk/ops/object.rs" ||
|
||||
f == root "/crates/ecstore/src/set_disk/core/io_primitives.rs" ||
|
||||
f == root "/crates/ecstore/src/disk/local.rs"
|
||||
}
|
||||
function is_gate_file(f) {
|
||||
return scope == "crate" ? is_crate_file(f) : is_ec_critical_file(f)
|
||||
}
|
||||
function pct(hit, found) {
|
||||
return found == 0 ? 0 : (hit / found) * 100
|
||||
}
|
||||
function flush() {
|
||||
if (file == "") {
|
||||
return
|
||||
}
|
||||
if (is_crate_file(file)) {
|
||||
crate_hit += hit
|
||||
crate_found += found
|
||||
}
|
||||
if (is_ec_critical_file(file)) {
|
||||
ec_hit += hit
|
||||
ec_found += found
|
||||
}
|
||||
if (is_gate_file(file)) {
|
||||
gate_hit += hit
|
||||
gate_found += found
|
||||
}
|
||||
if (is_crate_file(file)) {
|
||||
printf "%s\t%.2f\t%d\t%d\n", file, pct(hit, found), hit, found >> files
|
||||
}
|
||||
}
|
||||
BEGIN {
|
||||
print "file\tline_coverage\thit\tfound" > files
|
||||
}
|
||||
/^SF:/ {
|
||||
flush()
|
||||
file = substr($0, 4)
|
||||
hit = 0
|
||||
found = 0
|
||||
next
|
||||
}
|
||||
/^LH:/ { hit += substr($0, 4); next }
|
||||
/^LF:/ { found += substr($0, 4); next }
|
||||
END {
|
||||
flush()
|
||||
if (gate_found == 0) {
|
||||
exit 2
|
||||
}
|
||||
print "metric\tactual\tmin\ttarget\tscope" > summary
|
||||
printf "gate_line_coverage\t%.2f\t%s\t%s\t%s\n", pct(gate_hit, gate_found), min, target, scope >> summary
|
||||
printf "ec_critical_line_coverage\t%.2f\t%s\t%s\tec-critical\n", pct(ec_hit, ec_found), min, target >> summary
|
||||
printf "crate_line_coverage\t%.2f\t%s\t%s\tcrate\n", pct(crate_hit, crate_found), min, target >> summary
|
||||
}
|
||||
' min="$UNIT_COVERAGE_MIN" target="$UNIT_COVERAGE_TARGET" "$lcov_path"
|
||||
}
|
||||
|
||||
run_coverage() {
|
||||
if [[ "$SKIP_COVERAGE" == "true" ]]; then
|
||||
record_skip "ecstore-coverage" "--skip-coverage was set"
|
||||
return
|
||||
fi
|
||||
|
||||
local label="ecstore-unit-coverage"
|
||||
local log="$OUT_DIR/logs/$(safe_label "$label").log"
|
||||
local coverage_dir="$OUT_DIR/coverage/ecstore"
|
||||
local lcov_path="$coverage_dir/lcov.info"
|
||||
local coverage_summary="$coverage_dir/summary.tsv"
|
||||
local coverage_files="$coverage_dir/files.tsv"
|
||||
local cmd=(cargo llvm-cov -p rustfs-ecstore --lib --lcov --output-path "$lcov_path" -- --test-threads=1)
|
||||
local root
|
||||
root="$(pwd)"
|
||||
mkdir -p "$coverage_dir"
|
||||
|
||||
{
|
||||
printf 'label=%s\n' "$label"
|
||||
printf 'unit_coverage_min=%s\n' "$UNIT_COVERAGE_MIN"
|
||||
printf 'unit_coverage_target=%s\n' "$UNIT_COVERAGE_TARGET"
|
||||
printf 'unit_coverage_scope=%s\n' "$UNIT_COVERAGE_SCOPE"
|
||||
printf 'command='
|
||||
quote_command "${cmd[@]}"
|
||||
printf '\n'
|
||||
} >"$log"
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
printf 'DRY-RUN: '
|
||||
quote_command "${cmd[@]}"
|
||||
printf '\n'
|
||||
printf 'DRY_RUN\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
return
|
||||
fi
|
||||
|
||||
if ! command -v cargo-llvm-cov >/dev/null 2>&1; then
|
||||
printf 'ERROR: cargo-llvm-cov is required for the ECStore unit coverage gate\n' | tee -a "$log"
|
||||
printf 'FAIL\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
return 127
|
||||
fi
|
||||
|
||||
printf 'RUN: %s\n' "$label"
|
||||
if "${cmd[@]}" 2>&1 | tee -a "$log"; then
|
||||
:
|
||||
else
|
||||
local status=$?
|
||||
printf 'FAIL\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
return "$status"
|
||||
fi
|
||||
|
||||
if ! write_coverage_reports "$lcov_path" "$coverage_summary" "$coverage_files" "$root"; then
|
||||
printf 'ERROR: unable to compute scoped coverage from %s\n' "$lcov_path" | tee -a "$log"
|
||||
printf 'FAIL\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local line_coverage
|
||||
line_coverage="$(awk -F '\t' '$1 == "gate_line_coverage" { print $2 }' "$coverage_summary")"
|
||||
printf 'gate_line_coverage=%s%% scope=%s min=%s%% target=%s%%\n' \
|
||||
"$line_coverage" "$UNIT_COVERAGE_SCOPE" "$UNIT_COVERAGE_MIN" "$UNIT_COVERAGE_TARGET" | tee -a "$log"
|
||||
printf 'coverage_summary=%s\ncoverage_files=%s\n' "$coverage_summary" "$coverage_files" | tee -a "$log"
|
||||
|
||||
if awk -v actual="$line_coverage" -v min="$UNIT_COVERAGE_MIN" 'BEGIN { exit !(actual + 0 >= min + 0) }'; then
|
||||
printf 'PASS\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
else
|
||||
printf 'FAIL: %s coverage %s%% is below %s%%\n' "$UNIT_COVERAGE_SCOPE" "$line_coverage" "$UNIT_COVERAGE_MIN" | tee -a "$log"
|
||||
printf 'FAIL\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
run_full_profile() {
|
||||
run_quick_profile
|
||||
run_fixture_gate
|
||||
run_fixture_steps
|
||||
run_s3_subset
|
||||
run_coverage
|
||||
}
|
||||
|
||||
run_destructive_profile() {
|
||||
run_full_profile
|
||||
if [[ "$SKIP_E2E" == "true" ]]; then
|
||||
record_skip "e2e-destructive" "--skip-e2e was set"
|
||||
return
|
||||
fi
|
||||
|
||||
run_step "e2e-cluster-concurrency" cargo test --package e2e_test cluster_concurrency_test -- --nocapture
|
||||
run_step "e2e-stale-multipart-cleanup-cluster" cargo test --package e2e_test stale_multipart_cleanup_cluster_test -- --nocapture
|
||||
run_step "e2e-delete-marker-migration-semantics" cargo test --package e2e_test delete_marker_migration_semantics_test -- --nocapture
|
||||
}
|
||||
|
||||
run_fuzz_profile() {
|
||||
local max_total_time="${MAX_TOTAL_TIME:-60}"
|
||||
run_step "fuzz-storage-inputs" env MAX_TOTAL_TIME="$max_total_time" ./scripts/fuzz/run.sh
|
||||
}
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
validate_args
|
||||
setup_workspace
|
||||
write_metadata
|
||||
write_blackbox_matrix
|
||||
|
||||
case "$PROFILE" in
|
||||
quick) run_quick_profile ;;
|
||||
full) run_full_profile ;;
|
||||
destructive) run_destructive_profile ;;
|
||||
fuzz) run_fuzz_profile ;;
|
||||
esac
|
||||
|
||||
printf 'summary=%s\n' "$SUMMARY"
|
||||
printf 'artifacts=%s\n' "$OUT_DIR"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user