gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-08 04:58:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): preserve erasure writer error identity (#7455)

Browse Source 
* test(ecstore): pin erasure writer error identity

* fix(ecstore): preserve reduced erasure writer errors

* test(ecstore): size error fixtures for their shard payloads

* chore(ecstore): tighten formatted error baseline
This commit is contained in:
Zhengchao An
2026-09-08 12:39:50 +08:00
committed by GitHub
parent e65788f2e8
commit 0a9f0f59a7
2 changed files with 109 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/erasure/coding/encode.rs
+108 -34
View File
@@ -431,7 +431,7 @@ impl<'a> MultiWriter<'a> {
errs = ?self.errs,
"Erasure encode write quorum unavailable: {summary_text}"
);
Err(std::io::Error::other(format!("Failed to write data: {summary_text}")))
Err(write_err.into())
}
async fn shutdown_writer(writer_opt: &mut Option<BitrotWriterWrapper>, err: &mut Option<Error>) {
@@ -503,7 +503,7 @@ impl<'a> MultiWriter<'a> {
errs = ?self.errs,
"Erasure encode shutdown quorum unavailable: {summary_text}"
);
Err(std::io::Error::other(format!("Failed to shutdown writers: {summary_text}")))
Err(write_err.into())
}
}
@@ -1002,6 +1002,7 @@ impl Erasure {
mod tests {
use super::*;
use crate::erasure::coding::{BitrotWriterWrapper, CustomWriter};
use crate::error::StorageError;
use rustfs_rio::HardLimitReader;
use rustfs_utils::HashAlgorithm;
use std::future::Future;
@@ -1451,7 +1452,14 @@ mod tests {
Ok(_) => panic!("writer quorum failure should fail the encode pipeline"),
Err(err) => err,
};
assert!(err.to_string().contains("Failed to write data"));
let err = StorageError::from(err);
assert!(matches!(
&err,
StorageError::Io(source)
if source.kind() == std::io::ErrorKind::Other
&& source.to_string() == "injected write failure after producer blocks"
));
assert!(!err.is_quorum_error());
tokio::time::timeout(Duration::from_secs(1), reader_dropped)
.await
.expect("writer failure should abort the blocked producer")
@@ -1644,7 +1652,7 @@ mod tests {
#[tokio::test]
async fn multi_writer_short_write_fails_before_shutdown() {
let mut writers = vec![Some(bitrot_writer(ShortWriteWriter, 16))];
let mut writers = vec![Some(bitrot_writer(ShortWriteWriter, 32))];
let err = {
let mut writer = MultiWriter::new(&mut writers, 1);
writer
@@ -1653,63 +1661,93 @@ mod tests {
.expect_err("short writes must fail the shard writer")
};
assert!(err.to_string().contains("Failed to write data"));
let err = StorageError::from(err);
assert!(matches!(&err, StorageError::Io(source) if source.kind() == std::io::ErrorKind::WriteZero));
assert!(!err.is_quorum_error());
assert!(writers[0].is_none(), "short-write shard must be removed before commit");
}
#[tokio::test]
async fn multi_writer_reports_fallback_summary_when_only_offline_writers_remain() {
let mut writers = vec![None, None];
let err = {
let (err, summary) = {
let mut writer = MultiWriter::new(&mut writers, 1);
writer
let err = writer
.write(vec![Bytes::from_static(b"offline-a"), Bytes::from_static(b"offline-b")])
.await
.expect_err("offline writers cannot satisfy write quorum")
.expect_err("offline writers cannot satisfy write quorum");
let summary = build_write_quorum_failure_summary(&writer.errs, OBJECT_OP_IGNORED_ERRS, writer.write_quorum);
(err, format_write_quorum_failure(&summary))
};
let err = err.to_string();
assert!(err.contains("Failed to write data"));
assert!(err.contains("offline-disks=2/2"));
assert!(err.contains("required=1"));
assert_eq!(
err.get_ref().and_then(|source| source.downcast_ref::<Error>()),
Some(&Error::ErasureWriteQuorum),
);
let err = StorageError::from(err);
assert_eq!(err, StorageError::ErasureWriteQuorum);
assert!(err.is_quorum_error());
assert!(summary.contains("offline-disks=2/2"));
assert!(summary.contains("required=1"));
let shutdown_err = {
let (shutdown_err, summary) = {
let mut writer = MultiWriter::new(&mut writers, 1);
writer
let err = writer
.shutdown()
.await
.expect_err("offline writers cannot satisfy shutdown quorum")
.expect_err("offline writers cannot satisfy shutdown quorum");
let summary = build_write_quorum_failure_summary(&writer.errs, OBJECT_OP_IGNORED_ERRS, writer.write_quorum);
(err, format_write_quorum_failure(&summary))
};
let shutdown_err = shutdown_err.to_string();
assert!(shutdown_err.contains("Failed to shutdown writers"));
assert!(shutdown_err.contains("offline-disks=2/2"));
assert!(shutdown_err.contains("required=1"));
assert_eq!(
shutdown_err.get_ref().and_then(|source| source.downcast_ref::<Error>()),
Some(&Error::ErasureWriteQuorum),
);
let shutdown_err = StorageError::from(shutdown_err);
assert_eq!(shutdown_err, StorageError::ErasureWriteQuorum);
assert!(shutdown_err.is_quorum_error());
assert!(summary.contains("offline-disks=2/2"));
assert!(summary.contains("required=1"));
}
#[tokio::test]
async fn multi_writer_reports_quorum_failure_when_quorum_exceeds_writer_count() {
let committed = Arc::new(Mutex::new(Vec::new()));
let mut writers = vec![Some(bitrot_writer(DeferredCommitWriter::new(committed), 16))];
let mut writers = vec![Some(bitrot_writer(DeferredCommitWriter::new(committed), 32))];
let mut writer = MultiWriter::new(&mut writers, 2);
let err = writer
.write(vec![Bytes::from_static(b"quorum impossible")])
.await
.expect_err("write quorum above writer count must fail");
let err = err.to_string();
assert!(err.contains("Failed to write data"));
assert!(err.contains("required=2"));
assert!(err.contains("erasure write quorum"));
assert_eq!(
err.get_ref().and_then(|source| source.downcast_ref::<Error>()),
Some(&Error::ErasureWriteQuorum),
);
let err = StorageError::from(err);
assert_eq!(err, StorageError::ErasureWriteQuorum);
assert!(err.is_quorum_error());
let summary = build_write_quorum_failure_summary(&writer.errs, OBJECT_OP_IGNORED_ERRS, writer.write_quorum);
let summary = format_write_quorum_failure(&summary);
assert!(summary.contains("required=2"));
assert!(summary.contains("erasure write quorum"));
let shutdown_err = writer
.shutdown()
.await
.expect_err("shutdown quorum above writer count must fail");
let shutdown_err = shutdown_err.to_string();
assert!(shutdown_err.contains("Failed to shutdown writers"));
assert!(shutdown_err.contains("required=2"));
assert!(shutdown_err.contains("erasure write quorum"));
assert_eq!(
shutdown_err.get_ref().and_then(|source| source.downcast_ref::<Error>()),
Some(&Error::ErasureWriteQuorum),
);
let shutdown_err = StorageError::from(shutdown_err);
assert_eq!(shutdown_err, StorageError::ErasureWriteQuorum);
assert!(shutdown_err.is_quorum_error());
let summary = build_write_quorum_failure_summary(&writer.errs, OBJECT_OP_IGNORED_ERRS, writer.write_quorum);
let summary = format_write_quorum_failure(&summary);
assert!(summary.contains("required=2"));
assert!(summary.contains("erasure write quorum"));
}
// The production wiring (`MultiWriter::new`) must arm a real deadline by
@@ -1794,7 +1832,13 @@ mod tests {
.write(four_shards())
.await
.expect_err("two stalled writers must fail the write quorum instead of hanging");
assert!(err.to_string().contains("Failed to write data"));
assert_eq!(
err.get_ref().and_then(|source| source.downcast_ref::<Error>()),
Some(&Error::ErasureWriteQuorum),
);
let err = StorageError::from(err);
assert_eq!(err, StorageError::ErasureWriteQuorum);
assert!(err.is_quorum_error());
}
// A small object whose bytes were fully buffered leaves `write` succeeding
@@ -1839,7 +1883,13 @@ mod tests {
.shutdown()
.await
.expect_err("two shutdown stalls must fail the shutdown quorum instead of hanging");
assert!(err.to_string().contains("Failed to shutdown writers"));
assert_eq!(
err.get_ref().and_then(|source| source.downcast_ref::<Error>()),
Some(&Error::ErasureWriteQuorum),
);
let err = StorageError::from(err);
assert_eq!(err, StorageError::ErasureWriteQuorum);
assert!(err.is_quorum_error());
}
// A slow-but-honest writer that keeps completing shards (delay < stall
@@ -2121,7 +2171,13 @@ mod tests {
.await
.expect_err("streaming encode must fail when write quorum is unavailable");
assert!(err.to_string().contains("Failed to write data"));
assert_eq!(
err.get_ref().and_then(|source| source.downcast_ref::<Error>()),
Some(&Error::ErasureWriteQuorum),
);
let err = StorageError::from(err);
assert_eq!(err, StorageError::ErasureWriteQuorum);
assert!(err.is_quorum_error());
}
#[tokio::test]
@@ -2145,7 +2201,13 @@ mod tests {
.await
.expect_err("write quorum failure must fail the inline encode");
assert!(err.to_string().contains("Failed to write data"));
assert_eq!(
err.get_ref().and_then(|source| source.downcast_ref::<Error>()),
Some(&Error::ErasureWriteQuorum),
);
let err = StorageError::from(err);
assert_eq!(err, StorageError::ErasureWriteQuorum);
assert!(err.is_quorum_error());
assert!(
committed.lock().expect("committed buffer should be lockable").is_empty(),
"successful writer must not be committed when write quorum fails before shutdown"
@@ -2173,7 +2235,13 @@ mod tests {
.await
.expect_err("shutdown quorum failure must fail the inline encode");
assert!(err.to_string().contains("Failed to shutdown writers"));
let err = StorageError::from(err);
assert!(matches!(
&err,
StorageError::Io(source)
if source.kind() == std::io::ErrorKind::Other && source.to_string() == "injected shutdown failure"
));
assert!(!err.is_quorum_error());
assert!(
!committed.lock().expect("committed buffer should be lockable").is_empty(),
"the successful writer should have committed before shutdown quorum failure was reported"
@@ -2395,7 +2463,13 @@ mod tests {
.await
.expect_err("batched encode must fail when write quorum is unavailable");
assert!(err.to_string().contains("Failed to write data"));
assert_eq!(
err.get_ref().and_then(|source| source.downcast_ref::<Error>()),
Some(&Error::ErasureWriteQuorum),
);
let err = StorageError::from(err);
assert_eq!(err, StorageError::ErasureWriteQuorum);
assert!(err.is_quorum_error());
}
#[tokio::test]
scripts/error-other-format-baseline.txt
+1 -1
View File
@@ -33,7 +33,7 @@
1|crates/ecstore/src/disk/mod.rs
5|crates/ecstore/src/erasure/codec/bridge.rs
1|crates/ecstore/src/erasure/coding/decode_reader.rs
10|crates/ecstore/src/erasure/coding/encode.rs
8|crates/ecstore/src/erasure/coding/encode.rs
25|crates/ecstore/src/erasure/coding/erasure.rs
3|crates/ecstore/src/layout/disks_layout.rs
2|crates/ecstore/src/layout/endpoint.rs