perf(ecstore): improve erasure write diagnostics and single-block performance (#3280)

* docs(object-capacity): add localized crate docs * fix(ecstore): improve quorum and transport diagnostics * perf(ecstore): add safe single-block write fast path * refactor(ecstore): collapse layered small write paths * chore(docs): keep issue 662 design note tracked * fix(docs): restore issue 662 design note * chore(docs): keep issue 662 design local only * feat(obs): add internode reliability metrics and dashboard * feat(obs): extend internode diagnostics and service logging * fix(docs): use AGENTS guide filename * perf(ecstore): reuse owned buffer in small encode * fix(ecstore): tighten small write diagnostics --------- Co-authored-by: cxymds <Cxymds@qq.com>
2026-08-09 06:39:25 +00:00 · 2026-06-08 17:45:56 +08:00
parent ddd35badad
commit ed73952cb6
16 changed files with 2455 additions and 123 deletions
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+use rustfs_rio::{InternodeHttpError, InternodeHttpErrorKind};
 use std::hash::{Hash, Hasher};
 use std::io::{self};
 use std::path::PathBuf;
@@ -181,6 +182,26 @@ impl DiskError {
        matches!(err, &DiskError::FileVersionNotFound)
    }

+    pub fn is_retryable_internode_write_failure(&self) -> bool {
+        match self {
+            DiskError::Io(io_error) => io_error
+                .get_ref()
+                .and_then(|source| source.downcast_ref::<InternodeHttpError>())
+                .is_some_and(|err| err.kind().is_retryable()),
+            _ => false,
+        }
+    }
+
+    pub fn internode_http_error_kind(&self) -> Option<InternodeHttpErrorKind> {
+        match self {
+            DiskError::Io(io_error) => io_error
+                .get_ref()
+                .and_then(|source| source.downcast_ref::<InternodeHttpError>())
+                .map(InternodeHttpError::kind),
+            _ => None,
+        }
+    }
+
    // /// If all errors are of the same fatal disk error type, returns the corresponding error.
    // /// Otherwise, returns Ok.
    // pub fn check_disk_fatal_errs(errs: &[Option<Error>]) -> Result<()> {
@@ -241,7 +262,10 @@ impl From<rustfs_filemeta::Error> for DiskError {

 impl From<std::io::Error> for DiskError {
    fn from(e: std::io::Error) -> Self {
-        e.downcast::<DiskError>().unwrap_or_else(DiskError::Io)
+        match e.downcast::<DiskError>() {
+            Ok(disk_error) => disk_error,
+            Err(io_error) => DiskError::Io(io_error),
+        }
    }
 }

@@ -14,6 +14,19 @@

 use crate::disk::error::Error;

+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WriteQuorumFailureSummary {
+    pub required: usize,
+    pub achieved: usize,
+    pub failed: usize,
+    pub total: usize,
+    pub offline_disks: usize,
+    pub ignored_failures: usize,
+    pub retryable_failures: usize,
+    pub dominant_error: Option<Error>,
+    pub dominant_error_label: &'static str,
+}
+
 pub static OBJECT_OP_IGNORED_ERRS: &[Error] = &[
    Error::DiskNotFound,
    Error::FaultyDisk,
@@ -77,6 +90,58 @@ pub fn reduce_errs(errors: &[Option<Error>], ignored_errs: &[Error]) -> (usize,
    }
 }

+pub fn build_write_quorum_failure_summary(
+    errors: &[Option<Error>],
+    ignored_errs: &[Error],
+    quorum: usize,
+) -> WriteQuorumFailureSummary {
+    let total = errors.len();
+    let achieved = errors.iter().filter(|err| err.is_none()).count();
+    let failed = total.saturating_sub(achieved);
+    let offline_disks = count_errs(errors, &Error::DiskNotFound);
+    let ignored_failures = errors
+        .iter()
+        .filter_map(|err| err.as_ref())
+        .filter(|err| is_ignored_err(ignored_errs, err))
+        .count();
+    let retryable_failures = count_retryable_failures(errors);
+    let (_, dominant_error) = reduce_errs(errors, ignored_errs);
+    let dominant_error_label = dominant_error_label(errors, ignored_errs, dominant_error.as_ref());
+
+    WriteQuorumFailureSummary {
+        required: quorum,
+        achieved,
+        failed,
+        total,
+        offline_disks,
+        ignored_failures,
+        retryable_failures,
+        dominant_error,
+        dominant_error_label,
+    }
+}
+
+fn dominant_error_label(errors: &[Option<Error>], ignored_errs: &[Error], dominant_error: Option<&Error>) -> &'static str {
+    let Some(dominant_error) = dominant_error else {
+        return "nil_dominated";
+    };
+
+    if dominant_error == &Error::DiskNotFound {
+        return "disk_not_found";
+    }
+    if dominant_error == &Error::ShortWrite {
+        return "short_write";
+    }
+
+    errors
+        .iter()
+        .filter_map(|err| err.as_ref())
+        .find(|err| !is_ignored_err(ignored_errs, err) && *err == dominant_error)
+        .and_then(Error::internode_http_error_kind)
+        .map(|kind| kind.metric_label())
+        .unwrap_or("other_error")
+}
+
 pub fn is_ignored_err(ignored_errs: &[Error], err: &Error) -> bool {
    ignored_errs.iter().any(|e| e == err)
 }
@@ -85,6 +150,14 @@ pub fn count_errs(errors: &[Option<Error>], err: &Error) -> usize {
    errors.iter().filter(|&e| e.as_ref() == Some(err)).count()
 }

+pub fn count_retryable_failures(errors: &[Option<Error>]) -> usize {
+    errors
+        .iter()
+        .filter_map(|err| err.as_ref())
+        .filter(|err| err.is_retryable_internode_write_failure())
+        .count()
+}
+
 pub fn is_all_buckets_not_found(errs: &[Option<Error>]) -> bool {
    for err in errs.iter() {
        if let Some(err) = err {
@@ -163,6 +236,51 @@ mod tests {
        assert!(!is_ignored_err(&ignored, &e2));
    }

+    #[test]
+    fn test_build_write_quorum_failure_summary() {
+        let retryable = Error::from(rustfs_rio::new_test_internode_http_io_error(
+            rustfs_rio::InternodeHttpErrorKind::ConnectionReset,
+        ));
+        let non_retryable = err_io("other");
+        let errors = vec![
+            None,
+            None,
+            None,
+            None,
+            None,
+            Some(retryable),
+            Some(non_retryable),
+            Some(Error::DiskNotFound),
+        ];
+
+        let summary = build_write_quorum_failure_summary(&errors, OBJECT_OP_IGNORED_ERRS, 6);
+        assert_eq!(summary.required, 6);
+        assert_eq!(summary.achieved, 5);
+        assert_eq!(summary.failed, 3);
+        assert_eq!(summary.total, 8);
+        assert_eq!(summary.offline_disks, 1);
+        assert_eq!(summary.ignored_failures, 1);
+        assert_eq!(summary.retryable_failures, 1);
+        assert_eq!(summary.dominant_error, None);
+        assert_eq!(summary.dominant_error_label, "nil_dominated");
+    }
+
+    #[test]
+    fn test_build_write_quorum_failure_summary_preserves_internode_label() {
+        let retryable_a = Error::from(rustfs_rio::new_test_internode_http_io_error(
+            rustfs_rio::InternodeHttpErrorKind::ConnectionReset,
+        ));
+        let retryable_b = Error::from(rustfs_rio::new_test_internode_http_io_error(
+            rustfs_rio::InternodeHttpErrorKind::ConnectionReset,
+        ));
+        let errors = vec![Some(retryable_a), Some(retryable_b)];
+
+        let summary = build_write_quorum_failure_summary(&errors, OBJECT_OP_IGNORED_ERRS, 2);
+
+        assert_eq!(summary.retryable_failures, 2);
+        assert_eq!(summary.dominant_error_label, "connection_reset");
+    }
+
    #[test]
    fn test_reduce_errs_nil_tiebreak() {
        // Error::Nil and another error have the same count, should prefer Nil