chore(ci): update error format ratchet baseline

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

crates/ecstore/src/api/mod.rs

@@ -416,8 +416,8 @@ pub mod disk {
 
 pub mod error {
     pub use crate::error::{
-        Error, Result, StorageError, classify_system_path_failure_reason, is_err_bucket_not_found, is_err_object_not_found,
-        is_err_version_not_found,
+        Error, PoolMetadataError, PoolMetadataFailure, Result, StorageError, classify_system_path_failure_reason,
+        is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
     };
 }
 

crates/ecstore/src/error/mod.rs

@@ -23,17 +23,59 @@ use s3s::S3ErrorCode;
 pub type Error = StorageError;
 pub type Result<T> = core::result::Result<T, Error>;
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PoolMetadataFailure {
+    ReadUnavailable,
+    RecoveryRequired,
+    TransactionUnknown,
+    FenceLost,
+}
+
+impl PoolMetadataFailure {
+    fn recovery_hint(self) -> &'static str {
+        match self {
+            Self::ReadUnavailable => "read unavailable; retry after the replicas are readable",
+            Self::TransactionUnknown => "writes remain blocked pending fenced transaction recovery",
+            Self::RecoveryRequired | Self::FenceLost => {
+                "writes remain blocked after a recovery-required replica state; restart after all replicas are readable and consistent, with compatible formats"
+            }
+        }
+    }
+
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::ReadUnavailable => "read_unavailable",
+            Self::RecoveryRequired => "recovery_required",
+            Self::TransactionUnknown => "transaction_unknown",
+            Self::FenceLost => "fence_lost",
+        }
+    }
+}
+
+/// Local control-plane context. Keep the existing storage error wire codes;
+/// the HTTP boundary recognizes this typed source, not an error-message prefix.
+#[derive(Debug, Clone, thiserror::Error)]
+#[error("{operation}: pool metadata {hint} ({reason}, {phase}): {detail}", hint = kind.recovery_hint(), reason = kind.as_str(), detail = source.as_ref().map(ToString::to_string).unwrap_or_default())]
+pub struct PoolMetadataError {
+    pub kind: PoolMetadataFailure,
+    pub operation: String,
+    pub phase: &'static str,
+    pub since: time::OffsetDateTime,
+    #[source]
+    pub source: Option<std::sync::Arc<StorageError>>,
+}
+
 /// Keeps high-cardinality diagnostic detail in the error source while making
 /// the rendered `io::Error` stable for quorum aggregation.
 #[derive(Debug)]
 struct StableIoContextError {
-    message: &'static str,
+    message: std::borrow::Cow<'static, str>,
     source: Box<dyn std::error::Error + Send + Sync>,
 }
 
 impl std::fmt::Display for StableIoContextError {
     fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        formatter.write_str(self.message)
+        formatter.write_str(&self.message)
     }
 }
 
@@ -48,7 +90,7 @@ where
     E: Into<Box<dyn std::error::Error + Send + Sync>>,
 {
     std::io::Error::other(StableIoContextError {
-        message,
+        message: message.into(),
         source: source.into(),
     })
 }
@@ -300,6 +342,22 @@ impl From<crate::erasure::coding::ErasureConstructionError> for StorageError {
 }
 
 impl StorageError {
+    pub fn pool_metadata_failure(&self) -> Option<&PoolMetadataError> {
+        let mut current: Option<&(dyn std::error::Error + 'static)> = Some(self);
+        while let Some(error) = current {
+            if let Some(context) = error.downcast_ref::<PoolMetadataError>() {
+                return Some(context);
+            }
+            // io::Error::source skips its boxed context itself.
+            current = if let Some(io) = error.downcast_ref::<std::io::Error>() {
+                io.get_ref().map(|inner| inner as &(dyn std::error::Error + 'static))
+            } else {
+                error.source()
+            };
+        }
+        None
+    }
+
     pub fn other<E>(error: E) -> Self
     where
         E: Into<Box<dyn std::error::Error + Send + Sync>>,
@@ -548,7 +606,19 @@ impl PartialEq for StorageError {
 impl Clone for StorageError {
     fn clone(&self) -> Self {
         match self {
-            StorageError::Io(e) => StorageError::Io(std::io::Error::new(e.kind(), e.to_string())),
+            StorageError::Io(e) => {
+                if let Some(context) = self.pool_metadata_failure() {
+                    Self::Io(std::io::Error::new(
+                        e.kind(),
+                        StableIoContextError {
+                            message: e.to_string().into(),
+                            source: Box::new(context.clone()),
+                        },
+                    ))
+                } else {
+                    StorageError::Io(std::io::Error::new(e.kind(), e.to_string()))
+                }
+            }
             StorageError::FaultyDisk => StorageError::FaultyDisk,
             StorageError::DiskFull => StorageError::DiskFull,
             StorageError::VolumeNotFound => StorageError::VolumeNotFound,

crates/ecstore/src/store/heal.rs

@@ -2353,14 +2353,19 @@ mod tests {
             .await
             .expect("quorum boundary heal should return a mapped result");
         *store.pools[0].disk_set[0].disks.write().await = original_quorum_disks;
-        let quorum_err_text = quorum_err.as_ref().map(ToString::to_string);
-        assert!(
-            quorum_err_text.as_deref().is_some_and(|err| {
-                err.contains("target capacity admission failed")
-                    && err.contains("pool metadata update cannot overwrite an unreadable replica")
-            }),
-            "heal must fail closed when capacity admission cannot verify pool metadata, got {quorum_err:?}"
+        let quorum_err = quorum_err
+            .as_ref()
+            .expect("heal must fail closed when capacity admission cannot verify pool metadata");
+        let quorum_failure = quorum_err
+            .pool_metadata_failure()
+            .expect("capacity admission failure should preserve typed pool metadata context");
+        assert_eq!(
+            quorum_failure.kind,
+            crate::error::PoolMetadataFailure::ReadUnavailable,
+            "read-only capacity admission failure must remain retryable"
         );
+        assert_eq!(quorum_failure.operation, "target capacity admission failed");
+        assert_eq!(quorum_failure.phase, "pool_read");
         assert!(
             store.pool_meta_writes_ready().await,
             "read-only capacity admission failure must not latch the pool metadata writer"

crates/ecstore/src/store/init.rs

@@ -14,8 +14,8 @@
 
 use super::*;
 use crate::core::pools::{
-    PoolMetaBootstrapAuthority, PoolMetaReplicaState, PoolMetaWriteState, load_pool_meta_identity_observing,
-    local_decommission_queue_prefix, persist_pool_meta_identity_for_startup, pool_meta_has_active_decommission,
+    PoolMetaBootstrapAuthority, PoolMetaReplicaState, PoolMetaWriteState, local_decommission_queue_prefix,
+    persist_pool_meta_identity_for_startup, pool_meta_has_active_decommission,
 };
 use crate::runtime::instance::InstanceContext;
 use crate::runtime::sources as runtime_sources;
@@ -153,14 +153,11 @@ async fn load_pool_meta_for_startup<S>(
 where
     S: EcstoreObjectIO,
 {
-    load_pool_meta_identity_observing(pools.clone(), write_state)
-        .await
-        .map_err(|err| Error::other(format!("store init failed during load_pool_meta_identity: {err}")))?;
     let mut meta = PoolMeta::default();
     let replica_state = meta
-        .load_no_lock_from_replicas_observing(pools, write_state)
+        .load_for_startup_observing(pools, write_state)
         .await
-        .map_err(|err| Error::other(format!("store init failed during load_pool_meta: {err}")))?;
+        .map_err(|err| Error::other_with_context("store init failed during load_pool_meta", err))?;
     write_state.observe_replicas(replica_state);
     write_state
         .ensure_missing_metadata_can_initialize()
@@ -769,6 +766,33 @@ impl ECStore {
             });
         }
 
+        let recovery_store = self.clone();
+        let recovery_rx = rx.clone();
+        tokio::spawn(async move {
+            let mut delay = std::time::Duration::from_secs(5);
+            loop {
+                tokio::select! {
+                    _ = recovery_rx.cancelled() => return,
+                    _ = tokio::time::sleep(delay) => {}
+                }
+                let result = tokio::select! {
+                    _ = recovery_rx.cancelled() => return,
+                    result = tokio::time::timeout(std::time::Duration::from_secs(30), recovery_store.recover_pool_meta_transaction()) => result,
+                };
+                delay = match result {
+                    Ok(Ok(_)) => std::time::Duration::from_secs(5),
+                    failure => {
+                        let error = match failure {
+                            Ok(Err(error)) => error,
+                            _ => Error::Timeout,
+                        };
+                        recovery_store.record_pool_meta_recovery_failure(error);
+                        (delay * 2).min(std::time::Duration::from_secs(60))
+                    }
+                };
+            }
+        });
+
         runtime_sources::init_bucket_monitor_for_current_endpoints();
         crate::bucket::bucket_target_sys::BucketTargetSys::get().start_heartbeat();
 
@@ -2493,6 +2517,106 @@ mod tests {
         shutdown.cancel();
     }
 
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn pool_metadata_preflight_recovery_preserves_single_and_multi_pool_public_mutations() {
+        for layout in [vec![4], vec![4, 4]] {
+            let temp_dir = tempfile::tempdir().unwrap();
+            let (_ctx, store, shutdown) =
+                without_storage_class_env(build_isolated_test_store(temp_dir.path(), "pool-meta-retry", &layout)).await;
+            crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
+            let bucket = format!("pool-meta-retry-{}", Uuid::new_v4());
+            store.make_bucket(&bucket, &MakeBucketOptions::default()).await.unwrap();
+            let mut saved_disks = Vec::new();
+            for set in &store.pools[0].disk_set {
+                let mut disks = set.disks.write().await;
+                let count = disks.len();
+                saved_disks.push((set.clone(), std::mem::replace(&mut *disks, vec![None; count])));
+            }
+            let indices = (0..layout.len()).collect::<Vec<_>>();
+            let err = store.save_current_pool_meta_for_test(&indices).await.unwrap_err();
+            assert_eq!(
+                err.pool_metadata_failure().unwrap().kind,
+                crate::error::PoolMetadataFailure::ReadUnavailable
+            );
+            for (set, disks) in saved_disks {
+                *set.disks.write().await = disks;
+            }
+            store.save_current_pool_meta_for_test(&indices).await.unwrap();
+            assert!(store.pool_meta_writes_ready().await);
+
+            let payload = b"pool metadata recovery payload".to_vec();
+            store
+                .put_object(&bucket, "put", &mut PutObjReader::from_vec(payload.clone()), &ObjectOptions::default())
+                .await
+                .unwrap();
+            let mut reader = store
+                .get_object_reader(&bucket, "put", None, HeaderMap::new(), &ObjectOptions::default())
+                .await
+                .unwrap();
+            let mut actual = Vec::new();
+            reader.stream.read_to_end(&mut actual).await.unwrap();
+            assert_eq!(actual, payload);
+            drop(reader);
+            store.delete_object(&bucket, "put", ObjectOptions::default()).await.unwrap();
+            assert!(crate::error::is_err_object_not_found(
+                &store
+                    .get_object_info(&bucket, "put", &ObjectOptions::default())
+                    .await
+                    .unwrap_err()
+            ));
+
+            let upload = store
+                .new_multipart_upload(&bucket, "multipart", &ObjectOptions::default())
+                .await
+                .unwrap();
+            let part = store
+                .put_object_part(
+                    &bucket,
+                    "multipart",
+                    &upload.upload_id,
+                    1,
+                    &mut PutObjReader::from_vec(payload.clone()),
+                    &ObjectOptions::default(),
+                )
+                .await
+                .unwrap();
+            store
+                .clone()
+                .complete_multipart_upload(
+                    &bucket,
+                    "multipart",
+                    &upload.upload_id,
+                    vec![crate::storage_api_contracts::multipart::CompletePart {
+                        part_num: part.part_num,
+                        etag: part.etag,
+                        ..Default::default()
+                    }],
+                    &ObjectOptions::default(),
+                )
+                .await
+                .unwrap();
+            let mut reader = store
+                .get_object_reader(&bucket, "multipart", None, HeaderMap::new(), &ObjectOptions::default())
+                .await
+                .unwrap();
+            actual.clear();
+            reader.stream.read_to_end(&mut actual).await.unwrap();
+            assert_eq!(actual, payload);
+            drop(reader);
+            let upload = store
+                .new_multipart_upload(&bucket, "abort", &ObjectOptions::default())
+                .await
+                .unwrap();
+            store
+                .abort_multipart_upload(&bucket, "abort", &upload.upload_id, &ObjectOptions::default())
+                .await
+                .unwrap();
+            assert!(store.pool_meta_writes_ready().await);
+            shutdown.cancel();
+        }
+    }
+
     #[cfg(feature = "test-util")]
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     #[serial_test::serial(storage_class_env)]

docs/architecture/readiness-matrix.md

@@ -40,6 +40,9 @@ an unknown or unsupported peer-health snapshot degrades readiness with
 - Liveness reports process availability and must not depend on storage, IAM,
   lock quorum, or peer health.
 - Node readiness reports local dependency readiness.
+- A blocked pool metadata writer degrades node and cluster-write readiness with
+  `pool_metadata_blocked`. Metadata save-gate inspection is bounded to 100 ms;
+  contention reports `pool_metadata_check_timeout` without installing a block.
 - Cluster write readiness requires write quorum and the runtime dependency
   readiness used by `FullReady`.
 - Cluster read readiness may use the read-quorum path and cluster-health

docs/operations/pool-metadata-recovery.md

@@ -32,6 +32,27 @@ A V3 update first conditionally writes a pending generation containing the last
 
 Do not hand-edit a pending record or select a replica only because it is in pool zero. Preserve all copies when escalating recovery.
 
+## Runtime write recovery
+
+A runtime metadata or identity read failure before any write is dispatched rejects that operation but does not permanently block subsequent retries. Errors retain their typed cause; pool metadata unavailability reaches S3 as `503 ServiceUnavailable`, without exposing internal error details. Cancellation during preflight or a rejected first conditional write is also retryable. After any identity, prepare, or commit write may have started, cancellation, an uncertain write result, or abandoned runtime publication blocks further metadata-dependent mutations.
+
+The node checks for interrupted pool metadata transactions every five seconds. Failed recovery attempts back off to at most sixty seconds; each attempt has a thirty-second budget and stops on shutdown. Healthy nodes do not read metadata for this worker. Recovery:
+
+1. Cancels old decommission workers and waits for their supervisors to drain them. It does not cancel a separate rebalance operation; an attached rebalance worker prevents recovery until quiescent.
+2. Holds the local start/movement gates and distributed `pool.bin` write fence, validates the initialized deployment identity and unchanged pool topology, and selects the authoritative durable transaction.
+3. Repairs pending, missing, or lagging copies using conditional writes, then rereads and verifies convergence. A prepare-only first V3 migration commits the predecessor as V3, preserving the observed format floor.
+4. Invalidates old movement snapshots, installs the verified durable state, rechecks the fence, and only then clears the block. Speculative in-memory progress is never used as the recovery source. The existing decommission supervisor resumes eligible work afterward.
+
+An unreadable replica, lost fence, or conditional-write conflict leaves the original block in place. Recovery never initializes an all-missing metadata set. Corruption, incompatible layouts, conflicting identities/epochs/transactions, and topology changes require operator reconciliation; restore readability and consistency using the procedures below. Blocks originating in startup validation or storage-format heal are not cleared by the pool transaction worker: restart only after repairing the underlying condition. There is no force-clear switch. If an attached rebalance worker cannot quiesce, collect its status and restart the affected node after verifying the durable metadata; do not manually detach its worker token.
+
+### Diagnostics
+
+- The first block emits `decommission_state` with `state=pool_metadata_blocked`, `reason`, `phase`, and `blocked_since`. A change in recovery failure classification emits `state=pool_metadata_recovery_pending`; successful recovery emits `state=pool_metadata_recovered` with the original timestamp.
+- `rustfs_pool_metadata_blocks_total{reason}` and `rustfs_pool_metadata_recoveries_total` count block and recovery transitions. The original cause and phase remain attached to local typed errors; storage/RPC error numbers and on-disk formats are unchanged.
+- Node and cluster-write readiness include `pool_metadata_blocked`. Waiting for the metadata save mutex is bounded to 100 ms and reports `pool_metadata_check_timeout`, not a persistent block. Cluster probes retain their existing cache and overall timeout behavior. Liveness and cluster-read quorum checks are unchanged.
+
+If a block persists, inspect the first block and subsequent recovery phase, restore disk/peer readability, and verify every metadata and identity copy before restarting. Do not delete metadata to make readiness green.
+
 ## Disk replacement and metadata erasure
 
 1. Keep a quorum of nodes online and verify the cluster is ready.
@@ -39,4 +60,4 @@ Do not hand-edit a pending record or select a replica only because it is in pool
 3. Restore storage formats and the `pool.bin.identity` marker from the same deployment before rejoining it.
 4. Start the node and wait for it to load the verified committed generation and repair its replicas before touching another node.
 
-An initialized identity with every `pool.bin` missing is recovery required, as are existing storage formats with neither identity nor `pool.bin`. Format creation alone is not fresh-cluster proof: only the elected first topology node may create a durable `initialized=false` bootstrap identity with a fresh-bootstrap nonce, and only after every configured disk explicitly responds that it is unformatted. An unreachable peer, a non-elected distributed node, or an existing format is not sufficient proof. All-missing `pool.bin` replicas are accepted only by the same startup that proved the fresh topology and persisted that pending identity; when every `pool.bin` is missing, a later startup must recover even if the pending identity survived. This prevents a wiped or lagging node from rebuilding empty state and overwriting the cluster. Runtime reload, rebalance activation, and rebalance worker admission all fail closed and latch the same recovery gate until the node is restarted with readable metadata.
+An initialized identity with every `pool.bin` missing is recovery required, as are existing storage formats with neither identity nor `pool.bin`. Format creation alone is not fresh-cluster proof: only the elected first topology node may create a durable `initialized=false` bootstrap identity with a fresh-bootstrap nonce, and only after every configured disk explicitly responds that it is unformatted. An unreachable peer, a non-elected distributed node, or an existing format is not sufficient proof. All-missing `pool.bin` replicas are accepted only by the same startup that proved the fresh topology and persisted that pending identity; when every `pool.bin` is missing, a later startup must recover even if the pending identity survived. This prevents a wiped or lagging node from rebuilding empty state and overwriting the cluster. Runtime reload, rebalance activation, and rebalance worker admission fail closed on this missing-authority condition; a clean probe alone cannot clear it.

rustfs/src/error.rs

@@ -333,6 +333,13 @@ impl From<ApiError> for S3Error {
 
 impl From<StorageError> for ApiError {
     fn from(err: StorageError) -> Self {
+        if err.pool_metadata_failure().is_some() {
+            return ApiError {
+                code: S3ErrorCode::ServiceUnavailable,
+                message: ApiError::error_code_to_message(&S3ErrorCode::ServiceUnavailable),
+                source: Some(Box::new(err)),
+            };
+        }
         if let StorageError::Io(ref io_err) = err
             && let Some(inner) = io_err.get_ref()
         {
@@ -848,6 +855,35 @@ mod tests {
         assert_eq!(api_error.message, "The service is unavailable. Please retry.");
     }
 
+    #[test]
+    fn pool_metadata_failures_map_to_503_and_preserve_typed_private_context() {
+        use crate::storage_api::error::{PoolMetadataError, PoolMetadataFailure};
+        for kind in [
+            PoolMetadataFailure::ReadUnavailable,
+            PoolMetadataFailure::RecoveryRequired,
+            PoolMetadataFailure::TransactionUnknown,
+            PoolMetadataFailure::FenceLost,
+        ] {
+            let error = StorageError::other(PoolMetadataError {
+                kind,
+                operation: "pool metadata test".to_owned(),
+                phase: "prepare_cas",
+                since: time::OffsetDateTime::now_utc(),
+                source: Some(std::sync::Arc::new(StorageError::other("private disk failure"))),
+            });
+            let error = StorageError::Io(std::io::Error::new(std::io::ErrorKind::TimedOut, error));
+            let cloned = error.clone();
+            assert_eq!(cloned, error, "cloning must preserve the outer I/O kind and message");
+            assert_eq!(cloned.pool_metadata_failure().unwrap().kind, kind);
+            let api = ApiError::from(cloned);
+            assert_eq!(api.code, S3ErrorCode::ServiceUnavailable);
+            assert_eq!(api.message, "The service is unavailable. Please retry.");
+            assert!(!api.message.contains("private"));
+            let source = api.source.as_ref().unwrap().downcast_ref::<StorageError>().unwrap();
+            assert_eq!(source.pool_metadata_failure().unwrap().kind, kind);
+        }
+    }
+
     #[test]
     fn test_unknown_authoritative_quota_usage_maps_to_retryable_error() {
         let api_error = ApiError::from(QuotaError::UsageUnavailable {

rustfs/src/storage/storage_api.rs

@@ -479,6 +479,8 @@ pub(crate) mod ecstore_error {
     pub(crate) use rustfs_ecstore::api::error::{
         Error, Result, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
     };
+    #[cfg(test)]
+    pub(crate) use rustfs_ecstore::api::error::{PoolMetadataError, PoolMetadataFailure};
 }
 
 pub(crate) mod ecstore_event {

rustfs/src/storage_api.rs

@@ -81,6 +81,8 @@ pub(crate) mod error {
         }
     }
 
+    #[cfg(test)]
+    pub(crate) use crate::storage::storage_api::ecstore_error::{PoolMetadataError, PoolMetadataFailure};
     pub(crate) use crate::storage::storage_api::{QuotaError, StorageError};
 }
 

scripts/error-other-format-baseline.txt

@@ -25,7 +25,7 @@
 10|crates/ecstore/src/cluster/rpc/remote_disk.rs
 6|crates/ecstore/src/config/com.rs
 14|crates/ecstore/src/config/storageclass.rs
-182|crates/ecstore/src/core/pools.rs
+181|crates/ecstore/src/core/pools.rs
 7|crates/ecstore/src/data_movement/mod.rs
 2|crates/ecstore/src/data_usage/local_snapshot.rs
 12|crates/ecstore/src/data_usage/mod.rs
@@ -66,7 +66,7 @@
 3|crates/ecstore/src/set_disk/read.rs
 5|crates/ecstore/src/store/bucket.rs
 1|crates/ecstore/src/store/heal_walk.rs
-12|crates/ecstore/src/store/init.rs
+10|crates/ecstore/src/store/init.rs
 3|crates/ecstore/src/store/multipart.rs
 6|crates/ecstore/src/store/object.rs
 5|crates/ecstore/src/store/rebalance/support.rs