fix(ecstore): persist unresolved decommission entries (#6415)

* fix(ecstore): persist unresolved decommission entries

* fix(ecstore): type decommission completion result

* fix(ecstore): allow intentional decommission listing signatures under strict clippy

The sftp/swift feature-matrix clippy gates run with -D warnings and
flag the unresolved-entry resolver (large Err payload by design, 8
context parameters) and the decommission listing driver (9 args).
Document why and align with the existing decommission_entry precedent.

crates/config/src/constants/object.rs

@@ -168,6 +168,19 @@ pub const DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED: bool = false;
 const _: () = assert!(!DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_WRITE);
 const _: () = assert!(!DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED);
 
+/// Request writing pool metadata version 2.
+///
+/// This remains ineffective until [`ENV_POOL_META_V2_FLEET_CONFIRMED`] is also enabled.
+pub const ENV_POOL_META_V2_WRITE: &str = "RUSTFS_POOL_META_V2_WRITE";
+pub const DEFAULT_POOL_META_V2_WRITE: bool = false;
+
+/// Operator-attested confirmation that every pool metadata reader and writer understands version 2.
+pub const ENV_POOL_META_V2_FLEET_CONFIRMED: &str = "RUSTFS_POOL_META_V2_FLEET_CONFIRMED";
+pub const DEFAULT_POOL_META_V2_FLEET_CONFIRMED: bool = false;
+
+const _: () = assert!(!DEFAULT_POOL_META_V2_WRITE);
+const _: () = assert!(!DEFAULT_POOL_META_V2_FLEET_CONFIRMED);
+
 // =============================================================================
 // Concurrent Request Fix - Timeout and Backpressure Configuration
 // =============================================================================
@@ -736,4 +749,10 @@ mod remote_version_state_tests {
             "RUSTFS_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED"
         );
     }
+
+    #[test]
+    fn pool_meta_v2_gate_uses_stable_environment_names() {
+        assert_eq!(super::ENV_POOL_META_V2_WRITE, "RUSTFS_POOL_META_V2_WRITE");
+        assert_eq!(super::ENV_POOL_META_V2_FLEET_CONFIRMED, "RUSTFS_POOL_META_V2_FLEET_CONFIRMED");
+    }
 }

crates/ecstore/src/api/mod.rs

@@ -243,8 +243,8 @@ pub mod cache {
 
 pub mod capacity {
     pub use crate::core::pools::{
-        PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free, path2_bucket_object,
+        DecommissionUnresolvedEntry, PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
-        path2_bucket_object_with_base_path,
+        path2_bucket_object, path2_bucket_object_with_base_path,
     };
     pub use crate::store::utils::is_reserved_or_invalid_bucket;
 }

crates/ecstore/src/store/init.rs

@@ -637,7 +637,7 @@ mod tests {
     };
     use crate::{
         bucket::replication::{ReplicationState, ReplicationStatusType, replication_statuses_map},
-        core::pools::{POOL_META_FORMAT, POOL_META_VERSION, PoolDecommissionInfo, PoolMeta, PoolStatus},
+        core::pools::{POOL_META_VERSION, PoolDecommissionInfo, PoolMeta, PoolStatus},
         disk::endpoint::Endpoint,
         error::{Error, Result, StorageError},
         io_support::rio::{WritePlan, compression_metadata_value},
@@ -651,7 +651,6 @@ mod tests {
             range::HTTPRangeSpec,
         },
     };
-    use byteorder::{LittleEndian, WriteBytesExt};
     #[cfg(feature = "test-util")]
     use futures::{StreamExt as _, TryStreamExt as _};
     use http::HeaderMap;
@@ -681,13 +680,7 @@ mod tests {
     use tokio_util::sync::CancellationToken;
 
     fn startup_pool_meta_payload(meta: &PoolMeta) -> Vec<u8> {
-        let mut data = Vec::new();
+        meta.encode_config_data_for_test().expect("pool metadata should encode")
-        data.write_u16::<LittleEndian>(POOL_META_FORMAT)
-            .expect("pool metadata format should encode");
-        data.write_u16::<LittleEndian>(POOL_META_VERSION)
-            .expect("pool metadata version should encode");
-        data.extend(rmp_serde::to_vec(meta).expect("legacy pool metadata payload should encode"));
-        data
     }
 
     #[derive(Debug)]

docs/operations/rolling-restart.md

@@ -31,6 +31,15 @@ failure pattern reported in rustfs/rustfs#4304.
 > older build is not supported: older readers ignore the sidecar and can
 > report an object checksum in place of the requested part checksum.
 
+> [!WARNING]
+> Writing pool metadata version 2 remains inactive unless both
+> `RUSTFS_POOL_META_V2_WRITE=true` and
+> `RUSTFS_POOL_META_V2_FLEET_CONFIRMED=true`. Leave either setting disabled
+> until every node that can read or write `pool.bin` supports version 2. Once a node
+> observes or writes version 2 it will not downgrade the file, and older
+> binaries or rollback builds cannot read it. Unresolved decommission entries
+> fail closed instead of being written in the version 1 format.
+
 ## TL;DR
 
 - **Rolling restart (no downtime):** restart **one node at a time**, and wait

rustfs/src/app/admin_usecase.rs

@@ -16,7 +16,8 @@
 
 use super::storage_api::admin_usecase::admin::get_server_info;
 use super::storage_api::admin_usecase::capacity::{
-    PoolDecommissionInfo, PoolStatus, RebalStatus, get_total_usable_capacity, get_total_usable_capacity_free,
+    DecommissionUnresolvedEntry, PoolDecommissionInfo, PoolStatus, RebalStatus, get_total_usable_capacity,
+    get_total_usable_capacity_free,
 };
 use super::storage_api::admin_usecase::contract::StorageAdminApi;
 use super::storage_api::admin_usecase::contract::bucket::{BucketOperations as _, BucketOptions};
@@ -107,6 +108,8 @@ pub struct AdminPoolDecommissionInfo {
     pub bytes_failed: usize,
     #[serde(rename = "waitingReason")]
     pub waiting_reason: Option<String>,
+    #[serde(rename = "unresolvedEntries", skip_serializing_if = "Vec::is_empty")]
+    pub unresolved_entries: Vec<DecommissionUnresolvedEntry>,
 }
 
 #[derive(Debug, Clone, serde::Serialize)]
@@ -619,6 +622,7 @@ impl DefaultAdminUsecase {
             bytes_done: info.bytes_done,
             bytes_failed: info.bytes_failed,
             waiting_reason,
+            unresolved_entries: info.unresolved_entries,
         }
     }
 
@@ -676,7 +680,7 @@ impl DefaultAdminUsecase {
 
 #[cfg(test)]
 mod tests {
-    use super::super::storage_api::admin_usecase::capacity::{PoolDecommissionInfo, PoolStatus};
+    use super::super::storage_api::admin_usecase::capacity::{DecommissionUnresolvedEntry, PoolDecommissionInfo, PoolStatus};
     use super::*;
     use time::OffsetDateTime;
     use tracing_subscriber::{Layer, Registry, layer::Context, prelude::*};
@@ -987,6 +991,17 @@ mod tests {
                     items_decommission_failed: 1,
                     bytes_done: 1024,
                     bytes_failed: 64,
+                    unresolved_entries: vec![DecommissionUnresolvedEntry {
+                        bucket: "bucket-a".to_string(),
+                        object: "prefix/unresolved.txt".to_string(),
+                        pool_index: 3,
+                        set_index: 1,
+                        source_generation: OffsetDateTime::UNIX_EPOCH,
+                        candidate_count: 2,
+                        disk_error_count: 1,
+                        observed_at: OffsetDateTime::UNIX_EPOCH,
+                        reason: "metadata_resolution_failed".to_string(),
+                    }],
                     ..Default::default()
                 }),
             },
@@ -1010,6 +1025,13 @@ mod tests {
         assert_eq!(value["decommissionInfo"]["objectsDecommissionedFailed"], 1);
         assert_eq!(value["decommissionInfo"]["bytesDecommissioned"], 1024);
         assert_eq!(value["decommissionInfo"]["bytesDecommissionedFailed"], 64);
+        assert_eq!(value["decommissionInfo"]["unresolvedEntries"][0]["bucket"], "bucket-a");
+        assert_eq!(value["decommissionInfo"]["unresolvedEntries"][0]["object"], "prefix/unresolved.txt");
+        assert_eq!(
+            value["decommissionInfo"]["unresolvedEntries"][0]["sourceGeneration"],
+            "1970-01-01T00:00:00Z"
+        );
+        assert_eq!(value["decommissionInfo"]["unresolvedEntries"][0]["reason"], "metadata_resolution_failed");
         assert_eq!(value["decommissionInfo"]["waitingReason"], "queued");
     }
 

rustfs/src/app/storage_api.rs

@@ -31,6 +31,7 @@ pub(crate) mod admin {
 }
 
 pub(crate) mod capacity {
+    pub(crate) type DecommissionUnresolvedEntry = crate::storage::storage_api::ecstore_capacity::DecommissionUnresolvedEntry;
     pub(crate) type PoolDecommissionInfo = crate::storage::storage_api::ecstore_capacity::PoolDecommissionInfo;
     pub(crate) type PoolStatus = crate::storage::storage_api::ecstore_capacity::PoolStatus;
     pub(crate) type RebalStatus = crate::storage::storage_api::ecstore_rebalance::RebalStatus;

rustfs/src/storage/storage_api.rs

@@ -399,7 +399,7 @@ pub(crate) mod ecstore_bucket {
 
 pub(crate) mod ecstore_capacity {
     pub(crate) use rustfs_ecstore::api::capacity::{
-        PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
+        DecommissionUnresolvedEntry, PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
         is_reserved_or_invalid_bucket,
     };
 }