feat(rpc): dual-write a typed not-initialized code on control-plane responses (#6684)

2026-08-27 15:37:02 +00:00 · 2026-08-27 03:15:32 +08:00
parent f4cc919401
commit a42046b79c
9 changed files with 383 additions and 95 deletions
@@ -113,6 +113,21 @@ fn peer_failure_without_details(op: &str, bucket: Option<&str>) -> Error {
    }
 }

+/// Decode a control-plane response failure. Peers at or above the typed
+/// `ControlPlaneErrorCode` change (backlog#1845) carry a machine-readable
+/// discriminant beside the legacy `error_info` string; prefer it, then fall
+/// back to the string, then to the detail-free per-op failure.
+/// RUSTFS_COMPAT_TODO(not-initialized-error-code-v1): string fallback for peers that predate the typed wire code. Remove after the minimum supported RustFS peer version always sends error_code.
+fn control_plane_failure(op: &str, bucket: Option<&str>, error_code: Option<i32>, error_info: Option<String>) -> Error {
+    if error_code == Some(rustfs_protos::proto_gen::node_service::ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32) {
+        return Error::RemoteNotInitialized;
+    }
+    match error_info {
+        Some(msg) => Error::other(msg),
+        None => peer_failure_without_details(op, bucket),
+    }
+}
+
 fn decode_bucket_stats_response(response: GetBucketStatsDataResponse) -> Result<BucketStats> {
    if !response.success {
        return Err(Error::other(
@@ -845,10 +860,12 @@ impl PeerRestClient {

        let response = client.local_storage_info(request).await?.into_inner();
        if !response.success {
-            if let Some(msg) = response.error_info {
-                return Err(Error::other(msg));
-            }
-            return Err(peer_failure_without_details("local_storage_info", None));
+            return Err(control_plane_failure(
+                "local_storage_info",
+                None,
+                response.error_code,
+                response.error_info,
+            ));
        }
        let data = response.storage_info;

@@ -1235,11 +1252,10 @@ impl PeerRestClient {
                    Err(status) => return Err(status.into()),
                };
                if !response.success {
-                    return Err(Error::other(
-                        response
-                            .error_info
-                            .unwrap_or_else(|| "peer background heal status failed without an error".to_string()),
-                    ));
+                    return Err(match (response.error_code, response.error_info) {
+                        (None, None) => Error::other("peer background heal status failed without an error"),
+                        (error_code, error_info) => control_plane_failure("background_heal_status", None, error_code, error_info),
+                    });
                }
                Ok(Some(response.bg_heal_state.to_vec()))
            }
@@ -1267,11 +1283,12 @@ impl PeerRestClient {
                    Err(status) => return Err(status.into()),
                };
                if !response.success {
-                    return Err(Error::other(
-                        response
-                            .error_info
-                            .unwrap_or_else(|| "peer replacement recovery status failed without an error".to_string()),
-                    ));
+                    return Err(match (response.error_code, response.error_info) {
+                        (None, None) => Error::other("peer replacement recovery status failed without an error"),
+                        (error_code, error_info) => {
+                            control_plane_failure("replacement_recovery_status", None, error_code, error_info)
+                        }
+                    });
                }
                Ok(Some(response.recovery_status.to_vec()))
            }
@@ -1489,10 +1506,12 @@ impl PeerRestClient {

        let response = client.load_bucket_metadata(request).await?.into_inner();
        if !response.success {
-            if let Some(msg) = response.error_info {
-                return Err(Error::other(msg));
-            }
-            return Err(peer_failure_without_details("load_bucket_metadata", Some(bucket)));
+            return Err(control_plane_failure(
+                "load_bucket_metadata",
+                Some(bucket),
+                response.error_code,
+                response.error_info,
+            ));
        }
        Ok(())
    }
@@ -1531,10 +1550,7 @@ impl PeerRestClient {

                let response = client.delete_policy(request).await?.into_inner();
                if !response.success {
-                    if let Some(msg) = response.error_info {
-                        return Err(Error::other(msg));
-                    }
-                    return Err(peer_failure_without_details("delete_policy", None));
+                    return Err(control_plane_failure("delete_policy", None, response.error_code, response.error_info));
                }
                Ok(())
            }
@@ -1554,10 +1570,7 @@ impl PeerRestClient {

                let response = client.load_policy(request).await?.into_inner();
                if !response.success {
-                    if let Some(msg) = response.error_info {
-                        return Err(Error::other(msg));
-                    }
-                    return Err(peer_failure_without_details("load_policy", None));
+                    return Err(control_plane_failure("load_policy", None, response.error_code, response.error_info));
                }
                Ok(())
            }
@@ -1579,10 +1592,12 @@ impl PeerRestClient {

                let response = client.load_policy_mapping(request).await?.into_inner();
                if !response.success {
-                    if let Some(msg) = response.error_info {
-                        return Err(Error::other(msg));
-                    }
-                    return Err(peer_failure_without_details("load_policy_mapping", None));
+                    return Err(control_plane_failure(
+                        "load_policy_mapping",
+                        None,
+                        response.error_code,
+                        response.error_info,
+                    ));
                }
                Ok(())
            }
@@ -1602,10 +1617,7 @@ impl PeerRestClient {

                let response = client.delete_user(request).await?.into_inner();
                if !response.success {
-                    if let Some(msg) = response.error_info {
-                        return Err(Error::other(msg));
-                    }
-                    return Err(peer_failure_without_details("delete_user", None));
+                    return Err(control_plane_failure("delete_user", None, response.error_code, response.error_info));
                }
                Ok(())
            }
@@ -1625,10 +1637,12 @@ impl PeerRestClient {

                let response = client.delete_service_account(request).await?.into_inner();
                if !response.success {
-                    if let Some(msg) = response.error_info {
-                        return Err(Error::other(msg));
-                    }
-                    return Err(peer_failure_without_details("delete_service_account", None));
+                    return Err(control_plane_failure(
+                        "delete_service_account",
+                        None,
+                        response.error_code,
+                        response.error_info,
+                    ));
                }
                Ok(())
            }
@@ -1649,10 +1663,7 @@ impl PeerRestClient {

                let response = client.load_user(request).await?.into_inner();
                if !response.success {
-                    if let Some(msg) = response.error_info {
-                        return Err(Error::other(msg));
-                    }
-                    return Err(peer_failure_without_details("load_user", None));
+                    return Err(control_plane_failure("load_user", None, response.error_code, response.error_info));
                }
                Ok(())
            }
@@ -1672,10 +1683,12 @@ impl PeerRestClient {

                let response = client.load_service_account(request).await?.into_inner();
                if !response.success {
-                    if let Some(msg) = response.error_info {
-                        return Err(Error::other(msg));
-                    }
-                    return Err(peer_failure_without_details("load_service_account", None));
+                    return Err(control_plane_failure(
+                        "load_service_account",
+                        None,
+                        response.error_code,
+                        response.error_info,
+                    ));
                }
                Ok(())
            }
@@ -1695,10 +1708,7 @@ impl PeerRestClient {

                let response = client.load_group(request).await?.into_inner();
                if !response.success {
-                    if let Some(msg) = response.error_info {
-                        return Err(Error::other(msg));
-                    }
-                    return Err(peer_failure_without_details("load_group", None));
+                    return Err(control_plane_failure("load_group", None, response.error_code, response.error_info));
                }
                Ok(())
            }
@@ -1716,10 +1726,12 @@ impl PeerRestClient {

                let response = client.reload_site_replication_config(request).await?.into_inner();
                if !response.success {
-                    if let Some(msg) = response.error_info {
-                        return Err(Error::other(msg));
-                    }
-                    return Err(peer_failure_without_details("reload_site_replication_config", None));
+                    return Err(control_plane_failure(
+                        "reload_site_replication_config",
+                        None,
+                        response.error_code,
+                        response.error_info,
+                    ));
                }
                Ok(())
            }
@@ -1987,10 +1999,7 @@ impl PeerRestClient {

                let response = client.reload_pool_meta(request).await?.into_inner();
                if !response.success {
-                    if let Some(msg) = response.error_info {
-                        return Err(Error::other(msg));
-                    }
-                    return Err(peer_failure_without_details("reload_pool_meta", None));
+                    return Err(control_plane_failure("reload_pool_meta", None, response.error_code, response.error_info));
                }

                Ok(())
@@ -2011,10 +2020,7 @@ impl PeerRestClient {

                let response = client.stop_rebalance(request).await?.into_inner();
                if !response.success {
-                    if let Some(msg) = response.error_info {
-                        return Err(Error::other(msg));
-                    }
-                    return Err(peer_failure_without_details("stop_rebalance", None));
+                    return Err(control_plane_failure("stop_rebalance", None, response.error_code, response.error_info));
                }

                Ok(())
@@ -2045,10 +2051,12 @@ impl PeerRestClient {
                    "peer rebalance metadata response"
                );
                if !response.success {
-                    if let Some(msg) = response.error_info {
-                        return Err(Error::other(msg));
-                    }
-                    return Err(peer_failure_without_details("load_rebalance_meta", None));
+                    return Err(control_plane_failure(
+                        "load_rebalance_meta",
+                        None,
+                        response.error_code,
+                        response.error_info,
+                    ));
                }

                Ok(())
@@ -2073,10 +2081,12 @@ impl PeerRestClient {

                let response = client.start_decommission(request).await?.into_inner();
                if !response.success {
-                    if let Some(msg) = response.error_info {
-                        return Err(Error::other(msg));
-                    }
-                    return Err(peer_failure_without_details("start_decommission", None));
+                    return Err(control_plane_failure(
+                        "start_decommission",
+                        None,
+                        response.error_code,
+                        response.error_info,
+                    ));
                }

                Ok(())
@@ -2097,10 +2107,12 @@ impl PeerRestClient {

                let response = client.cancel_decommission(request).await?.into_inner();
                if !response.success {
-                    if let Some(msg) = response.error_info {
-                        return Err(Error::other(msg));
-                    }
-                    return Err(peer_failure_without_details("decommission_cancel", None));
+                    return Err(control_plane_failure(
+                        "decommission_cancel",
+                        None,
+                        response.error_code,
+                        response.error_info,
+                    ));
                }

                Ok(())
@@ -2121,10 +2133,12 @@ impl PeerRestClient {

                let response = client.clear_decommission(request).await?.into_inner();
                if !response.success {
-                    if let Some(msg) = response.error_info {
-                        return Err(Error::other(msg));
-                    }
-                    return Err(peer_failure_without_details("clear_decommission", None));
+                    return Err(control_plane_failure(
+                        "clear_decommission",
+                        None,
+                        response.error_code,
+                        response.error_info,
+                    ));
                }

                Ok(())
@@ -2180,7 +2194,7 @@ impl PeerRestClient {
            Err(status) => return tier_config_reload_status_outcome(status),
        };
        if !response.success {
-            return tier_config_reload_remote_failure(response.error_info);
+            return tier_config_reload_remote_failure(response.error_code, response.error_info);
        }

        TierConfigReloadOutcome::Success
@@ -2239,7 +2253,13 @@ fn is_tier_config_reload_connection_failure(err: &Error) -> bool {
 /// reload every `TIER_CONFIG_RELOAD_RETRY_CAP`, and `Terminal` stays reachable
 /// for transport and gRPC status failures, which is where a genuinely
 /// unrecoverable peer surfaces.
-fn tier_config_reload_remote_failure(error_info: Option<String>) -> TierConfigReloadOutcome {
+fn tier_config_reload_remote_failure(error_code: Option<i32>, error_info: Option<String>) -> TierConfigReloadOutcome {
+    // Remote rejections are transient by design (see the doc comment above);
+    // the typed not-initialized code keeps the error typed for downstream
+    // classifiers instead of a bare string (backlog#1845).
+    if error_code == Some(rustfs_protos::proto_gen::node_service::ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32) {
+        return TierConfigReloadOutcome::TransientRetrySameChannel(Error::RemoteNotInitialized);
+    }
    TierConfigReloadOutcome::TransientRetrySameChannel(Error::other(error_info.unwrap_or_default()))
 }

@@ -2278,6 +2298,63 @@ mod tests {
    use temp_env::async_with_vars;
    use tracing_subscriber::{Registry, fmt::MakeWriter, layer::SubscriberExt};

+    #[test]
+    fn control_plane_failure_prefers_typed_not_initialized_code() {
+        use rustfs_protos::proto_gen::node_service::ControlPlaneErrorCode;
+        let code = Some(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32);
+
+        // Typed code wins even when the legacy string is present (dual-write).
+        let err = control_plane_failure("load_bucket_metadata", Some("b"), code, Some("errServerNotInitialized".to_string()));
+        assert!(matches!(err, Error::RemoteNotInitialized));
+        assert!(crate::error::is_err_not_initialized(&err), "typed variant must satisfy the predicate");
+
+        // Legacy peers: no code, string only — the substring fallback still classifies.
+        let err = control_plane_failure("load_bucket_metadata", Some("b"), None, Some("errServerNotInitialized".to_string()));
+        assert!(crate::error::is_err_not_initialized(&err), "legacy string form must keep classifying");
+
+        // No code, no string: detail-free per-op failure, not misread as not-initialized.
+        let err = control_plane_failure("load_bucket_metadata", Some("b"), None, None);
+        assert!(!crate::error::is_err_not_initialized(&err));
+        assert!(err.to_string().contains("load_bucket_metadata"));
+
+        // Unspecified code behaves like no code.
+        let err = control_plane_failure(
+            "load_bucket_metadata",
+            None,
+            Some(ControlPlaneErrorCode::ControlPlaneErrorUnspecified as i32),
+            Some("boom".to_string()),
+        );
+        assert!(!matches!(err, Error::RemoteNotInitialized));
+        assert_eq!(err.to_string(), "Io error: boom");
+    }
+
+    #[test]
+    fn control_plane_not_initialized_wire_value_is_pinned() {
+        // The discriminant is wire contract: old peers ignore it, but a renumber
+        // would silently flip classification on mixed-version clusters.
+        use rustfs_protos::proto_gen::node_service::ControlPlaneErrorCode;
+        assert_eq!(ControlPlaneErrorCode::ControlPlaneErrorUnspecified as i32, 0);
+        assert_eq!(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32, 1);
+    }
+
+    #[test]
+    fn tier_config_reload_remote_failure_keeps_typed_not_initialized() {
+        use rustfs_protos::proto_gen::node_service::ControlPlaneErrorCode;
+        let outcome = tier_config_reload_remote_failure(
+            Some(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32),
+            Some("errServerNotInitialized".to_string()),
+        );
+        match outcome {
+            TierConfigReloadOutcome::TransientRetrySameChannel(err) => {
+                assert!(matches!(err, Error::RemoteNotInitialized));
+            }
+            TierConfigReloadOutcome::TransientReconnect(err) | TierConfigReloadOutcome::Terminal(err) => {
+                panic!("not-initialized must stay retry-same-channel, got {err}")
+            }
+            TierConfigReloadOutcome::Success => panic!("a rejection cannot classify as success"),
+        }
+    }
+
    #[test]
    fn scanner_publication_lease_response_rejects_stale_generation_and_session() {
        let token = Uuid::new_v4();
@@ -3031,11 +3108,11 @@ mod tests {
        // retired: the channel is healthy, so the rejection reflects remote state
        // that the next attempt can find healed.
        assert!(matches!(
-            tier_config_reload_remote_failure(Some("backend unavailable".to_string())),
+            tier_config_reload_remote_failure(None, Some("backend unavailable".to_string())),
            TierConfigReloadOutcome::TransientRetrySameChannel(_)
        ));
        assert!(matches!(
-            tier_config_reload_remote_failure(Some("errServerNotInitialized".to_string())),
+            tier_config_reload_remote_failure(None, Some("errServerNotInitialized".to_string())),
            TierConfigReloadOutcome::TransientRetrySameChannel(_)
        ));
        assert!(matches!(
@@ -3080,7 +3157,7 @@ mod tests {
        ] {
            assert!(
                matches!(
-                    tier_config_reload_remote_failure(Some(error_info.to_string())),
+                    tier_config_reload_remote_failure(None, Some(error_info.to_string())),
                    TierConfigReloadOutcome::TransientRetrySameChannel(_)
                ),
                "a peer that rejected the apply must stay retryable so it converges: {error_info}"
@@ -3089,7 +3166,7 @@ mod tests {

        // An absent error message is still a rejection, not a reason to stop.
        assert!(matches!(
-            tier_config_reload_remote_failure(None),
+            tier_config_reload_remote_failure(None, None),
            TierConfigReloadOutcome::TransientRetrySameChannel(_)
        ));

@@ -228,6 +228,14 @@ pub enum StorageError {
    /// during quorum aggregation (backlog#1845).
    #[error("remote rpc client unavailable: {0}")]
    RemoteClientUnavailable(String),
+
+    /// A peer answered a control-plane RPC but its storage/IAM layer is not
+    /// initialized yet. Typed form of the legacy "errServerNotInitialized"
+    /// error_info string (backlog#1845); the wire carries it as
+    /// `ControlPlaneErrorCode::ControlPlaneErrorNotInitialized` alongside the
+    /// legacy string for rolling-upgrade compatibility.
+    #[error("remote peer not initialized")]
+    RemoteNotInitialized,
 }

 impl From<crate::erasure::coding::ErasureConstructionError> for StorageError {
@@ -581,6 +589,7 @@ impl Clone for StorageError {
                limit: *limit,
            },
            StorageError::RemoteClientUnavailable(detail) => StorageError::RemoteClientUnavailable(detail.clone()),
+            StorageError::RemoteNotInitialized => StorageError::RemoteNotInitialized,
        }
    }
 }
@@ -670,6 +679,7 @@ impl StorageError {
            StorageError::NamespaceLockQuorumUnavailable { .. } => StorageErrorCode::NamespaceLockQuorumUnavailable,
            StorageError::QuotaExceeded { .. } => StorageErrorCode::QuotaExceeded,
            StorageError::RemoteClientUnavailable(_) => StorageErrorCode::RemoteClientUnavailable,
+            StorageError::RemoteNotInitialized => StorageErrorCode::RemoteNotInitialized,
        }
    }

@@ -800,6 +810,7 @@ impl StorageError {
                limit: Default::default(),
            }),
            StorageErrorCode::RemoteClientUnavailable => Some(StorageError::RemoteClientUnavailable(Default::default())),
+            StorageErrorCode::RemoteNotInitialized => Some(StorageError::RemoteNotInitialized),
        }
    }
 }
@@ -943,7 +954,13 @@ pub fn is_err_operation_canceled(err: &Error) -> bool {

 #[allow(dead_code, reason = "predicate asserted by this file's tests (backlog#1823)")]
 pub fn is_err_not_initialized(err: &Error) -> bool {
-    err.to_string().contains("errServerNotInitialized") || err.to_string().contains("ServerNotInitialized")
+    // Typed-first: peers at or above the ControlPlaneErrorCode change decode to
+    // the typed variant. The substring form only matches legacy peers' string
+    // responses.
+    // RUSTFS_COMPAT_TODO(not-initialized-error-code-v1): substring fallback for peers that predate the typed wire code. Remove after the minimum supported RustFS peer version always sends error_code.
+    matches!(err, StorageError::RemoteNotInitialized)
+        || err.to_string().contains("errServerNotInitialized")
+        || err.to_string().contains("ServerNotInitialized")
 }

 /// Strict "not found" predicate that only matches genuine object/version/volume
@@ -142,7 +142,8 @@ pub struct DeleteRequest {
    pub path: ::prost::alloc::string::String,
    #[prost(string, tag = "4")]
    pub options: ::prost::alloc::string::String,
-    /// Optional scanner publication lease token.
+    /// Optional scanner publication lease token. When present, the target binds
+    /// the complete delete operation to its movement read admission.
    #[prost(bytes = "bytes", tag = "5")]
    pub scanner_publication_lease_token: ::prost::bytes::Bytes,
 }
@@ -396,6 +397,9 @@ pub struct RenameDataRequest {
    pub dst_path: ::prost::alloc::string::String,
    #[prost(bytes = "bytes", tag = "7")]
    pub file_info_bin: ::prost::bytes::Bytes,
+    /// Optional target-side scanner publication lease.  Empty preserves the
+    /// legacy rename request body; a non-empty token is checked at the target's
+    /// rename linearization point.
    #[prost(bytes = "bytes", tag = "8")]
    pub scanner_publication_lease_token: ::prost::bytes::Bytes,
 }
@@ -719,7 +723,7 @@ pub struct DeleteVersionsRequest {
    #[prost(bytes = "bytes", tag = "6")]
    pub opts_bin: ::prost::bytes::Bytes,
 }
-#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+#[derive(Clone, PartialEq, ::prost::Message)]
 pub struct DeleteVersionsResponse {
    #[prost(bool, tag = "1")]
    pub success: bool,
@@ -841,6 +845,8 @@ pub struct LocalStorageInfoResponse {
    pub storage_info: ::prost::bytes::Bytes,
    #[prost(string, optional, tag = "3")]
    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "4")]
+    pub error_code: ::core::option::Option<i32>,
 }
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ServerInfoRequest {
@@ -1043,6 +1049,8 @@ pub struct LoadBucketMetadataResponse {
    pub success: bool,
    #[prost(string, optional, tag = "2")]
    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")]
+    pub error_code: ::core::option::Option<i32>,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct DeleteBucketMetadataRequest {
@@ -1067,6 +1075,8 @@ pub struct DeletePolicyResponse {
    pub success: bool,
    #[prost(string, optional, tag = "2")]
    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")]
+    pub error_code: ::core::option::Option<i32>,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct LoadPolicyRequest {
@@ -1079,6 +1089,8 @@ pub struct LoadPolicyResponse {
    pub success: bool,
    #[prost(string, optional, tag = "2")]
    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")]
+    pub error_code: ::core::option::Option<i32>,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct LoadPolicyMappingRequest {
@@ -1095,6 +1107,8 @@ pub struct LoadPolicyMappingResponse {
    pub success: bool,
    #[prost(string, optional, tag = "2")]
    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")]
+    pub error_code: ::core::option::Option<i32>,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct DeleteUserRequest {
@@ -1107,6 +1121,8 @@ pub struct DeleteUserResponse {
    pub success: bool,
    #[prost(string, optional, tag = "2")]
    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")]
+    pub error_code: ::core::option::Option<i32>,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct DeleteServiceAccountRequest {
@@ -1119,6 +1135,8 @@ pub struct DeleteServiceAccountResponse {
    pub success: bool,
    #[prost(string, optional, tag = "2")]
    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")]
+    pub error_code: ::core::option::Option<i32>,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct LoadUserRequest {
@@ -1133,6 +1151,8 @@ pub struct LoadUserResponse {
    pub success: bool,
    #[prost(string, optional, tag = "2")]
    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")]
+    pub error_code: ::core::option::Option<i32>,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct LoadServiceAccountRequest {
@@ -1145,6 +1165,8 @@ pub struct LoadServiceAccountResponse {
    pub success: bool,
    #[prost(string, optional, tag = "2")]
    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")]
+    pub error_code: ::core::option::Option<i32>,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct LoadGroupRequest {
@@ -1157,6 +1179,8 @@ pub struct LoadGroupResponse {
    pub success: bool,
    #[prost(string, optional, tag = "2")]
    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")]
+    pub error_code: ::core::option::Option<i32>,
 }
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ReloadSiteReplicationConfigRequest {}
@@ -1166,6 +1190,8 @@ pub struct ReloadSiteReplicationConfigResponse {
    pub success: bool,
    #[prost(string, optional, tag = "2")]
    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")]
+    pub error_code: ::core::option::Option<i32>,
 }
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct SignalServiceRequest {
@@ -1218,11 +1244,17 @@ pub struct ScannerActivityResponse {
    pub dirty_usage_generation: u64,
    #[prost(bool, tag = "9")]
    pub dirty_usage_pending: bool,
+    /// v7 fields.  They are optional so v6 peers can continue to decode the
+    /// response shape while newer readers fail closed when they are absent.
    #[prost(uint64, optional, tag = "10")]
    pub movement_generation: ::core::option::Option<u64>,
    #[prost(bool, optional, tag = "11")]
    pub publication_blocked: ::core::option::Option<bool>,
 }
+/// A short-lived storage-owned read admission used only around a final
+/// scanner metadata publication.  It is intentionally separate from the
+/// ScannerActivity observation wire so v6/v7 rolling compatibility remains
+/// unchanged.
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ScannerPublicationLeaseRequest {
    #[prost(bytes = "bytes", tag = "1")]
@@ -1231,8 +1263,14 @@ pub struct ScannerPublicationLeaseRequest {
    pub expected_movement_generation: u64,
    #[prost(uint64, tag = "3")]
    pub ttl_ms: u64,
+    /// The activity instance is a process session nonce.  It is intentionally
+    /// separate from the storage-owned deployment identity returned by the
+    /// lease response so a restart cannot reuse an old session token.
    #[prost(string, tag = "4")]
    pub expected_session_id: ::prost::alloc::string::String,
+    /// A non-empty token turns the acquire RPC into an in-place validation of an
+    /// existing lease.  Keeping this on the existing RPC lets old peers reject
+    /// the proof without changing the v7 activity wire shape.
    #[prost(bytes = "bytes", tag = "5")]
    pub token: ::prost::bytes::Bytes,
 }
@@ -1288,6 +1326,8 @@ pub struct BackgroundHealStatusResponse {
    pub bg_heal_state: ::prost::bytes::Bytes,
    #[prost(string, optional, tag = "3")]
    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "4")]
+    pub error_code: ::core::option::Option<i32>,
 }
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ReplacementRecoveryStatusRequest {}
@@ -1299,6 +1339,8 @@ pub struct ReplacementRecoveryStatusResponse {
    pub recovery_status: ::prost::bytes::Bytes,
    #[prost(string, optional, tag = "3")]
    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "4")]
+    pub error_code: ::core::option::Option<i32>,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct HealControlRequest {
@@ -1356,6 +1398,8 @@ pub struct ReloadPoolMetaResponse {
    pub success: bool,
    #[prost(string, optional, tag = "2")]
    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")]
+    pub error_code: ::core::option::Option<i32>,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct StopRebalanceRequest {
@@ -1368,6 +1412,8 @@ pub struct StopRebalanceResponse {
    pub success: bool,
    #[prost(string, optional, tag = "2")]
    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")]
+    pub error_code: ::core::option::Option<i32>,
 }
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct LoadRebalanceMetaRequest {
@@ -1380,6 +1426,8 @@ pub struct LoadRebalanceMetaResponse {
    pub success: bool,
    #[prost(string, optional, tag = "2")]
    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")]
+    pub error_code: ::core::option::Option<i32>,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct StartDecommissionRequest {
@@ -1392,6 +1440,8 @@ pub struct StartDecommissionResponse {
    pub success: bool,
    #[prost(string, optional, tag = "2")]
    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")]
+    pub error_code: ::core::option::Option<i32>,
 }
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct CancelDecommissionRequest {
@@ -1404,6 +1454,8 @@ pub struct CancelDecommissionResponse {
    pub success: bool,
    #[prost(string, optional, tag = "2")]
    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")]
+    pub error_code: ::core::option::Option<i32>,
 }
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ClearDecommissionRequest {
@@ -1416,6 +1468,8 @@ pub struct ClearDecommissionResponse {
    pub success: bool,
    #[prost(string, optional, tag = "2")]
    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")]
+    pub error_code: ::core::option::Option<i32>,
 }
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct LoadTransitionTierConfigRequest {}
@@ -1425,6 +1479,8 @@ pub struct LoadTransitionTierConfigResponse {
    pub success: bool,
    #[prost(string, optional, tag = "2")]
    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")]
+    pub error_code: ::core::option::Option<i32>,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct TierMutationPrepareRequest {
@@ -1486,6 +1542,38 @@ pub struct GetLiveEventsResponse {
    #[prost(string, optional, tag = "5")]
    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
 }
+/// Typed control-plane error discriminants carried alongside the legacy
+/// error_info string on control-plane responses. Rolling-upgrade compat:
+/// old peers ignore the field and keep reading error_info.
+/// RUSTFS_COMPAT_TODO(not-initialized-error-code-v1): legacy string dual-write. Remove after the minimum supported RustFS peer version always sends error_code.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
+#[repr(i32)]
+pub enum ControlPlaneErrorCode {
+    ControlPlaneErrorUnspecified = 0,
+    /// The peer answered but its storage/IAM layer is not initialized yet
+    /// (legacy string form: "errServerNotInitialized").
+    ControlPlaneErrorNotInitialized = 1,
+}
+impl ControlPlaneErrorCode {
+    /// String value of the enum field names used in the ProtoBuf definition.
+    ///
+    /// The values are not transformed in any way and thus are considered stable
+    /// (if the ProtoBuf definition does not change) and safe for programmatic use.
+    pub fn as_str_name(&self) -> &'static str {
+        match self {
+            Self::ControlPlaneErrorUnspecified => "CONTROL_PLANE_ERROR_UNSPECIFIED",
+            Self::ControlPlaneErrorNotInitialized => "CONTROL_PLANE_ERROR_NOT_INITIALIZED",
+        }
+    }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "CONTROL_PLANE_ERROR_UNSPECIFIED" => Some(Self::ControlPlaneErrorUnspecified),
+            "CONTROL_PLANE_ERROR_NOT_INITIALIZED" => Some(Self::ControlPlaneErrorNotInitialized),
+            _ => None,
+        }
+    }
+}
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
 #[repr(i32)]
 pub enum TierMutationPeerState {
@@ -3301,16 +3389,12 @@ pub mod node_service_server {
        ) -> std::result::Result<tonic::Response<super::ScannerActivityResponse>, tonic::Status>;
        async fn acquire_scanner_publication_lease(
            &self,
-            _request: tonic::Request<super::ScannerPublicationLeaseRequest>,
-        ) -> std::result::Result<tonic::Response<super::ScannerPublicationLeaseResponse>, tonic::Status> {
-            Err(tonic::Status::unimplemented("scanner publication leases are unsupported"))
-        }
+            request: tonic::Request<super::ScannerPublicationLeaseRequest>,
+        ) -> std::result::Result<tonic::Response<super::ScannerPublicationLeaseResponse>, tonic::Status>;
        async fn release_scanner_publication_lease(
            &self,
-            _request: tonic::Request<super::ScannerPublicationLeaseReleaseRequest>,
-        ) -> std::result::Result<tonic::Response<super::ScannerPublicationLeaseReleaseResponse>, tonic::Status> {
-            Err(tonic::Status::unimplemented("scanner publication leases are unsupported"))
-        }
+            request: tonic::Request<super::ScannerPublicationLeaseReleaseRequest>,
+        ) -> std::result::Result<tonic::Response<super::ScannerPublicationLeaseReleaseResponse>, tonic::Status>;
        async fn background_heal_status(
            &self,
            request: tonic::Request<super::BackgroundHealStatusRequest>,
@@ -21,6 +21,17 @@ message Error {
  string error_info = 2;
 }

+// Typed control-plane error discriminants carried alongside the legacy
+// error_info string on control-plane responses. Rolling-upgrade compat:
+// old peers ignore the field and keep reading error_info.
+// RUSTFS_COMPAT_TODO(not-initialized-error-code-v1): legacy string dual-write. Remove after the minimum supported RustFS peer version always sends error_code.
+enum ControlPlaneErrorCode {
+  CONTROL_PLANE_ERROR_UNSPECIFIED = 0;
+  // The peer answered but its storage/IAM layer is not initialized yet
+  // (legacy string form: "errServerNotInitialized").
+  CONTROL_PLANE_ERROR_NOT_INITIALIZED = 1;
+}
+
 message PingRequest {
  uint64 version = 1;
  bytes body = 2;
@@ -580,6 +591,7 @@ message LocalStorageInfoResponse {
  bool success = 1;
  bytes storage_info = 2;
  optional string error_info = 3;
+  optional ControlPlaneErrorCode error_code = 4;
 }

 message ServerInfoRequest {
@@ -726,6 +738,7 @@ message LoadBucketMetadataRequest {
 message LoadBucketMetadataResponse {
  bool success = 1;
  optional string error_info = 2;
+  optional ControlPlaneErrorCode error_code = 3;
 }

 message DeleteBucketMetadataRequest {
@@ -744,6 +757,7 @@ message DeletePolicyRequest {
 message DeletePolicyResponse {
  bool success = 1;
  optional string error_info = 2;
+  optional ControlPlaneErrorCode error_code = 3;
 }

 message LoadPolicyRequest {
@@ -753,6 +767,7 @@ message LoadPolicyRequest {
 message LoadPolicyResponse {
  bool success = 1;
  optional string error_info = 2;
+  optional ControlPlaneErrorCode error_code = 3;
 }

 message LoadPolicyMappingRequest {
@@ -764,6 +779,7 @@ message LoadPolicyMappingRequest {
 message LoadPolicyMappingResponse {
  bool success = 1;
  optional string error_info = 2;
+  optional ControlPlaneErrorCode error_code = 3;
 }

 message DeleteUserRequest {
@@ -773,6 +789,7 @@ message DeleteUserRequest {
 message DeleteUserResponse {
  bool success = 1;
  optional string error_info = 2;
+  optional ControlPlaneErrorCode error_code = 3;
 }

 message DeleteServiceAccountRequest {
@@ -782,6 +799,7 @@ message DeleteServiceAccountRequest {
 message DeleteServiceAccountResponse {
  bool success = 1;
  optional string error_info = 2;
+  optional ControlPlaneErrorCode error_code = 3;
 }

 message LoadUserRequest {
@@ -792,6 +810,7 @@ message LoadUserRequest {
 message LoadUserResponse {
  bool success = 1;
  optional string error_info = 2;
+  optional ControlPlaneErrorCode error_code = 3;
 }

 message LoadServiceAccountRequest {
@@ -801,6 +820,7 @@ message LoadServiceAccountRequest {
 message LoadServiceAccountResponse {
  bool success = 1;
  optional string error_info = 2;
+  optional ControlPlaneErrorCode error_code = 3;
 }

 message LoadGroupRequest {
@@ -810,6 +830,7 @@ message LoadGroupRequest {
 message LoadGroupResponse {
  bool success = 1;
  optional string error_info = 2;
+  optional ControlPlaneErrorCode error_code = 3;
 }

 message ReloadSiteReplicationConfigRequest {}
@@ -817,6 +838,7 @@ message ReloadSiteReplicationConfigRequest {}
 message ReloadSiteReplicationConfigResponse {
  bool success = 1;
  optional string error_info = 2;
+  optional ControlPlaneErrorCode error_code = 3;
 }

 message SignalServiceRequest {
@@ -907,6 +929,7 @@ message BackgroundHealStatusResponse {
  bool success = 1;
  bytes bg_heal_state = 2;
  optional string error_info = 3;
+  optional ControlPlaneErrorCode error_code = 4;
 }

 message ReplacementRecoveryStatusRequest {}
@@ -915,6 +938,7 @@ message ReplacementRecoveryStatusResponse {
  bool success = 1;
  bytes recovery_status = 2;
  optional string error_info = 3;
+  optional ControlPlaneErrorCode error_code = 4;
 }

 message HealControlRequest {
@@ -955,6 +979,7 @@ message ReloadPoolMetaRequest {}
 message ReloadPoolMetaResponse {
  bool success = 1;
  optional string error_info = 2;
+  optional ControlPlaneErrorCode error_code = 3;
 }

 message StopRebalanceRequest {
@@ -964,6 +989,7 @@ message StopRebalanceRequest {
 message StopRebalanceResponse {
  bool success = 1;
  optional string error_info = 2;
+  optional ControlPlaneErrorCode error_code = 3;
 }

 message LoadRebalanceMetaRequest {
@@ -973,6 +999,7 @@ message LoadRebalanceMetaRequest {
 message LoadRebalanceMetaResponse {
  bool success = 1;
  optional string error_info = 2;
+  optional ControlPlaneErrorCode error_code = 3;
 }

 message StartDecommissionRequest {
@@ -982,6 +1009,7 @@ message StartDecommissionRequest {
 message StartDecommissionResponse {
  bool success = 1;
  optional string error_info = 2;
+  optional ControlPlaneErrorCode error_code = 3;
 }

 message CancelDecommissionRequest {
@@ -991,6 +1019,7 @@ message CancelDecommissionRequest {
 message CancelDecommissionResponse {
  bool success = 1;
  optional string error_info = 2;
+  optional ControlPlaneErrorCode error_code = 3;
 }

 message ClearDecommissionRequest {
@@ -1000,6 +1029,7 @@ message ClearDecommissionRequest {
 message ClearDecommissionResponse {
  bool success = 1;
  optional string error_info = 2;
+  optional ControlPlaneErrorCode error_code = 3;
 }

 message LoadTransitionTierConfigRequest {}
@@ -1007,6 +1037,7 @@ message LoadTransitionTierConfigRequest {}
 message LoadTransitionTierConfigResponse {
  bool success = 1;
  optional string error_info = 2;
+  optional ControlPlaneErrorCode error_code = 3;
 }

 message TierMutationPrepareRequest {
@@ -105,6 +105,7 @@ pub enum StorageErrorCode {
    InvalidPath,
    QuotaExceeded,
    RemoteClientUnavailable,
+    RemoteNotInitialized,
 }

 impl StorageErrorCode {
@@ -192,6 +193,7 @@ impl StorageErrorCode {
            Self::InvalidPath => 0x52,
            Self::QuotaExceeded => 0x53,
            Self::RemoteClientUnavailable => 0x54,
+            Self::RemoteNotInitialized => 0x55,
        }
    }

@@ -279,6 +281,7 @@ impl StorageErrorCode {
            0x52 => Some(Self::InvalidPath),
            0x53 => Some(Self::QuotaExceeded),
            0x54 => Some(Self::RemoteClientUnavailable),
+            0x55 => Some(Self::RemoteNotInitialized),
            _ => None,
        }
    }
@@ -355,6 +358,7 @@ mod tests {
        (StorageErrorCode::NamespaceLockQuorumUnavailable, 0x42),
        (StorageErrorCode::QuotaExceeded, 0x53),
        (StorageErrorCode::RemoteClientUnavailable, 0x54),
+        (StorageErrorCode::RemoteNotInitialized, 0x55),
    ];

    const DISK_PRESERVATION_ERROR_CODES: &[(StorageErrorCode, u32)] = &[
@@ -34,6 +34,7 @@ for later deletion.
 - `tonic-013-status-render` peer RPC failure classification: internode failures that reach a node only as text (a peer's error_info payload, a status flattened through format!) are classified by matching the rendering of an Unavailable gRPC status. Releases up to 1.0.0-alpha.38 shipped tonic 0.13, which rendered that status as "status: Unavailable, message: ..."; tonic 0.14 renders it as "code: 'The service is currently unavailable', message: ...". Both forms are matched so an older peer's relayed text still marks an unreachable peer offline. Remove the tonic 0.13 form after the minimum supported RustFS peer version ships tonic 0.14 or later.
 - `rustfs-5063` pre-beta.9 Local KMS recovery: persisted Local KMS configs from beta.8 and earlier predate the explicit insecure-development flag, and encrypted key files use the legacy SHA-256 KDF. Remove the config fallback after supported upgrades have rewritten or explicitly resaved all pre-beta.9 configs with the development-default field, and remove the legacy KDF after supported upgrades have rewritten all pre-beta.9 Local KMS key files with explicit at-rest protection.
 - `sse-local-dek-json-v1` legacy local SSE DEK decoding: releases before the JSON envelope wrote wrapped DEKs as `base64(nonce):base64(ciphertext)`, so readers retain that decoder while all new writes use the versioned JSON envelope. Remove the colon decoder after the minimum supported direct-upgrade release writes JSON envelopes and migration tooling has rewritten every retained legacy object.
+- `not-initialized-error-code-v1` typed control-plane not-initialized wire code: control-plane RPC responses historically signaled an uninitialized peer only through the literal error_info string "errServerNotInitialized" (one drift site says "storage layer not initialized"). Responses now dual-carry a typed ControlPlaneErrorCode beside the legacy string, and clients prefer the code; the string stays populated and the client substring fallback (is_err_not_initialized, control_plane_failure) stays in place so mixed-version clusters keep classifying older peers' responses. Remove the substring fallback (and stop populating error_info for this case) after the minimum supported RustFS peer version always sends error_code.
 - `multipart-compression-default-off-window` staged multipart disk-compression rollout: releases before the resumable legacy decompressor fail transient reads of compressed objects under mid-payload suspension, so multipart uploads advertise the compression marker only when RUSTFS_COMPRESSION_MULTIPART_ENABLED is set in addition to RUSTFS_COMPRESSION_ENABLED, keeping rolling upgrades from creating new compressed multipart objects while pre-fix nodes may still serve reads. Flip the default to enabled (and retire the extra switch) after the minimum supported direct-upgrade release ships the resumable decompressor.

 ## Review Checklist
@@ -385,10 +385,12 @@ fn background_rebalance_start_error_message(result: StorageResult<()>) -> Option
 fn stop_rebalance_response(result: StorageResult<()>) -> StopRebalanceResponse {
    match result {
        Ok(_) => StopRebalanceResponse {
+            error_code: None,
            success: true,
            error_info: None,
        },
        Err(err) => StopRebalanceResponse {
+            error_code: None,
            success: false,
            error_info: Some(err.to_string()),
        },
@@ -1498,6 +1500,7 @@ impl Node for NodeService {
        let policy = request.policy_name;
        if policy.is_empty() {
            return Ok(Response::new(DeletePolicyResponse {
+                error_code: None,
                success: false,
                error_info: Some("policy name is missing".to_string()),
            }));
@@ -1507,17 +1510,20 @@ impl Node for NodeService {
            return Ok(Response::new(DeletePolicyResponse {
                success: false,
                error_info: Some("errServerNotInitialized".to_string()),
+                error_code: Some(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32),
            }));
        };

        let resp = iam_sys.delete_policy(&policy, false).await;
        if let Err(err) = resp {
            return Ok(Response::new(DeletePolicyResponse {
+                error_code: None,
                success: false,
                error_info: Some(err.to_string()),
            }));
        }
        Ok(Response::new(DeletePolicyResponse {
+            error_code: None,
            success: true,
            error_info: None,
        }))
@@ -1529,6 +1535,7 @@ impl Node for NodeService {
        let policy = request.policy_name;
        if policy.is_empty() {
            return Ok(Response::new(LoadPolicyResponse {
+                error_code: None,
                success: false,
                error_info: Some("policy name is missing".to_string()),
            }));
@@ -1537,17 +1544,20 @@ impl Node for NodeService {
            return Ok(Response::new(LoadPolicyResponse {
                success: false,
                error_info: Some("errServerNotInitialized".to_string()),
+                error_code: Some(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32),
            }));
        };

        let resp = iam_sys.load_policy(&policy).await;
        if let Err(err) = resp {
            return Ok(Response::new(LoadPolicyResponse {
+                error_code: None,
                success: false,
                error_info: Some(err.to_string()),
            }));
        }
        Ok(Response::new(LoadPolicyResponse {
+            error_code: None,
            success: true,
            error_info: None,
        }))
@@ -1562,12 +1572,14 @@ impl Node for NodeService {
        let user_or_group = request.user_or_group;
        if user_or_group.is_empty() {
            return Ok(Response::new(LoadPolicyMappingResponse {
+                error_code: None,
                success: false,
                error_info: Some("user_or_group name is missing".to_string()),
            }));
        }
        let Some(user_type) = UserType::from_u64(request.user_type) else {
            return Ok(Response::new(LoadPolicyMappingResponse {
+                error_code: None,
                success: false,
                error_info: Some("invalid user type".to_string()),
            }));
@@ -1577,16 +1589,19 @@ impl Node for NodeService {
            return Ok(Response::new(LoadPolicyMappingResponse {
                success: false,
                error_info: Some("errServerNotInitialized".to_string()),
+                error_code: Some(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32),
            }));
        };
        let resp = iam_sys.load_policy_mapping(&user_or_group, user_type, is_group).await;
        if let Err(err) = resp {
            return Ok(Response::new(LoadPolicyMappingResponse {
+                error_code: None,
                success: false,
                error_info: Some(err.to_string()),
            }));
        }
        Ok(Response::new(LoadPolicyMappingResponse {
+            error_code: None,
            success: true,
            error_info: None,
        }))
@@ -1598,6 +1613,7 @@ impl Node for NodeService {
        let access_key = request.access_key;
        if access_key.is_empty() {
            return Ok(Response::new(DeleteUserResponse {
+                error_code: None,
                success: false,
                error_info: Some("access_key name is missing".to_string()),
            }));
@@ -1606,17 +1622,20 @@ impl Node for NodeService {
            return Ok(Response::new(DeleteUserResponse {
                success: false,
                error_info: Some("errServerNotInitialized".to_string()),
+                error_code: Some(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32),
            }));
        };

        let resp = iam_sys.delete_user(&access_key, false).await;
        if let Err(err) = resp {
            return Ok(Response::new(DeleteUserResponse {
+                error_code: None,
                success: false,
                error_info: Some(err.to_string()),
            }));
        }
        Ok(Response::new(DeleteUserResponse {
+            error_code: None,
            success: true,
            error_info: None,
        }))
@@ -1631,6 +1650,7 @@ impl Node for NodeService {
        let access_key = request.access_key;
        if access_key.is_empty() {
            return Ok(Response::new(DeleteServiceAccountResponse {
+                error_code: None,
                success: false,
                error_info: Some("access_key name is missing".to_string()),
            }));
@@ -1644,6 +1664,7 @@ impl Node for NodeService {
            return Ok(Response::new(DeleteServiceAccountResponse {
                success: false,
                error_info: Some("errServerNotInitialized".to_string()),
+                error_code: Some(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32),
            }));
        };
        // This legacy RPC is a cache notification. Reloading shared state keeps a
@@ -1651,11 +1672,13 @@ impl Node for NodeService {
        let resp = iam_sys.load_service_account(&access_key).await;
        if let Err(err) = resp {
            return Ok(Response::new(DeleteServiceAccountResponse {
+                error_code: None,
                success: false,
                error_info: Some(err.to_string()),
            }));
        }
        Ok(Response::new(DeleteServiceAccountResponse {
+            error_code: None,
            success: true,
            error_info: None,
        }))
@@ -1668,6 +1691,7 @@ impl Node for NodeService {
        let temp = request.temp;
        if access_key.is_empty() {
            return Ok(Response::new(LoadUserResponse {
+                error_code: None,
                success: false,
                error_info: Some("access_key name is missing".to_string()),
            }));
@@ -1677,6 +1701,7 @@ impl Node for NodeService {
            return Ok(Response::new(LoadUserResponse {
                success: false,
                error_info: Some("errServerNotInitialized".to_string()),
+                error_code: Some(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32),
            }));
        };

@@ -1685,12 +1710,14 @@ impl Node for NodeService {
        let resp = iam_sys.load_user(&access_key, user_type).await;
        if let Err(err) = resp {
            return Ok(Response::new(LoadUserResponse {
+                error_code: None,
                success: false,
                error_info: Some(err.to_string()),
            }));
        }

        Ok(Response::new(LoadUserResponse {
+            error_code: None,
            success: true,
            error_info: None,
        }))
@@ -1705,6 +1732,7 @@ impl Node for NodeService {
        let access_key = request.access_key;
        if access_key.is_empty() {
            return Ok(Response::new(LoadServiceAccountResponse {
+                error_code: None,
                success: false,
                error_info: Some("access_key name is missing".to_string()),
            }));
@@ -1714,18 +1742,21 @@ impl Node for NodeService {
            return Ok(Response::new(LoadServiceAccountResponse {
                success: false,
                error_info: Some("errServerNotInitialized".to_string()),
+                error_code: Some(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32),
            }));
        };

        let resp = iam_sys.load_service_account(&access_key).await;
        if let Err(err) = resp {
            return Ok(Response::new(LoadServiceAccountResponse {
+                error_code: None,
                success: false,
                error_info: Some(err.to_string()),
            }));
        }

        Ok(Response::new(LoadServiceAccountResponse {
+            error_code: None,
            success: true,
            error_info: None,
        }))
@@ -1737,6 +1768,7 @@ impl Node for NodeService {
        let group = request.group;
        if group.is_empty() {
            return Ok(Response::new(LoadGroupResponse {
+                error_code: None,
                success: false,
                error_info: Some("group name is missing".to_string()),
            }));
@@ -1746,17 +1778,20 @@ impl Node for NodeService {
            return Ok(Response::new(LoadGroupResponse {
                success: false,
                error_info: Some("errServerNotInitialized".to_string()),
+                error_code: Some(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32),
            }));
        };

        let resp = iam_sys.load_group(&group).await;
        if let Err(err) = resp {
            return Ok(Response::new(LoadGroupResponse {
+                error_code: None,
                success: false,
                error_info: Some(err.to_string()),
            }));
        }
        Ok(Response::new(LoadGroupResponse {
+            error_code: None,
            success: true,
            error_info: None,
        }))
@@ -1771,14 +1806,17 @@ impl Node for NodeService {
            return Ok(Response::new(ReloadSiteReplicationConfigResponse {
                success: false,
                error_info: Some("errServerNotInitialized".to_string()),
+                error_code: Some(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32),
            }));
        };
        match reload_site_replication_runtime_state().await {
            Ok(()) => Ok(Response::new(ReloadSiteReplicationConfigResponse {
+                error_code: None,
                success: true,
                error_info: None,
            })),
            Err(err) => Ok(Response::new(ReloadSiteReplicationConfigResponse {
+                error_code: None,
                success: false,
                error_info: Some(err.to_string()),
            })),
@@ -2098,6 +2136,7 @@ impl Node for NodeService {
                success: false,
                bg_heal_state: Bytes::new(),
                error_info: Some("storage layer not initialized".to_string()),
+                error_code: Some(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32),
            }));
        }
        let snapshot = heal::capture_node_heal_status(rustfs_scanner::scanner::BackgroundHealInfo::default()).await;
@@ -2106,11 +2145,13 @@ impl Node for NodeService {
                success: true,
                bg_heal_state: bg_heal_state.into(),
                error_info: None,
+                error_code: None,
            })),
            Err(err) => Ok(Response::new(BackgroundHealStatusResponse {
                success: false,
                bg_heal_state: Bytes::new(),
                error_info: Some(err),
+                error_code: None,
            })),
        }
    }
@@ -2124,16 +2165,19 @@ impl Node for NodeService {
                success: false,
                recovery_status: Bytes::new(),
                error_info: Some("storage layer not initialized".to_string()),
+                error_code: Some(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32),
            }));
        }
        let snapshot = heal::capture_node_replacement_recovery_status().await;
        match heal::encode_node_replacement_recovery_status(&snapshot) {
            Ok(recovery_status) => Ok(Response::new(ReplacementRecoveryStatusResponse {
+                error_code: None,
                success: true,
                recovery_status: recovery_status.into(),
                error_info: None,
            })),
            Err(err) => Ok(Response::new(ReplacementRecoveryStatusResponse {
+                error_code: None,
                success: false,
                recovery_status: Bytes::new(),
                error_info: Some(err),
@@ -2164,6 +2208,7 @@ impl Node for NodeService {
            return Ok(Response::new(ReloadPoolMetaResponse {
                success: false,
                error_info: Some("errServerNotInitialized".to_string()),
+                error_code: Some(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32),
            }));
        };
        // Recover missing workers only after the reload merged newer state; a
@@ -2171,19 +2216,23 @@ impl Node for NodeService {
        match store.reload_pool_meta().await {
            Ok(true) => match store.spawn_missing_local_decommission_routines().await {
                Ok(_) => Ok(Response::new(ReloadPoolMetaResponse {
+                    error_code: None,
                    success: true,
                    error_info: None,
                })),
                Err(err) => Ok(Response::new(ReloadPoolMetaResponse {
+                    error_code: None,
                    success: false,
                    error_info: Some(err.to_string()),
                })),
            },
            Ok(false) => Ok(Response::new(ReloadPoolMetaResponse {
+                error_code: None,
                success: true,
                error_info: None,
            })),
            Err(err) => Ok(Response::new(ReloadPoolMetaResponse {
+                error_code: None,
                success: false,
                error_info: Some(err.to_string()),
            })),
@@ -2196,6 +2245,7 @@ impl Node for NodeService {
            return Ok(Response::new(StopRebalanceResponse {
                success: false,
                error_info: Some("errServerNotInitialized".to_string()),
+                error_code: Some(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32),
            }));
        };

@@ -2219,6 +2269,7 @@ impl Node for NodeService {
            return Ok(Response::new(LoadRebalanceMetaResponse {
                success: false,
                error_info: Some("errServerNotInitialized".to_string()),
+                error_code: Some(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32),
            }));
        };

@@ -2242,6 +2293,7 @@ impl Node for NodeService {
                    "node rpc background task failed"
                );
                return Ok(Response::new(LoadRebalanceMetaResponse {
+                    error_code: None,
                    success: false,
                    error_info: Some(message),
                }));
@@ -2249,6 +2301,7 @@ impl Node for NodeService {
        }

        Ok(Response::new(LoadRebalanceMetaResponse {
+            error_code: None,
            success: true,
            error_info: None,
        }))
@@ -2263,6 +2316,7 @@ impl Node for NodeService {
            return Ok(Response::new(StartDecommissionResponse {
                success: false,
                error_info: Some("errServerNotInitialized".to_string()),
+                error_code: Some(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32),
            }));
        };

@@ -2276,10 +2330,12 @@ impl Node for NodeService {

        match store.decommission(CancellationToken::new(), indices).await {
            Ok(()) => Ok(Response::new(StartDecommissionResponse {
+                error_code: None,
                success: true,
                error_info: None,
            })),
            Err(err) => Ok(Response::new(StartDecommissionResponse {
+                error_code: None,
                success: false,
                error_info: Some(err.to_string()),
            })),
@@ -2295,6 +2351,7 @@ impl Node for NodeService {
            return Ok(Response::new(CancelDecommissionResponse {
                success: false,
                error_info: Some("errServerNotInitialized".to_string()),
+                error_code: Some(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32),
            }));
        };

@@ -2302,6 +2359,7 @@ impl Node for NodeService {
            .map_err(|_| Status::invalid_argument("decommission pool index exceeds local range"))?;
        if let Err(err) = ensure_rpc_decommission_local_leader(&store, idx) {
            return Ok(Response::new(CancelDecommissionResponse {
+                error_code: None,
                success: false,
                error_info: Some(err.to_string()),
            }));
@@ -2309,10 +2367,12 @@ impl Node for NodeService {

        match store.decommission_cancel(idx).await {
            Ok(()) => Ok(Response::new(CancelDecommissionResponse {
+                error_code: None,
                success: true,
                error_info: None,
            })),
            Err(err) => Ok(Response::new(CancelDecommissionResponse {
+                error_code: None,
                success: false,
                error_info: Some(err.to_string()),
            })),
@@ -2328,6 +2388,7 @@ impl Node for NodeService {
            return Ok(Response::new(ClearDecommissionResponse {
                success: false,
                error_info: Some("errServerNotInitialized".to_string()),
+                error_code: Some(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32),
            }));
        };

@@ -2335,6 +2396,7 @@ impl Node for NodeService {
            .map_err(|_| Status::invalid_argument("decommission pool index exceeds local range"))?;
        if let Err(err) = ensure_rpc_decommission_local_leader(&store, idx) {
            return Ok(Response::new(ClearDecommissionResponse {
+                error_code: None,
                success: false,
                error_info: Some(err.to_string()),
            }));
@@ -2342,10 +2404,12 @@ impl Node for NodeService {

        match store.clear_decommission(idx).await {
            Ok(()) => Ok(Response::new(ClearDecommissionResponse {
+                error_code: None,
                success: true,
                error_info: None,
            })),
            Err(err) => Ok(Response::new(ClearDecommissionResponse {
+                error_code: None,
                success: false,
                error_info: Some(err.to_string()),
            })),
@@ -2361,15 +2425,18 @@ impl Node for NodeService {
            return Ok(Response::new(LoadTransitionTierConfigResponse {
                success: false,
                error_info: Some("errServerNotInitialized".to_string()),
+                error_code: Some(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32),
            }));
        };

        match reload_transition_tier_config(store).await {
            Ok(_) => Ok(Response::new(LoadTransitionTierConfigResponse {
+                error_code: None,
                success: true,
                error_info: None,
            })),
            Err(err) => Ok(Response::new(LoadTransitionTierConfigResponse {
+                error_code: None,
                success: false,
                error_info: Some(err.to_string()),
            })),
@@ -60,6 +60,7 @@ impl NodeService {
        let bucket = request.bucket;
        if bucket.is_empty() {
            return Ok(Response::new(LoadBucketMetadataResponse {
+                error_code: None,
                success: false,
                error_info: Some("bucket name is missing".to_string()),
            }));
@@ -69,6 +70,7 @@ impl NodeService {
            return Ok(Response::new(LoadBucketMetadataResponse {
                success: false,
                error_info: Some("errServerNotInitialized".to_string()),
+                error_code: Some(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32),
            }));
        };

@@ -78,11 +80,13 @@ impl NodeService {
                    rustfs_scanner::record_scanner_maintenance_change(&bucket);
                }
                Ok(Response::new(LoadBucketMetadataResponse {
+                    error_code: None,
                    success: true,
                    error_info: None,
                }))
            }
            Err(err) => Ok(Response::new(LoadBucketMetadataResponse {
+                error_code: None,
                success: false,
                error_info: Some(err.to_string()),
            })),
@@ -228,17 +228,20 @@ impl NodeService {
                success: false,
                storage_info: Bytes::new(),
                error_info: Some("errServerNotInitialized".to_string()),
+                error_code: Some(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32),
            }));
        };

        let info = StorageAdminApi::local_storage_info(store.as_ref()).await;
        match encode_msgpack_map(&info) {
            Ok(buf) => Ok(Response::new(LocalStorageInfoResponse {
+                error_code: None,
                success: true,
                storage_info: buf.into(),
                error_info: None,
            })),
            Err(err) => Ok(Response::new(LocalStorageInfoResponse {
+                error_code: None,
                success: false,
                storage_info: Bytes::new(),
                error_info: Some(err.to_string()),