fix(tier): lock tier config mutations (#5080)

* fix(tier): lock tier config mutations

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(tier): add mutation RPC auth contract (#5082)

Co-authored-by: heihutu <heihutu@gmail.com>

* fix(tier): add peer mutation handler core (#5084)

Co-authored-by: heihutu <heihutu@gmail.com>

* fix(tier): add mutation control rpc service (#5087)

Co-authored-by: heihutu <heihutu@gmail.com>

* fix(tier): recover prepared mutation drains (#5093)

Recover prepared tier mutation intent records into the local tier runtime so a restarted peer fails closed before issuing new remote-tier operation leases or conflicting admin publishes.

Reconcile the recovered block map on each scan so committed, aborted, or removed intents clear stale local blocks instead of wedging the peer until process restart.

Co-authored-by: heihutu <heihutu@gmail.com>

* fix(tier): prove zero references before tier removal (#5092)

Signed-off-by: houseme <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>

* fix(tier): clear peer mutation runtime blocks (#5094)

Install prepared mutation runtime blocks when peer prepare requests are applied or replayed so followers fail closed immediately before restart recovery.

Clear the in-memory block once peer commit or abort reaches a durable terminal state, including delayed duplicate prepare requests that observe a committed or aborted record.

Co-authored-by: heihutu <heihutu@gmail.com>

---------

Signed-off-by: houseme <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>

rustfs/src/server/http.rs

@@ -57,6 +57,7 @@ use rustfs_keystone::KeystoneAuthLayer;
 use rustfs_protocols::SwiftService;
 use rustfs_protos::proto_gen::node_service::{
     heal_control_service_server::HealControlServiceServer, node_service_server::NodeServiceServer,
+    tier_mutation_control_service_server::TierMutationControlServiceServer,
 };
 use rustfs_trusted_proxies::ClientInfo;
 use rustfs_utils::net::parse_and_resolve_address;
@@ -128,6 +129,9 @@ const EVENT_PEER_ADDR_UNAVAILABLE: &str = "peer_addr_unavailable";
 const EVENT_RPC_SIGNATURE_VERIFICATION_FAILED: &str = "rpc_signature_verification_failed";
 const EVENT_GRPC_TRACE_CONTEXT_PROPAGATION_FAILED: &str = "grpc_trace_context_propagation_failed";
 const HEAL_CONTROL_TONIC_RPC_PATH: &str = "/node_service.HealControlService/HealControl";
+const TIER_MUTATION_PREPARE_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/PrepareTierMutation";
+const TIER_MUTATION_COMMIT_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/CommitTierMutation";
+const TIER_MUTATION_ABORT_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/AbortTierMutation";
 
 static ACTIVE_HTTP_REQUESTS: AtomicU64 = AtomicU64::new(0);
 
@@ -1322,7 +1326,19 @@ fn process_connection(
             .max_encoding_message_size(heal_control_max_message_size),
             check_auth,
         );
-        let rpc_service = RpcRequestPathService::new(Routes::new(node_service).add_service(heal_control_service).prepare());
+        let tier_mutation_control_max_message_size = rustfs_protos::TIER_MUTATION_RPC_MAX_MESSAGE_SIZE;
+        let tier_mutation_control_service = InterceptedService::new(
+            TierMutationControlServiceServer::new(storage::tonic_service::make_tier_mutation_control_server())
+                .max_decoding_message_size(tier_mutation_control_max_message_size)
+                .max_encoding_message_size(tier_mutation_control_max_message_size),
+            check_auth,
+        );
+        let rpc_service = RpcRequestPathService::new(
+            Routes::new(node_service)
+                .add_service(heal_control_service)
+                .add_service(tier_mutation_control_service)
+                .prepare(),
+        );
 
         #[cfg(feature = "swift")]
         let http_service = SwiftService::new(true, None, s3_service);
@@ -1861,6 +1877,9 @@ fn check_auth(req: Request<()>) -> std::result::Result<Request<()>, Status> {
         .strip_prefix(TONIC_RPC_PREFIX)
         .and_then(|suffix| suffix.strip_prefix('/'))
         .or_else(|| (target.uri.path() == HEAL_CONTROL_TONIC_RPC_PATH).then_some("HealControl"))
+        .or_else(|| (target.uri.path() == TIER_MUTATION_PREPARE_TONIC_RPC_PATH).then_some("PrepareTierMutation"))
+        .or_else(|| (target.uri.path() == TIER_MUTATION_COMMIT_TONIC_RPC_PATH).then_some("CommitTierMutation"))
+        .or_else(|| (target.uri.path() == TIER_MUTATION_ABORT_TONIC_RPC_PATH).then_some("AbortTierMutation"))
         .filter(|method| !method.is_empty() && !method.contains('/'))
         .ok_or_else(|| Status::unauthenticated("Invalid RPC request path"))?;
     debug_assert!(!rpc_method.is_empty());
@@ -2279,6 +2298,23 @@ mod tests {
         });
         assert!(check_auth(heal_request).is_ok(), "heal control service path should authenticate");
 
+        let tier_headers = storage::gen_tonic_signature_headers(
+            "127.0.0.1:9000",
+            "node_service.TierMutationControlService",
+            "PrepareTierMutation",
+            None,
+        )
+        .expect("tier mutation auth headers should build");
+        let mut tier_request = Request::new(());
+        tier_request.metadata_mut().as_mut().extend(tier_headers);
+        tier_request.extensions_mut().insert(RpcRequestTarget {
+            uri: TIER_MUTATION_PREPARE_TONIC_RPC_PATH
+                .parse()
+                .expect("tier mutation path should parse"),
+            method: Method::POST,
+        });
+        assert!(check_auth(tier_request).is_ok(), "tier mutation control service path should authenticate");
+
         let replay_headers = storage::gen_tonic_signature_headers("127.0.0.1:9000", "node_service.NodeService", "Ping", None)
             .expect("node service auth headers should build");
         let mut cross_service_replay = Request::new(());
@@ -2292,6 +2328,21 @@ mod tests {
             "node service signature must not replay to heal control"
         );
 
+        let replay_headers = storage::gen_tonic_signature_headers("127.0.0.1:9000", "node_service.NodeService", "Ping", None)
+            .expect("node service auth headers should build");
+        let mut cross_service_replay = Request::new(());
+        cross_service_replay.metadata_mut().as_mut().extend(replay_headers);
+        cross_service_replay.extensions_mut().insert(RpcRequestTarget {
+            uri: TIER_MUTATION_PREPARE_TONIC_RPC_PATH
+                .parse()
+                .expect("tier mutation path should parse"),
+            method: Method::POST,
+        });
+        assert!(
+            check_auth(cross_service_replay).is_err(),
+            "node service signature must not replay to tier mutation control"
+        );
+
         rustfs_common::set_global_local_node_name("127.0.0.1:9001").await;
         let mut replay_to_other_node = Request::new(());
         replay_to_other_node.metadata_mut().as_mut().extend(headers.clone());

rustfs/src/storage/rpc/mod.rs

@@ -16,7 +16,10 @@ pub mod http_service;
 pub mod node_service;
 
 pub use http_service::InternodeRpcService;
-pub use node_service::{HealControlRpcService, NodeService, make_heal_control_server, make_server};
+pub use node_service::{
+    HealControlRpcService, NodeService, TierMutationControlRpcService, make_heal_control_server, make_server,
+    make_tier_mutation_control_server,
+};
 
 use rmp_serde::Serializer;
 use serde::Serialize;

rustfs/src/storage/rpc/node_service.rs

@@ -16,6 +16,7 @@ use crate::admin::service::{
     config::{reload_dynamic_config_runtime_state, reload_runtime_config_snapshot},
     site_replication::reload_site_replication_runtime_state,
 };
+use crate::storage::storage_api::ecstore_tier::tier_mutation_peer::{self, TierMutationPeerState as EcTierMutationPeerState};
 #[cfg(test)]
 use crate::storage::storage_api::rpc_consumer::node_service::STORAGE_CLASS_SUB_SYS;
 #[cfg(test)]
@@ -54,6 +55,7 @@ use tokio_stream::wrappers::ReceiverStream;
 use tokio_util::sync::CancellationToken;
 use tonic::{Request, Response, Status, Streaming};
 use tracing::{debug, error, info, warn};
+use uuid::Uuid;
 
 pub(crate) mod heal;
 
@@ -68,6 +70,10 @@ const EVENT_RPC_RESPONSE_EMITTED: &str = "rpc_response_emitted";
 const EVENT_RPC_BACKGROUND_TASK_SPAWNED: &str = "rpc_background_task_spawned";
 const EVENT_RPC_BACKGROUND_TASK_FAILED: &str = "rpc_background_task_failed";
 const HEAL_CONTROL_REPLAY_CACHE_MAX_ENTRIES: usize = 4096;
+const TIER_MUTATION_PEER_STATE_UNSPECIFIED_WIRE: i32 = 0;
+const TIER_MUTATION_PEER_STATE_PREPARED_WIRE: i32 = 1;
+const TIER_MUTATION_PEER_STATE_COMMITTED_WIRE: i32 = 2;
+const TIER_MUTATION_PEER_STATE_ABORTED_WIRE: i32 = 3;
 
 #[derive(Debug)]
 struct HealControlReplayEntry {
@@ -484,6 +490,203 @@ async fn execute_heal_control_envelope_with_manager(
     Ok(result)
 }
 
+#[derive(Clone, Default)]
+pub struct TierMutationControlRpcService {
+    context: Option<Arc<runtime_sources::AppContext>>,
+}
+
+impl std::fmt::Debug for TierMutationControlRpcService {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("TierMutationControlRpcService")
+            .field("context_present", &self.context.is_some())
+            .finish()
+    }
+}
+
+pub fn make_tier_mutation_control_server() -> TierMutationControlRpcService {
+    TierMutationControlRpcService {
+        context: runtime_sources::current_app_context(),
+    }
+}
+
+#[cfg(test)]
+pub(crate) fn make_tier_mutation_control_server_for_context(
+    context: Option<Arc<runtime_sources::AppContext>>,
+) -> TierMutationControlRpcService {
+    TierMutationControlRpcService { context }
+}
+
+impl TierMutationControlRpcService {
+    fn resolve_object_store(&self) -> Option<Arc<ECStore>> {
+        let context = self.context.clone().or_else(runtime_sources::current_app_context);
+        runtime_sources::current_object_store_handle_for_context(context.as_deref())
+    }
+
+    async fn execute_tier_mutation(
+        &self,
+        request: &Request<()>,
+        version: u32,
+        phase: rustfs_protos::TierMutationRpcPhase,
+        mutation_id: &str,
+        canonical_payload: &Bytes,
+    ) -> Result<Response<TierMutationControlResponse>, Status> {
+        validate_tier_mutation_payload_size(phase, canonical_payload.len())?;
+        let mutation_id = parse_tier_mutation_id(mutation_id)?;
+        let body = rustfs_protos::canonical_tier_mutation_rpc_body(version, phase, mutation_id, canonical_payload)
+            .map_err(|_| Status::invalid_argument("tier mutation request length cannot be represented"))?;
+        verify_tonic_canonical_body_digest(request, &body)
+            .map_err(|err| Status::permission_denied(format!("tier mutation authentication failed: {err}")))?;
+        let store = self
+            .resolve_object_store()
+            .ok_or_else(|| Status::failed_precondition("tier mutation object store is not initialized"))?;
+
+        match tier_mutation_peer::handle_tier_mutation_peer_request(store, version, phase, mutation_id, canonical_payload).await {
+            Ok(outcome) => tier_mutation_control_response(TierMutationControlResponseInput {
+                version,
+                phase,
+                mutation_id,
+                canonical_payload,
+                success: true,
+                state: tier_mutation_peer_state_to_proto_wire(outcome.state),
+                applied: outcome.applied,
+                error_info: None,
+            }),
+            Err(err) => tier_mutation_control_response(TierMutationControlResponseInput {
+                version,
+                phase,
+                mutation_id,
+                canonical_payload,
+                success: false,
+                state: TIER_MUTATION_PEER_STATE_UNSPECIFIED_WIRE,
+                applied: false,
+                error_info: Some(err.to_string()),
+            }),
+        }
+    }
+}
+
+#[tonic::async_trait]
+impl tier_mutation_control_service_server::TierMutationControlService for TierMutationControlRpcService {
+    async fn prepare_tier_mutation(
+        &self,
+        request: Request<TierMutationPrepareRequest>,
+    ) -> Result<Response<TierMutationControlResponse>, Status> {
+        let (metadata, extensions, inner) = request.into_parts();
+        let request = Request::from_parts(metadata, extensions, ());
+        self.execute_tier_mutation(
+            &request,
+            inner.version,
+            rustfs_protos::TierMutationRpcPhase::Prepare,
+            &inner.mutation_id,
+            &inner.canonical_payload,
+        )
+        .await
+    }
+
+    async fn commit_tier_mutation(
+        &self,
+        request: Request<TierMutationCommitRequest>,
+    ) -> Result<Response<TierMutationControlResponse>, Status> {
+        let (metadata, extensions, inner) = request.into_parts();
+        let request = Request::from_parts(metadata, extensions, ());
+        self.execute_tier_mutation(
+            &request,
+            inner.version,
+            rustfs_protos::TierMutationRpcPhase::Commit,
+            &inner.mutation_id,
+            &inner.canonical_payload,
+        )
+        .await
+    }
+
+    async fn abort_tier_mutation(
+        &self,
+        request: Request<TierMutationAbortRequest>,
+    ) -> Result<Response<TierMutationControlResponse>, Status> {
+        let (metadata, extensions, inner) = request.into_parts();
+        let request = Request::from_parts(metadata, extensions, ());
+        self.execute_tier_mutation(
+            &request,
+            inner.version,
+            rustfs_protos::TierMutationRpcPhase::Abort,
+            &inner.mutation_id,
+            &inner.canonical_payload,
+        )
+        .await
+    }
+}
+
+fn parse_tier_mutation_id(mutation_id: &str) -> Result<Uuid, Status> {
+    let parsed = Uuid::parse_str(mutation_id).map_err(|_| Status::invalid_argument("tier mutation id is invalid"))?;
+    if parsed.to_string() != mutation_id {
+        return Err(Status::invalid_argument("tier mutation id is not canonical"));
+    }
+    Ok(parsed)
+}
+
+fn validate_tier_mutation_payload_size(phase: rustfs_protos::TierMutationRpcPhase, payload_len: usize) -> Result<(), Status> {
+    let limit = match phase {
+        rustfs_protos::TierMutationRpcPhase::Prepare => rustfs_protos::TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE,
+        rustfs_protos::TierMutationRpcPhase::Commit => rustfs_protos::TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE,
+        rustfs_protos::TierMutationRpcPhase::Abort => {
+            if payload_len != 0 {
+                return Err(Status::invalid_argument("tier mutation abort payload must be empty"));
+            }
+            return Ok(());
+        }
+        _ => return Err(Status::invalid_argument("tier mutation rpc phase is unsupported")),
+    };
+    if payload_len > limit {
+        return Err(Status::invalid_argument("tier mutation payload exceeds size limit"));
+    }
+    Ok(())
+}
+
+struct TierMutationControlResponseInput<'a> {
+    version: u32,
+    phase: rustfs_protos::TierMutationRpcPhase,
+    mutation_id: Uuid,
+    canonical_payload: &'a [u8],
+    success: bool,
+    state: i32,
+    applied: bool,
+    error_info: Option<String>,
+}
+
+fn tier_mutation_control_response(
+    input: TierMutationControlResponseInput<'_>,
+) -> Result<Response<TierMutationControlResponse>, Status> {
+    let canonical_response =
+        rustfs_protos::canonical_tier_mutation_rpc_response_body(rustfs_protos::TierMutationRpcResponseProofInput {
+            version: input.version,
+            phase: input.phase,
+            mutation_id: input.mutation_id,
+            canonical_payload: input.canonical_payload,
+            success: input.success,
+            state: input.state,
+            applied: input.applied,
+            error_info: input.error_info.as_deref(),
+        })
+        .map_err(|_| Status::internal("tier mutation response length cannot be represented"))?;
+    let response_proof = sign_tonic_rpc_response_proof(&canonical_response)
+        .map_err(|_| Status::internal("tier mutation response proof is unavailable"))?;
+    Ok(Response::new(TierMutationControlResponse {
+        success: input.success,
+        state: input.state,
+        applied: input.applied,
+        error_info: input.error_info,
+        response_proof: response_proof.into(),
+    }))
+}
+
+fn tier_mutation_peer_state_to_proto_wire(state: EcTierMutationPeerState) -> i32 {
+    match state {
+        EcTierMutationPeerState::Prepared => TIER_MUTATION_PEER_STATE_PREPARED_WIRE,
+        EcTierMutationPeerState::Committed => TIER_MUTATION_PEER_STATE_COMMITTED_WIRE,
+        EcTierMutationPeerState::Aborted => TIER_MUTATION_PEER_STATE_ABORTED_WIRE,
+    }
+}
+
 #[tonic::async_trait]
 impl heal_control_service_server::HealControlService for HealControlRpcService {
     async fn heal_control(&self, request: Request<HealControlRequest>) -> Result<Response<HealControlResponse>, Status> {
@@ -1618,7 +1821,8 @@ mod tests {
         PEER_RESTSUB_SYS, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, STORAGE_CLASS_SUB_SYS,
         admit_heal_control_replay, background_rebalance_start_error_message, execute_heal_control_envelope_with_manager,
         initialize_heal_topology_fingerprint, make_heal_control_server, make_heal_control_server_with_cache, make_server,
-        remove_heal_control_replay, scanner_activity_response, stop_rebalance_response,
+        make_tier_mutation_control_server_for_context, remove_heal_control_replay, scanner_activity_response,
+        stop_rebalance_response,
     };
     use crate::storage::rpc::node_service::heal::heal_topology_fingerprint;
     use crate::storage::storage_api::rpc_consumer::node_service::{HealBucketInfo, HealEndpoint};
@@ -1643,12 +1847,14 @@ mod tests {
         MakeVolumesRequest, Mss, PingRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest, ReadVersionRequest,
         ReadXlRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest, RenameFileRequest,
         RenamePartRequest, ScannerActivityRequest, ServerInfoRequest, SignalServiceRequest, StartProfilingRequest,
-        StatVolumeRequest, StopRebalanceRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest,
-        WriteAllRequest, WriteMetadataRequest, WriteRequest,
+        StatVolumeRequest, StopRebalanceRequest, TierMutationPeerState, TierMutationPrepareRequest,
+        UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
+        WriteRequest,
         heal_control_service_client::HealControlServiceClient,
         heal_control_service_server::{HealControlService as _, HealControlServiceServer},
         node_service_client::NodeServiceClient,
         node_service_server::NodeServiceServer,
+        tier_mutation_control_service_server::TierMutationControlService as _,
     };
     use std::{collections::HashMap, sync::Arc};
     use time::OffsetDateTime;
@@ -1967,6 +2173,24 @@ mod tests {
             .insert("x-rustfs-rpc-auth-version", "2".parse().expect("valid metadata value"));
     }
 
+    fn signed_tier_prepare_request(mutation_id: uuid::Uuid, canonical_payload: Bytes) -> Request<TierMutationPrepareRequest> {
+        let mut request = Request::new(TierMutationPrepareRequest {
+            version: rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
+            mutation_id: mutation_id.to_string(),
+            canonical_payload,
+        });
+        let body = rustfs_protos::canonical_tier_mutation_rpc_body(
+            request.get_ref().version,
+            rustfs_protos::TierMutationRpcPhase::Prepare,
+            mutation_id,
+            &request.get_ref().canonical_payload,
+        )
+        .expect("small request should encode");
+        set_tonic_canonical_body_digest(&mut request, &body).expect("digest metadata should encode");
+        mark_v2_authenticated(&mut request);
+        request
+    }
+
     #[tokio::test]
     async fn heal_control_requires_body_bound_auth_before_topology_validation() {
         let service = make_heal_control_server();
@@ -2004,6 +2228,134 @@ mod tests {
         assert_eq!(unavailable.code(), tonic::Code::FailedPrecondition);
     }
 
+    #[tokio::test]
+    async fn tier_mutation_control_requires_body_bound_auth_before_store_lookup() {
+        let service = make_tier_mutation_control_server_for_context(None);
+        let mutation_id = uuid::Uuid::new_v4();
+        let unsigned = service
+            .prepare_tier_mutation(Request::new(TierMutationPrepareRequest {
+                version: rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
+                mutation_id: mutation_id.to_string(),
+                canonical_payload: Bytes::from_static(b"intent"),
+            }))
+            .await
+            .expect_err("unsigned request must fail before store lookup");
+        assert_eq!(unsigned.code(), tonic::Code::PermissionDenied);
+
+        let mut tampered = signed_tier_prepare_request(mutation_id, Bytes::from_static(b"intent"));
+        let other_body = rustfs_protos::canonical_tier_mutation_rpc_body(
+            rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
+            rustfs_protos::TierMutationRpcPhase::Commit,
+            mutation_id,
+            b"intent",
+        )
+        .expect("small request should encode");
+        set_tonic_canonical_body_digest(&mut tampered, &other_body).expect("digest metadata should encode");
+        let tampered = service
+            .prepare_tier_mutation(tampered)
+            .await
+            .expect_err("phase replay must fail body-bound authentication");
+        assert_eq!(tampered.code(), tonic::Code::PermissionDenied);
+
+        let signed = signed_tier_prepare_request(mutation_id, Bytes::from_static(b"intent"));
+        let unavailable = service
+            .prepare_tier_mutation(signed)
+            .await
+            .expect_err("authenticated request still requires initialized object store");
+        assert_eq!(unavailable.code(), tonic::Code::FailedPrecondition);
+    }
+
+    #[tokio::test]
+    async fn tier_mutation_control_requires_canonical_mutation_id() {
+        let service = make_tier_mutation_control_server_for_context(None);
+        let mutation_id = uuid::Uuid::new_v4().to_string().to_uppercase();
+        let error = service
+            .prepare_tier_mutation(Request::new(TierMutationPrepareRequest {
+                version: rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
+                mutation_id,
+                canonical_payload: Bytes::from_static(b"intent"),
+            }))
+            .await
+            .expect_err("uppercase UUID must not pass canonical request binding");
+        assert_eq!(error.code(), tonic::Code::InvalidArgument);
+    }
+
+    #[tokio::test]
+    async fn tier_mutation_control_rejects_oversized_prepare_before_auth_and_store_lookup() {
+        let service = make_tier_mutation_control_server_for_context(None);
+        let mutation_id = uuid::Uuid::new_v4();
+        let oversized = Bytes::from(vec![0; rustfs_protos::TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE + 1]);
+        let error = service
+            .prepare_tier_mutation(Request::new(TierMutationPrepareRequest {
+                version: rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
+                mutation_id: mutation_id.to_string(),
+                canonical_payload: oversized,
+            }))
+            .await
+            .expect_err("oversized prepare must fail before digest construction");
+        assert_eq!(error.code(), tonic::Code::InvalidArgument);
+    }
+
+    #[test]
+    fn tier_mutation_peer_state_wire_constants_match_generated_proto() {
+        assert_eq!(
+            super::TIER_MUTATION_PEER_STATE_UNSPECIFIED_WIRE,
+            TierMutationPeerState::Unspecified as i32
+        );
+        assert_eq!(super::TIER_MUTATION_PEER_STATE_PREPARED_WIRE, TierMutationPeerState::Prepared as i32);
+        assert_eq!(super::TIER_MUTATION_PEER_STATE_COMMITTED_WIRE, TierMutationPeerState::Committed as i32);
+        assert_eq!(super::TIER_MUTATION_PEER_STATE_ABORTED_WIRE, TierMutationPeerState::Aborted as i32);
+    }
+
+    #[test]
+    fn tier_mutation_control_response_proof_binds_request_and_result() {
+        let _ = rustfs_credentials::set_global_rpc_secret("tier-mutation-control-response-proof-test-secret".to_string());
+        let mutation_id = uuid::Uuid::new_v4();
+        let payload = b"canonical-intent-record";
+        let response = super::tier_mutation_control_response(super::TierMutationControlResponseInput {
+            version: rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
+            phase: rustfs_protos::TierMutationRpcPhase::Prepare,
+            mutation_id,
+            canonical_payload: payload,
+            success: false,
+            state: TierMutationPeerState::Unspecified as i32,
+            applied: false,
+            error_info: Some("store failed".to_string()),
+        })
+        .expect("response proof should be signed")
+        .into_inner();
+        let canonical =
+            rustfs_protos::canonical_tier_mutation_rpc_response_body(rustfs_protos::TierMutationRpcResponseProofInput {
+                version: rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
+                phase: rustfs_protos::TierMutationRpcPhase::Prepare,
+                mutation_id,
+                canonical_payload: payload,
+                success: false,
+                state: TierMutationPeerState::Unspecified as i32,
+                applied: false,
+                error_info: Some("store failed"),
+            })
+            .expect("small mutation response should encode");
+        crate::storage::storage_api::verify_tonic_rpc_response_proof(&canonical, &response.response_proof)
+            .expect("proof must authenticate the exact response");
+
+        let tampered =
+            rustfs_protos::canonical_tier_mutation_rpc_response_body(rustfs_protos::TierMutationRpcResponseProofInput {
+                version: rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
+                phase: rustfs_protos::TierMutationRpcPhase::Prepare,
+                mutation_id,
+                canonical_payload: payload,
+                success: true,
+                state: TierMutationPeerState::Unspecified as i32,
+                applied: false,
+                error_info: Some("store failed"),
+            })
+            .expect("small mutation response should encode");
+        let error = crate::storage::storage_api::verify_tonic_rpc_response_proof(&tampered, &response.response_proof)
+            .expect_err("proof must reject a tampered success flag");
+        assert_eq!(error.to_string(), "Invalid RPC response proof");
+    }
+
     #[tokio::test]
     async fn heal_control_rejects_oversized_command_before_canonical_copy() {
         let service = make_heal_control_server();

rustfs/src/storage/storage_api.rs

@@ -333,7 +333,9 @@ pub(crate) mod timeout_wrapper_consumer {
 pub(crate) mod tonic_service_consumer {
     #[cfg(test)]
     pub(crate) use super::super::tonic_service::{heal_topology_fingerprint, make_heal_control_server_for_source};
-    pub(crate) use super::super::tonic_service::{make_heal_control_server_with_cache, make_server};
+    pub(crate) use super::super::tonic_service::{
+        make_heal_control_server_with_cache, make_server, make_tier_mutation_control_server,
+    };
 }
 
 #[cfg(test)]
@@ -512,7 +514,7 @@ pub(crate) mod ecstore_storage {
 
 pub(crate) mod ecstore_tier {
     pub(crate) use rustfs_ecstore::api::tier::tier::{TierConfigMgr, TierConfigUpdateError};
-    pub(crate) use rustfs_ecstore::api::tier::{tier, tier_admin, tier_config, tier_handlers};
+    pub(crate) use rustfs_ecstore::api::tier::{tier, tier_admin, tier_config, tier_handlers, tier_mutation_peer};
     // Shared lifecycle/tier test utilities behind ecstore's `test-util` feature
     // (rustfs/backlog#1148 ilm-6). Only linked into test builds.
     #[cfg(test)]

rustfs/src/storage/tonic_service.rs

@@ -15,6 +15,6 @@
 pub(crate) use crate::storage::rpc::node_service::make_heal_control_server_with_cache;
 #[cfg(test)]
 pub(crate) use crate::storage::rpc::node_service::{heal::heal_topology_fingerprint, make_heal_control_server_for_source};
-pub use crate::storage::rpc::{make_heal_control_server, make_server};
+pub use crate::storage::rpc::{make_heal_control_server, make_server, make_tier_mutation_control_server};
 #[allow(dead_code)]
 pub type NodeService = crate::storage::rpc::NodeService;

rustfs/src/storage_api.rs

@@ -135,7 +135,7 @@ pub(crate) mod server {
                 heal_topology_fingerprint, make_heal_control_server_for_source,
             };
             pub(crate) use crate::storage::storage_api::tonic_service_consumer::{
-                make_heal_control_server_with_cache, make_server,
+                make_heal_control_server_with_cache, make_server, make_tier_mutation_control_server,
             };
         }
     }