feat(rpc): bind canonical body digest into internode mutating disk RPC signatures (#5234)

* feat(rpc): bind canonical body digest into internode mutating disk RPC signatures

Binds a domain-separated, length-prefixed canonical request-body digest into the
v2 HMAC signature scope for every mutating NodeService disk RPC, so an on-path
attacker on the default-plaintext internode channel can no longer tamper with a
mutation payload (or strip the msgpack `_bin` field to force the JSON fallback
decode) without invalidating the signature.

Covers 13 mutating disk RPCs: RenameData, DeleteVersion, DeleteVersions,
WriteMetadata, UpdateMetadata, WriteAll, Delete, DeletePaths, RenameFile,
RenamePart, DeleteVolume, MakeVolume, MakeVolumes. The digest covers both the
msgpack `_bin` payloads and their JSON compatibility copies. Gated fail-open by
default (RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT) with a convergence counter, so
rolling upgrades are byte-for-byte unaffected; the replay-cache capacity is now
configurable and overflow fails closed with a metric.

Refs https://github.com/rustfs/backlog/issues/1327

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* fix(rpc): satisfy architecture-migration compat-marker guard

Put the removal condition on the RUSTFS_COMPAT_TODO marker line itself, and
stop backticking env-var/metric names in the cleanup-register entry so the
guard's id extractor only sees the task-id.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

crates/ecstore/src/api/mod.rs

@@ -411,7 +411,8 @@ pub mod rpc {
         TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
         node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
         set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_rpc_signature,
-        verify_tonic_canonical_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
+        verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
+        verify_tonic_rpc_signature,
     };
 }
 

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -61,7 +61,6 @@ const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
 const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned";
 const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
 const REPLAY_CACHE_RETENTION: Duration = Duration::from_secs(601);
-const MAX_REPLAY_PROTECTED_NONCES: usize = 65_536;
 const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3";
 pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
 static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock<bool> = LazyLock::new(|| {
@@ -70,6 +69,22 @@ static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock<bool> = LazyLock::new(|| {
         rustfs_config::DEFAULT_INTERNODE_RPC_SIGNATURE_STRICT,
     )
 });
+static INTERNODE_RPC_BODY_DIGEST_STRICT: LazyLock<bool> = LazyLock::new(|| {
+    get_env_bool(
+        rustfs_config::ENV_INTERNODE_RPC_BODY_DIGEST_STRICT,
+        rustfs_config::DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT,
+    )
+});
+// Sized for peak legitimate body-bound mutation RPS x the retention window; overflow fails closed
+// and increments the replay-cache overflow counter. Clamped to at least 1 so a misconfigured zero
+// cannot disable replay protection by rejecting every body-bound request.
+static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
+    rustfs_utils::get_env_usize(
+        rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
+        rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
+    )
+    .max(1)
+});
 static RPC_SECRET_RESOLUTION_LOG_ONCE: Once = Once::new();
 
 #[derive(Default)]
@@ -110,6 +125,10 @@ impl RpcNonceCache {
             return Err(std::io::Error::other("RPC request replay detected"));
         }
         if self.nonces.len() >= capacity {
+            // Fail closed and alert: only legitimately signed traffic can fill the cache, so a
+            // sustained overflow means RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY is undersized
+            // for this node's peak mutation rate and writes are being refused.
+            global_internode_metrics().record_replay_cache_overflow();
             return Err(std::io::Error::other("RPC replay cache capacity exceeded"));
         }
         self.nonces.insert(nonce);
@@ -334,7 +353,7 @@ fn check_and_record_nonce(nonce: Uuid, signed_at: i64) -> std::io::Result<()> {
     let expires_at = now
         .checked_add(REPLAY_CACHE_RETENTION)
         .ok_or_else(|| std::io::Error::other("RPC replay expiry overflow"))?;
-    cache.check_and_record(nonce, signed_at, now, wall_time, expires_at, MAX_REPLAY_PROTECTED_NONCES)
+    cache.check_and_record(nonce, signed_at, now, wall_time, expires_at, *REPLAY_CACHE_CAPACITY)
 }
 
 /// Build headers with authentication signature
@@ -447,6 +466,50 @@ pub fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canoni
     Ok(())
 }
 
+/// Verify a mutating disk RPC's canonical body digest with a rolling-upgrade fallback.
+///
+/// When the request carries a real (non-`UNSIGNED-PAYLOAD`) content SHA-256 it is verified exactly
+/// like [`verify_tonic_canonical_body_digest`]. The digest value is a member of the signed v2
+/// scope, so within the v2 lane it cannot be stripped or altered without invalidating the signature
+/// `check_auth` already enforced. When the request carries no digest — a peer that predates
+/// body-digest signing, or an attacker who downgraded the request to the legacy signature by
+/// dropping every v2 header — the request is accepted and counted on the body-digest fallback
+/// counter unless `RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT` is enabled. That switch is what actually
+/// closes on-path body tampering for covered handlers: it rejects every digestless mutation,
+/// including v1-downgraded ones. It converges independently of the signature-strict switch
+/// (<https://github.com/rustfs/backlog/issues/1327>).
+pub fn verify_tonic_mutation_body_digest<T>(request: &tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
+    verify_tonic_mutation_body_digest_with_strictness(request, canonical_body, *INTERNODE_RPC_BODY_DIGEST_STRICT)
+}
+
+/// [`verify_tonic_mutation_body_digest`] with the strict gate injected as a parameter, so both
+/// rollout postures are unit-testable without racing on process-global environment variables.
+fn verify_tonic_mutation_body_digest_with_strictness<T>(
+    request: &tonic::Request<T>,
+    canonical_body: &[u8],
+    strict: bool,
+) -> std::io::Result<()> {
+    let digest = request
+        .metadata()
+        .get(RPC_CONTENT_SHA256_HEADER)
+        .and_then(|value| value.to_str().ok());
+    match digest {
+        Some(digest) if digest != UNSIGNED_PAYLOAD => verify_tonic_canonical_body_digest(request, canonical_body),
+        _ => {
+            // RUSTFS_COMPAT_TODO(disk-mutation-body-digest): accept digestless peers during rolling upgrades. Remove after the
+            // minimum supported RustFS peer version body-binds every mutating disk RPC.
+            if strict {
+                return Err(std::io::Error::other("RPC mutation requires a body-bound v2 signature"));
+            }
+            // Count only ACCEPTED digestless mutations: this counter is the convergence gate that
+            // must read zero fleet-wide across a release window before
+            // `RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT` may be enabled.
+            global_internode_metrics().record_body_digest_fallback();
+            Ok(())
+        }
+    }
+}
+
 fn has_v2_auth_headers(headers: &HeaderMap) -> bool {
     [
         RPC_AUTH_VERSION_HEADER,
@@ -1410,6 +1473,129 @@ mod tests {
         assert!(cache.nonces.contains(&nonce_b));
     }
 
+    // The `rpc_body_digest_fallback_counter` serial group covers every test that drives (or
+    // asserts on) the process-global body-digest fallback counter, so exact-delta assertions
+    // cannot race with each other.
+    #[test]
+    #[serial_test::serial(rpc_body_digest_fallback_counter)]
+    fn digestless_mutation_is_accepted_and_counted_while_strict_gate_is_off() {
+        let request = tonic::Request::new(());
+        let before = global_internode_metrics().snapshot().body_digest_fallback_total;
+
+        assert!(
+            verify_tonic_mutation_body_digest_with_strictness(&request, b"canonical-mutation-body", false).is_ok(),
+            "a digestless peer must keep mutating while the strict gate is off"
+        );
+
+        let after = global_internode_metrics().snapshot().body_digest_fallback_total;
+        assert_eq!(
+            after,
+            before + 1,
+            "an accepted digestless mutation must increment the body-digest fallback counter exactly once"
+        );
+    }
+
+    #[test]
+    #[serial_test::serial(rpc_body_digest_fallback_counter)]
+    fn strict_mutation_gate_rejects_digestless_but_keeps_body_bound() {
+        let before = global_internode_metrics().snapshot().body_digest_fallback_total;
+
+        let digestless = tonic::Request::new(());
+        let error = verify_tonic_mutation_body_digest_with_strictness(&digestless, b"body", true)
+            .expect_err("strict mode must reject a mutation without a body digest");
+        assert_eq!(error.to_string(), "RPC mutation requires a body-bound v2 signature");
+
+        let mut unsigned = tonic::Request::new(());
+        unsigned
+            .metadata_mut()
+            .as_mut()
+            .insert(RPC_CONTENT_SHA256_HEADER, HeaderValue::from_static(UNSIGNED_PAYLOAD));
+        let error = verify_tonic_mutation_body_digest_with_strictness(&unsigned, b"body", true)
+            .expect_err("strict mode must reject an explicitly unsigned mutation payload");
+        assert_eq!(error.to_string(), "RPC mutation requires a body-bound v2 signature");
+
+        let mut bound = tonic::Request::new(());
+        set_tonic_canonical_body_digest(&mut bound, b"body").expect("digest metadata should encode");
+        bound
+            .metadata_mut()
+            .as_mut()
+            .insert(RPC_AUTH_VERSION_HEADER, HeaderValue::from_static(RPC_AUTH_VERSION_V2));
+        assert!(
+            verify_tonic_mutation_body_digest_with_strictness(&bound, b"body", true).is_ok(),
+            "strict mode must keep accepting body-bound mutations"
+        );
+        let tampered = verify_tonic_mutation_body_digest_with_strictness(&bound, b"tampered-body", true)
+            .expect_err("a tampered canonical body must fail even in strict mode");
+        assert_eq!(tampered.to_string(), "RPC content SHA-256 mismatch");
+
+        let after = global_internode_metrics().snapshot().body_digest_fallback_total;
+        assert_eq!(
+            after, before,
+            "neither strict rejections nor bound verifications are digestless fallbacks"
+        );
+    }
+
+    #[test]
+    #[serial_test::serial(rpc_body_digest_fallback_counter)]
+    fn mutation_digest_default_posture_is_fail_open_digestless_accept() {
+        // The public entry point resolves strictness from the environment, whose compile-time
+        // default is pinned to false in `rustfs_config`. A digestless peer therefore keeps
+        // mutating through the default build with no configuration at all.
+        let request = tonic::Request::new(());
+        assert!(
+            verify_tonic_mutation_body_digest_with_strictness(
+                &request,
+                b"canonical-mutation-body",
+                rustfs_config::DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT,
+            )
+            .is_ok(),
+            "the default strict posture must accept digestless mutations"
+        );
+    }
+
+    #[test]
+    fn rename_data_mutation_contract_binds_method_nonce_and_body() {
+        ensure_test_rpc_secret();
+        let message = rustfs_protos::proto_gen::node_service::RenameDataRequest {
+            disk: "http://node-a:9000/data/rustfs0".to_string(),
+            src_volume: ".rustfs.sys/multipart".to_string(),
+            src_path: "uploads/object".to_string(),
+            file_info: "{\"volume\":\"bucket\"}".to_string(),
+            dst_volume: "bucket".to_string(),
+            dst_path: "object".to_string(),
+            file_info_bin: vec![0x81, 0xA1, 0x76, 0x01].into(),
+        };
+        let body = rustfs_protos::canonical_rename_data_request_body(&message).expect("small request should encode");
+        let mut request = tonic::Request::new(());
+        set_tonic_canonical_body_digest(&mut request, &body).expect("canonical body digest should be attached");
+        let content_sha256 = request
+            .metadata()
+            .get(RPC_CONTENT_SHA256_HEADER)
+            .and_then(|value| value.to_str().ok());
+        let headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "RenameData", content_sha256)
+            .expect("body-bound auth headers should build");
+        request.metadata_mut().as_mut().extend(headers.clone());
+
+        assert!(
+            verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/RenameData", &headers).is_ok(),
+            "the rename_data signature must bind destination, method, nonce, and body digest"
+        );
+        let replay = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/RenameData", &headers)
+            .expect_err("reusing a consumed rename_data nonce must fail");
+        assert_eq!(replay.to_string(), "RPC request replay detected");
+        let transplant = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/DeleteVersion", &headers)
+            .expect_err("a rename_data signature must not authenticate a different method");
+        assert_eq!(transplant.to_string(), "Invalid RPC v2 signature");
+
+        assert!(verify_tonic_mutation_body_digest(&request, &body).is_ok());
+        let mut tampered = message;
+        tampered.file_info_bin = Vec::new().into();
+        let tampered_body = rustfs_protos::canonical_rename_data_request_body(&tampered).expect("small request should encode");
+        let stripped = verify_tonic_mutation_body_digest(&request, &tampered_body)
+            .expect_err("stripping the msgpack payload to force the JSON fallback decode must fail");
+        assert_eq!(stripped.to_string(), "RPC content SHA-256 mismatch");
+    }
+
     #[test]
     fn nonce_cache_rejects_replay_after_wall_clock_regression() {
         let now = Instant::now();

crates/ecstore/src/cluster/rpc/mod.rs

@@ -31,7 +31,8 @@ pub use client::{
 pub use http_auth::{
     TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience,
     set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_ns_scanner_capability,
-    verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
+    verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
+    verify_tonic_rpc_signature,
 };
 #[cfg(test)]
 pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -16,6 +16,7 @@ use crate::cluster::rpc::client::{
     TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
     node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
 };
+use crate::cluster::rpc::http_auth::set_tonic_canonical_body_digest;
 use crate::cluster::rpc::internode_data_transport::{
     InternodeDataTransport, NsScannerCapabilityRequest, NsScannerStreamRequest, ReadStreamRequest, WalkDirStreamRequest,
     WriteStreamRequest,
@@ -99,6 +100,18 @@ const LOG_SUBSYSTEM_REMOTE_DISK: &str = "remote_disk";
 const EVENT_REMOTE_DISK_HEALTH: &str = "remote_disk_health";
 const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc";
 
+/// Bind a mutating disk RPC to its canonical body: the digest lands in the request metadata, and
+/// the signing interceptor folds it (plus a replay-protected nonce) into the v2 signature scope
+/// (backlog#1327).
+fn attach_mutation_body_digest<T>(
+    request: &mut Request<T>,
+    canonical_body: std::result::Result<Vec<u8>, std::num::TryFromIntError>,
+    op: &'static str,
+) -> Result<()> {
+    let canonical_body = canonical_body.map_err(|_| Error::other(format!("{op} request length cannot be represented")))?;
+    set_tonic_canonical_body_digest(request, &canonical_body).map_err(Error::other)
+}
+
 fn decode_volume_infos(volume_infos: Vec<String>) -> Result<Vec<VolumeInfo>> {
     volume_infos
         .into_iter()
@@ -1339,10 +1352,12 @@ impl DiskAPI for RemoteDisk {
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
-                let request = Request::new(MakeVolumeRequest {
+                let mut request = Request::new(MakeVolumeRequest {
                     disk: self.endpoint.to_string(),
                     volume: volume.to_string(),
                 });
+                let canonical_body = rustfs_protos::canonical_make_volume_request_body(request.get_ref());
+                attach_mutation_body_digest(&mut request, canonical_body, "make_volume")?;
 
                 let response = client.make_volume(request).await?.into_inner();
 
@@ -1376,10 +1391,12 @@ impl DiskAPI for RemoteDisk {
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
-                let request = Request::new(MakeVolumesRequest {
+                let mut request = Request::new(MakeVolumesRequest {
                     disk: self.endpoint.to_string(),
                     volumes: volumes.iter().map(|s| (*s).to_string()).collect(),
                 });
+                let canonical_body = rustfs_protos::canonical_make_volumes_request_body(request.get_ref());
+                attach_mutation_body_digest(&mut request, canonical_body, "make_volumes")?;
 
                 let response = client.make_volumes(request).await?.into_inner();
 
@@ -1489,11 +1506,13 @@ impl DiskAPI for RemoteDisk {
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
-                let request = Request::new(DeleteVolumeRequest {
+                let mut request = Request::new(DeleteVolumeRequest {
                     disk: self.endpoint.to_string(),
                     volume: volume.to_string(),
                     force: force_delete,
                 });
+                let canonical_body = rustfs_protos::canonical_delete_volume_request_body(request.get_ref());
+                attach_mutation_body_digest(&mut request, canonical_body, "delete_volume")?;
 
                 let response = client.delete_volume(request).await?.into_inner();
 
@@ -1542,7 +1561,7 @@ impl DiskAPI for RemoteDisk {
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
-                let request = Request::new(DeleteVersionRequest {
+                let mut request = Request::new(DeleteVersionRequest {
                     disk: self.endpoint.to_string(),
                     volume: volume.to_string(),
                     path: path.to_string(),
@@ -1552,6 +1571,8 @@ impl DiskAPI for RemoteDisk {
                     file_info_bin: file_info_bin.into(),
                     opts_bin: opts_bin.into(),
                 });
+                let canonical_body = rustfs_protos::canonical_delete_version_request_body(request.get_ref());
+                attach_mutation_body_digest(&mut request, canonical_body, "delete_version")?;
 
                 let response = client.delete_version(request).await?.into_inner();
 
@@ -1643,7 +1664,7 @@ impl DiskAPI for RemoteDisk {
             }
         };
 
-        let request = Request::new(DeleteVersionsRequest {
+        let mut request = Request::new(DeleteVersionsRequest {
             disk: self.endpoint.to_string(),
             volume: volume.to_string(),
             versions: versions_str,
@@ -1651,6 +1672,14 @@ impl DiskAPI for RemoteDisk {
             versions_bin,
             opts_bin: opts_bin.into(),
         });
+        let canonical_body = rustfs_protos::canonical_delete_versions_request_body(request.get_ref());
+        if let Err(err) = attach_mutation_body_digest(&mut request, canonical_body, "delete_versions") {
+            let mut errors = Vec::with_capacity(versions.len());
+            for _ in 0..versions.len() {
+                errors.push(Some(err.clone()));
+            }
+            return errors;
+        }
 
         // TODO: use Error not string
 
@@ -1719,11 +1748,13 @@ impl DiskAPI for RemoteDisk {
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
-                let request = Request::new(DeletePathsRequest {
+                let mut request = Request::new(DeletePathsRequest {
                     disk: self.endpoint.to_string(),
                     volume: volume.to_string(),
                     paths: paths.clone(),
                 });
+                let canonical_body = rustfs_protos::canonical_delete_paths_request_body(request.get_ref());
+                attach_mutation_body_digest(&mut request, canonical_body, "delete_paths")?;
 
                 let response = client.delete_paths(request).await?.into_inner();
 
@@ -1762,13 +1793,15 @@ impl DiskAPI for RemoteDisk {
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
-                let request = Request::new(WriteMetadataRequest {
+                let mut request = Request::new(WriteMetadataRequest {
                     disk,
                     volume: volume.to_string(),
                     path: path.to_string(),
                     file_info,
                     file_info_bin: file_info_bin.into(),
                 });
+                let canonical_body = rustfs_protos::canonical_write_metadata_request_body(request.get_ref());
+                attach_mutation_body_digest(&mut request, canonical_body, "write_metadata")?;
 
                 let response = client.write_metadata(request).await?.into_inner();
 
@@ -1840,7 +1873,7 @@ impl DiskAPI for RemoteDisk {
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
-                let request = Request::new(UpdateMetadataRequest {
+                let mut request = Request::new(UpdateMetadataRequest {
                     disk,
                     volume: volume.to_string(),
                     path: path.to_string(),
@@ -1849,6 +1882,8 @@ impl DiskAPI for RemoteDisk {
                     file_info_bin: file_info_bin.into(),
                     opts_bin: opts_bin.into(),
                 });
+                let canonical_body = rustfs_protos::canonical_update_metadata_request_body(request.get_ref());
+                attach_mutation_body_digest(&mut request, canonical_body, "update_metadata")?;
 
                 let response = client.update_metadata(request).await?.into_inner();
 
@@ -2096,7 +2131,7 @@ impl DiskAPI for RemoteDisk {
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
-                let request = Request::new(RenameDataRequest {
+                let mut request = Request::new(RenameDataRequest {
                     disk: self.endpoint.to_string(),
                     src_volume: src_volume.to_string(),
                     src_path: src_path.to_string(),
@@ -2105,6 +2140,8 @@ impl DiskAPI for RemoteDisk {
                     dst_path: dst_path.to_string(),
                     file_info_bin: file_info_bin.into(),
                 });
+                let canonical_body = rustfs_protos::canonical_rename_data_request_body(request.get_ref());
+                attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?;
 
                 let response = client.rename_data(request).await?.into_inner();
 
@@ -2361,13 +2398,15 @@ impl DiskAPI for RemoteDisk {
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
-                let request = Request::new(RenameFileRequest {
+                let mut request = Request::new(RenameFileRequest {
                     disk: self.endpoint.to_string(),
                     src_volume: src_volume.to_string(),
                     src_path: src_path.to_string(),
                     dst_volume: dst_volume.to_string(),
                     dst_path: dst_path.to_string(),
                 });
+                let canonical_body = rustfs_protos::canonical_rename_file_request_body(request.get_ref());
+                attach_mutation_body_digest(&mut request, canonical_body, "rename_file")?;
 
                 let response = client.rename_file(request).await?.into_inner();
 
@@ -2404,7 +2443,7 @@ impl DiskAPI for RemoteDisk {
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
-                let request = Request::new(RenamePartRequest {
+                let mut request = Request::new(RenamePartRequest {
                     disk: self.endpoint.to_string(),
                     src_volume: src_volume.to_string(),
                     src_path: src_path.to_string(),
@@ -2412,6 +2451,8 @@ impl DiskAPI for RemoteDisk {
                     dst_path: dst_path.to_string(),
                     meta,
                 });
+                let canonical_body = rustfs_protos::canonical_rename_part_request_body(request.get_ref());
+                attach_mutation_body_digest(&mut request, canonical_body, "rename_part")?;
 
                 let response = client.rename_part(request).await?.into_inner();
 
@@ -2449,12 +2490,14 @@ impl DiskAPI for RemoteDisk {
                     .get_client()
                     .await
                     .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
-                let request = Request::new(DeleteRequest {
+                let mut request = Request::new(DeleteRequest {
                     disk: self.endpoint.to_string(),
                     volume: volume.to_string(),
                     path: path.to_string(),
                     options,
                 });
+                let canonical_body = rustfs_protos::canonical_delete_request_body(request.get_ref());
+                attach_mutation_body_digest(&mut request, canonical_body, "delete")?;
 
                 let response = client.delete(request).await?.into_inner();
 
@@ -2665,12 +2708,14 @@ impl DiskAPI for RemoteDisk {
                     crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_error();
                     Error::other(format!("can not get client, err: {err}"))
                 })?;
-                let request = Request::new(WriteAllRequest {
+                let mut request = Request::new(WriteAllRequest {
                     disk,
                     volume: volume.to_string(),
                     path: path.to_string(),
                     data,
                 });
+                let canonical_body = rustfs_protos::canonical_write_all_request_body(request.get_ref());
+                attach_mutation_body_digest(&mut request, canonical_body, "write_all")?;
 
                 crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_request();
                 let response = match client.write_all(request).await {