fix(sse): validate copy source customer algorithm

# Conflicts:
#	rustfs/src/storage/sse.rs

.github/workflows/ci.yml

@@ -372,7 +372,7 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build debug binary
-        run: cargo build -p rustfs --bins
+        run: cargo build -p rustfs --bins --features e2e-test-hooks
 
       - name: Upload debug binary
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
@@ -402,7 +402,7 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build debug binary with rio-v2
-        run: cargo build -p rustfs --bins --features rio-v2
+        run: cargo build -p rustfs --bins --features rio-v2,e2e-test-hooks
 
       - name: Upload debug binary
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6

Cargo.lock

@@ -3671,12 +3671,13 @@ dependencies = [
  "flatbuffers",
  "flate2",
  "futures",
+ "hex",
  "http 1.4.2",
  "http-body-util",
  "hyper",
  "hyper-util",
  "local-ip-address",
- "md5",
+ "md-5 0.11.0",
  "opentelemetry-proto",
  "prost 0.14.4",
  "rand 0.10.2",
@@ -8890,7 +8891,7 @@ dependencies = [
  "libmimalloc-sys",
  "libsystemd",
  "matchit 0.9.2",
- "md5",
+ "md-5 0.11.0",
  "metrics",
  "metrics-util",
  "mimalloc",
@@ -9105,7 +9106,6 @@ dependencies = [
 name = "rustfs-ecstore"
 version = "1.0.0-beta.11"
 dependencies = [
- "aes-gcm",
  "arc-swap",
  "async-channel",
  "async-recursion",
@@ -9121,7 +9121,6 @@ dependencies = [
  "byteorder",
  "bytes",
  "bytesize",
- "chacha20poly1305",
  "chrono",
  "criterion",
  "enumset",
@@ -9174,7 +9173,6 @@ dependencies = [
  "rustfs-erasure-codec",
  "rustfs-filemeta",
  "rustfs-io-metrics",
- "rustfs-kms",
  "rustfs-lifecycle",
  "rustfs-lock",
  "rustfs-madmin",
@@ -9449,9 +9447,10 @@ dependencies = [
  "async-trait",
  "base64 0.23.0",
  "chacha20poly1305",
+ "hex",
  "insta",
  "jiff",
- "md5",
+ "md-5 0.11.0",
  "moka",
  "rand 0.10.2",
  "reqwest",
@@ -9460,6 +9459,7 @@ dependencies = [
  "serde",
  "serde_json",
  "sha2 0.11.0",
+ "subtle",
  "temp-env",
  "tempfile",
  "thiserror 2.0.19",
@@ -9717,7 +9717,7 @@ dependencies = [
  "hyper-util",
  "ipnetwork",
  "libunftp",
- "md5",
+ "md-5 0.11.0",
  "percent-encoding",
  "proptest",
  "quick-xml",
@@ -11829,6 +11829,7 @@ dependencies = [
  "futures-util",
  "libc",
  "pin-project-lite",
+ "slab",
  "tokio",
 ]
 

Cargo.toml

@@ -265,7 +265,6 @@ memmap2 = "0.9.11"
 lz4 = "1.28.1"
 matchit = "0.9.2"
 md-5 = "0.11.0"
-md5 = "0.8.1"
 mime_guess = "2.0.5"
 moka = { version = "0.12.15" }
 netif = "0.1.6"

crates/concurrency/Cargo.toml

@@ -10,6 +10,9 @@ description = "Shared concurrency contract types for RustFS - workload admission
 keywords = ["rustfs", "concurrency", "admission", "backpressure", "workers"]
 categories = ["concurrency", "filesystem"]
 
+[lints]
+workspace = true
+
 [dependencies]
 # Internal crates
 rustfs-io-core = { workspace = true }

crates/config/src/constants/object.rs

@@ -116,6 +116,27 @@ pub const ENV_OBJECT_GET_SKIP_BITROT_VERIFY: &str = "RUSTFS_OBJECT_GET_SKIP_BITR
 /// Default: bitrot verification is enabled on GetObject reads (do not skip).
 pub const DEFAULT_OBJECT_GET_SKIP_BITROT_VERIFY: bool = false;
 
+/// Request writing the complete remote-tier version state into object metadata.
+///
+/// This remains ineffective until
+/// [`ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED`] is also enabled.
+pub const ENV_TIER_REMOTE_VERSION_STATE_WRITE: &str = "RUSTFS_TIER_REMOTE_VERSION_STATE_WRITE";
+pub const DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE: bool = false;
+
+/// Operator-attested fleet-wide confirmation for
+/// [`ENV_TIER_REMOTE_VERSION_STATE_WRITE`].
+///
+/// This flag is an operational contract, not automatic capability discovery.
+/// Operators may enable it only after every node that can write or read
+/// transitioned object metadata supports the remote version-state schema and
+/// semantics. Keeping the confirmation separate makes a single-node request or
+/// a writer whose local opt-in is removed fail closed.
+pub const ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: &str = "RUSTFS_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED";
+pub const DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: bool = false;
+
+const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE);
+const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED);
+
 // =============================================================================
 // Concurrent Request Fix - Timeout and Backpressure Configuration
 // =============================================================================
@@ -617,3 +638,15 @@ pub const ENV_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY: &str = "RUSTFS_OBJ
 
 /// Default read-ahead disable concurrency threshold: 4.
 pub const DEFAULT_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY: usize = 4;
+
+#[cfg(test)]
+mod remote_version_state_tests {
+    #[test]
+    fn remote_version_state_gate_uses_stable_environment_names() {
+        assert_eq!(super::ENV_TIER_REMOTE_VERSION_STATE_WRITE, "RUSTFS_TIER_REMOTE_VERSION_STATE_WRITE");
+        assert_eq!(
+            super::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
+            "RUSTFS_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED"
+        );
+    }
+}

crates/data-usage/src/data_usage.rs

@@ -506,8 +506,35 @@ pub struct ReplicationStats {
 }
 
 impl ReplicationStats {
+    pub fn is_empty(&self) -> bool {
+        let Self {
+            pending_size,
+            replicated_size,
+            failed_size,
+            failed_count,
+            pending_count,
+            missed_threshold_size,
+            after_threshold_size,
+            missed_threshold_count,
+            after_threshold_count,
+            replicated_count,
+        } = self;
+
+        *pending_size == 0
+            && *replicated_size == 0
+            && *failed_size == 0
+            && *failed_count == 0
+            && *pending_count == 0
+            && *missed_threshold_size == 0
+            && *after_threshold_size == 0
+            && *missed_threshold_count == 0
+            && *after_threshold_count == 0
+            && *replicated_count == 0
+    }
+
+    #[deprecated(note = "use is_empty instead")]
     pub fn empty(&self) -> bool {
-        self.replicated_size == 0 && self.failed_size == 0 && self.failed_count == 0
+        self.is_empty()
     }
 }
 
@@ -520,16 +547,19 @@ pub struct ReplicationAllStats {
 }
 
 impl ReplicationAllStats {
+    pub fn is_empty(&self) -> bool {
+        let Self {
+            replica_size,
+            replica_count,
+            targets,
+        } = self;
+
+        *replica_size == 0 && *replica_count == 0 && targets.values().all(ReplicationStats::is_empty)
+    }
+
+    #[deprecated(note = "use is_empty instead")]
     pub fn empty(&self) -> bool {
-        if self.replica_size != 0 && self.replica_count != 0 {
-            return false;
-        }
-        for v in self.targets.values() {
-            if !v.empty() {
-                return false;
-            }
-        }
-        true
+        self.is_empty()
     }
 }
 
@@ -783,7 +813,7 @@ impl DataUsageCache {
                     return Some(root);
                 }
                 let mut flat = self.flatten(&root);
-                if flat.replication_stats.as_ref().is_some_and(|stats| stats.empty()) {
+                if flat.replication_stats.as_ref().is_some_and(ReplicationAllStats::is_empty) {
                     flat.replication_stats = None;
                 }
                 Some(flat)
@@ -1582,6 +1612,126 @@ mod tests {
         assert_eq!(map["BETWEEN_1024B_AND_1_MB"], u64::MAX);
     }
 
+    #[test]
+    fn replication_stats_empty_checks_every_field() {
+        type SetField = fn(&mut ReplicationStats);
+
+        let cases: [(&str, SetField); 10] = [
+            ("pending_size", |stats| stats.pending_size = 1),
+            ("replicated_size", |stats| stats.replicated_size = 1),
+            ("failed_size", |stats| stats.failed_size = 1),
+            ("failed_count", |stats| stats.failed_count = 1),
+            ("pending_count", |stats| stats.pending_count = 1),
+            ("missed_threshold_size", |stats| stats.missed_threshold_size = 1),
+            ("after_threshold_size", |stats| stats.after_threshold_size = 1),
+            ("missed_threshold_count", |stats| stats.missed_threshold_count = 1),
+            ("after_threshold_count", |stats| stats.after_threshold_count = 1),
+            ("replicated_count", |stats| stats.replicated_count = 1),
+        ];
+
+        assert!(ReplicationStats::default().is_empty());
+        for (field, set_nonzero) in cases {
+            let mut stats = ReplicationStats::default();
+            set_nonzero(&mut stats);
+            assert!(!stats.is_empty(), "{field} must make replication stats non-empty");
+        }
+    }
+
+    #[test]
+    fn replication_all_stats_empty_checks_aggregate_fields_independently() {
+        let cases = [
+            (
+                "replica_size",
+                ReplicationAllStats {
+                    replica_size: 1,
+                    ..Default::default()
+                },
+            ),
+            (
+                "replica_count",
+                ReplicationAllStats {
+                    replica_count: 1,
+                    ..Default::default()
+                },
+            ),
+        ];
+
+        assert!(ReplicationAllStats::default().is_empty());
+        for (field, stats) in cases {
+            assert!(!stats.is_empty(), "{field} must make aggregate replication stats non-empty");
+        }
+
+        let empty_targets = ReplicationAllStats {
+            targets: HashMap::from([("arn:test:empty".to_string(), ReplicationStats::default())]),
+            ..Default::default()
+        };
+        assert!(empty_targets.is_empty(), "all-empty targets must keep aggregate stats empty");
+
+        let stats = ReplicationAllStats {
+            targets: HashMap::from([
+                ("arn:test:empty".to_string(), ReplicationStats::default()),
+                (
+                    "arn:test:non-empty".to_string(),
+                    ReplicationStats {
+                        pending_count: 1,
+                        ..Default::default()
+                    },
+                ),
+            ]),
+            ..Default::default()
+        };
+        assert!(!stats.is_empty(), "a non-empty target must make aggregate replication stats non-empty");
+    }
+
+    #[test]
+    fn size_recursive_prunes_empty_and_preserves_pending_replication_stats() {
+        let root = hash_path("bucket");
+        let child = hash_path("bucket/child");
+        let mut cache = DataUsageCache::default();
+        cache.replace_hashed(&root, &None, &DataUsageEntry::default());
+        cache.replace_hashed(
+            &child,
+            &Some(root.clone()),
+            &DataUsageEntry {
+                replication_stats: Some(ReplicationAllStats::default()),
+                ..Default::default()
+            },
+        );
+
+        assert!(
+            cache
+                .size_recursive("bucket")
+                .expect("bucket usage should flatten")
+                .replication_stats
+                .is_none()
+        );
+
+        cache.replace_hashed(
+            &child,
+            &Some(root.clone()),
+            &DataUsageEntry {
+                replication_stats: Some(ReplicationAllStats {
+                    targets: HashMap::from([(
+                        "arn:test:pending".to_string(),
+                        ReplicationStats {
+                            pending_count: 1,
+                            ..Default::default()
+                        },
+                    )]),
+                    ..Default::default()
+                }),
+                ..Default::default()
+            },
+        );
+
+        let flattened = cache.size_recursive("bucket").expect("bucket usage should flatten");
+        let replication = flattened
+            .replication_stats
+            .expect("pending-only replication stats must survive pruning");
+
+        assert_eq!(replication.targets["arn:test:pending"].pending_count, 1);
+    }
+
     #[test]
     fn test_data_usage_cache_merge_adds_missing_child() {
         let mut base = DataUsageCache::default();

crates/e2e_test/Cargo.toml

@@ -70,7 +70,8 @@ walkdir.workspace = true
 base64 = { workspace = true }
 rand = { workspace = true, features = ["serde"] }
 chrono = { workspace = true, features = ["serde"] }
-md5 = { workspace = true }
+hex = { workspace = true }
+md-5 = { workspace = true }
 opentelemetry-proto = { workspace = true }
 prost.workspace = true
 sha2 = { workspace = true }

crates/e2e_test/src/checksum_upload_test.rs

@@ -24,9 +24,10 @@ mod tests {
     use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart};
     use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
     use base64::Engine;
+    use md5::{Digest as Md5Digest, Md5};
     use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
     use serial_test::serial;
-    use sha2::{Digest, Sha256};
+    use sha2::Sha256;
     use tracing::info;
 
     fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
@@ -70,7 +71,9 @@ mod tests {
     }
 
     fn content_md5_base64(body: &[u8]) -> String {
-        let digest = md5::compute(body);
+        let mut hasher = Md5::new();
+        hasher.update(body);
+        let digest = hasher.finalize();
         base64::engine::general_purpose::STANDARD.encode(digest.as_slice())
     }
 

crates/e2e_test/src/fake_s3_target/mod.rs

@@ -25,6 +25,7 @@ use hyper::body::Incoming;
 use hyper::server::conn::http1;
 use hyper::service::service_fn;
 use hyper_util::rt::{TokioIo, TokioTimer};
+use md5::{Digest as Md5Digest, Md5};
 use s3s::access::{S3Access, S3AccessContext};
 use s3s::auth::SimpleAuth;
 use s3s::dto::{
@@ -827,13 +828,25 @@ fn ensure_body_growth(current: usize, added: usize) -> S3Result {
 
 async fn md5_digest(body: Bytes, permit: OwnedSemaphorePermit) -> S3Result<([u8; 16], OwnedSemaphorePermit)> {
     if body.len() < 1024 * 1024 {
-        return Ok((md5::compute(body).0, permit));
+        return Ok((md5_bytes(body), permit));
     }
-    tokio::task::spawn_blocking(move || (md5::compute(body).0, permit))
+    tokio::task::spawn_blocking(move || (md5_bytes(body), permit))
         .await
         .map_err(|error| s3s::s3_error!(InternalError, "MD5 worker failed: {error}"))
 }
 
+fn md5_bytes(input: impl AsRef<[u8]>) -> [u8; 16] {
+    let mut hasher = Md5::new();
+    hasher.update(input.as_ref());
+    hasher.finalize().into()
+}
+
+fn md5_hex(input: impl AsRef<[u8]>) -> String {
+    let mut hasher = Md5::new();
+    hasher.update(input.as_ref());
+    hex::encode(hasher.finalize())
+}
+
 fn ensure_store_budget(state: &StoreState, removed_bytes: usize, added_bytes: usize, adds_version: bool) -> S3Result {
     let total_bytes = state
         .total_bytes
@@ -1005,7 +1018,7 @@ impl S3 for FakeBackend {
             Some(value) => value,
             None => {
                 let (digest, _body_permit) = md5_digest(body.clone(), _body_permit).await?;
-                format!("{:x}", md5::Digest(digest))
+                hex::encode(digest)
             }
         };
         let version = ObjectVersion {
@@ -1208,7 +1221,7 @@ impl S3 for FakeBackend {
         }
         let body = collect_stream(input.body, input.content_length, fault.as_ref(), &self.control).await?;
         let (digest, _body_permit) = md5_digest(body.clone(), _body_permit).await?;
-        let e_tag = format!("{:x}", md5::Digest(digest));
+        let e_tag = hex::encode(digest);
         let mut state = lock(&self.store);
         let existing_bytes = state
             .uploads
@@ -1336,7 +1349,7 @@ impl S3 for FakeBackend {
             .collect();
         let (body, digests, _body_permits) = assemble_multipart(assembly_parts, total_len, _body_permits).await?;
         let part_count = requested.len();
-        let e_tag = source_etag(&headers)?.unwrap_or_else(|| format!("{:x}-{part_count}", md5::compute(digests)));
+        let e_tag = source_etag(&headers)?.unwrap_or_else(|| format!("{}-{part_count}", md5_hex(digests)));
         let version = ObjectVersion {
             version_id: upload.version_id.clone(),
             body,

crates/e2e_test/src/kms/common.rs

@@ -30,6 +30,7 @@ use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::ServerSideEncryption;
 use base64::{Engine, engine::general_purpose::STANDARD as BASE64};
 use http::header::{CONTENT_TYPE, HOST};
+use md5::{Digest as Md5Digest, Md5};
 use rustfs_signer::constants::UNSIGNED_PAYLOAD;
 use rustfs_signer::sign_v4;
 use s3s::Body;
@@ -68,7 +69,9 @@ pub fn skip_if_kms_admin_tool_unavailable(test_name: &str) -> bool {
 }
 
 pub fn sse_customer_key_md5_base64(key: &str) -> String {
-    BASE64.encode(md5::compute(key).0)
+    let mut hasher = Md5::new();
+    hasher.update(key.as_bytes());
+    BASE64.encode(hasher.finalize())
 }
 
 pub async fn kms_admin_request(

crates/e2e_test/src/kms/kms_edge_cases_test.rs

@@ -25,12 +25,18 @@ use super::common::{LocalKMSTestEnvironment, sse_customer_key_md5_base64};
 use crate::common::{TEST_BUCKET, init_logging};
 use aws_sdk_s3::types::ServerSideEncryption;
 use base64::Engine;
-use md5::compute;
+use md5::{Digest as Md5Digest, Md5};
 use serial_test::serial;
 use std::sync::Arc;
 use tokio::sync::Semaphore;
 use tracing::{info, warn};
 
+fn md5_hex(input: impl AsRef<[u8]>) -> String {
+    let mut hasher = Md5::new();
+    hasher.update(input.as_ref());
+    hex::encode(hasher.finalize())
+}
+
 /// Test encryption of zero-byte files (empty files)
 #[tokio::test]
 #[serial]
@@ -294,7 +300,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
     info!("🔍 Testing invalid SSE-C key length");
     let invalid_short_key = "short"; // Too short
     let invalid_key_b64 = base64::engine::general_purpose::STANDARD.encode(invalid_short_key);
-    let invalid_key_md5 = format!("{:x}", compute(invalid_short_key));
+    let invalid_key_md5 = md5_hex(invalid_short_key);
 
     let invalid_key_result = s3_client
         .put_object()

crates/e2e_test/src/multipart_auth_test.rs

@@ -26,6 +26,7 @@ use chrono::{Duration as ChronoDuration, Utc};
 use flate2::{Compression, write::GzEncoder};
 use http::HeaderValue;
 use http::header::{CONTENT_TYPE, HOST};
+use md5::{Digest as Md5Digest, Md5};
 use rustfs_signer::constants::UNSIGNED_PAYLOAD;
 use rustfs_signer::sign_v4;
 use s3s::Body;
@@ -50,7 +51,15 @@ fn encode_post_policy(conditions: Vec<serde_json::Value>) -> String {
 }
 
 fn sse_customer_key_md5_base64(key: &str) -> String {
-    base64::engine::general_purpose::STANDARD.encode(md5::compute(key).0)
+    let mut hasher = Md5::new();
+    hasher.update(key.as_bytes());
+    base64::engine::general_purpose::STANDARD.encode(hasher.finalize())
+}
+
+fn md5_hex(input: impl AsRef<[u8]>) -> String {
+    let mut hasher = Md5::new();
+    hasher.update(input.as_ref());
+    hex::encode(hasher.finalize())
 }
 
 /// Env var consumed by the local SSE-S3 DEK provider when KMS is not configured.
@@ -5664,7 +5673,7 @@ async fn test_signed_put_object_extract_returns_archive_etag() -> Result<(), Box
     client.create_bucket().bucket(bucket).send().await?;
 
     let archive = make_tar(&[("alpha.txt", b"alpha-body")], &[]).await;
-    let expected_etag = format!("\"{:x}\"", md5::compute(&archive));
+    let expected_etag = format!("\"{}\"", md5_hex(&archive));
 
     let response = client
         .put_object()

crates/e2e_test/src/reliant/grpc_lock_server.rs

@@ -23,7 +23,8 @@ use rustfs_protos::{
     proto_gen::node_service::{
         BatchGenerallyLockRequest, BatchGenerallyLockResponse, BatchReadVersionRequest, BatchReadVersionResponse,
         GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult, PingRequest, PingResponse,
-        node_service_server::NodeService,
+        SnapshotLeaseMutationResponse, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
+        SnapshotLeaseResponse, node_service_server::NodeService,
     },
 };
 use std::pin::Pin;
@@ -104,6 +105,27 @@ impl NodeService for MinimalLockNodeService {
         Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
     }
 
+    async fn acquire_snapshot_lease(
+        &self,
+        _request: Request<SnapshotLeaseRequest>,
+    ) -> Result<Response<SnapshotLeaseResponse>, Status> {
+        Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
+    }
+
+    async fn renew_snapshot_lease(
+        &self,
+        _request: Request<SnapshotLeaseRenewRequest>,
+    ) -> Result<Response<SnapshotLeaseResponse>, Status> {
+        Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
+    }
+
+    async fn release_snapshot_lease(
+        &self,
+        _request: Request<SnapshotLeaseReleaseRequest>,
+    ) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
+        Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
+    }
+
     async fn lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
         let request = request.into_inner();
         let args: LockRequest = match serde_json::from_str(&request.args) {

crates/e2e_test/src/reliant/tiering.rs

@@ -95,6 +95,7 @@ const MANUAL_ASYNC_PARALLEL_OBJECTS: usize = 64;
 const MANUAL_ACTIVE_CANCEL_OBJECTS: usize = 512;
 const MANUAL_RESTART_CANCEL_OBJECTS: usize = 512;
 const MANUAL_ACTIVE_CANCEL_RUNNING_TIMEOUT: StdDuration = StdDuration::from_secs(15);
+const MANUAL_TRANSITION_CANCEL_BARRIER_ENV: &str = "RUSTFS_E2E_MANUAL_TRANSITION_CANCEL_BARRIER";
 const MANUAL_ASYNC_CONFLICT_TERMINAL_TIMEOUT: StdDuration = StdDuration::from_secs(90);
 const MANUAL_RESTART_RECOVERY_TIMEOUT: StdDuration = StdDuration::from_secs(80);
 const OBJECT_KEY: &str = "tier/鲁A12345/report.bin";
@@ -1684,8 +1685,15 @@ async fn test_manual_transition_async_active_cancel_reports_terminal_cancelled()
     cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
 
     let mut hot = RustFSTestEnvironment::new().await?;
-    hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
-        .await?;
+    hot.start_rustfs_server_with_env(
+        vec![],
+        &[
+            ("RUSTFS_SCANNER_ENABLED", "false"),
+            ("RUSTFS_SCANNER_CYCLE", "3600"),
+            (MANUAL_TRANSITION_CANCEL_BARRIER_ENV, "1"),
+        ],
+    )
+    .await?;
     let hot_client = hot.create_s3_client();
     add_rustfs_tier(&hot, &cold).await?;
 

crates/e2e_test/src/ssec_copy_test.rs

@@ -22,6 +22,7 @@ use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
 use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
 use base64::Engine;
+use md5::{Digest as Md5Digest, Md5};
 use std::collections::HashMap;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::{Arc, Mutex};
@@ -102,10 +103,12 @@ impl Intercept for ResponseHeaderCapture {
 
 fn customer_key(byte: u8) -> CustomerKey {
     let raw = [byte; 32];
+    let mut hasher = Md5::new();
+    hasher.update(raw);
     CustomerKey {
         raw: String::from_utf8_lossy(&raw).into_owned(),
         encoded: base64::engine::general_purpose::STANDARD.encode(raw),
-        md5: base64::engine::general_purpose::STANDARD.encode(md5::compute(raw).0),
+        md5: base64::engine::general_purpose::STANDARD.encode(hasher.finalize()),
     }
 }
 

crates/ecstore/Cargo.toml

@@ -57,7 +57,6 @@ rustfs-policy.workspace = true
 rustfs-protos.workspace = true
 rustfs-replication.workspace = true
 rustfs-lifecycle.workspace = true
-rustfs-kms.workspace = true
 rustfs-s3-types = { workspace = true }
 rustfs-data-usage.workspace = true
 rustfs-object-capacity.workspace = true
@@ -124,8 +123,6 @@ libc.workspace = true
 rustix = { workspace = true, features = ["process", "fs"] }
 rustfs-madmin.workspace = true
 reqwest = { workspace = true }
-aes-gcm = { workspace = true, features = ["rand_core"] }
-chacha20poly1305.workspace = true
 aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 urlencoding = { workspace = true }
 smallvec = { workspace = true, features = ["serde"] }

crates/ecstore/src/api/mod.rs

@@ -127,8 +127,8 @@ pub mod bucket {
             get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
             get_object_lock_config, get_public_access_block_config, get_quota_config, get_replication_config,
             get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
-            init_bucket_metadata_sys, list_bucket_targets, remove_bucket_metadata, set_bucket_metadata, update,
-            update_bucket_targets_under_transaction_lock, update_config_with,
+            init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata,
+            update, update_bucket_targets_under_transaction_lock, update_config_with,
         };
     }
 
@@ -305,7 +305,8 @@ pub mod disk {
         CheckPartsResp, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore,
         FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
         PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
-        STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk, validate_batch_read_version_item_count,
+        STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk,
+        validate_batch_read_version_item_count,
     };
     pub use bytes::Bytes;
     pub use endpoint::Endpoint;
@@ -381,10 +382,12 @@ pub mod notification {
 
 pub mod object {
     pub use crate::object_api::{
-        BLOCK_SIZE_V2, ERASURE_ALGORITHM, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource,
-        GetObjectReader, ObjectInfo, ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader, StreamConsumer,
-        get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
-        register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
+        BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,
+        GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, ObjectEncryptionResolver, ObjectInfo,
+        ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode,
+        ReadEncryptionRequest, StreamConsumer, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook,
+        register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook,
+        unregister_object_mutation_hook,
     };
     pub use crate::store::PreparedGetObjectReader;
 }

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -8974,7 +8974,7 @@ mod tests {
         .await
         .expect("first worker result should persist");
         assert_eq!(first.state, ManualTransitionJobState::Running);
-        assert_eq!(first.report.transition_completed, 1);
+        assert_eq!(first.report.transition_completed, 0);
         assert_eq!(first.report.transition_failed, 0);
 
         let duplicate = record_manual_transition_worker_result(
@@ -8987,11 +8987,11 @@ mod tests {
         .await
         .expect("duplicate worker result should be idempotent");
         assert_eq!(duplicate.state, ManualTransitionJobState::Running);
-        assert_eq!(duplicate.report.transition_completed, 1);
+        assert_eq!(duplicate.report.transition_completed, 0);
         assert_eq!(duplicate.report.transition_failed, 0);
 
         let second_key = manual_transition_worker_result_task_key(&bucket, "logs/b", None);
-        let final_record = record_manual_transition_worker_result(
+        let pending_record = record_manual_transition_worker_result(
             ecstore.clone(),
             job_id,
             &second_key,
@@ -9000,7 +9000,14 @@ mod tests {
         )
         .await
         .expect("second distinct worker result should persist");
+        assert_eq!(pending_record.state, ManualTransitionJobState::Running);
+        assert_eq!(pending_record.report.transition_completed, 0);
+        assert_eq!(pending_record.report.transition_failed, 0);
 
+        let final_record =
+            reconcile_manual_transition_worker_results(ecstore.clone(), job_id, ManualTransitionQueueSnapshot::default())
+                .await
+                .expect("worker result journal should reconcile");
         assert_eq!(final_record.state, ManualTransitionJobState::Partial);
         assert_eq!(final_record.report.transition_completed, 1);
         assert_eq!(final_record.report.transition_failed, 1);
@@ -9035,7 +9042,7 @@ mod tests {
             .expect("worker result job record should save");
 
         let task_key = manual_transition_worker_result_task_key(&bucket, "logs/fail", None);
-        let final_record = record_manual_transition_worker_result_with_reason(
+        let pending_record = record_manual_transition_worker_result_with_reason(
             ecstore.clone(),
             job_id,
             &task_key,
@@ -9045,7 +9052,12 @@ mod tests {
         )
         .await
         .expect("worker result with failure reason should persist");
+        assert!(pending_record.report.tier_failure_by_reason.is_empty());
+        assert_eq!(pending_record.report.transition_failed, 0);
 
+        let final_record = reconcile_manual_transition_worker_results(ecstore, job_id, ManualTransitionQueueSnapshot::default())
+            .await
+            .expect("worker failure reason should reconcile");
         assert_eq!(
             final_record
                 .report
@@ -11306,23 +11318,25 @@ mod tests {
 
         // Distinct payloads with distinct sizes: a mixed-generation reassembly
         // would produce bytes matching none of them (or fail the read outright).
-        let candidates: Vec<Vec<u8>> = (0..3)
+        let candidates: Vec<Vec<u8>> = (0..2)
             .map(|g| {
                 let len = 4096 + g * 512;
                 vec![b'a' + g as u8; len]
             })
             .collect();
 
-        let commit_barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
-        let start = Arc::new(tokio::sync::Barrier::new(candidates.len() + 1));
+        let commit_barrier = MultipartCommitBarrier::install_for_arrivals(
+            &bucket,
+            object,
+            MultipartCommitPause::PutPartBeforeLockAcquire,
+            candidates.len(),
+        );
         let mut tasks = tokio::task::JoinSet::new();
         for payload in candidates.iter().cloned() {
             let store = ecstore.clone();
             let bucket = bucket.clone();
             let upload_id = upload.upload_id.clone();
-            let start = Arc::clone(&start);
             tasks.spawn(async move {
-                start.wait().await;
                 let mut data = PutObjReader::from_vec(payload.clone());
                 store
                     .put_object_part(&bucket, object, &upload_id, 1, &mut data, &ObjectOptions::default())
@@ -11330,11 +11344,10 @@ mod tests {
                     .map(|info| (info, payload))
             });
         }
-        start.wait().await;
 
-        // The first writer holds the uploadId commit lock while the other
-        // resends reach the same critical section. Releasing it proves the
-        // handoff without depending on saturated CI disk latency.
+        // Both writers finish streaming before racing for the uploadId commit
+        // lock. Two generations are sufficient to exercise the mixed-shard
+        // hazard, while each waiter sits behind at most one cross-disk rename.
         commit_barrier.wait_until_paused().await;
         commit_barrier.release();
 

crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs

@@ -1646,40 +1646,14 @@ pub async fn record_manual_transition_worker_result_with_reason(
     job_id: Uuid,
     task_key: &str,
     result: ManualTransitionWorkerResult,
-    queue_snapshot: ManualTransitionQueueSnapshot,
+    _queue_snapshot: ManualTransitionQueueSnapshot,
     failure_reason: Option<ManualTransitionWorkerFailureReason>,
 ) -> EcstoreResult<ManualTransitionJobRecord> {
     let result_record = ManualTransitionWorkerResultRecord::new_with_reason(job_id, task_key, result, failure_reason);
     if !save_manual_transition_worker_result_if_absent(api.clone(), &result_record).await? {
         return load_manual_transition_job_record(api, job_id).await;
     }
-
-    for _ in 0..4 {
-        let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
-        if record.is_terminal() {
-            return Ok(record);
-        }
-        record.record_worker_result_with_reason(result, queue_snapshot, failure_reason);
-        match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
-            Ok(()) => {
-                if record.is_terminal() {
-                    delete_manual_transition_scope_admission_if_current(
-                        api.clone(),
-                        &record.scope_key,
-                        record.job_id,
-                        record.lease_id,
-                    )
-                    .await?;
-                } else {
-                    renew_manual_transition_scope_admission_from_job(api, &record).await?;
-                }
-                return Ok(record);
-            }
-            Err(Error::PreconditionFailed) => continue,
-            Err(err) => return Err(err),
-        }
-    }
-    Err(Error::PreconditionFailed)
+    load_manual_transition_job_record(api, job_id).await
 }
 
 pub async fn renew_manual_transition_job_lease(

crates/ecstore/src/bucket/metadata_sys.rs

@@ -23,7 +23,7 @@ use crate::error::{Error, Result, is_err_bucket_not_found};
 use crate::runtime::sources as runtime_sources;
 use crate::storage_api_contracts::heal::HealOperations as _;
 use crate::storage_api_contracts::namespace::NamespaceLocking as _;
-use crate::store::ECStore;
+use crate::store::{ECStore, await_bucket_namespace_operation};
 use futures::future::join_all;
 use rustfs_common::heal_channel::HealOpts;
 use rustfs_policy::policy::BucketPolicy;
@@ -37,13 +37,19 @@ use std::collections::HashSet;
 use std::time::Duration;
 use std::{collections::HashMap, sync::Arc};
 use time::OffsetDateTime;
-use tokio::sync::RwLock;
+use tokio::sync::{Mutex, RwLock};
 use tokio::time::sleep;
 use tokio_util::sync::CancellationToken;
 use tracing::{error, warn};
 
 const BUCKET_METADATA_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60);
 
+#[derive(Clone, Copy)]
+enum MetadataLoadMode {
+    Initial,
+    Refresh,
+}
+
 pub async fn init_bucket_metadata_sys(api: Arc<ECStore>, buckets: Vec<String>) {
     // The metadata system is inherently per-store (it holds the store handle
     // and that store's bucket cache), so it lives on the store's own instance
@@ -85,6 +91,20 @@ pub async fn set_bucket_metadata(bucket: String, bm: BucketMetadata) -> Result<(
     Ok(())
 }
 
+/// Peer LoadBucketMetadata entry point; see
+/// [`BucketMetadataSys::reload_from_store`] for the caching contract.
+///
+/// The outer write guard spans the disk load, mirroring [`update`]: every
+/// other cache installer holds this lock (read or write), so the snapshot
+/// read here can never land after — and roll back — a newer concurrent
+/// install, and the install-plus-registry-sync sequence stays atomic
+/// against concurrent removes and reloads.
+pub async fn reload_bucket_metadata(bucket: &str) -> Result<()> {
+    let sys = get_bucket_metadata_sys()?;
+    let lock = sys.write().await;
+    lock.reload_from_store(bucket).await
+}
+
 /// Drop a bucket's cached metadata from the in-memory map.
 ///
 /// This is the counterpart to [`set_bucket_metadata`] and is invoked when a
@@ -140,7 +160,8 @@ async fn refresh_buckets_metadata_once(sys: Arc<RwLock<BucketMetadataSys>>) {
 
     for chunk in buckets.chunks(count) {
         let sys = sys.read().await;
-        sys.concurrent_load(chunk, &mut failed_buckets).await;
+        sys.concurrent_load(chunk, &mut failed_buckets, MetadataLoadMode::Refresh)
+            .await;
     }
 
     if !failed_buckets.is_empty() {
@@ -461,6 +482,9 @@ const ABSENT_BUCKET_METADATA_MAX_ENTRIES: u64 = 10_000;
 #[derive(Debug)]
 pub struct BucketMetadataSys {
     metadata_map: RwLock<HashMap<String, Arc<BucketMetadata>>>,
+    metadata_publish_lock: Mutex<()>,
+    #[cfg(test)]
+    lazy_load_lock_probe: std::sync::atomic::AtomicBool,
     /// Buckets recently observed to have no persisted metadata. Serving the
     /// fabricated default from here (instead of re-reading disk) keeps the
     /// per-request cost of repeated lookups for such names bounded — without
@@ -476,6 +500,9 @@ impl BucketMetadataSys {
     pub fn new(api: Arc<ECStore>) -> Self {
         Self {
             metadata_map: RwLock::new(HashMap::new()),
+            metadata_publish_lock: Mutex::new(()),
+            #[cfg(test)]
+            lazy_load_lock_probe: std::sync::atomic::AtomicBool::new(false),
             absent_metadata: moka::future::Cache::builder()
                 .max_capacity(ABSENT_BUCKET_METADATA_MAX_ENTRIES)
                 .time_to_live(ABSENT_BUCKET_METADATA_TTL)
@@ -502,11 +529,13 @@ impl BucketMetadataSys {
 
         loop {
             if buckets.len() < count {
-                self.concurrent_load(buckets, &mut failed_buckets).await;
+                self.concurrent_load(buckets, &mut failed_buckets, MetadataLoadMode::Initial)
+                    .await;
                 break;
             }
 
-            self.concurrent_load(&buckets[..count], &mut failed_buckets).await;
+            self.concurrent_load(&buckets[..count], &mut failed_buckets, MetadataLoadMode::Initial)
+                .await;
 
             buckets = &buckets[count..]
         }
@@ -517,7 +546,7 @@ impl BucketMetadataSys {
         Ok(())
     }
 
-    async fn concurrent_load(&self, buckets: &[String], failed_buckets: &mut HashSet<String>) {
+    async fn concurrent_load(&self, buckets: &[String], failed_buckets: &mut HashSet<String>, mode: MetadataLoadMode) {
         let mut futures = Vec::new();
 
         for bucket in buckets.iter() {
@@ -525,16 +554,55 @@ impl BucketMetadataSys {
             let bucket = bucket.clone();
             futures.push(async move {
                 sleep(Duration::from_millis(30)).await;
-                let _ = api
-                    .heal_bucket(
-                        &bucket,
-                        &HealOpts {
-                            recreate: true,
-                            ..Default::default()
-                        },
-                    )
-                    .await;
-                load_bucket_metadata_parse_with_presence(self.api.clone(), bucket.as_str(), true).await
+                match mode {
+                    MetadataLoadMode::Initial => {
+                        let _ = api
+                            .heal_bucket(
+                                &bucket,
+                                &HealOpts {
+                                    recreate: true,
+                                    ..Default::default()
+                                },
+                            )
+                            .await;
+                        let (bm, persisted) =
+                            load_bucket_metadata_parse_with_presence(self.api.clone(), bucket.as_str(), true).await?;
+                        if persisted {
+                            self.set(bucket, Arc::new(bm)).await;
+                        } else {
+                            let _publish_guard = self.metadata_publish_lock.lock().await;
+                            let mut map = self.metadata_map.write().await;
+                            map.entry(bucket).or_insert_with(|| Arc::new(bm));
+                        }
+                    }
+                    MetadataLoadMode::Refresh => {
+                        let expected = self.metadata_map.read().await.get(&bucket).cloned();
+                        let heal_lock = api.new_ns_lock(&bucket, &bucket).await?;
+                        let heal_guard = heal_lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
+                        await_bucket_namespace_operation(
+                            Some(&heal_guard),
+                            &bucket,
+                            "bucket metadata refresh heal",
+                            api.heal_bucket(&bucket, &HealOpts::default()),
+                        )
+                        .await?;
+                        drop(heal_guard);
+                        let (bm, persisted) =
+                            load_bucket_metadata_parse_with_presence(self.api.clone(), bucket.as_str(), true).await?;
+                        let publish_lock = api.new_ns_lock(&bucket, &bucket).await?;
+                        let guard = publish_lock
+                            .get_read_lock(crate::set_disk::get_lock_acquire_timeout())
+                            .await?;
+                        if guard.is_lock_lost() {
+                            return Err(Error::other(format!(
+                                "bucket namespace lock was lost before bucket metadata refresh publish: {bucket}"
+                            )));
+                        }
+                        self.publish_refresh_if_unchanged(&bucket, expected.as_ref(), bm, persisted)
+                            .await;
+                    }
+                }
+                Ok::<(), Error>(())
             });
         }
 
@@ -542,26 +610,7 @@ impl BucketMetadataSys {
 
         for (idx, res) in results.into_iter().enumerate() {
             match res {
-                Ok((bm, persisted)) => {
-                    if let Some(bucket) = buckets.get(idx) {
-                        if persisted {
-                            self.set(bucket.clone(), Arc::new(bm)).await;
-                        } else {
-                            // A fabricated default (no persisted metadata
-                            // readable right now) must never REPLACE an
-                            // existing entry: the periodic refresh would
-                            // otherwise downgrade a lock-enabled bucket to an
-                            // authoritative "no lock" default on a transient
-                            // ConfigNotFound, disabling the object-lock
-                            // delete gate and wiping its target/durability
-                            // sync state. Insert-if-vacant keeps the startup
-                            // behavior for legacy buckets without a metadata
-                            // file, atomically under the map write lock.
-                            let mut map = self.metadata_map.write().await;
-                            map.entry(bucket.clone()).or_insert_with(|| Arc::new(bm));
-                        }
-                    }
-                }
+                Ok(()) => {}
                 Err(e) => {
                     error!("Unable to load bucket metadata, will be retried: {:?}", e);
                     if let Some(bucket) = buckets.get(idx) {
@@ -572,6 +621,32 @@ impl BucketMetadataSys {
         }
     }
 
+    async fn publish_refresh_if_unchanged(
+        &self,
+        bucket: &str,
+        expected: Option<&Arc<BucketMetadata>>,
+        metadata: BucketMetadata,
+        persisted: bool,
+    ) {
+        if !persisted {
+            return;
+        }
+        let _publish_guard = self.metadata_publish_lock.lock().await;
+        let metadata = Arc::new(metadata);
+        let mut map = self.metadata_map.write().await;
+        let unchanged = expected
+            .zip(map.get(bucket))
+            .is_some_and(|(expected, current)| Arc::ptr_eq(expected, current));
+        if !unchanged {
+            return;
+        }
+        map.insert(bucket.to_string(), Arc::clone(&metadata));
+        drop(map);
+        self.absent_metadata.invalidate(bucket).await;
+        sync_bucket_target_sys(bucket, &metadata).await;
+        sync_bucket_durability(bucket, &metadata);
+    }
+
     pub async fn get(&self, bucket: &str) -> Result<Arc<BucketMetadata>> {
         if is_meta_bucketname(bucket) {
             return Err(Error::ConfigNotFound);
@@ -587,6 +662,7 @@ impl BucketMetadataSys {
 
     pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
         if !is_meta_bucketname(&bucket) {
+            let _publish_guard = self.metadata_publish_lock.lock().await;
             let mut map = self.metadata_map.write().await;
             map.insert(bucket.clone(), bm.clone());
             drop(map);
@@ -597,6 +673,43 @@ impl BucketMetadataSys {
         }
     }
 
+    /// Reload `bucket`'s metadata from this system's own store and cache it,
+    /// refusing to treat a load miss as authoritative (the peer
+    /// LoadBucketMetadata notification path, [`reload_bucket_metadata`]).
+    ///
+    /// Only metadata actually read from persisted storage reaches the cache.
+    /// On a miss the fabricated default is discarded and an error is
+    /// returned: installing it would let a transient ConfigNotFound during
+    /// the notification overwrite a lock-enabled bucket's cached metadata
+    /// with an authoritative "no Object Lock" default, disabling the
+    /// batch-delete retention gate (`object_lock_delete_check_required`) on
+    /// this node until the next refresh. A miss is also not treated as
+    /// deletion: bucket deletion propagates through the dedicated
+    /// DeleteBucketMetadata notification ([`remove_bucket_metadata`]), which
+    /// is best-effort — a reload racing it can still re-install a just
+    /// deleted bucket's entry (pre-existing, bounded by the next delete or
+    /// restart) — but a reload miss removing entries would turn every
+    /// transient quorum dip into dropped metadata and spurious
+    /// target/durability teardown.
+    ///
+    /// The peer-visible error text is deliberately fixed: the notifying peer
+    /// matches error strings against network-failure needles
+    /// (`is_network_like_error`), so interpolating a caller-controlled
+    /// bucket name here could mark a healthy peer offline.
+    ///
+    /// Lock order: the caller holds the outer metadata-sys guard, and the
+    /// load acquires the namespace lock on the bucket's metadata config
+    /// object — the same `outer guard → meta-config namespace lock` order
+    /// `update`'s load takes; no path acquires these in reverse.
+    pub(crate) async fn reload_from_store(&self, bucket: &str) -> Result<()> {
+        let (bm, persisted) = load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true).await?;
+        if !persisted {
+            return Err(Error::other("no persisted bucket metadata readable; peer cache left unchanged"));
+        }
+        self.set(bucket.to_string(), Arc::new(bm)).await;
+        Ok(())
+    }
+
     /// Remove a bucket's cached metadata from the in-memory map.
     ///
     /// Returns `true` if an entry was present. Reserved meta buckets are ignored.
@@ -604,6 +717,7 @@ impl BucketMetadataSys {
         if is_meta_bucketname(bucket) {
             return false;
         }
+        let _publish_guard = self.metadata_publish_lock.lock().await;
         let mut map = self.metadata_map.write().await;
         let removed = map.remove(bucket).is_some();
         drop(map);
@@ -735,7 +849,24 @@ impl BucketMetadataSys {
                 return Ok((Arc::new(bm), true));
             }
 
-            let (bm, persisted) = match load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true).await {
+            let lock = self.api.new_ns_lock(bucket, bucket).await?;
+            let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
+            #[cfg(test)]
+            if self.lazy_load_lock_probe.load(std::sync::atomic::Ordering::Relaxed) {
+                let competing = self.api.new_ns_lock(bucket, bucket).await?;
+                assert!(
+                    competing.get_write_lock(Duration::from_millis(20)).await.is_err(),
+                    "lazy metadata IO must start while the bucket namespace read lock is held"
+                );
+            }
+            let (bm, persisted) = match await_bucket_namespace_operation(
+                Some(&guard),
+                bucket,
+                "lazy bucket metadata load",
+                Box::pin(load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true)),
+            )
+            .await
+            {
                 Ok(res) => res,
                 Err(err) => {
                     return if *self.initialized.read().await {
@@ -759,9 +890,33 @@ impl BucketMetadataSys {
             // defaults for buckets listed on disk — legacy buckets without a
             // metadata file — but never lets one replace an existing entry.)
             if persisted {
+                await_bucket_namespace_operation(
+                    Some(&guard),
+                    bucket,
+                    "lazy bucket metadata existence check",
+                    Box::pin(async {
+                        self.api
+                            .peer_sys
+                            .get_bucket_info(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
+                            .await
+                            .map(|_| ())
+                            .map_err(Into::into)
+                    }),
+                )
+                .await?;
+                if guard.is_lock_lost() {
+                    return Err(Error::other(format!(
+                        "bucket namespace lock was lost before lazy bucket metadata publish: {bucket}"
+                    )));
+                }
+                let _publish_guard = self.metadata_publish_lock.lock().await;
                 let mut map = self.metadata_map.write().await;
+                if let Some(current) = map.get(bucket) {
+                    return Ok((Arc::clone(current), true));
+                }
                 map.insert(bucket.to_string(), bm.clone());
                 drop(map);
+                self.absent_metadata.invalidate(bucket).await;
                 sync_bucket_target_sys(bucket, &bm).await;
                 sync_bucket_durability(bucket, &bm);
             } else {
@@ -1047,12 +1202,13 @@ mod tests {
     /// Pins the fail-closed caching contract of the lazy `get_config` path
     /// and the refresh no-replace rule: fabricated defaults are returned but
     /// never served by the map-only `get()`, persisted metadata is cached on
-    /// lazy load (superseding a recorded absence), and a refresh-load miss
-    /// never replaces an existing entry.
+    /// lazy load (superseding a recorded absence), a refresh-load miss never
+    /// replaces an existing entry or heals a deleted bucket, and initial load
+    /// still heals buckets discovered from storage.
     #[tokio::test]
     async fn get_config_never_caches_fabricated_defaults_as_authoritative() {
-        let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
-        let sys = BucketMetadataSys::new(ecstore);
+        let (dirs, ecstore) = isolated_store_over_temp_disks().await;
+        let sys = Arc::new(BucketMetadataSys::new(ecstore));
 
         // (a) Miss: the fabricated default is returned but not cached.
         let (bm, _) = sys
@@ -1078,6 +1234,9 @@ mod tests {
         let mut persisted = BucketMetadata::new("absent-bucket");
         persisted.policy_config_json = b"persisted-marker".to_vec();
         sys.persist_and_set(persisted).await.expect("metadata should persist");
+        for dir in &dirs {
+            std::fs::create_dir_all(dir.path().join("absent-bucket")).expect("persisted bucket directory should be created");
+        }
         sys.metadata_map.write().await.clear();
         let _ = sys
             .get_config("absent-bucket")
@@ -1089,14 +1248,47 @@ mod tests {
             .expect("lazily loaded persisted metadata must be cached");
         assert_eq!(cached.policy_config_json, b"persisted-marker".to_vec());
 
-        // (c) A refresh-load miss (no persisted metadata readable) must not
-        // replace an existing entry.
+        // (c) Persisted metadata left behind after physical deletion must not
+        // be lazily republished as a live bucket generation.
+        let mut deleted_lazy = BucketMetadata::new("deleted-lazy-bucket");
+        deleted_lazy.policy_config_json = b"stale-generation".to_vec();
+        sys.persist_and_set(deleted_lazy)
+            .await
+            .expect("stale metadata should persist");
+        sys.metadata_map.write().await.remove("deleted-lazy-bucket");
+        assert!(
+            sys.get_config("deleted-lazy-bucket").await.is_err(),
+            "lazy load must fail when the physical bucket no longer exists"
+        );
+        assert!(sys.get("deleted-lazy-bucket").await.is_err());
+
+        // (d) The namespace generation fence must be acquired before lazy
+        // metadata IO, so a writer can replace the generation atomically.
+        let fenced_bucket = "fenced-lazy-bucket";
+        for dir in &dirs {
+            std::fs::create_dir_all(dir.path().join(fenced_bucket)).unwrap();
+        }
+        let mut old_fenced = BucketMetadata::new(fenced_bucket);
+        old_fenced.policy_config_json = b"old-fenced-generation".to_vec();
+        sys.persist_and_set(old_fenced).await.unwrap();
+        sys.metadata_map.write().await.remove(fenced_bucket);
+        sys.lazy_load_lock_probe.store(true, std::sync::atomic::Ordering::Relaxed);
+        let (loaded, _) = sys.get_config(fenced_bucket).await.unwrap();
+        sys.lazy_load_lock_probe.store(false, std::sync::atomic::Ordering::Relaxed);
+        assert_eq!(loaded.policy_config_json, b"old-fenced-generation".to_vec());
+
+        // (e) A refresh-load miss for a bucket that still exists must not
+        // replace an existing entry with a fabricated default.
         let mut kept = BucketMetadata::new("kept-bucket");
         kept.policy_config_json = b"kept-marker".to_vec();
         sys.set("kept-bucket".to_string(), Arc::new(kept)).await;
+        for dir in &dirs {
+            std::fs::create_dir_all(dir.path().join("kept-bucket")).expect("kept bucket directory should be created");
+        }
         let mut failed = HashSet::new();
         let refresh_targets = vec!["kept-bucket".to_string()];
-        sys.concurrent_load(&refresh_targets, &mut failed).await;
+        sys.concurrent_load(&refresh_targets, &mut failed, MetadataLoadMode::Refresh)
+            .await;
         let kept = sys
             .get("kept-bucket")
             .await
@@ -1106,6 +1298,50 @@ mod tests {
             b"kept-marker".to_vec(),
             "a fabricated refresh default must not replace real metadata"
         );
+
+        // (f) A stale cache entry for a physically deleted bucket must not
+        // recreate the bucket during periodic refresh.
+        sys.set("deleted-bucket".to_string(), Arc::new(BucketMetadata::new("deleted-bucket")))
+            .await;
+        let deleted_targets = vec!["deleted-bucket".to_string()];
+        sys.concurrent_load(&deleted_targets, &mut failed, MetadataLoadMode::Refresh)
+            .await;
+        assert!(
+            dirs.iter().all(|dir| !dir.path().join("deleted-bucket").exists()),
+            "periodic refresh must not recreate a bucket from stale cached metadata"
+        );
+
+        // (g) Metadata loaded for an old bucket generation must not replace
+        // metadata published by delete plus same-name recreation.
+        let old = Arc::new(BucketMetadata::new("recreated-bucket"));
+        sys.set("recreated-bucket".to_string(), Arc::clone(&old)).await;
+        let mut recreated = BucketMetadata::new("recreated-bucket");
+        recreated.policy_config_json = b"new-generation".to_vec();
+        sys.set("recreated-bucket".to_string(), Arc::new(recreated)).await;
+        let mut stale = BucketMetadata::new("recreated-bucket");
+        stale.policy_config_json = b"old-generation".to_vec();
+        sys.publish_refresh_if_unchanged("recreated-bucket", Some(&old), stale, true)
+            .await;
+        assert_eq!(sys.get("recreated-bucket").await.unwrap().policy_config_json, b"new-generation".to_vec());
+
+        // (f) Refresh retains periodic healing for a partially missing bucket.
+        sys.set("partial-bucket".to_string(), Arc::new(BucketMetadata::new("partial-bucket")))
+            .await;
+        for dir in dirs.iter().take(3) {
+            std::fs::create_dir_all(dir.path().join("partial-bucket")).unwrap();
+        }
+        sys.concurrent_load(&["partial-bucket".to_string()], &mut failed, MetadataLoadMode::Refresh)
+            .await;
+        assert!(dirs.iter().all(|dir| dir.path().join("partial-bucket").is_dir()));
+
+        // (g) Initial discovery retains the historical unconditional heal.
+        let initial_targets = vec!["initial-bucket".to_string()];
+        sys.concurrent_load(&initial_targets, &mut failed, MetadataLoadMode::Initial)
+            .await;
+        assert!(
+            dirs.iter().all(|dir| dir.path().join("initial-bucket").is_dir()),
+            "initial load must heal buckets discovered from storage"
+        );
     }
 
     #[tokio::test]
@@ -1130,12 +1366,18 @@ mod tests {
     #[tokio::test]
     async fn update_config_with_persists_tagging_rewrite_across_disk_reload() {
         use crate::bucket::metadata::BUCKET_TAGGING_CONFIG;
+        use crate::storage_api_contracts::bucket::MakeBucketOptions;
         use s3s::dto::Tag;
 
         let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
-        let mut sys = BucketMetadataSys::new(ecstore);
 
         let bucket = "swift-tagging-bucket";
+        ecstore
+            .peer_sys
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket volume should be created");
+        let mut sys = BucketMetadataSys::new(ecstore);
         sys.persist_and_set(BucketMetadata::new(bucket))
             .await
             .expect("initial metadata should persist");
@@ -1240,6 +1482,71 @@ mod tests {
         assert_eq!(tags.tag_set.len(), WRITERS, "every concurrent rewrite must survive: {tags:?}");
     }
 
+    /// Pins the peer reload-notification contract (`reload_from_store`, the
+    /// LoadBucketMetadata RPC path): only metadata actually read from
+    /// persisted storage enters the cache. A load miss errors out and leaves
+    /// the cache untouched — it must neither install a fabricated default
+    /// for an unknown bucket nor replace an existing entry, since a
+    /// transient ConfigNotFound during the notification would otherwise
+    /// downgrade a lock-enabled bucket to an authoritative "no Object Lock"
+    /// default and disable the batch-delete retention gate on this peer.
+    #[tokio::test]
+    async fn peer_reload_never_caches_fabricated_defaults_as_authoritative() {
+        let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
+        let sys = BucketMetadataSys::new(ecstore.clone());
+
+        // (a) Miss with no cached entry: the reload fails and installs nothing.
+        let err = sys
+            .reload_from_store("reload-bucket")
+            .await
+            .expect_err("a reload miss must be reported to the notifying peer");
+        assert!(
+            err.to_string().contains("no persisted bucket metadata readable"),
+            "the miss must surface through the dedicated non-persisted branch, got: {err}"
+        );
+        assert!(
+            sys.get("reload-bucket").await.is_err(),
+            "a reload miss must not install a fabricated default"
+        );
+
+        // (b) Miss with an existing entry: the reload fails and the entry
+        // (standing in for a lock-enabled bucket's metadata) survives intact.
+        let mut kept = BucketMetadata::new("reload-bucket");
+        kept.object_lock_config_xml = b"<ObjectLockConfiguration/>".to_vec();
+        sys.set("reload-bucket".to_string(), Arc::new(kept)).await;
+        assert!(sys.reload_from_store("reload-bucket").await.is_err());
+        let cached = sys
+            .get("reload-bucket")
+            .await
+            .expect("existing entry must survive a reload miss");
+        assert_eq!(
+            cached.object_lock_config_xml,
+            b"<ObjectLockConfiguration/>".to_vec(),
+            "a reload miss must not replace the cached entry with a fabricated default"
+        );
+
+        // (c) Persisted metadata reloads over a stale cached entry: the
+        // reload converges the cache to disk truth.
+        let mut persisted = BucketMetadata::new("reload-bucket");
+        persisted.policy_config_json = b"persisted-marker".to_vec();
+        sys.persist_and_set(persisted).await.expect("metadata should persist");
+        let mut stale = BucketMetadata::new("reload-bucket");
+        stale.policy_config_json = b"stale-cache-marker".to_vec();
+        sys.set("reload-bucket".to_string(), Arc::new(stale)).await;
+        sys.reload_from_store("reload-bucket")
+            .await
+            .expect("persisted metadata should reload");
+        let cached = sys
+            .get("reload-bucket")
+            .await
+            .expect("reloaded persisted metadata must be cached");
+        assert_eq!(
+            cached.policy_config_json,
+            b"persisted-marker".to_vec(),
+            "a reload must converge the cache to the persisted disk state"
+        );
+    }
+
     fn target(bucket: &str, id: &str) -> BucketTarget {
         BucketTarget {
             source_bucket: bucket.to_string(),

crates/ecstore/src/client/utils.rs

@@ -46,16 +46,6 @@ lazy_static! {
         m.insert("x-amz-replication-status".to_string(), true);
         m
     };
-    static ref SSE_HEADERS: HashMap<String, bool> = {
-        let mut m = HashMap::new();
-        m.insert("x-amz-server-side-encryption".to_string(), true);
-        m.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), true);
-        m.insert("x-amz-server-side-encryption-context".to_string(), true);
-        m.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), true);
-        m.insert("x-amz-server-side-encryption-customer-key".to_string(), true);
-        m.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), true);
-        m
-    };
 }
 
 pub fn is_standard_query_value(qs_key: &str) -> bool {
@@ -70,16 +60,12 @@ pub fn is_standard_header(header_key: &str) -> bool {
     *SUPPORTED_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
 }
 
-pub fn is_sse_header(header_key: &str) -> bool {
-    *SSE_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
-}
-
 pub fn is_amz_header(header_key: &str) -> bool {
     let key = header_key.to_lowercase();
     key.starts_with("x-amz-meta-")
         || key.starts_with("x-amz-grant-")
         || key == "x-amz-acl"
-        || is_sse_header(header_key)
+        || rustfs_utils::http::is_sse_header(header_key)
         || key.starts_with("x-amz-checksum-")
 }
 

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -443,6 +443,16 @@ pub fn set_tonic_canonical_body_digest<T>(request: &mut tonic::Request<T>, canon
     Ok(())
 }
 
+pub fn set_tonic_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(
+    request: &mut tonic::Request<T>,
+) -> std::io::Result<()> {
+    let canonical_body = request
+        .get_ref()
+        .canonical_body()
+        .map_err(|_| std::io::Error::other("RPC mutation body length cannot be represented"))?;
+    set_tonic_canonical_body_digest(request, &canonical_body)
+}
+
 pub fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
     let version = request
         .metadata()
@@ -466,7 +476,7 @@ pub fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canoni
     Ok(())
 }
 
-/// Verify a mutating disk RPC's canonical body digest with a rolling-upgrade fallback.
+/// Verify a mutating RPC's canonical body digest with a rolling-upgrade fallback.
 ///
 /// When the request carries a real (non-`UNSIGNED-PAYLOAD`) content SHA-256 it is verified exactly
 /// like [`verify_tonic_canonical_body_digest`]. The digest value is a member of the signed v2
@@ -497,7 +507,7 @@ fn verify_tonic_mutation_body_digest_with_strictness<T>(
         Some(digest) if digest != UNSIGNED_PAYLOAD => verify_tonic_canonical_body_digest(request, canonical_body),
         _ => {
             // RUSTFS_COMPAT_TODO(disk-mutation-body-digest): accept digestless peers during rolling upgrades. Remove after the
-            // minimum supported RustFS peer version body-binds every mutating disk RPC.
+            // minimum supported RustFS peer version body-binds every mutating RPC.
             if strict {
                 return Err(std::io::Error::other("RPC mutation requires a body-bound v2 signature"));
             }
@@ -677,11 +687,28 @@ mod tests {
     use crate::cluster::rpc::context_propagation::REQUEST_ID_HEADER;
     use crate::runtime::sources as runtime_sources;
     use http::{HeaderMap, Method};
+    use rustfs_protos::{
+        CanonicalMutationBody as _, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
+        proto_gen::node_service::{Mss, SignalServiceRequest},
+    };
+    use std::collections::HashMap;
     use std::io::{self, Write};
     use std::sync::{Arc, Mutex};
     use time::OffsetDateTime;
     use tracing_subscriber::fmt::MakeWriter;
 
+    fn signal_service_request(signal: &str, sub_system: &str, dry_run: &str) -> SignalServiceRequest {
+        SignalServiceRequest {
+            vars: Some(Mss {
+                value: HashMap::from([
+                    (PEER_RESTSIGNAL.to_string(), signal.to_string()),
+                    (PEER_RESTSUB_SYS.to_string(), sub_system.to_string()),
+                    (PEER_RESTDRY_RUN.to_string(), dry_run.to_string()),
+                ]),
+            }),
+        }
+    }
+
     #[derive(Clone, Default)]
     struct CapturedLogs {
         buffer: Arc<Mutex<Vec<u8>>>,
@@ -1596,6 +1623,72 @@ mod tests {
         assert_eq!(stripped.to_string(), "RPC content SHA-256 mismatch");
     }
 
+    #[test]
+    fn signal_service_mutation_contract_rejects_tampering_and_replay() {
+        ensure_test_rpc_secret();
+        let body = signal_service_request("2", "scanner", "false")
+            .canonical_body()
+            .expect("small signal request should encode");
+        let mut request = tonic::Request::new(());
+        set_tonic_canonical_body_digest(&mut request, &body).expect("canonical body digest should be attached");
+        let content_sha256 = request
+            .metadata()
+            .get(RPC_CONTENT_SHA256_HEADER)
+            .and_then(|value| value.to_str().ok());
+        let headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "SignalService", content_sha256)
+            .expect("body-bound auth headers should build");
+        request.metadata_mut().as_mut().extend(headers.clone());
+
+        assert!(
+            verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/SignalService", &headers).is_ok(),
+            "the first body-bound signal request must authenticate"
+        );
+        assert!(verify_tonic_mutation_body_digest(&request, &body).is_ok());
+
+        let tampered = signal_service_request("1", "scanner", "false")
+            .canonical_body()
+            .expect("small signal request should encode");
+        let error = verify_tonic_mutation_body_digest(&request, &tampered)
+            .expect_err("changing the signal must invalidate the signed digest");
+        assert_eq!(error.to_string(), "RPC content SHA-256 mismatch");
+
+        let replay = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/SignalService", &headers)
+            .expect_err("reusing the signal nonce must fail");
+        assert_eq!(replay.to_string(), "RPC request replay detected");
+    }
+
+    #[test]
+    #[serial_test::serial(rpc_body_digest_fallback_counter)]
+    fn signal_service_mutation_contract_preserves_rollout_fallback_and_strictness() {
+        let body = signal_service_request("2", "scanner", "false")
+            .canonical_body()
+            .expect("small signal request should encode");
+        let before = global_internode_metrics().snapshot().body_digest_fallback_total;
+        let digestless = tonic::Request::new(());
+
+        assert!(
+            verify_tonic_mutation_body_digest_with_strictness(&digestless, &body, false).is_ok(),
+            "old peers must remain compatible while the rollout gate is open"
+        );
+        assert_eq!(
+            global_internode_metrics().snapshot().body_digest_fallback_total,
+            before + 1,
+            "accepted digestless signal requests must be visible in the fallback metric"
+        );
+
+        let error = verify_tonic_mutation_body_digest_with_strictness(&digestless, &body, true)
+            .expect_err("strict mode must reject a digestless signal request");
+        assert_eq!(error.to_string(), "RPC mutation requires a body-bound v2 signature");
+
+        let mut bound = tonic::Request::new(());
+        set_tonic_canonical_body_digest(&mut bound, &body).expect("canonical body digest should be attached");
+        bound
+            .metadata_mut()
+            .as_mut()
+            .insert(RPC_AUTH_VERSION_HEADER, HeaderValue::from_static(RPC_AUTH_VERSION_V2));
+        assert!(verify_tonic_mutation_body_digest_with_strictness(&bound, &body, true).is_ok());
+    }
+
     #[test]
     fn nonce_cache_rejects_replay_after_wall_clock_regression() {
         let now = Instant::now();

crates/ecstore/src/cluster/rpc/mod.rs

@@ -30,9 +30,9 @@ pub use client::{
 };
 pub use http_auth::{
     TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience,
-    set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_ns_scanner_capability,
-    verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
-    verify_tonic_rpc_signature,
+    set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof,
+    verify_ns_scanner_capability, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
+    verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
 };
 #[cfg(test)]
 pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -16,7 +16,7 @@ use crate::cluster::rpc::client::{
     TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
     is_network_like_status, message_has_network_needle, node_service_time_out_client, tier_mutation_control_time_out_client,
 };
-use crate::cluster::rpc::{set_tonic_canonical_body_digest, verify_tonic_rpc_response_proof};
+use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, verify_tonic_rpc_response_proof};
 use crate::error::{Error, Result};
 use crate::storage_api_contracts::internode::{
     SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
@@ -50,6 +50,7 @@ use rustfs_protos::proto_gen::node_service::{
     TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
     tier_mutation_control_service_client::TierMutationControlServiceClient,
 };
+pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
 use rustfs_protos::{TierMutationRpcPhase, evict_failed_connection};
 use rustfs_utils::XHost;
 use serde::{Deserialize, Serialize as _};
@@ -69,9 +70,6 @@ use tonic::transport::Channel;
 use tracing::{debug, info, warn};
 use uuid::Uuid;
 
-pub const PEER_RESTSIGNAL: &str = "signal";
-pub const PEER_RESTSUB_SYS: &str = "sub-sys";
-pub const PEER_RESTDRY_RUN: &str = "dry-run";
 pub const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = 1;
 pub const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = 2;
 const BACKGROUND_HEAL_STATUS_MAX_MESSAGE_SIZE: usize = 64 * 1024;
@@ -1160,10 +1158,11 @@ impl PeerRestClient {
         self.finalize_result(
             async {
                 let mut client = self.get_client().await?;
-                let request = Request::new(LoadBucketMetadataRequest {
+                let mut request = Request::new(LoadBucketMetadataRequest {
                     bucket: bucket.to_string(),
                     scanner_maintenance_change,
                 });
+                set_tonic_mutation_body_digest(&mut request)?;
 
                 let response = client.load_bucket_metadata(request).await?.into_inner();
                 if !response.success {
@@ -1183,9 +1182,10 @@ impl PeerRestClient {
         self.finalize_result(
             async {
                 let mut client = self.get_client().await?;
-                let request = Request::new(DeleteBucketMetadataRequest {
+                let mut request = Request::new(DeleteBucketMetadataRequest {
                     bucket: bucket.to_string(),
                 });
+                set_tonic_mutation_body_digest(&mut request)?;
 
                 let response = client.delete_bucket_metadata(request).await?.into_inner();
                 if !response.success {
@@ -1205,9 +1205,10 @@ impl PeerRestClient {
         self.finalize_result(
             async {
                 let mut client = self.get_client().await?;
-                let request = Request::new(DeletePolicyRequest {
+                let mut request = Request::new(DeletePolicyRequest {
                     policy_name: policy.to_string(),
                 });
+                set_tonic_mutation_body_digest(&mut request)?;
 
                 let response = client.delete_policy(request).await?.into_inner();
                 if !response.success {
@@ -1227,9 +1228,10 @@ impl PeerRestClient {
         self.finalize_result(
             async {
                 let mut client = self.get_client().await?;
-                let request = Request::new(LoadPolicyRequest {
+                let mut request = Request::new(LoadPolicyRequest {
                     policy_name: policy.to_string(),
                 });
+                set_tonic_mutation_body_digest(&mut request)?;
 
                 let response = client.load_policy(request).await?.into_inner();
                 if !response.success {
@@ -1249,11 +1251,12 @@ impl PeerRestClient {
         self.finalize_result(
             async {
                 let mut client = self.get_client().await?;
-                let request = Request::new(LoadPolicyMappingRequest {
+                let mut request = Request::new(LoadPolicyMappingRequest {
                     user_or_group: user_or_group.to_string(),
                     user_type,
                     is_group,
                 });
+                set_tonic_mutation_body_digest(&mut request)?;
 
                 let response = client.load_policy_mapping(request).await?.into_inner();
                 if !response.success {
@@ -1273,9 +1276,10 @@ impl PeerRestClient {
         self.finalize_result(
             async {
                 let mut client = self.get_client().await?;
-                let request = Request::new(DeleteUserRequest {
+                let mut request = Request::new(DeleteUserRequest {
                     access_key: access_key.to_string(),
                 });
+                set_tonic_mutation_body_digest(&mut request)?;
 
                 let response = client.delete_user(request).await?.into_inner();
                 if !response.success {
@@ -1295,9 +1299,10 @@ impl PeerRestClient {
         self.finalize_result(
             async {
                 let mut client = self.get_client().await?;
-                let request = Request::new(DeleteServiceAccountRequest {
+                let mut request = Request::new(DeleteServiceAccountRequest {
                     access_key: access_key.to_string(),
                 });
+                set_tonic_mutation_body_digest(&mut request)?;
 
                 let response = client.delete_service_account(request).await?.into_inner();
                 if !response.success {
@@ -1317,10 +1322,11 @@ impl PeerRestClient {
         self.finalize_result(
             async {
                 let mut client = self.get_client().await?;
-                let request = Request::new(LoadUserRequest {
+                let mut request = Request::new(LoadUserRequest {
                     access_key: access_key.to_string(),
                     temp,
                 });
+                set_tonic_mutation_body_digest(&mut request)?;
 
                 let response = client.load_user(request).await?.into_inner();
                 if !response.success {
@@ -1340,9 +1346,10 @@ impl PeerRestClient {
         self.finalize_result(
             async {
                 let mut client = self.get_client().await?;
-                let request = Request::new(LoadServiceAccountRequest {
+                let mut request = Request::new(LoadServiceAccountRequest {
                     access_key: access_key.to_string(),
                 });
+                set_tonic_mutation_body_digest(&mut request)?;
 
                 let response = client.load_service_account(request).await?.into_inner();
                 if !response.success {
@@ -1362,9 +1369,10 @@ impl PeerRestClient {
         self.finalize_result(
             async {
                 let mut client = self.get_client().await?;
-                let request = Request::new(LoadGroupRequest {
+                let mut request = Request::new(LoadGroupRequest {
                     group: group.to_string(),
                 });
+                set_tonic_mutation_body_digest(&mut request)?;
 
                 let response = client.load_group(request).await?.into_inner();
                 if !response.success {
@@ -1384,7 +1392,8 @@ impl PeerRestClient {
         self.finalize_result(
             async {
                 let mut client = self.get_client().await?;
-                let request = Request::new(ReloadSiteReplicationConfigRequest {});
+                let mut request = Request::new(ReloadSiteReplicationConfigRequest {});
+                set_tonic_mutation_body_digest(&mut request)?;
 
                 let response = client.reload_site_replication_config(request).await?.into_inner();
                 if !response.success {
@@ -1408,9 +1417,10 @@ impl PeerRestClient {
                 vars.insert(PEER_RESTSIGNAL.to_string(), sig.to_string());
                 vars.insert(PEER_RESTSUB_SYS.to_string(), sub_sys.to_string());
                 vars.insert(PEER_RESTDRY_RUN.to_string(), dry_run.to_string());
-                let request = Request::new(SignalServiceRequest {
+                let mut request = Request::new(SignalServiceRequest {
                     vars: Some(Mss { value: vars }),
                 });
+                set_tonic_mutation_body_digest(&mut request)?;
 
                 let response = client.signal_service(request).await?.into_inner();
                 if !response.success {
@@ -1479,7 +1489,8 @@ impl PeerRestClient {
         self.finalize_result(
             async {
                 let mut client = self.get_client().await?;
-                let request = Request::new(ReloadPoolMetaRequest {});
+                let mut request = Request::new(ReloadPoolMetaRequest {});
+                set_tonic_mutation_body_digest(&mut request)?;
 
                 let response = client.reload_pool_meta(request).await?.into_inner();
                 if !response.success {
@@ -1500,9 +1511,10 @@ impl PeerRestClient {
         self.finalize_result(
             async {
                 let mut client = self.get_client().await?;
-                let request = Request::new(StopRebalanceRequest {
+                let mut request = Request::new(StopRebalanceRequest {
                     expected_rebalance_id: expected_rebalance_id.unwrap_or_default().to_string(),
                 });
+                set_tonic_mutation_body_digest(&mut request)?;
 
                 let response = client.stop_rebalance(request).await?.into_inner();
                 if !response.success {
@@ -1523,7 +1535,8 @@ impl PeerRestClient {
         self.finalize_result(
             async {
                 let mut client = self.get_client().await?;
-                let request = Request::new(LoadRebalanceMetaRequest { start_rebalance });
+                let mut request = Request::new(LoadRebalanceMetaRequest { start_rebalance });
+                set_tonic_mutation_body_digest(&mut request)?;
 
                 let response = client.load_rebalance_meta(request).await?.into_inner();
 
@@ -1562,7 +1575,8 @@ impl PeerRestClient {
                     })
                     .collect::<Result<Vec<_>>>()?;
                 let mut client = self.get_client().await?;
-                let request = Request::new(StartDecommissionRequest { pool_indices });
+                let mut request = Request::new(StartDecommissionRequest { pool_indices });
+                set_tonic_mutation_body_digest(&mut request)?;
 
                 let response = client.start_decommission(request).await?.into_inner();
                 if !response.success {
@@ -1585,7 +1599,8 @@ impl PeerRestClient {
                 let pool_index = u32::try_from(pool_index)
                     .map_err(|_| Error::other(format!("decommission pool index {pool_index} exceeds RPC range")))?;
                 let mut client = self.get_client().await?;
-                let request = Request::new(CancelDecommissionRequest { pool_index });
+                let mut request = Request::new(CancelDecommissionRequest { pool_index });
+                set_tonic_mutation_body_digest(&mut request)?;
 
                 let response = client.cancel_decommission(request).await?.into_inner();
                 if !response.success {
@@ -1608,7 +1623,8 @@ impl PeerRestClient {
                 let pool_index = u32::try_from(pool_index)
                     .map_err(|_| Error::other(format!("decommission pool index {pool_index} exceeds RPC range")))?;
                 let mut client = self.get_client().await?;
-                let request = Request::new(ClearDecommissionRequest { pool_index });
+                let mut request = Request::new(ClearDecommissionRequest { pool_index });
+                set_tonic_mutation_body_digest(&mut request)?;
 
                 let response = client.clear_decommission(request).await?.into_inner();
                 if !response.success {
@@ -1628,10 +1644,14 @@ impl PeerRestClient {
     pub async fn load_transition_tier_config(&self) -> Result<()> {
         match self.load_transition_tier_config_outcome().await {
             TierConfigReloadOutcome::Success => Ok(()),
-            TierConfigReloadOutcome::TransientReconnect(err) | TierConfigReloadOutcome::TransientRetrySameChannel(err) => {
-                self.finalize_result(Err(err)).await
-            }
-            TierConfigReloadOutcome::Terminal(err) => Err(err),
+            // Only a reconnect-class failure says anything about the channel.
+            // `finalize_result` marks the peer offline and evicts its connection
+            // whenever the message looks network-like, and a peer that answered
+            // and rejected the apply can easily report one ("release RPC failed:
+            // transport error"). Routing those through here would gate a healthy,
+            // responding peer out of every unrelated RPC.
+            TierConfigReloadOutcome::TransientReconnect(err) => self.finalize_result(Err(err)).await,
+            TierConfigReloadOutcome::TransientRetrySameChannel(err) | TierConfigReloadOutcome::Terminal(err) => Err(err),
         }
     }
 
@@ -1657,6 +1677,9 @@ impl PeerRestClient {
             Err(err) => return tier_config_reload_connection_outcome(err),
         };
         let mut request = Request::new(LoadTransitionTierConfigRequest {});
+        if let Err(err) = set_tonic_mutation_body_digest(&mut request) {
+            return TierConfigReloadOutcome::Terminal(Error::other(err));
+        }
         request.set_timeout(rustfs_protos::heal_control_execution_timeout());
 
         let response = match client.load_transition_tier_config(request).await {
@@ -1710,13 +1733,24 @@ fn is_tier_config_reload_connection_failure(err: &Error) -> bool {
     message_has_network_needle(&message)
 }
 
+/// Classifies a reload the peer answered but refused to apply.
+///
+/// The peer replied, so the channel is healthy and only the remote apply
+/// failed. Those failures are transient by nature: the reload reads the tier
+/// mutation intents and takes the distributed tier-config lock, both of which
+/// fail while any other node is restarting or while the lock quorum is briefly
+/// disturbed. Retiring the worker on the first such rejection leaves that peer
+/// pinned to the old configuration with nothing left to heal it, so it answers
+/// `TierNotFound` for a tier the rest of the cluster already committed until a
+/// second admin mutation happens to spawn a fresh worker.
+///
+/// Convergence is the whole point of this path, so a rejection is retried on
+/// the same channel. The worker's exponential backoff caps the cost at one
+/// reload every `TIER_CONFIG_RELOAD_RETRY_CAP`, and `Terminal` stays reachable
+/// for transport and gRPC status failures, which is where a genuinely
+/// unrecoverable peer surfaces.
 fn tier_config_reload_remote_failure(error_info: Option<String>) -> TierConfigReloadOutcome {
-    let error_info = error_info.unwrap_or_default();
-    if matches!(error_info.as_str(), "errServerNotInitialized" | "ServerNotInitialized") {
-        TierConfigReloadOutcome::TransientRetrySameChannel(Error::other(error_info))
-    } else {
-        TierConfigReloadOutcome::Terminal(Error::other(error_info))
-    }
+    TierConfigReloadOutcome::TransientRetrySameChannel(Error::other(error_info.unwrap_or_default()))
 }
 
 fn tier_config_reload_status_outcome(status: tonic::Status) -> TierConfigReloadOutcome {
@@ -1726,6 +1760,14 @@ fn tier_config_reload_status_outcome(status: tonic::Status) -> TierConfigReloadO
         TierConfigReloadOutcome::TransientReconnect(status.into())
     } else if status.code() == Code::Unknown && status.message().starts_with("Service was not ready:") {
         TierConfigReloadOutcome::TransientRetrySameChannel(status.into())
+    } else if status.code() == Code::Unknown
+        && is_tier_config_reload_connection_failure(&Error::other(status.message().to_string()))
+    {
+        // tonic reports a connection dropped mid-call as `Unknown` carrying the
+        // transport error text rather than as `Unavailable`, which is what a peer
+        // restarting under an active mutation produces. Reconnect and retry, so
+        // the restart does not permanently retire this peer's reload worker.
+        TierConfigReloadOutcome::TransientReconnect(status.into())
     } else {
         TierConfigReloadOutcome::Terminal(status.into())
     }
@@ -2279,9 +2321,12 @@ mod tests {
             tier_config_reload_status_outcome(tonic::Status::cancelled("request cancelled")),
             TierConfigReloadOutcome::Terminal(_)
         ));
+        // A peer that answered and then refused the apply is retried rather than
+        // retired: the channel is healthy, so the rejection reflects remote state
+        // that the next attempt can find healed.
         assert!(matches!(
             tier_config_reload_remote_failure(Some("backend unavailable".to_string())),
-            TierConfigReloadOutcome::Terminal(_)
+            TierConfigReloadOutcome::TransientRetrySameChannel(_)
         ));
         assert!(matches!(
             tier_config_reload_remote_failure(Some("errServerNotInitialized".to_string())),
@@ -2305,6 +2350,50 @@ mod tests {
         ));
     }
 
+    /// A tier mutation issued while another node restarts must still converge on
+    /// the nodes that stayed up. Those peers answer the reload RPC and reject the
+    /// apply, because reloading reads the tier mutation intents and takes the
+    /// distributed tier-config lock while the lock quorum is still disturbed.
+    /// Classifying those rejections as terminal retired the reload worker on its
+    /// first attempt and pinned the peer to the previous configuration, so it
+    /// served `TierNotFound` for an already-committed tier until an unrelated
+    /// second admin mutation spawned a new worker.
+    #[test]
+    fn tier_config_reload_retries_peers_that_reject_the_apply_mid_restart() {
+        for error_info in [
+            "Lock acquisition timeout for resource '.rustfs.sys/config/tier-config.bin.lock' after 5s",
+            "Resource '.rustfs.sys/config/tier-config.bin.lock' is already locked by node-3",
+            "Internal error: release RPC failed: transport error",
+            "save_config_with_opts: err: PreconditionFailed",
+            "erasure read quorum",
+        ] {
+            assert!(
+                matches!(
+                    tier_config_reload_remote_failure(Some(error_info.to_string())),
+                    TierConfigReloadOutcome::TransientRetrySameChannel(_)
+                ),
+                "a peer that rejected the apply must stay retryable so it converges: {error_info}"
+            );
+        }
+
+        // An absent error message is still a rejection, not a reason to stop.
+        assert!(matches!(
+            tier_config_reload_remote_failure(None),
+            TierConfigReloadOutcome::TransientRetrySameChannel(_)
+        ));
+
+        // tonic surfaces a connection dropped mid-call as `Unknown`, not `Unavailable`.
+        assert!(matches!(
+            tier_config_reload_status_outcome(tonic::Status::unknown("transport error")),
+            TierConfigReloadOutcome::TransientReconnect(_)
+        ));
+        // An `Unknown` that is not transport-shaped stays terminal.
+        assert!(matches!(
+            tier_config_reload_status_outcome(tonic::Status::unknown("peer response unknown")),
+            TierConfigReloadOutcome::Terminal(_)
+        ));
+    }
+
     #[tokio::test]
     async fn tier_config_reload_single_attempt_clears_offline_gate_without_redial() {
         let client = test_peer_client();

crates/ecstore/src/cluster/rpc/peer_s3_client.rs

@@ -16,6 +16,7 @@ use crate::bucket::metadata_sys;
 use crate::cluster::rpc::client::{
     TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
 };
+use crate::cluster::rpc::set_tonic_mutation_body_digest;
 use crate::disk::error::DiskError;
 use crate::disk::error::{Error, Result};
 use crate::disk::error_reduce::{BUCKET_OP_IGNORED_ERRS, is_all_buckets_not_found, reduce_write_quorum_errs};
@@ -89,6 +90,22 @@ fn reduce_pool_write_quorum_errs(per_pool_errs: &[Option<Error>]) -> Option<Erro
     reduce_write_quorum_errs(per_pool_errs, BUCKET_OP_IGNORED_ERRS, pool_write_quorum(per_pool_errs.len()))
 }
 
+fn resolve_heal_bucket_mode(opts: &mut HealOpts, pool_errs: &[Option<Error>]) -> Result<()> {
+    if opts.recreate {
+        return Ok(());
+    }
+    if let Some(err) = pool_errs
+        .iter()
+        .flatten()
+        .find(|err| **err != Error::DiskNotFound && **err != Error::VolumeNotFound)
+    {
+        return Err(err.clone());
+    }
+    opts.remove = is_all_buckets_not_found(pool_errs);
+    opts.recreate = !opts.remove;
+    Ok(())
+}
+
 #[async_trait]
 pub trait PeerS3Client: Debug + Sync + Send + 'static {
     async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem>;
@@ -159,10 +176,7 @@ impl S3PeerSys {
             pool_errs.push(reduce_pool_write_quorum_errs(&per_pool_errs));
         }
 
-        if !opts.recreate {
-            opts.remove = is_all_buckets_not_found(&pool_errs);
-            opts.recreate = !opts.remove;
-        }
+        resolve_heal_bucket_mode(&mut opts, &pool_errs)?;
 
         let mut futures = Vec::new();
         let heal_bucket_results = Arc::new(RwLock::new(vec![HealResultItem::default(); self.clients.len()]));
@@ -917,10 +931,11 @@ impl PeerS3Client for RemotePeerS3Client {
             || async {
                 let options: String = serde_json::to_string(opts)?;
                 let mut client = self.get_client().await?;
-                let request = Request::new(HealBucketRequest {
+                let mut request = Request::new(HealBucketRequest {
                     bucket: bucket.to_string(),
                     options,
                 });
+                set_tonic_mutation_body_digest(&mut request)?;
                 let response = client.heal_bucket(request).await?.into_inner();
                 if !response.success {
                     return if let Some(err) = response.error {
@@ -973,10 +988,11 @@ impl PeerS3Client for RemotePeerS3Client {
             || async {
                 let options = serde_json::to_string(opts)?;
                 let mut client = self.get_client().await?;
-                let request = Request::new(MakeBucketRequest {
+                let mut request = Request::new(MakeBucketRequest {
                     name: bucket.to_string(),
                     options,
                 });
+                set_tonic_mutation_body_digest(&mut request)?;
                 let response = client.make_bucket(request).await?.into_inner();
 
                 if !response.success {
@@ -1027,10 +1043,11 @@ impl PeerS3Client for RemotePeerS3Client {
                 let options = serde_json::to_string(opts)?;
                 let mut client = self.get_client().await?;
 
-                let request = Request::new(DeleteBucketRequest {
+                let mut request = Request::new(DeleteBucketRequest {
                     bucket: bucket.to_string(),
                     options,
                 });
+                set_tonic_mutation_body_digest(&mut request)?;
                 let response = client.delete_bucket(request).await?.into_inner();
                 if !response.success {
                     return if let Some(err) = response.error {
@@ -1618,6 +1635,30 @@ mod tests {
         assert_eq!(err, Error::VolumeExists);
     }
 
+    #[test]
+    fn heal_bucket_mode_fails_closed_on_incomplete_topology() {
+        let mut opts = HealOpts::default();
+        assert_eq!(
+            resolve_heal_bucket_mode(&mut opts, &[Some(Error::ErasureWriteQuorum)]),
+            Err(Error::ErasureWriteQuorum)
+        );
+        assert!(!opts.recreate);
+        assert!(!opts.remove);
+    }
+
+    #[test]
+    fn heal_bucket_mode_distinguishes_deleted_and_partial_buckets() {
+        let mut deleted = HealOpts::default();
+        resolve_heal_bucket_mode(&mut deleted, &[Some(Error::VolumeNotFound)]).unwrap();
+        assert!(deleted.remove);
+        assert!(!deleted.recreate);
+
+        let mut partial = HealOpts::default();
+        resolve_heal_bucket_mode(&mut partial, &[None, Some(Error::VolumeNotFound)]).unwrap();
+        assert!(!partial.remove);
+        assert!(partial.recreate);
+    }
+
     #[tokio::test]
     async fn test_make_bucket_reduces_quorum_by_pool_participants() {
         let peer_sys = S3PeerSys {

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -25,7 +25,7 @@ use crate::disk::error::{Error, Result};
 use crate::disk::{
     BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation,
     DiskOption, FileInfoVersions, FileReader, FileWriter, PartTransactionAction, ReadMultipleReq, ReadMultipleResp, ReadOptions,
-    RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one,
+    RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one,
     disk_store::{
         DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE,
         get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout,
@@ -50,8 +50,9 @@ use rustfs_protos::proto_gen::node_service::{
     DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest,
     MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest,
     ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest,
-    RenameFileRequest, SettlePartTransactionRequest, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest,
-    WriteAllRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
+    RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
+    SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
+    WriteMetadataRequest, node_service_client::NodeServiceClient,
 };
 use serde::{Serialize, de::DeserializeOwned};
 use std::{
@@ -100,6 +101,18 @@ const LOG_COMPONENT_ECSTORE: &str = "ecstore";
 const LOG_SUBSYSTEM_REMOTE_DISK: &str = "remote_disk";
 const EVENT_REMOTE_DISK_HEALTH: &str = "remote_disk_health";
 const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc";
+const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1;
+pub const REMOTE_SNAPSHOT_LEASE_TTL: Duration = Duration::from_secs(60);
+
+fn snapshot_lease_token_from_response(response: SnapshotLeaseResponse) -> Result<SnapshotLeaseToken> {
+    if !response.success {
+        return Err(response.error.unwrap_or_default().into());
+    }
+    if response.protocol_version != SNAPSHOT_LEASE_PROTOCOL_VERSION {
+        return Err(Error::other("remote snapshot lease protocol is incompatible"));
+    }
+    SnapshotLeaseToken::from_slice(&response.token)
+}
 
 /// Bind a mutating disk RPC to its canonical body: the digest lands in the request metadata, and
 /// the signing interceptor folds it (plus a replay-protected nonce) into the v2 signature scope
@@ -1784,6 +1797,81 @@ impl DiskAPI for RemoteDisk {
         .await
     }
 
+    async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> Result<SnapshotLeaseToken> {
+        self.execute_with_timeout(
+            || async {
+                let mut client = self
+                    .get_client()
+                    .await
+                    .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
+                let mut request = Request::new(SnapshotLeaseRequest {
+                    disk: self.endpoint.to_string(),
+                    volume: volume.to_string(),
+                    path: path.to_string(),
+                    ttl_ms: u64::try_from(REMOTE_SNAPSHOT_LEASE_TTL.as_millis())
+                        .map_err(|_| Error::other("snapshot lease TTL cannot be represented"))?,
+                });
+                let canonical_body = rustfs_protos::canonical_snapshot_lease_request_body(request.get_ref());
+                attach_mutation_body_digest(&mut request, canonical_body, "acquire_snapshot_lease")?;
+                let response = client.acquire_snapshot_lease(request).await?.into_inner();
+                snapshot_lease_token_from_response(response)
+            },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
+    async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<SnapshotLeaseToken> {
+        self.execute_with_timeout(
+            || async {
+                let mut client = self
+                    .get_client()
+                    .await
+                    .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
+                let mut request = Request::new(SnapshotLeaseRenewRequest {
+                    disk: self.endpoint.to_string(),
+                    volume: volume.to_string(),
+                    path: path.to_string(),
+                    token: token.as_bytes().to_vec().into(),
+                    ttl_ms: u64::try_from(REMOTE_SNAPSHOT_LEASE_TTL.as_millis())
+                        .map_err(|_| Error::other("snapshot lease TTL cannot be represented"))?,
+                });
+                let canonical_body = rustfs_protos::canonical_snapshot_lease_renew_request_body(request.get_ref());
+                attach_mutation_body_digest(&mut request, canonical_body, "renew_snapshot_lease")?;
+                let response = client.renew_snapshot_lease(request).await?.into_inner();
+                snapshot_lease_token_from_response(response)
+            },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
+    async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<()> {
+        self.execute_with_timeout(
+            || async {
+                let mut client = self
+                    .get_client()
+                    .await
+                    .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
+                let mut request = Request::new(SnapshotLeaseReleaseRequest {
+                    disk: self.endpoint.to_string(),
+                    volume: volume.to_string(),
+                    path: path.to_string(),
+                    token: token.as_bytes().to_vec().into(),
+                });
+                let canonical_body = rustfs_protos::canonical_snapshot_lease_release_request_body(request.get_ref());
+                attach_mutation_body_digest(&mut request, canonical_body, "release_snapshot_lease")?;
+                let response = client.release_snapshot_lease(request).await?.into_inner();
+                if !response.success {
+                    return Err(response.error.unwrap_or_default().into());
+                }
+                Ok(())
+            },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
     #[tracing::instrument(level = "trace", skip_all)]
     async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
         trace!(
@@ -2930,6 +3018,34 @@ mod tests {
 
     static INIT: Once = Once::new();
 
+    #[test]
+    fn snapshot_lease_response_requires_current_protocol_and_valid_token() {
+        let token = SnapshotLeaseToken::new();
+        let response = SnapshotLeaseResponse {
+            success: true,
+            token: token.as_bytes().to_vec().into(),
+            protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
+            error: None,
+        };
+        assert_eq!(snapshot_lease_token_from_response(response).unwrap(), token);
+
+        let incompatible = SnapshotLeaseResponse {
+            success: true,
+            token: token.as_bytes().to_vec().into(),
+            protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION + 1,
+            error: None,
+        };
+        assert!(snapshot_lease_token_from_response(incompatible).is_err());
+
+        let malformed = SnapshotLeaseResponse {
+            success: true,
+            token: Bytes::from_static(b"not-a-uuid"),
+            protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
+            error: None,
+        };
+        assert!(snapshot_lease_token_from_response(malformed).is_err());
+    }
+
     #[test]
     fn list_volumes_decode_rejects_a_malformed_entry() {
         let valid = serde_json::to_string(&VolumeInfo {

crates/ecstore/src/cluster/rpc/remote_locker.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use crate::cluster::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
+use crate::cluster::rpc::set_tonic_mutation_body_digest;
 use async_trait::async_trait;
 use bytes::Bytes;
 use rustfs_lock::{
@@ -313,10 +314,11 @@ impl LockClient for RemoteClient {
         info!("remote acquire_exclusive for {}", request.resource);
         let mut client = self.get_client().await?;
         let resource_summary = request.resource.to_string();
-        let req = Request::new(GenerallyLockRequest {
+        let mut req = Request::new(GenerallyLockRequest {
             args: serde_json::to_string(&request)
                 .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
         });
+        set_tonic_mutation_body_digest(&mut req)?;
 
         let resp = match self.execute_rpc("lock", &resource_summary, client.lock(req)).await {
             Ok(resp) => resp.into_inner(),
@@ -347,7 +349,7 @@ impl LockClient for RemoteClient {
 
         let mut client = self.get_client().await?;
         let resource_summary = Self::summarize_resources(requests);
-        let req = Request::new(BatchGenerallyLockRequest {
+        let mut req = Request::new(BatchGenerallyLockRequest {
             args: requests
                 .iter()
                 .map(|request| {
@@ -355,6 +357,7 @@ impl LockClient for RemoteClient {
                 })
                 .collect::<Result<Vec<_>>>()?,
         });
+        set_tonic_mutation_body_digest(&mut req)?;
 
         let resp = match self
             .execute_rpc("lock_batch", &resource_summary, client.lock_batch(req))
@@ -395,7 +398,8 @@ impl LockClient for RemoteClient {
             .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
         let mut client = self.get_client().await?;
         let resource_summary = unlock_request.resource.to_string();
-        let req = Request::new(GenerallyLockRequest { args: request_string });
+        let mut req = Request::new(GenerallyLockRequest { args: request_string });
+        set_tonic_mutation_body_digest(&mut req)?;
         let resp = self
             .execute_rpc("release", &resource_summary, client.un_lock(req))
             .await?
@@ -414,7 +418,7 @@ impl LockClient for RemoteClient {
         let unlock_requests = lock_ids.iter().map(Self::create_unlock_request).collect::<Vec<_>>();
         let mut client = self.get_client().await?;
         let resource_summary = Self::summarize_resources(&unlock_requests);
-        let req = Request::new(BatchGenerallyLockRequest {
+        let mut req = Request::new(BatchGenerallyLockRequest {
             args: unlock_requests
                 .iter()
                 .map(|request| {
@@ -422,6 +426,7 @@ impl LockClient for RemoteClient {
                 })
                 .collect::<Result<Vec<_>>>()?,
         });
+        set_tonic_mutation_body_digest(&mut req)?;
 
         let resp = self
             .execute_rpc("release_batch", &resource_summary, client.un_lock_batch(req))
@@ -440,10 +445,11 @@ impl LockClient for RemoteClient {
         let refresh_request = Self::create_unlock_request(lock_id);
         let mut client = self.get_client().await?;
         let resource_summary = refresh_request.resource.to_string();
-        let req = Request::new(GenerallyLockRequest {
+        let mut req = Request::new(GenerallyLockRequest {
             args: serde_json::to_string(&refresh_request)
                 .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
         });
+        set_tonic_mutation_body_digest(&mut req)?;
         let resp = self
             .execute_rpc("refresh", &resource_summary, client.refresh(req))
             .await?
@@ -459,10 +465,11 @@ impl LockClient for RemoteClient {
         let force_request = Self::create_unlock_request(lock_id);
         let mut client = self.get_client().await?;
         let resource_summary = force_request.resource.to_string();
-        let req = Request::new(GenerallyLockRequest {
+        let mut req = Request::new(GenerallyLockRequest {
             args: serde_json::to_string(&force_request)
                 .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
         });
+        set_tonic_mutation_body_digest(&mut req)?;
         let resp = self
             .execute_rpc("force_release", &resource_summary, client.force_un_lock(req))
             .await?
@@ -483,10 +490,11 @@ impl LockClient for RemoteClient {
         let mut client = self.get_client().await?;
 
         // Try to acquire a very short-lived lock to test availability
-        let req = Request::new(GenerallyLockRequest {
+        let mut req = Request::new(GenerallyLockRequest {
             args: serde_json::to_string(&status_request)
                 .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
         });
+        set_tonic_mutation_body_digest(&mut req)?;
 
         // Try exclusive lock first with very short timeout
         let resp = match self.execute_rpc("check_status", &resource_summary, client.lock(req)).await {
@@ -497,10 +505,11 @@ impl LockClient for RemoteClient {
         if resp.success {
             // If we successfully acquired the lock, the resource was free.
             // Immediately release it on a best-effort basis.
-            let release_req = Request::new(GenerallyLockRequest {
+            let mut release_req = Request::new(GenerallyLockRequest {
                 args: serde_json::to_string(&status_request)
                     .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
             });
+            set_tonic_mutation_body_digest(&mut release_req)?;
             let _ = self
                 .execute_rpc("check_status_release", &resource_summary, client.un_lock(release_req))
                 .await;

crates/ecstore/src/disk/disk_store.rs

@@ -1365,6 +1365,14 @@ impl DiskAPI for LocalDiskWrapper {
         .await
     }
 
+    async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<SnapshotLeaseToken> {
+        self.track_disk_health(
+            || async { self.disk.renew_snapshot_lease(volume, path, token).await },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
     async fn delete_data_dir(&self, volume: &str, path: &str, opts: DeleteOptions) -> Result<DataDirDeleteStatus> {
         self.track_disk_health(
             || async { self.disk.delete_data_dir(volume, path, opts).await },

crates/ecstore/src/disk/local.rs

@@ -7908,6 +7908,23 @@ impl DiskAPI for LocalDisk {
         }
     }
 
+    async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<SnapshotLeaseToken> {
+        let key = SnapshotLeaseKey {
+            volume: volume.to_string(),
+            path: path.to_string(),
+        };
+        let mut registry = self.snapshot_leases.lock().await;
+        let Some(entry) = registry.entries.get_mut(&key) else {
+            return Err(DiskError::FileNotFound);
+        };
+        if entry.deleting || !entry.tokens.remove(&token) {
+            return Err(DiskError::FileNotFound);
+        }
+        let renewed = SnapshotLeaseToken::new();
+        entry.tokens.insert(renewed);
+        Ok(renewed)
+    }
+
     async fn delete_data_dir(&self, volume: &str, path: &str, opts: DeleteOptions) -> Result<DataDirDeleteStatus> {
         let key = SnapshotLeaseKey {
             volume: volume.to_string(),
@@ -14957,6 +14974,13 @@ mod test {
             .acquire_snapshot_lease(volume, &data_dir)
             .await
             .expect("second lease should be acquired");
+        let renewed = disk
+            .renew_snapshot_lease(volume, &data_dir, first)
+            .await
+            .expect("first lease should renew atomically");
+        disk.release_snapshot_lease(volume, &data_dir, first)
+            .await
+            .expect("the superseded token should be idempotent");
         let status = disk
             .delete_data_dir(
                 volume,
@@ -14976,9 +15000,9 @@ mod test {
             Bytes::from_static(b"later")
         );
 
-        disk.release_snapshot_lease(volume, &data_dir, first)
+        disk.release_snapshot_lease(volume, &data_dir, renewed)
             .await
-            .expect("first lease release should succeed");
+            .expect("renewed lease release should succeed");
         assert!(
             disk.read_all(volume, &first_part).await.is_ok(),
             "one remaining lease must keep the data directory"

crates/ecstore/src/disk/mod.rs

@@ -79,6 +79,18 @@ impl SnapshotLeaseToken {
     pub fn new() -> Self {
         Self(Uuid::new_v4())
     }
+
+    pub fn from_slice(bytes: &[u8]) -> Result<Self> {
+        let uuid = Uuid::from_slice(bytes).map_err(|_| Error::other("invalid snapshot lease token"))?;
+        if uuid.is_nil() {
+            return Err(Error::other("invalid snapshot lease token"));
+        }
+        Ok(Self(uuid))
+    }
+
+    pub fn as_bytes(&self) -> &[u8; 16] {
+        self.0.as_bytes()
+    }
 }
 
 impl Default for SnapshotLeaseToken {
@@ -284,6 +296,13 @@ impl DiskAPI for Disk {
         }
     }
 
+    async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<SnapshotLeaseToken> {
+        match self {
+            Disk::Local(local_disk) => local_disk.renew_snapshot_lease(volume, path, token).await,
+            Disk::Remote(remote_disk) => remote_disk.renew_snapshot_lease(volume, path, token).await,
+        }
+    }
+
     async fn delete_data_dir(&self, volume: &str, path: &str, opts: DeleteOptions) -> Result<DataDirDeleteStatus> {
         match self {
             Disk::Local(local_disk) => local_disk.delete_data_dir(volume, path, opts).await,
@@ -694,6 +713,9 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
     async fn release_snapshot_lease(&self, _volume: &str, _path: &str, _token: SnapshotLeaseToken) -> Result<()> {
         Err(Error::other("snapshot leases are not supported by this disk"))
     }
+    async fn renew_snapshot_lease(&self, _volume: &str, _path: &str, _token: SnapshotLeaseToken) -> Result<SnapshotLeaseToken> {
+        Err(Error::other("snapshot leases are not supported by this disk"))
+    }
     async fn delete_data_dir(&self, volume: &str, path: &str, opts: DeleteOptions) -> Result<DataDirDeleteStatus> {
         self.delete(volume, path, opts).await?;
         Ok(DataDirDeleteStatus::Deleted)

crates/ecstore/src/object_api/encryption.rs

@@ -0,0 +1,80 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use async_trait::async_trait;
+use http::{HeaderMap, HeaderValue};
+use std::collections::HashMap;
+use std::error::Error;
+use std::fmt::{Display, Formatter};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ReadEncryptionMode {
+    Direct { base_nonce: [u8; 12] },
+    Object,
+}
+
+pub struct ReadEncryptionMaterial {
+    pub key_bytes: [u8; 32],
+    pub mode: ReadEncryptionMode,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum EncryptionResolutionErrorKind {
+    InvalidRequest,
+    InvalidMetadata,
+    ServiceUnavailable,
+    DecryptionFailed,
+}
+
+#[derive(Debug)]
+pub struct EncryptionResolutionError {
+    kind: EncryptionResolutionErrorKind,
+    message: String,
+}
+
+impl EncryptionResolutionError {
+    pub fn new(kind: EncryptionResolutionErrorKind, message: impl Into<String>) -> Self {
+        Self {
+            kind,
+            message: message.into(),
+        }
+    }
+
+    pub fn kind(&self) -> EncryptionResolutionErrorKind {
+        self.kind
+    }
+}
+
+impl Display for EncryptionResolutionError {
+    fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result {
+        formatter.write_str(&self.message)
+    }
+}
+
+impl Error for EncryptionResolutionError {}
+
+pub struct ReadEncryptionRequest<'a> {
+    pub bucket: &'a str,
+    pub object: &'a str,
+    pub metadata: &'a HashMap<String, String>,
+    pub headers: &'a HeaderMap<HeaderValue>,
+}
+
+#[async_trait]
+pub trait ObjectEncryptionResolver: Send + Sync {
+    async fn resolve_read_material(
+        &self,
+        request: ReadEncryptionRequest<'_>,
+    ) -> Result<Option<ReadEncryptionMaterial>, EncryptionResolutionError>;
+}

crates/ecstore/src/object_api/mod.rs

@@ -84,6 +84,7 @@ pub(crate) fn legacy_encrypted_range_seek_enabled() -> bool {
 }
 
 mod body_cache_hook;
+mod encryption;
 mod hook_slot;
 mod object_mutation_hook;
 mod readers;
@@ -98,6 +99,10 @@ pub use body_cache_hook::{
 pub(crate) use body_cache_hook::{
     get_object_body_cache_hook, get_object_body_cache_hook_suppressed, without_get_object_body_cache_hook,
 };
+pub use encryption::{
+    EncryptionResolutionError, EncryptionResolutionErrorKind, ObjectEncryptionResolver, ReadEncryptionMaterial,
+    ReadEncryptionMode, ReadEncryptionRequest,
+};
 pub(crate) use object_mutation_hook::notify_object_mutation;
 pub use object_mutation_hook::{ObjectMutationHook, register_object_mutation_hook, unregister_object_mutation_hook};
 pub use readers::*;

crates/ecstore/src/object_api/types.rs

@@ -273,29 +273,9 @@ impl ObjectInfo {
     }
 
     pub fn is_encrypted(&self) -> bool {
-        // Corresponding to the logic in rustfs/src/sse.rs/encryption_material_to_metadata function
-        use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};
-
-        self.user_defined.keys().any(|key| {
-            let lower = key.to_ascii_lowercase();
-            lower.starts_with("x-minio-encryption-")
-                || lower.starts_with("x-minio-internal-server-side-encryption-")
-                || matches!(
-                    lower.as_str(),
-                    "x-minio-internal-encrypted-multipart"
-                        | "x-rustfs-encryption-key"
-                        | "x-rustfs-encryption-algorithm"
-                        | "x-rustfs-encryption-iv"
-                        | "x-rustfs-encryption-key-id"
-                        | "x-rustfs-encryption-context"
-                        | "x-rustfs-encryption-tag"
-                        | "x-amz-server-side-encryption-aws-kms-key-id"
-                        | SSEC_ALGORITHM_HEADER
-                        | SSEC_KEY_HEADER
-                        | SSEC_KEY_MD5_HEADER
-                        | "x-amz-server-side-encryption"
-                )
-        })
+        self.user_defined
+            .keys()
+            .any(|key| rustfs_utils::http::is_object_encryption_marker(key))
     }
 
     /// Maximum inline size for non-versioned objects (128 KiB).
@@ -339,26 +319,7 @@ impl ObjectInfo {
     }
 
     pub fn encryption_original_size(&self) -> std::io::Result<Option<i64>> {
-        let actual_size = rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE);
-        if let Some(size_str) = self
-            .user_defined
-            .get("x-rustfs-encryption-original-size")
-            .map(String::as_str)
-            .or_else(|| {
-                self.user_defined
-                    .get("x-amz-server-side-encryption-customer-original-size")
-                    .map(String::as_str)
-            })
-            .or(actual_size.as_deref())
-            && !size_str.is_empty()
-        {
-            let size = size_str
-                .parse::<i64>()
-                .map_err(|e| std::io::Error::other(format!("Failed to parse encryption original size: {e}")))?;
-            return Ok(Some(size));
-        }
-
-        Ok(None)
+        rustfs_utils::http::get_object_encryption_original_size(&self.user_defined)
     }
 
     pub fn decrypted_size(&self) -> std::io::Result<i64> {
@@ -388,9 +349,6 @@ impl ObjectInfo {
             return Ok(actual_size);
         }
 
-        // Check if object is encrypted
-        // Managed SSE stores original size in x-rustfs-encryption-original-size metadata
-        // SSE-C stores original size in x-amz-server-side-encryption-customer-original-size
         if let Some(size) = self.encryption_original_size()? {
             return Ok(size);
         }
@@ -881,6 +839,19 @@ mod tests {
         assert!(!object.is_inline_fast_path_eligible(), "transitioned objects must fall back");
     }
 
+    #[test]
+    fn minio_internal_encryption_metadata_is_not_treated_as_plaintext() {
+        let object = ObjectInfo {
+            user_defined: Arc::new(HashMap::from([(
+                "X-Minio-Internal-Server-Side-Encryption-Sealed-Key".to_string(),
+                "sealed".to_string(),
+            )])),
+            ..Default::default()
+        };
+
+        assert!(object.is_encrypted());
+    }
+
     #[test]
     fn versions_after_marker_handles_null_version_marker() {
         let first_version = Uuid::parse_str("11111111-2222-3333-4444-555555555555").unwrap();

crates/ecstore/src/runtime/instance.rs

@@ -46,6 +46,7 @@ use crate::bucket::metadata_sys::BucketMetadataSys;
 use crate::bucket::replication::{DynReplicationPool, ReplicationStats};
 use crate::disk::DiskStore;
 use crate::layout::endpoints::{EndpointServerPools, SetupType};
+use crate::object_api::ObjectEncryptionResolver;
 use crate::services::event_notification::EventNotifier;
 use crate::services::tier::tier::TierConfigMgr;
 use rustfs_lock::{GlobalLockManager, get_global_lock_manager};
@@ -159,6 +160,8 @@ pub struct InstanceContext {
     /// workers (scanner/heal/tier/lifecycle) without touching another instance.
     /// Replaces the process-global cancel-token static.
     background_cancel_token: OnceLock<CancellationToken>,
+    /// Resolves object-encryption material at the application boundary.
+    object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
     tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
     transition_transaction_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
     #[cfg(test)]
@@ -197,6 +200,7 @@ impl InstanceContext {
             local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
             bucket_metadata_sys: std::sync::Mutex::new(None),
             background_cancel_token: OnceLock::new(),
+            object_encryption_resolver: OnceLock::new(),
             tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
             transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
             #[cfg(test)]
@@ -209,6 +213,19 @@ impl InstanceContext {
         self.lock_manager.clone()
     }
 
+    /// Install the application-owned object-encryption resolver once.
+    pub fn set_object_encryption_resolver(
+        &self,
+        resolver: Arc<dyn ObjectEncryptionResolver>,
+    ) -> Result<(), Arc<dyn ObjectEncryptionResolver>> {
+        self.object_encryption_resolver.set(resolver)
+    }
+
+    /// Return the configured object-encryption resolver, if startup installed one.
+    pub fn object_encryption_resolver(&self) -> Option<&dyn ObjectEncryptionResolver> {
+        self.object_encryption_resolver.get().map(Arc::as_ref)
+    }
+
     /// Set this instance's S3 region.
     ///
     /// Write-once: panics on a second write, preserving the startup fail-fast

crates/ecstore/src/runtime/sources.rs

@@ -46,7 +46,6 @@ use crate::{
 use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
 use rustfs_config::server_config::{Config, get_global_server_config, set_global_server_config};
 use rustfs_io_metrics::internode_metrics::global_internode_metrics;
-use rustfs_kms::{ObjectEncryptionService, get_global_encryption_service};
 use rustfs_lock::client::LockClient;
 use s3s::dto::BucketLifecycleConfiguration;
 use s3s::region::Region;
@@ -105,10 +104,6 @@ pub(crate) fn record_erasure_write_quorum_failure(stage: &'static str, dominant_
     global_internode_metrics().record_erasure_write_quorum_failure(stage, dominant_error);
 }
 
-pub(crate) async fn object_encryption_service() -> Option<Arc<ObjectEncryptionService>> {
-    get_global_encryption_service().await
-}
-
 pub fn object_store_handle() -> Option<Arc<ECStore>> {
     resolve_object_store_handle()
 }

crates/ecstore/src/services/notification_sys.rs

@@ -1344,8 +1344,11 @@ async fn run_tier_config_reload_worker<F, Fut>(
                 }
                 TierConfigReloadFinish::Pending => retry_attempt = 0,
             },
-            TierConfigReloadOutcome::Terminal(_) => match sys.finish_tier_config_reload_worker(&host) {
+            TierConfigReloadOutcome::Terminal(err) => match sys.finish_tier_config_reload_worker(&host) {
                 TierConfigReloadFinish::Completed => {
+                    // This peer keeps the previous tier configuration for good, so record
+                    // why. Dropping the error here hides the only evidence of a divergent
+                    // node behind an outcome label that cannot be acted on.
                     warn!(
                         event = EVENT_NOTIFICATION_PEER_PROPAGATION,
                         component = LOG_COMPONENT_ECSTORE,
@@ -1353,6 +1356,7 @@ async fn run_tier_config_reload_worker<F, Fut>(
                         action = "reload_transition_tier_config",
                         host,
                         outcome = "terminal",
+                        error = ?err,
                         "tier configuration reload stopped after a terminal outcome"
                     );
                     return;

crates/ecstore/src/set_disk/metadata.rs

@@ -505,9 +505,7 @@ impl SetDisks {
     }
 
     fn file_info_has_encryption_metadata(meta: &FileInfo) -> bool {
-        meta.metadata
-            .keys()
-            .any(|name| http::is_encryption_metadata_key(name) || http::is_sse_header(name))
+        meta.metadata.keys().any(|name| http::is_object_encryption_marker(name))
     }
 
     fn starts_with_ignore_ascii_case(value: &str, prefix: &str) -> bool {
@@ -555,7 +553,7 @@ impl SetDisks {
         }
     }
 
-    fn file_info_quorum_hash(meta: &FileInfo) -> [u8; 32] {
+    pub(super) fn file_info_quorum_hash(meta: &FileInfo) -> [u8; 32] {
         let mut hasher = Sha256::new();
         Self::update_file_info_quorum_hash(&mut hasher, meta);
         let digest = hasher.finalize();

crates/ecstore/src/set_disk/mod.rs

@@ -143,15 +143,17 @@ use rustfs_object_capacity::capacity_scope::{
     CapacityScope, CapacityScopeDisk, current_dirty_generation, record_capacity_scope, record_global_dirty_scope,
 };
 use rustfs_s3_types::EventName;
+#[cfg(test)]
+use rustfs_utils::http::SSEC_ALGORITHM_HEADER;
 use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
 use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
 use rustfs_utils::http::headers::{
     CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _,
 };
 use rustfs_utils::http::{
-    SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE,
-    SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, contains_key_str,
-    get_header_map, get_str, insert_str, is_encryption_metadata_key, remove_header_map,
+    SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC,
+    SUFFIX_RESTORE_OPERATION_ID, contains_key_str, get_header_map, get_str, insert_str, is_object_encryption_marker,
+    remove_header_map,
 };
 use rustfs_utils::{
     HashAlgorithm,
@@ -407,10 +409,7 @@ pub(crate) fn strip_internal_multipart_metadata(metadata: &mut HashMap<String, S
 }
 
 fn should_persist_encryption_original_size(metadata: &HashMap<String, String>) -> bool {
-    metadata.keys().any(|key| is_encryption_metadata_key(key))
-        || metadata.contains_key(SSEC_ALGORITHM_HEADER)
-        || metadata.contains_key(SSEC_KEY_HEADER)
-        || metadata.contains_key(SSEC_KEY_MD5_HEADER)
+    metadata.keys().any(|key| is_object_encryption_marker(key))
 }
 
 /// Per-set memoized capacity dirty scope.

crates/ecstore/src/set_disk/ops/heal.rs

@@ -92,6 +92,69 @@ struct PartFailureSummary {
     bitrot_failure: bool,
 }
 
+#[derive(Clone)]
+struct RecoverableMetaCandidate {
+    identity: [u8; 32],
+    file_info: FileInfo,
+    data_count: usize,
+    local_payload: bool,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum DanglingDeleteSafety {
+    UnsafeToDelete,
+    NoRecoverableCandidate,
+}
+
+#[cfg(test)]
+struct DanglingCheckPartsFailure {
+    key: DanglingCheckPartsFailureKey,
+}
+
+#[cfg(test)]
+type DanglingCheckPartsFailureKey = (String, String, usize);
+
+#[cfg(test)]
+type DanglingCheckPartsFailures = HashMap<DanglingCheckPartsFailureKey, DiskError>;
+
+#[cfg(test)]
+fn dangling_check_parts_failures() -> &'static std::sync::Mutex<DanglingCheckPartsFailures> {
+    static FAILURES: std::sync::OnceLock<std::sync::Mutex<DanglingCheckPartsFailures>> = std::sync::OnceLock::new();
+    FAILURES.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
+}
+
+#[cfg(test)]
+impl DanglingCheckPartsFailure {
+    fn install(bucket: &str, object: &str, disk_index: usize, error: DiskError) -> Self {
+        let key = (bucket.to_string(), object.to_string(), disk_index);
+        let previous = dangling_check_parts_failures()
+            .lock()
+            .expect("dangling check-parts failure registry should not poison")
+            .insert(key.clone(), error);
+        assert!(previous.is_none(), "dangling check-parts failure already installed");
+        Self { key }
+    }
+}
+
+#[cfg(test)]
+impl Drop for DanglingCheckPartsFailure {
+    fn drop(&mut self) {
+        dangling_check_parts_failures()
+            .lock()
+            .expect("dangling check-parts failure registry should not poison")
+            .remove(&self.key);
+    }
+}
+
+#[cfg(test)]
+fn injected_dangling_check_parts_error(bucket: &str, object: &str, disk_index: usize) -> Option<DiskError> {
+    dangling_check_parts_failures()
+        .lock()
+        .expect("dangling check-parts failure registry should not poison")
+        .get(&(bucket.to_string(), object.to_string(), disk_index))
+        .cloned()
+}
+
 fn first_unhealthy_part_summary(
     data_errs_by_part: &HashMap<usize, Vec<usize>>,
     parts: &[ObjectPartInfo],
@@ -125,39 +188,17 @@ impl SetDisks {
         version_id: &str,
         opts: &HealOpts,
     ) -> disk::error::Result<(HealResultItem, Option<DiskError>)> {
-        // `allow_meta_regen` is true on the first pass: a version whose data shards
-        // physically survive (>= data_blocks) but whose xl.meta fell below
-        // read-quorum is RESCUED (missing xl.meta regenerated) rather than
-        // dangling-deleted. The re-drive after a rescue sets it false so the
-        // regeneration can happen at most once (no unbounded recursion).
-        Box::pin(self.heal_object_with_regen(bucket, object, version_id, opts, true)).await
-    }
-
-    /// Best-effort orphan-data-dir reclaim for an object that is healthy on this
-    /// set. Wraps [`Self::reclaim_orphan_data_dirs`] with the shared logging so
-    /// both `heal_object` exits — the already-healthy early return and the
-    /// post-heal tail — reclaim identically. Never fails the heal: delete errors
-    /// are logged and swallowed. Callers must gate this on `!opts.dry_run`.
-    async fn reclaim_orphan_data_dirs_best_effort(&self, bucket: &str, object: &str) {
-        match self.reclaim_orphan_data_dirs(bucket, object).await {
-            Ok(removed) if removed > 0 => {
-                info!(bucket, object, removed, "heal_object: reclaimed orphaned data directories");
-            }
-            Ok(_) => {}
-            Err(e) => {
-                warn!(bucket, object, error = %e, "heal_object: orphan data-dir reclaim failed");
-            }
-        }
+        Box::pin(self.heal_object_with_explicit_version_regen(bucket, object, version_id, opts, true)).await
     }
 
     #[allow(clippy::too_many_lines)]
-    async fn heal_object_with_regen(
+    async fn heal_object_with_explicit_version_regen(
         &self,
         bucket: &str,
         object: &str,
         version_id: &str,
         opts: &HealOpts,
-        allow_meta_regen: bool,
+        allow_explicit_version_regen: bool,
     ) -> disk::error::Result<(HealResultItem, Option<DiskError>)> {
         info!(?opts, "Starting heal_object");
 
@@ -354,22 +395,6 @@ impl SetDisks {
                             }
                         }
 
-                        // DATA-SAFETY GUARD (backlog#920, decision 1): before any
-                        // dangling delete, if the version's DATA shards physically
-                        // survive on >= data_blocks disks it is RECONSTRUCTABLE.
-                        // Regenerate the missing xl.meta from a surviving valid
-                        // FileInfo and re-drive the heal instead of destroying a
-                        // recoverable version. Torn writes (< data_blocks data
-                        // shards) fall through to the existing dangling behavior.
-                        if cannot_heal
-                            && allow_meta_regen
-                            && self
-                                .try_regenerate_recoverable_meta(bucket, object, &parts_metadata, &errs, &disks)
-                                .await?
-                        {
-                            return Box::pin(self.heal_object_with_regen(bucket, object, version_id, opts, false)).await;
-                        }
-
                         if cannot_heal {
                             let total_disks = parts_metadata.len();
                             let healthy_count = total_disks.saturating_sub(disks_to_heal_count);
@@ -422,6 +447,20 @@ impl SetDisks {
                                 );
                             }
 
+                            // `disks_with_all_parts` normalizes conflicting entries
+                            // in `parts_metadata` to defaults. Re-read only before
+                            // destructive cleanup so the guard sees every original
+                            // identity.
+                            let (delete_guard_metadata, delete_guard_errs) =
+                                Self::read_all_fileinfo(&disks, "", bucket, object, version_id, true, true, false).await?;
+                            if self
+                                .dangling_delete_safety(bucket, object, &delete_guard_metadata, &delete_guard_errs, &disks)
+                                .await?
+                                == DanglingDeleteSafety::UnsafeToDelete
+                            {
+                                return Ok((result, Some(cannot_heal_err)));
+                            }
+
                             // Allow for dangling deletes, on versions that have DataDir missing etc.
                             // this would end up restoring the correct readable versions.
                             return match self
@@ -837,17 +876,25 @@ impl SetDisks {
                 }
             }
             Err(err) => {
-                // DATA-SAFETY GUARD (backlog#920, decision 1): meta quorum failed,
-                // but the version's DATA may still physically survive on enough
-                // disks (xl.meta lost on > parity disks while part files remain).
-                // Rescue it by regenerating the missing xl.meta and re-driving heal
-                // instead of dangling-deleting a reconstructable version.
-                if allow_meta_regen
+                if allow_explicit_version_regen
+                    && !version_id.is_empty()
                     && self
-                        .try_regenerate_recoverable_meta(bucket, object, &parts_metadata, &errs, &disks)
+                        .try_regenerate_explicit_version_meta(bucket, object, version_id, &parts_metadata, &errs, &disks)
                         .await?
                 {
-                    return Box::pin(self.heal_object_with_regen(bucket, object, version_id, opts, false)).await;
+                    return Box::pin(self.heal_object_with_explicit_version_regen(bucket, object, version_id, opts, false)).await;
+                }
+
+                if self
+                    .dangling_delete_safety(bucket, object, &parts_metadata, &errs, &disks)
+                    .await?
+                    == DanglingDeleteSafety::UnsafeToDelete
+                {
+                    return Ok((
+                        self.default_heal_result(FileInfo::default(), &errs, bucket, object, version_id)
+                            .await,
+                        Some(err),
+                    ));
                 }
 
                 let data_errs_by_part = HashMap::new();
@@ -883,129 +930,226 @@ impl SetDisks {
         }
     }
 
-    /// backlog#920 (decision 1): rescue a version that meta-quorum logic would
-    /// otherwise dangling-DELETE, when its DATA is still reconstructable.
-    ///
-    /// Returns `Ok(true)` if the version was rescued (missing xl.meta regenerated
-    /// on at least one disk, so a re-driven heal can reconstruct it), `Ok(false)`
-    /// to fall through to the existing dangling-delete behavior.
-    ///
-    /// Recoverability is computed by physically probing part files across ALL
-    /// disks in the set with `check_parts` — including disks whose xl.meta is
-    /// absent (a lost xl.meta does not lose the sibling `part.*` data). If at
-    /// least `data_blocks` disks hold every part of a surviving valid FileInfo,
-    /// the object is EC-reconstructable, so we regenerate that FileInfo's xl.meta
-    /// on every disk whose metadata is absent (via `write_metadata`, which merges
-    /// into any existing xl.meta). Delete markers, remote/transitioned versions,
-    /// and genuine torn writes (< `data_blocks` surviving data shards) are NOT
-    /// rescued — they keep the current dangling-delete-after-grace behavior, so no
-    /// regression on those paths.
-    async fn try_regenerate_recoverable_meta(
+    async fn try_regenerate_explicit_version_meta(
+        &self,
+        bucket: &str,
+        object: &str,
+        version_id: &str,
+        parts_metadata: &[FileInfo],
+        errs: &[Option<DiskError>],
+        disks: &[Option<DiskStore>],
+    ) -> disk::error::Result<bool> {
+        let Ok(version_id) = Uuid::parse_str(version_id) else {
+            return Ok(false);
+        };
+        let candidates = parts_metadata
+            .iter()
+            .zip(errs.iter())
+            .filter_map(|(file_info, err)| {
+                (err.is_none()
+                    && file_info_is_valid_for_metadata(file_info)
+                    && file_info.version_id == Some(version_id)
+                    && file_info.has_valid_erasure_geometry()
+                    && !file_info.deleted
+                    && !file_info.is_remote()
+                    && file_info.data_dir.is_some()
+                    && !file_info.parts.is_empty()
+                    && file_info.erasure.data_blocks > 0
+                    && file_info
+                        .erasure
+                        .data_blocks
+                        .checked_add(file_info.erasure.parity_blocks)
+                        .is_some_and(|shards| shards == disks.len()))
+                .then_some(file_info)
+            })
+            .collect::<Vec<_>>();
+        let Some(candidate) = candidates.first().copied() else {
+            return Ok(false);
+        };
+        let identity = Self::file_info_quorum_hash(candidate);
+        if candidates
+            .iter()
+            .any(|file_info| Self::file_info_quorum_hash(file_info) != identity)
+        {
+            return Ok(false);
+        }
+
+        let mut available = 0usize;
+        for disk in disks {
+            let Some(disk) = disk else {
+                return Ok(false);
+            };
+            match disk.check_parts(bucket, object, candidate).await {
+                Ok(response)
+                    if !response.results.is_empty() && response.results.iter().all(|result| *result == CHECK_PART_SUCCESS) =>
+                {
+                    available += 1;
+                }
+                Ok(_)
+                | Err(
+                    DiskError::FileNotFound
+                    | DiskError::FileVersionNotFound
+                    | DiskError::PathNotFound
+                    | DiskError::VolumeNotFound,
+                ) => {}
+                Err(_) => return Ok(false),
+            }
+        }
+        if available < candidate.erasure.data_blocks {
+            return Ok(false);
+        }
+
+        let mut wrote = 0usize;
+        for (index, disk) in disks.iter().enumerate() {
+            let Some(disk) = disk else {
+                return Ok(false);
+            };
+            let metadata_absent = matches!(
+                errs.get(index).and_then(Option::as_ref),
+                Some(DiskError::FileNotFound | DiskError::FileVersionNotFound)
+            );
+            if !metadata_absent {
+                continue;
+            }
+            let Some(&shard_index) = candidate.erasure.distribution.get(index) else {
+                return Ok(false);
+            };
+            let mut regenerated = candidate.clone();
+            regenerated.fresh = false;
+            regenerated.erasure.index = shard_index;
+            match disk.write_metadata("", bucket, object, regenerated).await {
+                Ok(()) => wrote += 1,
+                Err(error) => {
+                    warn!(
+                        bucket,
+                        object,
+                        disk_index = index,
+                        error = %error,
+                        "failed to regenerate recoverable xl.meta"
+                    );
+                }
+            }
+        }
+        Ok(wrote > 0)
+    }
+
+    /// Best-effort orphan-data-dir reclaim for an object that is healthy on this
+    /// set. Wraps [`Self::reclaim_orphan_data_dirs`] with the shared logging so
+    /// both `heal_object` exits — the already-healthy early return and the
+    /// post-heal tail — reclaim identically. Never fails the heal: delete errors
+    /// are logged and swallowed. Callers must gate this on `!opts.dry_run`.
+    async fn reclaim_orphan_data_dirs_best_effort(&self, bucket: &str, object: &str) {
+        match self.reclaim_orphan_data_dirs(bucket, object).await {
+            Ok(removed) if removed > 0 => {
+                info!(bucket, object, removed, "heal_object: reclaimed orphaned data directories");
+            }
+            Ok(_) => {}
+            Err(e) => {
+                warn!(bucket, object, error = %e, "heal_object: orphan data-dir reclaim failed");
+            }
+        }
+    }
+
+    /// Prevent dangling cleanup when surviving state cannot prove that deletion
+    /// is safe. Part presence proves only recoverability, never commit: the write
+    /// path can durably rename data before xl.meta is committed.
+    async fn dangling_delete_safety(
         &self,
         bucket: &str,
         object: &str,
         parts_metadata: &[FileInfo],
         errs: &[Option<DiskError>],
         disks: &[Option<DiskStore>],
-    ) -> disk::error::Result<bool> {
-        // A surviving valid, non-deleted, non-remote data FileInfo to rebuild from.
-        let Some(surviving) = parts_metadata
+    ) -> disk::error::Result<DanglingDeleteSafety> {
+        if disks.iter().any(Option::is_none)
+            || errs.iter().flatten().any(|err| {
+                !matches!(
+                    err,
+                    DiskError::FileNotFound
+                        | DiskError::FileVersionNotFound
+                        | DiskError::PathNotFound
+                        | DiskError::VolumeNotFound
+                )
+            })
+        {
+            return Ok(DanglingDeleteSafety::UnsafeToDelete);
+        }
+
+        let mut candidates = Vec::<RecoverableMetaCandidate>::with_capacity(parts_metadata.len());
+        for (fi, err) in parts_metadata.iter().zip(errs.iter()) {
+            if err.is_some() || !file_info_is_valid_for_metadata(fi) {
+                continue;
+            }
+
+            let identity = Self::file_info_quorum_hash(fi);
+            if !candidates.iter().any(|candidate| candidate.identity == identity) {
+                let local_payload = fi.has_valid_erasure_geometry()
+                    && !fi.deleted
+                    && !fi.is_remote()
+                    && fi.data_dir.is_some()
+                    && !fi.parts.is_empty()
+                    && fi.erasure.data_blocks > 0
+                    && fi
+                        .erasure
+                        .data_blocks
+                        .checked_add(fi.erasure.parity_blocks)
+                        .is_some_and(|shards| shards == disks.len());
+                candidates.push(RecoverableMetaCandidate {
+                    identity,
+                    file_info: fi.clone(),
+                    data_count: 0,
+                    local_payload,
+                });
+            }
+        }
+
+        if candidates
             .iter()
-            .find(|fi| fi.has_valid_erasure_geometry() && !fi.deleted && !fi.is_remote())
-            .cloned()
-        else {
-            return Ok(false);
-        };
-
-        // Without a data_dir + parts there is no data to prove recoverable.
-        if surviving.data_dir.is_none() || surviving.parts.is_empty() {
-            return Ok(false);
-        }
-        let data_blocks = surviving.erasure.data_blocks;
-        if data_blocks == 0 {
-            return Ok(false);
+            .any(|candidate| candidate.file_info.deleted || candidate.file_info.is_remote())
+            || candidates.len() > 1
+        {
+            return Ok(DanglingDeleteSafety::UnsafeToDelete);
         }
 
-        // Physically probe part presence on EVERY online disk using the surviving
-        // FileInfo's data_dir/parts. `check_parts` stats `object/<data_dir>/part.N`
-        // directly, so it counts disks that still hold the data even if their
-        // xl.meta was deleted.
-        let mut available = 0usize;
-        for disk in disks.iter().flatten() {
-            if let Ok(resp) = disk.check_parts(bucket, object, &surviving).await
-                && !resp.results.is_empty()
-                && resp.results.iter().all(|r| *r == CHECK_PART_SUCCESS)
-            {
-                available += 1;
-            }
-        }
+        for candidate in candidates.iter_mut().filter(|candidate| candidate.local_payload) {
+            for (disk_index, disk) in disks.iter().enumerate() {
+                let Some(disk) = disk else {
+                    return Ok(DanglingDeleteSafety::UnsafeToDelete);
+                };
+                #[cfg(test)]
+                let check_result = match injected_dangling_check_parts_error(bucket, object, disk_index) {
+                    Some(error) => Err(error),
+                    None => disk.check_parts(bucket, object, &candidate.file_info).await,
+                };
+                #[cfg(not(test))]
+                let check_result = disk.check_parts(bucket, object, &candidate.file_info).await;
 
-        // Torn write: fewer than data_blocks surviving data shards is genuinely
-        // unrecoverable — preserve the current dangling behavior (no resurrection).
-        if available < data_blocks {
-            debug!(
-                bucket,
-                object,
-                available,
-                data_blocks,
-                "heal_object: version not reconstructable (torn write), keeping dangling behavior"
-            );
-            return Ok(false);
-        }
-
-        // Reconstructable: regenerate the surviving xl.meta on every disk whose
-        // metadata is absent so the version regains read-quorum. Each disk gets its
-        // OWN shard index: the disk at physical position `index` holds shard
-        // `distribution[index]` (mirrors `shuffle_disks` + the write path's
-        // `erasure.index = shuffled_pos + 1`). Copying the surviving disk's index
-        // verbatim would write an inconsistent xl.meta that the re-heal then treats
-        // as corrupt.
-        let distribution = &surviving.erasure.distribution;
-        let mut wrote = 0usize;
-        for (index, disk) in disks.iter().enumerate() {
-            let Some(disk) = disk else { continue };
-            let meta_absent = matches!(
-                errs.get(index).and_then(Option::as_ref),
-                Some(DiskError::FileNotFound | DiskError::FileVersionNotFound)
-            ) || !parts_metadata.get(index).map(FileInfo::is_valid).unwrap_or(false);
-            if !meta_absent {
-                continue;
-            }
-            // Without a known shard index for this position we cannot write a
-            // consistent xl.meta; leave it for the normal heal to reconstruct.
-            let Some(&shard_index) = distribution.get(index) else {
-                continue;
-            };
-            let mut regen = surviving.clone();
-            regen.fresh = false; // merge into any existing xl.meta on the disk
-            regen.erasure.index = shard_index;
-            match disk.write_metadata("", bucket, object, regen).await {
-                Ok(()) => wrote += 1,
-                Err(e) => {
-                    warn!(
-                        bucket,
-                        object,
-                        disk_index = index,
-                        error = %e,
-                        "heal_object: failed to regenerate recoverable xl.meta on disk"
-                    );
+                match check_result {
+                    Ok(resp) if !resp.results.is_empty() && resp.results.iter().all(|result| *result == CHECK_PART_SUCCESS) => {
+                        candidate.data_count += 1;
+                    }
+                    Ok(_) => {}
+                    Err(
+                        DiskError::FileNotFound
+                        | DiskError::FileVersionNotFound
+                        | DiskError::PathNotFound
+                        | DiskError::VolumeNotFound,
+                    ) => {}
+                    Err(_) => return Ok(DanglingDeleteSafety::UnsafeToDelete),
                 }
             }
         }
 
-        if wrote == 0 {
-            return Ok(false);
-        }
-
-        info!(
-            bucket,
-            object,
-            available,
-            data_blocks,
-            regenerated_meta_disks = wrote,
-            "heal_object: rescued reconstructable sub-quorum version by regenerating xl.meta"
-        );
-        Ok(true)
+        Ok(
+            if candidates
+                .iter()
+                .any(|candidate| candidate.local_payload && candidate.data_count >= candidate.file_info.erasure.data_blocks)
+            {
+                DanglingDeleteSafety::UnsafeToDelete
+            } else {
+                DanglingDeleteSafety::NoRecoverableCandidate
+            },
+        )
     }
 
     pub(in crate::set_disk) async fn heal_object_dir_locked(
@@ -1393,7 +1537,7 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
 
 #[cfg(test)]
 mod heal_result_report_tests {
-    use super::SetDisks;
+    use super::{DanglingCheckPartsFailure, DanglingDeleteSafety, SetDisks};
     use super::{HEAL_RENAME_INCOMPLETE, HealRenameFailureScope};
     use crate::disk::endpoint::Endpoint;
     use crate::disk::error::DiskError;
@@ -1405,10 +1549,12 @@ mod heal_result_report_tests {
     use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
     use crate::{config::storageclass, store::init_format::save_format_file};
     use rustfs_common::heal_channel::{DriveState, HealOpts, HealScanMode};
-    use rustfs_filemeta::{BLOCK_SIZE_V2, FileInfo};
+    use rustfs_filemeta::{BLOCK_SIZE_V2, FileInfo, ObjectPartInfo, TRANSITION_COMPLETE};
     use std::sync::Arc;
     use tempfile::TempDir;
+    use time::OffsetDateTime;
     use tokio::sync::RwLock;
+    use uuid::Uuid;
 
     async fn real_disk() -> (TempDir, Endpoint, DiskStore) {
         let dir = tempfile::tempdir().expect("tempdir should be created");
@@ -1446,6 +1592,68 @@ mod heal_result_report_tests {
         .await
     }
 
+    fn meta_regen_test_fileinfo(object: &str, data_dir: Uuid, mod_time: i64, disk_index: usize) -> FileInfo {
+        let mut fi = FileInfo::new(object, 2, 2);
+        fi.data_dir = Some(data_dir);
+        fi.mod_time = Some(OffsetDateTime::from_unix_timestamp(mod_time).expect("test timestamp should parse"));
+        fi.size = 1;
+        fi.parts = vec![ObjectPartInfo {
+            number: 1,
+            size: 1,
+            actual_size: 1,
+            ..Default::default()
+        }];
+        fi.erasure.index = fi.erasure.distribution[disk_index];
+        fi
+    }
+
+    async fn meta_regen_test_set(
+        bucket: &str,
+        object: &str,
+        data_dirs: &[(Uuid, usize)],
+    ) -> (Vec<TempDir>, Arc<SetDisks>, Vec<Option<DiskStore>>) {
+        let mut temp_dirs = Vec::new();
+        let mut endpoints = Vec::new();
+        let mut disks = Vec::new();
+        for disk_index in 0..4 {
+            let (temp_dir, endpoint, disk) = real_disk().await;
+            disk.make_volume(bucket).await.expect("test bucket should be created");
+            for (data_dir, shard_count) in data_dirs {
+                if disk_index >= *shard_count {
+                    continue;
+                }
+                let part_dir = temp_dir.path().join(bucket).join(object).join(data_dir.to_string());
+                tokio::fs::create_dir_all(&part_dir)
+                    .await
+                    .expect("test data directory should be created");
+                tokio::fs::write(part_dir.join("part.1"), [1u8; 2])
+                    .await
+                    .expect("test data shard should be written");
+            }
+            temp_dirs.push(temp_dir);
+            endpoints.push(endpoint);
+            disks.push(Some(disk));
+        }
+
+        let set = set_disks_with(disks.clone(), endpoints, 2).await;
+        (temp_dirs, set, disks)
+    }
+
+    async fn seed_meta_regen_test_metadata(
+        disks: &[Option<DiskStore>],
+        disk_index: usize,
+        bucket: &str,
+        object: &str,
+        file_info: &FileInfo,
+    ) {
+        disks[disk_index]
+            .as_ref()
+            .expect("metadata test disk should be online")
+            .write_metadata("", bucket, object, file_info.clone())
+            .await
+            .expect("test metadata should be written");
+    }
+
     async fn formatted_single_disk_no_parity_set() -> (TempDir, Arc<SetDisks>) {
         let format = FormatV3::new(1, 1);
         let dir = tempfile::tempdir().expect("tempdir should be created");
@@ -1753,6 +1961,312 @@ mod heal_result_report_tests {
         assert_eq!(result.before.drives[3].state, DriveState::Ok.to_string());
     }
 
+    #[tokio::test]
+    async fn dangling_delete_guard_preserves_conflicting_identities_without_writing_metadata() {
+        let bucket = "bucket-delete-guard-conflict";
+        let object = "object.bin";
+        let old_data_dir = Uuid::parse_str("99999999-9999-9999-9999-999999999999").expect("old data dir should parse");
+        let new_data_dir = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa").expect("new data dir should parse");
+        let (_temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[(old_data_dir, 4), (new_data_dir, 2)]).await;
+        let version_id = Uuid::parse_str("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb").expect("version id should parse");
+        let mut metadata = vec![
+            meta_regen_test_fileinfo(object, old_data_dir, 9, 0),
+            meta_regen_test_fileinfo(object, new_data_dir, 10, 1),
+            FileInfo::default(),
+            FileInfo::default(),
+        ];
+        metadata[0].version_id = Some(version_id);
+        metadata[1].version_id = Some(version_id);
+        assert_eq!(
+            metadata[0].version_id, metadata[1].version_id,
+            "the conflicting candidates must share one version id"
+        );
+        seed_meta_regen_test_metadata(&disks, 0, bucket, object, &metadata[0]).await;
+        seed_meta_regen_test_metadata(&disks, 1, bucket, object, &metadata[1]).await;
+        let errs = vec![None, None, Some(DiskError::FileNotFound), Some(DiskError::FileNotFound)];
+
+        assert!(
+            set.dangling_delete_safety(bucket, object, &metadata, &errs, &disks)
+                .await
+                .expect("conflicting identities should be classified")
+                == DanglingDeleteSafety::UnsafeToDelete
+        );
+        let reversed = vec![
+            metadata[1].clone(),
+            metadata[0].clone(),
+            FileInfo::default(),
+            FileInfo::default(),
+        ];
+        assert!(
+            set.dangling_delete_safety(bucket, object, &reversed, &errs, &disks)
+                .await
+                .expect("reversed identities should be classified")
+                == DanglingDeleteSafety::UnsafeToDelete
+        );
+        let version_id = version_id.to_string();
+        assert!(
+            !set.try_regenerate_explicit_version_meta(bucket, object, &version_id, &metadata, &errs, &disks)
+                .await
+                .expect("conflicting explicit-version candidates should be rejected"),
+            "an explicit version must not select between conflicting metadata identities"
+        );
+        for disk_index in [2, 3] {
+            assert!(
+                matches!(
+                    disks[disk_index]
+                        .as_ref()
+                        .expect("test disk should be online")
+                        .read_version("", bucket, object, "", &ReadOptions::default())
+                        .await,
+                    Err(DiskError::FileNotFound)
+                ),
+                "the delete guard must not manufacture metadata on missing disks"
+            );
+        }
+        let old = disks[0]
+            .as_ref()
+            .expect("first test disk should be online")
+            .read_version("", bucket, object, "", &ReadOptions::default())
+            .await
+            .expect("old metadata should remain readable");
+        let new = disks[1]
+            .as_ref()
+            .expect("second test disk should be online")
+            .read_version("", bucket, object, "", &ReadOptions::default())
+            .await
+            .expect("new metadata should remain readable");
+        assert_eq!(old.data_dir, Some(old_data_dir));
+        assert_eq!(new.data_dir, Some(new_data_dir));
+    }
+
+    #[tokio::test]
+    async fn heal_meta_quorum_failure_preserves_reconstructable_uncommitted_candidate() {
+        let bucket = "bucket-delete-guard-reconstructable";
+        let object = "object.bin";
+        let data_dir = Uuid::parse_str("33333333-3333-3333-3333-333333333333").expect("data dir should parse");
+        let (_temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[(data_dir, 2)]).await;
+        let metadata = [
+            meta_regen_test_fileinfo(object, data_dir, 3, 0),
+            FileInfo::default(),
+            FileInfo::default(),
+            FileInfo::default(),
+        ];
+        seed_meta_regen_test_metadata(&disks, 0, bucket, object, &metadata[0]).await;
+        let (observed_metadata, observed_errs) = SetDisks::read_all_fileinfo(&disks, "", bucket, object, "", true, true, false)
+            .await
+            .expect("test metadata should be readable across the set");
+        assert_eq!(
+            set.dangling_delete_safety(bucket, object, &observed_metadata, &observed_errs, &disks)
+                .await
+                .expect("observed reconstructable candidate should be classified"),
+            DanglingDeleteSafety::UnsafeToDelete
+        );
+
+        let (_, err) = set
+            .heal_object(
+                bucket,
+                object,
+                "",
+                &HealOpts {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("unsafe dangling state should be reported without deletion");
+        assert_eq!(err, Some(DiskError::FileNotFound));
+        let surviving = disks[0]
+            .as_ref()
+            .expect("first test disk should be online")
+            .read_version("", bucket, object, "", &ReadOptions::default())
+            .await
+            .expect("the only metadata copy must be preserved");
+        assert_eq!(surviving.data_dir, Some(data_dir));
+        assert!(
+            matches!(
+                disks[1]
+                    .as_ref()
+                    .expect("second test disk should be online")
+                    .read_version("", bucket, object, "", &ReadOptions::default())
+                    .await,
+                Err(DiskError::FileNotFound)
+            ),
+            "the delete guard must not propagate metadata"
+        );
+    }
+
+    #[tokio::test]
+    async fn heal_meta_quorum_failure_preserves_candidate_when_required_shard_disk_is_offline() {
+        let bucket = "bucket-delete-guard-offline";
+        let object = "object.bin";
+        let data_dir = Uuid::parse_str("44444444-4444-4444-4444-444444444444").expect("data dir should parse");
+        let (temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[(data_dir, 2)]).await;
+        let metadata = meta_regen_test_fileinfo(object, data_dir, 4, 0);
+        seed_meta_regen_test_metadata(&disks, 0, bucket, object, &metadata).await;
+        set.disks.write().await[1] = None;
+
+        let (_, err) = set
+            .heal_object(
+                bucket,
+                object,
+                "",
+                &HealOpts {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("offline shard state should be reported without deletion");
+        assert_eq!(err, Some(DiskError::FileNotFound));
+        let surviving = disks[0]
+            .as_ref()
+            .expect("first test disk should be online")
+            .read_version("", bucket, object, "", &ReadOptions::default())
+            .await
+            .expect("offline uncertainty must preserve the surviving metadata");
+        assert_eq!(surviving.data_dir, Some(data_dir));
+        assert!(
+            temp_dirs[0]
+                .path()
+                .join(bucket)
+                .join(object)
+                .join(data_dir.to_string())
+                .join("part.1")
+                .is_file(),
+            "offline uncertainty must preserve the last online shard"
+        );
+    }
+
+    #[tokio::test]
+    async fn heal_meta_quorum_failure_preserves_candidate_when_part_probe_times_out() {
+        let bucket = "bucket-delete-guard-timeout";
+        let object = "object.bin";
+        let data_dir = Uuid::parse_str("55555555-5555-5555-5555-555555555555").expect("data dir should parse");
+        let (temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[(data_dir, 2)]).await;
+        let metadata = meta_regen_test_fileinfo(object, data_dir, 5, 0);
+        seed_meta_regen_test_metadata(&disks, 0, bucket, object, &metadata).await;
+        let _failure = DanglingCheckPartsFailure::install(bucket, object, 1, DiskError::Timeout);
+
+        let (_, err) = set
+            .heal_object(
+                bucket,
+                object,
+                "",
+                &HealOpts {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("part probe timeout should be reported without deletion");
+        assert_eq!(err, Some(DiskError::FileNotFound));
+        let surviving = disks[0]
+            .as_ref()
+            .expect("first test disk should be online")
+            .read_version("", bucket, object, "", &ReadOptions::default())
+            .await
+            .expect("probe uncertainty must preserve the surviving metadata");
+        assert_eq!(surviving.data_dir, Some(data_dir));
+        assert!(
+            temp_dirs[0]
+                .path()
+                .join(bucket)
+                .join(object)
+                .join(data_dir.to_string())
+                .join("part.1")
+                .is_file(),
+            "probe uncertainty must preserve the last confirmed shard"
+        );
+    }
+
+    #[tokio::test]
+    async fn dangling_delete_guard_ignores_set_incompatible_geometry() {
+        let bucket = "bucket-delete-guard-short-geometry";
+        let object = "object.bin";
+        let data_dir = Uuid::parse_str("abababab-abab-abab-abab-abababababab").expect("data dir should parse");
+        let (_temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[(data_dir, 1)]).await;
+        let mut candidate = FileInfo::new(object, 1, 0);
+        candidate.data_dir = Some(data_dir);
+        candidate.mod_time = Some(OffsetDateTime::from_unix_timestamp(18).expect("timestamp should parse"));
+        candidate.size = 1;
+        candidate.parts = vec![ObjectPartInfo {
+            number: 1,
+            size: 1,
+            actual_size: 1,
+            ..Default::default()
+        }];
+        candidate.erasure.index = candidate.erasure.distribution[0];
+        seed_meta_regen_test_metadata(&disks, 0, bucket, object, &candidate).await;
+        let metadata = vec![candidate, FileInfo::default(), FileInfo::default(), FileInfo::default()];
+        let errs = vec![
+            None,
+            Some(DiskError::FileNotFound),
+            Some(DiskError::FileNotFound),
+            Some(DiskError::FileNotFound),
+        ];
+
+        assert!(
+            set.dangling_delete_safety(bucket, object, &metadata, &errs, &disks)
+                .await
+                .expect("set-incompatible geometry should be classified")
+                == DanglingDeleteSafety::NoRecoverableCandidate
+        );
+    }
+
+    #[tokio::test]
+    async fn dangling_delete_guard_preserves_delete_marker_and_remote_metadata() {
+        let bucket = "bucket-delete-guard-nonlocal";
+        let object = "object.bin";
+        let (_temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[]).await;
+        let marker = FileInfo {
+            name: object.to_string(),
+            version_id: Some(Uuid::parse_str("eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee").expect("version id should parse")),
+            deleted: true,
+            mod_time: Some(OffsetDateTime::from_unix_timestamp(14).expect("marker timestamp should parse")),
+            ..Default::default()
+        };
+        let remote_dir = Uuid::parse_str("89898989-8989-8989-8989-898989898989").expect("remote data dir should parse");
+        let mut remote = meta_regen_test_fileinfo(object, remote_dir, 15, 1);
+        remote.transition_status = TRANSITION_COMPLETE.to_string();
+        remote.transition_tier = "WARM".to_string();
+        remote.transitioned_objname = "remote/object.bin".to_string();
+
+        for metadata in [marker, remote] {
+            let candidates = vec![metadata, FileInfo::default(), FileInfo::default(), FileInfo::default()];
+            let errs = vec![
+                None,
+                Some(DiskError::FileNotFound),
+                Some(DiskError::FileNotFound),
+                Some(DiskError::FileNotFound),
+            ];
+            assert_eq!(
+                set.dangling_delete_safety(bucket, object, &candidates, &errs, &disks)
+                    .await
+                    .expect("non-local metadata should be classified"),
+                DanglingDeleteSafety::UnsafeToDelete
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn dangling_delete_guard_preserves_metadata_read_uncertainty() {
+        let bucket = "bucket-delete-guard-read-error";
+        let object = "object.bin";
+        let (_temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[]).await;
+        let metadata = vec![FileInfo::default(); disks.len()];
+
+        for read_error in [DiskError::Timeout, DiskError::DiskAccessDenied, DiskError::DiskNotFound] {
+            let mut errs = vec![Some(DiskError::FileNotFound); disks.len()];
+            errs[0] = Some(read_error);
+            assert_eq!(
+                set.dangling_delete_safety(bucket, object, &metadata, &errs, &disks)
+                    .await
+                    .expect("metadata read uncertainty should be classified"),
+                DanglingDeleteSafety::UnsafeToDelete
+            );
+        }
+    }
+
     #[tokio::test]
     async fn heal_no_parity_bitrot_reports_unrecoverable_integrity_failure() {
         let (dir, set) = formatted_single_disk_no_parity_set().await;

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -27,7 +27,7 @@ use crate::multipart_listing::paginate_multipart_listing;
 use futures::{StreamExt, stream};
 use std::future::Future;
 #[cfg(test)]
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::atomic::{AtomicUsize, Ordering};
 use std::time::Duration;
 use tokio::task::JoinSet;
 
@@ -36,6 +36,7 @@ const MULTIPART_LIST_IO_CONCURRENCY: usize = 16;
 #[cfg(test)]
 #[derive(Clone, Copy, PartialEq, Eq)]
 pub(crate) enum MultipartCommitPause {
+    PutPartBeforeLockAcquire,
     PutPartBeforeLockLost,
     PutPartAfterRename,
     BeforeLockLost,
@@ -47,9 +48,10 @@ struct MultipartCommitBarrierState {
     bucket: String,
     object: String,
     pause: MultipartCommitPause,
-    armed: AtomicBool,
+    expected_arrivals: usize,
+    arrivals: AtomicUsize,
     arrived: tokio::sync::Notify,
-    release: tokio::sync::Notify,
+    release: tokio::sync::Semaphore,
 }
 
 #[cfg(test)]
@@ -64,13 +66,24 @@ static MULTIPART_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc
 #[cfg(test)]
 impl MultipartCommitBarrier {
     pub(crate) fn install(bucket: &str, object: &str, pause: MultipartCommitPause) -> Self {
+        Self::install_for_arrivals(bucket, object, pause, 1)
+    }
+
+    pub(crate) fn install_for_arrivals(
+        bucket: &str,
+        object: &str,
+        pause: MultipartCommitPause,
+        expected_arrivals: usize,
+    ) -> Self {
+        assert!(expected_arrivals > 0, "multipart commit barrier must wait for at least one arrival");
         let state = Arc::new(MultipartCommitBarrierState {
             bucket: bucket.to_string(),
             object: object.to_string(),
             pause,
-            armed: AtomicBool::new(true),
+            expected_arrivals,
+            arrivals: AtomicUsize::new(0),
             arrived: tokio::sync::Notify::new(),
-            release: tokio::sync::Notify::new(),
+            release: tokio::sync::Semaphore::new(0),
         });
         let mut slot = MULTIPART_COMMIT_BARRIER
             .get_or_init(|| std::sync::Mutex::new(None))
@@ -83,20 +96,28 @@ impl MultipartCommitBarrier {
     }
 
     pub(crate) async fn wait_until_paused(&self) {
-        tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
-            .await
-            .expect("multipart completion should reach the deterministic commit barrier");
+        tokio::time::timeout(Duration::from_secs(30), async {
+            loop {
+                let arrived = self.state.arrived.notified();
+                if self.state.arrivals.load(Ordering::Acquire) >= self.state.expected_arrivals {
+                    return;
+                }
+                arrived.await;
+            }
+        })
+        .await
+        .expect("multipart completion should reach the deterministic commit barrier");
     }
 
     pub(crate) fn release(&self) {
-        self.state.release.notify_one();
+        self.state.release.add_permits(self.state.expected_arrivals);
     }
 }
 
 #[cfg(test)]
 impl Drop for MultipartCommitBarrier {
     fn drop(&mut self) {
-        self.state.release.notify_one();
+        self.release();
         let mut slot = MULTIPART_COMMIT_BARRIER
             .get_or_init(|| std::sync::Mutex::new(None))
             .lock()
@@ -117,10 +138,20 @@ async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartComm
         .filter(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause == pause)
         .cloned();
     if let Some(barrier) = barrier
-        && barrier.armed.swap(false, Ordering::AcqRel)
+        && let Ok(previous) = barrier.arrivals.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| {
+            (current < barrier.expected_arrivals).then_some(current + 1)
+        })
     {
-        barrier.arrived.notify_one();
-        barrier.release.notified().await;
+        let arrival = previous + 1;
+        if arrival == barrier.expected_arrivals {
+            barrier.arrived.notify_one();
+        }
+        barrier
+            .release
+            .acquire()
+            .await
+            .expect("multipart commit barrier should remain open")
+            .forget();
     }
 }
 
@@ -693,6 +724,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
 
             let part_path = format!("{}/{}/{}", upload_id_path, fi.data_dir.unwrap_or_default(), part_suffix);
 
+            #[cfg(test)]
+            pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockAcquire).await;
             // Serialize only the commit (rename_part), not the whole upload. Each
             // concurrent stream writes to its own unique temp dir (see `tmp_part`
             // above), so the encode/stream phase never conflicts and must stay

crates/ecstore/src/set_disk/ops/object.rs

@@ -42,6 +42,7 @@ use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppress
 use crate::services::tier::tier::{TierConfigMgr, TierOperationLease};
 use crate::store::ECStore;
 use futures::FutureExt as _;
+use http::HeaderValue;
 use std::future::Future;
 
 fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result<coding::Erasure> {
@@ -49,6 +50,17 @@ fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result<coding::Er
         .map_err(Error::from)
 }
 
+async fn get_object_reader_with_context(
+    ctx: &InstanceContext,
+    reader: Box<dyn AsyncRead + Unpin + Send + Sync>,
+    range: Option<HTTPRangeSpec>,
+    object_info: &ObjectInfo,
+    opts: &ObjectOptions,
+    headers: &HeaderMap<HeaderValue>,
+) -> Result<(GetObjectReader, usize, i64)> {
+    GetObjectReader::new_with_resolver(reader, range, object_info, opts, headers, ctx.object_encryption_resolver()).await
+}
+
 /// Length of the full plaintext body when — and only when — this read's output
 /// is exactly the object's complete plaintext, so the app-layer body cache may
 /// serve it in place of the erasure read.
@@ -707,7 +719,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
                             size_bucket,
                         );
                         record_get_object_reader_path_observation(GET_OBJECT_PATH_CODEC_STREAMING, object_class, size_bucket);
-                        let (mut reader, _offset, _length) = GetObjectReader::new(stream, range, &object_info, opts, &h).await?;
+                        let (mut reader, _offset, _length) =
+                            get_object_reader_with_context(&self.ctx, stream, range, &object_info, opts, &h).await?;
                         // Carry the hook probe result so the app layer skips its
                         // now-redundant lookup on the streaming miss path (ODC-16).
                         reader.body_source = body_source;
@@ -745,7 +758,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
         let (rd, wd) = tokio::io::duplex(duplex_buffer_size);
         debug!(bucket, object, duplex_buffer_size, "Created duplex pipe for object data transfer");
 
-        let (mut reader, offset, length) = GetObjectReader::new(Box::new(rd), range, &object_info, opts, &h).await?;
+        let (mut reader, offset, length) =
+            get_object_reader_with_context(&self.ctx, Box::new(rd), range, &object_info, opts, &h).await?;
         // Carry the hook probe result so the app layer skips its now-redundant
         // lookup on the streaming miss path (ODC-16).
         reader.body_source = body_source;
@@ -2315,23 +2329,50 @@ async fn pause_transition_commit(bucket: &str, object: &str, pause: TransitionCo
 
 fn persisted_transition_version(
     remote_version: &str,
+) -> std::io::Result<(Option<String>, rustfs_filemeta::TransitionVersionState)> {
+    persisted_transition_version_with_gate(remote_version, remote_version_state_writer_enabled())
+}
+
+fn remote_version_state_writer_enabled() -> bool {
+    remote_version_state_writer_enabled_for(
+        rustfs_utils::get_env_bool(
+            rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE,
+            rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE,
+        ),
+        rustfs_utils::get_env_bool(
+            rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
+            rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
+        ),
+    )
+}
+
+fn remote_version_state_writer_enabled_for(requested: bool, fleet_confirmed: bool) -> bool {
+    requested && fleet_confirmed
+}
+
+fn persisted_transition_version_with_gate(
+    remote_version: &str,
+    remote_version_state_writer_enabled: bool,
 ) -> std::io::Result<(Option<String>, rustfs_filemeta::TransitionVersionState)> {
     if remote_version.is_empty() {
         return Ok((None, rustfs_filemeta::TransitionVersionState::KnownDisabled));
     }
-    let version_id = Uuid::parse_str(remote_version).map_err(|_| {
-        std::io::Error::new(
-            std::io::ErrorKind::Unsupported,
-            "opaque remote tier versions require the cluster capability gate",
-        )
-    })?;
-    if version_id.is_nil() {
-        return Err(std::io::Error::new(
+
+    match Uuid::parse_str(remote_version) {
+        Ok(version_id) if version_id.is_nil() => Err(std::io::Error::new(
             std::io::ErrorKind::InvalidData,
             "remote tier returned a nil object version ID",
-        ));
+        )),
+        Ok(_) => Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::Exact)),
+        Err(_) if !remote_version_state_writer_enabled => Err(std::io::Error::new(
+            std::io::ErrorKind::Unsupported,
+            "opaque remote tier versions require the operator-attested fleet gate",
+        )),
+        Err(_) if remote_version == "null" => {
+            Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::SuspendedNull))
+        }
+        Err(_) => Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::Exact)),
     }
-    Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::Exact))
 }
 
 #[cfg(test)]
@@ -2569,7 +2610,10 @@ mod transition_upload_completion_tests {
 
 #[cfg(test)]
 mod transition_version_id_tests {
-    use super::{TransitionUploadCandidate, persisted_transition_version};
+    use super::{
+        TransitionUploadCandidate, persisted_transition_version, persisted_transition_version_with_gate,
+        remote_version_state_writer_enabled_for,
+    };
     use rustfs_filemeta::TransitionVersionState;
     use uuid::Uuid;
 
@@ -2608,6 +2652,44 @@ mod transition_version_id_tests {
             "opaque-version-token"
         );
     }
+
+    #[test]
+    fn remote_version_state_writer_requires_request_and_fleet_confirmation() {
+        for (case, requested, fleet_confirmed, expected) in [
+            ("old defaults", false, false, false),
+            ("missing fleet confirmation", true, false, false),
+            ("missing local opt-in", false, true, false),
+            ("explicitly unconfirmed fleet", true, false, false),
+            ("rolled-back writer", false, true, false),
+            ("fully upgraded fleet", true, true, true),
+        ] {
+            assert_eq!(remote_version_state_writer_enabled_for(requested, fleet_confirmed), expected, "{case}");
+        }
+    }
+
+    #[test]
+    fn fleet_gate_enables_null_and_opaque_remote_version_states() {
+        for (remote_version, expected) in [
+            ("null", (Some("null".to_string()), TransitionVersionState::SuspendedNull)),
+            (
+                "opaque-version-token",
+                (Some("opaque-version-token".to_string()), TransitionVersionState::Exact),
+            ),
+        ] {
+            assert!(
+                persisted_transition_version_with_gate(remote_version, false).is_err(),
+                "missing fleet confirmation must reject {remote_version:?}"
+            );
+            assert_eq!(
+                persisted_transition_version_with_gate(remote_version, true).expect("fleet-confirmed state must be persisted"),
+                expected
+            );
+        }
+        assert_eq!(
+            persisted_transition_version_with_gate("", true).expect("empty remote version identifies an unversioned tier"),
+            (None, TransitionVersionState::KnownDisabled)
+        );
+    }
 }
 
 impl SetDisks {
@@ -4386,6 +4468,61 @@ mod erasure_construction_tests {
     }
 }
 
+#[cfg(test)]
+mod object_encryption_resolver_wiring_tests {
+    use super::*;
+    use crate::object_api::{EncryptionResolutionError, ObjectEncryptionResolver, ReadEncryptionMaterial, ReadEncryptionRequest};
+    use std::io::Cursor;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+
+    struct CountingResolver {
+        calls: AtomicUsize,
+    }
+
+    #[async_trait::async_trait]
+    impl ObjectEncryptionResolver for CountingResolver {
+        async fn resolve_read_material(
+            &self,
+            _request: ReadEncryptionRequest<'_>,
+        ) -> std::result::Result<Option<ReadEncryptionMaterial>, EncryptionResolutionError> {
+            self.calls.fetch_add(1, Ordering::Relaxed);
+            Ok(None)
+        }
+    }
+
+    #[tokio::test]
+    async fn get_object_reader_forwards_instance_resolver() {
+        let resolver = Arc::new(CountingResolver {
+            calls: AtomicUsize::new(0),
+        });
+        let ctx = InstanceContext::new();
+        assert!(
+            ctx.set_object_encryption_resolver(resolver.clone()).is_ok(),
+            "fresh context should accept resolver"
+        );
+        let object_info = ObjectInfo {
+            bucket: "bucket".to_string(),
+            name: "object".to_string(),
+            size: 1,
+            user_defined: Arc::new(HashMap::from([("x-amz-server-side-encryption".to_string(), "AES256".to_string())])),
+            ..Default::default()
+        };
+
+        let result = get_object_reader_with_context(
+            &ctx,
+            Box::new(Cursor::new(Vec::<u8>::new())),
+            None,
+            &object_info,
+            &ObjectOptions::default(),
+            &HeaderMap::new(),
+        )
+        .await;
+
+        assert!(result.is_err(), "resolver returning no material must fail closed");
+        assert_eq!(resolver.calls.load(Ordering::Relaxed), 1);
+    }
+}
+
 #[cfg(test)]
 pub(in crate::set_disk::ops) mod hermetic_set_disks_support {
     //! Shared hermetic `SetDisks` construction for the ops tests below: the
@@ -7166,7 +7303,6 @@ mod transition_source_identity_matrix_tests {
     async fn transition_source_identity_field_matrix_rejects_single_field_drift() {
         #[derive(Clone, Copy, Debug)]
         enum IdentityField {
-            VersionId,
             DataDir,
             ModTime,
             Size,
@@ -7182,7 +7318,6 @@ mod transition_source_identity_matrix_tests {
         let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
 
         for (index, field) in [
-            IdentityField::VersionId,
             IdentityField::DataDir,
             IdentityField::ModTime,
             IdentityField::Size,
@@ -7236,10 +7371,6 @@ mod transition_source_identity_matrix_tests {
 
             let mut changed = source.clone();
             match field {
-                IdentityField::VersionId => {
-                    changed.version_id = Some(Uuid::new_v4());
-                    changed.fresh = true;
-                }
                 IdentityField::DataDir => changed.data_dir = Some(Uuid::new_v4()),
                 IdentityField::ModTime => {
                     changed.mod_time = changed.mod_time.map(|value| value + time::Duration::nanoseconds(1));
@@ -7280,15 +7411,12 @@ mod transition_source_identity_matrix_tests {
             );
 
             match field {
-                IdentityField::VersionId => assert_ne!(source.version_id, persisted.version_id),
                 IdentityField::DataDir => assert_ne!(source.data_dir, persisted.data_dir),
                 IdentityField::ModTime => assert_ne!(source.mod_time, persisted.mod_time),
                 IdentityField::Size => assert_ne!(source.size, persisted.size),
                 IdentityField::Etag => assert_ne!(get_raw_etag(&source.metadata), get_raw_etag(&persisted.metadata)),
             }
-            if !matches!(field, IdentityField::VersionId) {
-                assert_eq!(source.version_id, persisted.version_id);
-            }
+            assert_eq!(source.version_id, persisted.version_id);
             if !matches!(field, IdentityField::DataDir) {
                 assert_eq!(source.data_dir, persisted.data_dir);
             }

crates/ecstore/src/store/bucket.rs

@@ -87,7 +87,7 @@ fn bucket_deleted_marker_volume(bucket: &str) -> String {
     format!("{RUSTFS_META_BUCKET}/{}", bucket_deleted_marker_prefix(bucket))
 }
 
-async fn await_bucket_namespace_operation<T, F>(
+pub(crate) async fn await_bucket_namespace_operation<T, F>(
     guard: Option<&rustfs_lock::NamespaceLockGuard>,
     bucket: &str,
     operation: &'static str,

crates/ecstore/src/store/mod.rs

@@ -141,6 +141,7 @@ fn should_enqueue_transition_immediately(oi: &ObjectInfo) -> bool {
 const MAX_UPLOADS_LIST: usize = 10000;
 
 mod bucket;
+pub(crate) use bucket::await_bucket_namespace_operation;
 mod heal;
 mod heal_walk;
 pub use heal_walk::HealWalkVersion;

crates/ecstore/tests/README.md

@@ -2,7 +2,7 @@
 
 ## MinIO-generated encrypted fixtures
 
-`minio_generated_read_test.rs` validates the `bitrot -> GetObjectReader` path against raw MinIO backend data captured by
+`rustfs/src/storage/minio_generated_read_test.rs` validates the `bitrot -> GetObjectReader` path against raw MinIO backend data captured by
 `.\rustfs\scripts\minio_fixture_lab\lab.py`.
 
 It currently covers multipart fixtures for:
@@ -20,5 +20,5 @@ Example:
 ```powershell
 $env:RUSTFS_MINIO_FIXTURE_ROOT = '.\rustfs\tmp\minio-fixture-lab-local-key'
 $env:RUSTFS_MINIO_STATIC_KMS_KEY_B64 = '<base64-32-byte-local-minio-kms-key>'
-cargo +1.97.1 test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored
+cargo +1.97.1 test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored
 ```

crates/filemeta/src/filemeta.rs

@@ -356,6 +356,14 @@ impl FileMeta {
             .versions
             .partition_point(|existing| existing.header.sorts_before(&new_shallow.header));
         self.versions.insert(insert_pos, new_shallow);
+        // `partition_point` only returns the canonical slot when `versions` is
+        // already canonically ordered, and nothing establishes that: `FileMeta::load`
+        // replays the on-disk order verbatim, and metadata written before the
+        // canonical-order fix ordered equal-`mod_time` ties by insertion rather than
+        // by `sorts_before`. Re-sort so an insert leaves the list canonical either
+        // way, matching what `set_idx` already does on the replace path. Cheap: after
+        // a correctly placed insert the slice is a single sorted run.
+        self.sort_by_mod_time();
         Ok(())
 
         // if !ver.valid() {
@@ -1215,6 +1223,57 @@ mod test {
         assert!(fm.versions[0].header.sorts_before(&fm.versions[1].header));
     }
 
+    /// `add_version_filemata` positions an inserted version with
+    /// `partition_point(sorts_before)`, which only yields the canonical slot when
+    /// `versions` is already canonically ordered. Nothing establishes that
+    /// precondition on load: `FileMeta::unmarshal_msg` pushes versions in file
+    /// order, and metadata written before the canonical-order fix ordered
+    /// equal-`mod_time` ties by insertion instead of by `sorts_before`. An insert
+    /// into such a list must still leave it canonical, the same way `set_idx`
+    /// already re-sorts on the replace path.
+    #[test]
+    fn add_version_filemata_canonicalizes_versions_loaded_out_of_order() {
+        let mod_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid test timestamp");
+        let first = version_for_ordering(VersionType::Object, Uuid::from_u128(70), mod_time, 70);
+        let second = version_for_ordering(VersionType::Object, Uuid::from_u128(80), mod_time, 80);
+        let (early, late) = if first.header().sorts_before(&second.header()) {
+            (first, second)
+        } else {
+            (second, first)
+        };
+
+        // Persist the pair the wrong way round to emulate a pre-canonical-order
+        // xl.meta, bypassing add_version_filemata so the bad order really lands on
+        // the wire.
+        let mut legacy = FileMeta::new();
+        legacy
+            .versions
+            .push(FileMetaShallowVersion::try_from(late).expect("shallow later version"));
+        legacy
+            .versions
+            .push(FileMetaShallowVersion::try_from(early).expect("shallow earlier version"));
+
+        let mut loaded =
+            FileMeta::load(&legacy.marshal_msg().expect("serialize legacy metadata")).expect("reload legacy metadata");
+        assert!(
+            !loaded.versions[0].header.sorts_before(&loaded.versions[1].header),
+            "fixture must reach add_version_filemata non-canonically ordered"
+        );
+
+        loaded
+            .add_version_filemata(version_for_ordering(VersionType::Delete, Uuid::from_u128(90), mod_time, 90))
+            .expect("insert equal-time delete marker");
+
+        assert_eq!(loaded.versions.len(), 3);
+        assert!(
+            loaded
+                .versions
+                .windows(2)
+                .all(|pair| pair[0].header.sorts_before(&pair[1].header)),
+            "insert must leave the version list canonically ordered"
+        );
+    }
+
     /// Regression for backlog#799 B16: replication reset state must be
     /// persisted under both internal prefixes, never as a bare ARN. A bare-ARN
     /// key (produced by `ObjectInfo::replication_state`) has no internal prefix,

crates/heal/Cargo.toml

@@ -26,6 +26,9 @@ documentation = "https://docs.rs/rustfs-heal/latest/rustfs_heal/"
 keywords = ["RustFS", "heal", "erasure-coding", "Minio"]
 categories = ["web-programming", "development-tools", "filesystem"]
 
+[lints]
+workspace = true
+
 [dependencies]
 rustfs-config = { workspace = true }
 rustfs-concurrency = { workspace = true }
@@ -53,7 +56,7 @@ serial_test = { workspace = true }
 tempfile = { workspace = true }
 walkdir = { workspace = true }
 http = { workspace = true }
-temp-env = { workspace = true }
+temp-env = { workspace = true, features = ["async_closure"] }
 tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
 
 [lib]

crates/heal/src/heal/channel.rs

@@ -482,7 +482,7 @@ impl HealChannelProcessor {
                     request_id: client_token,
                     success: false,
                     data: None,
-                    error: Some(error_text.clone()),
+                    error: Some(error_text),
                 };
                 let _ = response_tx.send(Ok(response.clone()));
                 self.publish_response(response);

crates/heal/src/heal/event.rs

@@ -576,7 +576,7 @@ mod tests {
         assert_eq!(handler.event_count(), 1);
 
         handler.add_event(event.clone());
-        handler.add_event(event.clone());
+        handler.add_event(event);
         assert_eq!(handler.event_count(), 3);
     }
 
@@ -593,7 +593,7 @@ mod tests {
 
         handler.add_event(event.clone());
         handler.add_event(event.clone());
-        handler.add_event(event.clone()); // Should remove oldest
+        handler.add_event(event); // Should remove oldest
 
         assert_eq!(handler.event_count(), 2);
     }
@@ -610,7 +610,7 @@ mod tests {
         };
 
         handler.add_event(event.clone());
-        handler.add_event(event.clone());
+        handler.add_event(event);
 
         let events = handler.get_events();
         assert_eq!(events.len(), 2);

crates/heal/src/heal/manager.rs

@@ -2892,7 +2892,7 @@ fn update_task_running_metric_for_task(active_heals: &HashMap<String, Arc<HealTa
     gauge!(
         "rustfs_heal_task_running",
         "type" => type_label.to_string(),
-        "set" => set_label.clone()
+        "set" => set_label
     )
     .set(count as f64);
 }
@@ -3482,7 +3482,7 @@ mod tests {
         );
 
         assert_eq!(queue.push(blocked), QueuePushOutcome::Accepted);
-        assert_eq!(queue.push(runnable.clone()), QueuePushOutcome::Accepted);
+        assert_eq!(queue.push(runnable), QueuePushOutcome::Accepted);
 
         let mut running = HashMap::new();
         running.insert("pool_0_set_1".to_string(), 1);

crates/heal/tests/heal_b920_subquorum_union_test.rs

@@ -28,6 +28,7 @@ use rustfs_heal::heal::storage::{
 };
 use serial_test::serial;
 use std::{
+    future::Future,
     path::{Path, PathBuf},
     sync::Arc,
 };
@@ -52,15 +53,9 @@ fn versioned_test_data(seed: u8) -> Vec<u8> {
         .collect()
 }
 
-/// Disable the dangling-delete grace window so the destructive path is genuinely
-/// LIVE in these tests: without the decision-1 guard, a recoverable version WOULD
-/// be dangling-deleted here. With grace at its 1h default the delete path would be
-/// masked and the test would prove nothing.
-fn disable_dangling_grace() {
-    // Safe under nextest: each test runs in its own process and is `#[serial]`.
-    unsafe {
-        std::env::set_var(GRACE_ENV, "0");
-    }
+/// Disable the grace window while the destructive heal decision is evaluated.
+async fn with_dangling_grace_disabled<T>(future: impl Future<Output = T>) -> T {
+    temp_env::async_with_vars([(GRACE_ENV, Some("0"))], future).await
 }
 
 /// Build a real N-disk single-set `ECStore` + `ECStoreHealStorage` via the
@@ -250,7 +245,6 @@ mod serial_tests {
     #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
     #[serial]
     async fn union_meta_lost_data_present_is_repaired_not_destroyed() {
-        disable_dangling_grace();
         let (disk_paths, ecstore, heal_storage) = heal_env_n(8).await;
         let bucket = "b920-meta-lost";
         let object = "obj.bin";
@@ -268,10 +262,10 @@ mod serial_tests {
         }
 
         // Heal the version through the real heal storage (Deep).
-        let (_result, error) = heal_storage
-            .heal_object(bucket, object, Some(&v1), &deep_heal_opts())
-            .await
-            .expect("heal_object call must not itself error");
+        let (_result, error) =
+            with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &deep_heal_opts()))
+                .await
+                .expect("heal_object call must not itself error");
         assert!(
             error.is_none(),
             "recoverable version must heal without error (must NOT be dangling-deleted): {error:?}"
@@ -300,7 +294,6 @@ mod serial_tests {
     #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
     #[serial]
     async fn deep_heal_torn_minority_is_dangling_deleted_with_grace_zero() {
-        disable_dangling_grace();
         let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
         let bucket = "b920-torn";
         let object = "obj.bin";
@@ -318,10 +311,10 @@ mod serial_tests {
         // is a real destructive action, not a no-op).
         assert_eq!(count_part_files(&object_dir(&disk_paths[0], bucket, object)), 1);
 
-        let (_result, error) = heal_storage
-            .heal_object(bucket, object, Some(&v1), &deep_heal_opts())
-            .await
-            .expect("heal_object call must not itself error");
+        let (_result, error) =
+            with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &deep_heal_opts()))
+                .await
+                .expect("heal_object call must not itself error");
         // A dangling delete reports the version as gone (FileVersionNotFound),
         // proving the destructive path fired for a genuinely torn write.
         assert!(error.is_some(), "a torn (< data_blocks) version must NOT be silently treated as healed");
@@ -349,7 +342,6 @@ mod serial_tests {
     #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
     #[serial]
     async fn deep_heal_restores_subquorum_but_reconstructable_version_wider_set() {
-        disable_dangling_grace();
         let (disk_paths, ecstore, heal_storage) = heal_env_n(8).await;
         let bucket = "b920-reconstruct";
         let object = "obj.bin";
@@ -373,10 +365,10 @@ mod serial_tests {
             "disk-walk must enumerate the reconstructable sub-quorum version"
         );
 
-        let (_result, error) = heal_storage
-            .heal_object(bucket, object, Some(&v1), &deep_heal_opts())
-            .await
-            .expect("heal_object call must not itself error");
+        let (_result, error) =
+            with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &deep_heal_opts()))
+                .await
+                .expect("heal_object call must not itself error");
         assert!(error.is_none(), "reconstructable version must heal cleanly: {error:?}");
 
         // part.* + xl.meta physically restored on the 4 wiped disks.
@@ -463,7 +455,6 @@ mod serial_tests {
     #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
     #[serial]
     async fn offline_disk_during_walk_does_not_dangling_delete() {
-        disable_dangling_grace();
         let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
         let bucket = "b920-offline";
         let object = "obj.bin";
@@ -491,8 +482,7 @@ mod serial_tests {
             remove: false,
             ..Default::default()
         };
-        let (_result, error) = heal_storage
-            .heal_object(bucket, object, Some(&v1), &normal_opts)
+        let (_result, error) = with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &normal_opts))
             .await
             .expect("heal_object call must not itself error");
         assert!(error.is_none(), "quorum-present object must not be destroyed: {error:?}");
@@ -510,7 +500,6 @@ mod serial_tests {
     #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
     #[serial]
     async fn deep_heal_keeps_present_ec2_plus_2_shards_healthy() {
-        disable_dangling_grace();
         let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
         let bucket = "b1044-deep-verify";
         let object = "obj.bin";
@@ -524,10 +513,10 @@ mod serial_tests {
             assert_eq!(count_part_files(&object_dir(disk, bucket, object)), 1, "intact shard must remain present");
         }
 
-        let (_result, error) = heal_storage
-            .heal_object(bucket, object, Some(&v1), &deep_heal_opts())
-            .await
-            .expect("deep heal_object call must not itself error");
+        let (_result, error) =
+            with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &deep_heal_opts()))
+                .await
+                .expect("deep heal_object call must not itself error");
         assert!(error.is_none(), "Deep heal must retain the three intact EC2+2 shards: {error:?}");
 
         assert!(

crates/heal/tests/heal_integration_test.rs

@@ -390,8 +390,10 @@ mod serial_tests {
 
         // ─── 1️⃣ delete format.json on one disk ──────────────
         let format_path = disk_paths[0].join(".rustfs.sys").join("format.json");
-        std::fs::remove_dir_all(&disk_paths[0]).expect("failed to delete all contents under disk_paths[0]");
+        let failed_disk_path = disk_paths[0].with_extension("failed");
+        std::fs::rename(&disk_paths[0], &failed_disk_path).expect("failed to detach disk_paths[0]");
         std::fs::create_dir_all(&disk_paths[0]).expect("failed to recreate disk_paths[0] directory");
+        assert!(!format_path.exists(), "replacement disk already contains format.json before heal");
         println!("✅ Deleted format.json on disk: {:?}", disk_paths[0]);
 
         let (_format_result, format_error) = heal_storage.heal_format(false).await.expect("failed to run heal_format");

crates/kms/AGENTS.md

@@ -25,3 +25,25 @@ For local KMS end-to-end tests, keep proxy bypass settings:
 NO_PROXY=127.0.0.1,localhost HTTP_PROXY= HTTPS_PROXY= http_proxy= https_proxy= \
 cargo test --package e2e_test test_local_kms_end_to_end -- --nocapture --test-threads=1
 ```
+
+## Local Key Export for SSE-S3 Migration Tests
+
+Use the read-only `local_kms_key_decrypt` example to export an AES-256 Local
+KMS key as the base64 value expected by `RUSTFS_SSE_S3_MASTER_KEY`:
+
+```bash
+export RUSTFS_KMS_LOCAL_MASTER_KEY='<local-kms-at-rest-master-key>'
+export RUSTFS_SSE_S3_MASTER_KEY="$(
+  cargo run -q -p rustfs-kms --example local_kms_key_decrypt -- \
+    /absolute/path/to/<key-id>.key
+)"
+```
+
+For a `plaintext-dev-only` Local KMS key file,
+`RUSTFS_KMS_LOCAL_MASTER_KEY` is not required.
+
+The example writes only the base64-encoded 32-byte key to stdout. Diagnostics
+go to stderr. Never paste its output into logs, shell history, issue comments,
+or committed configuration. The export path must remain read-only and must
+reuse `LocalKmsClient` decoding so current Argon2id and legacy key-file
+compatibility stay aligned with the backend.

crates/kms/Cargo.toml

@@ -44,7 +44,9 @@ argon2 = { workspace = true }
 chacha20poly1305 = { workspace = true }
 rand = { workspace = true, features = ["serde"] }
 base64 = { workspace = true }
+hex = { workspace = true }
 sha2 = { workspace = true }
+subtle = { workspace = true }
 zeroize = { workspace = true, features = ["derive"] }
 
 # Configuration and storage
@@ -55,7 +57,7 @@ tempfile = { workspace = true }
 moka = { workspace = true, features = ["future"] }
 
 # Additional dependencies
-md5 = { workspace = true }
+md-5 = { workspace = true }
 arc-swap = { workspace = true }
 rustfs-utils = { workspace = true }
 rustfs-security-governance = { workspace = true }

crates/kms/examples/local_kms_key_decrypt.rs

@@ -0,0 +1,112 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
+use rustfs_kms::{LocalConfig, backends::local::LocalKmsClient};
+use std::io::{self, Write};
+use std::path::{Path, PathBuf};
+use zeroize::Zeroizing;
+
+const LOCAL_KMS_MASTER_KEY_ENV: &str = "RUSTFS_KMS_LOCAL_MASTER_KEY";
+
+fn usage(program: &str) -> String {
+    format!(
+        "Usage: {program} <local-kms-key-file>\n\
+         Reads {LOCAL_KMS_MASTER_KEY_ENV} when the key file is encrypted.\n\
+         Writes only the base64-encoded 32-byte key to stdout."
+    )
+}
+
+fn resolve_key_file(path: &Path) -> Result<(PathBuf, String), String> {
+    let canonical = std::fs::canonicalize(path).map_err(|error| format!("cannot open Local KMS key file: {error}"))?;
+    if canonical.extension().and_then(|extension| extension.to_str()) != Some("key") {
+        return Err("Local KMS key file must have a .key extension".to_string());
+    }
+    let key_dir = canonical
+        .parent()
+        .ok_or_else(|| "Local KMS key file must have a parent directory".to_string())?
+        .to_path_buf();
+    let key_id = canonical
+        .file_stem()
+        .and_then(|stem| stem.to_str())
+        .filter(|stem| !stem.is_empty())
+        .ok_or_else(|| "Local KMS key file name must contain a valid UTF-8 key ID".to_string())?
+        .to_string();
+    Ok((key_dir, key_id))
+}
+
+async fn run() -> Result<(), String> {
+    let mut args = std::env::args();
+    let program = args.next().unwrap_or_else(|| "local_kms_key_decrypt".to_string());
+    let Some(key_file) = args.next() else {
+        return Err(usage(&program));
+    };
+    if args.next().is_some() {
+        return Err(usage(&program));
+    }
+
+    let (key_dir, key_id) = resolve_key_file(Path::new(&key_file))?;
+    let master_key = std::env::var(LOCAL_KMS_MASTER_KEY_ENV).ok().filter(|value| !value.is_empty());
+    let client = LocalKmsClient::new_for_key_export(LocalConfig {
+        key_dir,
+        master_key,
+        file_permissions: Some(0o600),
+    })
+    .await
+    .map_err(|error| error.to_string())?;
+    let key_material = client
+        .decrypt_key_material_for_export(&key_id)
+        .await
+        .map_err(|error| error.to_string())?;
+    let encoded = Zeroizing::new(BASE64_STANDARD.encode(key_material.as_ref()));
+
+    let mut stdout = io::stdout().lock();
+    writeln!(stdout, "{}", encoded.as_str()).map_err(|error| format!("failed to write decrypted key: {error}"))
+}
+
+#[tokio::main]
+async fn main() {
+    if let Err(error) = run().await {
+        let _ = writeln!(io::stderr().lock(), "local_kms_key_decrypt: {error}");
+        std::process::exit(1);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn resolve_key_file_extracts_directory_and_key_id() {
+        let directory = tempfile::tempdir().expect("create temporary directory");
+        let key_file = directory.path().join("migration-key.key");
+        std::fs::write(&key_file, b"{}").expect("create key file");
+
+        let (key_dir, key_id) = resolve_key_file(&key_file).expect("resolve key file");
+
+        assert_eq!(key_dir, directory.path().canonicalize().expect("canonical directory"));
+        assert_eq!(key_id, "migration-key");
+    }
+
+    #[test]
+    fn resolve_key_file_rejects_non_key_extension() {
+        let directory = tempfile::tempdir().expect("create temporary directory");
+        let key_file = directory.path().join("migration-key.json");
+        std::fs::write(&key_file, b"{}").expect("create key file");
+
+        let error = resolve_key_file(&key_file).expect_err("non-key file must be rejected");
+
+        assert!(error.contains(".key"));
+    }
+}

crates/kms/src/backends/local.rs

@@ -35,8 +35,8 @@ use std::collections::HashMap;
 use std::path::{Component, Path, PathBuf};
 use std::time::Duration;
 use tokio::fs;
-use tokio::sync::RwLock;
 use tracing::{debug, warn};
+use zeroize::Zeroizing;
 
 /// Reject key identifiers that would not name a single file directly inside the key
 /// directory.
@@ -77,8 +77,6 @@ const LOCAL_KMS_ARGON2_P_COST: u32 = 1;
 /// Local KMS client that stores keys in local files
 pub struct LocalKmsClient {
     config: LocalConfig,
-    /// In-memory cache of loaded keys for performance
-    key_cache: RwLock<HashMap<String, MasterKeyInfo>>,
     /// Master encryption key for encrypting stored keys
     master_cipher: Option<Aes256Gcm>,
     /// Legacy pre-beta.9 master cipher for reading pre-Argon2 key files
@@ -139,9 +137,49 @@ impl LocalKmsClient {
             (None, None)
         };
 
+        let client = Self {
+            config,
+            master_cipher,
+            legacy_master_cipher,
+            dek_crypto: AesDekCrypto::new(),
+        };
+        client.validate_existing_keys().await?;
+        Ok(client)
+    }
+
+    /// Open a Local KMS key directory without creating or modifying any files.
+    ///
+    /// This constructor is restricted to explicit key-export tooling. Normal
+    /// backend operation must use [`Self::new`].
+    pub async fn new_for_key_export(config: LocalConfig) -> Result<Self> {
+        if !fs::try_exists(&config.key_dir).await? {
+            return Err(KmsError::configuration_error("Local KMS key directory does not exist"));
+        }
+
+        let (master_cipher, legacy_master_cipher) = if let Some(ref master_key) = config.master_key {
+            let legacy_key = Self::derive_legacy_master_key(master_key)?;
+            let legacy_master_cipher = Aes256Gcm::new(&legacy_key);
+            let salt_path = Self::master_key_salt_path(&config);
+            let master_cipher = if fs::try_exists(&salt_path).await? {
+                let salt = fs::read(&salt_path).await?;
+                let salt: [u8; LOCAL_KMS_MASTER_KEY_SALT_LEN] = salt.try_into().map_err(|_| {
+                    KmsError::configuration_error(format!(
+                        "Local KMS master key salt at {} must be exactly {} bytes",
+                        salt_path.display(),
+                        LOCAL_KMS_MASTER_KEY_SALT_LEN
+                    ))
+                })?;
+                Aes256Gcm::new(&Self::derive_master_key(master_key, &salt)?)
+            } else {
+                Aes256Gcm::new(&legacy_key)
+            };
+            (Some(master_cipher), Some(legacy_master_cipher))
+        } else {
+            (None, None)
+        };
+
         Ok(Self {
             config,
-            key_cache: RwLock::new(HashMap::new()),
             master_cipher,
             legacy_master_cipher,
             dek_crypto: AesDekCrypto::new(),
@@ -193,10 +231,35 @@ impl LocalKmsClient {
 
         let mut salt = [0u8; LOCAL_KMS_MASTER_KEY_SALT_LEN];
         rand::rng().fill(&mut salt[..]);
-        fs::write(&salt_path, salt).await?;
-        Self::set_file_permissions(&salt_path, config.file_permissions).await?;
-        debug!(path = ?salt_path, "Local KMS master key salt created");
-        Ok(salt)
+        let temp_path = config
+            .key_dir
+            .join(format!("{LOCAL_KMS_MASTER_KEY_SALT_FILE}.tmp-{}", uuid::Uuid::new_v4()));
+        fs::write(&temp_path, salt).await?;
+        Self::set_file_permissions(&temp_path, config.file_permissions).await?;
+        match fs::hard_link(&temp_path, &salt_path).await {
+            Ok(()) => {
+                if let Err(error) = fs::remove_file(&temp_path).await {
+                    warn!(path = ?temp_path, %error, "Failed to remove Local KMS salt temporary file");
+                }
+                debug!(path = ?salt_path, "Local KMS master key salt created");
+                Ok(salt)
+            }
+            Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
+                let _ = fs::remove_file(&temp_path).await;
+                let bytes = fs::read(&salt_path).await?;
+                bytes.try_into().map_err(|_| {
+                    KmsError::configuration_error(format!(
+                        "Local KMS master key salt at {} must be exactly {} bytes",
+                        salt_path.display(),
+                        LOCAL_KMS_MASTER_KEY_SALT_LEN
+                    ))
+                })
+            }
+            Err(error) => {
+                let _ = fs::remove_file(&temp_path).await;
+                Err(error.into())
+            }
+        }
     }
 
     #[cfg(unix)]
@@ -242,6 +305,12 @@ impl LocalKmsClient {
 
         let content = fs::read(&key_path).await?;
         let stored_key: StoredMasterKey = serde_json::from_slice(&content)?;
+        if stored_key.key_id != key_id {
+            return Err(KmsError::invalid_key(format!(
+                "Local KMS key file identity mismatch: expected {key_id:?}, found {:?}",
+                stored_key.key_id
+            )));
+        }
 
         let encrypted_bytes = BASE64
             .decode(&stored_key.encrypted_key_material)
@@ -326,8 +395,43 @@ impl LocalKmsClient {
     /// Save a master key to disk
     async fn save_master_key(&self, master_key: &MasterKeyInfo, key_material: &[u8]) -> Result<()> {
         let key_path = self.master_key_path(&master_key.key_id)?;
+        let content = self.encode_master_key(master_key, key_material)?;
+        let temp_path = key_path.with_extension(format!("tmp-{}", uuid::Uuid::new_v4()));
+        fs::write(&temp_path, &content).await?;
+        Self::set_file_permissions(&temp_path, self.config.file_permissions).await?;
+        fs::rename(&temp_path, &key_path).await?;
 
-        // Encrypt key material if master cipher is available
+        debug!(key_id = %master_key.key_id, path = ?key_path, "Local KMS master key saved");
+        Ok(())
+    }
+
+    async fn save_new_master_key(&self, master_key: &MasterKeyInfo, key_material: &[u8]) -> Result<()> {
+        let key_path = self.master_key_path(&master_key.key_id)?;
+        let content = self.encode_master_key(master_key, key_material)?;
+        let temp_path = key_path.with_extension(format!("tmp-{}", uuid::Uuid::new_v4()));
+        fs::write(&temp_path, &content).await?;
+        Self::set_file_permissions(&temp_path, self.config.file_permissions).await?;
+
+        match fs::hard_link(&temp_path, &key_path).await {
+            Ok(()) => {
+                if let Err(error) = fs::remove_file(&temp_path).await {
+                    warn!(path = ?temp_path, %error, "Failed to remove Local KMS key temporary file");
+                }
+                debug!(key_id = %master_key.key_id, path = ?key_path, "Local KMS master key created");
+                Ok(())
+            }
+            Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
+                let _ = fs::remove_file(&temp_path).await;
+                Err(KmsError::key_already_exists(&master_key.key_id))
+            }
+            Err(error) => {
+                let _ = fs::remove_file(&temp_path).await;
+                Err(error.into())
+            }
+        }
+    }
+
+    fn encode_master_key(&self, master_key: &MasterKeyInfo, key_material: &[u8]) -> Result<Vec<u8>> {
         let (encrypted_key_material, nonce, at_rest_protection) = if let Some(ref cipher) = self.master_cipher {
             let mut nonce_bytes = [0u8; 12];
             rand::rng().fill(&mut nonce_bytes[..]);
@@ -362,17 +466,7 @@ impl LocalKmsClient {
             at_rest_protection,
         };
 
-        let content = serde_json::to_vec_pretty(&stored_key)?;
-
-        // Write to temporary file first, then rename for atomicity
-        let temp_path = key_path.with_extension("tmp");
-        fs::write(&temp_path, &content).await?;
-        Self::set_file_permissions(&temp_path, self.config.file_permissions).await?;
-
-        fs::rename(&temp_path, &key_path).await?;
-
-        debug!(key_id = %master_key.key_id, path = ?key_path, "Local KMS master key saved");
-        Ok(())
+        serde_json::to_vec_pretty(&stored_key).map_err(Into::into)
     }
 
     /// Get the actual key material for a master key
@@ -381,6 +475,37 @@ impl LocalKmsClient {
         Ok(key_material)
     }
 
+    /// Decrypt an AES-256 Local KMS key for explicit migration tooling.
+    ///
+    /// The returned buffer is zeroized on drop. Callers must treat the value as
+    /// plaintext key material and avoid logging or persisting it.
+    pub async fn decrypt_key_material_for_export(&self, key_id: &str) -> Result<Zeroizing<[u8; 32]>> {
+        let (stored_key, key_material) = self.decode_stored_key(key_id).await?;
+        if stored_key.algorithm != "AES_256" {
+            return Err(KmsError::unsupported_algorithm(stored_key.algorithm));
+        }
+        let actual = key_material.len();
+        let key_material = key_material.try_into().map_err(|_| KmsError::invalid_key_size(32, actual))?;
+        Ok(Zeroizing::new(key_material))
+    }
+
+    async fn validate_existing_keys(&self) -> Result<()> {
+        let mut entries = fs::read_dir(&self.config.key_dir).await?;
+        while let Some(entry) = entries.next_entry().await? {
+            let path = entry.path();
+            if path.extension().is_none_or(|extension| extension != "key") {
+                continue;
+            }
+
+            let key_id = path
+                .file_stem()
+                .and_then(|stem| stem.to_str())
+                .ok_or_else(|| KmsError::configuration_error("Local KMS key file name must be valid UTF-8"))?;
+            self.decode_stored_key(key_id).await?;
+        }
+        Ok(())
+    }
+
     /// Encrypt data using a master key
     async fn encrypt_with_master_key(&self, key_id: &str, plaintext: &[u8]) -> Result<(Vec<u8>, Vec<u8>)> {
         // Load the actual master key material
@@ -513,11 +638,7 @@ impl KmsClient for LocalKmsClient {
         let master_key = MasterKeyInfo::new_with_description(key_id.to_string(), algorithm.to_string(), Some(created_by), None);
 
         // Save to disk
-        self.save_master_key(&master_key, &key_material).await?;
-
-        // Cache the key
-        let mut cache = self.key_cache.write().await;
-        cache.insert(key_id.to_string(), master_key.clone());
+        self.save_new_master_key(&master_key, &key_material).await?;
 
         debug!(key_id, "Local KMS master key created");
         Ok(master_key)
@@ -526,23 +647,7 @@ impl KmsClient for LocalKmsClient {
     async fn describe_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<KeyInfo> {
         debug!("Describing key: {}", key_id);
 
-        // Check cache first
-        {
-            let cache = self.key_cache.read().await;
-            if let Some(master_key) = cache.get(key_id) {
-                return Ok(master_key.clone().into());
-            }
-        }
-
-        // Load from disk
         let master_key = self.load_master_key(key_id).await?;
-
-        // Update cache
-        {
-            let mut cache = self.key_cache.write().await;
-            cache.insert(key_id.to_string(), master_key.clone());
-        }
-
         Ok(master_key.into())
     }
 
@@ -602,10 +707,6 @@ impl KmsClient for LocalKmsClient {
         let key_material = self.get_key_material(key_id).await?;
         self.save_master_key(&master_key, &key_material).await?;
 
-        // Update cache
-        let mut cache = self.key_cache.write().await;
-        cache.insert(key_id.to_string(), master_key);
-
         debug!(key_id, "Local KMS key enabled");
         Ok(())
     }
@@ -621,10 +722,6 @@ impl KmsClient for LocalKmsClient {
         let key_material = self.get_key_material(key_id).await?;
         self.save_master_key(&master_key, &key_material).await?;
 
-        // Update cache
-        let mut cache = self.key_cache.write().await;
-        cache.insert(key_id.to_string(), master_key);
-
         debug!(key_id, "Local KMS key disabled");
         Ok(())
     }
@@ -646,10 +743,6 @@ impl KmsClient for LocalKmsClient {
         let key_material = self.get_key_material(key_id).await?;
         self.save_master_key(&master_key, &key_material).await?;
 
-        // Update cache
-        let mut cache = self.key_cache.write().await;
-        cache.insert(key_id.to_string(), master_key);
-
         debug!(key_id, "Local KMS key deletion scheduled");
         Ok(())
     }
@@ -665,31 +758,17 @@ impl KmsClient for LocalKmsClient {
         let key_material = self.get_key_material(key_id).await?;
         self.save_master_key(&master_key, &key_material).await?;
 
-        // Update cache
-        let mut cache = self.key_cache.write().await;
-        cache.insert(key_id.to_string(), master_key);
-
         debug!(key_id, "Local KMS key deletion canceled");
         Ok(())
     }
 
     async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> {
-        debug!("Rotating key: {}", key_id);
-
-        let mut master_key = self.load_master_key(key_id).await?;
-        master_key.version += 1;
-        master_key.rotated_at = Some(Zoned::now());
-
-        // Generate new key material
-        let key_material = generate_key_material(&master_key.algorithm)?;
-        self.save_master_key(&master_key, &key_material).await?;
-
-        // Update cache
-        let mut cache = self.key_cache.write().await;
-        cache.insert(key_id.to_string(), master_key.clone());
-
-        debug!(key_id, "Local KMS key rotated");
-        Ok(master_key)
+        if !fs::try_exists(self.master_key_path(key_id)?).await? {
+            return Err(KmsError::key_not_found(key_id));
+        }
+        Err(KmsError::invalid_operation(
+            "Local KMS key rotation is unavailable until historical key versions can be retained",
+        ))
     }
 
     async fn health_check(&self) -> Result<()> {
@@ -745,11 +824,6 @@ impl KmsBackend for LocalKmsBackend {
     async fn create_key(&self, request: CreateKeyRequest) -> Result<CreateKeyResponse> {
         let key_id = request.key_name.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
 
-        // `save_master_key` writes through a temp file and renames over the destination, so
-        // creating a key under an existing name would replace its material and silently
-        // destroy the ability to decrypt everything wrapped under it. The sibling
-        // `KmsClient::create_key` has always refused this; the backend path did not, and
-        // this is the path the admin API uses.
         if self.client.master_key_path(&key_id)?.exists() {
             return Err(KmsError::key_already_exists(&key_id));
         }
@@ -767,11 +841,8 @@ impl KmsBackend for LocalKmsBackend {
                 request.description.clone(),
             );
 
-            // Save to disk and cache
-            self.client.save_master_key(&master_key, &key_material).await?;
-
-            let mut cache = self.client.key_cache.write().await;
-            cache.insert(key_id.clone(), master_key.clone());
+            // Save to disk
+            self.client.save_new_master_key(&master_key, &key_material).await?;
 
             master_key
         };
@@ -888,10 +959,6 @@ impl KmsBackend for LocalKmsBackend {
                 .await
                 .map_err(|e| KmsError::internal_error(format!("Failed to delete key file: {e}")))?;
 
-            // Remove from cache
-            let mut cache = self.client.key_cache.write().await;
-            cache.remove(key_id);
-
             debug!(key_id, "Local KMS key deleted immediately");
 
             // Return success response for immediate deletion
@@ -935,10 +1002,6 @@ impl KmsBackend for LocalKmsBackend {
 
         self.client.save_master_key(&master_key, &existing_key_material).await?;
 
-        // Update cache
-        let mut cache = self.client.key_cache.write().await;
-        cache.insert(key_id.to_string(), master_key.clone());
-
         // Convert master_key to KeyMetadata for response
         let key_metadata = KeyMetadata {
             key_id: master_key.key_id.clone(),
@@ -986,10 +1049,6 @@ impl KmsBackend for LocalKmsBackend {
 
         self.client.save_master_key(&master_key, &existing_key_material).await?;
 
-        // Update cache
-        let mut cache = self.client.key_cache.write().await;
-        cache.insert(key_id.to_string(), master_key.clone());
-
         // Convert master_key to KeyMetadata for response
         let key_metadata = KeyMetadata {
             key_id: master_key.key_id.clone(),
@@ -1189,6 +1248,64 @@ mod tests {
         .expect("stored encrypted key should deserialize");
         assert_eq!(stored.at_rest_protection, StoredKeyProtection::EncryptedMasterKey);
         assert_eq!(stored.nonce.len(), 12);
+
+        let wrong_master_error = match LocalKmsClient::new(LocalConfig {
+            key_dir: client.config.key_dir.clone(),
+            master_key: Some("wrong-master-key".to_string()),
+            file_permissions: Some(0o600),
+        })
+        .await
+        {
+            Ok(_) => panic!("wrong master key must fail initialization"),
+            Err(error) => error,
+        };
+        assert!(matches!(wrong_master_error, KmsError::CryptographicError { .. }));
+    }
+
+    #[tokio::test]
+    async fn key_export_uses_existing_local_decryption_path_without_writing_files() {
+        let (client, _temp_dir) = create_test_client().await;
+        let key_id = "export-key";
+        client
+            .create_key(key_id, "AES_256", None)
+            .await
+            .expect("create encrypted key");
+        let expected = client.get_key_material(key_id).await.expect("load expected key material");
+        let salt_path = LocalKmsClient::master_key_salt_path(&client.config);
+        let salt_before = fs::read(&salt_path).await.expect("read existing salt");
+
+        let export_client = LocalKmsClient::new_for_key_export(client.config.clone())
+            .await
+            .expect("open read-only export client");
+        let exported = export_client
+            .decrypt_key_material_for_export(key_id)
+            .await
+            .expect("decrypt key for export");
+
+        assert_eq!(exported.as_ref(), expected.as_slice());
+        assert_eq!(fs::read(&salt_path).await.expect("read unchanged salt"), salt_before);
+    }
+
+    #[tokio::test]
+    async fn key_export_accepts_plaintext_dev_only_key_without_master_key() {
+        let (client, _temp_dir) = create_dev_mode_client().await;
+        let key_id = "plaintext-export-key";
+        client
+            .create_key(key_id, "AES_256", None)
+            .await
+            .expect("create plaintext-dev-only key");
+        let expected = client.get_key_material(key_id).await.expect("load expected key material");
+
+        let export_client = LocalKmsClient::new_for_key_export(client.config.clone())
+            .await
+            .expect("open read-only export client");
+        let exported = export_client
+            .decrypt_key_material_for_export(key_id)
+            .await
+            .expect("export plaintext-dev-only key");
+
+        assert_eq!(exported.as_ref(), expected.as_slice());
+        assert!(!LocalKmsClient::master_key_salt_path(&client.config).exists());
     }
 
     #[tokio::test]
@@ -1228,17 +1345,57 @@ mod tests {
             master_key: None,
             file_permissions: Some(0o600),
         };
-        let client_without_master = LocalKmsClient::new(config)
-            .await
-            .expect("client without master key should still initialize in dev-mode tests");
-
-        let err = client_without_master
-            .describe_key("encrypted-key", None)
-            .await
-            .expect_err("encrypted key should require a master key to read");
+        let err = match LocalKmsClient::new(config).await {
+            Ok(_) => panic!("initialization must reject an unreadable encrypted key"),
+            Err(error) => error,
+        };
         assert!(err.to_string().contains("requires a configured master key"));
     }
 
+    #[tokio::test]
+    async fn local_key_rotation_is_rejected_without_overwriting_key_material() {
+        let (client, _temp_dir) = create_test_client().await;
+        let key_id = "rotation-key";
+        client.create_key(key_id, "AES_256", None).await.expect("create key");
+        let original_material = client.get_key_material(key_id).await.expect("load original material");
+
+        let error = client
+            .rotate_key(key_id, None)
+            .await
+            .expect_err("rotation must remain unavailable without historical key versions");
+
+        assert!(matches!(error, KmsError::InvalidOperation { .. }));
+        assert_eq!(
+            client.get_key_material(key_id).await.expect("reload original material"),
+            original_material
+        );
+    }
+
+    #[tokio::test]
+    async fn startup_rejects_key_file_with_mismatched_embedded_id() {
+        let (client, temp_dir) = create_dev_mode_client().await;
+        client.create_key("file-name", "AES_256", None).await.expect("create key");
+        let key_path = client.master_key_path("file-name").expect("valid key id");
+        let mut stored: serde_json::Value =
+            serde_json::from_slice(&fs::read(&key_path).await.expect("read key file")).expect("decode key file");
+        stored["key_id"] = serde_json::json!("embedded-name");
+        fs::write(&key_path, serde_json::to_vec_pretty(&stored).expect("encode mismatched key"))
+            .await
+            .expect("write mismatched key");
+
+        let error = match LocalKmsClient::new(LocalConfig {
+            key_dir: temp_dir.path().to_path_buf(),
+            master_key: None,
+            file_permissions: Some(0o600),
+        })
+        .await
+        {
+            Ok(_) => panic!("mismatched key identity must fail initialization"),
+            Err(error) => error,
+        };
+        assert!(matches!(error, KmsError::InvalidKey { .. }));
+    }
+
     #[tokio::test]
     async fn test_load_master_key_accepts_legacy_rfc3339_timestamp() {
         let (client, _temp_dir) = create_dev_mode_client().await;
@@ -1329,17 +1486,6 @@ mod tests {
         )
         .await
         .expect("write beta.5 fixture");
-        let mut explicit_protection = stored_key.clone();
-        let explicit_object = explicit_protection.as_object_mut().expect("beta.5 fixture is a JSON object");
-        explicit_object.insert("key_id".to_string(), serde_json::json!("beta5-explicit-key"));
-        explicit_object.insert("at_rest_protection".to_string(), serde_json::json!("encrypted-master-key"));
-        fs::write(
-            temp_dir.path().join("beta5-explicit-key.key"),
-            serde_json::to_vec_pretty(&explicit_protection).expect("serialize explicit-protection fixture"),
-        )
-        .await
-        .expect("write explicit-protection fixture");
-
         let client = LocalKmsClient::new(LocalConfig {
             key_dir: temp_dir.path().to_path_buf(),
             master_key: Some("beta5-test-master-key".to_string()),
@@ -1353,24 +1499,34 @@ mod tests {
             .await
             .expect("decrypt beta.5 SHA-256 protected key");
         assert_eq!(material, vec![0x42; 32]);
+
+        let mut explicit_protection = stored_key.clone();
+        let explicit_object = explicit_protection.as_object_mut().expect("beta.5 fixture is a JSON object");
+        explicit_object.insert("key_id".to_string(), serde_json::json!("beta5-explicit-key"));
+        explicit_object.insert("at_rest_protection".to_string(), serde_json::json!("encrypted-master-key"));
+        fs::write(
+            temp_dir.path().join("beta5-explicit-key.key"),
+            serde_json::to_vec_pretty(&explicit_protection).expect("serialize explicit-protection fixture"),
+        )
+        .await
+        .expect("write explicit-protection fixture");
         let explicit_error = client
             .get_key_material("beta5-explicit-key")
             .await
             .expect_err("explicit current protection must not fall back to the beta.5 KDF");
         assert!(matches!(explicit_error, KmsError::CryptographicError { .. }));
 
-        let wrong_key_client = LocalKmsClient::new(LocalConfig {
+        let wrong_key_error = match LocalKmsClient::new(LocalConfig {
             key_dir: temp_dir.path().to_path_buf(),
             master_key: Some("wrong-beta5-master-key".to_string()),
             file_permissions: Some(0o600),
         })
         .await
-        .expect("initialize local KMS with wrong beta.5 master key");
-        let error = wrong_key_client
-            .get_key_material("beta5-key")
-            .await
-            .expect_err("wrong beta.5 master key must not decrypt the fixture");
-        assert!(matches!(error, KmsError::CryptographicError { .. }));
+        {
+            Ok(_) => panic!("wrong beta.5 master key must fail initialization"),
+            Err(error) => error,
+        };
+        assert!(matches!(wrong_key_error, KmsError::CryptographicError { .. }));
     }
 
     /// R03-CAN-072 / R03-CAN-073: key identifiers arrive from request input, so every path
@@ -1429,9 +1585,7 @@ mod tests {
         }
     }
 
-    /// R07-CAN-103: `save_master_key` writes a temp file and renames over the destination,
-    /// so creating a key under an existing name would replace its material and silently
-    /// destroy the ability to decrypt anything wrapped under it.
+    /// R07-CAN-103: creating a duplicate key must preserve its original material.
     #[tokio::test]
     async fn backend_create_key_refuses_to_replace_existing_key_material() {
         let temp_dir = TempDir::new().expect("Failed to create temp dir");
@@ -1469,4 +1623,33 @@ mod tests {
             .expect("original key material must survive the refused create");
         assert_eq!(original, after, "existing key material must not be replaced");
     }
+
+    #[tokio::test]
+    async fn concurrent_backend_create_allows_only_one_writer() {
+        let temp_dir = TempDir::new().expect("create key directory");
+        let config = || LocalConfig {
+            key_dir: temp_dir.path().to_path_buf(),
+            master_key: Some("test-master-key".to_string()),
+            file_permissions: Some(0o600),
+        };
+        let first = LocalKmsBackend {
+            client: LocalKmsClient::new(config()).await.expect("create first client"),
+        };
+        let second = LocalKmsBackend {
+            client: LocalKmsClient::new(config()).await.expect("create second client"),
+        };
+        let request = || CreateKeyRequest {
+            key_name: Some("concurrent-key".to_string()),
+            ..Default::default()
+        };
+
+        let (first_result, second_result) = tokio::join!(first.create_key(request()), second.create_key(request()));
+        assert_ne!(first_result.is_ok(), second_result.is_ok(), "exactly one create must succeed");
+        let error = first_result
+            .err()
+            .or_else(|| second_result.err())
+            .expect("one create must fail");
+        assert!(matches!(error, KmsError::KeyAlreadyExists { .. }));
+        assert_eq!(first.client.get_key_material("concurrent-key").await.expect("load key").len(), 32);
+    }
 }

crates/kms/src/lib.rs

@@ -89,7 +89,7 @@ pub use error::{KmsError, KmsUnavailableError, Result};
 pub use manager::KmsManager;
 pub use service::{DataKey, ObjectEncryptionService};
 pub use service_manager::{
-    KmsServiceManager, KmsServiceStatus, get_global_encryption_service, get_global_kms_service_manager,
+    KmsServiceManager, KmsServiceStatus, KmsStartOutcome, get_global_encryption_service, get_global_kms_service_manager,
     init_global_kms_service_manager,
 };
 pub use types::*;
@@ -177,8 +177,8 @@ mod tests {
         let service1 = manager.get_encryption_service().await.expect("Service should be available");
 
         // Reconfigure to new service (zero-downtime)
-        let temp_dir2 = TempDir::new().expect("Failed to create temp dir");
-        let config2 = KmsConfig::local(temp_dir2.path().to_path_buf()).with_insecure_development_defaults();
+        let mut config2 = config1;
+        config2.timeout = std::time::Duration::from_secs(45);
         manager.reconfigure(config2).await.expect("Reconfiguration should succeed");
 
         // Verify version 2

crates/kms/src/service.rs

@@ -20,6 +20,7 @@ use crate::manager::KmsManager;
 use crate::types::*;
 use base64::Engine;
 use jiff::Zoned;
+use md5::{Digest as Md5Digest, Md5};
 use rand::random;
 use std::collections::HashMap;
 use std::io::Cursor;
@@ -27,6 +28,12 @@ use tokio::io::{AsyncRead, AsyncReadExt};
 use tracing::debug;
 use zeroize::Zeroize;
 
+fn md5_hex(input: impl AsRef<[u8]>) -> String {
+    let mut hasher = Md5::new();
+    hasher.update(input.as_ref());
+    hex::encode(hasher.finalize())
+}
+
 /// Data key for object encryption
 /// SECURITY: This struct automatically zeros sensitive key material when dropped
 #[derive(Debug, Clone)]
@@ -486,8 +493,7 @@ impl ObjectEncryptionService {
 
         // Validate key MD5 if provided
         if let Some(expected_md5) = customer_key_md5 {
-            let actual_md5 = md5::compute(customer_key);
-            let actual_md5_hex = format!("{actual_md5:x}");
+            let actual_md5_hex = md5_hex(customer_key);
             if actual_md5_hex != expected_md5.to_lowercase() {
                 return Err(KmsError::validation_error("Customer key MD5 mismatch"));
             }

crates/kms/src/service_manager.rs

@@ -20,17 +20,67 @@ use crate::error::{KmsError, Result};
 use crate::manager::KmsManager;
 use crate::service::ObjectEncryptionService;
 use arc_swap::ArcSwap;
+use sha2::{Digest, Sha256};
+use std::future::Future;
 use std::sync::{
     Arc, OnceLock,
     atomic::{AtomicU64, Ordering},
 };
-use tokio::sync::{Mutex, RwLock};
+use subtle::ConstantTimeEq;
+use tokio::sync::Mutex;
 use tracing::{debug, error, info, warn};
 
 const LOG_COMPONENT_KMS: &str = "kms";
 const LOG_SUBSYSTEM_SERVICE: &str = "service";
 const EVENT_KMS_SERVICE_STATE: &str = "kms_service_state";
 
+fn local_master_key_fingerprint(master_key: Option<&str>) -> [u8; 32] {
+    let mut digest = Sha256::new();
+    digest.update([u8::from(master_key.is_some())]);
+    if let Some(master_key) = master_key {
+        digest.update(master_key.as_bytes());
+    }
+    digest.finalize().into()
+}
+
+fn validate_local_transition(current: Option<&KmsConfig>, new: &KmsConfig) -> Result<()> {
+    let Some(current) = current else {
+        return Ok(());
+    };
+    let BackendConfig::Local(current_local) = &current.backend_config else {
+        return Ok(());
+    };
+    let BackendConfig::Local(new_local) = &new.backend_config else {
+        return Err(KmsError::configuration_error("Local KMS backend cannot be changed after configuration"));
+    };
+
+    if current_local.key_dir != new_local.key_dir {
+        return Err(KmsError::configuration_error(
+            "Local KMS key directory cannot be changed after configuration",
+        ));
+    }
+    if current_local.file_permissions != new_local.file_permissions {
+        return Err(KmsError::configuration_error(
+            "Local KMS file permissions cannot be changed after configuration",
+        ));
+    }
+    if current.allow_insecure_dev_defaults != new.allow_insecure_dev_defaults {
+        return Err(KmsError::configuration_error(
+            "Local KMS development mode cannot be changed after configuration",
+        ));
+    }
+
+    let current_master_key = local_master_key_fingerprint(current_local.master_key.as_deref());
+    let new_master_key = local_master_key_fingerprint(new_local.master_key.as_deref());
+    if !bool::from(current_master_key.ct_eq(&new_master_key)) {
+        return Err(KmsError::configuration_error(
+            "Local KMS master key cannot be changed after configuration",
+        ));
+    }
+
+    Ok(())
+}
+
 /// KMS service status
 #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
 pub enum KmsServiceStatus {
@@ -44,6 +94,13 @@ pub enum KmsServiceStatus {
     Error(String),
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum KmsStartOutcome {
+    Started,
+    Restarted,
+    AlreadyRunning,
+}
+
 /// Service version information for zero-downtime reconfiguration
 #[derive(Clone)]
 struct ServiceVersion {
@@ -55,16 +112,17 @@ struct ServiceVersion {
     manager: Arc<KmsManager>,
 }
 
+#[derive(Clone)]
+struct RuntimeState {
+    config: Option<KmsConfig>,
+    status: KmsServiceStatus,
+    current_service: Option<ServiceVersion>,
+}
+
 /// Dynamic KMS service manager with versioned services for zero-downtime reconfiguration
 pub struct KmsServiceManager {
-    /// Current service version (if running)
-    /// Uses ArcSwap for atomic, lock-free service switching
-    /// This allows instant atomic updates without blocking readers
-    current_service: ArcSwap<Option<ServiceVersion>>,
-    /// Current configuration
-    config: Arc<RwLock<Option<KmsConfig>>>,
-    /// Current status
-    status: Arc<RwLock<KmsServiceStatus>>,
+    /// Atomically published configuration, status, and current service.
+    state: ArcSwap<RuntimeState>,
     /// Version counter (monotonically increasing)
     version_counter: Arc<AtomicU64>,
     /// Mutex to protect lifecycle operations (start, stop, reconfigure)
@@ -76,9 +134,11 @@ impl KmsServiceManager {
     /// Create a new KMS service manager (not configured)
     pub fn new() -> Self {
         Self {
-            current_service: ArcSwap::from_pointee(None),
-            config: Arc::new(RwLock::new(None)),
-            status: Arc::new(RwLock::new(KmsServiceStatus::NotConfigured)),
+            state: ArcSwap::from_pointee(RuntimeState {
+                config: None,
+                status: KmsServiceStatus::NotConfigured,
+                current_service: None,
+            }),
             version_counter: Arc::new(AtomicU64::new(0)),
             lifecycle_mutex: Arc::new(Mutex::new(())),
         }
@@ -86,39 +146,67 @@ impl KmsServiceManager {
 
     /// Get current service status
     pub async fn get_status(&self) -> KmsServiceStatus {
-        self.status.read().await.clone()
+        self.state.load().status.clone()
     }
 
     /// Get current configuration (if any)
     pub async fn get_config(&self) -> Option<KmsConfig> {
-        self.config.read().await.clone()
+        self.state.load().config.clone()
     }
 
     /// Get configuration for status and management responses without static key material.
     pub async fn get_redacted_config(&self) -> Option<KmsConfig> {
-        let mut config = self.config.read().await.clone()?;
+        let mut config = self.state.load().config.clone()?;
+        Self::redact_config(&mut config);
+        Some(config)
+    }
+
+    /// Get status and redacted configuration from the same published snapshot.
+    pub async fn get_redacted_state(&self) -> (KmsServiceStatus, Option<KmsConfig>) {
+        let state = self.state.load();
+        let mut config = state.config.clone();
+        if let Some(config) = &mut config {
+            Self::redact_config(config);
+        }
+        (state.status.clone(), config)
+    }
+
+    fn redact_config(config: &mut KmsConfig) {
         if let BackendConfig::Static(static_config) = &mut config.backend_config {
             use zeroize::Zeroize;
             static_config.secret_key.zeroize();
         }
-        Some(config)
     }
 
     /// Configure KMS with new configuration
     pub async fn configure(&self, new_config: KmsConfig) -> Result<()> {
+        self.configure_with_persistence(new_config, || async { Ok(()) }).await
+    }
+
+    /// Configure KMS and publish the in-memory state only after persistence succeeds.
+    ///
+    /// The persistence callback runs under the lifecycle lock and must not call
+    /// another lifecycle method on this manager.
+    pub async fn configure_with_persistence<Persist, PersistFuture>(&self, new_config: KmsConfig, persist: Persist) -> Result<()>
+    where
+        Persist: FnOnce() -> PersistFuture,
+        PersistFuture: Future<Output = Result<()>>,
+    {
         new_config.validate()?;
-
-        // Update configuration
-        {
-            let mut config = self.config.write().await;
-            *config = Some(new_config.clone());
-        }
-
-        // Update status
-        {
-            let mut status = self.status.write().await;
-            *status = KmsServiceStatus::Configured;
+        let _guard = self.lifecycle_mutex.lock().await;
+        let current = self.state.load_full();
+        validate_local_transition(current.config.as_ref(), &new_config)?;
+        if current.current_service.is_some() {
+            return Err(KmsError::configuration_error(
+                "Cannot configure KMS while it is running; use reconfigure instead",
+            ));
         }
+        persist().await?;
+        self.state.store(Arc::new(RuntimeState {
+            config: Some(new_config),
+            status: KmsServiceStatus::Configured,
+            current_service: None,
+        }));
 
         debug!(
             event = EVENT_KMS_SERVICE_STATE,
@@ -136,19 +224,35 @@ impl KmsServiceManager {
         self.start_internal().await
     }
 
+    /// Start or restart KMS with the running-state decision serialized with the lifecycle action.
+    pub async fn start_or_restart(&self, force: bool) -> Result<KmsStartOutcome> {
+        let _guard = self.lifecycle_mutex.lock().await;
+        let running = self.state.load().current_service.is_some();
+        if running && !force {
+            return Ok(KmsStartOutcome::AlreadyRunning);
+        }
+        self.start_internal().await?;
+        Ok(if running {
+            KmsStartOutcome::Restarted
+        } else {
+            KmsStartOutcome::Started
+        })
+    }
+
     /// Internal start implementation (called within lifecycle mutex)
     async fn start_internal(&self) -> Result<()> {
-        let config = {
-            let config_guard = self.config.read().await;
-            match config_guard.as_ref() {
-                Some(config) => config.clone(),
-                None => {
-                    let err_msg = "Cannot start KMS: no configuration provided";
-                    error!("{}", err_msg);
-                    let mut status = self.status.write().await;
-                    *status = KmsServiceStatus::Error(err_msg.to_string());
-                    return Err(KmsError::configuration_error(err_msg));
-                }
+        let state = self.state.load_full();
+        let config = match state.config.as_ref() {
+            Some(config) => config.clone(),
+            None => {
+                let err_msg = "Cannot start KMS: no configuration provided";
+                error!("{}", err_msg);
+                self.state.store(Arc::new(RuntimeState {
+                    config: None,
+                    status: KmsServiceStatus::Error(err_msg.to_string()),
+                    current_service: None,
+                }));
+                return Err(KmsError::configuration_error(err_msg));
             }
         };
 
@@ -161,17 +265,9 @@ impl KmsServiceManager {
             "KMS service starting"
         );
 
-        match self.create_service_version(&config).await {
+        match self.create_healthy_service_version(&config).await {
             Ok(service_version) => {
-                // Atomically update to new service version (lock-free, instant)
-                // ArcSwap::store() is a true atomic operation using CAS
-                self.current_service.store(Arc::new(Some(service_version)));
-
-                // Update status
-                {
-                    let mut status = self.status.write().await;
-                    *status = KmsServiceStatus::Running;
-                }
+                self.publish_running(config, service_version);
 
                 debug!(
                     event = EVENT_KMS_SERVICE_STATE,
@@ -185,13 +281,24 @@ impl KmsServiceManager {
             Err(e) => {
                 let err_msg = format!("Failed to create KMS backend: {e}");
                 error!("{}", err_msg);
-                let mut status = self.status.write().await;
-                *status = KmsServiceStatus::Error(err_msg.clone());
+                if state.current_service.is_none() {
+                    self.state.store(Arc::new(RuntimeState {
+                        config: state.config.clone(),
+                        status: KmsServiceStatus::Error(err_msg.clone()),
+                        current_service: None,
+                    }));
+                }
                 Err(KmsError::backend_error(&err_msg))
             }
         }
     }
 
+    /// Replace the running service without exposing a stopped interval.
+    pub async fn restart(&self) -> Result<()> {
+        let _guard = self.lifecycle_mutex.lock().await;
+        self.start_internal().await
+    }
+
     /// Stop KMS service
     ///
     /// Note: This stops accepting new operations, but existing operations using
@@ -213,15 +320,16 @@ impl KmsServiceManager {
 
         // Atomically clear current service version (lock-free, instant)
         // Note: Existing Arc references will keep the service alive until operations complete
-        self.current_service.store(Arc::new(None));
-
-        // Update status (keep configuration)
-        {
-            let mut status = self.status.write().await;
-            if !matches!(*status, KmsServiceStatus::NotConfigured) {
-                *status = KmsServiceStatus::Configured;
-            }
-        }
+        let state = self.state.load_full();
+        self.state.store(Arc::new(RuntimeState {
+            config: state.config.clone(),
+            status: if state.config.is_some() {
+                KmsServiceStatus::Configured
+            } else {
+                KmsServiceStatus::NotConfigured
+            },
+            current_service: None,
+        }));
 
         debug!(
             event = EVENT_KMS_SERVICE_STATE,
@@ -244,6 +352,22 @@ impl KmsServiceManager {
     /// This ensures zero downtime during reconfiguration, even for long-running
     /// operations like encrypting large files.
     pub async fn reconfigure(&self, new_config: KmsConfig) -> Result<()> {
+        self.reconfigure_with_persistence(new_config, || async { Ok(()) }).await
+    }
+
+    /// Reconfigure KMS after the candidate is healthy and persistence succeeds.
+    ///
+    /// The persistence callback runs under the lifecycle lock and must not call
+    /// another lifecycle method on this manager.
+    pub async fn reconfigure_with_persistence<Persist, PersistFuture>(
+        &self,
+        new_config: KmsConfig,
+        persist: Persist,
+    ) -> Result<()>
+    where
+        Persist: FnOnce() -> PersistFuture,
+        PersistFuture: Future<Output = Result<()>>,
+    {
         let _guard = self.lifecycle_mutex.lock().await;
 
         debug!(
@@ -254,30 +378,18 @@ impl KmsServiceManager {
             "KMS service reconfiguring"
         );
         new_config.validate()?;
-
-        // Configure with new config
-        {
-            let mut config = self.config.write().await;
-            *config = Some(new_config.clone());
-        }
+        validate_local_transition(self.state.load().config.as_ref(), &new_config)?;
 
         // Create new service version without stopping old one
         // This allows existing operations to continue while new operations use new service
-        match self.create_service_version(&new_config).await {
+        match self.create_healthy_service_version(&new_config).await {
             Ok(new_service_version) => {
                 // Get old version for logging (lock-free read)
-                let old_version = self.current_service.load().as_ref().as_ref().map(|sv| sv.version);
+                let old_version = self.state.load().current_service.as_ref().map(|sv| sv.version);
 
-                // Atomically switch to new service version (lock-free, instant CAS operation)
-                // This is a true atomic operation - no waiting for locks, instant switch
-                // Old service will be dropped when no more Arc references exist
-                self.current_service.store(Arc::new(Some(new_service_version.clone())));
+                persist().await?;
 
-                // Update status
-                {
-                    let mut status = self.status.write().await;
-                    *status = KmsServiceStatus::Running;
-                }
+                self.publish_running(new_config, new_service_version.clone());
 
                 if let Some(old_ver) = old_version {
                     info!(
@@ -304,8 +416,6 @@ impl KmsServiceManager {
             Err(e) => {
                 let err_msg = format!("Failed to reconfigure KMS: {e}");
                 error!("{}", err_msg);
-                let mut status = self.status.write().await;
-                *status = KmsServiceStatus::Error(err_msg.clone());
                 Err(KmsError::backend_error(&err_msg))
             }
         }
@@ -316,7 +426,7 @@ impl KmsServiceManager {
     /// Returns the manager from the current service version.
     /// Uses lock-free atomic load for optimal performance.
     pub async fn get_manager(&self) -> Option<Arc<KmsManager>> {
-        self.current_service.load().as_ref().as_ref().map(|sv| sv.manager.clone())
+        self.state.load().current_service.as_ref().map(|sv| sv.manager.clone())
     }
 
     /// Get encryption service (if running)
@@ -326,7 +436,7 @@ impl KmsServiceManager {
     /// This ensures new operations always use the latest service version,
     /// while existing operations continue using their Arc references.
     pub async fn get_encryption_service(&self) -> Option<Arc<ObjectEncryptionService>> {
-        self.current_service.load().as_ref().as_ref().map(|sv| sv.service.clone())
+        self.state.load().current_service.as_ref().map(|sv| sv.service.clone())
     }
 
     /// Get current service version number
@@ -334,14 +444,16 @@ impl KmsServiceManager {
     /// Useful for monitoring and debugging.
     /// Uses lock-free atomic load.
     pub async fn get_service_version(&self) -> Option<u64> {
-        self.current_service.load().as_ref().as_ref().map(|sv| sv.version)
+        self.state.load().current_service.as_ref().map(|sv| sv.version)
     }
 
     /// Health check for the KMS service
     pub async fn health_check(&self) -> Result<bool> {
-        let manager = self.get_manager().await;
-        match manager {
-            Some(manager) => {
+        let checked_state = self.state.load_full();
+        match checked_state.current_service.as_ref() {
+            Some(service_version) => {
+                let manager = service_version.manager.clone();
+                let checked_version = service_version.version;
                 // Perform health check on the backend
                 match manager.health_check().await {
                     Ok(healthy) => {
@@ -352,9 +464,8 @@ impl KmsServiceManager {
                     }
                     Err(e) => {
                         error!("KMS health check error: {}", e);
-                        // Update status to error
-                        let mut status = self.status.write().await;
-                        *status = KmsServiceStatus::Error(format!("Health check failed: {e}"));
+                        let _guard = self.lifecycle_mutex.lock().await;
+                        self.mark_health_error_if_current(checked_version, &e);
                         Err(e)
                     }
                 }
@@ -413,6 +524,33 @@ impl KmsServiceManager {
             manager: kms_manager,
         })
     }
+
+    async fn create_healthy_service_version(&self, config: &KmsConfig) -> Result<ServiceVersion> {
+        let service_version = self.create_service_version(config).await?;
+        if !service_version.manager.health_check().await? {
+            return Err(KmsError::backend_error("KMS backend health check failed"));
+        }
+        Ok(service_version)
+    }
+
+    fn publish_running(&self, config: KmsConfig, service_version: ServiceVersion) {
+        self.state.store(Arc::new(RuntimeState {
+            config: Some(config),
+            status: KmsServiceStatus::Running,
+            current_service: Some(service_version),
+        }));
+    }
+
+    fn mark_health_error_if_current(&self, checked_version: u64, error: &KmsError) {
+        let current = self.state.load_full();
+        if current.current_service.as_ref().map(|version| version.version) == Some(checked_version) {
+            self.state.store(Arc::new(RuntimeState {
+                config: current.config.clone(),
+                status: KmsServiceStatus::Error(format!("Health check failed: {error}")),
+                current_service: current.current_service.clone(),
+            }));
+        }
+    }
 }
 
 impl Default for KmsServiceManager {
@@ -445,6 +583,11 @@ pub async fn get_global_encryption_service() -> Option<Arc<ObjectEncryptionServi
 #[cfg(test)]
 mod tests {
     use super::*;
+    use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
+
+    fn static_config(key_id: &str, fill: u8) -> KmsConfig {
+        KmsConfig::static_kms(key_id.to_string(), BASE64_STANDARD.encode([fill; 32]))
+    }
 
     #[tokio::test]
     async fn configure_rejects_insecure_development_defaults_before_state_update() {
@@ -462,8 +605,6 @@ mod tests {
 
     #[tokio::test]
     async fn redacted_config_omits_static_key_material() {
-        use base64::Engine as _;
-
         let manager = KmsServiceManager::new();
         let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]);
         manager
@@ -477,4 +618,235 @@ mod tests {
         };
         assert!(static_config.secret_key.is_empty());
     }
+
+    #[tokio::test]
+    async fn configure_persistence_failure_leaves_state_unchanged() {
+        let manager = KmsServiceManager::new();
+
+        let result = manager
+            .configure_with_persistence(static_config("key-a", 0x11), || async { Err(KmsError::backend_error("persist failed")) })
+            .await;
+
+        assert!(result.is_err());
+        assert_eq!(manager.get_status().await, KmsServiceStatus::NotConfigured);
+        assert!(manager.get_config().await.is_none());
+        assert!(manager.get_encryption_service().await.is_none());
+    }
+
+    #[tokio::test]
+    async fn configure_rejects_running_service_without_changing_snapshot() {
+        let manager = KmsServiceManager::new();
+        manager.configure(static_config("key-a", 0x11)).await.expect("configure");
+        manager.start().await.expect("start");
+        let version = manager.get_service_version().await;
+
+        let result = manager.configure(static_config("key-b", 0x22)).await;
+
+        assert!(result.is_err());
+        assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
+        assert_eq!(manager.get_service_version().await, version);
+        assert_eq!(
+            manager.get_config().await.and_then(|config| config.default_key_id),
+            Some("key-a".to_string())
+        );
+    }
+
+    #[tokio::test]
+    async fn reconfigure_persistence_failure_keeps_old_running_snapshot() {
+        let manager = KmsServiceManager::new();
+        manager.configure(static_config("key-a", 0x11)).await.expect("configure");
+        manager.start().await.expect("start");
+        let old_version = manager.get_service_version().await;
+        let old_service = manager.get_encryption_service().await.expect("old service");
+
+        let result = manager
+            .reconfigure_with_persistence(static_config("key-b", 0x22), || async {
+                Err(KmsError::backend_error("persist failed"))
+            })
+            .await;
+
+        assert!(result.is_err());
+        assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
+        assert_eq!(manager.get_service_version().await, old_version);
+        assert_eq!(
+            manager.get_config().await.and_then(|config| config.default_key_id),
+            Some("key-a".to_string())
+        );
+        assert!(Arc::ptr_eq(
+            &old_service,
+            &manager.get_encryption_service().await.expect("old service remains")
+        ));
+    }
+
+    #[tokio::test]
+    async fn reconfigure_candidate_failure_keeps_old_running_snapshot() {
+        let manager = KmsServiceManager::new();
+        manager.configure(static_config("key-a", 0x11)).await.expect("configure");
+        manager.start().await.expect("start");
+        let old_version = manager.get_service_version().await;
+        let invalid_parent = tempfile::NamedTempFile::new().expect("temporary file");
+        let invalid_config = KmsConfig::local(invalid_parent.path().join("keys")).with_insecure_development_defaults();
+
+        let result = manager.reconfigure(invalid_config).await;
+
+        assert!(result.is_err());
+        assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
+        assert_eq!(manager.get_service_version().await, old_version);
+        assert_eq!(
+            manager.get_config().await.and_then(|config| config.default_key_id),
+            Some("key-a".to_string())
+        );
+    }
+
+    #[tokio::test]
+    async fn restart_never_unpublishes_the_running_service() {
+        let manager = Arc::new(KmsServiceManager::new());
+        manager.configure(static_config("key-a", 0x11)).await.expect("configure");
+        manager.start().await.expect("start");
+        let old_version = manager.get_service_version().await.expect("old version");
+        let restarting = {
+            let manager = manager.clone();
+            tokio::spawn(async move { manager.restart().await })
+        };
+
+        while !restarting.is_finished() {
+            assert!(manager.get_encryption_service().await.is_some());
+            tokio::task::yield_now().await;
+        }
+        restarting.await.expect("restart task").expect("restart");
+
+        assert!(manager.get_encryption_service().await.is_some());
+        assert!(manager.get_service_version().await.expect("new version") > old_version);
+        assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
+    }
+
+    #[tokio::test]
+    async fn start_or_restart_decides_under_the_lifecycle_lock() {
+        let manager = KmsServiceManager::new();
+        manager.configure(static_config("key-a", 0x11)).await.expect("configure");
+
+        assert_eq!(manager.start_or_restart(false).await.expect("initial start"), KmsStartOutcome::Started);
+        let first_version = manager.get_service_version().await.expect("first version");
+        assert_eq!(
+            manager.start_or_restart(false).await.expect("already running"),
+            KmsStartOutcome::AlreadyRunning
+        );
+        assert_eq!(manager.get_service_version().await, Some(first_version));
+        assert_eq!(manager.start_or_restart(true).await.expect("forced restart"), KmsStartOutcome::Restarted);
+        assert!(manager.get_service_version().await.expect("restarted version") > first_version);
+    }
+
+    #[tokio::test]
+    async fn stale_health_failure_cannot_poison_new_service_status() {
+        let manager = KmsServiceManager::new();
+        manager.configure(static_config("key-a", 0x11)).await.expect("configure");
+        manager.start().await.expect("start");
+        let old_version = manager.get_service_version().await.expect("old version");
+        manager.restart().await.expect("restart");
+
+        manager.mark_health_error_if_current(old_version, &KmsError::backend_error("stale failure"));
+
+        assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
+    }
+
+    #[tokio::test]
+    async fn forbidden_local_master_key_change_preserves_running_config_and_service() {
+        use crate::types::{CreateKeyRequest, KeyUsage};
+        use std::collections::HashMap;
+        use tempfile::TempDir;
+
+        let key_dir = TempDir::new().expect("create local KMS directory");
+        let config = |master_key: &str| {
+            let mut config = KmsConfig::local(key_dir.path().to_path_buf());
+            let BackendConfig::Local(local) = &mut config.backend_config else {
+                panic!("local constructor must create local backend config");
+            };
+            local.master_key = Some(master_key.to_string());
+            config.allow_insecure_dev_defaults = true;
+            config
+        };
+        let manager = KmsServiceManager::new();
+        manager
+            .configure(config("working-master-key"))
+            .await
+            .expect("configure local KMS");
+        manager.start().await.expect("start local KMS");
+        manager
+            .get_manager()
+            .await
+            .expect("running KMS manager")
+            .create_key(CreateKeyRequest {
+                key_name: Some("existing-key".to_string()),
+                key_usage: KeyUsage::EncryptDecrypt,
+                description: None,
+                policy: None,
+                tags: HashMap::new(),
+                origin: None,
+            })
+            .await
+            .expect("create encrypted key");
+        let service_version = manager.get_service_version().await;
+
+        let error = manager
+            .reconfigure(config("wrong-master-key"))
+            .await
+            .expect_err("local master key change must be rejected");
+
+        assert!(error.to_string().contains("master key cannot be changed"));
+        assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
+        assert_eq!(manager.get_service_version().await, service_version);
+        let current = manager.get_config().await.expect("working config must remain");
+        let BackendConfig::Local(local) = current.backend_config else {
+            panic!("working config must remain local");
+        };
+        assert_eq!(local.master_key.as_deref(), Some("working-master-key"));
+    }
+
+    #[tokio::test]
+    async fn configure_cannot_replace_existing_local_backend() {
+        use base64::Engine as _;
+        use tempfile::TempDir;
+
+        let key_dir = TempDir::new().expect("create local KMS directory");
+        let mut local = KmsConfig::local(key_dir.path().to_path_buf());
+        local.allow_insecure_dev_defaults = true;
+        let manager = KmsServiceManager::new();
+        manager.configure(local.clone()).await.expect("configure local KMS");
+
+        let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]);
+        let error = manager
+            .configure(KmsConfig::static_kms("static-key".to_string(), encoded_key))
+            .await
+            .expect_err("existing local backend must be immutable");
+
+        assert!(error.to_string().contains("backend cannot be changed"));
+        let current = manager.get_config().await.expect("local config must remain");
+        assert!(matches!(current.backend_config, BackendConfig::Local(_)));
+    }
+
+    #[tokio::test]
+    async fn reconfigure_allows_safe_local_runtime_settings_only() {
+        use tempfile::TempDir;
+
+        let key_dir = TempDir::new().expect("create local KMS directory");
+        let mut initial = KmsConfig::local(key_dir.path().to_path_buf());
+        initial.allow_insecure_dev_defaults = true;
+        let manager = KmsServiceManager::new();
+        manager.configure(initial.clone()).await.expect("configure local KMS");
+        manager.start().await.expect("start local KMS");
+
+        let mut updated = initial;
+        updated.default_key_id = Some("evaluation-key".to_string());
+        updated.timeout = std::time::Duration::from_secs(45);
+        updated.enable_cache = false;
+        manager
+            .reconfigure(updated.clone())
+            .await
+            .expect("update safe local settings");
+
+        let current = manager.get_config().await.expect("updated config");
+        assert_eq!(current.default_key_id, updated.default_key_id);
+        assert_eq!(current.timeout, updated.timeout);
+        assert!(!current.enable_cache);
+    }
 }

crates/protocols/Cargo.toml

@@ -44,7 +44,7 @@ swift = [
     "dep:tokio-util",
     "dep:serde",
     "dep:urlencoding",
-    "dep:md5",
+    "dep:md-5",
     "dep:quick-xml",
     "dep:hmac",
     "dep:sha1",
@@ -108,7 +108,7 @@ http-body-util = { workspace = true, optional = true }
 tokio-util = { workspace = true, optional = true, features = ["rt", "io", "compat"] }
 serde = { workspace = true, optional = true, features = ["derive"] }
 urlencoding = { workspace = true, optional = true }
-md5 = { workspace = true, optional = true }
+md-5 = { workspace = true, optional = true }
 quick-xml = { workspace = true, optional = true, features = ["serialize"] }
 hmac = { workspace = true, optional = true }
 sha1 = { workspace = true, optional = true }

crates/protocols/src/swift/account.rs

@@ -14,11 +14,11 @@
 
 //! Swift account operations and validation
 
+use super::metadata_update::{ACCOUNT_META_TAG_PREFIX, MetadataUpdate};
 use super::storage_api::account::{BucketOperations, MakeBucketOptions};
 use super::{SwiftError, SwiftResult};
-use super::{get_swift_bucket_metadata, resolve_swift_object_store_handle, update_swift_bucket_tagging, validate_metadata};
+use super::{get_swift_bucket_metadata, resolve_swift_object_store_handle, update_swift_bucket_tagging};
 use rustfs_credentials::Credentials;
-use s3s::dto::{Tag, Tagging};
 use sha2::{Digest, Sha256};
 use std::collections::HashMap;
 
@@ -131,9 +131,8 @@ pub async fn get_account_metadata(account: &str, _credentials: &Option<Credentia
     if let Some(tagging) = &bucket_meta.tagging_config {
         for tag in &tagging.tag_set {
             if let (Some(key), Some(value)) = (&tag.key, &tag.value)
-                && let Some(meta_key) = key.strip_prefix("swift-account-meta-")
+                && let Some(meta_key) = key.strip_prefix(ACCOUNT_META_TAG_PREFIX)
             {
-                // Strip "swift-account-meta-" prefix
                 metadata.insert(meta_key.to_string(), value.clone());
             }
         }
@@ -147,6 +146,12 @@ pub async fn get_account_metadata(account: &str, _credentials: &Option<Credentia
 /// Updates account-level metadata such as TempURL keys.
 /// Only updates swift-account-meta-* tags, preserving other tags.
 ///
+/// Swift account POST is additive: `update` names the items to write and the
+/// items to drop, and everything else keeps its stored value. Replacing the
+/// whole set instead would make an unrelated POST — setting a quota, say —
+/// delete the account's TempURL signing key, permanently invalidating every
+/// outstanding TempURL and FormPost signature for the account.
+///
 /// The caller must own the account: this metadata holds the account's TempURL
 /// signing key, so writing it for someone else's account would let the writer
 /// mint valid pre-signed URLs against that account's objects. Reads
@@ -155,11 +160,11 @@ pub async fn get_account_metadata(account: &str, _credentials: &Option<Credentia
 ///
 /// # Arguments
 /// * `account` - Account identifier
-/// * `metadata` - Metadata key-value pairs to store (keys will be prefixed with `swift-account-meta-`)
+/// * `update` - Metadata items to set and to remove (names are prefixed with `swift-account-meta-`)
 /// * `credentials` - Keystone credentials of the caller
 pub async fn update_account_metadata(
     account: &str,
-    metadata: &HashMap<String, String>,
+    update: &MetadataUpdate,
     credentials: &Option<Credentials>,
 ) -> SwiftResult<()> {
     let Some(credentials) = credentials.as_ref() else {
@@ -171,8 +176,15 @@ pub async fn update_account_metadata(
 
     // These tags are persisted into the bucket metadata file, which every
     // later config write rewrites in full — so unbounded metadata inflates
-    // the cost of unrelated writes for the life of the account.
-    validate_metadata(metadata)?;
+    // the cost of unrelated writes for the life of the account. The item
+    // count is capped against the merged result, inside the rewrite.
+    update.validate()?;
+
+    // An update that names no item changes nothing, so there is no reason to
+    // bring the account's metadata bucket into existence for it.
+    if update.is_empty() {
+        return Ok(());
+    }
 
     let bucket_name = get_account_metadata_bucket_name(account);
 
@@ -190,32 +202,10 @@ pub async fn update_account_metadata(
             .map_err(|e| SwiftError::InternalServerError(format!("Failed to create account metadata bucket: {}", e)))?;
     }
 
-    // Rewrite the persisted tags: replace swift-account-meta-* tags with the
-    // new metadata while preserving other tags. An empty result clears the
-    // tagging config.
-    update_swift_bucket_tagging(bucket_name, |current| {
-        let mut tagging = current.cloned().unwrap_or_else(|| Tagging { tag_set: vec![] });
-
-        tagging.tag_set.retain(|tag| {
-            if let Some(key) = &tag.key {
-                !key.starts_with("swift-account-meta-")
-            } else {
-                true
-            }
-        });
-
-        for (key, value) in metadata {
-            tagging.tag_set.push(Tag {
-                key: Some(format!("swift-account-meta-{}", key)),
-                value: Some(value.clone()),
-            });
-        }
-
-        tagging
-    })
-    .await?;
-
-    Ok(())
+    // Merge into the persisted tags: only the swift-account-meta-* items this
+    // update names change, and non-Swift tags are left alone. An empty result
+    // clears the tagging config.
+    update_swift_bucket_tagging(bucket_name, |current| update.apply_to_tags(current, ACCOUNT_META_TAG_PREFIX)).await
 }
 
 /// Get TempURL key for account

crates/protocols/src/swift/container.rs

@@ -17,15 +17,13 @@
 //! This module implements Swift container CRUD operations and container-bucket translation.
 
 use super::account::validate_account_access;
+use super::metadata_update::{CONTAINER_META_TAG_PREFIX, MetadataUpdate};
 use super::storage_api::container::{
     BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, ListOperations as _, MakeBucketOptions,
 };
 use super::types::Container;
 use super::{SwiftError, SwiftResult};
-use super::{
-    get_swift_bucket_metadata, get_swift_bucket_usage, resolve_swift_object_store_handle, update_swift_bucket_tagging,
-    validate_metadata,
-};
+use super::{get_swift_bucket_metadata, get_swift_bucket_usage, resolve_swift_object_store_handle, update_swift_bucket_tagging};
 use rustfs_credentials::Credentials;
 use s3s::dto::{Tag, Tagging};
 use sha2::{Digest, Sha256};
@@ -62,30 +60,6 @@ fn sanitize_storage_error<E: std::fmt::Display>(operation: &str, error: E) -> Sw
     SwiftError::InternalServerError(format!("{} operation failed", operation))
 }
 
-/// Convert Swift container metadata to S3 tags
-///
-/// Swift container metadata uses X-Container-Meta-* headers.
-/// We store these as S3 tags with "swift-meta-" prefix to distinguish from regular bucket tags.
-///
-/// Example: X-Container-Meta-Color: Blue → S3 Tag: swift-meta-color=Blue
-fn swift_metadata_to_s3_tags(metadata: &std::collections::HashMap<String, String>) -> Option<Tagging> {
-    let mut tags = Vec::new();
-
-    for (key, value) in metadata {
-        // Store with "swift-meta-" prefix to namespace container metadata
-        tags.push(Tag {
-            key: Some(format!("swift-meta-{}", key.to_lowercase())),
-            value: Some(value.clone()),
-        });
-    }
-
-    if tags.is_empty() {
-        None
-    } else {
-        Some(Tagging { tag_set: tags })
-    }
-}
-
 /// Convert S3 tags back to Swift container metadata
 ///
 /// Extracts only tags with "swift-meta-" prefix, which represent Swift container metadata.
@@ -94,11 +68,9 @@ fn s3_tags_to_swift_metadata(tagging: &Tagging) -> std::collections::HashMap<Str
     let mut metadata = std::collections::HashMap::new();
 
     for tag in &tagging.tag_set {
-        // Only process tags with "swift-meta-" prefix
         if let (Some(key), Some(value)) = (&tag.key, &tag.value)
-            && let Some(meta_key) = key.strip_prefix("swift-meta-")
+            && let Some(meta_key) = key.strip_prefix(CONTAINER_META_TAG_PREFIX)
         {
-            // Skip "swift-meta-"
             metadata.insert(meta_key.to_string(), value.clone());
         }
     }
@@ -473,12 +445,15 @@ pub async fn get_container_metadata(account: &str, container: &str, credentials:
 /// - Returns 204 No Content on success
 /// - Returns 404 Not Found if container doesn't exist
 /// - Metadata is provided via X-Container-Meta-* headers
+/// - The update is additive: items the request does not name keep their stored
+///   value, and removal is explicit, via `X-Remove-Container-Meta-{name}` or an
+///   empty value
 #[allow(dead_code)] // Used by handler
 pub async fn update_container_metadata(
     account: &str,
     container: &str,
     credentials: &Credentials,
-    metadata: std::collections::HashMap<String, String>,
+    update: MetadataUpdate,
 ) -> SwiftResult<()> {
     // Validate account access and extract project_id
     let project_id = validate_account_access(account, credentials)?;
@@ -488,8 +463,9 @@ pub async fn update_container_metadata(
 
     // These tags are persisted into the bucket metadata file, which every
     // later config write rewrites in full — so unbounded metadata inflates
-    // the cost of unrelated writes for the life of the container.
-    validate_metadata(&metadata)?;
+    // the cost of unrelated writes for the life of the container. The item
+    // count is capped against the merged result, inside the rewrite.
+    update.validate()?;
 
     // Create mapper with default config (tenant prefixing enabled)
     let mapper = ContainerMapper::default();
@@ -502,7 +478,9 @@ pub async fn update_container_metadata(
         return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
     };
 
-    // Verify container exists
+    // Verify container exists. Checked before the empty-update shortcut below,
+    // so a POST to a container that does not exist still answers 404 whether
+    // or not it carried metadata.
     store
         .get_bucket_info(&bucket_name, &BucketOptions::default())
         .await
@@ -514,32 +492,19 @@ pub async fn update_container_metadata(
             }
         })?;
 
-    // Rewrite the persisted tags: replace swift-meta-* tags with the new
-    // metadata while preserving non-Swift tags. An empty result clears the
-    // tagging config.
-    update_swift_bucket_tagging(bucket_name, |current| {
-        let mut tagging = current.cloned().unwrap_or_else(|| Tagging { tag_set: vec![] });
+    // An update that names no item leaves stored metadata alone, so skip the
+    // persisted write and the peer reload it triggers rather than rewriting
+    // the config to its current value. The handler reaches here on every
+    // container POST, including ACL-only and versioning-only ones.
+    if update.is_empty() {
+        return Ok(());
+    }
 
-        tagging.tag_set.retain(|tag| {
-            if let Some(key) = &tag.key {
-                !key.starts_with("swift-meta-")
-            } else {
-                true // Keep tags with no key (shouldn't happen, but be safe)
-            }
-        });
-
-        if let Some(mut new_tagging) = swift_metadata_to_s3_tags(&metadata) {
-            tagging.tag_set.append(&mut new_tagging.tag_set);
-        }
-        // If metadata.is_empty() and swift_metadata_to_s3_tags returns None,
-        // we've already removed swift-meta-* tags above, so only non-Swift
-        // tags remain
-
-        tagging
-    })
-    .await?;
-
-    Ok(())
+    // Merge into the persisted tags: only the swift-meta-* items this update
+    // names change, so the container's other metadata — and the ACL and
+    // versioning tags sharing this tag set — survive. An empty result clears
+    // the tagging config.
+    update_swift_bucket_tagging(bucket_name, |current| update.apply_to_tags(current, CONTAINER_META_TAG_PREFIX)).await
 }
 
 /// Delete a container
@@ -814,7 +779,7 @@ pub async fn enable_versioning(
             value: Some(archive_container.to_string()), // Store Swift container name, not S3 bucket name
         });
 
-        tagging
+        Ok(tagging)
     })
     .await?;
 
@@ -871,7 +836,7 @@ pub async fn disable_versioning(account: &str, container: &str, credentials: &Cr
             .tag_set
             .retain(|tag| tag.key.as_deref() != Some("swift-versions-location"));
 
-        tagging
+        Ok(tagging)
     })
     .await?;
 
@@ -1026,7 +991,7 @@ pub async fn set_container_acl(
             });
         }
 
-        tagging
+        Ok(tagging)
     })
     .await?;
 
@@ -1374,64 +1339,6 @@ mod tests {
         assert!(first_char.is_ascii_lowercase() || first_char.is_ascii_digit());
     }
 
-    #[test]
-    fn test_swift_metadata_to_s3_tags() {
-        let mut metadata = std::collections::HashMap::new();
-        metadata.insert("color".to_string(), "blue".to_string());
-        metadata.insert("description".to_string(), "test container".to_string());
-
-        let tagging = swift_metadata_to_s3_tags(&metadata).unwrap();
-        assert_eq!(tagging.tag_set.len(), 2);
-
-        // Verify tags have swift-meta- prefix
-        let color_tag = tagging
-            .tag_set
-            .iter()
-            .find(|t| t.key.as_deref() == Some("swift-meta-color"))
-            .expect("color tag not found");
-        assert_eq!(color_tag.value.as_deref(), Some("blue"));
-
-        let desc_tag = tagging
-            .tag_set
-            .iter()
-            .find(|t| t.key.as_deref() == Some("swift-meta-description"))
-            .expect("description tag not found");
-        assert_eq!(desc_tag.value.as_deref(), Some("test container"));
-    }
-
-    #[test]
-    fn test_swift_metadata_to_s3_tags_empty() {
-        let metadata = std::collections::HashMap::new();
-        let tagging = swift_metadata_to_s3_tags(&metadata);
-        assert!(tagging.is_none());
-    }
-
-    #[test]
-    fn test_swift_metadata_to_s3_tags_case_normalization() {
-        let mut metadata = std::collections::HashMap::new();
-        metadata.insert("Color".to_string(), "Red".to_string());
-        metadata.insert("PRIORITY".to_string(), "High".to_string());
-
-        let tagging = swift_metadata_to_s3_tags(&metadata).unwrap();
-
-        // Keys should be lowercased
-        assert!(tagging.tag_set.iter().any(|t| t.key.as_deref() == Some("swift-meta-color")));
-        assert!(
-            tagging
-                .tag_set
-                .iter()
-                .any(|t| t.key.as_deref() == Some("swift-meta-priority"))
-        );
-
-        // Values should be preserved as-is
-        let color_tag = tagging
-            .tag_set
-            .iter()
-            .find(|t| t.key.as_deref() == Some("swift-meta-color"))
-            .unwrap();
-        assert_eq!(color_tag.value.as_deref(), Some("Red"));
-    }
-
     #[test]
     fn test_s3_tags_to_swift_metadata() {
         let tagging = Tagging {
@@ -1487,91 +1394,80 @@ mod tests {
 
     #[test]
     fn test_metadata_roundtrip() {
-        // Test that we can convert metadata -> tags -> metadata without loss
-        let mut original_metadata = std::collections::HashMap::new();
-        original_metadata.insert("color".to_string(), "blue".to_string());
-        original_metadata.insert("owner".to_string(), "alice".to_string());
-        original_metadata.insert("priority".to_string(), "high".to_string());
+        // What a POST writes must be what a HEAD reads back: run the items
+        // through the tag-merge write path and the tag-parse read path.
+        let update = MetadataUpdate::default()
+            .set("color", "blue")
+            .set("owner", "alice")
+            .set("priority", "high");
 
-        let tagging = swift_metadata_to_s3_tags(&original_metadata).unwrap();
-        let recovered_metadata = s3_tags_to_swift_metadata(&tagging);
+        let tagging = update
+            .apply_to_tags(None, CONTAINER_META_TAG_PREFIX)
+            .expect("merge should be accepted");
+        let recovered = s3_tags_to_swift_metadata(&tagging);
 
-        assert_eq!(recovered_metadata.len(), original_metadata.len());
-        for (key, value) in &original_metadata {
-            assert_eq!(
-                recovered_metadata.get(&key.to_lowercase()),
-                Some(value),
-                "Metadata key {} not preserved in roundtrip",
-                key
-            );
-        }
+        assert_eq!(recovered.len(), 3);
+        assert_eq!(recovered.get("color").map(String::as_str), Some("blue"));
+        assert_eq!(recovered.get("owner").map(String::as_str), Some("alice"));
+        assert_eq!(recovered.get("priority").map(String::as_str), Some("high"));
     }
 
     #[test]
     fn test_tag_preservation_merge_with_existing() {
-        // Test merging Swift metadata with existing non-Swift tags
-        let mut existing_tagging = Tagging {
+        // A container metadata POST shares its tag set with the container ACL
+        // and versioning tags, and with whatever S3 tags the bucket carries.
+        // Only the swift-meta-* items the POST names may change.
+        let existing = Tagging {
             tag_set: vec![
                 Tag {
                     key: Some("swift-meta-color".to_string()),
                     value: Some("blue".to_string()),
                 },
+                Tag {
+                    key: Some("swift-acl-read".to_string()),
+                    value: Some(".r:*".to_string()),
+                },
+                Tag {
+                    key: Some("swift-versions-location".to_string()),
+                    value: Some("archive".to_string()),
+                },
                 Tag {
                     key: Some("env".to_string()),
                     value: Some("production".to_string()),
                 },
-                Tag {
-                    key: Some("team".to_string()),
-                    value: Some("backend".to_string()),
-                },
             ],
         };
 
-        // Remove old swift-meta-* tags
-        existing_tagging.tag_set.retain(|tag| {
-            if let Some(key) = &tag.key {
-                !key.starts_with("swift-meta-")
-            } else {
-                true
-            }
-        });
+        let merged = MetadataUpdate::default()
+            .set("description", "test")
+            .apply_to_tags(Some(&existing), CONTAINER_META_TAG_PREFIX)
+            .expect("merge should be accepted");
+        let tag = |key: &str| {
+            merged
+                .tag_set
+                .iter()
+                .find(|t| t.key.as_deref() == Some(key))
+                .and_then(|t| t.value.as_deref())
+        };
 
-        // Add new Swift metadata
-        let mut new_metadata = std::collections::HashMap::new();
-        new_metadata.insert("description".to_string(), "test".to_string());
-        let mut new_tagging = swift_metadata_to_s3_tags(&new_metadata).unwrap();
-
-        // Merge
-        existing_tagging.tag_set.append(&mut new_tagging.tag_set);
-
-        // Verify: should have env, team, and new swift-meta-description
-        assert_eq!(existing_tagging.tag_set.len(), 3);
-
-        let has_env = existing_tagging.tag_set.iter().any(|t| t.key.as_deref() == Some("env"));
-        let has_team = existing_tagging.tag_set.iter().any(|t| t.key.as_deref() == Some("team"));
-        let has_description = existing_tagging
-            .tag_set
-            .iter()
-            .any(|t| t.key.as_deref() == Some("swift-meta-description"));
-
-        assert!(has_env, "env tag should be preserved");
-        assert!(has_team, "team tag should be preserved");
-        assert!(has_description, "swift-meta-description should be added");
+        assert_eq!(tag("swift-meta-description"), Some("test"), "the new item should be added");
+        assert_eq!(tag("swift-meta-color"), Some("blue"), "an unnamed item should be preserved");
+        assert_eq!(tag("swift-acl-read"), Some(".r:*"), "the ACL tag should be preserved");
+        assert_eq!(tag("swift-versions-location"), Some("archive"), "the versioning tag should be preserved");
+        assert_eq!(tag("env"), Some("production"), "non-Swift tags should be preserved");
+        assert_eq!(merged.tag_set.len(), 5);
     }
 
     #[test]
     fn test_tag_preservation_remove_only_swift() {
-        // Test that clearing Swift metadata preserves non-Swift tags
-        let mut existing_tagging = Tagging {
+        // Removing the last container metadata item leaves the other tags —
+        // and so must not clear the tagging config.
+        let existing = Tagging {
             tag_set: vec![
                 Tag {
                     key: Some("swift-meta-color".to_string()),
                     value: Some("blue".to_string()),
                 },
-                Tag {
-                    key: Some("swift-meta-owner".to_string()),
-                    value: Some("alice".to_string()),
-                },
                 Tag {
                     key: Some("env".to_string()),
                     value: Some("production".to_string()),
@@ -1583,37 +1479,28 @@ mod tests {
             ],
         };
 
-        // Remove swift-meta-* tags (simulating empty metadata update)
-        existing_tagging.tag_set.retain(|tag| {
-            if let Some(key) = &tag.key {
-                !key.starts_with("swift-meta-")
-            } else {
-                true
-            }
-        });
+        let merged = MetadataUpdate::default()
+            .remove("color")
+            .apply_to_tags(Some(&existing), CONTAINER_META_TAG_PREFIX)
+            .expect("merge should be accepted");
 
-        // Verify: should only have env and cost-center
-        assert_eq!(existing_tagging.tag_set.len(), 2);
-
-        let has_env = existing_tagging.tag_set.iter().any(|t| t.key.as_deref() == Some("env"));
-        let has_cost_center = existing_tagging
-            .tag_set
-            .iter()
-            .any(|t| t.key.as_deref() == Some("cost-center"));
-        let has_swift_meta = existing_tagging
-            .tag_set
-            .iter()
-            .any(|t| t.key.as_ref().is_some_and(|k| k.starts_with("swift-meta-")));
-
-        assert!(has_env, "env tag should be preserved");
-        assert!(has_cost_center, "cost-center tag should be preserved");
-        assert!(!has_swift_meta, "all swift-meta-* tags should be removed");
+        assert_eq!(merged.tag_set.len(), 2);
+        assert!(merged.tag_set.iter().any(|t| t.key.as_deref() == Some("env")));
+        assert!(merged.tag_set.iter().any(|t| t.key.as_deref() == Some("cost-center")));
+        assert!(
+            !merged
+                .tag_set
+                .iter()
+                .any(|t| t.key.as_ref().is_some_and(|k| k.starts_with(CONTAINER_META_TAG_PREFIX))),
+            "the removed item should be gone"
+        );
     }
 
     #[test]
     fn test_tag_preservation_empty_after_swift_removal() {
-        // Test that if only Swift tags exist, clearing them results in empty tagging
-        let mut existing_tagging = Tagging {
+        // Removing every item when nothing else is tagged empties the tag set,
+        // which is how the caller knows to clear the tagging config.
+        let existing = Tagging {
             tag_set: vec![
                 Tag {
                     key: Some("swift-meta-color".to_string()),
@@ -1626,38 +1513,13 @@ mod tests {
             ],
         };
 
-        // Remove swift-meta-* tags
-        existing_tagging.tag_set.retain(|tag| {
-            if let Some(key) = &tag.key {
-                !key.starts_with("swift-meta-")
-            } else {
-                true
-            }
-        });
+        let merged = MetadataUpdate::default()
+            .remove("color")
+            .remove("owner")
+            .apply_to_tags(Some(&existing), CONTAINER_META_TAG_PREFIX)
+            .expect("merge should be accepted");
 
-        // Verify: should be empty
-        assert!(
-            existing_tagging.tag_set.is_empty(),
-            "tagging should be empty after removing all swift-meta-* tags"
-        );
-    }
-
-    #[test]
-    fn test_tag_preservation_no_existing_tags() {
-        // Test adding Swift metadata when no tags exist
-        let existing_tagging = Tagging { tag_set: vec![] };
-
-        let mut new_metadata = std::collections::HashMap::new();
-        new_metadata.insert("color".to_string(), "blue".to_string());
-        let mut new_tagging = swift_metadata_to_s3_tags(&new_metadata).unwrap();
-
-        let mut merged = existing_tagging.clone();
-        merged.tag_set.append(&mut new_tagging.tag_set);
-
-        // Verify: should have only the new Swift tag
-        assert_eq!(merged.tag_set.len(), 1);
-        assert_eq!(merged.tag_set[0].key.as_deref(), Some("swift-meta-color"));
-        assert_eq!(merged.tag_set[0].value.as_deref(), Some("blue"));
+        assert!(merged.tag_set.is_empty(), "tagging should be empty after removing every swift-meta-* tag");
     }
 
     // Object Versioning Tests

crates/protocols/src/swift/handler.rs

@@ -20,6 +20,7 @@
 
 use super::container;
 use super::dlo;
+use super::metadata_update::MetadataUpdate;
 use super::object;
 use super::slo;
 use super::tempurl;
@@ -321,32 +322,15 @@ async fn handle_authenticated_request(
                     Err(SwiftError::NotImplemented("Swift Account HEAD operation not yet implemented".to_string()))
                 }
                 Method::POST => {
-                    // Account metadata update - extract headers
-                    let mut metadata = std::collections::HashMap::new();
+                    // Account metadata update. Additive, per Swift: the request
+                    // names the items to set (X-Account-Meta-*) and the items to
+                    // drop (X-Remove-Account-Meta-*, or an empty value), and
+                    // everything it does not name keeps its stored value. The
+                    // TempURL signing key lives here, so a replacing POST would
+                    // invalidate every outstanding signature for the account.
+                    let update = MetadataUpdate::from_account_headers(&headers);
 
-                    // Extract X-Account-Meta-* headers
-                    for (key, value) in &headers {
-                        let key_str = key.as_str();
-                        if let Some(meta_key) = key_str.strip_prefix("x-account-meta-") {
-                            // Strip "x-account-meta-"
-                            if let Ok(value_str) = value.to_str() {
-                                metadata.insert(meta_key.to_string(), value_str.to_string());
-                            }
-                        }
-                    }
-
-                    // Special handling for TempURL key headers
-                    // X-Account-Meta-Temp-URL-Key or X-Account-Meta-Temp-Url-Key
-                    if let Some(tempurl_key) = headers
-                        .get("x-account-meta-temp-url-key")
-                        .or_else(|| headers.get("x-account-meta-temp-Url-key"))
-                        && let Ok(key_str) = tempurl_key.to_str()
-                    {
-                        metadata.insert("temp-url-key".to_string(), key_str.to_string());
-                    }
-
-                    // Update account metadata
-                    super::account::update_account_metadata(&account, &metadata, &credentials_opt).await?;
+                    super::account::update_account_metadata(&account, &update, &credentials_opt).await?;
 
                     let trans_id = generate_trans_id();
                     Response::builder()
@@ -581,17 +565,15 @@ async fn handle_authenticated_request(
                         container::set_container_acl(&account, &container, new_read, new_write, &credentials).await?;
                     }
 
-                    // Update container metadata - now we have access to request headers
-                    let mut metadata = std::collections::HashMap::new();
-                    for (name, value) in headers.iter() {
-                        if let Some(meta_key) = name.as_str().strip_prefix("x-container-meta-")
-                            && let Ok(value_str) = value.to_str()
-                        {
-                            metadata.insert(meta_key.to_string(), value_str.to_string());
-                        }
-                    }
+                    // Update container metadata. Additive, per Swift: items the
+                    // request does not name keep their stored value, and removal
+                    // is explicit (X-Remove-Container-Meta-*, or an empty value).
+                    // Still called when the request named no item — an ACL-only
+                    // or versioning-only POST — because this is what reports 404
+                    // for a container that does not exist.
+                    let update = MetadataUpdate::from_container_headers(&headers);
 
-                    container::update_container_metadata(&account, &container, &credentials, metadata).await?;
+                    container::update_container_metadata(&account, &container, &credentials, update).await?;
 
                     let trans_id = generate_trans_id();
                     Response::builder()

crates/protocols/src/swift/metadata_update.rs

@@ -0,0 +1,410 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Account and container metadata updates
+//!
+//! Swift account and container POSTs are *additive*: an item the request does
+//! not mention keeps its stored value, and removal is explicit — either
+//! `X-Remove-{Account,Container}-Meta-{name}`, or the item sent with an empty
+//! value. Object POST is the one that replaces the whole set; `swift::object`
+//! handles that separately and must keep doing so.
+//!
+//! Getting this wrong is not a cosmetic divergence. Account metadata holds the
+//! TempURL signing key, so a POST that set an unrelated item while dropping
+//! the rest would invalidate every outstanding TempURL and FormPost signature
+//! for the account.
+//!
+//! This module turns a request's headers into the items to write and the items
+//! to drop, and applies that to the bucket tag set the metadata is persisted
+//! in.
+
+use super::{MAX_METADATA_COUNT, MAX_METADATA_VALUE_SIZE, SwiftError, SwiftResult};
+use axum::http::HeaderMap;
+use s3s::dto::{Tag, Tagging};
+use std::collections::{BTreeMap, BTreeSet};
+
+/// Request header prefix carrying an account metadata item.
+const ACCOUNT_META_HEADER_PREFIX: &str = "x-account-meta-";
+/// Request header prefix removing an account metadata item.
+const ACCOUNT_META_REMOVE_HEADER_PREFIX: &str = "x-remove-account-meta-";
+/// Request header prefix carrying a container metadata item.
+const CONTAINER_META_HEADER_PREFIX: &str = "x-container-meta-";
+/// Request header prefix removing a container metadata item.
+const CONTAINER_META_REMOVE_HEADER_PREFIX: &str = "x-remove-container-meta-";
+
+/// Bucket-tag namespace holding account metadata items.
+pub(crate) const ACCOUNT_META_TAG_PREFIX: &str = "swift-account-meta-";
+/// Bucket-tag namespace holding container metadata items.
+///
+/// Deliberately narrower than the `swift-` tags around it: the container ACL
+/// (`swift-acl-*`) and versioning (`swift-versions-location`) tags share this
+/// tag set and must survive a metadata POST.
+pub(crate) const CONTAINER_META_TAG_PREFIX: &str = "swift-meta-";
+
+/// The metadata changes carried by one account or container POST.
+///
+/// Item names are held lowercased. Swift metadata names are case-insensitive,
+/// HTTP header names arrive lowercased anyway, and a removal has to match the
+/// name a previous POST stored — so normalizing once here is what makes
+/// `X-Remove-Container-Meta-Color` find a stored `color`.
+///
+/// Ordered rather than hashed so the persisted tag set — and therefore the
+/// serialized XML — comes out in a stable order for a given update.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct MetadataUpdate {
+    /// Items to write, by name.
+    items: BTreeMap<String, String>,
+    /// Names to drop.
+    removals: BTreeSet<String>,
+}
+
+impl MetadataUpdate {
+    /// Write `name` = `value`.
+    pub fn set(mut self, name: &str, value: &str) -> Self {
+        let name = name.to_lowercase();
+        self.removals.remove(&name);
+        self.items.insert(name, value.to_string());
+        self
+    }
+
+    /// Drop `name`, if it is stored.
+    pub fn remove(mut self, name: &str) -> Self {
+        let name = name.to_lowercase();
+        self.items.remove(&name);
+        self.removals.insert(name);
+        self
+    }
+
+    /// Parse the metadata headers of an account POST.
+    pub(crate) fn from_account_headers(headers: &HeaderMap) -> Self {
+        Self::from_headers(headers, ACCOUNT_META_HEADER_PREFIX, ACCOUNT_META_REMOVE_HEADER_PREFIX)
+    }
+
+    /// Parse the metadata headers of a container POST.
+    pub(crate) fn from_container_headers(headers: &HeaderMap) -> Self {
+        Self::from_headers(headers, CONTAINER_META_HEADER_PREFIX, CONTAINER_META_REMOVE_HEADER_PREFIX)
+    }
+
+    /// Parse metadata headers under `item_prefix`, and removals under
+    /// `remove_prefix`. Both prefixes are lowercase, matching how `http`
+    /// normalizes header names.
+    ///
+    /// An item sent with an empty value is a removal — the deletion path
+    /// Swift clients use when they do not send a dedicated remove header.
+    /// A name carried by both header forms is removed: the explicit removal
+    /// wins, as it does in Swift, where a remove header is rewritten into an
+    /// empty-valued item header.
+    fn from_headers(headers: &HeaderMap, item_prefix: &str, remove_prefix: &str) -> Self {
+        let mut update = Self::default();
+
+        for (name, value) in headers {
+            let Some(item) = name.as_str().strip_prefix(item_prefix) else {
+                continue;
+            };
+            // A value that is not valid UTF-8 cannot be stored as a tag.
+            // Skipping it matches how the rest of the Swift handlers treat
+            // unreadable header values.
+            let Ok(value) = value.to_str() else {
+                continue;
+            };
+
+            update = if value.is_empty() {
+                update.remove(item)
+            } else {
+                update.set(item, value)
+            };
+        }
+
+        for name in headers.keys() {
+            if let Some(item) = name.as_str().strip_prefix(remove_prefix) {
+                update = update.remove(item);
+            }
+        }
+
+        update
+    }
+
+    /// Whether this update changes anything.
+    ///
+    /// An ACL-only or versioning-only container POST produces an empty update:
+    /// it names no metadata item, so it must leave stored metadata alone.
+    pub(crate) fn is_empty(&self) -> bool {
+        self.items.is_empty() && self.removals.is_empty()
+    }
+
+    /// Reject oversized values before anything is persisted.
+    ///
+    /// The item *count* is not checked here — an additive POST has to be
+    /// measured against the merged result, which only [`Self::apply_to_tags`]
+    /// can see.
+    pub(crate) fn validate(&self) -> SwiftResult<()> {
+        for (name, value) in &self.items {
+            if value.len() > MAX_METADATA_VALUE_SIZE {
+                return Err(SwiftError::BadRequest(format!(
+                    "Metadata value for '{}' too large: {} bytes (max: {} bytes)",
+                    name,
+                    value.len(),
+                    MAX_METADATA_VALUE_SIZE
+                )));
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Merge this update into the persisted tag set.
+    ///
+    /// `prefix` is the tag namespace holding the items. Tags outside it — the
+    /// container ACL and versioning tags, plus any S3 tags the bucket carries
+    /// — are untouched, and so are the namespaced items this update does not
+    /// name.
+    ///
+    /// The item-count cap is checked against the merged result rather than the
+    /// request: because POSTs are additive, a client could otherwise walk past
+    /// it one header at a time. This runs inside the bucket metadata write
+    /// guard, so the count it checks is the one about to be persisted.
+    pub(crate) fn apply_to_tags(&self, current: Option<&Tagging>, prefix: &str) -> SwiftResult<Tagging> {
+        let mut tagging = current.cloned().unwrap_or_else(|| Tagging { tag_set: vec![] });
+
+        tagging.tag_set.retain(|tag| match item_name(tag, prefix) {
+            Some(name) => !self.items.contains_key(name) && !self.removals.contains(name),
+            None => true,
+        });
+
+        let merged = tagging.tag_set.iter().filter(|tag| item_name(tag, prefix).is_some()).count() + self.items.len();
+        if merged > MAX_METADATA_COUNT {
+            return Err(SwiftError::BadRequest(format!(
+                "Too many metadata headers: {} (max: {})",
+                merged, MAX_METADATA_COUNT
+            )));
+        }
+
+        for (name, value) in &self.items {
+            tagging.tag_set.push(Tag {
+                key: Some(format!("{}{}", prefix, name)),
+                value: Some(value.clone()),
+            });
+        }
+
+        Ok(tagging)
+    }
+}
+
+/// The metadata item name a tag carries, or `None` if the tag does not belong
+/// to this namespace.
+fn item_name<'a>(tag: &'a Tag, prefix: &str) -> Option<&'a str> {
+    tag.key.as_deref()?.strip_prefix(prefix)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use axum::http::{HeaderName, HeaderValue};
+
+    fn headers(pairs: &[(&str, &str)]) -> HeaderMap {
+        let mut map = HeaderMap::new();
+        for (name, value) in pairs {
+            map.insert(
+                HeaderName::from_bytes(name.as_bytes()).expect("test header name should parse"),
+                HeaderValue::from_str(value).expect("test header value should parse"),
+            );
+        }
+        map
+    }
+
+    fn tags(pairs: &[(&str, &str)]) -> Tagging {
+        Tagging {
+            tag_set: pairs
+                .iter()
+                .map(|(key, value)| Tag {
+                    key: Some((*key).to_string()),
+                    value: Some((*value).to_string()),
+                })
+                .collect(),
+        }
+    }
+
+    fn tag_value<'a>(tagging: &'a Tagging, key: &str) -> Option<&'a str> {
+        tagging
+            .tag_set
+            .iter()
+            .find(|tag| tag.key.as_deref() == Some(key))
+            .and_then(|tag| tag.value.as_deref())
+    }
+
+    #[test]
+    fn from_headers_collects_items_and_ignores_unrelated_headers() {
+        let update = MetadataUpdate::from_container_headers(&headers(&[
+            ("x-container-meta-color", "blue"),
+            ("x-container-read", ".r:*"),
+            ("content-type", "text/plain"),
+        ]));
+
+        assert_eq!(update, MetadataUpdate::default().set("color", "blue"));
+    }
+
+    #[test]
+    fn from_headers_lowercases_item_names() {
+        // `http` normalizes header names on the way in, so the mixed case a
+        // client sends is already gone by the time a handler sees it; the
+        // lowercasing here is what makes a directly built update match.
+        let update = MetadataUpdate::from_container_headers(&headers(&[("X-Container-Meta-Color", "Blue")]));
+
+        assert_eq!(update, MetadataUpdate::default().set("COLOR", "Blue"));
+    }
+
+    #[test]
+    fn empty_value_is_a_removal() {
+        let update = MetadataUpdate::from_container_headers(&headers(&[("x-container-meta-color", "")]));
+
+        assert_eq!(update, MetadataUpdate::default().remove("color"));
+    }
+
+    #[test]
+    fn remove_header_drops_the_item() {
+        let update = MetadataUpdate::from_account_headers(&headers(&[("x-remove-account-meta-temp-url-key", "x")]));
+
+        assert_eq!(update, MetadataUpdate::default().remove("temp-url-key"));
+    }
+
+    #[test]
+    fn remove_header_wins_over_a_value_for_the_same_item() {
+        let update = MetadataUpdate::from_container_headers(&headers(&[
+            ("x-container-meta-color", "blue"),
+            ("x-remove-container-meta-color", "x"),
+        ]));
+
+        assert_eq!(update, MetadataUpdate::default().remove("color"));
+    }
+
+    #[test]
+    fn a_removal_header_is_not_mistaken_for_an_item() {
+        // "x-remove-container-meta-color" must not also parse as the item
+        // "remove-container-meta-color" or similar.
+        let update = MetadataUpdate::from_container_headers(&headers(&[("x-remove-container-meta-color", "x")]));
+
+        assert!(update.items.is_empty(), "a removal header must not set an item");
+    }
+
+    #[test]
+    fn a_request_with_no_metadata_headers_is_empty() {
+        assert!(MetadataUpdate::from_container_headers(&headers(&[("x-container-read", ".r:*")])).is_empty());
+        assert!(MetadataUpdate::default().is_empty());
+        assert!(!MetadataUpdate::default().remove("color").is_empty());
+    }
+
+    #[test]
+    fn apply_preserves_items_the_update_does_not_name() {
+        let current = tags(&[
+            ("swift-meta-color", "blue"),
+            ("swift-meta-season", "summer"),
+            ("swift-acl-read", ".r:*"),
+            ("swift-versions-location", "archive"),
+            ("unrelated-s3-tag", "keep"),
+        ]);
+
+        let merged = MetadataUpdate::default()
+            .set("mood", "calm")
+            .apply_to_tags(Some(&current), CONTAINER_META_TAG_PREFIX)
+            .expect("merge should be accepted");
+
+        assert_eq!(tag_value(&merged, "swift-meta-color"), Some("blue"));
+        assert_eq!(tag_value(&merged, "swift-meta-season"), Some("summer"));
+        assert_eq!(tag_value(&merged, "swift-meta-mood"), Some("calm"));
+        assert_eq!(tag_value(&merged, "swift-acl-read"), Some(".r:*"));
+        assert_eq!(tag_value(&merged, "swift-versions-location"), Some("archive"));
+        assert_eq!(tag_value(&merged, "unrelated-s3-tag"), Some("keep"));
+    }
+
+    #[test]
+    fn apply_overwrites_a_named_item_exactly_once() {
+        let current = tags(&[("swift-meta-color", "blue")]);
+
+        let merged = MetadataUpdate::default()
+            .set("color", "red")
+            .apply_to_tags(Some(&current), CONTAINER_META_TAG_PREFIX)
+            .expect("merge should be accepted");
+
+        assert_eq!(merged.tag_set.len(), 1, "overwriting must not duplicate the tag");
+        assert_eq!(tag_value(&merged, "swift-meta-color"), Some("red"));
+    }
+
+    #[test]
+    fn apply_drops_only_the_removed_item() {
+        let current = tags(&[
+            ("swift-meta-color", "blue"),
+            ("swift-meta-season", "summer"),
+            ("swift-acl-read", ".r:*"),
+        ]);
+
+        let merged = MetadataUpdate::default()
+            .remove("color")
+            .apply_to_tags(Some(&current), CONTAINER_META_TAG_PREFIX)
+            .expect("merge should be accepted");
+
+        assert_eq!(tag_value(&merged, "swift-meta-color"), None);
+        assert_eq!(tag_value(&merged, "swift-meta-season"), Some("summer"));
+        assert_eq!(tag_value(&merged, "swift-acl-read"), Some(".r:*"));
+    }
+
+    #[test]
+    fn apply_to_an_untagged_bucket_starts_from_nothing() {
+        let merged = MetadataUpdate::default()
+            .set("color", "blue")
+            .apply_to_tags(None, ACCOUNT_META_TAG_PREFIX)
+            .expect("merge should be accepted");
+
+        assert_eq!(tag_value(&merged, "swift-account-meta-color"), Some("blue"));
+    }
+
+    #[test]
+    fn apply_caps_the_merged_item_count_not_the_request() {
+        let current = Tagging {
+            tag_set: (0..MAX_METADATA_COUNT)
+                .map(|i| Tag {
+                    key: Some(format!("{}item{}", CONTAINER_META_TAG_PREFIX, i)),
+                    value: Some("v".to_string()),
+                })
+                .collect(),
+        };
+
+        // One more item than the container already stores: rejected, even
+        // though the request itself carries a single header.
+        let err = MetadataUpdate::default()
+            .set("overflow", "v")
+            .apply_to_tags(Some(&current), CONTAINER_META_TAG_PREFIX)
+            .expect_err("exceeding the item cap must be rejected");
+        assert!(matches!(err, SwiftError::BadRequest(_)), "expected BadRequest, got {err:?}");
+
+        // Overwriting an item already counted stays at the cap.
+        MetadataUpdate::default()
+            .set("item0", "v2")
+            .apply_to_tags(Some(&current), CONTAINER_META_TAG_PREFIX)
+            .expect("overwriting an existing item must not trip the cap");
+    }
+
+    #[test]
+    fn validate_rejects_oversized_values() {
+        let err = MetadataUpdate::default()
+            .set("color", &"b".repeat(MAX_METADATA_VALUE_SIZE + 1))
+            .validate()
+            .expect_err("an oversized value must be rejected");
+        assert!(matches!(err, SwiftError::BadRequest(_)), "expected BadRequest, got {err:?}");
+
+        MetadataUpdate::default()
+            .set("color", &"b".repeat(MAX_METADATA_VALUE_SIZE))
+            .validate()
+            .expect("a value at the limit must be accepted");
+    }
+}

crates/protocols/src/swift/mod.rs

@@ -44,6 +44,7 @@ pub mod expiration;
 pub mod expiration_worker;
 pub mod formpost;
 pub mod handler;
+pub mod metadata_update;
 pub mod object;
 pub mod quota;
 pub mod router;
@@ -57,6 +58,7 @@ pub mod types;
 pub mod versioning;
 
 pub use errors::{SwiftError, SwiftResult};
+pub use metadata_update::MetadataUpdate;
 pub use router::{SwiftRoute, SwiftRouter};
 
 /// Maximum number of metadata headers allowed per resource (Swift standard)
@@ -71,9 +73,10 @@ pub(crate) const MAX_METADATA_VALUE_SIZE: usize = 256;
 /// - Total number of metadata entries doesn't exceed MAX_METADATA_COUNT
 /// - Individual metadata values don't exceed MAX_METADATA_VALUE_SIZE
 ///
-/// Applies to object, container and account metadata alike: all three are
-/// persisted, and container/account metadata additionally lands in the
-/// bucket metadata file that every later config write rewrites in full.
+/// This is the object-metadata form, where a POST replaces the whole set and
+/// the request is therefore the complete result. Account and container POSTs
+/// are additive, so their count has to be measured against the merged state
+/// instead — see [`MetadataUpdate::apply_to_tags`].
 ///
 /// Returns error if limits are exceeded.
 pub(crate) fn validate_metadata(metadata: &std::collections::HashMap<String, String>) -> SwiftResult<()> {

crates/protocols/src/swift/slo.rs

@@ -21,6 +21,7 @@
 use super::storage_api::large_object::HTTPRangeSpec;
 use super::{SwiftError, object};
 use axum::http::{HeaderMap, Response, StatusCode};
+use md5::{Digest as Md5Digest, Md5};
 use rustfs_credentials::Credentials;
 use s3s::Body;
 use serde::{Deserialize, Serialize};
@@ -81,9 +82,9 @@ impl SLOManifest {
             etag_concat.push_str(etag);
         }
 
-        // Calculate MD5 hash
-        let hash = md5::compute(etag_concat.as_bytes());
-        format!("\"{:x}-{}\"", hash, self.segments.len())
+        let mut hasher = Md5::new();
+        hasher.update(etag_concat.as_bytes());
+        format!("\"{}-{}\"", hex::encode(hasher.finalize()), self.segments.len())
     }
 
     /// Validate manifest against actual segments

crates/protocols/src/swift/storage_api.rs

@@ -88,6 +88,10 @@ pub(crate) async fn get_swift_bucket_metadata(bucket: &str) -> SwiftStorageResul
 /// disk-truth reloads. Peers are then told to reload, matching what the S3
 /// handlers do after a config write.
 ///
+/// A rewrite may also refuse the update outright, for limits that can only be
+/// judged against the state being merged into. That verdict is the client's
+/// answer, so it is returned as-is rather than folded into a storage error.
+///
 /// Storage failures are logged in full and reported to the client as a
 /// generic error: these now carry real disk and quorum detail, which does not
 /// belong in a Swift response body. The one exception is an unreadable
@@ -95,8 +99,12 @@ pub(crate) async fn get_swift_bucket_metadata(bucket: &str) -> SwiftStorageResul
 /// to act on it.
 pub(crate) async fn update_swift_bucket_tagging<F>(bucket: String, rewrite: F) -> SwiftResult<()>
 where
-    F: FnOnce(Option<&Tagging>) -> Tagging + Send,
+    F: FnOnce(Option<&Tagging>) -> SwiftResult<Tagging> + Send,
 {
+    // Carries a rewrite's refusal back out past the storage error the config
+    // write has to fail with to abort the transaction.
+    let mut rejected = None;
+
     let result = update_config_with(&bucket, BUCKET_TAGGING_CONFIG, |bm| {
         // Merging onto an unparseable tag set would silently drop every tag
         // the bucket has — including the container ACL and versioning tags —
@@ -106,7 +114,14 @@ where
             return Err(SwiftStorageError::other(UNREADABLE_TAGGING_SENTINEL));
         }
 
-        let tagging = rewrite(bm.tagging_config.as_ref());
+        let tagging = match rewrite(bm.tagging_config.as_ref()) {
+            Ok(tagging) => tagging,
+            Err(err) => {
+                rejected = Some(err);
+                return Err(SwiftStorageError::other("swift: tagging rewrite rejected the update"));
+            }
+        };
+
         if tagging.tag_set.is_empty() {
             Ok(Vec::new())
         } else {
@@ -118,6 +133,12 @@ where
     })
     .await;
 
+    // Nothing was written, but that is the rewrite's own decision about the
+    // request rather than a storage fault, so it is not logged as one.
+    if let Some(err) = rejected {
+        return Err(err);
+    }
+
     if let Err(err) = result {
         let unreadable = err.to_string().contains(UNREADABLE_TAGGING_SENTINEL);
         tracing::error!(

crates/protocols/tests/swift_metadata_persistence.rs

@@ -16,15 +16,20 @@
 //! metadata file, not just the in-memory cache. The metadata has to survive
 //! the disk-truth reloads performed by peer LoadBucketMetadata notifications
 //! and the periodic refresh loop — and, transitively, a process restart.
+//!
+//! Durability is what makes the *content* of those writes matter, so this file
+//! also covers Swift's additive account/container POST semantics: an item the
+//! request does not name keeps its stored value, and removal is explicit. The
+//! two are tested together because a reload is the only way to tell a real
+//! merge from one that happened to look right in the cache.
 
 #![cfg(feature = "swift")]
 
 use std::collections::HashMap;
 
 use rustfs_credentials::Credentials;
-use rustfs_protocols::swift::SwiftError;
 use rustfs_protocols::swift::container::{ContainerMapper, update_container_metadata};
-use rustfs_protocols::swift::{account, container};
+use rustfs_protocols::swift::{MetadataUpdate, SwiftError, account, container};
 use rustfs_test_utils::TestECStoreEnv;
 use serde_json::json;
 use sha2::{Digest, Sha256};
@@ -97,6 +102,8 @@ async fn swift_metadata_writes_are_durable() {
 
     posts_survive_disk_truth_reload(&env).await;
     tag_writers_preserve_each_others_state(&env).await;
+    unrelated_account_posts_preserve_the_tempurl_key().await;
+    acl_only_posts_leave_container_metadata_alone(&env).await;
     versioning_writes_reject_missing_containers().await;
 }
 
@@ -109,11 +116,14 @@ async fn posts_survive_disk_truth_reload(env: &TestECStoreEnv) {
     let bucket = ContainerMapper::default().swift_to_s3_bucket(swift_container, project_id);
     env.make_bucket(&bucket, false).await;
 
-    let mut metadata = HashMap::new();
-    metadata.insert("color".to_string(), "blue".to_string());
-    update_container_metadata(&swift_account, swift_container, &credentials, metadata)
-        .await
-        .expect("container metadata POST should succeed");
+    update_container_metadata(
+        &swift_account,
+        swift_container,
+        &credentials,
+        MetadataUpdate::default().set("color", "blue"),
+    )
+    .await
+    .expect("container metadata POST should succeed");
 
     reload_bucket_metadata_from_disk(&bucket).await;
 
@@ -124,22 +134,47 @@ async fn posts_survive_disk_truth_reload(env: &TestECStoreEnv) {
         "container metadata POST must survive a disk-truth metadata reload"
     );
 
-    // A follow-up POST replaces the Swift metadata and that replacement must
-    // survive a reload too (the rewrite merges against disk state, so the
-    // previous value must actually be gone).
-    let mut metadata = HashMap::new();
-    metadata.insert("season".to_string(), "summer".to_string());
-    update_container_metadata(&swift_account, swift_container, &credentials, metadata)
-        .await
-        .expect("second container metadata POST should succeed");
+    // A follow-up POST names a different item. Swift POSTs are additive, so
+    // the new item lands and the first one stays — and both are still there
+    // after a reload, which is what proves the merge ran against disk state
+    // rather than looking right in the cache.
+    update_container_metadata(
+        &swift_account,
+        swift_container,
+        &credentials,
+        MetadataUpdate::default().set("season", "summer"),
+    )
+    .await
+    .expect("second container metadata POST should succeed");
 
     reload_bucket_metadata_from_disk(&bucket).await;
 
     let container_meta = persisted_container_metadata(&bucket).await;
     assert_eq!(container_meta.get("season").map(String::as_str), Some("summer"));
+    assert_eq!(
+        container_meta.get("color").map(String::as_str),
+        Some("blue"),
+        "a POST that does not name an item must not delete it"
+    );
+
+    // X-Remove-Container-Meta-Color is the deletion path, and it has to be
+    // durable in the other direction: the removed item must not come back
+    // from the persisted tags on reload.
+    update_container_metadata(&swift_account, swift_container, &credentials, MetadataUpdate::default().remove("color"))
+        .await
+        .expect("container metadata removal should succeed");
+
+    reload_bucket_metadata_from_disk(&bucket).await;
+
+    let container_meta = persisted_container_metadata(&bucket).await;
     assert!(
         !container_meta.contains_key("color"),
-        "replaced container metadata must not resurrect on reload"
+        "an explicitly removed item must not resurrect on reload"
+    );
+    assert_eq!(
+        container_meta.get("season").map(String::as_str),
+        Some("summer"),
+        "removing one item must not disturb the others"
     );
 
     // --- Container versioning POST (X-Versions-Location) ---
@@ -163,11 +198,13 @@ async fn posts_survive_disk_truth_reload(env: &TestECStoreEnv) {
     );
 
     // --- Account metadata POST (TempURL keys etc.) ---
-    let mut account_meta = HashMap::new();
-    account_meta.insert("temp-url-key".to_string(), "s3cr3t".to_string());
-    account::update_account_metadata(&swift_account, &account_meta, &Some(credentials.clone()))
-        .await
-        .expect("account metadata POST should succeed");
+    account::update_account_metadata(
+        &swift_account,
+        &MetadataUpdate::default().set("temp-url-key", "s3cr3t"),
+        &Some(credentials.clone()),
+    )
+    .await
+    .expect("account metadata POST should succeed");
 
     reload_bucket_metadata_from_disk(&account_metadata_bucket_name(&swift_account)).await;
 
@@ -196,9 +233,7 @@ async fn tag_writers_preserve_each_others_state(env: &TestECStoreEnv) {
     env.make_bucket(&ContainerMapper::default().swift_to_s3_bucket(archive, project_id), false)
         .await;
 
-    let mut metadata = HashMap::new();
-    metadata.insert("color".to_string(), "blue".to_string());
-    update_container_metadata(&swift_account, container, &credentials, metadata)
+    update_container_metadata(&swift_account, container, &credentials, MetadataUpdate::default().set("color", "blue"))
         .await
         .expect("container metadata POST should succeed");
     container::enable_versioning(&swift_account, container, archive, &credentials)
@@ -252,6 +287,111 @@ async fn tag_writers_preserve_each_others_state(env: &TestECStoreEnv) {
     assert!(!acl.read.is_empty(), "disable_versioning must not disturb the ACL");
 }
 
+/// The failure additive POSTs exist to prevent. Account metadata holds the
+/// TempURL signing key, so a POST that replaced the whole set — setting a
+/// quota, say — would delete that key and permanently invalidate every
+/// outstanding TempURL and FormPost signature for the account. Durable
+/// storage made that loss permanent instead of a cache blip, so the check
+/// runs against reloaded disk state.
+///
+/// Relies on the ambient store the caller's `TestECStoreEnv` published; the
+/// account metadata bucket is created by the write path itself.
+async fn unrelated_account_posts_preserve_the_tempurl_key() {
+    let project_id = "swiftmergeproj";
+    let swift_account = format!("AUTH_{project_id}");
+    let credentials = Some(keystone_credentials(project_id));
+    let account_bucket = account_metadata_bucket_name(&swift_account);
+
+    account::update_account_metadata(&swift_account, &MetadataUpdate::default().set("temp-url-key", "s3cr3t"), &credentials)
+        .await
+        .expect("TempURL key POST should succeed");
+
+    // A later POST that has nothing to do with the TempURL key.
+    account::update_account_metadata(&swift_account, &MetadataUpdate::default().set("quota-bytes", "100"), &credentials)
+        .await
+        .expect("quota POST should succeed");
+
+    reload_bucket_metadata_from_disk(&account_bucket).await;
+
+    let loaded = account::get_account_metadata(&swift_account, &None)
+        .await
+        .expect("account metadata should load");
+    assert_eq!(
+        loaded.get("temp-url-key").map(String::as_str),
+        Some("s3cr3t"),
+        "an unrelated account POST must not delete the TempURL signing key"
+    );
+    assert_eq!(loaded.get("quota-bytes").map(String::as_str), Some("100"));
+
+    // And the lookup that actually breaks when the key is lost still resolves.
+    assert_eq!(
+        account::get_tempurl_key(&swift_account, &None)
+            .await
+            .expect("TempURL key should load")
+            .as_deref(),
+        Some("s3cr3t"),
+        "TempURL signature validation must still find the key"
+    );
+
+    // X-Remove-Account-Meta-Quota-Bytes drops that item and nothing else.
+    account::update_account_metadata(&swift_account, &MetadataUpdate::default().remove("quota-bytes"), &credentials)
+        .await
+        .expect("account metadata removal should succeed");
+
+    reload_bucket_metadata_from_disk(&account_bucket).await;
+
+    let loaded = account::get_account_metadata(&swift_account, &None)
+        .await
+        .expect("account metadata should load");
+    assert!(
+        !loaded.contains_key("quota-bytes"),
+        "an explicitly removed item must not resurrect on reload"
+    );
+    assert_eq!(
+        loaded.get("temp-url-key").map(String::as_str),
+        Some("s3cr3t"),
+        "removing one item must not disturb the TempURL key"
+    );
+}
+
+/// An ACL-only or versioning-only container POST carries no `X-Container-Meta-*`
+/// header, but the handler still runs the metadata step on every POST. That
+/// step must leave stored metadata alone rather than treating "named nothing"
+/// as "wants everything gone".
+async fn acl_only_posts_leave_container_metadata_alone(env: &TestECStoreEnv) {
+    let project_id = "swiftaclonlyproj";
+    let swift_account = format!("AUTH_{project_id}");
+    let credentials = keystone_credentials(project_id);
+    let container = "gallery";
+    let bucket = ContainerMapper::default().swift_to_s3_bucket(container, project_id);
+    env.make_bucket(&bucket, false).await;
+
+    update_container_metadata(&swift_account, container, &credentials, MetadataUpdate::default().set("color", "blue"))
+        .await
+        .expect("container metadata POST should succeed");
+
+    // What the handler does for `POST … -H 'X-Container-Read: .r:*'`: set the
+    // ACL, then run the metadata step with an update that names no item.
+    container::set_container_acl(&swift_account, container, Some(".r:*"), None, &credentials)
+        .await
+        .expect("ACL-only POST should succeed");
+    update_container_metadata(&swift_account, container, &credentials, MetadataUpdate::default())
+        .await
+        .expect("the metadata step of an ACL-only POST should succeed");
+
+    reload_bucket_metadata_from_disk(&bucket).await;
+
+    assert_eq!(
+        persisted_container_metadata(&bucket).await.get("color").map(String::as_str),
+        Some("blue"),
+        "an ACL-only POST must not wipe X-Container-Meta-*"
+    );
+    let acl = container::get_container_acl(&swift_account, container, &credentials)
+        .await
+        .expect("container ACL should load");
+    assert!(!acl.read.is_empty(), "the ACL that POST set must be stored");
+}
+
 /// A container that does not exist must not get metadata persisted for it:
 /// the metadata loader turns "nothing on disk" into a fresh default, so an
 /// unguarded rewrite would create an orphan metadata file and cache a
@@ -270,6 +410,16 @@ async fn versioning_writes_reject_missing_containers() {
         "expected NotFound for a missing container, got {err:?}"
     );
 
+    // Naming no item skips the persisted write, but must not skip the
+    // existence check that turns a POST to a missing container into a 404.
+    let err = update_container_metadata(&swift_account, missing, &credentials, MetadataUpdate::default())
+        .await
+        .expect_err("a metadata POST to a missing container must fail");
+    assert!(
+        matches!(err, SwiftError::NotFound(_)),
+        "expected NotFound for a missing container, got {err:?}"
+    );
+
     let bucket = ContainerMapper::default().swift_to_s3_bucket(missing, project_id);
     assert!(
         get_bucket_metadata(&bucket).await.is_err(),
@@ -291,8 +441,7 @@ async fn account_metadata_write_rejects_foreign_and_anonymous_callers() {
     let victim_account = "AUTH_victimproject";
     let attacker_credentials = keystone_credentials("attackerproject");
 
-    let mut poisoned = HashMap::new();
-    poisoned.insert("temp-url-key".to_string(), "attacker-key".to_string());
+    let poisoned = MetadataUpdate::default().set("temp-url-key", "attacker-key");
 
     let err = account::update_account_metadata(victim_account, &poisoned, &Some(attacker_credentials))
         .await

crates/protos/src/generated/proto_gen/node_service.rs

@@ -484,6 +484,59 @@ pub struct DeletePathsResponse {
     pub error: ::core::option::Option<Error>,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct SnapshotLeaseRequest {
+    #[prost(string, tag = "1")]
+    pub disk: ::prost::alloc::string::String,
+    #[prost(string, tag = "2")]
+    pub volume: ::prost::alloc::string::String,
+    #[prost(string, tag = "3")]
+    pub path: ::prost::alloc::string::String,
+    #[prost(uint64, tag = "4")]
+    pub ttl_ms: u64,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct SnapshotLeaseRenewRequest {
+    #[prost(string, tag = "1")]
+    pub disk: ::prost::alloc::string::String,
+    #[prost(string, tag = "2")]
+    pub volume: ::prost::alloc::string::String,
+    #[prost(string, tag = "3")]
+    pub path: ::prost::alloc::string::String,
+    #[prost(bytes = "bytes", tag = "4")]
+    pub token: ::prost::bytes::Bytes,
+    #[prost(uint64, tag = "5")]
+    pub ttl_ms: u64,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct SnapshotLeaseReleaseRequest {
+    #[prost(string, tag = "1")]
+    pub disk: ::prost::alloc::string::String,
+    #[prost(string, tag = "2")]
+    pub volume: ::prost::alloc::string::String,
+    #[prost(string, tag = "3")]
+    pub path: ::prost::alloc::string::String,
+    #[prost(bytes = "bytes", tag = "4")]
+    pub token: ::prost::bytes::Bytes,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct SnapshotLeaseResponse {
+    #[prost(bool, tag = "1")]
+    pub success: bool,
+    #[prost(bytes = "bytes", tag = "2")]
+    pub token: ::prost::bytes::Bytes,
+    #[prost(uint32, tag = "3")]
+    pub protocol_version: u32,
+    #[prost(message, optional, tag = "4")]
+    pub error: ::core::option::Option<Error>,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct SnapshotLeaseMutationResponse {
+    #[prost(bool, tag = "1")]
+    pub success: bool,
+    #[prost(message, optional, tag = "2")]
+    pub error: ::core::option::Option<Error>,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ReadMetadataRequest {
     #[prost(string, tag = "1")]
     pub disk: ::prost::alloc::string::String,
@@ -1595,6 +1648,51 @@ pub mod node_service_client {
                 .insert(GrpcMethod::new("node_service.NodeService", "Delete"));
             self.inner.unary(req, path, codec).await
         }
+        pub async fn acquire_snapshot_lease(
+            &mut self,
+            request: impl tonic::IntoRequest<super::SnapshotLeaseRequest>,
+        ) -> std::result::Result<tonic::Response<super::SnapshotLeaseResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
+            let codec = tonic_prost::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/AcquireSnapshotLease");
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(GrpcMethod::new("node_service.NodeService", "AcquireSnapshotLease"));
+            self.inner.unary(req, path, codec).await
+        }
+        pub async fn renew_snapshot_lease(
+            &mut self,
+            request: impl tonic::IntoRequest<super::SnapshotLeaseRenewRequest>,
+        ) -> std::result::Result<tonic::Response<super::SnapshotLeaseResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
+            let codec = tonic_prost::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RenewSnapshotLease");
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(GrpcMethod::new("node_service.NodeService", "RenewSnapshotLease"));
+            self.inner.unary(req, path, codec).await
+        }
+        pub async fn release_snapshot_lease(
+            &mut self,
+            request: impl tonic::IntoRequest<super::SnapshotLeaseReleaseRequest>,
+        ) -> std::result::Result<tonic::Response<super::SnapshotLeaseMutationResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
+            let codec = tonic_prost::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReleaseSnapshotLease");
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(GrpcMethod::new("node_service.NodeService", "ReleaseSnapshotLease"));
+            self.inner.unary(req, path, codec).await
+        }
         pub async fn verify_file(
             &mut self,
             request: impl tonic::IntoRequest<super::VerifyFileRequest>,
@@ -2784,6 +2882,18 @@ pub mod node_service_server {
             &self,
             request: tonic::Request<super::DeleteRequest>,
         ) -> std::result::Result<tonic::Response<super::DeleteResponse>, tonic::Status>;
+        async fn acquire_snapshot_lease(
+            &self,
+            request: tonic::Request<super::SnapshotLeaseRequest>,
+        ) -> std::result::Result<tonic::Response<super::SnapshotLeaseResponse>, tonic::Status>;
+        async fn renew_snapshot_lease(
+            &self,
+            request: tonic::Request<super::SnapshotLeaseRenewRequest>,
+        ) -> std::result::Result<tonic::Response<super::SnapshotLeaseResponse>, tonic::Status>;
+        async fn release_snapshot_lease(
+            &self,
+            request: tonic::Request<super::SnapshotLeaseReleaseRequest>,
+        ) -> std::result::Result<tonic::Response<super::SnapshotLeaseMutationResponse>, tonic::Status>;
         async fn verify_file(
             &self,
             request: tonic::Request<super::VerifyFileRequest>,
@@ -3426,6 +3536,90 @@ pub mod node_service_server {
                     };
                     Box::pin(fut)
                 }
+                "/node_service.NodeService/AcquireSnapshotLease" => {
+                    #[allow(non_camel_case_types)]
+                    struct AcquireSnapshotLeaseSvc<T: NodeService>(pub Arc<T>);
+                    impl<T: NodeService> tonic::server::UnaryService<super::SnapshotLeaseRequest> for AcquireSnapshotLeaseSvc<T> {
+                        type Response = super::SnapshotLeaseResponse;
+                        type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
+                        fn call(&mut self, request: tonic::Request<super::SnapshotLeaseRequest>) -> Self::Future {
+                            let inner = Arc::clone(&self.0);
+                            let fut = async move { <T as NodeService>::acquire_snapshot_lease(&inner, request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let max_decoding_message_size = self.max_decoding_message_size;
+                    let max_encoding_message_size = self.max_encoding_message_size;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let method = AcquireSnapshotLeaseSvc(inner);
+                        let codec = tonic_prost::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(accept_compression_encodings, send_compression_encodings)
+                            .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
+                "/node_service.NodeService/RenewSnapshotLease" => {
+                    #[allow(non_camel_case_types)]
+                    struct RenewSnapshotLeaseSvc<T: NodeService>(pub Arc<T>);
+                    impl<T: NodeService> tonic::server::UnaryService<super::SnapshotLeaseRenewRequest> for RenewSnapshotLeaseSvc<T> {
+                        type Response = super::SnapshotLeaseResponse;
+                        type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
+                        fn call(&mut self, request: tonic::Request<super::SnapshotLeaseRenewRequest>) -> Self::Future {
+                            let inner = Arc::clone(&self.0);
+                            let fut = async move { <T as NodeService>::renew_snapshot_lease(&inner, request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let max_decoding_message_size = self.max_decoding_message_size;
+                    let max_encoding_message_size = self.max_encoding_message_size;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let method = RenewSnapshotLeaseSvc(inner);
+                        let codec = tonic_prost::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(accept_compression_encodings, send_compression_encodings)
+                            .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
+                "/node_service.NodeService/ReleaseSnapshotLease" => {
+                    #[allow(non_camel_case_types)]
+                    struct ReleaseSnapshotLeaseSvc<T: NodeService>(pub Arc<T>);
+                    impl<T: NodeService> tonic::server::UnaryService<super::SnapshotLeaseReleaseRequest> for ReleaseSnapshotLeaseSvc<T> {
+                        type Response = super::SnapshotLeaseMutationResponse;
+                        type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
+                        fn call(&mut self, request: tonic::Request<super::SnapshotLeaseReleaseRequest>) -> Self::Future {
+                            let inner = Arc::clone(&self.0);
+                            let fut = async move { <T as NodeService>::release_snapshot_lease(&inner, request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let max_decoding_message_size = self.max_decoding_message_size;
+                    let max_encoding_message_size = self.max_encoding_message_size;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let method = ReleaseSnapshotLeaseSvc(inner);
+                        let codec = tonic_prost::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(accept_compression_encodings, send_compression_encodings)
+                            .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
                 "/node_service.NodeService/VerifyFile" => {
                     #[allow(non_camel_case_types)]
                     struct VerifyFileSvc<T: NodeService>(pub Arc<T>);

crates/protos/src/lib.rs

@@ -447,10 +447,23 @@ impl CanonicalBodyBuilder {
         self.push_bytes(field.as_bytes())
     }
 
+    fn push_optional_str(&mut self, field: Option<&str>) -> Result<(), std::num::TryFromIntError> {
+        self.body.push(u8::from(field.is_some()));
+        self.push_str(field.unwrap_or_default())
+    }
+
     fn push_bool(&mut self, field: bool) {
         self.body.push(u8::from(field));
     }
 
+    fn push_u32(&mut self, field: u32) {
+        self.body.extend_from_slice(&field.to_be_bytes());
+    }
+
+    fn push_u64(&mut self, field: u64) {
+        self.body.extend_from_slice(&field.to_be_bytes());
+    }
+
     fn push_count(&mut self, count: usize) -> Result<(), std::num::TryFromIntError> {
         self.body.extend_from_slice(&u64::try_from(count)?.to_be_bytes());
         Ok(())
@@ -461,6 +474,211 @@ impl CanonicalBodyBuilder {
     }
 }
 
+pub const PEER_RESTSIGNAL: &str = "signal";
+pub const PEER_RESTSUB_SYS: &str = "sub-sys";
+pub const PEER_RESTDRY_RUN: &str = "dry-run";
+
+/// A stable semantic body for a side-effecting unary RPC.
+///
+/// Implementations deliberately enumerate handler-consumed fields instead of re-encoding the
+/// protobuf message, whose unknown fields and map order are not a mixed-version contract.
+pub trait CanonicalMutationBody {
+    fn canonical_body(&self) -> Result<Vec<u8>, std::num::TryFromIntError>;
+}
+
+macro_rules! impl_canonical_mutation_body {
+    ($request:ty, $domain:expr, |$value:ident, $body:ident| $fields:block) => {
+        impl CanonicalMutationBody for $request {
+            fn canonical_body(&self) -> Result<Vec<u8>, std::num::TryFromIntError> {
+                let $value = self;
+                let mut $body = CanonicalBodyBuilder::new($domain);
+                $fields
+                Ok($body.finish())
+            }
+        }
+    };
+    ($request:ty, $domain:expr) => {
+        impl CanonicalMutationBody for $request {
+            fn canonical_body(&self) -> Result<Vec<u8>, std::num::TryFromIntError> {
+                Ok(CanonicalBodyBuilder::new($domain).finish())
+            }
+        }
+    };
+}
+
+impl_canonical_mutation_body!(
+    proto_gen::node_service::SignalServiceRequest,
+    b"rustfs-signal-service-request-v1\0",
+    |request, body| {
+        let vars = request.vars.as_ref().map(|vars| &vars.value);
+        body.push_optional_str(vars.and_then(|vars| vars.get(PEER_RESTSIGNAL).map(String::as_str)))?;
+        body.push_optional_str(vars.and_then(|vars| vars.get(PEER_RESTSUB_SYS).map(String::as_str)))?;
+        body.push_optional_str(vars.and_then(|vars| vars.get(PEER_RESTDRY_RUN).map(String::as_str)))?;
+    }
+);
+impl_canonical_mutation_body!(
+    proto_gen::node_service::HealBucketRequest,
+    b"rustfs-heal-bucket-request-v1\0",
+    |request, body| {
+        body.push_str(&request.bucket)?;
+        body.push_str(&request.options)?;
+    }
+);
+impl_canonical_mutation_body!(
+    proto_gen::node_service::MakeBucketRequest,
+    b"rustfs-make-bucket-request-v1\0",
+    |request, body| {
+        body.push_str(&request.name)?;
+        body.push_str(&request.options)?;
+    }
+);
+impl_canonical_mutation_body!(
+    proto_gen::node_service::DeleteBucketRequest,
+    b"rustfs-delete-bucket-request-v1\0",
+    |request, body| {
+        body.push_str(&request.bucket)?;
+        body.push_str(&request.options)?;
+    }
+);
+impl_canonical_mutation_body!(
+    proto_gen::node_service::GenerallyLockRequest,
+    b"rustfs-lock-request-v1\0",
+    |request, body| {
+        body.push_str(&request.args)?;
+    }
+);
+impl_canonical_mutation_body!(
+    proto_gen::node_service::BatchGenerallyLockRequest,
+    b"rustfs-lock-batch-request-v1\0",
+    |request, body| {
+        body.push_count(request.args.len())?;
+        for arg in &request.args {
+            body.push_str(arg)?;
+        }
+    }
+);
+impl_canonical_mutation_body!(
+    proto_gen::node_service::LoadBucketMetadataRequest,
+    b"rustfs-load-bucket-metadata-request-v1\0",
+    |request, body| {
+        body.push_str(&request.bucket)?;
+        body.push_bool(request.scanner_maintenance_change);
+    }
+);
+impl_canonical_mutation_body!(
+    proto_gen::node_service::DeleteBucketMetadataRequest,
+    b"rustfs-delete-bucket-metadata-request-v1\0",
+    |request, body| {
+        body.push_str(&request.bucket)?;
+    }
+);
+impl_canonical_mutation_body!(
+    proto_gen::node_service::DeletePolicyRequest,
+    b"rustfs-delete-policy-request-v1\0",
+    |request, body| {
+        body.push_str(&request.policy_name)?;
+    }
+);
+impl_canonical_mutation_body!(
+    proto_gen::node_service::LoadPolicyRequest,
+    b"rustfs-load-policy-request-v1\0",
+    |request, body| {
+        body.push_str(&request.policy_name)?;
+    }
+);
+impl_canonical_mutation_body!(
+    proto_gen::node_service::LoadPolicyMappingRequest,
+    b"rustfs-load-policy-mapping-request-v1\0",
+    |request, body| {
+        body.push_str(&request.user_or_group)?;
+        body.push_u64(request.user_type);
+        body.push_bool(request.is_group);
+    }
+);
+impl_canonical_mutation_body!(
+    proto_gen::node_service::DeleteUserRequest,
+    b"rustfs-delete-user-request-v1\0",
+    |request, body| {
+        body.push_str(&request.access_key)?;
+    }
+);
+impl_canonical_mutation_body!(
+    proto_gen::node_service::DeleteServiceAccountRequest,
+    b"rustfs-delete-service-account-request-v1\0",
+    |request, body| {
+        body.push_str(&request.access_key)?;
+    }
+);
+impl_canonical_mutation_body!(
+    proto_gen::node_service::LoadUserRequest,
+    b"rustfs-load-user-request-v1\0",
+    |request, body| {
+        body.push_str(&request.access_key)?;
+        body.push_bool(request.temp);
+    }
+);
+impl_canonical_mutation_body!(
+    proto_gen::node_service::LoadServiceAccountRequest,
+    b"rustfs-load-service-account-request-v1\0",
+    |request, body| {
+        body.push_str(&request.access_key)?;
+    }
+);
+impl_canonical_mutation_body!(
+    proto_gen::node_service::LoadGroupRequest,
+    b"rustfs-load-group-request-v1\0",
+    |request, body| {
+        body.push_str(&request.group)?;
+    }
+);
+impl_canonical_mutation_body!(
+    proto_gen::node_service::ReloadSiteReplicationConfigRequest,
+    b"rustfs-reload-site-replication-config-request-v1\0"
+);
+impl_canonical_mutation_body!(proto_gen::node_service::ReloadPoolMetaRequest, b"rustfs-reload-pool-meta-request-v1\0");
+impl_canonical_mutation_body!(
+    proto_gen::node_service::StopRebalanceRequest,
+    b"rustfs-stop-rebalance-request-v1\0",
+    |request, body| {
+        body.push_str(&request.expected_rebalance_id)?;
+    }
+);
+impl_canonical_mutation_body!(
+    proto_gen::node_service::LoadRebalanceMetaRequest,
+    b"rustfs-load-rebalance-meta-request-v1\0",
+    |request, body| {
+        body.push_bool(request.start_rebalance);
+    }
+);
+impl_canonical_mutation_body!(
+    proto_gen::node_service::StartDecommissionRequest,
+    b"rustfs-start-decommission-request-v1\0",
+    |request, body| {
+        body.push_count(request.pool_indices.len())?;
+        for pool_index in &request.pool_indices {
+            body.push_u32(*pool_index);
+        }
+    }
+);
+impl_canonical_mutation_body!(
+    proto_gen::node_service::CancelDecommissionRequest,
+    b"rustfs-cancel-decommission-request-v1\0",
+    |request, body| {
+        body.push_u32(request.pool_index);
+    }
+);
+impl_canonical_mutation_body!(
+    proto_gen::node_service::ClearDecommissionRequest,
+    b"rustfs-clear-decommission-request-v1\0",
+    |request, body| {
+        body.push_u32(request.pool_index);
+    }
+);
+impl_canonical_mutation_body!(
+    proto_gen::node_service::LoadTransitionTierConfigRequest,
+    b"rustfs-load-transition-tier-config-request-v1\0"
+);
+
 // Canonical request bodies for the mutating NodeService disk RPCs (backlog#1327 body-digest
 // binding). Each covers every semantic wire field — including both the msgpack `_bin` payload and
 // its JSON compatibility copy — so tampering with either encoding, or stripping `_bin` to force
@@ -575,6 +793,40 @@ pub fn canonical_delete_paths_request_body(
     Ok(body.finish())
 }
 
+pub fn canonical_snapshot_lease_request_body(
+    request: &proto_gen::node_service::SnapshotLeaseRequest,
+) -> Result<Vec<u8>, std::num::TryFromIntError> {
+    let mut body = CanonicalBodyBuilder::new(b"rustfs-snapshot-lease-request-v1\0");
+    body.push_str(&request.disk)?;
+    body.push_str(&request.volume)?;
+    body.push_str(&request.path)?;
+    body.push_u64(request.ttl_ms);
+    Ok(body.finish())
+}
+
+pub fn canonical_snapshot_lease_renew_request_body(
+    request: &proto_gen::node_service::SnapshotLeaseRenewRequest,
+) -> Result<Vec<u8>, std::num::TryFromIntError> {
+    let mut body = CanonicalBodyBuilder::new(b"rustfs-snapshot-lease-renew-request-v1\0");
+    body.push_str(&request.disk)?;
+    body.push_str(&request.volume)?;
+    body.push_str(&request.path)?;
+    body.push_bytes(&request.token)?;
+    body.push_u64(request.ttl_ms);
+    Ok(body.finish())
+}
+
+pub fn canonical_snapshot_lease_release_request_body(
+    request: &proto_gen::node_service::SnapshotLeaseReleaseRequest,
+) -> Result<Vec<u8>, std::num::TryFromIntError> {
+    let mut body = CanonicalBodyBuilder::new(b"rustfs-snapshot-lease-release-request-v1\0");
+    body.push_str(&request.disk)?;
+    body.push_str(&request.volume)?;
+    body.push_str(&request.path)?;
+    body.push_bytes(&request.token)?;
+    Ok(body.finish())
+}
+
 pub fn canonical_rename_file_request_body(
     request: &proto_gen::node_service::RenameFileRequest,
 ) -> Result<Vec<u8>, std::num::TryFromIntError> {
@@ -662,7 +914,8 @@ mod disk_mutation_canonical_tests {
     use super::proto_gen::node_service::{
         DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, MakeVolumeRequest,
         MakeVolumesRequest, PreparePartTransactionRequest, RenameDataRequest, RenameFileRequest, RenamePartRequest,
-        SettlePartTransactionRequest, UpdateMetadataRequest, WriteAllRequest, WriteMetadataRequest,
+        SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
+        UpdateMetadataRequest, WriteAllRequest, WriteMetadataRequest,
     };
     use super::*;
 
@@ -1020,6 +1273,58 @@ mod disk_mutation_canonical_tests {
         );
     }
 
+    #[test]
+    fn snapshot_lease_canonical_bodies_bind_every_field() {
+        let acquire = SnapshotLeaseRequest {
+            disk: "d".into(),
+            volume: "v".into(),
+            path: "p".into(),
+            ttl_ms: 60_000,
+        };
+        let mut acquire_bodies = vec![canonical_snapshot_lease_request_body(&acquire).unwrap()];
+        for mutate in [
+            |r: &mut SnapshotLeaseRequest| r.disk = "d2".into(),
+            |r: &mut SnapshotLeaseRequest| r.volume = "v2".into(),
+            |r: &mut SnapshotLeaseRequest| r.path = "p2".into(),
+            |r: &mut SnapshotLeaseRequest| r.ttl_ms = 60_001,
+        ] {
+            let mut request = acquire.clone();
+            mutate(&mut request);
+            acquire_bodies.push(canonical_snapshot_lease_request_body(&request).unwrap());
+        }
+        assert_all_distinct(&acquire_bodies);
+
+        let renew = SnapshotLeaseRenewRequest {
+            disk: "d".into(),
+            volume: "v".into(),
+            path: "p".into(),
+            token: vec![1; 16].into(),
+            ttl_ms: 60_000,
+        };
+        let mut changed_token = renew.clone();
+        changed_token.token = vec![2; 16].into();
+        let mut changed_ttl = renew.clone();
+        changed_ttl.ttl_ms += 1;
+        assert_all_distinct(&[
+            canonical_snapshot_lease_renew_request_body(&renew).unwrap(),
+            canonical_snapshot_lease_renew_request_body(&changed_token).unwrap(),
+            canonical_snapshot_lease_renew_request_body(&changed_ttl).unwrap(),
+        ]);
+
+        let release = SnapshotLeaseReleaseRequest {
+            disk: "d".into(),
+            volume: "v".into(),
+            path: "p".into(),
+            token: vec![1; 16].into(),
+        };
+        let mut changed_release = release.clone();
+        changed_release.token = vec![2; 16].into();
+        assert_ne!(
+            canonical_snapshot_lease_release_request_body(&release).unwrap(),
+            canonical_snapshot_lease_release_request_body(&changed_release).unwrap()
+        );
+    }
+
     #[test]
     fn disk_mutation_canonical_domains_are_distinct_per_message() {
         // The same field values must never authenticate one RPC's request as another's.
@@ -1045,6 +1350,154 @@ mod disk_mutation_canonical_tests {
     }
 }
 
+#[cfg(test)]
+mod non_disk_mutation_canonical_tests {
+    use super::*;
+    use proto_gen::node_service::*;
+    use std::collections::HashMap;
+
+    macro_rules! assert_fields_bound {
+        ($request:ty, {$($field:ident: $value:expr),+ $(,)?}) => {{
+            let baseline = <$request>::default();
+            let expected = baseline.canonical_body().expect("baseline canonical body should encode");
+            $(
+                let mut variant = baseline.clone();
+                variant.$field = $value;
+                assert_ne!(
+                    expected,
+                    variant.canonical_body().expect("variant canonical body should encode"),
+                    concat!(stringify!($request), " omitted field ", stringify!($field)),
+                );
+            )+
+        }};
+    }
+
+    fn signal_request(signal: Option<&str>, sub_system: Option<&str>, dry_run: Option<&str>) -> SignalServiceRequest {
+        let mut value = HashMap::new();
+        if let Some(signal) = signal {
+            value.insert(PEER_RESTSIGNAL.to_string(), signal.to_string());
+        }
+        if let Some(sub_system) = sub_system {
+            value.insert(PEER_RESTSUB_SYS.to_string(), sub_system.to_string());
+        }
+        if let Some(dry_run) = dry_run {
+            value.insert(PEER_RESTDRY_RUN.to_string(), dry_run.to_string());
+        }
+        SignalServiceRequest {
+            vars: Some(Mss { value }),
+        }
+    }
+
+    #[test]
+    fn signal_service_canonical_body_is_versioned_and_binds_every_semantic_field() {
+        let request = signal_request(Some("2"), Some("scanner"), Some("false"));
+        let baseline = request.canonical_body().expect("small signal request should encode");
+        assert!(baseline.starts_with(b"rustfs-signal-service-request-v1\0"));
+
+        for variant in [
+            signal_request(Some("1"), Some("scanner"), Some("false")),
+            signal_request(Some("2"), Some("heal"), Some("false")),
+            signal_request(Some("2"), Some("scanner"), Some("true")),
+            signal_request(None, Some("scanner"), Some("false")),
+            signal_request(Some("2"), None, Some("false")),
+            signal_request(Some("2"), Some("scanner"), None),
+        ] {
+            assert_ne!(baseline, variant.canonical_body().expect("small signal request should encode"));
+        }
+
+        let mut ignored_field = request;
+        ignored_field
+            .vars
+            .as_mut()
+            .expect("signal vars should exist")
+            .value
+            .insert("future-field".to_string(), "ignored".to_string());
+        assert_eq!(
+            baseline,
+            ignored_field
+                .canonical_body()
+                .expect("unknown signal field should not affect the semantic body"),
+        );
+    }
+
+    #[test]
+    fn signal_service_canonical_body_distinguishes_missing_and_empty_fields() {
+        assert_ne!(
+            signal_request(None, Some("scanner"), Some("false"))
+                .canonical_body()
+                .expect("missing signal should encode"),
+            signal_request(Some(""), Some("scanner"), Some("false"))
+                .canonical_body()
+                .expect("empty signal should encode"),
+        );
+    }
+
+    #[test]
+    fn bucket_and_lock_canonical_bodies_bind_every_semantic_field() {
+        assert_fields_bound!(HealBucketRequest, { bucket: "bucket".into(), options: "opts".into() });
+        assert_fields_bound!(MakeBucketRequest, { name: "bucket".into(), options: "opts".into() });
+        assert_fields_bound!(DeleteBucketRequest, { bucket: "bucket".into(), options: "opts".into() });
+        assert_fields_bound!(GenerallyLockRequest, { args: "lock".into() });
+        assert_fields_bound!(BatchGenerallyLockRequest, { args: vec!["first".into(), "second".into()] });
+
+        let first = BatchGenerallyLockRequest {
+            args: vec!["first".into(), "second".into()],
+        };
+        let reversed = BatchGenerallyLockRequest {
+            args: vec!["second".into(), "first".into()],
+        };
+        assert_ne!(first.canonical_body().unwrap(), reversed.canonical_body().unwrap());
+    }
+
+    #[test]
+    fn metadata_and_iam_canonical_bodies_bind_every_semantic_field() {
+        assert_fields_bound!(LoadBucketMetadataRequest, {
+            bucket: "bucket".into(),
+            scanner_maintenance_change: true,
+        });
+        assert_fields_bound!(DeleteBucketMetadataRequest, { bucket: "bucket".into() });
+        assert_fields_bound!(DeletePolicyRequest, { policy_name: "policy".into() });
+        assert_fields_bound!(LoadPolicyRequest, { policy_name: "policy".into() });
+        assert_fields_bound!(LoadPolicyMappingRequest, {
+            user_or_group: "user".into(),
+            user_type: 1,
+            is_group: true,
+        });
+        assert_fields_bound!(DeleteUserRequest, { access_key: "user".into() });
+        assert_fields_bound!(DeleteServiceAccountRequest, { access_key: "service".into() });
+        assert_fields_bound!(LoadUserRequest, { access_key: "user".into(), temp: true });
+        assert_fields_bound!(LoadServiceAccountRequest, { access_key: "service".into() });
+        assert_fields_bound!(LoadGroupRequest, { group: "group".into() });
+    }
+
+    #[test]
+    fn control_plane_canonical_bodies_bind_every_semantic_field() {
+        assert_fields_bound!(StopRebalanceRequest, { expected_rebalance_id: "rebalance".into() });
+        assert_fields_bound!(LoadRebalanceMetaRequest, { start_rebalance: true });
+        assert_fields_bound!(StartDecommissionRequest, { pool_indices: vec![1, 2] });
+        assert_fields_bound!(CancelDecommissionRequest, { pool_index: 1 });
+        assert_fields_bound!(ClearDecommissionRequest, { pool_index: 1 });
+
+        let first = StartDecommissionRequest {
+            pool_indices: vec![1, 2],
+        };
+        let reversed = StartDecommissionRequest {
+            pool_indices: vec![2, 1],
+        };
+        assert_ne!(first.canonical_body().unwrap(), reversed.canonical_body().unwrap());
+
+        let empty_domains = [
+            ReloadSiteReplicationConfigRequest::default().canonical_body().unwrap(),
+            ReloadPoolMetaRequest::default().canonical_body().unwrap(),
+            LoadTransitionTierConfigRequest::default().canonical_body().unwrap(),
+        ];
+        assert!(empty_domains.iter().all(|body| !body.is_empty()));
+        assert_ne!(empty_domains[0], empty_domains[1]);
+        assert_ne!(empty_domains[0], empty_domains[2]);
+        assert_ne!(empty_domains[1], empty_domains[2]);
+    }
+}
+
 #[cfg(test)]
 mod scanner_activity_tests {
     use super::{

crates/protos/src/node.proto

@@ -342,6 +342,40 @@ message DeletePathsResponse {
   optional Error error = 2;
 }
 
+message SnapshotLeaseRequest {
+  string disk = 1;
+  string volume = 2;
+  string path = 3;
+  uint64 ttl_ms = 4;
+}
+
+message SnapshotLeaseRenewRequest {
+  string disk = 1;
+  string volume = 2;
+  string path = 3;
+  bytes token = 4;
+  uint64 ttl_ms = 5;
+}
+
+message SnapshotLeaseReleaseRequest {
+  string disk = 1;
+  string volume = 2;
+  string path = 3;
+  bytes token = 4;
+}
+
+message SnapshotLeaseResponse {
+  bool success = 1;
+  bytes token = 2;
+  uint32 protocol_version = 3;
+  optional Error error = 4;
+}
+
+message SnapshotLeaseMutationResponse {
+  bool success = 1;
+  optional Error error = 2;
+}
+
 message ReadMetadataRequest {
   string disk = 1;
   string volume = 2;
@@ -954,104 +988,107 @@ message GetLiveEventsResponse {
 
 service NodeService {
 /* -------------------------------meta service-------------------------- */
-  rpc Ping(PingRequest) returns (PingResponse) {};
-  rpc HealBucket(HealBucketRequest) returns (HealBucketResponse) {};
-  rpc ListBucket(ListBucketRequest) returns (ListBucketResponse) {};
-  rpc MakeBucket(MakeBucketRequest) returns (MakeBucketResponse) {};
-  rpc GetBucketInfo(GetBucketInfoRequest) returns (GetBucketInfoResponse) {};
-  rpc DeleteBucket(DeleteBucketRequest) returns (DeleteBucketResponse) {};
+  rpc Ping(PingRequest) returns (PingResponse) {}; // auth-policy: read-only
+  rpc HealBucket(HealBucketRequest) returns (HealBucketResponse) {}; // auth-policy: body-bound
+  rpc ListBucket(ListBucketRequest) returns (ListBucketResponse) {}; // auth-policy: read-only
+  rpc MakeBucket(MakeBucketRequest) returns (MakeBucketResponse) {}; // auth-policy: body-bound
+  rpc GetBucketInfo(GetBucketInfoRequest) returns (GetBucketInfoResponse) {}; // auth-policy: read-only
+  rpc DeleteBucket(DeleteBucketRequest) returns (DeleteBucketResponse) {}; // auth-policy: body-bound
 
 /* -------------------------------disk service-------------------------- */
 
-  rpc ReadAll(ReadAllRequest) returns (ReadAllResponse) {};
-  rpc WriteAll(WriteAllRequest) returns (WriteAllResponse) {};
-  rpc Delete(DeleteRequest) returns (DeleteResponse) {};
-  rpc VerifyFile(VerifyFileRequest) returns (VerifyFileResponse) {};
-  rpc ReadParts(ReadPartsRequest) returns (ReadPartsResponse) {};
-  rpc CheckParts(CheckPartsRequest) returns (CheckPartsResponse) {};
-  rpc PreparePartTransaction(PreparePartTransactionRequest) returns (PreparePartTransactionResponse) {};
-  rpc RenamePart(RenamePartRequest) returns (RenamePartResponse) {};
-  rpc SettlePartTransaction(SettlePartTransactionRequest) returns (SettlePartTransactionResponse) {};
-  rpc RenameFile(RenameFileRequest) returns (RenameFileResponse) {};
-  rpc Write(WriteRequest) returns (WriteResponse) {};
-  rpc WriteStream(stream WriteRequest) returns (stream WriteResponse) {};
+  rpc ReadAll(ReadAllRequest) returns (ReadAllResponse) {}; // auth-policy: read-only
+  rpc WriteAll(WriteAllRequest) returns (WriteAllResponse) {}; // auth-policy: body-bound
+  rpc Delete(DeleteRequest) returns (DeleteResponse) {}; // auth-policy: body-bound
+  rpc AcquireSnapshotLease(SnapshotLeaseRequest) returns (SnapshotLeaseResponse) {}; // auth-policy: body-bound
+  rpc RenewSnapshotLease(SnapshotLeaseRenewRequest) returns (SnapshotLeaseResponse) {}; // auth-policy: body-bound
+  rpc ReleaseSnapshotLease(SnapshotLeaseReleaseRequest) returns (SnapshotLeaseMutationResponse) {}; // auth-policy: body-bound
+  rpc VerifyFile(VerifyFileRequest) returns (VerifyFileResponse) {}; // auth-policy: read-only
+  rpc ReadParts(ReadPartsRequest) returns (ReadPartsResponse) {}; // auth-policy: read-only
+  rpc CheckParts(CheckPartsRequest) returns (CheckPartsResponse) {}; // auth-policy: read-only
+  rpc PreparePartTransaction(PreparePartTransactionRequest) returns (PreparePartTransactionResponse) {}; // auth-policy: body-bound
+  rpc RenamePart(RenamePartRequest) returns (RenamePartResponse) {}; // auth-policy: body-bound
+  rpc SettlePartTransaction(SettlePartTransactionRequest) returns (SettlePartTransactionResponse) {}; // auth-policy: body-bound
+  rpc RenameFile(RenameFileRequest) returns (RenameFileResponse) {}; // auth-policy: body-bound
+  rpc Write(WriteRequest) returns (WriteResponse) {}; // auth-policy: unimplemented
+  rpc WriteStream(stream WriteRequest) returns (stream WriteResponse) {}; // auth-policy: streaming
 //  rpc Append(AppendRequest) returns (AppendResponse) {};
-  rpc ReadAt(stream ReadAtRequest) returns (stream ReadAtResponse) {};
-  rpc ListDir(ListDirRequest) returns (ListDirResponse) {};
-  rpc WalkDir(WalkDirRequest) returns (stream WalkDirResponse) {};
-  rpc RenameData(RenameDataRequest) returns (RenameDataResponse) {};
-  rpc MakeVolumes(MakeVolumesRequest) returns (MakeVolumesResponse) {};
-  rpc MakeVolume(MakeVolumeRequest) returns (MakeVolumeResponse) {};
-  rpc ListVolumes(ListVolumesRequest) returns (ListVolumesResponse) {};
-  rpc StatVolume(StatVolumeRequest) returns (StatVolumeResponse) {};
-  rpc DeletePaths(DeletePathsRequest) returns (DeletePathsResponse) {};
-  rpc UpdateMetadata(UpdateMetadataRequest) returns (UpdateMetadataResponse) {};
-  rpc ReadMetadata(ReadMetadataRequest) returns (ReadMetadataResponse) {};
-  rpc WriteMetadata(WriteMetadataRequest) returns (WriteMetadataResponse) {};
-  rpc ReadVersion(ReadVersionRequest) returns (ReadVersionResponse) {};
-  rpc BatchReadVersion(BatchReadVersionRequest) returns (BatchReadVersionResponse) {};
-  rpc ReadXL(ReadXLRequest) returns (ReadXLResponse) {};
-  rpc DeleteVersion(DeleteVersionRequest) returns (DeleteVersionResponse) {};
-  rpc DeleteVersions(DeleteVersionsRequest) returns (DeleteVersionsResponse) {};
-  rpc ReadMultiple(ReadMultipleRequest) returns (ReadMultipleResponse) {};
-  rpc DeleteVolume(DeleteVolumeRequest) returns (DeleteVolumeResponse) {};
-  rpc DiskInfo(DiskInfoRequest) returns (DiskInfoResponse) {};
+  rpc ReadAt(stream ReadAtRequest) returns (stream ReadAtResponse) {}; // auth-policy: streaming
+  rpc ListDir(ListDirRequest) returns (ListDirResponse) {}; // auth-policy: read-only
+  rpc WalkDir(WalkDirRequest) returns (stream WalkDirResponse) {}; // auth-policy: streaming
+  rpc RenameData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
+  rpc MakeVolumes(MakeVolumesRequest) returns (MakeVolumesResponse) {}; // auth-policy: body-bound
+  rpc MakeVolume(MakeVolumeRequest) returns (MakeVolumeResponse) {}; // auth-policy: body-bound
+  rpc ListVolumes(ListVolumesRequest) returns (ListVolumesResponse) {}; // auth-policy: read-only
+  rpc StatVolume(StatVolumeRequest) returns (StatVolumeResponse) {}; // auth-policy: read-only
+  rpc DeletePaths(DeletePathsRequest) returns (DeletePathsResponse) {}; // auth-policy: body-bound
+  rpc UpdateMetadata(UpdateMetadataRequest) returns (UpdateMetadataResponse) {}; // auth-policy: body-bound
+  rpc ReadMetadata(ReadMetadataRequest) returns (ReadMetadataResponse) {}; // auth-policy: read-only
+  rpc WriteMetadata(WriteMetadataRequest) returns (WriteMetadataResponse) {}; // auth-policy: body-bound
+  rpc ReadVersion(ReadVersionRequest) returns (ReadVersionResponse) {}; // auth-policy: read-only
+  rpc BatchReadVersion(BatchReadVersionRequest) returns (BatchReadVersionResponse) {}; // auth-policy: read-only
+  rpc ReadXL(ReadXLRequest) returns (ReadXLResponse) {}; // auth-policy: read-only
+  rpc DeleteVersion(DeleteVersionRequest) returns (DeleteVersionResponse) {}; // auth-policy: body-bound
+  rpc DeleteVersions(DeleteVersionsRequest) returns (DeleteVersionsResponse) {}; // auth-policy: body-bound
+  rpc ReadMultiple(ReadMultipleRequest) returns (ReadMultipleResponse) {}; // auth-policy: read-only
+  rpc DeleteVolume(DeleteVolumeRequest) returns (DeleteVolumeResponse) {}; // auth-policy: body-bound
+  rpc DiskInfo(DiskInfoRequest) returns (DiskInfoResponse) {}; // auth-policy: read-only
   
 
 /* -------------------------------lock service-------------------------- */
 
-  rpc Lock(GenerallyLockRequest) returns (GenerallyLockResponse) {};
-  rpc UnLock(GenerallyLockRequest) returns (GenerallyLockResponse) {};
-  rpc ForceUnLock(GenerallyLockRequest) returns (GenerallyLockResponse) {};
-  rpc Refresh(GenerallyLockRequest) returns (GenerallyLockResponse) {};
-  rpc LockBatch(BatchGenerallyLockRequest) returns (BatchGenerallyLockResponse) {};
-  rpc UnLockBatch(BatchGenerallyLockRequest) returns (BatchGenerallyLockResponse) {};
+  rpc Lock(GenerallyLockRequest) returns (GenerallyLockResponse) {}; // auth-policy: body-bound
+  rpc UnLock(GenerallyLockRequest) returns (GenerallyLockResponse) {}; // auth-policy: body-bound
+  rpc ForceUnLock(GenerallyLockRequest) returns (GenerallyLockResponse) {}; // auth-policy: body-bound
+  rpc Refresh(GenerallyLockRequest) returns (GenerallyLockResponse) {}; // auth-policy: body-bound
+  rpc LockBatch(BatchGenerallyLockRequest) returns (BatchGenerallyLockResponse) {}; // auth-policy: body-bound
+  rpc UnLockBatch(BatchGenerallyLockRequest) returns (BatchGenerallyLockResponse) {}; // auth-policy: body-bound
 
 /* -------------------------------peer rest service-------------------------- */
 
-  rpc LocalStorageInfo(LocalStorageInfoRequest) returns (LocalStorageInfoResponse) {};
-  rpc ServerInfo(ServerInfoRequest) returns (ServerInfoResponse) {};
-  rpc GetCpus(GetCpusRequest) returns (GetCpusResponse) {};
-  rpc GetNetInfo(GetNetInfoRequest) returns (GetNetInfoResponse) {};
-  rpc GetPartitions(GetPartitionsRequest) returns (GetPartitionsResponse) {};
-  rpc GetOsInfo(GetOsInfoRequest) returns (GetOsInfoResponse) {};
-  rpc GetSELinuxInfo(GetSELinuxInfoRequest) returns (GetSELinuxInfoResponse) {};
-  rpc GetSysConfig(GetSysConfigRequest) returns (GetSysConfigResponse) {};
-  rpc GetSysErrors(GetSysErrorsRequest) returns (GetSysErrorsResponse) {};
-  rpc GetMemInfo(GetMemInfoRequest) returns (GetMemInfoResponse) {};
-  rpc GetMetrics(GetMetricsRequest) returns (GetMetricsResponse) {};
-  rpc GetProcInfo(GetProcInfoRequest) returns (GetProcInfoResponse) {};
-  rpc StartProfiling(StartProfilingRequest) returns (StartProfilingResponse) {};
-  rpc DownloadProfileData(DownloadProfileDataRequest) returns (DownloadProfileDataResponse) {};
-  rpc GetBucketStats(GetBucketStatsDataRequest) returns (GetBucketStatsDataResponse) {};
-  rpc GetSRMetrics(GetSRMetricsDataRequest) returns (GetSRMetricsDataResponse) {};
-  rpc GetAllBucketStats(GetAllBucketStatsRequest) returns (GetAllBucketStatsResponse) {};
-  rpc LoadBucketMetadata(LoadBucketMetadataRequest) returns (LoadBucketMetadataResponse) {};
-  rpc DeleteBucketMetadata(DeleteBucketMetadataRequest) returns (DeleteBucketMetadataResponse) {};
-  rpc DeletePolicy(DeletePolicyRequest) returns (DeletePolicyResponse) {};
-  rpc LoadPolicy(LoadPolicyRequest) returns (LoadPolicyResponse) {};
-  rpc LoadPolicyMapping(LoadPolicyMappingRequest) returns (LoadPolicyMappingResponse) {};
-  rpc DeleteUser(DeleteUserRequest) returns (DeleteUserResponse) {};
-  rpc DeleteServiceAccount(DeleteServiceAccountRequest) returns (DeleteServiceAccountResponse) {};
-  rpc LoadUser(LoadUserRequest) returns (LoadUserResponse) {};
-  rpc LoadServiceAccount(LoadServiceAccountRequest) returns (LoadServiceAccountResponse) {};
-  rpc LoadGroup(LoadGroupRequest) returns (LoadGroupResponse) {};
-  rpc ReloadSiteReplicationConfig(ReloadSiteReplicationConfigRequest) returns (ReloadSiteReplicationConfigResponse) {};
+  rpc LocalStorageInfo(LocalStorageInfoRequest) returns (LocalStorageInfoResponse) {}; // auth-policy: read-only
+  rpc ServerInfo(ServerInfoRequest) returns (ServerInfoResponse) {}; // auth-policy: read-only
+  rpc GetCpus(GetCpusRequest) returns (GetCpusResponse) {}; // auth-policy: read-only
+  rpc GetNetInfo(GetNetInfoRequest) returns (GetNetInfoResponse) {}; // auth-policy: read-only
+  rpc GetPartitions(GetPartitionsRequest) returns (GetPartitionsResponse) {}; // auth-policy: read-only
+  rpc GetOsInfo(GetOsInfoRequest) returns (GetOsInfoResponse) {}; // auth-policy: read-only
+  rpc GetSELinuxInfo(GetSELinuxInfoRequest) returns (GetSELinuxInfoResponse) {}; // auth-policy: read-only
+  rpc GetSysConfig(GetSysConfigRequest) returns (GetSysConfigResponse) {}; // auth-policy: read-only
+  rpc GetSysErrors(GetSysErrorsRequest) returns (GetSysErrorsResponse) {}; // auth-policy: read-only
+  rpc GetMemInfo(GetMemInfoRequest) returns (GetMemInfoResponse) {}; // auth-policy: read-only
+  rpc GetMetrics(GetMetricsRequest) returns (GetMetricsResponse) {}; // auth-policy: read-only
+  rpc GetProcInfo(GetProcInfoRequest) returns (GetProcInfoResponse) {}; // auth-policy: read-only
+  rpc StartProfiling(StartProfilingRequest) returns (StartProfilingResponse) {}; // auth-policy: unimplemented
+  rpc DownloadProfileData(DownloadProfileDataRequest) returns (DownloadProfileDataResponse) {}; // auth-policy: unimplemented
+  rpc GetBucketStats(GetBucketStatsDataRequest) returns (GetBucketStatsDataResponse) {}; // auth-policy: read-only
+  rpc GetSRMetrics(GetSRMetricsDataRequest) returns (GetSRMetricsDataResponse) {}; // auth-policy: unimplemented
+  rpc GetAllBucketStats(GetAllBucketStatsRequest) returns (GetAllBucketStatsResponse) {}; // auth-policy: unimplemented
+  rpc LoadBucketMetadata(LoadBucketMetadataRequest) returns (LoadBucketMetadataResponse) {}; // auth-policy: body-bound
+  rpc DeleteBucketMetadata(DeleteBucketMetadataRequest) returns (DeleteBucketMetadataResponse) {}; // auth-policy: body-bound
+  rpc DeletePolicy(DeletePolicyRequest) returns (DeletePolicyResponse) {}; // auth-policy: body-bound
+  rpc LoadPolicy(LoadPolicyRequest) returns (LoadPolicyResponse) {}; // auth-policy: body-bound
+  rpc LoadPolicyMapping(LoadPolicyMappingRequest) returns (LoadPolicyMappingResponse) {}; // auth-policy: body-bound
+  rpc DeleteUser(DeleteUserRequest) returns (DeleteUserResponse) {}; // auth-policy: body-bound
+  rpc DeleteServiceAccount(DeleteServiceAccountRequest) returns (DeleteServiceAccountResponse) {}; // auth-policy: body-bound
+  rpc LoadUser(LoadUserRequest) returns (LoadUserResponse) {}; // auth-policy: body-bound
+  rpc LoadServiceAccount(LoadServiceAccountRequest) returns (LoadServiceAccountResponse) {}; // auth-policy: body-bound
+  rpc LoadGroup(LoadGroupRequest) returns (LoadGroupResponse) {}; // auth-policy: body-bound
+  rpc ReloadSiteReplicationConfig(ReloadSiteReplicationConfigRequest) returns (ReloadSiteReplicationConfigResponse) {}; // auth-policy: body-bound
   // rpc VerifyBinary() returns () {};
   // rpc CommitBinary() returns () {};
-  rpc SignalService(SignalServiceRequest) returns (SignalServiceResponse) {};
-  rpc ScannerActivity(ScannerActivityRequest) returns (ScannerActivityResponse) {};
-  rpc BackgroundHealStatus(BackgroundHealStatusRequest) returns (BackgroundHealStatusResponse) {};
-  rpc GetMetacacheListing(GetMetacacheListingRequest) returns (GetMetacacheListingResponse) {};
-  rpc UpdateMetacacheListing(UpdateMetacacheListingRequest) returns (UpdateMetacacheListingResponse) {};
-  rpc ReloadPoolMeta(ReloadPoolMetaRequest) returns (ReloadPoolMetaResponse) {};
-  rpc StopRebalance(StopRebalanceRequest) returns (StopRebalanceResponse) {};
-  rpc LoadRebalanceMeta(LoadRebalanceMetaRequest) returns (LoadRebalanceMetaResponse) {};
-  rpc StartDecommission(StartDecommissionRequest) returns (StartDecommissionResponse) {};
-  rpc CancelDecommission(CancelDecommissionRequest) returns (CancelDecommissionResponse) {};
-  rpc ClearDecommission(ClearDecommissionRequest) returns (ClearDecommissionResponse) {};
-  rpc LoadTransitionTierConfig(LoadTransitionTierConfigRequest) returns (LoadTransitionTierConfigResponse) {};
-  rpc GetLiveEvents(GetLiveEventsRequest) returns (GetLiveEventsResponse) {};
+  rpc SignalService(SignalServiceRequest) returns (SignalServiceResponse) {}; // auth-policy: body-bound
+  rpc ScannerActivity(ScannerActivityRequest) returns (ScannerActivityResponse) {}; // auth-policy: body-bound
+  rpc BackgroundHealStatus(BackgroundHealStatusRequest) returns (BackgroundHealStatusResponse) {}; // auth-policy: read-only
+  rpc GetMetacacheListing(GetMetacacheListingRequest) returns (GetMetacacheListingResponse) {}; // auth-policy: unimplemented
+  rpc UpdateMetacacheListing(UpdateMetacacheListingRequest) returns (UpdateMetacacheListingResponse) {}; // auth-policy: unimplemented
+  rpc ReloadPoolMeta(ReloadPoolMetaRequest) returns (ReloadPoolMetaResponse) {}; // auth-policy: body-bound
+  rpc StopRebalance(StopRebalanceRequest) returns (StopRebalanceResponse) {}; // auth-policy: body-bound
+  rpc LoadRebalanceMeta(LoadRebalanceMetaRequest) returns (LoadRebalanceMetaResponse) {}; // auth-policy: body-bound
+  rpc StartDecommission(StartDecommissionRequest) returns (StartDecommissionResponse) {}; // auth-policy: body-bound
+  rpc CancelDecommission(CancelDecommissionRequest) returns (CancelDecommissionResponse) {}; // auth-policy: body-bound
+  rpc ClearDecommission(ClearDecommissionRequest) returns (ClearDecommissionResponse) {}; // auth-policy: body-bound
+  rpc LoadTransitionTierConfig(LoadTransitionTierConfigRequest) returns (LoadTransitionTierConfigResponse) {}; // auth-policy: body-bound
+  rpc GetLiveEvents(GetLiveEventsRequest) returns (GetLiveEventsResponse) {}; // auth-policy: read-only
 }
 
 service HealControlService {

crates/rio-v2/tests/minio_fixture_lab/README.md

@@ -137,7 +137,7 @@ tests read):
 ./capture_via_docker.sh
 
 RUSTFS_MINIO_STATIC_KMS_KEY_B64=IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g= \
-  cargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored
+  cargo test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored
 ```
 
 This is exactly what the nightly `minio-interop` GitHub Actions workflow runs

crates/scanner/src/data_usage_define.rs

@@ -698,7 +698,7 @@ impl DataUsageCache {
                 let mut visited = HashSet::new();
                 visited.insert(hash_path(path).key());
                 let mut flat = self.flatten_with_guard(root, &mut visited, 0);
-                if flat.replication_stats.as_ref().is_some_and(|stats| stats.empty()) {
+                if flat.replication_stats.as_ref().is_some_and(|stats| stats.is_empty()) {
                     flat.replication_stats = None;
                 }
                 Some(flat)
@@ -1574,6 +1574,7 @@ mod tests {
     use super::*;
     use crate::storage_api::scanner_io::{HTTPRangeSpec, ObjectIO};
     use crate::{ScannerGetObjectReader, ScannerPutObjReader};
+    use rustfs_data_usage::{ReplicationAllStats, ReplicationStats};
     use serde_json::Value;
     use std::io::Cursor;
     use std::pin::Pin;
@@ -2767,6 +2768,55 @@ mod tests {
         assert!(flat.children.is_empty());
     }
 
+    #[test]
+    fn size_recursive_prunes_empty_and_preserves_threshold_replication_stats() {
+        let root = hash_path("bucket");
+        let child = hash_path("bucket/child");
+        let mut cache = DataUsageCache::default();
+        cache.replace_hashed(&root, &None, &DataUsageEntry::default());
+        cache.replace_hashed(
+            &child,
+            &Some(root.clone()),
+            &DataUsageEntry {
+                replication_stats: Some(ReplicationAllStats::default()),
+                ..Default::default()
+            },
+        );
+
+        assert!(
+            cache
+                .size_recursive("bucket")
+                .expect("scanner bucket usage should flatten")
+                .replication_stats
+                .is_none()
+        );
+
+        cache.replace_hashed(
+            &child,
+            &Some(root.clone()),
+            &DataUsageEntry {
+                replication_stats: Some(ReplicationAllStats {
+                    targets: HashMap::from([(
+                        "arn:test:threshold".to_string(),
+                        ReplicationStats {
+                            after_threshold_count: 1,
+                            ..Default::default()
+                        },
+                    )]),
+                    ..Default::default()
+                }),
+                ..Default::default()
+            },
+        );
+
+        let flattened = cache.size_recursive("bucket").expect("scanner bucket usage should flatten");
+        let replication = flattened
+            .replication_stats
+            .expect("threshold-only replication stats must survive pruning");
+
+        assert_eq!(replication.targets["arn:test:threshold"].after_threshold_count, 1);
+    }
+
     #[test]
     fn checked_flatten_rejects_dangling_child() {
         let root_key = hash_path("bucket").key();

crates/utils/src/http/header_compat.rs

@@ -25,6 +25,11 @@ const RUSTFS_PREFIX: &str = "x-rustfs-";
 const MINIO_PREFIX: &str = "x-minio-";
 const MINIO_ENCRYPTION_PREFIX: &str = "x-minio-encryption-";
 const RUSTFS_ENCRYPTION_PREFIX: &str = "x-rustfs-encryption-";
+const MINIO_INTERNAL_ENCRYPTION_PREFIX: &str = "x-minio-internal-server-side-encryption-";
+const MINIO_INTERNAL_ENCRYPTED_MULTIPART: &str = "x-minio-internal-encrypted-multipart";
+const RUSTFS_ENCRYPTION_ORIGINAL_SIZE: &str = "x-rustfs-encryption-original-size";
+const MINIO_ENCRYPTION_ORIGINAL_SIZE: &str = "x-minio-encryption-original-size";
+const SSEC_ORIGINAL_SIZE: &str = "x-amz-server-side-encryption-customer-original-size";
 
 // Suffix constants (part after x-rustfs- or x-minio-). Use with get_header/insert_header.
 pub const SUFFIX_FORCE_DELETE: &str = "force-delete";
@@ -40,11 +45,49 @@ pub const SUFFIX_SOURCE_REPLICATION_REQUEST: &str = "source-replication-request"
 pub const SUFFIX_SOURCE_REPLICATION_CHECK: &str = "source-replication-check";
 pub const SUFFIX_REPLICATION_SSEC_CRC: &str = "replication-ssec-crc";
 
-/// Returns true if the key is an internal encryption metadata key (x-rustfs-encryption-* or
-/// x-minio-encryption-*). Case-insensitive for metadata filtering.
+/// Returns true if the key is object-encryption metadata understood by RustFS or MinIO.
+/// Case-insensitive for metadata filtering.
 pub fn is_encryption_metadata_key(key: &str) -> bool {
     let lower = key.to_lowercase();
-    lower.starts_with(RUSTFS_ENCRYPTION_PREFIX) || lower.starts_with(MINIO_ENCRYPTION_PREFIX)
+    lower.starts_with(RUSTFS_ENCRYPTION_PREFIX)
+        || lower.starts_with(MINIO_ENCRYPTION_PREFIX)
+        || lower.starts_with(MINIO_INTERNAL_ENCRYPTION_PREFIX)
+        || lower == MINIO_INTERNAL_ENCRYPTED_MULTIPART
+}
+
+/// Returns true when a metadata key proves that object data is encrypted.
+///
+/// Original-size metadata alone is not proof: older plaintext objects can
+/// retain that compatibility field after metadata migration.
+pub fn is_object_encryption_marker(key: &str) -> bool {
+    (is_encryption_metadata_key(key)
+        && !key.eq_ignore_ascii_case(RUSTFS_ENCRYPTION_ORIGINAL_SIZE)
+        && !key.eq_ignore_ascii_case(MINIO_ENCRYPTION_ORIGINAL_SIZE))
+        || super::is_sse_header(key)
+}
+
+/// Reads the logical object size recorded by encryption metadata.
+pub fn get_object_encryption_original_size(metadata: &std::collections::HashMap<String, String>) -> std::io::Result<Option<i64>> {
+    let actual_size = super::get_str(metadata, super::SUFFIX_ACTUAL_SIZE);
+    let size = get_case_insensitive(metadata, RUSTFS_ENCRYPTION_ORIGINAL_SIZE)
+        .or_else(|| get_case_insensitive(metadata, SSEC_ORIGINAL_SIZE))
+        .or(actual_size.as_deref());
+
+    let Some(size) = size.filter(|size| !size.is_empty()) else {
+        return Ok(None);
+    };
+    size.parse::<i64>()
+        .map(Some)
+        .map_err(|error| std::io::Error::other(format!("Failed to parse encryption original size: {error}")))
+}
+
+fn get_case_insensitive<'a>(metadata: &'a std::collections::HashMap<String, String>, key: &str) -> Option<&'a str> {
+    metadata.get(key).map(String::as_str).or_else(|| {
+        metadata
+            .iter()
+            .find(|(candidate, _)| candidate.eq_ignore_ascii_case(key))
+            .map(|(_, value)| value.as_str())
+    })
 }
 
 fn rustfs_key(suffix: &str) -> String {
@@ -106,10 +149,37 @@ mod tests {
         assert!(is_encryption_metadata_key("x-rustfs-encryption-iv"));
         assert!(is_encryption_metadata_key("X-Rustfs-Encryption-Key"));
         assert!(is_encryption_metadata_key("x-minio-encryption-iv"));
+        assert!(is_encryption_metadata_key("X-Minio-Internal-Server-Side-Encryption-Sealed-Key"));
+        assert!(is_encryption_metadata_key("X-Minio-Internal-Encrypted-Multipart"));
         assert!(!is_encryption_metadata_key("x-amz-meta-custom"));
         assert!(!is_encryption_metadata_key("x-rustfs-internal-healing"));
     }
 
+    #[test]
+    fn object_encryption_marker_excludes_size_only_metadata() {
+        assert!(!is_object_encryption_marker(RUSTFS_ENCRYPTION_ORIGINAL_SIZE));
+        assert!(is_object_encryption_marker("X-Minio-Internal-Server-Side-Encryption-Sealed-Key"));
+        assert!(is_object_encryption_marker("x-amz-server-side-encryption"));
+    }
+
+    #[test]
+    fn object_encryption_original_size_is_case_insensitive() {
+        let metadata = std::collections::HashMap::from([(
+            "X-Amz-Server-Side-Encryption-Customer-Original-Size".to_string(),
+            "42".to_string(),
+        )]);
+        assert_eq!(get_object_encryption_original_size(&metadata).expect("valid size"), Some(42));
+    }
+
+    #[test]
+    fn object_encryption_original_size_prefers_rustfs_metadata() {
+        let metadata = std::collections::HashMap::from([
+            (SSEC_ORIGINAL_SIZE.to_string(), "21".to_string()),
+            (RUSTFS_ENCRYPTION_ORIGINAL_SIZE.to_string(), "42".to_string()),
+        ]);
+        assert_eq!(get_object_encryption_original_size(&metadata).expect("valid size"), Some(42));
+    }
+
     #[test]
     fn test_get_header() {
         let mut headers = HeaderMap::new();

docs/testing/ecstore-validation-suite-design.md

@@ -358,7 +358,7 @@ Fixture-backed tests should run when the fixture path is present:
 
 ```bash
 cargo test -p rustfs-ecstore --test legacy_bitrot_read_test -- --nocapture
-cargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored --nocapture
+cargo test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored --nocapture
 ```
 
 ## Multi-Expert Adversarial Review Summary

rustfs/Cargo.toml

@@ -47,6 +47,7 @@ io-scheduler-debug = []  # Enable debug information in I/O scheduler
 tracing-chunk-debug = []  # Enable per-chunk tracing in data plane (high noise, for debugging only)
 full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp", "pyroscope"]
 manual-test-runners = []
+e2e-test-hooks = []
 rio-v2 = ["rustfs-ecstore/rio-v2"]
 pyroscope = ["rustfs-obs/pyroscope"]
 # Tokio runtime telemetry. Requires `--cfg tokio_unstable`; use `make build-profiling`.
@@ -124,7 +125,7 @@ tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "sig
 tokio-rustls = { workspace = true, default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
 aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 tokio-stream.workspace = true
-tokio-util = { workspace = true, features = ["io", "compat"] }
+tokio-util = { workspace = true, features = ["io", "compat", "time"] }
 tonic = { workspace = true, features = ["gzip", "deflate"] }
 tower = { workspace = true, features = ["timeout"] }
 tower-http = { workspace = true, features = ["trace", "compression-full", "cors", "catch-panic", "timeout", "limit", "request-id", "add-extension"] }
@@ -163,7 +164,7 @@ const-str = { workspace = true, features = ["std", "proc"] }
 datafusion = { workspace = true, default-features = false, features = ["parquet", "recursive_protection", "sql"] }
 hex-simd.workspace = true
 matchit = { workspace = true }
-md5.workspace = true
+md-5 = { workspace = true }
 mime_guess = { workspace = true }
 percent-encoding = { workspace = true }
 pin-project-lite.workspace = true

rustfs/src/admin/handlers/ilm_transition.rs

@@ -58,6 +58,8 @@ const LOG_SUBSYSTEM_ILM_TRANSITION: &str = "ilm_transition";
 const EVENT_ADMIN_ILM_TRANSITION_STATE: &str = "admin_ilm_transition_state";
 
 static ACTIVE_MANUAL_TRANSITION_SCOPES: OnceLock<Mutex<Vec<ManualTransitionRunScope>>> = OnceLock::new();
+#[cfg(feature = "e2e-test-hooks")]
+const E2E_MANUAL_TRANSITION_CANCEL_BARRIER_ENV: &str = "RUSTFS_E2E_MANUAL_TRANSITION_CANCEL_BARRIER";
 static ACTIVE_MANUAL_TRANSITION_JOBS: OnceLock<Mutex<HashMap<Uuid, CancellationToken>>> = OnceLock::new();
 static MANUAL_TRANSITION_OWNER_ID: OnceLock<String> = OnceLock::new();
 
@@ -748,6 +750,10 @@ async fn start_manual_transition_job(
     let job_heartbeat_shutdown_token = heartbeat_shutdown_token.clone();
     spawn_manual_transition_job_heartbeat(store, job_id, scan_cancel_token, heartbeat_shutdown_token);
     tokio::spawn(async move {
+        #[cfg(feature = "e2e-test-hooks")]
+        if std::env::var_os(E2E_MANUAL_TRANSITION_CANCEL_BARRIER_ENV).is_some() {
+            job_scan_cancel_token.cancelled().await;
+        }
         let result = enqueue_transition_for_existing_objects_scoped(run_store.clone(), &bucket, run_options).await;
         if let Some(final_record) = finalize_manual_transition_job(run_store.clone(), job_id, result).await
             && final_record.is_terminal()

rustfs/src/admin/handlers/kms_dynamic.rs

@@ -79,10 +79,40 @@ fn kms_service_control_actions() -> Vec<Action> {
     vec![Action::KmsAction(KmsAction::ServiceControlAction)]
 }
 
-fn normalize_configure_request_auth(
+fn normalize_configure_request_secrets(
     request: &mut ConfigureKmsRequest,
     existing_config: Option<&KmsConfig>,
 ) -> Result<(), String> {
+    if existing_config.is_some_and(|config| matches!(&config.backend_config, rustfs_kms::BackendConfig::Local(_)))
+        && !matches!(request, ConfigureKmsRequest::Local(_))
+    {
+        return Err("Changing from the Local KMS backend is not supported".to_string());
+    }
+
+    if let ConfigureKmsRequest::Local(request) = request
+        && let Some(KmsConfig {
+            backend_config: rustfs_kms::BackendConfig::Local(existing),
+            allow_insecure_dev_defaults,
+            ..
+        }) = existing_config
+    {
+        if request.key_dir != existing.key_dir {
+            return Err("Changing the Local KMS key directory is not supported".to_string());
+        }
+        match request.file_permissions {
+            Some(permissions) if Some(permissions) != existing.file_permissions => {
+                return Err("Changing Local KMS file permissions is not supported".to_string());
+            }
+            None => request.file_permissions = existing.file_permissions,
+            Some(_) => {}
+        }
+        if request.master_key.as_deref().is_some_and(|master_key| !master_key.is_empty()) {
+            return Err("Changing the Local KMS master key is not supported".to_string());
+        }
+        request.master_key.clone_from(&existing.master_key);
+        request.allow_insecure_dev_defaults = Some(*allow_insecure_dev_defaults);
+    }
+
     let needs_existing_auth = match request {
         ConfigureKmsRequest::VaultKv2(req) => token_is_blank(&req.auth_method),
         ConfigureKmsRequest::VaultTransit(req) => token_is_blank(&req.auth_method),
@@ -350,9 +380,9 @@ impl Operation for ConfigureKmsHandler {
         );
 
         let service_manager = kms_service_manager_from_context();
-        let existing_config = service_manager.get_redacted_config().await;
+        let existing_config = service_manager.get_config().await;
 
-        if let Err(e) = normalize_configure_request_auth(&mut configure_request, existing_config.as_ref()) {
+        if let Err(e) = normalize_configure_request_secrets(&mut configure_request, existing_config.as_ref()) {
             return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(e))));
         }
 
@@ -363,36 +393,27 @@ impl Operation for ConfigureKmsHandler {
         // Convert request to KmsConfig
         let kms_config = configure_request.to_kms_config();
 
-        // Configure the service
-        let (success, message, status) = match service_manager.configure(kms_config.clone()).await {
+        let persisted_config = kms_config.clone();
+        let (success, message, status) = match service_manager
+            .configure_with_persistence(kms_config, || async move {
+                save_kms_config(&persisted_config)
+                    .await
+                    .map_err(|error| rustfs_kms::KmsError::backend_error(format!("Failed to persist KMS configuration: {error}")))
+            })
+            .await
+        {
             Ok(()) => {
-                // Persist the configuration to cluster storage
-                if let Err(e) = save_kms_config(&kms_config).await {
-                    let error_msg = format!("KMS configured in memory but failed to persist: {e}");
-                    error!(
-                        component = LOG_COMPONENT_ADMIN,
-                        subsystem = LOG_SUBSYSTEM_KMS,
-                        event = "kms_service_state",
-                        operation = "configure",
-                        state = "persist_failed",
-                        error = %e,
-                        "admin kms dynamic state"
-                    );
-                    let status = service_manager.get_status().await;
-                    (false, error_msg, status)
-                } else {
-                    let status = service_manager.get_status().await;
-                    info!(
-                        component = LOG_COMPONENT_ADMIN,
-                        subsystem = LOG_SUBSYSTEM_KMS,
-                        event = "kms_service_state",
-                        operation = "configure",
-                        state = "configured",
-                        status = ?status,
-                        "admin kms dynamic state"
-                    );
-                    (true, "KMS configured successfully".to_string(), status)
-                }
+                let status = service_manager.get_status().await;
+                info!(
+                    component = LOG_COMPONENT_ADMIN,
+                    subsystem = LOG_SUBSYSTEM_KMS,
+                    event = "kms_service_state",
+                    operation = "configure",
+                    state = "configured",
+                    status = ?status,
+                    "admin kms dynamic state"
+                );
+                (true, "KMS configured successfully".to_string(), status)
             }
             Err(e) => {
                 let error_msg = format!("Failed to configure KMS: {e}");
@@ -499,125 +520,61 @@ impl Operation for StartKmsHandler {
         );
 
         let service_manager = kms_service_manager_from_context();
-
-        // Check if already running and force flag
-        let current_status = service_manager.get_status().await;
-        if matches!(current_status, KmsServiceStatus::Running) && !start_request.force.unwrap_or(false) {
-            warn!(
-                component = LOG_COMPONENT_ADMIN,
-                subsystem = LOG_SUBSYSTEM_KMS,
-                event = "kms_service_state",
-                operation = "start",
-                state = "already_running",
-                "admin kms dynamic state"
-            );
-            let response = StartKmsResponse {
-                success: false,
-                message: "KMS service is already running. Use force=true to restart.".to_string(),
-                status: current_status,
-            };
-            let json_response = match serde_json::to_string(&response) {
-                Ok(json) => json,
-                Err(e) => {
-                    error!(
-                        component = LOG_COMPONENT_ADMIN,
-                        subsystem = LOG_SUBSYSTEM_KMS,
-                        event = EVENT_ADMIN_KMS_DYNAMIC_STATE,
-                        operation = "start",
-                        result = "response_serialize_failed",
-                        error = %e,
-                        "admin kms dynamic state"
-                    );
-                    return Ok(S3Response::new((
-                        StatusCode::INTERNAL_SERVER_ERROR,
-                        Body::from("Serialization error".to_string()),
-                    )));
-                }
-            };
-            return Ok(S3Response::new((StatusCode::OK, Body::from(json_response))));
-        }
-
-        // Start the service (or restart if force=true)
-        let (success, message, status) =
-            if start_request.force.unwrap_or(false) && matches!(current_status, KmsServiceStatus::Running) {
-                // Force restart
-                match service_manager.stop().await {
-                    Ok(()) => match service_manager.start().await {
-                        Ok(()) => {
-                            let status = service_manager.get_status().await;
-                            info!(
-                                component = LOG_COMPONENT_ADMIN,
-                                subsystem = LOG_SUBSYSTEM_KMS,
-                                event = "kms_service_state",
-                                operation = "restart",
-                                state = "running",
-                                status = ?status,
-                                "admin kms dynamic state"
-                            );
-                            (true, "KMS service restarted successfully".to_string(), status)
-                        }
-                        Err(e) => {
-                            let error_msg = format!("Failed to restart KMS service: {e}");
-                            error!(
-                                component = LOG_COMPONENT_ADMIN,
-                                subsystem = LOG_SUBSYSTEM_KMS,
-                                event = "kms_service_state",
-                                operation = "restart",
-                                state = "start_failed",
-                                error = %e,
-                                "admin kms dynamic state"
-                            );
-                            let status = service_manager.get_status().await;
-                            (false, error_msg, status)
-                        }
-                    },
-                    Err(e) => {
-                        let error_msg = format!("Failed to stop KMS service for restart: {e}");
-                        error!(
-                            component = LOG_COMPONENT_ADMIN,
-                            subsystem = LOG_SUBSYSTEM_KMS,
-                            event = "kms_service_state",
-                            operation = "restart",
-                            state = "stop_failed",
-                            error = %e,
-                            "admin kms dynamic state"
-                        );
-                        let status = service_manager.get_status().await;
-                        (false, error_msg, status)
-                    }
-                }
-            } else {
-                // Normal start
-                match service_manager.start().await {
-                    Ok(()) => {
-                        let status = service_manager.get_status().await;
-                        info!(
-                            component = LOG_COMPONENT_ADMIN,
-                            subsystem = LOG_SUBSYSTEM_KMS,
-                            event = "kms_service_state",
-                            operation = "start",
-                            state = "running",
-                            status = ?status,
-                            "admin kms dynamic state"
-                        );
-                        (true, "KMS service started successfully".to_string(), status)
-                    }
-                    Err(e) => {
-                        let error_msg = format!("Failed to start KMS service: {e}");
-                        error!(
-                            component = LOG_COMPONENT_ADMIN,
-                            subsystem = LOG_SUBSYSTEM_KMS,
-                            event = "kms_service_state",
-                            operation = "start",
-                            state = "start_failed",
-                            error = %e,
-                            "admin kms dynamic state"
-                        );
-                        let status = service_manager.get_status().await;
-                        (false, error_msg, status)
-                    }
-                }
-            };
+        let force = start_request.force.unwrap_or(false);
+        let (success, message, status) = match service_manager.start_or_restart(force).await {
+            Ok(rustfs_kms::KmsStartOutcome::Started) => {
+                let status = service_manager.get_status().await;
+                info!(
+                    component = LOG_COMPONENT_ADMIN,
+                    subsystem = LOG_SUBSYSTEM_KMS,
+                    event = "kms_service_state",
+                    operation = "start",
+                    state = "running",
+                    status = ?status,
+                    "admin kms dynamic state"
+                );
+                (true, "KMS service started successfully".to_string(), status)
+            }
+            Ok(rustfs_kms::KmsStartOutcome::Restarted) => {
+                let status = service_manager.get_status().await;
+                info!(
+                    component = LOG_COMPONENT_ADMIN,
+                    subsystem = LOG_SUBSYSTEM_KMS,
+                    event = "kms_service_state",
+                    operation = "restart",
+                    state = "running",
+                    status = ?status,
+                    "admin kms dynamic state"
+                );
+                (true, "KMS service restarted successfully".to_string(), status)
+            }
+            Ok(rustfs_kms::KmsStartOutcome::AlreadyRunning) => {
+                let status = service_manager.get_status().await;
+                warn!(
+                    component = LOG_COMPONENT_ADMIN,
+                    subsystem = LOG_SUBSYSTEM_KMS,
+                    event = "kms_service_state",
+                    operation = "start",
+                    state = "already_running",
+                    "admin kms dynamic state"
+                );
+                (false, "KMS service is already running. Use force=true to restart.".to_string(), status)
+            }
+            Err(e) => {
+                let error_msg = format!("Failed to start or restart KMS service: {e}");
+                error!(
+                    component = LOG_COMPONENT_ADMIN,
+                    subsystem = LOG_SUBSYSTEM_KMS,
+                    event = "kms_service_state",
+                    operation = "start",
+                    state = "start_failed",
+                    error = %e,
+                    "admin kms dynamic state"
+                );
+                let status = service_manager.get_status().await;
+                (false, error_msg, status)
+            }
+        };
 
         let response = StartKmsResponse {
             success,
@@ -774,8 +731,7 @@ impl Operation for GetKmsStatusHandler {
 
         let service_manager = kms_service_manager_from_context();
 
-        let status = service_manager.get_status().await;
-        let config = service_manager.get_redacted_config().await;
+        let (status, config) = service_manager.get_redacted_state().await;
 
         // Get backend type and health status
         let backend_type = config.as_ref().map(|c| c.backend.clone());
@@ -895,9 +851,9 @@ impl Operation for ReconfigureKmsHandler {
         );
 
         let service_manager = kms_service_manager_from_context();
-        let existing_config = service_manager.get_redacted_config().await;
+        let existing_config = service_manager.get_config().await;
 
-        if let Err(e) = normalize_configure_request_auth(&mut configure_request, existing_config.as_ref()) {
+        if let Err(e) = normalize_configure_request_secrets(&mut configure_request, existing_config.as_ref()) {
             return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(e))));
         }
 
@@ -908,36 +864,27 @@ impl Operation for ReconfigureKmsHandler {
         // Convert request to KmsConfig
         let kms_config = configure_request.to_kms_config();
 
-        // Reconfigure the service (stops, reconfigures, and starts)
-        let (success, message, status) = match service_manager.reconfigure(kms_config.clone()).await {
+        let persisted_config = kms_config.clone();
+        let (success, message, status) = match service_manager
+            .reconfigure_with_persistence(kms_config, || async move {
+                save_kms_config(&persisted_config)
+                    .await
+                    .map_err(|error| rustfs_kms::KmsError::backend_error(format!("Failed to persist KMS configuration: {error}")))
+            })
+            .await
+        {
             Ok(()) => {
-                // Persist the configuration to cluster storage
-                if let Err(e) = save_kms_config(&kms_config).await {
-                    let error_msg = format!("KMS reconfigured in memory but failed to persist: {e}");
-                    error!(
-                        component = LOG_COMPONENT_ADMIN,
-                        subsystem = LOG_SUBSYSTEM_KMS,
-                        event = "kms_service_state",
-                        operation = "reconfigure",
-                        state = "persist_failed",
-                        error = %e,
-                        "admin kms dynamic state"
-                    );
-                    let status = service_manager.get_status().await;
-                    (false, error_msg, status)
-                } else {
-                    let status = service_manager.get_status().await;
-                    info!(
-                        component = LOG_COMPONENT_ADMIN,
-                        subsystem = LOG_SUBSYSTEM_KMS,
-                        event = "kms_service_state",
-                        operation = "reconfigure",
-                        state = "reconfigured",
-                        status = ?status,
-                        "admin kms dynamic state"
-                    );
-                    (true, "KMS reconfigured and restarted successfully".to_string(), status)
-                }
+                let status = service_manager.get_status().await;
+                info!(
+                    component = LOG_COMPONENT_ADMIN,
+                    subsystem = LOG_SUBSYSTEM_KMS,
+                    event = "kms_service_state",
+                    operation = "reconfigure",
+                    state = "reconfigured",
+                    status = ?status,
+                    "admin kms dynamic state"
+                );
+                (true, "KMS reconfigured and restarted successfully".to_string(), status)
             }
             Err(e) => {
                 let error_msg = format!("Failed to reconfigure KMS: {e}");
@@ -986,8 +933,12 @@ impl Operation for ReconfigureKmsHandler {
 
 #[cfg(test)]
 mod tests {
-    use super::{decode_persisted_kms_config, ensure_kms_config_persistable, kms_configure_actions, kms_service_control_actions};
+    use super::{
+        decode_persisted_kms_config, ensure_kms_config_persistable, kms_configure_actions, kms_service_control_actions,
+        normalize_configure_request_secrets,
+    };
     use rustfs_policy::policy::action::{Action, AdminAction, KmsAction};
+    use std::path::PathBuf;
     use tempfile::TempDir;
 
     fn assert_has_action(actions: &[Action], action: Action) {
@@ -1084,4 +1035,140 @@ mod tests {
 
         assert!(ensure_kms_config_persistable(&config).is_err());
     }
+
+    #[test]
+    fn local_reconfigure_preserves_hidden_master_key_for_same_directory() {
+        let key_dir = PathBuf::from("/var/lib/rustfs/kms");
+        let mut existing = rustfs_kms::KmsConfig::local(key_dir.clone());
+        let rustfs_kms::BackendConfig::Local(existing_local) = &mut existing.backend_config else {
+            panic!("local constructor must create local backend config");
+        };
+        existing_local.master_key = Some("stored-master-key".to_string());
+
+        let mut request = rustfs_kms::ConfigureKmsRequest::Local(rustfs_kms::ConfigureLocalKmsRequest {
+            key_dir,
+            master_key: None,
+            file_permissions: Some(0o600),
+            default_key_id: Some("experience-key".to_string()),
+            timeout_seconds: Some(30),
+            retry_attempts: Some(3),
+            enable_cache: Some(true),
+            max_cached_keys: Some(1000),
+            cache_ttl_seconds: Some(3600),
+            allow_insecure_dev_defaults: Some(false),
+        });
+
+        normalize_configure_request_secrets(&mut request, Some(&existing)).expect("normalize local request");
+        let rustfs_kms::ConfigureKmsRequest::Local(request) = request else {
+            panic!("request must remain local");
+        };
+        assert_eq!(request.master_key.as_deref(), Some("stored-master-key"));
+    }
+
+    #[test]
+    fn local_reconfigure_preserves_unspecified_legacy_file_permissions() {
+        let key_dir = PathBuf::from("/var/lib/rustfs/kms");
+        let mut existing = rustfs_kms::KmsConfig::local(key_dir.clone());
+        let rustfs_kms::BackendConfig::Local(existing_local) = &mut existing.backend_config else {
+            panic!("local constructor must create local backend config");
+        };
+        existing_local.master_key = Some("stored-master-key".to_string());
+        existing_local.file_permissions = None;
+
+        let mut request = rustfs_kms::ConfigureKmsRequest::Local(rustfs_kms::ConfigureLocalKmsRequest {
+            key_dir,
+            master_key: None,
+            file_permissions: None,
+            default_key_id: Some("experience-key".to_string()),
+            timeout_seconds: None,
+            retry_attempts: None,
+            enable_cache: None,
+            max_cached_keys: None,
+            cache_ttl_seconds: None,
+            allow_insecure_dev_defaults: Some(false),
+        });
+
+        normalize_configure_request_secrets(&mut request, Some(&existing)).expect("normalize legacy local request");
+        let rustfs_kms::ConfigureKmsRequest::Local(request) = request else {
+            panic!("request must remain local");
+        };
+        assert!(request.file_permissions.is_none());
+    }
+
+    #[test]
+    fn local_reconfigure_does_not_reuse_master_key_for_different_directory() {
+        let mut existing = rustfs_kms::KmsConfig::local(PathBuf::from("/var/lib/rustfs/kms"));
+        let rustfs_kms::BackendConfig::Local(existing_local) = &mut existing.backend_config else {
+            panic!("local constructor must create local backend config");
+        };
+        existing_local.master_key = Some("stored-master-key".to_string());
+
+        let mut request = rustfs_kms::ConfigureKmsRequest::Local(rustfs_kms::ConfigureLocalKmsRequest {
+            key_dir: PathBuf::from("/var/lib/rustfs/other-kms"),
+            master_key: None,
+            file_permissions: Some(0o600),
+            default_key_id: None,
+            timeout_seconds: None,
+            retry_attempts: None,
+            enable_cache: None,
+            max_cached_keys: None,
+            cache_ttl_seconds: None,
+            allow_insecure_dev_defaults: Some(false),
+        });
+
+        let error = normalize_configure_request_secrets(&mut request, Some(&existing))
+            .expect_err("changing the local key directory must be rejected");
+        assert_eq!(error, "Changing the Local KMS key directory is not supported");
+    }
+
+    #[test]
+    fn local_reconfigure_rejects_file_permission_and_master_key_changes() {
+        let key_dir = PathBuf::from("/var/lib/rustfs/kms");
+        let mut existing = rustfs_kms::KmsConfig::local(key_dir.clone());
+        let rustfs_kms::BackendConfig::Local(existing_local) = &mut existing.backend_config else {
+            panic!("local constructor must create local backend config");
+        };
+        existing_local.master_key = Some("stored-master-key".to_string());
+        existing_local.file_permissions = Some(0o600);
+
+        let request = |master_key, file_permissions| {
+            rustfs_kms::ConfigureKmsRequest::Local(rustfs_kms::ConfigureLocalKmsRequest {
+                key_dir: key_dir.clone(),
+                master_key,
+                file_permissions,
+                default_key_id: None,
+                timeout_seconds: None,
+                retry_attempts: None,
+                enable_cache: None,
+                max_cached_keys: None,
+                cache_ttl_seconds: None,
+                allow_insecure_dev_defaults: Some(false),
+            })
+        };
+
+        let mut permissions_change = request(None, Some(0o666));
+        let permissions_error = normalize_configure_request_secrets(&mut permissions_change, Some(&existing))
+            .expect_err("changing file permissions must be rejected");
+        assert_eq!(permissions_error, "Changing Local KMS file permissions is not supported");
+
+        let mut master_key_change = request(Some("replacement-master-key".to_string()), Some(0o600));
+        let master_key_error = normalize_configure_request_secrets(&mut master_key_change, Some(&existing))
+            .expect_err("changing the master key must be rejected");
+        assert_eq!(master_key_error, "Changing the Local KMS master key is not supported");
+
+        let mut backend_change = rustfs_kms::ConfigureKmsRequest::Static(rustfs_kms::ConfigureStaticKmsRequest {
+            key_id: "static-key".to_string(),
+            secret_key: "not-used-by-normalization".to_string(),
+            default_key_id: None,
+            timeout_seconds: None,
+            retry_attempts: None,
+            enable_cache: None,
+            max_cached_keys: None,
+            cache_ttl_seconds: None,
+            allow_insecure_dev_defaults: None,
+        });
+        let backend_error = normalize_configure_request_secrets(&mut backend_change, Some(&existing))
+            .expect_err("changing from the local backend must be rejected");
+        assert_eq!(backend_error, "Changing from the Local KMS backend is not supported");
+    }
 }

rustfs/src/app/context/global.rs

@@ -18,13 +18,12 @@ use super::handles::{
     IamHandle, KmsHandle, default_action_credential_interface, default_boot_time_interface, default_bucket_metadata_interface,
     default_bucket_monitor_interface, default_buffer_config_interface, default_deployment_id_interface,
     default_endpoints_interface, default_expiry_state_interface, default_federated_identity_interface,
-    default_internode_metrics_interface, default_kms_runtime_interface, default_local_node_name_interface,
-    default_lock_client_interface, default_lock_clients_interface, default_notification_system_interface,
-    default_notify_interface, default_outbound_tls_runtime_interface, default_performance_metrics_interface,
-    default_region_interface, default_replication_pool_interface, default_replication_stats_interface,
-    default_runtime_port_interface, default_s3select_db_interface, default_scanner_metrics_interface,
-    default_server_config_interface, default_storage_class_interface, default_tier_config_interface,
-    default_transition_state_interface,
+    default_internode_metrics_interface, default_local_node_name_interface, default_lock_client_interface,
+    default_lock_clients_interface, default_notification_system_interface, default_notify_interface,
+    default_outbound_tls_runtime_interface, default_performance_metrics_interface, default_region_interface,
+    default_replication_pool_interface, default_replication_stats_interface, default_runtime_port_interface,
+    default_s3select_db_interface, default_scanner_metrics_interface, default_server_config_interface,
+    default_storage_class_interface, default_tier_config_interface, default_transition_state_interface,
 };
 use super::interfaces::{
     ActionCredentialInterface, BootTimeInterface, BucketMetadataInterface, BucketMonitorInterface, BufferConfigInterface,
@@ -80,6 +79,7 @@ pub struct AppContext {
 impl AppContext {
     pub fn new(object_store: Arc<ECStore>, iam: Arc<dyn IamInterface>, kms: Arc<dyn KmsInterface>) -> Self {
         let object_data_cache = ObjectDataCacheAdapter::from_env_or_disabled();
+        let kms_runtime = Arc::new(crate::app::context::handles::KmsRuntimeHandle::new(kms.handle()));
         // Let ecstore probe this cache inside get_object_reader, after
         // metadata resolution but before the erasure data read (backlog#802).
         crate::app::object_data_cache::register_object_data_cache_body_hook(Arc::clone(&object_data_cache));
@@ -94,7 +94,7 @@ impl AppContext {
             iam,
             federated_identity: default_federated_identity_interface(),
             kms,
-            kms_runtime: default_kms_runtime_interface(),
+            kms_runtime,
             outbound_tls_runtime: default_outbound_tls_runtime_interface(),
             notify: default_notify_interface(),
             notification_system: default_notification_system_interface(),

rustfs/src/app/context/handles.rs

@@ -128,12 +128,19 @@ impl KmsInterface for KmsHandle {
 }
 
 /// Default KMS runtime interface adapter.
-#[derive(Default)]
-pub struct KmsRuntimeHandle;
+pub struct KmsRuntimeHandle {
+    kms: Option<Arc<KmsServiceManager>>,
+}
+
+impl KmsRuntimeHandle {
+    pub fn new(kms: Arc<KmsServiceManager>) -> Self {
+        Self { kms: Some(kms) }
+    }
+}
 
 impl KmsRuntimeInterface for KmsRuntimeHandle {
     fn service_manager(&self) -> Option<Arc<KmsServiceManager>> {
-        runtime_sources::kms_service_manager()
+        self.kms.clone()
     }
 }
 
@@ -485,7 +492,9 @@ pub fn default_notification_system_interface() -> Arc<dyn NotificationSystemInte
 }
 
 pub fn default_kms_runtime_interface() -> Arc<dyn KmsRuntimeInterface> {
-    Arc::new(KmsRuntimeHandle)
+    Arc::new(KmsRuntimeHandle {
+        kms: runtime_sources::kms_service_manager(),
+    })
 }
 
 pub fn default_outbound_tls_runtime_interface() -> Arc<dyn OutboundTlsRuntimeInterface> {
@@ -607,10 +616,10 @@ pub fn default_buffer_config_interface() -> Arc<dyn BufferConfigInterface> {
 #[cfg(test)]
 mod tests {
     use super::{
-        ServerConfigHandle, default_federated_identity_interface, federated_identity_interface,
-        publish_default_federated_identity_service, runtime_sources,
+        KmsRuntimeHandle, KmsServiceManager, ServerConfigHandle, default_federated_identity_interface,
+        federated_identity_interface, publish_default_federated_identity_service, runtime_sources,
     };
-    use crate::app::context::interfaces::ServerConfigInterface;
+    use crate::app::context::interfaces::{KmsRuntimeInterface, ServerConfigInterface};
     use rustfs_config::server_config::Config;
     use rustfs_iam::{
         federation::{FederatedIdentityRegistry, FederatedIdentityService, oidc::StandardOidcAdapter},
@@ -733,4 +742,19 @@ mod tests {
             "handle B must serve its own credentials"
         );
     }
+
+    #[test]
+    fn kms_runtime_handles_keep_injected_managers_isolated() {
+        let manager_a = Arc::new(KmsServiceManager::new());
+        let manager_b = Arc::new(KmsServiceManager::new());
+        let handle_a = KmsRuntimeHandle::new(manager_a.clone());
+        let handle_b = KmsRuntimeHandle::new(manager_b.clone());
+
+        assert!(Arc::ptr_eq(&handle_a.service_manager().expect("manager A"), &manager_a));
+        assert!(Arc::ptr_eq(&handle_b.service_manager().expect("manager B"), &manager_b));
+        assert!(!Arc::ptr_eq(
+            &handle_a.service_manager().expect("manager A"),
+            &handle_b.service_manager().expect("manager B")
+        ));
+    }
 }

rustfs/src/app/lifecycle_transition_api_test.rs

@@ -60,6 +60,7 @@ use uuid::Uuid;
 static GLOBAL_ENV: OnceLock<(Vec<PathBuf>, Arc<ECStore>)> = OnceLock::new();
 static INIT: Once = Once::new();
 const TRANSITION_WAIT_TIMEOUT: Duration = Duration::from_secs(15);
+const RESTORE_SUSPENDED_WAIT_TIMEOUT: Duration = Duration::from_secs(15);
 const ENV_GET_CODEC_STREAMING_ENABLE: &str = "RUSTFS_GET_CODEC_STREAMING_ENABLE";
 const ENV_GET_CODEC_STREAMING_ROLLOUT: &str = "RUSTFS_GET_CODEC_STREAMING_ROLLOUT";
 const ENV_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED: &str = "RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED";
@@ -161,6 +162,23 @@ async fn create_test_bucket(ecstore: &Arc<ECStore>, bucket_name: &str) {
         .expect("Failed to create test bucket");
 }
 
+async fn suspend_test_bucket(bucket: &str) {
+    DefaultBucketUsecase::from_global()
+        .execute_put_bucket_versioning(build_request(
+            PutBucketVersioningInput::builder()
+                .bucket(bucket.to_string())
+                .versioning_configuration(VersioningConfiguration {
+                    status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
+                    ..Default::default()
+                })
+                .build()
+                .expect("suspended versioning request should build"),
+            Method::PUT,
+        ))
+        .await
+        .expect("bucket versioning should be suspended");
+}
+
 async fn upload_test_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str, data: &[u8]) -> ObjectInfo {
     let mut reader = PutObjReader::from_vec(data.to_vec());
     (**ecstore)
@@ -338,6 +356,45 @@ async fn wait_for_transition(ecstore: &Arc<ECStore>, bucket: &str, object: &str,
     }
 }
 
+async fn wait_for_restore_completion(
+    ecstore: &Arc<ECStore>,
+    backend: &MockWarmBackend,
+    bucket: &str,
+    object: &str,
+    timeout: Duration,
+) -> Result<ObjectInfo, String> {
+    let deadline = tokio::time::Instant::now() + timeout;
+    let mut last_state = None;
+
+    loop {
+        if tokio::time::Instant::now() >= deadline {
+            let tier_gets = backend.get_count().await;
+            let op_log = backend.op_log().await;
+            return Err(format!(
+                "restore copy-back should complete within {timeout:?}; tier_gets={tier_gets}, op_log={op_log:?}; last observed state: {}",
+                last_state.unwrap_or_else(|| "no object info observed".to_string())
+            ));
+        }
+
+        match (**ecstore).get_object_info(bucket, object, &ObjectOptions::default()).await {
+            Ok(info) => {
+                if !info.restore_ongoing && info.restore_expires.is_some() {
+                    return Ok(info);
+                }
+                last_state = Some(format!(
+                    "restore_ongoing={}, restore_expires={:?}, transitioned_status={}",
+                    info.restore_ongoing, info.restore_expires, info.transitioned_object.status
+                ));
+            }
+            Err(err) => {
+                last_state = Some(format!("get_object_info failed: {err}"));
+            }
+        }
+
+        tokio::time::sleep(Duration::from_millis(500)).await;
+    }
+}
+
 // SAFETY: this helper is used only by `#[serial]` tests and runs under the single-threaded Tokio
 // runtime (`worker_threads = 1`), so no concurrent test can mutate process environment during the
 // `env::set_var` / `env::remove_var` window.
@@ -2084,9 +2141,9 @@ async fn restore_object_usecase_reports_ongoing_conflict() {
 
     create_test_bucket(&ecstore, bucket.as_str()).await;
     let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
+    assert!(uploaded.version_id.is_none(), "fixture must create the null version");
     let _ = transition_uploaded_object_directly(&ecstore, bucket.as_str(), object, &tier_name, &uploaded).await;
     backend.clear_op_log().await;
-
     let get_barrier = backend.arm_get_barrier().await;
 
     let restore_request = || RestoreRequest {
@@ -2138,6 +2195,73 @@ async fn restore_object_usecase_reports_ongoing_conflict() {
     get_barrier.release();
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[serial]
+#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#4879"]
+async fn restore_object_usecase_completes_suspended_null_version_in_place() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultObjectUsecase::from_global();
+    let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+    let backend = register_mock_tier(&tier_name).await;
+    let bucket = format!("test-api-restore-suspended-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "test/restore/suspended-null.bin";
+    let payload: Vec<u8> = (0..128 * 1024).map(|i| (i % 251) as u8).collect();
+
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+    let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
+    assert!(uploaded.version_id.is_none(), "fixture must create the null version");
+    let _ = transition_uploaded_object_directly(&ecstore, bucket.as_str(), object, &tier_name, &uploaded).await;
+    suspend_test_bucket(bucket.as_str()).await;
+    backend.clear_op_log().await;
+    let tier_gets_before_restore = backend.get_count().await;
+    let get_barrier = backend.arm_get_barrier().await;
+
+    Box::pin(
+        usecase.execute_restore_object(build_request(
+            RestoreObjectInput::builder()
+                .bucket(bucket.clone())
+                .key(object.to_string())
+                .restore_request(Some(RestoreRequest {
+                    days: Some(1),
+                    description: None,
+                    glacier_job_parameters: None,
+                    output_location: None,
+                    select_parameters: None,
+                    tier: None,
+                    type_: None,
+                }))
+                .build()
+                .expect("restore request should build"),
+            Method::POST,
+        )),
+    )
+    .await
+    .expect("suspended null-version restore should be accepted");
+
+    get_barrier.wait_until_paused().await;
+    get_barrier.release();
+    let completed = wait_for_restore_completion(&ecstore, &backend, bucket.as_str(), object, RESTORE_SUSPENDED_WAIT_TIMEOUT)
+        .await
+        .unwrap_or_else(|err| panic!("{err}"));
+
+    assert!(!completed.restore_ongoing, "the original null version must complete in place");
+    assert!(completed.restore_expires.is_some(), "completed restore must carry an expiry");
+    assert!(
+        completed.version_id.is_none() || completed.version_id.is_some_and(|version_id| version_id.is_nil()),
+        "suspended restore must remain on the null version"
+    );
+    assert_eq!(
+        live_object_version_count(&ecstore, bucket.as_str(), object).await,
+        1,
+        "suspended restore must not create a UUID version"
+    );
+    assert_eq!(
+        backend.get_count().await - tier_gets_before_restore,
+        1,
+        "suspended restore copy-back must fetch the tier exactly once"
+    );
+}
+
 /// backlog#1304: the restore-accept compare-and-set itself, under real
 /// concurrency. Two POST ?restore for the same transitioned object race each
 /// other from separate tasks; the accept guard must let exactly one through —

rustfs/src/app/object_usecase.rs

@@ -116,7 +116,7 @@ use crate::table_catalog;
 use bytes::Bytes;
 use futures::{Stream, StreamExt};
 use http::{HeaderMap, HeaderValue, StatusCode};
-use md5::Context as Md5Context;
+use md5::{Digest as Md5Digest, Md5};
 use metrics::{counter, histogram};
 use pin_project_lite::pin_project;
 use rustfs_concurrency::GetObjectQueueSnapshot;
@@ -801,7 +801,7 @@ pin_project! {
     struct ExtractArchiveEtagReader<R> {
         #[pin]
         inner: R,
-        md5: Md5Context,
+        md5: Md5,
         finished: bool,
         etag: Arc<Mutex<Option<String>>>,
     }
@@ -1854,7 +1854,7 @@ impl<R> ExtractArchiveEtagReader<R> {
     fn new(inner: R, etag: Arc<Mutex<Option<String>>>) -> Self {
         Self {
             inner,
-            md5: Md5Context::new(),
+            md5: Md5::new(),
             finished: false,
             etag,
         }
@@ -1870,11 +1870,11 @@ impl<R: AsyncRead> AsyncRead for ExtractArchiveEtagReader<R> {
             Poll::Ready(Ok(())) => {
                 let filled = &buf.filled()[before..];
                 if !filled.is_empty() {
-                    this.md5.consume(filled);
+                    this.md5.update(filled);
                 } else if !*this.finished {
                     *this.finished = true;
                     if let Ok(mut etag) = this.etag.lock() {
-                        *etag = Some(format!("{:x}", this.md5.clone().finalize()));
+                        *etag = Some(hex_simd::encode_to_string(this.md5.clone().finalize(), hex_simd::AsciiCase::Lower));
                     }
                 }
                 Poll::Ready(Ok(()))
@@ -7783,7 +7783,12 @@ impl DefaultObjectUsecase {
         let bucket_clone = bucket.clone();
         let object_clone = object.clone();
         let rreq_clone = rreq.clone();
-        let version_id_clone = obj_info_.version_id.map(|v| v.to_string());
+        let version_id_clone = obj_info_
+            .version_id
+            .map(|v| v.to_string())
+            .or_else(|| (opts.versioned || opts.version_suspended).then(|| Uuid::nil().to_string()));
+        let versioned = opts.versioned;
+        let version_suspended = opts.version_suspended;
         let mut restore_operation_metadata = HashMap::new();
         if let Some(id) = restore_operation_id {
             insert_str(&mut restore_operation_metadata, SUFFIX_RESTORE_OPERATION_ID, id.to_string());
@@ -7797,6 +7802,8 @@ impl DefaultObjectUsecase {
                     ..Default::default()
                 },
                 version_id: version_id_clone,
+                versioned,
+                version_suspended,
                 user_defined: restore_operation_metadata,
                 ..Default::default()
             };

rustfs/src/storage/minio_generated_read_test.rs

@@ -4,14 +4,13 @@ use std::fs;
 use std::io::Cursor;
 use std::path::{Path, PathBuf};
 
-mod storage_api;
-
+use super::sse::SseObjectEncryptionResolver;
+use super::storage_api::ecstore_test_support::{
+    DiskAPI as _, DiskOption, Endpoint, Erasure, GetObjectReader, ObjectInfo, ObjectOptions, create_bitrot_reader, new_disk,
+};
 use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info};
 use serde::Deserialize;
 use sha2::{Digest, Sha256};
-use storage_api::minio_generated_read::{
-    DiskAPI as _, DiskOption, Endpoint, Erasure, GetObjectReader, ObjectInfo, ObjectOptions, create_bitrot_reader, new_disk,
-};
 use temp_env::async_with_vars;
 use tokio::io::{AsyncReadExt, AsyncWrite};
 
@@ -49,7 +48,7 @@ impl AsyncWrite for VecAsyncWriter {
 fn fixture_root() -> PathBuf {
     std::env::var_os("RUSTFS_MINIO_FIXTURE_ROOT")
         .map(PathBuf::from)
-        .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../rio-v2/tests/fixtures/minio-generated"))
+        .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../crates/rio-v2/tests/fixtures/minio-generated"))
 }
 
 fn case_dir(case_id: &str) -> PathBuf {
@@ -137,12 +136,14 @@ async fn read_fixture_plaintext(encrypted: Vec<u8>, object_info: ObjectInfo, kms
             ("RUSTFS_SSE_S3_MASTER_KEY", None::<String>),
         ],
         async move {
-            let (mut reader, offset, length) = GetObjectReader::new(
+            let resolver = SseObjectEncryptionResolver;
+            let (mut reader, offset, length) = GetObjectReader::new_with_resolver(
                 Box::new(Cursor::new(encrypted)),
                 None,
                 &object_info,
                 &ObjectOptions::default(),
                 &http::HeaderMap::new(),
+                Some(&resolver),
             )
             .await
             .map_err(|err| format!("construct GetObjectReader from MinIO raw fixture: {err:?}"))?;
@@ -219,11 +220,12 @@ async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, fil
             readers.push(reader);
         }
 
-        let erasure = Erasure::new(
+        let erasure = Erasure::try_new(
             file_info.erasure.data_blocks,
             file_info.erasure.parity_blocks,
             file_info.erasure.block_size,
-        );
+        )
+        .expect("fixture erasure geometry");
         let mut writer = VecAsyncWriter::default();
         let (written, err) = erasure.decode(&mut writer, readers, 0, part.size, part.size).await;
         if let Some(err) = err {

rustfs/src/storage/mod.rs

@@ -36,6 +36,8 @@ mod ecfs_extend;
 mod ecfs_test;
 pub(crate) mod head_prefix;
 #[cfg(test)]
+mod minio_generated_read_test;
+#[cfg(test)]
 mod multi_factor_scheduler_integration_test;
 pub(crate) mod runtime_sources;
 #[cfg(test)]
@@ -69,9 +71,9 @@ pub(crate) use storage_api::{
     get_lock_acquire_timeout, get_public_access_block_config, head_prefix_consumer, helper_consumer, init_background_replication,
     init_bucket_metadata_sys, init_ecstore_config, init_local_disks_with_instance_ctx, init_lock_clients,
     is_all_buckets_not_found, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, is_valid_storage_class,
-    load_bucket_metadata, options_consumer, prewarm_local_disk_id_map_with_instance_ctx, read_config, record_replication_proxy,
-    rpc_consumer, runtime_sources_consumer, s3_api_consumer, serialize, set_bucket_metadata, table_catalog_path_hash,
-    to_s3s_etag, topology_snapshot_from_endpoint_pools_with_capabilities, try_migrate_bucket_metadata, try_migrate_iam_config,
+    options_consumer, prewarm_local_disk_id_map_with_instance_ctx, read_config, record_replication_proxy, rpc_consumer,
+    runtime_sources_consumer, s3_api_consumer, serialize, table_catalog_path_hash, to_s3s_etag,
+    topology_snapshot_from_endpoint_pools_with_capabilities, try_migrate_bucket_metadata, try_migrate_iam_config,
     try_migrate_server_config, update_bucket_metadata_config, verify_rpc_signature, wrap_reader,
 };
 

rustfs/src/storage/rpc/node_service.rs

@@ -28,7 +28,9 @@ use crate::storage::storage_api::rpc_consumer::node_service::{
     reload_transition_tier_config,
 };
 use crate::storage::storage_api::runtime_sources_consumer::{EndpointServerPools, runtime_sources};
-use crate::storage::storage_api::{sign_tonic_rpc_response_proof, verify_tonic_canonical_body_digest};
+use crate::storage::storage_api::{
+    sign_tonic_rpc_response_proof, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
+};
 use bytes::Bytes;
 use futures::Stream;
 use futures_util::future::join_all;
@@ -40,6 +42,7 @@ use rustfs_filemeta::MetacacheReader;
 use rustfs_iam::store::UserType;
 use rustfs_lock::LockClient;
 use rustfs_protos::{
+    CanonicalMutationBody,
     models::{PingBody, PingBodyBuilder},
     proto_gen::node_service::{node_service_server::NodeService as Node, *},
 };
@@ -87,6 +90,15 @@ fn signal_service_response(success: bool, error_info: Option<String>) -> Respons
     })
 }
 
+fn verify_node_mutation_body<T: CanonicalMutationBody>(request: &Request<T>, operation: &'static str) -> Result<(), Status> {
+    let canonical_body = request
+        .get_ref()
+        .canonical_body()
+        .map_err(|_| Status::invalid_argument(format!("{operation} request length cannot be represented")))?;
+    verify_tonic_mutation_body_digest(request, &canonical_body)
+        .map_err(|err| Status::permission_denied(format!("{operation} authentication failed: {err}")))
+}
+
 fn supports_dynamic_config_rpc(sub_system: &str) -> bool {
     NOTIFY_SUB_SYSTEMS.contains(&sub_system)
         || matches!(
@@ -343,6 +355,7 @@ mod metrics;
 pub struct NodeService {
     local_peer: LocalPeerS3Client,
     context: Option<Arc<runtime_sources::AppContext>>,
+    snapshot_lease_expiry: disk::SnapshotLeaseExpiryScheduler,
 }
 
 impl std::fmt::Debug for NodeService {
@@ -361,7 +374,11 @@ pub fn make_server() -> NodeService {
 
 pub fn make_server_for_context(context: Option<Arc<runtime_sources::AppContext>>) -> NodeService {
     let local_peer = LocalPeerS3Client::new(None, None);
-    NodeService { local_peer, context }
+    NodeService {
+        local_peer,
+        context,
+        snapshot_lease_expiry: disk::SnapshotLeaseExpiryScheduler::new(),
+    }
 }
 
 #[derive(Clone, Debug, Default)]
@@ -885,6 +902,7 @@ impl Node for NodeService {
     }
 
     async fn heal_bucket(&self, request: Request<HealBucketRequest>) -> Result<Response<HealBucketResponse>, Status> {
+        verify_node_mutation_body(&request, "heal bucket")?;
         self.handle_heal_bucket(request).await
     }
 
@@ -893,6 +911,7 @@ impl Node for NodeService {
     }
 
     async fn make_bucket(&self, request: Request<MakeBucketRequest>) -> Result<Response<MakeBucketResponse>, Status> {
+        verify_node_mutation_body(&request, "make bucket")?;
         self.handle_make_bucket(request).await
     }
 
@@ -901,6 +920,7 @@ impl Node for NodeService {
     }
 
     async fn delete_bucket(&self, request: Request<DeleteBucketRequest>) -> Result<Response<DeleteBucketResponse>, Status> {
+        verify_node_mutation_body(&request, "delete bucket")?;
         self.handle_delete_bucket(request).await
     }
 
@@ -1124,6 +1144,24 @@ impl Node for NodeService {
     async fn delete_paths(&self, request: Request<DeletePathsRequest>) -> Result<Response<DeletePathsResponse>, Status> {
         self.handle_delete_paths(request).await
     }
+    async fn acquire_snapshot_lease(
+        &self,
+        request: Request<SnapshotLeaseRequest>,
+    ) -> Result<Response<SnapshotLeaseResponse>, Status> {
+        self.handle_acquire_snapshot_lease(request).await
+    }
+    async fn renew_snapshot_lease(
+        &self,
+        request: Request<SnapshotLeaseRenewRequest>,
+    ) -> Result<Response<SnapshotLeaseResponse>, Status> {
+        self.handle_renew_snapshot_lease(request).await
+    }
+    async fn release_snapshot_lease(
+        &self,
+        request: Request<SnapshotLeaseReleaseRequest>,
+    ) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
+        self.handle_release_snapshot_lease(request).await
+    }
     async fn read_metadata(&self, request: Request<ReadMetadataRequest>) -> Result<Response<ReadMetadataResponse>, Status> {
         self.handle_read_metadata(request).await
     }
@@ -1172,18 +1210,22 @@ impl Node for NodeService {
     }
 
     async fn lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
+        verify_node_mutation_body(&request, "lock")?;
         self.handle_lock(request).await
     }
 
     async fn un_lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
+        verify_node_mutation_body(&request, "unlock")?;
         self.handle_un_lock(request).await
     }
 
     async fn force_un_lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
+        verify_node_mutation_body(&request, "force unlock")?;
         self.handle_force_un_lock(request).await
     }
 
     async fn refresh(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
+        verify_node_mutation_body(&request, "refresh lock")?;
         self.handle_refresh(request).await
     }
 
@@ -1191,6 +1233,7 @@ impl Node for NodeService {
         &self,
         request: Request<BatchGenerallyLockRequest>,
     ) -> Result<Response<BatchGenerallyLockResponse>, Status> {
+        verify_node_mutation_body(&request, "lock batch")?;
         self.handle_lock_batch(request).await
     }
 
@@ -1198,6 +1241,7 @@ impl Node for NodeService {
         &self,
         request: Request<BatchGenerallyLockRequest>,
     ) -> Result<Response<BatchGenerallyLockResponse>, Status> {
+        verify_node_mutation_body(&request, "unlock batch")?;
         self.handle_un_lock_batch(request).await
     }
 
@@ -1317,6 +1361,7 @@ impl Node for NodeService {
         &self,
         request: Request<LoadBucketMetadataRequest>,
     ) -> Result<Response<LoadBucketMetadataResponse>, Status> {
+        verify_node_mutation_body(&request, "load bucket metadata")?;
         self.handle_load_bucket_metadata(request).await
     }
 
@@ -1324,10 +1369,12 @@ impl Node for NodeService {
         &self,
         request: Request<DeleteBucketMetadataRequest>,
     ) -> Result<Response<DeleteBucketMetadataResponse>, Status> {
+        verify_node_mutation_body(&request, "delete bucket metadata")?;
         self.handle_delete_bucket_metadata(request).await
     }
 
     async fn delete_policy(&self, request: Request<DeletePolicyRequest>) -> Result<Response<DeletePolicyResponse>, Status> {
+        verify_node_mutation_body(&request, "delete policy")?;
         let request = request.into_inner();
         let policy = request.policy_name;
         if policy.is_empty() {
@@ -1358,6 +1405,7 @@ impl Node for NodeService {
     }
 
     async fn load_policy(&self, request: Request<LoadPolicyRequest>) -> Result<Response<LoadPolicyResponse>, Status> {
+        verify_node_mutation_body(&request, "load policy")?;
         let request = request.into_inner();
         let policy = request.policy_name;
         if policy.is_empty() {
@@ -1390,6 +1438,7 @@ impl Node for NodeService {
         &self,
         request: Request<LoadPolicyMappingRequest>,
     ) -> Result<Response<LoadPolicyMappingResponse>, Status> {
+        verify_node_mutation_body(&request, "load policy mapping")?;
         let request = request.into_inner();
         let user_or_group = request.user_or_group;
         if user_or_group.is_empty() {
@@ -1425,6 +1474,7 @@ impl Node for NodeService {
     }
 
     async fn delete_user(&self, request: Request<DeleteUserRequest>) -> Result<Response<DeleteUserResponse>, Status> {
+        verify_node_mutation_body(&request, "delete user")?;
         let request = request.into_inner();
         let access_key = request.access_key;
         if access_key.is_empty() {
@@ -1457,6 +1507,7 @@ impl Node for NodeService {
         &self,
         request: Request<DeleteServiceAccountRequest>,
     ) -> Result<Response<DeleteServiceAccountResponse>, Status> {
+        verify_node_mutation_body(&request, "delete service account")?;
         let request = request.into_inner();
         let access_key = request.access_key;
         if access_key.is_empty() {
@@ -1492,6 +1543,7 @@ impl Node for NodeService {
     }
 
     async fn load_user(&self, request: Request<LoadUserRequest>) -> Result<Response<LoadUserResponse>, Status> {
+        verify_node_mutation_body(&request, "load user")?;
         let request = request.into_inner();
         let access_key = request.access_key;
         let temp = request.temp;
@@ -1529,6 +1581,7 @@ impl Node for NodeService {
         &self,
         request: Request<LoadServiceAccountRequest>,
     ) -> Result<Response<LoadServiceAccountResponse>, Status> {
+        verify_node_mutation_body(&request, "load service account")?;
         let request = request.into_inner();
         let access_key = request.access_key;
         if access_key.is_empty() {
@@ -1560,6 +1613,7 @@ impl Node for NodeService {
     }
 
     async fn load_group(&self, request: Request<LoadGroupRequest>) -> Result<Response<LoadGroupResponse>, Status> {
+        verify_node_mutation_body(&request, "load group")?;
         let request = request.into_inner();
         let group = request.group;
         if group.is_empty() {
@@ -1591,8 +1645,9 @@ impl Node for NodeService {
 
     async fn reload_site_replication_config(
         &self,
-        _request: Request<ReloadSiteReplicationConfigRequest>,
+        request: Request<ReloadSiteReplicationConfigRequest>,
     ) -> Result<Response<ReloadSiteReplicationConfigResponse>, Status> {
+        verify_node_mutation_body(&request, "reload site replication config")?;
         let Some(_store) = self.resolve_object_store() else {
             return Ok(Response::new(ReloadSiteReplicationConfigResponse {
                 success: false,
@@ -1612,6 +1667,7 @@ impl Node for NodeService {
     }
 
     async fn signal_service(&self, request: Request<SignalServiceRequest>) -> Result<Response<SignalServiceResponse>, Status> {
+        verify_node_mutation_body(&request, "signal service")?;
         let request = request.into_inner();
         let vars = match request.vars {
             Some(vars) => vars.value,
@@ -1812,8 +1868,9 @@ impl Node for NodeService {
 
     async fn reload_pool_meta(
         &self,
-        _request: Request<ReloadPoolMetaRequest>,
+        request: Request<ReloadPoolMetaRequest>,
     ) -> Result<Response<ReloadPoolMetaResponse>, Status> {
+        verify_node_mutation_body(&request, "reload pool metadata")?;
         let Some(store) = self.resolve_object_store() else {
             return Ok(Response::new(ReloadPoolMetaResponse {
                 success: false,
@@ -1839,6 +1896,7 @@ impl Node for NodeService {
     }
 
     async fn stop_rebalance(&self, request: Request<StopRebalanceRequest>) -> Result<Response<StopRebalanceResponse>, Status> {
+        verify_node_mutation_body(&request, "stop rebalance")?;
         let Some(store) = self.resolve_object_store() else {
             return Ok(Response::new(StopRebalanceResponse {
                 success: false,
@@ -1859,6 +1917,7 @@ impl Node for NodeService {
         &self,
         request: Request<LoadRebalanceMetaRequest>,
     ) -> Result<Response<LoadRebalanceMetaResponse>, Status> {
+        verify_node_mutation_body(&request, "load rebalance metadata")?;
         let LoadRebalanceMetaRequest { start_rebalance } = request.into_inner();
         let Some(store) = self.resolve_object_store() else {
             log_load_rebalance_meta_rejected!("server_not_initialized", start_rebalance);
@@ -1904,6 +1963,7 @@ impl Node for NodeService {
         &self,
         request: Request<StartDecommissionRequest>,
     ) -> Result<Response<StartDecommissionResponse>, Status> {
+        verify_node_mutation_body(&request, "start decommission")?;
         let Some(store) = runtime_sources::current_object_store_handle() else {
             return Ok(Response::new(StartDecommissionResponse {
                 success: false,
@@ -1935,6 +1995,7 @@ impl Node for NodeService {
         &self,
         request: Request<CancelDecommissionRequest>,
     ) -> Result<Response<CancelDecommissionResponse>, Status> {
+        verify_node_mutation_body(&request, "cancel decommission")?;
         let Some(store) = runtime_sources::current_object_store_handle() else {
             return Ok(Response::new(CancelDecommissionResponse {
                 success: false,
@@ -1967,6 +2028,7 @@ impl Node for NodeService {
         &self,
         request: Request<ClearDecommissionRequest>,
     ) -> Result<Response<ClearDecommissionResponse>, Status> {
+        verify_node_mutation_body(&request, "clear decommission")?;
         let Some(store) = runtime_sources::current_object_store_handle() else {
             return Ok(Response::new(ClearDecommissionResponse {
                 success: false,
@@ -1997,8 +2059,9 @@ impl Node for NodeService {
 
     async fn load_transition_tier_config(
         &self,
-        _request: Request<LoadTransitionTierConfigRequest>,
+        request: Request<LoadTransitionTierConfigRequest>,
     ) -> Result<Response<LoadTransitionTierConfigResponse>, Status> {
+        verify_node_mutation_body(&request, "load transition tier config")?;
         let Some(store) = self.resolve_object_store() else {
             return Ok(Response::new(LoadTransitionTierConfigResponse {
                 success: false,
@@ -2049,21 +2112,24 @@ mod tests {
         sys::NewServiceAccountOpts,
     };
     use rustfs_kms::KmsServiceManager;
+    use rustfs_protos::CanonicalMutationBody as _;
     use rustfs_protos::models::PingBodyBuilder;
     use rustfs_protos::proto_gen::node_service::{
-        BackgroundHealStatusRequest, CheckPartsRequest, DeleteBucketMetadataRequest, DeleteBucketRequest, DeletePathsRequest,
-        DeletePolicyRequest, DeleteRequest, DeleteServiceAccountRequest, DeleteUserRequest, DeleteVersionRequest,
-        DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, DownloadProfileDataRequest, GenerallyLockRequest,
-        GetAllBucketStatsRequest, GetBucketInfoRequest, GetBucketStatsDataRequest, GetCpusRequest, GetMemInfoRequest,
-        GetMetacacheListingRequest, GetMetricsRequest, GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest,
-        GetProcInfoRequest, GetSeLinuxInfoRequest, GetSrMetricsDataRequest, GetSysConfigRequest, GetSysErrorsRequest,
-        HealBucketRequest, HealControlRequest, ListBucketRequest, ListDirRequest, ListVolumesRequest, LoadBucketMetadataRequest,
-        LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest,
+        BackgroundHealStatusRequest, BatchGenerallyLockRequest, CancelDecommissionRequest, CheckPartsRequest,
+        ClearDecommissionRequest, DeleteBucketMetadataRequest, DeleteBucketRequest, DeletePathsRequest, DeletePolicyRequest,
+        DeleteRequest, DeleteServiceAccountRequest, DeleteUserRequest, DeleteVersionRequest, DeleteVersionsRequest,
+        DeleteVolumeRequest, DiskInfoRequest, DownloadProfileDataRequest, GenerallyLockRequest, GetAllBucketStatsRequest,
+        GetBucketInfoRequest, GetBucketStatsDataRequest, GetCpusRequest, GetMemInfoRequest, GetMetacacheListingRequest,
+        GetMetricsRequest, GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest, GetProcInfoRequest, GetSeLinuxInfoRequest,
+        GetSrMetricsDataRequest, GetSysConfigRequest, GetSysErrorsRequest, HealBucketRequest, HealControlRequest,
+        ListBucketRequest, ListDirRequest, ListVolumesRequest, LoadBucketMetadataRequest, LoadGroupRequest,
+        LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest,
         LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, MakeBucketRequest, MakeVolumeRequest,
         MakeVolumesRequest, Mss, PingRequest, PreparePartTransactionRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest,
         ReadVersionRequest, ReadXlRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest,
         RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ServerInfoRequest, SettlePartTransactionRequest,
-        SignalServiceRequest, StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationPeerState,
+        SignalServiceRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
+        StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationPeerState,
         TierMutationPrepareRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
         WriteMetadataRequest, WriteRequest,
         heal_control_service_client::HealControlServiceClient,
@@ -2072,13 +2138,79 @@ mod tests {
         node_service_server::NodeServiceServer,
         tier_mutation_control_service_server::TierMutationControlService as _,
     };
-    use std::{collections::HashMap, sync::Arc};
+    use std::{
+        collections::{HashMap, HashSet},
+        sync::Arc,
+    };
     use time::OffsetDateTime;
     use tokio::net::TcpListener;
     use tokio::time::Duration;
     use tokio_stream::wrappers::TcpListenerStream;
     use tonic::{Request, Response, Status};
 
+    const DISK_MUTATION_RPC_METHODS: [&str; 18] = [
+        "renamedata",
+        "deleteversion",
+        "deleteversions",
+        "writemetadata",
+        "updatemetadata",
+        "writeall",
+        "delete",
+        "deletepaths",
+        "renamefile",
+        "renamepart",
+        "prepareparttransaction",
+        "settleparttransaction",
+        "deletevolume",
+        "makevolume",
+        "makevolumes",
+        "acquiresnapshotlease",
+        "renewsnapshotlease",
+        "releasesnapshotlease",
+    ];
+
+    fn normalized_rpc_method(method: &str) -> String {
+        method.replace('_', "").to_ascii_lowercase()
+    }
+
+    fn node_service_auth_policies() -> HashMap<String, &'static str> {
+        const POLICY_MARKER: &str = "// auth-policy: ";
+
+        let schema = include_str!("../../../../crates/protos/src/node.proto");
+        let service = schema
+            .split_once("service NodeService {")
+            .expect("NodeService must exist in node.proto")
+            .1
+            .split_once("\n}")
+            .expect("NodeService must have a closing brace")
+            .0;
+        let mut policies = HashMap::new();
+        for declaration in service.lines().filter_map(|line| line.trim().strip_prefix("rpc ")) {
+            let (rpc, policy) = declaration
+                .split_once(POLICY_MARKER)
+                .expect("every NodeService RPC must declare an auth-policy beside its proto definition");
+            let method = rpc.split_once('(').expect("RPC declaration must have a request type").0;
+            assert!(
+                policies.insert(normalized_rpc_method(method), policy.trim()).is_none(),
+                "duplicate NodeService RPC {method}",
+            );
+        }
+        assert!(!policies.is_empty(), "NodeService must declare RPC methods");
+        policies
+    }
+
+    #[test]
+    fn every_node_service_rpc_declares_an_auth_policy() {
+        const VALID_POLICIES: [&str; 4] = ["body-bound", "read-only", "streaming", "unimplemented"];
+
+        for (method, policy) in node_service_auth_policies() {
+            assert!(
+                VALID_POLICIES.contains(&policy),
+                "NodeService RPC {method} has unsupported auth-policy {policy:?}",
+            );
+        }
+    }
+
     struct HealControlMockStorage;
 
     #[async_trait::async_trait]
@@ -2462,9 +2594,14 @@ mod tests {
     async fn every_mutating_handler_enforces_its_body_digest() {
         let service = make_server();
         let disk = "http://node-a:9000/data/rustfs0".to_string();
+        let mut covered_methods = HashSet::new();
 
         macro_rules! assert_gated {
             ($method:ident, $msg:expr, $canonical:path) => {{
+                assert!(
+                    covered_methods.insert(normalized_rpc_method(stringify!($method))),
+                    concat!("duplicate disk mutation test for ", stringify!($method)),
+                );
                 let msg = $msg;
 
                 // Correct digest: the gate passes and the handler proceeds to the (unknown) disk
@@ -2580,6 +2717,37 @@ mod tests {
             },
             rustfs_protos::canonical_delete_request_body
         );
+        assert_gated!(
+            acquire_snapshot_lease,
+            SnapshotLeaseRequest {
+                disk: disk.clone(),
+                volume: "v".into(),
+                path: "p".into(),
+                ttl_ms: 60_000,
+            },
+            rustfs_protos::canonical_snapshot_lease_request_body
+        );
+        assert_gated!(
+            renew_snapshot_lease,
+            SnapshotLeaseRenewRequest {
+                disk: disk.clone(),
+                volume: "v".into(),
+                path: "p".into(),
+                token: vec![1; 16].into(),
+                ttl_ms: 60_000,
+            },
+            rustfs_protos::canonical_snapshot_lease_renew_request_body
+        );
+        assert_gated!(
+            release_snapshot_lease,
+            SnapshotLeaseReleaseRequest {
+                disk: disk.clone(),
+                volume: "v".into(),
+                path: "p".into(),
+                token: vec![1; 16].into(),
+            },
+            rustfs_protos::canonical_snapshot_lease_release_request_body
+        );
         assert_gated!(
             delete_paths,
             DeletePathsRequest {
@@ -2659,6 +2827,12 @@ mod tests {
             },
             rustfs_protos::canonical_make_volumes_request_body
         );
+
+        let expected_methods = DISK_MUTATION_RPC_METHODS.into_iter().map(String::from).collect();
+        assert_eq!(
+            covered_methods, expected_methods,
+            "the disk mutation exclusion set must exactly match handlers exercised by the independent digest test",
+        );
     }
 
     #[tokio::test]
@@ -4416,6 +4590,31 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_load_bucket_metadata_failure_skips_scanner_maintenance() {
+        let service = create_test_node_service();
+        let maintenance_generation = rustfs_scanner::scanner_maintenance_generation();
+
+        let request = Request::new(LoadBucketMetadataRequest {
+            bucket: "reload-miss-scanner-guard-bucket".to_string(),
+            scanner_maintenance_change: true,
+        });
+
+        let response = service.load_bucket_metadata(request).await.expect("rpc should reply");
+        let load_response = response.into_inner();
+
+        // Whether the reload fails on missing server state or on the absent
+        // persisted metadata, a failed reload must report failure and must
+        // not tell the scanner a maintenance change landed.
+        assert!(!load_response.success);
+        assert!(load_response.error_info.is_some());
+        assert_eq!(
+            rustfs_scanner::scanner_maintenance_generation(),
+            maintenance_generation,
+            "a failed metadata reload must not advance scanner maintenance activity"
+        );
+    }
+
     #[tokio::test]
     #[ignore = "requires isolated global object layer state"]
     async fn test_load_bucket_metadata_no_object_layer() {
@@ -4726,6 +4925,128 @@ mod tests {
         assert_eq!(signal_response.error_info.as_deref(), Some("unsupported service signal: 99"));
     }
 
+    #[tokio::test]
+    async fn signal_service_body_digest_gate_runs_before_request_handling() {
+        let service = create_test_node_service();
+        let mut vars = HashMap::new();
+        vars.insert(PEER_RESTSIGNAL.to_string(), "99".to_string());
+        vars.insert(PEER_RESTSUB_SYS.to_string(), "scanner".to_string());
+        vars.insert(PEER_RESTDRY_RUN.to_string(), "false".to_string());
+        let message = SignalServiceRequest {
+            vars: Some(Mss { value: vars }),
+        };
+
+        let mut other = message.clone();
+        other
+            .vars
+            .as_mut()
+            .expect("signal vars should exist")
+            .value
+            .insert(PEER_RESTSIGNAL.to_string(), "1".to_string());
+        let mut tampered = Request::new(message.clone());
+        let other_body = other.canonical_body().expect("small signal request should encode");
+        set_tonic_canonical_body_digest(&mut tampered, &other_body).expect("digest metadata should encode");
+        mark_v2_authenticated(&mut tampered);
+        let error = service
+            .signal_service(tampered)
+            .await
+            .expect_err("a tampered signal request must fail before handler logic");
+        assert_eq!(error.code(), tonic::Code::PermissionDenied);
+
+        let mut signed = Request::new(message);
+        let body = signed.get_ref().canonical_body().expect("small signal request should encode");
+        set_tonic_canonical_body_digest(&mut signed, &body).expect("digest metadata should encode");
+        mark_v2_authenticated(&mut signed);
+        let response = service
+            .signal_service(signed)
+            .await
+            .expect("a correctly body-bound signal request must reach handler logic")
+            .into_inner();
+        assert!(!response.success);
+        assert_eq!(response.error_info.as_deref(), Some("unsupported service signal: 99"));
+    }
+
+    #[tokio::test]
+    async fn every_non_disk_mutation_rejects_a_mismatched_body_digest() {
+        let service = create_test_node_service();
+        let mut covered_methods = HashSet::new();
+
+        macro_rules! assert_tampered {
+            ($method:ident, $message:expr) => {{
+                assert!(
+                    covered_methods.insert(normalized_rpc_method(stringify!($method))),
+                    concat!("duplicate non-disk mutation test for ", stringify!($method)),
+                );
+                let mut request = Request::new($message);
+                set_tonic_canonical_body_digest(&mut request, b"unrelated-canonical-body")
+                    .expect("digest metadata should encode");
+                mark_v2_authenticated(&mut request);
+                let error = service
+                    .$method(request)
+                    .await
+                    .expect_err(concat!(stringify!($method), " must reject a mismatched body digest"));
+                assert_eq!(
+                    error.code(),
+                    tonic::Code::PermissionDenied,
+                    concat!(stringify!($method), " must authenticate before any mutation"),
+                );
+            }};
+        }
+
+        assert_tampered!(heal_bucket, HealBucketRequest::default());
+        assert_tampered!(make_bucket, MakeBucketRequest::default());
+        assert_tampered!(delete_bucket, DeleteBucketRequest::default());
+        assert_tampered!(lock, GenerallyLockRequest::default());
+        assert_tampered!(un_lock, GenerallyLockRequest::default());
+        assert_tampered!(force_un_lock, GenerallyLockRequest::default());
+        assert_tampered!(refresh, GenerallyLockRequest::default());
+        assert_tampered!(lock_batch, BatchGenerallyLockRequest::default());
+        assert_tampered!(un_lock_batch, BatchGenerallyLockRequest::default());
+        assert_tampered!(load_bucket_metadata, LoadBucketMetadataRequest::default());
+        assert_tampered!(delete_bucket_metadata, DeleteBucketMetadataRequest::default());
+        assert_tampered!(delete_policy, DeletePolicyRequest::default());
+        assert_tampered!(load_policy, LoadPolicyRequest::default());
+        assert_tampered!(load_policy_mapping, LoadPolicyMappingRequest::default());
+        assert_tampered!(delete_user, DeleteUserRequest::default());
+        assert_tampered!(delete_service_account, DeleteServiceAccountRequest::default());
+        assert_tampered!(load_user, LoadUserRequest::default());
+        assert_tampered!(load_service_account, LoadServiceAccountRequest::default());
+        assert_tampered!(load_group, LoadGroupRequest::default());
+        assert_tampered!(reload_site_replication_config, ReloadSiteReplicationConfigRequest::default());
+        assert_tampered!(signal_service, SignalServiceRequest::default());
+        assert_tampered!(
+            scanner_activity,
+            ScannerActivityRequest {
+                challenge: vec![7; 16].into(),
+                protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,
+                acknowledge_instance_id: String::new(),
+                acknowledge_dirty_usage_generation: 0,
+            }
+        );
+        assert_tampered!(reload_pool_meta, ReloadPoolMetaRequest::default());
+        assert_tampered!(stop_rebalance, StopRebalanceRequest::default());
+        assert_tampered!(load_rebalance_meta, LoadRebalanceMetaRequest::default());
+        assert_tampered!(start_decommission, StartDecommissionRequest::default());
+        assert_tampered!(cancel_decommission, CancelDecommissionRequest::default());
+        assert_tampered!(clear_decommission, ClearDecommissionRequest::default());
+        assert_tampered!(load_transition_tier_config, LoadTransitionTierConfigRequest::default());
+
+        let body_bound_methods: HashSet<_> = node_service_auth_policies()
+            .into_iter()
+            .filter_map(|(method, policy)| (policy == "body-bound").then_some(method))
+            .collect();
+        let disk_methods: HashSet<_> = DISK_MUTATION_RPC_METHODS.into_iter().map(String::from).collect();
+        assert!(
+            disk_methods.is_subset(&body_bound_methods),
+            "every independently tested disk mutation must remain declared body-bound",
+        );
+        let expected_methods: HashSet<_> = body_bound_methods.difference(&disk_methods).cloned().collect();
+        assert_eq!(
+            covered_methods, expected_methods,
+            "proto body-bound non-disk RPCs must exactly match handlers exercised by mismatch tests",
+        );
+    }
+
     #[tokio::test]
     async fn test_scanner_activity_requires_body_bound_auth_before_storage_lookup() {
         let service = create_test_node_service();
@@ -4790,26 +5111,6 @@ mod tests {
             .expect_err("a signed malformed challenge must fail before storage lookup");
         assert_eq!(malformed_current.code(), tonic::Code::InvalidArgument);
 
-        let mut tampered = Request::new(ScannerActivityRequest {
-            challenge: vec![7; 16].into(),
-            protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,
-            acknowledge_instance_id: String::new(),
-            acknowledge_dirty_usage_generation: 0,
-        });
-        let other = ScannerActivityRequest {
-            challenge: vec![8; 16].into(),
-            ..tampered.get_ref().clone()
-        };
-        let other_canonical =
-            rustfs_protos::canonical_scanner_activity_request_body(&other).expect("scanner activity request should encode");
-        set_tonic_canonical_body_digest(&mut tampered, &other_canonical).expect("digest metadata should encode");
-        mark_v2_authenticated(&mut tampered);
-        let tampered = service
-            .scanner_activity(tampered)
-            .await
-            .expect_err("tampered activity challenge must fail before storage lookup");
-        assert_eq!(tampered.code(), tonic::Code::PermissionDenied);
-
         let mut downgraded = Request::new(ScannerActivityRequest {
             challenge: vec![7; 16].into(),
             protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,

rustfs/src/storage/rpc/node_service/bucket.rs

@@ -17,7 +17,7 @@ use crate::storage::storage_api::rpc_consumer::node_service::contract::bucket::{
     BucketOptions, DeleteBucketOptions, MakeBucketOptions,
 };
 use crate::storage::storage_api::rpc_consumer::node_service::{
-    DiskError, StoragePeerS3ClientExt as _, load_bucket_metadata, remove_bucket_metadata, set_bucket_metadata,
+    DiskError, StoragePeerS3ClientExt as _, reload_bucket_metadata, remove_bucket_metadata,
 };
 use rustfs_common::heal_channel::HealOpts;
 use rustfs_protos::proto_gen::node_service::*;
@@ -66,21 +66,15 @@ impl NodeService {
             }));
         }
 
-        let Some(store) = self.resolve_object_store() else {
+        let Some(_store) = self.resolve_object_store() else {
             return Ok(Response::new(LoadBucketMetadataResponse {
                 success: false,
                 error_info: Some("errServerNotInitialized".to_string()),
             }));
         };
 
-        match load_bucket_metadata(store, &bucket).await {
-            Ok(meta) => {
-                if let Err(err) = set_bucket_metadata(bucket.clone(), meta).await {
-                    return Ok(Response::new(LoadBucketMetadataResponse {
-                        success: false,
-                        error_info: Some(err.to_string()),
-                    }));
-                };
+        match reload_bucket_metadata(&bucket).await {
+            Ok(()) => {
                 if scanner_maintenance_change {
                     rustfs_scanner::record_scanner_maintenance_change(&bucket);
                 }

rustfs/src/storage/rpc/node_service/disk.rs

@@ -14,11 +14,12 @@
 
 use super::NodeService;
 use crate::storage::storage_api::rpc_consumer::node_service::{
-    BatchReadVersionReq, BatchReadVersionResp, DeleteOptions, DiskError, DiskInfoOptions, FileInfoVersions, ReadMultipleReq,
-    ReadMultipleResp, ReadOptions, StorageDiskRpcExt as _, UpdateMetadataOpts, validate_batch_read_version_item_count,
+    BatchReadVersionReq, BatchReadVersionResp, DeleteOptions, DiskError, DiskInfoOptions, DiskStore, FileInfoVersions,
+    ReadMultipleReq, ReadMultipleResp, ReadOptions, StorageDiskRpcExt as _, UpdateMetadataOpts,
+    validate_batch_read_version_item_count,
 };
 use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
-use crate::storage::storage_api::{PartTransactionAction, verify_tonic_mutation_body_digest};
+use crate::storage::storage_api::{PartTransactionAction, SnapshotLeaseToken, verify_tonic_mutation_body_digest};
 use bytes::Bytes;
 use rustfs_filemeta::FileInfo;
 use rustfs_io_metrics::internode_metrics::{
@@ -28,13 +29,96 @@ use rustfs_io_metrics::internode_metrics::{
 };
 use rustfs_protos::proto_gen::node_service::*;
 use serde::de::DeserializeOwned;
-use std::io::Cursor;
+use std::{collections::HashMap, io::Cursor, time::Duration};
+use tokio::sync::mpsc;
+use tokio_util::time::DelayQueue;
 use tonic::{Request, Response, Status};
 use tracing::debug;
 
 /// Initial capacity hint (bytes) for msgpack encode buffers, sized to cover a typical single-
 /// version `FileInfo` without repeated growth reallocations. Larger payloads still grow as needed.
 const MSGPACK_ENCODE_CAPACITY_HINT: usize = 512;
+const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1;
+const SNAPSHOT_LEASE_MIN_TTL: Duration = Duration::from_secs(5);
+const SNAPSHOT_LEASE_MAX_TTL: Duration = Duration::from_secs(5 * 60);
+
+struct SnapshotLeaseExpiry {
+    disk: DiskStore,
+    volume: String,
+    path: String,
+    token: SnapshotLeaseToken,
+}
+
+pub(super) struct SnapshotLeaseExpiryScheduler {
+    tx: mpsc::UnboundedSender<SnapshotLeaseExpiryCommand>,
+}
+
+enum SnapshotLeaseExpiryCommand {
+    Schedule(SnapshotLeaseExpiry, Duration),
+    Cancel(SnapshotLeaseToken),
+}
+
+impl SnapshotLeaseExpiryScheduler {
+    pub(super) fn new() -> Self {
+        let (tx, mut rx) = mpsc::unbounded_channel();
+        tokio::spawn(async move {
+            let mut expirations: DelayQueue<SnapshotLeaseExpiry> = DelayQueue::new();
+            let mut keys = HashMap::new();
+            loop {
+                tokio::select! {
+                    Some(command) = rx.recv() => {
+                        match command {
+                            SnapshotLeaseExpiryCommand::Schedule(expiry, ttl) => {
+                                if let Some(key) = keys.remove(&expiry.token) {
+                                    expirations.remove(&key);
+                                }
+                                let token = expiry.token;
+                                let key = expirations.insert(expiry, ttl);
+                                keys.insert(token, key);
+                            }
+                            SnapshotLeaseExpiryCommand::Cancel(token) => {
+                                if let Some(key) = keys.remove(&token) {
+                                    expirations.remove(&key);
+                                }
+                            }
+                        }
+                    }
+                    Some(expired) = futures_util::StreamExt::next(&mut expirations), if !expirations.is_empty() => {
+                        let expiry = expired.into_inner();
+                        keys.remove(&expiry.token);
+                        let _ = expiry
+                            .disk
+                            .release_snapshot_lease(&expiry.volume, &expiry.path, expiry.token)
+                            .await;
+                    }
+                    else => break,
+                }
+            }
+        });
+        Self { tx }
+    }
+
+    fn schedule(&self, expiry: SnapshotLeaseExpiry, ttl: Duration) -> Result<(), SnapshotLeaseExpiry> {
+        self.tx
+            .send(SnapshotLeaseExpiryCommand::Schedule(expiry, ttl))
+            .map_err(|err| match err.0 {
+                SnapshotLeaseExpiryCommand::Schedule(expiry, _) => expiry,
+                SnapshotLeaseExpiryCommand::Cancel(_) => unreachable!(),
+            })
+    }
+
+    fn cancel(&self, token: SnapshotLeaseToken) {
+        let _ = self.tx.send(SnapshotLeaseExpiryCommand::Cancel(token));
+    }
+}
+
+fn snapshot_lease_ttl(ttl_ms: u64) -> Result<Duration, Status> {
+    let ttl = Duration::from_millis(ttl_ms);
+    if !(SNAPSHOT_LEASE_MIN_TTL..=SNAPSHOT_LEASE_MAX_TTL).contains(&ttl) {
+        return Err(Status::invalid_argument("snapshot lease TTL is outside the supported range"));
+    }
+    Ok(ttl)
+}
 
 fn decode_msgpack_or_json<T: DeserializeOwned>(
     binary: &[u8],
@@ -165,6 +249,146 @@ fn encode_batch_read_version_response_payloads(
 }
 
 impl NodeService {
+    pub(super) async fn handle_acquire_snapshot_lease(
+        &self,
+        request: Request<SnapshotLeaseRequest>,
+    ) -> Result<Response<SnapshotLeaseResponse>, Status> {
+        verify_disk_mutation_digest(
+            &request,
+            rustfs_protos::canonical_snapshot_lease_request_body(request.get_ref()),
+            "acquire_snapshot_lease",
+        )?;
+        let request = request.into_inner();
+        let ttl = snapshot_lease_ttl(request.ttl_ms)?;
+        let Some(disk) = self.find_disk(&request.disk).await else {
+            return Ok(Response::new(SnapshotLeaseResponse {
+                success: false,
+                token: Bytes::new(),
+                protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
+                error: Some(DiskError::other("cannot find disk").into()),
+            }));
+        };
+        match disk.acquire_snapshot_lease(&request.volume, &request.path).await {
+            Ok(token) => {
+                if let Err(expiry) = self.snapshot_lease_expiry.schedule(
+                    SnapshotLeaseExpiry {
+                        disk,
+                        volume: request.volume,
+                        path: request.path,
+                        token,
+                    },
+                    ttl,
+                ) {
+                    let _ = expiry
+                        .disk
+                        .release_snapshot_lease(&expiry.volume, &expiry.path, expiry.token)
+                        .await;
+                    return Err(Status::internal("snapshot lease expiry scheduler is unavailable"));
+                }
+                Ok(Response::new(SnapshotLeaseResponse {
+                    success: true,
+                    token: Bytes::copy_from_slice(token.as_bytes()),
+                    protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
+                    error: None,
+                }))
+            }
+            Err(err) => Ok(Response::new(SnapshotLeaseResponse {
+                success: false,
+                token: Bytes::new(),
+                protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
+                error: Some(err.into()),
+            })),
+        }
+    }
+
+    pub(super) async fn handle_renew_snapshot_lease(
+        &self,
+        request: Request<SnapshotLeaseRenewRequest>,
+    ) -> Result<Response<SnapshotLeaseResponse>, Status> {
+        verify_disk_mutation_digest(
+            &request,
+            rustfs_protos::canonical_snapshot_lease_renew_request_body(request.get_ref()),
+            "renew_snapshot_lease",
+        )?;
+        let request = request.into_inner();
+        let ttl = snapshot_lease_ttl(request.ttl_ms)?;
+        let token =
+            SnapshotLeaseToken::from_slice(&request.token).map_err(|_| Status::invalid_argument("invalid lease token"))?;
+        let Some(disk) = self.find_disk(&request.disk).await else {
+            return Ok(Response::new(SnapshotLeaseResponse {
+                success: false,
+                token: Bytes::new(),
+                protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
+                error: Some(DiskError::other("cannot find disk").into()),
+            }));
+        };
+        match disk.renew_snapshot_lease(&request.volume, &request.path, token).await {
+            Ok(renewed) => {
+                if let Err(expiry) = self.snapshot_lease_expiry.schedule(
+                    SnapshotLeaseExpiry {
+                        disk,
+                        volume: request.volume,
+                        path: request.path,
+                        token: renewed,
+                    },
+                    ttl,
+                ) {
+                    let _ = expiry
+                        .disk
+                        .release_snapshot_lease(&expiry.volume, &expiry.path, expiry.token)
+                        .await;
+                    return Err(Status::internal("snapshot lease expiry scheduler is unavailable"));
+                }
+                self.snapshot_lease_expiry.cancel(token);
+                Ok(Response::new(SnapshotLeaseResponse {
+                    success: true,
+                    token: Bytes::copy_from_slice(renewed.as_bytes()),
+                    protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
+                    error: None,
+                }))
+            }
+            Err(err) => Ok(Response::new(SnapshotLeaseResponse {
+                success: false,
+                token: Bytes::new(),
+                protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
+                error: Some(err.into()),
+            })),
+        }
+    }
+
+    pub(super) async fn handle_release_snapshot_lease(
+        &self,
+        request: Request<SnapshotLeaseReleaseRequest>,
+    ) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
+        verify_disk_mutation_digest(
+            &request,
+            rustfs_protos::canonical_snapshot_lease_release_request_body(request.get_ref()),
+            "release_snapshot_lease",
+        )?;
+        let request = request.into_inner();
+        let token =
+            SnapshotLeaseToken::from_slice(&request.token).map_err(|_| Status::invalid_argument("invalid lease token"))?;
+        let Some(disk) = self.find_disk(&request.disk).await else {
+            return Ok(Response::new(SnapshotLeaseMutationResponse {
+                success: false,
+                error: Some(DiskError::other("cannot find disk").into()),
+            }));
+        };
+        match disk.release_snapshot_lease(&request.volume, &request.path, token).await {
+            Ok(()) => {
+                self.snapshot_lease_expiry.cancel(token);
+                Ok(Response::new(SnapshotLeaseMutationResponse {
+                    success: true,
+                    error: None,
+                }))
+            }
+            Err(err) => Ok(Response::new(SnapshotLeaseMutationResponse {
+                success: false,
+                error: Some(err.into()),
+            })),
+        }
+    }
+
     pub(super) async fn handle_disk_info(&self, request: Request<DiskInfoRequest>) -> Result<Response<DiskInfoResponse>, Status> {
         let request = request.into_inner();
         if let Some(disk) = self.find_disk(&request.disk).await {
@@ -1368,8 +1592,9 @@ impl NodeService {
 #[cfg(test)]
 mod tests {
     use super::{
-        compat_response_json, decode_msgpack_or_json, encode_batch_read_version_response_payloads, encode_msgpack,
-        encode_msgpack_named, encode_read_multiple_response_payloads,
+        SNAPSHOT_LEASE_MAX_TTL, SNAPSHOT_LEASE_MIN_TTL, compat_response_json, decode_msgpack_or_json,
+        encode_batch_read_version_response_payloads, encode_msgpack, encode_msgpack_named,
+        encode_read_multiple_response_payloads, snapshot_lease_ttl,
     };
     use crate::storage::storage_api::ReadMultipleResp;
     use crate::storage::storage_api::rpc_consumer::node_service::BatchReadVersionResp;
@@ -1382,6 +1607,14 @@ mod tests {
         count: u32,
     }
 
+    #[test]
+    fn snapshot_lease_ttl_rejects_values_outside_server_bounds() {
+        assert!(snapshot_lease_ttl(4_999).is_err());
+        assert_eq!(snapshot_lease_ttl(5_000).unwrap(), SNAPSHOT_LEASE_MIN_TTL);
+        assert_eq!(snapshot_lease_ttl(300_000).unwrap(), SNAPSHOT_LEASE_MAX_TTL);
+        assert!(snapshot_lease_ttl(300_001).is_err());
+    }
+
     #[test]
     fn decode_msgpack_or_json_prefers_binary_payload() {
         let payload = SamplePayload {

rustfs/src/storage/sse.rs

@@ -70,6 +70,10 @@
 //! ```
 
 use super::StorageError;
+use super::storage_api::ecstore_object::{
+    EncryptionResolutionError, EncryptionResolutionErrorKind, ObjectEncryptionResolver, ReadEncryptionMaterial,
+    ReadEncryptionMode, ReadEncryptionRequest,
+};
 use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
 #[cfg(feature = "rio-v2")]
 use aes_gcm::aead::Payload;
@@ -84,6 +88,7 @@ use chacha20poly1305::ChaCha20Poly1305;
 #[cfg(feature = "rio-v2")]
 use hmac::{Hmac, Mac};
 use http::{HeaderMap, HeaderValue};
+use md5::{Digest as Md5Digest, Md5};
 use rand::Rng;
 #[cfg(feature = "rio-v2")]
 use rand::RngExt;
@@ -135,6 +140,16 @@ const SEALED_KEY_SIZE: usize = DARE_HEADER_SIZE + 32 + DARE_TAG_SIZE;
 #[cfg(feature = "rio-v2")]
 const OBJECT_KEY_DERIVATION_CONTEXT: &[u8] = b"object-encryption-key generation";
 
+fn md5_bytes(input: impl AsRef<[u8]>) -> [u8; 16] {
+    let mut hasher = Md5::new();
+    hasher.update(input.as_ref());
+    hasher.finalize().into()
+}
+
+fn md5_base64(input: impl AsRef<[u8]>) -> String {
+    BASE64_STANDARD.encode(md5_bytes(input))
+}
+
 use super::Error;
 use super::get_bucket_sse_config;
 use crate::error::ApiError;
@@ -144,6 +159,7 @@ use rustfs_utils::http::headers::{
 };
 use rustfs_utils::path::path_join_buf;
 use s3s::dto::{SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId};
+use std::borrow::Cow;
 
 // ============================================================================
 // High-Level SSE Configuration
@@ -641,6 +657,23 @@ pub(crate) fn validate_sse_headers_for_read(metadata: &HashMap<String, String>,
 }
 
 pub(crate) fn map_get_object_reader_error(err: StorageError) -> ApiError {
+    if let StorageError::Io(io_error) = &err
+        && let Some(resolution_error) = io_error
+            .get_ref()
+            .and_then(|source| source.downcast_ref::<EncryptionResolutionError>())
+    {
+        let code = match resolution_error.kind() {
+            EncryptionResolutionErrorKind::InvalidRequest => S3ErrorCode::InvalidRequest,
+            EncryptionResolutionErrorKind::ServiceUnavailable => S3ErrorCode::ServiceUnavailable,
+            _ => S3ErrorCode::InternalError,
+        };
+        return ApiError {
+            code,
+            message: resolution_error.to_string(),
+            source: Some(Box::new(err)),
+        };
+    }
+
     if let Some(message) = map_ssec_get_object_reader_error_message(&err) {
         return ApiError {
             code: S3ErrorCode::InvalidRequest,
@@ -763,6 +796,117 @@ pub enum EncryptionKeyKind {
     Object,
 }
 
+pub(crate) struct SseObjectEncryptionResolver;
+
+#[async_trait]
+impl ObjectEncryptionResolver for SseObjectEncryptionResolver {
+    async fn resolve_read_material(
+        &self,
+        request: ReadEncryptionRequest<'_>,
+    ) -> Result<Option<ReadEncryptionMaterial>, EncryptionResolutionError> {
+        let metadata = normalize_encryption_metadata_case(request.metadata)?;
+        let (customer_algorithm, customer_key, customer_key_md5) =
+            extract_ssec_params_from_headers(request.headers).map_err(map_encryption_resolution_error)?;
+        if let Some(stored_algorithm) = metadata.get("x-amz-server-side-encryption-customer-algorithm") {
+            let request_algorithm = customer_algorithm.as_ref().ok_or_else(|| {
+                map_encryption_resolution_error(ssec_invalid_request(
+                    "The object was stored using a form of Server Side Encryption. \
+                     The correct parameters must be provided to retrieve the object.",
+                ))
+            })?;
+            if stored_algorithm != request_algorithm.as_str() {
+                return Err(map_encryption_resolution_error(ssec_invalid_request(
+                    "The provided encryption parameters did not match the ones used originally to encrypt the object.",
+                )));
+            }
+        }
+        let material = sse_decryption(DecryptionRequest {
+            bucket: request.bucket,
+            key: request.object,
+            metadata: &metadata,
+            sse_customer_key: customer_key.as_ref(),
+            sse_customer_key_md5: customer_key_md5.as_ref(),
+        })
+        .await
+        .map_err(map_encryption_resolution_error)?;
+
+        Ok(material.map(|material| ReadEncryptionMaterial {
+            key_bytes: material.key_bytes,
+            mode: match material.key_kind {
+                EncryptionKeyKind::Direct => ReadEncryptionMode::Direct {
+                    base_nonce: material.base_nonce,
+                },
+                EncryptionKeyKind::Object => ReadEncryptionMode::Object,
+            },
+        }))
+    }
+}
+
+fn normalize_encryption_metadata_case(
+    metadata: &HashMap<String, String>,
+) -> Result<Cow<'_, HashMap<String, String>>, EncryptionResolutionError> {
+    const CANONICAL_KEYS: &[&str] = &[
+        "x-amz-server-side-encryption",
+        "x-amz-server-side-encryption-aws-kms-key-id",
+        "x-amz-server-side-encryption-customer-algorithm",
+        "x-amz-server-side-encryption-customer-key-md5",
+        SSEC_ORIGINAL_SIZE_HEADER,
+        INTERNAL_ENCRYPTION_KEY_ID_HEADER,
+        INTERNAL_ENCRYPTION_KEY_HEADER,
+        INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
+        INTERNAL_ENCRYPTION_IV_HEADER,
+        "x-rustfs-encryption-context",
+        "x-rustfs-encryption-tag",
+        INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER,
+        MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER,
+        MINIO_INTERNAL_ENCRYPTION_IV_HEADER,
+        MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
+        MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER,
+        MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
+        MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER,
+        MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER,
+        MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER,
+    ];
+
+    let needs_normalization = metadata.keys().any(|key| {
+        CANONICAL_KEYS
+            .iter()
+            .any(|canonical| key != canonical && key.eq_ignore_ascii_case(canonical))
+    });
+    if !needs_normalization {
+        return Ok(Cow::Borrowed(metadata));
+    }
+
+    let mut normalized = metadata.clone();
+    for canonical in CANONICAL_KEYS {
+        let mut matching_values = metadata
+            .iter()
+            .filter_map(|(key, value)| key.eq_ignore_ascii_case(canonical).then_some(value));
+        let Some(value) = matching_values.next() else {
+            continue;
+        };
+        if matching_values.any(|candidate| candidate != value) {
+            return Err(EncryptionResolutionError::new(
+                EncryptionResolutionErrorKind::InvalidMetadata,
+                format!("conflicting object encryption metadata for {canonical}"),
+            ));
+        }
+        if !normalized.contains_key(*canonical) {
+            normalized.insert((*canonical).to_string(), value.clone());
+        }
+    }
+    Ok(Cow::Owned(normalized))
+}
+
+fn map_encryption_resolution_error(error: ApiError) -> EncryptionResolutionError {
+    let kind = match error.code {
+        S3ErrorCode::InvalidArgument | S3ErrorCode::InvalidRequest => EncryptionResolutionErrorKind::InvalidRequest,
+        S3ErrorCode::ServiceUnavailable => EncryptionResolutionErrorKind::ServiceUnavailable,
+        _ => EncryptionResolutionErrorKind::DecryptionFailed,
+    };
+    EncryptionResolutionError::new(kind, error.message)
+}
+
 #[derive(Debug, Clone)]
 pub struct ManagedSealedKey {
     #[cfg(feature = "rio-v2")]
@@ -2527,7 +2671,7 @@ pub fn validate_ssec_params(params: SsecParams) -> Result<ValidatedSsecParams, A
         )));
     }
 
-    let computed_md5 = BASE64_STANDARD.encode(md5::compute(&key_bytes).0);
+    let computed_md5 = md5_base64(&key_bytes);
     if computed_md5 != params.key_md5 {
         return Err(ssec_invalid_request(
             "The calculated MD5 hash of the key did not match the hash that was provided.",
@@ -2552,9 +2696,9 @@ pub fn validate_ssec_params(params: SsecParams) -> Result<ValidatedSsecParams, A
 /// 2. Different objects get different nonces
 pub fn generate_ssec_nonce(bucket: &str, key: &str) -> [u8; 12] {
     let nonce_source = format!("{bucket}-{key}");
-    let nonce_hash = md5::compute(nonce_source.as_bytes());
+    let nonce_hash = md5_bytes(nonce_source.as_bytes());
     let mut nonce = [0u8; 12];
-    nonce.copy_from_slice(&nonce_hash.0[..12]);
+    nonce.copy_from_slice(&nonce_hash[..12]);
     nonce
 }
 
@@ -2631,19 +2775,20 @@ fn ssec_invalid_request(message: &str) -> ApiError {
 mod tests {
     use super::{
         ApiError, DataKey, DecryptionRequest, EncryptionKeyKind, EncryptionMaterial, EncryptionRequest,
-        INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_IV_HEADER, INTERNAL_ENCRYPTION_KEY_HEADER,
-        INTERNAL_ENCRYPTION_KEY_ID_HEADER, KmsSseDekProvider, KmsUnavailableError, MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
-        MINIO_INTERNAL_ENCRYPTION_IV_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER,
-        MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER,
-        MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
-        MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER, PrepareEncryptionRequest, SSEC_ORIGINAL_SIZE_HEADER, SSEType,
-        SseDekProvider, SsecParams, StorageError, TestSseDekProvider, apply_managed_decryption_material,
-        apply_managed_encryption_material, encryption_material_to_metadata, extract_server_side_encryption_from_headers,
-        extract_ssec_params_from_headers, extract_ssekms_context_from_headers, generate_ssec_nonce, is_managed_sse,
-        kms_operation_error, map_get_object_reader_error, mark_encrypted_multipart_metadata, normalize_managed_metadata,
-        reset_sse_dek_provider, resolve_effective_kms_key_id, sse_decryption, sse_encryption, sse_prepare_encryption,
-        strip_managed_encryption_metadata, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read,
-        validate_ssec_params, verify_ssec_key_match,
+        EncryptionResolutionErrorKind, INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_IV_HEADER,
+        INTERNAL_ENCRYPTION_KEY_HEADER, INTERNAL_ENCRYPTION_KEY_ID_HEADER, KmsSseDekProvider, KmsUnavailableError,
+        MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER, MINIO_INTERNAL_ENCRYPTION_IV_HEADER,
+        MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER,
+        MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER,
+        MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER,
+        ObjectEncryptionResolver, PrepareEncryptionRequest, ReadEncryptionMode, ReadEncryptionRequest, SSEC_ORIGINAL_SIZE_HEADER,
+        SSEType, SseDekProvider, SseObjectEncryptionResolver, SsecParams, StorageError, TestSseDekProvider,
+        apply_managed_decryption_material, apply_managed_encryption_material, encryption_material_to_metadata,
+        extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, extract_ssekms_context_from_headers,
+        generate_ssec_nonce, is_managed_sse, kms_operation_error, map_get_object_reader_error, mark_encrypted_multipart_metadata,
+        md5_base64, normalize_managed_metadata, reset_sse_dek_provider, resolve_effective_kms_key_id, sse_decryption,
+        sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata, validate_sse_headers_for_read,
+        validate_sse_headers_for_write, validate_ssec_for_read, validate_ssec_params, verify_ssec_key_match,
     };
     #[cfg(feature = "rio-v2")]
     use super::{
@@ -2698,11 +2843,154 @@ mod tests {
     use tokio::sync::Mutex;
 
     static SSE_TEST_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
+    static SSE_TEST_KMS_KEY_DIR: OnceLock<tempfile::TempDir> = OnceLock::new();
 
     async fn lock_sse_test_state() -> tokio::sync::MutexGuard<'static, ()> {
         SSE_TEST_LOCK.get_or_init(|| Mutex::new(())).lock().await
     }
 
+    async fn configure_test_global_local_kms() -> Arc<rustfs_kms::KmsServiceManager> {
+        let key_dir = SSE_TEST_KMS_KEY_DIR.get_or_init(|| tempfile::TempDir::new().expect("create KMS key directory"));
+        let manager = rustfs_kms::init_global_kms_service_manager();
+        manager
+            .reconfigure(rustfs_kms::KmsConfig::local(key_dir.path().to_path_buf()).with_insecure_development_defaults())
+            .await
+            .expect("configure test KMS service");
+        manager
+    }
+
+    #[tokio::test]
+    async fn object_encryption_resolver_returns_ssec_read_material() {
+        let key = [0x31; 32];
+        let key_b64 = BASE64_STANDARD.encode(key);
+        let key_md5 = md5_base64(key);
+        let nonce = [0x42; 12];
+        let metadata = HashMap::from([
+            ("X-Amz-Server-Side-Encryption-Customer-Algorithm".to_string(), "AES256".to_string()),
+            ("X-Amz-Server-Side-Encryption-Customer-Key-Md5".to_string(), key_md5.clone()),
+            ("X-Rustfs-Encryption-Iv".to_string(), BASE64_STANDARD.encode(nonce)),
+        ]);
+        let mut headers = HeaderMap::new();
+        headers.insert("x-amz-server-side-encryption-customer-algorithm", HeaderValue::from_static("AES256"));
+        headers.insert(
+            "x-amz-server-side-encryption-customer-key",
+            HeaderValue::from_str(&key_b64).expect("base64 key is a valid header"),
+        );
+        headers.insert(
+            "x-amz-server-side-encryption-customer-key-md5",
+            HeaderValue::from_str(&key_md5).expect("base64 MD5 is a valid header"),
+        );
+
+        let material = SseObjectEncryptionResolver
+            .resolve_read_material(ReadEncryptionRequest {
+                bucket: "bucket",
+                object: "object",
+                metadata: &metadata,
+                headers: &headers,
+            })
+            .await
+            .expect("SSE-C material should resolve")
+            .expect("SSE-C metadata should produce material");
+
+        assert_eq!(material.key_bytes, key);
+        assert_eq!(material.mode, ReadEncryptionMode::Direct { base_nonce: nonce });
+    }
+
+    #[tokio::test]
+    async fn object_encryption_resolver_rejects_missing_or_invalid_ssec_algorithm() {
+        let key = [0x31; 32];
+        let key_b64 = BASE64_STANDARD.encode(key);
+        let key_md5 = md5_base64(key);
+        let metadata = HashMap::from([
+            ("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
+            ("x-amz-server-side-encryption-customer-key-md5".to_string(), key_md5.clone()),
+        ]);
+
+        for algorithm in [None, Some("AES128")] {
+            let mut headers = HeaderMap::new();
+            if let Some(algorithm) = algorithm {
+                headers.insert("x-amz-server-side-encryption-customer-algorithm", HeaderValue::from_static(algorithm));
+            }
+            headers.insert(
+                "x-amz-server-side-encryption-customer-key",
+                HeaderValue::from_str(&key_b64).expect("base64 key is a valid header"),
+            );
+            headers.insert(
+                "x-amz-server-side-encryption-customer-key-md5",
+                HeaderValue::from_str(&key_md5).expect("base64 MD5 is a valid header"),
+            );
+
+            let result = SseObjectEncryptionResolver
+                .resolve_read_material(ReadEncryptionRequest {
+                    bucket: "bucket",
+                    object: "object",
+                    metadata: &metadata,
+                    headers: &headers,
+                })
+                .await;
+            let error = match result {
+                Err(error) => error,
+                Ok(_) => panic!("missing or invalid SSE-C algorithm must fail closed"),
+            };
+
+            assert_eq!(error.kind(), EncryptionResolutionErrorKind::InvalidRequest);
+        }
+    }
+
+    #[tokio::test]
+    async fn object_encryption_resolver_classifies_missing_ssec_key_as_invalid_request() {
+        let metadata = HashMap::from([("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string())]);
+        let result = SseObjectEncryptionResolver
+            .resolve_read_material(ReadEncryptionRequest {
+                bucket: "bucket",
+                object: "object",
+                metadata: &metadata,
+                headers: &HeaderMap::new(),
+            })
+            .await;
+        let error = match result {
+            Err(error) => error,
+            Ok(_) => panic!("missing SSE-C key must fail closed"),
+        };
+
+        assert_eq!(error.kind(), EncryptionResolutionErrorKind::InvalidRequest);
+    }
+
+    #[tokio::test]
+    async fn object_encryption_resolver_rejects_conflicting_metadata_case_variants() {
+        let metadata = HashMap::from([
+            ("x-rustfs-encryption-key".to_string(), "first".to_string()),
+            ("X-Rustfs-Encryption-Key".to_string(), "second".to_string()),
+        ]);
+        let result = SseObjectEncryptionResolver
+            .resolve_read_material(ReadEncryptionRequest {
+                bucket: "bucket",
+                object: "object",
+                metadata: &metadata,
+                headers: &HeaderMap::new(),
+            })
+            .await;
+        let error = match result {
+            Err(error) => error,
+            Ok(_) => panic!("conflicting metadata aliases must fail closed"),
+        };
+
+        assert_eq!(error.kind(), EncryptionResolutionErrorKind::InvalidMetadata);
+    }
+
+    #[test]
+    fn normalize_encryption_metadata_case_accepts_lowercase_minio_internal_keys() {
+        let lowercase_key = MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_ascii_lowercase();
+        let metadata = HashMap::from([(lowercase_key, "sealed-key".to_string())]);
+
+        let normalized = super::normalize_encryption_metadata_case(&metadata).expect("metadata aliases should normalize");
+
+        assert_eq!(
+            normalized.get(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER),
+            Some(&"sealed-key".to_string())
+        );
+    }
+
     struct UnavailableSseDekProvider;
 
     #[async_trait::async_trait]
@@ -2882,7 +3170,7 @@ mod tests {
     #[test]
     fn test_validate_ssec_params_success() {
         let key = BASE64_STANDARD.encode([42u8; 32]);
-        let key_md5 = BASE64_STANDARD.encode(md5::compute([42u8; 32]).0);
+        let key_md5 = md5_base64([42u8; 32]);
 
         let params = SsecParams {
             algorithm: "AES256".to_string(),
@@ -2899,7 +3187,7 @@ mod tests {
     #[test]
     fn test_validate_ssec_params_wrong_algorithm() {
         let key = BASE64_STANDARD.encode([42u8; 32]);
-        let key_md5 = BASE64_STANDARD.encode(md5::compute([42u8; 32]).0);
+        let key_md5 = md5_base64([42u8; 32]);
 
         let params = SsecParams {
             algorithm: "AES128".to_string(), // Wrong algorithm
@@ -2914,7 +3202,7 @@ mod tests {
     #[test]
     fn test_validate_ssec_params_wrong_key_length() {
         let key = BASE64_STANDARD.encode([42u8; 16]); // Only 16 bytes
-        let key_md5 = BASE64_STANDARD.encode(md5::compute([42u8; 16]).0);
+        let key_md5 = md5_base64([42u8; 16]);
 
         let params = SsecParams {
             algorithm: "AES256".to_string(),
@@ -2946,7 +3234,7 @@ mod tests {
         let bucket = "test-bucket";
         let key = "test-key";
         let sse_key = BASE64_STANDARD.encode([42u8; 32]);
-        let sse_key_md5 = BASE64_STANDARD.encode(md5::compute([42u8; 32]).0);
+        let sse_key_md5 = md5_base64([42u8; 32]);
         let content_size = 1024;
 
         let request_missing_md5 = EncryptionRequest {
@@ -2999,7 +3287,7 @@ mod tests {
     async fn test_sse_prepare_encryption_rejects_partial_ssec_headers() {
         let bucket = "test-bucket";
         let key = "test-key";
-        let sse_key_md5 = BASE64_STANDARD.encode(md5::compute([42u8; 32]).0);
+        let sse_key_md5 = md5_base64([42u8; 32]);
 
         let request_missing_algorithm = PrepareEncryptionRequest {
             bucket,
@@ -3029,7 +3317,7 @@ mod tests {
     async fn test_sse_prepare_encryption_rejects_ssec_headers_without_customer_key() {
         let bucket = "test-bucket";
         let key = "test-key";
-        let sse_key_md5 = BASE64_STANDARD.encode(md5::compute([42u8; 32]).0);
+        let sse_key_md5 = md5_base64([42u8; 32]);
 
         let request = PrepareEncryptionRequest {
             bucket,
@@ -3087,7 +3375,7 @@ mod tests {
         let key = "object";
         let customer_key_bytes = [0x24u8; 32];
         let customer_key = BASE64_STANDARD.encode(customer_key_bytes);
-        let customer_key_md5 = BASE64_STANDARD.encode(md5::compute(customer_key_bytes).0);
+        let customer_key_md5 = md5_base64(customer_key_bytes);
 
         let metadata_one = ssec_direct_put_metadata(bucket, key, &customer_key, &customer_key_md5).await;
         let metadata_two = ssec_direct_put_metadata(bucket, key, &customer_key, &customer_key_md5).await;
@@ -3128,7 +3416,7 @@ mod tests {
         let key = "object";
         let customer_key_bytes = [0x24u8; 32];
         let customer_key = BASE64_STANDARD.encode(customer_key_bytes);
-        let customer_key_md5 = BASE64_STANDARD.encode(md5::compute(customer_key_bytes).0);
+        let customer_key_md5 = md5_base64(customer_key_bytes);
 
         let mut metadata = HashMap::new();
         metadata.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string());
@@ -3160,7 +3448,7 @@ mod tests {
         let key = "object";
         let customer_key_bytes = [0x51u8; 32];
         let customer_key = BASE64_STANDARD.encode(customer_key_bytes);
-        let customer_key_md5 = BASE64_STANDARD.encode(md5::compute(customer_key_bytes).0);
+        let customer_key_md5 = md5_base64(customer_key_bytes);
         let plaintext = b"attack at dawn - sse-c round trip".to_vec();
 
         let metadata = ssec_direct_put_metadata(bucket, key, &customer_key, &customer_key_md5).await;
@@ -3199,7 +3487,7 @@ mod tests {
         let key = "object";
         let customer_key_bytes = [0x33u8; 32];
         let customer_key = BASE64_STANDARD.encode(customer_key_bytes);
-        let customer_key_md5 = BASE64_STANDARD.encode(md5::compute(customer_key_bytes).0);
+        let customer_key_md5 = md5_base64(customer_key_bytes);
 
         let material = sse_prepare_encryption(PrepareEncryptionRequest {
             bucket,
@@ -3261,7 +3549,7 @@ mod tests {
         let key = "test-key";
         let customer_key_bytes = [0x24u8; 32];
         let customer_key = BASE64_STANDARD.encode(customer_key_bytes);
-        let sse_key_md5 = BASE64_STANDARD.encode(md5::compute(customer_key_bytes).0);
+        let sse_key_md5 = md5_base64(customer_key_bytes);
 
         let request = PrepareEncryptionRequest {
             bucket,
@@ -3356,7 +3644,7 @@ mod tests {
         let key = "test-key";
         let content_size = 1024;
         let sse_key = BASE64_STANDARD.encode([42u8; 32]);
-        let sse_key_md5 = BASE64_STANDARD.encode(md5::compute([42u8; 32]).0);
+        let sse_key_md5 = md5_base64([42u8; 32]);
 
         let request = EncryptionRequest {
             bucket,
@@ -3422,18 +3710,11 @@ mod tests {
     #[cfg(feature = "rio-v2")]
     #[tokio::test]
     async fn test_sse_kms_roundtrip_persists_and_uses_minio_context() {
-        use rustfs_kms::config::KmsConfig;
         use rustfs_kms::types::{CreateKeyRequest, KeyUsage};
-        use tempfile::TempDir;
         let _guard = lock_sse_test_state().await;
 
         reset_sse_dek_provider();
-        let manager = rustfs_kms::init_global_kms_service_manager();
-        let temp_dir = TempDir::new().expect("temp dir");
-        manager
-            .reconfigure(KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults())
-            .await
-            .expect("kms reconfigure should succeed");
+        let manager = configure_test_global_local_kms().await;
         manager
             .get_encryption_service()
             .await
@@ -3725,6 +4006,19 @@ mod tests {
 
                 assert_eq!(decrypted.key_kind, EncryptionKeyKind::Object);
                 assert_eq!(decrypted.key_bytes, material.key_bytes);
+
+                let resolved = SseObjectEncryptionResolver
+                    .resolve_read_material(ReadEncryptionRequest {
+                        bucket: "bucket",
+                        object: "object",
+                        metadata: &metadata,
+                        headers: &HeaderMap::new(),
+                    })
+                    .await
+                    .expect("managed resolver")
+                    .expect("managed material");
+                assert_eq!(resolved.mode, ReadEncryptionMode::Object);
+                assert_eq!(resolved.key_bytes, material.key_bytes);
             },
         )
         .await;
@@ -3745,7 +4039,7 @@ mod tests {
     async fn test_ssec_rio_v2_uses_sealed_object_key_metadata_roundtrip() {
         let customer_key_bytes = [0x42u8; 32];
         let customer_key = BASE64_STANDARD.encode(customer_key_bytes);
-        let customer_key_md5 = BASE64_STANDARD.encode(md5::compute(customer_key_bytes).0);
+        let customer_key_md5 = md5_base64(customer_key_bytes);
 
         let material = sse_encryption(EncryptionRequest {
             bucket: "bucket",
@@ -3785,6 +4079,29 @@ mod tests {
 
         assert_eq!(decrypted.key_kind, EncryptionKeyKind::Object);
         assert_eq!(decrypted.key_bytes, material.key_bytes);
+
+        let mut headers = HeaderMap::new();
+        headers.insert("x-amz-server-side-encryption-customer-algorithm", HeaderValue::from_static("AES256"));
+        headers.insert(
+            "x-amz-server-side-encryption-customer-key",
+            HeaderValue::from_str(&customer_key).expect("customer key header"),
+        );
+        headers.insert(
+            "x-amz-server-side-encryption-customer-key-md5",
+            HeaderValue::from_str(&customer_key_md5).expect("customer key MD5 header"),
+        );
+        let resolved = SseObjectEncryptionResolver
+            .resolve_read_material(ReadEncryptionRequest {
+                bucket: "bucket",
+                object: "object",
+                metadata: &metadata,
+                headers: &headers,
+            })
+            .await
+            .expect("SSE-C resolver")
+            .expect("SSE-C material");
+        assert_eq!(resolved.mode, ReadEncryptionMode::Object);
+        assert_eq!(resolved.key_bytes, material.key_bytes);
     }
 
     #[cfg(feature = "rio-v2")]
@@ -3825,7 +4142,7 @@ mod tests {
             ssekms_context: None,
             sse_customer_algorithm: Some("AES256".to_string()),
             sse_customer_key: Some(BASE64_STANDARD.encode(key_bytes)),
-            sse_customer_key_md5: Some(BASE64_STANDARD.encode(md5::compute(key_bytes).0)),
+            sse_customer_key_md5: Some(md5_base64(key_bytes)),
             content_size: 1,
         }
     }
@@ -3834,10 +4151,7 @@ mod tests {
         let key_bytes = [key_byte; 32];
         HashMap::from([
             ("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
-            (
-                "x-amz-server-side-encryption-customer-key-md5".to_string(),
-                BASE64_STANDARD.encode(md5::compute(key_bytes).0),
-            ),
+            ("x-amz-server-side-encryption-customer-key-md5".to_string(), md5_base64(key_bytes)),
         ])
     }
 
@@ -4262,17 +4576,9 @@ mod tests {
 
     #[tokio::test]
     async fn test_managed_decryption_selects_provider_from_persisted_dek() {
-        use rustfs_kms::config::KmsConfig;
-        use tempfile::TempDir;
-
         let _guard = lock_sse_test_state().await;
         reset_sse_dek_provider();
-        let manager = rustfs_kms::init_global_kms_service_manager();
-        let key_dir = TempDir::new().expect("create KMS key directory");
-        manager
-            .reconfigure(KmsConfig::local(key_dir.path().to_path_buf()).with_insecure_development_defaults())
-            .await
-            .expect("start test KMS service");
+        let manager = configure_test_global_local_kms().await;
 
         let local_master_key = [7u8; 32];
         let local_provider = TestSseDekProvider::new_with_key(local_master_key);
@@ -4340,9 +4646,6 @@ mod tests {
     /// the local-provider cache.
     #[tokio::test]
     async fn test_kms_envelope_never_routes_to_cached_local_provider() {
-        use rustfs_kms::config::KmsConfig;
-        use tempfile::TempDir;
-
         let _guard = lock_sse_test_state().await;
         reset_sse_dek_provider();
 
@@ -4354,12 +4657,7 @@ mod tests {
             .expect("write local provider into local cache") = Some(Arc::new(TestSseDekProvider::new_with_key(local_master_key)));
 
         // 2. Start a KMS service (dynamic enable).
-        let manager = rustfs_kms::init_global_kms_service_manager();
-        let key_dir = TempDir::new().expect("create KMS key directory");
-        manager
-            .reconfigure(KmsConfig::local(key_dir.path().to_path_buf()).with_insecure_development_defaults())
-            .await
-            .expect("start test KMS service");
+        let manager = configure_test_global_local_kms().await;
 
         // 3. Construct a KMS JSON envelope — the persisted format of a KMS-wrapped DEK.
         //    is_data_key_envelope() will return true for this payload.
@@ -4446,32 +4744,16 @@ mod tests {
 
     #[tokio::test]
     async fn test_kms_sse_dek_provider_uses_latest_reconfigured_service() {
+        use base64::Engine as _;
         use rustfs_kms::config::KmsConfig;
-        use rustfs_kms::types::{CreateKeyRequest, KeyUsage};
-        use tempfile::TempDir;
         let _guard = lock_sse_test_state().await;
 
-        let manager = rustfs_kms::init_global_kms_service_manager();
+        let manager = Arc::new(rustfs_kms::KmsServiceManager::new());
 
-        let first_dir = TempDir::new().expect("first temp dir");
         manager
-            .reconfigure(KmsConfig::local(first_dir.path().to_path_buf()).with_insecure_development_defaults())
+            .reconfigure(KmsConfig::static_kms("first-key".to_string(), BASE64_STANDARD.encode([0x11; 32])))
             .await
             .expect("first KMS reconfigure should succeed");
-        manager
-            .get_encryption_service()
-            .await
-            .expect("first encryption service should exist")
-            .create_key(CreateKeyRequest {
-                key_name: Some("first-key".to_string()),
-                key_usage: KeyUsage::EncryptDecrypt,
-                description: None,
-                policy: None,
-                tags: HashMap::new(),
-                origin: None,
-            })
-            .await
-            .expect("first key should be created");
 
         let provider = KmsSseDekProvider::new_with_service_manager(manager.clone())
             .await
@@ -4482,25 +4764,10 @@ mod tests {
             .await
             .expect("provider should use the initial service");
 
-        let second_dir = TempDir::new().expect("second temp dir");
         manager
-            .reconfigure(KmsConfig::local(second_dir.path().to_path_buf()).with_insecure_development_defaults())
+            .reconfigure(KmsConfig::static_kms("second-key".to_string(), BASE64_STANDARD.encode([0x22; 32])))
             .await
             .expect("second KMS reconfigure should succeed");
-        manager
-            .get_encryption_service()
-            .await
-            .expect("second encryption service should exist")
-            .create_key(CreateKeyRequest {
-                key_name: Some("second-key".to_string()),
-                key_usage: KeyUsage::EncryptDecrypt,
-                description: None,
-                policy: None,
-                tags: HashMap::new(),
-                origin: None,
-            })
-            .await
-            .expect("second key should be created");
 
         provider
             .generate_sse_dek(&context, "second-key")
@@ -4578,7 +4845,7 @@ mod tests {
     fn test_validate_ssec_for_read_wrong_key() {
         // Key A is used to "encrypt" the object (stored MD5 is from key A).
         let key_a = [42u8; 32];
-        let stored_md5 = BASE64_STANDARD.encode(md5::compute(key_a).0);
+        let stored_md5 = md5_base64(key_a);
 
         let mut metadata = HashMap::new();
         metadata.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string());
@@ -4587,7 +4854,7 @@ mod tests {
         // Key B is a different key; its MD5 won't match stored MD5.
         let key_b = [99u8; 32];
         let key_b_b64 = BASE64_STANDARD.encode(key_b);
-        let key_b_md5 = BASE64_STANDARD.encode(md5::compute(key_b).0);
+        let key_b_md5 = md5_base64(key_b);
 
         let err = validate_ssec_for_read(&metadata, Some(&key_b_b64), Some(&key_b_md5)).unwrap_err();
         assert_eq!(err.code, S3ErrorCode::InvalidRequest);
@@ -4597,7 +4864,7 @@ mod tests {
     fn test_validate_ssec_for_read_correct_key() {
         let key_bytes = [42u8; 32];
         let key_b64 = BASE64_STANDARD.encode(key_bytes);
-        let key_md5 = BASE64_STANDARD.encode(md5::compute(key_bytes).0);
+        let key_md5 = md5_base64(key_bytes);
 
         let mut metadata = HashMap::new();
         metadata.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string());
@@ -4613,7 +4880,7 @@ mod tests {
         // DIFFERENT key. The server must recompute MD5 from the key bytes and
         // reject the request because the recomputed MD5 won't match the header.
         let real_key = [42u8; 32];
-        let stored_md5 = BASE64_STANDARD.encode(md5::compute(real_key).0);
+        let stored_md5 = md5_base64(real_key);
 
         let mut metadata = HashMap::new();
         metadata.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string());
@@ -4715,6 +4982,15 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_map_get_object_reader_error_preserves_typed_service_unavailable() {
+        let resolution_error =
+            super::EncryptionResolutionError::new(EncryptionResolutionErrorKind::ServiceUnavailable, "KMS unavailable");
+        let err = map_get_object_reader_error(StorageError::other(resolution_error));
+        assert_eq!(err.code, S3ErrorCode::ServiceUnavailable);
+        assert_eq!(err.message, "KMS unavailable");
+    }
+
     #[test]
     fn test_map_get_object_reader_error_leaves_non_ssec_errors_unchanged() {
         let err = map_get_object_reader_error(StorageError::other("plain io failure"));
@@ -4725,7 +5001,7 @@ mod tests {
     #[test]
     fn test_validate_ssec_params_returns_invalid_request_on_bad_algorithm() {
         let key = BASE64_STANDARD.encode([42u8; 32]);
-        let key_md5 = BASE64_STANDARD.encode(md5::compute([42u8; 32]).0);
+        let key_md5 = md5_base64([42u8; 32]);
         let params = SsecParams {
             algorithm: "AES128".to_string(),
             key,
@@ -4811,7 +5087,7 @@ mod tests {
             ssekms_context: None,
             sse_customer_algorithm: Some("unsupported-algo".to_string()),
             sse_customer_key: Some(sse_key),
-            sse_customer_key_md5: Some(BASE64_STANDARD.encode(md5::compute([42u8; 32]).0)),
+            sse_customer_key_md5: Some(md5_base64([42u8; 32])),
             content_size: 1024,
         };
         let err = sse_encryption(request_unsupported_algorithm).await.unwrap_err();

rustfs/src/storage/storage_api.rs

@@ -236,8 +236,8 @@ pub(crate) mod rpc_consumer {
             ECStore, Error, FileInfoVersions, LocalPeerS3Client, MetricType, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
             ReadMultipleReq, ReadMultipleResp, ReadOptions, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
             StorageDiskRpcExt, StoragePeerS3ClientExt, UpdateMetadataOpts, all_local_disk_path, collect_local_metrics,
-            find_local_disk_by_ref, get_local_server_property, load_bucket_metadata, reload_transition_tier_config,
-            remove_bucket_metadata, set_bucket_metadata, validate_batch_read_version_item_count,
+            find_local_disk_by_ref, get_local_server_property, reload_bucket_metadata, reload_transition_tier_config,
+            remove_bucket_metadata, validate_batch_read_version_item_count,
         };
         pub(crate) type StorageResult<T> = super::super::Result<T>;
 
@@ -431,7 +431,7 @@ pub(crate) mod ecstore_disk {
     pub(crate) use rustfs_ecstore::api::disk::{
         BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskStore,
         FileInfoVersions, FileReader, FileWriter, OldCurrentSize, PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq,
-        ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
+        ReadMultipleResp, ReadOptions, RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
         get_object_disk_read_timeout, validate_batch_read_version_item_count,
     };
     pub(crate) use rustfs_ecstore::api::disk::{endpoint, error, error_reduce};
@@ -510,12 +510,21 @@ pub(crate) mod ecstore_object {
     #[cfg(test)]
     pub(crate) use rustfs_ecstore::api::object::GetObjectBodySource;
     pub(crate) use rustfs_ecstore::api::object::{
-        GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, ObjectMutationHook, get_object_body_cache_plaintext_len,
-        lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
-        unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
+        EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup,
+        ObjectEncryptionResolver, ObjectMutationHook, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
+        get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
+        register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
     };
 }
 
+#[cfg(all(test, feature = "rio-v2"))]
+pub(crate) mod ecstore_test_support {
+    pub(crate) use rustfs_ecstore::api::bitrot::create_bitrot_reader;
+    pub(crate) use rustfs_ecstore::api::disk::{DiskAPI, DiskOption, endpoint::Endpoint, new_disk};
+    pub(crate) use rustfs_ecstore::api::erasure::Erasure;
+    pub(crate) use rustfs_ecstore::api::object::{GetObjectReader, ObjectInfo, ObjectOptions};
+}
+
 pub(crate) mod ecstore_set_disk {
     pub(crate) use rustfs_ecstore::api::set_disk::{DEFAULT_READ_BUFFER_SIZE, get_lock_acquire_timeout, is_valid_storage_class};
 }
@@ -606,6 +615,7 @@ pub(crate) type ExpiryState = ecstore_bucket::lifecycle::bucket_lifecycle_ops::E
 pub(crate) type FileInfoVersions = ecstore_disk::FileInfoVersions;
 pub(crate) type FileReader = ecstore_disk::FileReader;
 pub(crate) type FileWriter = ecstore_disk::FileWriter;
+pub(crate) type SnapshotLeaseToken = ecstore_disk::SnapshotLeaseToken;
 pub(crate) type FS = super::ecfs::FS;
 pub(crate) type HashReader = ecstore_rio::HashReader;
 pub(crate) type InstanceContext = ecstore_runtime::InstanceContext;
@@ -945,13 +955,21 @@ pub(crate) async fn init_local_disks(endpoint_pools: EndpointServerPools) -> Res
 /// The process-level bootstrap instance context that single-instance startup
 /// threads through the storage foundation (Phase 5 follow-up, backlog#1052).
 pub(crate) fn bootstrap_instance_ctx() -> Arc<InstanceContext> {
-    ecstore_runtime::bootstrap_ctx()
+    let context = ecstore_runtime::bootstrap_ctx();
+    configure_object_encryption_resolver(&context);
+    context
 }
 
 /// Construct a fresh per-server instance context (backlog#1052 S5): a second
 /// embedded server owns its own erasure/region/endpoint/deployment id cells.
 pub(crate) fn new_instance_ctx() -> Arc<InstanceContext> {
-    Arc::new(InstanceContext::new())
+    let context = Arc::new(InstanceContext::new());
+    configure_object_encryption_resolver(&context);
+    context
+}
+
+fn configure_object_encryption_resolver(context: &InstanceContext) {
+    let _ = context.set_object_encryption_resolver(Arc::new(super::sse::SseObjectEncryptionResolver));
 }
 
 pub(crate) fn init_lock_clients(endpoint_pools: EndpointServerPools) {
@@ -1075,6 +1093,9 @@ pub(crate) trait StorageDiskRpcExt {
     ) -> DiskResult<()>;
     async fn read_metadata(&self, volume: &str, path: &str) -> DiskResult<bytes::Bytes>;
     async fn delete_paths(&self, volume: &str, paths: &[String]) -> DiskResult<()>;
+    async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> DiskResult<SnapshotLeaseToken>;
+    async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<SnapshotLeaseToken>;
+    async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<()>;
     async fn stat_volume(&self, volume: &str) -> DiskResult<VolumeInfo>;
     async fn list_volumes(&self) -> DiskResult<Vec<VolumeInfo>>;
     async fn make_volume(&self, volume: &str) -> DiskResult<()>;
@@ -1201,6 +1222,18 @@ where
         ecstore_disk::DiskAPI::delete_paths(self, volume, paths).await
     }
 
+    async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> DiskResult<SnapshotLeaseToken> {
+        ecstore_disk::DiskAPI::acquire_snapshot_lease(self, volume, path).await
+    }
+
+    async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<SnapshotLeaseToken> {
+        ecstore_disk::DiskAPI::renew_snapshot_lease(self, volume, path, token).await
+    }
+
+    async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<()> {
+        ecstore_disk::DiskAPI::release_snapshot_lease(self, volume, path, token).await
+    }
+
     async fn stat_volume(&self, volume: &str) -> DiskResult<VolumeInfo> {
         ecstore_disk::DiskAPI::stat_volume(self, volume).await
     }
@@ -1349,10 +1382,6 @@ impl StoragePeerS3ClientExt for LocalPeerS3Client {
     }
 }
 
-pub(crate) async fn load_bucket_metadata(api: Arc<ECStore>, bucket: &str) -> Result<BucketMetadata> {
-    ecstore_bucket::metadata::load_bucket_metadata(api, bucket).await
-}
-
 #[cfg(test)]
 pub(crate) fn bucket_metadata_sys_initialized() -> bool {
     ecstore_bucket::metadata_sys::get_global_bucket_metadata_sys().is_some()
@@ -1425,10 +1454,15 @@ pub(crate) async fn get_bucket_website_config(bucket: &str) -> Result<(s3s::dto:
     ecstore_bucket::metadata_sys::get_website_config(bucket).await
 }
 
+#[cfg(test)]
 pub(crate) async fn set_bucket_metadata(bucket: String, bm: BucketMetadata) -> Result<()> {
     ecstore_bucket::metadata_sys::set_bucket_metadata(bucket, bm).await
 }
 
+pub(crate) async fn reload_bucket_metadata(bucket: &str) -> Result<()> {
+    ecstore_bucket::metadata_sys::reload_bucket_metadata(bucket).await
+}
+
 pub(crate) async fn remove_bucket_metadata(bucket: &str) -> Result<bool> {
     ecstore_bucket::metadata_sys::remove_bucket_metadata(bucket).await
 }
@@ -1713,7 +1747,7 @@ pub(crate) async fn init_compression_total_memory_from_backend(store: Arc<ECStor
 mod tests {
     use super::{
         apply_active_resync_intents, bucket_targets_metadata_lock_shard, ecstore_bucket, lock_bucket_targets_metadata,
-        scanner_maintenance_config_file,
+        new_instance_ctx, scanner_maintenance_config_file,
     };
     use std::time::Duration;
 
@@ -1744,6 +1778,16 @@ mod tests {
         );
     }
 
+    #[test]
+    fn fresh_instance_context_installs_object_encryption_resolver() {
+        assert!(new_instance_ctx().object_encryption_resolver().is_some());
+    }
+
+    #[test]
+    fn bootstrap_instance_context_installs_object_encryption_resolver() {
+        assert!(super::bootstrap_instance_ctx().object_encryption_resolver().is_some());
+    }
+
     #[test]
     fn scanner_maintenance_config_only_includes_scanner_owned_work() {
         assert!(scanner_maintenance_config_file(ecstore_bucket::metadata::BUCKET_LIFECYCLE_CONFIG));

scripts/run_ecstore_validation_suite.sh

@@ -317,7 +317,7 @@ write_blackbox_matrix() {
 	    printf 'quick\theal degraded erasure disk rebuild\tblack-box\tcargo test --package e2e_test heal_erasure_disk_rebuild_test -- --nocapture\tnone\t%s\n' "$e2e_status"
 	    printf 'quick\tnamespace lock quorum under EC ops\tblack-box\tcargo test --package e2e_test namespace_lock_quorum_test -- --nocapture\tnone\t%s\n' "$e2e_status"
 	    printf 'full\tlegacy bitrot read fixture restore\tfixture\tcargo test -p rustfs-ecstore --test legacy_bitrot_read_test -- --nocapture\tRUSTFS_LEGACY_TEST_ROOT,RUSTFS_LEGACY_TEST_DISK\t%s\n' "$legacy_status"
-	    printf 'full\tMinIO generated encrypted read and negative restore fixture\tfixture\tcargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored --nocapture\tRUSTFS_MINIO_FIXTURE_ROOT,RUSTFS_MINIO_STATIC_KMS_KEY_B64\t%s\n' "$minio_status"
+	    printf 'full\tMinIO generated encrypted read and negative restore fixture\tfixture\tcargo test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored --nocapture\tRUSTFS_MINIO_FIXTURE_ROOT,RUSTFS_MINIO_STATIC_KMS_KEY_B64\t%s\n' "$minio_status"
 	    printf 'full\tS3 multipart range versioning delete subset\tblack-box\tenv TESTEXPR=\"multipart or range or versioning or delete\" DEPLOY_MODE=build MAXFAIL=0 ./scripts/s3-tests/run.sh\tnone\t%s\n' "$s3_status"
 	    printf 'destructive\tdistributed cluster concurrency\tblack-box\tcargo test --package e2e_test cluster_concurrency_test -- --nocapture\tnone\t%s\n' "$destructive_status"
 	    printf 'destructive\tstale multipart cleanup cluster\tblack-box\tcargo test --package e2e_test stale_multipart_cleanup_cluster_test -- --nocapture\tnone\t%s\n' "$destructive_status"
@@ -377,7 +377,7 @@ run_fixture_steps() {
 
 	  if fixture_available; then
 	    run_step "ecstore-minio-generated-read-fixture" \
-	      cargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored --nocapture
+	      cargo test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored --nocapture
 	  elif [[ "$REQUIRE_FIXTURES" == "true" ]]; then
 	    echo "ERROR: $(minio_fixture_missing_reason)" >&2
 	    exit 1