Merge branch 'main' into fix/s3-select-error-semantics

.config/nextest.toml

@@ -218,7 +218,7 @@ test-group = 'ecstore-serial-flaky'
 # the nightly profile derives its set as "the replication module MINUS this
 # allowlist", so any new replication test lands in nightly by default (never
 # silently unrun) until it is explicitly blessed as fast here. Keep the two
-# regexes byte-identical. Count invariant: 20 here + 43 nightly = 63 total
+# regexes byte-identical. Count invariant: 20 here + 49 nightly = 69 total
 # (authority: `cargo nextest list`; docs/testing/e2e-suite-inventory.md).
 # HISTORY (2026-07-11): the 20 fast tests were briefly pulled out of this lane
 # (#4724) because they set a loopback (127.0.0.1) replication target that the
@@ -344,7 +344,7 @@ path = "junit.xml"
 #     object_lambda) — too heavy for the merge budget; they run in ci-7's
 #     nightly 4-node lane.
 #   * replication_extension_test — repl-1 already splits it into the PR
-#     `e2e-smoke` (20 fast) and `e2e-repl-nightly` (43 slow) lanes and reserves
+#     `e2e-smoke` (20 fast) and `e2e-repl-nightly` (49 slow) lanes and reserves
 #     it for those, so e2e-full does not double-run it.
 #   * #[ignore]d tests — nextest skips them by default (no --run-ignored); the
 #     manual-localhost:9000 reliant/policy tests are ci-13's migration.

crates/config/README.md

@@ -97,6 +97,14 @@ Current guidance:
   - enables minimal payload mode for GET health responses (`status`, `ready` only).
 - `RUSTFS_HEALTH_READINESS_CACHE_TTL_MS`
   - TTL for readiness cache evaluation.
+- `RUSTFS_HEALTH_OBJECT_PROGRESS_ENABLE`
+  - withdraws readiness when bounded object read/write stages stop completing while requests remain active.
+  - default is `true`.
+- `RUSTFS_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS`
+  - maximum time without completion in a bounded object stage before readiness is withdrawn.
+  - default is `30000`; `0` uses the default.
+  - the effective value is at least 5 seconds longer than `RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT`.
+  - this readiness SLO is independent of disk read/write failure deadlines and may withdraw traffic before those deadlines expire.
 - `RUSTFS_HEALTH_COMPAT_BUSY_CHECK_ENABLE`
   - enables busy protection behavior for health probes.
   - default is `false`.

crates/config/src/constants/health.rs

@@ -22,6 +22,19 @@ pub const DEFAULT_HEALTH_ENDPOINT_ENABLE: bool = true;
 pub const ENV_HEALTH_READINESS_CACHE_TTL_MS: &str = "RUSTFS_HEALTH_READINESS_CACHE_TTL_MS";
 pub const DEFAULT_HEALTH_READINESS_CACHE_TTL_MS: u64 = 1000;
 
+/// Enable readiness withdrawal when bounded object read/write stages stop
+/// completing while requests remain active.
+pub const ENV_HEALTH_OBJECT_PROGRESS_ENABLE: &str = "RUSTFS_HEALTH_OBJECT_PROGRESS_ENABLE";
+pub const DEFAULT_HEALTH_OBJECT_PROGRESS_ENABLE: bool = true;
+
+/// Requested time without completion in a bounded object stage before local
+/// readiness is withdrawn (milliseconds). A value of `0` uses the default;
+/// runtime adds a safety floor based on the object-lock acquisition timeout.
+pub const ENV_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS: &str = "RUSTFS_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS";
+pub const DEFAULT_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS: u64 = 30_000;
+/// Additional time beyond the configured object-lock acquisition deadline.
+pub const HEALTH_OBJECT_PROGRESS_LOCK_MARGIN_MS: u64 = 5_000;
+
 /// Timeout for cluster health readiness collectors (milliseconds).
 /// This bounds expensive storage and lock quorum checks used by cluster probes.
 pub const ENV_HEALTH_CLUSTER_TIMEOUT_MS: &str = "RUSTFS_HEALTH_CLUSTER_TIMEOUT_MS";

crates/e2e_test/src/fake_s3_target/mod.rs

@@ -30,10 +30,10 @@ use s3s::access::{S3Access, S3AccessContext};
 use s3s::auth::SimpleAuth;
 use s3s::dto::{
     AbortMultipartUploadInput, AbortMultipartUploadOutput, CompleteMultipartUploadInput, CompleteMultipartUploadOutput,
-    CreateMultipartUploadInput, CreateMultipartUploadOutput, DeleteObjectInput, DeleteObjectOutput, ETag,
+    CreateMultipartUploadInput, CreateMultipartUploadOutput, DeleteMarkerEntry, DeleteObjectInput, DeleteObjectOutput, ETag,
     GetBucketVersioningInput, GetBucketVersioningOutput, GetObjectInput, GetObjectOutput, HeadBucketInput, HeadBucketOutput,
-    HeadObjectInput, HeadObjectOutput, PutObjectInput, PutObjectOutput, StreamingBlob, Timestamp, TimestampFormat,
+    HeadObjectInput, HeadObjectOutput, ListObjectVersionsInput, ListObjectVersionsOutput, ObjectVersionId, PutObjectInput,
-    UploadPartInput, UploadPartOutput,
+    PutObjectOutput, StreamingBlob, Timestamp, TimestampFormat, UploadPartInput, UploadPartOutput,
 };
 use s3s::service::{S3Service, S3ServiceBuilder};
 use s3s::validation::{AwsNameValidation, NameValidation};
@@ -91,6 +91,7 @@ pub enum Operation {
     GetObject,
     HeadObject,
     DeleteObject,
+    ListObjectVersions,
     CreateMultipartUpload,
     UploadPart,
     CompleteMultipartUpload,
@@ -109,6 +110,8 @@ pub enum FaultAction {
     /// already have buffered the rest of the current frame; the journal reports
     /// the threshold, and the backend never receives or stores the request.
     DisconnectAfterBytes(usize),
+    /// Apply the request, then close the connection before returning its response.
+    DisconnectAfterResponse,
     /// Drain a request body in fixed-size slices, sleeping after every slice.
     SlowDrain { chunk_bytes: usize, delay: Duration },
     /// Store the request normally but replace the response ETag.
@@ -142,6 +145,7 @@ struct ControlState {
 #[derive(Default)]
 struct StoreState {
     assign_own_version_ids: bool,
+    assign_own_multipart_version_ids: bool,
     buckets: HashMap<String, BucketState>,
     uploads: HashMap<String, MultipartState>,
     total_bytes: usize,
@@ -390,6 +394,16 @@ impl FakeS3Target {
         lock(&self.backend.store).assign_own_version_ids = enabled;
     }
 
+    /// Mint own version ids for the multipart path only — models a target
+    /// that adopts PutObject version ids but not CreateMultipartUpload ones.
+    pub fn assign_own_multipart_version_ids(&self, enabled: bool) {
+        lock(&self.backend.store).assign_own_multipart_version_ids = enabled;
+    }
+
+    pub fn active_multipart_upload_count(&self) -> usize {
+        lock(&self.backend.store).uploads.len()
+    }
+
     /// Queue `times` copies of a fault for one operation.
     pub fn inject(&self, operation: Operation, action: FaultAction, times: usize) {
         if times == 0 {
@@ -659,6 +673,7 @@ fn parse_request(method: &Method, uri: &Uri) -> ParsedRequest {
     let operation = match (method, key.is_some()) {
         (&Method::HEAD, false) => Operation::HeadBucket,
         (&Method::GET, false) if query.contains_key("versioning") => Operation::GetBucketVersioning,
+        (&Method::GET, false) if query.contains_key("versions") => Operation::ListObjectVersions,
         (&Method::PUT, true) if upload_id.is_some() && part_number.is_some() => Operation::UploadPart,
         (&Method::PUT, true) if upload_id.is_some() || query.contains_key("partNumber") => Operation::Unknown,
         (&Method::POST, true) if query.contains_key("uploads") => Operation::CreateMultipartUpload,
@@ -810,7 +825,10 @@ async fn apply_non_body_fault(fault: Option<&RequestFault>, control: &Mutex<Cont
             update_consumed(control, fault.expect("matched fault").sequence, 0);
             Err(scripted_disconnect_error())
         }
-        Some(FaultAction::SlowDrain { .. }) | Some(FaultAction::WrongEtag) | None => Ok(()),
+        Some(FaultAction::SlowDrain { .. })
+        | Some(FaultAction::WrongEtag)
+        | Some(FaultAction::DisconnectAfterResponse)
+        | None => Ok(()),
     }
 }
 
@@ -851,7 +869,7 @@ async fn collect_stream(
             Some(FaultAction::SlowDrain { chunk_bytes, delay }) => {
                 return collect_stream_slow(body, capacity, *chunk_bytes, *delay).await;
             }
-            Some(FaultAction::WrongEtag) | None => {}
+            Some(FaultAction::WrongEtag) | Some(FaultAction::DisconnectAfterResponse) | None => {}
         }
 
         let mut output = BytesMut::with_capacity(capacity);
@@ -913,6 +931,9 @@ fn apply_response_fault<T>(mut response: S3Response<T>, fault: Option<&RequestFa
     if fault.is_some_and(|fault| fault.action == FaultAction::WrongEtag) {
         response.headers.insert(ETAG, HeaderValue::from_static(WRONG_ETAG));
     }
+    if fault.is_some_and(|fault| fault.action == FaultAction::DisconnectAfterResponse) {
+        response.headers.insert(DISCONNECT_HEADER, HeaderValue::from_static("true"));
+    }
     response
 }
 
@@ -1101,6 +1122,63 @@ impl S3 for FakeBackend {
         ))
     }
 
+    /// Prefix + max-keys subset only — enough for the replication-check probe
+    /// key allocation. No pagination markers or delimiter folding.
+    async fn list_object_versions(
+        &self,
+        req: S3Request<ListObjectVersionsInput>,
+    ) -> S3Result<S3Response<ListObjectVersionsOutput>> {
+        let fault = request_fault(&req);
+        apply_non_body_fault(fault.as_ref(), &self.control).await?;
+        let state = lock(&self.store);
+        let Some(bucket_state) = state.buckets.get(&req.input.bucket) else {
+            return Err(s3s::s3_error!(NoSuchBucket, "bucket does not exist"));
+        };
+        let prefix = req.input.prefix.as_deref().unwrap_or_default();
+        let max_keys = req.input.max_keys.unwrap_or(1000).max(0) as usize;
+
+        let mut keys: Vec<&String> = bucket_state.objects.keys().filter(|key| key.starts_with(prefix)).collect();
+        keys.sort();
+
+        let mut versions = Vec::new();
+        let mut delete_markers = Vec::new();
+        'keys: for key in keys {
+            for version in bucket_state.objects[key].iter().rev() {
+                if versions.len() + delete_markers.len() >= max_keys {
+                    break 'keys;
+                }
+                if version.delete_marker {
+                    delete_markers.push(DeleteMarkerEntry {
+                        key: Some(key.clone()),
+                        version_id: Some(ObjectVersionId::from(version.version_id.clone())),
+                        last_modified: Some(version.last_modified.clone()),
+                        ..Default::default()
+                    });
+                } else {
+                    versions.push(s3s::dto::ObjectVersion {
+                        key: Some(key.clone()),
+                        version_id: Some(ObjectVersionId::from(version.version_id.clone())),
+                        last_modified: Some(version.last_modified.clone()),
+                        e_tag: Some(ETag::Strong(version.e_tag.clone())),
+                        size: Some(version.body.len() as i64),
+                        ..Default::default()
+                    });
+                }
+            }
+        }
+        drop(state);
+
+        Ok(apply_response_fault(
+            S3Response::new(ListObjectVersionsOutput {
+                name: Some(req.input.bucket),
+                versions: Some(versions),
+                delete_markers: Some(delete_markers),
+                ..Default::default()
+            }),
+            fault.as_ref(),
+        ))
+    }
+
     async fn put_object(&self, req: S3Request<PutObjectInput>) -> S3Result<S3Response<PutObjectOutput>> {
         let fault = request_fault(&req);
         let _body_permit = timeout(MAX_FAULT_DURATION, Arc::clone(&self.body_limit).acquire_owned())
@@ -1155,7 +1233,7 @@ impl S3 for FakeBackend {
                 content_type: version.content_type,
                 metadata: version.metadata,
                 e_tag: Some(ETag::Strong(version.e_tag)),
-                last_modified: Some(version.last_modified),
+                last_modified: Some(version.last_modified.clone()),
                 version_id: Some(version.version_id),
                 ..Default::default()
             }),
@@ -1177,7 +1255,7 @@ impl S3 for FakeBackend {
                 content_type: version.content_type,
                 metadata: version.metadata,
                 e_tag: Some(ETag::Strong(version.e_tag)),
-                last_modified: Some(version.last_modified),
+                last_modified: Some(version.last_modified.clone()),
                 version_id: Some(version.version_id),
                 ..Default::default()
             }),
@@ -1290,7 +1368,8 @@ impl S3 for FakeBackend {
         let upload_id = Uuid::new_v4().to_string();
         // Read the flag before the mutable borrow of `state.uploads` below
         // (and never re-lock the store: the mutex is not reentrant).
-        let version_id = new_version_id(&headers, state.assign_own_version_ids)?;
+        let mint_own = state.assign_own_version_ids || state.assign_own_multipart_version_ids;
+        let version_id = new_version_id(&headers, mint_own)?;
         state.uploads.insert(
             upload_id.clone(),
             MultipartState {

crates/e2e_test/src/replication_extension_test.rs

@@ -629,6 +629,17 @@ async fn put_bucket_replication_with_delete_statuses(
     target_arn: &str,
     delete_marker_status: &str,
     version_delete_status: Option<&str>,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    put_bucket_replication_with_statuses(env, bucket, target_arn, delete_marker_status, version_delete_status, "Enabled").await
+}
+
+async fn put_bucket_replication_with_statuses(
+    env: &RustFSTestEnvironment,
+    bucket: &str,
+    target_arn: &str,
+    delete_marker_status: &str,
+    version_delete_status: Option<&str>,
+    existing_object_status: &str,
 ) -> Result<(), Box<dyn Error + Send + Sync>> {
     let delete_replication = version_delete_status
         .map(|status| format!("<DeleteReplication><Status>{status}</Status></DeleteReplication>"))
@@ -645,7 +656,7 @@ async fn put_bucket_replication_with_delete_statuses(
     </DeleteMarkerReplication>
     {delete_replication}
     <ExistingObjectReplication>
-      <Status>Enabled</Status>
+      <Status>{existing_object_status}</Status>
     </ExistingObjectReplication>
     <Destination>
       <Bucket>{target_arn}</Bucket>
@@ -2595,6 +2606,9 @@ async fn test_replication_check_succeeds_with_remote_target() -> Result<(), Box<
     assert_eq!(payload["Targets"].as_array().map(Vec::len), Some(1));
     assert_eq!(payload["Targets"][0]["Status"], "OK");
     assert_eq!(payload["Targets"][0]["Phases"]["Put"]["Status"], "OK");
+    // A RustFS target adopts the source version id, so the P1-19
+    // version-identity probe passes.
+    assert_eq!(payload["Targets"][0]["Phases"]["VersionFidelity"]["Status"], "OK");
     assert_eq!(payload["Targets"][0]["Phases"]["DeleteMarker"]["Status"], "OK");
     assert_eq!(payload["Targets"][0]["Phases"]["VersionDelete"]["Status"], "OK");
     assert_eq!(payload["Targets"][0]["Phases"]["Cleanup"]["Status"], "OK");
@@ -7489,6 +7503,334 @@ async fn test_scanner_never_cascades_inbound_replicas() -> TestResult {
     Ok(())
 }
 
+/// P1-19 review follow-up: multipart fixes the target version at initiate
+/// and only reports it on completion, so a target can adopt PutObject
+/// version ids and still mint its own there — the check must not report OK
+/// while multipart deletes and heals would silently miss.
+#[tokio::test]
+#[serial]
+async fn test_replication_check_flags_multipart_only_version_minting_target() -> TestResult {
+    init_logging();
+
+    let target = FakeS3Target::start().await?;
+    let target_bucket = "multipart-fidelity-dst";
+    target.create_bucket(target_bucket);
+    // PutObject mirrors the source version id; CreateMultipartUpload does not.
+    target.assign_own_multipart_version_ids(true);
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    let mut env_vars = replication_fast_env();
+    env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
+    env_vars.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]);
+    source_env.start_rustfs_server_with_env(vec![], &env_vars).await?;
+
+    let source_bucket = "multipart-fidelity-src";
+    let source_client = source_env.create_s3_client();
+    source_client.create_bucket().bucket(source_bucket).send().await?;
+    enable_bucket_versioning(&source_env, source_bucket).await?;
+
+    let target_arn = set_replication_target_with_options(
+        &source_env,
+        source_bucket,
+        ReplicationTargetOptions {
+            endpoint: &target.address(),
+            access_key: FAKE_ACCESS_KEY,
+            secret_key: FAKE_SECRET_KEY,
+            target_bucket,
+            secure: false,
+            skip_tls_verify: false,
+            ca_cert_pem: None,
+        },
+    )
+    .await?;
+    put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
+
+    let response = run_replication_check(&source_env, source_bucket).await?;
+    assert_eq!(response.status(), StatusCode::OK);
+    let payload: serde_json::Value = response.json().await?;
+
+    assert_eq!(payload["Status"], "FAILED", "multipart drift must fail the check: {payload}");
+    let target_report = &payload["Targets"][0];
+    let fidelity = &target_report["Phases"]["VersionFidelity"];
+    assert_eq!(fidelity["Status"], "FAILED", "{payload}");
+    assert_eq!(fidelity["Code"], "BucketRemoteTargetVersionMismatch", "{payload}");
+    assert!(
+        fidelity["Error"]
+            .as_str()
+            .is_some_and(|error| error.contains("CreateMultipartUpload")),
+        "the failure must name the multipart path: {payload}"
+    );
+    // The PutObject leg mirrored, so it is the multipart probe that failed.
+    assert_eq!(target_report["Phases"]["Put"]["Status"], "OK", "{payload}");
+    assert_eq!(target_report["Phases"]["DeleteMarker"]["Status"], "SKIPPED", "{payload}");
+    assert_eq!(target_report["Phases"]["Cleanup"]["Status"], "OK", "{payload}");
+
+    let probe_key = target
+        .requests()
+        .into_iter()
+        .find(|record| record.operation == FakeTargetOperation::PutObject)
+        .and_then(|record| record.key)
+        .ok_or("the probe PUT never reached the fake target")?;
+    assert!(
+        target.stored_versions(target_bucket, &probe_key).is_empty(),
+        "both probe versions must be cleaned up on the mismatching target"
+    );
+
+    target.shutdown().await;
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_replication_check_aborts_failed_multipart_probes() -> TestResult {
+    init_logging();
+
+    let target = FakeS3Target::start().await?;
+    let target_bucket = "multipart-cleanup-dst";
+    target.create_bucket(target_bucket);
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    let mut env_vars = replication_fast_env();
+    env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
+    env_vars.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]);
+    source_env.start_rustfs_server_with_env(vec![], &env_vars).await?;
+
+    let source_bucket = "multipart-cleanup-src";
+    let source_client = source_env.create_s3_client();
+    source_client.create_bucket().bucket(source_bucket).send().await?;
+    enable_bucket_versioning(&source_env, source_bucket).await?;
+
+    let target_arn = set_replication_target_with_options(
+        &source_env,
+        source_bucket,
+        ReplicationTargetOptions {
+            endpoint: &target.address(),
+            access_key: FAKE_ACCESS_KEY,
+            secret_key: FAKE_SECRET_KEY,
+            target_bucket,
+            secure: false,
+            skip_tls_verify: false,
+            ca_cert_pem: None,
+        },
+    )
+    .await?;
+    put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
+
+    for failed_operation in [FakeTargetOperation::UploadPart, FakeTargetOperation::CompleteMultipartUpload] {
+        target.clear_faults();
+        target.take_requests();
+        target.inject(failed_operation, FakeTargetFault::Status(StatusCode::SERVICE_UNAVAILABLE), 16);
+
+        let response = run_replication_check(&source_env, source_bucket).await?;
+        assert_eq!(response.status(), StatusCode::OK);
+        let payload: serde_json::Value = response.json().await?;
+        assert_eq!(
+            payload["Status"], "FAILED",
+            "the injected multipart failure must fail the check: {payload}"
+        );
+
+        let requests = target.requests();
+        assert!(
+            requests.iter().any(|request| {
+                request.operation == failed_operation
+                    && request.fault == Some(FakeTargetFault::Status(StatusCode::SERVICE_UNAVAILABLE))
+            }),
+            "the check must reach the injected {failed_operation:?} failure: {requests:?}"
+        );
+        assert!(
+            requests
+                .iter()
+                .any(|request| request.operation == FakeTargetOperation::AbortMultipartUpload),
+            "the failed {failed_operation:?} probe must be aborted: {requests:?}"
+        );
+        assert_eq!(
+            target.active_multipart_upload_count(),
+            0,
+            "the failed {failed_operation:?} probe must not leave multipart state"
+        );
+        assert_eq!(payload["Targets"][0]["Phases"]["Cleanup"]["Status"], "OK", "{payload}");
+    }
+
+    target.clear_faults();
+    target.take_requests();
+    target.inject(FakeTargetOperation::CompleteMultipartUpload, FakeTargetFault::DisconnectAfterResponse, 16);
+
+    let response = run_replication_check(&source_env, source_bucket).await?;
+    assert_eq!(response.status(), StatusCode::OK);
+    let payload: serde_json::Value = response.json().await?;
+    let target_report = &payload["Targets"][0];
+    assert_eq!(target_report["Status"], "FAILED", "{payload}");
+    assert_eq!(target_report["Phases"]["VersionFidelity"]["Status"], "FAILED", "{payload}");
+    assert_eq!(
+        target_report["Phases"]["Cleanup"]["Status"], "OK",
+        "NoSuchUpload after an ambiguous complete means the multipart artifact is gone: {payload}"
+    );
+    let requests = target.requests();
+    let completed_key = requests
+        .iter()
+        .find(|request| {
+            request.operation == FakeTargetOperation::CompleteMultipartUpload
+                && request.fault == Some(FakeTargetFault::DisconnectAfterResponse)
+        })
+        .and_then(|request| request.key.as_deref())
+        .expect("the scripted complete response disconnect must be observed");
+    assert!(
+        requests
+            .iter()
+            .any(|request| request.operation == FakeTargetOperation::AbortMultipartUpload),
+        "the ambiguous complete must still attempt abort: {requests:?}"
+    );
+    assert_eq!(target.active_multipart_upload_count(), 0);
+    assert!(
+        target.stored_versions(target_bucket, completed_key).is_empty(),
+        "outer cleanup must remove the object committed before the response disconnect"
+    );
+
+    target.clear_faults();
+    target.take_requests();
+    target.inject(FakeTargetOperation::UploadPart, FakeTargetFault::Status(StatusCode::FORBIDDEN), 16);
+    target.inject(
+        FakeTargetOperation::AbortMultipartUpload,
+        FakeTargetFault::Status(StatusCode::SERVICE_UNAVAILABLE),
+        16,
+    );
+
+    let response = run_replication_check(&source_env, source_bucket).await?;
+    assert_eq!(response.status(), StatusCode::OK);
+    let payload: serde_json::Value = response.json().await?;
+    let target_report = &payload["Targets"][0];
+    assert_eq!(target_report["Status"], "FAILED", "{payload}");
+    assert_eq!(target_report["Phases"]["VersionFidelity"]["Status"], "FAILED", "{payload}");
+    assert_eq!(
+        target_report["Phases"]["Cleanup"]["Status"], "FAILED",
+        "an unremoved multipart probe must be reported as a cleanup failure: {payload}"
+    );
+    assert_eq!(
+        target_report["Error"], "s3:ReplicateObject permissions missing for replication user",
+        "the primary multipart error must remain the target error: {payload}"
+    );
+    assert_eq!(
+        target_report["Phases"]["VersionFidelity"]["Error"], "s3:ReplicateObject permissions missing for replication user",
+        "{payload}"
+    );
+    assert_eq!(
+        target_report["Phases"]["Cleanup"]["Error"], "failed to abort multipart replication probe",
+        "{payload}"
+    );
+    assert!(
+        target.requests().iter().any(|request| {
+            request.operation == FakeTargetOperation::AbortMultipartUpload
+                && request.fault == Some(FakeTargetFault::Status(StatusCode::SERVICE_UNAVAILABLE))
+        }),
+        "the abort failure must be observed"
+    );
+    assert_eq!(
+        target.active_multipart_upload_count(),
+        1,
+        "the report must match the retained multipart state"
+    );
+
+    target.shutdown().await;
+    Ok(())
+}
+
+/// P1-19 (backlog#1675): the supported replication contract is targets that
+/// adopt the source version id (RustFS/MinIO semantics). A target that mints
+/// its own version ids silently breaks every version-addressed operation that
+/// follows — version deletes and heal re-drives never match, diverging the
+/// two sides. replication-check must surface this explicitly: a
+/// VersionFidelity phase that compares the probe PUT's response version id
+/// against the sent source version id and fails with
+/// BucketRemoteTargetVersionMismatch — while still cleaning up the probe
+/// object via the version id the target actually assigned.
+#[tokio::test]
+#[serial]
+async fn test_replication_check_flags_version_minting_target() -> TestResult {
+    init_logging();
+
+    let target = FakeS3Target::start().await?;
+    let target_bucket = "version-fidelity-dst";
+    target.create_bucket(target_bucket);
+    target.assign_own_version_ids(true);
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    let mut env_vars = replication_fast_env();
+    env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
+    env_vars.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]);
+    source_env.start_rustfs_server_with_env(vec![], &env_vars).await?;
+
+    let source_bucket = "version-fidelity-src";
+    let source_client = source_env.create_s3_client();
+    source_client.create_bucket().bucket(source_bucket).send().await?;
+    enable_bucket_versioning(&source_env, source_bucket).await?;
+
+    let target_arn = set_replication_target_with_options(
+        &source_env,
+        source_bucket,
+        ReplicationTargetOptions {
+            endpoint: &target.address(),
+            access_key: FAKE_ACCESS_KEY,
+            secret_key: FAKE_SECRET_KEY,
+            target_bucket,
+            secure: false,
+            skip_tls_verify: false,
+            ca_cert_pem: None,
+        },
+    )
+    .await?;
+    put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
+
+    let response = run_replication_check(&source_env, source_bucket).await?;
+    assert_eq!(response.status(), StatusCode::OK);
+    let payload: serde_json::Value = response.json().await?;
+
+    assert_eq!(
+        payload["Status"], "FAILED",
+        "a version-minting target must fail the replication check: {payload}"
+    );
+    let target_report = &payload["Targets"][0];
+    assert_eq!(target_report["Status"], "FAILED", "target must be FAILED: {payload}");
+    let fidelity = &target_report["Phases"]["VersionFidelity"];
+    assert_eq!(fidelity["Status"], "FAILED", "VersionFidelity phase must fail: {payload}");
+    assert_eq!(
+        fidelity["Code"], "BucketRemoteTargetVersionMismatch",
+        "the failure must carry a machine-readable code: {payload}"
+    );
+    // The probe PUT itself succeeded (fidelity is judged from its response);
+    // the later mutation phases are pointless against a drifting target and
+    // must be skipped, but cleanup still runs.
+    assert_eq!(target_report["Phases"]["Put"]["Status"], "OK", "{payload}");
+    assert_eq!(target_report["Phases"]["DeleteMarker"]["Status"], "SKIPPED", "{payload}");
+    assert_eq!(target_report["Phases"]["Cleanup"]["Status"], "OK", "{payload}");
+
+    // The probe PUT must carry the source version as `?versionId=` — the
+    // exact shape live replication uses (P0-5), and the only shape MinIO
+    // consumes. The journal records the query value.
+    let probe_put = target
+        .requests()
+        .into_iter()
+        .find(|record| record.operation == FakeTargetOperation::PutObject)
+        .ok_or("the probe PUT never reached the fake target")?;
+    let probe_query_version = probe_put
+        .version_id
+        .as_deref()
+        .ok_or("the probe PUT must carry a versionId query")?;
+    assert!(
+        uuid::Uuid::parse_str(probe_query_version).is_ok(),
+        "the probe versionId query must be the source uuid, got {probe_query_version}"
+    );
+
+    // No probe residue: cleanup must address the version id the target
+    // actually assigned, not the source id (which never matched anything).
+    let probe_key = probe_put.key.ok_or("probe PUT journal record has no key")?;
+    assert!(
+        target.stored_versions(target_bucket, &probe_key).is_empty(),
+        "the probe object must be cleaned up on the mismatching target"
+    );
+
+    Ok(())
+}
+
 // --- P1-21 (backlog#1675): delayed delete-marker purge failure handling ---
 //
 // The fixtures below wire a versioned source bucket to a FakeS3Target with the
@@ -7818,5 +8160,239 @@ async fn test_delayed_delete_marker_purge_exhaustion_persists_to_mrf_and_replays
     );
 
     target.shutdown().await;
+
+    Ok(())
+}
+
+// --- P1-20 (backlog#1675): scanner existing-object compensation matrix ---
+//
+// Every case below inverts the order used by the rest of this file: objects
+// are written FIRST and the replication rule arrives afterwards, so the only
+// channel that can move the pre-existing objects is the data scanner's
+// existing-object resync pass. Negative cells ("never compensated") are
+// contracts and are asserted over multiple scanner cycles, always next to a
+// replicated control key that proves the scanner and the live path are
+// running — an absent key on a dead scanner proves nothing.
+
+/// Envs + buckets only: versioning, the remote target, and the rule variant
+/// are wired by each test (the null-version case must PUT before the source
+/// bucket becomes versioned). The source runs with FAST_SCANNER_ENV so
+/// existing keys are rescanned within seconds instead of 16 dir cycles.
+async fn build_scanner_compensation_pair(
+    source_bucket: &str,
+    target_bucket: &str,
+) -> Result<(RustFSTestEnvironment, RustFSTestEnvironment), Box<dyn Error + Send + Sync>> {
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    let mut source_process_env = replication_fast_env();
+    source_process_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
+    source_process_env.extend_from_slice(FAST_SCANNER_ENV);
+    source_env.start_rustfs_server_with_env(vec![], &source_process_env).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    source_env
+        .create_s3_client()
+        .create_bucket()
+        .bucket(source_bucket)
+        .send()
+        .await?;
+    target_env
+        .create_s3_client()
+        .create_bucket()
+        .bucket(target_bucket)
+        .send()
+        .await?;
+
+    Ok((source_env, target_env))
+}
+
+/// P1-20: objects that already exist when a rule with
+/// ExistingObjectReplication=Enabled arrives are compensated by the scanner's
+/// existing-object resync pass, whatever wrote them — plain PUT, CopyObject,
+/// or Snowball auto-extract. The pinned exception is a null-version object
+/// (written before the bucket became versioned): the scanner heal gate skips
+/// nil-version objects entirely (`scanner_folder.rs` heal_replication), so it
+/// must NEVER be compensated.
+#[tokio::test]
+#[serial]
+async fn test_scanner_compensates_existing_objects_across_write_paths() -> TestResult {
+    init_logging();
+    let source_bucket = "scanner-comp-src";
+    let target_bucket = "scanner-comp-dst";
+    let (source_env, target_env) = build_scanner_compensation_pair(source_bucket, target_bucket).await?;
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+
+    // Null-version cell: PUT before versioning; the object keeps the nil
+    // version id forever.
+    let null_key = "pre-versioning-null.txt";
+    source_client
+        .put_object()
+        .bucket(source_bucket)
+        .key(null_key)
+        .body(ByteStream::from_static(b"null version payload"))
+        .send()
+        .await?;
+
+    enable_bucket_versioning(&source_env, source_bucket).await?;
+    enable_bucket_versioning(&target_env, target_bucket).await?;
+
+    // Pre-existing objects from three write paths, all before any replication
+    // config exists (their replication status stays Empty).
+    let plain_key = "existing-plain.txt";
+    let plain_payload = "existing plain payload";
+    source_client
+        .put_object()
+        .bucket(source_bucket)
+        .key(plain_key)
+        .body(ByteStream::from_static(plain_payload.as_bytes()))
+        .send()
+        .await?;
+
+    let copy_key = "existing-copy.txt";
+    source_client
+        .copy_object()
+        .bucket(source_bucket)
+        .key(copy_key)
+        .copy_source(format!("{source_bucket}/{plain_key}"))
+        .send()
+        .await?;
+
+    let member_key = "snowball/existing-member.txt";
+    let member_payload: &[u8] = b"existing snowball member payload";
+    let mut builder = tokio_tar::Builder::new(std::io::Cursor::new(Vec::new()));
+    let mut header = tokio_tar::Header::new_gnu();
+    header.set_size(member_payload.len() as u64);
+    header.set_mode(0o644);
+    header.set_cksum();
+    builder
+        .append_data(&mut header, member_key, std::io::Cursor::new(member_payload))
+        .await?;
+    let archive = builder.into_inner().await?.into_inner();
+    source_client
+        .put_object()
+        .bucket(source_bucket)
+        .key("existing-members.tar")
+        .metadata("Snowball-Auto-Extract", "true")
+        .body(ByteStream::from(archive))
+        .send()
+        .await?;
+    // The extracted member must exist locally before the rule arrives, or it
+    // would replicate through the live path instead of the scanner.
+    let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
+    loop {
+        if source_client
+            .head_object()
+            .bucket(source_bucket)
+            .key(member_key)
+            .send()
+            .await
+            .is_ok()
+        {
+            break;
+        }
+        if tokio::time::Instant::now() >= deadline {
+            return Err("snowball member was never extracted on the source".into());
+        }
+        sleep(Duration::from_millis(200)).await;
+    }
+
+    // Only now wire the remote target and the Enabled rule.
+    let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
+    put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
+
+    // Control key written after the rule replicates through the live path.
+    let control_key = "control-live.txt";
+    let control_payload = "control live payload";
+    source_client
+        .put_object()
+        .bucket(source_bucket)
+        .key(control_key)
+        .body(ByteStream::from_static(control_payload.as_bytes()))
+        .send()
+        .await?;
+    wait_for_replicated_object(&target_client, target_bucket, control_key, control_payload).await?;
+
+    // Scanner compensation for each pre-existing write path.
+    wait_for_replicated_object(&target_client, target_bucket, plain_key, plain_payload).await?;
+    wait_for_replicated_object(&target_client, target_bucket, copy_key, plain_payload).await?;
+    wait_for_replicated_object(&target_client, target_bucket, member_key, std::str::from_utf8(member_payload)?).await?;
+
+    // Null-version contract: with every sibling compensated (scanner proven
+    // live), the nil-version object must stay absent across further cycles.
+    assert_replication_key_absent(&target_client, target_bucket, null_key, Duration::from_secs(6)).await?;
+
+    Ok(())
+}
+
+/// P1-20: ExistingObjectReplication=Disabled is a contract, not a delay — the
+/// scanner must NEVER compensate objects that predate the rule, while objects
+/// written after the rule replicate normally (the setting only gates the
+/// existing-object resync path).
+#[tokio::test]
+#[serial]
+async fn test_scanner_never_compensates_when_existing_object_replication_disabled() -> TestResult {
+    init_logging();
+    let source_bucket = "scanner-disabled-src";
+    let target_bucket = "scanner-disabled-dst";
+    let (source_env, mut target_env) = build_scanner_compensation_pair(source_bucket, target_bucket).await?;
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+
+    enable_bucket_versioning(&source_env, source_bucket).await?;
+    enable_bucket_versioning(&target_env, target_bucket).await?;
+
+    let existing_key = "existing-disabled.txt";
+    source_client
+        .put_object()
+        .bucket(source_bucket)
+        .key(existing_key)
+        .body(ByteStream::from_static(b"existing disabled payload"))
+        .send()
+        .await?;
+
+    let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
+    put_bucket_replication_with_statuses(&source_env, source_bucket, &target_arn, "Enabled", None, "Disabled").await?;
+
+    // The live path is unaffected by the Disabled existing-object setting.
+    let control_key = "control-live.txt";
+    let control_payload = "control live payload";
+    source_client
+        .put_object()
+        .bucket(source_bucket)
+        .key(control_key)
+        .body(ByteStream::from_static(control_payload.as_bytes()))
+        .send()
+        .await?;
+    wait_for_replicated_object(&target_client, target_bucket, control_key, control_payload).await?;
+
+    // Scanner-only witness. A live-path control key alone would let this test
+    // pass while the existing-object scanner is disabled or wedged, so make
+    // the scanner itself observable: an object whose replication FAILED while
+    // the target was down can only be re-driven by the data scanner's
+    // replication heal pass (see FAST_SCANNER_ENV), and that pass is NOT
+    // gated by ExistingObjectReplication. The witness lives in the same
+    // bucket and prefix as the pre-existing key, so a heal pass that reached
+    // it necessarily walked the pre-existing key in the same scan.
+    let witness_key = "scanner-witness.txt";
+    let witness_payload = "scanner witness payload";
+    target_env.stop_server();
+    source_client
+        .put_object()
+        .bucket(source_bucket)
+        .key(witness_key)
+        .body(ByteStream::from_static(witness_payload.as_bytes()))
+        .send()
+        .await?;
+    wait_for_source_replication_status(&source_client, source_bucket, witness_key, "FAILED", false).await?;
+    target_env.restart_server_preserving_data(vec![], &[]).await?;
+    let target_client = target_env.create_s3_client();
+    wait_for_replicated_object(&target_client, target_bucket, witness_key, witness_payload).await?;
+
+    // The scanner demonstrably swept this bucket; the pre-existing key must
+    // still be absent, and stay absent over further cycles.
+    assert_replication_key_absent(&target_client, target_bucket, existing_key, Duration::from_secs(6)).await?;
+
     Ok(())
 }

crates/ecstore/src/api/mod.rs

@@ -32,7 +32,7 @@ pub mod bucket {
     pub mod bucket_target_sys {
         pub use crate::bucket::bucket_target_sys::{
             AdvancedPutOptions, BucketTargetError, BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError,
-            TargetClient,
+            TargetClient, append_version_id_query,
         };
     }
 
@@ -281,7 +281,7 @@ pub mod config {
     pub mod com {
         pub use crate::config::com::{
             COMMA_SEPARATED_LISTS, CONFIG_PREFIX, ENV_CONFIG_RECOVER_ON_CORRUPTION, STORAGE_CLASS_SUB_SYS,
-            ServerConfigCorruptError, ServerConfigSaveResult, ServerConfigSnapshot, delete_config,
+            ServerConfigCorruptError, ServerConfigSaveResult, ServerConfigSnapshot, delete_config, delete_config_no_lock,
             is_server_config_corrupt_error, lookup_configs, read_config, read_config_no_lock, read_config_with_metadata,
             read_config_without_migrate, read_config_without_migrate_no_lock, read_existing_server_config_no_lock,
             read_server_config_snapshot, save_config, save_config_no_lock, save_config_with_opts, save_server_config,

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -1450,7 +1450,7 @@ fn resolve_put_api_version_id(source_version_id: &str) -> Option<&str> {
 /// member, so the query is spliced in via `map_request`, which runs at
 /// `modify_before_signing`: the parameter becomes part of the SigV4 canonical
 /// request.
-fn append_version_id_query(uri: &str, version_id: &str) -> String {
+pub fn append_version_id_query(uri: &str, version_id: &str) -> String {
     let separator = if uri.contains('?') { '&' } else { '?' };
     format!("{uri}{separator}versionId={}", urlencoding::encode(version_id))
 }
@@ -1861,6 +1861,9 @@ impl TargetClient {
         }
     }
 
+    /// On success returns the version id the target assigned (from
+    /// `x-amz-version-id`), letting callers audit the version-identity
+    /// contract — a target that adopts the source version echoes it back.
     pub async fn put_object(
         &self,
         bucket: &str,
@@ -1868,7 +1871,7 @@ impl TargetClient {
         size: i64,
         body: ByteStream,
         opts: &PutObjectOptions,
-    ) -> Result<(), S3ClientError> {
+    ) -> Result<Option<String>, S3ClientError> {
         let mut headers = opts.header();
 
         let builder = self.client.put_object();
@@ -1903,7 +1906,7 @@ impl TargetClient {
             .send()
             .await
         {
-            Ok(_) => Ok(()),
+            Ok(output) => Ok(output.version_id().map(ToOwned::to_owned)),
             Err(e) => match e {
                 SdkError::ServiceError(service_err) => {
                     let err = service_err.into_err();

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -74,10 +74,10 @@ use rustfs_utils::http::{
 use rustfs_utils::{DEFAULT_SIP_HASH_KEY, get_env_usize, sip_hash};
 #[cfg(test)]
 use s3s::dto::ReplicationConfiguration;
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::fmt::Display;
-use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, LazyLock, Mutex as StdMutex};
 use time::OffsetDateTime;
 use time::format_description::well_known::Rfc3339;
 use tokio::io::AsyncRead;
@@ -105,6 +105,7 @@ const EVENT_RESYNC_RUNTIME_CHANNEL_FAILED: &str = "replication_resync_runtime_ch
 const EVENT_DELETE_MARKER_PURGE_FAILED: &str = "replication_delete_marker_purge_failed";
 const EVENT_DELETE_MARKER_PURGE_MRF: &str = "replication_delete_marker_purge_mrf";
 const METRIC_DELETE_MARKER_PURGE_TOTAL: &str = "rustfs_replication_delete_marker_purge_total";
+const EVENT_REPLICATION_VERSION_IDENTITY_DRIFT: &str = "replication_version_identity_drift";
 const REPLICATION_TARGET_OFFLINE_ERROR_MARKERS: &[&str] = &[
     "dispatch failure",
     "timeouterror",
@@ -186,6 +187,57 @@ fn is_head_proxy_failure(err: &SdkError<HeadObjectError>) -> bool {
     should_count_head_proxy_failure(is_not_found, code, raw_status)
 }
 
+const METRIC_VERSION_IDENTITY_DRIFT_TOTAL: &str = "rustfs_replication_version_identity_drift_total";
+
+/// Targets that already produced a version-identity-drift warning this
+/// process lifetime, by ARN. Deduping is advisory only (the metric still
+/// counts every drifting PUT), so a reconfigured target re-warning only
+/// after a restart is acceptable.
+static VERSION_IDENTITY_WARNED_ARNS: LazyLock<StdMutex<HashSet<String>>> = LazyLock::new(|| StdMutex::new(HashSet::new()));
+
+/// Runtime half of the P1-19 version-identity contract (the explicit probe
+/// lives in replication-check's VersionFidelity phase): every replication PUT
+/// response reveals whether the target adopted the source version id. A
+/// target minting its own ids silently breaks version-addressed deletes and
+/// heal, so surface it — once per target — instead of letting the divergence
+/// accumulate unseen.
+/// Pure drift judgment: the contract only applies when the source addressed a
+/// real (non-nil) version uuid, and drift means the target answered with
+/// anything else — including nothing at all.
+fn version_identity_drifted(source_version_id: &str, assigned_version_id: Option<&str>) -> bool {
+    if source_version_id.is_empty() {
+        return false;
+    }
+    // A nil source uuid travels as the literal "null" (unversioned-source
+    // semantics); no identity contract applies to it.
+    if Uuid::parse_str(source_version_id).map(|uuid| uuid.is_nil()).unwrap_or(true) {
+        return false;
+    }
+    assigned_version_id != Some(source_version_id)
+}
+
+fn audit_target_version_identity(tgt_client: &TargetClient, source_version_id: &str, assigned_version_id: Option<&str>) {
+    if !version_identity_drifted(source_version_id, assigned_version_id) {
+        return;
+    }
+    counter!(METRIC_VERSION_IDENTITY_DRIFT_TOTAL).increment(1);
+    let mut warned = VERSION_IDENTITY_WARNED_ARNS
+        .lock()
+        .unwrap_or_else(|poisoned| poisoned.into_inner());
+    if warned.insert(tgt_client.arn.clone()) {
+        warn!(
+            event = EVENT_REPLICATION_VERSION_IDENTITY_DRIFT,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+            arn = %tgt_client.arn,
+            endpoint = %tgt_client.endpoint,
+            sent_version_id = %source_version_id,
+            assigned_version_id = assigned_version_id.unwrap_or("<none>"),
+            "Replication target does not adopt source version ids; version-addressed replication cannot converge (run ?replication-check for details)"
+        );
+    }
+}
+
 async fn record_proxy_request(bucket: &str, api: &str, is_err: bool) {
     if let Some(stats) = runtime_sources::replication_stats() {
         stats.inc_proxy(bucket, api, is_err).await;
@@ -2872,6 +2924,13 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
             let result = tgt_client
                 .put_object(&tgt_client.bucket, &object, transfer_size, byte_stream, &put_opts)
                 .await
+                .map(|assigned_version_id| {
+                    audit_target_version_identity(
+                        &tgt_client,
+                        &put_opts.internal.source_version_id,
+                        assigned_version_id.as_deref(),
+                    )
+                })
                 .map_err(|e| std::io::Error::other(e.to_string()));
             record_proxy_request(&bucket, "PutObject", result.is_err()).await;
             if has_tagging_replication {
@@ -3279,6 +3338,13 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
             let result = tgt_client
                 .put_object(&tgt_client.bucket, &object, transfer_size, byte_stream, &put_opts)
                 .await
+                .map(|assigned_version_id| {
+                    audit_target_version_identity(
+                        &tgt_client,
+                        &put_opts.internal.source_version_id,
+                        assigned_version_id.as_deref(),
+                    )
+                })
                 .map_err(|e| std::io::Error::other(e.to_string()));
             record_proxy_request(&bucket, "PutObject", result.is_err()).await;
             if has_tagging_replication {
@@ -3470,15 +3536,27 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
 
     let actual_size = replication_multipart_complete_actual_size(&object_info.user_defined);
 
-    cli.complete_multipart_upload(
+    let completed = cli
-        dst_bucket,
+        .complete_multipart_upload(
-        object,
+            dst_bucket,
-        &upload_id,
+            object,
-        uploaded_parts,
+            &upload_id,
-        &replication_complete_multipart_options(actual_size, object_info.etag.clone().unwrap_or_default(), object_info.mod_time),
+            uploaded_parts,
-    )
+            &replication_complete_multipart_options(
-    .await
+                actual_size,
-    .map_err(|e| std::io::Error::other(e.to_string()))?;
+                object_info.etag.clone().unwrap_or_default(),
+                object_info.mod_time,
+            ),
+        )
+        .await
+        .map_err(|e| std::io::Error::other(e.to_string()))?;
+
+    // Multipart decides the target version at initiate time and only reveals
+    // it on completion, so this is where the identity contract is observable
+    // for this path. A target can mirror PutObject version ids and still mint
+    // its own here, which would leave multipart deletes and heals addressing
+    // a version that never existed.
+    audit_target_version_identity(&cli, &put_opts.internal.source_version_id, completed.version_id());
 
     Ok(())
 }
@@ -3525,6 +3603,27 @@ mod tests {
         ReplicationTargetStore::register_test_target(target).await;
     }
 
+    /// P1-19 runtime spot-check exemption matrix: drift only applies when the
+    /// source addressed a real version uuid.
+    #[test]
+    fn test_version_identity_drift_judgment() {
+        let source = "6fa459ea-ee8a-3ca4-894e-db77e160355e";
+        for (sent, got, expected) in [
+            (source, Some(source), false),
+            (source, Some("0e304ce5-33e9-4b8a-9b12-9e40a53e6ded"), true),
+            (source, None, true),
+            ("", None, false),
+            ("null", Some("anything"), false),
+            ("00000000-0000-0000-0000-000000000000", Some("anything"), false),
+        ] {
+            assert_eq!(
+                version_identity_drifted(sent, got),
+                expected,
+                "sent {sent:?} got {got:?} must judge drift = {expected}"
+            );
+        }
+    }
+
     #[test]
     fn resync_admission_configuration_is_bounded() {
         assert_eq!(ENV_REPL_RESYNC_MAX_JOBS, "RUSTFS_REPL_RESYNC_MAX_JOBS");

crates/ecstore/src/cluster/rpc/client.rs

@@ -15,12 +15,12 @@
 #[cfg(test)]
 use crate::cluster::rpc::http_auth::RPC_REPLAY_SCOPE_VERSION_HEADER;
 use crate::cluster::rpc::http_auth::{
-    RPC_AUTH_VERSION_HEADER, RPC_AUTH_VERSION_V2, RPC_BOOT_EPOCH_CHALLENGE_HEADER, RPC_BOOT_EPOCH_HEADER,
+    AuthenticatedPeerReplayCapabilities, RPC_AUTH_VERSION_HEADER, RPC_AUTH_VERSION_V2, RPC_BOOT_EPOCH_CHALLENGE_HEADER,
-    RPC_BOOT_EPOCH_PROOF_HEADER, RPC_CONTENT_SHA256_HEADER, TIMESTAMP_HEADER,
+    RPC_CONTENT_SHA256_HEADER, RPC_REPLAY_CACHE_CAPABILITY_HEADER, RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER,
-};
+    RollingMutationBodyDigest, TIMESTAMP_HEADER, internode_rpc_body_digest_strict,
-use crate::cluster::rpc::{
+    verify_tonic_peer_replay_capabilities_response,
-    gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, verify_tonic_boot_epoch_response,
 };
+use crate::cluster::rpc::{gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience};
 #[cfg(test)]
 use crate::cluster::rpc::{tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers};
 use crate::disk::error::{DiskError, Error as DiskErrorType, RpcStatusError};
@@ -233,7 +233,22 @@ pub struct ReplayScopeChannel<S> {
 /// The channel type used by internode clients after v2 authentication and replay-scope handling.
 pub type AuthenticatedChannel = ReplayScopeChannel<Channel>;
 
-static PEER_BOOT_EPOCHS: LazyLock<Mutex<HashMap<String, Uuid>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum PeerReplayCapability {
+    Capable { boot_epoch: Uuid },
+    Revoked,
+}
+
+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
+struct PeerReplayState {
+    boot_epoch: Option<Uuid>,
+    cache_capability: Option<PeerReplayCapability>,
+}
+
+#[derive(Clone, Copy, Debug)]
+struct PeerReplayStateSnapshot(PeerReplayState);
+
+static PEER_REPLAY_STATES: LazyLock<Mutex<HashMap<String, PeerReplayState>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
 
 impl<S> ReplayScopeChannel<S> {
     fn new(inner: S, audience: Option<String>) -> Self {
@@ -241,13 +256,67 @@ impl<S> ReplayScopeChannel<S> {
     }
 }
 
-fn cached_peer_boot_epoch(audience: &str) -> Option<Uuid> {
+fn peer_replay_state(audience: &str) -> PeerReplayState {
-    PEER_BOOT_EPOCHS.lock().ok().and_then(|epochs| epochs.get(audience).copied())
+    PEER_REPLAY_STATES
+        .lock()
+        .ok()
+        .and_then(|states| states.get(audience).copied())
+        .unwrap_or_default()
 }
 
-fn remember_peer_boot_epoch(audience: String, epoch: Uuid) {
+fn apply_peer_replay_response(
-    if let Ok(mut epochs) = PEER_BOOT_EPOCHS.lock() {
+    audience: String,
-        epochs.insert(audience, epoch);
+    sent_state: PeerReplayState,
+    response: std::io::Result<AuthenticatedPeerReplayCapabilities>,
+) {
+    if let Ok(mut states) = PEER_REPLAY_STATES.lock() {
+        let current_state = states.get(&audience).copied().unwrap_or_default();
+        let mut next_state = current_state;
+        if let Ok(response) = &response
+            && sent_state.boot_epoch == current_state.boot_epoch
+        {
+            next_state.boot_epoch = Some(response.boot_epoch);
+        }
+
+        if sent_state.boot_epoch == current_state.boot_epoch {
+            let response_capability = response
+                .as_ref()
+                .ok()
+                .filter(|response| response.dynamic_replay_cache)
+                .map(|response| response.boot_epoch);
+            match (sent_state.cache_capability, current_state.cache_capability, response_capability) {
+                (None, None, Some(boot_epoch))
+                | (Some(PeerReplayCapability::Revoked), Some(PeerReplayCapability::Revoked), Some(boot_epoch)) => {
+                    next_state.cache_capability = Some(PeerReplayCapability::Capable { boot_epoch });
+                }
+                (
+                    Some(PeerReplayCapability::Capable {
+                        boot_epoch: sent_boot_epoch,
+                    }),
+                    Some(PeerReplayCapability::Capable {
+                        boot_epoch: current_boot_epoch,
+                    }),
+                    Some(response_boot_epoch),
+                ) if sent_boot_epoch == current_boot_epoch => {
+                    next_state.cache_capability = Some(PeerReplayCapability::Capable {
+                        boot_epoch: response_boot_epoch,
+                    });
+                }
+                (
+                    Some(PeerReplayCapability::Capable {
+                        boot_epoch: sent_boot_epoch,
+                    }),
+                    Some(PeerReplayCapability::Capable {
+                        boot_epoch: current_boot_epoch,
+                    }),
+                    None,
+                ) if sent_boot_epoch == current_boot_epoch => {
+                    next_state.cache_capability = Some(PeerReplayCapability::Revoked);
+                }
+                _ => {}
+            }
+        }
+        states.insert(audience, next_state);
     }
 }
 
@@ -276,6 +345,11 @@ where
                 == Some(RPC_AUTH_VERSION_V2)
         });
         let challenge = authenticated.then(Uuid::new_v4);
+        let sent_state = request
+            .extensions()
+            .get::<PeerReplayStateSnapshot>()
+            .map(|snapshot| snapshot.0)
+            .unwrap_or_default();
         if let (Some(audience), Some(challenge)) = (self.audience.as_deref(), challenge) {
             // The challenge is independently HMAC-authenticated by the response proof. It is not
             // part of v2 so old peers ignore it, while a new peer can safely advertise its epoch.
@@ -284,7 +358,7 @@ where
                 challenge.to_string().parse().expect("UUID must be a valid header value"),
             );
             if let (Some(boot_epoch), Some(timestamp), Some(content_sha256)) = (
-                cached_peer_boot_epoch(audience),
+                sent_state.boot_epoch,
                 request.headers().get(TIMESTAMP_HEADER).and_then(|value| value.to_str().ok()),
                 request
                     .headers()
@@ -303,16 +377,21 @@ where
         Box::pin(async move {
             let response = future.await?;
             if let (Some(audience), Some(challenge)) = (audience, challenge) {
-                match verify_tonic_boot_epoch_response(&audience, challenge, response.headers()) {
+                let response_state = verify_tonic_peer_replay_capabilities_response(&audience, challenge, response.headers());
-                    Ok(epoch) => remember_peer_boot_epoch(audience, epoch),
+                if let Err(error) = &response_state
-                    Err(error)
+                    && (response.headers().contains_key(RPC_REPLAY_CACHE_CAPABILITY_HEADER)
-                        if response.headers().contains_key(RPC_BOOT_EPOCH_HEADER)
+                        || response.headers().contains_key(RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER))
-                            || response.headers().contains_key(RPC_BOOT_EPOCH_PROOF_HEADER) =>
+                {
-                    {
+                    debug!(
-                        debug!(error = %error, "peer boot epoch response proof was rejected")
+                        event = "internode_rpc_capability_proof_rejected",
-                    }
+                        component = "ecstore",
-                    Err(_) => {}
+                        subsystem = "rpc_client",
+                        result = "rejected",
+                        error = %error,
+                        "internode RPC capability proof rejected"
+                    )
                 }
+                apply_peer_replay_response(audience, sent_state, response_state);
             }
             Ok(response)
         })
@@ -321,6 +400,7 @@ where
 
 pub struct TonicSignatureInterceptor {
     audience: Option<String>,
+    body_digest_strict: bool,
 }
 
 impl tonic::service::Interceptor for TonicSignatureInterceptor {
@@ -337,9 +417,31 @@ impl tonic::service::Interceptor for TonicSignatureInterceptor {
             .metadata()
             .get(RPC_CONTENT_SHA256_HEADER)
             .and_then(|value| value.to_str().ok());
+        // RUSTFS_COMPAT_TODO(disk-mutation-body-digest): use cache-free v2 for peers without an authenticated boot epoch. Remove after every supported peer advertises the authenticated dynamic replay-cache capability and body-digest strict mode is the default.
+        // beta.11 verifies v2 body digests but stores their nonces in a fixed-size cache.
+        let rolling_mutation = req.extensions().get::<RollingMutationBodyDigest>().is_some();
+        let peer_state = PEER_REPLAY_STATES
+            .lock()
+            .map_err(|_| tonic::Status::unauthenticated("RPC peer capability state unavailable"))?
+            .get(audience)
+            .copied()
+            .unwrap_or_default();
+        let content_sha256 = if content_sha256.is_some() {
+            if peer_state.cache_capability == Some(PeerReplayCapability::Revoked) {
+                return Err(tonic::Status::unauthenticated("RPC peer replay capability changed"));
+            }
+            if rolling_mutation && !self.body_digest_strict && peer_state.boot_epoch.is_none() {
+                None
+            } else {
+                content_sha256
+            }
+        } else {
+            content_sha256
+        };
         let headers = gen_tonic_signature_headers(audience, method.service(), method.method(), content_sha256)
             .map_err(|_| tonic::Status::unauthenticated("No valid auth token"))?;
         req.metadata_mut().as_mut().extend(headers);
+        req.extensions_mut().insert(PeerReplayStateSnapshot(peer_state));
         inject_trace_context_into_metadata(req.metadata_mut());
         inject_request_id_into_metadata(req.metadata_mut());
         Ok(req)
@@ -347,7 +449,10 @@ impl tonic::service::Interceptor for TonicSignatureInterceptor {
 }
 
 pub fn gen_tonic_signature_interceptor() -> TonicSignatureInterceptor {
-    TonicSignatureInterceptor { audience: None }
+    TonicSignatureInterceptor {
+        audience: None,
+        body_digest_strict: internode_rpc_body_digest_strict(),
+    }
 }
 
 pub struct NoOpInterceptor;
@@ -409,6 +514,7 @@ mod tests {
     #[derive(Clone)]
     struct EpochProofService {
         audience: String,
+        include_capability: bool,
         seen_headers: std::sync::Arc<Mutex<Vec<http::HeaderMap>>>,
     }
 
@@ -430,29 +536,97 @@ mod tests {
                 .expect("client challenge must be syntactically valid")
                 .expect("authenticated client request must carry a boot epoch challenge");
             let mut response = HttpResponse::new(());
-            response.headers_mut().extend(
+            let mut headers = tonic_boot_epoch_response_headers(&self.audience, challenge)
-                tonic_boot_epoch_response_headers(&self.audience, challenge)
+                .expect("test server must be able to sign an epoch proof");
-                    .expect("test server must be able to sign an epoch proof"),
+            if !self.include_capability {
-            );
+                headers.remove(RPC_REPLAY_CACHE_CAPABILITY_HEADER);
+                headers.remove(RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER);
+            }
+            response.headers_mut().extend(headers);
             std::future::ready(Ok(response))
         }
     }
 
+    #[derive(Clone)]
+    struct MissingProofService;
+
+    impl Service<HttpRequest<()>> for MissingProofService {
+        type Response = HttpResponse<()>;
+        type Error = std::convert::Infallible;
+        type Future = std::future::Ready<Result<Self::Response, Self::Error>>;
+
+        fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn call(&mut self, _request: HttpRequest<()>) -> Self::Future {
+            std::future::ready(Ok(HttpResponse::new(())))
+        }
+    }
+
     fn ensure_test_rpc_secret() {
         runtime_sources::ensure_test_rpc_secret();
     }
 
     fn test_request() -> tonic::Request<()> {
+        test_request_for("Ping")
+    }
+
+    fn test_request_for(method: &'static str) -> tonic::Request<()> {
         let mut request = tonic::Request::new(());
         request
             .extensions_mut()
-            .insert(tonic::GrpcMethod::new("node_service.NodeService", "Ping"));
+            .insert(tonic::GrpcMethod::new("node_service.NodeService", method));
         request
     }
 
     fn test_interceptor() -> TonicSignatureInterceptor {
+        test_interceptor_for("node-a:9000", false)
+    }
+
+    fn test_interceptor_for(audience: &str, body_digest_strict: bool) -> TonicSignatureInterceptor {
         TonicSignatureInterceptor {
-            audience: Some("node-a:9000".to_string()),
+            audience: Some(audience.to_string()),
+            body_digest_strict,
+        }
+    }
+
+    fn clear_peer_capability(audience: &str) {
+        PEER_REPLAY_STATES
+            .lock()
+            .expect("peer capability cache lock must not be poisoned")
+            .remove(audience);
+    }
+
+    fn rolling_mutation_request(method: &'static str) -> tonic::Request<()> {
+        let mut request = tonic::Request::new(rustfs_protos::proto_gen::node_service::GenerallyLockRequest {
+            args: "canonical mutation request".to_string(),
+        });
+        request
+            .extensions_mut()
+            .insert(tonic::GrpcMethod::new("node_service.NodeService", method));
+        crate::cluster::rpc::set_tonic_rolling_mutation_body_digest(&mut request).expect("test mutation digest must be attached");
+        request.map(|_| ())
+    }
+
+    fn replay_scope_request(audience: &str, method: &'static str) -> HttpRequest<()> {
+        let mut request = HttpRequest::builder()
+            .uri(format!("/node_service.NodeService/{method}"))
+            .body(())
+            .expect("test RPC request must build");
+        request.headers_mut().extend(
+            gen_tonic_signature_headers(audience, "node_service.NodeService", method, None).expect("v2 test headers must mint"),
+        );
+        request
+            .extensions_mut()
+            .insert(PeerReplayStateSnapshot(peer_replay_state(audience)));
+        request
+    }
+
+    fn authenticated_peer_response(boot_epoch: Uuid, dynamic_replay_cache: bool) -> AuthenticatedPeerReplayCapabilities {
+        AuthenticatedPeerReplayCapabilities {
+            boot_epoch,
+            dynamic_replay_cache,
         }
     }
 
@@ -567,6 +741,431 @@ mod tests {
         );
     }
 
+    #[test]
+    fn unknown_peer_mutations_use_cache_free_unsigned_v2() {
+        ensure_test_rpc_secret();
+        let audience = "legacy-body-digest-client-test:9000";
+        clear_peer_capability(audience);
+        let mut interceptor = test_interceptor_for(audience, false);
+        for method in ["Lock", "WriteAll"] {
+            let request = interceptor
+                .call(rolling_mutation_request(method))
+                .expect("interceptor call should succeed");
+
+            assert_eq!(
+                request
+                    .metadata()
+                    .get("x-rustfs-content-sha256")
+                    .and_then(|value| value.to_str().ok()),
+                Some("UNSIGNED-PAYLOAD")
+            );
+            assert_eq!(
+                request
+                    .metadata()
+                    .get("x-rustfs-rpc-nonce")
+                    .and_then(|value| value.to_str().ok()),
+                Some("unsigned")
+            );
+            assert!(
+                crate::cluster::rpc::verify_tonic_rpc_signature(
+                    audience,
+                    &format!("/node_service.NodeService/{method}"),
+                    request.metadata().as_ref(),
+                )
+                .is_ok(),
+                "the cache-free request must retain valid audience- and method-bound v2 authentication"
+            );
+        }
+    }
+
+    #[test]
+    fn unknown_peer_exact_body_contract_remains_body_bound() {
+        ensure_test_rpc_secret();
+        let audience = "exact-body-contract-client-test:9000";
+        clear_peer_capability(audience);
+        let mut interceptor = test_interceptor_for(audience, false);
+        let mut request = test_request_for("ScannerActivity");
+        crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, b"exact scanner activity body")
+            .expect("test exact body digest must be attached");
+        let expected_digest = request
+            .metadata()
+            .get("x-rustfs-content-sha256")
+            .and_then(|value| value.to_str().ok())
+            .expect("test request must carry its digest")
+            .to_string();
+
+        let request = interceptor.call(request).expect("interceptor call should succeed");
+
+        assert_eq!(
+            request
+                .metadata()
+                .get("x-rustfs-content-sha256")
+                .and_then(|value| value.to_str().ok()),
+            Some(expected_digest.as_str())
+        );
+        assert!(
+            crate::cluster::rpc::verify_tonic_rpc_signature(
+                audience,
+                "/node_service.NodeService/ScannerActivity",
+                request.metadata().as_ref(),
+            )
+            .is_ok()
+        );
+    }
+
+    #[test]
+    fn unknown_peer_iam_mutation_helper_remains_body_bound() {
+        ensure_test_rpc_secret();
+        let audience = "exact-iam-mutation-client-test:9000";
+        clear_peer_capability(audience);
+        let mut interceptor = test_interceptor_for(audience, false);
+        let mut request = tonic::Request::new(rustfs_protos::proto_gen::node_service::DeleteUserRequest {
+            access_key: "target-access-key".to_string(),
+        });
+        request
+            .extensions_mut()
+            .insert(tonic::GrpcMethod::new("node_service.NodeService", "DeleteUser"));
+        crate::cluster::rpc::set_tonic_mutation_body_digest(&mut request).expect("test IAM mutation digest must be attached");
+        let request = request.map(|_| ());
+        let expected_digest = request
+            .metadata()
+            .get("x-rustfs-content-sha256")
+            .and_then(|value| value.to_str().ok())
+            .expect("test IAM mutation must carry its digest")
+            .to_string();
+
+        let request = interceptor.call(request).expect("interceptor call should succeed");
+
+        assert_eq!(
+            request
+                .metadata()
+                .get("x-rustfs-content-sha256")
+                .and_then(|value| value.to_str().ok()),
+            Some(expected_digest.as_str())
+        );
+        assert!(
+            crate::cluster::rpc::verify_tonic_rpc_signature(
+                audience,
+                "/node_service.NodeService/DeleteUser",
+                request.metadata().as_ref(),
+            )
+            .is_ok(),
+            "IAM mutations must remain body-bound before capability discovery"
+        );
+    }
+
+    #[test]
+    fn authenticated_replay_cache_capability_enables_body_binding() {
+        ensure_test_rpc_secret();
+        let audience = "body-digest-capable-client-test:9000";
+        clear_peer_capability(audience);
+        let seen_headers = std::sync::Arc::new(Mutex::new(Vec::new()));
+        let service = EpochProofService {
+            audience: audience.to_string(),
+            include_capability: true,
+            seen_headers,
+        };
+        let mut channel = ReplayScopeChannel::new(service, Some(audience.to_string()));
+        futures::executor::block_on(channel.call(replay_scope_request(audience, "Ping")))
+            .expect("authenticated capability probe must complete");
+
+        let mut interceptor = test_interceptor_for(audience, false);
+        let request = rolling_mutation_request("Lock");
+        let expected_digest = request
+            .metadata()
+            .get("x-rustfs-content-sha256")
+            .and_then(|value| value.to_str().ok())
+            .expect("test mutation must carry its digest")
+            .to_string();
+
+        let request = interceptor.call(request).expect("interceptor call should succeed");
+
+        assert_eq!(
+            request
+                .metadata()
+                .get("x-rustfs-content-sha256")
+                .and_then(|value| value.to_str().ok()),
+            Some(expected_digest.as_str())
+        );
+        let nonce = request
+            .metadata()
+            .get("x-rustfs-rpc-nonce")
+            .and_then(|value| value.to_str().ok())
+            .and_then(|value| Uuid::parse_str(value).ok())
+            .expect("capable peer body-bound mutation must carry a UUID nonce");
+        assert!(!nonce.is_nil());
+        assert!(
+            crate::cluster::rpc::verify_tonic_rpc_signature(
+                audience,
+                "/node_service.NodeService/Lock",
+                request.metadata().as_ref(),
+            )
+            .is_ok(),
+            "the body-bound request must retain valid audience- and method-bound v2 authentication"
+        );
+        clear_peer_capability(audience);
+    }
+
+    #[test]
+    fn invalid_capability_proof_does_not_enable_body_binding() {
+        ensure_test_rpc_secret();
+        let audience = "invalid-capability-client-test:9000";
+        clear_peer_capability(audience);
+        let service = EpochProofService {
+            audience: "wrong-capability-audience:9000".to_string(),
+            include_capability: true,
+            seen_headers: std::sync::Arc::new(Mutex::new(Vec::new())),
+        };
+        let mut channel = ReplayScopeChannel::new(service, Some(audience.to_string()));
+        futures::executor::block_on(channel.call(replay_scope_request(audience, "Ping")))
+            .expect("invalid capability response must still complete");
+
+        let mut interceptor = test_interceptor_for(audience, false);
+        let request = interceptor
+            .call(rolling_mutation_request("Lock"))
+            .expect("legacy-compatible mutation must still be signed");
+
+        assert_eq!(
+            request
+                .metadata()
+                .get("x-rustfs-content-sha256")
+                .and_then(|value| value.to_str().ok()),
+            Some("UNSIGNED-PAYLOAD")
+        );
+    }
+
+    #[test]
+    fn legacy_boot_proof_keeps_mutations_body_bound_and_enables_non_ping_v3() {
+        ensure_test_rpc_secret();
+        let audience = "legacy-boot-proof-client-test:9000";
+        clear_peer_capability(audience);
+        let seen_headers = std::sync::Arc::new(Mutex::new(Vec::new()));
+        let service = EpochProofService {
+            audience: audience.to_string(),
+            include_capability: false,
+            seen_headers: seen_headers.clone(),
+        };
+        let mut channel = ReplayScopeChannel::new(service, Some(audience.to_string()));
+        futures::executor::block_on(channel.call(replay_scope_request(audience, "Ping")))
+            .expect("legacy boot proof response must complete");
+
+        let state = peer_replay_state(audience);
+        assert!(state.boot_epoch.is_some(), "authenticated legacy proof must enable replay-scoped v3");
+        assert_eq!(state.cache_capability, None);
+
+        let mut interceptor = test_interceptor_for(audience, false);
+        let request = rolling_mutation_request("Lock");
+        let expected_digest = request
+            .metadata()
+            .get("x-rustfs-content-sha256")
+            .and_then(|value| value.to_str().ok())
+            .expect("test mutation must carry its digest")
+            .to_string();
+        let request = interceptor.call(request).expect("legacy-compatible mutation must be signed");
+        assert_eq!(
+            request
+                .metadata()
+                .get("x-rustfs-content-sha256")
+                .and_then(|value| value.to_str().ok()),
+            Some(expected_digest.as_str())
+        );
+        let (metadata, extensions, body) = request.into_parts();
+        let mut request = HttpRequest::new(body);
+        *request.uri_mut() = "/node_service.NodeService/Lock".parse().expect("test RPC URI must parse");
+        *request.headers_mut() = metadata.into_headers();
+        *request.extensions_mut() = extensions;
+        futures::executor::block_on(channel.call(request)).expect("legacy strict-compatible lock request must complete");
+
+        let headers = seen_headers.lock().expect("test header capture lock must not be poisoned");
+        assert!(
+            headers[1].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
+            "authenticated legacy boot proof must enable v3 on a non-Ping request"
+        );
+    }
+
+    #[test]
+    fn reordered_capability_responses_cannot_undo_newer_state() {
+        let audience = "reordered-capability-client-test:9000";
+        let epoch_one = Uuid::new_v4();
+        let epoch_two = Uuid::new_v4();
+        clear_peer_capability(audience);
+
+        let unknown = PeerReplayState::default();
+        apply_peer_replay_response(audience.to_string(), unknown, Ok(authenticated_peer_response(epoch_one, true)));
+        apply_peer_replay_response(audience.to_string(), unknown, Err(std::io::Error::other("delayed legacy response")));
+        let epoch_one_state = PeerReplayState {
+            boot_epoch: Some(epoch_one),
+            cache_capability: Some(PeerReplayCapability::Capable { boot_epoch: epoch_one }),
+        };
+        assert_eq!(peer_replay_state(audience), epoch_one_state);
+
+        apply_peer_replay_response(audience.to_string(), epoch_one_state, Err(std::io::Error::other("rollback response")));
+        apply_peer_replay_response(audience.to_string(), epoch_one_state, Ok(authenticated_peer_response(epoch_one, true)));
+        assert_eq!(
+            peer_replay_state(audience),
+            PeerReplayState {
+                boot_epoch: Some(epoch_one),
+                cache_capability: Some(PeerReplayCapability::Revoked),
+            }
+        );
+
+        let revoked = peer_replay_state(audience);
+        apply_peer_replay_response(audience.to_string(), revoked, Ok(authenticated_peer_response(epoch_two, true)));
+        apply_peer_replay_response(audience.to_string(), epoch_one_state, Ok(authenticated_peer_response(epoch_one, true)));
+        assert_eq!(
+            peer_replay_state(audience),
+            PeerReplayState {
+                boot_epoch: Some(epoch_two),
+                cache_capability: Some(PeerReplayCapability::Capable { boot_epoch: epoch_two }),
+            }
+        );
+        clear_peer_capability(audience);
+    }
+
+    #[test]
+    fn stale_capability_response_cannot_cross_a_new_boot_epoch() {
+        let audience = "cross-epoch-capability-client-test:9000";
+        let epoch_one = Uuid::new_v4();
+        let epoch_two = Uuid::new_v4();
+        let epoch_three = Uuid::new_v4();
+        clear_peer_capability(audience);
+
+        let revoked_epoch_one = PeerReplayState {
+            boot_epoch: Some(epoch_one),
+            cache_capability: Some(PeerReplayCapability::Revoked),
+        };
+        PEER_REPLAY_STATES
+            .lock()
+            .expect("peer replay state lock must not be poisoned")
+            .insert(audience.to_string(), revoked_epoch_one);
+
+        apply_peer_replay_response(
+            audience.to_string(),
+            revoked_epoch_one,
+            Ok(authenticated_peer_response(epoch_three, false)),
+        );
+        apply_peer_replay_response(audience.to_string(), revoked_epoch_one, Ok(authenticated_peer_response(epoch_two, true)));
+
+        assert_eq!(
+            peer_replay_state(audience),
+            PeerReplayState {
+                boot_epoch: Some(epoch_three),
+                cache_capability: Some(PeerReplayCapability::Revoked),
+            },
+            "a stale dynamic-cache proof must not cross a newer authenticated boot epoch"
+        );
+        clear_peer_capability(audience);
+    }
+
+    #[test]
+    fn interceptor_snapshot_prevents_delayed_legacy_response_from_revoking_capability() {
+        ensure_test_rpc_secret();
+        let audience = "capability-snapshot-client-test:9000";
+        clear_peer_capability(audience);
+        let boot_epoch = Uuid::new_v4();
+        let mut interceptor = test_interceptor_for(audience, false);
+        let request = interceptor
+            .call(rolling_mutation_request("Lock"))
+            .expect("legacy-compatible request must pass the interceptor");
+        assert_eq!(
+            request
+                .extensions()
+                .get::<PeerReplayStateSnapshot>()
+                .map(|snapshot| snapshot.0),
+            Some(PeerReplayState::default()),
+            "interceptor must preserve its unknown-state admission snapshot"
+        );
+
+        let capable_state = PeerReplayState {
+            boot_epoch: Some(boot_epoch),
+            cache_capability: Some(PeerReplayCapability::Capable { boot_epoch }),
+        };
+        PEER_REPLAY_STATES
+            .lock()
+            .expect("peer capability cache lock must not be poisoned")
+            .insert(audience.to_string(), capable_state);
+        let (metadata, extensions, body) = request.into_parts();
+        let mut request = HttpRequest::new(body);
+        *request.uri_mut() = "/node_service.NodeService/Lock".parse().expect("test RPC URI must parse");
+        *request.headers_mut() = metadata.into_headers();
+        *request.extensions_mut() = extensions;
+        let mut channel = ReplayScopeChannel::new(MissingProofService, Some(audience.to_string()));
+
+        futures::executor::block_on(channel.call(request)).expect("in-flight request response must complete");
+
+        assert_eq!(peer_replay_state(audience), capable_state);
+        clear_peer_capability(audience);
+    }
+
+    #[test]
+    fn strict_mode_keeps_unknown_peer_mutations_body_bound() {
+        ensure_test_rpc_secret();
+        let audience = "strict-body-digest-client-test:9000";
+        clear_peer_capability(audience);
+        let mut interceptor = test_interceptor_for(audience, true);
+        let request = rolling_mutation_request("WriteAll");
+        let expected_digest = request
+            .metadata()
+            .get("x-rustfs-content-sha256")
+            .and_then(|value| value.to_str().ok())
+            .expect("test mutation must carry its digest")
+            .to_string();
+
+        let request = interceptor.call(request).expect("interceptor call should succeed");
+
+        assert_eq!(
+            request
+                .metadata()
+                .get("x-rustfs-content-sha256")
+                .and_then(|value| value.to_str().ok()),
+            Some(expected_digest.as_str())
+        );
+        let nonce = request
+            .metadata()
+            .get("x-rustfs-rpc-nonce")
+            .and_then(|value| value.to_str().ok())
+            .and_then(|value| Uuid::parse_str(value).ok())
+            .expect("strict body-bound mutation must carry a UUID nonce");
+        assert!(!nonce.is_nil());
+        assert!(
+            crate::cluster::rpc::verify_tonic_rpc_signature(
+                audience,
+                "/node_service.NodeService/WriteAll",
+                request.metadata().as_ref(),
+            )
+            .is_ok()
+        );
+    }
+
+    #[test]
+    fn missing_capability_after_pin_fails_closed() {
+        ensure_test_rpc_secret();
+        let audience = "revoked-capability-client-test:9000";
+        let boot_epoch = Uuid::new_v4();
+        PEER_REPLAY_STATES
+            .lock()
+            .expect("peer capability cache lock must not be poisoned")
+            .insert(
+                audience.to_string(),
+                PeerReplayState {
+                    boot_epoch: Some(boot_epoch),
+                    cache_capability: Some(PeerReplayCapability::Capable { boot_epoch }),
+                },
+            );
+        let mut channel = ReplayScopeChannel::new(MissingProofService, Some(audience.to_string()));
+        futures::executor::block_on(channel.call(replay_scope_request(audience, "Ping")))
+            .expect("legacy response must complete before capability rejection");
+
+        let mut interceptor = test_interceptor_for(audience, false);
+        let error = interceptor
+            .call(rolling_mutation_request("Lock"))
+            .expect_err("a peer that loses its pinned capability must fail closed");
+
+        assert_eq!(error.code(), tonic::Code::Unauthenticated);
+        assert_eq!(error.message(), "RPC peer replay capability changed");
+        clear_peer_capability(audience);
+    }
+
     #[test]
     fn test_signature_interceptor_binds_audience_from_peer_uri() {
         let interceptor = TonicInterceptor::Signature(gen_tonic_signature_interceptor())
@@ -583,27 +1182,15 @@ mod tests {
     fn replay_scope_channel_uses_epoch_proof_before_sending_v3() {
         ensure_test_rpc_secret();
         let audience = "replay-scope-client-test:9000";
-        PEER_BOOT_EPOCHS
+        clear_peer_capability(audience);
-            .lock()
-            .expect("peer epoch cache lock must not be poisoned")
-            .remove(audience);
         let seen_headers = std::sync::Arc::new(Mutex::new(Vec::new()));
         let service = EpochProofService {
             audience: audience.to_string(),
+            include_capability: true,
             seen_headers: seen_headers.clone(),
         };
         let mut channel = ReplayScopeChannel::new(service, Some(audience.to_string()));
-        let make_request = || {
+        let make_request = || replay_scope_request(audience, "Ping");
-            let mut request = HttpRequest::builder()
-                .uri("/node_service.NodeService/Ping")
-                .body(())
-                .expect("test RPC request must build");
-            request.headers_mut().extend(
-                gen_tonic_signature_headers(audience, "node_service.NodeService", "Ping", None)
-                    .expect("v2 test headers must mint"),
-            );
-            request
-        };
 
         futures::executor::block_on(channel.call(make_request())).expect("first request must complete");
         futures::executor::block_on(channel.call(make_request())).expect("second request must complete");
@@ -619,10 +1206,7 @@ mod tests {
             headers[1].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
             "the second request must carry the replay-scoped v3 signature"
         );
-        PEER_BOOT_EPOCHS
+        clear_peer_capability(audience);
-            .lock()
-            .expect("peer epoch cache lock must not be poisoned")
-            .remove(audience);
     }
 
     #[test]

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -73,10 +73,14 @@ pub const RPC_REPLAY_SCOPE_NONCE_HEADER: &str = "x-rustfs-rpc-replay-nonce";
 pub const RPC_BOOT_EPOCH_HEADER: &str = "x-rustfs-rpc-boot-epoch";
 pub const RPC_BOOT_EPOCH_CHALLENGE_HEADER: &str = "x-rustfs-rpc-boot-epoch-challenge";
 pub const RPC_BOOT_EPOCH_PROOF_HEADER: &str = "x-rustfs-rpc-boot-epoch-proof";
+pub(crate) const RPC_REPLAY_CACHE_CAPABILITY_HEADER: &str = "x-rustfs-rpc-replay-cache-capability";
+pub(crate) const RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER: &str = "x-rustfs-rpc-replay-cache-capability-proof";
 const RPC_REPLAY_SCOPE_VERSION_V3: &str = "3";
 const RPC_RESPONSE_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-response-proof-v1\0";
 const RPC_REPLAY_SCOPE_DOMAIN: &[u8] = b"rustfs-rpc-replay-scope-v3\0";
 const RPC_BOOT_EPOCH_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-boot-epoch-proof-v1\0";
+const RPC_REPLAY_CACHE_CAPABILITY_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-replay-cache-capability-proof-v1\0";
+const RPC_REPLAY_CACHE_CAPABILITY_V1: &str = "dynamic-replay-cache-v1";
 const HTTP_PUT_FILE_AUTH_DOMAIN: &[u8] = b"rustfs-http-put-file-auth-v1\0";
 const HTTP_PUT_FILE_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-http-put-file-capability-v1\0";
 const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
@@ -85,8 +89,9 @@ const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
 const REPLAY_CACHE_RETENTION: Duration = Duration::from_secs(601);
 const REPLAY_CACHE_RETENTION_SECS: usize = 601;
 const REPLAY_CACHE_ENTRY_BYTES_ESTIMATE: u64 = 128;
-const REPLAY_CACHE_AUTO_MEMORY_PERCENT: u64 = 8;
+// Keep 16 CPU / 32 GiB field nodes at the 32M cap without requiring an env override.
-const REPLAY_CACHE_AUTO_RPC_RPS_PER_CPU: usize = 2048;
+const REPLAY_CACHE_AUTO_MEMORY_PERCENT: u64 = 13;
+const REPLAY_CACHE_AUTO_RPC_RPS_PER_CPU: usize = 4096;
 const REPLAY_CACHE_AUTO_MAX_CAPACITY: usize = 33_554_432;
 const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3";
 pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
@@ -102,6 +107,10 @@ static INTERNODE_RPC_BODY_DIGEST_STRICT: LazyLock<bool> = LazyLock::new(|| {
         rustfs_config::DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT,
     )
 });
+
+pub(crate) fn internode_rpc_body_digest_strict() -> bool {
+    *INTERNODE_RPC_BODY_DIGEST_STRICT
+}
 static INTERNODE_RPC_REPLAY_SCOPE_STRICT: LazyLock<bool> = LazyLock::new(|| {
     get_env_bool(
         rustfs_config::ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
@@ -789,6 +798,50 @@ fn verify_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_e
         .map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid RPC boot epoch proof"))
 }
 
+fn update_replay_cache_capability_proof(mac: &mut HmacSha256, audience: &str, challenge: Uuid, boot_epoch: Uuid) {
+    mac.update(RPC_REPLAY_CACHE_CAPABILITY_PROOF_DOMAIN);
+    for part in [
+        audience.as_bytes(),
+        b"|",
+        challenge.as_bytes(),
+        b"|",
+        boot_epoch.as_bytes(),
+        b"|",
+        RPC_REPLAY_CACHE_CAPABILITY_V1.as_bytes(),
+    ] {
+        mac.update(part);
+    }
+}
+
+fn generate_replay_cache_capability_proof(
+    secret: &str,
+    audience: &str,
+    challenge: Uuid,
+    boot_epoch: Uuid,
+) -> std::io::Result<String> {
+    let mut mac =
+        <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
+    update_replay_cache_capability_proof(&mut mac, audience, challenge, boot_epoch);
+    Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
+}
+
+fn verify_replay_cache_capability_proof(
+    secret: &str,
+    audience: &str,
+    challenge: Uuid,
+    boot_epoch: Uuid,
+    proof: &str,
+) -> std::io::Result<()> {
+    let proof = general_purpose::STANDARD
+        .decode(proof)
+        .map_err(|_| std::io::Error::other("Invalid RPC replay cache capability proof"))?;
+    let mut mac =
+        <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
+    update_replay_cache_capability_proof(&mut mac, audience, challenge, boot_epoch);
+    mac.verify_slice(&proof)
+        .map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid RPC replay cache capability proof"))
+}
+
 fn non_nil_uuid(value: &str, name: &str) -> std::io::Result<Uuid> {
     let value = Uuid::parse_str(value).map_err(|_| std::io::Error::other(format!("Invalid {name}")))?;
     (!value.is_nil())
@@ -871,15 +924,34 @@ pub fn tonic_boot_epoch_challenge(headers: &HeaderMap) -> std::io::Result<Option
 /// Build the authenticated response headers for a client boot-epoch challenge.
 pub fn tonic_boot_epoch_response_headers(audience: &str, challenge: Uuid) -> std::io::Result<HeaderMap> {
     let boot_epoch = tonic_rpc_boot_epoch();
-    let proof = generate_boot_epoch_proof(&get_shared_secret()?, audience, challenge, boot_epoch)?;
+    let secret = get_shared_secret()?;
+    let proof = generate_boot_epoch_proof(&secret, audience, challenge, boot_epoch)?;
+    let capability_proof = generate_replay_cache_capability_proof(&secret, audience, challenge, boot_epoch)?;
     let mut headers = HeaderMap::new();
     headers.insert(RPC_BOOT_EPOCH_HEADER, header_value(&boot_epoch.to_string(), RPC_BOOT_EPOCH_HEADER)?);
     headers.insert(RPC_BOOT_EPOCH_PROOF_HEADER, header_value(&proof, RPC_BOOT_EPOCH_PROOF_HEADER)?);
+    headers.insert(
+        RPC_REPLAY_CACHE_CAPABILITY_HEADER,
+        HeaderValue::from_static(RPC_REPLAY_CACHE_CAPABILITY_V1),
+    );
+    headers.insert(
+        RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER,
+        header_value(&capability_proof, RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER)?,
+    );
     Ok(headers)
 }
 
 /// Verify the server boot-epoch response for a challenge generated by this client.
 pub fn verify_tonic_boot_epoch_response(audience: &str, challenge: Uuid, headers: &HeaderMap) -> std::io::Result<Uuid> {
+    verify_tonic_boot_epoch_response_with_secret(&get_shared_secret()?, audience, challenge, headers)
+}
+
+fn verify_tonic_boot_epoch_response_with_secret(
+    secret: &str,
+    audience: &str,
+    challenge: Uuid,
+    headers: &HeaderMap,
+) -> std::io::Result<Uuid> {
     let boot_epoch = headers
         .get(RPC_BOOT_EPOCH_HEADER)
         .and_then(|value| value.to_str().ok())
@@ -889,10 +961,47 @@ pub fn verify_tonic_boot_epoch_response(audience: &str, challenge: Uuid, headers
         .get(RPC_BOOT_EPOCH_PROOF_HEADER)
         .and_then(|value| value.to_str().ok())
         .ok_or_else(|| std::io::Error::other("Missing RPC boot epoch proof"))?;
-    verify_boot_epoch_proof(&get_shared_secret()?, audience, challenge, boot_epoch, proof)?;
+    verify_boot_epoch_proof(secret, audience, challenge, boot_epoch, proof)?;
     Ok(boot_epoch)
 }
 
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub(crate) struct AuthenticatedPeerReplayCapabilities {
+    pub(crate) boot_epoch: Uuid,
+    pub(crate) dynamic_replay_cache: bool,
+}
+
+pub(crate) fn verify_tonic_peer_replay_capabilities_response(
+    audience: &str,
+    challenge: Uuid,
+    headers: &HeaderMap,
+) -> std::io::Result<AuthenticatedPeerReplayCapabilities> {
+    let secret = get_shared_secret()?;
+    let boot_epoch = verify_tonic_boot_epoch_response_with_secret(&secret, audience, challenge, headers)?;
+    let capability = headers.get(RPC_REPLAY_CACHE_CAPABILITY_HEADER);
+    let proof = headers.get(RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER);
+    if capability.is_none() && proof.is_none() {
+        return Ok(AuthenticatedPeerReplayCapabilities {
+            boot_epoch,
+            dynamic_replay_cache: false,
+        });
+    }
+    let capability = capability
+        .and_then(|value| value.to_str().ok())
+        .ok_or_else(|| std::io::Error::other("Missing RPC replay cache capability"))?;
+    if capability != RPC_REPLAY_CACHE_CAPABILITY_V1 {
+        return Err(std::io::Error::other("Unsupported RPC replay cache capability"));
+    }
+    let proof = proof
+        .and_then(|value| value.to_str().ok())
+        .ok_or_else(|| std::io::Error::other("Missing RPC replay cache capability proof"))?;
+    verify_replay_cache_capability_proof(&secret, audience, challenge, boot_epoch, proof)?;
+    Ok(AuthenticatedPeerReplayCapabilities {
+        boot_epoch,
+        dynamic_replay_cache: true,
+    })
+}
+
 fn valid_content_sha256(value: &str) -> bool {
     value == UNSIGNED_PAYLOAD
         || (value.len() == 64
@@ -1103,6 +1212,23 @@ pub fn set_tonic_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(
     set_tonic_canonical_body_digest(request, &canonical_body)
 }
 
+pub fn set_tonic_rolling_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(
+    request: &mut tonic::Request<T>,
+) -> std::io::Result<()> {
+    set_tonic_mutation_body_digest(request)?;
+    request.extensions_mut().insert(RollingMutationBodyDigest);
+    Ok(())
+}
+
+pub fn set_tonic_rolling_canonical_body_digest<T>(request: &mut tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
+    set_tonic_canonical_body_digest(request, canonical_body)?;
+    request.extensions_mut().insert(RollingMutationBodyDigest);
+    Ok(())
+}
+
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct RollingMutationBodyDigest;
+
 pub fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
     let version = request
         .metadata()
@@ -1139,7 +1265,7 @@ pub fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canoni
 /// including v1-downgraded ones. It converges independently of the signature-strict switch
 /// (<https://github.com/rustfs/backlog/issues/1327>).
 pub fn verify_tonic_mutation_body_digest<T>(request: &tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
-    verify_tonic_mutation_body_digest_with_strictness(request, canonical_body, *INTERNODE_RPC_BODY_DIGEST_STRICT)
+    verify_tonic_mutation_body_digest_with_strictness(request, canonical_body, internode_rpc_body_digest_strict())
 }
 
 /// [`verify_tonic_mutation_body_digest`] with the strict gate injected as a parameter, so both
@@ -2192,6 +2318,23 @@ mod tests {
         assert!(verify_tonic_boot_epoch_response("node-a:9000", Uuid::new_v4(), &headers).is_err());
     }
 
+    #[test]
+    fn replay_cache_capability_proof_binds_audience_challenge_epoch_and_value() {
+        ensure_test_rpc_secret();
+        let challenge = Uuid::new_v4();
+        let headers = tonic_boot_epoch_response_headers("node-a:9000", challenge).expect("capability headers should build");
+        let capabilities = verify_tonic_peer_replay_capabilities_response("node-a:9000", challenge, &headers)
+            .expect("matching capability proof should verify");
+        assert_eq!(capabilities.boot_epoch, tonic_rpc_boot_epoch());
+        assert!(capabilities.dynamic_replay_cache);
+        assert!(verify_tonic_peer_replay_capabilities_response("node-b:9000", challenge, &headers).is_err());
+        assert!(verify_tonic_peer_replay_capabilities_response("node-a:9000", Uuid::new_v4(), &headers).is_err());
+
+        let mut changed_capability = headers;
+        changed_capability.insert(RPC_REPLAY_CACHE_CAPABILITY_HEADER, HeaderValue::from_static("dynamic-replay-cache-v2"));
+        assert!(verify_tonic_peer_replay_capabilities_response("node-a:9000", challenge, &changed_capability).is_err());
+    }
+
     #[test]
     fn tonic_rpc_auth_failure_reason_maps_security_relevant_errors() {
         for (message, reason) in [
@@ -2552,21 +2695,27 @@ mod tests {
 
         assert_eq!(decision.source, ReplayCacheCapacitySource::Auto);
         assert_eq!(decision.memory_basis, Some(MemoryBasis::Host));
-        assert_eq!(decision.memory_based_capacity, 10_737_418);
+        assert_eq!(decision.memory_based_capacity, 17_448_304);
-        assert_eq!(decision.cpu_based_capacity, 9_846_784);
+        assert_eq!(decision.cpu_based_capacity, 19_693_568);
-        assert_eq!(decision.capacity, 9_846_784);
+        assert_eq!(decision.capacity, 17_448_304);
     }
 
     #[test]
-    fn replay_cache_capacity_auto_uses_resource_model_on_larger_nodes() {
+    fn replay_cache_capacity_auto_uses_32m_on_field_sized_nodes() {
         let gib = 1024_u64 * 1024 * 1024;
         let decision =
             replay_cache_capacity_decision(rustfs_utils::EnvParseOutcome::Absent, 16, Some(32 * gib), Some(MemoryBasis::Host));
 
         assert_eq!(decision.source, ReplayCacheCapacitySource::Auto);
-        assert_eq!(decision.memory_based_capacity, 21_474_836);
+        assert_eq!(decision.memory_based_capacity, 34_896_609);
-        assert_eq!(decision.cpu_based_capacity, 19_693_568);
+        assert_eq!(decision.cpu_based_capacity, 39_387_136);
-        assert_eq!(decision.capacity, 19_693_568);
+        assert_eq!(decision.capacity, REPLAY_CACHE_AUTO_MAX_CAPACITY);
+
+        let observed_field_node =
+            replay_cache_capacity_decision(rustfs_utils::EnvParseOutcome::Absent, 16, Some(31 * gib), Some(MemoryBasis::Host));
+        assert_eq!(observed_field_node.memory_based_capacity, 33_806_090);
+        assert_eq!(observed_field_node.cpu_based_capacity, 39_387_136);
+        assert_eq!(observed_field_node.capacity, REPLAY_CACHE_AUTO_MAX_CAPACITY);
     }
 
     #[test]
@@ -2598,7 +2747,7 @@ mod tests {
         let decision = replay_cache_capacity_decision(rustfs_utils::EnvParseOutcome::Invalid, 8, None, None);
 
         assert_eq!(decision.source, ReplayCacheCapacitySource::AutoInvalidEnv);
-        assert_eq!(decision.capacity, 9_846_784);
+        assert_eq!(decision.capacity, 19_693_568);
     }
 
     fn check_test_nonce_record(cache: &mut RpcNonceCache, record: RpcNonceRecord<'_>) -> std::io::Result<()> {

crates/ecstore/src/cluster/rpc/mod.rs

@@ -34,11 +34,12 @@ pub use client::{
 pub use http_auth::{
     TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, gen_signature_headers,
     gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
-    set_tonic_mutation_body_digest, sign_ns_scanner_capability, sign_put_file_capability, sign_tonic_rpc_response_proof,
+    set_tonic_mutation_body_digest, set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest,
-    tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability,
+    sign_ns_scanner_capability, sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge,
-    verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
+    tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_put_file_auth_trailer,
-    verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
+    verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
-    verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
+    verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
+    verify_tonic_rpc_signature_with_bootstrap,
 };
 #[cfg(test)]
 pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -16,7 +16,6 @@ use crate::cluster::rpc::client::{
     AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error,
     node_service_time_out_client, node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
 };
-use crate::cluster::rpc::http_auth::set_tonic_canonical_body_digest;
 use crate::cluster::rpc::internode_data_transport::{
     InternodeDataTransport, NsScannerCapabilityRequest, NsScannerStreamRequest, ReadStreamRequest, WalkDirStreamRequest,
     WriteStreamRequest,
@@ -123,7 +122,7 @@ fn attach_mutation_body_digest<T>(
     op: &'static str,
 ) -> Result<()> {
     let canonical_body = canonical_body.map_err(|_| Error::other(format!("{op} request length cannot be represented")))?;
-    set_tonic_canonical_body_digest(request, &canonical_body).map_err(Error::other)
+    crate::cluster::rpc::set_tonic_rolling_canonical_body_digest(request, &canonical_body).map_err(Error::other)
 }
 
 fn decode_volume_infos(volume_infos: Vec<String>) -> Result<Vec<VolumeInfo>> {
@@ -3029,6 +3028,22 @@ mod tests {
 
     static INIT: Once = Once::new();
 
+    #[test]
+    fn disk_mutation_digest_marks_rolling_compatibility() {
+        let mut request = Request::new(());
+
+        attach_mutation_body_digest(&mut request, Ok(b"canonical disk mutation".to_vec()), "WriteAll")
+            .expect("disk mutation digest must be attached");
+
+        assert!(
+            request
+                .extensions()
+                .get::<crate::cluster::rpc::http_auth::RollingMutationBodyDigest>()
+                .is_some(),
+            "remote-disk mutations must reach the cache-free compatibility gate"
+        );
+    }
+
     // `#[serial(internode_metrics)]` marks every test that observes
     // `global_internode_metrics()`. Those counters are a process-wide singleton:
     // some of these tests snapshot a counter, run one decode, and assert on the

crates/ecstore/src/cluster/rpc/remote_locker.rs

@@ -15,7 +15,7 @@
 use crate::cluster::rpc::client::{
     AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client,
 };
-use crate::cluster::rpc::set_tonic_mutation_body_digest;
+use crate::cluster::rpc::set_tonic_rolling_mutation_body_digest;
 use async_trait::async_trait;
 use bytes::Bytes;
 use rustfs_lock::{
@@ -33,6 +33,10 @@ use tonic::Request;
 use tonic::service::interceptor::InterceptedService;
 use tracing::{debug, info, warn};
 
+fn attach_lock_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(request: &mut Request<T>) -> std::io::Result<()> {
+    set_tonic_rolling_mutation_body_digest(request)
+}
+
 /// Remote lock client implementation
 #[derive(Debug, Clone)]
 pub struct RemoteClient {
@@ -319,7 +323,7 @@ impl LockClient for RemoteClient {
             args: serde_json::to_string(&request)
                 .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
         });
-        set_tonic_mutation_body_digest(&mut req)?;
+        attach_lock_mutation_body_digest(&mut req)?;
 
         let resp = match self.execute_rpc("lock", &resource_summary, client.lock(req)).await {
             Ok(resp) => resp.into_inner(),
@@ -358,7 +362,7 @@ impl LockClient for RemoteClient {
                 })
                 .collect::<Result<Vec<_>>>()?,
         });
-        set_tonic_mutation_body_digest(&mut req)?;
+        attach_lock_mutation_body_digest(&mut req)?;
 
         let resp = match self
             .execute_rpc("lock_batch", &resource_summary, client.lock_batch(req))
@@ -400,7 +404,7 @@ impl LockClient for RemoteClient {
         let mut client = self.get_client().await?;
         let resource_summary = unlock_request.resource.to_string();
         let mut req = Request::new(GenerallyLockRequest { args: request_string });
-        set_tonic_mutation_body_digest(&mut req)?;
+        attach_lock_mutation_body_digest(&mut req)?;
         let resp = self
             .execute_rpc("release", &resource_summary, client.un_lock(req))
             .await?
@@ -427,7 +431,7 @@ impl LockClient for RemoteClient {
                 })
                 .collect::<Result<Vec<_>>>()?,
         });
-        set_tonic_mutation_body_digest(&mut req)?;
+        attach_lock_mutation_body_digest(&mut req)?;
 
         let resp = self
             .execute_rpc("release_batch", &resource_summary, client.un_lock_batch(req))
@@ -450,7 +454,7 @@ impl LockClient for RemoteClient {
             args: serde_json::to_string(&refresh_request)
                 .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
         });
-        set_tonic_mutation_body_digest(&mut req)?;
+        attach_lock_mutation_body_digest(&mut req)?;
         let resp = self
             .execute_rpc("refresh", &resource_summary, client.refresh(req))
             .await?
@@ -470,7 +474,7 @@ impl LockClient for RemoteClient {
             args: serde_json::to_string(&force_request)
                 .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
         });
-        set_tonic_mutation_body_digest(&mut req)?;
+        attach_lock_mutation_body_digest(&mut req)?;
         let resp = self
             .execute_rpc("force_release", &resource_summary, client.force_un_lock(req))
             .await?
@@ -495,7 +499,7 @@ impl LockClient for RemoteClient {
             args: serde_json::to_string(&status_request)
                 .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
         });
-        set_tonic_mutation_body_digest(&mut req)?;
+        attach_lock_mutation_body_digest(&mut req)?;
 
         // Try exclusive lock first with very short timeout
         let resp = match self.execute_rpc("check_status", &resource_summary, client.lock(req)).await {
@@ -510,7 +514,7 @@ impl LockClient for RemoteClient {
                 args: serde_json::to_string(&status_request)
                     .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
             });
-            set_tonic_mutation_body_digest(&mut release_req)?;
+            attach_lock_mutation_body_digest(&mut release_req)?;
             let _ = self
                 .execute_rpc("check_status_release", &resource_summary, client.un_lock(release_req))
                 .await;
@@ -626,6 +630,31 @@ mod tests {
             .with_priority(LockPriority::Normal)
     }
 
+    #[test]
+    fn lock_mutation_helper_marks_single_and_batch_requests_for_rolling_auth() {
+        let mut single = Request::new(GenerallyLockRequest {
+            args: "single-lock".to_string(),
+        });
+        attach_lock_mutation_body_digest(&mut single).expect("single lock digest must be attached");
+        assert!(
+            single
+                .extensions()
+                .get::<crate::cluster::rpc::http_auth::RollingMutationBodyDigest>()
+                .is_some()
+        );
+
+        let mut batch = Request::new(BatchGenerallyLockRequest {
+            args: vec!["batch-lock".to_string()],
+        });
+        attach_lock_mutation_body_digest(&mut batch).expect("batch lock digest must be attached");
+        assert!(
+            batch
+                .extensions()
+                .get::<crate::cluster::rpc::http_auth::RollingMutationBodyDigest>()
+                .is_some()
+        );
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn test_remote_client_acquire_lock_uses_rpc_timeout_and_evicts_connection() {

crates/ecstore/src/config/com.rs

@@ -584,6 +584,44 @@ where
     .await
 }
 
+/// `delete_config` with `no_lock` set — for callers already holding the
+/// config object's namespace lock (e.g. inside `with_config_object_write_lock`),
+/// where the locked variant would self-deadlock.
+pub async fn delete_config_no_lock<S>(api: Arc<S>, file: &str) -> Result<()>
+where
+    S: ObjectOperations<
+            Error = Error,
+            ObjectInfo = ObjectInfo,
+            ObjectOptions = ObjectOptions,
+            FileInfo = FileInfo,
+            ObjectToDelete = ObjectToDelete,
+            DeletedObject = DeletedObject,
+        >,
+{
+    match api
+        .delete_object(
+            RUSTFS_META_BUCKET,
+            file,
+            ObjectOptions {
+                delete_prefix: true,
+                delete_prefix_object: true,
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+    {
+        Ok(_) => Ok(()),
+        Err(err) => {
+            if err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _)) {
+                Err(Error::ConfigNotFound)
+            } else {
+                Err(err)
+            }
+        }
+    }
+}
+
 #[instrument(skip(api))]
 pub async fn delete_config<S>(api: Arc<S>, file: &str) -> Result<()>
 where

crates/ecstore/src/disk/local.rs

@@ -7890,6 +7890,11 @@ impl DiskAPI for LocalDisk {
                 std::io::Write::write_all(&mut new_meta, &meta)?;
                 if durability.syncs_commit_metadata() {
                     new_meta.sync_data()?;
+                }
+                // Windows rejects renaming a directory while one of its children is
+                // still open, even when the child handle shares delete access.
+                drop(new_meta);
+                if durability.syncs_commit_metadata() {
                     os::fsync_dir_std(&staging_path)?;
                 }
                 std::fs::rename(&staging_path, &transaction_path)?;
@@ -8096,7 +8101,7 @@ impl DiskAPI for LocalDisk {
         let durability = effective_durability(dst_volume);
         if durability.syncs_data_shards() && !src_is_dir {
             let src = src_file_path.clone();
-            tokio::task::spawn_blocking(move || std::fs::File::open(&src)?.sync_data())
+            tokio::task::spawn_blocking(move || os::sync_file(&src))
                 .await
                 .map_err(DiskError::from)?
                 .map_err(to_file_error)?;
@@ -11566,6 +11571,116 @@ mod test {
         );
     }
 
+    #[cfg(windows)]
+    #[tokio::test]
+    #[allow(clippy::await_holding_lock)]
+    async fn test_rename_part_commits_realistic_windows_multipart_path() {
+        use crate::disk::RUSTFS_META_MULTIPART_BUCKET;
+        use tempfile::tempdir;
+
+        let _mode = durability_mode_override::set(DurabilityMode::Strict);
+        assert_eq!(effective_durability(RUSTFS_META_MULTIPART_BUCKET), DurabilityMode::Strict);
+        let dir = tempdir().expect("temp dir should be created");
+        let root = dir.path().join("realistic-windows-multipart-root");
+        fs::create_dir_all(&root).await.expect("disk root should be created");
+        let endpoint = Endpoint::try_from(root.to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
+        let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
+        ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
+        ensure_test_volume(&disk, RUSTFS_META_MULTIPART_BUCKET).await;
+
+        let src_path = "upload/part.1";
+        let dst_path = concat!(
+            "6f897928dfe04a87a269ccd9f5a5897d9cbbdf6b55e4d903ef3cbc1125c0cb8f/",
+            "8f897819-2604-4f3d-b843-c32a45d198b2x1786372838834745500/",
+            "58ba822c-06e4-4332-81cc-be2c9d921900/part.1"
+        );
+        let transaction_path = disk
+            .io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, &crate::disk::part_transaction_path(dst_path))
+            .expect("transaction path should resolve");
+        let deepest_marker = transaction_path
+            .parent()
+            .expect("transaction path should have a parent")
+            .join(".part-txn-00000000-0000-0000-0000-000000000000")
+            .join(PART_TRANSACTION_OLD_DATA_ABSENT);
+        assert!(
+            deepest_marker.as_os_str().len() > 260,
+            "regression path must cross the traditional Windows MAX_PATH boundary: {deepest_marker:?}"
+        );
+        let payload = Bytes::from_static(b"part payload");
+        let meta = Bytes::from_static(b"part metadata");
+        disk.write_all(RUSTFS_META_TMP_BUCKET, src_path, payload.clone())
+            .await
+            .expect("source part should be written");
+
+        disk.prepare_part_transaction(RUSTFS_META_TMP_BUCKET, src_path, RUSTFS_META_MULTIPART_BUCKET, dst_path, meta.clone())
+            .await
+            .expect("realistic Windows part transaction should be prepared");
+        disk.rename_part(RUSTFS_META_TMP_BUCKET, src_path, RUSTFS_META_MULTIPART_BUCKET, dst_path, meta.clone())
+            .await
+            .expect("realistic Windows part should be committed");
+        disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, dst_path, PartTransactionAction::Commit)
+            .await
+            .expect("realistic Windows part transaction should be settled");
+
+        assert_eq!(
+            disk.read_all(RUSTFS_META_MULTIPART_BUCKET, dst_path)
+                .await
+                .expect("destination part should be readable"),
+            payload
+        );
+        assert_eq!(
+            disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &format!("{dst_path}.meta"))
+                .await
+                .expect("destination metadata should be readable"),
+            meta
+        );
+
+        let replacement_payload = Bytes::from_static(b"replacement part payload");
+        let replacement_meta = Bytes::from_static(b"replacement part metadata");
+        disk.write_all(RUSTFS_META_TMP_BUCKET, src_path, replacement_payload.clone())
+            .await
+            .expect("replacement source part should be written");
+        disk.prepare_part_transaction(
+            RUSTFS_META_TMP_BUCKET,
+            src_path,
+            RUSTFS_META_MULTIPART_BUCKET,
+            dst_path,
+            replacement_meta.clone(),
+        )
+        .await
+        .expect("replacement Windows part transaction should be prepared");
+        disk.rename_part(
+            RUSTFS_META_TMP_BUCKET,
+            src_path,
+            RUSTFS_META_MULTIPART_BUCKET,
+            dst_path,
+            replacement_meta.clone(),
+        )
+        .await
+        .expect("replacement Windows part should be committed");
+        disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, dst_path, PartTransactionAction::Commit)
+            .await
+            .expect("replacement Windows part transaction should be settled");
+
+        assert_eq!(
+            disk.read_all(RUSTFS_META_MULTIPART_BUCKET, dst_path)
+                .await
+                .expect("replacement destination part should be readable"),
+            replacement_payload
+        );
+        assert_eq!(
+            disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &format!("{dst_path}.meta"))
+                .await
+                .expect("replacement destination metadata should be readable"),
+            replacement_meta
+        );
+        assert!(
+            matches!(disk.read_all(RUSTFS_META_TMP_BUCKET, src_path).await, Err(DiskError::FileNotFound)),
+            "successful replacement must remove its source part"
+        );
+        assert!(!transaction_path.exists(), "settled replacement must remove its transaction directory");
+    }
+
     #[tokio::test]
     async fn test_part_transaction_rolls_back_data_published_before_metadata() {
         use tempfile::tempdir;

crates/ecstore/src/disk/os.rs

@@ -497,7 +497,7 @@ pub(crate) mod file_sync_probe {
     }
 }
 
-fn sync_file(path: &Path) -> io::Result<()> {
+pub(crate) fn sync_file(path: &Path) -> io::Result<()> {
     #[cfg(test)]
     let _probe = file_sync_probe::enter(path);
     #[cfg(test)]

crates/replication/src/operation.rs

@@ -635,6 +635,38 @@ mod tests {
         assert!(!should_use_existing_delete_replication_info(false, false));
     }
 
+    /// P1-20 truth-table pin (rustfs/backlog#1675): without any reset in play
+    /// (no per-target reset header on the object, empty reset id on the
+    /// target) the existing-object resync decision compensates exactly the
+    /// never-replicated objects — Empty replicates, any recorded status does
+    /// not.
+    #[test]
+    fn resync_target_without_reset_replicates_only_empty_status() {
+        let user_defined = HashMap::new();
+        let object = ReplicationResyncTargetObject {
+            mod_time: Some(OffsetDateTime::UNIX_EPOCH + Duration::seconds(10)),
+            user_defined: &user_defined,
+        };
+
+        for (status, expected) in [
+            (ReplicationStatusType::Empty, true),
+            (ReplicationStatusType::Completed, false),
+            // "COMPLETE" on disk parses to this legacy variant, so objects
+            // written by older versions reach the decision through it.
+            (ReplicationStatusType::CompletedLegacy, false),
+            (ReplicationStatusType::Pending, false),
+            (ReplicationStatusType::Failed, false),
+            (ReplicationStatusType::Replica, false),
+        ] {
+            let label = format!("{status:?}");
+            let decision = resync_target_for_object(&object, "arn:target", "", None, status);
+            assert_eq!(
+                decision.replicate, expected,
+                "existing-object resync without a reset must replicate only never-replicated objects (status {label})"
+            );
+        }
+    }
+
     #[test]
     fn resync_target_includes_object_at_reset_before_boundary() {
         let reset_before = OffsetDateTime::UNIX_EPOCH + Duration::seconds(30);

crates/replication/src/queue.rs

@@ -360,6 +360,48 @@ mod tests {
         );
     }
 
+    /// P1-20 truth-table pin (rustfs/backlog#1675): when no target replicates
+    /// — the decision is empty because ExistingObjectReplication is Disabled
+    /// for a never-replicated object, or because the object is an inbound
+    /// REPLICA (must_replicate returns an empty decision for those) — the
+    /// heal pass must skip entirely, whatever the recorded status says. The
+    /// scanner never compensates these objects.
+    #[test]
+    fn heal_queue_action_skips_when_no_target_replicates() {
+        for status in [
+            ReplicationStatusType::Empty,
+            ReplicationStatusType::Failed,
+            ReplicationStatusType::Replica,
+        ] {
+            let mut roi = ReplicateObjectInfo {
+                bucket: "bucket".to_string(),
+                name: "object".to_string(),
+                replication_status: status,
+                dsc: ReplicateDecision::new(),
+                ..Default::default()
+            };
+
+            let action = replication_heal_queue_action(&mut roi);
+
+            assert!(
+                matches!(action, ReplicationHealQueueAction::Skip),
+                "an empty replicate decision must skip heal queueing (status {:?})",
+                roi.replication_status
+            );
+        }
+    }
+
+    /// P1-20 truth-table pin: a Completed object with no resync decision has
+    /// nothing left to heal — the scanner must not requeue it.
+    #[test]
+    fn heal_queue_action_skips_completed_object_without_resync() {
+        let mut roi = replicate_object_info(ReplicationStatusType::Completed);
+
+        let action = replication_heal_queue_action(&mut roi);
+
+        assert!(matches!(action, ReplicationHealQueueAction::Skip));
+    }
+
     #[test]
     fn heal_queue_action_routes_failed_objects_to_heal_queue() {
         let mut roi = replicate_object_info(ReplicationStatusType::Failed);

crates/scanner/src/scanner_io.rs

@@ -3881,7 +3881,7 @@ impl ScannerIODisk for Disk {
         Ok(size_summary)
     }
 
-    #[tracing::instrument(skip(self, budget, updates, cache))]
+    #[tracing::instrument(skip(self, budget, updates, cache, set_disks))]
     async fn nsscanner_disk(
         self: Arc<Self>,
         ctx: CancellationToken,

docs/architecture/compat-cleanup-register.md

@@ -23,7 +23,7 @@ for later deletion.
 - `ns-scanner-rpc-v3` namespace scanner capability and activity handshake: old peers and legacy internode transports lack the authenticated startup-epoch handshake. The oldest peers send an empty activity request and receive a field-empty protocol-0 response. Protocol v4 binds the challenge and response topology but cannot authenticate distributed dirty-usage state. Protocol v5 binds the request version, acknowledgement target and generation, and the response dirty-usage state, but predates set-scoped scanner cache locks. Current protocol v6 additionally fences scanner cache lock-domain changes, so distributed scanner cycles publish usage only after every peer reports protocol v6 state. Servers retain protocol-0 and protocol-v4 codecs for rolling upgrades, while protocol-v5 peers are treated as previous-version peers that cannot safely participate in the new cache lock domain. Scanner selection treats HTTP 404/405/426 and the legacy MethodNotAllowed default as an explicit lack of remote scanner v3 support and assigns those disks to coordinator-driven workers; transient capability failures remain incomplete and do not activate the fallback. Remove the coordinator fallback after the minimum supported RustFS peer version implements namespace scanner protocol v3, remove protocol-0 activity requests and responses after every supported peer implements authenticated scanner activity protocol v4, remove the protocol-v4 activity codec after every supported peer implements protocol v5, and remove protocol-v5 previous-version rejection after every supported peer implements protocol v6; future protocol revisions must keep the same dual-version server/codec window before changing the advertised version.
 - `#4648` walk-dir stream completion capability: old clients can append fallback output to an already-used metacache writer after a terminal body error, so servers emit terminal walk errors only to clients that sign the `walk_dir_stream_completion=error-v1` query capability and its request-body digest. Remove the legacy clean-EOF path after the minimum supported RustFS peer version always advertises this capability.
 - `heal-rpc-auth-v2` internode gRPC authentication: servers temporarily accept legacy prefix signatures so old peers remain available during rolling upgrades. Remove the legacy fallback after the minimum supported RustFS peer version sends v2 authentication on every internode gRPC request.
-- `disk-mutation-body-digest` internode mutating disk RPCs: servers temporarily accept mutating disk RPCs (RenameData, DeleteVersion, DeleteVersions, WriteMetadata, UpdateMetadata, WriteAll, Delete, DeletePaths, RenameFile, RenamePart, DeleteVolume, MakeVolume, MakeVolumes) that carry no signature-bound canonical body digest, so peers from releases that predate body-digest signing remain available during rolling upgrades. Accepted digestless mutations increment the internode body-digest fallback counter; that counter must read zero fleet-wide across a release window before RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT is enabled. Because body-bound requests now consume replay-cache nonces on the receiver, deploy the raised RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY default fleet-wide before enabling strict mode, and watch the internode replay-cache overflow counter for undersized capacity during the rollout. Remove the digestless fallback after the minimum supported RustFS peer version body-binds every mutating disk RPC.
+- `disk-mutation-body-digest` rolling-compatible internode mutations: servers temporarily accept high-frequency lock and disk mutations that carry no signature-bound canonical body digest, so older peers remain available during rolling upgrades. Clients use the authenticated but cache-free UNSIGNED-PAYLOAD v2 lane only for those explicitly marked mutations when a peer has no authenticated boot-epoch proof, as on beta.11. A peer with an authenticated boot epoch but no dynamic-cache capability, as on beta.12 or an unpatched RC1, receives body-bound v3 requests; a patched peer additionally pins the separately HMAC-bound dynamic-replay-cache-v1 capability. After that capability is pinned, a missing or invalid capability proof fails closed instead of silently permitting a rollback; an intentional rollback requires restarting the client process to clear the in-memory pin. IAM, service-control, heal-control, tier-mutation, and scanner-activity contracts remain body-bound. Accepted digestless mutations increment the internode body-digest fallback counter; that counter must read zero fleet-wide across a release window before RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT is enabled. Because body-bound requests consume replay-cache nonces on the receiver, deploy the raised RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY default fleet-wide before enabling strict mode, and watch the internode replay-cache overflow counter for undersized capacity during the rollout. Remove both client and server fallbacks after every supported peer advertises the authenticated replay-cache capability, body-binds every rolling-compatible mutation, and body-digest strict mode is the default.
 - `heal-status-rpc-v1` node heal status capability: new peers treat an unimplemented BackgroundHealStatus RPC as an explicitly incomplete rolling-upgrade response. Remove the fallback after the minimum supported RustFS peer version implements BackgroundHealStatus.
 - `replacement-recovery-status-v1` replacement recovery status capability: new peers treat an unimplemented ReplacementRecoveryStatus RPC as an explicitly non-definitive rolling-upgrade response, so Admin v4 cannot claim distributed replacement completion from old peers. Remove the fallback after the minimum supported RustFS peer version implements ReplacementRecoveryStatus.
 - `backlog-1316` legacy encrypted multipart range seek: the feature remains opt-in until every server that can initiate, write, or complete multipart uploads supports the candidate-to-final marker protocol and uploadId commit lock, and pre-upgrade multipart uploads have drained. Remove the RUSTFS_ENCRYPTED_RANGE_SEEK switch after the minimum supported release does so; keep the quorum marker and malformed-layout full-read guards permanently.

docs/operations/replication-check.md

@@ -42,6 +42,7 @@ target results remain present when another target fails.
         "Versioning": { "Status": "OK" },
         "ObjectLock": { "Status": "OK" },
         "Put": { "Status": "OK" },
+        "VersionFidelity": { "Status": "OK" },
         "DeleteMarker": { "Status": "OK" },
         "VersionDelete": { "Status": "OK" },
         "Cleanup": {
@@ -58,3 +59,19 @@ Phase states are `OK`, `FAILED`, or `SKIPPED`. Errors are single-line, bounded
 to 512 bytes, and omit remote messages, endpoints, credentials, signatures, and
 authorization material. A cleanup failure is always explicit; it is never
 reported as a successful check.
+
+`VersionFidelity` pins the version-identity contract on **both** write paths:
+the probe PUT carries a source version id (header plus `?versionId=` query,
+the exact shape live replication uses) and the target must answer with the
+same id, and a second probe repeats it through CreateMultipartUpload ->
+UploadPart -> CompleteMultipartUpload, where the target fixes the version at
+initiate and only reports it on completion. A target can adopt PutObject ids
+and still mint its own for multipart, which would leave multipart deletes and
+heals addressing a version that never existed; the failure message names the
+path that drifted. Targets that
+mint their own version ids break every version-addressed operation that
+follows (version deletes, heal re-drives), so the phase fails with the
+machine-readable extension key `"Code": "BucketRemoteTargetVersionMismatch"`,
+the later mutation phases are skipped, and cleanup still removes the probe via
+the version id the target actually assigned. `Code` only appears on failures
+that callers are expected to branch on; Go decoders ignore the unknown key.

rustfs/src/admin/console.rs

@@ -28,7 +28,7 @@ use crate::server::{
 };
 use crate::version::build;
 use axum::{
-    Json, Router,
+    Extension, Json, Router,
     body::Body,
     extract::Request,
     middleware,
@@ -632,13 +632,22 @@ fn setup_console_middleware_stack(
 /// # Returns:
 /// - A `Response` containing the health check result.
 #[instrument]
-async fn health_check(method: Method, uri: Uri) -> Response {
+async fn health_check(
+    method: Method,
+    uri: Uri,
+    server_ctx: Option<Extension<Arc<crate::runtime_sources::ServerContextSlot>>>,
+) -> Response {
     let probe = if uri.path().strip_prefix(CONSOLE_PREFIX) == Some(HEALTH_READY_PATH) {
         HealthProbe::Readiness
     } else {
         HealthProbe::Liveness
     };
-    let readiness_report = collect_probe_readiness(probe).await;
+    let app_context = match server_ctx {
+        Some(Extension(server_ctx)) => server_ctx.installed_app_context(),
+        None => crate::runtime_sources::current_app_context(),
+    };
+    let object_traffic_health = app_context.map(|context| context.object_traffic_health());
+    let readiness_report = collect_probe_readiness(probe, object_traffic_health.as_deref()).await;
     let uptime = std::time::SystemTime::now()
         .duration_since(std::time::UNIX_EPOCH)
         .unwrap_or_default()
@@ -919,6 +928,44 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn console_readiness_uses_the_request_server_object_progress() {
+        temp_env::async_with_vars([(rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("false"))], async {
+            let object_traffic_health =
+                Arc::new(crate::app::object_traffic_health::ObjectTrafficHealth::enabled_for_test(Duration::ZERO));
+            let stalled = object_traffic_health
+                .track_write_storage()
+                .expect("write tracking must be enabled");
+            let app_context =
+                crate::app::gating_test_env::app_context_with_object_traffic_health(Arc::clone(&object_traffic_health)).await;
+            let server_ctx = crate::runtime_sources::ServerContextSlot::new();
+            assert!(server_ctx.install(app_context));
+
+            let response = health_check(
+                Method::GET,
+                format!("{CONSOLE_PREFIX}{HEALTH_READY_PATH}")
+                    .parse()
+                    .expect("console readiness URI"),
+                Some(Extension(server_ctx)),
+            )
+            .await;
+
+            assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
+            let body = response
+                .into_body()
+                .collect()
+                .await
+                .expect("console readiness body")
+                .to_bytes();
+            let payload: serde_json::Value = serde_json::from_slice(&body).expect("console readiness JSON");
+            assert_eq!(payload["ready"], false);
+            assert_eq!(payload["degradedReasons"], serde_json::json!(["object_write_stalled"]));
+            drop(stalled);
+        })
+        .await;
+    }
+
     // setup_console_middleware_stack reads ENV_HEALTH_ENDPOINT_ENABLE (see above).
     #[tokio::test]
     #[serial]

rustfs/src/admin/handlers/health.rs

@@ -14,6 +14,7 @@
 
 use super::profile::{TriggerProfileCPU, TriggerProfileMemory};
 use crate::admin::router::{AdminOperation, Operation, S3Router};
+use crate::admin::runtime_sources::app_context_from_req;
 use crate::server::{
     HEALTH_PREFIX, HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, build_health_response_parts,
     collect_probe_readiness, probe_from_path,
@@ -51,6 +52,7 @@ pub struct HealthCheckHandler {}
 #[async_trait::async_trait]
 impl Operation for HealthCheckHandler {
     async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let object_traffic_health = app_context_from_req(&req).map(|context| context.object_traffic_health());
         // Extract the original HTTP Method (encapsulated by s3s into S3Request)
         let method = req.method;
 
@@ -66,7 +68,7 @@ impl Operation for HealthCheckHandler {
         }
 
         let probe = probe_from_path(req.uri.path());
-        let readiness_report = collect_probe_readiness(probe).await;
+        let readiness_report = collect_probe_readiness(probe, object_traffic_health.as_deref()).await;
 
         let response_parts =
             build_health_response_parts(method.clone(), probe, readiness_report.as_ref(), "rustfs-endpoint", None, None);

rustfs/src/admin/mod.rs

@@ -24,6 +24,7 @@ pub mod router;
 pub(crate) mod runtime_sources;
 pub mod service;
 pub mod site_replication_identity;
+pub(crate) mod site_replication_state;
 pub(crate) mod storage_api;
 pub mod utils;
 

rustfs/src/admin/router.rs

@@ -17,7 +17,7 @@ use super::storage_api::bucket::metadata_sys;
 use super::storage_api::bucket::replication::{self, BucketReplicationResyncStatus, BucketStats, ReplicationStatusType};
 use super::storage_api::bucket::target::{BucketTarget, BucketTargetType, BucketTargets};
 use super::storage_api::bucket::target_sys::{
-    BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError, TargetClient,
+    BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError, TargetClient, append_version_id_query,
 };
 use super::storage_api::bucket::versioning_sys::BucketVersioningSys;
 use super::storage_api::bucket::{AdminReplicationConfigExt as _, AdminVersioningConfigExt as _};
@@ -42,6 +42,7 @@ use crate::server::{
 };
 use crate::storage::storage_api::lock_bucket_targets_metadata;
 use aws_sdk_s3::primitives::ByteStream as AwsByteStream;
+use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
 use bytes::Bytes;
 use futures::{Stream, StreamExt};
 use http::HeaderValue;
@@ -206,6 +207,9 @@ struct ReplicationResetStatusTarget {
 
 const REPLICATION_CHECK_PROBE_PREFIX: &str = ".rustfs.sys/replication-check/";
 const REPLICATION_CHECK_ERROR_MAX_BYTES: usize = 512;
+/// RustFS extension code (no madmin analogue): the target does not adopt the
+/// source version id, breaking the version-identity replication contract.
+const REPLICATION_CHECK_CODE_VERSION_MISMATCH: &str = "BucketRemoteTargetVersionMismatch";
 
 #[derive(Debug, Clone, serde::Serialize)]
 struct ReplicationCheckResponse {
@@ -245,6 +249,8 @@ struct ReplicationCheckPhases {
     object_lock: ReplicationCheckPhaseStatus,
     #[serde(rename = "Put")]
     put: ReplicationCheckPhaseStatus,
+    #[serde(rename = "VersionFidelity")]
+    version_fidelity: ReplicationCheckPhaseStatus,
     #[serde(rename = "DeleteMarker")]
     delete_marker: ReplicationCheckPhaseStatus,
     #[serde(rename = "VersionDelete")]
@@ -259,6 +265,11 @@ struct ReplicationCheckPhaseStatus {
     status: &'static str,
     #[serde(rename = "Error", skip_serializing_if = "Option::is_none")]
     error: Option<String>,
+    /// Machine-readable failure code (RustFS extension key; Go decoders
+    /// ignore unknown keys). Only set for failures that a caller is expected
+    /// to branch on, e.g. `BucketRemoteTargetVersionMismatch`.
+    #[serde(rename = "Code", skip_serializing_if = "Option::is_none")]
+    code: Option<&'static str>,
 }
 
 impl Default for ReplicationCheckPhaseStatus {
@@ -266,6 +277,7 @@ impl Default for ReplicationCheckPhaseStatus {
         Self {
             status: "SKIPPED",
             error: None,
+            code: None,
         }
     }
 }
@@ -275,6 +287,7 @@ impl ReplicationCheckPhaseStatus {
         Self {
             status: "OK",
             error: None,
+            code: None,
         }
     }
 
@@ -282,6 +295,14 @@ impl ReplicationCheckPhaseStatus {
         Self {
             status: "FAILED",
             error: Some(bound_replication_check_error(error.into())),
+            code: None,
+        }
+    }
+
+    fn failed_with_code(error: impl Into<String>, code: &'static str) -> Self {
+        Self {
+            code: Some(code),
+            ..Self::failed(error)
         }
     }
 }
@@ -2046,12 +2067,39 @@ fn fail_replication_check_target(result: &mut ReplicationCheckTargetStatus, erro
     }
 }
 
+/// The probe PUT reports both sides of the version-identity contract: the
+/// source version id it sent (header + `?versionId=` query, the exact shape
+/// live replication uses) and the version id the target answered with.
+struct ReplicationProbePutOutcome {
+    sent_version_id: String,
+    response_version_id: Option<String>,
+}
+
+struct ReplicationProbeMultipartError {
+    primary: S3ClientError,
+    cleanup_error: Option<String>,
+}
+
+impl From<S3ClientError> for ReplicationProbeMultipartError {
+    fn from(primary: S3ClientError) -> Self {
+        Self {
+            primary,
+            cleanup_error: None,
+        }
+    }
+}
+
 #[async_trait::async_trait]
 trait ReplicationProbeOperations {
-    async fn put(&mut self) -> Result<Option<String>, S3ClientError>;
+    async fn put(&mut self) -> Result<ReplicationProbePutOutcome, S3ClientError>;
+    /// Multipart decides the target version at initiate time and only reports
+    /// it on completion, so the identity contract has to be probed separately
+    /// there: a target can adopt PutObject version ids and still mint its own
+    /// for CreateMultipartUpload.
+    async fn multipart_put(&mut self) -> Result<ReplicationProbePutOutcome, ReplicationProbeMultipartError>;
     async fn create_delete_marker(&mut self, version_id: Option<&str>) -> Result<Option<String>, S3ClientError>;
     async fn delete_version(&mut self, version_id: Option<&str>) -> Result<(), S3ClientError>;
-    async fn cleanup(&mut self, known_version_ids: [Option<&str>; 2]) -> Result<(), String>;
+    async fn cleanup(&mut self, known_version_ids: [Option<&str>; 3]) -> Result<(), String>;
 }
 
 struct RemoteReplicationProbeOperations<'a> {
@@ -2063,10 +2111,14 @@ struct RemoteReplicationProbeOperations<'a> {
 
 #[async_trait::async_trait]
 impl ReplicationProbeOperations for RemoteReplicationProbeOperations<'_> {
-    async fn put(&mut self) -> Result<Option<String>, S3ClientError> {
+    async fn put(&mut self) -> Result<ReplicationProbePutOutcome, S3ClientError> {
         put_replication_probe_object(self.client, self.bucket, self.key, self.time).await
     }
 
+    async fn multipart_put(&mut self) -> Result<ReplicationProbePutOutcome, ReplicationProbeMultipartError> {
+        multipart_put_replication_probe_object(self.client, self.bucket, self.key, self.time).await
+    }
+
     async fn create_delete_marker(&mut self, version_id: Option<&str>) -> Result<Option<String>, S3ClientError> {
         delete_replication_probe_object(
             self.client,
@@ -2090,20 +2142,49 @@ impl ReplicationProbeOperations for RemoteReplicationProbeOperations<'_> {
         .map(|_| ())
     }
 
-    async fn cleanup(&mut self, known_version_ids: [Option<&str>; 2]) -> Result<(), String> {
+    async fn cleanup(&mut self, known_version_ids: [Option<&str>; 3]) -> Result<(), String> {
         cleanup_replication_probe(self.client, self.bucket, self.key, known_version_ids).await
     }
 }
 
+/// `None` when the target adopted the source version id on this path.
+fn version_fidelity_error(api: &str, outcome: &ReplicationProbePutOutcome) -> Option<String> {
+    if outcome.response_version_id.as_deref() == Some(outcome.sent_version_id.as_str()) {
+        return None;
+    }
+    Some(format!(
+        "target assigned version id {} instead of adopting the source version id {} on {api}; \
+         version-addressed replication (version deletes, heal) cannot converge on this target",
+        outcome.response_version_id.as_deref().unwrap_or("<none>"),
+        outcome.sent_version_id,
+    ))
+}
+
 async fn execute_replication_probe(result: &mut ReplicationCheckTargetStatus, operations: &mut impl ReplicationProbeOperations) {
     let mut probe_version_id = None;
+    let mut multipart_probe_version_id = None;
     let mut delete_marker_version_id = None;
     let mut cleanup_required = true;
+    let mut multipart_cleanup_error = None;
 
     match operations.put().await {
-        Ok(version_id) => {
+        Ok(outcome) => {
-            probe_version_id = version_id;
             result.phases.put = ReplicationCheckPhaseStatus::passed();
+            // P1-19 version-identity contract: replication only converges on
+            // targets that adopt the source version id — version-addressed
+            // deletes and heal re-drives never match a minted id. Judge it
+            // from the probe PUT's own response; on mismatch the later
+            // mutation phases are pointless (they address by version id), but
+            // cleanup still runs against whatever id the target assigned.
+            match version_fidelity_error("PutObject", &outcome) {
+                None => result.phases.version_fidelity = ReplicationCheckPhaseStatus::passed(),
+                Some(error) => {
+                    result.phases.version_fidelity =
+                        ReplicationCheckPhaseStatus::failed_with_code(&error, REPLICATION_CHECK_CODE_VERSION_MISMATCH);
+                    fail_replication_check_target(result, error);
+                }
+            }
+            probe_version_id = outcome.response_version_id;
         }
         Err(err) => {
             let error = format_replication_check_client_error(&err, ReplicationCheckFailureContext::ReplicateObject);
@@ -2115,7 +2196,29 @@ async fn execute_replication_probe(result: &mut ReplicationCheckTargetStatus, op
         }
     }
 
-    if result.phases.put.status == "OK" {
+    // The multipart path fixes the target version at initiate and only
+    // reports it on completion, so a target can adopt PutObject ids and still
+    // mint its own here — probe it before declaring the contract met.
+    if result.phases.version_fidelity.status == "OK" {
+        match operations.multipart_put().await {
+            Ok(outcome) => {
+                multipart_probe_version_id = outcome.response_version_id.clone();
+                if let Some(error) = version_fidelity_error("CreateMultipartUpload", &outcome) {
+                    result.phases.version_fidelity =
+                        ReplicationCheckPhaseStatus::failed_with_code(&error, REPLICATION_CHECK_CODE_VERSION_MISMATCH);
+                    fail_replication_check_target(result, error);
+                }
+            }
+            Err(err) => {
+                let error = format_replication_check_client_error(&err.primary, ReplicationCheckFailureContext::ReplicateObject);
+                result.phases.version_fidelity = ReplicationCheckPhaseStatus::failed(&error);
+                fail_replication_check_target(result, error);
+                multipart_cleanup_error = err.cleanup_error;
+            }
+        }
+    }
+
+    if result.phases.put.status == "OK" && result.phases.version_fidelity.status == "OK" {
         match operations.create_delete_marker(probe_version_id.as_deref()).await {
             Ok(version_id) => {
                 delete_marker_version_id = version_id;
@@ -2138,19 +2241,31 @@ async fn execute_replication_probe(result: &mut ReplicationCheckTargetStatus, op
         }
     }
 
-    if cleanup_required {
+    let cleanup_result = if cleanup_required {
-        match operations
+        operations
-            .cleanup([probe_version_id.as_deref(), delete_marker_version_id.as_deref()])
+            .cleanup([
+                probe_version_id.as_deref(),
+                multipart_probe_version_id.as_deref(),
+                delete_marker_version_id.as_deref(),
+            ])
             .await
-        {
-            Ok(()) => result.phases.cleanup = ReplicationCheckPhaseStatus::passed(),
-            Err(error) => {
-                result.phases.cleanup = ReplicationCheckPhaseStatus::failed(&error);
-                fail_replication_check_target(result, format!("probe cleanup failed: {error}"));
-            }
-        }
     } else {
+        Ok(())
+    };
+
+    let mut cleanup_errors = Vec::new();
+    if let Some(error) = multipart_cleanup_error {
+        cleanup_errors.push(error);
+    }
+    if let Err(error) = cleanup_result {
+        cleanup_errors.push(error);
+    }
+    if cleanup_errors.is_empty() {
         result.phases.cleanup = ReplicationCheckPhaseStatus::passed();
+    } else {
+        let error = cleanup_errors.join("; ");
+        result.phases.cleanup = ReplicationCheckPhaseStatus::failed(&error);
+        fail_replication_check_target(result, format!("probe cleanup failed: {error}"));
     }
 }
 
@@ -2225,13 +2340,7 @@ fn build_replication_probe_remove_options(now: OffsetDateTime, replication_delet
     }
 }
 
-async fn put_replication_probe_object(
+fn build_replication_probe_headers(options: &PutObjectOptions) -> HeaderMap {
-    target_client: &TargetClient,
-    target_bucket: &str,
-    probe_key: &str,
-    now: OffsetDateTime,
-) -> Result<Option<String>, S3ClientError> {
-    let options = build_replication_probe_put_options(now);
     let mut headers = HeaderMap::new();
     insert_header(&mut headers, SUFFIX_SOURCE_VERSION_ID, &options.internal.source_version_id);
     insert_header(
@@ -2245,8 +2354,166 @@ async fn put_replication_probe_object(
         HeaderName::from_static("x-amz-replication-status"),
         HeaderValue::from_static(ReplicationStatusType::Replica.as_str()),
     );
+    headers
+}
 
-    target_client
+/// Probe the identity contract on the multipart path: initiate carrying the
+/// source version as `?versionId=` (where the target fixes the version),
+/// upload one small part, and read the version the completion reports.
+async fn multipart_put_replication_probe_object(
+    target_client: &TargetClient,
+    target_bucket: &str,
+    probe_key: &str,
+    now: OffsetDateTime,
+) -> Result<ReplicationProbePutOutcome, ReplicationProbeMultipartError> {
+    let options = build_replication_probe_put_options(now);
+    let sent_version_id = options.internal.source_version_id.clone();
+    let headers = build_replication_probe_headers(&options);
+
+    let initiate_headers = headers.clone();
+    let initiate_version_id = sent_version_id.clone();
+    let created = target_client
+        .client
+        .create_multipart_upload()
+        .bucket(target_bucket)
+        .key(probe_key)
+        .customize()
+        .map_request(move |mut req| {
+            for (key, value) in initiate_headers.clone() {
+                req.headers_mut().insert(key.expect("operation should succeed"), value);
+            }
+            let uri = append_version_id_query(req.uri(), &initiate_version_id);
+            req.set_uri(uri).map_err(std::io::Error::other)?;
+            Result::<_, std::io::Error>::Ok(req)
+        })
+        .send()
+        .await
+        .map_err(S3ClientError::from)
+        .map_err(ReplicationProbeMultipartError::from)?;
+    let upload_id = created
+        .upload_id()
+        .ok_or_else(|| S3ClientError::new("target multipart initiate returned no upload id"))
+        .map_err(ReplicationProbeMultipartError::from)?
+        .to_string();
+
+    let uploaded = match target_client
+        .client
+        .upload_part()
+        .bucket(target_bucket)
+        .key(probe_key)
+        .upload_id(&upload_id)
+        .part_number(1)
+        .content_length(8)
+        .body(AwsByteStream::from_static(b"aaaaaaaa"))
+        .send()
+        .await
+    {
+        Ok(uploaded) => uploaded,
+        Err(error) => {
+            return Err(abort_failed_replication_probe_multipart(
+                target_client,
+                target_bucket,
+                probe_key,
+                &upload_id,
+                S3ClientError::from(error),
+            )
+            .await);
+        }
+    };
+
+    let completed_part = CompletedPart::builder()
+        .part_number(1)
+        .set_e_tag(uploaded.e_tag().map(ToOwned::to_owned))
+        .build();
+    let complete_headers = headers.clone();
+    let completed = match target_client
+        .client
+        .complete_multipart_upload()
+        .bucket(target_bucket)
+        .key(probe_key)
+        .upload_id(&upload_id)
+        .multipart_upload(
+            CompletedMultipartUpload::builder()
+                .set_parts(Some(vec![completed_part]))
+                .build(),
+        )
+        .customize()
+        .map_request(move |mut req| {
+            for (key, value) in complete_headers.clone() {
+                req.headers_mut().insert(key.expect("operation should succeed"), value);
+            }
+            Result::<_, std::io::Error>::Ok(req)
+        })
+        .send()
+        .await
+    {
+        Ok(completed) => completed,
+        Err(error) => {
+            return Err(abort_failed_replication_probe_multipart(
+                target_client,
+                target_bucket,
+                probe_key,
+                &upload_id,
+                S3ClientError::from(error),
+            )
+            .await);
+        }
+    };
+
+    Ok(ReplicationProbePutOutcome {
+        sent_version_id,
+        response_version_id: completed.version_id().map(ToOwned::to_owned),
+    })
+}
+
+async fn abort_failed_replication_probe_multipart(
+    target_client: &TargetClient,
+    target_bucket: &str,
+    probe_key: &str,
+    upload_id: &str,
+    primary_error: S3ClientError,
+) -> ReplicationProbeMultipartError {
+    match target_client
+        .client
+        .abort_multipart_upload()
+        .bucket(target_bucket)
+        .key(probe_key)
+        .upload_id(upload_id)
+        .send()
+        .await
+    {
+        Ok(_) => ReplicationProbeMultipartError::from(primary_error),
+        Err(error) => {
+            let abort_error = S3ClientError::from(error);
+            if abort_error.code.as_deref() == Some("NoSuchUpload") {
+                ReplicationProbeMultipartError::from(primary_error)
+            } else {
+                ReplicationProbeMultipartError {
+                    primary: primary_error,
+                    cleanup_error: Some("failed to abort multipart replication probe".to_string()),
+                }
+            }
+        }
+    }
+}
+
+async fn put_replication_probe_object(
+    target_client: &TargetClient,
+    target_bucket: &str,
+    probe_key: &str,
+    now: OffsetDateTime,
+) -> Result<ReplicationProbePutOutcome, S3ClientError> {
+    let options = build_replication_probe_put_options(now);
+    let sent_version_id = options.internal.source_version_id.clone();
+    let headers = build_replication_probe_headers(&options);
+
+    // Carry the source version as `?versionId=` exactly like a live
+    // replication PUT (P0-5 shape): the probe must exercise the query the
+    // real data path relies on, and the response tells us whether the target
+    // adopts the id. The probe id is always a fresh non-nil UUID, so the
+    // null-version mapping in the live path does not apply here.
+    let query_version_id = sent_version_id.clone();
+    let response = target_client
         .client
         .put_object()
         .bucket(target_bucket)
@@ -2259,12 +2526,18 @@ async fn put_replication_probe_object(
             for (key, value) in headers.clone() {
                 req.headers_mut().insert(key.expect("operation should succeed"), value);
             }
+            let uri = append_version_id_query(req.uri(), &query_version_id);
+            req.set_uri(uri).map_err(std::io::Error::other)?;
             Result::<_, std::io::Error>::Ok(req)
         })
         .send()
         .await
-        .map(|output| output.version_id().map(ToOwned::to_owned))
+        .map_err(S3ClientError::from)?;
-        .map_err(S3ClientError::from)
+
+    Ok(ReplicationProbePutOutcome {
+        sent_version_id,
+        response_version_id: response.version_id().map(ToOwned::to_owned),
+    })
 }
 
 async fn delete_replication_probe_object(
@@ -3431,6 +3704,12 @@ mod tests {
     #[derive(Default)]
     struct ScriptedReplicationProbe {
         put_error: Option<&'static str>,
+        /// Version id the scripted target answers with on PUT; None models a
+        /// mirroring target that echoes the sent source version id.
+        minted_version_id: Option<&'static str>,
+        /// Same, for the multipart leg: a target may mirror PutObject ids and
+        /// still mint its own at CreateMultipartUpload.
+        minted_multipart_version_id: Option<&'static str>,
         delete_marker_error: Option<&'static str>,
         version_delete_error: Option<&'static str>,
         cleanup_error: Option<&'static str>,
@@ -3449,14 +3728,25 @@ mod tests {
 
     #[async_trait::async_trait]
     impl ReplicationProbeOperations for ScriptedReplicationProbe {
-        async fn put(&mut self) -> Result<Option<String>, S3ClientError> {
+        async fn put(&mut self) -> Result<ReplicationProbePutOutcome, S3ClientError> {
             self.calls.push("put");
             match self.put_error {
                 Some(code) => Err(scripted_probe_error(code)),
-                None => Ok(Some("object-version".to_string())),
+                None => Ok(ReplicationProbePutOutcome {
+                    sent_version_id: "object-version".to_string(),
+                    response_version_id: Some(self.minted_version_id.unwrap_or("object-version").to_string()),
+                }),
             }
         }
 
+        async fn multipart_put(&mut self) -> Result<ReplicationProbePutOutcome, ReplicationProbeMultipartError> {
+            self.calls.push("multipart-put");
+            Ok(ReplicationProbePutOutcome {
+                sent_version_id: "multipart-version".to_string(),
+                response_version_id: Some(self.minted_multipart_version_id.unwrap_or("multipart-version").to_string()),
+            })
+        }
+
         async fn create_delete_marker(&mut self, _version_id: Option<&str>) -> Result<Option<String>, S3ClientError> {
             self.calls.push("delete-marker");
             match self.delete_marker_error {
@@ -3473,7 +3763,7 @@ mod tests {
             }
         }
 
-        async fn cleanup(&mut self, known_version_ids: [Option<&str>; 2]) -> Result<(), String> {
+        async fn cleanup(&mut self, known_version_ids: [Option<&str>; 3]) -> Result<(), String> {
             self.calls.push("cleanup");
             self.cleanup_ids = known_version_ids
                 .into_iter()
@@ -3486,6 +3776,44 @@ mod tests {
         }
     }
 
+    /// P1-19: a target that mints its own version ids must fail the
+    /// VersionFidelity phase with the machine-readable mismatch code, skip
+    /// the version-addressed mutation phases (they cannot mean anything on a
+    /// drifting target), and still clean up using the id the target actually
+    /// assigned — the source-derived id would never match.
+    #[tokio::test]
+    async fn replication_probe_flags_version_minting_target() {
+        let mut result = replication_check_target("arn:a", "OK", None);
+        let mut operations = ScriptedReplicationProbe {
+            minted_version_id: Some("target-minted-version"),
+            ..Default::default()
+        };
+
+        execute_replication_probe(&mut result, &mut operations).await;
+
+        assert_eq!(operations.calls, ["put", "cleanup"]);
+        assert_eq!(result.status, "FAILED");
+        assert_eq!(result.phases.put.status, "OK");
+        assert_eq!(result.phases.version_fidelity.status, "FAILED");
+        assert_eq!(result.phases.version_fidelity.code, Some(REPLICATION_CHECK_CODE_VERSION_MISMATCH));
+        assert_eq!(result.phases.delete_marker.status, "SKIPPED");
+        assert_eq!(result.phases.version_delete.status, "SKIPPED");
+        assert_eq!(result.phases.cleanup.status, "OK");
+        assert_eq!(operations.cleanup_ids, [Some("target-minted-version".to_string()), None, None]);
+    }
+
+    #[tokio::test]
+    async fn replication_probe_passes_version_fidelity_for_mirroring_target() {
+        let mut result = replication_check_target("arn:a", "OK", None);
+        let mut operations = ScriptedReplicationProbe::default();
+
+        execute_replication_probe(&mut result, &mut operations).await;
+
+        assert_eq!(result.status, "OK");
+        assert_eq!(result.phases.version_fidelity.status, "OK");
+        assert_eq!(result.phases.version_fidelity.code, None);
+    }
+
     #[tokio::test]
     async fn replication_probe_attempts_cleanup_after_ambiguous_put_failure() {
         let mut result = replication_check_target("arn:a", "OK", None);
@@ -3514,8 +3842,15 @@ mod tests {
 
         execute_replication_probe(&mut result, &mut operations).await;
 
-        assert_eq!(operations.calls, ["put", "delete-marker", "version-delete", "cleanup"]);
+        assert_eq!(operations.calls, ["put", "multipart-put", "delete-marker", "version-delete", "cleanup"]);
-        assert_eq!(operations.cleanup_ids, [Some("object-version".to_string()), None]);
+        assert_eq!(
+            operations.cleanup_ids,
+            [
+                Some("object-version".to_string()),
+                Some("multipart-version".to_string()),
+                None
+            ]
+        );
         assert_eq!(result.phases.delete_marker.status, "FAILED");
         assert_eq!(result.phases.version_delete.status, "OK");
         assert_eq!(result.phases.cleanup.status, "OK");
@@ -3532,10 +3867,14 @@ mod tests {
 
         execute_replication_probe(&mut result, &mut operations).await;
 
-        assert_eq!(operations.calls, ["put", "delete-marker", "version-delete", "cleanup"]);
+        assert_eq!(operations.calls, ["put", "multipart-put", "delete-marker", "version-delete", "cleanup"]);
         assert_eq!(
             operations.cleanup_ids,
-            [Some("object-version".to_string()), Some("marker-version".to_string())]
+            [
+                Some("object-version".to_string()),
+                Some("multipart-version".to_string()),
+                Some("marker-version".to_string())
+            ]
         );
         assert_eq!(result.phases.version_delete.status, "FAILED");
         assert_eq!(result.phases.cleanup.status, "FAILED");

rustfs/src/admin/service/site_replication.rs

@@ -16,14 +16,14 @@ use crate::admin::runtime_sources::{AppContext, current_app_context, current_obj
 use crate::admin::site_replication_identity::{
     deployment_id_for_endpoint, mark_unknown_peer_sync_enabled, normalize_peer_map_by_identity_with,
 };
-use crate::admin::storage_api::config::{read_admin_config, save_admin_config};
+use crate::admin::site_replication_state::{SITE_REPLICATION_STATE_PATH, with_site_replication_state_lock_on};
 use crate::admin::storage_api::error::Error as StorageError;
+use crate::storage::storage_api::{read_config_no_lock, save_config_no_lock};
 use rustfs_madmin::PeerInfo;
 use s3s::{S3Error, S3ErrorCode, S3Result};
 use serde_json::{Map, Value};
 use tracing::info;
 
-const SITE_REPLICATION_STATE_PATH: &str = "config/site-replication/state.json";
 const SYNC_STATE_INITIALIZED_FIELD: &str = "sync_state_initialized";
 
 fn normalize_peers_map(peers: &Map<String, Value>, initialize_sync_state: bool) -> Map<String, Value> {
@@ -113,25 +113,36 @@ pub async fn reload_site_replication_runtime_state_for_context(context: Option<&
         return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
     };
 
-    match read_admin_config(store.clone(), SITE_REPLICATION_STATE_PATH).await {
+    // The whole read -> normalize -> save is one RMW: run it inside the
-        Ok(data) => {
+    // shared state transaction boundary (P1-15) so a cluster-wide reload
-            if let Some(normalized) =
+    // fan-out cannot overwrite a concurrent state writer. IO must be the
-                normalize_site_replication_state_json(&data).map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e))?
+    // no-lock variants — the boundary already holds the object lock.
-            {
+    let lock_store = store.clone();
-                save_admin_config(store, SITE_REPLICATION_STATE_PATH, normalized)
+    with_site_replication_state_lock_on(lock_store, move || async move {
-                    .await
+        match read_config_no_lock(store.clone(), SITE_REPLICATION_STATE_PATH).await {
-                    .map_err(|e| {
+            Ok(data) => {
-                        S3Error::with_message(S3ErrorCode::InternalError, format!("normalize site replication state failed: {e}"))
+                if let Some(normalized) = normalize_site_replication_state_json(&data)
-                    })?;
+                    .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e))?
+                {
+                    save_config_no_lock(store, SITE_REPLICATION_STATE_PATH, normalized)
+                        .await
+                        .map_err(|e| {
+                            S3Error::with_message(
+                                S3ErrorCode::InternalError,
+                                format!("normalize site replication state failed: {e}"),
+                            )
+                        })?;
+                }
+                Ok(())
             }
-            Ok(())
+            Err(StorageError::ConfigNotFound) => Ok(()),
+            Err(err) => Err(S3Error::with_message(
+                S3ErrorCode::InternalError,
+                format!("failed to load site replication state: {err}"),
+            )),
         }
-        Err(StorageError::ConfigNotFound) => Ok(()),
+    })
-        Err(err) => Err(S3Error::with_message(
+    .await
-            S3ErrorCode::InternalError,
-            format!("failed to load site replication state: {err}"),
-        )),
-    }
 }
 
 pub async fn reload_site_replication_runtime_state() -> S3Result<()> {

rustfs/src/admin/site_replication_state.rs

@@ -0,0 +1,110 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Locking primitive for the site-replication state object (P1-15,
+//! rustfs/backlog#1675 B2).
+//!
+//! `config/site-replication/state.json` is mutated by read-modify-write
+//! sequences spread over many call sites: admin handlers, the retry-event
+//! writers on every hook broadcast path, and the service-side reload driven
+//! over node RPC. Historically only some of them held the process-local
+//! mutex and none held a distributed lock across the whole RMW, so
+//! concurrent writers overwrote each other (single-process for the unlocked
+//! writers, cross-node for everyone).
+//!
+//! `with_site_replication_state_lock` is the single transaction boundary:
+//! it holds the process-local mutex AND the distributed config-object write
+//! lock (the pattern proven by the repair state,
+//! `update_site_replication_repair_state`) for the duration of the caller's
+//! closure. All IO inside the closure must use the `*_no_lock` config
+//! helpers — the locked variants would self-deadlock on the same object
+//! lock. Do not perform peer network calls or take other config locks
+//! inside the closure.
+//!
+//! The process-local mutex is transitional: call sites still outside this
+//! primitive serialize against migrated ones through it. Once every RMW
+//! call site goes through here (P1-15 PR2) it will be removed, leaving the
+//! object lock as the only mechanism.
+//!
+//! Lock order (unchanged from the historical comment next to the mutex):
+//! lifecycle -> bucket operation -> repair admission -> state (process
+//! mutex, then state object lock) -> per-bucket metadata.
+
+use crate::admin::storage_api::runtime::ECStore;
+use crate::storage::storage_api::with_config_object_write_lock;
+use s3s::{S3Error, S3ErrorCode, S3Result};
+use std::sync::Arc;
+
+use super::runtime_sources::current_object_store_handle;
+
+/// Config object holding the whole site-replication state, including the
+/// retry-event queue. Shared by the typed handler-side accessors and the
+/// byte-level tolerant reload on the service side.
+pub(crate) const SITE_REPLICATION_STATE_PATH: &str = "config/site-replication/state.json";
+
+/// Transitional process-local mutex — see the module docs. Stays private to
+/// this module (owner-local static, enforced by
+/// `scripts/check_architecture_migration_rules.sh`); callers go through
+/// [`site_replication_state_process_guard`].
+static SITE_REPLICATION_STATE_LOCK: std::sync::LazyLock<tokio::sync::Mutex<()>> =
+    std::sync::LazyLock::new(|| tokio::sync::Mutex::new(()));
+
+/// Owner helper for the transitional process mutex: the RMW call sites in
+/// `handlers::site_replication` that PR2 has not migrated to
+/// [`with_site_replication_state_lock`] yet hold this guard so they stay
+/// mutually exclusive with the migrated ones. Removed together with the
+/// mutex once every call site runs inside the transaction boundary.
+pub(crate) async fn site_replication_state_process_guard() -> tokio::sync::MutexGuard<'static, ()> {
+    SITE_REPLICATION_STATE_LOCK.lock().await
+}
+
+/// Run `operation` under the site-replication state transaction boundary:
+/// process mutex first, then the distributed state-object write lock.
+pub(crate) async fn with_site_replication_state_lock<T, F, Fut>(operation: F) -> S3Result<T>
+where
+    T: Send + 'static,
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = S3Result<T>> + Send + 'static,
+{
+    let store =
+        current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?;
+    with_site_replication_state_lock_on(store, operation).await
+}
+
+/// Context-store variant for callers that resolve their store from an
+/// explicit [`AppContext`] (the service-side reload driven over node RPC).
+pub(crate) async fn with_site_replication_state_lock_on<T, F, Fut>(store: Arc<ECStore>, operation: F) -> S3Result<T>
+where
+    T: Send + 'static,
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = S3Result<T>> + Send + 'static,
+{
+    let _process_guard = SITE_REPLICATION_STATE_LOCK.lock().await;
+    with_site_replication_state_object_lock(store, operation).await
+}
+
+/// The distributed half of the boundary on its own: the state-object write
+/// lock, without the process mutex. This is the only thing that serializes
+/// writers in *different* processes (the mutex cannot), so it is also what
+/// the separate-nodes regression test drives.
+pub(crate) async fn with_site_replication_state_object_lock<T, F, Fut>(store: Arc<ECStore>, operation: F) -> S3Result<T>
+where
+    T: Send + 'static,
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = S3Result<T>> + Send + 'static,
+{
+    with_config_object_write_lock(store, SITE_REPLICATION_STATE_PATH.to_string(), operation)
+        .await
+        .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("lock site replication state failed: {e}")))?
+}

rustfs/src/admin/storage_api.rs

@@ -183,6 +183,7 @@ pub(crate) mod bandwidth {
 }
 
 pub(crate) mod bucket_target_sys {
+    pub(crate) use super::ecstore_bucket::bucket_target_sys::append_version_id_query;
     pub(crate) type AdvancedPutOptions = super::ecstore_bucket::bucket_target_sys::AdvancedPutOptions;
     pub(crate) type BucketTargetError = super::ecstore_bucket::bucket_target_sys::BucketTargetError;
     pub(crate) type BucketTargetSys = super::ecstore_bucket::bucket_target_sys::BucketTargetSys;

rustfs/src/app/context/global.rs

@@ -34,6 +34,7 @@ use super::interfaces::{
     ScannerMetricsInterface, ServerConfigInterface, StorageClassInterface, TierConfigInterface, TransitionStateInterface,
 };
 use crate::app::object_data_cache::ObjectDataCacheAdapter;
+use crate::app::object_traffic_health::ObjectTrafficHealth;
 use rustfs_iam::{federation::FederatedIdentityService, store::object::ObjectStore, sys::IamSys};
 use rustfs_kms::KmsServiceManager;
 use std::sync::{Arc, OnceLock};
@@ -74,6 +75,7 @@ pub struct AppContext {
     storage_class: Arc<dyn StorageClassInterface>,
     buffer_config: Arc<dyn BufferConfigInterface>,
     object_data_cache: Arc<ObjectDataCacheAdapter>,
+    object_traffic_health: Arc<ObjectTrafficHealth>,
 }
 
 impl AppContext {
@@ -122,6 +124,7 @@ impl AppContext {
             storage_class: default_storage_class_interface(),
             buffer_config: default_buffer_config_interface(),
             object_data_cache,
+            object_traffic_health: Arc::new(ObjectTrafficHealth::from_env()),
         }
     }
 
@@ -137,6 +140,10 @@ impl AppContext {
         self.object_store.clone()
     }
 
+    pub(crate) fn object_traffic_health(&self) -> Arc<ObjectTrafficHealth> {
+        Arc::clone(&self.object_traffic_health)
+    }
+
     pub fn iam(&self) -> Arc<dyn IamInterface> {
         self.iam.clone()
     }
@@ -342,9 +349,15 @@ impl AppContext {
             storage_class: interfaces.storage_class,
             buffer_config: interfaces.buffer_config,
             object_data_cache: ObjectDataCacheAdapter::disabled_arc(),
+            object_traffic_health: Arc::new(ObjectTrafficHealth::from_env()),
         }
     }
 
+    pub(crate) fn with_test_object_traffic_health(mut self, object_traffic_health: Arc<ObjectTrafficHealth>) -> Self {
+        self.object_traffic_health = object_traffic_health;
+        self
+    }
+
     pub(crate) fn with_test_runtime_config_interfaces(
         mut self,
         server_config: Arc<dyn ServerConfigInterface>,

rustfs/src/app/gating_test_env.rs

@@ -25,6 +25,7 @@
 use super::storage_api::test::bucket::metadata_sys;
 use super::storage_api::test::contract::bucket::{BucketOperations, BucketOptions};
 use super::storage_api::test::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints};
+use super::{context::AppContext, object_traffic_health::ObjectTrafficHealth};
 use std::path::PathBuf;
 use std::sync::{Arc, OnceLock};
 use tempfile::TempDir;
@@ -32,6 +33,7 @@ use tokio::fs;
 use tokio_util::sync::CancellationToken;
 
 static SHARED_GATING_ENV: OnceLock<(Vec<PathBuf>, Arc<ECStore>, TempDir)> = OnceLock::new();
+static SHARED_GATING_INIT: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
 
 /// Return a shared 4-disk `ECStore` with bucket metadata initialized.
 ///
@@ -42,6 +44,10 @@ pub(crate) async fn shared_gating_ecstore() -> Arc<ECStore> {
     if let Some((_paths, store, _)) = SHARED_GATING_ENV.get() {
         return store.clone();
     }
+    let _init_guard = SHARED_GATING_INIT.lock().await;
+    if let Some((_paths, store, _)) = SHARED_GATING_ENV.get() {
+        return store.clone();
+    }
 
     let temp_dir = TempDir::new().expect("create temp dir for gating test env");
     let temp_path = temp_dir.path().to_path_buf();
@@ -101,6 +107,28 @@ pub(crate) async fn shared_gating_ecstore() -> Arc<ECStore> {
     ecstore
 }
 
+pub(crate) async fn shared_gating_ambient() -> Arc<AppContext> {
+    let store = shared_gating_ecstore().await;
+    if let Some(ambient) = crate::runtime_sources::current_app_context() {
+        return ambient;
+    }
+
+    let _init_guard = SHARED_GATING_INIT.lock().await;
+    if crate::runtime_sources::current_app_context().is_none() {
+        super::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
+    }
+    crate::runtime_sources::current_app_context().expect("object traffic test context must be installed")
+}
+
+pub(crate) fn app_context_from_current_environment(ambient: &AppContext) -> AppContext {
+    AppContext::new(ambient.object_store(), ambient.iam(), ambient.kms())
+}
+
+pub(crate) async fn app_context_with_object_traffic_health(object_traffic_health: Arc<ObjectTrafficHealth>) -> Arc<AppContext> {
+    let ambient = shared_gating_ambient().await;
+    Arc::new(app_context_from_current_environment(&ambient).with_test_object_traffic_health(object_traffic_health))
+}
+
 /// Like [`shared_gating_ecstore`], but also returns the backing disk paths so
 /// tests can remove on-disk shards and simulate the object data vanishing
 /// mid-stream.

rustfs/src/app/mod.rs

@@ -21,6 +21,7 @@ pub mod context;
 pub(crate) mod metadata_route;
 pub mod multipart_usecase;
 pub(crate) mod object_data_cache;
+pub(crate) mod object_traffic_health;
 pub mod object_usecase;
 pub(crate) mod runtime_sources;
 mod select_object;

rustfs/src/app/object_traffic_health.rs

@@ -0,0 +1,335 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::time::{Duration, Instant};
+
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub(crate) struct ObjectTrafficSnapshot {
+    pub(crate) read_stalled: bool,
+    pub(crate) write_stalled: bool,
+}
+
+/// Detects bounded object stages that stop returning from foreground requests.
+/// Both success and error returns are progress: dependency correctness remains
+/// the responsibility of the existing readiness checks.
+#[derive(Debug)]
+pub(crate) struct ObjectTrafficHealth {
+    started_at: Instant,
+    stall_after_ms: u64,
+    enabled: bool,
+    read_metadata: OperationProgress,
+    read_storage: OperationProgress,
+    write_storage: OperationProgress,
+}
+
+impl ObjectTrafficHealth {
+    pub(crate) fn from_env() -> Self {
+        let enabled = rustfs_utils::get_env_bool(
+            rustfs_config::ENV_HEALTH_OBJECT_PROGRESS_ENABLE,
+            rustfs_config::DEFAULT_HEALTH_OBJECT_PROGRESS_ENABLE,
+        );
+        let configured_timeout_ms = rustfs_utils::get_env_u64(
+            rustfs_config::ENV_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS,
+            rustfs_config::DEFAULT_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS,
+        );
+        let requested_timeout_ms = if configured_timeout_ms == 0 {
+            rustfs_config::DEFAULT_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS
+        } else {
+            configured_timeout_ms
+        };
+        let minimum_timeout_ms = duration_ms_saturating(crate::storage::get_lock_acquire_timeout())
+            .saturating_add(rustfs_config::HEALTH_OBJECT_PROGRESS_LOCK_MARGIN_MS);
+        let stall_after_ms = requested_timeout_ms.max(minimum_timeout_ms);
+
+        Self::new(enabled, stall_after_ms)
+    }
+
+    fn new(enabled: bool, stall_after_ms: u64) -> Self {
+        Self {
+            started_at: Instant::now(),
+            stall_after_ms,
+            enabled,
+            read_metadata: OperationProgress::default(),
+            read_storage: OperationProgress::default(),
+            write_storage: OperationProgress::default(),
+        }
+    }
+
+    pub(crate) fn track_read_metadata(&self) -> Option<ObjectTrafficProgressGuard<'_>> {
+        self.track(&self.read_metadata)
+    }
+
+    pub(crate) fn track_read_storage(&self) -> Option<ObjectTrafficProgressGuard<'_>> {
+        self.track(&self.read_storage)
+    }
+
+    pub(crate) fn track_write_storage(&self) -> Option<ObjectTrafficProgressGuard<'_>> {
+        self.track(&self.write_storage)
+    }
+
+    pub(crate) fn snapshot(&self) -> ObjectTrafficSnapshot {
+        if !self.enabled {
+            return ObjectTrafficSnapshot::default();
+        }
+
+        let now_ms = self.now_ms();
+        ObjectTrafficSnapshot {
+            read_stalled: self.read_metadata.is_stalled_at(now_ms, self.stall_after_ms)
+                || self.read_storage.is_stalled_at(now_ms, self.stall_after_ms),
+            write_stalled: self.write_storage.is_stalled_at(now_ms, self.stall_after_ms),
+        }
+    }
+
+    fn track<'a>(&'a self, progress: &'a OperationProgress) -> Option<ObjectTrafficProgressGuard<'a>> {
+        if !self.enabled || !progress.begin_at(self.now_ms()) {
+            return None;
+        }
+        Some(ObjectTrafficProgressGuard { health: self, progress })
+    }
+
+    fn now_ms(&self) -> u64 {
+        duration_ms_saturating(self.started_at.elapsed())
+    }
+
+    #[cfg(test)]
+    pub(crate) fn enabled_for_test(stall_after: Duration) -> Self {
+        Self::new(true, duration_ms_saturating(stall_after))
+    }
+
+    #[cfg(test)]
+    pub(crate) fn read_storage_stalled_for_test(&self) -> bool {
+        self.read_storage.is_stalled_at(self.now_ms(), self.stall_after_ms)
+    }
+}
+
+#[derive(Debug, Default)]
+struct OperationProgress {
+    active: AtomicU64,
+    last_progress_ms: AtomicU64,
+}
+
+impl OperationProgress {
+    fn begin_at(&self, now_ms: u64) -> bool {
+        let mut active = self.active.load(Ordering::Relaxed);
+        loop {
+            let Some(next) = active.checked_add(1) else {
+                return false;
+            };
+            if active == 0 {
+                self.last_progress_ms.fetch_max(now_ms, Ordering::Relaxed);
+            }
+            match self
+                .active
+                .compare_exchange_weak(active, next, Ordering::Release, Ordering::Relaxed)
+            {
+                Ok(_) => return true,
+                Err(observed) => active = observed,
+            }
+        }
+    }
+
+    fn complete_at(&self, now_ms: u64) {
+        self.last_progress_ms.fetch_max(now_ms, Ordering::Relaxed);
+        let previous = self.active.fetch_sub(1, Ordering::Release);
+        debug_assert!(previous > 0, "object traffic progress guard underflow");
+    }
+
+    fn is_stalled_at(&self, now_ms: u64, stall_after_ms: u64) -> bool {
+        self.active.load(Ordering::Acquire) > 0
+            && now_ms.saturating_sub(self.last_progress_ms.load(Ordering::Relaxed)) >= stall_after_ms
+    }
+}
+
+#[must_use = "dropping the guard records operation completion"]
+pub(crate) struct ObjectTrafficProgressGuard<'a> {
+    health: &'a ObjectTrafficHealth,
+    progress: &'a OperationProgress,
+}
+
+impl Drop for ObjectTrafficProgressGuard<'_> {
+    fn drop(&mut self) {
+        self.progress.complete_at(self.health.now_ms());
+    }
+}
+
+fn duration_ms_saturating(duration: Duration) -> u64 {
+    duration
+        .as_secs()
+        .saturating_mul(1_000)
+        .saturating_add(u64::from(duration.subsec_millis()))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn an_active_operation_stalls_at_the_exact_boundary() {
+        let progress = OperationProgress::default();
+        assert!(progress.begin_at(10));
+
+        assert!(!progress.is_stalled_at(39, 30));
+        assert!(progress.is_stalled_at(40, 30));
+    }
+
+    #[test]
+    fn later_arrivals_do_not_hide_an_existing_stall() {
+        let progress = OperationProgress::default();
+        assert!(progress.begin_at(10));
+        assert!(progress.begin_at(35));
+
+        assert!(progress.is_stalled_at(40, 30));
+    }
+
+    #[test]
+    fn a_completion_resets_progress_until_the_remaining_operation_stalls() {
+        let progress = OperationProgress::default();
+        assert!(progress.begin_at(10));
+        assert!(progress.begin_at(20));
+
+        progress.complete_at(35);
+
+        assert!(!progress.is_stalled_at(64, 30));
+        assert!(progress.is_stalled_at(65, 30));
+        progress.complete_at(65);
+        assert!(!progress.is_stalled_at(u64::MAX, 30));
+    }
+
+    #[test]
+    fn a_stale_begin_timestamp_cannot_overwrite_newer_progress() {
+        let progress = OperationProgress::default();
+        assert!(progress.begin_at(10));
+        progress.complete_at(100);
+
+        assert!(progress.begin_at(10));
+        assert!(!progress.is_stalled_at(129, 30));
+        assert!(progress.is_stalled_at(130, 30));
+    }
+
+    #[test]
+    #[serial_test::serial]
+    fn environment_configuration_is_sanitized() {
+        temp_env::with_vars(
+            [
+                (rustfs_config::ENV_HEALTH_OBJECT_PROGRESS_ENABLE, Some("false")),
+                (rustfs_config::ENV_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS, Some("1")),
+            ],
+            || {
+                let minimum_timeout_ms = duration_ms_saturating(crate::storage::get_lock_acquire_timeout())
+                    .saturating_add(rustfs_config::HEALTH_OBJECT_PROGRESS_LOCK_MARGIN_MS);
+                let health = ObjectTrafficHealth::from_env();
+                assert!(!health.enabled);
+                assert_eq!(health.stall_after_ms, minimum_timeout_ms);
+            },
+        );
+
+        temp_env::with_vars([(rustfs_config::ENV_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS, Some("0"))], || {
+            let minimum_timeout_ms = duration_ms_saturating(crate::storage::get_lock_acquire_timeout())
+                .saturating_add(rustfs_config::HEALTH_OBJECT_PROGRESS_LOCK_MARGIN_MS);
+            let health = ObjectTrafficHealth::from_env();
+            assert_eq!(
+                health.stall_after_ms,
+                rustfs_config::DEFAULT_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS.max(minimum_timeout_ms)
+            );
+        });
+    }
+
+    #[test]
+    fn read_and_write_progress_are_independent() {
+        let health = ObjectTrafficHealth::enabled_for_test(Duration::ZERO);
+        let read = health.track_read_storage().expect("read tracking must be enabled");
+
+        assert_eq!(
+            health.snapshot(),
+            ObjectTrafficSnapshot {
+                read_stalled: true,
+                write_stalled: false,
+            }
+        );
+
+        drop(read);
+        let write = health.track_write_storage().expect("write tracking must be enabled");
+        assert_eq!(
+            health.snapshot(),
+            ObjectTrafficSnapshot {
+                read_stalled: false,
+                write_stalled: true,
+            }
+        );
+        drop(write);
+        assert_eq!(health.snapshot(), ObjectTrafficSnapshot::default());
+    }
+
+    #[test]
+    fn disabled_tracking_never_withdraws_readiness() {
+        let health = ObjectTrafficHealth::new(false, 0);
+
+        assert!(health.track_read_metadata().is_none());
+        assert!(health.track_read_storage().is_none());
+        assert!(health.track_write_storage().is_none());
+        assert_eq!(health.snapshot(), ObjectTrafficSnapshot::default());
+    }
+
+    #[test]
+    fn metadata_completions_do_not_hide_a_storage_stall() {
+        let health = ObjectTrafficHealth::enabled_for_test(Duration::ZERO);
+        let storage = health.track_read_storage().expect("read storage tracking must be enabled");
+        let metadata = health.track_read_metadata().expect("read metadata tracking must be enabled");
+        drop(metadata);
+
+        assert!(health.snapshot().read_stalled);
+        drop(storage);
+    }
+
+    #[tokio::test]
+    async fn aborting_a_tracked_future_clears_the_active_operation() {
+        let health = std::sync::Arc::new(ObjectTrafficHealth::enabled_for_test(Duration::ZERO));
+        let task_health = std::sync::Arc::clone(&health);
+        let task = tokio::spawn(async move {
+            let _progress = task_health.track_read_storage().expect("read tracking must be enabled");
+            std::future::pending::<()>().await;
+        });
+
+        if tokio::time::timeout(Duration::from_secs(2), async {
+            while !health.snapshot().read_stalled {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .is_err()
+        {
+            task.abort();
+            let _ = task.await;
+            panic!("tracked future did not publish an active operation");
+        }
+        task.abort();
+        assert!(task.await.expect_err("tracked task must be cancelled").is_cancelled());
+        assert!(!health.snapshot().read_stalled);
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn app_context_honors_the_disabled_progress_environment() {
+        let ambient = crate::app::gating_test_env::shared_gating_ambient().await;
+        temp_env::async_with_vars([(rustfs_config::ENV_HEALTH_OBJECT_PROGRESS_ENABLE, Some("false"))], async {
+            let context = crate::app::gating_test_env::app_context_from_current_environment(&ambient);
+            assert!(context.object_traffic_health().track_read_storage().is_none());
+        })
+        .await;
+
+        let installed = crate::app::runtime_sources::current_app_context().expect("test AppContext must remain installed");
+        assert!(std::sync::Arc::ptr_eq(&ambient, &installed));
+    }
+}

rustfs/src/app/object_usecase.rs

@@ -222,6 +222,7 @@ use crate::app::object_data_cache::{
 };
 #[cfg(test)]
 use crate::app::object_data_cache::{ColdFillRole, ColdFillWaitOutcome, scope_cold_fill_disk_permit_owner_for_test};
+use crate::app::object_traffic_health::ObjectTrafficHealth;
 
 type S3StdError = Box<dyn std::error::Error + Send + Sync + 'static>;
 
@@ -2951,6 +2952,8 @@ fn normalize_delete_objects_version_id(
 
 #[cfg(test)]
 type DeleteSnapshotTestHook = (String, Arc<tokio::sync::Barrier>, Arc<tokio::sync::Barrier>);
+#[cfg(test)]
+type PutPostStoreTestHook = (String, Arc<tokio::sync::Barrier>, Arc<tokio::sync::Barrier>);
 
 #[cfg(test)]
 static DELETE_SNAPSHOT_TEST_HOOK: OnceLock<Mutex<Option<DeleteSnapshotTestHook>>> = OnceLock::new();
@@ -2958,6 +2961,8 @@ static DELETE_SNAPSHOT_TEST_HOOK: OnceLock<Mutex<Option<DeleteSnapshotTestHook>>
 static DELETE_SOURCE_TEST_HOOK: OnceLock<Mutex<Option<DeleteSnapshotTestHook>>> = OnceLock::new();
 #[cfg(test)]
 static DELETE_OBJECTS_AUTH_TEST_HOOK: OnceLock<Mutex<Option<DeleteSnapshotTestHook>>> = OnceLock::new();
+#[cfg(test)]
+static PUT_POST_STORE_TEST_HOOK: OnceLock<Mutex<Option<PutPostStoreTestHook>>> = OnceLock::new();
 
 #[cfg(test)]
 pub(crate) fn install_delete_snapshot_test_hook(
@@ -3052,6 +3057,33 @@ async fn wait_for_delete_objects_auth_test_hook(bucket: &str) {
     }
 }
 
+#[cfg(test)]
+fn install_put_post_store_test_hook(bucket: String, entered: Arc<tokio::sync::Barrier>, resume: Arc<tokio::sync::Barrier>) {
+    *PUT_POST_STORE_TEST_HOOK
+        .get_or_init(|| Mutex::new(None))
+        .lock()
+        .expect("PUT post-store test hook lock should not be poisoned") = Some((bucket, entered, resume));
+}
+
+#[cfg(test)]
+async fn wait_for_put_post_store_test_hook(bucket: &str) {
+    let hook = {
+        let mut slot = PUT_POST_STORE_TEST_HOOK
+            .get_or_init(|| Mutex::new(None))
+            .lock()
+            .expect("PUT post-store test hook lock should not be poisoned");
+        if slot.as_ref().is_some_and(|(expected_bucket, _, _)| expected_bucket == bucket) {
+            slot.take()
+        } else {
+            None
+        }
+    };
+    if let Some((_bucket, entered, resume)) = hook {
+        entered.wait().await;
+        resume.wait().await;
+    }
+}
+
 fn build_put_object_expiration_header(event: &lifecycle::Event) -> Option<String> {
     if !event.action.delete() {
         return None;
@@ -3895,6 +3927,14 @@ pub struct DefaultObjectUsecase {
     get_object_timeout_policy: Option<GetObjectTimeoutPolicy>,
 }
 
+async fn track_object_read_setup<F>(health: Option<&ObjectTrafficHealth>, future: F) -> F::Output
+where
+    F: std::future::Future,
+{
+    let _progress = health.and_then(ObjectTrafficHealth::track_read_storage);
+    future.await
+}
+
 impl DefaultObjectUsecase {
     fn should_use_large_put_concurrency_tuning(size: i64) -> bool {
         size >= DEFAULT_PUT_LARGE_CONCURRENCY_TUNING_MIN_SIZE_BYTES
@@ -3951,6 +3991,13 @@ impl DefaultObjectUsecase {
         current_object_data_cache_for_context(self.context.as_deref())
     }
 
+    fn object_traffic_health(&self) -> Option<Arc<ObjectTrafficHealth>> {
+        self.context
+            .as_ref()
+            .map(|context| context.object_traffic_health())
+            .or_else(|| current_app_context().map(|context| context.object_traffic_health()))
+    }
+
     fn base_buffer_size(&self) -> usize {
         self.context
             .clone()
@@ -4351,6 +4398,7 @@ impl DefaultObjectUsecase {
         rs: Option<HTTPRangeSpec>,
         opts: &ObjectOptions,
         part_number: Option<usize>,
+        object_traffic_health: Option<Arc<ObjectTrafficHealth>>,
     ) -> S3Result<GetObjectPreparedRead> {
         let read_start = std::time::Instant::now();
         let read_stage_start = rustfs_io_metrics::get_stage_metrics_enabled().then_some(read_start);
@@ -4366,10 +4414,12 @@ impl DefaultObjectUsecase {
                 key,
             )
             .await?;
-            let reader = store
+            let reader = track_object_read_setup(
-                .get_object_reader(bucket, key, rs.clone(), req.headers.clone(), opts)
+                object_traffic_health.as_deref(),
-                .await
+                store.get_object_reader(bucket, key, rs.clone(), req.headers.clone(), opts),
-                .map_err(map_get_object_reader_error)?;
+            )
+            .await
+            .map_err(map_get_object_reader_error)?;
             let read_setup =
                 Self::finish_get_object_read(req, manager, bucket, key, rs, part_number, read_start, reader, true).await?;
             return Ok(GetObjectPreparedRead { io_planning, read_setup });
@@ -4390,10 +4440,12 @@ impl DefaultObjectUsecase {
             .await?,
         );
         let mut prepared = Some(
-            store
+            track_object_read_setup(
-                .prepare_get_object_reader(bucket, key, rs.clone(), HeaderMap::new(), opts)
+                object_traffic_health.as_deref(),
-                .await
+                store.prepare_get_object_reader(bucket, key, rs.clone(), HeaderMap::new(), opts),
-                .map_err(map_get_object_reader_error)?,
+            )
+            .await
+            .map_err(map_get_object_reader_error)?,
         );
         let mut cache_fill_allowed = true;
         let mut legacy_hook_missed = false;
@@ -4494,6 +4546,7 @@ impl DefaultObjectUsecase {
                 let headers = &req.headers;
                 let store = &store;
                 let range = &rs;
+                let object_traffic_health = &object_traffic_health;
                 move |producer| {
                     let adapter = Arc::clone(adapter);
                     let engine_plan = engine_plan.clone();
@@ -4503,6 +4556,7 @@ impl DefaultObjectUsecase {
                     let bucket = bucket.to_owned();
                     let key = key.to_owned();
                     let opts = opts.clone();
+                    let object_traffic_health = object_traffic_health.as_ref().map(Arc::clone);
                     async move {
                         let producer_deadline = producer.deadline();
                         let cancellation = producer.cancellation_token();
@@ -4548,7 +4602,10 @@ impl DefaultObjectUsecase {
                             }
                         };
 
-                        let prepare = store.prepare_get_object_reader(&bucket, &key, range.clone(), HeaderMap::new(), &opts);
+                        let prepare = track_object_read_setup(
+                            object_traffic_health.as_deref(),
+                            store.prepare_get_object_reader(&bucket, &key, range.clone(), HeaderMap::new(), &opts),
+                        );
                         let prepared = match match await_cold_fill_startup(prepare, &cancellation, producer_deadline).await {
                             Ok(result) => result,
                             Err(ColdFillStartupWaitError::Cancelled) => {
@@ -4602,7 +4659,8 @@ impl DefaultObjectUsecase {
                             || {
                                 #[cfg(test)]
                                 record_cold_fill_reader_open_for_test(&reader_open_plan);
-                                prepared.with_headers(h).into_reader()
+                                let open_reader = prepared.with_headers(h).into_reader();
+                                async move { track_object_read_setup(object_traffic_health.as_deref(), open_reader).await }
                             },
                             ColdFillProducerExecution {
                                 expected,
@@ -4647,11 +4705,12 @@ impl DefaultObjectUsecase {
             let io_planning = metadata_admission
                 .take()
                 .ok_or_else(|| s3_error!(InternalError, "prepared metadata admission is unavailable"))?;
-            let reader = prepared
+            let reader = track_object_read_setup(
-                .with_headers(req.headers.clone())
+                object_traffic_health.as_deref(),
-                .into_reader()
+                prepared.with_headers(req.headers.clone()).into_reader(),
-                .await
+            )
-                .map_err(map_get_object_reader_error)?;
+            .await
+            .map_err(map_get_object_reader_error)?;
             (io_planning, reader)
         } else {
             let io_planning = Self::acquire_get_object_io_planning(
@@ -4665,19 +4724,25 @@ impl DefaultObjectUsecase {
             )
             .await?;
             let reader = if legacy_hook_missed {
-                store
+                let prepared = track_object_read_setup(
-                    .prepare_get_object_reader(bucket, key, rs.clone(), HeaderMap::new(), opts)
+                    object_traffic_health.as_deref(),
-                    .await
+                    store.prepare_get_object_reader(bucket, key, rs.clone(), HeaderMap::new(), opts),
-                    .map_err(map_get_object_reader_error)?
+                )
-                    .with_headers(req.headers.clone())
+                .await
-                    .into_reader()
+                .map_err(map_get_object_reader_error)?;
-                    .await
+                track_object_read_setup(
-                    .map_err(map_get_object_reader_error)?
+                    object_traffic_health.as_deref(),
+                    prepared.with_headers(req.headers.clone()).into_reader(),
+                )
+                .await
+                .map_err(map_get_object_reader_error)?
             } else {
-                store
+                track_object_read_setup(
-                    .get_object_reader(bucket, key, rs.clone(), req.headers.clone(), opts)
+                    object_traffic_health.as_deref(),
-                    .await
+                    store.get_object_reader(bucket, key, rs.clone(), req.headers.clone(), opts),
-                    .map_err(map_get_object_reader_error)?
+                )
+                .await
+                .map_err(map_get_object_reader_error)?
             };
             (io_planning, reader)
         };
@@ -5485,8 +5550,8 @@ impl DefaultObjectUsecase {
             debug!("Zero-copy write enabled for {} byte object (bucket={}, key={})", size, bucket, key);
         }
 
-        let use_small_eager_put_path =
+        let use_empty_or_small_eager_put_path = size == 0
-            should_use_small_eager_put_path(size, &req.headers, server_side_encryption_requested, should_compress, false);
+            || should_use_small_eager_put_path(size, &req.headers, server_side_encryption_requested, should_compress, false);
         let zero_copy_eager_put_path_status =
             zero_copy_eager_put_path_status(size, &req.headers, server_side_encryption_requested, should_compress, false);
         let use_zero_copy_eager_put_path = zero_copy_eager_put_path_status == PUT_EAGER_STATUS_ELIGIBLE;
@@ -5498,7 +5563,7 @@ impl DefaultObjectUsecase {
             "stream_compressed"
         } else if use_zero_copy_eager_put_path {
             "zero_copy_eager"
-        } else if use_small_eager_put_path {
+        } else if use_empty_or_small_eager_put_path {
             "small_eager"
         } else {
             "streaming"
@@ -5712,7 +5777,7 @@ impl DefaultObjectUsecase {
                 let eager_body = read_zero_copy_put_body_exact(body, actual_size as usize).await?;
                 rustfs_io_metrics::record_zero_copy_write(actual_size as usize, zero_copy_start.elapsed().as_secs_f64() * 1000.0);
                 HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?
-            } else if use_small_eager_put_path {
+            } else if use_empty_or_small_eager_put_path {
                 if (actual_size as usize) <= POOL_BYPASS_MAX_SIZE {
                     // Bypass BytesPool for very small objects to avoid Small-tier
                     // Mutex contention under high concurrency. Direct allocation
@@ -5866,6 +5931,14 @@ impl DefaultObjectUsecase {
             }
         });
 
+        let object_traffic_health = if use_zero_copy_eager_put_path || use_empty_or_small_eager_put_path {
+            self.object_traffic_health()
+        } else {
+            None
+        };
+        let object_traffic_progress = object_traffic_health
+            .as_deref()
+            .and_then(ObjectTrafficHealth::track_write_storage);
         let (obj_info, backfilled_old_current_size) = match store
             .put_object_with_old_current_size(&bucket, &key, &mut reader, &opts)
             .await
@@ -5912,6 +5985,9 @@ impl DefaultObjectUsecase {
                 return result;
             }
         };
+        drop(object_traffic_progress);
+        #[cfg(test)]
+        wait_for_put_post_store_test_hook(&bucket).await;
 
         maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await;
         let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &key).await;
@@ -6352,6 +6428,10 @@ impl DefaultObjectUsecase {
         // naming nonexistent buckets fail before the versioning lookup in
         // get_opts. The store comes from the request-bound server context
         // (backlog#1052 S6), not the process-global handle.
+        let object_traffic_health = self.object_traffic_health();
+        let object_metadata_progress = object_traffic_health
+            .as_deref()
+            .and_then(ObjectTrafficHealth::track_read_metadata);
         let store_lookup_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
         let Some(store) = self.object_store() else {
             lifecycle.finish_err();
@@ -6392,6 +6472,7 @@ impl DefaultObjectUsecase {
             rs,
             opts,
         } = request_context;
+        drop(object_metadata_progress);
 
         let manager = get_concurrency_manager();
 
@@ -6407,6 +6488,7 @@ impl DefaultObjectUsecase {
                 rs,
                 &opts,
                 part_number,
+                object_traffic_health,
             )
             .await
         {
@@ -11142,6 +11224,278 @@ mod tests {
         (store, context)
     }
 
+    #[tokio::test]
+    #[serial_test::serial(body_cache_hook)]
+    async fn object_progress_tracks_real_get_and_small_put_lock_waits() {
+        use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
+
+        let object_traffic_health = Arc::new(ObjectTrafficHealth::enabled_for_test(Duration::ZERO));
+        let context = temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_DATA_CACHE_ENABLE, Some("false"))], async {
+            crate::app::gating_test_env::app_context_with_object_traffic_health(Arc::clone(&object_traffic_health)).await
+        })
+        .await;
+        let store = context.object_store();
+        let bucket = format!("object-progress-{}", Uuid::new_v4());
+        let object = "object.bin";
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("object progress bucket must be created");
+        put_real_cold_fill_object(&store, &bucket, object, b"initial").await;
+
+        let metadata_entered = Arc::new(tokio::sync::Barrier::new(2));
+        let metadata_resume = Arc::new(tokio::sync::Barrier::new(2));
+        crate::storage::options::install_versioning_config_test_hook(
+            bucket.clone(),
+            Arc::clone(&metadata_entered),
+            Arc::clone(&metadata_resume),
+        );
+        let metadata_input = GetObjectInput::builder()
+            .bucket(bucket.clone())
+            .key(object.to_string())
+            .build()
+            .expect("metadata GET input must build");
+        let metadata_usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
+        let metadata_get = tokio::spawn(async move {
+            metadata_usecase
+                .execute_get_object(build_request(metadata_input, Method::GET))
+                .await
+        });
+        tokio::time::timeout(Duration::from_secs(2), metadata_entered.wait())
+            .await
+            .expect("GET must enter the bucket metadata stage");
+        assert!(object_traffic_health.snapshot().read_stalled);
+        assert!(!metadata_get.is_finished(), "GET must still be waiting in bucket metadata");
+        metadata_resume.wait().await;
+        let metadata_response = tokio::time::timeout(Duration::from_secs(10), metadata_get)
+            .await
+            .expect("metadata GET must finish after release")
+            .expect("metadata GET task must join")
+            .expect("metadata GET must succeed after release");
+        assert!(!object_traffic_health.snapshot().read_stalled);
+        drop(metadata_response);
+
+        let read_lock = store
+            .new_ns_lock(&bucket, object)
+            .await
+            .expect("read test namespace lock must be created")
+            .get_write_lock(Duration::from_secs(5))
+            .await
+            .expect("read test namespace lock must be held");
+        let get_input = GetObjectInput::builder()
+            .bucket(bucket.clone())
+            .key(object.to_string())
+            .build()
+            .expect("GET input must build");
+        let get_usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
+        let get = tokio::spawn(async move { get_usecase.execute_get_object(build_request(get_input, Method::GET)).await });
+        tokio::time::timeout(Duration::from_secs(2), async {
+            while !object_traffic_health.read_storage_stalled_for_test() {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("blocked GET must publish a storage stall");
+        assert!(!get.is_finished(), "GET must still be waiting for the held namespace lock");
+        drop(read_lock);
+        let get_response = tokio::time::timeout(Duration::from_secs(10), get)
+            .await
+            .expect("GET must finish after releasing the lock")
+            .expect("GET task must join")
+            .expect("GET must succeed after releasing the lock");
+        assert!(!object_traffic_health.snapshot().read_stalled);
+        drop(get_response);
+
+        let write_lock = store
+            .new_ns_lock(&bucket, object)
+            .await
+            .expect("write test namespace lock must be created")
+            .get_write_lock(Duration::from_secs(5))
+            .await
+            .expect("write test namespace lock must be held");
+        let post_store_entered = Arc::new(tokio::sync::Barrier::new(2));
+        let post_store_resume = Arc::new(tokio::sync::Barrier::new(2));
+        install_put_post_store_test_hook(bucket.clone(), Arc::clone(&post_store_entered), Arc::clone(&post_store_resume));
+        let payload = Bytes::from_static(b"replacement");
+        let put_input = PutObjectInput::builder()
+            .bucket(bucket)
+            .key(object.to_string())
+            .body(Some(StreamingBlob::from(s3s::Body::from(payload.clone()))))
+            .content_length(Some(i64::try_from(payload.len()).expect("test payload length must fit i64")))
+            .build()
+            .expect("PUT input must build");
+        let put_usecase = DefaultObjectUsecase::with_context(Some(context));
+        let put = tokio::spawn(async move {
+            put_usecase
+                .execute_put_object(&FS::new(), build_request(put_input, Method::PUT))
+                .await
+        });
+        tokio::time::timeout(Duration::from_secs(2), async {
+            while !object_traffic_health.snapshot().write_stalled {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("blocked small PUT must publish a storage stall");
+        assert!(!put.is_finished(), "PUT must still be waiting for the held namespace lock");
+        drop(write_lock);
+        tokio::time::timeout(Duration::from_secs(10), post_store_entered.wait())
+            .await
+            .expect("PUT must reach the first post-store hook");
+        assert!(!object_traffic_health.snapshot().write_stalled);
+        assert!(!put.is_finished(), "PUT must remain blocked after the store guard has ended");
+        post_store_resume.wait().await;
+        tokio::time::timeout(Duration::from_secs(10), put)
+            .await
+            .expect("PUT must finish after releasing the lock")
+            .expect("PUT task must join")
+            .expect("PUT must succeed after releasing the lock");
+        let recovered = object_traffic_health.snapshot();
+        assert!(!recovered.read_stalled);
+        assert!(!recovered.write_stalled);
+    }
+
+    #[tokio::test]
+    async fn object_progress_tracks_zero_byte_and_zero_copy_put_lock_waits() {
+        use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
+
+        let object_traffic_health = Arc::new(ObjectTrafficHealth::enabled_for_test(Duration::ZERO));
+        let context =
+            crate::app::gating_test_env::app_context_with_object_traffic_health(Arc::clone(&object_traffic_health)).await;
+        let store = context.object_store();
+        let bucket = format!("progress-buffered-{}", Uuid::new_v4());
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("buffered PUT progress bucket must be created");
+
+        let extra_body_object = "zero-byte-extra.bin";
+        let extra_body_input = PutObjectInput::builder()
+            .bucket(bucket.clone())
+            .key(extra_body_object.to_string())
+            .body(Some(StreamingBlob::from(s3s::Body::from(Bytes::from_static(b"x")))))
+            .content_length(Some(88))
+            .build()
+            .expect("zero-byte extra-body PUT input must build");
+        let extra_body_usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
+        let mut extra_body_request = build_request(extra_body_input, Method::PUT);
+        extra_body_request.headers = streaming_headers(Some("0"));
+        let extra_body_err = extra_body_usecase
+            .execute_put_object(&FS::new(), extra_body_request)
+            .await
+            .expect_err("decoded zero-byte PUT with body data must fail");
+        assert_eq!(extra_body_err.code(), &S3ErrorCode::UnexpectedContent);
+        assert!(!object_traffic_health.snapshot().write_stalled);
+        let lookup_err = store
+            .get_object_info(&bucket, extra_body_object, &ObjectOptions::default())
+            .await
+            .expect_err("rejected zero-byte PUT must not create an object");
+        assert!(is_err_object_not_found(&lookup_err));
+
+        let zero_object = "zero-byte.bin";
+        let zero_write_lock = store
+            .new_ns_lock(&bucket, zero_object)
+            .await
+            .expect("zero-byte PUT namespace lock must be created")
+            .get_write_lock(Duration::from_secs(30))
+            .await
+            .expect("zero-byte PUT namespace lock must be held");
+        let (body_polled_tx, body_polled_rx) = tokio::sync::oneshot::channel();
+        let (body_release_tx, body_release_rx) = tokio::sync::oneshot::channel();
+        let pending_zero_body = StreamingBlob::wrap(futures::stream::once(async move {
+            body_polled_tx.send(()).expect("zero-byte body poll signal must be received");
+            body_release_rx.await.expect("zero-byte body EOF must be released");
+            Ok::<Bytes, std::io::Error>(Bytes::new())
+        }));
+        let zero_input = PutObjectInput::builder()
+            .bucket(bucket.clone())
+            .key(zero_object.to_string())
+            .body(Some(pending_zero_body))
+            .content_length(Some(87))
+            .build()
+            .expect("zero-byte PUT input must build");
+        let zero_usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
+        let mut zero_request = build_request(zero_input, Method::PUT);
+        zero_request.headers = streaming_headers(Some("0"));
+        let zero_put = tokio::spawn(async move { zero_usecase.execute_put_object(&FS::new(), zero_request).await });
+
+        tokio::time::timeout(Duration::from_secs(30), body_polled_rx)
+            .await
+            .expect("zero-byte PUT body must be polled for EOF")
+            .expect("zero-byte PUT body poll signal must be sent");
+        assert!(!object_traffic_health.snapshot().write_stalled);
+        assert!(!zero_put.is_finished(), "zero-byte PUT must still be waiting for request EOF");
+
+        body_release_tx.send(()).expect("zero-byte PUT body EOF must be released");
+        tokio::time::timeout(Duration::from_secs(30), async {
+            while !object_traffic_health.snapshot().write_stalled {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("fully received zero-byte PUT must publish a storage stall");
+        assert!(!zero_put.is_finished(), "zero-byte PUT must still be waiting for the held namespace lock");
+
+        drop(zero_write_lock);
+        tokio::time::timeout(Duration::from_secs(30), zero_put)
+            .await
+            .expect("zero-byte PUT must finish after releasing the lock")
+            .expect("zero-byte PUT task must join")
+            .expect("zero-byte PUT must succeed after releasing the lock");
+        assert!(!object_traffic_health.snapshot().write_stalled);
+
+        let zero_copy_object = "zero-copy-eager.jpg";
+        let zero_copy_payload = Bytes::from(vec![b'z'; 1024 * 1024 + 1]);
+        let zero_copy_size = i64::try_from(zero_copy_payload.len()).expect("zero-copy payload length must fit i64");
+        let zero_copy_headers = HeaderMap::new();
+        assert!(!is_disk_compressible(&zero_copy_headers, zero_copy_object));
+        assert_eq!(
+            zero_copy_eager_put_path_status(zero_copy_size, &zero_copy_headers, false, false, false),
+            PUT_EAGER_STATUS_ELIGIBLE,
+            "test payload must exercise the production zero-copy eager path",
+        );
+        let zero_copy_write_lock = store
+            .new_ns_lock(&bucket, zero_copy_object)
+            .await
+            .expect("zero-copy PUT namespace lock must be created")
+            .get_write_lock(Duration::from_secs(30))
+            .await
+            .expect("zero-copy PUT namespace lock must be held");
+        let zero_copy_input = PutObjectInput::builder()
+            .bucket(bucket)
+            .key(zero_copy_object.to_string())
+            .body(Some(StreamingBlob::from(s3s::Body::from(zero_copy_payload))))
+            .content_length(Some(zero_copy_size))
+            .build()
+            .expect("zero-copy PUT input must build");
+        let zero_copy_usecase = DefaultObjectUsecase::with_context(Some(context));
+        let zero_copy_put = tokio::spawn(async move {
+            zero_copy_usecase
+                .execute_put_object(&FS::new(), build_request(zero_copy_input, Method::PUT))
+                .await
+        });
+
+        tokio::time::timeout(Duration::from_secs(30), async {
+            while !object_traffic_health.snapshot().write_stalled {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("blocked zero-copy eager PUT must publish a storage stall");
+        assert!(
+            !zero_copy_put.is_finished(),
+            "zero-copy PUT must still be waiting for the held namespace lock"
+        );
+
+        drop(zero_copy_write_lock);
+        tokio::time::timeout(Duration::from_secs(30), zero_copy_put)
+            .await
+            .expect("zero-copy PUT must finish after releasing the lock")
+            .expect("zero-copy PUT task must join")
+            .expect("zero-copy PUT must succeed after releasing the lock");
+        assert!(!object_traffic_health.snapshot().write_stalled);
+    }
+
     async fn put_real_cold_fill_object(store: &Arc<ECStore>, bucket: &str, object: &str, body: &[u8]) -> ObjectInfo {
         let mut reader = PutObjReader::from_vec(body.to_vec());
         store

rustfs/src/server/health.rs

@@ -12,11 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use super::readiness::{DependencyReadinessReport, ReadinessDegradedReason};
+use super::readiness::{DependencyReadinessReport, ReadinessDegradedReason, record_readiness_overlay_reason};
 use super::{
     HEALTH_READY_PATH, MINIO_HEALTH_CLUSTER_PATH, MINIO_HEALTH_CLUSTER_READ_PATH, MINIO_HEALTH_READY_PATH,
     collect_cluster_read_health_report, collect_cluster_write_health_report, collect_node_readiness_report,
 };
+use crate::app::object_traffic_health::{ObjectTrafficHealth, ObjectTrafficSnapshot};
 use http::{Method, StatusCode};
 use rustfs_kms::ProbeStatus;
 use rustfs_kms::probe::{DEFAULT_PROBE_INTERVAL, ENV_KMS_PROBE_INTERVAL_SECS, MIN_PROBE_INTERVAL};
@@ -70,11 +71,33 @@ pub(crate) struct HealthPayloadContext<'a> {
     pub(crate) include_dependency_details: bool,
 }
 
-pub(crate) async fn collect_probe_readiness(probe: HealthProbe) -> Option<DependencyReadinessReport> {
+pub(crate) async fn collect_probe_readiness(
-    match readiness_source_for_probe(probe)? {
+    probe: HealthProbe,
-        HealthReadinessSource::Node => Some(collect_node_readiness_report().await),
+    object_traffic_health: Option<&ObjectTrafficHealth>,
-        HealthReadinessSource::ClusterWrite => Some(collect_cluster_write_health_report().await),
+) -> Option<DependencyReadinessReport> {
-        HealthReadinessSource::ClusterRead => Some(collect_cluster_read_health_report().await),
+    let mut report = match readiness_source_for_probe(probe)? {
+        HealthReadinessSource::Node => collect_node_readiness_report().await,
+        HealthReadinessSource::ClusterWrite => collect_cluster_write_health_report().await,
+        HealthReadinessSource::ClusterRead => collect_cluster_read_health_report().await,
+    };
+    if probe == HealthProbe::Readiness
+        && let Some(object_traffic_health) = object_traffic_health
+    {
+        apply_object_traffic_snapshot(&mut report, object_traffic_health.snapshot());
+    }
+    Some(report)
+}
+
+fn apply_object_traffic_snapshot(report: &mut DependencyReadinessReport, snapshot: ObjectTrafficSnapshot) {
+    if snapshot.read_stalled {
+        let reason = ReadinessDegradedReason::ObjectReadStalled;
+        report.degraded_reasons.push(reason);
+        record_readiness_overlay_reason(reason);
+    }
+    if snapshot.write_stalled {
+        let reason = ReadinessDegradedReason::ObjectWriteStalled;
+        report.degraded_reasons.push(reason);
+        record_readiness_overlay_reason(reason);
     }
 }
 
@@ -300,13 +323,19 @@ pub(crate) fn build_health_response_parts(
             ),
         };
 
-    if probe == HealthProbe::Readiness && matches!(kms_ready, Some(false)) {
+    let object_traffic_stalled = degraded_reasons.iter().any(|reason| {
+        matches!(
+            reason,
+            ReadinessDegradedReason::ObjectReadStalled | ReadinessDegradedReason::ObjectWriteStalled
+        )
+    });
+    if probe == HealthProbe::Readiness && (object_traffic_stalled || matches!(kms_ready, Some(false))) {
         health = HealthCheckState {
             status_code: StatusCode::SERVICE_UNAVAILABLE,
             status: "degraded",
             ready: false,
         };
-        if !degraded_reasons.contains(&ReadinessDegradedReason::KmsNotReady) {
+        if matches!(kms_ready, Some(false)) && !degraded_reasons.contains(&ReadinessDegradedReason::KmsNotReady) {
             degraded_reasons.push(ReadinessDegradedReason::KmsNotReady);
         }
     }
@@ -365,6 +394,8 @@ pub(crate) fn build_health_payload(ctx: HealthPayloadContext<'_>) -> Value {
 mod tests {
     use super::super::readiness::DependencyReadiness;
     use super::*;
+    use metrics_util::MetricKind;
+    use metrics_util::debugging::{DebugValue, DebuggingRecorder};
     use rustfs_kms::{ProbeFailureKind, ProbeResult};
     use serial_test::serial;
     use temp_env::with_var;
@@ -394,6 +425,106 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn readiness_collects_object_stalls_and_recovers_on_completion() {
+        let object_traffic_health = ObjectTrafficHealth::enabled_for_test(Duration::ZERO);
+        let read = object_traffic_health
+            .track_read_storage()
+            .expect("read tracking must be enabled");
+        let write = object_traffic_health
+            .track_write_storage()
+            .expect("write tracking must be enabled");
+
+        let stalled = collect_probe_readiness(HealthProbe::Readiness, Some(&object_traffic_health))
+            .await
+            .expect("readiness must have a dependency report");
+        assert!(stalled.degraded_reasons.contains(&ReadinessDegradedReason::ObjectReadStalled));
+        assert!(
+            stalled
+                .degraded_reasons
+                .contains(&ReadinessDegradedReason::ObjectWriteStalled)
+        );
+
+        drop(read);
+        drop(write);
+        let recovered = collect_probe_readiness(HealthProbe::Readiness, Some(&object_traffic_health))
+            .await
+            .expect("readiness must have a dependency report");
+        assert!(
+            !recovered
+                .degraded_reasons
+                .contains(&ReadinessDegradedReason::ObjectReadStalled)
+        );
+        assert!(
+            !recovered
+                .degraded_reasons
+                .contains(&ReadinessDegradedReason::ObjectWriteStalled)
+        );
+    }
+
+    #[test]
+    #[serial]
+    fn an_object_stall_degrades_readiness_without_changing_dependency_details() {
+        with_var(rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("false"), || {
+            let mut report = ready_report();
+            report.degraded_reasons.push(ReadinessDegradedReason::ObjectReadStalled);
+
+            let parts =
+                build_health_response_parts(Method::GET, HealthProbe::Readiness, Some(&report), "rustfs-endpoint", None, None);
+
+            assert_eq!(parts.status_code, StatusCode::SERVICE_UNAVAILABLE);
+            let payload = parts.payload.expect("GET should include payload");
+            assert_eq!(payload["ready"], false);
+            assert_eq!(payload["details"]["storage"]["ready"], true);
+            assert_eq!(payload["degradedReasons"], json!(["object_read_stalled"]));
+        });
+    }
+
+    #[test]
+    fn object_stalls_do_not_change_liveness() {
+        let mut report = ready_report();
+        report.degraded_reasons.push(ReadinessDegradedReason::ObjectWriteStalled);
+
+        let parts =
+            build_health_response_parts(Method::HEAD, HealthProbe::Liveness, Some(&report), "rustfs-endpoint", None, None);
+
+        assert_eq!(parts.status_code, StatusCode::OK);
+        assert!(parts.payload.is_none());
+    }
+
+    #[test]
+    fn object_stall_overlay_records_the_final_readiness_metrics() {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        metrics::with_local_recorder(&recorder, || {
+            let mut report = ready_report();
+            apply_object_traffic_snapshot(
+                &mut report,
+                ObjectTrafficSnapshot {
+                    read_stalled: true,
+                    write_stalled: false,
+                },
+            );
+        });
+
+        let entries = snapshotter.snapshot().into_vec();
+        let ready = entries.iter().find_map(|(composite, _, _, value)| {
+            (composite.kind() == MetricKind::Gauge && composite.key().name() == "rustfs_runtime_readiness_ready").then_some(value)
+        });
+        assert!(matches!(ready, Some(DebugValue::Gauge(value)) if value.into_inner() == 0.0));
+
+        let degraded = entries.iter().find_map(|(composite, _, _, value)| {
+            (composite.kind() == MetricKind::Counter
+                && composite.key().name() == "rustfs_runtime_readiness_degraded_total"
+                && composite
+                    .key()
+                    .labels()
+                    .any(|label| label.key() == "reason" && label.value() == "object_read_stalled"))
+            .then_some(value)
+        });
+        assert!(matches!(degraded, Some(DebugValue::Counter(1))));
+    }
+
     #[tokio::test(start_paused = true)]
     async fn a_fresh_successful_round_keeps_the_service_ready() {
         let round_at = Instant::now();

rustfs/src/server/http.rs

@@ -1604,7 +1604,7 @@ fn process_connection(
                 .option_layer(if is_console { Some(RedirectLayer) } else { None })
                 .layer(BodylessStatusFixLayer)
                 .layer(HeadRequestBodyFixLayer)
-                .layer(PublicHealthEndpointLayer)
+                .layer(PublicHealthEndpointLayer::new(Arc::clone(&server_ctx)))
                 .option_layer((!server_domains_configured && !is_console).then_some(VirtualHostStyleHintLayer))
                 .layer(DoubleSlashListBucketsCompatLayer)
                 .service(service)
@@ -1701,7 +1701,7 @@ fn process_connection(
                 .option_layer(if is_console { Some(RedirectLayer) } else { None })
                 .layer(BodylessStatusFixLayer)
                 .layer(HeadRequestBodyFixLayer)
-                .layer(PublicHealthEndpointLayer)
+                .layer(PublicHealthEndpointLayer::new(Arc::clone(&server_ctx)))
                 .option_layer((!server_domains_configured && !is_console).then_some(VirtualHostStyleHintLayer))
                 .layer(DoubleSlashListBucketsCompatLayer)
                 .service(service)

rustfs/src/server/layer.rs

@@ -14,6 +14,7 @@
 
 use super::runtime_sources;
 use crate::admin::console::is_console_path;
+use crate::app::object_traffic_health::ObjectTrafficHealth;
 use crate::error::ApiError;
 use crate::server::RemoteAddr;
 use crate::server::cors;
@@ -1238,19 +1239,31 @@ where
 }
 
 #[derive(Clone)]
-pub struct PublicHealthEndpointLayer;
+pub struct PublicHealthEndpointLayer {
+    server_ctx: Arc<crate::runtime_sources::ServerContextSlot>,
+}
+
+impl PublicHealthEndpointLayer {
+    pub fn new(server_ctx: Arc<crate::runtime_sources::ServerContextSlot>) -> Self {
+        Self { server_ctx }
+    }
+}
 
 impl<S> Layer<S> for PublicHealthEndpointLayer {
     type Service = PublicHealthEndpointService<S>;
 
     fn layer(&self, inner: S) -> Self::Service {
-        PublicHealthEndpointService { inner }
+        PublicHealthEndpointService {
+            inner,
+            server_ctx: Arc::clone(&self.server_ctx),
+        }
     }
 }
 
 #[derive(Clone)]
 pub struct PublicHealthEndpointService<S> {
     inner: S,
+    server_ctx: Arc<crate::runtime_sources::ServerContextSlot>,
 }
 
 fn health_endpoint_enabled() -> bool {
@@ -1318,6 +1331,7 @@ async fn health_kms_ready() -> bool {
 async fn build_public_health_http_response<RestBody, GrpcBody>(
     method: Method,
     path: String,
+    object_traffic_health: Option<Arc<ObjectTrafficHealth>>,
 ) -> Response<HybridBody<RestBody, GrpcBody>>
 where
     RestBody: From<Bytes>,
@@ -1342,7 +1356,7 @@ where
             .expect("failed to build health busy response");
     }
 
-    let readiness_report = collect_probe_readiness(probe).await;
+    let readiness_report = collect_probe_readiness(probe, object_traffic_health.as_deref()).await;
     let kms_ready = if probe == HealthProbe::Readiness && health_compat_kms_ready_check_enabled() {
         Some(health_kms_ready().await)
     } else {
@@ -1388,7 +1402,11 @@ where
         if is_public_health_endpoint_request(method, path) {
             let method = method.clone();
             let path = path.to_owned();
-            return Box::pin(async move { Ok(build_public_health_http_response(method, path).await) });
+            let object_traffic_health = self
+                .server_ctx
+                .installed_app_context()
+                .map(|context| context.object_traffic_health());
+            return Box::pin(async move { Ok(build_public_health_http_response(method, path, object_traffic_health).await) });
         }
 
         let mut inner = self.inner.clone();
@@ -2185,6 +2203,17 @@ mod tests {
     use temp_env::{async_with_vars, with_var};
     use tracing_subscriber::{Registry, fmt::MakeWriter, layer::SubscriberExt};
 
+    fn public_health_layer() -> PublicHealthEndpointLayer {
+        PublicHealthEndpointLayer::new(crate::runtime_sources::ServerContextSlot::new())
+    }
+
+    async fn public_health_layer_with_tracker(object_traffic_health: Arc<ObjectTrafficHealth>) -> PublicHealthEndpointLayer {
+        let app_context = crate::app::gating_test_env::app_context_with_object_traffic_health(object_traffic_health).await;
+        let server_ctx = crate::runtime_sources::ServerContextSlot::new();
+        assert!(server_ctx.install(app_context));
+        PublicHealthEndpointLayer::new(server_ctx)
+    }
+
     #[derive(Clone, Debug)]
     struct CaptureService;
 
@@ -2651,7 +2680,7 @@ mod tests {
         async_with_vars([(rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, Some("true"))], async {
             let inner = CountingHybridService::default();
             let calls = inner.calls();
-            let mut service = PublicHealthEndpointLayer.layer(inner);
+            let mut service = public_health_layer().layer(inner);
 
             let response = service
                 .call(
@@ -2846,7 +2875,7 @@ mod tests {
         async_with_vars([(rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, Some("true"))], async {
             let inner = CountingHybridService::default();
             let calls = inner.calls();
-            let mut service = PublicHealthEndpointLayer.layer(inner);
+            let mut service = public_health_layer().layer(inner);
 
             let response = service
                 .call(
@@ -2874,7 +2903,7 @@ mod tests {
         async_with_vars([(rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, Some("true"))], async {
             let inner = CountingHybridService::default();
             let calls = inner.calls();
-            let mut service = PublicHealthEndpointLayer.layer(inner);
+            let mut service = public_health_layer().layer(inner);
 
             let response = service
                 .call(
@@ -2899,7 +2928,7 @@ mod tests {
         async_with_vars([(rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, Some("true"))], async {
             let inner = CountingHybridService::default();
             let calls = inner.calls();
-            let mut service = PublicHealthEndpointLayer.layer(inner);
+            let mut service = public_health_layer().layer(inner);
 
             let response = service
                 .call(
@@ -2924,7 +2953,7 @@ mod tests {
         async_with_vars([(rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, Some("true"))], async {
             let inner = CountingHybridService::default();
             let calls = inner.calls();
-            let mut service = PublicHealthEndpointLayer.layer(inner);
+            let mut service = public_health_layer().layer(inner);
 
             let response = service
                 .call(
@@ -2946,13 +2975,107 @@ mod tests {
         .await;
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn public_readiness_aliases_use_the_installed_object_progress() {
+        async_with_vars(
+            [
+                (rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, Some("true")),
+                (rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("false")),
+            ],
+            async {
+                let object_traffic_health = Arc::new(ObjectTrafficHealth::enabled_for_test(Duration::ZERO));
+                let stalled = object_traffic_health
+                    .track_read_storage()
+                    .expect("read tracking must be enabled");
+                let inner = CountingHybridService::default();
+                let calls = inner.calls();
+                let mut service = public_health_layer_with_tracker(Arc::clone(&object_traffic_health))
+                    .await
+                    .layer(inner);
+
+                let response = service
+                    .call(
+                        Request::builder()
+                            .method(Method::GET)
+                            .uri(HEALTH_READY_PATH)
+                            .body(Full::<Bytes>::from(Bytes::new()))
+                            .expect("canonical readiness request"),
+                    )
+                    .await
+                    .expect("canonical readiness response");
+                assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
+                let body = BodyExt::collect(response.into_body())
+                    .await
+                    .expect("readiness body")
+                    .to_bytes();
+                let payload: serde_json::Value = serde_json::from_slice(&body).expect("readiness JSON");
+                assert_eq!(payload["ready"], false);
+                assert_eq!(payload["degradedReasons"], serde_json::json!(["object_read_stalled"]));
+
+                let response = service
+                    .call(
+                        Request::builder()
+                            .method(Method::HEAD)
+                            .uri(MINIO_HEALTH_READY_PATH)
+                            .body(Full::<Bytes>::from(Bytes::new()))
+                            .expect("MinIO readiness request"),
+                    )
+                    .await
+                    .expect("MinIO readiness response");
+                assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
+                assert!(
+                    BodyExt::collect(response.into_body())
+                        .await
+                        .expect("HEAD body")
+                        .to_bytes()
+                        .is_empty()
+                );
+
+                let response = service
+                    .call(
+                        Request::builder()
+                            .method(Method::GET)
+                            .uri(HEALTH_COMPAT_LIVE_PATH)
+                            .body(Full::<Bytes>::from(Bytes::new()))
+                            .expect("liveness request"),
+                    )
+                    .await
+                    .expect("liveness response");
+                assert_eq!(response.status(), StatusCode::OK);
+                assert_eq!(calls.load(Ordering::SeqCst), 0);
+
+                drop(stalled);
+                let response = service
+                    .call(
+                        Request::builder()
+                            .method(Method::HEAD)
+                            .uri(HEALTH_READY_PATH)
+                            .body(Full::<Bytes>::from(Bytes::new()))
+                            .expect("recovered readiness request"),
+                    )
+                    .await
+                    .expect("recovered readiness response");
+                assert_eq!(response.status(), StatusCode::OK);
+                assert!(
+                    BodyExt::collect(response.into_body())
+                        .await
+                        .expect("HEAD body")
+                        .to_bytes()
+                        .is_empty()
+                );
+            },
+        )
+        .await;
+    }
+
     #[tokio::test]
     #[serial]
     async fn public_health_endpoint_layer_handles_minio_health_cluster_before_inner_service() {
         async_with_vars([(rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, Some("true"))], async {
             let inner = CountingHybridService::default();
             let calls = inner.calls();
-            let mut service = PublicHealthEndpointLayer.layer(inner);
+            let mut service = public_health_layer().layer(inner);
 
             let response = service
                 .call(
@@ -2977,7 +3100,7 @@ mod tests {
         async_with_vars([(rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, Some("true"))], async {
             let inner = CountingHybridService::default();
             let calls = inner.calls();
-            let mut service = PublicHealthEndpointLayer.layer(inner);
+            let mut service = public_health_layer().layer(inner);
 
             let response = service
                 .call(
@@ -3002,7 +3125,7 @@ mod tests {
         async_with_vars([(rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, Some("false"))], async {
             let inner = CountingHybridService::default();
             let calls = inner.calls();
-            let mut service = PublicHealthEndpointLayer.layer(inner);
+            let mut service = public_health_layer().layer(inner);
 
             let response = service
                 .call(
@@ -3027,7 +3150,7 @@ mod tests {
         async_with_vars([(rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, Some("false"))], async {
             let inner = CountingHybridService::default();
             let calls = inner.calls();
-            let mut service = PublicHealthEndpointLayer.layer(inner);
+            let mut service = public_health_layer().layer(inner);
 
             let response = service
                 .call(
@@ -3069,7 +3192,7 @@ mod tests {
         async_with_vars([(rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, Some("false"))], async {
             let inner = CountingHybridService::default();
             let calls = inner.calls();
-            let mut service = PublicHealthEndpointLayer.layer(inner);
+            let mut service = public_health_layer().layer(inner);
 
             let response = service
                 .call(
@@ -3092,7 +3215,7 @@ mod tests {
     async fn public_health_endpoint_layer_forwards_non_health_requests() {
         let inner = CountingHybridService::default();
         let calls = inner.calls();
-        let mut service = PublicHealthEndpointLayer.layer(inner);
+        let mut service = public_health_layer().layer(inner);
 
         let response = service
             .call(

rustfs/src/server/readiness.rs

@@ -88,6 +88,8 @@ pub enum ReadinessDegradedReason {
     IamNotReady,
     LockQuorumUnavailable,
     KmsNotReady,
+    ObjectReadStalled,
+    ObjectWriteStalled,
     ClusterHealthTimeout,
     PeerHealthUnavailable,
     StorageAndIamUnavailable,
@@ -103,6 +105,8 @@ impl ReadinessDegradedReason {
             ReadinessDegradedReason::IamNotReady => "iam_not_ready",
             ReadinessDegradedReason::LockQuorumUnavailable => "lock_quorum_unavailable",
             ReadinessDegradedReason::KmsNotReady => "kms_not_ready",
+            ReadinessDegradedReason::ObjectReadStalled => "object_read_stalled",
+            ReadinessDegradedReason::ObjectWriteStalled => "object_write_stalled",
             ReadinessDegradedReason::ClusterHealthTimeout => "cluster_health_timeout",
             ReadinessDegradedReason::PeerHealthUnavailable => "peer_health_unavailable",
             ReadinessDegradedReason::StorageAndIamUnavailable => "storage_and_iam_unavailable",
@@ -714,6 +718,11 @@ fn record_readiness_report(report: &DependencyReadinessReport) {
     }
 }
 
+pub(crate) fn record_readiness_overlay_reason(reason: ReadinessDegradedReason) {
+    gauge!(METRIC_RUNTIME_READINESS_READY).set(0.0);
+    counter!(METRIC_RUNTIME_READINESS_DEGRADED_TOTAL, "reason" => reason.as_str()).increment(1);
+}
+
 fn dependency_readiness_report_from_readiness(readiness: DependencyReadiness) -> DependencyReadinessReport {
     DependencyReadinessReport {
         degraded_reasons: degraded_reasons(readiness),

rustfs/src/storage/options.rs

@@ -59,11 +59,51 @@ use s3s::dto::VersioningConfiguration;
 #[cfg(test)]
 pub(crate) static VERSIONING_CONFIG_LOOKUPS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
 
+#[cfg(test)]
+type VersioningConfigTestHook = (String, std::sync::Arc<tokio::sync::Barrier>, std::sync::Arc<tokio::sync::Barrier>);
+
+#[cfg(test)]
+static VERSIONING_CONFIG_TEST_HOOK: std::sync::OnceLock<std::sync::Mutex<Option<VersioningConfigTestHook>>> =
+    std::sync::OnceLock::new();
+
+#[cfg(test)]
+pub(crate) fn install_versioning_config_test_hook(
+    bucket: String,
+    entered: std::sync::Arc<tokio::sync::Barrier>,
+    resume: std::sync::Arc<tokio::sync::Barrier>,
+) {
+    *VERSIONING_CONFIG_TEST_HOOK
+        .get_or_init(|| std::sync::Mutex::new(None))
+        .lock()
+        .expect("versioning config test hook lock should not be poisoned") = Some((bucket, entered, resume));
+}
+
+#[cfg(test)]
+async fn wait_for_versioning_config_test_hook(bucket: &str) {
+    let hook = {
+        let mut slot = VERSIONING_CONFIG_TEST_HOOK
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("versioning config test hook lock should not be poisoned");
+        if slot.as_ref().is_some_and(|(expected_bucket, _, _)| expected_bucket == bucket) {
+            slot.take()
+        } else {
+            None
+        }
+    };
+    if let Some((_bucket, entered, resume)) = hook {
+        entered.wait().await;
+        resume.wait().await;
+    }
+}
+
 /// Fetch the bucket's versioning configuration once so callers can derive
 /// enabled/suspended state without repeated metadata-sys lookups per request.
 pub(crate) async fn bucket_versioning_config(bucket: &str) -> VersioningConfiguration {
     #[cfg(test)]
     VERSIONING_CONFIG_LOOKUPS.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+    #[cfg(test)]
+    wait_for_versioning_config_test_hook(bucket).await;
     match BucketVersioningSys::get(bucket).await {
         Ok(cfg) => cfg,
         Err(err) => {

rustfs/src/storage/storage_api.rs

@@ -1034,6 +1034,10 @@ pub(crate) async fn save_config_no_lock(api: Arc<ECStore>, file: &str, data: Vec
     ecstore_config::com::save_config_no_lock(api, file, data).await
 }
 
+pub(crate) async fn delete_config_no_lock(api: Arc<ECStore>, file: &str) -> Result<()> {
+    ecstore_config::com::delete_config_no_lock(api, file).await
+}
+
 pub(crate) async fn with_config_object_write_lock<F, Fut, T>(api: Arc<ECStore>, object: String, operation: F) -> Result<T>
 where
     F: FnOnce() -> Fut + Send + 'static,

scripts/check_logging_guardrails.sh

@@ -842,6 +842,34 @@ for file in "${disk_logging_files[@]}"; do
   fi
 done
 
+# `set_disks` expands every Disk through Debug, including raw format bytes and
+# the full per-operation metrics ring. Keep it out of the INFO scanner span.
+scanner_disk_skip_pattern='#\[(tracing::)?instrument\([^]]*skip\([^)]*\bset_disks\b[^)]*\)[^]]*\)\][[:space:]]*async fn nsscanner_disk\b'
+if ! rg -U "$scanner_disk_skip_pattern" crates/scanner/src/scanner_io.rs >/dev/null; then
+  echo "❌ logging guardrail violation: nsscanner_disk must skip set_disks in its tracing instrumentation" >&2
+  exit 1
+fi
+
+for fixture in \
+  $'#[tracing::instrument(skip(self, budget, updates, cache, set_disks))]\nasync fn nsscanner_disk('; do
+  if ! printf '%s\n' "$fixture" | rg -U "$scanner_disk_skip_pattern" >/dev/null; then
+    echo "❌ logging guardrail self-test failed: safe nsscanner_disk span was rejected" >&2
+    echo "$fixture" >&2
+    exit 1
+  fi
+done
+
+for fixture in \
+  $'#[tracing::instrument(skip(self, budget, updates, cache))]\nasync fn nsscanner_disk(' \
+  $'#[tracing::instrument(skip(self, budget, updates, cache), fields(set_disks = set_disks.len()))]\nasync fn nsscanner_disk(' \
+  $'#[tracing::instrument(skip(self, budget, updates, cache, set_disks_count))]\nasync fn nsscanner_disk('; do
+  if printf '%s\n' "$fixture" | rg -U "$scanner_disk_skip_pattern" >/dev/null; then
+    echo "❌ logging guardrail self-test failed: unsafe nsscanner_disk span was accepted" >&2
+    echo "$fixture" >&2
+    exit 1
+  fi
+done
+
 # `forbidden_patterns` above only retires log lines that already shipped, so a
 # newly written sentence-style log passes every check in this script — which is
 # how one reaches review in the first place (PR #5822 added

scripts/check_s3s_footprint.sh

@@ -22,10 +22,13 @@ set -euo pipefail
 
 cd "$(dirname "$0")/.."
 
-# Baselines verified on 2026-08-06. Lower-only; see header.
+# Baselines verified on 2026-08-11. Lower-only; see header.
-S3S_IMPORT_FILES_BASELINE=236
+# Excludes crates/e2e_test/ — test infrastructure legitimately uses s3s
-S3_ERROR_LINES_BASELINE=1678
+# to verify S3 behavior and does not widen the production s3s surface.
+S3S_IMPORT_FILES_BASELINE=213
+S3_ERROR_LINES_BASELINE=1617
 S3S_PATH_PATTERN='(^|[^"[:alnum:]_])s3s::'
+E2E_TEST_GLOB='--glob=!crates/e2e_test/**'
 
 TMP_DIR="$(mktemp -d)"
 trap 'rm -rf "$TMP_DIR"' EXIT
@@ -42,8 +45,8 @@ run_rg_to() {
     fi
 }
 
-run_rg_to "$TMP_DIR/import_files" -l "$S3S_PATH_PATTERN" --type rust
+run_rg_to "$TMP_DIR/import_files" -l "$S3S_PATH_PATTERN" --type rust $E2E_TEST_GLOB
-run_rg_to "$TMP_DIR/error_lines" -c 's3_error!' --type rust
+run_rg_to "$TMP_DIR/error_lines" -c 's3_error!' --type rust $E2E_TEST_GLOB
 
 s3s_import_files="$(grep -c . "$TMP_DIR/import_files" || true)"
 s3_error_lines="$(awk -F: '{sum += $NF} END {print sum + 0}' "$TMP_DIR/error_lines")"
@@ -76,9 +79,9 @@ check_ratchet() {
 }
 
 check_ratchet "files importing s3s" "$s3s_import_files" "$S3S_IMPORT_FILES_BASELINE" \
-    "rg -l '$S3S_PATH_PATTERN' --type rust"
+    "rg -l '$S3S_PATH_PATTERN' --type rust $E2E_TEST_GLOB"
 check_ratchet "s3_error! invocation lines" "$s3_error_lines" "$S3_ERROR_LINES_BASELINE" \
-    "rg -c 's3_error!' --type rust"
+    "rg -c 's3_error!' --type rust $E2E_TEST_GLOB"
 
 if ((status != 0)); then
     exit 1