merge: sync main and preserve prepared delete ownership

Eliminate redundant delete recovery inspections, inline-object data-dir scans, redundant directory fsyncs, and Fsync/Foreground worker-lane ping-pong during single-key and batch S3 DELETE commits.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: d85aaf47-2a4b-4c2c-97a3-dbe8c7717e2a

.config/e2e-distributed-selection.txt

@@ -1,2 +1,2 @@
-sha256-linux=e31182a075f38b8a6680f4563ed674f0cb92d7ad9e764d3da667c99510656325
-sha256-darwin=e31182a075f38b8a6680f4563ed674f0cb92d7ad9e764d3da667c99510656325
+sha256-linux=74a1d6975e2d6dfe01363585cb9bfa3232d79556a154d4f599653fe2ed1ce696
+sha256-darwin=74a1d6975e2d6dfe01363585cb9bfa3232d79556a154d4f599653fe2ed1ce696

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=359b16f0bba8baa6e995fd3701062a61088fd4c0196ade3bac9e8fda7a050833
-sha256-linux=137098f8bc31a5ca38cac35a5b628346c1082b0277ec3982e2e0ffb69d0ea2e4
+sha256-darwin=b53fe6d46cdffedfad6a83da0dfffa9aec10cd7c8ace53469f129587bc9f4d50
+sha256-linux=6be90f5c3886e9852ac27d51f2d47e6d2fa8bb424e0d03ee931268dcd0020db8

.config/e2e-smoke-selection.txt

@@ -1 +1 @@
-sha256=eaa140bb7c7fff74c92a0cd191805f7fdee3174eaa6a649a1b807e1e6b497a31
+sha256=002ec5d973f82288e50a096a2c8b3b38749f62a7a3e169775566874d8ee98b4a

.github/workflows/e2e-upgrade.yml

@@ -25,6 +25,7 @@ on:
       - "crates/e2e_test/src/upgrade_compatibility_test.rs"
       - "crates/ecstore/**"
       - "crates/filemeta/**"
+      - "crates/heal/**"
       - "crates/kms/**"
       - "crates/storage-api/**"
       - "rustfs/**"
@@ -160,6 +161,7 @@ jobs:
           )
           if [[ "$UPGRADE_SOURCE_VERSION" == "1.0.0-rc.5" ]]; then
             cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
+            cases+=(checkpoint-upgrade:rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects)
           else
             cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects)
           fi

.github/workflows/windows-filesystem.yml

@@ -25,6 +25,8 @@ on:
       - "crates/ecstore/src/object_api/write_commit_context.rs"
       - "crates/ecstore/src/bucket/utils.rs"
       - "crates/ecstore/Cargo.toml"
+      - "crates/storage-api/src/bucket.rs"
+      - "crates/storage-api/Cargo.toml"
       - "crates/utils/src/dunce.rs"
       - "crates/utils/src/os/**"
       - "crates/utils/Cargo.toml"
@@ -48,6 +50,8 @@ on:
       - "crates/ecstore/src/object_api/write_commit_context.rs"
       - "crates/ecstore/src/bucket/utils.rs"
       - "crates/ecstore/Cargo.toml"
+      - "crates/storage-api/src/bucket.rs"
+      - "crates/storage-api/Cargo.toml"
       - "crates/utils/src/dunce.rs"
       - "crates/utils/src/os/**"
       - "crates/utils/Cargo.toml"
@@ -140,6 +144,17 @@ jobs:
         shell: pwsh
         run: "cargo test -p rustfs-ecstore --lib bucket::utils:: -- --nocapture"
 
+      - name: Test artifact prefix Windows file-name rules
+        shell: pwsh
+        run: |
+          $testName = 'bucket::tests::artifact_prefix_preserves_host_file_name_rules'
+          cargo test --locked -p rustfs-storage-api --lib $testName -- --exact --nocapture --color never 2>&1 |
+            Tee-Object -Variable testOutput
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+          if (($testOutput -join "`n") -notmatch 'test result: ok\. 1 passed; 0 failed; 0 ignored;') {
+            throw 'Expected exactly one passing artifact prefix test on Windows'
+          }
+
       - name: Test Windows file-name rules
         shell: pwsh
         run: "cargo test -p rustfs-utils --no-default-features --features path --lib dunce:: -- --nocapture"

Cargo.toml

@@ -252,9 +252,9 @@ atoi = "3.1.1"
 atomic_enum = "0.3.0"
 aws-config = { version = "1.13.0" }
 aws-credential-types = { version = "1.3.1" }
-aws-sdk-kms = { default-features = false, version = "1.125.0" }
-aws-sdk-s3 = { default-features = false, version = "1.154.0" }
-aws-sdk-sts = { default-features = false, version = "1.121.0" }
+aws-sdk-kms = { default-features = false, version = "1.126.0" }
+aws-sdk-s3 = { default-features = false, version = "1.155.0" }
+aws-sdk-sts = { default-features = false, version = "1.122.0" }
 aws-smithy-async = { version = "1.3.1" }
 aws-smithy-http-client = { default-features = false, version = "1.5.1" }
 aws-smithy-runtime-api = { version = "1.19.1" }
@@ -272,7 +272,7 @@ crossbeam-utils = "0.8.23"
 datafusion = { default-features = false, version = "55.2.0" }
 derive_builder = "0.20.2"
 ed25519-dalek = "3.0.0"
-enumset = "1.1.14"
+enumset = "1.1.15"
 faster-hex = "1.0.0"
 flate2 = "1.1.10"
 glob = "0.3.4"
@@ -317,7 +317,7 @@ rustfs-erasure-codec = { version = "9.0.0" }
 reed-solomon-simd = "3.1.0"
 regex = { version = "1.13.1" }
 rumqttc = { package = "rumqttc-next", version = "0.34.0" }
-redis = { version = "1.7.1" }
+redis = { version = "1.8.0" }
 rustify = { version = "0.7", default-features = false }
 rustix = { version = "1.1.5" }
 rust-embed = { version = "8.13.0" }
@@ -327,7 +327,7 @@ s3s-sigv4 = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fc
 serial_test = "4.0.1"
 shadow-rs = { default-features = false, version = "2.0.0" }
 siphasher = "1.0.4"
-smallvec = { version = "1.16.3" }
+smallvec = { version = "1.16.4" }
 compact_str = "0.10.0"
 snap = "1.1.2"
 starshard = { version = "2.4.0" }
@@ -346,7 +346,7 @@ tracing-subscriber = { version = "0.3.23" }
 transform-stream = "0.3.1"
 url = "2.5.8"
 urlencoding = "2.1.3"
-uuid = { version = "1.28.0" }
+uuid = { version = "1.29.0" }
 vaultrs = { version = "0.8.0" }
 tar = "0.4.46"
 walkdir = "2.5.0"
@@ -354,7 +354,7 @@ winapi-util = "0.1.11"
 windows = { version = "0.62.2" }
 windows-sys = "0.61.2"
 xxhash-rust = { version = "0.8.19" }
-zip = "9.0.5"
+zip = "9.1.0"
 zstd = "0.14.3"
 
 # Observability and Metrics
@@ -368,7 +368,7 @@ opentelemetry-proto = { version = "0.33.1", default-features = false, features =
 opentelemetry_sdk = { version = "0.33.1" }
 opentelemetry-semantic-conventions = { version = "0.33.1" }
 opentelemetry-stdout = { version = "0.33.1" }
-pyroscope = { version = "2.1.4" }
+pyroscope = { version = "2.1.5" }
 
 # FTP and SFTP
 libunftp = { version = "0.23.1" }
@@ -382,9 +382,9 @@ russh-sftp = "3.0.1"
 dav-server = "0.11.0"
 
 # Performance Analysis and Memory Profiling
-rustfs-mimalloc = { version = "0.7.1" }
+rustfs-mimalloc = { version = "0.7.2" }
 # Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
-hotpath = { version = "0.28.7", default-features = false }
+hotpath = { version = "0.29.1", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.49" }
 
@@ -421,3 +421,19 @@ inherits = "release"
 inherits = "release"
 debug = true
 strip = "none"
+
+# Pin upstream fixes until compatible crates.io releases are available.
+[patch.crates-io]
+# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
+# Upstream: openidconnect-rs#244 and oauth2-rs#352.
+openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
+oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }
+aws-sigv4 = { git = "https://github.com/smithy-lang/smithy-rs", rev = "5c1c4bca2ea7af97496e077c9f1a718eaac64bf8" }
+
+# Keep the signer's path dependencies on the SDK's existing registry types.
+[patch."https://github.com/smithy-lang/smithy-rs"]
+aws-credential-types = { version = "=1.3.1" }
+aws-smithy-eventstream = { version = "=0.61.6" }
+aws-smithy-http = { version = "=0.64.2" }
+aws-smithy-runtime-api = { version = "=1.19.1" }
+aws-smithy-types = { version = "=1.9.0" }

crates/concurrency/src/adaptive.rs

@@ -0,0 +1,957 @@
+// Copyright 2026 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+//! Mutable admission capacity without replacing or revoking held permits.
+
+use std::num::NonZeroU16;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
+use std::time::Duration;
+use tokio::sync::{Notify, Semaphore};
+use tokio::time::Instant;
+
+/// A resource owner publishes a capacity; running units retain their permits
+/// after a reduction. Lease slots are reserved once, with no per-unit Arc
+/// allocation. Tokio wait queues may lazily initialize platform mutex storage
+/// on first contention. It does not discover devices, start
+/// workers, or decide resource policy. Cancellation drops the acquire future.
+///
+/// Uncontended admission is atomic. Parked work takes a FIFO admission turn,
+/// so repeated fast acquisitions cannot starve an older waiter. Source/age
+/// priority still belongs to the caller's scheduler, before joining this queue.
+pub struct AdaptiveAdmission {
+    // One CAS observes both capacity and active count, so a concurrent shrink
+    // cannot admit a new unit using an already superseded capacity.
+    state: AtomicU64,
+    origin: Instant,
+    valid_until_millis: AtomicU64,
+    // Only the FIFO turn holder subscribes. A stored notification is harmless:
+    // capacity and freshness are always checked again after waking.
+    changed: Notify,
+    queued: AtomicUsize,
+    // FIFO admission turn, not a mutable-state lock. No storage lock or
+    // admitted permit may be held while joining. This private semaphore is
+    // never closed; release and resize do not acquire it.
+    turn: Semaphore,
+    maximum: u32,
+    slots: Box<[PermitSlots]>,
+    next_chunk: AtomicUsize,
+}
+
+struct PermitSlots {
+    free: AtomicU64,
+    references: [AtomicUsize; 64],
+}
+
+struct QueuedAdmission<'a>(&'a AdaptiveAdmission);
+
+impl Drop for QueuedAdmission<'_> {
+    fn drop(&mut self) {
+        self.0.queued.fetch_sub(1, Ordering::AcqRel);
+    }
+}
+
+/// One active unit. Clones share that unit without allocating; the final drop
+/// returns it to the same owner, including after resize or waiter cancellation.
+#[must_use = "hold the permit for the admitted unit's complete lifetime"]
+pub struct AdaptivePermit {
+    owner: Arc<AdaptiveAdmission>,
+    slot: usize,
+}
+
+impl Clone for AdaptivePermit {
+    fn clone(&self) -> Self {
+        // Every slot reference owns a strong owner reference. Clone the Arc
+        // first, so its overflow protection also bounds the slot refcount.
+        let owner = Arc::clone(&self.owner);
+        owner.slots[self.slot / 64].references[self.slot % 64].fetch_add(1, Ordering::Relaxed);
+        Self { owner, slot: self.slot }
+    }
+}
+
+impl std::fmt::Debug for AdaptivePermit {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("AdaptivePermit")
+            .field("slot", &self.slot)
+            .field("owner", &self.owner.snapshot())
+            .finish()
+    }
+}
+
+/// One coherent view of configured capacity and actual active units.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct AdaptiveAdmissionSnapshot {
+    /// Current cap on new admissions. Zero pauses all new units.
+    pub limit: u32,
+    /// Existing units, which may temporarily exceed a reduced limit.
+    pub active: u32,
+}
+
+/// Node scheduling capacity, separate from physical I/O and buffer ownership.
+/// Its non-cloneable units end with the async operation; a native tail must
+/// retain its own physical-resource unit, never this scheduling unit.
+pub struct NodeAdmission {
+    capacity: Arc<AdaptiveAdmission>,
+    completed: AtomicU64,
+}
+
+/// One async operation's scheduling unit. Dropping a cancelled or failed
+/// operation releases capacity without claiming completed work.
+#[must_use = "retain the unit until the async operation ends"]
+pub struct NodePermit {
+    _capacity: AdaptivePermit,
+    owner: Arc<NodeAdmission>,
+}
+
+impl NodeAdmission {
+    /// Construct one node owner with no capacity until a resource sample arrives.
+    pub fn new(maximum: NonZeroU16) -> Arc<Self> {
+        Arc::new(Self {
+            capacity: AdaptiveAdmission::new(maximum, 0),
+            completed: AtomicU64::new(0),
+        })
+    }
+
+    /// Publish feedback with this owner's actual completed-work counter.
+    pub fn update(&self, controller: &mut FeedbackController, observation: Option<FeedbackObservation>) -> FeedbackDecision {
+        let observation = observation.map(|mut observation| {
+            observation.completed_units = self.completed();
+            observation
+        });
+        controller.update(&self.capacity, observation)
+    }
+
+    /// Current leased scheduling cap and held async units.
+    pub fn snapshot(&self) -> AdaptiveAdmissionSnapshot {
+        self.capacity.snapshot()
+    }
+
+    /// Completed operations, or unknown after the counter exhausts its range.
+    pub fn completed(&self) -> Option<u64> {
+        let value = self.completed.load(Ordering::Acquire);
+        (value != u64::MAX).then_some(value)
+    }
+
+    /// Try without waiting, including when the caller already owns a lock.
+    pub fn try_acquire(self: &Arc<Self>) -> Option<NodePermit> {
+        self.capacity.try_acquire().map(|capacity| NodePermit {
+            _capacity: capacity,
+            owner: self.clone(),
+        })
+    }
+
+    /// Wait in FIFO order. Callers must provide a deadline/cancellation and
+    /// acquire before storage locks, not while retaining another node unit.
+    pub async fn acquire(self: &Arc<Self>) -> NodePermit {
+        NodePermit {
+            _capacity: self.capacity.acquire().await,
+            owner: self.clone(),
+        }
+    }
+}
+
+impl NodePermit {
+    /// Count a completed operation exactly once, then release its node unit.
+    /// This counter is scheduling progress, not an authoritative heal receipt.
+    pub fn complete(self) {
+        let _ = self
+            .owner
+            .completed
+            .try_update(Ordering::AcqRel, Ordering::Acquire, |value| value.checked_add(1));
+    }
+}
+
+/// Confidence-qualified pressure for one complete sampling window.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ResourcePressure {
+    /// Missing, stale, reset, or insufficient observations; never evidence of idle.
+    Unknown,
+    /// A resource or foreground-protection threshold was reached.
+    Busy,
+    /// All required observations were fresh and below their low watermarks.
+    Stable,
+}
+
+/// Resource-owner input to the controller, after applying operator and kernel
+/// caps. This describes one owner; callers must not duplicate a device's budget
+/// for aliases, pools, or remote requesters.
+#[derive(Debug, Clone, Copy)]
+pub struct FeedbackObservation {
+    /// Identity of the resource/telemetry generation. Replacement resets probes.
+    pub generation: u64,
+    /// Upper bound imposed by resources and explicit configuration; zero blocks.
+    pub max_concurrency: u32,
+    /// Complete-window pressure, including the confidence of its sources.
+    pub pressure: ResourcePressure,
+    /// Cumulative completed work in this generation, not just scheduled work.
+    pub completed_units: Option<u64>,
+}
+
+/// Why the latest concurrency limit was selected.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum FeedbackReason {
+    /// The resource cap is unavailable; no new unit can be bounded safely.
+    Unavailable,
+    /// A changed owner or reset work counter requires a new baseline.
+    NewBaseline,
+    /// The resource or operator cap reduced the permitted concurrency.
+    CapReduced,
+    /// Pressure caused an immediate multiplicative reduction.
+    Pressure,
+    /// Incomplete telemetry retains at most one maintenance unit.
+    LowConfidence,
+    /// Waiting for consecutive stable windows, work progress and cooldown.
+    Recovering,
+    /// One additional unit is being probed after a stable recovery period.
+    Probe,
+}
+
+/// One controller decision, suitable for metrics without dynamic labels.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct FeedbackDecision {
+    /// New admissions use this cap; existing permits remain valid.
+    pub limit: u32,
+    /// Decision cause, distinct from a proof of work completion or storage health.
+    pub reason: FeedbackReason,
+}
+
+/// Single-owner additive probe/multiplicative decrease controller. It adjusts
+/// only concurrency. Every source must be fresh before declaring `Stable`.
+///
+/// The sampler runs every second. A gap over two seconds resets recovery; eight
+/// consecutive stable seconds and observed completed work allow one extra unit.
+/// Pressure or lower caps apply immediately. This policy is not a performance
+/// claim; its parameters require the same workload/foreground-SLO acceptance.
+pub struct FeedbackController {
+    previous: Option<(Instant, FeedbackObservation)>,
+    stable_since: Option<Instant>,
+    last_change: Instant,
+    limit: u32,
+}
+
+impl Default for FeedbackController {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl FeedbackController {
+    /// Start without a resource observation or an admission grant.
+    pub fn new() -> Self {
+        Self {
+            previous: None,
+            stable_since: None,
+            last_change: Instant::now(),
+            limit: 0,
+        }
+    }
+
+    /// Consume one sample and publish a three-second lease to the same gate.
+    /// If this function stops being called, admission expires independently.
+    pub fn update(&mut self, gate: &AdaptiveAdmission, observation: Option<FeedbackObservation>) -> FeedbackDecision {
+        let observation = observation.map(|mut observation| {
+            observation.max_concurrency = observation.max_concurrency.min(gate.maximum);
+            observation
+        });
+        let decision = self.decide(Instant::now(), observation);
+        gate.set_sampled_limit(decision.limit, Duration::from_secs(3));
+        decision
+    }
+
+    fn decide(&mut self, now: Instant, observation: Option<FeedbackObservation>) -> FeedbackDecision {
+        let Some(observation) = observation else {
+            self.previous = None;
+            self.stable_since = None;
+            self.last_change = now;
+            self.limit = 0;
+            return FeedbackDecision {
+                limit: 0,
+                reason: FeedbackReason::Unavailable,
+            };
+        };
+        let previous = self.previous.replace((now, observation));
+        let contiguous = previous.is_some_and(|(at, old)| {
+            now.saturating_duration_since(at) <= Duration::from_secs(2)
+                && old.generation == observation.generation
+                && matches!((old.completed_units, observation.completed_units), (Some(a), Some(b)) if b >= a)
+        });
+        let advanced = previous
+            .is_some_and(|(_, old)| matches!((old.completed_units, observation.completed_units), (Some(a), Some(b)) if b > a));
+        let old_limit = self.limit;
+        let reason = if observation.max_concurrency == 0 {
+            self.limit = 0;
+            self.stable_since = None;
+            FeedbackReason::CapReduced
+        } else if observation.pressure == ResourcePressure::Busy {
+            self.limit = (self.limit / 2).max(1).min(observation.max_concurrency);
+            self.stable_since = None;
+            // Repeated busy windows restart cooldown, even at the minimum cap.
+            self.last_change = now;
+            FeedbackReason::Pressure
+        } else if observation.max_concurrency < self.limit {
+            self.limit = observation.max_concurrency;
+            self.stable_since = None;
+            FeedbackReason::CapReduced
+        } else if observation.pressure == ResourcePressure::Unknown || observation.completed_units.is_none() {
+            self.limit = 1.min(observation.max_concurrency);
+            self.stable_since = None;
+            FeedbackReason::LowConfidence
+        } else if !contiguous || self.limit == 0 {
+            self.limit = 1.min(observation.max_concurrency);
+            self.stable_since = None;
+            self.last_change = now;
+            FeedbackReason::NewBaseline
+        } else {
+            let since = *self.stable_since.get_or_insert(now);
+            if advanced
+                && now.saturating_duration_since(since) >= Duration::from_secs(8)
+                && now.saturating_duration_since(self.last_change) >= Duration::from_secs(8)
+                && self.limit < observation.max_concurrency
+            {
+                self.limit += 1;
+                FeedbackReason::Probe
+            } else {
+                FeedbackReason::Recovering
+            }
+        };
+        if old_limit != self.limit {
+            self.last_change = now;
+        }
+        FeedbackDecision {
+            limit: self.limit,
+            reason,
+        }
+    }
+}
+
+impl AdaptiveAdmission {
+    /// Construct a bounded owner, with zero initial capacity allowed for unknown
+    /// input. The immutable maximum reserves at most 65,535 lease slots once;
+    /// acquiring or cloning a permit does not allocate a per-unit Arc.
+    pub fn new(maximum: NonZeroU16, limit: u32) -> Arc<Self> {
+        let maximum_u32 = u32::from(maximum.get());
+        let maximum = usize::from(maximum.get());
+        let slots = (0..maximum.div_ceil(64))
+            .map(|chunk| {
+                let count = (maximum - chunk * 64).min(64);
+                PermitSlots {
+                    free: AtomicU64::new(if count == 64 { u64::MAX } else { (1_u64 << count) - 1 }),
+                    references: std::array::from_fn(|_| AtomicUsize::new(0)),
+                }
+            })
+            .collect::<Vec<_>>()
+            .into_boxed_slice();
+        Arc::new(Self {
+            state: AtomicU64::new(u64::from(limit.min(maximum_u32)) << 32),
+            origin: Instant::now(),
+            valid_until_millis: AtomicU64::new(u64::MAX),
+            changed: Notify::new(),
+            queued: AtomicUsize::new(0),
+            turn: Semaphore::new(1),
+            maximum: maximum_u32,
+            slots,
+            next_chunk: AtomicUsize::new(0),
+        })
+    }
+
+    /// Publish a cap bounded by the immutable maximum, without losing held work
+    /// or changing owner identity.
+    pub fn set_limit(&self, limit: u32) {
+        let limit = limit.min(self.maximum);
+        let _ = self.state.try_update(Ordering::AcqRel, Ordering::Acquire, |state| {
+            Some((u64::from(limit) << 32) | (state & u64::from(u32::MAX)))
+        });
+        self.changed.notify_one();
+    }
+
+    /// Publish a cap with a finite observation lifetime. Expired observations
+    /// block new admissions even if the sampling worker has stopped. Existing
+    /// permits are retained. A single resource owner must serialize publishers.
+    ///
+    /// Capacity is published before freshness, so a concurrent reader can
+    /// temporarily reject a fresh update but cannot use an increased lifetime
+    /// with the previous, potentially larger capacity.
+    pub fn set_sampled_limit(&self, limit: u32, valid_for: Duration) {
+        self.set_limit(limit);
+        let deadline = self
+            .elapsed_millis()
+            .saturating_add(u64::try_from(valid_for.as_millis()).unwrap_or(u64::MAX));
+        self.valid_until_millis.store(deadline, Ordering::Release);
+        self.changed.notify_one();
+    }
+
+    fn elapsed_millis(&self) -> u64 {
+        u64::try_from(self.origin.elapsed().as_millis()).unwrap_or(u64::MAX)
+    }
+
+    fn fresh(&self) -> bool {
+        self.elapsed_millis() < self.valid_until_millis.load(Ordering::Acquire)
+    }
+
+    /// Read the cap and active count from the same atomic observation.
+    pub fn snapshot(&self) -> AdaptiveAdmissionSnapshot {
+        // Acquire freshness first: a newly published lease must not be paired
+        // with capacity loaded before its preceding reduction.
+        let fresh = self.fresh();
+        let state = self.state.load(Ordering::Acquire);
+        AdaptiveAdmissionSnapshot {
+            limit: if fresh {
+                u32::try_from(state >> 32).unwrap_or(u32::MAX)
+            } else {
+                0
+            },
+            active: u32::try_from(state & u64::from(u32::MAX)).unwrap_or(u32::MAX),
+        }
+    }
+
+    /// Admit immediately if capacity remains, without a per-unit allocation.
+    /// The uncontended path uses atomics; refunding a racing acquisition may
+    /// wake the FIFO head through Tokio's notification queue.
+    pub fn try_acquire(self: &Arc<Self>) -> Option<AdaptivePermit> {
+        if self.queued.load(Ordering::Acquire) != 0 {
+            return None;
+        }
+        let permit = self.acquire_capacity()?;
+        // A waiter can register between the first queue check and the capacity
+        // CAS. Refund before exposing the permit rather than stealing its turn.
+        (self.queued.load(Ordering::Acquire) == 0).then_some(permit)
+    }
+
+    fn acquire_capacity(self: &Arc<Self>) -> Option<AdaptivePermit> {
+        if !self.fresh() {
+            return None;
+        }
+        let state = self.state.load(Ordering::Acquire);
+        if state & u64::from(u32::MAX) >= state >> 32 {
+            return None;
+        }
+        let slot = self.reserve_slot()?;
+        if self
+            .state
+            .try_update(Ordering::AcqRel, Ordering::Acquire, |state| {
+                let active = state & u64::from(u32::MAX);
+                (active < state >> 32).then(|| state + 1)
+            })
+            .is_err()
+        {
+            self.release_slot(slot);
+            self.changed.notify_one();
+            return None;
+        }
+        self.slots[slot / 64].references[slot % 64].store(1, Ordering::Release);
+        let permit = AdaptivePermit {
+            owner: self.clone(),
+            slot,
+        };
+        // The capacity CAS can retry across an observation expiry. Recheck
+        // before giving ownership to the caller; dropping refunds this unit.
+        self.fresh().then_some(permit)
+    }
+
+    fn reserve_slot(&self) -> Option<usize> {
+        let first = self.next_chunk.fetch_add(1, Ordering::Relaxed) % self.slots.len();
+        for offset in 0..self.slots.len() {
+            let chunk_index = (first + offset) % self.slots.len();
+            let free = &self.slots[chunk_index].free;
+            if let Ok(bits) = free.try_update(Ordering::AcqRel, Ordering::Acquire, |bits| (bits != 0).then(|| bits & (bits - 1)))
+            {
+                // trailing_zeros is in 0..64 because the zero word is rejected.
+                return Some(chunk_index * 64 + bits.trailing_zeros() as usize);
+            }
+        }
+        None
+    }
+
+    fn release_slot(&self, slot: usize) {
+        self.slots[slot / 64].free.fetch_or(1_u64 << (slot % 64), Ordering::Release);
+    }
+
+    /// Wait for capacity. The future owns no permit until it returns; dropping
+    /// a pending future leaves capacity untouched. Wrap in the caller's existing
+    /// cancellation/deadline select before acquiring any storage commit locks.
+    pub async fn acquire(self: &Arc<Self>) -> AdaptivePermit {
+        if let Some(permit) = self.try_acquire() {
+            return permit;
+        }
+        self.queued.fetch_add(1, Ordering::AcqRel);
+        let _queued = QueuedAdmission(self);
+        // Retain the successful permit inside its Result. No code can close
+        // this private semaphore, so acquisition cannot return an error.
+        let _turn = self.turn.acquire().await;
+        loop {
+            let notified = self.changed.notified();
+            tokio::pin!(notified);
+            // Register before checking capacity to avoid a lost release/resize.
+            notified.as_mut().enable();
+            if let Some(permit) = self.acquire_capacity() {
+                return permit;
+            }
+            notified.await;
+        }
+    }
+}
+
+impl Drop for AdaptivePermit {
+    fn drop(&mut self) {
+        if self.owner.slots[self.slot / 64].references[self.slot % 64].fetch_sub(1, Ordering::AcqRel) != 1 {
+            return;
+        }
+        // Publish the free slot before exposing another admission capacity.
+        self.owner.release_slot(self.slot);
+        self.owner.state.fetch_sub(1, Ordering::AcqRel);
+        self.owner.changed.notify_one();
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn node_progress_requires_explicit_completion_and_never_wraps() {
+        let node = NodeAdmission::new(NonZeroU16::new(2).expect("two slots"));
+        node.capacity.set_limit(2);
+        drop(node.acquire().await);
+        assert_eq!(node.completed(), Some(0), "cancellation is not progress");
+        node.acquire().await.complete();
+        assert_eq!(node.completed(), Some(1));
+        node.completed.store(u64::MAX - 1, Ordering::Release);
+        node.acquire().await.complete();
+        node.acquire().await.complete();
+        assert_eq!(node.completed(), None, "exhausted progress stays unknown");
+        assert_eq!(node.snapshot().active, 0);
+    }
+
+    #[tokio::test]
+    async fn cancelled_node_work_does_not_reserve_capacity_for_unrelated_native_tail() {
+        let node = NodeAdmission::new(NonZeroU16::new(1).expect("one slot"));
+        node.capacity.set_limit(1);
+        let physical = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one disk unit"), 1);
+        let scheduling = node.acquire().await;
+        let native_tail = physical.acquire().await;
+        drop(scheduling);
+        assert_eq!(node.completed(), Some(0));
+        assert!(node.try_acquire().is_some(), "healthy resources can still be scheduled");
+        assert!(physical.try_acquire().is_none(), "the stalled disk remains charged");
+        drop(native_tail);
+        assert!(physical.try_acquire().is_some());
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn node_feedback_expires_without_revoking_held_work_or_fabricating_progress() {
+        let node = NodeAdmission::new(NonZeroU16::new(2).expect("two slots"));
+        let mut controller = FeedbackController::new();
+        let input = FeedbackObservation {
+            generation: 1,
+            max_concurrency: 2,
+            pressure: ResourcePressure::Stable,
+            completed_units: Some(99),
+        };
+        node.update(&mut controller, Some(input));
+        let held = node.acquire().await;
+        assert_eq!(node.completed(), Some(0));
+        tokio::time::advance(Duration::from_secs(4)).await;
+        assert_eq!(node.snapshot(), AdaptiveAdmissionSnapshot { limit: 0, active: 1 });
+        assert!(node.try_acquire().is_none());
+        held.complete();
+        assert_eq!(node.completed(), Some(1));
+        assert_eq!(node.snapshot().active, 0);
+    }
+
+    #[tokio::test]
+    async fn shared_unit_stays_admitted_until_native_worker_releases_its_last_clone() {
+        let gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one unit"), 1);
+        let outer = gate.acquire().await;
+        let native = outer.clone();
+        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
+        let (release_tx, release_rx) = tokio::sync::oneshot::channel();
+        let worker = tokio::task::spawn_blocking(move || {
+            started_tx.send(()).expect("worker started");
+            release_rx.blocking_recv().expect("release native worker");
+            drop(native);
+        });
+        started_rx.await.expect("native worker is live");
+        drop(outer);
+        assert_eq!(gate.snapshot().active, 1);
+        assert!(gate.try_acquire().is_none());
+        gate.set_limit(0);
+        assert_eq!(gate.snapshot().active, 1);
+        release_tx.send(()).expect("release tail");
+        worker.await.expect("native tail completed");
+        assert_eq!(gate.snapshot().active, 0);
+        gate.set_limit(1);
+        assert!(gate.try_acquire().is_some());
+    }
+
+    #[tokio::test]
+    async fn bounded_slots_cross_word_boundaries_and_recycle_only_after_final_drop() {
+        let maximum = NonZeroU16::new(65).expect("partial second chunk");
+        let gate = AdaptiveAdmission::new(maximum, u32::MAX);
+        assert_eq!(gate.snapshot().limit, 65);
+        for _ in 0..4 {
+            let units: Vec<_> = (0..65).map(|_| gate.try_acquire().expect("free slot")).collect();
+            let tails = units.clone();
+            assert_eq!(gate.snapshot().active, 65);
+            assert!(gate.try_acquire().is_none());
+            drop(units);
+            assert_eq!(gate.snapshot().active, 65);
+            gate.set_limit(0);
+            for (remaining, tail) in (0..65).rev().zip(tails) {
+                drop(tail);
+                assert_eq!(gate.snapshot().active, remaining);
+            }
+            gate.set_limit(u32::MAX);
+            assert_eq!(gate.snapshot().limit, 65);
+        }
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    async fn shared_units_survive_concurrent_clone_drop_reuse_and_resize() {
+        let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded owner"), 8);
+        let mut workers = Vec::new();
+        for _ in 0..16 {
+            let gate = gate.clone();
+            workers.push(tokio::spawn(async move {
+                for _ in 0..64 {
+                    let root = gate.acquire().await;
+                    let one = root.clone();
+                    let two = one.clone();
+                    drop(root);
+                    tokio::task::yield_now().await;
+                    drop(two);
+                    tokio::task::yield_now().await;
+                    drop(one);
+                }
+            }));
+        }
+        for limit in [0, 1, 8, 2, 0, 8] {
+            gate.set_limit(limit);
+            tokio::task::yield_now().await;
+        }
+        tokio::time::timeout(Duration::from_secs(5), async {
+            for worker in workers {
+                worker.await.expect("lease worker");
+            }
+        })
+        .await
+        .expect("all lease workers complete");
+        assert_eq!(gate.snapshot(), AdaptiveAdmissionSnapshot { limit: 8, active: 0 });
+        let full: Vec<_> = (0..8).map(|_| gate.try_acquire().expect("all slots returned")).collect();
+        assert!(gate.try_acquire().is_none());
+        drop(full);
+    }
+
+    #[tokio::test]
+    async fn queued_work_cannot_be_starved_by_new_fast_acquisitions() {
+        use std::task::{Context, Poll, Waker};
+        let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 1);
+        let held = gate.acquire().await;
+        let mut first = Box::pin(gate.acquire());
+        let mut second = Box::pin(gate.acquire());
+        let mut context = Context::from_waker(Waker::noop());
+        assert!(first.as_mut().poll(&mut context).is_pending());
+        assert!(second.as_mut().poll(&mut context).is_pending());
+        drop(held);
+        assert!(gate.try_acquire().is_none(), "new work must not steal a parked worker's turn");
+        assert!(second.as_mut().poll(&mut context).is_pending(), "later work cannot pass the queue head");
+        let Poll::Ready(first_permit) = first.as_mut().poll(&mut context) else {
+            panic!("queue head must run")
+        };
+        drop(first_permit);
+        let Poll::Ready(second_permit) = second.as_mut().poll(&mut context) else {
+            panic!("second waiter must run")
+        };
+        drop(second_permit);
+        assert!(gate.try_acquire().is_some(), "drained queue restores the atomic fast path");
+    }
+
+    #[tokio::test]
+    async fn cancelling_queued_work_preserves_fifo_capacity_and_fast_path() {
+        use std::task::{Context, Poll, Waker};
+        let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);
+        let mut head = Box::pin(gate.acquire());
+        let mut middle = Box::pin(gate.acquire());
+        let mut tail = Box::pin(gate.acquire());
+        let mut context = Context::from_waker(Waker::noop());
+        assert!(head.as_mut().poll(&mut context).is_pending());
+        assert!(middle.as_mut().poll(&mut context).is_pending());
+        assert!(tail.as_mut().poll(&mut context).is_pending());
+        drop(middle);
+        drop(head);
+        gate.set_limit(1);
+        assert!(
+            gate.try_acquire().is_none(),
+            "cancelled head must not expose the surviving waiter's capacity"
+        );
+        let Poll::Ready(permit) = tail.as_mut().poll(&mut context) else {
+            panic!("surviving waiter must run")
+        };
+        drop(permit);
+        assert!(gate.try_acquire().is_some());
+
+        gate.set_limit(0);
+        let mut cancelled = Box::pin(gate.acquire());
+        assert!(cancelled.as_mut().poll(&mut context).is_pending());
+        drop(cancelled);
+        gate.set_limit(1);
+        assert!(
+            gate.try_acquire().is_some(),
+            "cancelling the final waiter must not strand an admission turn"
+        );
+    }
+
+    #[tokio::test]
+    async fn cancelling_a_notified_head_hands_capacity_to_the_next_waiter() {
+        use std::task::{Context, Poll, Waker};
+        let gate = AdaptiveAdmission::new(NonZeroU16::new(2).expect("two units"), 0);
+        let mut head = Box::pin(gate.acquire());
+        let mut next = Box::pin(gate.acquire());
+        let mut context = Context::from_waker(Waker::noop());
+        assert!(head.as_mut().poll(&mut context).is_pending());
+        assert!(next.as_mut().poll(&mut context).is_pending());
+        gate.set_limit(1);
+        // Cancel after notification, before the head consumes its capacity.
+        drop(head);
+        let Poll::Ready(permit) = next.as_mut().poll(&mut context) else {
+            panic!("successor must observe capacity without another notification")
+        };
+        assert_eq!(gate.snapshot(), AdaptiveAdmissionSnapshot { limit: 1, active: 1 });
+        drop(permit);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn one_resize_serves_multiple_fifo_turns_but_stored_wakes_cannot_bypass_expiry() {
+        use std::task::{Context, Poll, Waker};
+        let gate = AdaptiveAdmission::new(NonZeroU16::new(2).expect("two units"), 0);
+        let mut first = Box::pin(gate.acquire());
+        let mut second = Box::pin(gate.acquire());
+        let mut context = Context::from_waker(Waker::noop());
+        assert!(first.as_mut().poll(&mut context).is_pending());
+        assert!(second.as_mut().poll(&mut context).is_pending());
+        gate.set_sampled_limit(2, Duration::from_secs(1));
+        let Poll::Ready(one) = first.as_mut().poll(&mut context) else {
+            panic!("first turn must observe resize")
+        };
+        let Poll::Ready(two) = second.as_mut().poll(&mut context) else {
+            panic!("second turn must observe resize without releasing the first unit")
+        };
+        tokio::time::advance(Duration::from_secs(1)).await;
+        drop(one);
+        drop(two);
+        let mut expired = Box::pin(gate.acquire());
+        assert!(expired.as_mut().poll(&mut context).is_pending(), "stored wake is not a fresh lease");
+        gate.set_sampled_limit(1, Duration::from_secs(1));
+        let Poll::Ready(renewed) = expired.as_mut().poll(&mut context) else {
+            panic!("fresh observation must wake expired admission")
+        };
+        drop(renewed);
+        assert_eq!(gate.snapshot().active, 0);
+    }
+    use std::future::Future;
+    use std::task::Poll;
+
+    #[tokio::test]
+    async fn shrink_preserves_all_held_units_and_blocks_until_below_new_limit() {
+        let owner = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 3);
+        let first = owner.acquire().await;
+        let second = owner.acquire().await;
+        let third = owner.acquire().await;
+        owner.set_limit(1);
+        assert_eq!(owner.snapshot(), AdaptiveAdmissionSnapshot { limit: 1, active: 3 });
+        drop(first);
+        assert!(owner.try_acquire().is_none());
+        drop(second);
+        assert!(owner.try_acquire().is_none());
+        drop(third);
+        let fourth = owner.try_acquire().expect("last old permit returned to original owner");
+        assert_eq!(owner.snapshot(), AdaptiveAdmissionSnapshot { limit: 1, active: 1 });
+        drop(fourth);
+        assert_eq!(owner.snapshot().active, 0);
+    }
+
+    #[tokio::test]
+    async fn zero_capacity_and_cancelled_waiter_do_not_leak_or_bypass() {
+        let owner = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);
+        let mut waiting = Box::pin(owner.acquire());
+        std::future::poll_fn(|cx| {
+            assert!(waiting.as_mut().poll(cx).is_pending());
+            Poll::Ready(())
+        })
+        .await;
+        drop(waiting);
+        assert_eq!(owner.snapshot(), AdaptiveAdmissionSnapshot { limit: 0, active: 0 });
+        assert!(owner.try_acquire().is_none());
+        owner.set_limit(1);
+        let permit = owner.acquire().await;
+        assert!(owner.try_acquire().is_none());
+        drop(permit);
+        assert_eq!(owner.snapshot().active, 0);
+    }
+
+    #[tokio::test]
+    async fn resize_wakes_a_registered_waiter_without_an_unrelated_release() {
+        let owner = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);
+        let mut waiting = Box::pin(owner.acquire());
+        std::future::poll_fn(|cx| {
+            assert!(waiting.as_mut().poll(cx).is_pending());
+            Poll::Ready(())
+        })
+        .await;
+        owner.set_limit(1);
+        let permit = tokio::time::timeout(std::time::Duration::from_secs(1), waiting)
+            .await
+            .expect("resize notification");
+        assert_eq!(owner.snapshot().active, 1);
+        drop(permit);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    async fn concurrent_resize_release_and_acquire_keep_the_owner_count_exact() {
+        let owner = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 8);
+        let mut tasks = Vec::new();
+        for _ in 0..16 {
+            let owner = owner.clone();
+            tasks.push(tokio::spawn(async move {
+                for _ in 0..100 {
+                    let permit = owner.acquire().await;
+                    tokio::task::yield_now().await;
+                    drop(permit);
+                }
+            }));
+        }
+        for limit in [1, 0, 4, 2, 8, 0, u32::MAX, 1] {
+            owner.set_limit(limit);
+            tokio::task::yield_now().await;
+        }
+        owner.set_limit(8);
+        tokio::time::timeout(std::time::Duration::from_secs(5), async {
+            for task in tasks {
+                task.await.expect("admission task");
+            }
+        })
+        .await
+        .expect("all waiters make progress");
+        assert_eq!(owner.snapshot(), AdaptiveAdmissionSnapshot { limit: 8, active: 0 });
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn expired_observation_blocks_new_work_without_revoking_held_permits() {
+        let owner = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);
+        owner.set_sampled_limit(2, Duration::from_secs(2));
+        let held = owner.acquire().await;
+        tokio::time::advance(Duration::from_secs(2)).await;
+        assert!(owner.try_acquire().is_none());
+        assert_eq!(owner.snapshot(), AdaptiveAdmissionSnapshot { limit: 0, active: 1 });
+        drop(held);
+        assert_eq!(owner.snapshot().active, 0);
+        owner.set_sampled_limit(1, Duration::from_secs(2));
+        let fresh = owner.try_acquire().expect("fresh observation reopens same owner");
+        owner.set_sampled_limit(8, Duration::ZERO);
+        assert!(owner.try_acquire().is_none());
+        drop(fresh);
+        assert_eq!(owner.snapshot().active, 0);
+    }
+
+    fn healthy(completed: u64) -> FeedbackObservation {
+        FeedbackObservation {
+            generation: 1,
+            max_concurrency: 8,
+            pressure: ResourcePressure::Stable,
+            completed_units: Some(completed),
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn controller_grows_one_unit_per_cooldown_and_cuts_pressure_immediately() {
+        let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);
+        let mut controller = FeedbackController::new();
+        assert_eq!(controller.update(&gate, Some(healthy(0))).limit, 1);
+        for completed in 1..=25 {
+            tokio::time::advance(Duration::from_secs(1)).await;
+            let limit = controller.update(&gate, Some(healthy(completed))).limit;
+            assert_eq!(
+                limit,
+                if completed < 9 {
+                    1
+                } else if completed < 17 {
+                    2
+                } else if completed < 25 {
+                    3
+                } else {
+                    4
+                }
+            );
+        }
+        let held = gate.try_acquire().expect("existing work");
+        let mut pressure = healthy(26);
+        pressure.pressure = ResourcePressure::Busy;
+        assert_eq!(
+            controller.update(&gate, Some(pressure)),
+            FeedbackDecision {
+                limit: 2,
+                reason: FeedbackReason::Pressure
+            }
+        );
+        pressure.max_concurrency = 0;
+        assert_eq!(controller.update(&gate, Some(pressure)).limit, 0);
+        assert_eq!(gate.snapshot().active, 1);
+        assert!(gate.try_acquire().is_none());
+        drop(held);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn controller_missing_low_sample_stalled_or_changed_sources_never_expand() {
+        for kind in 0..5 {
+            let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);
+            let mut controller = FeedbackController::new();
+            for tick in 0..32 {
+                let mut observation = healthy(tick);
+                match kind {
+                    0 => observation.pressure = ResourcePressure::Unknown,
+                    1 => observation.completed_units = None,
+                    2 => observation.completed_units = Some(0),
+                    3 => observation.generation = tick,
+                    _ => {}
+                }
+                tokio::time::advance(Duration::from_secs(if kind == 4 { 3 } else { 1 })).await;
+                assert!(controller.update(&gate, Some(observation)).limit <= 1, "kind {kind}");
+            }
+            assert_eq!(
+                controller.update(&gate, None),
+                FeedbackDecision {
+                    limit: 0,
+                    reason: FeedbackReason::Unavailable
+                }
+            );
+            assert!(gate.try_acquire().is_none());
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn controller_resumes_after_zero_cap_without_requiring_impossible_progress() {
+        let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);
+        let mut controller = FeedbackController::new();
+        let mut sample = healthy(0);
+        sample.max_concurrency = 0;
+        assert_eq!(controller.update(&gate, Some(sample)).limit, 0);
+        tokio::time::advance(Duration::from_secs(1)).await;
+        sample.max_concurrency = 8;
+        assert_eq!(controller.update(&gate, Some(sample)).limit, 1);
+        assert!(gate.try_acquire().is_some(), "minimum progress restarts inside the restored cap");
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn controller_applies_pressure_reduction_and_lower_cap_in_the_same_window() {
+        let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);
+        let mut controller = FeedbackController::new();
+        for completed in 0..=25 {
+            controller.update(&gate, Some(healthy(completed)));
+            tokio::time::advance(Duration::from_secs(1)).await;
+        }
+        assert_eq!(gate.snapshot().limit, 4);
+        let mut sample = healthy(26);
+        sample.max_concurrency = 3;
+        sample.pressure = ResourcePressure::Busy;
+        assert_eq!(controller.update(&gate, Some(sample)).limit, 2);
+    }
+}

crates/concurrency/src/foreground.rs

@@ -0,0 +1,568 @@
+// Copyright 2026 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+//! Bounded foreground completion windows for background resource feedback.
+//! Recording uses fixed atomic storage, without an allocation, mutex or task
+//! per request. A single reader rotates banks and waits for the retired bank's
+//! writers before reading it. Delayed writers delay publication, not the
+//! recorded window boundary. Percentiles are intervals, never exact values.
+
+use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
+use std::time::{Duration, Instant};
+
+const SUB_BUCKETS: usize = 64;
+const FINITE_BUCKETS: usize = 27 * SUB_BUCKETS;
+const BUCKETS: usize = FINITE_BUCKETS + 1;
+const MAX_MICROS: u64 = u32::MAX as u64;
+
+/// Bounded foreground categories; no bucket, object or request identifiers.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum ForegroundClass {
+    /// Object body reads.
+    Read,
+    /// Object writes, including multipart and copy operations.
+    Write,
+    /// Metadata and listing operations.
+    Metadata,
+    /// Other operations, including rejection before typed dispatch.
+    Other,
+}
+
+impl ForegroundClass {
+    /// Stable order shared by window summaries.
+    pub const ALL: [Self; 4] = [Self::Read, Self::Write, Self::Metadata, Self::Other];
+
+    /// Fixed metric label.
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            Self::Read => "read",
+            Self::Write => "write",
+            Self::Metadata => "metadata",
+            Self::Other => "other",
+        }
+    }
+
+    const fn index(self) -> usize {
+        match self {
+            Self::Read => 0,
+            Self::Write => 1,
+            Self::Metadata => 2,
+            Self::Other => 3,
+        }
+    }
+}
+
+/// Outcome at the observed response-body boundary, not a client ACK.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum CompletionOutcome {
+    /// Successful response body completed.
+    Success,
+    /// Error response, service error, or body-stream failure.
+    Failed,
+    /// The request/body was dropped before completion.
+    Cancelled,
+}
+
+/// Inclusive bounds containing the observed percentile.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct LatencyBounds {
+    /// Inclusive lower bound in nanoseconds.
+    pub lower_nanos: u64,
+    /// Inclusive upper bound; absent for the overflow bucket.
+    pub upper_nanos: Option<u64>,
+}
+
+/// One class in a complete observation window.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub struct ClassCompletions {
+    /// All finishes, including errors and cancellation.
+    pub finished: u64,
+    /// Error responses, service errors and body-stream failures.
+    pub failed: u64,
+    /// Censored observations; these must not be treated as successful latency.
+    pub cancelled: u64,
+    /// P99 of non-cancelled finishes, when at least 100 samples exist.
+    /// This count permits computing the rank; it is not statistical confidence.
+    pub p99: Option<LatencyBounds>,
+}
+
+/// Clock readings bracketing an atomic event-window cut. A scheduler pause
+/// between these readings is visible, rather than disguised as precise timing.
+#[derive(Clone, Copy, Debug)]
+pub struct WindowBoundary {
+    /// Read immediately before the event cut.
+    pub earliest: Instant,
+    /// Read immediately after the event cut.
+    pub latest: Instant,
+}
+
+/// Complete windows retain their actual cut brackets, even if publication waits.
+#[derive(Clone, Copy, Debug)]
+pub struct ForegroundWindow {
+    /// Start of this event window.
+    pub start: WindowBoundary,
+    /// Cut of the retired bank, possibly older than publication time.
+    pub end: WindowBoundary,
+    /// Request starts in this window, including pre-dispatch rejection.
+    pub started: Option<u64>,
+    /// Requests still alive at the cut; absent after accounting overflow.
+    pub in_flight: Option<u64>,
+    /// Per-class completions in ForegroundClass::ALL order.
+    pub classes: [Option<ClassCompletions>; 4],
+    /// False after a counter overflow or an inconsistent event count.
+    pub valid: bool,
+}
+
+struct ClassBank {
+    latencies: [AtomicU64; BUCKETS],
+    failed: AtomicU64,
+    cancelled: AtomicU64,
+}
+
+impl ClassBank {
+    fn new() -> Self {
+        Self {
+            latencies: std::array::from_fn(|_| AtomicU64::new(0)),
+            failed: AtomicU64::new(0),
+            cancelled: AtomicU64::new(0),
+        }
+    }
+}
+
+struct Bank {
+    writers: AtomicUsize,
+    started: AtomicU64,
+    overflowed: AtomicBool,
+    classes: [ClassBank; 4],
+}
+
+impl Bank {
+    fn new() -> Self {
+        Self {
+            writers: AtomicUsize::new(0),
+            started: AtomicU64::new(0),
+            overflowed: AtomicBool::new(false),
+            classes: std::array::from_fn(|_| ClassBank::new()),
+        }
+    }
+
+    fn increment(&self, counter: &AtomicU64) {
+        if counter.fetch_add(1, Ordering::Relaxed) == u64::MAX {
+            self.overflowed.store(true, Ordering::Relaxed);
+        }
+    }
+}
+
+struct Writer<'a>(&'a Bank);
+
+impl Drop for Writer<'_> {
+    fn drop(&mut self) {
+        self.0.writers.fetch_sub(1, Ordering::SeqCst);
+    }
+}
+
+/// One process-wide recorder. Its fixed storage is allocated with the owner.
+/// A request holds only a reference, timestamps and bounded enum state.
+pub struct ForegroundRecorder {
+    created_at: Instant,
+    epoch: AtomicU64,
+    reader_taken: AtomicBool,
+    banks: [Bank; 2],
+}
+
+impl Default for ForegroundRecorder {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl ForegroundRecorder {
+    /// Construct empty accounting for one lifetime of the owning process.
+    pub fn new() -> Self {
+        Self {
+            created_at: Instant::now(),
+            epoch: AtomicU64::new(0),
+            reader_taken: AtomicBool::new(false),
+            banks: std::array::from_fn(|_| Bank::new()),
+        }
+    }
+
+    fn writer(&self) -> Writer<'_> {
+        loop {
+            let epoch = self.epoch.load(Ordering::SeqCst);
+            let bank = &self.banks[usize::from(epoch & 1 != 0)];
+            bank.writers.fetch_add(1, Ordering::SeqCst);
+            let writer = Writer(bank);
+            // Sequential ordering joins registration, the reader's cut, and
+            // its zero-writer check. A late old-epoch registrant cannot write
+            // into a bank that the reader has begun draining or reusing.
+            if self.epoch.load(Ordering::SeqCst) == epoch {
+                return writer;
+            }
+        }
+    }
+
+    /// Begin at the external service-call boundary. Drop records cancellation
+    /// unless finish has already recorded a response-body outcome.
+    pub fn start(&self) -> ForegroundObservation<'_> {
+        let started_at = Instant::now();
+        let writer = self.writer();
+        writer.0.increment(&writer.0.started);
+        ForegroundObservation {
+            recorder: self,
+            started_at: Some(started_at),
+            class: ForegroundClass::Other,
+        }
+    }
+
+    fn finish(&self, class: ForegroundClass, elapsed: Duration, outcome: CompletionOutcome) {
+        let writer = self.writer();
+        let bank = writer.0;
+        let class = &bank.classes[class.index()];
+        if outcome == CompletionOutcome::Cancelled {
+            bank.increment(&class.cancelled);
+            return;
+        }
+        let bucket = latency_bucket(elapsed);
+        bank.increment(&class.latencies[bucket]);
+        if outcome == CompletionOutcome::Failed {
+            bank.increment(&class.failed);
+        }
+    }
+
+    /// Claim the sole reader. It cannot be replaced after being dropped:
+    /// resetting its carried in-flight count would falsify later windows.
+    pub fn take_reader(&self) -> Option<ForegroundReader<'_>> {
+        self.reader_taken
+            .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
+            .ok()?;
+        Some(ForegroundReader {
+            recorder: self,
+            start: WindowBoundary {
+                earliest: self.created_at,
+                latest: self.created_at,
+            },
+            retired: None,
+            in_flight: Some(0),
+        })
+    }
+}
+
+/// Non-cloneable request ownership, moved from the service future to its body.
+pub struct ForegroundObservation<'a> {
+    recorder: &'a ForegroundRecorder,
+    started_at: Option<Instant>,
+    class: ForegroundClass,
+}
+
+impl ForegroundObservation<'_> {
+    /// Set the first dispatched operation's bounded category.
+    pub fn set_class(&mut self, class: ForegroundClass) {
+        self.class = class;
+    }
+
+    /// Finish once. Later calls or Drop do not produce another observation.
+    pub fn finish(&mut self, outcome: CompletionOutcome) {
+        if let Some(started_at) = self.started_at.take() {
+            self.recorder.finish(self.class, started_at.elapsed(), outcome);
+        }
+    }
+}
+
+impl Drop for ForegroundObservation<'_> {
+    fn drop(&mut self) {
+        self.finish(CompletionOutcome::Cancelled);
+    }
+}
+
+/// Single-reader state; never acquired by a request path.
+pub struct ForegroundReader<'a> {
+    recorder: &'a ForegroundRecorder,
+    start: WindowBoundary,
+    retired: Option<(usize, WindowBoundary)>,
+    in_flight: Option<u64>,
+}
+
+impl ForegroundReader<'_> {
+    /// Cut a new window, or finish publishing an earlier cut. A live retired
+    /// writer returns None without blocking, discarding data or moving its cut.
+    pub fn sample(&mut self) -> Option<ForegroundWindow> {
+        if self.retired.is_none() {
+            let epoch = self.recorder.epoch.load(Ordering::SeqCst);
+            let next = epoch.checked_add(1)?;
+            let earliest = Instant::now();
+            self.recorder.epoch.store(next, Ordering::SeqCst);
+            let cut = WindowBoundary {
+                earliest,
+                latest: Instant::now(),
+            };
+            self.retired = Some((usize::from(epoch & 1 != 0), cut));
+        }
+        let (index, end) = self.retired?;
+        let bank = &self.recorder.banks[index];
+        if bank.writers.load(Ordering::SeqCst) != 0 {
+            return None;
+        }
+
+        let started = bank.started.swap(0, Ordering::Relaxed);
+        let overflowed = bank.overflowed.swap(false, Ordering::Relaxed);
+        let classes = std::array::from_fn(|index| {
+            let class = &bank.classes[index];
+            let failed = class.failed.swap(0, Ordering::Relaxed);
+            let cancelled = class.cancelled.swap(0, Ordering::Relaxed);
+            let buckets = std::array::from_fn::<_, BUCKETS, _>(|i| class.latencies[i].swap(0, Ordering::Relaxed));
+            let non_cancelled = buckets.iter().try_fold(0_u64, |sum, count| sum.checked_add(*count));
+            let finished = non_cancelled.and_then(|total| total.checked_add(cancelled));
+            let finished = finished.filter(|_| !overflowed && non_cancelled.is_some_and(|total| failed <= total))?;
+            Some(ClassCompletions {
+                finished,
+                failed,
+                cancelled,
+                p99: non_cancelled
+                    .filter(|total| *total >= 100)
+                    .and_then(|total| percentile_99(&buckets, total)),
+            })
+        });
+        let started = (!overflowed).then_some(started);
+        let finished = classes
+            .iter()
+            .try_fold(0_u64, |sum, class| sum.checked_add(class.as_ref()?.finished));
+        self.in_flight = self
+            .in_flight
+            .zip(finished)
+            .and_then(|(active, finished)| active.checked_add(started?)?.checked_sub(finished));
+        let window = ForegroundWindow {
+            start: self.start,
+            end,
+            started,
+            in_flight: self.in_flight,
+            classes,
+            valid: self.in_flight.is_some(),
+        };
+        self.start = end;
+        self.retired = None;
+        Some(window)
+    }
+}
+
+fn latency_bucket(elapsed: Duration) -> usize {
+    let micros = elapsed.as_nanos().div_ceil(1_000);
+    let Ok(micros) = u64::try_from(micros) else {
+        return FINITE_BUCKETS;
+    };
+    if micros > MAX_MICROS {
+        return FINITE_BUCKETS;
+    }
+    if micros < SUB_BUCKETS as u64 {
+        return usize::try_from(micros).unwrap_or(FINITE_BUCKETS);
+    }
+    let shift = 63 - micros.leading_zeros() - 6;
+    usize::try_from((u64::from(shift) + 1) * SUB_BUCKETS as u64 + (micros >> shift) - SUB_BUCKETS as u64)
+        .unwrap_or(FINITE_BUCKETS)
+}
+
+fn bucket_bounds(index: usize) -> LatencyBounds {
+    if index == FINITE_BUCKETS {
+        return LatencyBounds {
+            lower_nanos: MAX_MICROS * 1_000 + 1,
+            upper_nanos: None,
+        };
+    }
+    let (lower, upper) = if index < SUB_BUCKETS {
+        (index as u64, index as u64)
+    } else {
+        let shift = index / SUB_BUCKETS - 1;
+        let lower = (SUB_BUCKETS + index % SUB_BUCKETS) as u64 * (1_u64 << shift);
+        (lower, lower + (1_u64 << shift) - 1)
+    };
+    LatencyBounds {
+        lower_nanos: lower.saturating_sub(1) * 1_000 + u64::from(lower != 0),
+        upper_nanos: Some(upper * 1_000),
+    }
+}
+
+fn percentile_99(buckets: &[u64; BUCKETS], total: u64) -> Option<LatencyBounds> {
+    let rank = (u128::from(total) * 99).div_ceil(100);
+    let mut cumulative = 0_u128;
+    for (index, count) in buckets.iter().enumerate() {
+        cumulative += u128::from(*count);
+        if cumulative >= rank {
+            return Some(bucket_bounds(index));
+        }
+    }
+    None
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::sync::Arc;
+
+    #[test]
+    fn percentile_intervals_contain_each_bucket_edge_and_neighbour() {
+        for index in 0..BUCKETS {
+            let bounds = bucket_bounds(index);
+            assert_eq!(latency_bucket(Duration::from_nanos(bounds.lower_nanos)), index);
+            if let Some(upper) = bounds.upper_nanos {
+                assert!(upper >= bounds.lower_nanos);
+                assert_eq!(latency_bucket(Duration::from_nanos(upper)), index);
+                assert_eq!(latency_bucket(Duration::from_nanos(upper + 1)), index + 1);
+            }
+        }
+        assert_eq!(latency_bucket(Duration::MAX), FINITE_BUCKETS);
+    }
+
+    #[test]
+    fn body_ownership_carries_in_flight_counts_across_windows() {
+        let recorder = ForegroundRecorder::new();
+        let mut reader = recorder.take_reader().expect("first reader");
+        let mut request = recorder.start();
+        request.set_class(ForegroundClass::Read);
+        let first = reader.sample().expect("cut with a live request");
+        assert!(first.valid);
+        assert_eq!(first.started, Some(1));
+        assert_eq!(first.in_flight, Some(1));
+        assert_eq!(first.classes[0].expect("valid read count").finished, 0);
+
+        request.finish(CompletionOutcome::Success);
+        request.finish(CompletionOutcome::Failed);
+        drop(request);
+        let second = reader.sample().expect("body finished");
+        assert!(second.valid);
+        assert_eq!(second.started, Some(0));
+        assert_eq!(second.in_flight, Some(0));
+        let read = second.classes[0].expect("valid read summary");
+        assert_eq!((read.finished, read.failed, read.cancelled, read.p99), (1, 0, 0, None));
+        assert_eq!(first.end.earliest, second.start.earliest);
+        assert_eq!(first.end.latest, second.start.latest);
+    }
+
+    #[test]
+    fn cancellations_are_counted_without_fabricating_success_latencies() {
+        let recorder = ForegroundRecorder::new();
+        let mut reader = recorder.take_reader().expect("first reader");
+        for _ in 0..150 {
+            let mut request = recorder.start();
+            request.set_class(ForegroundClass::Write);
+        }
+        let window = reader.sample().expect("cancelled requests");
+        let write = window.classes[1].expect("known write count");
+        assert!(window.valid);
+        assert_eq!(window.in_flight, Some(0));
+        assert_eq!((write.finished, write.cancelled, write.failed, write.p99), (150, 150, 0, None));
+    }
+
+    #[test]
+    fn delayed_retired_writer_preserves_the_original_cut_and_next_bank() {
+        let recorder = ForegroundRecorder::new();
+        let mut reader = recorder.take_reader().expect("first reader");
+        let writer = recorder.writer();
+        assert!(reader.sample().is_none(), "never read a live writer's bank");
+        let original_cut = reader.retired.expect("pending cut").1;
+        let request = recorder.start(); // a start after the cut belongs to the new bank
+        writer.0.increment(&writer.0.started);
+        drop(writer);
+        let previous = reader.sample().expect("retired writer drained");
+        assert_eq!(previous.end.earliest, original_cut.earliest);
+        assert_eq!(previous.end.latest, original_cut.latest);
+        assert_eq!(previous.started, Some(1));
+        assert_eq!(previous.in_flight, Some(1));
+        drop(request);
+        let current = reader.sample().expect("next bank");
+        assert_eq!(current.started, Some(1));
+        assert_eq!(current.in_flight, Some(1));
+        assert_eq!(current.classes[3].expect("unclassified cancellation").cancelled, 1);
+    }
+
+    #[test]
+    fn completed_percentile_uses_rank_and_never_turns_maximum_into_p99() {
+        let recorder = ForegroundRecorder::new();
+        let mut reader = recorder.take_reader().expect("first reader");
+        for i in 0..100 {
+            let writer = recorder.writer();
+            writer.0.increment(&writer.0.started);
+            drop(writer);
+            recorder.finish(
+                ForegroundClass::Read,
+                if i == 99 {
+                    Duration::from_secs(60)
+                } else {
+                    Duration::from_millis(2)
+                },
+                if i == 99 {
+                    CompletionOutcome::Failed
+                } else {
+                    CompletionOutcome::Success
+                },
+            );
+        }
+        let window = reader.sample().expect("complete window");
+        let read = window.classes[0].expect("read summary");
+        assert_eq!((read.finished, read.failed, read.cancelled), (100, 1, 0));
+        let p99 = read.p99.expect("100 non-cancelled samples");
+        assert!(p99.lower_nanos <= 2_000_000);
+        assert!(p99.upper_nanos.is_some_and(|upper| (2_000_000..3_000_000).contains(&upper)));
+    }
+
+    #[test]
+    fn overflow_stays_unknown_and_reader_cannot_reset_event_accounting() {
+        let recorder = ForegroundRecorder::new();
+        {
+            let mut reader = recorder.take_reader().expect("first reader");
+            recorder.banks[0].started.store(u64::MAX, Ordering::Relaxed);
+            drop(recorder.start());
+            let invalid = reader.sample().expect("overflow window");
+            assert!(!invalid.valid);
+            assert_eq!(invalid.started, None);
+            assert_eq!(invalid.in_flight, None);
+            assert_eq!(invalid.classes, [None; 4]);
+            let later = reader.sample().expect("later empty window");
+            assert!(!later.valid, "a lost carried count cannot recover by inventing zero active work");
+        }
+        assert!(recorder.take_reader().is_none(), "reader replacement would lose live request ownership");
+    }
+
+    #[test]
+    fn concurrent_rotation_preserves_every_start_finish_and_cancellation() {
+        let recorder = Arc::new(ForegroundRecorder::new());
+        let mut reader = recorder.take_reader().expect("first reader");
+        let workers: Vec<_> = (0..4)
+            .map(|_| {
+                let recorder = Arc::clone(&recorder);
+                std::thread::spawn(move || {
+                    for index in 0..10_000 {
+                        let mut request = recorder.start();
+                        request.set_class(ForegroundClass::Read);
+                        if index % 10 != 0 {
+                            request.finish(CompletionOutcome::Success);
+                        }
+                    }
+                })
+            })
+            .collect();
+        let (mut started, mut finished, mut cancelled) = (0, 0, 0);
+        let mut add_window = |window: ForegroundWindow| {
+            assert!(window.valid, "concurrent event cuts must stay balanced");
+            started += window.started.expect("valid start count");
+            let reads = window.classes[0].expect("valid read count");
+            finished += reads.finished;
+            cancelled += reads.cancelled;
+        };
+        while workers.iter().any(|worker| !worker.is_finished()) {
+            if let Some(window) = reader.sample() {
+                add_window(window);
+            }
+            std::thread::yield_now();
+        }
+        for worker in workers {
+            worker.join().expect("writer must finish");
+        }
+        add_window(reader.sample().expect("drain after writers join"));
+        // A previously retired window may have been published by the last
+        // sample; cut once more to include the current, still open bank.
+        let final_window = reader.sample().expect("final open bank");
+        assert_eq!(final_window.in_flight, Some(0));
+        add_window(final_window);
+        assert_eq!((started, finished, cancelled), (40_000, 40_000, 4_000));
+    }
+}

crates/concurrency/src/lib.rs

@@ -36,8 +36,10 @@
 // Re-exported io-core type used by downstream timeout implementations.
 pub use rustfs_io_core::OperationProgress;
 
+pub mod adaptive;
 mod backpressure;
 mod deadlock;
+pub mod foreground;
 mod queue;
 pub mod workers;
 pub mod workload;

crates/concurrency/src/workers.rs

@@ -49,12 +49,28 @@ impl Workers {
 
     /// Acquire a worker slot, waiting until one becomes available.
     pub async fn take(&self) {
+        struct AcquireCancelGuard<'a> {
+            workers: &'a Workers,
+            completed: bool,
+        }
+        impl Drop for AcquireCancelGuard<'_> {
+            fn drop(&mut self) {
+                if !self.completed {
+                    self.workers.in_use.send_modify(|_| {});
+                }
+            }
+        }
+        let mut guard = AcquireCancelGuard {
+            workers: self,
+            completed: false,
+        };
         let permit = match self.semaphore.acquire().await {
             Ok(permit) => permit,
             // The semaphore is owned by `self` and never closed.
             Err(_) => unreachable!("worker semaphore is never closed"),
         };
         permit.forget(); // returned manually via give()
+        guard.completed = true;
         self.in_use.send_modify(|held| *held += 1);
         trace!(
             event = "worker_slot.acquire",
@@ -82,6 +98,7 @@ impl Workers {
         });
         if released {
             self.semaphore.add_permits(1);
+            self.in_use.send_modify(|_| {});
         }
         trace!(
             event = "worker_slot.release",
@@ -99,7 +116,9 @@ impl Workers {
     pub async fn wait(&self) {
         let mut rx = self.in_use.subscribe();
         // The sender is owned by `self`, so it outlives this borrow.
-        let _ = rx.wait_for(|&held| held == 0).await;
+        let _ = rx
+            .wait_for(|&held| held == 0 && self.semaphore.available_permits() == self.limit)
+            .await;
         debug!(
             event = "worker_slot.wait",
             component = "concurrency",
@@ -114,7 +133,9 @@ impl Workers {
 
     /// Return the current number of available worker slots.
     pub async fn available(&self) -> usize {
-        self.limit.saturating_sub(*self.in_use.borrow())
+        self.limit
+            .saturating_sub(*self.in_use.borrow())
+            .min(self.semaphore.available_permits())
     }
 }
 
@@ -217,4 +238,36 @@ mod tests {
         }
         assert_eq!(workers.available().await, 0);
     }
+
+    #[tokio::test]
+    async fn test_wait_does_not_complete_while_permit_is_in_transit_to_queued_taker() {
+        use std::future::Future;
+        use std::pin::pin;
+        use std::task::{Context, Poll, Waker};
+
+        let workers = Workers::new(1).unwrap();
+        workers.take().await;
+
+        let mut taker = pin!(workers.take());
+        let mut cx = Context::from_waker(Waker::noop());
+        assert!(matches!(taker.as_mut().poll(&mut cx), Poll::Pending));
+
+        let mut waiter = pin!(workers.wait());
+        assert!(matches!(waiter.as_mut().poll(&mut cx), Poll::Pending));
+
+        // Hand off the slot to the queued taker before `taker` is polled again.
+        workers.give().await;
+        assert_eq!(workers.available().await, 0);
+        assert!(
+            matches!(waiter.as_mut().poll(&mut cx), Poll::Pending),
+            "wait() must not complete while a permit is in transit to a woken taker"
+        );
+
+        assert!(matches!(taker.as_mut().poll(&mut cx), Poll::Ready(())));
+        assert!(matches!(waiter.as_mut().poll(&mut cx), Poll::Pending));
+
+        workers.give().await;
+        assert!(matches!(waiter.as_mut().poll(&mut cx), Poll::Ready(())));
+        assert_eq!(workers.available().await, 1);
+    }
 }

crates/config/README.md

@@ -116,6 +116,7 @@ when restoring `/rustfs/console` for a custom OEM build.
 - `RUSTFS_DATA_SCANNER_START_DELAY_SECS` (deprecated alias for compatibility)
 - `RUSTFS_SCANNER_IDLE_MODE` (canonical)
 - `RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS` (canonical)
+- `RUSTFS_SCANNER_RECOVERY_BACKOFF_CAP_SECS` (canonical)
 - `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` (canonical)
 - `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` (canonical)
 - `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` (canonical)
@@ -127,6 +128,12 @@ Scanner cycle budget controls:
 
 - When `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` is unset, the finite default is 1800 seconds (30 minutes), matching the scanner benchmark guidance.
 - An explicit `0` preserves the compatibility behavior of an unbounded runtime budget. Object and directory budgets likewise remain unbounded when explicitly set to `0`.
+
+Scanner recovery retry controls:
+
+- `RUSTFS_SCANNER_RECOVERY_BACKOFF_CAP_SECS` caps the wait between automatic startup recovery retries. The persistent admin key is `scanner.recovery_backoff_cap`; the environment takes precedence over the persistent value, and the unset default remains 600 seconds (10 minutes). Values must be positive integer seconds within the platform timer range; `0`, malformed values, and overflow are rejected. A valid environment override does not hide an invalid persistent value from `admin config set` validation.
+- The cap bounds the existing exponential retry schedule. Updating the persistent value applies to the next computed retry; it does not reschedule a wait already in progress. Retryable observations continue retrying after the short retry budget. Admin scanner status reports the effective value and source at `runtime_config.recovery_backoff_cap_seconds`.
+
 Scanner walker controls:
 
 - `RUSTFS_SCANNER_WALKERS_PER_DISK` sets how many bucket walkers each local disk of an erasure set runs; the default `1` walks one bucket per disk at a time, which is the right value for rotational media. On flash, `2` to `4` lets one disk walk several buckets concurrently, which matters most for single-disk deployments with many buckets, where the walk is otherwise fully serial. Values outside `1` through `16` are rejected by `admin config set`; an out-of-range or non-numeric environment value is logged as a scanner runtime config error (`state = "startup_apply_failed"`) and the scanner keeps the default. The walkers of one disk share that disk's scanner admission, so peers still see one scan session per physical disk, and `RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS` still caps the total.

crates/config/src/constants/internode.rs

@@ -276,6 +276,9 @@ pub const DEFAULT_INTERNODE_BULK_CHANNELS: usize = 2;
 pub const ENV_INTERNODE_HTTP_TUNING_PROFILE: &str = "RUSTFS_INTERNODE_HTTP_TUNING_PROFILE";
 pub const DEFAULT_INTERNODE_HTTP_TUNING_PROFILE: &str = "legacy";
 
+/// Internode HTTP data-plane protocol: `http1` (default) or `auto` (TLS ALPN).
+pub const ENV_INTERNODE_HTTP_PROTOCOL: &str = "RUSTFS_INTERNODE_HTTP_PROTOCOL";
+
 /// Internode HTTP connection pool maximum idle connections per host.
 pub const ENV_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST: &str = "RUSTFS_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST";
 
@@ -287,8 +290,9 @@ pub const ENV_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_HT
 /// NOTE (backlog#805-C3): all `RUSTFS_INTERNODE_HTTP2_*` tuning (window sizes,
 /// adaptive window, keepalive timeout) only takes effect when the internode
 /// connection actually negotiates HTTP/2, which happens via TLS-ALPN. Over
-/// plaintext internode transport the connection is HTTP/1.1 and these settings
-/// are silently inert — enable internode TLS to use them.
+/// plaintext internode transport or with the default `http1` protocol policy,
+/// these settings are inert. Select `RUSTFS_INTERNODE_HTTP_PROTOCOL=auto` and
+/// enable internode TLS to use them.
 pub const ENV_INTERNODE_HTTP2_INITIAL_STREAM_WINDOW_SIZE: &str = "RUSTFS_INTERNODE_HTTP2_INITIAL_STREAM_WINDOW_SIZE";
 
 /// Internode HTTP/2 initial connection window size in bytes.

crates/config/src/constants/scanner.rs

@@ -53,6 +53,9 @@ pub const SCANNER_IDLE_MODE: &str = "idle_mode";
 /// Scanner config key controlling scanner cache save timeout in seconds.
 pub const SCANNER_CACHE_SAVE_TIMEOUT: &str = "cache_save_timeout";
 
+/// Scanner config key capping automatic recovery retry waits in seconds.
+pub const SCANNER_RECOVERY_BACKOFF_CAP: &str = "recovery_backoff_cap";
+
 /// Scanner config key capping concurrent scanner set tasks.
 pub const SCANNER_MAX_CONCURRENT_SET_SCANS: &str = "max_concurrent_set_scans";
 
@@ -93,6 +96,7 @@ pub const SCANNER_KEYS: &[&str] = &[
     SCANNER_BITROT_CYCLE,
     SCANNER_IDLE_MODE,
     SCANNER_CACHE_SAVE_TIMEOUT,
+    SCANNER_RECOVERY_BACKOFF_CAP,
     SCANNER_MAX_CONCURRENT_SET_SCANS,
     SCANNER_MAX_CONCURRENT_DISK_SCANS,
     SCANNER_WALKERS_PER_DISK,
@@ -216,6 +220,14 @@ pub const ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS: &str = "RUSTFS_SCANNER_CACHE_SAVE
 /// Default scanner cache save timeout in seconds.
 pub const DEFAULT_SCANNER_CACHE_SAVE_TIMEOUT_SECS: u64 = 14;
 
+/// Environment variable that caps automatic scanner recovery retry waits.
+/// The value must be a positive integer number of seconds within the timer range.
+/// - Example: `export RUSTFS_SCANNER_RECOVERY_BACKOFF_CAP_SECS=600`
+pub const ENV_SCANNER_RECOVERY_BACKOFF_CAP_SECS: &str = "RUSTFS_SCANNER_RECOVERY_BACKOFF_CAP_SECS";
+
+/// Default maximum wait between automatic scanner recovery retries.
+pub const DEFAULT_SCANNER_RECOVERY_BACKOFF_CAP_SECS: u64 = 10 * 60;
+
 /// Environment variable that caps concurrent scanner set tasks.
 /// A value of `0` keeps the existing topology-based concurrency.
 /// - Example: `export RUSTFS_SCANNER_MAX_CONCURRENT_SET_SCANS=2`

crates/e2e_test/Cargo.toml

@@ -107,7 +107,7 @@ flate2.workspace = true
 http.workspace = true
 http-body-util.workspace = true
 hyper = { workspace = true, features = ["http2", "http1", "server"] }
-hyper-util = { workspace = true, features = ["tokio", "server-auto", "server-graceful", "tracing"] }
+hyper-util = { workspace = true, features = ["tokio", "server-auto", "server-graceful", "client-legacy", "tracing"] }
 reqwest = { workspace = true, features = ["json", "multipart", "stream"] }
 rustfs-signer.workspace = true
 
@@ -147,5 +147,6 @@ clap = { workspace = true, features = ["derive", "env"] }
 
 [dev-dependencies]
 aws-smithy-http-client = { workspace = true, default-features = false, features = ["test-util"] }
+rustfs-heal.workspace = true
 tempfile.workspace = true
 tokio = { workspace = true, features = ["test-util"] }

crates/e2e_test/README.md

@@ -200,8 +200,8 @@ python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run
 python3 scripts/e2e_binary.py run --binary "$RUSTFS_E2E_STARTUP_CAS_BINARY" --features e2e-test-hooks -- cargo nextest run --profile e2e-full -p e2e_test
 python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-nightly -p e2e_test
 
-# Distributed 4-node 4-disk lane uses the fault-test hooks server; the
-# scanner_heal_topology_ fault cases fail closed without it.
+# Distributed 4-node 4-disk lane includes physical-worker fault injection and
+# scanner_heal_topology_ cases; both require the fault-test hooks server.
 # Upgrade cases require RUSTFS_UPGRADE_SOURCE_BINARY and fail closed without it.
 python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
 python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-distributed -p e2e_test

crates/e2e_test/src/common.rs

@@ -34,7 +34,7 @@ use rustfs_signer::sign_v4;
 use s3s::Body;
 use serde_json;
 use std::fs as stdfs;
-use std::io::ErrorKind;
+use std::io::{self, ErrorKind};
 use std::net::{Ipv6Addr, SocketAddr};
 use std::path::{Path, PathBuf};
 use std::process::{Child, Command, Stdio};
@@ -1272,6 +1272,7 @@ pub struct RustFSTestClusterEnvironment {
     /// Optional socket proxies used for the corresponding node's volume
     /// endpoints. Proxies must be installed before [`Self::start`].
     volume_proxy_addresses: Vec<Option<SocketAddr>>,
+    tls_ca_pem: Option<Vec<u8>>,
 }
 
 impl RustFSTestClusterEnvironment {
@@ -1312,7 +1313,10 @@ impl RustFSTestClusterEnvironment {
     pub async fn with_topology(topology: ClusterTopology) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
         topology.validate()?;
 
-        let temp_dir = format!("/tmp/rustfs_cluster_test_{}", Uuid::new_v4());
+        let temp_dir = std::env::temp_dir()
+            .join(format!("rustfs_cluster_test_{}", Uuid::new_v4()))
+            .to_string_lossy()
+            .into_owned();
         fs::create_dir_all(&temp_dir).await?;
 
         // Map every node to its owning pool index.
@@ -1374,9 +1378,35 @@ impl RustFSTestClusterEnvironment {
             node_capture_log_paths: vec![None; topology.node_count],
             topology,
             volume_proxy_addresses: vec![None; node_count],
+            tls_ca_pem: None,
         })
     }
 
+    /// Configure verified HTTPS for both volume endpoints and S3 clients.
+    /// The bundle must be generated before any node starts.
+    pub async fn enable_tls(&mut self, directory: &Path) -> std::io::Result<()> {
+        if self.nodes.iter().any(|node| node.process.is_some()) {
+            return Err(std::io::Error::new(
+                ErrorKind::InvalidInput,
+                "configure cluster TLS before starting nodes",
+            ));
+        }
+        let directory = fs::canonicalize(directory).await?;
+        let ca_pem = fs::read(directory.join("ca.crt")).await?;
+        reqwest::Certificate::from_pem(&ca_pem).map_err(|error| std::io::Error::new(ErrorKind::InvalidData, error))?;
+        self.set_env(
+            "RUSTFS_TLS_PATH",
+            directory
+                .to_str()
+                .ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "TLS directory is not UTF-8"))?,
+        );
+        for node in &mut self.nodes {
+            node.url = format!("https://{}", node.address);
+        }
+        self.tls_ca_pem = Some(ca_pem);
+        Ok(())
+    }
+
     /// Add an extra environment variable applied to every cluster node process.
     pub fn set_env<K, V>(&mut self, key: K, value: V)
     where
@@ -1467,6 +1497,29 @@ impl RustFSTestClusterEnvironment {
         Ok(proxy)
     }
 
+    /// Route internode traffic through a method-aware HTTP proxy.
+    pub(crate) async fn start_rpc_proxy_for_node(
+        &mut self,
+        node_idx: usize,
+    ) -> io::Result<crate::rpc_fault_proxy::RpcFaultProxy> {
+        self.ensure_node_index(node_idx)
+            .map_err(|error| io::Error::new(ErrorKind::InvalidInput, error))?;
+        if self.volume_proxy_addresses[node_idx].is_some() {
+            return Err(io::Error::new(
+                ErrorKind::AlreadyExists,
+                format!("a volume proxy is already configured for node {node_idx}"),
+            ));
+        }
+        let target = self.nodes[node_idx]
+            .address
+            .parse::<SocketAddr>()
+            .map_err(|error| io::Error::new(ErrorKind::InvalidInput, error))?;
+        let listen = SocketAddr::from((Ipv6Addr::LOCALHOST, target.port()));
+        let proxy = crate::rpc_fault_proxy::RpcFaultProxy::start_on(listen, target).await?;
+        self.volume_proxy_addresses[node_idx] = Some(proxy.local_addr());
+        Ok(proxy)
+    }
+
     fn volume_address(&self, node_idx: usize) -> String {
         self.volume_proxy_addresses[node_idx]
             .map(|address| address.to_string())
@@ -1475,6 +1528,7 @@ impl RustFSTestClusterEnvironment {
 
     fn build_volumes_arg(&self) -> String {
         let pools = self.topology.normalized_pools();
+        let scheme = if self.tls_ca_pem.is_some() { "https" } else { "http" };
 
         if pools.len() <= 1 {
             // Single pool: explicit enumeration of every drive on every node.
@@ -1484,7 +1538,7 @@ impl RustFSTestClusterEnvironment {
                 .enumerate()
                 .flat_map(|(node_idx, n)| {
                     let address = self.volume_address(node_idx);
-                    n.data_dirs.iter().map(move |dir| format!("http://{}{}", address, dir))
+                    n.data_dirs.iter().map(move |dir| format!("{scheme}://{address}{dir}"))
                 })
                 .collect::<Vec<_>>()
                 .join(" ");
@@ -1504,7 +1558,7 @@ impl RustFSTestClusterEnvironment {
                     .and_then(|d| d.rsplit_once('/').map(|(parent, _)| parent))
                     .unwrap_or(&node.data_dir);
                 format!(
-                    "http://{}{}/drive{{0...{}}}",
+                    "{scheme}://{}{}/drive{{0...{}}}",
                     self.volume_address(node_idx),
                     base,
                     self.topology.drives_per_node - 1
@@ -1693,13 +1747,20 @@ impl RustFSTestClusterEnvironment {
         if node_idx >= self.nodes.len() {
             return Err("node_idx is invalid".into());
         }
-        Ok(Client::from_conf(build_test_s3_config(
-            &self.nodes[node_idx].url,
-            &self.access_key,
-            &self.secret_key,
-            None,
-            "cluster-test",
-        )))
+        let mut config =
+            build_test_s3_config(&self.nodes[node_idx].url, &self.access_key, &self.secret_key, None, "cluster-test");
+        if let Some(ca_pem) = &self.tls_ca_pem {
+            use aws_smithy_http_client::tls::{Provider, TlsContext, TrustStore, rustls_provider::CryptoMode};
+            let tls = TlsContext::builder()
+                .with_trust_store(TrustStore::empty().with_pem_certificate(ca_pem.clone()))
+                .build()?;
+            let client = SmithyHttpClientBuilder::new()
+                .tls_provider(Provider::Rustls(CryptoMode::AwsLc))
+                .tls_context(tls)
+                .build_https();
+            config = config.to_builder().http_client(client).build();
+        }
+        Ok(Client::from_conf(config))
     }
 
     /// Create S3 clients for all nodes in the RustFS cluster and collect them into a vector.
@@ -1824,7 +1885,7 @@ impl RustFSTestClusterEnvironment {
         }
 
         self.nodes.push(ClusterNode {
-            url: format!("http://{address}"),
+            url: format!("{}://{address}", if self.tls_ca_pem.is_some() { "https" } else { "http" }),
             address,
             data_dir: data_dirs[0].clone(),
             data_dirs,
@@ -2438,6 +2499,7 @@ mod tests {
             node_capture_log_paths: vec![None; topology.node_count],
             topology,
             volume_proxy_addresses: vec![None; node_count],
+            tls_ca_pem: None,
         }
     }
 
@@ -2455,8 +2517,8 @@ mod tests {
         );
     }
 
-    #[test]
-    fn volumes_single_pool_multidrive_enumerates_every_drive() {
+    #[tokio::test]
+    async fn volumes_single_pool_multidrive_enumerates_every_drive() {
         // 4 nodes x 2 drives -> 8 explicit endpoints, one legacy pool. No
         // ellipses, so the server parser keeps this as a single DistErasure pool.
         let env = fake_cluster(ClusterTopology::single_pool_multidrive(4, 2));
@@ -2468,6 +2530,25 @@ mod tests {
             .join(" ");
         assert_eq!(env.build_volumes_arg(), expected);
         assert_eq!(env.build_volumes_arg().split(' ').count(), 8);
+
+        let directory = tempfile::tempdir().expect("TLS fixture directory");
+        crate::tls_gen::run(crate::tls_gen::Args {
+            out_dir: directory.path().to_owned(),
+            days: 1,
+            force: false,
+        })
+        .expect("generate TLS bundle");
+        for topology in [
+            ClusterTopology::single_pool_multidrive(4, 4),
+            ClusterTopology::per_node_pools(4, vec![vec![0], vec![1]]),
+        ] {
+            let mut cluster = fake_cluster(topology);
+            let plaintext_volumes = cluster.build_volumes_arg();
+            cluster.enable_tls(directory.path()).await.expect("configure TLS");
+            assert_eq!(cluster.build_volumes_arg(), plaintext_volumes.replace("http://", "https://"));
+            assert!(cluster.nodes.iter().all(|node| node.url.starts_with("https://")));
+            assert!(cluster.create_s3_client(0).is_ok(), "trusted SDK client should build");
+        }
     }
 
     #[test]
@@ -2555,6 +2636,22 @@ mod tests {
         proxy.shutdown().await;
     }
 
+    #[tokio::test]
+    async fn cluster_data_dirs_use_os_temporary_directory() {
+        let cluster = RustFSTestClusterEnvironment::new(1).await.expect("allocate isolated cluster");
+        let root = PathBuf::from(&cluster.temp_dir);
+        assert!(
+            root.starts_with(std::env::temp_dir()),
+            "cluster must honor the selected temporary filesystem"
+        );
+        for node in &cluster.nodes {
+            assert!(Path::new(&node.data_dir).starts_with(&root));
+            assert!(node.data_dirs.iter().all(|drive| Path::new(drive).starts_with(&root)));
+        }
+        drop(cluster);
+        assert!(!root.exists(), "dropping the fixture must remove its owned directories");
+    }
+
     #[test]
     fn cluster_node_env_supports_per_node_overrides() {
         let mut env = fake_cluster(ClusterTopology::single_pool(4));

crates/e2e_test/src/data_usage_test.rs

@@ -14,6 +14,7 @@
 
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+use futures::{StreamExt, TryStreamExt, stream};
 use rustfs_data_usage::DataUsageInfo;
 use tokio::time::{Duration, sleep};
 
@@ -74,16 +75,23 @@ async fn data_usage_reports_all_objects() -> Result<(), Box<dyn std::error::Erro
     // Create bucket and upload objects
     client.create_bucket().bucket(TEST_BUCKET).send().await?;
 
-    for i in 0..1000 {
-        let key = format!("obj-{i:04}");
-        client
-            .put_object()
-            .bucket(TEST_BUCKET)
-            .key(key)
-            .body(ByteStream::from_static(b"hello-world"))
-            .send()
-            .await?;
-    }
+    // Keep the full fixture and await every durable PUT before checking publication.
+    stream::iter(0..1000)
+        .map(Ok)
+        .try_for_each_concurrent(16, |i| {
+            let client = &client;
+            async move {
+                client
+                    .put_object()
+                    .bucket(TEST_BUCKET)
+                    .key(format!("obj-{i:04}"))
+                    .body(ByteStream::from_static(b"hello-world"))
+                    .send()
+                    .await
+                    .map(|_| ())
+            }
+        })
+        .await?;
 
     let usage = wait_for_bucket_usage(&env, TEST_BUCKET, |usage| {
         usage

crates/e2e_test/src/delete_objects_versioning_test.rs

@@ -24,6 +24,163 @@
 //! `xl.meta` immediately visible, and the old file-cache shortcut has been
 //! removed from the read path.
 
+use aws_sdk_s3::Client;
+use aws_sdk_s3::config::retry::RetryConfig;
+use aws_sdk_s3::types::{BucketVersioningStatus, Delete, ObjectIdentifier, VersioningConfiguration};
+use std::collections::BTreeMap;
+
+async fn delete_exact_marker_versions(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    versions: &[(String, bool)],
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let objects = versions
+        .iter()
+        .map(|(version, _)| ObjectIdentifier::builder().key(key).version_id(version).build())
+        .collect::<Result<Vec<_>, _>>()?;
+    let response = client
+        .delete_objects()
+        .bucket(bucket)
+        .delete(Delete::builder().set_objects(Some(objects)).build()?)
+        .send()
+        .await?;
+    assert!(response.errors().is_empty(), "explicit batch returned errors: {response:?}");
+    let mut actual = BTreeMap::new();
+    for deleted in response.deleted() {
+        assert_eq!(deleted.key(), Some(key));
+        *actual
+            .entry((
+                deleted.version_id().ok_or("Deleted entry omitted version ID")?.to_string(),
+                deleted.delete_marker().unwrap_or(false),
+                deleted.delete_marker_version_id().map(str::to_string),
+            ))
+            .or_insert(0_usize) += 1;
+    }
+    let mut expected = BTreeMap::new();
+    for (version, marker) in versions {
+        *expected
+            .entry((version.clone(), *marker, marker.then(|| version.clone())))
+            .or_insert(0_usize) += 1;
+    }
+    assert_eq!(actual, expected, "every requested occurrence must report its own version identity");
+    Ok(())
+}
+
+pub(crate) async fn assert_explicit_marker_batch_cleanup(
+    clients: &[Client],
+    bucket: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    assert!(!clients.is_empty());
+    let clients: Vec<_> = clients
+        .iter()
+        .map(|client| {
+            Client::from_conf(
+                client
+                    .config()
+                    .to_builder()
+                    .retry_config(RetryConfig::standard().with_max_attempts(1))
+                    .build(),
+            )
+        })
+        .collect();
+    let writer = &clients[0];
+    let marker_writer = &clients[1 % clients.len()];
+    let deleter = &clients[2 % clients.len()];
+    let key = "same-key-history";
+    writer.create_bucket().bucket(bucket).send().await?;
+    writer
+        .put_bucket_versioning()
+        .bucket(bucket)
+        .versioning_configuration(
+            VersioningConfiguration::builder()
+                .status(BucketVersioningStatus::Enabled)
+                .build(),
+        )
+        .send()
+        .await?;
+    let mut data = Vec::new();
+    for byte in [41_u8, 73_u8] {
+        let body = vec![byte; 1024 * 1024];
+        let uploaded = writer
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(body.clone().into())
+            .send()
+            .await?;
+        data.push((uploaded.version_id().ok_or("versioned PUT omitted ID")?.to_string(), body));
+    }
+    let mut markers = Vec::with_capacity(16);
+    for _ in 0..16 {
+        let deleted = marker_writer.delete_object().bucket(bucket).key(key).send().await?;
+        assert_eq!(deleted.delete_marker(), Some(true));
+        markers.push(deleted.version_id().ok_or("marker creation omitted ID")?.to_string());
+    }
+    let mut requested: Vec<_> = markers.iter().map(|id| (id.clone(), true)).collect();
+    requested.push((markers[0].clone(), true));
+    requested.push((markers[15].clone(), true));
+    delete_exact_marker_versions(deleter, bucket, key, &requested).await?;
+    for client in &clients {
+        let versions = client.list_object_versions().bucket(bucket).send().await?;
+        assert!(versions.delete_markers().is_empty(), "marker survived successful batch: {versions:?}");
+        let mut actual: Vec<_> = versions
+            .versions()
+            .iter()
+            .filter_map(|version| version.version_id())
+            .collect();
+        actual.sort_unstable();
+        let mut expected: Vec<_> = data.iter().map(|(id, _)| id.as_str()).collect();
+        expected.sort_unstable();
+        assert_eq!(actual, expected);
+        let current = client.get_object().bucket(bucket).key(key).send().await?;
+        assert_eq!(current.body.collect().await?.into_bytes().as_ref(), data[1].1.as_slice());
+        let historical = client
+            .get_object()
+            .bucket(bucket)
+            .key(key)
+            .version_id(&data[0].0)
+            .send()
+            .await?;
+        assert_eq!(historical.body.collect().await?.into_bytes().as_ref(), data[0].1.as_slice());
+    }
+    let uuid_marker = marker_writer.delete_object().bucket(bucket).key(key).send().await?;
+    let uuid_marker = uuid_marker.version_id().ok_or("marker creation omitted UUID")?.to_string();
+    writer
+        .put_bucket_versioning()
+        .bucket(bucket)
+        .versioning_configuration(
+            VersioningConfiguration::builder()
+                .status(BucketVersioningStatus::Suspended)
+                .build(),
+        )
+        .send()
+        .await?;
+    let null_marker = marker_writer.delete_object().bucket(bucket).key(key).send().await?;
+    assert_eq!(null_marker.delete_marker(), Some(true));
+    assert_eq!(null_marker.version_id(), Some("null"));
+    let requested = [
+        (uuid_marker.clone(), true),
+        (data[1].0.clone(), false),
+        ("null".to_string(), true),
+        (data[0].0.clone(), false),
+        (uuid_marker, true),
+    ];
+    delete_exact_marker_versions(deleter, bucket, key, &requested).await?;
+    for client in &clients {
+        let versions = client.list_object_versions().bucket(bucket).send().await?;
+        assert!(versions.versions().is_empty() && versions.delete_markers().is_empty(), "{versions:?}");
+    }
+    clients
+        .last()
+        .expect("nonempty clients")
+        .delete_bucket()
+        .bucket(bucket)
+        .send()
+        .await?;
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use crate::common::{RustFSTestEnvironment, init_logging};
@@ -35,6 +192,16 @@ mod tests {
         env.create_s3_client()
     }
 
+    #[tokio::test]
+    async fn test_explicit_delete_marker_batch_removes_versions_without_client_retries()
+    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
+            .await?;
+        super::assert_explicit_marker_batch_cleanup(&[env.create_s3_client()], "explicit-marker-batch").await
+    }
+
     /// Regression test for Issue #1878.
     ///
     /// Verifies that after calling `delete_objects` (the batch plural API) on

crates/e2e_test/src/distributed/artifact_delete_test.rs

@@ -0,0 +1,378 @@
+// Copyright 2026 RustFS Team
+// SPDX-License-Identifier: Apache-2.0
+
+#![cfg(unix)]
+
+use super::artifact_seal_test::{assert_exact_version, confirm_seal, create_artifact_attempt};
+use super::harness::{
+    DistCluster, DistLayout, TestResult, enable_versioning, payload_for, s3_probe_client, unique_bucket, wait_for_ready,
+};
+use crate::chaos::{VersionShardCensus, census_object_version_on_disk};
+use crate::common::init_logging;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::types::{Delete, ObjectIdentifier};
+use serde_json::json;
+use std::collections::{BTreeMap, BTreeSet};
+use std::os::unix::fs::MetadataExt;
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+use tokio::time::timeout;
+
+const PUBLIC_INTERNAL_ERROR: &str = "We encountered an internal error, please try again.";
+
+#[derive(Debug, PartialEq, Eq)]
+pub(super) struct PhysicalEntry {
+    identity: (u64, u64),
+    changed: (i64, i64),
+    modified: (i64, i64),
+    bytes: Option<Vec<u8>>,
+}
+
+// Include directory identity/ctime as well as file bytes. Creating and removing
+// a temporary marker/rollback file cannot hide behind a restored final xl.meta.
+// Reads change atime, which is deliberately excluded.
+pub(super) fn physical_snapshot(dist: &DistCluster, bucket: &str) -> TestResult<BTreeMap<(usize, PathBuf), PhysicalEntry>> {
+    let mut snapshot = BTreeMap::new();
+    for (node, instance) in dist.cluster.nodes.iter().enumerate() {
+        let root = Path::new(&instance.data_dirs[0]).join(bucket);
+        for entry in walkdir::WalkDir::new(&root) {
+            let entry = entry?;
+            let metadata = std::fs::symlink_metadata(entry.path())?;
+            assert!(!metadata.file_type().is_symlink(), "unexpected fixture symlink");
+            snapshot.insert(
+                (node, entry.path().strip_prefix(&root)?.to_path_buf()),
+                PhysicalEntry {
+                    identity: (metadata.dev(), metadata.ino()),
+                    changed: (metadata.ctime(), metadata.ctime_nsec()),
+                    modified: (metadata.mtime(), metadata.mtime_nsec()),
+                    bytes: if metadata.is_dir() {
+                        None
+                    } else {
+                        Some(std::fs::read(entry.path())?)
+                    },
+                },
+            );
+        }
+    }
+    Ok(snapshot)
+}
+
+struct Version {
+    key: String,
+    id: String,
+    body: Vec<u8>,
+    manifests: Vec<VersionShardCensus>,
+}
+
+impl Version {
+    async fn put(dist: &DistCluster, bucket: &str, key: &str, seed: &str) -> TestResult<Self> {
+        let body = payload_for(seed, 2 * 1024 * 1024);
+        let output = timeout(
+            Duration::from_secs(30),
+            dist.client(1)?
+                .put_object()
+                .bucket(bucket)
+                .key(key)
+                .body(body.clone().into())
+                .send(),
+        )
+        .await??;
+        let id = output.version_id().ok_or("fixture PUT omitted version ID")?.to_owned();
+        let manifests = dist
+            .cluster
+            .nodes
+            .iter()
+            .map(|node| census_object_version_on_disk(Path::new(&node.data_dirs[0]), bucket, key, Some(&id)))
+            .collect::<TestResult<Vec<_>>>()?;
+        assert!(
+            manifests
+                .iter()
+                .all(|manifest| manifest.is_complete() && !manifest.expected_part_numbers.is_empty())
+        );
+        Ok(Self {
+            key: key.to_owned(),
+            id,
+            body,
+            manifests,
+        })
+    }
+}
+
+fn target(key: &str, version: Option<&str>) -> TestResult<ObjectIdentifier> {
+    Ok(ObjectIdentifier::builder()
+        .key(key)
+        .set_version_id(version.map(str::to_owned))
+        .build()?)
+}
+
+async fn assert_fixture(
+    dist: &DistCluster,
+    bucket: &str,
+    prefix: &str,
+    versions: &[Version],
+    marker: (&str, &str),
+) -> TestResult {
+    let expected: BTreeSet<_> = versions
+        .iter()
+        .map(|version| (version.key.clone(), version.id.clone()))
+        .collect();
+    for node in 0..dist.cluster.nodes.len() {
+        let client = dist.client(node)?;
+        let listed = timeout(
+            Duration::from_secs(30),
+            client.list_object_versions().bucket(bucket).prefix(prefix).send(),
+        )
+        .await??;
+        let actual: BTreeSet<_> = listed
+            .versions()
+            .iter()
+            .map(|version| {
+                (
+                    version.key().unwrap_or_default().to_owned(),
+                    version.version_id().unwrap_or_default().to_owned(),
+                )
+            })
+            .collect();
+        assert_eq!(listed.versions().len(), versions.len());
+        assert_eq!(actual, expected, "delete changed exact data versions on node {node}");
+        assert_eq!(listed.delete_markers().len(), 1);
+        assert_eq!(listed.delete_markers()[0].key(), Some(marker.0));
+        assert_eq!(listed.delete_markers()[0].version_id(), Some(marker.1));
+        assert_eq!(listed.delete_markers()[0].is_latest(), Some(true));
+        for version in versions {
+            assert_exact_version(&client, bucket, &version.key, &version.id, &version.body).await?;
+            let disk = Path::new(&dist.cluster.nodes[node].data_dirs[0]);
+            assert!(
+                census_object_version_on_disk(disk, bucket, &version.key, Some(&version.id))?
+                    .matches_manifest(&version.manifests[node])
+            );
+        }
+    }
+    Ok(())
+}
+
+async fn assert_single_denied(dist: &DistCluster, node: usize, bucket: &str, key: &str, version: Option<&str>) -> TestResult {
+    let before = physical_snapshot(dist, bucket)?;
+    // One attempt: a later retry must not conceal a first local mutation.
+    let client = s3_probe_client(&dist.client(node)?);
+    let error = timeout(
+        Duration::from_secs(30),
+        client
+            .delete_object()
+            .bucket(bucket)
+            .key(key)
+            .set_version_id(version.map(str::to_owned))
+            .send(),
+    )
+    .await?
+    .expect_err("ordinary deletion succeeded in a managed bucket");
+    let service = error
+        .as_service_error()
+        .ok_or("delete failed without an S3 service rejection")?;
+    assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(500));
+    assert_eq!(service.code(), Some("InternalError"));
+    assert_eq!(service.message(), Some(PUBLIC_INTERNAL_ERROR));
+    assert!(
+        physical_snapshot(dist, bucket)? == before,
+        "first delete through node {node} wrote or rolled back managed data"
+    );
+    Ok(())
+}
+
+async fn assert_batch_denied(dist: &DistCluster, node: usize, bucket: &str, objects: &[ObjectIdentifier]) -> TestResult {
+    let before = physical_snapshot(dist, bucket)?;
+    let client = s3_probe_client(&dist.client(node)?);
+    let response = timeout(
+        Duration::from_secs(30),
+        client
+            .delete_objects()
+            .bucket(bucket)
+            .delete(Delete::builder().set_objects(Some(objects.to_vec())).quiet(false).build()?)
+            .send(),
+    )
+    .await??;
+    assert!(response.deleted().is_empty(), "managed batch reported a successful delete");
+    assert_eq!(response.errors().len(), objects.len(), "batch must reject every requested occurrence");
+    for (error, requested) in response.errors().iter().zip(objects) {
+        assert_eq!(error.key(), Some(requested.key()));
+        assert_eq!(error.version_id(), requested.version_id());
+        assert_eq!(error.code(), Some("InternalError"));
+        assert_eq!(error.message(), Some(PUBLIC_INTERNAL_ERROR));
+    }
+    assert!(
+        physical_snapshot(dist, bucket)? == before,
+        "first batch through node {node} wrote or rolled back managed data"
+    );
+    Ok(())
+}
+
+fn copy_existing_object(source: &Path, destination: &Path) -> TestResult {
+    assert!(!destination.exists());
+    for entry in walkdir::WalkDir::new(source) {
+        let entry = entry?;
+        let target = destination.join(entry.path().strip_prefix(source)?);
+        if entry.file_type().is_dir() {
+            std::fs::create_dir_all(&target)?;
+        } else {
+            assert!(entry.file_type().is_file());
+            std::fs::copy(entry.path(), target)?;
+        }
+    }
+    Ok(())
+}
+
+async fn ordinary_delete_control(dist: &DistCluster, bucket: &str) -> TestResult {
+    let client = s3_probe_client(&dist.client(0)?);
+    let key = "ordinary-observer-control.bin";
+    let version = Version::put(dist, bucket, key, key).await?;
+    let before = physical_snapshot(dist, bucket)?;
+    let deleted = client.delete_object().bucket(bucket).key(key).send().await?;
+    assert_eq!(deleted.delete_marker(), Some(true));
+    let marker = deleted.version_id().ok_or("ordinary delete omitted marker ID")?;
+    assert!(
+        physical_snapshot(dist, bucket)? != before,
+        "physical observer missed real marker creation"
+    );
+    client
+        .delete_object()
+        .bucket(bucket)
+        .key(key)
+        .version_id(marker)
+        .send()
+        .await?;
+    // The logical version inventory is restored, but ctime/identity still
+    // expose the two writes. This checks the observer used by managed negatives.
+    let listed = client.list_object_versions().bucket(bucket).prefix(key).send().await?;
+    assert_eq!(listed.versions().len(), 1);
+    assert_eq!(listed.versions()[0].version_id(), Some(version.id.as_str()));
+    assert!(listed.delete_markers().is_empty());
+    assert!(
+        physical_snapshot(dist, bucket)? != before,
+        "marker round-trip hid transient physical writes"
+    );
+    assert_exact_version(&client, bucket, key, &version.id, &version.body).await?;
+    client
+        .delete_object()
+        .bucket(bucket)
+        .key(key)
+        .version_id(&version.id)
+        .send()
+        .await?;
+    assert!(
+        client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix(key)
+            .send()
+            .await?
+            .versions()
+            .is_empty()
+    );
+
+    let current = Version::put(dist, bucket, "ordinary-batch-current.bin", "current").await?;
+    let exact = Version::put(dist, bucket, "ordinary-batch-exact.bin", "exact").await?;
+    let response = client
+        .delete_objects()
+        .bucket(bucket)
+        .delete(
+            Delete::builder()
+                .objects(target(&current.key, None)?)
+                .objects(target(&exact.key, Some(&exact.id))?)
+                .objects(target(&exact.key, Some(&exact.id))?)
+                .build()?,
+        )
+        .send()
+        .await?;
+    assert!(response.errors().is_empty());
+    assert_eq!(response.deleted().len(), 3);
+    let listed = client
+        .list_object_versions()
+        .bucket(bucket)
+        .prefix(&current.key)
+        .send()
+        .await?;
+    assert_eq!(listed.versions().len(), 1);
+    assert_eq!(listed.delete_markers().len(), 1);
+    assert!(
+        client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix(&exact.key)
+            .send()
+            .await?
+            .versions()
+            .is_empty()
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_artifact_delete_entrypoints_preserve_versions_and_physical_state() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
+    let (identity, admin) = create_artifact_attempt(&dist).await?;
+    let bucket = identity["bucket"]
+        .as_str()
+        .ok_or("managed receipt omitted bucket")?
+        .to_owned();
+    let prefix = identity["attempt_prefix"]
+        .as_str()
+        .ok_or("managed receipt omitted prefix")?
+        .to_owned();
+    let key = format!("{prefix}native.bin");
+    let mut versions = vec![
+        Version::put(&dist, &bucket, &key, "native-old").await?,
+        Version::put(&dist, &bucket, &key, "native-new").await?,
+    ];
+    let ordinary = unique_bucket("delete-control");
+    dist.create_bucket(&ordinary).await?;
+    enable_versioning(&dist.client(0)?, &ordinary).await?;
+
+    // A new managed bucket cannot legitimately create a marker. Model existing
+    // persisted state from before adoption/upgrade using a real S3-generated
+    // ordinary object+marker, copied disk-for-disk in the same erasure set.
+    // Only this object's files move; bucket authority/configuration is untouched.
+    let marker_key = format!("{prefix}existing-marker.bin");
+    versions.push(Version::put(&dist, &ordinary, &marker_key, "existing-data").await?);
+    let output = dist
+        .client(2)?
+        .delete_object()
+        .bucket(&ordinary)
+        .key(&marker_key)
+        .send()
+        .await?;
+    assert_eq!(output.delete_marker(), Some(true));
+    let marker = output.version_id().ok_or("ordinary fixture omitted marker ID")?.to_owned();
+    dist.cluster.stop();
+    for node in &dist.cluster.nodes {
+        let disk = Path::new(&node.data_dirs[0]);
+        copy_existing_object(&disk.join(&ordinary).join(&marker_key), &disk.join(&bucket).join(&marker_key))?;
+    }
+    dist.cluster.start().await?;
+    wait_for_ready(&dist.cluster).await?;
+    assert_fixture(&dist, &bucket, &prefix, &versions, (&marker_key, &marker)).await?;
+    ordinary_delete_control(&dist, &ordinary).await?;
+    let batch = vec![
+        target(&key, None)?,
+        target(&key, Some(&versions[0].id))?,
+        target(&key, Some(&versions[1].id))?,
+        target(&marker_key, Some(&marker))?,
+        target(&key, Some(&versions[0].id))?,
+        target(&marker_key, Some(&versions[2].id))?,
+    ];
+    for phase in ["open", "sealed", "restarted"] {
+        if phase == "sealed" {
+            confirm_seal(&dist, &admin, &json!({"version": 1, "target": identity}).to_string(), &identity).await?;
+        } else if phase == "restarted" {
+            dist.restart_with_current_binary().await?;
+        }
+        for node in 0..dist.cluster.nodes.len() {
+            assert_single_denied(&dist, node, &bucket, &key, None).await?;
+            assert_single_denied(&dist, node, &bucket, &key, Some(&versions[0].id)).await?;
+            assert_single_denied(&dist, node, &bucket, &marker_key, Some(&marker)).await?;
+            assert_batch_denied(&dist, node, &bucket, &batch).await?;
+        }
+        assert_fixture(&dist, &bucket, &prefix, &versions, (&marker_key, &marker)).await?;
+    }
+    Ok(())
+}

crates/e2e_test/src/distributed/artifact_inventory_test.rs

@@ -0,0 +1,434 @@
+// Copyright 2026 RustFS Team
+// SPDX-License-Identifier: Apache-2.0
+
+#![cfg(unix)]
+
+use super::artifact_delete_test::physical_snapshot;
+use super::artifact_seal_test::{ArtifactAdmin, create_artifact_attempt};
+use super::harness::{DistCluster, DistLayout, TestResult, wait_for_ready};
+use crate::common::{
+    AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
+    signed_s3_request,
+};
+use futures::{StreamExt as _, TryStreamExt as _};
+use http::{Method, StatusCode};
+use serde::Deserialize;
+use serde_json::{Value, json};
+use std::{collections::BTreeSet, time::Duration};
+use tokio::time::timeout;
+use uuid::Uuid;
+
+const ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/inventory";
+const BUDGET: Duration = Duration::from_secs(45);
+
+struct Reader {
+    access: String,
+    secret: String,
+}
+
+impl Reader {
+    async fn create(dist: &DistCluster, target: &Value, action: &str) -> TestResult<Self> {
+        let reader = Self {
+            access: format!("inventory{}", &Uuid::new_v4().simple().to_string()[..12]),
+            secret: Uuid::new_v4().simple().to_string(),
+        };
+        let policy_name = format!("inventory{}", &Uuid::new_v4().simple().to_string()[..12]);
+        let bucket = target["bucket"].as_str().ok_or("target bucket")?;
+        let prefix = target["attempt_prefix"]
+            .as_str()
+            .ok_or("target prefix")?
+            .trim_end_matches('/');
+        let resource = if action == "s3:ListBucket" {
+            format!("arn:aws:s3:::{bucket}")
+        } else {
+            format!("arn:aws:s3:::{bucket}/{prefix}")
+        };
+        let policy =
+            json!({"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":action,"Resource":resource}]}).to_string();
+        let url = &dist.cluster.nodes[0].url;
+        let access = &dist.cluster.access_key;
+        let secret = &dist.cluster.secret_key;
+        admin_create_user_via(AdminTransport::Signed, url, access, secret, &reader.access, &reader.secret).await?;
+        admin_add_canned_policy_via(AdminTransport::Signed, url, access, secret, &policy_name, &policy).await?;
+        admin_attach_user_policy_via(AdminTransport::Signed, url, access, secret, &policy_name, &reader.access).await?;
+        Ok(reader)
+    }
+
+    async fn request(&self, dist: &DistCluster, target: &Value, invocation: Uuid) -> TestResult<reqwest::Response> {
+        timeout(
+            BUDGET,
+            signed_s3_request(
+                Method::POST,
+                &format!("{}{ROUTE}", dist.cluster.nodes[0].url),
+                Some(json!({"version":1,"invocation_id":invocation,"target":target}).to_string()),
+                Some("application/json"),
+                &self.access,
+                &self.secret,
+            ),
+        )
+        .await?
+    }
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+struct Frame {
+    version: u32,
+    invocation_id: Uuid,
+    target: Value,
+    sequence: u64,
+    payload: Payload,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
+enum Payload {
+    Observation {
+        disk_id: Uuid,
+        key: String,
+        version_id: Uuid,
+        delete_marker: bool,
+    },
+    Complete {
+        members: usize,
+        observations: u64,
+    },
+}
+
+async fn read_frames(mut response: reqwest::Response) -> TestResult<(Vec<Frame>, bool)> {
+    let mut pending = Vec::new();
+    let mut frames = Vec::new();
+    let mut clean = true;
+    loop {
+        match timeout(BUDGET, response.chunk()).await? {
+            Ok(Some(chunk)) => {
+                pending.extend_from_slice(&chunk);
+                while let Some(end) = pending.iter().position(|byte| *byte == b'\n') {
+                    assert!(end < 1024 * 1024, "HTTP inventory line exceeds encoded bound");
+                    let line: Vec<_> = pending.drain(..=end).collect();
+                    frames.push(serde_json::from_slice(&line[..end])?);
+                }
+                assert!(pending.len() < 1024 * 1024, "unbounded incomplete HTTP line");
+            }
+            Ok(None) => break,
+            Err(_) => {
+                clean = false;
+                break;
+            }
+        }
+    }
+    Ok((frames, clean && pending.is_empty()))
+}
+
+async fn seal(dist: &DistCluster, user: &ArtifactAdmin, target: &Value) -> TestResult {
+    let (status, body) = user
+        .request(
+            dist,
+            0,
+            "/rustfs/admin/v3/plugin-artifacts/seal-writes",
+            json!({"version":1,"target":target}).to_string(),
+        )
+        .await?;
+    assert_eq!(status, StatusCode::OK, "actual all-member closure: {body}");
+    assert_eq!(serde_json::from_str::<Value>(&body)?["all_writes_sealed"], true);
+    Ok(())
+}
+
+fn exact_inventory(frames: &[Frame], target: &Value, invocation: Uuid, versions: &BTreeSet<(String, Uuid, bool)>) {
+    let mut observed = BTreeSet::new();
+    let mut disks = BTreeSet::new();
+    let mut count = 0u64;
+    for (sequence, frame) in frames.iter().enumerate() {
+        assert_eq!(frame.version, 1);
+        assert_eq!(frame.invocation_id, invocation);
+        assert_eq!(&frame.target, target);
+        assert_eq!(frame.sequence, u64::try_from(sequence).expect("bounded fixture sequence"));
+        match &frame.payload {
+            Payload::Observation {
+                disk_id,
+                key,
+                version_id,
+                delete_marker,
+            } => {
+                assert!(!disk_id.is_nil() && !version_id.is_nil());
+                assert!(
+                    observed.insert((*disk_id, key.clone(), *version_id, *delete_marker)),
+                    "duplicate physical observation"
+                );
+                disks.insert(*disk_id);
+                count += 1;
+            }
+            Payload::Complete { members, observations } => {
+                assert_eq!(sequence + 1, frames.len(), "Complete must be the last HTTP frame");
+                assert_eq!(*members, 4);
+                assert_eq!(*observations, count);
+            }
+        }
+    }
+    assert!(
+        matches!(frames.last().map(|frame| &frame.payload), Some(Payload::Complete { .. })),
+        "A03I3A_HTTP_COMPLETE"
+    );
+    if !versions.is_empty() {
+        assert_eq!(disks.len(), 4);
+    }
+    for disk in disks {
+        let actual: BTreeSet<_> = observed
+            .iter()
+            .filter(|(id, ..)| *id == disk)
+            .map(|(_, key, version, marker)| (key.clone(), *version, *marker))
+            .collect();
+        assert_eq!(&actual, versions, "A03I3A_EXACT_VERSIONS");
+    }
+    assert_eq!(observed.len(), versions.len() * 4);
+}
+
+#[tokio::test]
+async fn artifact_inventory_http_exact_versions() -> TestResult {
+    crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
+    let (target, admin) = create_artifact_attempt(&dist).await?;
+    let bucket = target["bucket"].as_str().ok_or("bucket")?;
+    let prefix = target["attempt_prefix"].as_str().ok_or("prefix")?;
+    let mut versions = BTreeSet::new();
+    for (name, body) in [
+        ("same", b"first".to_vec()),
+        ("same", b"second".to_vec()),
+        ("zero", Vec::new()),
+    ] {
+        let key = format!("{prefix}{name}");
+        let output = timeout(BUDGET, dist.client(1)?.put_object().bucket(bucket).key(&key).body(body.into()).send()).await??;
+        let version = Uuid::parse_str(output.version_id().ok_or("actual PUT version")?)?;
+        versions.insert((key, version, false));
+    }
+    seal(&dist, &admin, &target).await?;
+    // A wildcard action is still restricted to this exact attempt resource.
+    // The separate authority case exercises the dedicated action alone.
+    let reader = Reader::create(&dist, &target, "admin:*").await?;
+    let before = physical_snapshot(&dist, bucket)?;
+    let invocation = Uuid::new_v4();
+    let response = reader.request(&dist, &target, invocation).await?;
+    assert_eq!(response.status(), StatusCode::OK, "A03I3A_HTTP_ROUTE");
+    assert_eq!(
+        response.headers().get("content-type").and_then(|value| value.to_str().ok()),
+        Some("application/x-ndjson")
+    );
+    let (frames, clean) = read_frames(response).await?;
+    assert!(clean, "Complete requires clean HTTP EOF");
+    assert!(frames.len() > 2, "real multi-frame HTTP body");
+    exact_inventory(&frames, &target, invocation, &versions);
+    assert!(
+        before == physical_snapshot(&dist, bucket)?,
+        "inventory mutated physical artifact namespace"
+    );
+    let (empty, empty_admin) = create_artifact_attempt(&dist).await?;
+    seal(&dist, &empty_admin, &empty).await?;
+    let empty_reader = Reader::create(&dist, &empty, "admin:*").await?;
+    let invocation = Uuid::new_v4();
+    let response = empty_reader.request(&dist, &empty, invocation).await?;
+    assert_eq!(response.status(), StatusCode::OK);
+    let (frames, clean) = read_frames(response).await?;
+    assert!(clean);
+    exact_inventory(&frames, &empty, invocation, &BTreeSet::new());
+    Ok(())
+}
+
+#[tokio::test]
+async fn artifact_inventory_http_scoped_authority() -> TestResult {
+    crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
+    let (target, admin) = create_artifact_attempt(&dist).await?;
+    seal(&dist, &admin, &target).await?;
+    let reader = Reader::create(&dist, &target, "admin:InventoryPluginArtifactAttempt").await?;
+    let invocation = Uuid::new_v4();
+    let response = reader.request(&dist, &target, invocation).await?;
+    assert_eq!(response.status(), StatusCode::OK, "dedicated inventory authority");
+    let (frames, clean) = read_frames(response).await?;
+    assert!(clean);
+    exact_inventory(&frames, &target, invocation, &BTreeSet::new());
+    let mut other_prefix = target.clone();
+    other_prefix["attempt_prefix"] = json!(format!("outputs/6169/{}/{}/", Uuid::new_v4(), Uuid::new_v4()));
+    let mut other_bucket = target.clone();
+    other_bucket["bucket"] = json!("another-artifact-bucket");
+    for denied in [&other_prefix, &other_bucket] {
+        assert_eq!(
+            reader.request(&dist, denied, Uuid::new_v4()).await?.status(),
+            StatusCode::FORBIDDEN,
+            "A03I3A_SCOPED_AUTHORITY"
+        );
+    }
+    let (status, _) = admin
+        .request(
+            &dist,
+            0,
+            ROUTE,
+            json!({"version":1,"invocation_id":Uuid::new_v4(),"target":target}).to_string(),
+        )
+        .await?;
+    assert_eq!(status, StatusCode::FORBIDDEN, "A03I3A_SCOPED_AUTHORITY seal-only user");
+    let listed = Reader::create(&dist, &target, "s3:ListBucket").await?;
+    dist.client_with_credentials(0, &listed.access, &listed.secret)?
+        .list_objects_v2()
+        .bucket(target["bucket"].as_str().ok_or("bucket")?)
+        .send()
+        .await?;
+    assert_eq!(
+        listed.request(&dist, &target, Uuid::new_v4()).await?.status(),
+        StatusCode::FORBIDDEN,
+        "A03I3A_SCOPED_AUTHORITY S3 list-only user"
+    );
+    let root_response = signed_s3_request(
+        Method::POST,
+        &format!("{}{ROUTE}", dist.cluster.nodes[0].url),
+        Some(json!({"version":1,"invocation_id":Uuid::new_v4(),"target":target}).to_string()),
+        Some("application/json"),
+        &dist.cluster.access_key,
+        &dist.cluster.secret_key,
+    )
+    .await?;
+    assert_eq!(root_response.status(), StatusCode::FORBIDDEN, "original scoped user required");
+    for body in [
+        json!({"version":1,"invocation_id":Uuid::nil(),"target":target}),
+        json!({"version":1,"invocation_id":Uuid::new_v4(),"target":target,"extra":true}),
+    ] {
+        let response = signed_s3_request(
+            Method::POST,
+            &format!("{}{ROUTE}", dist.cluster.nodes[0].url),
+            Some(body.to_string()),
+            Some("application/json"),
+            &reader.access,
+            &reader.secret,
+        )
+        .await?;
+        assert_eq!(response.status(), StatusCode::BAD_REQUEST, "strict inventory request");
+    }
+    Ok(())
+}
+
+async fn assert_incomplete(response: reqwest::Response) -> TestResult<Vec<Frame>> {
+    assert_eq!(
+        response.status(),
+        StatusCode::OK,
+        "valid request must exercise the streaming failure boundary"
+    );
+    let (frames, _) = read_frames(response).await?;
+    assert!(
+        !frames.iter().any(|frame| matches!(frame.payload, Payload::Complete { .. })),
+        "A03I3A_NO_FALSE_COMPLETE"
+    );
+    Ok(frames)
+}
+
+#[tokio::test]
+async fn artifact_inventory_http_incomplete_has_no_terminal() -> TestResult {
+    crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
+    let (target, admin) = create_artifact_attempt(&dist).await?;
+    let bucket = target["bucket"].as_str().ok_or("bucket")?;
+    let prefix = target["attempt_prefix"].as_str().ok_or("prefix")?;
+    // Real PUTs produce more data than a stalled HTTP/2 flow-control window.
+    futures::stream::iter(0..256)
+        .map(|index| {
+            let client = dist.client(1);
+            let key = format!("{prefix}slow/{index:04}/{}/{}", "a".repeat(190), "b".repeat(190));
+            async move {
+                let output = client?
+                    .put_object()
+                    .bucket(bucket)
+                    .key(key)
+                    .body(vec![1].into())
+                    .send()
+                    .await?;
+                assert!(!Uuid::parse_str(output.version_id().ok_or("actual PUT version")?)?.is_nil());
+                Ok::<_, Box<dyn std::error::Error + Send + Sync>>(())
+            }
+        })
+        .buffer_unordered(8)
+        .try_collect::<Vec<_>>()
+        .await?;
+    seal(&dist, &admin, &target).await?;
+    let reader = Reader::create(&dist, &target, "admin:InventoryPluginArtifactAttempt").await?;
+    dist.cluster.stop_node(3)?;
+    assert_incomplete(reader.request(&dist, &target, Uuid::new_v4()).await?).await?;
+    dist.cluster.start_node(3).await?;
+    wait_for_ready(&dist.cluster).await?;
+    let mut wrong = target.clone();
+    wrong["bucket_incarnation"] = json!(Uuid::new_v4());
+    assert_incomplete(reader.request(&dist, &wrong, Uuid::new_v4()).await?).await?;
+    // The configured endpoint map is ordered. Damage its final receiver, so
+    // earlier real observations are emitted before the physical scan fails.
+    let node = dist
+        .cluster
+        .nodes
+        .iter()
+        .max_by_key(|node| format!("{}{}", node.url, node.data_dirs[0]))
+        .ok_or("configured members")?;
+    let metadata = std::path::Path::new(&node.data_dirs[0])
+        .join(bucket)
+        .join(format!("{prefix}slow/0000/{}/{}", "a".repeat(190), "b".repeat(190)))
+        .join("xl.meta");
+    let original = std::fs::read(&metadata)?;
+    std::fs::write(&metadata, b"corrupt inventory metadata")?;
+    let before = physical_snapshot(&dist, bucket)?;
+    let outcome = assert_incomplete(reader.request(&dist, &target, Uuid::new_v4()).await?).await;
+    let after = physical_snapshot(&dist, bucket)?;
+    std::fs::write(&metadata, original)?;
+    let frames = outcome?;
+    assert!(before == after, "failed inventory repaired or mutated a physical object");
+    assert!(
+        frames
+            .iter()
+            .any(|frame| matches!(frame.payload, Payload::Observation { .. })),
+        "late physical failure must follow real HTTP observations"
+    );
+    let client = reqwest::Client::builder()
+        .no_proxy()
+        .http2_prior_knowledge()
+        .http2_initial_stream_window_size(1)
+        .http2_initial_connection_window_size(1)
+        .http2_adaptive_window(false)
+        .build()?;
+    let url = format!("{}{ROUTE}", dist.cluster.nodes[0].url);
+    let request = |invocation| {
+        json!({"version":1,"invocation_id":invocation,"target":target})
+            .to_string()
+            .into_bytes()
+    };
+    let response = crate::common::signed_request_with_client(
+        &client,
+        Method::POST,
+        &url,
+        &reader.access,
+        &reader.secret,
+        Some(request(Uuid::new_v4())),
+        Some("application/json"),
+    )
+    .await?;
+    assert_eq!(response.status(), StatusCode::OK);
+    // Do not poll the body: its one-byte HTTP/2 receive window stalls transport.
+    // This waits beyond the fixed30s service budget, not for readiness.
+    tokio::time::sleep(Duration::from_secs(35)).await;
+    assert_incomplete(response).await?;
+    let response = crate::common::signed_request_with_client(
+        &client,
+        Method::POST,
+        &url,
+        &reader.access,
+        &reader.secret,
+        Some(request(Uuid::new_v4())),
+        Some("application/json"),
+    )
+    .await?;
+    assert_eq!(response.status(), StatusCode::OK);
+    drop(response);
+    let response = timeout(Duration::from_secs(10), reader.request(&dist, &target, Uuid::new_v4())).await??;
+    let (frames, clean) = timeout(Duration::from_secs(10), read_frames(response)).await??;
+    assert!(
+        clean && matches!(frames.last().map(|frame| &frame.payload), Some(Payload::Complete { .. })),
+        "fresh scan must succeed promptly after an actual disconnected HTTP body"
+    );
+    Ok(())
+}

crates/e2e_test/src/distributed/artifact_put_tail_test.rs

@@ -0,0 +1,269 @@
+// Copyright 2026 RustFS Team
+// SPDX-License-Identifier: Apache-2.0
+
+use super::artifact_repair_tail_test::PhysicalBarrier;
+use super::artifact_seal_test::{assert_exact_version, confirm_seal, create_artifact_attempt};
+use super::harness::{DistCluster, DistLayout, TestResult, payload_for, wait_until};
+use crate::chaos::census_object_version_on_disk;
+use crate::common::init_logging;
+use crate::storage_api::artifact_rpc::{
+    DiskError, TONIC_RPC_PREFIX, gen_tonic_signature_headers, set_tonic_canonical_body_digest,
+};
+use http::StatusCode;
+use rustfs_filemeta::FileMeta;
+use rustfs_protos::proto_gen::node_service::{RenameDataRequest, node_service_client::NodeServiceClient};
+use serde_json::{Value, json};
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+use tokio::time::timeout;
+use tonic::Request;
+use uuid::Uuid;
+
+const PUT_SEAL_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-attempt";
+
+fn signed_put_request(body: RenameDataRequest, audience: &str) -> TestResult<Request<RenameDataRequest>> {
+    let canonical = rustfs_protos::canonical_artifact_rename_data_request_body(&body)?;
+    let mut request = Request::new(body);
+    request.set_timeout(Duration::from_secs(15));
+    set_tonic_canonical_body_digest(&mut request, &canonical)?;
+    let digest = request
+        .metadata()
+        .get("x-rustfs-content-sha256")
+        .ok_or("missing canonical digest")?
+        .to_str()?
+        .to_owned();
+    request.metadata_mut().as_mut().extend(gen_tonic_signature_headers(
+        audience,
+        TONIC_RPC_PREFIX.trim_start_matches('/'),
+        "ArtifactRenameData",
+        Some(&digest),
+    )?);
+    Ok(request)
+}
+
+fn publication_snapshot(paths: &[&Path]) -> TestResult<BTreeMap<PathBuf, Option<Vec<u8>>>> {
+    let mut snapshot = BTreeMap::new();
+    for root in paths {
+        for entry in walkdir::WalkDir::new(root) {
+            let entry = entry?;
+            let contents = if entry.file_type().is_dir() {
+                None
+            } else {
+                Some(std::fs::read(entry.path())?)
+            };
+            snapshot.insert(entry.path().to_path_buf(), contents);
+        }
+    }
+    Ok(snapshot)
+}
+
+async fn assert_raw_put_closed(body: &RenameDataRequest, url: &str, audience: &str, paths: &[&Path]) -> TestResult {
+    let before = publication_snapshot(paths)?;
+    // Re-sign the exact original request; replay protection is not the rejection.
+    let request = signed_put_request(body.clone(), audience)?;
+    let response = timeout(Duration::from_secs(20), async {
+        let mut client = NodeServiceClient::connect(url.to_owned()).await?;
+        Ok::<_, Box<dyn std::error::Error + Send + Sync>>(client.artifact_rename_data(request).await?.into_inner())
+    })
+    .await??;
+    assert!(!response.success, "sealed attempt admitted another ordinary PUT");
+    let error = response.error.ok_or("rejected raw PUT omitted error")?;
+    assert_eq!(error.code, DiskError::Io(std::io::ErrorKind::PermissionDenied.into()).to_u32());
+    assert_eq!(error.error_info, "io error artifact attempt mutation domain is closed");
+    assert_eq!(publication_snapshot(paths)?, before, "rejected raw PUT mutated staged or published files");
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_artifact_put_seal_waits_for_disconnected_physical_tail() -> TestResult {
+    crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
+    init_logging();
+    let _ = rustfs_credentials::set_global_rpc_secret("rustfs-internode-signature-e2e-secret".to_owned());
+    let rpc_secret = rustfs_credentials::try_get_rpc_token()?;
+    let dist = DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_RPC_SECRET", &rpc_secret)]).await?;
+    let (target, user) = create_artifact_attempt(&dist).await?;
+    let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
+    let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
+    let key = format!("{prefix}detached-put.bin");
+    let payload = payload_for(&key, 2 * 1024 * 1024);
+    let output = timeout(
+        Duration::from_secs(30),
+        dist.client(1)?
+            .put_object()
+            .bucket(bucket)
+            .key(&key)
+            .body(payload.clone().into())
+            .send(),
+    )
+    .await??;
+    let version = output.version_id().ok_or("managed PUT omitted version ID")?;
+    let request_body = json!({"version": 1, "target": target}).to_string();
+    let baselines = dist
+        .cluster
+        .nodes
+        .iter()
+        .map(|node| census_object_version_on_disk(Path::new(&node.data_dirs[0]), bucket, &key, Some(version)))
+        .collect::<TestResult<Vec<_>>>()?;
+    assert!(
+        baselines
+            .iter()
+            .all(|baseline| baseline.is_complete() && !baseline.expected_part_numbers.is_empty())
+    );
+
+    let node = &dist.cluster.nodes[3];
+    let disk = PathBuf::from(&node.data_dirs[0]).canonicalize()?;
+    let object = disk.join(bucket).join(&key);
+    let metadata_bytes = std::fs::read(object.join("xl.meta"))?;
+    let fi = FileMeta::load(&metadata_bytes)?.into_fileinfo(bucket, &key, version, true, false, true)?;
+    let admitted_file_info = serde_json::to_value(&fi)?;
+    assert!(!fi.is_healing(), "ordinary publication must not carry the repair marker");
+    let data_dir = fi
+        .data_dir
+        .ok_or("physical-tail fixture requires external shards")?
+        .to_string();
+    let source = Uuid::new_v4().to_string();
+    let staging = disk.join(".rustfs.sys/tmp").join(&source);
+    std::fs::create_dir_all(&staging)?;
+    let destination = object.join(&data_dir);
+    // Replay the receiver publication of a real ordinary PUT while its attempt
+    // remains open. Preserve its referenced version metadata. The admitted
+    // owned operation may still finish metadata publication after RPC detach.
+    std::fs::rename(&destination, staging.join(&data_dir))?;
+    let barrier = PhysicalBarrier::arm(&disk, &destination)?;
+    let format: Value = serde_json::from_slice(&std::fs::read(disk.join(".rustfs.sys/format.json"))?)?;
+    let disk_id = Uuid::parse_str(format["xl"]["this"].as_str().ok_or("disk format omitted xl.this")?)?;
+    let incarnation = Uuid::parse_str(target["bucket_incarnation"].as_str().ok_or("target omitted incarnation")?)?;
+    let put = RenameDataRequest {
+        disk: disk_id.to_string(),
+        src_volume: ".rustfs.sys/tmp".to_owned(),
+        src_path: source,
+        dst_volume: bucket.to_owned(),
+        dst_path: key.clone(),
+        file_info: serde_json::to_string(&fi)?,
+        file_info_bin: Default::default(),
+        scanner_publication_lease_token: Default::default(),
+        bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
+    };
+    let request = signed_put_request(put.clone(), &node.address)?;
+    // Direct RPC takes no coordinator attempt lock and owns its own connection.
+    let url = node.url.clone();
+    let mut rpc = tokio::spawn(async move {
+        timeout(Duration::from_secs(30), async move {
+            let mut client = NodeServiceClient::connect(url).await?;
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>(client.artifact_rename_data(request).await?)
+        })
+        .await
+    });
+    tokio::select! {
+        result = &mut rpc => {
+            let response = result???.into_inner();
+            panic!(
+                "ordinary PUT receiver completed before the physical barrier: success={}, error={:?}",
+                response.success, response.error
+            );
+        }
+        arrived = barrier.observed("arrived") => arrived?,
+    }
+    assert!(!destination.exists(), "physical PUT escaped its barrier");
+    rpc.abort();
+    assert!(rpc.await.expect_err("disconnect the ordinary PUT waiter").is_cancelled());
+    // The ordinary PUT wrapper retains an owned task. Observe its RPC-facing
+    // waiter dropping, rather than cancellation of the continuing disk task.
+    barrier.observed("rpc-waiter-dropped").await?;
+
+    let seal = user.request(&dist, 0, PUT_SEAL_ROUTE, request_body.clone());
+    tokio::pin!(seal);
+    let closed = wait_until(
+        Duration::from_secs(15),
+        || async {
+            Ok(walkdir::WalkDir::new(disk.join(".rustfs-artifact-put-seals"))
+                .min_depth(2)
+                .max_depth(2)
+                .into_iter()
+                .filter_map(Result::ok)
+                .any(|entry| entry.file_name() == "closed"))
+        },
+        "PUT receiver persisted closing before physical drain",
+    );
+    tokio::pin!(closed);
+    tokio::select! {
+        result = &mut seal => {
+            let (status, response) = result?;
+            panic!("PUT seal returned before its blocked receiver closed admission: {status} {response}");
+        }
+        result = &mut closed => result?,
+    }
+    let (status, response) = timeout(Duration::from_secs(40), &mut seal).await??;
+    assert_eq!(
+        status,
+        StatusCode::SERVICE_UNAVAILABLE,
+        "PUT seal confirmed an unfinished physical tail: {response}"
+    );
+    assert!(response.contains("artifact operation unconfirmed"));
+    assert!(!destination.exists());
+    assert_raw_put_closed(&put, &node.url, &node.address, &[&staging, &object]).await?;
+    assert!(!destination.exists(), "a second admission published while the worker was paused");
+
+    barrier.release()?;
+    wait_until(
+        Duration::from_secs(15),
+        || async { Ok(destination.join("part.1").is_file()) },
+        "detached ordinary PUT worker finished",
+    )
+    .await?;
+    confirm_seal(&dist, &user, &request_body, &target).await?;
+    // Seal confirmation is the boundary: already admitted work may re-encode
+    // metadata before this point, but the complete FileInfo must be unchanged.
+    let published_objects: Vec<_> = dist
+        .cluster
+        .nodes
+        .iter()
+        .map(|node| PathBuf::from(&node.data_dirs[0]).join(bucket).join(&key))
+        .collect();
+    let published_paths: Vec<_> = published_objects.iter().map(PathBuf::as_path).collect();
+    let sealed_snapshot = publication_snapshot(&published_paths)?;
+    let sealed_file_info =
+        FileMeta::load(&std::fs::read(object.join("xl.meta"))?)?.into_fileinfo(bucket, &key, version, true, false, true)?;
+    assert_eq!(
+        serde_json::to_value(sealed_file_info)?,
+        admitted_file_info,
+        "admitted PUT changed FileInfo semantics"
+    );
+    for (index, baseline) in baselines.iter().enumerate() {
+        assert!(
+            census_object_version_on_disk(Path::new(&dist.cluster.nodes[index].data_dirs[0]), bucket, &key, Some(version))?
+                .matches_manifest(baseline)
+        );
+        assert_exact_version(&dist.client(index)?, bucket, &key, version, &payload).await?;
+    }
+
+    // Stage the exact request again. A source metadata sentinel makes a missing
+    // admission check observable even if the existing destination later causes
+    // ENOTEMPTY: ordinary rename rewrites source metadata before that syscall.
+    let staged_data = staging.join(&data_dir);
+    std::fs::create_dir_all(&staged_data)?;
+    for entry in std::fs::read_dir(&destination)? {
+        let entry = entry?;
+        assert!(entry.file_type()?.is_file(), "unexpected nested shard fixture");
+        std::fs::copy(entry.path(), staged_data.join(entry.file_name()))?;
+    }
+    std::fs::write(staging.join("xl.meta"), b"closed-admission-must-preserve-this-staged-metadata")?;
+    assert_raw_put_closed(&put, &node.url, &node.address, &[&staging, &object]).await?;
+    let listed = dist
+        .client(0)?
+        .list_object_versions()
+        .bucket(bucket)
+        .prefix(&key)
+        .send()
+        .await?;
+    assert_eq!(listed.versions().len(), 1, "raw PUT retries created another version");
+    assert_eq!(listed.versions()[0].version_id(), Some(version));
+    assert!(listed.delete_markers().is_empty());
+    confirm_seal(&dist, &user, &request_body, &target).await?;
+    assert!(
+        publication_snapshot(&published_paths)? == sealed_snapshot,
+        "published files changed after the first successful physical-drain confirmation"
+    );
+    Ok(())
+}

crates/e2e_test/src/distributed/artifact_repair_tail_test.rs

@@ -0,0 +1,218 @@
+// Copyright 2026 RustFS Team
+// SPDX-License-Identifier: Apache-2.0
+
+use super::artifact_seal_test::{assert_exact_version, confirm_seal, create_artifact_attempt};
+use super::harness::{DistCluster, DistLayout, TestResult, payload_for, wait_until};
+use crate::chaos::census_object_version_on_disk;
+use crate::common::init_logging;
+use crate::storage_api::artifact_rpc::{TONIC_RPC_PREFIX, gen_tonic_signature_headers, set_tonic_canonical_body_digest};
+use http::StatusCode;
+use rustfs_filemeta::FileMeta;
+use rustfs_protos::proto_gen::node_service::{
+    ArtifactRepairRenameDataRequest, RenameDataRequest, node_service_client::NodeServiceClient,
+};
+use serde_json::{Value, json};
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+use tokio::time::timeout;
+use tonic::Request;
+use uuid::Uuid;
+
+const REPAIR_SEAL_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-repairs";
+
+pub(super) struct PhysicalBarrier(PathBuf);
+
+impl PhysicalBarrier {
+    pub(super) fn arm(disk: &Path, destination: &Path) -> TestResult<Self> {
+        let path = disk.join(".rustfs.sys/e2e-physical-rename-barrier");
+        std::fs::write(&path, destination.to_str().ok_or("non-UTF8 fixture destination")?)?;
+        Ok(Self(path))
+    }
+
+    pub(super) async fn observed(&self, stage: &str) -> TestResult {
+        wait_until(Duration::from_secs(15), || async { Ok(self.0.with_extension(stage).is_file()) }, stage).await
+    }
+
+    pub(super) fn release(&self) -> TestResult {
+        std::fs::write(self.0.with_extension("release"), b"release physical worker")?;
+        Ok(())
+    }
+}
+
+impl Drop for PhysicalBarrier {
+    fn drop(&mut self) {
+        // Release even on assertion failure; the worker also has its own deadline.
+        let _ = self.release();
+        let _ = std::fs::remove_file(&self.0);
+    }
+}
+
+#[tokio::test]
+async fn four_node_artifact_repair_seal_waits_for_disconnected_physical_tail() -> TestResult {
+    crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks"));
+    init_logging();
+    // Match the other raw internode signature tests sharing the OnceLock.
+    let _ = rustfs_credentials::set_global_rpc_secret("rustfs-internode-signature-e2e-secret".to_owned());
+    let rpc_secret = rustfs_credentials::try_get_rpc_token()?;
+    let dist = DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_RPC_SECRET", &rpc_secret)]).await?;
+    let (target, user) = create_artifact_attempt(&dist).await?;
+    let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
+    let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
+    let key = format!("{prefix}detached-rename.bin");
+    let body = payload_for(&key, 2 * 1024 * 1024);
+    let output = timeout(
+        Duration::from_secs(30),
+        dist.client(1)?
+            .put_object()
+            .bucket(bucket)
+            .key(&key)
+            .body(body.clone().into())
+            .send(),
+    )
+    .await??;
+    let version = output.version_id().ok_or("managed PUT omitted version ID")?;
+    let request_body = json!({"version": 1, "target": target}).to_string();
+    confirm_seal(&dist, &user, &request_body, &target).await?;
+
+    let node = &dist.cluster.nodes[3];
+    let disk = PathBuf::from(&node.data_dirs[0]).canonicalize()?;
+    let object = disk.join(bucket).join(&key);
+    let metadata = FileMeta::load(&std::fs::read(object.join("xl.meta"))?)?;
+    let mut fi = metadata.into_fileinfo(bucket, &key, version, true, false, true)?;
+    fi.set_healing();
+    let data_dir = fi
+        .data_dir
+        .ok_or("physical-tail fixture requires external shard files")?
+        .to_string();
+    let baseline = census_object_version_on_disk(&disk, bucket, &key, Some(version))?;
+    assert!(baseline.is_complete() && !baseline.expected_part_numbers.is_empty());
+    let source = Uuid::new_v4().to_string();
+    let staging = disk.join(".rustfs.sys/tmp").join(&source);
+    std::fs::create_dir_all(&staging)?;
+    let destination = object.join(&data_dir);
+    std::fs::rename(&destination, staging.join(&data_dir))?;
+    let barrier = PhysicalBarrier::arm(&disk, &destination)?;
+    let format: Value = serde_json::from_slice(&std::fs::read(disk.join(".rustfs.sys/format.json"))?)?;
+    let disk_id = Uuid::parse_str(format["xl"]["this"].as_str().ok_or("disk format omitted xl.this")?)?;
+    let incarnation = Uuid::parse_str(target["bucket_incarnation"].as_str().ok_or("target omitted incarnation")?)?;
+    let repair = ArtifactRepairRenameDataRequest {
+        rename: Some(RenameDataRequest {
+            disk: disk_id.to_string(),
+            src_volume: ".rustfs.sys/tmp".to_owned(),
+            src_path: source,
+            dst_volume: bucket.to_owned(),
+            dst_path: key.clone(),
+            file_info: serde_json::to_string(&fi)?,
+            file_info_bin: Default::default(),
+            scanner_publication_lease_token: Default::default(),
+            bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
+        }),
+        expected_disk_id: disk_id.as_bytes().to_vec().into(),
+    };
+    let canonical = rustfs_protos::canonical_artifact_repair_rename_data_request_body(&repair)?;
+    let mut request = Request::new(repair);
+    request.set_timeout(Duration::from_secs(15));
+    set_tonic_canonical_body_digest(&mut request, &canonical)?;
+    let digest = request
+        .metadata()
+        .get("x-rustfs-content-sha256")
+        .ok_or("missing canonical digest")?
+        .to_str()?
+        .to_owned();
+    let headers = gen_tonic_signature_headers(
+        &node.address,
+        TONIC_RPC_PREFIX.trim_start_matches('/'),
+        "ArtifactRepairRenameData",
+        Some(&digest),
+    )?;
+    request.metadata_mut().as_mut().extend(headers);
+    // An independent channel and direct receiver RPC take no coordinator attempt
+    // lock. Dropping this task closes the client stream while rename is paused.
+    let url = node.url.clone();
+    let mut rpc = tokio::spawn(async move {
+        timeout(Duration::from_secs(30), async move {
+            let mut client = NodeServiceClient::connect(url).await?;
+            client
+                .artifact_repair_rename_data(request)
+                .await
+                .map_err(|error| -> Box<dyn std::error::Error + Send + Sync> { Box::new(error) })
+        })
+        .await
+    });
+    tokio::select! {
+        result = &mut rpc => {
+            let response = result???.into_inner();
+            panic!(
+                "repair receiver completed before the physical barrier: success={}, error={:?}",
+                response.success, response.error
+            );
+        }
+        arrived = barrier.observed("arrived") => arrived?,
+    }
+    assert!(!destination.exists(), "physical publication ran before its barrier");
+    rpc.abort();
+    assert!(rpc.await.expect_err("cancel the disconnected RPC waiter").is_cancelled());
+    // This acknowledgement is emitted by the receiver's dropped async waiter,
+    // not by the client and not by a delayed response.
+    barrier.observed("waiter-dropped").await?;
+
+    let seal = user.request(&dist, 0, REPAIR_SEAL_ROUTE, request_body.clone());
+    tokio::pin!(seal);
+    let closed = wait_until(
+        Duration::from_secs(15),
+        || async {
+            Ok(walkdir::WalkDir::new(disk.join(".rustfs-artifact-repair-seals"))
+                .min_depth(2)
+                .max_depth(2)
+                .into_iter()
+                .filter_map(Result::ok)
+                .any(|entry| entry.file_name() == "closed"))
+        },
+        "receiver persisted repair closure while its physical worker remains paused",
+    );
+    tokio::pin!(closed);
+    tokio::select! {
+        result = &mut seal => {
+            let (status, response) = result?;
+            panic!("repair seal returned before the blocked receiver closed admission: {status} {response}");
+        }
+        result = &mut closed => result?,
+    }
+    // Cover a complete receiver drain timeout, so early physical-owner release
+    // causes a successful response here and fails this runtime assertion.
+    let (status, response) = timeout(Duration::from_secs(40), &mut seal).await??;
+    assert_eq!(
+        status,
+        StatusCode::SERVICE_UNAVAILABLE,
+        "seal confirmed an unfinished physical tail: {response}"
+    );
+    assert!(response.contains("artifact operation unconfirmed"));
+    assert!(!destination.exists(), "worker escaped the physical barrier");
+    barrier.release()?;
+    wait_until(
+        Duration::from_secs(15),
+        || async { Ok(destination.join("part.1").is_file()) },
+        "detached physical rename completes after release",
+    )
+    .await?;
+    wait_until(
+        Duration::from_secs(45),
+        || async {
+            let (status, response) = user.request(&dist, 0, REPAIR_SEAL_ROUTE, request_body.clone()).await?;
+            if status == StatusCode::SERVICE_UNAVAILABLE && response.contains("artifact operation unconfirmed") {
+                return Ok(false);
+            }
+            assert_eq!(status, StatusCode::OK, "repair seal did not confirm after physical drain: {response}");
+            assert_eq!(
+                serde_json::from_str::<Value>(&response)?,
+                json!({"version": 1, "target": target, "attempt_repairs_sealed": true})
+            );
+            Ok(true)
+        },
+        "exact retry confirms all four receivers after physical drain",
+    )
+    .await?;
+    assert!(census_object_version_on_disk(&disk, bucket, &key, Some(version))?.matches_manifest(&baseline));
+    assert_exact_version(&dist.client(0)?, bucket, &key, version, &body).await?;
+    Ok(())
+}

crates/e2e_test/src/distributed/artifact_repair_test.rs

@@ -0,0 +1,327 @@
+// Copyright 2026 RustFS Team
+// SPDX-License-Identifier: Apache-2.0
+
+use super::artifact_seal_test::{ArtifactAdmin, assert_exact_version, confirm_seal, create_artifact_attempt};
+use super::harness::{
+    DistCluster, DistLayout, TestResult, enable_versioning, payload_for, unique_bucket, wait_for_ready, wait_until,
+};
+use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post, start_root_heal_when_control_ready};
+use crate::common::init_logging;
+use http::StatusCode;
+use serde_json::{Value, json};
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+use tokio::time::timeout;
+
+const REPAIR_SEAL_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-repairs";
+const HEAL_BODY: &str =
+    r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
+const HEAL_BUDGET: Duration = Duration::from_secs(120);
+
+pub(super) struct RepairObject {
+    pub(super) key: String,
+    pub(super) version: String,
+    pub(super) body: Vec<u8>,
+    manifests: Vec<VersionShardCensus>,
+}
+
+fn roots(dist: &DistCluster) -> Vec<PathBuf> {
+    dist.cluster
+        .nodes
+        .iter()
+        .map(|node| PathBuf::from(&node.data_dirs[0]))
+        .collect()
+}
+
+impl RepairObject {
+    pub(super) async fn put(dist: &DistCluster, bucket: &str, key: String) -> TestResult<Self> {
+        let body = payload_for(&key, 2 * 1024 * 1024);
+        let output = timeout(
+            Duration::from_secs(30),
+            dist.client(1)?
+                .put_object()
+                .bucket(bucket)
+                .key(&key)
+                .body(body.clone().into())
+                .send(),
+        )
+        .await??;
+        let version = output.version_id().ok_or("managed PUT omitted version ID")?.to_owned();
+        let manifests = roots(dist)
+            .iter()
+            .map(|disk| census_object_version_on_disk(disk, bucket, &key, Some(&version)))
+            .collect::<TestResult<Vec<_>>>()?;
+        for manifest in &manifests {
+            assert!(manifest.is_complete(), "repair needs a complete baseline: {manifest:?}");
+            assert!(!manifest.expected_part_numbers.is_empty(), "repair must use actual shard files");
+        }
+        Ok(Self {
+            key,
+            version,
+            body,
+            manifests,
+        })
+    }
+
+    pub(super) fn part(&self, disk: &Path, bucket: &str, node: usize) -> TestResult<PathBuf> {
+        let data_dir = self.manifests[node]
+            .data_dir
+            .as_ref()
+            .ok_or("external shard omitted data directory")?;
+        Ok(disk.join(bucket).join(&self.key).join(data_dir).join("part.1"))
+    }
+
+    pub(super) async fn assert_repaired(&self, dist: &DistCluster, bucket: &str) -> TestResult {
+        wait_until(
+            HEAL_BUDGET,
+            || async {
+                for (disk, baseline) in roots(dist).iter().zip(&self.manifests) {
+                    if !census_object_version_on_disk(disk, bucket, &self.key, Some(&self.version))?.matches_manifest(baseline) {
+                        return Ok(false);
+                    }
+                }
+                Ok(true)
+            },
+            "every configured receiver restored the exact version and original shard fingerprints",
+        )
+        .await?;
+        for node in 0..dist.cluster.nodes.len() {
+            assert_exact_version(&dist.client(node)?, bucket, &self.key, &self.version, &self.body).await?;
+        }
+        Ok(())
+    }
+}
+
+pub(super) async fn start_heal(dist: &DistCluster, bucket: &str, prefix: &str, remove: bool) -> TestResult<String> {
+    let url = format!("{}/rustfs/admin/v3/heal/{bucket}/{prefix}?forceStart=true", dist.cluster.nodes[0].url);
+    let body = if remove {
+        HEAL_BODY.replace("\"remove\":false", "\"remove\":true")
+    } else {
+        HEAL_BODY.to_owned()
+    };
+    let response = timeout(
+        HEAL_BUDGET,
+        start_root_heal_when_control_ready(&url, &body, &dist.cluster.access_key, &dist.cluster.secret_key),
+    )
+    .await??;
+    let response: Value = serde_json::from_str(&response)?;
+    Ok(response["clientToken"]
+        .as_str()
+        .ok_or("heal omitted its task token")?
+        .to_owned())
+}
+
+pub(super) async fn finish_heal(dist: &DistCluster, token: &str, rejected_key: Option<&str>) -> TestResult {
+    let url = format!("{}/rustfs/admin/v3/heal/?clientToken={token}", dist.cluster.nodes[0].url);
+    wait_until(
+        HEAL_BUDGET,
+        || async {
+            let response = signed_admin_post(&url, None, &dist.cluster.access_key, &dist.cluster.secret_key).await?;
+            let status: Value = serde_json::from_str(&response)?;
+            if status["summary"] == "running" {
+                return Ok(false);
+            }
+            if let Some(key) = rejected_key {
+                assert_eq!(status["summary"], "stopped", "closed repair must report failure: {status}");
+                assert!(
+                    status["outcome"]["counters"]["failed"]
+                        .as_u64()
+                        .is_some_and(|count| count > 0)
+                );
+                assert!(
+                    status["outcome"]["objects"].as_array().is_some_and(|objects| objects
+                        .iter()
+                        .any(|object| { object["identity"]["object"] == key && object["disposition"]["state"] == "failed" })),
+                    "heal must actually attempt the rejected object: {status}"
+                );
+            } else {
+                assert_eq!(status["summary"], "finished", "heal did not complete: {status}");
+            }
+            Ok(true)
+        },
+        "exact admin heal task reached its expected terminal outcome",
+    )
+    .await
+}
+
+async fn repair_seal(dist: &DistCluster, user: &ArtifactAdmin, target: &Value, confirmed: bool) -> TestResult {
+    wait_until(
+        Duration::from_secs(45),
+        || async {
+            let request = json!({"version": 1, "target": target}).to_string();
+            let (status, body) = user.request(dist, 0, REPAIR_SEAL_ROUTE, request).await?;
+            if !confirmed {
+                assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "offline receiver repair seal: {body}");
+                assert!(body.contains("<Code>ServiceUnavailable</Code>"));
+                assert!(body.contains("artifact operation unconfirmed"));
+                return Ok(true);
+            }
+            if status == StatusCode::SERVICE_UNAVAILABLE
+                && body.contains("<Code>ServiceUnavailable</Code>")
+                && body.contains("<Message>artifact operation unconfirmed; retry the same identity</Message>")
+            {
+                return Ok(false);
+            }
+            assert_eq!(status, StatusCode::OK, "repair seal failed: {body}");
+            assert_eq!(
+                serde_json::from_str::<Value>(&body)?,
+                json!({"version": 1, "target": target, "attempt_repairs_sealed": true})
+            );
+            Ok(true)
+        },
+        "repair seal confirms every configured receiver",
+    )
+    .await
+}
+
+#[tokio::test]
+async fn four_node_artifact_shard_and_minority_metadata_repair_after_put_seal() -> TestResult {
+    init_logging();
+    // Sub-quorum metadata recovery supports genuine legacy shards only.
+    let mut dist =
+        DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_SHARD_INTEGRITY_WRITE", "false")]).await?;
+    let (target, user) = create_artifact_attempt(&dist).await?;
+    let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
+    let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
+    let shard = RepairObject::put(&dist, bucket, format!("{prefix}shard.bin")).await?;
+    let metadata = RepairObject::put(&dist, bucket, format!("{prefix}metadata.bin")).await?;
+    let later = dist
+        .client(2)?
+        .put_object()
+        .bucket(bucket)
+        .key(&shard.key)
+        .body(b"newer-version".to_vec().into())
+        .send()
+        .await?;
+    let later_version = later.version_id().ok_or("second PUT omitted version ID")?.to_owned();
+    assert_ne!(shard.version, later_version);
+    confirm_seal(&dist, &user, &json!({"version": 1, "target": target}).to_string(), &target).await?;
+
+    dist.cluster.stop();
+    let disks = roots(&dist);
+    std::fs::remove_file(shard.part(&disks[3], bucket, 3)?)?;
+    for disk in &disks[1..] {
+        std::fs::remove_file(disk.join(bucket).join(&metadata.key).join("xl.meta"))?;
+    }
+    assert!(!census_object_version_on_disk(&disks[3], bucket, &shard.key, Some(&shard.version))?.is_complete());
+    assert!(disks[0].join(bucket).join(&metadata.key).join("xl.meta").is_file());
+    assert!(
+        disks[1..]
+            .iter()
+            .all(|disk| !disk.join(bucket).join(&metadata.key).join("xl.meta").exists())
+    );
+    dist.cluster.start().await?;
+    wait_for_ready(&dist.cluster).await?;
+    let token = start_heal(&dist, bucket, prefix, false).await?;
+    shard.assert_repaired(&dist, bucket).await?;
+    metadata.assert_repaired(&dist, bucket).await?;
+    finish_heal(&dist, &token, None).await?;
+    assert_exact_version(&dist.client(3)?, bucket, &shard.key, &later_version, b"newer-version").await?;
+    let listed = dist
+        .client(0)?
+        .list_object_versions()
+        .bucket(bucket)
+        .prefix(prefix)
+        .send()
+        .await?;
+    assert_eq!(listed.versions().len(), 3, "repair must not publish replacement versions");
+    assert!(listed.delete_markers().is_empty());
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_artifact_repair_seal_denies_shard_and_metadata_repair_after_restart() -> TestResult {
+    init_logging();
+    let mut dist =
+        DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_SHARD_INTEGRITY_WRITE", "false")]).await?;
+    let (target, user) = create_artifact_attempt(&dist).await?;
+    let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
+    let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
+    let shard = RepairObject::put(&dist, bucket, format!("{prefix}shard.bin")).await?;
+    let metadata = RepairObject::put(&dist, bucket, format!("{prefix}metadata.bin")).await?;
+    confirm_seal(&dist, &user, &json!({"version": 1, "target": target}).to_string(), &target).await?;
+    repair_seal(&dist, &user, &target, true).await?;
+    let ordinary_bucket = unique_bucket("repair-control");
+    dist.create_bucket(&ordinary_bucket).await?;
+    enable_versioning(&dist.client(0)?, &ordinary_bucket).await?;
+    let ordinary = RepairObject::put(&dist, &ordinary_bucket, "ordinary.bin".to_owned()).await?;
+    let disks = roots(&dist);
+    let part = shard.part(&disks[3], bucket, 3)?;
+    std::fs::remove_file(&part)?;
+    for disk in &disks[1..] {
+        std::fs::remove_file(disk.join(bucket).join(&metadata.key).join("xl.meta"))?;
+    }
+    for restarted in [false, true] {
+        if restarted {
+            dist.restart_with_current_binary().await?;
+        }
+        // The same receivers must still execute an ordinary bucket repair.
+        std::fs::remove_file(ordinary.part(&disks[3], &ordinary_bucket, 3)?)?;
+        let control = start_heal(&dist, &ordinary_bucket, &ordinary.key, false).await?;
+        ordinary.assert_repaired(&dist, &ordinary_bucket).await?;
+        finish_heal(&dist, &control, None).await?;
+        for object in [&shard, &metadata] {
+            let token = start_heal(&dist, bucket, &object.key, false).await?;
+            finish_heal(&dist, &token, Some(&object.key)).await?;
+        }
+        assert!(!part.exists(), "sealed repair recreated the missing shard");
+        assert!(
+            disks[1..]
+                .iter()
+                .all(|disk| !disk.join(bucket).join(&metadata.key).join("xl.meta").exists())
+        );
+        assert_exact_version(&dist.client(2)?, bucket, &shard.key, &shard.version, &shard.body).await?;
+    }
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_artifact_repair_seal_requires_every_receiver_on_exact_retry() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
+    let (target, user) = create_artifact_attempt(&dist).await?;
+    let request = json!({"version": 1, "target": target}).to_string();
+    confirm_seal(&dist, &user, &request, &target).await?;
+    // Repeat after global confirmation: its persisted receipt cannot replace a
+    // fresh acknowledgement from an offline configured receiver.
+    for _ in 0..2 {
+        dist.cluster.stop_node(3)?;
+        repair_seal(&dist, &user, &target, false).await?;
+        dist.cluster.start_node(3).await?;
+        wait_for_ready(&dist.cluster).await?;
+        repair_seal(&dist, &user, &target, true).await?;
+    }
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_artifact_dangling_repair_preserves_surviving_metadata() -> TestResult {
+    init_logging();
+    let mut dist =
+        DistCluster::start_with_env(DistLayout::FourNodeFourDisk, &[("RUSTFS_SHARD_INTEGRITY_WRITE", "false")]).await?;
+    let (target, user) = create_artifact_attempt(&dist).await?;
+    let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
+    let prefix = target["attempt_prefix"].as_str().ok_or("target omitted prefix")?;
+    let object = RepairObject::put(&dist, bucket, format!("{prefix}dangling.bin")).await?;
+    confirm_seal(&dist, &user, &json!({"version": 1, "target": target}).to_string(), &target).await?;
+    dist.cluster.stop();
+    let disks = roots(&dist);
+    let survivor = disks[0].join(bucket).join(&object.key).join("xl.meta");
+    let before = std::fs::read(&survivor)?;
+    for (node, disk) in disks.iter().enumerate() {
+        std::fs::remove_file(object.part(disk, bucket, node)?)?;
+        if node != 0 {
+            std::fs::remove_file(disk.join(bucket).join(&object.key).join("xl.meta"))?;
+        }
+    }
+    dist.cluster.start().await?;
+    wait_for_ready(&dist.cluster).await?;
+    let token = start_heal(&dist, bucket, &object.key, true).await?;
+    finish_heal(&dist, &token, Some(&object.key)).await?;
+    assert_eq!(
+        std::fs::read(survivor)?,
+        before,
+        "distributed repair must not authorize managed dangling deletion"
+    );
+    Ok(())
+}

crates/e2e_test/src/distributed/artifact_seal_test.rs

@@ -0,0 +1,347 @@
+// Copyright 2026 RustFS Team
+// SPDX-License-Identifier: Apache-2.0
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_object_bytes, put_object, s3_probe_client, unique_bucket, wait_for_ready,
+    wait_until,
+};
+use crate::common::{
+    AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, admin_request, init_logging,
+};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use http::{Method, StatusCode};
+use serde_json::{Value, json};
+use std::time::Duration;
+use tokio::time::timeout;
+use uuid::Uuid;
+
+const BUCKET_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/buckets";
+const INSPECT_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/buckets/inspect";
+const SEAL_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-attempt";
+const ADMIN_BUDGET: Duration = Duration::from_secs(45);
+
+pub(super) struct ArtifactAdmin {
+    access_key: String,
+    secret_key: String,
+}
+
+impl ArtifactAdmin {
+    pub(super) async fn request(
+        &self,
+        dist: &DistCluster,
+        node: usize,
+        route: &str,
+        body: String,
+    ) -> TestResult<(StatusCode, String)> {
+        timeout(
+            ADMIN_BUDGET,
+            admin_request(
+                &dist.cluster.nodes[node].url,
+                Method::POST,
+                route,
+                Some(body),
+                &self.access_key,
+                &self.secret_key,
+            ),
+        )
+        .await?
+    }
+}
+
+async fn create_artifact_admin(dist: &DistCluster, bucket: &str, prefix: &str) -> TestResult<ArtifactAdmin> {
+    let user = ArtifactAdmin {
+        access_key: format!("artifact{}", &Uuid::new_v4().simple().to_string()[..12]),
+        secret_key: Uuid::new_v4().simple().to_string(),
+    };
+    let policy_name = format!("artifact{}", &Uuid::new_v4().simple().to_string()[..12]);
+    let policy = json!({
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Action": ["admin:CreatePluginArtifactBucket", "admin:GetPluginArtifactBucket"],
+                "Resource": format!("arn:aws:s3:::{bucket}"),
+            },
+            {
+                "Effect": "Allow",
+                "Action": ["admin:SealPluginArtifactAttempt", "admin:SealPluginArtifactRepairs", "admin:SealPluginArtifactWrites"],
+                "Resource": format!("arn:aws:s3:::{bucket}/{}", prefix.trim_end_matches('/')),
+            },
+        ],
+    })
+    .to_string();
+    let url = &dist.cluster.nodes[0].url;
+    let access = &dist.cluster.access_key;
+    let secret = &dist.cluster.secret_key;
+    admin_create_user_via(AdminTransport::Signed, url, access, secret, &user.access_key, &user.secret_key).await?;
+    admin_add_canned_policy_via(AdminTransport::Signed, url, access, secret, &policy_name, &policy).await?;
+    admin_attach_user_policy_via(AdminTransport::Signed, url, access, secret, &policy_name, &user.access_key).await?;
+    Ok(user)
+}
+
+pub(super) async fn create_artifact_attempt(dist: &DistCluster) -> TestResult<(Value, ArtifactAdmin)> {
+    let bucket = unique_bucket("artifact");
+    let prefix = format!("outputs/6169/{}/{}/", Uuid::new_v4(), Uuid::new_v4());
+    let user = create_artifact_admin(dist, &bucket, &prefix).await?;
+    let body = json!({"version": 1, "bucket": bucket, "base_prefix": "outputs/"}).to_string();
+    let (status, response) = user.request(dist, 0, BUCKET_ROUTE, body).await?;
+    assert_eq!(status, StatusCode::OK, "artifact bucket creation failed: {response}");
+    let receipt: Value = serde_json::from_str(&response)?;
+    assert_eq!(receipt["version"], 1);
+    assert_eq!(receipt["bucket"], bucket);
+    assert_eq!(receipt["base_prefix"], "outputs/");
+    assert_eq!(receipt["versioning_enabled"], true);
+    let incarnation = receipt["bucket_incarnation"]
+        .as_str()
+        .ok_or("artifact receipt omitted incarnation")?;
+    assert!(!Uuid::parse_str(incarnation)?.is_nil());
+    let target = json!({
+        "bucket": bucket,
+        "bucket_incarnation": incarnation,
+        "attempt_prefix": prefix,
+    });
+    assert_bucket_identity(dist, &user, 1, &target).await?;
+    Ok((target, user))
+}
+
+async fn assert_bucket_identity(dist: &DistCluster, user: &ArtifactAdmin, node: usize, target: &Value) -> TestResult {
+    let body = json!({"version": 1, "bucket": target["bucket"], "base_prefix": "outputs/"}).to_string();
+    wait_until(
+        ADMIN_BUDGET,
+        || async {
+            let (status, response) = user.request(dist, node, INSPECT_ROUTE, body.clone()).await?;
+            // IAM peers may not have received the new user and policy yet.
+            // Storage failures must fail immediately, including the old gate's 503.
+            if matches!(status, StatusCode::FORBIDDEN | StatusCode::UNAUTHORIZED)
+                && (response.contains("<Code>AccessDenied</Code>") || response.contains("<Code>InvalidAccessKeyId</Code>"))
+            {
+                return Ok(false);
+            }
+            assert_eq!(status, StatusCode::OK, "cross-node artifact inspect failed: {response}");
+            assert_eq!(
+                serde_json::from_str::<Value>(&response)?,
+                json!({
+                    "version": 1,
+                    "bucket": target["bucket"],
+                    "base_prefix": "outputs/",
+                    "bucket_incarnation": target["bucket_incarnation"],
+                    "versioning_enabled": true,
+                }),
+                "inspect must return the created managed bucket identity"
+            );
+            Ok(true)
+        },
+        "artifact IAM user and policy propagation",
+    )
+    .await
+}
+
+pub(super) async fn confirm_seal(dist: &DistCluster, user: &ArtifactAdmin, request: &str, target: &Value) -> TestResult {
+    wait_until(
+        ADMIN_BUDGET,
+        || async {
+            let (status, response) = user.request(dist, 0, SEAL_ROUTE, request.to_owned()).await?;
+            // HTTP readiness does not confirm that remote disk handles have
+            // recovered. Retry only the API's explicit same-identity response.
+            if status == StatusCode::SERVICE_UNAVAILABLE
+                && response.contains("<Code>ServiceUnavailable</Code>")
+                && response.contains("<Message>artifact operation unconfirmed; retry the same identity</Message>")
+            {
+                return Ok(false);
+            }
+            assert_eq!(status, StatusCode::OK, "artifact seal failed: {response}");
+            let receipt: Value = serde_json::from_str(&response)?;
+            assert_eq!(receipt, json!({"version": 1, "target": target, "ordinary_put_sealed": true}));
+            Ok(true)
+        },
+        "exact artifact seal confirmation after receiver recovery",
+    )
+    .await
+}
+
+async fn assert_seal_unconfirmed(dist: &DistCluster, user: &ArtifactAdmin, request: &str) -> TestResult {
+    let (status, body) = user.request(dist, 0, SEAL_ROUTE, request.to_owned()).await?;
+    assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "offline receiver seal response: {body}");
+    assert!(
+        body.contains("<Code>ServiceUnavailable</Code>"),
+        "seal failure lost its service code: {body}"
+    );
+    assert!(
+        body.contains("artifact operation unconfirmed"),
+        "seal failure lost its retry contract: {body}"
+    );
+    Ok(())
+}
+
+pub(super) async fn assert_exact_version(client: &Client, bucket: &str, key: &str, version: &str, body: &[u8]) -> TestResult {
+    let output = timeout(
+        Duration::from_secs(30),
+        client.get_object().bucket(bucket).key(key).version_id(version).send(),
+    )
+    .await??;
+    assert_eq!(output.version_id(), Some(version));
+    let actual = timeout(Duration::from_secs(30), output.body.collect()).await??.into_bytes();
+    assert_eq!(actual.as_ref(), body);
+    Ok(())
+}
+
+async fn assert_put_denied(client: &Client, ordinary_bucket: &str, bucket: &str, key: &str) -> TestResult {
+    // The same receiver must still accept a normal write before its sealed probe.
+    put_object(client, ordinary_bucket, key, b"ordinary-control".to_vec()).await?;
+    assert_object_bytes(client, ordinary_bucket, key, b"ordinary-control").await?;
+    let client = s3_probe_client(client);
+    let result = timeout(
+        Duration::from_secs(30),
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(b"must-not-publish".to_vec().into())
+            .send(),
+    )
+    .await?;
+    let error = result.expect_err("sealed artifact attempt accepted a new PUT");
+    let service = error
+        .as_service_error()
+        .ok_or("sealed PUT failed without a service rejection")?;
+    assert_eq!(
+        error.raw_response().map(|response| response.status().as_u16()),
+        Some(500),
+        "sealed PUT must return its current HTTP status"
+    );
+    assert_eq!(service.code(), Some("InternalError"), "sealed PUT must return its current service code");
+    // StorageError::Io is redacted at the API boundary; the sealed reason is
+    // not exposed. Success controls and exact inventory checks prove the effect.
+    assert_eq!(
+        service.message(),
+        Some("We encountered an internal error, please try again."),
+        "sealed PUT must retain the public redacted message"
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_artifact_put_seal_survives_full_restart() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
+    let (target, user) = create_artifact_attempt(&dist).await?;
+    let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
+    let prefix = target["attempt_prefix"].as_str().ok_or("target omitted attempt prefix")?;
+    let key = format!("{prefix}result.bin");
+    let mut versions = Vec::new();
+    for (node, body) in [(1, b"first-version".as_slice()), (2, b"second-version".as_slice())] {
+        let client = dist.client(node)?;
+        let output = timeout(
+            Duration::from_secs(30),
+            client.put_object().bucket(bucket).key(&key).body(body.to_vec().into()).send(),
+        )
+        .await??;
+        let version = output.version_id().ok_or("artifact PUT omitted version ID")?.to_owned();
+        assert!(!Uuid::parse_str(&version)?.is_nil());
+        assert_exact_version(&dist.client(3)?, bucket, &key, &version, body).await?;
+        versions.push((version, body.to_vec()));
+    }
+    assert_ne!(versions[0].0, versions[1].0);
+    let ordinary = unique_bucket("ordinary");
+    dist.create_bucket(&ordinary).await?;
+    put_object(&dist.client(2)?, &ordinary, "normal.bin", b"before-seal".to_vec()).await?;
+    assert_object_bytes(&dist.client(3)?, &ordinary, "normal.bin", b"before-seal").await?;
+
+    let request = json!({"version": 1, "target": target}).to_string();
+    confirm_seal(&dist, &user, &request, &target).await?;
+    let other_prefix = format!("outputs/6169/{}/{}/", Uuid::new_v4(), Uuid::new_v4());
+    for phase in ["sealed", "restarted"] {
+        if phase == "restarted" {
+            dist.cluster.stop();
+            dist.cluster.start().await?;
+            wait_for_ready(&dist.cluster).await?;
+        }
+        assert_bucket_identity(&dist, &user, 2, &target).await?;
+        for node in 0..dist.cluster.nodes.len() {
+            // A second attempt in the managed bucket must remain writable;
+            // a broken bucket-wide write path cannot stand in for sealing.
+            let client = dist.client(node)?;
+            let other_key = format!("{other_prefix}{phase}-{node}.bin");
+            let output = timeout(
+                Duration::from_secs(30),
+                client
+                    .put_object()
+                    .bucket(bucket)
+                    .key(&other_key)
+                    .body(b"other-attempt".to_vec().into())
+                    .send(),
+            )
+            .await??;
+            let version = output.version_id().ok_or("other attempt PUT omitted version ID")?;
+            assert!(!Uuid::parse_str(version)?.is_nil());
+            assert_exact_version(
+                &dist.client((node + 1) % dist.cluster.nodes.len())?,
+                bucket,
+                &other_key,
+                version,
+                b"other-attempt",
+            )
+            .await?;
+            assert_put_denied(&dist.client(node)?, &ordinary, bucket, &format!("{prefix}{phase}-{node}.bin")).await?;
+            for (version, body) in &versions {
+                assert_exact_version(&dist.client(node)?, bucket, &key, version, body).await?;
+            }
+        }
+        let listed = dist
+            .client(0)?
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix(prefix)
+            .send()
+            .await?;
+        assert_eq!(listed.versions().len(), 2, "denied PUT left an artifact version: {listed:?}");
+        assert!(listed.delete_markers().is_empty(), "artifact seal created a delete marker");
+        put_object(&dist.client(1)?, &ordinary, "normal.bin", phase.as_bytes().to_vec()).await?;
+        assert_object_bytes(&dist.client(2)?, &ordinary, "normal.bin", phase.as_bytes()).await?;
+    }
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_artifact_seal_requires_every_configured_receiver_on_exact_retry() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
+    let (target, user) = create_artifact_attempt(&dist).await?;
+    let bucket = target["bucket"].as_str().ok_or("target omitted bucket")?;
+    let prefix = target["attempt_prefix"].as_str().ok_or("target omitted attempt prefix")?;
+    let key = format!("{prefix}existing.bin");
+    let output = dist
+        .client(1)?
+        .put_object()
+        .bucket(bucket)
+        .key(&key)
+        .body(b"existing".to_vec().into())
+        .send()
+        .await?;
+    let version = output.version_id().ok_or("artifact PUT omitted version ID")?.to_owned();
+    assert_exact_version(&dist.client(3)?, bucket, &key, &version, b"existing").await?;
+    let ordinary = unique_bucket("ordinary");
+    dist.create_bucket(&ordinary).await?;
+    let request = json!({"version": 1, "target": target}).to_string();
+
+    // The second iteration starts from a fully acknowledged global seal. Its
+    // persisted record cannot replace a fresh acknowledgement from every disk.
+    for _ in 0..2 {
+        dist.cluster.stop_node(3)?;
+        assert_seal_unconfirmed(&dist, &user, &request).await?;
+        dist.cluster.start_node(3).await?;
+        wait_for_ready(&dist.cluster).await?;
+        confirm_seal(&dist, &user, &request, &target).await?;
+    }
+    assert_put_denied(&dist.client(3)?, &ordinary, bucket, &format!("{prefix}late.bin")).await?;
+    assert_exact_version(&dist.client(2)?, bucket, &key, &version, b"existing").await?;
+    let listed = timeout(
+        Duration::from_secs(30),
+        dist.client(0)?.list_object_versions().bucket(bucket).prefix(prefix).send(),
+    )
+    .await??;
+    assert_eq!(listed.versions().len(), 1, "rejected PUT added a version");
+    assert!(listed.delete_markers().is_empty(), "seal added a delete marker");
+    Ok(())
+}

crates/e2e_test/src/distributed/artifact_writes_test.rs

@@ -0,0 +1,235 @@
+// Copyright 2026 RustFS Team
+// SPDX-License-Identifier: Apache-2.0
+
+#![cfg(unix)]
+
+use super::artifact_delete_test::physical_snapshot;
+use super::artifact_repair_test::{RepairObject, finish_heal, start_heal};
+use super::artifact_seal_test::{ArtifactAdmin, assert_exact_version, confirm_seal, create_artifact_attempt};
+use super::harness::{
+    DistCluster, DistLayout, TestResult, enable_versioning, s3_probe_client, unique_bucket, wait_for_ready, wait_until,
+};
+use crate::common::init_logging;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::types::{
+    ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention,
+    ObjectLockRetentionMode,
+};
+use http::StatusCode;
+use serde_json::{Value, json};
+use std::path::Path;
+use std::time::Duration;
+
+const WRITES_ROUTE: &str = "/rustfs/admin/v3/plugin-artifacts/seal-writes";
+const INTERNAL_MESSAGE: &str = "We encountered an internal error, please try again.";
+
+async fn seal_writes(dist: &DistCluster, user: &ArtifactAdmin, target: &Value, confirmed: bool) -> TestResult {
+    wait_until(Duration::from_secs(45), || async {
+        let (status, response) = user.request(dist, 0, WRITES_ROUTE, json!({"version":1,"target":target}).to_string()).await?;
+        if status == StatusCode::SERVICE_UNAVAILABLE && response.contains("artifact operation unconfirmed; retry the same identity") {
+            return Ok(!confirmed);
+        }
+        assert!(confirmed, "offline configured receiver produced a complete receipt: {status} {response}");
+        assert_eq!(status, StatusCode::OK, "combined writes seal: {response}");
+        assert_eq!(serde_json::from_str::<Value>(&response)?, json!({"version":1,"target":target,"ordinary_put_sealed":true,"attempt_repairs_sealed":true,"all_writes_sealed":true}));
+        Ok(true)
+    }, "combined seal acknowledges both domains on every receiver").await
+}
+
+async fn assert_put_closed(dist: &DistCluster, bucket: &str, key: &str) -> TestResult {
+    let before = physical_snapshot(dist, bucket)?;
+    let client = s3_probe_client(&dist.client(2)?);
+    let error = client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(b"must-not-publish".to_vec().into())
+        .send()
+        .await
+        .expect_err("combined seal admitted a new ordinary PUT");
+    let service = error.as_service_error().ok_or("PUT was not a service rejection")?;
+    assert_eq!(service.code(), Some("InternalError"));
+    assert_eq!(service.message(), Some(INTERNAL_MESSAGE));
+    assert!(
+        physical_snapshot(dist, bucket)? == before,
+        "rejected PUT changed the managed physical namespace"
+    );
+    Ok(())
+}
+
+async fn lock_metadata_entrypoints(dist: &DistCluster, managed: &Value, object: &RepairObject, ordinary: &str) -> TestResult {
+    let bucket = managed["bucket"].as_str().ok_or("bucket identity")?;
+    let client = s3_probe_client(&dist.client(0)?);
+    for bucket in [bucket, ordinary] {
+        client
+            .put_object_lock_configuration()
+            .bucket(bucket)
+            .object_lock_configuration(
+                ObjectLockConfiguration::builder()
+                    .object_lock_enabled(ObjectLockEnabled::Enabled)
+                    .build(),
+            )
+            .send()
+            .await?;
+        wait_until(
+            Duration::from_secs(30),
+            || async {
+                for node in 0..4 {
+                    let output = match dist.client(node)?.get_object_lock_configuration().bucket(bucket).send().await {
+                        Ok(output) => output,
+                        Err(error)
+                            if error.as_service_error().and_then(ProvideErrorMetadata::code)
+                                == Some("ObjectLockConfigurationNotFoundError") =>
+                        {
+                            return Ok(false);
+                        }
+                        Err(error) => return Err(error.into()),
+                    };
+                    if output
+                        .object_lock_configuration()
+                        .and_then(|configuration| configuration.object_lock_enabled())
+                        != Some(&ObjectLockEnabled::Enabled)
+                    {
+                        return Ok(false);
+                    }
+                }
+                Ok(true)
+            },
+            "actual Object Lock configuration is enabled on all request nodes",
+        )
+        .await?;
+    }
+    let control = RepairObject::put(dist, ordinary, "lock-control.bin".to_owned()).await?;
+    let retention = ObjectLockRetention::builder()
+        .mode(ObjectLockRetentionMode::Governance)
+        .retain_until_date(aws_sdk_s3::primitives::DateTime::from_secs(chrono::Utc::now().timestamp() + 86400))
+        .build();
+    let hold = ObjectLockLegalHold::builder().status(ObjectLockLegalHoldStatus::On).build();
+    client
+        .put_object_retention()
+        .bucket(ordinary)
+        .key(&control.key)
+        .version_id(&control.version)
+        .retention(retention.clone())
+        .send()
+        .await?;
+    assert_eq!(
+        client
+            .get_object_retention()
+            .bucket(ordinary)
+            .key(&control.key)
+            .version_id(&control.version)
+            .send()
+            .await?
+            .retention()
+            .and_then(|value| value.mode()),
+        Some(&ObjectLockRetentionMode::Governance)
+    );
+    client
+        .put_object_legal_hold()
+        .bucket(ordinary)
+        .key(&control.key)
+        .version_id(&control.version)
+        .legal_hold(hold.clone())
+        .send()
+        .await?;
+    assert_eq!(
+        client
+            .get_object_legal_hold()
+            .bucket(ordinary)
+            .key(&control.key)
+            .version_id(&control.version)
+            .send()
+            .await?
+            .legal_hold()
+            .and_then(|value| value.status()),
+        Some(&ObjectLockLegalHoldStatus::On)
+    );
+    let before = physical_snapshot(dist, bucket)?;
+    let error = client
+        .put_object_retention()
+        .bucket(bucket)
+        .key(&object.key)
+        .version_id(&object.version)
+        .retention(retention)
+        .send()
+        .await
+        .expect_err("managed Retention must reach and fail its unsupported metadata write");
+    let service = error.as_service_error().ok_or("retention was not a service rejection")?;
+    assert_eq!(service.code(), Some("InternalError"));
+    assert_eq!(service.message(), Some(INTERNAL_MESSAGE));
+    assert!(physical_snapshot(dist, bucket)? == before, "retention changed managed metadata");
+    let error = client
+        .put_object_legal_hold()
+        .bucket(bucket)
+        .key(&object.key)
+        .version_id(&object.version)
+        .legal_hold(hold)
+        .send()
+        .await
+        .expect_err("managed LegalHold must reach and fail its unsupported metadata write");
+    let service = error.as_service_error().ok_or("legal hold was not a service rejection")?;
+    assert_eq!(service.code(), Some("InternalError"));
+    assert_eq!(service.message(), Some("Io error: artifact bucket does not support this write operation"));
+    assert!(physical_snapshot(dist, bucket)? == before, "legal hold changed managed metadata");
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_artifact_writes_seal_first_partial_retry_offline_and_restart() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourNodeFourDisk).await?;
+    let ordinary = unique_bucket("writes-control");
+    dist.create_bucket(&ordinary).await?;
+    enable_versioning(&dist.client(0)?, &ordinary).await?;
+    let (fresh, fresh_user) = create_artifact_attempt(&dist).await?;
+    let fresh_bucket = fresh["bucket"].as_str().ok_or("fresh bucket")?;
+    let fresh_prefix = fresh["attempt_prefix"].as_str().ok_or("fresh prefix")?;
+    let first = RepairObject::put(&dist, fresh_bucket, format!("{fresh_prefix}existing.bin")).await?;
+    lock_metadata_entrypoints(&dist, &fresh, &first, &ordinary).await?;
+    seal_writes(&dist, &fresh_user, &fresh, true).await?;
+    seal_writes(&dist, &fresh_user, &fresh, true).await?;
+    assert_put_closed(&dist, fresh_bucket, &format!("{fresh_prefix}closed.bin")).await?;
+
+    let (partial, partial_user) = create_artifact_attempt(&dist).await?;
+    let bucket = partial["bucket"].as_str().ok_or("partial bucket")?;
+    let prefix = partial["attempt_prefix"].as_str().ok_or("partial prefix")?;
+    let repaired = RepairObject::put(&dist, bucket, format!("{prefix}repair.bin")).await?;
+    // Enter the combined operation with only its ordinary domain closed.
+    confirm_seal(&dist, &partial_user, &json!({"version":1,"target":partial}).to_string(), &partial).await?;
+    let part = repaired.part(Path::new(&dist.cluster.nodes[3].data_dirs[0]), bucket, 3)?;
+    std::fs::remove_file(&part)?;
+    let token = start_heal(&dist, bucket, &repaired.key, false).await?;
+    repaired.assert_repaired(&dist, bucket).await?;
+    finish_heal(&dist, &token, None).await?;
+    for _ in 0..2 {
+        dist.cluster.stop_node(3)?;
+        seal_writes(&dist, &partial_user, &partial, false).await?;
+        dist.cluster.start_node(3).await?;
+        wait_for_ready(&dist.cluster).await?;
+        seal_writes(&dist, &partial_user, &partial, true).await?;
+    }
+    let ordinary_object = RepairObject::put(&dist, &ordinary, "repair-control.bin".to_owned()).await?;
+    for restarted in [false, true] {
+        if restarted {
+            dist.restart_with_current_binary().await?;
+        }
+        seal_writes(&dist, &partial_user, &partial, true).await?;
+        assert_put_closed(&dist, bucket, &format!("{prefix}closed-{restarted}.bin")).await?;
+        if part.exists() {
+            std::fs::remove_file(&part)?;
+        }
+        let before = physical_snapshot(&dist, bucket)?;
+        let token = start_heal(&dist, bucket, &repaired.key, false).await?;
+        finish_heal(&dist, &token, Some(&repaired.key)).await?;
+        assert!(!part.exists(), "combined seal admitted a remote repair publication");
+        assert!(physical_snapshot(&dist, bucket)? == before, "rejected repair changed metadata or shards");
+        assert_exact_version(&dist.client(2)?, bucket, &repaired.key, &repaired.version, &repaired.body).await?;
+        let ordinary_part = ordinary_object.part(Path::new(&dist.cluster.nodes[3].data_dirs[0]), &ordinary, 3)?;
+        std::fs::remove_file(ordinary_part)?;
+        let token = start_heal(&dist, &ordinary, &ordinary_object.key, false).await?;
+        ordinary_object.assert_repaired(&dist, &ordinary).await?;
+        finish_heal(&dist, &token, None).await?;
+    }
+    Ok(())
+}

crates/e2e_test/src/distributed/harness.rs

@@ -34,7 +34,7 @@ use crate::common::{
 };
 use crate::replication_extension_test::LOOPBACK_REPLICATION_TARGET_ENV;
 use aws_sdk_s3::Client;
-use aws_sdk_s3::config::retry::RetryConfig;
+use aws_sdk_s3::config::{Credentials, retry::RetryConfig};
 use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
 use aws_sdk_s3::operation::{get_object::GetObjectError, put_object::PutObjectError};
 use aws_sdk_s3::primitives::ByteStream;
@@ -185,13 +185,12 @@ impl DistCluster {
         if node_idx >= self.cluster.nodes.len() {
             return Err("node_idx is invalid".into());
         }
-        Ok(Client::from_conf(build_test_s3_config(
-            &self.cluster.nodes[node_idx].url,
-            access_key,
-            secret_key,
-            None,
-            "cluster-iam-test",
-        )))
+        let config = self.cluster.create_s3_client(node_idx)?.config().to_builder();
+        Ok(Client::from_conf(
+            config
+                .credentials_provider(Credentials::new(access_key, secret_key, None, None, "cluster-iam-test"))
+                .build(),
+        ))
     }
 
     /// Stop the fleet, drop retired pools from the command line, and restart
@@ -347,7 +346,7 @@ fn relocate_pool_storage(cluster: &mut RustFSTestClusterEnvironment, node_idx: u
     Ok(pool_run_root)
 }
 
-fn configure_node_logs(cluster: &mut RustFSTestClusterEnvironment, capture_dir: Option<&Path>) -> TestResult {
+pub(super) fn configure_node_logs(cluster: &mut RustFSTestClusterEnvironment, capture_dir: Option<&Path>) -> TestResult {
     if let Some(directory) = capture_dir {
         std::fs::create_dir_all(directory)?;
         for node_idx in 0..cluster.nodes.len() {

crates/e2e_test/src/distributed/internode_tls_test.rs

@@ -0,0 +1,286 @@
+// Copyright 2026 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+use super::harness::{DistCluster, DistLayout, TestResult, unique_bucket};
+use crate::tls_gen;
+use aws_sdk_s3::{Client, config::retry::RetryConfig, primitives::ByteStream};
+use rand::{Rng, SeedableRng, rngs::StdRng};
+use sha2::{Digest, Sha256};
+use std::io::{Read, Seek, SeekFrom, Write};
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+use tokio::task::JoinSet;
+
+const MIB: u64 = 1024 * 1024;
+const READ_TIMEOUT: Duration = Duration::from_secs(180);
+
+struct ExpectedRead {
+    range: Option<(u64, u64)>,
+    length: u64,
+    sha256: String,
+}
+
+struct ObjectFixture {
+    key: String,
+    path: PathBuf,
+    length: u64,
+    reads: Vec<ExpectedRead>,
+}
+
+fn fixture(directory: &Path, length: u64) -> TestResult<ObjectFixture> {
+    let key = format!("object-{length}");
+    let path = directory.join(&key);
+    let mut file = std::fs::File::create(&path)?;
+    let mut rng = StdRng::seed_from_u64(length);
+    let mut block = vec![0; usize::try_from(MIB)?];
+    let mut remaining = length;
+    let mut digest = Sha256::new();
+    while remaining != 0 {
+        let count = usize::try_from(remaining.min(MIB))?;
+        rng.fill_bytes(&mut block[..count]);
+        file.write_all(&block[..count])?;
+        digest.update(&block[..count]);
+        remaining -= u64::try_from(count)?;
+    }
+    drop(file);
+    let mut reads = vec![ExpectedRead {
+        range: None,
+        length,
+        sha256: hex_simd::encode_to_string(digest.finalize(), hex_simd::AsciiCase::Lower),
+    }];
+    // Unaligned ranges cross TLS frames and EC blocks; the tail detects
+    // truncation and incorrect range offsets independently of a full GET.
+    for (start, end) in [(17, (MIB + 31).min(length - 1)), (length.saturating_sub(65537), length - 1)] {
+        let mut input = std::fs::File::open(&path)?;
+        input.seek(SeekFrom::Start(start))?;
+        let range_length = end - start + 1;
+        let mut hasher = Sha256::new();
+        let mut input = input.take(range_length);
+        loop {
+            let count = input.read(&mut block)?;
+            if count == 0 {
+                break;
+            }
+            hasher.update(&block[..count]);
+        }
+        reads.push(ExpectedRead {
+            range: Some((start, end)),
+            length: range_length,
+            sha256: hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower),
+        });
+    }
+    Ok(ObjectFixture {
+        key,
+        path,
+        length,
+        reads,
+    })
+}
+
+async fn verified_get(
+    client: &Client,
+    node: usize,
+    bucket: &str,
+    object: &ObjectFixture,
+    expected: &ExpectedRead,
+    concurrent: bool,
+) -> TestResult<serde_json::Value> {
+    let started = Instant::now();
+    let mut received = 0u64;
+    let mut stage = "response headers";
+    tokio::time::timeout(READ_TIMEOUT, async {
+        let mut request = client.get_object().bucket(bucket).key(&object.key);
+        if let Some((start, end)) = expected.range {
+            request = request.range(format!("bytes={start}-{end}"));
+        }
+        let mut output = request.send().await?;
+        assert_eq!(output.content_length(), Some(i64::try_from(expected.length)?));
+        if let Some((start, end)) = expected.range {
+            assert_eq!(output.content_range(), Some(format!("bytes {start}-{end}/{}", object.length).as_str()));
+        } else {
+            assert_eq!(output.content_range(), None);
+        }
+        stage = "response body";
+        let mut digest = Sha256::new();
+        while let Some(chunk) = output.body.next().await {
+            let chunk = chunk?;
+            received += u64::try_from(chunk.len())?;
+            assert!(received <= expected.length, "node {node}: GET exceeded the requested length");
+            digest.update(&chunk);
+        }
+        let actual_sha256 = hex_simd::encode_to_string(digest.finalize(), hex_simd::AsciiCase::Lower);
+        assert_eq!(received, expected.length, "node {node}: truncated GET {}", object.key);
+        assert_eq!(actual_sha256, expected.sha256, "node {node}: corrupt GET {}", object.key);
+        eprintln!(
+            "TLS acceptance: GET node={node}, key={}, range={:?}, concurrent={concurrent}, bytes={received}, elapsed={:?}",
+            object.key, expected.range, started.elapsed()
+        );
+        Ok(serde_json::json!({
+            "operation": "get", "node": node, "key": object.key,
+            "range": expected.range, "bytes": received, "sha256": actual_sha256,
+            "concurrent": concurrent, "elapsed_ms": started.elapsed().as_secs_f64() * 1000.0,
+        }))
+    })
+    .await
+    .map_err(|error| {
+        std::io::Error::new(
+            std::io::ErrorKind::TimedOut,
+            format!(
+                "TLS GET timed out: node={node}, key={}, range={:?}, concurrent={concurrent}, stage={stage}, received={received}/{}, elapsed={:?}: {error}",
+                object.key, expected.range, expected.length, started.elapsed()
+            ),
+        )
+    })?
+}
+
+/// This case always exercises the full acceptance sizes; do not substitute
+/// tiny objects and count that as a large distributed-read pass.
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn four_node_tls_large_and_range_gets_preserve_bytes() -> TestResult {
+    let mode = std::env::var("RUSTFS_E2E_INTERNODE_HTTP_MODE").unwrap_or_else(|_| "http1".into());
+    let protocol = match mode.as_str() {
+        "http1" => "http1",
+        "auto" | "unpooled" => "auto",
+        _ => return Err("RUSTFS_E2E_INTERNODE_HTTP_MODE must be http1, auto or unpooled".into()),
+    };
+    let mut dist = DistCluster::new_stopped_with_env(
+        DistLayout::FourByFour,
+        &[
+            ("RUSTFS_INTERNODE_HTTP_PROTOCOL", protocol),
+            ("RUSTFS_SCANNER_ENABLED", "true"),
+            ("RUSTFS_SCANNER_START_DELAY_SECS", "0"),
+            ("RUSTFS_SCANNER_CYCLE", "1"),
+            ("RUSTFS_HEAL_ENABLED", "true"),
+            ("RUST_LOG", "warn"),
+        ],
+    )
+    .await?;
+    if mode == "unpooled" {
+        dist.cluster.set_env("RUSTFS_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST", "0");
+    }
+    let directory = PathBuf::from(&dist.cluster.temp_dir);
+    let tls_directory = directory.join("tls");
+    tls_gen::run(tls_gen::Args {
+        out_dir: tls_directory.clone(),
+        days: 1,
+        force: false,
+    })?;
+    dist.cluster.enable_tls(&tls_directory).await?;
+    let report_path = std::env::var_os("RUSTFS_INTERNODE_ACCEPTANCE_REPORT").map(PathBuf::from);
+    if let Some(path) = &report_path
+        && (path.exists() || path.with_extension("pids.json").exists())
+    {
+        return Err("acceptance report paths must be unused".into());
+    }
+    let log_directory = report_path.as_ref().map_or_else(
+        || {
+            std::env::var_os("RUSTFS_E2E_LOG_DIR")
+                .map_or_else(|| directory.join("logs"), |root| PathBuf::from(root).join(unique_bucket("internode-tls")))
+        },
+        |path| path.with_extension("logs"),
+    );
+    std::fs::create_dir(&log_directory)?;
+    dist.capture_logs_under(&log_directory)?;
+    dist.cluster.start().await?;
+    let clients = dist
+        .clients()?
+        .into_iter()
+        .map(|client| Client::from_conf(client.config().to_builder().retry_config(RetryConfig::disabled()).build()))
+        .collect::<Vec<_>>();
+    let bucket = unique_bucket("internode-tls");
+    clients[0].create_bucket().bucket(&bucket).send().await?;
+    let pids = dist
+        .cluster
+        .nodes
+        .iter()
+        .map(|node| node.process.as_ref().map(std::process::Child::id))
+        .collect::<Vec<_>>();
+    if let Some(path) = &report_path {
+        std::fs::write(path.with_extension("pids.json"), serde_json::to_vec(&pids)?)?;
+    }
+
+    let mut results = Vec::new();
+    for length in [32 * 1024, 100 * MIB, 1024 * MIB] {
+        let fixture_directory = directory.clone();
+        let object = Arc::new(tokio::task::spawn_blocking(move || fixture(&fixture_directory, length)).await??);
+        eprintln!("TLS acceptance: fixture key={}, bytes={length}", object.key);
+        let started = Instant::now();
+        tokio::time::timeout(
+            READ_TIMEOUT,
+            clients[0]
+                .put_object()
+                .bucket(&bucket)
+                .key(&object.key)
+                .body(ByteStream::from_path(&object.path).await?)
+                .send(),
+        )
+        .await
+        .map_err(|error| {
+            std::io::Error::new(
+                std::io::ErrorKind::TimedOut,
+                format!(
+                    "TLS PUT timed out: key={}, bytes={length}, elapsed={:?}: {error}",
+                    object.key,
+                    started.elapsed()
+                ),
+            )
+        })??;
+        eprintln!("TLS acceptance: PUT key={}, bytes={length}, elapsed={:?}", object.key, started.elapsed());
+        results.push(
+            serde_json::json!({"operation": "put", "bytes": length, "elapsed_ms": started.elapsed().as_secs_f64() * 1000.0}),
+        );
+        if length == 32 * 1024 {
+            let alternate = dist.client_with_credentials(0, &dist.cluster.access_key, &dist.cluster.secret_key)?;
+            let alternate = Client::from_conf(alternate.config().to_builder().retry_config(RetryConfig::disabled()).build());
+            results.push(verified_get(&alternate, 0, &bucket, &object, &object.reads[0], false).await?);
+            let invalid = dist.client_with_credentials(0, "invalid-tls-user", "invalid-tls-secret-password")?;
+            let invalid = Client::from_conf(invalid.config().to_builder().retry_config(RetryConfig::disabled()).build());
+            let denied = invalid
+                .list_buckets()
+                .send()
+                .await
+                .expect_err("alternate credentials must not inherit the root identity");
+            assert_eq!(denied.raw_response().map(|response| response.status().as_u16()), Some(403));
+        }
+        for expected in &object.reads {
+            for (node, client) in clients.iter().enumerate() {
+                results.push(verified_get(client, node, &bucket, &object, expected, false).await?);
+            }
+        }
+        // Each reader owns a Tokio task and its cooperative budget. Polling
+        // every body and synchronous digest in one parent task can let an
+        // always-ready reader starve its siblings and their deadlines.
+        let mut readers = JoinSet::new();
+        for (node, client) in clients.iter().enumerate() {
+            let client = client.clone();
+            let bucket = bucket.clone();
+            let object = Arc::clone(&object);
+            readers.spawn(async move { verified_get(&client, node, &bucket, &object, &object.reads[0], true).await });
+        }
+        while let Some(result) = readers.join_next().await {
+            results.push(result??);
+        }
+        tokio::fs::remove_file(&object.path).await?;
+    }
+    for node in 0..clients.len() {
+        let log = tokio::fs::read_to_string(log_directory.join(format!("node-{node}.log"))).await?;
+        assert!(
+            !log.contains("internode body stalled"),
+            "node {node}: internode body stall during verified reads"
+        );
+    }
+    let report = serde_json::json!({
+        "schema": 1, "mode": mode, "tls_verified": true, "nodes": 4, "drives_per_node": 4,
+        "source_revision": env!("RUSTFS_E2E_BUILD_COMMIT"),
+        "source_dirty": env!("RUSTFS_E2E_BUILD_DIRTY"),
+        "scanner_enabled": true, "heal_enabled": true,
+        "scanner_progress_verified": false, "heal_progress_verified": false,
+        "results": results,
+    });
+    if let Some(path) = report_path {
+        std::fs::write(path, serde_json::to_vec_pretty(&report)?)?;
+    }
+    eprintln!("internode_tls_acceptance={report}");
+    Ok(())
+}

crates/e2e_test/src/distributed/mod.rs

@@ -18,6 +18,13 @@
 //! `.github/workflows/e2e-distributed.yml`. Excluded from `e2e-full` because
 //! each case starts four real `rustfs` processes.
 
+mod artifact_delete_test;
+mod artifact_inventory_test;
+mod artifact_put_tail_test;
+mod artifact_repair_tail_test;
+mod artifact_repair_test;
+mod artifact_seal_test;
+mod artifact_writes_test;
 mod chaos_test;
 mod concurrency_stability_test;
 mod concurrent_data_movement_test;
@@ -26,6 +33,8 @@ mod expand_decommission_rebalance_test;
 mod extra_test;
 mod harness;
 mod heal_test;
+mod internode_tls_test;
+mod multipart_fence_acceptance_test;
 mod object_lock_test;
 mod observability_test;
 mod replication_delete_marker_test;

crates/e2e_test/src/distributed/multipart_fence_acceptance_test.rs

@@ -0,0 +1,825 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    TestResult, assert_object_bytes, cluster_admin_ok, configure_node_logs, payload_for, retrying_put, sha256_hex, unique_bucket,
+    wait_until,
+};
+use crate::common::{ClusterTopology, RustFSTestClusterEnvironment};
+use crate::rpc_fault_proxy::{RpcFaultProxy, RpcPause};
+use aws_sdk_s3::{
+    Client,
+    config::retry::RetryConfig,
+    error::{ProvideErrorMetadata, SdkError},
+    operation::create_multipart_upload::{CreateMultipartUploadError, CreateMultipartUploadOutput},
+    primitives::ByteStream,
+    types::{CompletedMultipartUpload, CompletedPart},
+};
+use futures::{StreamExt as _, TryStreamExt as _, stream};
+use std::time::{Duration, Instant};
+use tokio::task::JoinHandle;
+
+const DEADLINE: Duration = Duration::from_secs(45);
+
+async fn fixture(
+    drives_per_node: usize,
+    storage_class: &str,
+) -> TestResult<(RustFSTestClusterEnvironment, Vec<RpcFaultProxy>, Vec<Client>, String)> {
+    let mut cluster =
+        RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(4, drives_per_node)).await?;
+    for (name, value) in [
+        ("RUSTFS_STORAGE_CLASS_STANDARD", storage_class),
+        ("RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT", "5"),
+        ("RUSTFS_OBJECT_LOCK_RPC_TIMEOUT_MS", "3000"),
+        ("RUSTFS_DRIVE_METADATA_TIMEOUT_SECS", "2"),
+        ("RUSTFS_DRIVE_MAX_TIMEOUT_DURATION", "5"),
+        // Keep disk-health ejection out of a method-specific quorum test.
+        ("RUSTFS_DRIVE_SUSPECT_FAILURE_THRESHOLD", "1000"),
+        ("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false"),
+        ("NO_PROXY", "127.0.0.1,::1,localhost"),
+        ("HTTP_PROXY", ""),
+        ("HTTPS_PROXY", ""),
+        ("RUST_LOG", "warn"),
+    ] {
+        cluster.set_env(name, value);
+    }
+    let mut proxies = Vec::new();
+    for node in 0..4 {
+        proxies.push(cluster.start_rpc_proxy_for_node(node).await?);
+    }
+    configure_node_logs(&mut cluster, None)?;
+    cluster.start().await?;
+    let clients = cluster
+        .create_all_clients()?
+        .into_iter()
+        .map(|client| {
+            Client::from_conf(
+                client
+                    .config()
+                    .to_builder()
+                    .retry_config(RetryConfig::standard().with_max_attempts(1))
+                    .build(),
+            )
+        })
+        .collect();
+    let bucket = unique_bucket("mpu-fence");
+    cluster.create_test_bucket(&bucket).await?;
+    Ok((cluster, proxies, clients, bucket))
+}
+
+fn create(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+) -> JoinHandle<Result<CreateMultipartUploadOutput, SdkError<CreateMultipartUploadError>>> {
+    let request = client.create_multipart_upload().bucket(bucket).key(key);
+    tokio::spawn(async move { request.send().await })
+}
+
+async fn wait_for_faults(pauses: &[RpcPause], methods: &[&str]) -> TestResult {
+    if methods.iter().any(|method| matches!(*method, "Lock" | "LockBatch")) {
+        // Adaptive lock fan-out may not contact an unnecessary voter. All
+        // selected peers remain gated; require an actual scoped request hit.
+        let (result, _, _) = futures::future::select_all(pauses.iter().map(|pause| Box::pin(pause.wait_for_hits(1)))).await;
+        result?;
+    } else {
+        // Storage mutations fan out eagerly. Every selected peer must really
+        // receive this upload's RPC, so an unrelated failure cannot fake quorum.
+        for pause in pauses {
+            pause.wait_for_hits(1).await?;
+        }
+    }
+    Ok(())
+}
+
+fn assert_unavailable<E: std::fmt::Debug + ProvideErrorMetadata>(error: &SdkError<E>, cause: &str, retry: Option<&str>) {
+    let response = error.raw_response().expect("an S3 service response");
+    assert_eq!(response.status().as_u16(), 503, "{error:?}");
+    assert_eq!(error.as_service_error().and_then(|e| e.meta().code()), Some("ServiceUnavailable"));
+    assert_eq!(response.headers().get("x-rustfs-error-cause"), Some(cause));
+    assert_eq!(response.headers().get("retry-after"), retry);
+    assert!(
+        response
+            .headers()
+            .get("content-type")
+            .is_some_and(|v| v.starts_with("application/xml"))
+    );
+    let request_id = response.headers().get("x-amz-request-id").expect("request correlation ID");
+    assert!(!request_id.is_empty());
+    eprintln!("MPU acceptance: cause={cause}, retry={retry:?}, request_id={request_id}");
+}
+
+async fn abort(client: &Client, bucket: &str, key: &str, upload: &str) -> TestResult {
+    client
+        .abort_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload)
+        .send()
+        .await?;
+    let missing = client
+        .list_parts()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload)
+        .send()
+        .await
+        .expect_err("an aborted upload must no longer admit ListParts");
+    assert_eq!(missing.as_service_error().and_then(|e| e.meta().code()), Some("NoSuchUpload"));
+    Ok(())
+}
+
+// Fault removal does not synchronously clear existing server leases or drive
+// health state. Keep the injected operation single-attempt, then require
+// bounded recovery of the exact cleanup API instead of assuming instant heal.
+async fn recover_after_fault<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
+where
+    E: std::fmt::Debug + ProvideErrorMetadata,
+    F: FnMut() -> Fut,
+    Fut: std::future::Future<Output = Result<T, SdkError<E>>>,
+{
+    let started = Instant::now();
+    let mut attempts = 0;
+    let mut last_failure = String::new();
+    let outcome = tokio::time::timeout(DEADLINE, async {
+        loop {
+            attempts += 1;
+            let result = call().await;
+            let retry = result.as_ref().err().and_then(|error| {
+                let response = error.raw_response()?;
+                if response.status().as_u16() != 503
+                    || error.as_service_error()?.meta().code() != Some("ServiceUnavailable")
+                    || !matches!(
+                        response.headers().get("x-rustfs-error-cause"),
+                        Some(
+                            "namespace_lock_timeout"
+                                | "namespace_lock_unavailable"
+                                | "storage_write_quorum"
+                                | "storage_read_quorum"
+                        )
+                    )
+                {
+                    return None;
+                }
+                let delay = match response.headers().get("retry-after") {
+                    Some("1") => Duration::from_secs(1),
+                    None => Duration::from_millis(100),
+                    Some(_) => return None,
+                };
+                last_failure = format!(
+                    "cause={:?}, request_id={:?}",
+                    response.headers().get("x-rustfs-error-cause"),
+                    response.headers().get("x-amz-request-id")
+                );
+                Some(delay)
+            });
+            match retry {
+                Some(delay) => tokio::time::sleep(delay).await,
+                None => return result,
+            }
+        }
+    })
+    .await;
+    eprintln!(
+        "MPU recovery: operation={operation}, attempts={attempts}, elapsed={:?}, last={last_failure}",
+        started.elapsed()
+    );
+    outcome.map_err(|_| format!("{operation} did not recover within {DEADLINE:?}: {last_failure}").into())
+}
+
+async fn abort_after_fault(client: &Client, bucket: &str, key: &str, upload: &str, allow_missing: bool) -> TestResult {
+    match recover_after_fault(&format!("Abort {key}"), || {
+        client
+            .abort_multipart_upload()
+            .bucket(bucket)
+            .key(key)
+            .upload_id(upload)
+            .send()
+    })
+    .await?
+    {
+        Ok(_) => {}
+        Err(error) if allow_missing && error.as_service_error().and_then(|e| e.meta().code()) == Some("NoSuchUpload") => {}
+        Err(error) => return Err(error.into()),
+    }
+    let missing = recover_after_fault(&format!("ListParts {key}"), || {
+        client.list_parts().bucket(bucket).key(key).upload_id(upload).send()
+    })
+    .await?
+    .expect_err("a recovered Abort must remove the upload");
+    assert_eq!(missing.as_service_error().and_then(|e| e.meta().code()), Some("NoSuchUpload"));
+    Ok(())
+}
+
+async fn round_trip(clients: &[Client], bucket: &str, key: &str, size: usize) -> TestResult {
+    let upload = clients[0].create_multipart_upload().bucket(bucket).key(key).send().await?;
+    let upload_id = upload.upload_id().ok_or("missing upload ID")?;
+    let body = payload_for(key, size);
+    let mut parts: Vec<_> = stream::iter(body.chunks(8 * 1024 * 1024).enumerate())
+        .map(|(part, bytes)| async move {
+            let number = i32::try_from(part + 1)?;
+            let result = clients[part % clients.len()]
+                .upload_part()
+                .bucket(bucket)
+                .key(key)
+                .upload_id(upload_id)
+                .part_number(number)
+                .body(ByteStream::from(bytes.to_vec()))
+                .send()
+                .await?;
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>(
+                CompletedPart::builder()
+                    .part_number(number)
+                    .e_tag(result.e_tag().ok_or("missing part ETag")?)
+                    .build(),
+            )
+        })
+        .buffer_unordered(4)
+        .try_collect()
+        .await?;
+    parts.sort_by_key(|part| part.part_number());
+    clients[1]
+        .complete_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload_id)
+        .multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(parts)).build())
+        .send()
+        .await?;
+    for client in clients {
+        assert_object_bytes(client, bucket, key, &body).await?;
+    }
+    clients[0].delete_object().bucket(bucket).key(key).send().await?;
+    Ok(())
+}
+
+async fn assert_no_uploads(clients: &[Client], bucket: &str) -> TestResult {
+    for client in clients {
+        let uploads = client.list_multipart_uploads().bucket(bucket).send().await?;
+        assert!(uploads.uploads().is_empty(), "unexpected residual uploads: {:?}", uploads.uploads());
+    }
+    Ok(())
+}
+
+async fn assert_no_uploads_after_fault(clients: &[Client], bucket: &str) -> TestResult {
+    for client in clients {
+        let uploads = recover_after_fault("ListMultipartUploads after fault", || {
+            client.list_multipart_uploads().bucket(bucket).send()
+        })
+        .await??;
+        assert!(uploads.uploads().is_empty(), "unexpected residual uploads: {:?}", uploads.uploads());
+    }
+    Ok(())
+}
+
+async fn assert_object_bytes_after_fault(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
+    let output = recover_after_fault(&format!("GetObject {key} after fault"), || {
+        client.get_object().bucket(bucket).key(key).send()
+    })
+    .await??;
+    // A successful response must have a complete, correct body. Streaming
+    // failures are not retried or converted into a successful comparison.
+    let bytes = output.body.collect().await?.into_bytes();
+    if bytes.as_ref() != expected {
+        return Err(format!("object {key} bytes differ: expected {} bytes, got {}", expected.len(), bytes.len()).into());
+    }
+    Ok(())
+}
+
+// This fixture inspects RustFS RPCs, so use the server-generated upload's
+// canonical suffix and fail explicitly if its encoding changes.
+fn upload_scope(upload_id: &str) -> TestResult<String> {
+    let decoded = String::from_utf8(base64_simd::URL_SAFE_NO_PAD.decode_to_vec(upload_id.as_bytes())?)?;
+    let (_, suffix) = decoded.split_once('.').ok_or("server upload ID lacks deployment delimiter")?;
+    if suffix.is_empty() {
+        return Err("server upload ID has an empty suffix".into());
+    }
+    Ok(suffix.to_owned())
+}
+
+fn upload_path_scope(bucket: &str, key: &str, upload_id: &str) -> TestResult<String> {
+    Ok(format!(
+        "{}/{}",
+        sha256_hex(format!("{bucket}/{key}").as_bytes()),
+        upload_scope(upload_id)?
+    ))
+}
+
+async fn with_paused_peers<T: Send + 'static>(
+    proxies: &[RpcFaultProxy],
+    methods: &[&'static str],
+    limits: &[usize],
+    scope: &str,
+    request: impl std::future::Future<Output = T> + Send + 'static,
+) -> TestResult<T> {
+    let pauses: Vec<_> = proxies[1..=limits.len()]
+        .iter()
+        .zip(limits)
+        .map(|(proxy, limit)| proxy.pause(methods, *limit, Some(scope)))
+        .collect::<Result<_, _>>()?;
+    let task = tokio::spawn(request);
+    wait_for_faults(&pauses, methods).await?;
+    let result = tokio::time::timeout(DEADLINE, task).await??;
+    for pause in &pauses {
+        pause.discard_pending();
+    }
+    drop(pauses);
+    Ok(result)
+}
+
+fn quorum_outcome<T: std::fmt::Debug, E: std::fmt::Debug + ProvideErrorMetadata>(
+    result: Result<T, SdkError<E>>,
+    at_quorum: bool,
+    locks: bool,
+) -> TestResult<Option<T>> {
+    if at_quorum {
+        Ok(Some(result.map_err(|error| format!("exactly quorum must succeed: {error:?}"))?))
+    } else {
+        assert_unavailable(
+            &result.expect_err("quorum-minus-one must fail"),
+            if locks {
+                "namespace_lock_timeout"
+            } else {
+                "storage_write_quorum"
+            },
+            if locks { Some("1") } else { None },
+        );
+        Ok(None)
+    }
+}
+
+async fn wait_for_lock_peer_recovery(proxies: &[RpcFaultProxy], client: &Client, bucket: &str) -> TestResult {
+    let key = unique_bucket("peer-recovery");
+    let observers = proxies[1..]
+        .iter()
+        .map(|proxy| proxy.pause(&["Lock", "LockBatch"], usize::MAX, Some(&key)))
+        .collect::<Result<Vec<_>, _>>()?;
+    for observer in &observers {
+        observer.release();
+    }
+    wait_until(
+        DEADLINE,
+        || async {
+            retrying_put(client, bucket, &key, vec![0x42; 1024], DEADLINE).await?;
+            // Quorum success alone can leave one peer's request breaker open.
+            // Two replies require a half-open peer to finish its first probe
+            // before admitting another RPC; every peer must rejoin the matrix.
+            Ok(observers.iter().all(|observer| observer.forwarded() >= 2))
+        },
+        "all lock peers recover after fault retirement",
+    )
+    .await
+}
+
+async fn wait_for_storage_peer_recovery(
+    proxies: &[RpcFaultProxy],
+    client: &Client,
+    bucket: &str,
+    upload: &str,
+    drives_per_node: usize,
+) -> TestResult {
+    let scope = upload_scope(upload)?;
+    tokio::time::timeout(
+        DEADLINE,
+        wait_until(
+            DEADLINE,
+            || async {
+                let observers = proxies[1..]
+                    .iter()
+                    .map(|proxy| proxy.pause(&["ReadVersion"], usize::MAX, Some(&scope)))
+                    .collect::<Result<Vec<_>, _>>()?;
+                for observer in &observers {
+                    observer.release();
+                }
+                // A fresh absent key forces a full metadata read without mutating
+                // this upload. Count distinct configured disk routes, not retries
+                // from the same healthy disk or just lock-RPC reachability.
+                let key = format!("storage-ready-{}", uuid::Uuid::new_v4());
+                let _ = client.list_parts().bucket(bucket).key(key).upload_id(upload).send().await;
+                Ok(observers
+                    .iter()
+                    .all(|observer| observer.forwarded_routes() >= drives_per_node))
+            },
+            "all storage disk routes recover after fault retirement",
+        ),
+    )
+    .await
+    .map_err(|_| "storage route recovery deadline exceeded".to_string())?
+}
+
+async fn phase_quorum_matrix(
+    proxies: &[RpcFaultProxy],
+    clients: &[Client],
+    bucket: &str,
+    locks: bool,
+    drives_per_node: usize,
+) -> TestResult {
+    for phase in ["create", "part", "complete", "abort"] {
+        // Ordinary Create holds only the shared bucket lifecycle fence.
+        // Four-node read quorum is two; commit/write quorum is three.
+        let unavailable_at_quorum = if locks && phase == "create" { 2 } else { 1 };
+        for unavailable in [unavailable_at_quorum, unavailable_at_quorum + 1] {
+            let key = format!("{locks}-{phase}-{unavailable}");
+            let mut limits = vec![usize::MAX; unavailable];
+            // EC12+4: one peer contributes four shards; one packet on the
+            // second peer moves exactly twelve available shards to eleven.
+            if !locks && drives_per_node == 4 && unavailable > 1 {
+                limits[unavailable - 1] = 1;
+            }
+            let available_votes = if locks {
+                4 - unavailable
+            } else {
+                4 * drives_per_node - limits.iter().map(|limit| drives_per_node.min(*limit)).sum::<usize>()
+            };
+            let methods: &[&'static str] = if locks {
+                &["Lock", "LockBatch"]
+            } else {
+                match phase {
+                    "create" => &["WriteMetadata"],
+                    "part" => &["PreparePartTransaction"],
+                    "complete" => &["RenameData", "RenameDataAtIncarnation", "RenameDataWithQuotaFence"],
+                    "abort" => &["Delete"],
+                    _ => unreachable!(),
+                }
+            };
+            eprintln!("MPU quorum acceptance: phase={phase}, lock_fault={locks}, available_votes={available_votes}");
+            if phase == "create" {
+                let request = clients[0].create_multipart_upload().bucket(bucket).key(&key);
+                let result = with_paused_peers(proxies, methods, &limits, bucket, async move { request.send().await }).await?;
+                if let Some(upload) = quorum_outcome(result, unavailable == unavailable_at_quorum, locks)? {
+                    let upload_id = upload.upload_id().ok_or("missing upload ID")?;
+                    wait_for_storage_peer_recovery(proxies, &clients[0], bucket, upload_id, drives_per_node).await?;
+                    abort_after_fault(&clients[0], bucket, &key, upload_id, false).await?;
+                }
+            } else {
+                let upload = clients[0]
+                    .create_multipart_upload()
+                    .bucket(bucket)
+                    .key(&key)
+                    .send()
+                    .await
+                    .map_err(|error| format!("setup Create failed for {key}: {error:?}"))?;
+                let upload_id = upload.upload_id().ok_or("missing upload ID")?;
+                let scope = if !locks && phase == "abort" {
+                    upload_path_scope(bucket, &key, upload_id)?
+                } else {
+                    upload_scope(upload_id)?
+                };
+                let body = payload_for(&key, if drives_per_node == 4 { 9 * 1024 * 1024 } else { 1024 * 1024 });
+                let initial = if phase != "part" {
+                    Some(
+                        clients[0]
+                            .upload_part()
+                            .bucket(bucket)
+                            .key(&key)
+                            .upload_id(upload_id)
+                            .part_number(1)
+                            .body(ByteStream::from(body.clone()))
+                            .send()
+                            .await?,
+                    )
+                } else {
+                    None
+                };
+                match phase {
+                    "part" => {
+                        let request = clients[0]
+                            .upload_part()
+                            .bucket(bucket)
+                            .key(&key)
+                            .upload_id(upload_id)
+                            .part_number(1)
+                            .body(ByteStream::from(body.clone()));
+                        let result =
+                            with_paused_peers(proxies, methods, &limits, &scope, async move { request.send().await }).await?;
+                        if let Some(part) = quorum_outcome(result, unavailable == unavailable_at_quorum, locks)? {
+                            let completion = clients[0]
+                                .complete_multipart_upload()
+                                .bucket(bucket)
+                                .key(&key)
+                                .upload_id(upload_id)
+                                .multipart_upload(
+                                    CompletedMultipartUpload::builder()
+                                        .parts(
+                                            CompletedPart::builder()
+                                                .part_number(1)
+                                                .e_tag(part.e_tag().ok_or("missing ETag")?)
+                                                .build(),
+                                        )
+                                        .build(),
+                                );
+                            recover_after_fault(&format!("Complete {key} after part fault"), || completion.clone().send())
+                                .await??;
+                            for reader in clients {
+                                assert_object_bytes_after_fault(reader, bucket, &key, &body).await?;
+                            }
+                        }
+                    }
+                    "complete" => {
+                        let part = initial.as_ref().expect("complete fixture has one part");
+                        let request = clients[0]
+                            .complete_multipart_upload()
+                            .bucket(bucket)
+                            .key(&key)
+                            .upload_id(upload_id)
+                            .multipart_upload(
+                                CompletedMultipartUpload::builder()
+                                    .parts(
+                                        CompletedPart::builder()
+                                            .part_number(1)
+                                            .e_tag(part.e_tag().ok_or("missing ETag")?)
+                                            .build(),
+                                    )
+                                    .build(),
+                            );
+                        let result =
+                            with_paused_peers(proxies, methods, &limits, &scope, async move { request.send().await }).await?;
+                        if quorum_outcome(result, unavailable == unavailable_at_quorum, locks)?.is_some() {
+                            for reader in clients {
+                                assert_object_bytes_after_fault(reader, bucket, &key, &body).await?;
+                            }
+                        }
+                    }
+                    "abort" => {
+                        let request = clients[0]
+                            .abort_multipart_upload()
+                            .bucket(bucket)
+                            .key(&key)
+                            .upload_id(upload_id);
+                        let result =
+                            with_paused_peers(proxies, methods, &limits, &scope, async move { request.send().await }).await?;
+                        quorum_outcome(result, unavailable == unavailable_at_quorum, locks)?;
+                    }
+                    _ => unreachable!(),
+                }
+                wait_for_storage_peer_recovery(proxies, &clients[0], bucket, upload_id, drives_per_node).await?;
+                // A rejected Complete must retain authority for ordinary
+                // Abort. NoSuchUpload must not hide partial staging loss.
+                let allow_missing = phase != "complete" || unavailable == unavailable_at_quorum;
+                abort_after_fault(&clients[0], bucket, &key, upload_id, allow_missing).await?;
+                recover_after_fault(&format!("DeleteObject {key} after {phase}"), || {
+                    clients[0].delete_object().bucket(bucket).key(&key).send()
+                })
+                .await??;
+            }
+            wait_for_lock_peer_recovery(proxies, &clients[0], bucket).await?;
+            round_trip(clients, bucket, "recovered", 9 * 1024 * 1024).await?;
+            assert_no_uploads_after_fault(clients, bucket).await?;
+        }
+    }
+    Ok(())
+}
+
+#[tokio::test]
+async fn metadata_delay_preserves_write_quorum_and_rejects_quorum_minus_one() -> TestResult {
+    let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
+    phase_quorum_matrix(&proxies, &clients, &bucket, false, 1).await
+}
+
+#[tokio::test]
+async fn lock_delay_preserves_admission_quorum_and_reports_timeout() -> TestResult {
+    let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
+    phase_quorum_matrix(&proxies, &clients, &bucket, true, 1).await
+}
+
+#[tokio::test]
+async fn sixteen_shard_multipart_metadata_requires_twelve_votes() -> TestResult {
+    let (cluster, proxies, clients, bucket) = fixture(4, "EC:4").await?;
+    let storage = cluster_admin_ok(&cluster, http::Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
+    let storage: serde_json::Value = serde_json::from_str(&storage)?;
+    let storage: rustfs_madmin::StorageInfo = serde_json::from_value(storage["info"].clone())?;
+    assert_eq!(storage.disks.len(), 16);
+    assert_eq!(storage.backend.total_sets, vec![1]);
+    assert_eq!(storage.backend.drives_per_set, vec![16]);
+    assert_eq!(storage.backend.standard_sc_parity, Some(4));
+    phase_quorum_matrix(&proxies, &clients, &bucket, false, 4).await
+}
+
+#[tokio::test]
+async fn cancelled_metadata_wait_releases_fence_for_bucket_recreation() -> TestResult {
+    let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
+    let pauses: Vec<_> = proxies[1..]
+        .iter()
+        .map(|proxy| proxy.pause(&["WriteMetadata"], usize::MAX, Some(&bucket)))
+        .collect::<Result<_, _>>()?;
+    let task = create(&clients[0], &bucket, "cancelled");
+    wait_for_faults(&pauses, &["WriteMetadata"]).await?;
+    task.abort();
+    assert!(task.await.expect_err("cancelled S3 future").is_cancelled());
+    drop(pauses);
+    // Deletion must wait for admitted server work to finish, then erase its
+    // generation. Client cancellation alone does not prove server cancellation.
+    tokio::time::timeout(DEADLINE, clients[1].delete_bucket().bucket(&bucket).send()).await??;
+    clients[2].create_bucket().bucket(&bucket).send().await?;
+    assert_no_uploads(&clients, &bucket).await?;
+    round_trip(&clients, &bucket, "new-generation", 9 * 1024 * 1024).await?;
+    assert_no_uploads(&clients, &bucket).await
+}
+
+/// Same logical geometry as the incident: one pool, four 16-shard sets,
+/// EC12+4, 45 MiB files, 8 MiB parts, two files and four parts in flight.
+#[tokio::test]
+async fn eight_node_eight_drive_multipart_load_leaves_no_residual_uploads() -> TestResult {
+    let rounds = std::env::var("RUSTFS_E2E_MPU_LOAD_ROUNDS")
+        .unwrap_or_else(|_| "3".into())
+        .parse::<usize>()?;
+    if !(1..=256).contains(&rounds) {
+        return Err("RUSTFS_E2E_MPU_LOAD_ROUNDS must be in 1..=256".into());
+    }
+    let mut cluster = RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(8, 8)).await?;
+    for (name, value) in [
+        ("RUSTFS_STORAGE_CLASS_STANDARD", "EC:4"),
+        ("TOKIO_WORKER_THREADS", "2"),
+        ("RAYON_NUM_THREADS", "2"),
+        ("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false"),
+        ("NO_PROXY", "127.0.0.1,::1,localhost"),
+        ("HTTP_PROXY", ""),
+        ("HTTPS_PROXY", ""),
+        ("RUST_LOG", "warn"),
+    ] {
+        cluster.set_env(name, value);
+    }
+    configure_node_logs(&mut cluster, None)?;
+    cluster.start().await?;
+    let clients: Vec<_> = cluster
+        .create_all_clients()?
+        .into_iter()
+        .map(|client| {
+            Client::from_conf(
+                client
+                    .config()
+                    .to_builder()
+                    .retry_config(RetryConfig::standard().with_max_attempts(1))
+                    .build(),
+            )
+        })
+        .collect();
+    let storage = cluster_admin_ok(&cluster, http::Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
+    let storage: serde_json::Value = serde_json::from_str(&storage)?;
+    let storage: rustfs_madmin::StorageInfo = serde_json::from_value(storage["info"].clone())?;
+    assert_eq!(storage.disks.len(), 64);
+    assert_eq!(storage.backend.total_sets, vec![4]);
+    assert_eq!(storage.backend.drives_per_set, vec![16]);
+    assert_eq!(storage.backend.standard_sc_parity, Some(4));
+    let bucket = unique_bucket("mpu-8x8");
+    cluster.create_test_bucket(&bucket).await?;
+    let started = Instant::now();
+    for round in 0..rounds {
+        let first = format!("round-{round}-first");
+        let second = format!("round-{round}-second");
+        tokio::try_join!(
+            round_trip(&clients, &bucket, &first, 45 * 1024 * 1024),
+            round_trip(&clients, &bucket, &second, 45 * 1024 * 1024),
+        )?;
+        let aborted = clients[round % clients.len()]
+            .create_multipart_upload()
+            .bucket(&bucket)
+            .key("aborted")
+            .send()
+            .await?;
+        let upload_id = aborted.upload_id().ok_or("missing upload ID")?;
+        clients[(round + 1) % clients.len()]
+            .upload_part()
+            .bucket(&bucket)
+            .key("aborted")
+            .upload_id(upload_id)
+            .part_number(1)
+            .body(ByteStream::from(vec![0xa5; 8 * 1024 * 1024]))
+            .send()
+            .await?;
+        abort(&clients[(round + 2) % clients.len()], &bucket, "aborted", upload_id).await?;
+        assert_no_uploads(&clients, &bucket).await?;
+        eprintln!(
+            "MPU 8x8 acceptance: round={}/{rounds}, completed={}, aborted={}, elapsed={:?}",
+            round + 1,
+            2 * (round + 1),
+            round + 1,
+            started.elapsed()
+        );
+    }
+    Ok(())
+}
+
+async fn resume_at_rpc<T: Send + 'static>(
+    proxies: &[RpcFaultProxy],
+    methods: &[&'static str],
+    scope: &str,
+    request: impl std::future::Future<Output = TestResult<T>> + Send + 'static,
+) -> TestResult<T> {
+    let pauses: Vec<_> = proxies[1..]
+        .iter()
+        .map(|proxy| proxy.pause(methods, usize::MAX, Some(scope)))
+        .collect::<Result<_, _>>()?;
+    eprintln!("MPU delay acceptance: methods={methods:?}, scope={scope}");
+    let task = tokio::spawn(request);
+    wait_for_faults(&pauses, methods).await?;
+    assert!(
+        !task.is_finished(),
+        "the selected commit must wait while only one shard/voter is available"
+    );
+    drop(pauses);
+    tokio::time::timeout(DEADLINE, task).await??
+}
+
+#[tokio::test]
+async fn each_multipart_phase_resumes_after_metadata_and_lock_delays() -> TestResult {
+    let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
+    for locks in [false, true] {
+        let key = format!("phase-{locks}");
+        let client = clients[0].clone();
+        let create = client.create_multipart_upload().bucket(&bucket).key(&key);
+        let upload = resume_at_rpc(
+            &proxies,
+            if locks { &["Lock", "LockBatch"] } else { &["WriteMetadata"] },
+            bucket.as_str(),
+            async move { Ok(create.send().await?) },
+        )
+        .await?;
+        let upload_id = upload.upload_id().ok_or("missing upload ID")?;
+        let scope = upload_scope(upload_id)?;
+        let body = payload_for(&key, 1024 * 1024);
+        let part = client
+            .upload_part()
+            .bucket(&bucket)
+            .key(&key)
+            .upload_id(upload_id)
+            .part_number(1)
+            .body(ByteStream::from(body.clone()));
+        let part = resume_at_rpc(
+            &proxies,
+            if locks {
+                &["Lock", "LockBatch"]
+            } else {
+                &["PreparePartTransaction"]
+            },
+            &scope,
+            async move { Ok(part.send().await?) },
+        )
+        .await?;
+        let complete = client
+            .complete_multipart_upload()
+            .bucket(&bucket)
+            .key(&key)
+            .upload_id(upload_id)
+            .multipart_upload(
+                CompletedMultipartUpload::builder()
+                    .parts(
+                        CompletedPart::builder()
+                            .part_number(1)
+                            .e_tag(part.e_tag().ok_or("missing ETag")?)
+                            .build(),
+                    )
+                    .build(),
+            );
+        resume_at_rpc(
+            &proxies,
+            if locks {
+                &["Lock", "LockBatch"]
+            } else {
+                &["RenameData", "RenameDataAtIncarnation", "RenameDataWithQuotaFence"]
+            },
+            &scope,
+            async move { Ok(complete.send().await?) },
+        )
+        .await?;
+        for reader in &clients {
+            assert_object_bytes(reader, &bucket, &key, &body).await?;
+        }
+        client.delete_object().bucket(&bucket).key(&key).send().await?;
+        let upload = client.create_multipart_upload().bucket(&bucket).key(&key).send().await?;
+        let upload_id = upload.upload_id().ok_or("missing upload ID")?;
+        let scope = upload_scope(upload_id)?;
+        let scope = if locks {
+            scope
+        } else {
+            upload_path_scope(&bucket, &key, upload_id)?
+        };
+        let request = client.abort_multipart_upload().bucket(&bucket).key(&key).upload_id(upload_id);
+        resume_at_rpc(&proxies, if locks { &["Lock", "LockBatch"] } else { &["Delete"] }, &scope, async move {
+            Ok(request.send().await?)
+        })
+        .await?;
+        let missing = client
+            .list_parts()
+            .bucket(&bucket)
+            .key(&key)
+            .upload_id(upload_id)
+            .send()
+            .await
+            .expect_err("delayed Abort must remove the upload");
+        assert_eq!(missing.as_service_error().and_then(|e| e.meta().code()), Some("NoSuchUpload"));
+        assert_no_uploads(&clients, &bucket).await?;
+    }
+    Ok(())
+}

crates/e2e_test/src/distributed/scanner_heal_topology_test.rs

@@ -528,7 +528,7 @@ async fn scanner_heal_topology_decommission_kill_before_source_cleanup_resumes_w
     run.movement_barrier.disarm()?;
     let restarted_pid = run.restart_killed_node(SEED_POOL).await?;
     let label = "after the killed leader resumed decommission";
-    let outcome = settle_decommission(&run, &manifest, label, MOVEMENT_COMPLETE_TIMEOUT).await;
+    let outcome = settle_decommission(&run, &manifest, label, DECOMMISSION_RESUME_COMPLETE_TIMEOUT).await;
     let strict_nodes = assert_process_durability_strict(&run.node_log_paths());
     record(
         &run,

crates/e2e_test/src/distributed/versioning_test.rs

@@ -17,6 +17,14 @@ use crate::common::init_logging;
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
 
+#[tokio::test]
+async fn four_node_explicit_delete_marker_batch_is_immediately_visible() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("explicit-marker-batch");
+    crate::delete_objects_versioning_test::assert_explicit_marker_batch_cleanup(&dist.clients()?, &bucket).await
+}
+
 #[tokio::test]
 async fn four_node_four_drive_versioning_put_list_get_delete_marker() -> TestResult {
     init_logging();

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -1206,7 +1206,9 @@ mod tests {
         let interval = interval
             .filter(|interval| *interval != 0)
             .ok_or("scanner schedule has no positive interval")?;
-        let next_cycle = cycle_end
+        // A new leader may already be sleeping before completing its first cycle.
+        let schedule_start = if cycle_end == 0 { now } else { cycle_end };
+        let next_cycle = schedule_start
             .checked_add(interval)
             .ok_or("scanner schedule timestamp overflows")?;
         let budget = next_cycle
@@ -1223,6 +1225,11 @@ mod tests {
             Ok(Duration::from_secs(259)),
         );
         assert_eq!(scanner_cycle_completion_budget(100, Some(1), 110, false), Ok(Duration::from_secs(60)));
+        // A newly elected leader can advertise a sleep before its first completed cycle.
+        assert_eq!(
+            scanner_cycle_completion_budget(0, Some(289), 1_792_913_211, false),
+            Ok(Duration::from_secs(349)),
+        );
     }
 
     #[test]
@@ -1243,7 +1250,7 @@ mod tests {
         previous_cycle_end: u64,
     ) -> Result<u64, Box<dyn Error + Send + Sync>> {
         let mut deadline = Instant::now() + Duration::from_secs(60);
-        let mut schedule_accounted = false;
+        let mut accounted_schedules = HashSet::new();
         loop {
             let mut latest_cycle_end = 0;
             let mut versions_observed = false;
@@ -1290,15 +1297,27 @@ mod tests {
                     return Err("scanner cycle timestamp is in the future".into());
                 }
                 let schedule = &status["cycle_schedule"];
-                if !schedule_accounted && schedule["execution_role"].as_str() == Some("leader") {
+                if schedule["execution_role"].as_str() == Some("leader") {
                     let active = metrics["current_cycle_active"]
                         .as_bool()
                         .ok_or("scanner status is missing its active-cycle flag")?;
-                    if active || (cycle_end != 0 && schedule["effective_interval_available"].as_bool() == Some(true)) {
-                        let interval = schedule["effective_interval_seconds"].as_u64();
-                        let budget = scanner_cycle_completion_budget(cycle_end, interval, now, active)?;
-                        deadline = Instant::now().checked_add(budget).ok_or("scanner deadline overflows")?;
-                        schedule_accounted = true;
+                    if active || schedule["effective_interval_available"].as_bool() == Some(true) {
+                        let interval = if active {
+                            None
+                        } else {
+                            Some(
+                                schedule["effective_interval_seconds"]
+                                    .as_u64()
+                                    .ok_or("scanner schedule is missing its effective interval")?,
+                            )
+                        };
+                        // Reconsider a changed leader or retry plan, but never renew
+                        // the deadline on repeated observations of the same sleep.
+                        if accounted_schedules.insert((node_index, cycle_end, interval)) {
+                            let budget = scanner_cycle_completion_budget(cycle_end, interval, now, active)?;
+                            let scheduled_deadline = Instant::now().checked_add(budget).ok_or("scanner deadline overflows")?;
+                            deadline = deadline.max(scheduled_deadline);
+                        }
                     }
                 }
                 // Superseded walks complete all sets while withholding quota authority.

crates/e2e_test/src/lib.rs

@@ -45,6 +45,9 @@ pub mod on_demand_migration;
 #[cfg(test)]
 pub mod fault_proxy;
 
+#[cfg(test)]
+mod rpc_fault_proxy;
+
 // Reliability tests built on the fault-injection harness
 #[cfg(test)]
 mod reliability_disk_fault_test;

crates/e2e_test/src/negative_sigv4_test.rs

@@ -230,6 +230,302 @@ fn encode_unsigned_aws_chunked_with_sha256_trailer(decoded: &[u8]) -> Vec<u8> {
     encoded
 }
 
+/// Sign the data and terminating chunks independently of the server decoder.
+fn encode_signed_aws_chunked(signer: &SigV4, headers: &SignedHeaders, decoded: &[u8], trailer: bool) -> Vec<u8> {
+    let mut previous = headers
+        .authorization
+        .rsplit_once("Signature=")
+        .expect("seed signature")
+        .1
+        .to_string();
+    let mut encoded = Vec::new();
+    for chunk in [decoded, b"".as_slice()] {
+        let string_to_sign = format!(
+            "AWS4-HMAC-SHA256-PAYLOAD\n{}\n{}\n{previous}\n{}\n{}",
+            headers.amz_date,
+            get_scope(REGION, signer.time, "s3"),
+            sha256_hex(b""),
+            sha256_hex(chunk),
+        );
+        previous = get_signature(get_signing_key(&signer.secret_key, REGION, signer.time, "s3"), &string_to_sign);
+        encoded.extend_from_slice(format!("{:x};chunk-signature={previous}\r\n", chunk.len()).as_bytes());
+        encoded.extend_from_slice(chunk);
+        encoded.extend_from_slice(b"\r\n");
+    }
+    if trailer {
+        let checksum = sha256_base64(decoded);
+        let canonical = format!("x-amz-checksum-sha256:{checksum}\n");
+        let string_to_sign = format!(
+            "AWS4-HMAC-SHA256-TRAILER\n{}\n{}\n{previous}\n{}",
+            headers.amz_date,
+            get_scope(REGION, signer.time, "s3"),
+            sha256_hex(canonical.as_bytes()),
+        );
+        let signature = get_signature(get_signing_key(&signer.secret_key, REGION, signer.time, "s3"), &string_to_sign);
+        encoded.extend_from_slice(
+            format!("x-amz-checksum-sha256:{checksum}\r\nx-amz-trailer-signature:{signature}\r\n\r\n").as_bytes(),
+        );
+    }
+    encoded
+}
+
+/// Exercise decoder failures through both storage publication boundaries.
+#[tokio::test]
+async fn aws_chunked_named_errors_reject_put_and_upload_part() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let mut env = RustFSTestEnvironment::new().await?;
+    setup(&mut env).await?;
+    let client = env.create_s3_client();
+    let decoded = b"private-chunked-fixture-body";
+    let mut failures = Vec::new();
+    for multipart in [false, true] {
+        for (case, expected_status, expected_code) in [
+            ("signature", 403, "SignatureDoesNotMatch"),
+            ("final-signature", 403, "SignatureDoesNotMatch"),
+            ("format", 400, "IncompleteBody"),
+            ("metadata-at-limit", 400, "IncompleteBody"),
+            ("metadata-over-limit", 400, "EntityTooLarge"),
+            ("chunk-over-limit", 400, "EntityTooLarge"),
+            ("trailers-over-limit", 400, "IncompleteBody"),
+            ("too-many-trailers", 400, "IncompleteBody"),
+        ] {
+            let key = format!("chunked-{case}-{multipart}.bin");
+            let upload_id = if multipart {
+                Some(
+                    client
+                        .create_multipart_upload()
+                        .bucket(BUCKET)
+                        .key(&key)
+                        .send()
+                        .await?
+                        .upload_id()
+                        .ok_or("missing upload ID")?
+                        .to_owned(),
+                )
+            } else {
+                None
+            };
+            let query = upload_id
+                .as_ref()
+                .map(|id| format!("partNumber=1&uploadId={}", urlencoding::encode(id)))
+                .unwrap_or_default();
+            let path = format!("/{BUCKET}/{key}");
+            let target = if query.is_empty() {
+                path.clone()
+            } else {
+                format!("{path}?{query}")
+            };
+            let signer = SigV4::new(&env);
+            let unsigned = matches!(case, "trailers-over-limit" | "too-many-trailers");
+            let mode = if unsigned {
+                UNSIGNED_PAYLOAD_TRAILER
+            } else {
+                "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
+            };
+            let length = decoded.len().to_string();
+            let mut extra = vec![
+                ("content-encoding", "aws-chunked"),
+                ("x-amz-decoded-content-length", length.as_str()),
+            ];
+            if unsigned {
+                extra.push(("x-amz-trailer", "x-amz-checksum-sha256"));
+            }
+            let headers = signer.sign_with_extra_headers("PUT", &path, &query, mode, &extra);
+            let encoded = match case {
+                "signature" | "final-signature" => {
+                    let mut body = encode_signed_aws_chunked(&signer, &headers, decoded, false);
+                    let offset = if case == "signature" {
+                        body.windows(2).position(|w| w == b"\r\n").expect("chunk header") + 2
+                    } else {
+                        body.len() - 5
+                    };
+                    body[offset] = if body[offset] == b'a' { b'b' } else { b'a' };
+                    body
+                }
+                "format" => b"invalid-chunk-size\r\n".to_vec(),
+                "metadata-at-limit" | "metadata-over-limit" => {
+                    let size = if case == "metadata-at-limit" { 1024 } else { 1025 };
+                    let mut body = vec![b'z'; size - 2];
+                    body.extend_from_slice(b"\r\n");
+                    body
+                }
+                "chunk-over-limit" => format!("10000001;chunk-signature={}\r\n", "0".repeat(64)).into_bytes(),
+                "trailers-over-limit" | "too-many-trailers" => {
+                    let mut body = format!("{:x}\r\n", decoded.len()).into_bytes();
+                    body.extend_from_slice(decoded);
+                    body.extend_from_slice(b"\r\n0\r\n");
+                    if case == "trailers-over-limit" {
+                        body.extend_from_slice(format!("x-private:{}\r\n\r\n", "v".repeat(16385 - 14)).as_bytes());
+                    } else {
+                        body.extend_from_slice("x-private:v\r\n".repeat(101).as_bytes());
+                        body.extend_from_slice(b"\r\n");
+                    }
+                    body
+                }
+                _ => unreachable!("all fixtures are enumerated"),
+            };
+            let mut request = local_http_client()
+                .put(format!("{}{target}", env.url))
+                .header("authorization", &headers.authorization)
+                .header("x-amz-content-sha256", &headers.content_sha256)
+                .header("x-amz-date", &headers.amz_date);
+            for (name, value) in extra {
+                request = request.header(name, value);
+            }
+            let response = request
+                .body(encoded)
+                .timeout(std::time::Duration::from_secs(20))
+                .send()
+                .await?;
+            let status = response.status().as_u16();
+            let body = response.text().await?;
+            eprintln!("aws-chunked {case} multipart={multipart}: HTTP {status}, {body}");
+            if status != expected_status || !body.contains(&format!("<Code>{expected_code}</Code>")) {
+                failures.push(format!(
+                    "{case} multipart={multipart}: expected {expected_status} {expected_code}, got {status} {body}"
+                ));
+            }
+            for private in [
+                "private-chunked-fixture-body",
+                "chunk-signature=",
+                "AwsChunkedStreamError",
+                "x-private",
+            ] {
+                if body.contains(private) {
+                    failures.push(format!("{case}: response exposed {private}"));
+                }
+            }
+            if let Some(id) = upload_id {
+                let parts = client.list_parts().bucket(BUCKET).key(&key).upload_id(&id).send().await?;
+                assert!(parts.parts().is_empty(), "{case}: rejected UploadPart published a part");
+                client
+                    .abort_multipart_upload()
+                    .bucket(BUCKET)
+                    .key(&key)
+                    .upload_id(id)
+                    .send()
+                    .await?;
+            }
+            let absent = client
+                .get_object()
+                .bucket(BUCKET)
+                .key(&key)
+                .send()
+                .await
+                .expect_err("a rejected body must not publish an object");
+            assert_eq!(absent.as_service_error().and_then(ProvideErrorMetadata::code), Some("NoSuchKey"));
+        }
+    }
+    assert!(failures.is_empty(), "{}", failures.join("\n"));
+    Ok(())
+}
+
+#[tokio::test]
+async fn aws_chunked_valid_signed_and_trailer_boundaries_succeed() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let mut env = RustFSTestEnvironment::new().await?;
+    setup(&mut env).await?;
+    let client = env.create_s3_client();
+    let decoded = b"valid-chunked-control";
+    for (case, multipart) in ["signed", "signed-trailer", "trailer-size-limit", "trailer-count-limit"]
+        .into_iter()
+        .flat_map(|case| [false, true].map(|multipart| (case, multipart)))
+    {
+        let key = format!("{case}-{multipart}");
+        let upload_id = if multipart {
+            Some(
+                client
+                    .create_multipart_upload()
+                    .bucket(BUCKET)
+                    .key(&key)
+                    .send()
+                    .await?
+                    .upload_id()
+                    .ok_or("missing upload ID")?
+                    .to_owned(),
+            )
+        } else {
+            None
+        };
+        let query = upload_id
+            .as_ref()
+            .map(|id| format!("partNumber=1&uploadId={}", urlencoding::encode(id)))
+            .unwrap_or_default();
+        let signer = SigV4::new(&env);
+        let path = format!("/{BUCKET}/{key}");
+        let target = if query.is_empty() {
+            path.clone()
+        } else {
+            format!("{path}?{query}")
+        };
+        let length = decoded.len().to_string();
+        let mode = match case {
+            "signed" => "STREAMING-AWS4-HMAC-SHA256-PAYLOAD",
+            "signed-trailer" => "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER",
+            _ => UNSIGNED_PAYLOAD_TRAILER,
+        };
+        let mut extra = vec![
+            ("content-encoding", "aws-chunked"),
+            ("x-amz-decoded-content-length", length.as_str()),
+        ];
+        if case != "signed" {
+            extra.push(("x-amz-trailer", "x-amz-checksum-sha256"));
+        }
+        let headers = signer.sign_with_extra_headers("PUT", &path, &query, mode, &extra);
+        let mut encoded = if case.starts_with("signed") {
+            encode_signed_aws_chunked(&signer, &headers, decoded, case == "signed-trailer")
+        } else {
+            encode_unsigned_aws_chunked_with_sha256_trailer(decoded)
+        };
+        if !case.starts_with("signed") {
+            encoded.truncate(encoded.len() - 2);
+            if case == "trailer-size-limit" {
+                let checksum_line_size = format!("x-amz-checksum-sha256:{}\r\n", sha256_base64(decoded)).len();
+                encoded.extend_from_slice(format!("x-padding:{}\r\n", "v".repeat(16384 - checksum_line_size - 14)).as_bytes());
+            } else {
+                encoded.extend_from_slice("x-padding:v\r\n".repeat(99).as_bytes());
+            }
+            encoded.extend_from_slice(b"\r\n");
+        }
+        let mut request = local_http_client()
+            .put(format!("{}{target}", env.url))
+            .header("authorization", &headers.authorization)
+            .header("x-amz-content-sha256", &headers.content_sha256)
+            .header("x-amz-date", &headers.amz_date);
+        for (name, value) in extra {
+            request = request.header(name, value);
+        }
+        let response = request
+            .body(encoded)
+            .timeout(std::time::Duration::from_secs(20))
+            .send()
+            .await?;
+        let status = response.status();
+        let etag = response
+            .headers()
+            .get("etag")
+            .and_then(|value| value.to_str().ok())
+            .map(str::to_owned);
+        let body = response.text().await?;
+        assert_eq!(status.as_u16(), 200, "{case}: {body}");
+        if let Some(id) = upload_id {
+            let part = aws_sdk_s3::types::CompletedPart::builder()
+                .part_number(1)
+                .e_tag(etag.ok_or("successful UploadPart omitted ETag")?)
+                .build();
+            client
+                .complete_multipart_upload()
+                .bucket(BUCKET)
+                .key(&key)
+                .upload_id(id)
+                .multipart_upload(aws_sdk_s3::types::CompletedMultipartUpload::builder().parts(part).build())
+                .send()
+                .await?;
+        }
+        let object = client.get_object().bucket(BUCKET).key(&key).send().await?;
+        assert_eq!(object.body.collect().await?.into_bytes().as_ref(), decoded, "{case}");
+    }
+    Ok(())
+}
+
 /// Positive control: a correctly hand-signed request must succeed. Without
 /// this, every negative assertion below could pass for the wrong reason (a
 /// broken signer that never produces a valid signature).

crates/e2e_test/src/reliant/grpc_lock_server.rs

@@ -541,6 +541,50 @@ impl NodeService for MinimalLockNodeService {
         Err(Status::unimplemented("lock-only test server"))
     }
 
+    async fn artifact_rename_data(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameDataResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
+    async fn artifact_repair_rename_data(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::ArtifactRepairRenameDataRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameDataResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
+    async fn artifact_repair_write_metadata(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::ArtifactRepairWriteMetadataRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::WriteMetadataResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
+    type InventoryArtifactAttemptStream =
+        Pin<Box<dyn Stream<Item = Result<rustfs_protos::proto_gen::node_service::ArtifactInventoryFrame, Status>> + Send>>;
+    async fn inventory_artifact_attempt(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::ArtifactInventoryRequest>,
+    ) -> Result<Response<Self::InventoryArtifactAttemptStream>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
+    async fn seal_artifact_attempt_repairs(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
+    async fn seal_artifact_attempt_puts(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
     async fn rename_data(
         &self,
         _request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,

crates/e2e_test/src/rpc_fault_proxy.rs

@@ -0,0 +1,614 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Method-selective faults without changing the server or gRPC body/trailers.
+
+use bytes::Bytes;
+use http_body_util::{BodyExt as _, Full, Limited, combinators::BoxBody};
+use hyper::{Request, body::Incoming, service::service_fn};
+use hyper_util::{
+    client::legacy::{Client, connect::HttpConnector},
+    rt::{TokioExecutor, TokioIo},
+    server::conn::auto,
+};
+use prost::Message as _;
+use rustfs_protos::proto_gen::node_service::{DeleteRequest, ReadVersionRequest};
+use std::{
+    collections::HashSet,
+    io,
+    net::SocketAddr,
+    sync::{
+        Arc, Mutex,
+        atomic::{AtomicUsize, Ordering},
+    },
+    time::Duration,
+};
+use tokio::{
+    net::TcpListener,
+    sync::{Notify, watch},
+    task::{JoinHandle, JoinSet},
+};
+
+type ProxyBody = BoxBody<Bytes, io::Error>;
+type ProxyClient = Client<HttpConnector, ProxyBody>;
+
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum Disposition {
+    Hold,
+    Forward,
+    Discard,
+}
+
+struct PauseState {
+    methods: Vec<&'static str>,
+    scope: Option<Vec<u8>>,
+    limit: usize,
+    matched: AtomicUsize,
+    blocked: AtomicUsize,
+    forwarded: AtomicUsize,
+    forwarded_routes: Mutex<HashSet<String>>,
+    reached: Notify,
+    release: watch::Sender<Disposition>,
+}
+
+type PauseSlot = Arc<Mutex<Option<Arc<PauseState>>>>;
+
+pub(crate) struct RpcPause {
+    state: Arc<PauseState>,
+    slot: PauseSlot,
+}
+
+impl RpcPause {
+    pub(crate) fn hits(&self) -> usize {
+        self.state.blocked.load(Ordering::SeqCst)
+    }
+
+    /// Selected RPCs whose backend returned a response, excluding discarded packets.
+    pub(crate) fn forwarded(&self) -> usize {
+        self.state.forwarded.load(Ordering::SeqCst)
+    }
+
+    pub(crate) fn forwarded_routes(&self) -> usize {
+        self.state.forwarded_routes.lock().expect("RPC route observations").len()
+    }
+
+    pub(crate) async fn wait_for_hits(&self, count: usize) -> io::Result<()> {
+        tokio::time::timeout(Duration::from_secs(20), async {
+            loop {
+                let notified = self.state.reached.notified();
+                if self.hits() >= count {
+                    return;
+                }
+                notified.await;
+            }
+        })
+        .await
+        .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "the selected RPC fault was not reached"))
+    }
+
+    pub(crate) fn release(&self) {
+        self.settle(Disposition::Forward);
+    }
+
+    /// Retire packets that outlived the caller's operation budget. A later
+    /// healthy request must not replay an expired lock grant or mutation.
+    pub(crate) fn discard_pending(&self) {
+        self.settle(Disposition::Discard);
+    }
+
+    fn settle(&self, disposition: Disposition) {
+        self.state.release.send_if_modified(|current| {
+            if *current == Disposition::Hold {
+                *current = disposition;
+                true
+            } else {
+                false
+            }
+        });
+    }
+}
+
+impl Drop for RpcPause {
+    fn drop(&mut self) {
+        self.release();
+        let mut slot = self.slot.lock().expect("RPC pause slot");
+        if slot.as_ref().is_some_and(|current| Arc::ptr_eq(current, &self.state)) {
+            *slot = None;
+        }
+    }
+}
+
+pub(crate) struct RpcFaultProxy {
+    address: SocketAddr,
+    slot: PauseSlot,
+    task: JoinHandle<()>,
+}
+
+impl RpcFaultProxy {
+    pub(crate) async fn start_on(address: SocketAddr, target: SocketAddr) -> io::Result<Self> {
+        let listener = TcpListener::bind(address).await?;
+        let address = listener.local_addr()?;
+        let slot: PauseSlot = Arc::new(Mutex::new(None));
+        let active = slot.clone();
+        let task = tokio::spawn(async move {
+            let h1: ProxyClient = Client::builder(TokioExecutor::new()).build_http();
+            let h2: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
+            let mut connections = JoinSet::new();
+            loop {
+                tokio::select! {
+                    accepted = listener.accept() => {
+                        let Ok((socket, _)) = accepted else { break };
+                        let h1 = h1.clone();
+                        let h2 = h2.clone();
+                        let active = active.clone();
+                        connections.spawn(async move {
+                            let service = service_fn(move |request: Request<Incoming>| {
+                                let client = if request.version() == http::Version::HTTP_2 { h2.clone() } else { h1.clone() };
+                                let pause = active.lock().expect("RPC pause slot").clone();
+                                forward_request(request, client, target, pause)
+
+                            });
+                            let _ = auto::Builder::new(TokioExecutor::new())
+                                .serve_connection(TokioIo::new(socket), service).await;
+                        });
+                    }
+                    _ = connections.join_next(), if !connections.is_empty() => {}
+                }
+            }
+        });
+        Ok(Self { address, slot, task })
+    }
+
+    pub(crate) fn local_addr(&self) -> SocketAddr {
+        self.address
+    }
+
+    /// Pause at most limit requests to the named unary methods; other streams
+    /// on the same HTTP/2 connection continue forwarding.
+    pub(crate) fn pause(&self, methods: &[&'static str], limit: usize, scope: Option<&str>) -> io::Result<RpcPause> {
+        let mut slot = self.slot.lock().expect("RPC pause slot");
+        if slot.is_some() || methods.is_empty() || limit == 0 || scope.is_some_and(str::is_empty) {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidInput,
+                "RPC pause needs an empty slot, methods and a positive limit",
+            ));
+        }
+        let (release, _) = watch::channel(Disposition::Hold);
+        let state = Arc::new(PauseState {
+            methods: methods.to_vec(),
+            scope: scope.map(|scope| scope.as_bytes().to_vec()),
+            limit,
+            matched: AtomicUsize::new(0),
+            blocked: AtomicUsize::new(0),
+            forwarded: AtomicUsize::new(0),
+            forwarded_routes: Mutex::new(HashSet::new()),
+            reached: Notify::new(),
+            release,
+        });
+        *slot = Some(state.clone());
+        Ok(RpcPause {
+            state,
+            slot: self.slot.clone(),
+        })
+    }
+}
+
+fn unary_payload(body: &[u8]) -> io::Result<&[u8]> {
+    if body.len() < 5 || body[0] != 0 {
+        return Err(io::Error::new(io::ErrorKind::InvalidData, "scoped RPC needs an uncompressed unary frame"));
+    }
+    let mut encoded_length = [0; 4];
+    encoded_length.copy_from_slice(&body[1..5]);
+    if u32::try_from(body.len() - 5).ok() != Some(u32::from_be_bytes(encoded_length)) {
+        return Err(io::Error::new(io::ErrorKind::InvalidData, "scoped RPC frame length mismatch"));
+    }
+    Ok(&body[5..])
+}
+
+fn scope_matches(method: &str, body: &[u8], scope: &[u8]) -> io::Result<bool> {
+    if method != "Delete" {
+        return Ok(body.windows(scope.len()).any(|window| window == scope));
+    }
+    let request = DeleteRequest::decode(unary_payload(body)?).map_err(io::Error::other)?;
+    let options: serde_json::Value = serde_json::from_str(&request.options).map_err(io::Error::other)?;
+    Ok(request.volume == ".rustfs.sys/multipart"
+        && request.path.trim_end_matches('/').as_bytes() == scope
+        && options.get("recursive").and_then(serde_json::Value::as_bool) == Some(true))
+}
+
+async fn forward_request(
+    request: Request<Incoming>,
+    client: ProxyClient,
+    target: SocketAddr,
+    pause: Option<Arc<PauseState>>,
+) -> io::Result<hyper::Response<Incoming>> {
+    let (mut parts, body) = request.into_parts();
+    let mut body = body.map_err(io::Error::other).boxed();
+    let mut observed = None;
+    if let Some(pause) = pause {
+        let method = parts.uri.path().strip_prefix("/node_service.NodeService/");
+        if let Some(method) = method.filter(|name| pause.methods.contains(name)) {
+            let mut route = None;
+            let matches = if let Some(scope) = pause.scope.as_deref() {
+                // Inspect bounded control routing while preserving the exact
+                // signed bytes and trailers forwarded to the backend.
+                let collected = Limited::new(body, 1024 * 1024).collect().await.map_err(io::Error::other)?;
+                let trailers = collected.trailers().cloned();
+                let bytes = collected.to_bytes();
+                let matches = scope_matches(method, &bytes, scope)?;
+                if matches && method == "ReadVersion" {
+                    let disk = ReadVersionRequest::decode(unary_payload(&bytes)?)
+                        .map_err(io::Error::other)?
+                        .disk;
+                    if !disk.is_empty() {
+                        route = Some(disk);
+                    }
+                }
+                body = Full::new(bytes)
+                    .map_err(|never| match never {})
+                    .with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
+                    .boxed();
+                matches
+            } else {
+                true
+            };
+            if matches && pause.matched.fetch_add(1, Ordering::SeqCst) < pause.limit {
+                let mut released = pause.release.subscribe();
+                pause.blocked.fetch_add(1, Ordering::SeqCst);
+                pause.reached.notify_one();
+                let disposition = loop {
+                    let disposition = *released.borrow_and_update();
+                    if disposition != Disposition::Hold {
+                        break disposition;
+                    }
+                    if released.changed().await.is_err() {
+                        break Disposition::Discard;
+                    }
+                };
+                if disposition == Disposition::Discard {
+                    return Err(io::Error::new(io::ErrorKind::ConnectionAborted, "injected RPC packet loss"));
+                }
+                observed = Some((pause, route));
+            }
+        }
+    }
+    if !parts.headers.contains_key(http::header::HOST)
+        && let Some(authority) = parts.uri.authority()
+    {
+        parts.headers.insert(
+            http::header::HOST,
+            http::HeaderValue::from_str(authority.as_str()).map_err(io::Error::other)?,
+        );
+    }
+    parts.uri = http::Uri::builder()
+        .scheme("http")
+        .authority(target.to_string())
+        .path_and_query(parts.uri.path_and_query().map_or("/", |value| value.as_str()))
+        .build()
+        .map_err(io::Error::other)?;
+    let response = client
+        .request(Request::from_parts(parts, body))
+        .await
+        .map_err(io::Error::other)?;
+    if let Some((pause, route)) = observed
+        && response.status().is_success()
+        && response.headers().get("grpc-status").is_none_or(|status| status == "0")
+    {
+        pause.forwarded.fetch_add(1, Ordering::SeqCst);
+        if let Some(route) = route {
+            pause.forwarded_routes.lock().expect("RPC route observations").insert(route);
+        }
+    }
+    Ok(response)
+}
+
+impl Drop for RpcFaultProxy {
+    fn drop(&mut self) {
+        self.task.abort();
+    }
+}
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use http_body_util::{Full, StreamBody};
+    use hyper::{Response, body::Frame};
+    use std::convert::Infallible;
+
+    struct Backend {
+        received: Arc<AtomicUsize>,
+        address: SocketAddr,
+        task: JoinHandle<()>,
+    }
+
+    impl Drop for Backend {
+        fn drop(&mut self) {
+            self.task.abort();
+        }
+    }
+
+    async fn backend() -> io::Result<Backend> {
+        let listener = TcpListener::bind("127.0.0.1:0").await?;
+        let address = listener.local_addr()?;
+        let received = Arc::new(AtomicUsize::new(0));
+        let received_task = received.clone();
+        let task = tokio::spawn(async move {
+            let mut children = JoinSet::new();
+            loop {
+                tokio::select! {
+                    accepted = listener.accept() => {
+                        let Ok((socket, _)) = accepted else { break };
+                        let received = received_task.clone();
+                        children.spawn(async move {
+                            let service = service_fn(move |request: Request<Incoming>| {
+                                let received = received.clone();
+                                async move {
+                                received.fetch_add(1, Ordering::SeqCst);
+                                let authority = request.headers().get(http::header::HOST).cloned();
+                                let body = request.into_body().collect().await?.to_bytes();
+                                let mut trailers = http::HeaderMap::new();
+                                trailers.insert("grpc-status", http::HeaderValue::from_static("0"));
+                                trailers.insert("x-proof", http::HeaderValue::from_static("unchanged-proof"));
+                                let stream = futures::stream::iter([
+                                    Ok::<_, Infallible>(Frame::data(body)),
+                                    Ok(Frame::trailers(trailers)),
+                                ]);
+                                let mut response = Response::new(StreamBody::new(stream));
+                                response.headers_mut().insert(http::header::TRAILER, http::HeaderValue::from_static("grpc-status, x-proof"));
+                                if let Some(authority) = authority { response.headers_mut().insert("x-original-authority", authority); }
+                                Ok::<_, hyper::Error>(response)
+                            }});
+                            let _ = auto::Builder::new(TokioExecutor::new()).serve_connection(TokioIo::new(socket), service).await;
+                        });
+                    }
+                    _ = children.join_next(), if !children.is_empty() => {}
+                }
+            }
+        });
+        Ok(Backend { received, address, task })
+    }
+
+    async fn call(client: &ProxyClient, address: SocketAddr, method: &str) -> io::Result<hyper::Response<Incoming>> {
+        call_body(client, address, method, Bytes::from_static(b"opaque signed protobuf")).await
+    }
+
+    async fn call_body(
+        client: &ProxyClient,
+        address: SocketAddr,
+        method: &str,
+        bytes: Bytes,
+    ) -> io::Result<hyper::Response<Incoming>> {
+        let body: ProxyBody = Full::new(bytes).map_err(|never| match never {}).boxed();
+        let request = Request::builder()
+            .method("POST")
+            .uri(format!("http://{address}/node_service.NodeService/{method}"))
+            .header(http::header::TE, "trailers")
+            .body(body)
+            .map_err(io::Error::other)?;
+        client.request(request).await.map_err(io::Error::other)
+    }
+
+    #[tokio::test]
+    async fn preserves_protocol_authority_payload_and_grpc_trailers() {
+        tokio::time::timeout(Duration::from_secs(10), async {
+            let backend = backend().await.expect("backend listener");
+            let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback address"), backend.address)
+                .await
+                .expect("proxy");
+            for h2 in [false, true] {
+                let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(h2).build_http();
+                let response = call(&client, proxy.local_addr(), "ReadVersion")
+                    .await
+                    .expect("transparent request");
+                assert_eq!(response.version(), if h2 { http::Version::HTTP_2 } else { http::Version::HTTP_11 });
+                assert_eq!(response.headers()["x-original-authority"], proxy.local_addr().to_string());
+                let collected = response.into_body().collect().await.expect("complete body and trailers");
+                let trailers = collected.trailers().expect("gRPC trailers must survive both proxy hops");
+                assert_eq!(trailers["grpc-status"], "0");
+                assert_eq!(trailers["x-proof"], "unchanged-proof");
+                assert_eq!(collected.to_bytes(), Bytes::from_static(b"opaque signed protobuf"));
+            }
+        })
+        .await
+        .expect("transparent proxy deadline");
+    }
+
+    #[tokio::test]
+    async fn isolates_rpc_methods_limits_injection_and_releases_on_drop() {
+        tokio::time::timeout(Duration::from_secs(10), async {
+            let backend = backend().await.expect("backend listener");
+            let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback address"), backend.address)
+                .await
+                .expect("proxy");
+            let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
+            let pause = proxy.pause(&["WriteMetadata"], 1, None).expect("arm one metadata request");
+            let task_client = client.clone();
+            let address = proxy.local_addr();
+            let blocked = tokio::spawn(async move { call(&task_client, address, "WriteMetadata").await });
+            pause.wait_for_hits(1).await.expect("metadata barrier reached");
+            for method in ["Lock", "WriteMetadata"] {
+                let response = call(&client, address, method)
+                    .await
+                    .expect("unselected or over-limit request must progress");
+                assert_eq!(
+                    response.into_body().collect().await.expect("forwarded body").to_bytes(),
+                    Bytes::from_static(b"opaque signed protobuf")
+                );
+            }
+            assert_eq!(pause.hits(), 1);
+            assert!(!blocked.is_finished(), "the selected request must still be held");
+            drop(pause);
+            assert_eq!(
+                blocked
+                    .await
+                    .expect("blocked request task")
+                    .expect("released request")
+                    .into_body()
+                    .collect()
+                    .await
+                    .expect("released body")
+                    .to_bytes(),
+                Bytes::from_static(b"opaque signed protobuf")
+            );
+        })
+        .await
+        .expect("selective fault deadline");
+    }
+    #[tokio::test]
+    async fn released_observer_requires_a_selected_backend_response() {
+        tokio::time::timeout(Duration::from_secs(10), async {
+            let backend = backend().await.expect("backend listener");
+            let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
+                .await
+                .expect("proxy listener");
+            let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
+            let observer = proxy
+                .pause(&["Lock"], usize::MAX, Some("recovery-bucket"))
+                .expect("RPC observer");
+            let unrelated = call_body(&client, proxy.local_addr(), "Lock", Bytes::from_static(b"other-bucket"))
+                .await
+                .expect("unrelated request");
+            unrelated.into_body().collect().await.expect("unrelated body");
+            assert_eq!(observer.forwarded(), 0, "unrelated traffic cannot prove recovery");
+            let task_client = client.clone();
+            let address = proxy.local_addr();
+            let request =
+                tokio::spawn(
+                    async move { call_body(&task_client, address, "Lock", Bytes::from_static(b"recovery-bucket")).await },
+                );
+            observer.wait_for_hits(1).await.expect("selected request arrived");
+            assert_eq!(observer.forwarded(), 0, "an intercepted request has not reached its backend");
+            observer.release();
+            let body = request
+                .await
+                .expect("request task")
+                .expect("forwarded request")
+                .into_body()
+                .collect()
+                .await
+                .expect("forwarded body");
+            assert_eq!(body.trailers().expect("preserved trailers")["x-proof"], "unchanged-proof");
+            assert_eq!(body.to_bytes(), Bytes::from_static(b"recovery-bucket"));
+            assert_eq!(observer.forwarded(), 1);
+            let another = call_body(&client, address, "Lock", Bytes::from_static(b"recovery-bucket"))
+                .await
+                .expect("observer forwards without pausing");
+            another.into_body().collect().await.expect("second body");
+            assert_eq!(observer.forwarded(), 2);
+            drop(observer);
+            let routes = proxy
+                .pause(&["ReadVersion"], usize::MAX, Some("recovery-route"))
+                .expect("disk-route observer");
+            routes.release();
+            for (disk, expected) in [("drive-a", 1), ("drive-a", 1), ("drive-b", 2)] {
+                let request = ReadVersionRequest {
+                    disk: disk.to_owned(),
+                    path: "recovery-route".to_owned(),
+                    ..Default::default()
+                }
+                .encode_to_vec();
+                let mut frame = vec![0];
+                frame.extend_from_slice(&u32::try_from(request.len()).expect("small frame").to_be_bytes());
+                frame.extend_from_slice(&request);
+                call_body(&client, address, "ReadVersion", Bytes::from(frame))
+                    .await
+                    .expect("routed read")
+                    .into_body()
+                    .collect()
+                    .await
+                    .expect("routed response");
+                assert_eq!(routes.forwarded_routes(), expected, "repeated traffic from one disk is only one voter");
+            }
+        })
+        .await
+        .expect("passive observer deadline");
+    }
+
+    #[tokio::test]
+    async fn scoped_fault_discards_expired_packets_without_replaying_them() {
+        tokio::time::timeout(Duration::from_secs(10), async {
+            let backend = backend().await.expect("backend listener");
+            let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
+                .await
+                .expect("proxy listener");
+            let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
+            assert!(proxy.pause(&["Lock"], 1, Some("")).is_err(), "empty scope must not match every body");
+            let pause = proxy
+                .pause(&["Lock"], usize::MAX, Some("specific-bucket"))
+                .expect("bucket-scoped lock fault");
+            let unrelated = call_body(&client, proxy.local_addr(), "Lock", Bytes::from_static(b"other-bucket"))
+                .await
+                .expect("unrelated bucket must progress");
+            let unrelated = unrelated.into_body().collect().await.expect("unrelated response");
+            assert_eq!(unrelated.trailers().expect("unchanged trailers")["x-proof"], "unchanged-proof");
+            assert_eq!(unrelated.to_bytes(), Bytes::from_static(b"other-bucket"));
+            assert_eq!(pause.hits(), 0);
+            let task_client = client.clone();
+            let address = proxy.local_addr();
+            let blocked =
+                tokio::spawn(
+                    async move { call_body(&task_client, address, "Lock", Bytes::from_static(b"specific-bucket")).await },
+                );
+            pause.wait_for_hits(1).await.expect("selected bucket reached");
+            assert!(!blocked.is_finished());
+            pause.discard_pending();
+            drop(pause); // Drop must not replace Discard with Forward.
+            assert!(blocked.await.expect("request task").is_err());
+            assert_eq!(backend.received.load(Ordering::SeqCst), 1, "discard must never execute at the peer");
+            let recovered = call(&client, address, "Lock").await.expect("future requests recover");
+            recovered.into_body().collect().await.expect("recovered response");
+            assert_eq!(backend.received.load(Ordering::SeqCst), 2);
+        })
+        .await
+        .expect("scoped discard deadline");
+    }
+    #[test]
+    fn delete_scope_matches_only_the_upload_root() {
+        let frame = |volume: &str, path: &str| {
+            let message = DeleteRequest {
+                volume: volume.to_owned(),
+                path: path.to_owned(),
+                options: r#"{"recursive":true}"#.to_owned(),
+                ..Default::default()
+            };
+            let payload = message.encode_to_vec();
+            let length = u32::try_from(payload.len()).expect("small control packet");
+            let mut bytes = vec![0];
+            bytes.extend_from_slice(&length.to_be_bytes());
+            bytes.extend_from_slice(&payload);
+            bytes
+        };
+        let root = b"hash/upload";
+        assert!(scope_matches("Delete", &frame(".rustfs.sys/multipart", "hash/upload"), root).expect("root frame"));
+        assert!(scope_matches("Delete", &frame(".rustfs.sys/multipart", "hash/upload/"), root).expect("directory frame"));
+        for path in ["hash/upload/part.1", "hash/upload/.abort-authority", "hash/upload-other"] {
+            assert!(!scope_matches("Delete", &frame(".rustfs.sys/multipart", path), root).expect("other path"));
+        }
+        assert!(!scope_matches("Delete", &frame(".rustfs.sys/tmp", "hash/upload"), root).expect("other volume"));
+        let non_recursive = DeleteRequest {
+            volume: ".rustfs.sys/multipart".to_owned(),
+            path: "hash/upload".to_owned(),
+            options: r#"{"recursive":false}"#.to_owned(),
+            ..Default::default()
+        }
+        .encode_to_vec();
+        let mut non_recursive_frame = vec![0];
+        non_recursive_frame.extend_from_slice(&u32::try_from(non_recursive.len()).expect("small frame").to_be_bytes());
+        non_recursive_frame.extend_from_slice(&non_recursive);
+        assert!(!scope_matches("Delete", &non_recursive_frame, root).expect("non-recursive frame"));
+        assert!(
+            scope_matches("Delete", &[0, 0, 0, 0, 1], root).is_err(),
+            "truncation must not become a match"
+        );
+    }
+}

crates/e2e_test/src/storage_api.rs

@@ -15,12 +15,14 @@
 #[cfg(test)]
 pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys;
 #[cfg(test)]
-pub(crate) use rustfs_ecstore::api::disk::{Endpoint, RUSTFS_META_BUCKET, VolumeInfo, WalkDirOptions};
+pub(crate) use rustfs_ecstore::api::disk::{
+    DiskError, DiskOption, Endpoint, RUSTFS_META_BUCKET, VolumeInfo, WalkDirOptions, new_disk,
+};
 pub(crate) use rustfs_ecstore::api::rpc::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
 #[cfg(test)]
 pub(crate) use rustfs_ecstore::api::rpc::{
     TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
-    verify_tonic_boot_epoch_response,
+    set_tonic_canonical_body_digest, verify_tonic_boot_epoch_response,
 };
 #[cfg(test)]
 pub(crate) use rustfs_ecstore::api::rpc::{gen_tonic_signature_interceptor, node_service_time_out_client};
@@ -36,6 +38,11 @@ pub(crate) mod grpc_lock {
     pub(crate) use super::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
 }
 
+#[cfg(test)]
+pub(crate) mod artifact_rpc {
+    pub(crate) use super::{DiskError, TONIC_RPC_PREFIX, gen_tonic_signature_headers, set_tonic_canonical_body_digest};
+}
+
 /// Signing/transport surface used by the cross-process internode RPC signature
 /// acceptance tests (backlog#1327). The signing helpers are what let a test mint
 /// the one legitimately signed request an on-path attacker is assumed to have
@@ -52,3 +59,8 @@ pub(crate) mod internode_rpc_signature {
 pub(crate) mod replication_extension {
     pub(crate) use super::BucketTargetSys;
 }
+
+#[cfg(test)]
+pub(crate) mod upgrade_compatibility {
+    pub(crate) use super::{DiskOption, Endpoint, new_disk};
+}

crates/e2e_test/src/upgrade_compatibility_test.rs

@@ -14,7 +14,7 @@
 
 use crate::common::{
     RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging, replication_fast_env, rustfs_binary_path,
-    signed_request,
+    signal_process, signed_request,
 };
 use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, Operation as FakeTargetOperation};
 use crate::on_demand_migration::common::{ODM_SERVER_ENV, OdmTestEnv, SeedObject, fake_source_client};
@@ -1372,6 +1372,353 @@ async fn rc5_rollback_requires_restoring_odm_configuration() -> TestResult {
     Ok(())
 }
 
+struct Rc5CheckpointCapture {
+    disk: PathBuf,
+    path: PathBuf,
+    task_id: String,
+    bytes: Vec<u8>,
+}
+
+async fn capture_rc5_checkpoint(drives: &[PathBuf], writer_pid: u32) -> Result<Rc5CheckpointCapture, BoxError> {
+    let has_signed_progress = |value: &Value| {
+        value["schema_version"].as_u64() == Some(6)
+            && value["successful_objects"].as_u64().is_some_and(|count| count > 0)
+            && value["processed_objects"]
+                .as_array()
+                .is_some_and(|objects| !objects.is_empty())
+            && value["integrity_digest"].as_str().is_some_and(|digest| !digest.is_empty())
+    };
+    tokio::time::timeout(Duration::from_secs(60), async {
+        loop {
+            for drive in drives {
+                let mut entries = match tokio::fs::read_dir(drive.join(".rustfs.sys/buckets")).await {
+                    Ok(entries) => entries,
+                    Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
+                    Err(error) => return Err(error.into()),
+                };
+                while let Some(entry) = entries.next_entry().await? {
+                    if !entry.file_name().to_string_lossy().ends_with("_ahm_checkpoint.json") {
+                        continue;
+                    }
+                    let metadata = entry.metadata().await?;
+                    if !metadata.is_file() || metadata.len() > 1024 * 1024 {
+                        return Err("the old checkpoint is not a bounded regular file".into());
+                    }
+                    let bytes = match tokio::fs::read(entry.path()).await {
+                        Ok(bytes) => bytes,
+                        Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
+                        Err(error) => return Err(error.into()),
+                    };
+                    let Ok(value) = serde_json::from_slice::<Value>(&bytes) else {
+                        continue;
+                    };
+                    if !has_signed_progress(&value) {
+                        continue;
+                    }
+
+                    // Freeze only this fixture's writer before copying its actual
+                    // bytes. The environment's normal stop path kills a stopped child.
+                    signal_process(writer_pid, "STOP")?;
+                    let bytes = tokio::fs::read(entry.path()).await?;
+                    let value: Value = serde_json::from_slice(&bytes)?;
+                    if !has_signed_progress(&value) {
+                        return Err("the stopped rc.5 writer did not retain signed checkpoint progress".into());
+                    }
+                    let task_id = value["task_id"].as_str().ok_or("the old checkpoint has no task id")?;
+                    uuid::Uuid::parse_str(task_id)?;
+                    if entry.file_name() != std::ffi::OsStr::new(&format!("{task_id}_ahm_checkpoint.json")) {
+                        return Err("the old checkpoint task id does not match its filename".into());
+                    }
+                    return Ok(Rc5CheckpointCapture {
+                        disk: drive.clone(),
+                        path: entry.path(),
+                        task_id: task_id.to_owned(),
+                        bytes,
+                    });
+                }
+            }
+            sleep(Duration::from_millis(5)).await;
+        }
+    })
+    .await
+    .map_err(|_| "rc.5 did not produce a signed checkpoint with progress within 60 seconds")?
+}
+
+fn rc5_checkpoint_object(index: usize) -> (String, Vec<u8>) {
+    (
+        format!("object-{index:05}.bin"),
+        format!("rc.5 checkpoint upgrade object {index:05}\n")
+            .repeat(256)
+            .into_bytes(),
+    )
+}
+
+fn checkpoint_binary_sha256(path: &Path) -> Result<String, BoxError> {
+    use sha2::Digest;
+    use std::io::Read;
+
+    let mut input = std::fs::File::open(path)?;
+    let mut digest = sha2::Sha256::new();
+    let mut buffer = [0_u8; 64 * 1024];
+    loop {
+        let count = input.read(&mut buffer)?;
+        if count == 0 {
+            break;
+        }
+        digest.update(&buffer[..count]);
+    }
+    Ok(hex_simd::encode_to_string(digest.finalize(), hex_simd::AsciiCase::Lower))
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[ignore = "requires the pinned 1.0.0-rc.5 release binary"]
+async fn rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects() -> TestResult {
+    use crate::storage_api::upgrade_compatibility::{DiskOption, Endpoint, new_disk};
+    use futures::{StreamExt, TryStreamExt, stream};
+    use rustfs_heal::heal::CheckpointManager;
+
+    const BUCKET: &str = "upgrade-rc5-checkpoint";
+    const OBJECTS: usize = 1600;
+
+    init_logging();
+    let previous_binary = source_binary()?;
+    assert_eq!(binary_source_revision(&previous_binary).await?, RC5_COMMIT);
+    let current_binary = rustfs_binary_path();
+    let evidence_dir = std::env::var_os(G09_EVIDENCE_DIR_ENV)
+        .map(|root| PathBuf::from(root).join(format!("rc5-checkpoint-upgrade-{}", uuid::Uuid::new_v4())));
+    if let Some(directory) = &evidence_dir {
+        std::fs::create_dir_all(directory)?;
+    }
+
+    // Declare the server after its extra drives so error unwinding stops it
+    // before the fixture directories are removed.
+    let fixture = tempfile::TempDir::new()?;
+    let extra_drives = (0..3)
+        .map(|index| {
+            let path = fixture.path().join(format!("drive-{index}"));
+            std::fs::create_dir(&path)?;
+            Ok(path.to_str().ok_or("fixture drive path is not UTF-8")?.to_owned())
+        })
+        .collect::<Result<Vec<_>, BoxError>>()?;
+    let mut env = RustFSTestEnvironment::new().await?;
+    let log_path = evidence_dir.as_deref().unwrap_or(fixture.path()).join("server.log");
+    env.capture_log_path = Some(log_path.to_str().ok_or("fixture log path is not UTF-8")?.to_owned());
+    let server_env = [
+        ("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true"),
+        ("RUSTFS_DURABILITY_MODE", "strict"),
+        ("RUSTFS_NEW_BUCKET_DURABILITY_MODE", "strict"),
+        ("RUST_LOG", "warn,rustfs::heal=debug"),
+    ];
+    env.start_rustfs_server_from_binary(&previous_binary, extra_drives.iter().map(String::as_str).collect(), &server_env)
+        .await?;
+    env.create_test_bucket(BUCKET).await?;
+    let (status, durability) = admin_request(
+        &env.url,
+        Method::GET,
+        &format!("/rustfs/admin/v3/bucket-durability/{BUCKET}"),
+        None,
+        &env.access_key,
+        &env.secret_key,
+    )
+    .await?;
+    assert_eq!(status, StatusCode::OK, "old bucket durability: {durability}");
+    assert_eq!(serde_json::from_str::<Value>(&durability)?["mode"], "strict");
+
+    let old_client = env.create_s3_client();
+    let manifest = stream::iter(0..OBJECTS)
+        .map(|index| {
+            let client = old_client.clone();
+            async move {
+                let (key, body) = rc5_checkpoint_object(index);
+                let entry = serde_json::json!({
+                    "bucket": BUCKET,
+                    "key": key,
+                    "length": body.len(),
+                    "sha256": rustfs_utils::hex_sha256(&body, str::to_owned),
+                });
+                client
+                    .put_object()
+                    .bucket(BUCKET)
+                    .key(&key)
+                    .body(ByteStream::from(body))
+                    .send()
+                    .await?;
+                Ok::<_, BoxError>(entry)
+            }
+        })
+        .buffered(4)
+        .try_collect::<Vec<_>>()
+        .await?;
+    let drives = extra_drives
+        .iter()
+        .map(PathBuf::from)
+        .chain(std::iter::once(PathBuf::from(&env.temp_dir)))
+        .collect::<Vec<_>>();
+    let writer_pid = env.process.as_ref().ok_or("rc.5 writer is not running")?.id();
+    let start_heal = async {
+        let (status, body) = tokio::time::timeout(
+            Duration::from_secs(15),
+            admin_request(
+                &env.url,
+                Method::POST,
+                "/rustfs/admin/v3/heal/",
+                Some(
+                    serde_json::json!({
+                        "recursive": true, "dryRun": false, "remove": false, "recreate": false,
+                        "scanMode": 2, "updateParity": false, "nolock": false, "pool": 0, "set": 0,
+                    })
+                    .to_string(),
+                ),
+                &env.access_key,
+                &env.secret_key,
+            ),
+        )
+        .await??;
+        if status != StatusCode::OK {
+            return Err(format!("rc.5 root heal was not accepted: {status} {body}").into());
+        }
+        Ok::<_, BoxError>(body)
+    };
+    let (capture, heal_start) = tokio::try_join!(capture_rc5_checkpoint(&drives, writer_pid), start_heal)?;
+    env.stop_server();
+    assert_eq!(
+        std::fs::read(&capture.path)?,
+        capture.bytes,
+        "stopping the old writer must preserve captured bytes"
+    );
+    let checkpoint_dir = capture.disk.join(".rustfs.sys/buckets");
+    let resume_path = checkpoint_dir.join(format!("{}_ahm_resume_state.json", capture.task_id));
+    let resume_bytes = std::fs::read(&resume_path)?;
+    let old_checkpoint: Value = serde_json::from_slice(&capture.bytes)?;
+    if let Some(directory) = &evidence_dir {
+        std::fs::write(directory.join("checkpoint.rc5.json"), &capture.bytes)?;
+        std::fs::write(directory.join("resume.rc5.json"), &resume_bytes)?;
+        std::fs::write(directory.join("manifest.json"), serde_json::to_vec_pretty(&manifest)?)?;
+        std::fs::write(
+            directory.join("source.json"),
+            serde_json::to_vec_pretty(&serde_json::json!({
+                "kind": "old-release-checkpoint-reader-upgrade",
+                "scope": "schema-6 reader authentication, safe rewind and object reads; not full G09 qualification",
+                "writer_revision": RC5_COMMIT,
+                "writer_binary_sha256": checkpoint_binary_sha256(&previous_binary)?,
+                "candidate_revision": current_source_revision()?,
+                "candidate_binary_sha256": checkpoint_binary_sha256(&current_binary)?,
+                "layout": "one process and four logical drives; not physical fault qualification",
+                "task_id": capture.task_id,
+                "checkpoint_sha256": rustfs_utils::hex_sha256(&capture.bytes, str::to_owned),
+                "resume_sha256": rustfs_utils::hex_sha256(&resume_bytes, str::to_owned),
+                "acknowledged_objects": manifest.len(),
+                "bucket_durability": serde_json::from_str::<Value>(&durability)?,
+                "heal_start": serde_json::from_str::<Value>(&heal_start)?,
+            }))?,
+        )?;
+    }
+
+    let blocked_path = checkpoint_dir.join(format!("{}_ahm_checkpoint.blocked", capture.task_id));
+    assert!(!blocked_path.try_exists()?, "the original checkpoint must not already be blocked");
+    let disk_options = DiskOption {
+        cleanup: false,
+        health_check: false,
+    };
+    // Reopen with the same member order passed to the old writer.
+    let disk_index = drives
+        .iter()
+        .position(|drive| drive == &capture.disk)
+        .ok_or("captured disk is not a fixture member")?;
+    let mut endpoint = Endpoint::try_from(capture.disk.to_str().ok_or("captured drive path is not UTF-8")?)?;
+    endpoint.set_pool_index(0);
+    endpoint.set_set_index(0);
+    endpoint.set_disk_index(disk_index);
+    let disk = new_disk(&endpoint, &disk_options).await?;
+    let manager = CheckpointManager::load_from_disk(disk.clone(), &capture.task_id).await?;
+    let rewound = manager.get_checkpoint().await;
+    assert_eq!(rewound.schema_version, 7);
+    assert_eq!(rewound.task_id, capture.task_id);
+    assert_eq!((rewound.current_bucket_index, rewound.current_object_index), (0, 0));
+    assert!(rewound.processed_objects.is_empty() && rewound.failed_objects.is_empty() && rewound.skipped_objects.is_empty());
+    assert_eq!(rewound.successful_objects, 0);
+    assert_eq!(rewound.failed_object_count, 0);
+    assert_eq!(rewound.skipped_object_count, 0);
+    assert_eq!(rewound.processed_bytes, 0);
+    assert_eq!((rewound.total_objects, rewound.total_bytes), (0, 0));
+    assert!(rewound.baseline_generation.is_none() && !rewound.baseline_known && !rewound.counter_unknown);
+    assert!(!blocked_path.try_exists()?, "an authentic old checkpoint must not be quarantined");
+    assert_eq!(
+        std::fs::read(&capture.path)?,
+        capture.bytes,
+        "the reader must not rewrite the old checkpoint"
+    );
+    assert_eq!(std::fs::read(&resume_path)?, resume_bytes, "the reader must not take over the old task");
+    drop(manager);
+    drop(disk);
+
+    // A separate copy with one digest byte changed proves that successful
+    // rewind above did not bypass authentication of the original wire bytes.
+    let corrupt_root = fixture.path().join("corrupt-reader-control");
+    let corrupt_metadata = corrupt_root.join(".rustfs.sys/buckets");
+    std::fs::create_dir_all(&corrupt_metadata)?;
+    let mut corrupted = capture.bytes.clone();
+    let digest = old_checkpoint["integrity_digest"]
+        .as_str()
+        .ok_or("old checkpoint digest is not a string")?;
+    let offset = corrupted
+        .windows(digest.len())
+        .position(|bytes| bytes == digest.as_bytes())
+        .ok_or("the digest is absent from the captured bytes")?;
+    corrupted[offset] = if corrupted[offset] == b'A' { b'B' } else { b'A' };
+    let corrupt_path = corrupt_metadata.join(format!("{}_ahm_checkpoint.json", capture.task_id));
+    std::fs::write(&corrupt_path, &corrupted)?;
+    let endpoint = Endpoint::try_from(corrupt_root.to_str().ok_or("control disk path is not UTF-8")?)?;
+    let corrupt_disk = new_disk(&endpoint, &disk_options).await?;
+    let Err(error) = CheckpointManager::load_from_disk(corrupt_disk.clone(), &capture.task_id).await else {
+        return Err("the current checkpoint reader accepted a corrupted signature".into());
+    };
+    assert!(matches!(error, rustfs_heal::Error::InvalidCheckpoint(ref reason) if reason.contains("digest")));
+    let corrupt_blocked = corrupt_metadata.join(format!("{}_ahm_checkpoint.blocked", capture.task_id));
+    assert!(corrupt_blocked.try_exists()?, "the corrupted copy must retain a blocked marker");
+    assert_eq!(std::fs::read(&corrupt_path)?, corrupted, "quarantine must preserve the invalid bytes");
+    if let Some(directory) = &evidence_dir {
+        std::fs::write(directory.join("checkpoint.corrupted.json"), &corrupted)?;
+        std::fs::copy(&corrupt_blocked, directory.join("checkpoint.corrupted.blocked"))?;
+    }
+    drop(corrupt_disk);
+
+    env.restart_server_preserving_data(extra_drives.iter().map(String::as_str).collect(), &server_env)
+        .await?;
+    let current_client = env.create_s3_client();
+    stream::iter(0..OBJECTS)
+        .map(|index| {
+            let client = current_client.clone();
+            async move {
+                let (key, expected) = rc5_checkpoint_object(index);
+                let (_, actual) = read_object(&client, BUCKET, &key, None).await?;
+                assert_eq!(actual, expected, "old object {key} changed during the upgrade");
+                Ok::<_, BoxError>(())
+            }
+        })
+        .buffer_unordered(4)
+        .try_collect::<Vec<_>>()
+        .await?;
+    env.stop_server();
+    if let Some(directory) = &evidence_dir {
+        std::fs::write(
+            directory.join("result.json"),
+            serde_json::to_vec_pretty(&serde_json::json!({
+                "status": "passed",
+                "scope": "old-release checkpoint reader and object-data upgrade only",
+                "authenticated_original_bytes": true,
+                "original_checkpoint_unchanged_after_read": true,
+                "old_successful_objects": old_checkpoint["successful_objects"],
+                "rewound_schema": rewound.schema_version,
+                "rewound_successful_objects": rewound.successful_objects,
+                "corrupt_digest_rejected_and_retained": true,
+                "verified_objects": OBJECTS,
+            }))?,
+        )?;
+    }
+    Ok(())
+}
+
 #[tokio::test]
 #[ignore = "requires a pinned previous RustFS release binary"]
 async fn direct_upgrade_from_previous_release_preserves_object_contracts() -> TestResult {

crates/ecstore/src/api/mod.rs

@@ -382,6 +382,43 @@ pub mod config {
             with_config_object_read_lock, with_config_object_write_lock, with_server_config_read_lock,
             with_server_config_write_lock,
         };
+
+        /// Observe persisted config bytes and their reader-associated metadata within a byte limit.
+        pub async fn read_config_observation(
+            api: std::sync::Arc<crate::store::ECStore>,
+            file: &str,
+            max_bytes: usize,
+        ) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
+            crate::config::com::read_config_observation(api, file, max_bytes).await
+        }
+
+        /// Observe bounded config bytes while the caller retains the object's write owner.
+        pub async fn read_config_observation_no_lock(
+            api: std::sync::Arc<crate::store::ECStore>,
+            file: &str,
+            max_bytes: usize,
+            guard: &crate::object_api::WriteCommitGuard,
+        ) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
+            crate::config::com::read_config_observation_no_lock(api, file, max_bytes, guard).await
+        }
+
+        /// Read config bytes and metadata without treating an empty payload as absent.
+        pub async fn read_config_preserve_empty_with_metadata_opts(
+            api: std::sync::Arc<crate::store::ECStore>,
+            file: &str,
+            opts: &crate::object_api::ObjectOptions,
+        ) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
+            crate::config::com::read_config_preserve_empty_with_metadata_opts(api, file, opts).await
+        }
+
+        pub async fn read_config_limited_preserve_empty_with_metadata_opts(
+            api: std::sync::Arc<crate::store::ECStore>,
+            file: &str,
+            opts: &crate::object_api::ObjectOptions,
+            max_bytes: usize,
+        ) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
+            crate::config::com::read_config_limited_preserve_empty_with_metadata_opts(api, file, opts, max_bytes).await
+        }
     }
 
     pub mod storageclass {
@@ -431,16 +468,19 @@ pub mod disk {
     pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
     #[cfg(all(feature = "test-util", not(windows)))]
     pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
+    pub use crate::disk::{ArtifactInventoryFrame, ArtifactInventoryVersion};
     pub use crate::disk::{
         BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
         CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration, CurrentDeleteTransition,
         DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
         DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DeleteRecoveryState,
         Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, FileReader, FileWriter,
-        GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
-        PartTransactionAction, QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp,
-        ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
-        is_quota_mutation_fence_path, new_disk, validate_batch_read_version_item_count,
+        GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEALING_MARKER_PATH, LegacyFileStream, LegacyFileWrite,
+        LegacyMetadataWrite, LegacyPartMutation, LegacyPartOperation, LegacyPathDelete, LegacyPathDeleteOperation,
+        LegacyRenameData, LegacyRenameFile, NsScannerOpenRequest, OldCurrentSize, PartTransactionAction,
+        QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
+        STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path,
+        new_disk, validate_batch_read_version_item_count,
     };
     pub use bytes::Bytes;
     pub use endpoint::Endpoint;
@@ -491,6 +531,8 @@ pub mod global {
 }
 
 pub mod runtime {
+    #[cfg(any(test, feature = "test-util"))]
+    pub use crate::runtime::global::set_object_store_resolver;
     pub use crate::runtime::instance::{InstanceContext, bootstrap_ctx};
     pub use crate::runtime::sources::{
         boot_time, bucket_monitor, deployment_id, endpoint_pools, expiry_state_handle, first_cluster_node_is_local,
@@ -587,12 +629,14 @@ pub mod rpc {
         decode_heal_bucket_rpc_options, encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers,
         gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
         node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
-        sign_ns_scanner_capability, sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability,
-        sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
-        tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation,
-        verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
-        verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned,
-        verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
+        sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
+        sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
+        tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
+        verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
+        verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer, verify_put_file_capability,
+        verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
+        verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
+        verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
     };
 }
 
@@ -631,6 +675,7 @@ pub mod store_list {
 }
 
 pub mod storage {
+    pub use crate::bucket::artifact::inventory::ArtifactAttemptInventoryFrame;
     pub use crate::core::pools::{HealLifecycleExpiryContext, POOL_META_NAME};
     pub use crate::store::HealWalkVersion;
     pub use crate::store::{

crates/ecstore/src/bucket/artifact/backup_retention.rs

@@ -0,0 +1,120 @@
+// Copyright 2026 RustFS Team
+// SPDX-License-Identifier: Apache-2.0
+
+use super::*;
+use crate::bucket::metadata::BUCKET_INCARNATION_FILE;
+use crate::config::com::read_config_observation;
+use crate::object_api::WriteCompletion;
+
+impl ECStore {
+    /// Permanently retain this managed bucket's versions for historical restores.
+    /// Lifecycle WRITE drains earlier exact deletes. The detached owner keeps
+    /// lifecycle -> metadata transaction locks until every metadata write drains.
+    pub async fn retain_artifact_backups(self: Arc<Self>, bucket: &str, incarnation: Uuid, base_prefix: &str) -> Result<()> {
+        let bucket = bucket.to_owned();
+        let base_prefix = base_prefix.to_owned();
+        let ctx = Arc::clone(&self.ctx);
+        ctx.run_detached_mutation(async move {
+            require_put_seal_topology(&self).await?;
+            if self.ctx.lock_manager().is_disabled() {
+                return Err(Error::other("artifact backup retention requires namespace locking"));
+            }
+            let lifecycle = self.acquire_bucket_lifecycle_write_lock(&bucket).await?;
+            let transaction = metadata_sys::acquire_bucket_metadata_transaction_lock_in(&self.ctx, &bucket).await?;
+            let mut metadata = self.backup_retention_metadata(&bucket, incarnation, &base_prefix).await?;
+            let check = || {
+                if lifecycle.is_lock_lost() || transaction.is_lock_lost() {
+                    Err(Error::other("artifact backup retention fence was lost"))
+                } else {
+                    Ok(())
+                }
+            };
+            check()?;
+            // Even an idempotent retry rewrites with TailDrained: an earlier
+            // interrupted call may have reached only some metadata replicas.
+            metadata.artifact_backup_retained = true;
+            let mut opts = ObjectOptions {
+                max_parity: true,
+                write_completion: WriteCompletion::TailDrained,
+                ..Default::default()
+            };
+            opts.add_bucket_lifecycle_lock_guard(&lifecycle);
+            opts.add_namespace_lock_guard(&transaction);
+            metadata.save_with_store_options(self.clone(), &opts).await?;
+            check()?;
+            let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
+            let sys = sys.read().await.clone();
+            sys.set_validated(metadata).await?;
+            check()
+        })
+        .await
+        .map_err(|_| Error::other("artifact backup retention owner failed"))?
+    }
+
+    /// Observe the immutable bucket identity and retention in one locked read.
+    pub async fn inspect_artifact_backup_retention(&self, bucket: &str, incarnation: Uuid, base_prefix: &str) -> Result<bool> {
+        require_put_seal_topology(self).await?;
+        if self.ctx.lock_manager().is_disabled() {
+            return Err(Error::other("artifact backup retention requires namespace locking"));
+        }
+        let lifecycle = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
+        let metadata = self.backup_retention_metadata(bucket, incarnation, base_prefix).await?;
+        if lifecycle.is_lock_lost() {
+            return Err(Error::other("artifact backup retention observation fence was lost"));
+        }
+        Ok(metadata.artifact_backup_retained)
+    }
+
+    async fn backup_retention_metadata(&self, bucket: &str, incarnation: Uuid, base_prefix: &str) -> Result<BucketMetadata> {
+        if incarnation.is_nil() || !crate::storage_api_contracts::bucket::valid_artifact_base_prefix(base_prefix) {
+            return Err(Error::other("artifact backup retention identity is invalid"));
+        }
+        let mut metadata = self.read_backup_retention_metadata(bucket).await?;
+        let store = metadata_sys::object_store_in(&self.ctx).await?;
+        let (sidecar, _) =
+            read_config_observation(store, &format!("{BUCKET_META_PREFIX}/{bucket}/{BUCKET_INCARNATION_FILE}"), 16).await?;
+        metadata.parse_all_configs()?;
+        if metadata.bucket_incarnation_id != incarnation
+            || Uuid::from_slice(&sidecar).ok() != Some(incarnation)
+            || metadata.artifact_base_prefix.as_deref() != Some(base_prefix)
+            || !metadata.bucket_creation_committed
+            || !metadata.versioning_config.as_ref().is_some_and(|versioning| {
+                versioning.enabled()
+                    && !versioning.exclude_folders.unwrap_or(false)
+                    && versioning.excluded_prefixes.as_ref().is_none_or(Vec::is_empty)
+            })
+        {
+            return Err(Error::other("artifact backup retention identity mismatch"));
+        }
+        metadata.bucket_incarnation_sidecar = true;
+        Ok(metadata)
+    }
+
+    /// Caller must hold the bucket lifecycle fence. An unreadable or missing
+    /// record cannot prove that historical backup protection was never enabled.
+    pub(crate) async fn require_no_artifact_backup_retention(&self, bucket: &str) -> Result<()> {
+        if self.read_backup_retention_metadata(bucket).await?.artifact_backup_retained {
+            return Err(Error::other("artifact versions are retained for backups"));
+        }
+        Ok(())
+    }
+
+    async fn read_backup_retention_metadata(&self, bucket: &str) -> Result<BucketMetadata> {
+        if bucket.is_empty()
+            || super::super::utils::has_bad_path_component(bucket)
+            || super::super::utils::is_meta_bucketname(bucket)
+        {
+            return Err(Error::other("artifact backup retention bucket is invalid"));
+        }
+        let store = metadata_sys::object_store_in(&self.ctx).await?;
+        if store.id != self.id {
+            return Err(Error::other("artifact backup retention belongs to another store"));
+        }
+        let (bytes, _) = read_config_observation(store, &BucketMetadata::new(bucket).save_file_path(), 64 * 1024 * 1024).await?;
+        let metadata = BucketMetadata::decode_file(&bytes)?;
+        if metadata.name != bucket {
+            return Err(Error::other("artifact backup retention bucket mismatch"));
+        }
+        Ok(metadata)
+    }
+}

crates/ecstore/src/bucket/artifact/delete.rs

@@ -0,0 +1,154 @@
+// Copyright 2026 RustFS Team
+// SPDX-License-Identifier: Apache-2.0
+
+use super::*;
+use crate::storage_api_contracts::object::ObjectOperations as _;
+
+/// In-process authority for exactly one drained managed version. No caller can
+/// construct this capability or serialize it into a disk/RPC request.
+#[doc(hidden)]
+pub struct ArtifactDeleteTarget {
+    store: Uuid,
+    record: ArtifactVersionRetirementRecord,
+    lifecycle: Arc<rustfs_lock::NamespaceLockGuard>,
+    attempt: WriteCommitGuard,
+    repair: WriteCommitGuard,
+}
+
+impl ArtifactDeleteTarget {
+    fn check_live(&self) -> Result<()> {
+        if self.lifecycle.is_lock_lost() || self.attempt.is_lock_lost() || self.repair.is_lock_lost() {
+            return Err(Error::other("artifact exact delete owned fence was lost"));
+        }
+        Ok(())
+    }
+
+    pub(crate) fn lifecycle(&self) -> Arc<rustfs_lock::NamespaceLockGuard> {
+        Arc::clone(&self.lifecycle)
+    }
+
+    pub(crate) fn validate(&self, store: &ECStore, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
+        if self.store != store.id
+            || bucket != self.record.bucket
+            || rustfs_utils::path::encode_dir_object(object) != self.record.object
+            || opts.expected_bucket_incarnation_id != Some(self.record.incarnation_id)
+            || opts.version_id.as_deref() != Some(self.record.version_id.to_string().as_str())
+            || !opts.versioned
+            || opts.version_suspended
+            || opts.delete_prefix
+            || opts.delete_marker
+            || opts.data_movement
+            || opts.lifecycle_delete_all.is_some()
+            || self.lifecycle.is_lock_lost()
+            || self.attempt.is_lock_lost()
+            || self.repair.is_lock_lost()
+        {
+            return Err(Error::other("artifact exact delete target or owned fence mismatch"));
+        }
+        Ok(())
+    }
+}
+
+impl ECStore {
+    /// Read-only capability observation; each mutation revalidates the topology.
+    pub async fn supports_artifact_exact_delete(&self) -> bool {
+        require_local_put_seal_topology(self).await.is_ok() && !self.ctx.lock_manager().is_disabled()
+    }
+
+    /// Delete an exact version in a fully sealed attempt. This authenticates the
+    /// storage identity and drains writers; the caller still owns GC eligibility
+    /// and cleanup IAM. Distributed receiver deletion is not yet supported.
+    pub async fn delete_artifact_object_version(
+        self: Arc<Self>,
+        bucket: &str,
+        expected_incarnation: Uuid,
+        attempt_prefix: &str,
+        object: &str,
+        version: Uuid,
+    ) -> Result<()> {
+        require_local_put_seal_topology(&self).await?;
+        if self.ctx.lock_manager().is_disabled() {
+            return Err(Error::other("artifact exact deletion requires namespace locking"));
+        }
+        let lifecycle = Arc::new(self.acquire_bucket_lifecycle_read_lock(bucket).await?);
+        self.require_no_artifact_backup_retention(bucket).await?;
+        let put = self
+            .inventory_artifact_record(bucket, expected_incarnation, attempt_prefix)
+            .await?;
+        let repair = ArtifactRepairSealRecord::new(&put);
+        let record = ArtifactVersionRetirementRecord::new(&put, object, version)?;
+        let object = object.to_owned();
+        let store = Arc::clone(&self);
+        // Lifecycle -> attempt -> repair seal -> metadata transaction -> object.
+        // The spawned owner, and the disk deletion tail's typed capability, keep
+        // these locks after cancellation of the HTTP caller.
+        with_config_object_write_lock(self, put.path(), move |attempt| async move {
+            if !put.exists(Arc::clone(&store), &attempt).await? {
+                return Err(Error::other("artifact exact deletion requires sealed PUTs"));
+            }
+            let lock = store.new_ns_lock(RUSTFS_META_BUCKET, &repair.path()).await?;
+            let repair_owner = WriteCommitGuard::acquire(&lock, get_lock_acquire_timeout()).await?;
+            if !repair.exists(Arc::clone(&store), &repair_owner).await? {
+                return Err(Error::other("artifact exact deletion requires sealed repairs"));
+            }
+            let receivers = store.capture_artifact_receivers().await?;
+            store
+                .close_artifact_receiver_snapshot(&put, crate::disk::local::ArtifactLeaseDomain::Put, &receivers)
+                .await?;
+            store
+                .close_artifact_receiver_snapshot(&put, crate::disk::local::ArtifactLeaseDomain::Repair, &receivers)
+                .await?;
+            store.verify_artifact_receiver_snapshot(&put, &receivers).await?;
+            let target = Arc::new(ArtifactDeleteTarget {
+                store: store.id,
+                record,
+                lifecycle,
+                attempt,
+                repair: repair_owner,
+            });
+            let mut opts = ObjectOptions {
+                versioned: true,
+                version_id: Some(version.to_string()),
+                expected_bucket_incarnation_id: Some(expected_incarnation),
+                artifact_delete_target: Some(Arc::clone(&target)),
+                delete_mutation_owner: Some(target.clone()),
+                ..Default::default()
+            };
+            opts.add_write_commit_guard(&target.attempt);
+            opts.add_write_commit_guard(&target.repair);
+            // These owners fence seal records, not the object. Ordinary DELETE
+            // must still acquire and retain its exact object WRITE owner.
+            opts.no_lock = false;
+            target.check_live()?;
+            // Preserve the normal exact DELETE's Object Lock, quorum, journal,
+            // cache and capacity behavior. Its success is logical deletion, not
+            // a claim that every offline shard has been physically erased.
+            match store.delete_object(&put.bucket, &object, opts).await {
+                Ok(info) => {
+                    // Match the S3 request layer's committed-generation quota
+                    // delta. A strict absence retry must not subtract twice.
+                    if !info.name.is_empty()
+                        && !info.delete_marker
+                        && let Ok(size) = crate::data_usage::quota_object_size(&info)
+                    {
+                        crate::data_usage::record_bucket_object_delete_memory(&put.bucket, size, false).await;
+                    }
+                }
+                Err(Error::VersionNotFound(bucket, object, version))
+                    if bucket == target.record.bucket
+                        && rustfs_utils::path::encode_dir_object(&object) == target.record.object
+                        && version == target.record.version_id.to_string() => {}
+                Err(Error::ObjectNotFound(bucket, object))
+                    if bucket == target.record.bucket
+                        && rustfs_utils::path::encode_dir_object(&object) == target.record.object => {}
+                Err(error) => return Err(error),
+            }
+            store.verify_artifact_receiver_snapshot(&put, &receivers).await?;
+            store
+                .inventory_artifact_record(&put.bucket, expected_incarnation, &put.attempt_prefix)
+                .await?;
+            target.check_live()
+        })
+        .await?
+    }
+}

crates/ecstore/src/bucket/artifact/inventory.rs

@@ -0,0 +1,207 @@
+// Copyright 2026 RustFS Team
+// SPDX-License-Identifier: Apache-2.0
+
+use super::*;
+use crate::disk::{ArtifactInventoryFrame, ArtifactInventoryVersion, Disk};
+use rustfs_protos::proto_gen::node_service::ArtifactInventoryPhase;
+use tokio::{sync::mpsc, time::Instant};
+
+/// Physical observations retain their member. Replicas are not extra business
+/// versions and this stream makes no quorum-readability or GC claim.
+#[derive(Debug)]
+pub enum ArtifactAttemptInventoryFrame {
+    Observations {
+        endpoint: String,
+        disk_id: Uuid,
+        versions: Vec<ArtifactInventoryVersion>,
+    },
+    Complete {
+        members: usize,
+        observations: u64,
+    },
+}
+
+impl ECStore {
+    /// Inventory one fully sealed attempt across every configured receiver.
+    /// Earlier output is staging only; only the final frame plus Ok is complete.
+    pub async fn inventory_artifact_attempt(
+        self: Arc<Self>,
+        bucket: &str,
+        incarnation: Uuid,
+        prefix: &str,
+        deadline: Instant,
+        output: mpsc::Sender<ArtifactAttemptInventoryFrame>,
+    ) -> Result<()> {
+        tokio::time::timeout_at(deadline, Box::pin(async move {
+            require_put_seal_topology(&self).await?;
+            if self.ctx.lock_manager().is_disabled() { return Err(Error::other("artifact inventory requires namespace locking")); }
+            let lifecycle = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
+            let put = self.inventory_artifact_record(bucket, incarnation, prefix).await?;
+            let repair = ArtifactRepairSealRecord::new(&put);
+            // Lifecycle -> attempt -> repair record. Receivers do not reenter
+            // distributed locks; actual two-domain disk owners guard their reads.
+            let store = self.clone();
+            let bucket = bucket.to_owned();
+            let prefix = prefix.to_owned();
+            with_config_object_write_lock(self, put.path(), move |attempt| async move {
+                tokio::time::timeout_at(deadline, async {
+                    if !put.exists(store.clone(), &attempt).await? { return Err(Error::other("artifact inventory requires PUT seal record")); }
+                    let lock = store.new_ns_lock(RUSTFS_META_BUCKET, &repair.path()).await?;
+                    let repair_owner = WriteCommitGuard::acquire(&lock, get_lock_acquire_timeout()).await?;
+                    if !repair.exists(store.clone(), &repair_owner).await? { return Err(Error::other("artifact inventory requires repair seal record")); }
+                    let receivers = store.capture_artifact_receivers().await?;
+                    let mut observations = 0u64;
+                    for (endpoint, (disk, expected)) in &receivers {
+                        if lifecycle.is_lock_lost() || attempt.is_lock_lost() || repair_owner.is_lock_lost() {
+                            return Err(Error::other("artifact inventory fence was lost"));
+                        }
+                        let (sender, mut frames) = mpsc::channel(2);
+                        let scan = async {
+                            match disk.as_ref() {
+                                Disk::Local(local) => local.get_disk().inventory_artifact_attempt(*expected, &bucket, &prefix, put.identity(), deadline, sender).await,
+                                Disk::Remote(remote) => remote.inventory_artifact_attempt(*expected, (&bucket, incarnation), &prefix, ArtifactInventoryPhase::ArtifactInventoryScan, deadline, sender).await,
+                            }
+                        };
+                        tokio::pin!(scan);
+                        let mut sequence = 0;
+                        let mut count = 0u64;
+                        let mut terminal = false;
+                        let mut ended = false;
+                        let mut done = false;
+                        while !ended || !done {
+                            tokio::select! {
+                                result = &mut scan, if !done => { result.map_err(Error::other)?; done = true; },
+                                frame = frames.recv(), if !ended => match frame {
+                                    Some(ArtifactInventoryFrame::Entries { sequence: actual, versions }) if !terminal && actual == sequence => {
+                                        count = count.checked_add(u64::try_from(versions.len()).map_err(Error::other)?).ok_or_else(|| Error::other("artifact inventory count overflow"))?;
+                                        output.send(ArtifactAttemptInventoryFrame::Observations { endpoint: endpoint.clone(), disk_id: *expected, versions }).await
+                                            .map_err(|_| Error::other("artifact inventory consumer closed"))?;
+                                        sequence += 1;
+                                    }
+                                    Some(ArtifactInventoryFrame::Complete { disk_id, incarnation: actual, attempt_prefix, entries })
+                                        if !terminal && disk_id == *expected && actual == incarnation && attempt_prefix == prefix && entries == count => terminal = true,
+                                    Some(_) => return Err(Error::other("artifact inventory member stream is invalid")),
+                                    None => ended = true,
+                                },
+                            }
+                        }
+                        if !terminal { return Err(Error::other("artifact inventory member is incomplete")); }
+                        observations = observations.checked_add(count).ok_or_else(|| Error::other("artifact inventory count overflow"))?;
+                    }
+                    store.verify_inventory_members(&put, &receivers, deadline).await?;
+                    let current = store.artifact_inventory_receiver_identity(&bucket, incarnation, &prefix).await?;
+                    if current.receiver_key != put.identity().receiver_key || lifecycle.is_lock_lost() || attempt.is_lock_lost() || repair_owner.is_lock_lost() {
+                        return Err(Error::other("artifact inventory authority changed"));
+                    }
+                    output.send(ArtifactAttemptInventoryFrame::Complete { members: receivers.len(), observations }).await
+                        .map_err(|_| Error::other("artifact inventory consumer closed"))?;
+                    Ok(())
+                }).await.map_err(|_| Error::other("artifact inventory deadline exceeded"))?
+            }).await?
+        })).await.map_err(|_| Error::other("artifact inventory deadline exceeded"))?
+    }
+
+    pub(super) async fn inventory_artifact_record(&self, bucket: &str, expected: Uuid, prefix: &str) -> Result<PutSealRecord> {
+        use crate::bucket::metadata::BUCKET_INCARNATION_FILE;
+        use crate::config::com::read_config_observation;
+        if expected.is_nil()
+            || crate::bucket::utils::has_bad_path_component(bucket)
+            || bucket.is_empty()
+            || crate::bucket::utils::is_meta_bucketname(bucket)
+        {
+            return Err(Error::other("artifact inventory authority is invalid"));
+        }
+        let store = metadata_sys::object_store_in(&self.ctx).await?;
+        if store.id != self.id {
+            return Err(Error::other("artifact inventory authority belongs to another store"));
+        }
+        // Do not enter metadata_sys's lazy bucket READ/migration path: the
+        // coordinator already owns lifecycle READ and a writer may be queued.
+        // Observation reads bypass cache and suppress physical read repair.
+        let path = BucketMetadata::new(bucket).save_file_path();
+        let (bytes, _) = read_config_observation(store.clone(), &path, 64 * 1024 * 1024).await?;
+        let mut metadata = BucketMetadata::decode_file(&bytes)?;
+        let (sidecar, _) =
+            read_config_observation(store, &format!("{BUCKET_META_PREFIX}/{bucket}/{BUCKET_INCARNATION_FILE}"), 16).await?;
+        if metadata.name != bucket
+            || !metadata.bucket_creation_committed
+            || metadata.bucket_incarnation_id != expected
+            || Uuid::from_slice(&sidecar).ok() != Some(expected)
+        {
+            return Err(Error::other("artifact inventory incarnation mismatch"));
+        }
+        metadata.parse_all_configs()?;
+        if !metadata.versioning_config.as_ref().is_some_and(|config| {
+            config.enabled()
+                && !config.exclude_folders.unwrap_or(false)
+                && config.excluded_prefixes.as_ref().is_none_or(Vec::is_empty)
+        }) {
+            return Err(Error::other("artifact inventory requires managed versioning"));
+        }
+        PutSealRecord::new(&metadata, prefix)
+    }
+
+    pub(crate) async fn artifact_inventory_receiver_identity(
+        &self,
+        bucket: &str,
+        expected: Uuid,
+        prefix: &str,
+    ) -> Result<ArtifactPutIdentity> {
+        Ok(self.inventory_artifact_record(bucket, expected, prefix).await?.identity())
+    }
+
+    async fn verify_inventory_members(
+        &self,
+        record: &PutSealRecord,
+        captured: &ArtifactReceiverSnapshot,
+        deadline: Instant,
+    ) -> Result<()> {
+        for final_pass in [false, true] {
+            let current = self.artifact_receiver_inventory().await?;
+            if current.len() != captured.len()
+                || current
+                    .iter()
+                    .any(|(endpoint, disk)| !captured.get(endpoint).is_some_and(|(old, _)| Arc::ptr_eq(old, disk)))
+            {
+                return Err(Error::other("artifact inventory membership changed"));
+            }
+            if final_pass {
+                for (disk, expected) in captured.values() {
+                    if disk.cached_disk_id().await != Some(*expected) {
+                        return Err(Error::other("artifact inventory expected identity changed"));
+                    }
+                }
+                break;
+            }
+            for (disk, expected) in captured.values() {
+                if disk.cached_disk_id().await != Some(*expected) {
+                    return Err(Error::other("artifact inventory expected identity changed"));
+                }
+                match disk.as_ref() {
+                    Disk::Local(local) => {
+                        let _verification = local
+                            .get_disk()
+                            .verify_artifact_inventory_closed(*expected, record.identity(), deadline)
+                            .await
+                            .map_err(Error::other)?;
+                    }
+                    Disk::Remote(remote) => {
+                        let (output, _unused) = mpsc::channel(1);
+                        remote
+                            .inventory_artifact_attempt(
+                                *expected,
+                                (&record.bucket, record.incarnation_id),
+                                &record.attempt_prefix,
+                                ArtifactInventoryPhase::ArtifactInventoryVerify,
+                                deadline,
+                                output,
+                            )
+                            .await
+                            .map_err(Error::other)?;
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+}

crates/ecstore/src/bucket/artifact/ordinary_classification.rs

@@ -0,0 +1,287 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::future::Future;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, Mutex, Weak};
+
+/// Shares only reads started after a caller arrives; retains no idle results.
+#[derive(Default)]
+pub(crate) struct OrdinaryClassificationRegistry {
+    gates: Mutex<HashMap<String, Weak<Gate>>>,
+    #[cfg(test)]
+    after_unregister: Option<std::sync::Barrier>,
+}
+
+#[derive(Default)]
+struct Gate {
+    started_generation: AtomicU64,
+    completed: tokio::sync::Mutex<Option<CompletedRead>>,
+}
+
+struct CompletedRead {
+    generation: u64,
+    allowed: bool,
+}
+
+struct Registration<'a> {
+    registry: &'a OrdinaryClassificationRegistry,
+    bucket: &'a str,
+    gate: Option<Arc<Gate>>,
+    observed_generation: u64,
+}
+
+impl OrdinaryClassificationRegistry {
+    pub(crate) async fn classify(&self, bucket: &str, read: impl Future<Output = bool>) -> bool {
+        self.register(bucket).classify(read).await
+    }
+
+    fn register<'a>(&'a self, bucket: &'a str) -> Registration<'a> {
+        // Callers requiring physical admission must register after acquiring it.
+        // Release the registry lock before awaiting the gate, then acquire the
+        // existing metadata locks.
+        let mut gates = self.gates.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
+        let gate = gates.get(bucket).and_then(Weak::upgrade).unwrap_or_else(|| {
+            let gate = Arc::new(Gate::default());
+            gates.insert(bucket.to_owned(), Arc::downgrade(&gate));
+            gate
+        });
+        let observed_generation = gate.started_generation.load(Ordering::Acquire);
+        Registration {
+            registry: self,
+            bucket,
+            gate: Some(gate),
+            observed_generation,
+        }
+    }
+}
+
+impl Registration<'_> {
+    async fn classify(self, read: impl Future<Output = bool>) -> bool {
+        let Some(gate) = self.gate.as_ref() else {
+            return false;
+        };
+        let mut completed = gate.completed.lock().await;
+        // Only a read started after this caller registered can authorize it.
+        // Arrivals during a read therefore wait for the next fresh read.
+        if let Some(result) = completed
+            .as_ref()
+            .filter(|result| result.generation > self.observed_generation)
+        {
+            return result.allowed;
+        }
+        let Ok(previous) = gate
+            .started_generation
+            .try_update(Ordering::AcqRel, Ordering::Acquire, |generation| generation.checked_add(1))
+        else {
+            return false;
+        };
+        // Keep the gate through the original bounded metadata read. Cancellation
+        // releases it without publishing a result; another caller can read again.
+        let allowed = read.await;
+        *completed = Some(CompletedRead {
+            generation: previous + 1,
+            allowed,
+        });
+        allowed
+    }
+}
+
+impl Drop for Registration<'_> {
+    fn drop(&mut self) {
+        let Some(gate) = self.gate.take() else {
+            return;
+        };
+        let mut gates = self.registry.gates.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
+        // Strong references are acquired and released under this registry lock,
+        // so concurrent final callers cannot both miss the last reference.
+        if Arc::strong_count(&gate) == 1
+            && gates
+                .get(self.bucket)
+                .is_some_and(|registered| registered.ptr_eq(&Arc::downgrade(&gate)))
+        {
+            gates.remove(self.bucket);
+        }
+        drop(gate);
+        drop(gates);
+        #[cfg(test)]
+        if let Some(barrier) = &self.registry.after_unregister {
+            barrier.wait();
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use futures::poll;
+    use std::task::Poll;
+    use tokio::sync::oneshot;
+
+    fn assert_empty(registry: &OrdinaryClassificationRegistry) {
+        assert!(registry.gates.lock().expect("classification registry").is_empty());
+    }
+
+    #[tokio::test]
+    async fn late_arrivals_share_only_the_next_fail_closed_read() {
+        let registry = OrdinaryClassificationRegistry::default();
+        let reads = AtomicU64::new(0);
+        let (release, waiting) = oneshot::channel();
+        let mut first = Box::pin(registry.classify("bucket", async {
+            reads.fetch_add(1, Ordering::Relaxed);
+            waiting.await.expect("release first read");
+            true
+        }));
+        assert!(poll!(first.as_mut()).is_pending());
+        let mut followers = (0..16)
+            .map(|_| {
+                Box::pin(registry.classify("bucket", async {
+                    reads.fetch_add(1, Ordering::Relaxed);
+                    false
+                }))
+            })
+            .collect::<Vec<_>>();
+        for follower in &mut followers {
+            assert!(poll!(follower.as_mut()).is_pending());
+        }
+        release.send(()).expect("first reader still alive");
+        assert_eq!(poll!(first.as_mut()), Poll::Ready(true));
+        for follower in &mut followers {
+            assert_eq!(poll!(follower.as_mut()), Poll::Ready(false));
+        }
+        assert_eq!(reads.load(Ordering::Relaxed), 2);
+        assert_empty(&registry);
+    }
+
+    #[tokio::test]
+    async fn successive_waves_always_read_again() {
+        let registry = OrdinaryClassificationRegistry::default();
+        let reads = AtomicU64::new(0);
+        for expected in [true, false, true] {
+            assert_eq!(
+                registry
+                    .classify("bucket", async {
+                        reads.fetch_add(1, Ordering::Relaxed);
+                        expected
+                    })
+                    .await,
+                expected
+            );
+            assert_empty(&registry);
+        }
+        assert_eq!(reads.load(Ordering::Relaxed), 3);
+    }
+
+    #[tokio::test]
+    async fn canceled_leader_does_not_publish_or_strand_waiters() {
+        let registry = OrdinaryClassificationRegistry::default();
+        let reads = AtomicU64::new(0);
+        let old_registration = registry.register("bucket");
+        assert!(
+            registry
+                .classify("bucket", async {
+                    reads.fetch_add(1, Ordering::Relaxed);
+                    true
+                })
+                .await
+        );
+        let mut leader = Box::pin(registry.classify("bucket", async {
+            reads.fetch_add(1, Ordering::Relaxed);
+            std::future::pending::<bool>().await
+        }));
+        assert!(poll!(leader.as_mut()).is_pending());
+        let mut old_waiter =
+            Box::pin(old_registration.classify(async { panic!("old waiter must reuse the completed post-registration read") }));
+        assert!(poll!(old_waiter.as_mut()).is_pending());
+        let mut late_waiter = Box::pin(registry.classify("bucket", async {
+            reads.fetch_add(1, Ordering::Relaxed);
+            false
+        }));
+        assert!(poll!(late_waiter.as_mut()).is_pending());
+        drop(leader);
+        assert_eq!(poll!(old_waiter.as_mut()), Poll::Ready(true));
+        assert_eq!(poll!(late_waiter.as_mut()), Poll::Ready(false));
+        assert_eq!(reads.load(Ordering::Relaxed), 3);
+        assert_empty(&registry);
+    }
+
+    #[tokio::test]
+    async fn canceled_follower_does_not_retain_the_gate() {
+        let registry = OrdinaryClassificationRegistry::default();
+        let (release, waiting) = oneshot::channel();
+        let mut leader = Box::pin(registry.classify("bucket", async {
+            waiting.await.expect("release leader");
+            true
+        }));
+        assert!(poll!(leader.as_mut()).is_pending());
+        let mut follower = Box::pin(registry.classify("bucket", async { panic!("canceled follower must not read") }));
+        assert!(poll!(follower.as_mut()).is_pending());
+        drop(follower);
+        assert_eq!(registry.gates.lock().expect("classification registry").len(), 1);
+        release.send(()).expect("leader still alive");
+        assert_eq!(poll!(leader.as_mut()), Poll::Ready(true));
+        assert_empty(&registry);
+    }
+
+    #[test]
+    fn concurrent_final_registrations_remove_the_gate() {
+        let registry = OrdinaryClassificationRegistry {
+            after_unregister: Some(std::sync::Barrier::new(2)),
+            ..Default::default()
+        };
+        let first = registry.register("bucket");
+        let second = registry.register("bucket");
+        // Both destructors reach the unlocked barrier before either returns.
+        // Moving the Arc release after this barrier leaves a stale registry key.
+        std::thread::scope(|scope| {
+            scope.spawn(move || drop(first));
+            scope.spawn(move || drop(second));
+        });
+        assert_empty(&registry);
+    }
+
+    #[tokio::test]
+    async fn buckets_and_stores_do_not_share_gates() {
+        let first_store = OrdinaryClassificationRegistry::default();
+        let second_store = OrdinaryClassificationRegistry::default();
+        let mut blocked = Box::pin(first_store.classify("first", std::future::pending()));
+        assert!(poll!(blocked.as_mut()).is_pending());
+        assert_eq!(poll!(Box::pin(first_store.classify("second", async { true }))), Poll::Ready(true));
+        assert_eq!(poll!(Box::pin(second_store.classify("first", async { false }))), Poll::Ready(false));
+        assert_empty(&second_store);
+        drop(blocked);
+        assert_empty(&first_store);
+    }
+
+    #[tokio::test]
+    async fn generation_exhaustion_fails_closed_without_reading() {
+        let registry = OrdinaryClassificationRegistry::default();
+        let registration = registry.register("bucket");
+        let gate = registration.gate.as_ref().expect("registered gate");
+        gate.started_generation.store(u64::MAX, Ordering::Release);
+        *gate.completed.lock().await = Some(CompletedRead {
+            generation: u64::MAX,
+            allowed: true,
+        });
+        assert!(
+            !registry
+                .classify("bucket", async { panic!("exhausted generation must deny without reading") })
+                .await
+        );
+        assert_eq!(gate.started_generation.load(Ordering::Acquire), u64::MAX);
+        drop(registration);
+        assert_empty(&registry);
+    }
+}

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -890,6 +890,7 @@ pub(crate) async fn cleanup_free_version_exact(
     let bucket_guard = api
         .acquire_bucket_lifecycle_read_lock(&oi.bucket)
         .await
+        .map(Arc::new)
         .map_err(std::io::Error::other)?;
     let (lease, delete_plan) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?;
     let local_object = encode_dir_object(&oi.name);
@@ -958,7 +959,7 @@ pub(crate) async fn cleanup_free_version_exact(
         };
         delete_request.set_tier_free_version();
         if let Err(err) = set
-            .delete_object_version(&oi.bucket, &local_object, &delete_request, false)
+            .delete_object_version_with_lifecycle_owner(&oi.bucket, &local_object, &delete_request, Arc::clone(&bucket_guard))
             .await
             && first_error.is_none()
         {
@@ -8496,6 +8497,104 @@ mod tests {
         }
     }
 
+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    #[serial]
+    async fn artifact_delete_entrypoints_free_version_cleaner_completes_with_queued_lifecycle_writer() {
+        let (disk_paths, ecstore) = setup_test_env().await;
+        let bucket = format!("free-version-queued-writer-{}", Uuid::new_v4());
+        let object = "free-version";
+        create_test_bucket(&ecstore, &bucket).await;
+        let (backend, identity) = register_recovery_mock_tier(&ecstore).await;
+        seed_recoverable_free_version(&disk_paths, &bucket, object, None, Some(identity)).await;
+        let oi = list_tier_free_versions(Arc::clone(&ecstore), 100, None, None, CancellationToken::new())
+            .await
+            .expect("list real persisted free version")
+            .items
+            .into_iter()
+            .find(|oi| oi.bucket == bucket)
+            .expect("free-version task must address this fixture");
+        assert!(oi.transitioned_object.free_version);
+        backend
+            .set_put_remote_version(Some(oi.transitioned_object.version_id.clone()))
+            .await;
+        let remote = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
+            .await
+            .expect("remote fixture lease");
+        remote
+            .put(&oi.transitioned_object.name, ReaderImpl::Body(Bytes::from_static(b"body")), 4)
+            .await
+            .expect("seed exact remote object");
+        drop(remote);
+
+        let barrier = Arc::new(super::FreeVersionPostRemoteDeleteTestBarrier::default());
+        let cleanup_barrier = Arc::clone(&barrier);
+        let cleanup_store = Arc::clone(&ecstore);
+        let cleanup_oi = oi.clone();
+        let mut cleanup = tokio::spawn(async move {
+            super::FREE_VERSION_POST_REMOTE_DELETE_TEST_BARRIER
+                .scope(cleanup_barrier, async move {
+                    super::cleanup_free_version_exact(cleanup_store, &cleanup_oi, &CancellationToken::new()).await
+                })
+                .await
+        });
+        tokio::time::timeout(StdDuration::from_secs(10), barrier.arrived.notified())
+            .await
+            .expect("cleaner owns lifecycle READ and has completed remote DELETE");
+        assert!(!backend.contains(&oi.transitioned_object.name).await);
+        for disk in &disk_paths {
+            assert!(
+                disk.join(&bucket).join(object).join(STORAGE_FORMAT_FILE).is_file(),
+                "local exact cleanup must still be pending at the boundary"
+            );
+        }
+
+        // Resolve first so the pending poll is the actual local lock wait.
+        // A later READ would normally be admitted alongside the cleaner's READ;
+        // its pending state independently proves this WRITE is now queued.
+        let lifecycle = ecstore
+            .new_ns_lock(&bucket, crate::storage_api_contracts::bucket::BUCKET_LIFECYCLE_LOCK_OBJECT)
+            .await
+            .expect("lifecycle namespace");
+        let mut writer = Box::pin(lifecycle.get_write_lock(StdDuration::from_secs(30)));
+        let writer_queued = futures::poll!(&mut writer).is_pending();
+        let mut later_reader = Box::pin(lifecycle.get_read_lock(StdDuration::from_secs(30)));
+        let later_reader_blocked = futures::poll!(&mut later_reader).is_pending();
+        drop(later_reader);
+        barrier.release.notify_one();
+        assert!(writer_queued, "cleaner must retain the real lifecycle READ owner");
+        assert!(later_reader_blocked, "queued lifecycle WRITE must block a later READ");
+
+        // Keep the same WRITE future queued, without cancelling/retrying it.
+        // Re-entering READ here would deadlock behind this waiting writer.
+        let finished = tokio::time::timeout(StdDuration::from_secs(10), &mut cleanup).await;
+        if finished.is_err() {
+            cleanup.abort();
+            drop(writer);
+            let _ = tokio::time::timeout(StdDuration::from_secs(5), &mut cleanup).await;
+            panic!("exact local free-version cleanup re-entered lifecycle READ behind the queued writer");
+        }
+        assert!(
+            finished
+                .expect("bounded cleaner completion")
+                .expect("cleaner task")
+                .expect("exact local cleanup")
+        );
+        let writer = tokio::time::timeout(StdDuration::from_secs(10), writer)
+            .await
+            .expect("queued lifecycle WRITE proceeds after real cleanup drains")
+            .expect("lifecycle WRITE acquired");
+        for disk in &disk_paths {
+            assert!(!disk.join(&bucket).join(object).exists(), "free version remains on a real local disk");
+        }
+        assert_eq!(backend.remove_count().await, 1, "cleaner must delete the exact remote tuple once");
+        drop(writer);
+        ecstore
+            .delete_bucket(&bucket, &DeleteBucketOptions::default())
+            .await
+            .expect("remove empty fixture bucket");
+    }
+
     #[cfg(feature = "test-util")]
     #[tokio::test]
     #[serial]
@@ -14276,9 +14375,14 @@ mod tests {
 
         crate::store::init_local_disks(endpoint_pools.clone()).await.unwrap();
 
-        let ecstore = ECStore::new("127.0.0.1:0".parse().unwrap(), endpoint_pools, CancellationToken::new())
-            .await
-            .unwrap();
+        // Tests start the recovery loop they observe. An automatic startup loop
+        // would race that loop for the same unconsumed worker queue.
+        let ecstore = temp_env::async_with_vars(
+            [(super::ENV_TIER_FREE_VERSION_RECOVERY_ENABLED, Some("false"))],
+            ECStore::new("127.0.0.1:0".parse().unwrap(), endpoint_pools, CancellationToken::new()),
+        )
+        .await
+        .unwrap();
 
         let buckets = ecstore
             .list_bucket(&BucketOptions {
@@ -14850,7 +14954,13 @@ mod tests {
     #[tokio::test]
     #[serial]
     async fn tier_free_version_recovery_loop_observes_tier_configuration_changes() {
+        let automatic_recovery_started = super::TIER_FREE_VERSION_RECOVERY_STARTED.get().copied();
         let (disk_paths, ecstore) = setup_test_env().await;
+        assert_eq!(
+            super::TIER_FREE_VERSION_RECOVERY_STARTED.get().copied(),
+            automatic_recovery_started,
+            "fixture must not start an unowned automatic recovery loop"
+        );
         let manager = ecstore.tier_config_mgr();
         let saved_tiers = std::mem::take(&mut manager.write().await.tiers);
         let runtime_state = install_unconsumed_runtime_expiry_worker(&ecstore, 1).await;
@@ -14860,6 +14970,8 @@ mod tests {
         create_test_bucket(&ecstore, &bucket).await;
         seed_recoverable_free_version(&disk_paths, &bucket, object, None, None).await;
 
+        let page_permits = Arc::new(tokio::sync::Semaphore::new(1));
+        let worker_permits = Arc::clone(&page_permits);
         let (page_tx, mut page_rx) = tokio::sync::mpsc::unbounded_channel();
         let cancel = CancellationToken::new();
         let loop_cancel = cancel.clone();
@@ -14872,7 +14984,16 @@ mod tests {
                 move |bucket, object, cancel| {
                     let api = Arc::clone(&recovery_store);
                     let page_tx = page_tx.clone();
+                    let page_permits = Arc::clone(&worker_permits);
                     async move {
+                        // Advance the scheduler clock only while real disk I/O is
+                        // parked. Otherwise its timeout can win while the blocking
+                        // filesystem operation is still running on wall-clock time.
+                        let permit = tokio::select! {
+                            _ = cancel.cancelled() => return Err(crate::error::Error::other("test recovery canceled")),
+                            permit = page_permits.acquire() => permit.expect("test controls page admission"),
+                        };
+                        permit.forget();
                         let stats =
                             super::recover_configured_tier_free_versions(api, bucket.clone(), object.clone(), cancel).await?;
                         page_tx
@@ -14900,13 +15021,14 @@ mod tests {
         tokio::task::yield_now().await;
         tokio::time::advance(TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL).await;
         tokio::time::resume();
+        page_permits.add_permits(1);
         let (bucket_marker, object_marker, active) = tokio::time::timeout(StdDuration::from_secs(30), page_rx.recv())
             .await
             .expect("new tier should enable the next recovery page")
             .expect("worker should remain running");
         assert!(bucket_marker.is_none());
         assert!(object_marker.is_none());
-        assert_eq!(active.enqueued, 1);
+        assert_eq!(active.enqueued, 1, "recovery page: {active:?}");
         let task = recovery_rx
             .lock()
             .await
@@ -14927,6 +15049,7 @@ mod tests {
         tokio::task::yield_now().await;
         tokio::time::advance(TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL).await;
         tokio::time::resume();
+        page_permits.add_permits(1);
         let (_, _, idle_again) = tokio::time::timeout(StdDuration::from_secs(30), page_rx.recv())
             .await
             .expect("empty configuration should still produce an idle check")

crates/ecstore/src/bucket/lifecycle/metadata_boundary.rs

@@ -63,7 +63,14 @@ pub(crate) async fn get_expiry_configs_for_options(
     bucket: &str,
     opts: &crate::object_api::ObjectOptions,
 ) -> Result<LifecycleExpiryConfigs> {
-    let bucket_incarnation_id = metadata_sys::get_bucket_incarnation_id_for_options_in(&api.ctx, bucket, opts).await?;
+    let bucket_incarnation_id = match opts
+        .object_lock_config_snapshot
+        .as_ref()
+        .and_then(|snapshot| snapshot.validated_bucket_incarnation_for(api.id, bucket, opts.expected_bucket_incarnation_id))
+    {
+        Some(incarnation_id) => incarnation_id,
+        None => metadata_sys::get_bucket_incarnation_id_for_options_in(&api.ctx, bucket, opts).await?,
+    };
     let metadata = get_authoritative_metadata(api, bucket, bucket_incarnation_id).await?;
     let table_bucket_enabled = metadata.table_bucket_enabled();
 
@@ -174,5 +181,36 @@ mod tests {
                 .unwrap()
         );
         assert!(get_expiry_configs(&store_b, bucket).await.unwrap().lifecycle.is_none());
+
+        let snapshot = store_a.object_lock_config_snapshot(bucket).await.unwrap();
+        assert_eq!(
+            snapshot.validated_bucket_incarnation_for(store_a.id, bucket, Some(configs.bucket_incarnation_id)),
+            Some(configs.bucket_incarnation_id)
+        );
+        assert_eq!(
+            snapshot.validated_bucket_incarnation_for(store_a.id, bucket, Some(uuid::Uuid::new_v4())),
+            None
+        );
+        assert_eq!(snapshot.validated_bucket_incarnation_for(store_b.id, bucket, None), None);
+        let opts = crate::object_api::ObjectOptions {
+            object_lock_config_snapshot: Some(snapshot.clone()),
+            expected_bucket_incarnation_id: Some(configs.bucket_incarnation_id),
+            ..Default::default()
+        };
+        assert_eq!(
+            get_expiry_configs_for_options(&store_a, bucket, &opts)
+                .await
+                .unwrap()
+                .bucket_incarnation_id,
+            configs.bucket_incarnation_id
+        );
+        let unguarded = crate::object_api::ObjectLockConfigSnapshot::for_store_bucket(
+            store_a.id,
+            bucket,
+            configs.bucket_incarnation_id,
+            time::OffsetDateTime::UNIX_EPOCH,
+            metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
+        );
+        assert_eq!(unguarded.validated_bucket_incarnation_for(store_a.id, bucket, None), None);
     }
 }

crates/ecstore/src/bucket/metadata.rs

@@ -437,6 +437,8 @@ pub struct BucketMetadata {
     pub created: OffsetDateTime,
     pub lock_enabled: bool, // While marked as unused, it may need to be retained
     pub bucket_incarnation_id: Uuid,
+    pub artifact_base_prefix: Option<String>,
+    pub artifact_backup_retained: bool,
     pub(crate) bucket_incarnation_sidecar: bool,
     pub(crate) bucket_creation_committed: bool,
     pub policy_config_json: Vec<u8>,
@@ -512,6 +514,8 @@ impl Default for BucketMetadata {
             created: OffsetDateTime::UNIX_EPOCH,
             lock_enabled: Default::default(),
             bucket_incarnation_id: Uuid::nil(),
+            artifact_base_prefix: None,
+            artifact_backup_retained: false,
             bucket_incarnation_sidecar: false,
             bucket_creation_committed: false,
             policy_config_json: Default::default(),
@@ -607,7 +611,7 @@ impl BucketMetadata {
     /// legacy migration path, which mints the sidecar first; it must not be
     /// rewritten by this migration.
     pub(crate) fn needs_bucket_creation_commit(&self) -> bool {
-        self.lock_enabled
+        (self.lock_enabled || self.artifact_base_prefix.is_some())
             && !self.bucket_creation_committed
             && self.bucket_incarnation_sidecar
             && !self.bucket_incarnation_id.is_nil()
@@ -704,6 +708,22 @@ impl BucketMetadata {
                     self.bucket_incarnation_id =
                         Uuid::from_slice(&bytes).map_err(|err| Error::other(format!("invalid BucketIncarnationID: {err}")))?;
                 }
+                "ArtifactBackupRetained" => {
+                    if self.artifact_backup_retained || !rmp::decode::read_bool(rd)? {
+                        return Err(Error::other("invalid artifact backup retention marker"));
+                    }
+                    self.artifact_backup_retained = true;
+                }
+                "ArtifactBasePrefix" => {
+                    if self.artifact_base_prefix.is_some() {
+                        return Err(Error::other("duplicate artifact namespace"));
+                    }
+                    let prefix = read_msgp_str(rd)?;
+                    if !crate::storage_api_contracts::bucket::valid_artifact_base_prefix(&prefix) {
+                        return Err(Error::other("invalid artifact namespace"));
+                    }
+                    self.artifact_base_prefix = Some(prefix);
+                }
                 "PolicyConfigJSON" | "PolicyConfigJson" => self.policy_config_json = read_msgp_bin(rd)?,
                 "NotificationConfigXML" | "NotificationConfigXml" => self.notification_config_xml = read_msgp_bin(rd)?,
                 "LifecycleConfigXML" | "LifecycleConfigXml" => self.lifecycle_config_xml = read_msgp_bin(rd)?,
@@ -766,7 +786,7 @@ impl BucketMetadata {
     /// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
     pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
         // Map size: MinIO fields (25) + RustFS extensions (22)
-        let map_len: u32 = 47;
+        let map_len: u32 = 47 + u32::from(self.artifact_base_prefix.is_some()) + u32::from(self.artifact_backup_retained);
         rmp::encode::write_map_len(wr, map_len)?;
 
         // MinIO field order (same as Go struct)
@@ -780,6 +800,15 @@ impl BucketMetadata {
         rmp::encode::write_bool(wr, self.lock_enabled)?;
 
         write_bin_field(wr, "BucketIncarnationID", self.bucket_incarnation_id.as_bytes())?;
+        if let Some(prefix) = &self.artifact_base_prefix {
+            rmp::encode::write_str(wr, "ArtifactBasePrefix")?;
+            rmp::encode::write_str(wr, prefix)?;
+        }
+
+        if self.artifact_backup_retained {
+            rmp::encode::write_str(wr, "ArtifactBackupRetained")?;
+            rmp::encode::write_bool(wr, true)?;
+        }
 
         write_bin_field(wr, "PolicyConfigJSON", &self.policy_config_json)?;
         write_bin_field(wr, "NotificationConfigXML", &self.notification_config_xml)?;
@@ -880,13 +909,28 @@ impl BucketMetadata {
         }
 
         match version {
-            BUCKET_METADATA_VERSION => {}
+            BUCKET_METADATA_VERSION | 2 => {}
             _ => return Err(Error::other("read_bucket_metadata: version invalid")),
         }
 
         Ok(())
     }
 
+    /// Version two is reserved for permanent artifact backup retention. Older
+    /// readers reject its header instead of skipping the protection field.
+    pub(crate) fn decode_file(bytes: &[u8]) -> Result<Self> {
+        Self::check_header(bytes)?;
+        let metadata = Self::unmarshal(&bytes[4..])?;
+        let retained_header = LittleEndian::read_u16(&bytes[2..4]) == 2;
+        if retained_header != metadata.artifact_backup_retained {
+            return Err(Error::other("artifact backup retention header mismatch"));
+        }
+        if metadata.artifact_backup_retained {
+            metadata.validate_artifact_configuration()?;
+        }
+        Ok(metadata)
+    }
+
     pub(crate) fn default_timestamps(&mut self) {
         if self.policy_config_updated_at == OffsetDateTime::UNIX_EPOCH {
             self.policy_config_updated_at = self.created
@@ -1025,6 +1069,9 @@ impl BucketMetadata {
                     }
                     Some(config)
                 };
+                if self.artifact_base_prefix.is_some() && !config.as_ref().is_some_and(artifact_versioning_enabled) {
+                    return Err(Error::other("artifact bucket versioning must remain enabled"));
+                }
                 self.versioning_config_xml = data;
                 self.versioning_config = config;
                 self.versioning_config_updated_at = updated;
@@ -1127,13 +1174,34 @@ impl BucketMetadata {
         &mut self,
         store: std::sync::Arc<crate::store::ECStore>,
         write_completion: WriteCompletion,
+    ) -> Result<()> {
+        let opts = crate::object_api::ObjectOptions {
+            max_parity: true,
+            write_completion,
+            ..Default::default()
+        };
+        self.save_with_store_options(store, &opts).await
+    }
+
+    pub(crate) async fn save_with_store_options(
+        &mut self,
+        store: Arc<ECStore>,
+        opts: &crate::object_api::ObjectOptions,
     ) -> Result<()> {
         self.parse_all_configs()?;
+        self.validate_artifact_update(store.clone()).await?;
         let mut buf: Vec<u8> = vec![0; 4];
 
         LittleEndian::write_u16(&mut buf[0..2], BUCKET_METADATA_FORMAT);
 
-        LittleEndian::write_u16(&mut buf[2..4], BUCKET_METADATA_VERSION);
+        LittleEndian::write_u16(
+            &mut buf[2..4],
+            if self.artifact_backup_retained {
+                2
+            } else {
+                BUCKET_METADATA_VERSION
+            },
+        );
 
         let data = self
             .marshal_msg()
@@ -1141,21 +1209,52 @@ impl BucketMetadata {
 
         buf.extend_from_slice(&data);
 
-        save_config_with_opts(
-            store,
-            self.save_file_path().as_str(),
-            buf,
-            &crate::object_api::ObjectOptions {
-                max_parity: true,
-                write_completion,
-                ..Default::default()
-            },
-        )
-        .await?;
+        save_config_with_opts(store, self.save_file_path().as_str(), buf, opts).await?;
 
         Ok(())
     }
 
+    fn validate_artifact_configuration(&self) -> Result<()> {
+        if self.artifact_backup_retained && (self.artifact_base_prefix.is_none() || !self.bucket_creation_committed) {
+            return Err(Error::other("artifact backup retention requires a committed managed bucket"));
+        }
+        if let Some(prefix) = &self.artifact_base_prefix {
+            if !crate::storage_api_contracts::bucket::valid_artifact_base_prefix(prefix) || self.bucket_incarnation_id.is_nil() {
+                return Err(Error::other("invalid artifact bucket identity"));
+            }
+            let versioning = deserialize::<VersioningConfiguration>(&self.versioning_config_xml)?;
+            if !artifact_versioning_enabled(&versioning) {
+                return Err(Error::other("artifact bucket versioning must remain enabled"));
+            }
+        }
+        Ok(())
+    }
+
+    /// Full metadata replacement must not change a generation's creation identity.
+    pub(crate) async fn validate_artifact_update(&self, store: Arc<ECStore>) -> Result<()> {
+        self.validate_artifact_configuration()?;
+        // An existing empty/corrupt metadata object is not permission to create an identity.
+        match read_config_preserve_empty(store, &self.save_file_path()).await {
+            Ok(data) => {
+                let previous = Self::decode_file(&data)?;
+                previous.validate_artifact_configuration()?;
+                if previous.artifact_backup_retained && !self.artifact_backup_retained {
+                    return Err(Error::other("artifact backup retention cannot be removed"));
+                }
+                // Deletion removes this record before recreation. A caller-supplied
+                // incarnation cannot turn an existing bucket into a new generation.
+                if previous.artifact_base_prefix != self.artifact_base_prefix
+                    || (previous.artifact_base_prefix.is_some() && previous.bucket_incarnation_id != self.bucket_incarnation_id)
+                {
+                    return Err(Error::other("artifact bucket identity is immutable"));
+                }
+            }
+            Err(Error::ConfigNotFound) => {}
+            Err(error) => return Err(error),
+        }
+        Ok(())
+    }
+
     fn parse_policy_config(&mut self) -> Result<()> {
         if !self.policy_config_json.is_empty() {
             self.policy_config = Some(serde_json::from_slice(&self.policy_config_json)?);
@@ -1199,6 +1298,7 @@ impl BucketMetadata {
     /// `rustfs_bucket_metadata_parse_failed_total`, reflected in
     /// `rustfs_bucket_metadata_unparsable_current`, and logged at error level.
     pub(super) fn parse_all_configs(&mut self) -> Result<()> {
+        self.validate_artifact_configuration()?;
         if let Err(e) = self.parse_policy_config() {
             tracing::warn!(
                 event = "bucket_metadata_parse_failed",
@@ -1464,6 +1564,10 @@ impl BucketMetadata {
     }
 }
 
+fn artifact_versioning_enabled(config: &VersioningConfiguration) -> bool {
+    config.enabled() && !config.exclude_folders.unwrap_or(false) && config.excluded_prefixes.as_ref().is_none_or(Vec::is_empty)
+}
+
 pub(crate) async fn load_bucket_incarnation(api: Arc<ECStore>, bucket: &str) -> Result<Option<Uuid>> {
     let path = format!("{BUCKET_META_PREFIX}/{bucket}/{BUCKET_INCARNATION_FILE}");
     let data = match read_config_preserve_empty(api, &path).await {
@@ -1560,9 +1664,7 @@ async fn read_bucket_metadata(api: Arc<ECStore>, bucket: &str) -> Result<BucketM
 
     let data = read_config(api, &file_path).await?;
 
-    BucketMetadata::check_header(&data)?;
-
-    let bm = BucketMetadata::unmarshal(&data[4..])?;
+    let bm = BucketMetadata::decode_file(&data)?;
 
     Ok(bm)
 }
@@ -1586,6 +1688,98 @@ where
 mod test {
     use super::*;
 
+    #[test]
+    fn artifact_metadata_roundtrip_and_corrupt_prefix_rejection() {
+        let mut metadata = BucketMetadata::new("artifacts");
+        metadata.artifact_base_prefix = Some("images/".to_owned());
+        let encoded = metadata.marshal_msg().expect("encode artifact metadata");
+        let decoded = BucketMetadata::unmarshal(&encoded).expect("decode artifact metadata");
+        assert_eq!(decoded.artifact_base_prefix, metadata.artifact_base_prefix, "ARTIFACT_METADATA_ROUNDTRIP");
+        assert_eq!(decoded.bucket_incarnation_id, metadata.bucket_incarnation_id);
+        let legacy = BucketMetadata::new("ordinary")
+            .marshal_msg()
+            .expect("encode ordinary metadata");
+        assert!(
+            BucketMetadata::unmarshal(&legacy)
+                .expect("decode ordinary metadata")
+                .artifact_base_prefix
+                .is_none()
+        );
+        for prefix in ["", "../", "outputs", "a b/"] {
+            let mut invalid = Vec::new();
+            rmp::encode::write_map_len(&mut invalid, 1).expect("encode map");
+            rmp::encode::write_str(&mut invalid, "ArtifactBasePrefix").expect("encode field");
+            rmp::encode::write_str(&mut invalid, prefix).expect("encode invalid prefix");
+            assert!(BucketMetadata::unmarshal(&invalid).is_err(), "ARTIFACT_CORRUPT_PREFIX");
+        }
+    }
+
+    #[test]
+    fn artifact_backup_retention_marker_rejects_false_and_duplicates() {
+        for count in [1, 2] {
+            let mut bytes = Vec::new();
+            rmp::encode::write_map_len(&mut bytes, count).unwrap();
+            for _ in 0..count {
+                rmp::encode::write_str(&mut bytes, "ArtifactBackupRetained").unwrap();
+                rmp::encode::write_bool(&mut bytes, count == 2).unwrap();
+            }
+            assert!(BucketMetadata::unmarshal(&bytes).is_err(), "O02P3_INVALID_RETENTION_MARKER");
+        }
+    }
+
+    #[test]
+    fn artifact_backup_retention_requires_matching_header_and_identity() {
+        let mut metadata = BucketMetadata::new("retained");
+        metadata.bucket_incarnation_id = Uuid::new_v4();
+        metadata.bucket_creation_committed = true;
+        metadata.artifact_base_prefix = Some("outputs/".to_owned());
+        metadata.versioning_config_xml = b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec();
+        let file = |metadata: &BucketMetadata, version: u16| {
+            let mut bytes = Vec::new();
+            bytes.extend_from_slice(&BUCKET_METADATA_FORMAT.to_le_bytes());
+            bytes.extend_from_slice(&version.to_le_bytes());
+            bytes.extend_from_slice(&metadata.marshal_msg().unwrap());
+            bytes
+        };
+        assert!(BucketMetadata::decode_file(&file(&metadata, 1)).is_ok(), "O02P3_ORDINARY_V1_COMPATIBLE");
+        assert!(BucketMetadata::decode_file(&file(&metadata, 2)).is_err(), "O02P3_MARKER_REQUIRED");
+        metadata.artifact_backup_retained = true;
+        assert!(
+            BucketMetadata::decode_file(&file(&metadata, 1)).is_err(),
+            "O02P3_PROTECTION_CANNOT_DOWNGRADE_HEADER"
+        );
+        assert!(
+            BucketMetadata::decode_file(&file(&metadata, 2))
+                .unwrap()
+                .artifact_backup_retained,
+            "O02P3_PROTECTED_V2_ROUNDTRIP"
+        );
+        metadata.artifact_base_prefix = None;
+        assert!(
+            BucketMetadata::decode_file(&file(&metadata, 2)).is_err(),
+            "O02P3_PROTECTION_REQUIRES_MANAGED_BUCKET"
+        );
+    }
+
+    #[test]
+    fn artifact_versioning_cannot_be_disabled_or_excluded() {
+        let mut metadata = BucketMetadata::new("artifacts");
+        metadata.artifact_base_prefix = Some("images/".to_owned());
+        for xml in [
+            "",
+            "<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>",
+            "<VersioningConfiguration><Status>Enabled</Status><ExcludeFolders>true</ExcludeFolders></VersioningConfiguration>",
+            "<VersioningConfiguration><Status>Enabled</Status><ExcludedPrefixes><Prefix>images/</Prefix></ExcludedPrefixes></VersioningConfiguration>",
+        ] {
+            assert!(
+                metadata
+                    .update_config(BUCKET_VERSIONING_CONFIG, xml.as_bytes().to_vec())
+                    .is_err(),
+                "ARTIFACT_VERSIONING_REQUIRED"
+            );
+        }
+    }
+
     /// rustfs/backlog#1734: `StorageError::clone` rebuilds I/O errors from
     /// their text. The unreadable-config refusal must stay typed across that
     /// clone, or a cloned error stops mapping to its retryable S3 response.

crates/ecstore/src/bucket/metadata_sys.rs

@@ -293,8 +293,10 @@ pub(super) fn get_bucket_metadata_sys() -> Result<Arc<RwLock<BucketMetadataSys>>
 pub async fn set_bucket_metadata(bucket: String, bm: BucketMetadata) -> Result<()> {
     let sys = get_bucket_metadata_sys()?;
     let lock = sys.write().await;
-    lock.set(bucket, Arc::new(bm)).await;
-    Ok(())
+    if bucket != bm.name {
+        return Err(Error::other("bucket metadata identity mismatch"));
+    }
+    lock.set_validated(bm).await
 }
 
 pub async fn reload_bucket_metadata(api: Arc<ECStore>, bucket: &str) -> Result<()> {
@@ -585,6 +587,15 @@ pub(crate) async fn get_cached_bucket_incarnation_id_in(
     sys.get_bucket_incarnation_id(bucket).await
 }
 
+pub(crate) async fn get_bucket_incarnation_metadata_in(
+    ctx: &crate::runtime::instance::InstanceContext,
+    bucket: &str,
+) -> Result<Arc<BucketMetadata>> {
+    let sys = bucket_metadata_sys_of(ctx)?;
+    let sys = sys.read().await.clone();
+    sys.get_bucket_incarnation_metadata(bucket).await
+}
+
 pub(crate) async fn set_bucket_metadata_in(ctx: &crate::runtime::instance::InstanceContext, bm: BucketMetadata) -> Result<()> {
     let sys = bucket_metadata_sys_of(ctx)?;
     let lock = sys.read().await;
@@ -2132,7 +2143,16 @@ impl BucketMetadataSys {
         }
     }
 
-    pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
+    pub(crate) async fn set_validated(&self, mut bm: BucketMetadata) -> Result<()> {
+        if bm.artifact_base_prefix.is_some() {
+            bm.parse_all_configs()?;
+        }
+        bm.validate_artifact_update(self.object_store()).await?;
+        self.set(bm.name.clone(), Arc::new(bm)).await;
+        Ok(())
+    }
+
+    async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
         if !is_meta_bucketname(&bucket) {
             let publish_lock = self.metadata_publish_lock(&bucket);
             let _publish_guard = publish_lock.lock().await;
@@ -2147,6 +2167,11 @@ impl BucketMetadataSys {
         }
     }
 
+    #[cfg(any(test, feature = "test-util"))]
+    pub async fn set_for_test(&self, bucket: String, bm: Arc<BucketMetadata>) {
+        self.set(bucket, bm).await;
+    }
+
     /// Remove a bucket's cached metadata from the in-memory map.
     ///
     /// Returns `true` if an entry was present. Reserved meta buckets are ignored.
@@ -2651,12 +2676,16 @@ impl BucketMetadataSys {
     }
 
     async fn get_bucket_incarnation_id(&self, bucket: &str) -> Result<Uuid> {
+        Ok(self.get_bucket_incarnation_metadata(bucket).await?.bucket_incarnation_id)
+    }
+
+    async fn get_bucket_incarnation_metadata(&self, bucket: &str) -> Result<Arc<BucketMetadata>> {
         let authority = self.get_metadata_authority(bucket).await?;
         if let BucketMetadataAuthority::Authoritative(metadata) = &authority
             && metadata.bucket_incarnation_sidecar
             && !metadata.bucket_incarnation_id.is_nil()
         {
-            return Ok(metadata.bucket_incarnation_id);
+            return Ok(Arc::clone(metadata));
         }
 
         match authority {
@@ -2665,7 +2694,7 @@ impl BucketMetadataSys {
                     BucketMetadataAuthority::Authoritative(metadata)
                         if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
                     {
-                        Ok(metadata.bucket_incarnation_id)
+                        Ok(metadata)
                     }
                     BucketMetadataAuthority::Authoritative(_) | BucketMetadataAuthority::Fabricated => {
                         Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}")))
@@ -2993,7 +3022,8 @@ impl BucketMetadataSys {
             load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
         )
         .await?;
-        if persisted && metadata.lock_enabled && !metadata.bucket_creation_committed {
+        if persisted && (metadata.lock_enabled || metadata.artifact_base_prefix.is_some()) && !metadata.bucket_creation_committed
+        {
             return Err(Error::ErasureWriteQuorum);
         }
         if persisted {

crates/ecstore/src/bucket/mod.rs

@@ -14,6 +14,7 @@
 
 // #730: bucket subsystems still contain staged ECStore migration code.
 
+pub mod artifact;
 pub mod bandwidth;
 pub mod bucket_target_sys;
 pub mod config_parse_mode;

crates/ecstore/src/bucket/quota/checker.rs

@@ -235,6 +235,8 @@ impl QuotaChecker {
         Ok(updated_at)
     }
 
+    /// Return quota configuration with optional scanner usage, independently
+    /// of the usage requirements enforced by write admission.
     pub async fn get_quota_stats(&self, bucket: &str) -> Result<(BucketQuota, Option<u64>), QuotaError> {
         // If bucket doesn't exist, return ConfigNotFound error
         if !self.bucket_exists(bucket).await {
@@ -244,37 +246,28 @@ impl QuotaChecker {
         }
 
         let quota = self.get_quota_config(bucket).await?;
-        let current_usage = if quota.uses_durable_reservations() {
-            self.report_only_usage(bucket).await
-        } else {
-            Some(self.get_real_time_usage(bucket).await?)
-        };
+        let current_usage = self.report_only_usage(bucket).await;
 
         Ok((quota, current_usage))
     }
 
-    /// Scanner usage for a bucket the reservation ledger admits is reporting
-    /// input only, so a failed lookup degrades to "unknown" and is counted
-    /// instead of becoming an error the write path would fail on.
+    /// Best-effort usage for statistics and durable admission reports.
+    /// Missing scanner coverage is unknown, not an error or a zero-sized bucket.
     async fn report_only_usage(&self, bucket: &str) -> Option<u64> {
         // The plain lookup: a report must not wait for an early snapshot reload.
         let usage = get_bucket_usage_memory(bucket).await;
-        match self.usage_or_degraded_baseline(bucket, usage).await {
-            Ok(usage) => Some(usage),
-            Err(err) => {
-                metrics::counter!(METRIC_QUOTA_USAGE_REPORT_UNAVAILABLE_TOTAL).increment(1);
-                debug!(
-                    event = EVENT_QUOTA_USAGE_REPORT,
-                    component = LOG_COMPONENT_ECSTORE,
-                    subsystem = LOG_SUBSYSTEM_QUOTA,
-                    state = "usage_unknown",
-                    bucket = %bucket,
-                    error = %err,
-                    "durable quota usage report degraded"
-                );
-                None
-            }
+        if usage.is_none() {
+            metrics::counter!(METRIC_QUOTA_USAGE_REPORT_UNAVAILABLE_TOTAL).increment(1);
+            debug!(
+                event = EVENT_QUOTA_USAGE_REPORT,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_QUOTA,
+                state = "usage_unknown",
+                bucket = %bucket,
+                "quota usage report degraded"
+            );
         }
+        usage
     }
 
     /// Re-save `bucket`'s quota under the sharded durable reservation
@@ -544,6 +537,8 @@ mod tests {
     use serial_test::serial;
     use uuid::Uuid;
 
+    mod reporting_retirement;
+
     /// Regression (PR #5307 / s3-tests `test_100_continue_error_retry`): a
     /// bucket with no persisted metadata has no quota, so the admission check
     /// must pass and let the request reach its NoSuchBucket answer. With the
@@ -744,8 +739,12 @@ mod tests {
             1,
             "fail-closed admission takes the early-reload lookup"
         );
-        assert!(matches!(checker.get_quota_stats(&legacy).await, Err(QuotaError::UsageUnavailable { .. })));
-        assert_eq!(crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&legacy), 2);
+        let (_, usage) = checker
+            .get_quota_stats(&legacy)
+            .await
+            .expect("legacy quota statistics are reporting only");
+        assert_eq!(usage, None);
+        assert_eq!(crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&legacy), 1);
         assert_eq!(
             crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&bucket),
             0,
@@ -896,7 +895,7 @@ mod tests {
             .update_bucket_metadata_config(&bucket, QUOTA_CONFIG_FILE, legacy_quota_json(8192))
             .await
             .expect("newer quota save");
-        sys.read().await.set(bucket.clone(), stale).await;
+        sys.read().await.set_for_test(bucket.clone(), stale).await;
         assert_eq!(
             checker
                 .migrate_reservation_protocol(&bucket)

crates/ecstore/src/bucket/quota/checker/tests/reporting_retirement.rs

@@ -0,0 +1,330 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::*;
+use crate::disk::DiskAPI as _;
+use crate::object_api::{ObjectOptions, PutObjReader};
+use crate::storage_api_contracts::object::ObjectIO as _;
+use crate::test_metrics::CapturingRecorder;
+use rustfs_data_usage::{BucketUsageInfo, DataUsageInfo};
+use std::time::SystemTime;
+
+async fn stop_reporting_fixture(store: &crate::store::ECStore) {
+    if let Some(cancel) = store.ctx.background_cancel_token() {
+        cancel.cancel();
+    }
+    for pool in &store.pools {
+        for set in &pool.disk_set {
+            let disks = set.disks.read().await.clone();
+            for disk in disks.iter().flatten() {
+                disk.close().await.expect("stop fixture disk monitoring");
+            }
+        }
+    }
+}
+
+async fn install_reporting_snapshot(
+    store: Arc<crate::store::ECStore>,
+    bucket: &str,
+    source: &str,
+    size: u64,
+) -> Option<(String, Vec<u8>)> {
+    if source == "missing" {
+        return None;
+    }
+    let mut info = DataUsageInfo {
+        last_update: Some(SystemTime::now()),
+        scanner_epoch: Some(1),
+        scanner_cycle: Some(1),
+        buckets_count: 1,
+        ..Default::default()
+    };
+    info.buckets_usage.insert(
+        bucket.to_owned(),
+        BucketUsageInfo {
+            size,
+            objects_count: 1,
+            versions_count: 1,
+            ..Default::default()
+        },
+    );
+    info.bucket_sizes.insert(bucket.to_owned(), size);
+    info.calculate_totals();
+    let object = match source {
+        "legacy" => rustfs_data_usage::LEGACY_DATA_USAGE_OBJECT_NAME,
+        "incomplete" => rustfs_data_usage::DATA_USAGE_OBJECT_NAME,
+        "complete" => {
+            info.usage_snapshot_complete = true;
+            rustfs_data_usage::DATA_USAGE_OBJECT_NAME
+        }
+        "observed" => {
+            info.usage_snapshot_complete = true;
+            info.usage_snapshot_converged = Some(false);
+            info.usage_snapshot_authoritative_baseline = Some(DataUsageInfo::default().snapshot_identity());
+            rustfs_data_usage::DATA_USAGE_OBSERVED_OBJECT_NAME
+        }
+        _ => panic!("unknown scanner fixture source"),
+    };
+    let path = format!("{}/{object}", crate::disk::BUCKET_META_PREFIX);
+    let bytes = serde_json::to_vec(&info).expect("scanner fixture encoding");
+    crate::config::com::save_config(store, &path, bytes.clone())
+        .await
+        .expect("persist scanner fixture");
+    if source == "complete" {
+        crate::data_usage::replace_bucket_usage_memory_from_info(&info).await;
+    }
+    Some((path, bytes))
+}
+
+#[tokio::test(flavor = "current_thread")]
+#[serial]
+async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger_admission() {
+    let recorder = CapturingRecorder::default();
+    let _recorder = metrics::set_default_local_recorder(&recorder);
+    for protocol in [1, 2] {
+        for source in ["missing", "legacy", "incomplete", "observed", "complete"] {
+            let (_dirs, store, sys) = isolated_durable_quota_store().await;
+            let checker = QuotaChecker::new(sys);
+            let bucket = format!("quota-report-{protocol}-{}", Uuid::new_v4().simple());
+            let quota = if protocol == 1 {
+                BucketQuota::new(Some(4096))
+            } else {
+                BucketQuota::new_sharded(4096)
+            };
+            create_bucket_with_quota_json(&store, &bucket, serde_json::to_vec(&quota).expect("quota fixture encoding")).await;
+            let snapshot = install_reporting_snapshot(store.clone(), &bucket, source, 1_000_000).await;
+            crate::data_usage::invalidate_data_usage_snapshot_cache().await;
+            let unavailable_before = recorder.counter_value(METRIC_QUOTA_USAGE_REPORT_UNAVAILABLE_TOTAL, &[]);
+            let check = checker
+                .check_quota(&bucket, QuotaOperation::PutObject, 512)
+                .await
+                .expect("reporting cannot fail ledger admission");
+            let (_, usage) = checker.get_quota_stats(&bucket).await.expect("best-effort quota statistics");
+            let preview = checker
+                .check_quota_with_usage_reporting(&bucket, QuotaOperation::PutObject, 512, true)
+                .await
+                .expect("admin preview uses the same optional report");
+
+            // Exercise the real storage entry point before testing the new
+            // reporting semantics: the same ledger must admit and refuse
+            // these writes even when a scanner floor claims a million bytes.
+            let mut too_large = PutObjReader::from_vec(vec![0x5a; 8192]);
+            assert!(
+                matches!(
+                    store
+                        .put_object(&bucket, "too-large", &mut too_large, &ObjectOptions::default())
+                        .await,
+                    Err(StorageError::QuotaExceeded { .. })
+                ),
+                "protocol={protocol}, source={source}"
+            );
+            let mut fits = PutObjReader::from_vec(vec![0x5a; 512]);
+            store
+                .put_object(&bucket, "fits", &mut fits, &ObjectOptions::default())
+                .await
+                .expect("ledger admits 512 bytes");
+            let mut fills = PutObjReader::from_vec(vec![0x5a; 3584]);
+            store
+                .put_object(&bucket, "fills", &mut fills, &ObjectOptions::default())
+                .await
+                .expect("ledger admits exactly the remaining quota");
+            let mut exceeds = PutObjReader::from_vec(vec![0x5a; 1]);
+            assert!(
+                matches!(
+                    store
+                        .put_object(&bucket, "exceeds", &mut exceeds, &ObjectOptions::default())
+                        .await,
+                    Err(StorageError::QuotaExceeded { .. })
+                ),
+                "ledger cannot undercount protocol={protocol}, source={source}"
+            );
+
+            assert!(check.allowed && check.uses_durable_reservations);
+            let expected_usage = (source == "complete").then_some(1_000_000);
+            assert_eq!(
+                check.current_usage, expected_usage,
+                "protocol={protocol}, source={source}: a floor is not a current report"
+            );
+            assert_eq!(check.remaining, expected_usage.map(|_| 0));
+            assert_eq!(usage, expected_usage, "statistics use the same best-effort reporting contract");
+            assert_eq!(preview.current_usage, expected_usage);
+            assert_eq!(preview.remaining, expected_usage.map(|_| 0));
+            assert_eq!(preview.allowed, source != "complete", "unknown preview is advisory, not a reservation");
+            assert_eq!(
+                recorder.counter_value(METRIC_QUOTA_USAGE_REPORT_UNAVAILABLE_TOTAL, &[]) - unavailable_before,
+                if source == "complete" { 0 } else { 3 },
+                "each unknown check/statistics/preview report is counted"
+            );
+            assert_eq!(crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&bucket), 0);
+            assert_eq!(persisted_quota(&store, &bucket).await.reservation_protocol, Some(protocol));
+            if let Some((path, original)) = snapshot {
+                assert_eq!(
+                    crate::config::com::read_config(store.clone(), &path)
+                        .await
+                        .expect("retained scanner evidence"),
+                    original
+                );
+            }
+            crate::data_usage::prepare_bucket_usage_for_namespace_change(&bucket, None)
+                .await
+                .expect("fixture cache cleanup");
+            crate::data_usage::invalidate_data_usage_snapshot_cache().await;
+            stop_reporting_fixture(&store).await;
+        }
+    }
+}
+
+#[tokio::test]
+#[serial]
+async fn snapshot_quota_admission_retains_degraded_floors_until_migration() {
+    for source in ["missing", "legacy", "incomplete", "observed"] {
+        let (_dirs, store, _) = isolated_durable_quota_store().await;
+        let bucket = format!("quota-compat-{}", Uuid::new_v4().simple());
+        create_bucket_with_quota_json(&store, &bucket, legacy_quota_json(4096)).await;
+        // An unregistered checker models a snapshot-protocol reader without
+        // starting the optional background migration during this assertion.
+        let checker = QuotaChecker::new(Arc::new(RwLock::new(BucketMetadataSys::new(store.clone()))));
+        install_reporting_snapshot(store.clone(), &bucket, source, 1000).await;
+        crate::data_usage::invalidate_data_usage_snapshot_cache().await;
+        let check = checker.check_quota(&bucket, QuotaOperation::PutObject, 512).await;
+        if source == "missing" {
+            assert!(matches!(check, Err(QuotaError::UsageUnavailable { .. })));
+        } else {
+            let check = check.expect("legacy admission retains its grounded compatibility fallback");
+            assert_eq!(check.current_usage, Some(1000));
+            assert!(check.allowed && !check.uses_durable_reservations);
+            assert!(
+                !checker
+                    .check_quota(&bucket, QuotaOperation::PutObject, 4096)
+                    .await
+                    .expect("legacy quota decision")
+                    .allowed
+            );
+        }
+        assert_eq!(persisted_quota(&store, &bucket).await.reservation_protocol, None);
+        crate::data_usage::prepare_bucket_usage_for_namespace_change(&bucket, None)
+            .await
+            .expect("fixture cache cleanup");
+        crate::data_usage::invalidate_data_usage_snapshot_cache().await;
+        stop_reporting_fixture(&store).await;
+    }
+}
+
+#[tokio::test(flavor = "current_thread")]
+#[serial]
+async fn quota_without_limit_keeps_the_zero_usage_io_fast_path() {
+    let (_dirs, store, sys) = isolated_durable_quota_store().await;
+    let bucket = format!("quota-unlimited-{}", Uuid::new_v4().simple());
+    create_bucket_with_quota_json(
+        &store,
+        &bucket,
+        serde_json::to_vec(&BucketQuota::new(None)).expect("unlimited quota encoding"),
+    )
+    .await;
+    let recorder = CapturingRecorder::default();
+    let _recorder = metrics::set_default_local_recorder(&recorder);
+    let check = QuotaChecker::new(sys)
+        .check_quota(&bucket, QuotaOperation::PutObject, u64::MAX)
+        .await
+        .expect("unlimited quota fast path");
+    assert!(check.allowed);
+    assert_eq!(check.current_usage, None);
+    assert_eq!(check.quota_limit, None);
+    assert_eq!(recorder.counter_value(METRIC_QUOTA_USAGE_REPORT_UNAVAILABLE_TOTAL, &[]), 0);
+    assert_eq!(crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&bucket), 0);
+    stop_reporting_fixture(&store).await;
+}
+
+#[tokio::test]
+#[serial]
+async fn quota_statistics_remain_available_without_a_complete_scanner_snapshot() {
+    use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
+
+    for configuration in ["unconfigured", "unlimited", "snapshot"] {
+        for source in ["missing", "legacy", "incomplete", "observed", "complete", "empty"] {
+            let (_dirs, store, _) = isolated_durable_quota_store().await;
+            let bucket = format!("quota-statistics-{}", Uuid::new_v4().simple());
+            match configuration {
+                "unconfigured" => store
+                    .make_bucket(&bucket, &MakeBucketOptions::default())
+                    .await
+                    .expect("create bucket without a quota configuration"),
+                "unlimited" => {
+                    create_bucket_with_quota_json(
+                        &store,
+                        &bucket,
+                        serde_json::to_vec(&BucketQuota::new(None)).expect("unlimited quota encoding"),
+                    )
+                    .await;
+                }
+                "snapshot" => create_bucket_with_quota_json(&store, &bucket, legacy_quota_json(4096)).await,
+                _ => unreachable!(),
+            }
+            // An unregistered checker keeps legacy migration out of the
+            // assertion: reporting must work before a protocol upgrade.
+            let metadata_sys = BucketMetadataSys::new(store.clone());
+            let (metadata, _) = metadata_sys.get_config(&bucket).await.expect("load existing bucket metadata");
+            metadata_sys.set_for_test(bucket.clone(), metadata).await;
+            let checker = QuotaChecker::new(Arc::new(RwLock::new(metadata_sys)));
+            let (snapshot_source, size) = if source == "empty" { ("complete", 0) } else { (source, 1000) };
+            install_reporting_snapshot(store.clone(), &bucket, snapshot_source, size).await;
+            crate::data_usage::invalidate_data_usage_snapshot_cache().await;
+
+            let statistics = checker.get_quota_stats(&bucket).await;
+            let early_reloads = crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&bucket);
+            let missing_bucket = format!("missing-{}", Uuid::new_v4().simple());
+            let missing_statistics = checker.get_quota_stats(&missing_bucket).await;
+            crate::data_usage::prepare_bucket_usage_for_namespace_change(&bucket, None)
+                .await
+                .expect("fixture cache cleanup");
+            crate::data_usage::invalidate_data_usage_snapshot_cache().await;
+            stop_reporting_fixture(&store).await;
+
+            let (quota, usage) = statistics.unwrap_or_else(|error| {
+                panic!("configuration={configuration}, source={source}: quota statistics must remain available: {error}")
+            });
+            assert_eq!(quota.quota, (configuration == "snapshot").then_some(4096));
+            assert_eq!(
+                usage,
+                (snapshot_source == "complete").then_some(size),
+                "configuration={configuration}, source={source}: only known usage is numeric"
+            );
+            assert_eq!(early_reloads, 0, "read-only statistics must not force an early snapshot reload");
+            assert!(
+                matches!(missing_statistics, Err(QuotaError::ConfigNotFound { bucket }) if bucket == missing_bucket),
+                "reporting must still reject a missing bucket"
+            );
+        }
+    }
+}
+
+#[tokio::test]
+async fn quota_statistics_preserve_configuration_errors() {
+    let (_dirs, store, _) = isolated_durable_quota_store().await;
+    let bucket = format!("quota-invalid-{}", Uuid::new_v4().simple());
+    create_bucket_with_quota_json(&store, &bucket, legacy_quota_json(4096)).await;
+    let metadata_sys = BucketMetadataSys::new(store.clone());
+    let (mut metadata, _) = metadata_sys.get_config(&bucket).await.expect("load existing bucket metadata");
+    Arc::make_mut(&mut metadata).quota_config_json = b"{not-json".to_vec();
+    metadata_sys.set_for_test(bucket.clone(), metadata).await;
+    let checker = QuotaChecker::new(Arc::new(RwLock::new(metadata_sys)));
+
+    let statistics = checker.get_quota_stats(&bucket).await;
+    stop_reporting_fixture(&store).await;
+
+    assert!(
+        matches!(statistics, Err(QuotaError::InvalidConfig { .. })),
+        "unavailable reporting must not hide malformed quota configuration"
+    );
+}

crates/ecstore/src/bucket/quota/reservation.rs

@@ -595,7 +595,7 @@ async fn reserve_sharded(context: QuotaContext, old_size: u64, new_size: u64) ->
         let allow_create = shard_may_be_created(&allocator, data.shard_index, bootstrap);
         let shard_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &data.shard_object).await?;
         let shard_guard = Arc::new(shard_lock.get_write_lock(get_lock_acquire_timeout()).await?);
-        let mut shard = load_current_shard_locked(
+        let (mut shard, shard_created) = load_current_shard_locked(
             Arc::clone(&store),
             &data.shard_object,
             bucket_incarnation,
@@ -640,7 +640,7 @@ async fn reserve_sharded(context: QuotaContext, old_size: u64, new_size: u64) ->
             });
         }
         let shard_was_empty = shard.grants.is_empty() && shard.reservations.is_empty() && shard.accounted_usage == 0;
-        if shard_was_empty {
+        if shard_created || shard_was_empty {
             fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &data.shard_object, None).await?;
             save_shard_locked(Arc::clone(&store), &data.shard_object, &shard, &shard_guard).await?;
         }
@@ -1510,18 +1510,18 @@ async fn load_current_shard_locked(
     shard_index: u16,
     allow_create: bool,
     allocator: &QuotaAllocatorLedger,
-) -> Result<QuotaShardLedger> {
-    let mut shard = match load_shard_locked(Arc::clone(&store), object).await {
-        Ok(shard) if shard.matches(bucket_incarnation, quota_revision, shard_index) => shard,
+) -> Result<(QuotaShardLedger, bool)> {
+    let (mut shard, created) = match load_shard_locked(Arc::clone(&store), object).await {
+        Ok(shard) if shard.matches(bucket_incarnation, quota_revision, shard_index) => (shard, false),
         Ok(_) => return Err(StorageError::PartMissingOrCorrupt),
         Err(StorageError::ConfigNotFound) if allow_create => {
-            QuotaShardLedger::new(bucket_incarnation, quota_revision, shard_index)
+            (QuotaShardLedger::new(bucket_incarnation, quota_revision, shard_index), true)
         }
         Err(StorageError::ConfigNotFound) => return Err(StorageError::PartMissingOrCorrupt),
         Err(err) => return Err(err),
     };
     shard.adopt_grants(allocator)?;
-    Ok(shard)
+    Ok((shard, created))
 }
 
 async fn save_allocator_locked(
@@ -1646,7 +1646,7 @@ async fn reap_sharded_reservations(
     .await?;
     let shard_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &data.shard_object).await?;
     let shard_guard = shard_lock.get_write_lock(get_lock_acquire_timeout()).await?;
-    let shard = load_current_shard_locked(
+    let (shard, _) = load_current_shard_locked(
         Arc::clone(&store),
         &data.shard_object,
         bucket_incarnation,
@@ -1709,7 +1709,7 @@ async fn reap_sharded_reservations(
     .await?;
     let shard_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &data.shard_object).await?;
     let shard_guard = Arc::new(shard_lock.get_write_lock(get_lock_acquire_timeout()).await?);
-    let mut shard = load_current_shard_locked(
+    let (mut shard, _) = load_current_shard_locked(
         Arc::clone(&store),
         &data.shard_object,
         bucket_incarnation,
@@ -1774,7 +1774,7 @@ async fn mark_commit_started_sharded(data: &ShardedReservationData) -> Result<()
         let allocator = load_allocator_locked(Arc::clone(&store), &data.allocator_object).await?;
         let shard_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &data.shard_object).await?;
         let shard_guard = Arc::new(shard_lock.get_write_lock(get_lock_acquire_timeout()).await?);
-        let mut shard = load_current_shard_locked(
+        let (mut shard, _) = load_current_shard_locked(
             Arc::clone(&store),
             &data.shard_object,
             data.bucket_incarnation,
@@ -1819,7 +1819,7 @@ async fn settle_sharded(data: &ShardedReservationData, committed: bool) -> Resul
         let allocator = load_allocator_locked(Arc::clone(&store), &data.allocator_object).await?;
         let shard_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &data.shard_object).await?;
         let shard_guard = Arc::new(shard_lock.get_write_lock(get_lock_acquire_timeout()).await?);
-        let mut shard = load_current_shard_locked(
+        let (mut shard, _) = load_current_shard_locked(
             Arc::clone(&store),
             &data.shard_object,
             data.bucket_incarnation,
@@ -2535,4 +2535,70 @@ mod tests {
             1
         );
     }
+
+    #[tokio::test]
+    #[serial]
+    async fn sharded_reservation_persists_a_populated_lazy_shard_before_refill() {
+        use crate::object_api::PutObjReader;
+        use crate::storage_api_contracts::object::ObjectIO;
+
+        let (_directories, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
+        metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
+        crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
+        let bucket = format!("quota-populated-bootstrap-{}", Uuid::new_v4().simple());
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create quota bucket");
+
+        let mut seed_shard = None;
+        let mut seed_name = None;
+        let mut candidate_name = None;
+        let mut bootstrap_name = None;
+        for index in 0..10_000 {
+            let object = format!("quota-populated-object-{index}");
+            let object_shard = shard_index(&object);
+            if seed_shard.is_none() {
+                seed_shard = Some(object_shard);
+                seed_name = Some(object);
+            } else if Some(object_shard) == seed_shard && candidate_name.is_none() {
+                candidate_name = Some(object);
+            } else if Some(object_shard) != seed_shard && bootstrap_name.is_none() {
+                bootstrap_name = Some(object);
+            }
+            if candidate_name.is_some() && bootstrap_name.is_some() {
+                break;
+            }
+        }
+        let seed_name = seed_name.expect("find a seeded shard");
+        let candidate_name = candidate_name.expect("find a second object in the seeded shard");
+        let bootstrap_name = bootstrap_name.expect("find an empty bootstrap shard");
+
+        let mut seed = PutObjReader::from_vec(vec![1]);
+        store
+            .put_object(&bucket, &seed_name, &mut seed, &ObjectOptions::default())
+            .await
+            .expect("seed existing usage before enabling v2");
+        store
+            .update_bucket_metadata_config(
+                &bucket,
+                "quota.json",
+                serde_json::to_vec(&super::super::BucketQuota::new_sharded(100_000)).expect("serialize v2 quota"),
+            )
+            .await
+            .expect("enable v2 quota");
+
+        let mut bootstrap = PutObjReader::from_vec(vec![2]);
+        store
+            .put_object(&bucket, &bootstrap_name, &mut bootstrap, &ObjectOptions::default())
+            .await
+            .expect("bootstrap an empty shard");
+
+        let mut candidate = PutObjReader::from_vec(vec![3]);
+        let result = store
+            .put_object(&bucket, &candidate_name, &mut candidate, &ObjectOptions::default())
+            .await
+            .expect("a populated shard must persist before refill and accept its first growth");
+        assert_eq!(result.size, 1);
+    }
 }

crates/ecstore/src/cache_value/metacache_set.rs

@@ -25,7 +25,7 @@ use std::{
     future::Future,
     io::ErrorKind,
     pin::Pin,
-    sync::{Arc, OnceLock, atomic::AtomicBool},
+    sync::{Arc, OnceLock},
     task::{Context, Poll},
     time::Duration,
 };
@@ -40,7 +40,7 @@ const LOG_COMPONENT_ECSTORE: &str = "ecstore";
 const LOG_SUBSYSTEM_METACACHE: &str = "metacache";
 const EVENT_METACACHE_LISTING: &str = "metacache_listing";
 
-pub type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
+pub type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = Result<(), DiskError>> + Send>> + Send + 'static>;
 pub type PartialFn =
     Box<dyn Fn(MetaCacheEntries, &[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
 pub(crate) type PartialResultFn = Box<
@@ -198,6 +198,8 @@ pub(crate) enum TestReaderBehavior {
         stopped: Arc<tokio::sync::Notify>,
     },
     IgnoreCancel,
+    /// Entries followed by a terminator trailer saying the producer stopped at its limit.
+    LimitReached(Vec<MetaCacheEntry>),
     ProducerError(DiskError),
     PrimaryErrorThenFallback(DiskError),
     PartialThenTimeout(Vec<MetaCacheEntry>),
@@ -236,8 +238,6 @@ pub struct ListPathRawOptions {
     pub skip_walkdir_total_timeout: bool,
     pub walkdir_timeout: Option<Duration>,
     pub walkdir_stall_timeout: Option<Duration>,
-    /// Shared terminal state for bounded producer walks.
-    pub producer_limit_reached: Option<Arc<AtomicBool>>,
     pub agreed: Option<AgreedFn>,
     pub partial: Option<PartialFn>,
     pub finished: Option<FinishedFn>,
@@ -248,7 +248,7 @@ pub struct ListPathRawOptions {
     #[cfg(test)]
     pub(crate) peek_timeout: Option<Duration>,
     #[cfg(test)]
-    pub(crate) test_disk_io_context_observed: Option<Arc<AtomicBool>>,
+    pub(crate) test_disk_io_context_observed: Option<Arc<std::sync::atomic::AtomicBool>>,
     // pub agreed: Option<Arc<dyn Fn(MetaCacheEntry) + Send + Sync>>,
     // pub partial: Option<Arc<dyn Fn(MetaCacheEntries, &[Option<Error>]) + Send + Sync>>,
     // pub finished: Option<Arc<dyn Fn(&[Option<Error>]) + Send + Sync>>,
@@ -273,7 +273,6 @@ impl Clone for ListPathRawOptions {
             skip_walkdir_total_timeout: self.skip_walkdir_total_timeout,
             walkdir_timeout: self.walkdir_timeout,
             walkdir_stall_timeout: self.walkdir_stall_timeout,
-            producer_limit_reached: self.producer_limit_reached.clone(),
             #[cfg(test)]
             test_reader_behaviors: self.test_reader_behaviors.clone(),
             #[cfg(test)]
@@ -301,22 +300,31 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
         skip_total_timeout: opts.skip_walkdir_total_timeout,
         timeout_ms: opts.walkdir_timeout.map(duration_millis),
         stall_timeout_ms: opts.walkdir_stall_timeout.map(duration_millis),
-        producer_limit_reached: opts.producer_limit_reached.clone(),
         ..Default::default()
     }
 }
 
+/// How a merged raw listing stream ended.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub struct ListPathRawOutcome {
+    /// At least one producer stopped at its per-disk limit instead of at the
+    /// end of its namespace, so the closed stream is not proof of exhaustion.
+    /// Carried in-band by the metacache stream trailer, so remote producers
+    /// report it as well.
+    pub limit_reached: bool,
+}
+
 pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
     let rx = rx.child_token();
     let _cancel_guard = rx.clone().drop_guard();
-    list_path_raw_inner(rx, opts, None, None).await
+    list_path_raw_inner(rx, opts, None, None).await.map(|_| ())
 }
 
 pub(crate) async fn list_path_raw_with_claim_tracker(
     rx: CancellationToken,
     opts: ListPathRawOptions,
     claim_tracker: FallbackClaimTracker,
-) -> disk::error::Result<()> {
+) -> disk::error::Result<ListPathRawOutcome> {
     let rx = rx.child_token();
     let _cancel_guard = rx.clone().drop_guard();
     list_path_raw_inner(rx, opts, Some(claim_tracker), None).await
@@ -327,7 +335,7 @@ pub(crate) async fn list_path_raw_with_partial_result(
     opts: ListPathRawOptions,
     claim_tracker: FallbackClaimTracker,
     partial_result: PartialResultFn,
-) -> disk::error::Result<()> {
+) -> disk::error::Result<ListPathRawOutcome> {
     let rx = rx.child_token();
     let _cancel_guard = rx.clone().drop_guard();
     list_path_raw_inner(rx, opts, Some(claim_tracker), Some(partial_result)).await
@@ -338,7 +346,7 @@ async fn list_path_raw_inner(
     opts: ListPathRawOptions,
     fallback_claim_tracker: Option<FallbackClaimTracker>,
     partial_result: Option<PartialResultFn>,
-) -> disk::error::Result<()> {
+) -> disk::error::Result<ListPathRawOutcome> {
     if opts.disks.is_empty() {
         return Err(DiskError::ErasureReadQuorum);
     }
@@ -393,6 +401,14 @@ async fn list_path_raw_inner(
                         out.close().await.expect("test entries should close");
                         return Ok(());
                     }
+                    TestReaderBehavior::LimitReached(entries) => {
+                        let mut out = rustfs_filemeta::MetacacheWriter::new(&mut wr);
+                        out.write(&entries).await.expect("test entries should be written");
+                        out.close_with_limit_reached(true)
+                            .await
+                            .expect("test entries should close at the limit");
+                        return Ok(());
+                    }
                     TestReaderBehavior::Stall => {
                         let _held_writer = wr;
                         cancel_rx_clone.cancelled().await;
@@ -531,6 +547,16 @@ async fn list_path_raw_inner(
                             last_err = None;
                             continue;
                         }
+                        TestReaderBehavior::LimitReached(entries) => {
+                            let mut out = rustfs_filemeta::MetacacheWriter::new(&mut wr);
+                            out.write(&entries).await.expect("test fallback entries should be written");
+                            out.close_with_limit_reached(true)
+                                .await
+                                .expect("test fallback entries should close at the limit");
+                            need_fallback = false;
+                            last_err = None;
+                            continue;
+                        }
                         TestReaderBehavior::ProducerError(err) | TestReaderBehavior::PrimaryErrorThenFallback(err) => {
                             last_err = Some(err);
                             continue;
@@ -699,6 +725,9 @@ async fn list_path_raw_inner(
         }
         let mut pending_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
         let mut peek_outcomes: Vec<Option<PeekOutcome>> = std::iter::repeat_with(|| None).take(readers.len()).collect();
+        // Sticky across rounds: once any reader's trailer says its producer
+        // stopped at the per-disk limit, the merged stream may hide more keys.
+        let mut limit_reached = false;
 
         loop {
             let mut current = MetaCacheEntry::default();
@@ -710,7 +739,7 @@ async fn list_path_raw_inner(
             // );
 
             if revjob_rx.is_cancelled() {
-                return Ok(());
+                return Ok(ListPathRawOutcome { limit_reached });
             }
 
             let mut top_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
@@ -760,6 +789,7 @@ async fn list_path_raw_inner(
                                     continue;
                                 }
                                 // eof
+                                limit_reached |= r.limit_reached() == Some(true);
                                 at_eof += 1;
                                 // warn!("list_path_raw: peek eof, disk: {}", i);
                                 continue;
@@ -833,7 +863,7 @@ async fn list_path_raw_inner(
                             *r = MetacacheReader::new(detached_rd);
                             continue;
                         }
-                        PeekOutcome::Cancelled => return Ok(()),
+                        PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached }),
                     }
                 };
 
@@ -892,7 +922,7 @@ async fn list_path_raw_inner(
                 if let Some(finished_fn) = opts.finished.as_ref() {
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(()),
+                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
                         _ = finished_fn(&errs) => {}
                     }
                 }
@@ -949,7 +979,7 @@ async fn list_path_raw_inner(
                 {
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(()),
+                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
                         _ = finished_fn(&errs) => {}
                     }
                 }
@@ -969,8 +999,8 @@ async fn list_path_raw_inner(
                     // warn!("list_path_raw: agreed_fn start, current: {:?}", &current.name);
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(()),
-                        _ = agreed_fn(current) => {}
+                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                        result = agreed_fn(current) => result?
                     }
                     // warn!("list_path_raw: agreed_fn done");
                 }
@@ -993,58 +1023,61 @@ async fn list_path_raw_inner(
             if let Some(partial_fn) = partial_result.as_ref() {
                 tokio::select! {
                     biased;
-                    _ = revjob_rx.cancelled() => return Ok(()),
+                    _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
                     result = partial_fn(MetaCacheEntries(top_entries), &errs) => result?,
                 }
             } else if let Some(partial_fn) = opts.partial.as_ref() {
                 tokio::select! {
                     biased;
-                    _ = revjob_rx.cancelled() => return Ok(()),
+                    _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
                     _ = partial_fn(MetaCacheEntries(top_entries), &errs) => {}
                 }
             }
         }
-        Ok(())
+        Ok(ListPathRawOutcome { limit_reached })
     });
 
     let merge_started = std::time::Instant::now();
     let Some(revjob) = revjobs.join_next().await else {
         return Err(DiskError::Unexpected);
     };
-    if let Err(err) = revjob.map_err(std::io::Error::other)? {
-        rustfs_io_metrics::record_stage_duration("metacache_merge_failed", merge_started.elapsed().as_secs_f64() * 1000.0);
-        if is_missing_path_error(&err) {
-            debug!(
-                event = EVENT_METACACHE_LISTING,
-                component = LOG_COMPONENT_ECSTORE,
-                subsystem = LOG_SUBSYSTEM_METACACHE,
-                bucket = %log_bucket,
-                path = %log_path,
-                state = "merge_job_missing_path",
-                error = ?err,
-                "Metacache merge job missing path skipped"
-            );
-        } else {
-            error!(
-                event = EVENT_METACACHE_LISTING,
-                component = LOG_COMPONENT_ECSTORE,
-                subsystem = LOG_SUBSYSTEM_METACACHE,
-                bucket = %log_bucket,
-                path = %log_path,
-                state = "merge_job_failed",
-                error = ?err,
-                "Metacache merge job failed"
-            );
-        }
-        cancel_rx.cancel();
-        jobs.abort_all();
+    let outcome = match revjob.map_err(std::io::Error::other)? {
+        Ok(outcome) => outcome,
+        Err(err) => {
+            rustfs_io_metrics::record_stage_duration("metacache_merge_failed", merge_started.elapsed().as_secs_f64() * 1000.0);
+            if is_missing_path_error(&err) {
+                debug!(
+                    event = EVENT_METACACHE_LISTING,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_METACACHE,
+                    bucket = %log_bucket,
+                    path = %log_path,
+                    state = "merge_job_missing_path",
+                    error = ?err,
+                    "Metacache merge job missing path skipped"
+                );
+            } else {
+                error!(
+                    event = EVENT_METACACHE_LISTING,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_METACACHE,
+                    bucket = %log_bucket,
+                    path = %log_path,
+                    state = "merge_job_failed",
+                    error = ?err,
+                    "Metacache merge job failed"
+                );
+            }
+            cancel_rx.cancel();
+            jobs.abort_all();
 
-        return Err(err);
-    }
+            return Err(err);
+        }
+    };
     if rx.is_cancelled() {
         cancel_rx.cancel();
         jobs.abort_all();
-        return Ok(());
+        return Ok(outcome);
     }
     rustfs_io_metrics::record_stage_duration("metacache_merge", merge_started.elapsed().as_secs_f64() * 1000.0);
 
@@ -1113,7 +1146,7 @@ async fn list_path_raw_inner(
     }
 
     // warn!("list_path_raw: done");
-    Ok(())
+    Ok(outcome)
 }
 
 #[inline]
@@ -1132,6 +1165,7 @@ mod tests {
     use rustfs_filemeta::MetacacheWriter;
     use rustfs_filemeta::{FileInfo, FileMeta, MetadataResolutionParams};
     use std::sync::Mutex;
+    use std::sync::atomic::AtomicBool;
     use time::OffsetDateTime;
     use uuid::Uuid;
 
@@ -1252,6 +1286,61 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn list_path_raw_reports_limit_reached_from_any_reader_trailer() {
+        let first = fallback_test_entry();
+        let second = MetaCacheEntry {
+            name: "bucket/object-2".to_string(),
+            ..fallback_test_entry()
+        };
+        // One bounded producer, one that ends naturally, one legacy empty
+        // stream without any trailer.
+        let outcome = list_path_raw_inner(
+            CancellationToken::new(),
+            ListPathRawOptions {
+                disks: vec![None, None, None],
+                min_disks: 1,
+                test_reader_behaviors: vec![
+                    TestReaderBehavior::LimitReached(vec![first.clone()]),
+                    TestReaderBehavior::Entries(vec![first, second]),
+                    TestReaderBehavior::Eof,
+                ],
+                ..Default::default()
+            },
+            None,
+            None,
+        )
+        .await
+        .expect("listing should complete");
+
+        assert!(
+            outcome.limit_reached,
+            "a reader whose trailer reports a limit stop must mark the merged stream as possibly incomplete"
+        );
+    }
+
+    #[tokio::test]
+    async fn list_path_raw_reports_natural_end_when_no_reader_trailer_says_limit() {
+        let outcome = list_path_raw_inner(
+            CancellationToken::new(),
+            ListPathRawOptions {
+                disks: vec![None, None],
+                min_disks: 1,
+                test_reader_behaviors: vec![
+                    TestReaderBehavior::Entries(vec![fallback_test_entry()]),
+                    TestReaderBehavior::Eof,
+                ],
+                ..Default::default()
+            },
+            None,
+            None,
+        )
+        .await
+        .expect("listing should complete");
+
+        assert!(!outcome.limit_reached, "natural and legacy stream ends must not claim a limit stop");
+    }
+
     #[tokio::test]
     async fn list_path_raw_returns_typed_quorum_error_for_multiple_drive_failures() {
         let err = list_path_raw(
@@ -1710,6 +1799,34 @@ mod tests {
             .expect("aborting the listing should drop the blocked producer");
     }
 
+    #[tokio::test]
+    async fn list_path_raw_propagates_agreed_callback_failure() {
+        let entry = MetaCacheEntry {
+            name: "bucket/object".to_owned(),
+            metadata: vec![1],
+            ..Default::default()
+        };
+        let result = timeout(
+            Duration::from_secs(1),
+            list_path_raw(
+                CancellationToken::new(),
+                ListPathRawOptions {
+                    disks: vec![None],
+                    min_disks: 1,
+                    test_reader_behaviors: vec![TestReaderBehavior::Entries(vec![entry])],
+                    agreed: Some(Box::new(|_| Box::pin(async { Err(DiskError::ErasureReadQuorum) }))),
+                    ..Default::default()
+                },
+            ),
+        )
+        .await
+        .expect("callback failure must terminate the walk");
+        assert_eq!(
+            result.expect_err("agreed resolution must not silently omit the key"),
+            DiskError::ErasureReadQuorum
+        );
+    }
+
     #[tokio::test]
     async fn list_path_raw_propagates_partial_callback_failure() {
         let first = MetaCacheEntry {
@@ -1811,6 +1928,7 @@ mod tests {
                     Box::pin(async move {
                         callback_started.notify_one();
                         std::future::pending::<()>().await;
+                        Ok(())
                     })
                 })),
                 ..Default::default()
@@ -1962,6 +2080,7 @@ mod tests {
                     let seen = seen_clone.clone();
                     Box::pin(async move {
                         seen.lock().expect("seen mutex poisoned").push(entry.name);
+                        Ok(())
                     })
                 })),
                 ..Default::default()
@@ -2004,6 +2123,7 @@ mod tests {
                     let seen = seen_clone.clone();
                     Box::pin(async move {
                         seen.lock().expect("seen mutex poisoned").push(entry.name);
+                        Ok(())
                     })
                 })),
                 ..Default::default()
@@ -2076,6 +2196,7 @@ mod tests {
                     let seen = agreed_seen.clone();
                     Box::pin(async move {
                         seen.lock().expect("seen mutex poisoned").push(entry.name);
+                        Ok(())
                     })
                 })),
                 partial: Some(Box::new(move |entries, _| {
@@ -2241,6 +2362,7 @@ mod tests {
                     let agreed_events = agreed_events.clone();
                     Box::pin(async move {
                         agreed_events.lock().expect("events lock").push((entry.name, 4usize, true));
+                        Ok(())
                     })
                 })),
                 partial: Some(Box::new(move |entries, _| {
@@ -2397,6 +2519,7 @@ mod tests {
                         let name = entry.name;
                         Box::pin(async move {
                             agreed_obs.lock().expect("obs lock").push((name, vec![Some(desc); 4], true));
+                            Ok(())
                         })
                     })),
                     partial: Some(Box::new(move |entries, _| {

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -94,6 +94,7 @@ const REPLAY_CACHE_AUTO_MEMORY_PERCENT: u64 = 13;
 const REPLAY_CACHE_AUTO_RPC_RPS_PER_CPU: usize = 4096;
 const REPLAY_CACHE_AUTO_MAX_CAPACITY: usize = 33_554_432;
 const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3";
+const NS_SCANNER_CHECKPOINT_COVERAGE_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-checkpoint-coverage-v1";
 const NS_SCANNER_TIER_REGISTRY_GENERATION_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-tier-registry-generation-v1";
 pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
 static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock<bool> = LazyLock::new(|| {
@@ -642,6 +643,7 @@ fn update_ns_scanner_capability_mac(
     challenge: Uuid,
     server_epoch: Uuid,
     supports_tier_registry_generation: bool,
+    supports_checkpoint_coverage: bool,
 ) {
     mac.update(NS_SCANNER_CAPABILITY_AUTH_DOMAIN);
     mac.update(&NS_SCANNER_PROTOCOL_VERSION.to_be_bytes());
@@ -653,6 +655,9 @@ fn update_ns_scanner_capability_mac(
         // that receives generation-fenced scanner work.
         mac.update(NS_SCANNER_TIER_REGISTRY_GENERATION_AUTH_DOMAIN);
     }
+    if supports_checkpoint_coverage {
+        mac.update(NS_SCANNER_CHECKPOINT_COVERAGE_AUTH_DOMAIN);
+    }
 }
 
 fn generate_ns_scanner_capability_proof(
@@ -660,13 +665,20 @@ fn generate_ns_scanner_capability_proof(
     challenge: Uuid,
     server_epoch: Uuid,
     supports_tier_registry_generation: bool,
+    supports_checkpoint_coverage: bool,
 ) -> std::io::Result<Vec<u8>> {
     if challenge.is_nil() || server_epoch.is_nil() {
         return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
     }
     let mut mac =
         <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
-    update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch, supports_tier_registry_generation);
+    update_ns_scanner_capability_mac(
+        &mut mac,
+        challenge,
+        server_epoch,
+        supports_tier_registry_generation,
+        supports_checkpoint_coverage,
+    );
     Ok(mac.finalize().into_bytes().to_vec())
 }
 
@@ -676,13 +688,20 @@ fn verify_ns_scanner_capability_proof(
     server_epoch: Uuid,
     proof: &[u8],
     supports_tier_registry_generation: bool,
+    supports_checkpoint_coverage: bool,
 ) -> std::io::Result<()> {
     if challenge.is_nil() || server_epoch.is_nil() {
         return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
     }
     let mut mac =
         <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
-    update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch, supports_tier_registry_generation);
+    update_ns_scanner_capability_mac(
+        &mut mac,
+        challenge,
+        server_epoch,
+        supports_tier_registry_generation,
+        supports_checkpoint_coverage,
+    );
     mac.verify_slice(proof)
         .map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid namespace scanner capability proof"))
 }
@@ -700,7 +719,22 @@ pub fn sign_ns_scanner_capability_with_tier_registry_generation(
     server_epoch: Uuid,
     supports_tier_registry_generation: bool,
 ) -> std::io::Result<Vec<u8>> {
-    generate_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, supports_tier_registry_generation)
+    sign_ns_scanner_capability_with_checkpoint_coverage(challenge, server_epoch, supports_tier_registry_generation, false)
+}
+
+pub fn sign_ns_scanner_capability_with_checkpoint_coverage(
+    challenge: Uuid,
+    server_epoch: Uuid,
+    supports_tier_registry_generation: bool,
+    supports_checkpoint_coverage: bool,
+) -> std::io::Result<Vec<u8>> {
+    generate_ns_scanner_capability_proof(
+        &get_shared_secret()?,
+        challenge,
+        server_epoch,
+        supports_tier_registry_generation,
+        supports_checkpoint_coverage,
+    )
 }
 
 pub fn verify_ns_scanner_capability_with_tier_registry_generation(
@@ -709,7 +743,30 @@ pub fn verify_ns_scanner_capability_with_tier_registry_generation(
     proof: &[u8],
     supports_tier_registry_generation: bool,
 ) -> std::io::Result<()> {
-    verify_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, proof, supports_tier_registry_generation)
+    verify_ns_scanner_capability_with_checkpoint_coverage(
+        challenge,
+        server_epoch,
+        proof,
+        supports_tier_registry_generation,
+        false,
+    )
+}
+
+pub fn verify_ns_scanner_capability_with_checkpoint_coverage(
+    challenge: Uuid,
+    server_epoch: Uuid,
+    proof: &[u8],
+    supports_tier_registry_generation: bool,
+    supports_checkpoint_coverage: bool,
+) -> std::io::Result<()> {
+    verify_ns_scanner_capability_proof(
+        &get_shared_secret()?,
+        challenge,
+        server_epoch,
+        proof,
+        supports_tier_registry_generation,
+        supports_checkpoint_coverage,
+    )
 }
 
 #[derive(Clone, Copy)]
@@ -1776,13 +1833,30 @@ mod tests {
         let secret = "test-scanner-capability-secret";
         let challenge = Uuid::new_v4();
         let server_epoch = Uuid::new_v4();
-        let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false)
+        let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false, false)
             .expect("capability proof should be generated");
 
-        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false).is_ok());
-        assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, false).is_err());
-        assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof, false).is_err());
-        assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof, false).is_err());
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, false).is_ok());
+        assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, false, false).is_err());
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof, false, false).is_err());
+        assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof, false, false).is_err());
+    }
+
+    #[test]
+    fn namespace_scanner_capability_proof_binds_checkpoint_coverage_support() {
+        let secret = "checkpoint-capability-test-secret";
+        let challenge = Uuid::new_v4();
+        let server_epoch = Uuid::new_v4();
+        let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, true)
+            .expect("checkpoint capability proof");
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, true).is_ok());
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, false).is_err());
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, true).is_err());
+        let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, false)
+            .expect("legacy generation capability proof");
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true, true).is_err());
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &[], true, true).is_err());
+        assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, true, true).is_err());
     }
 
     #[test]
@@ -1790,14 +1864,14 @@ mod tests {
         let secret = "test-scanner-capability-secret";
         let challenge = Uuid::new_v4();
         let server_epoch = Uuid::new_v4();
-        let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true)
+        let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, false)
             .expect("generation capability proof should be generated");
 
-        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true).is_ok());
-        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false).is_err());
-        let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false)
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, false).is_ok());
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, false).is_err());
+        let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false, false)
             .expect("legacy capability proof should be generated");
-        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true).is_err());
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true, false).is_err());
     }
 
     /// Security regression for GHSA-r5qv-rc46-hv8q (internode RPC fail-closed,

crates/ecstore/src/cluster/rpc/internode_data_transport.rs

@@ -13,18 +13,19 @@
 // limitations under the License.
 
 use crate::cluster::rpc::{
-    build_auth_headers, build_put_file_auth_trailer, verify_ns_scanner_capability_with_tier_registry_generation,
-    verify_put_file_capability,
+    build_auth_headers, build_put_file_auth_trailer, verify_ns_scanner_capability_with_checkpoint_coverage,
+    verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_capability,
 };
 use crate::disk::error::{Error, Result};
 use crate::disk::{FileReader, FileWriter};
 use crate::storage_api_contracts::internode::{
-    NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY,
-    NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY,
-    NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY,
-    NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY,
-    PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY,
-    PutFileCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
+    NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CHECKPOINT_COVERAGE_QUERY,
+    NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY,
+    NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY,
+    NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY, NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_V1,
+    PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY,
+    PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY,
+    WALK_DIR_STREAM_COMPLETION_V1,
 };
 use async_trait::async_trait;
 use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE};
@@ -250,6 +251,7 @@ pub struct NsScannerStreamRequest {
 pub struct NsScannerCapabilityRequest {
     pub endpoint: String,
     pub supports_tier_registry_generation: bool,
+    pub supports_checkpoint_coverage: bool,
 }
 
 /// Data-plane stream opener used by `RemoteDisk`.
@@ -289,6 +291,7 @@ pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug {
             server_epoch,
             proof: Vec::new(),
             supports_tier_registry_generation: None,
+            supports_checkpoint_coverage: None,
         })
     }
     // Interface facet nobody calls yet: every transport implements both, but no
@@ -379,6 +382,18 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
     }
 
     async fn probe_ns_scanner_capability(&self, request: NsScannerCapabilityRequest) -> Result<NsScannerCapabilityResponse> {
+        let request = if request.supports_checkpoint_coverage {
+            match self.probe_ns_scanner_capability_once(&request).await {
+                Ok(response) => return Ok(response),
+                Err(error) if ns_scanner_capability_error_allows_legacy(&error) => NsScannerCapabilityRequest {
+                    supports_checkpoint_coverage: false,
+                    ..request
+                },
+                Err(error) => return Err(error),
+            }
+        } else {
+            request
+        };
         if request.supports_tier_registry_generation {
             return match self.probe_ns_scanner_capability_once(&request).await {
                 Ok(response) => Ok(response),
@@ -390,6 +405,7 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
                     let legacy_request = NsScannerCapabilityRequest {
                         endpoint: request.endpoint.clone(),
                         supports_tier_registry_generation: false,
+                        supports_checkpoint_coverage: false,
                     };
                     match self.probe_ns_scanner_capability_once(&legacy_request).await {
                         Ok(mut response) => {
@@ -447,18 +463,32 @@ impl TcpHttpInternodeDataTransport {
         if response.version != NS_SCANNER_PROTOCOL_VERSION || response.server_epoch.is_nil() {
             return Err(Error::other("incompatible remote namespace scanner capability response"));
         }
-        if let Err(err) = verify_ns_scanner_capability_with_tier_registry_generation(
+        if let Err(err) = verify_ns_scanner_capability_with_checkpoint_coverage(
             challenge,
             response.server_epoch,
             &response.proof,
             request.supports_tier_registry_generation,
+            request.supports_checkpoint_coverage,
         ) {
+            if request.supports_checkpoint_coverage
+                && verify_ns_scanner_capability_with_tier_registry_generation(
+                    challenge,
+                    response.server_epoch,
+                    &response.proof,
+                    request.supports_tier_registry_generation,
+                )
+                .is_ok()
+            {
+                normalize_ns_scanner_capability_response(&mut response, request.supports_tier_registry_generation, false);
+                return Ok(response);
+            }
             // A permissive older peer can ignore the additive marker and
             // return a valid legacy-scope proof with HTTP 200. Accept that
             // response only after independently authenticating the legacy
             // scope; all other verification failures remain fail-closed.
             if request.supports_tier_registry_generation && ns_scanner_capability_legacy_proof_is_valid(challenge, &response) {
                 response.supports_tier_registry_generation = None;
+                response.supports_checkpoint_coverage = None;
                 return Ok(response);
             }
             return Err(Error::other(format!("remote namespace scanner capability authentication failed: {err}")));
@@ -468,7 +498,11 @@ impl TcpHttpInternodeDataTransport {
         // verified scope so an intermediary cannot strip or rewrite the field
         // and force a silent downgrade after a successful generation-bound
         // handshake.
-        normalize_ns_scanner_capability_response(&mut response, request.supports_tier_registry_generation);
+        normalize_ns_scanner_capability_response(
+            &mut response,
+            request.supports_tier_registry_generation,
+            request.supports_checkpoint_coverage,
+        );
         Ok(response)
     }
 
@@ -798,8 +832,13 @@ fn build_walk_dir_url(request: &WalkDirStreamRequest) -> String {
     )
 }
 
-fn normalize_ns_scanner_capability_response(response: &mut NsScannerCapabilityResponse, requested_generation_support: bool) {
+fn normalize_ns_scanner_capability_response(
+    response: &mut NsScannerCapabilityResponse,
+    requested_generation_support: bool,
+    requested_checkpoint_coverage: bool,
+) {
     response.supports_tier_registry_generation = requested_generation_support.then_some(true);
+    response.supports_checkpoint_coverage = requested_checkpoint_coverage.then_some(true);
 }
 
 fn ns_scanner_capability_legacy_proof_is_valid(challenge: Uuid, response: &NsScannerCapabilityResponse) -> bool {
@@ -832,7 +871,7 @@ fn build_ns_scanner_url(request: &NsScannerStreamRequest) -> String {
 
 fn build_ns_scanner_capability_url(request: &NsScannerCapabilityRequest, challenge: Uuid) -> String {
     format!(
-        "{}{}?{}={}&{}={}{}",
+        "{}{}?{}={}&{}={}{}{}",
         request.endpoint,
         NS_SCANNER_PATH,
         NS_SCANNER_PROTOCOL_VERSION_QUERY,
@@ -843,6 +882,11 @@ fn build_ns_scanner_capability_url(request: &NsScannerCapabilityRequest, challen
             format!("&{}=true", NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY)
         } else {
             String::new()
+        },
+        if request.supports_checkpoint_coverage {
+            format!("&{}=true", NS_SCANNER_CHECKPOINT_COVERAGE_QUERY)
+        } else {
+            String::new()
         }
     )
 }
@@ -1001,6 +1045,7 @@ mod tests {
             .probe_ns_scanner(NsScannerCapabilityRequest {
                 endpoint: "http://node1:9000".to_string(),
                 supports_tier_registry_generation: false,
+                supports_checkpoint_coverage: false,
             })
             .await
             .expect_err("legacy transport should report namespace scanner as unsupported");
@@ -1027,6 +1072,97 @@ mod tests {
         assert!(matches!(open_err, Error::MethodNotAllowed));
     }
 
+    #[tokio::test]
+    async fn namespace_scanner_checkpoint_capability_authenticates_new_and_legacy_peers() {
+        crate::runtime::sources::ensure_test_rpc_secret();
+        for peer in ["current", "strict-old", "permissive-old", "invalid-proof", "missing-proof"] {
+            let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
+                .await
+                .expect("capability listener");
+            let address = listener.local_addr().expect("capability listener address");
+            let server_epoch = Uuid::new_v4();
+            let server = tokio::spawn(async move {
+                let requests = if peer == "strict-old" { 2 } else { 1 };
+                for index in 0..requests {
+                    let (mut socket, _) = listener.accept().await.expect("capability request");
+                    let mut bytes = Vec::new();
+                    while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
+                        let mut chunk = [0; 1024];
+                        let read = socket.read(&mut chunk).await.expect("capability headers");
+                        assert!(read > 0 && bytes.len() < 16 * 1024, "bounded complete capability headers");
+                        bytes.extend_from_slice(&chunk[..read]);
+                    }
+                    let request = std::str::from_utf8(&bytes).expect("HTTP headers");
+                    let target = request.split_whitespace().nth(1).expect("HTTP request target");
+                    let url = reqwest::Url::parse(&format!("http://{address}{target}")).expect("capability URL");
+                    let query: HashMap<_, _> = url.query_pairs().into_owned().collect();
+                    assert_eq!(query.get(NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY).map(String::as_str), Some("true"));
+                    assert_eq!(query.contains_key(NS_SCANNER_CHECKPOINT_COVERAGE_QUERY), index == 0);
+                    if peer == "strict-old" && index == 0 {
+                        socket
+                            .write_all(b"HTTP/1.1 400 Bad Request\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
+                            .await
+                            .expect("legacy parser rejection");
+                        continue;
+                    }
+                    let challenge = query[NS_SCANNER_CAPABILITY_CHALLENGE_QUERY].parse().expect("challenge UUID");
+                    let proof = match peer {
+                        "invalid-proof" => vec![0; 32],
+                        "missing-proof" => Vec::new(),
+                        _ => crate::cluster::rpc::sign_ns_scanner_capability_with_checkpoint_coverage(
+                            challenge,
+                            server_epoch,
+                            true,
+                            peer == "current",
+                        )
+                        .expect("server capability proof"),
+                    };
+                    let body = rmp_serde::to_vec_named(&NsScannerCapabilityResponse {
+                        version: NS_SCANNER_PROTOCOL_VERSION,
+                        server_epoch,
+                        proof,
+                        supports_tier_registry_generation: Some(true),
+                        // Strip a supported bit, or forge it for an older peer.
+                        supports_checkpoint_coverage: (peer != "current").then_some(true),
+                    })
+                    .expect("capability response");
+                    socket
+                        .write_all(
+                            format!("HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", body.len()).as_bytes(),
+                        )
+                        .await
+                        .expect("capability response headers");
+                    socket.write_all(&body).await.expect("capability response body");
+                }
+                requests
+            });
+            let response = tokio::time::timeout(
+                Duration::from_secs(10),
+                TcpHttpInternodeDataTransport.probe_ns_scanner_capability(NsScannerCapabilityRequest {
+                    endpoint: format!("http://{address}"),
+                    supports_tier_registry_generation: true,
+                    supports_checkpoint_coverage: true,
+                }),
+            )
+            .await
+            .expect("bounded capability probe");
+            if peer.ends_with("proof") {
+                assert!(
+                    response
+                        .expect_err("unverified capability must fail closed")
+                        .to_string()
+                        .contains("authentication failed")
+                );
+            } else {
+                let response = response.expect("authenticated capability");
+                assert_eq!(response.server_epoch, server_epoch);
+                assert_eq!(response.supports_tier_registry_generation, Some(true));
+                assert_eq!(response.supports_checkpoint_coverage, (peer == "current").then_some(true));
+            }
+            assert_eq!(server.await.expect("capability server"), if peer == "strict-old" { 2 } else { 1 });
+        }
+    }
+
     #[tokio::test]
     async fn walk_dir_restores_explicit_remote_missing_error() {
         let _ = rustfs_credentials::set_global_rpc_secret("walk-dir-error-test-secret".to_string());
@@ -1769,6 +1905,7 @@ mod tests {
             &NsScannerCapabilityRequest {
                 endpoint: "http://node1:9000".to_string(),
                 supports_tier_registry_generation: false,
+                supports_checkpoint_coverage: false,
             },
             challenge,
         );
@@ -1788,6 +1925,7 @@ mod tests {
             &NsScannerCapabilityRequest {
                 endpoint: "http://node1:9000".to_string(),
                 supports_tier_registry_generation: true,
+                supports_checkpoint_coverage: false,
             },
             challenge,
         );
@@ -1826,13 +1964,14 @@ mod tests {
             server_epoch: Uuid::new_v4(),
             proof: Vec::new(),
             supports_tier_registry_generation: None,
+            supports_checkpoint_coverage: None,
         };
 
-        normalize_ns_scanner_capability_response(&mut response, true);
+        normalize_ns_scanner_capability_response(&mut response, true, true);
         assert_eq!(response.supports_tier_registry_generation, Some(true));
 
         response.supports_tier_registry_generation = Some(false);
-        normalize_ns_scanner_capability_response(&mut response, false);
+        normalize_ns_scanner_capability_response(&mut response, false, false);
         assert_eq!(response.supports_tier_registry_generation, None);
     }
 
@@ -1846,6 +1985,7 @@ mod tests {
             proof: crate::cluster::rpc::sign_ns_scanner_capability(challenge, Uuid::new_v4())
                 .expect("placeholder proof should be generated"),
             supports_tier_registry_generation: None,
+            supports_checkpoint_coverage: None,
         };
         // A proof bound to a different challenge cannot authorize the legacy
         // fallback, even though the response has the expected shape.

crates/ecstore/src/cluster/rpc/mod.rs

@@ -36,12 +36,13 @@ pub use http_auth::{
     TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, gen_signature_headers,
     gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
     set_tonic_mutation_body_digest, set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest,
-    sign_ns_scanner_capability, sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability,
-    sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
-    verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
-    verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
-    verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
-    verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
+    sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
+    sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
+    tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability,
+    verify_ns_scanner_capability_with_checkpoint_coverage, verify_ns_scanner_capability_with_tier_registry_generation,
+    verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
+    verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned,
+    verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
 };
 #[cfg(test)]
 pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -631,6 +631,17 @@ fn decode_cross_pool_fence_capability(expected_member: &str, result: &[u8]) -> R
     Ok((version, epoch))
 }
 
+fn decode_mrf_lifecycle_reader_capability(expected_member: &str, result: &[u8]) -> Result<(u8, Uuid)> {
+    let (&maximum_format_version, identity) = result
+        .split_first()
+        .ok_or_else(|| Error::other("peer returned a truncated MRF lifecycle reader capability"))?;
+    if maximum_format_version == 0 || expected_member.trim().is_empty() {
+        return Err(Error::other("peer returned an invalid MRF lifecycle reader capability"));
+    }
+    let epoch = decode_remote_version_state_capability(expected_member, identity)?;
+    Ok((maximum_format_version, epoch))
+}
+
 #[derive(Clone, Debug)]
 pub struct PeerLiveEventsBatch {
     pub events: Vec<u8>,
@@ -1964,6 +1975,16 @@ impl PeerRestClient {
         Ok((self.topology_member.clone(), supported_version, epoch))
     }
 
+    /// Reports one peer's reader support without authorizing a lifecycle writer or rollback.
+    pub async fn probe_mrf_lifecycle_reader(&self, topology_fingerprint: String) -> Result<(String, u8, Uuid)> {
+        let probe = rustfs_protos::mrf_lifecycle_reader_capability_probe(Uuid::new_v4().as_bytes());
+        let result = self
+            .heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
+            .await?;
+        let (maximum_format_version, epoch) = decode_mrf_lifecycle_reader_capability(&self.topology_member, &result)?;
+        Ok((self.topology_member.clone(), maximum_format_version, epoch))
+    }
+
     pub async fn probe_ilm_recovery_export(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
         let probe = rustfs_protos::ilm_recovery_export_capability_probe(Uuid::new_v4().as_bytes());
         let result = self
@@ -4333,6 +4354,49 @@ mod tests {
         }
     }
 
+    #[test]
+    fn mrf_lifecycle_reader_capability_decoder_preserves_the_advertised_version() {
+        let epoch = Uuid::new_v4();
+        for version in [1, 2, u8::MAX] {
+            let result = rustfs_protos::encode_mrf_lifecycle_reader_capability(version, "node-a:9000", epoch.as_bytes())
+                .expect("capability frame");
+            assert_eq!(
+                decode_mrf_lifecycle_reader_capability("node-a:9000", &result).expect("reported capability"),
+                (version, epoch)
+            );
+        }
+    }
+
+    #[test]
+    fn mrf_lifecycle_reader_capability_decoder_rejects_unbound_or_malformed_identity() {
+        let epoch = Uuid::new_v4();
+        let valid =
+            rustfs_protos::encode_mrf_lifecycle_reader_capability(2, "node-a:9000", epoch.as_bytes()).expect("capability frame");
+        let mut invalid = vec![Vec::new(), vec![2], valid[..valid.len() - 1].to_vec()];
+        let mut zero = valid.clone();
+        zero[0] = 0;
+        invalid.push(zero);
+        let mut trailing = valid.clone();
+        trailing.push(0);
+        invalid.push(trailing);
+        let mut length_overflow = valid.clone();
+        length_overflow[1..9].copy_from_slice(&u64::MAX.to_be_bytes());
+        invalid.push(length_overflow);
+        let mut invalid_utf8 = valid.clone();
+        invalid_utf8[9] = 0xff;
+        invalid.push(invalid_utf8);
+        invalid.push(
+            rustfs_protos::encode_mrf_lifecycle_reader_capability(2, "node-a:9000", Uuid::nil().as_bytes())
+                .expect("nil epoch fixture"),
+        );
+        for result in invalid {
+            assert!(decode_mrf_lifecycle_reader_capability("node-a:9000", &result).is_err());
+        }
+        assert!(decode_mrf_lifecycle_reader_capability("node-b:9000", &valid).is_err());
+        let empty = rustfs_protos::encode_mrf_lifecycle_reader_capability(2, "", epoch.as_bytes()).expect("empty member fixture");
+        assert!(decode_mrf_lifecycle_reader_capability("", &empty).is_err());
+    }
+
     #[test]
     fn remote_version_state_capability_decoder_fails_closed() {
         let epoch = Uuid::new_v4();

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -1143,12 +1143,19 @@ impl RemoteDisk {
         let probe = self.data_transport.probe_ns_scanner_capability(NsScannerCapabilityRequest {
             endpoint: self.endpoint.grid_host(),
             supports_tier_registry_generation: true,
+            supports_checkpoint_coverage: true,
         });
         let result = timeout(NS_SCANNER_CAPABILITY_PROBE_TIMEOUT, probe)
             .await
             .map_err(|_| DiskError::other("remote namespace scanner capability probe timed out"))?;
         match result {
-            Ok(response) if response.supports_tier_registry_generation == Some(true) => Ok(Some(response.server_epoch)),
+            // RUSTFS_COMPAT_TODO(ns-scanner-checkpoint-coverage): old strict readers cannot accept the coverage field. Remove after every supported peer authenticates both scanner capabilities.
+            Ok(response)
+                if response.supports_tier_registry_generation == Some(true)
+                    && response.supports_checkpoint_coverage == Some(true) =>
+            {
+                Ok(Some(response.server_epoch))
+            }
             Ok(_) => Ok(None),
             // RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): old peers and legacy transports lack the authenticated startup-epoch handshake. Remove after every supported peer implements namespace scanner protocol v3.
             Err(DiskError::MethodNotAllowed) => Ok(None),
@@ -2368,7 +2375,300 @@ fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
     file_info.validate_for_metadata_read().map_err(Into::into)
 }
 
+fn validate_artifact_repair_seal_response(
+    request: &rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest,
+    response: &rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsResponse,
+) -> Result<()> {
+    let canonical = rustfs_protos::canonical_seal_artifact_attempt_repairs_response_body(request, response)
+        .map_err(|_| Error::other("artifact repair seal response is too large"))?;
+    crate::cluster::rpc::verify_tonic_rpc_response_proof(&canonical, &response.response_proof)
+        .map_err(|_| Error::other("artifact repair seal response proof is invalid"))?;
+    if !response.success {
+        return Err(response.error.clone().unwrap_or_default().into());
+    }
+    let expected_disk = Uuid::from_slice(&request.expected_disk_id).ok().filter(|id| !id.is_nil());
+    if response.error.is_some() || expected_disk.is_none() || Uuid::from_slice(&response.disk_id).ok() != expected_disk {
+        return Err(Error::other("artifact repair seal response disk identity mismatch"));
+    }
+    Ok(())
+}
+
 impl RemoteDisk {
+    pub(crate) async fn rename_artifact_repair_data(
+        &self,
+        expected_disk: Uuid,
+        source: (&str, &str),
+        fi: &FileInfo,
+        destination: (&str, &str),
+        expected_incarnation: Uuid,
+    ) -> Result<RenameDataResp> {
+        use rustfs_protos::proto_gen::node_service::ArtifactRepairRenameDataRequest;
+        self.execute_with_timeout_for_op(
+            "artifact_repair_rename_data",
+            || async {
+                let body = ArtifactRepairRenameDataRequest {
+                    rename: Some(RenameDataRequest {
+                        disk: self.endpoint.to_string(),
+                        src_volume: source.0.to_owned(),
+                        src_path: source.1.to_owned(),
+                        dst_volume: destination.0.to_owned(),
+                        dst_path: destination.1.to_owned(),
+                        file_info: compat_json(fi)?,
+                        file_info_bin: encode_file_info_msgpack(fi)?.into(),
+                        scanner_publication_lease_token: Bytes::new(),
+                        bucket_incarnation_id: expected_incarnation.as_bytes().to_vec().into(),
+                    }),
+                    expected_disk_id: expected_disk.as_bytes().to_vec().into(),
+                };
+                let canonical = rustfs_protos::canonical_artifact_repair_rename_data_request_body(&body)
+                    .map_err(|_| Error::other("artifact repair rename request is too large"))?;
+                let mut request = Request::new(body);
+                crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
+                // Unimplemented is terminal: never downgrade repair publication to PUT or legacy RPCs.
+                let response = self
+                    .get_client()
+                    .await?
+                    .artifact_repair_rename_data(request)
+                    .await?
+                    .into_inner();
+                if !response.success {
+                    return Err(response.error.unwrap_or_default().into());
+                }
+                decode_msgpack_or_json::<RenameDataResp>(
+                    &response.rename_data_resp_bin,
+                    &response.rename_data_resp,
+                    "RenameDataResp",
+                )
+            },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
+    pub(crate) async fn write_artifact_repair_metadata(
+        &self,
+        expected_disk: Uuid,
+        target: (&str, &str),
+        fi: &FileInfo,
+        expected_incarnation: Uuid,
+    ) -> Result<()> {
+        use rustfs_protos::proto_gen::node_service::ArtifactRepairWriteMetadataRequest;
+        self.execute_with_timeout_for_op(
+            "artifact_repair_write_metadata",
+            || async {
+                let body = ArtifactRepairWriteMetadataRequest {
+                    metadata: Some(WriteMetadataRequest {
+                        disk: self.endpoint.to_string(),
+                        volume: target.0.to_owned(),
+                        path: target.1.to_owned(),
+                        file_info: compat_json(fi)?,
+                        file_info_bin: encode_file_info_msgpack(fi)?.into(),
+                        bucket_incarnation_id: expected_incarnation.as_bytes().to_vec().into(),
+                    }),
+                    expected_disk_id: expected_disk.as_bytes().to_vec().into(),
+                };
+                let canonical = rustfs_protos::canonical_artifact_repair_write_metadata_request_body(&body)
+                    .map_err(|_| Error::other("artifact repair metadata request is too large"))?;
+                let mut request = Request::new(body);
+                crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
+                let response = self
+                    .get_client()
+                    .await?
+                    .artifact_repair_write_metadata(request)
+                    .await?
+                    .into_inner();
+                if !response.success {
+                    return Err(response.error.unwrap_or_default().into());
+                }
+                Ok(())
+            },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
+    /// One authenticated scan (or explicitly distinct read-only identity probe).
+    /// Partial observations are never retried after output has begun.
+    pub async fn inventory_artifact_attempt(
+        &self,
+        expected_disk: Uuid,
+        target: (&str, Uuid),
+        prefix: &str,
+        phase: rustfs_protos::proto_gen::node_service::ArtifactInventoryPhase,
+        deadline: tokio::time::Instant,
+        output: tokio::sync::mpsc::Sender<crate::disk::ArtifactInventoryFrame>,
+    ) -> Result<()> {
+        use crate::disk::{ArtifactInventoryFrame, ArtifactInventoryVersion};
+        use rustfs_protos::{
+            artifact_inventory as protocol,
+            proto_gen::node_service::{ArtifactInventoryRequest, artifact_inventory_frame::Payload},
+        };
+        let (bucket, incarnation) = target;
+        let duration = deadline
+            .checked_duration_since(tokio::time::Instant::now())
+            .ok_or(Error::Timeout)?;
+        let body = ArtifactInventoryRequest {
+            protocol_version: protocol::VERSION,
+            invocation_id: Uuid::new_v4().as_bytes().to_vec().into(),
+            disk: self.endpoint.to_string(),
+            expected_disk_id: expected_disk.as_bytes().to_vec().into(),
+            bucket: bucket.to_owned(),
+            bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
+            attempt_prefix: prefix.to_owned(),
+            timeout_millis: u64::try_from(duration.as_millis().min(u128::from(protocol::MAX_TIMEOUT_MILLIS)))
+                .map_err(Error::other)?,
+            phase: phase.into(),
+        };
+        protocol::validate_request(&body).map_err(Error::other)?;
+        tokio::time::timeout_at(deadline, async {
+            let canonical = protocol::canonical_request(&body).map_err(Error::other)?;
+            let mut request = Request::new(body.clone());
+            request.set_timeout(duration);
+            crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
+            let mut stream = self
+                .get_client()
+                .await?
+                .max_decoding_message_size(protocol::MAX_FRAME_BYTES)
+                .inventory_artifact_attempt(request)
+                .await?
+                .into_inner();
+            let mut state = protocol::StreamState::new();
+            let mut terminal = None;
+            while let Some(frame) = stream.message().await? {
+                state
+                    .accept(&body, &frame, |canonical, proof| {
+                        crate::cluster::rpc::verify_tonic_rpc_response_proof(canonical, proof).is_ok()
+                    })
+                    .map_err(Error::other)?;
+                match frame.payload {
+                    Some(Payload::Entries(entries)) => {
+                        let versions = entries
+                            .versions
+                            .into_iter()
+                            .map(|entry| {
+                                Ok(ArtifactInventoryVersion {
+                                    key: entry.key,
+                                    version_id: Uuid::from_slice(&entry.version_id).map_err(Error::other)?,
+                                    delete_marker: entry.delete_marker,
+                                })
+                            })
+                            .collect::<Result<Vec<_>>>()?;
+                        output
+                            .send(ArtifactInventoryFrame::Entries {
+                                sequence: frame.sequence,
+                                versions,
+                            })
+                            .await
+                            .map_err(|_| Error::other("artifact inventory consumer closed"))?;
+                    }
+                    Some(Payload::Complete(complete)) => terminal = Some(complete.entries),
+                    Some(Payload::Verified(_)) => {}
+                    None => return Err(Error::other("artifact inventory payload is absent")),
+                }
+            }
+            // A valid Complete followed by a transport failure or extra frame is
+            // still incomplete. Publish the local terminal only after clean EOF.
+            state.finish().map_err(Error::other)?;
+            if let Some(entries) = terminal {
+                output
+                    .send(ArtifactInventoryFrame::Complete {
+                        disk_id: expected_disk,
+                        incarnation,
+                        attempt_prefix: prefix.to_owned(),
+                        entries,
+                    })
+                    .await
+                    .map_err(|_| Error::other("artifact inventory consumer closed"))?;
+            }
+            Ok(())
+        })
+        .await
+        .map_err(|_| Error::Timeout)?
+    }
+
+    pub(crate) async fn seal_artifact_puts(
+        &self,
+        expected_disk: Uuid,
+        bucket: &str,
+        incarnation: Uuid,
+        attempt_prefix: &str,
+    ) -> Result<()> {
+        use rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest;
+        self.execute_with_timeout_for_op(
+            "seal_artifact_puts",
+            || async {
+                let body = SealArtifactAttemptPutsRequest {
+                    disk: self.endpoint.to_string(),
+                    bucket: bucket.to_owned(),
+                    bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
+                    attempt_prefix: attempt_prefix.to_owned(),
+                    expected_disk_id: expected_disk.as_bytes().to_vec().into(),
+                };
+                let canonical = rustfs_protos::canonical_seal_artifact_attempt_puts_request_body(&body)
+                    .map_err(|_| Error::other("artifact seal request is too large"))?;
+                let mut request = Request::new(body.clone());
+                crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
+                let response = self
+                    .get_client()
+                    .await?
+                    .seal_artifact_attempt_puts(request)
+                    .await?
+                    .into_inner();
+                let canonical = rustfs_protos::canonical_seal_artifact_attempt_puts_response_body(&body, &response)
+                    .map_err(|_| Error::other("artifact seal response is too large"))?;
+                crate::cluster::rpc::verify_tonic_rpc_response_proof(&canonical, &response.response_proof)
+                    .map_err(|_| Error::other("artifact seal response proof is invalid"))?;
+                if !response.success {
+                    return Err(response.error.unwrap_or_default().into());
+                }
+                if response.error.is_some() || Uuid::from_slice(&response.disk_id).ok() != Some(expected_disk) {
+                    return Err(Error::other("artifact seal response disk identity mismatch"));
+                }
+                Ok(())
+            },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
+    pub(crate) async fn seal_artifact_repairs(
+        &self,
+        expected_disk: Uuid,
+        bucket: &str,
+        incarnation: Uuid,
+        attempt_prefix: &str,
+    ) -> Result<()> {
+        use rustfs_protos::proto_gen::node_service::SealArtifactAttemptPutsRequest;
+        if expected_disk.is_nil() || incarnation.is_nil() {
+            return Err(Error::other("artifact repair seal requires non-nil disk and incarnation identities"));
+        }
+        self.execute_with_timeout_for_op(
+            "seal_artifact_repairs",
+            || async {
+                let body = SealArtifactAttemptPutsRequest {
+                    disk: self.endpoint.to_string(),
+                    bucket: bucket.to_owned(),
+                    bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
+                    attempt_prefix: attempt_prefix.to_owned(),
+                    expected_disk_id: expected_disk.as_bytes().to_vec().into(),
+                };
+                let canonical = rustfs_protos::canonical_seal_artifact_attempt_repairs_request_body(&body)
+                    .map_err(|_| Error::other("artifact repair seal request is too large"))?;
+                let mut request = Request::new(body.clone());
+                crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
+                let response = self
+                    .get_client()
+                    .await?
+                    .seal_artifact_attempt_repairs(request)
+                    .await?
+                    .into_inner();
+                validate_artifact_repair_seal_response(&body, &response)
+            },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
     #[tracing::instrument(level = "trace", skip_all)]
     pub(crate) async fn rename_data_borrowed(
         &self,
@@ -2378,20 +2678,39 @@ impl RemoteDisk {
         dst_volume: &str,
         dst_path: &str,
     ) -> Result<RenameDataResp> {
-        self.rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
+        self.rename_data_borrowed_with_fence((src_volume, src_path), fi, (dst_volume, dst_path), None, None)
             .await
     }
 
     #[tracing::instrument(level = "trace", skip_all)]
     pub(crate) async fn rename_data_borrowed_with_fence(
         &self,
-        src_volume: &str,
-        src_path: &str,
+        source: (&str, &str),
         fi: &FileInfo,
-        dst_volume: &str,
-        dst_path: &str,
+        target: (&str, &str),
         scanner_publication_lease_token: Option<Uuid>,
+        artifact_incarnation: Option<Uuid>,
     ) -> Result<RenameDataResp> {
+        let (src_volume, src_path) = source;
+        let (dst_volume, dst_path) = target;
+        if let Some(scope) = crate::bucket::artifact::artifact_repair_scope(dst_volume, dst_path)
+            && let Some(identity) = scope.receiver_identity()
+        {
+            scope.check().map_err(Error::other)?;
+            if artifact_incarnation.is_some() || scanner_publication_lease_token.is_some() {
+                return Err(Error::other("artifact repair cannot use a publication lease"));
+            }
+            let expected = self
+                .get_disk_id()
+                .await?
+                .filter(|id| !id.is_nil())
+                .ok_or_else(|| Error::other("artifact repair disk identity is unavailable"))?;
+            let result = self
+                .rename_artifact_repair_data(expected, (src_volume, src_path), fi, (dst_volume, dst_path), identity.incarnation)
+                .await;
+            scope.check().map_err(Error::other)?;
+            return result;
+        }
         trace!(
             event = EVENT_REMOTE_DISK_RPC,
             component = LOG_COMPONENT_ECSTORE,
@@ -2414,8 +2733,11 @@ impl RemoteDisk {
                 let file_info_bin = encode_file_info_msgpack(fi)?;
                 let mut client = self.get_client().await?;
                 let mut request = Request::new(RenameDataRequest {
-                    bucket_incarnation_id: crate::store::bucket_heal_scope_for_object(dst_volume, dst_path)
-                        .map(|scope| scope.incarnation.as_bytes().to_vec().into())
+                    bucket_incarnation_id: artifact_incarnation
+                        .or_else(|| {
+                            crate::store::bucket_heal_scope_for_object(dst_volume, dst_path).map(|scope| scope.incarnation)
+                        })
+                        .map(|incarnation| incarnation.as_bytes().to_vec().into())
                         .unwrap_or_default(),
                     disk: self.endpoint.to_string(),
                     src_volume: src_volume.to_string(),
@@ -2428,7 +2750,11 @@ impl RemoteDisk {
                         .map(|token| token.as_bytes().to_vec().into())
                         .unwrap_or_default(),
                 });
-                let canonical_body = rustfs_protos::canonical_rename_data_request_body(request.get_ref());
+                let canonical_body = if artifact_incarnation.is_some() {
+                    rustfs_protos::canonical_artifact_rename_data_request_body(request.get_ref())
+                } else {
+                    rustfs_protos::canonical_rename_data_request_body(request.get_ref())
+                };
                 let incarnation_bound = !request.get_ref().bucket_incarnation_id.is_empty();
                 let quota_fenced = rustfs_utils::http::metadata_compat::contains_key_str(
                     &fi.metadata,
@@ -2451,7 +2777,9 @@ impl RemoteDisk {
                 let rpc_started = rustfs_io_metrics::put_stage_timer();
                 let response = rustfs_io_metrics::observe_put_stage_future(
                     async {
-                        if quota_fenced {
+                        if artifact_incarnation.is_some() {
+                            client.artifact_rename_data(request).await
+                        } else if quota_fenced {
                             // The method itself proves quota-token enforcement; an old peer must fail closed.
                             client.rename_data_with_quota_fence(request).await
                         } else if incarnation_bound {
@@ -3197,6 +3525,21 @@ impl DiskAPI for RemoteDisk {
 
     #[tracing::instrument(level = "trace", skip_all)]
     async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
+        if let Some(scope) = crate::bucket::artifact::artifact_repair_scope(volume, path)
+            && let Some(identity) = scope.receiver_identity()
+        {
+            scope.check().map_err(Error::other)?;
+            let expected = self
+                .get_disk_id()
+                .await?
+                .filter(|id| !id.is_nil())
+                .ok_or_else(|| Error::other("artifact repair disk identity is unavailable"))?;
+            let result = self
+                .write_artifact_repair_metadata(expected, (volume, path), &fi, identity.incarnation)
+                .await;
+            scope.check().map_err(Error::other)?;
+            return result;
+        }
         trace!(
             event = EVENT_REMOTE_DISK_RPC,
             component = LOG_COMPONENT_ECSTORE,
@@ -4546,6 +4889,83 @@ mod tests {
 
     static INIT: Once = Once::new();
 
+    #[tokio::test]
+    async fn artifact_repair_old_peer_unimplemented_never_downgrades_mutations() {
+        runtime_sources::ensure_test_rpc_secret();
+        let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
+            .await
+            .expect("loopback test peer");
+        let endpoint = Endpoint {
+            url: url::Url::parse(&format!("{}/data/rustfs0", peer.addr)).expect("test disk endpoint"),
+            is_local: false,
+            pool_idx: 0,
+            set_idx: 0,
+            disk_idx: 0,
+        };
+        let disk = RemoteDisk::new(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+            Arc::new(TcpHttpInternodeDataTransport),
+        )
+        .await
+        .expect("remote test disk");
+        let expected_disk = Uuid::new_v4();
+        let incarnation = Uuid::new_v4();
+        let info = FileInfo::default();
+        let rename = disk
+            .rename_artifact_repair_data(
+                expected_disk,
+                ("tmp", "staged"),
+                &info,
+                ("artifacts", "outputs/tenant/run/attempt/image"),
+                incarnation,
+            )
+            .await
+            .expect_err("old peer cannot publish repair data");
+        let metadata = disk
+            .write_artifact_repair_metadata(expected_disk, ("artifacts", "outputs/tenant/run/attempt/image"), &info, incarnation)
+            .await
+            .expect_err("old peer cannot publish repair metadata");
+        let seal = disk
+            .seal_artifact_repairs(expected_disk, "artifacts", incarnation, "outputs/tenant/run/attempt")
+            .await
+            .expect_err("old peer cannot confirm repair closure");
+        for error in [rename, metadata, seal] {
+            let DiskError::Io(error) = error else {
+                panic!("expected gRPC status error");
+            };
+            let status = error
+                .get_ref()
+                .and_then(|error| error.downcast_ref::<crate::disk::error::RpcStatusError>())
+                .expect("preserved gRPC status");
+            assert_eq!(status.status().code(), Code::Unimplemented);
+        }
+        let paths = peer.peer.rpc_paths.lock().expect("RPC inventory").clone();
+        for method in [
+            "ArtifactRepairRenameData",
+            "ArtifactRepairWriteMetadata",
+            "SealArtifactAttemptRepairs",
+        ] {
+            assert_eq!(
+                paths.iter().filter(|path| path.ends_with(&format!("/{method}"))).count(),
+                1,
+                "each repair method is sent once"
+            );
+        }
+        assert!(
+            paths.iter().all(|path| matches!(
+                path.rsplit('/').next(),
+                Some("ArtifactRepairRenameData" | "ArtifactRepairWriteMetadata" | "SealArtifactAttemptRepairs" | "Ping")
+            )),
+            "repair cannot downgrade to a legacy mutation: {paths:?}"
+        );
+        disk.cancel_token.cancel();
+        peer.stop().await;
+    }
+
     #[tokio::test]
     async fn quota_fence_old_peer_rejects_without_legacy_fallback() {
         runtime_sources::ensure_test_rpc_secret();
@@ -4641,6 +5061,93 @@ mod tests {
         peer.stop().await;
     }
 
+    #[test]
+    fn artifact_repair_seal_response_requires_proof_full_request_and_physical_identity() {
+        use rustfs_protos::proto_gen::node_service::{
+            Error as WireError, SealArtifactAttemptPutsRequest, SealArtifactAttemptPutsResponse,
+        };
+        runtime_sources::ensure_test_rpc_secret();
+        let request = SealArtifactAttemptPutsRequest {
+            disk: "disk-a".into(),
+            bucket: "artifacts".into(),
+            bucket_incarnation_id: vec![1; 16].into(),
+            attempt_prefix: "outputs/tenant/run/attempt".into(),
+            expected_disk_id: vec![2; 16].into(),
+        };
+        let mut response = SealArtifactAttemptPutsResponse {
+            success: true,
+            disk_id: request.expected_disk_id.clone(),
+            ..Default::default()
+        };
+        let sign = |request: &SealArtifactAttemptPutsRequest, response: &mut SealArtifactAttemptPutsResponse| {
+            let canonical = rustfs_protos::canonical_seal_artifact_attempt_repairs_response_body(request, response)
+                .expect("repair response canonical");
+            response.response_proof = crate::cluster::rpc::http_auth::sign_tonic_rpc_response_proof(&canonical)
+                .expect("repair response proof")
+                .into();
+        };
+        assert!(validate_artifact_repair_seal_response(&request, &response).is_err(), "proof is mandatory");
+        sign(&request, &mut response);
+        validate_artifact_repair_seal_response(&request, &response).expect("signed exact physical receiver");
+        for mutate in [
+            |r: &mut SealArtifactAttemptPutsRequest| r.disk = "other-disk".into(),
+            |r: &mut SealArtifactAttemptPutsRequest| r.bucket = "other-bucket".into(),
+            |r: &mut SealArtifactAttemptPutsRequest| r.bucket_incarnation_id = vec![3; 16].into(),
+            |r: &mut SealArtifactAttemptPutsRequest| r.attempt_prefix = "other-attempt".into(),
+            |r: &mut SealArtifactAttemptPutsRequest| r.expected_disk_id = vec![4; 16].into(),
+        ] {
+            let mut changed = request.clone();
+            mutate(&mut changed);
+            assert!(
+                validate_artifact_repair_seal_response(&changed, &response).is_err(),
+                "another request cannot reuse the proof"
+            );
+        }
+        let legacy = rustfs_protos::canonical_seal_artifact_attempt_puts_response_body(&request, &response).expect("PUT domain");
+        let mut legacy_response = response.clone();
+        legacy_response.response_proof = crate::cluster::rpc::http_auth::sign_tonic_rpc_response_proof(&legacy)
+            .expect("PUT proof")
+            .into();
+        assert!(
+            validate_artifact_repair_seal_response(&request, &legacy_response).is_err(),
+            "PUT proof cannot acknowledge repair seal"
+        );
+        for disk in [vec![], vec![0; 16], vec![3; 16], vec![1; 15]] {
+            let mut changed = response.clone();
+            changed.disk_id = disk.into();
+            sign(&request, &mut changed);
+            assert!(
+                validate_artifact_repair_seal_response(&request, &changed).is_err(),
+                "signed wrong or absent physical identity is invalid"
+            );
+        }
+        for mutate in [
+            |r: &mut SealArtifactAttemptPutsResponse| r.success = false,
+            |r: &mut SealArtifactAttemptPutsResponse| r.error = Some(WireError::default()),
+        ] {
+            let mut changed = response.clone();
+            mutate(&mut changed);
+            assert!(
+                validate_artifact_repair_seal_response(&request, &changed).is_err(),
+                "response fields are bound"
+            );
+            sign(&request, &mut changed);
+            assert!(
+                validate_artifact_repair_seal_response(&request, &changed).is_err(),
+                "signed failure or inconsistent success is invalid"
+            );
+        }
+        let mut nil_request = request;
+        nil_request.expected_disk_id = vec![0; 16].into();
+        let mut nil_response = response;
+        nil_response.disk_id = nil_request.expected_disk_id.clone();
+        sign(&nil_request, &mut nil_response);
+        assert!(
+            validate_artifact_repair_seal_response(&nil_request, &nil_response).is_err(),
+            "nil physical identity is never a receipt"
+        );
+    }
+
     #[test]
     fn recoverable_batch_delete_rejects_invalid_error_codes_and_length() {
         let errors = decode_delete_recovery_batch_errors(vec![None, Some(DiskError::FileNotFound.to_u32())], 2)
@@ -5008,6 +5515,7 @@ mod tests {
         calls: Arc<StdMutex<Vec<RecordedTransportCall>>>,
         ns_scanner_probe_status: Arc<StdMutex<Option<u16>>>,
         ns_scanner_generation_support: Arc<StdMutex<Option<bool>>>,
+        ns_scanner_checkpoint_support: Arc<StdMutex<Option<bool>>>,
     }
 
     impl Default for RecordingInternodeDataTransport {
@@ -5016,6 +5524,7 @@ mod tests {
                 calls: Arc::default(),
                 ns_scanner_probe_status: Arc::default(),
                 ns_scanner_generation_support: Arc::new(StdMutex::new(Some(true))),
+                ns_scanner_checkpoint_support: Arc::new(StdMutex::new(Some(true))),
             }
         }
     }
@@ -5029,6 +5538,7 @@ mod tests {
         format_data: Bytes,
         read_all_data: Bytes,
         next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
+        rpc_paths: Arc<StdMutex<Vec<String>>>,
         unsupported_methods: Arc<StdMutex<Vec<String>>>,
     }
 
@@ -5048,6 +5558,7 @@ mod tests {
                 format_data,
                 read_all_data,
                 next_format_read_gate: Arc::default(),
+                rpc_paths: Arc::default(),
                 unsupported_methods: Arc::default(),
             }
         }
@@ -5103,6 +5614,11 @@ mod tests {
         }
 
         fn call(&mut self, request: http::Request<B>) -> Self::Future {
+            self.peer
+                .rpc_paths
+                .lock()
+                .expect("RPC path inventory")
+                .push(request.uri().path().to_owned());
             match request.uri().path() {
                 "/node_service.NodeService/DiskInfo" => {
                     #[derive(Clone)]
@@ -5266,6 +5782,7 @@ mod tests {
                 calls: Arc::default(),
                 ns_scanner_probe_status: Arc::new(StdMutex::new(Some(status))),
                 ns_scanner_generation_support: Arc::new(StdMutex::new(Some(true))),
+                ns_scanner_checkpoint_support: Arc::new(StdMutex::new(Some(true))),
             }
         }
 
@@ -5274,6 +5791,7 @@ mod tests {
                 calls: Arc::default(),
                 ns_scanner_probe_status: Arc::default(),
                 ns_scanner_generation_support: Arc::new(StdMutex::new(support)),
+                ns_scanner_checkpoint_support: Arc::new(StdMutex::new(Some(true))),
             }
         }
 
@@ -5974,6 +6492,7 @@ mod tests {
                 server_epoch,
                 proof: Vec::new(),
                 supports_tier_registry_generation,
+                supports_checkpoint_coverage: *self.ns_scanner_checkpoint_support.lock().expect("checkpoint capability lock"),
             })
         }
 
@@ -9069,6 +9588,28 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn test_remote_disk_namespace_scanner_falls_back_until_checkpoint_support_is_authenticated() {
+        for support in [None, Some(false)] {
+            let transport = RecordingInternodeDataTransport::default();
+            *transport
+                .ns_scanner_checkpoint_support
+                .lock()
+                .expect("checkpoint capability lock") = support;
+            let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
+            assert_eq!(remote_disk.ns_scanner_server_epoch().await.expect("old peer fallback"), None);
+            assert!(matches!(transport.calls().as_slice(), [RecordedTransportCall::NsScannerProbe(_)]));
+            *transport
+                .ns_scanner_checkpoint_support
+                .lock()
+                .expect("checkpoint capability lock") = Some(true);
+            assert_eq!(
+                remote_disk.ns_scanner_server_epoch().await.expect("upgraded peer"),
+                Some(Uuid::from_u128(1))
+            );
+        }
+    }
+
     #[tokio::test]
     async fn test_remote_disk_namespace_scanner_capability_falls_back_without_generation_support() {
         for support in [None, Some(false)] {

crates/ecstore/src/config/com.rs

@@ -438,6 +438,64 @@ where
     Ok(data)
 }
 
+pub(crate) async fn read_config_observation<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<(Vec<u8>, ObjectInfo)>
+where
+    S: EcstoreObjectIO,
+{
+    let opts = ObjectOptions {
+        suppress_read_repair: true,
+        no_lock: false,
+        skip_verify_bitrot: false,
+        ..Default::default()
+    };
+    crate::object_api::without_get_object_body_cache_hook(read_config_limited_preserve_empty_with_metadata_opts(
+        api, file, &opts, max_bytes,
+    ))
+    .await
+}
+
+pub(crate) async fn read_config_observation_no_lock<S>(
+    api: Arc<S>,
+    file: &str,
+    max_bytes: usize,
+    guard: &crate::object_api::WriteCommitGuard,
+) -> Result<(Vec<u8>, ObjectInfo)>
+where
+    S: EcstoreObjectIO,
+{
+    if !guard.matches(RUSTFS_META_BUCKET, file) {
+        return Err(Error::InvalidArgument(
+            RUSTFS_META_BUCKET.to_owned(),
+            file.to_owned(),
+            "locked observation requires a matching write owner".to_owned(),
+        ));
+    }
+    let lock_lost_error = || Error::NamespaceLockQuorumUnavailable {
+        mode: "locked_config_observation",
+        bucket: RUSTFS_META_BUCKET.to_owned(),
+        object: file.to_owned(),
+        required: 1,
+        achieved: 0,
+    };
+    if guard.is_lock_lost() {
+        return Err(lock_lost_error());
+    }
+    let opts = ObjectOptions {
+        suppress_read_repair: true,
+        no_lock: true,
+        skip_verify_bitrot: false,
+        ..Default::default()
+    };
+    let result = crate::object_api::without_get_object_body_cache_hook(read_config_limited_preserve_empty_with_metadata_opts(
+        api, file, &opts, max_bytes,
+    ))
+    .await;
+    if guard.is_lock_lost() {
+        return Err(lock_lost_error());
+    }
+    result
+}
+
 pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
 where
     S: EcstoreObjectIO,
@@ -484,10 +542,22 @@ pub(crate) async fn read_config_preserve_empty<S>(api: Arc<S>, file: &str) -> Re
 where
     S: EcstoreObjectIO,
 {
-    let (data, _obj) = read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, None).await?;
+    let (data, _obj) = read_config_preserve_empty_with_metadata_opts(api, file, &ObjectOptions::default()).await?;
     Ok(data)
 }
 
+/// Preserve an empty config payload, its metadata, and caller-supplied read options.
+pub(crate) async fn read_config_preserve_empty_with_metadata_opts<S>(
+    api: Arc<S>,
+    file: &str,
+    opts: &ObjectOptions,
+) -> Result<(Vec<u8>, ObjectInfo)>
+where
+    S: EcstoreObjectIO,
+{
+    read_config_with_metadata_inner(api, file, opts, true, None).await
+}
+
 pub async fn read_config_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
 where
     S: ObjectIO<
@@ -516,15 +586,13 @@ pub(crate) async fn read_config_no_lock_preserve_empty_with_metadata<S>(api: Arc
 where
     S: EcstoreObjectIO,
 {
-    read_config_with_metadata_inner(
+    read_config_preserve_empty_with_metadata_opts(
         api,
         file,
         &ObjectOptions {
             no_lock: true,
             ..Default::default()
         },
-        true,
-        None,
     )
     .await
 }
@@ -2919,9 +2987,9 @@ mod tests {
         config_task_join_error, configs_semantically_equal, decode_server_config_blob, encode_server_config_blob,
         heal_config_descriptor, is_standard_object_server_config, lookup_configs, map_system_metadata_write_error,
         new_and_save_server_config, read_config, read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty,
-        read_config_with_metadata, read_config_without_migrate, read_server_config_snapshot, save_config_with_opts_inner,
-        save_server_config, save_server_config_snapshot, save_server_config_snapshot_with_generation,
-        server_config_transaction_lock_path, storage_class_kvs_mut,
+        read_config_preserve_empty_with_metadata_opts, read_config_with_metadata, read_config_without_migrate,
+        read_server_config_snapshot, save_config_with_opts_inner, save_server_config, save_server_config_snapshot,
+        save_server_config_snapshot_with_generation, server_config_transaction_lock_path, storage_class_kvs_mut,
     };
     use crate::config::{audit, heal, notify, oidc, scanner};
     use crate::disk::{RUSTFS_META_BUCKET, endpoint::Endpoint};
@@ -5396,6 +5464,7 @@ mod tests {
         heal_replacement: Option<Vec<u8>>,
         heal_calls: AtomicUsize,
         write_calls: AtomicUsize,
+        last_get_no_lock: AtomicBool,
         last_put_no_lock: AtomicBool,
         last_put_preconditions: Mutex<Option<HTTPPreconditions>>,
         revision: AtomicUsize,
@@ -5411,6 +5480,7 @@ mod tests {
                 heal_replacement,
                 heal_calls: AtomicUsize::new(0),
                 write_calls: AtomicUsize::new(0),
+                last_get_no_lock: AtomicBool::new(false),
                 last_put_no_lock: AtomicBool::new(false),
                 last_put_preconditions: Mutex::new(None),
                 revision: AtomicUsize::new(1),
@@ -5466,8 +5536,9 @@ mod tests {
             _object: &str,
             _range: Option<HTTPRangeSpec>,
             _h: HeaderMap,
-            _opts: &ObjectOptions,
+            opts: &ObjectOptions,
         ) -> Result<GetObjectReader> {
+            self.last_get_no_lock.store(opts.no_lock, Ordering::SeqCst);
             let data = match &*self.state.lock().expect("state lock poisoned") {
                 RecoveryReadState::Missing => return Err(Error::ConfigNotFound),
                 RecoveryReadState::Blob(data) => data.clone(),
@@ -5779,6 +5850,325 @@ mod tests {
         assert_eq!(store.write_calls.load(Ordering::SeqCst), 0);
     }
 
+    type ObservationEofGate = (Option<tokio::sync::oneshot::Sender<()>>, tokio::sync::oneshot::Receiver<()>);
+
+    struct ObservationStream {
+        inner: Cursor<Vec<u8>>,
+        consumed: Arc<AtomicUsize>,
+        fail: bool,
+        eof_gate: Option<ObservationEofGate>,
+    }
+
+    impl AsyncRead for ObservationStream {
+        fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
+            if self.fail {
+                return Poll::Ready(Err(std::io::Error::other("observation stream failed")));
+            }
+            if self.inner.position() == u64::try_from(self.inner.get_ref().len()).expect("fixture length")
+                && let Some((ready, release)) = self.eof_gate.as_mut()
+            {
+                if let Some(ready) = ready.take() {
+                    ready.send(()).expect("EOF observer remains active");
+                }
+                match std::future::Future::poll(Pin::new(release), cx) {
+                    Poll::Pending => return Poll::Pending,
+                    Poll::Ready(result) => result.expect("EOF gate released"),
+                }
+                self.eof_gate = None;
+            }
+            let before = buf.filled().len();
+            let result = Pin::new(&mut self.inner).poll_read(cx, buf);
+            self.consumed.fetch_add(buf.filled().len() - before, Ordering::SeqCst);
+            result
+        }
+    }
+
+    struct ObservationStore {
+        body: Vec<u8>,
+        declared_size: i64,
+        error: Option<Error>,
+        stream_error: bool,
+        gets: AtomicUsize,
+        eof_gate: std::sync::Mutex<Option<ObservationEofGate>>,
+        consumed: Arc<AtomicUsize>,
+        options: std::sync::Mutex<Option<(bool, bool, bool)>>,
+        writes: AtomicUsize,
+    }
+
+    impl Debug for ObservationStore {
+        fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+            f.debug_struct("ObservationStore").finish()
+        }
+    }
+
+    impl ObservationStore {
+        fn new(body: Vec<u8>, declared_size: i64) -> Self {
+            Self {
+                body,
+                declared_size,
+                error: None,
+                stream_error: false,
+                gets: AtomicUsize::new(0),
+                eof_gate: std::sync::Mutex::new(None),
+                consumed: Arc::new(AtomicUsize::new(0)),
+                options: std::sync::Mutex::new(None),
+                writes: AtomicUsize::new(0),
+            }
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl crate::storage_api_contracts::object::ObjectIO for ObservationStore {
+        type Error = Error;
+        type RangeSpec = HTTPRangeSpec;
+        type HeaderMap = HeaderMap;
+        type ObjectOptions = ObjectOptions;
+        type ObjectInfo = ObjectInfo;
+        type GetObjectReader = GetObjectReader;
+        type PutObjectReader = PutObjReader;
+
+        async fn get_object_reader(
+            &self,
+            bucket: &str,
+            object: &str,
+            range: Option<HTTPRangeSpec>,
+            _headers: HeaderMap,
+            opts: &ObjectOptions,
+        ) -> Result<GetObjectReader> {
+            assert_eq!(bucket, RUSTFS_META_BUCKET);
+            assert_eq!(object, "config/observation.bin");
+            self.gets.fetch_add(1, Ordering::SeqCst);
+            assert!(range.is_none());
+            *self.options.lock().expect("observation options") =
+                Some((opts.suppress_read_repair, opts.no_lock, opts.skip_verify_bitrot));
+            if let Some(error) = &self.error {
+                return Err(error.clone());
+            }
+            Ok(GetObjectReader {
+                stream: Box::new(ObservationStream {
+                    inner: Cursor::new(self.body.clone()),
+                    consumed: self.consumed.clone(),
+                    fail: self.stream_error,
+                    eof_gate: self.eof_gate.lock().expect("EOF gate").take(),
+                }),
+                object_info: ObjectInfo {
+                    size: self.declared_size,
+                    etag: Some("observation-etag".to_string()),
+                    ..Default::default()
+                },
+                buffered_body: None,
+                body_source: Default::default(),
+            })
+        }
+
+        async fn put_object(
+            &self,
+            _bucket: &str,
+            _object: &str,
+            _data: &mut PutObjReader,
+            _opts: &ObjectOptions,
+        ) -> Result<ObjectInfo> {
+            self.writes.fetch_add(1, Ordering::SeqCst);
+            Err(Error::other("observation must not write"))
+        }
+    }
+
+    #[tokio::test]
+    async fn locked_observation_preserves_options_and_typed_errors() {
+        let object = "config/observation.bin";
+        let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
+        let wrapper = rustfs_lock::NamespaceLockWrapper::new(
+            rustfs_lock::NamespaceLock::with_local_manager("locked-observation-options".to_owned(), manager.clone()),
+            rustfs_lock::ObjectKey::new(RUSTFS_META_BUCKET, object),
+            "locked-observation-options".to_owned(),
+        );
+        let guard = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
+            .await
+            .expect("matching real owner");
+        let store = Arc::new(ObservationStore::new(b"raw".to_vec(), 3));
+        let result = super::read_config_observation_no_lock(store.clone(), object, 3, &guard).await;
+        assert!(result.is_ok(), "I06L_READ_FLAGS: {result:?}");
+        assert!(*store.options.lock().expect("options") == Some((true, true, false)), "I06L_READ_FLAGS");
+        assert_eq!(result.expect("raw observation").0, b"raw");
+        assert_eq!(store.writes.load(Ordering::SeqCst), 0);
+        for (bucket, key) in [(RUSTFS_META_BUCKET, "config/other.bin"), ("wrong-bucket", object)] {
+            let wrapper = rustfs_lock::NamespaceLockWrapper::new(
+                rustfs_lock::NamespaceLock::with_local_manager("locked-observation-target".to_owned(), manager.clone()),
+                rustfs_lock::ObjectKey::new(bucket, key),
+                "locked-observation-target".to_owned(),
+            );
+            let wrong = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
+                .await
+                .expect("wrong-target real owner");
+            let store = Arc::new(ObservationStore::new(Vec::new(), 0));
+            let result = super::read_config_observation_no_lock(store.clone(), object, 8, &wrong).await;
+            assert!(matches!(result, Err(Error::InvalidArgument(..))), "I06L_TARGET_REJECTED");
+            assert!(store.gets.load(Ordering::SeqCst) == 0, "I06L_TARGET_REJECTED");
+        }
+        for error in [
+            Error::FileNotFound,
+            Error::ObjectNotFound(RUSTFS_META_BUCKET.to_owned(), object.to_owned()),
+            Error::ErasureReadQuorum,
+        ] {
+            let expected = if error == Error::FileNotFound || matches!(error, Error::ObjectNotFound(..)) {
+                Error::ConfigNotFound
+            } else {
+                error.clone()
+            };
+            let mut fixture = ObservationStore::new(Vec::new(), 0);
+            fixture.error = Some(error);
+            let store = Arc::new(fixture);
+            assert_eq!(
+                super::read_config_observation_no_lock(store.clone(), object, 8, &guard)
+                    .await
+                    .expect_err("typed error"),
+                expected
+            );
+            assert_eq!(store.writes.load(Ordering::SeqCst), 0);
+        }
+        let mut fixture = ObservationStore::new(Vec::new(), 0);
+        fixture.stream_error = true;
+        let store = Arc::new(fixture);
+        let err = super::read_config_observation_no_lock(store.clone(), object, 8, &guard)
+            .await
+            .expect_err("stream failure");
+        assert!(err.to_string().contains("observation stream failed"), "I06L_TYPED_STREAM_ERROR");
+        assert_eq!(store.writes.load(Ordering::SeqCst), 0);
+        let store = Arc::new(ObservationStore::new(vec![7; 64], 0));
+        let result = super::read_config_observation_no_lock(store.clone(), object, 8, &guard).await;
+        assert!(matches!(result, Err(Error::CorruptedFormat)), "I06L_STREAM_BOUND");
+        assert!(store.consumed.load(Ordering::SeqCst) == 9, "I06L_STREAM_BOUND");
+        assert_eq!(store.writes.load(Ordering::SeqCst), 0);
+    }
+
+    #[tokio::test]
+    async fn locked_observation_rejects_actual_owner_loss_before_and_after_read() {
+        let object = "config/observation.bin";
+        for after_read in [false, true] {
+            let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
+            let client = Arc::new(RefreshControlledClient::new(manager));
+            let wrapper = rustfs_lock::NamespaceLockWrapper::new(
+                rustfs_lock::NamespaceLock::new("locked-observation-loss".to_owned(), client.clone()),
+                rustfs_lock::ObjectKey::new(RUSTFS_META_BUCKET, object),
+                "locked-observation-loss".to_owned(),
+            );
+            let guard = crate::object_api::WriteCommitGuard::acquire(&wrapper, std::time::Duration::from_secs(5))
+                .await
+                .expect("real distributed owner");
+            let signal = guard.lock_lost_signal().expect("acquired distributed owner signal");
+            let store = Arc::new(ObservationStore::new(b"raw".to_vec(), 3));
+            if after_read {
+                let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
+                let (release_tx, release_rx) = tokio::sync::oneshot::channel();
+                *store.eof_gate.lock().expect("EOF gate") = Some((Some(ready_tx), release_rx));
+                let mut read = Box::pin(super::read_config_observation_no_lock(store.clone(), object, 3, &guard));
+                tokio::time::timeout(std::time::Duration::from_secs(60), async {
+                    tokio::select! {
+                        ready = ready_rx => ready.expect("read reached EOF"),
+                        result = &mut read => assert_eq!(false, true, "I06L_LOSS_AFTER_READ: returned before EOF gate: {result:?}"),
+                    }
+                }).await.expect("EOF watchdog");
+                client.reject_refreshes();
+                tokio::time::timeout(std::time::Duration::from_secs(60), signal.notified())
+                    .await
+                    .expect("actual owner loss watchdog");
+                assert!(guard.is_lock_lost(), "I06L_LOSS_AFTER_READ");
+                release_tx.send(()).expect("release EOF");
+                let result = tokio::time::timeout(std::time::Duration::from_secs(60), read)
+                    .await
+                    .expect("read watchdog");
+                assert!(
+                    matches!(
+                        result,
+                        Err(Error::NamespaceLockQuorumUnavailable {
+                            mode: "locked_config_observation",
+                            ..
+                        })
+                    ),
+                    "I06L_LOSS_AFTER_READ"
+                );
+                assert_eq!(store.gets.load(Ordering::SeqCst), 1);
+            } else {
+                client.reject_refreshes();
+                tokio::time::timeout(std::time::Duration::from_secs(60), signal.notified())
+                    .await
+                    .expect("actual owner loss watchdog");
+                assert!(guard.is_lock_lost(), "I06L_LOSS_BEFORE_GET");
+                let result = super::read_config_observation_no_lock(store.clone(), object, 3, &guard).await;
+                assert!(
+                    matches!(
+                        result,
+                        Err(Error::NamespaceLockQuorumUnavailable {
+                            mode: "locked_config_observation",
+                            ..
+                        })
+                    ),
+                    "I06L_LOSS_BEFORE_GET"
+                );
+                assert!(store.gets.load(Ordering::SeqCst) == 0, "I06L_LOSS_BEFORE_GET");
+            }
+            assert_eq!(store.writes.load(Ordering::SeqCst), 0);
+        }
+    }
+
+    #[tokio::test]
+    async fn observation_preserves_options_and_errors() {
+        for body in [Vec::new(), b"raw config".to_vec()] {
+            let store = Arc::new(ObservationStore::new(body.clone(), i64::try_from(body.len()).expect("fixture size")));
+            let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
+            assert!(result.is_ok(), "observation must preserve empty and nonempty objects: {result:?}");
+            let (observed, info) = result.expect("successful observation");
+            assert_eq!(observed, body);
+            assert_eq!(info.etag.as_deref(), Some("observation-etag"));
+            assert_eq!(
+                *store.options.lock().expect("options"),
+                Some((true, false, false)),
+                "I05_REPAIR_SUPPRESSED"
+            );
+            assert_eq!(store.writes.load(Ordering::SeqCst), 0);
+        }
+        for (source, expected) in [
+            (Error::FileNotFound, Error::ConfigNotFound),
+            (
+                Error::ObjectNotFound(RUSTFS_META_BUCKET.to_string(), "config/observation.bin".to_string()),
+                Error::ConfigNotFound,
+            ),
+            (Error::ErasureReadQuorum, Error::ErasureReadQuorum),
+        ] {
+            let mut fixture = ObservationStore::new(Vec::new(), 0);
+            fixture.error = Some(source);
+            let store = Arc::new(fixture);
+            let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
+            assert_eq!(result.expect_err("read error must propagate"), expected);
+            assert_eq!(store.writes.load(Ordering::SeqCst), 0);
+        }
+        let mut fixture = ObservationStore::new(Vec::new(), 0);
+        fixture.stream_error = true;
+        let store = Arc::new(fixture);
+        let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
+        let error = result.expect_err("stream error must propagate");
+        assert!(error.to_string().contains("observation stream failed"));
+        assert_eq!(store.writes.load(Ordering::SeqCst), 0);
+    }
+
+    #[tokio::test]
+    async fn observation_enforces_byte_limit() {
+        let store = Arc::new(ObservationStore::new(vec![7; 64], 0));
+        let result = super::read_config_observation(store.clone(), "config/observation.bin", 8).await;
+        assert!(result.is_err(), "I05_BOUND_ENFORCED");
+        assert_eq!(
+            result.expect_err("bounded observation error"),
+            Error::CorruptedFormat,
+            "I05_BOUND_ENFORCED"
+        );
+        assert_eq!(store.consumed.load(Ordering::SeqCst), 9, "I05_BOUND_ENFORCED: drain exactly limit + 1");
+        let store = Arc::new(ObservationStore::new(vec![7; 8], 8));
+        let result = super::read_config_observation(store.clone(), "config/observation.bin", 8).await;
+        assert!(result.is_ok(), "exact byte limit must succeed: {result:?}");
+        assert_eq!(result.expect("exact limit").0, vec![7; 8]);
+        assert_eq!(store.consumed.load(Ordering::SeqCst), 8);
+    }
+
     #[tokio::test]
     async fn read_config_preserve_empty_distinguishes_empty_object() {
         let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(Vec::new()), None));
@@ -5799,6 +6189,60 @@ mod tests {
         assert!(data.is_empty());
     }
 
+    #[tokio::test]
+    async fn read_config_preserve_empty_metadata_keeps_empty_and_options() {
+        let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(Vec::new()), None));
+        for no_lock in [false, true] {
+            let opts = ObjectOptions {
+                no_lock,
+                ..Default::default()
+            };
+            let (data, info) = read_config_preserve_empty_with_metadata_opts(store.clone(), "config/empty.json", &opts)
+                .await
+                .expect("an existing empty config must retain its payload and metadata");
+            assert!(data.is_empty());
+            assert_eq!(info.size, 0);
+            assert_eq!(info.actual_size, 0);
+            assert_eq!(info.etag.as_deref(), Some("config-1"));
+            assert_eq!(info.data_dir, Some(uuid::Uuid::from_u128(1)));
+            assert_eq!(store.last_get_no_lock.load(Ordering::SeqCst), no_lock);
+        }
+        let err = read_config(store, "config/empty.json")
+            .await
+            .expect_err("the existing reader must still treat empty configs as missing");
+        assert!(matches!(err, Error::ConfigNotFound));
+    }
+
+    #[tokio::test]
+    async fn read_config_preserve_empty_metadata_keeps_payload_and_metadata() {
+        let payload = b"original config bytes".to_vec();
+        let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(payload.clone()), None));
+        let (data, info) =
+            read_config_preserve_empty_with_metadata_opts(store, "config/nonempty.json", &ObjectOptions::default())
+                .await
+                .expect("a nonempty config must retain its payload and metadata");
+        assert_eq!(data, payload);
+        let size = i64::try_from(payload.len()).expect("test payload size fits i64");
+        assert_eq!(info.size, size);
+        assert_eq!(info.actual_size, size);
+        assert_eq!(info.etag.as_deref(), Some("config-1"));
+        assert_eq!(info.data_dir, Some(uuid::Uuid::from_u128(1)));
+    }
+
+    #[tokio::test]
+    async fn read_config_preserve_empty_metadata_preserves_read_errors() {
+        let missing = Arc::new(RecoveryMockStore::new(RecoveryReadState::Missing, None));
+        let err = read_config_preserve_empty_with_metadata_opts(missing, "config/missing.json", &ObjectOptions::default())
+            .await
+            .expect_err("physical absence must remain ConfigNotFound");
+        assert!(matches!(err, Error::ConfigNotFound));
+        let quorum = Arc::new(RecoveryMockStore::new(RecoveryReadState::QuorumError, None));
+        let err = read_config_preserve_empty_with_metadata_opts(quorum, "config/quorum.json", &ObjectOptions::default())
+            .await
+            .expect_err("a quorum failure must retain its typed read error");
+        assert!(matches!(err, Error::ErasureReadQuorum));
+    }
+
     #[async_trait::async_trait]
     impl StorageAdminApi for RecoveryMockStore {
         type BackendInfo = rustfs_madmin::BackendInfo;

crates/ecstore/src/config/scanner.rs

@@ -17,12 +17,13 @@ use rustfs_config::{
     DEFAULT_SCANNER_BITROT_CYCLE_SECS, DEFAULT_SCANNER_CACHE_SAVE_TIMEOUT_SECS, DEFAULT_SCANNER_CYCLE_MAX_DIRECTORIES,
     DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS, DEFAULT_SCANNER_CYCLE_MAX_OBJECTS, DEFAULT_SCANNER_DISK_IO_BUDGET,
     DEFAULT_SCANNER_IDLE_MODE, DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS, DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS,
-    DEFAULT_SCANNER_SPEED, DEFAULT_SCANNER_WALKERS_PER_DISK, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS, SCANNER_ALERT_EXCESS_FOLDERS,
-    SCANNER_ALERT_EXCESS_VERSION_SIZE, SCANNER_ALERT_EXCESS_VERSIONS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT,
-    SCANNER_CYCLE, SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY,
+    DEFAULT_SCANNER_RECOVERY_BACKOFF_CAP_SECS, DEFAULT_SCANNER_SPEED, DEFAULT_SCANNER_WALKERS_PER_DISK,
+    DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS, SCANNER_ALERT_EXCESS_FOLDERS, SCANNER_ALERT_EXCESS_VERSION_SIZE,
+    SCANNER_ALERT_EXCESS_VERSIONS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE,
+    SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY,
     SCANNER_DISK_IO_BUDGET_BYTES_PER_SEC, SCANNER_DISK_IO_BUDGET_IOPS, SCANNER_IDLE_MODE, SCANNER_MAX_CONCURRENT_DISK_SCANS,
-    SCANNER_MAX_CONCURRENT_SET_SCANS, SCANNER_MAX_WAIT, SCANNER_SPEED, SCANNER_START_DELAY, SCANNER_WALKERS_PER_DISK,
-    SCANNER_YIELD_EVERY_N_OBJECTS,
+    SCANNER_MAX_CONCURRENT_SET_SCANS, SCANNER_MAX_WAIT, SCANNER_RECOVERY_BACKOFF_CAP, SCANNER_SPEED, SCANNER_START_DELAY,
+    SCANNER_WALKERS_PER_DISK, SCANNER_YIELD_EVERY_N_OBJECTS,
 };
 use std::sync::LazyLock;
 
@@ -83,6 +84,11 @@ pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
             value: DEFAULT_SCANNER_CACHE_SAVE_TIMEOUT_SECS.to_string(),
             hidden_if_empty: false,
         },
+        KV {
+            key: SCANNER_RECOVERY_BACKOFF_CAP.to_owned(),
+            value: DEFAULT_SCANNER_RECOVERY_BACKOFF_CAP_SECS.to_string(),
+            hidden_if_empty: false,
+        },
         KV {
             key: SCANNER_MAX_CONCURRENT_SET_SCANS.to_owned(),
             value: DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS.to_string(),

crates/ecstore/src/core/pools.rs

@@ -196,6 +196,56 @@ const DECOMMISSION_RETRY_BACKOFF_CAP: std::time::Duration = std::time::Duration:
 const DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT: usize = 100;
 const DECOMMISSION_TERMINAL_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(1);
 const DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS: usize = 3;
+/// Relaunch delay after a durable pause. A first pause is relaunched by the
+/// next watchdog pass as before; each further pause on the same cause without
+/// progress holds the relaunch for this base, doubled per repeat.
+const DECOMMISSION_PAUSE_RELAUNCH_BASE: std::time::Duration = std::time::Duration::from_secs(30);
+const DECOMMISSION_PAUSE_RELAUNCH_MAX: std::time::Duration = std::time::Duration::from_secs(15 * 60);
+
+/// The pause record after `reason` paused a pool whose settled progress is
+/// `progress`. A pause keeps its exit: the pool is relaunched after the
+/// returned delay, never failed. Only a repeat of the same cause without
+/// progress in between grows the delay. Progress is what a resumed pass cannot
+/// redo: completed buckets and entries moved off the pool. Item counters are
+/// not, because a relisted retained entry is seen again on every pass.
+fn next_decommission_pause_record(
+    previous: Option<&crate::runtime::instance::DecommissionPauseRecord>,
+    reason: &str,
+    progress: u64,
+    now: tokio::time::Instant,
+) -> crate::runtime::instance::DecommissionPauseRecord {
+    let repeats = match previous {
+        Some(previous) if previous.reason == reason && previous.progress == progress => previous.repeats.saturating_add(1),
+        _ => 1,
+    };
+    let delay = if repeats < 2 {
+        std::time::Duration::ZERO
+    } else {
+        DECOMMISSION_PAUSE_RELAUNCH_BASE
+            .saturating_mul(1_u32 << (repeats - 2).min(5))
+            .min(DECOMMISSION_PAUSE_RELAUNCH_MAX)
+    };
+    crate::runtime::instance::DecommissionPauseRecord {
+        reason: reason.to_owned(),
+        progress,
+        repeats,
+        relaunch_after: now + delay,
+    }
+}
+
+/// Whether the watchdog must not relaunch this pool yet: it is still paused
+/// on the cause of its last recorded pause and that pause's delay has not
+/// run out.
+fn decommission_pause_relaunch_pending(
+    record: Option<&crate::runtime::instance::DecommissionPauseRecord>,
+    info: Option<&PoolDecommissionInfo>,
+    now: tokio::time::Instant,
+) -> bool {
+    record.is_some_and(|record| {
+        now < record.relaunch_after
+            && info.and_then(|info| info.capacity_blocked_reason.as_deref()) == Some(record.reason.as_str())
+    })
+}
 
 fn decommission_capacity_target_gate_retry_exhausted(attempt: usize) -> bool {
     attempt.saturating_add(1) >= DECOMMISSION_CAPACITY_TARGET_GATE_MAX_ATTEMPTS
@@ -719,6 +769,25 @@ pub(crate) fn local_decommission_queue_prefix(endpoints: &EndpointServerPools, i
     Ok(local)
 }
 
+/// Carry `current`'s progress counters for pool `idx` into `snapshot` when
+/// both describe the same decommission generation. Counters only grow within
+/// a generation, so the larger value of each is the newer one.
+fn carry_decommission_progress_counters(snapshot: &mut PoolMeta, current: &PoolMeta, idx: usize) {
+    let (Some(target), Some(source)) = (
+        snapshot.pools.get_mut(idx).and_then(|pool| pool.decommission.as_mut()),
+        current.pools.get(idx).and_then(|pool| pool.decommission.as_ref()),
+    ) else {
+        return;
+    };
+    if target.start_time.is_none() || target.start_time != source.start_time {
+        return;
+    }
+    target.items_decommissioned = target.items_decommissioned.max(source.items_decommissioned);
+    target.items_decommission_failed = target.items_decommission_failed.max(source.items_decommission_failed);
+    target.bytes_done = target.bytes_done.max(source.bytes_done);
+    target.bytes_failed = target.bytes_failed.max(source.bytes_failed);
+}
+
 fn resumable_decommission_queue_indices(meta: &PoolMeta) -> Vec<usize> {
     let mut indices = Vec::new();
     for (idx, pool) in meta.pools.iter().enumerate() {
@@ -5617,9 +5686,9 @@ enum DecommissionEntryAttemptOutcome {
 struct DeferredDecommissionEntry {
     name: String,
     attempts: usize,
-    /// Versions whose progress was already counted. A deferral after the
-    /// copy (source cleanup) has counted every version; the replay must not
-    /// count them again.
+    /// Versions whose progress was already counted. Versions are counted
+    /// only when their entry settles, so a deferred entry has counted none;
+    /// this guards the replay against counting a settled version twice.
     counted_versions: HashSet<(Option<uuid::Uuid>, bool)>,
 }
 
@@ -12362,6 +12431,22 @@ fn resolve_data_movement_lifecycle_expiry_result(action: IlmAction, apply_action
     )))
 }
 
+const DATA_MOVEMENT_DELETE_ALL_DEFERRED_REASON: &str = "delete_all_needs_bucket_write_lock";
+
+fn log_data_movement_lifecycle_expiry_deferred(bucket: &str, object: &str, action: IlmAction, reason: &str) {
+    warn!(
+        event = EVENT_DECOMMISSION_ENTRY,
+        component = LOG_COMPONENT_ECSTORE,
+        subsystem = LOG_SUBSYSTEM_POOLS,
+        state = "lifecycle_expiry_deferred",
+        bucket = %bucket,
+        object = %object,
+        action = ?action,
+        reason,
+        "Data movement deferred a version's lifecycle expiry; moving the version"
+    );
+}
+
 #[allow(clippy::too_many_arguments)]
 pub(crate) async fn should_skip_lifecycle_for_data_movement(
     store: Arc<ECStore>,
@@ -12421,6 +12506,21 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement(
             }
             let applied = !apply_actions
                 || match lock_lost_signal {
+                    // A delete-all is a prefix delete that takes the bucket
+                    // lifecycle write lock, which the lifecycle read locks held
+                    // by the inline apply and by the rebalance entry's bucket
+                    // incarnation fence would block until timeout. The version
+                    // is still live, so it moves like any other version and
+                    // lifecycle expires it on the target.
+                    Some(_) if action.delete_all() => {
+                        log_data_movement_lifecycle_expiry_deferred(
+                            bucket,
+                            &version.name,
+                            action,
+                            DATA_MOVEMENT_DELETE_ALL_DEFERRED_REASON,
+                        );
+                        return Ok(false);
+                    }
                     Some(signal) => {
                         apply_expiry_rule_for_data_movement(store, &event, event_source, &object_info, Some(signal)).await
                     }
@@ -12446,16 +12546,15 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement(
                             // The version is still live, so it moves like any
                             // other version and lifecycle expires it on the
                             // target.
-                            warn!(
-                                event = EVENT_DECOMMISSION_ENTRY,
-                                component = LOG_COMPONENT_ECSTORE,
-                                subsystem = LOG_SUBSYSTEM_POOLS,
-                                state = "lifecycle_expiry_deferred",
-                                bucket = %bucket,
-                                object = %version.name,
-                                action = ?action,
-                                reason = if delete_all { "delete_all_needs_bucket_write_lock" } else { "not_applied" },
-                                "Data movement deferred a transitioned version's lifecycle expiry; moving the version"
+                            log_data_movement_lifecycle_expiry_deferred(
+                                bucket,
+                                &version.name,
+                                action,
+                                if delete_all {
+                                    DATA_MOVEMENT_DELETE_ALL_DEFERRED_REASON
+                                } else {
+                                    "not_applied"
+                                },
                             );
                             return Ok(false);
                         }
@@ -15740,6 +15839,14 @@ impl ECStore {
         let (pool_meta_guard, mut snapshot) = self
             .acquire_pool_meta_write_guard(&mut save_guard, "decommission capacity pause failed")
             .await?;
+        // The pause is a checkpoint too. Without carrying the counters of this
+        // generation, everything counted since the last progress save would
+        // be dropped, and the relisted pass could not tell a version it
+        // already counted from one it never did.
+        {
+            let pool_meta = self.pool_meta.read().await;
+            carry_decommission_progress_counters(&mut snapshot, &pool_meta, idx);
+        }
         snapshot.mark_decommission_capacity_blocked(idx, decommission_pause_reason(err), OffsetDateTime::now_utc())?;
         let outcome = snapshot
             .save_no_lock_armed(
@@ -15750,12 +15857,40 @@ impl ECStore {
             )
             .await?;
         ensure_pool_meta_write_fence(&pool_meta_guard, "decommission capacity pause failed")?;
-        {
+        let (generation, completed_buckets) = {
             let mut pool_meta = self.pool_meta.write().await;
             publish_pool_meta_updates(&mut pool_meta, &outcome.committed, &[idx]);
-        }
+            pool_meta
+                .pools
+                .get(idx)
+                .and_then(|pool| pool.decommission.as_ref())
+                .map_or((None, 0), |info| (info.start_time, info.decommissioned_buckets.len()))
+        };
+        let cleaned_entries = generation.map_or(0, |generation| {
+            self.ctx
+                .with_decommission_count_ledger(idx, generation, |ledger| ledger.progress())
+        });
+        let progress = cleaned_entries.saturating_add(u64::try_from(completed_buckets).unwrap_or(u64::MAX));
         ensure_pool_meta_write_fence(&pool_meta_guard, "decommission capacity pause failed")?;
         outcome.disarm();
+        let reason = decommission_pause_reason(err);
+        let now = tokio::time::Instant::now();
+        let record = self
+            .ctx
+            .record_decommission_pause(idx, |previous| next_decommission_pause_record(previous, &reason, progress, now));
+        if record.repeats > 1 {
+            warn!(
+                event = EVENT_DECOMMISSION_STATE,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_POOLS,
+                state = "pause_relaunch_backoff",
+                pool_index = idx,
+                repeats = record.repeats,
+                retry_delay_secs = record.relaunch_after.saturating_duration_since(now).as_secs(),
+                reason = %reason,
+                "Decommission paused again on the same cause without progress; backing off its relaunch"
+            );
+        }
         Ok(())
     }
 
@@ -15856,10 +15991,24 @@ impl ECStore {
         rx: &CancellationToken,
         endpoints: &EndpointServerPools,
     ) -> Result<Vec<(usize, DecommissionCancelerGuard)>> {
-        let indices = {
+        let (indices, relaunch_pending) = {
             let pool_meta = self.pool_meta.read().await;
-            resumable_decommission_queue_indices(&pool_meta)
+            let indices = resumable_decommission_queue_indices(&pool_meta);
+            let now = tokio::time::Instant::now();
+            let relaunch_pending = indices.iter().any(|idx| {
+                decommission_pause_relaunch_pending(
+                    self.ctx.decommission_pause(*idx).as_ref(),
+                    pool_meta.pools.get(*idx).and_then(|pool| pool.decommission.as_ref()),
+                    now,
+                )
+            });
+            (indices, relaunch_pending)
         };
+        // A paused pool resumes in queue order; while its backoff runs, no
+        // later pool may start ahead of it.
+        if relaunch_pending {
+            return Ok(Vec::new());
+        }
         let indices = local_decommission_queue_prefix(endpoints, &indices)?;
         self.reserve_decommission_routines(rx, indices.as_slice()).await
     }
@@ -16855,28 +17004,36 @@ impl ECStore {
                         DecommissionCapacityAdmission::ScannerBacklog,
                     )?;
                 }
-                let results = join_all(
+                let persist = |read: &data_movement::scanner_backlog::ScannerPauseBacklogRetirementRead| {
+                    let target = Arc::clone(&self.pools[read.replica.pool_index].disk_set[read.replica.set_index]);
+                    let record = record.clone();
+                    let preconditions = read.preconditions();
+                    let object_fence = &object_fence;
+                    let pool_meta_guard = &pool_meta_guard;
+                    async move {
+                        let mut opts = ObjectOptions {
+                            no_lock: self.pools[0].disk_set[0].shares_namespace_lock_domain(&target).await,
+                            ..Default::default()
+                        };
+                        object_fence.add_namespace_lock_fence(&mut opts);
+                        opts.add_namespace_lock_guard(pool_meta_guard);
+                        persist_native_scanner_pause_backlog_replica(target, record, preconditions, opts, phase).await
+                    }
+                };
+                let survivors = || {
                     replicas
                         .iter()
                         .filter(|read| read.replica.pool_index != idx && read.replica.writable)
-                        .map(|read| {
-                            let target = Arc::clone(&self.pools[read.replica.pool_index].disk_set[read.replica.set_index]);
-                            let record = record.clone();
-                            let object_fence = &object_fence;
-                            let pool_meta_guard = &pool_meta_guard;
-                            async move {
-                                let mut opts = ObjectOptions {
-                                    no_lock: self.pools[0].disk_set[0].shares_namespace_lock_domain(&target).await,
-                                    ..Default::default()
-                                };
-                                object_fence.add_namespace_lock_fence(&mut opts);
-                                opts.add_namespace_lock_guard(pool_meta_guard);
-                                persist_native_scanner_pause_backlog_replica(target, record, read.preconditions(), opts, phase)
-                                    .await
-                            }
-                        }),
-                )
-                .await;
+                };
+                // Create survivors that read as missing first, one at a time, and
+                // stop at the first failure. A failed create then leaves every
+                // stored survivor unchanged, so the retry re-plans from the same
+                // authority; a survivor that only read as missing rejects its
+                // create before any stored survivor is overwritten.
+                for read in survivors().filter(|read| read.replica.data.is_none()) {
+                    persist(read).await?;
+                }
+                let results = join_all(survivors().filter(|read| read.replica.data.is_some()).map(persist)).await;
                 // Every dispatched native CAS has finished before an error can
                 // release the owned task's object and durable membership fences.
                 for result in results {
@@ -16919,6 +17076,7 @@ impl ECStore {
             &[],
             data_movement::SourceCleanupBucketFence {
                 expected_incarnation_id: None,
+                lifecycle_owner: None,
                 lifecycle_guard: None,
                 namespace_lock_lost_signal: pool_meta_guard.lock_lost_signal(),
                 object_mutation_fence: Some(&object_fence),
@@ -17187,7 +17345,7 @@ impl ECStore {
         let operation_gate = self.ctx.data_movement_operation_gate();
 
         let bucket_incarnation_fence = match expected_bucket_incarnation_id {
-            Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?),
+            Some(expected) => Some(Arc::new(self.acquire_bucket_incarnation_fence(&bucket, expected).await?)),
             None => None,
         };
 
@@ -17282,10 +17440,13 @@ impl ECStore {
         let mut decommissioned: usize = 0;
         let mut expired: usize = 0;
         let mut free_version_disposition = DecommissionFreeVersionDisposition::default();
+        // Counted when the entry settles; see the flush before the progress save.
+        let mut pending_counts: Vec<((Option<uuid::Uuid>, bool), usize, bool)> = Vec::with_capacity(fivs.versions.len());
         let mut cleanup_preflight_allowed_missing = Vec::new();
         let mut cleanup_preflight_vanished = Vec::new();
         let mut cleanup_transferred_tier_versions = Vec::new();
         let mut entry_blocked = false;
+        let mut source_cleaned = false;
 
         for version in fivs.versions.iter() {
             if self.decommission_cancel_requested(idx, &rx).await {
@@ -17385,18 +17546,7 @@ impl ECStore {
                     ));
                 }
 
-                if counted_versions.insert((version.version_id, version.deleted)) {
-                    let mut pool_meta = self.pool_meta.write().await;
-                    ensure_decommission_generation(&pool_meta, idx, generation)?;
-                    if let Err(err) = count_decommission_item(&mut pool_meta, idx, 0, !migrated && !consumed) {
-                        return Err(with_decommission_entry_context(
-                            "count_decommission_item",
-                            bucket.as_str(),
-                            entry.name.as_str(),
-                            err,
-                        ));
-                    }
-                }
+                pending_counts.push(((version.version_id, version.deleted), 0, !migrated && !consumed));
 
                 if migrated || consumed {
                     decommissioned += 1;
@@ -17655,18 +17805,7 @@ impl ECStore {
                     continue;
                 }
 
-                if counted_versions.insert((version.version_id, version.deleted)) {
-                    let mut pool_meta = self.pool_meta.write().await;
-                    ensure_decommission_generation(&pool_meta, idx, generation)?;
-                    if let Err(err) = count_decommission_item(&mut pool_meta, idx, 0, failure) {
-                        return Err(with_decommission_entry_context(
-                            "count_decommission_item",
-                            bucket.as_str(),
-                            entry.name.as_str(),
-                            err,
-                        ));
-                    }
-                }
+                pending_counts.push(((version.version_id, version.deleted), 0, failure));
 
                 if !failure {
                     decommissioned += 1;
@@ -18059,18 +18198,7 @@ impl ECStore {
                 continue;
             }
 
-            if counted_versions.insert((version.version_id, version.deleted)) {
-                let mut pool_meta = self.pool_meta.write().await;
-                ensure_decommission_generation(&pool_meta, idx, generation)?;
-                if let Err(err) = count_decommission_item(&mut pool_meta, idx, decommission_item_size(version.size), failure) {
-                    return Err(with_decommission_entry_context(
-                        "count_decommission_item",
-                        bucket.as_str(),
-                        entry.name.as_str(),
-                        err,
-                    ));
-                }
-            }
+            pending_counts.push(((version.version_id, version.deleted), decommission_item_size(version.size), failure));
 
             if failure {
                 break;
@@ -18153,9 +18281,8 @@ impl ECStore {
                         &cleanup_transferred_tier_versions,
                         data_movement::SourceCleanupBucketFence {
                             expected_incarnation_id: expected_bucket_incarnation_id,
-                            lifecycle_guard: bucket_incarnation_fence
-                                .as_ref()
-                                .and_then(|guard| guard.namespace_lock_guard()),
+                            lifecycle_owner: bucket_incarnation_fence.as_ref(),
+                            lifecycle_guard: None,
                             namespace_lock_lost_signal: None,
                             object_mutation_fence: Some(&source_cleanup_mutation_fence),
                         },
@@ -18165,7 +18292,7 @@ impl ECStore {
                 })
                 .await;
             match cleanup_result {
-                Ok(_) => {}
+                Ok(_) => source_cleaned = true,
                 Err(data_movement::SourceCleanupError::Storage(err)) => {
                     resolve_decommission_entry_cleanup_delete_result(
                         Err::<(), Error>(err),
@@ -18245,6 +18372,48 @@ impl ECStore {
             let mut pool_meta = self.pool_meta.write().await;
             ensure_decommission_generation(&pool_meta, idx, generation)?;
 
+            // Versions are counted once their entry settles: copied and cleaned
+            // up, or retained. An attempt that is retried, deferred or resumed
+            // after a pause recounted nothing, because nothing of it was
+            // counted. A retained entry is relisted by every resumed pass; the
+            // ledger counts its versions once per process (rustfs/backlog#2873).
+            let had_failure = pending_counts.iter().any(|(_, _, failed)| *failed);
+            let admitted = self.ctx.with_decommission_count_ledger(idx, generation, |ledger| {
+                if source_cleaned {
+                    ledger.record_cleaned_entry();
+                }
+                pending_counts
+                    .into_iter()
+                    .filter(|(version, _, _)| ledger.admit(bucket.as_str(), entry.name.as_str(), *version, source_cleaned))
+                    .collect::<Vec<_>>()
+            });
+            for (version, size, failure) in admitted {
+                if counted_versions.insert(version)
+                    && let Err(err) = count_decommission_item(&mut pool_meta, idx, size, failure)
+                {
+                    return Err(with_decommission_entry_context(
+                        "count_decommission_item",
+                        bucket.as_str(),
+                        entry.name.as_str(),
+                        err,
+                    ));
+                }
+            }
+
+            // Failure presence also selects the terminal state. A full
+            // diagnostic ledger may undercount versions, but cannot erase the
+            // last evidence that this generation encountered a failed entry.
+            if had_failure
+                && pool_meta.pools[idx]
+                    .decommission
+                    .as_ref()
+                    .is_some_and(|info| info.items_decommission_failed == 0)
+            {
+                count_decommission_item(&mut pool_meta, idx, 0, true).map_err(|err| {
+                    with_decommission_entry_context("count_decommission_failure", bucket.as_str(), entry.name.as_str(), err)
+                })?;
+            }
+
             if let Err(err) = track_decommission_current_object(&mut pool_meta, idx, bucket.as_str(), entry.name.as_str()) {
                 return Err(with_decommission_entry_context(
                     "track_decommission_current_object",
@@ -27276,6 +27445,81 @@ mod tests {
         assert_eq!(named(&[]), None);
     }
 
+    /// A copy that lost drive quorum stays the named pause cause when a
+    /// transient contention settles after it (rustfs/backlog#2873, defect F).
+    #[test]
+    fn decommission_deferral_cause_keeps_a_copy_quorum_loss() {
+        let quorum = decommission_copy_quorum_deferral(
+            "delete_marker_copy",
+            "bucket",
+            "object",
+            Error::InsufficientWriteQuorum("bucket".to_owned(), "object".to_owned()),
+        );
+        let busy = decommission_target_gate_busy_error(2);
+        let named = |errors: &[&Error]| {
+            let mut recorded = None;
+            for err in errors {
+                record_decommission_deferral_cause(&mut recorded, err);
+            }
+            recorded.map(|(_, cause)| cause)
+        };
+        assert_eq!(named(&[&quorum, &busy]), Some(quorum.to_string()));
+        assert_eq!(named(&[&busy, &quorum]), Some(quorum.to_string()));
+        let reason = ensure_decommission_deferred_drained("bucket", 0, 1, named(&[&quorum, &busy]).as_deref())
+            .expect_err("a remaining deferral pauses the set")
+            .to_string();
+        assert!(reason.contains("lost drive quorum"), "{reason}");
+    }
+
+    /// A first pause relaunches on the next watchdog pass; only a repeat of
+    /// the same cause without a counted item backs off, and the delay is
+    /// bounded. Progress or a new cause starts over.
+    #[test]
+    fn decommission_pause_relaunch_backs_off_only_on_repeats_without_progress() {
+        let now = tokio::time::Instant::now();
+        let first = next_decommission_pause_record(None, "cause", 7, now);
+        assert_eq!((first.repeats, first.relaunch_after), (1, now));
+
+        let mut record = first;
+        let mut delays = Vec::new();
+        for _ in 0..8 {
+            record = next_decommission_pause_record(Some(&record), "cause", 7, now);
+            delays.push(record.relaunch_after - now);
+        }
+        assert_eq!(delays[0], DECOMMISSION_PAUSE_RELAUNCH_BASE);
+        assert_eq!(delays[1], 2 * DECOMMISSION_PAUSE_RELAUNCH_BASE);
+        assert!(delays.windows(2).all(|pair| pair[0] <= pair[1]), "{delays:?}");
+        assert_eq!(*delays.last().unwrap(), DECOMMISSION_PAUSE_RELAUNCH_MAX);
+
+        let progressed = next_decommission_pause_record(Some(&record), "cause", 8, now);
+        assert_eq!((progressed.repeats, progressed.relaunch_after), (1, now));
+        let other = next_decommission_pause_record(Some(&record), "other cause", 7, now);
+        assert_eq!((other.repeats, other.relaunch_after), (1, now));
+    }
+
+    /// The hold applies only while the pool is still paused on the recorded
+    /// cause and its delay has not run out.
+    #[test]
+    fn decommission_pause_relaunch_hold_requires_the_same_live_pause() {
+        let now = tokio::time::Instant::now();
+        let first = next_decommission_pause_record(None, "cause", 7, now);
+        let repeated = next_decommission_pause_record(Some(&first), "cause", 7, now);
+        let paused = PoolDecommissionInfo {
+            capacity_blocked_reason: Some("cause".to_owned()),
+            ..Default::default()
+        };
+        assert!(decommission_pause_relaunch_pending(Some(&repeated), Some(&paused), now));
+        assert!(!decommission_pause_relaunch_pending(
+            Some(&repeated),
+            Some(&paused),
+            repeated.relaunch_after
+        ));
+        assert!(!decommission_pause_relaunch_pending(Some(&first), Some(&paused), now));
+        let resumed = PoolDecommissionInfo::default();
+        assert!(!decommission_pause_relaunch_pending(Some(&repeated), Some(&resumed), now));
+        assert!(!decommission_pause_relaunch_pending(None, Some(&paused), now));
+    }
+
     /// A set that still holds a deferred entry must not report success.
     #[test]
     fn decommission_deferred_drain_invariant_rejects_unresolved_entries() {
@@ -28429,7 +28673,13 @@ impl SetDisks {
                 min_disks: listing_quorum,
                 skip_walkdir_total_timeout: true,
                 walkdir_stall_timeout: Some(DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT),
-                agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry)))),
+                agreed: Some(Box::new(move |entry: MetaCacheEntry| {
+                    let future = cb1(entry);
+                    Box::pin(async move {
+                        future.await;
+                        Ok(())
+                    })
+                })),
                 partial: Some(Box::new(move |entries: MetaCacheEntries, errs: &[Option<DiskError>]| {
                     let resolver = resolver.clone();
                     let cb_func = cb_func.clone();
@@ -29158,6 +29408,7 @@ mod pools_tests {
             pool_meta_save_gate: tokio::sync::Mutex::new(super::PoolMetaWriteState::for_test_bootstrap()),
             ctx,
             bucket_fence_registry: Arc::default(),
+            ordinary_classification: Default::default(),
         })
     }
 

crates/ecstore/src/core/pools_test.rs

@@ -499,6 +499,7 @@ mod decommission_lock_order_tests {
             ),
             ctx,
             bucket_fence_registry: Arc::default(),
+            ordinary_classification: Default::default(),
         });
         crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
         (store, refresh_calls)
@@ -2447,6 +2448,481 @@ mod decommission_lock_order_tests {
         });
     }
 
+    /// Seed `object` in pool 0 with three 2 KiB data versions and a newest
+    /// delete marker; returns the data versions oldest first.
+    async fn seed_versioned_entry_with_marker(
+        store: &Arc<crate::store::ECStore>,
+        bucket: &str,
+        object: &str,
+    ) -> Vec<crate::object_api::ObjectInfo> {
+        store
+            .make_bucket(
+                bucket,
+                &MakeBucketOptions {
+                    versioning_enabled: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("create the versioned bucket");
+        let mut written = Vec::new();
+        for index in 0..3u8 {
+            written.push(
+                store.pools[0]
+                    .put_object(
+                        bucket,
+                        object,
+                        &mut PutObjReader::from_vec(vec![index; 2048]),
+                        &ObjectOptions {
+                            versioned: true,
+                            ..Default::default()
+                        },
+                    )
+                    .await
+                    .expect("write a source version"),
+            );
+        }
+        store.pools[0]
+            .delete_object(
+                bucket,
+                object,
+                ObjectOptions {
+                    versioned: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("write the source delete marker");
+        written
+    }
+
+    /// Fail every target write of `object` after the first, before it
+    /// commits, while `armed` is set.
+    fn fail_target_writes_after_the_first(
+        bucket: &str,
+        object: &'static str,
+        armed: Arc<AtomicBool>,
+    ) -> DecommissionTestFaultGuard {
+        let writes = Arc::new(AtomicUsize::new(0));
+        let bucket = bucket.to_owned();
+        DecommissionTestFaultGuard::install_with_error(
+            Arc::new(move |stage, entry_bucket, entry_object, _, _| {
+                stage == "before_target_commit"
+                    && entry_bucket == bucket
+                    && entry_object == object
+                    && armed.load(Ordering::Acquire)
+                    && writes.fetch_add(1, Ordering::AcqRel) >= 1
+            }),
+            Some(Arc::new(|_, bucket, object| {
+                crate::error::Error::InsufficientWriteQuorum(bucket.to_owned(), object.to_owned())
+            })),
+        )
+    }
+
+    async fn assert_every_version_readable(
+        store: &Arc<crate::store::ECStore>,
+        bucket: &str,
+        object: &str,
+        written: &[crate::object_api::ObjectInfo],
+    ) {
+        for (index, version) in written.iter().enumerate() {
+            let opts = ObjectOptions {
+                versioned: true,
+                version_id: version.version_id.map(|id| id.to_string()),
+                ..Default::default()
+            };
+            let mut reader = store
+                .get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
+                .await
+                .unwrap_or_else(|err| panic!("version {:?} must stay readable: {err:?}", version.version_id));
+            let mut body = Vec::new();
+            reader.stream.read_to_end(&mut body).await.expect("read the version body");
+            assert_eq!(body, vec![index as u8; 2048], "version {:?}", version.version_id);
+        }
+    }
+
+    fn decommission_info(pool_meta: &crate::core::pools::PoolMeta) -> crate::core::pools::PoolDecommissionInfo {
+        pool_meta.pools[0]
+            .decommission
+            .clone()
+            .expect("the decommission state should remain present")
+    }
+
+    /// Replace the in-memory pool metadata with what the replicas persisted,
+    /// as a restarted leader would load it.
+    async fn reload_persisted_pool_meta(store: &Arc<crate::store::ECStore>) {
+        let mut persisted = crate::core::pools::PoolMeta::default();
+        persisted
+            .load_no_lock_from_replicas(store.pools.clone())
+            .await
+            .expect("reload the persisted pool metadata");
+        *store.pool_meta.write().await = persisted;
+    }
+
+    /// Wait until the relaunched routine recorded its `repeats`-th pause and
+    /// released its worker slot.
+    async fn wait_for_relaunched_pause(store: &Arc<crate::store::ECStore>, repeats: u32) {
+        tokio::time::timeout(Duration::from_secs(60), async {
+            loop {
+                let paused = store
+                    .ctx
+                    .decommission_pause(0)
+                    .is_some_and(|record| record.repeats >= repeats);
+                if paused && !store.has_active_local_decommission_worker().await {
+                    return;
+                }
+                tokio::time::sleep(Duration::from_millis(50)).await;
+            }
+        })
+        .await
+        .unwrap_or_else(|_| panic!("the relaunched routine should pause again: record={:?}", store.ctx.decommission_pause(0)));
+    }
+
+    /// A partially moved entry pauses the decommission, and the resumed
+    /// routine, started from the persisted metadata, finds its first versions
+    /// already on the target. Every version is counted exactly once
+    /// (rustfs/backlog#2873, defect F follow-up).
+    #[test]
+    #[serial_test::serial]
+    fn decommission_counts_each_version_once_across_a_pause_and_resume() {
+        run_large_stack_async_test("decommission-pause-resume-count", || async {
+            // Serialize this accounting fixture so the deferred entry precedes
+            // the later checkpoint, regardless of worker scheduling.
+            let env = [
+                ("RUSTFS_DECOMMISSION_BUCKET_CONCURRENCY", Some("1")),
+                ("RUSTFS_DECOMMISSION_ENTRY_CONCURRENCY", Some("1")),
+            ];
+            temp_env::async_with_vars(env, async {
+                let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
+                let bucket = test_bucket("pause-resume-count");
+                let object = "versioned/03.bin";
+                let written = seed_versioned_entry_with_marker(&store, &bucket, object).await;
+                // Listed after the partially moved entry: its completion saves
+                // a progress checkpoint while that entry's first versions are
+                // moved and its later ones are still deferred.
+                let source_set = store.pools[0].get_disks_by_key(object).set_index;
+                let later = (0..1024)
+                    .map(|index| format!("z/after-{index:04}.bin"))
+                    .find(|key| store.pools[0].get_disks_by_key(key).set_index == source_set)
+                    .expect("a later key in the same source set");
+                store.pools[0]
+                    .put_object(
+                        &bucket,
+                        &later,
+                        &mut PutObjReader::from_vec(vec![0x7a; 16]),
+                        &ObjectOptions {
+                            versioned: true,
+                            ..Default::default()
+                        },
+                    )
+                    .await
+                    .expect("write the later entry");
+
+                // Counters when the test bucket's first entry starts: only the
+                // system objects are in them.
+                let baseline = Arc::new(StdMutex::new(None::<(String, usize, usize)>));
+                let armed = Arc::new(AtomicBool::new(true));
+                let writes = Arc::new(AtomicUsize::new(0));
+                let _fault = DecommissionTestFaultGuard::install_with_error(
+                    Arc::new({
+                        let store = Arc::clone(&store);
+                        let bucket = bucket.clone();
+                        let baseline = Arc::clone(&baseline);
+                        let armed = Arc::clone(&armed);
+                        move |stage, entry_bucket, entry_object, _, _| {
+                            if entry_bucket != bucket {
+                                return false;
+                            }
+                            if stage == "exact_versions_read" {
+                                let mut baseline = baseline.lock().expect("baseline lock");
+                                if baseline.is_none() {
+                                    // This synchronous fault hook must not miss
+                                    // the baseline when metadata is briefly busy.
+                                    let pool_meta = tokio::task::block_in_place(|| store.pool_meta.blocking_read());
+                                    let info = pool_meta.pools[0].decommission.as_ref().expect("active decommission");
+                                    *baseline = Some((entry_object.to_owned(), info.items_decommissioned, info.bytes_done));
+                                }
+                                return false;
+                            }
+                            stage == "before_target_commit"
+                                && entry_object == object
+                                && armed.load(Ordering::Acquire)
+                                && writes.fetch_add(1, Ordering::AcqRel) >= 1
+                        }
+                    }),
+                    Some(Arc::new(|_, bucket, object| {
+                        crate::error::Error::InsufficientWriteQuorum(bucket.to_owned(), object.to_owned())
+                    })),
+                );
+                let _checkpoint = crate::core::pools::DecommissionCleanupMutationGuard::install(Arc::new({
+                    let store = Arc::clone(&store);
+                    let bucket = bucket.clone();
+                    move |hook_bucket, hook_object, _| {
+                        let store = Arc::clone(&store);
+                        let due = hook_bucket == bucket && hook_object == later;
+                        Box::pin(async move {
+                            if due {
+                                let stale = time::OffsetDateTime::now_utc() - time::Duration::hours(1);
+                                let mut pool_meta = store.pool_meta.write().await;
+                                pool_meta.pools[0].last_update = stale;
+                                if let Some(info) = pool_meta.pools[0].decommission.as_mut() {
+                                    info.progress_save_last_at = Some(stale);
+                                }
+                            }
+                        })
+                    }
+                }));
+
+                start_transitioned_entry_decommission(&store, &bucket).await;
+                spawn_decommission_routine(&store, 1)
+                    .await
+                    .expect("the decommission routine task should join")
+                    .expect("a paused decommission routine returns without an error");
+                let paused = decommission_info(&*store.pool_meta.read().await);
+                assert!(!paused.complete && !paused.failed, "{paused:?}");
+                assert!(paused.capacity_blocked_reason.is_some(), "{paused:?}");
+                assert_eq!(paused.items_decommission_failed, 0, "{paused:?}");
+                let (first_entry, base_items, base_bytes) = baseline
+                    .lock()
+                    .expect("baseline lock")
+                    .clone()
+                    .expect("the test bucket's first entry started");
+                assert_eq!(first_entry, object, "the baseline precedes every entry of the test bucket");
+
+                reload_persisted_pool_meta(&store).await;
+                armed.store(false, Ordering::Release);
+                spawn_decommission_routine(&store, 1)
+                    .await
+                    .expect("the resumed routine task should join")
+                    .expect("the resumed routine should return");
+                let done = decommission_info(&*store.pool_meta.read().await);
+                assert!(done.complete && !done.failed && done.capacity_blocked_reason.is_none(), "{done:?}");
+                assert_eq!(
+                    (done.items_decommissioned, done.bytes_done),
+                    (base_items + 5, base_bytes + 3 * 2048 + 16),
+                    "the entry's four versions and the later entry, each counted once: {done:?}"
+                );
+                assert_eq!(done.items_decommission_failed, 0, "{done:?}");
+                assert!(exact_versions(&store, 0, &bucket, object).await.is_empty(), "the source is cleaned up");
+                assert_every_version_readable(&store, &bucket, object, &written).await;
+            })
+            .await;
+        });
+    }
+
+    /// Leave `object`'s target metadata unusable on two of the target set's
+    /// four drives, below the read and write quorum of EC 3+1: a directory
+    /// where `xl.meta` belongs fails every read and write there.
+    async fn break_target_metadata_on_two_drives(
+        temp_dirs: &[tempfile::TempDir],
+        store: &Arc<crate::store::ECStore>,
+        target: usize,
+        bucket: &str,
+        object: &str,
+    ) {
+        let set_index = store.pools[target].get_disks_by_key(object).set_index;
+        for disk in 0..2 {
+            let meta = temp_dirs[target * 8 + set_index * 4 + disk]
+                .path()
+                .join(bucket)
+                .join(object)
+                .join("xl.meta");
+            tokio::fs::remove_file(&meta)
+                .await
+                .unwrap_or_else(|err| panic!("remove {}: {err}", meta.display()));
+            tokio::fs::create_dir(&meta)
+                .await
+                .unwrap_or_else(|err| panic!("block {}: {err}", meta.display()));
+        }
+    }
+
+    /// A target that holds the entry's metadata on fewer drives than its
+    /// quorum is unreadable. The intact source keeps serving every version,
+    /// and a pass that meets the sub-quorum target defers the entry: it pauses
+    /// without a failed item and counts nothing of the entry.
+    #[test]
+    #[serial_test::serial]
+    fn decommission_half_written_target_never_shadows_the_source() {
+        run_large_stack_async_test("decommission-half-written-target", || async {
+            let (temp_dirs, store, _other_store) =
+                crate::services::rebalance::test_three_pool_stores_with_four_disk_sets_with_isolated_node_contexts(None).await;
+            let bucket = test_bucket("half-written-target");
+            let object = "versioned/03.bin";
+            let written = seed_versioned_entry_with_marker(&store, &bucket, object).await;
+            let armed = Arc::new(AtomicBool::new(true));
+            let _fault = fail_target_writes_after_the_first(&bucket, object, Arc::clone(&armed));
+
+            start_transitioned_entry_decommission(&store, &bucket).await;
+            spawn_decommission_routine(&store, 1)
+                .await
+                .expect("the decommission routine task should join")
+                .expect("a paused decommission routine returns without an error");
+            let mut target = None;
+            for pool in [1, 2] {
+                if !exact_versions(&store, pool, &bucket, object).await.is_empty() {
+                    target = Some(pool);
+                }
+            }
+            let target = target.expect("the marker and the newest version reached a target");
+            break_target_metadata_on_two_drives(&temp_dirs, &store, target, &bucket, object).await;
+            let moved = store.pools[target]
+                .get_object_info(
+                    &bucket,
+                    object,
+                    &ObjectOptions {
+                        versioned: true,
+                        version_id: written[2].version_id.map(|id| id.to_string()),
+                        ..Default::default()
+                    },
+                )
+                .await;
+            assert!(moved.is_err(), "the target copy must be unreadable: {moved:?}");
+            assert_every_version_readable(&store, &bucket, object, &written).await;
+
+            // No injected fault any more: only the sub-quorum target remains.
+            armed.store(false, Ordering::Release);
+            let before = decommission_info(&*store.pool_meta.read().await);
+            spawn_decommission_routine(&store, 1)
+                .await
+                .expect("the resumed routine task should join")
+                .expect("a paused decommission routine returns without an error");
+            let info = decommission_info(&*store.pool_meta.read().await);
+            assert!(!info.complete && !info.failed, "the entry is deferred, not settled: {info:?}");
+            let reason = info.capacity_blocked_reason.as_deref().unwrap_or_default();
+            assert!(reason.contains("lost drive quorum"), "{reason}");
+            assert_eq!(info.items_decommission_failed, 0, "{info:?}");
+            assert_eq!(
+                (info.items_decommissioned, info.bytes_done),
+                (before.items_decommissioned, before.bytes_done),
+                "a deferred entry counts nothing: {info:?}"
+            );
+            assert_eq!(exact_versions(&store, 0, &bucket, object).await.len(), 4, "the source is retained");
+            assert_every_version_readable(&store, &bucket, object, &written).await;
+        });
+    }
+
+    /// A bucket holding a permanently retained entry and an entry that keeps
+    /// losing target quorum pauses on the same cause every pass. The retained
+    /// entry is not counted again, so the watchdog's relaunch backs off; a
+    /// first pause is relaunched by the watchdog path and a repeated one is
+    /// held (rustfs/backlog#2873, defect F follow-up).
+    #[test]
+    #[serial_test::serial]
+    fn decommission_relaunch_backs_off_while_a_retained_entry_is_relisted() {
+        assert_retained_entry_relaunch_backoff(false);
+    }
+
+    #[test]
+    #[serial_test::serial]
+    fn decommission_full_count_ledger_preserves_failure_and_relaunch_backoff() {
+        assert_retained_entry_relaunch_backoff(true);
+    }
+
+    fn assert_retained_entry_relaunch_backoff(full_ledger: bool) {
+        run_large_stack_async_test("decommission-retained-backoff", move || async move {
+            let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
+            let bucket = test_bucket("retained-backoff");
+            let object = "versioned/03.bin";
+            let retained = "a/retained.bin";
+            seed_versioned_entry_with_marker(&store, &bucket, object).await;
+            store.pools[0]
+                .put_object(
+                    &bucket,
+                    retained,
+                    &mut PutObjReader::from_vec(vec![0x52; 64]),
+                    &ObjectOptions {
+                        versioned: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("write the retained entry");
+            let _fault = DecommissionTestFaultGuard::install_with_error(
+                Arc::new({
+                    let bucket = bucket.clone();
+                    move |stage, entry_bucket, entry_object, _, _| {
+                        entry_bucket == bucket
+                            && ((stage == "object_read" && entry_object == retained)
+                                || (stage == "before_target_commit" && entry_object == object))
+                    }
+                }),
+                Some(Arc::new(|stage, bucket, object| {
+                    if stage == "object_read" {
+                        crate::error::Error::other("injected permanent source read failure")
+                    } else {
+                        crate::error::Error::InsufficientWriteQuorum(bucket.to_owned(), object.to_owned())
+                    }
+                })),
+            );
+
+            start_transitioned_entry_decommission(&store, &bucket).await;
+            if full_ledger {
+                let generation = decommission_info(&*store.pool_meta.read().await)
+                    .start_time
+                    .expect("generation");
+                store.ctx.with_decommission_count_ledger(0, generation, |ledger| {
+                    for index in 0..2048 {
+                        ledger.admit("retained-ledger-fixture", &format!("key-{index}"), (None, false), false);
+                    }
+                    assert!(!ledger.admit("retained-ledger-fixture", "overflow", (None, false), false));
+                });
+            }
+            let pause_round = |round: u32| {
+                let store = Arc::clone(&store);
+                async move {
+                    let info = decommission_info(&*store.pool_meta.read().await);
+                    assert!(!info.complete && !info.failed, "round {round} pauses: {info:?}");
+                    assert!(info.capacity_blocked_reason.is_some(), "round {round}: {info:?}");
+                    assert_eq!(
+                        info.items_decommission_failed, 1,
+                        "the retained version is counted once (round {round}): {info:?}"
+                    );
+                    let record = store.ctx.decommission_pause(0).expect("the pause is recorded");
+                    assert_eq!(record.repeats, round, "round {round}: {record:?}");
+                    info
+                }
+            };
+
+            spawn_decommission_routine(&store, 1)
+                .await
+                .expect("the decommission routine task should join")
+                .expect("a paused decommission routine returns without an error");
+            let first = pause_round(1).await;
+            // The test helper keeps its worker slot; a finished routine frees it.
+            store.decommission_cancelers.write().await[0] = None;
+
+            // A first pause holds nothing: the watchdog path relaunches it.
+            store
+                .spawn_missing_local_decommission_routines()
+                .await
+                .expect("the watchdog relaunches a first pause");
+            assert!(
+                store.has_active_local_decommission_worker().await
+                    || store.ctx.decommission_pause(0).is_some_and(|record| record.repeats == 2),
+                "the first pause must be relaunched"
+            );
+            wait_for_relaunched_pause(&store, 2).await;
+            let second = pause_round(2).await;
+            assert_eq!(second.items_decommissioned, first.items_decommissioned, "{second:?}");
+
+            // A repeated pause is held: the watchdog path starts no worker.
+            store
+                .spawn_missing_local_decommission_routines()
+                .await
+                .expect("a held relaunch is not an error");
+            assert!(!store.has_active_local_decommission_worker().await, "a repeated pause must be held");
+            assert_eq!(store.ctx.decommission_pause(0).map(|record| record.repeats), Some(2));
+
+            spawn_decommission_routine(&store, 1)
+                .await
+                .expect("the third routine task should join")
+                .expect("a paused decommission routine returns without an error");
+            let third = pause_round(3).await;
+            assert_eq!(third.items_decommissioned, first.items_decommissioned, "{third:?}");
+            assert_eq!(exact_versions(&store, 0, &bucket, retained).await.len(), 1, "the retained source stays");
+        });
+    }
+
     /// Only a target write is deferred. A source version that cannot be read
     /// keeps its per-object failure, so an unreadable source ends the
     /// decommission as failed instead of pausing and resuming forever.

crates/ecstore/src/core/sets.rs

@@ -99,6 +99,33 @@ pub struct Sets {
     ctx: Arc<InstanceContext>,
 }
 
+#[cfg(test)]
+impl Sets {
+    pub(crate) fn from_test_set(
+        set: Arc<SetDisks>,
+        endpoints: PoolEndpoints,
+        format: FormatV3,
+        pool_idx: usize,
+        parity_count: usize,
+        ctx: Arc<InstanceContext>,
+    ) -> Arc<Self> {
+        Arc::new(Self {
+            id: format.id,
+            disk_set: vec![set],
+            pool_idx,
+            set_count: 1,
+            set_drive_count: endpoints.drives_per_set,
+            endpoints,
+            format,
+            parity_count,
+            default_parity_count: parity_count,
+            distribution_algo: DistributionAlgoVersion::V1,
+            exit_signal: None,
+            ctx,
+        })
+    }
+}
+
 impl Drop for Sets {
     fn drop(&mut self) {
         if let Some(exit_signal) = self.exit_signal.take() {
@@ -800,6 +827,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
             mod_time: dst_opts.mod_time,
             http_preconditions: dst_opts.http_preconditions.clone(),
             quota_admission: dst_opts.quota_admission,
+            expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
+            bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
+            bucket_incarnation_fence: dst_opts.bucket_incarnation_fence.clone(),
+            object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
             ..Default::default()
         };
 

crates/ecstore/src/data_movement/mod.rs

@@ -31,7 +31,7 @@ use crate::storage_api_contracts::{
     namespace::NamespaceLocking as _,
     object::{HTTPPreconditions, ObjectOperations as _},
 };
-use crate::store::{DecommissionFixedReadAnchor, ECStore, SourceCleanupMutationFence};
+use crate::store::{BucketIncarnationFenceGuard, DecommissionFixedReadAnchor, ECStore, SourceCleanupMutationFence};
 use bytes::Bytes;
 use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo};
 use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
@@ -1278,6 +1278,7 @@ pub(crate) enum SourceCleanupError {
 #[derive(Clone, Default)]
 pub(crate) struct SourceCleanupBucketFence<'a> {
     pub(crate) expected_incarnation_id: Option<uuid::Uuid>,
+    pub(crate) lifecycle_owner: Option<&'a Arc<BucketIncarnationFenceGuard>>,
     pub(crate) lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
     pub(crate) namespace_lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
     pub(crate) object_mutation_fence: Option<&'a SourceCleanupMutationFence>,
@@ -1486,9 +1487,10 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
         )
     };
 
-    if bucket_fence
-        .lifecycle_guard
-        .is_some_and(rustfs_lock::NamespaceLockGuard::is_lock_lost)
+    if bucket_fence.lifecycle_owner.is_some_and(|owner| owner.is_lock_lost())
+        || bucket_fence
+            .lifecycle_guard
+            .is_some_and(rustfs_lock::NamespaceLockGuard::is_lock_lost)
     {
         return Err(SourceCleanupError::Storage(Error::other(format!(
             "{op_label}: bucket incarnation fence was lost before source cleanup"
@@ -1527,6 +1529,9 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
     if let Some(object_mutation_fence) = bucket_fence.object_mutation_fence {
         object_mutation_fence.add_namespace_lock_fence(&mut opts);
     }
+    if let Some(owner) = bucket_fence.lifecycle_owner {
+        owner.attach_owner(&mut opts);
+    }
     if let Some(bucket_lifecycle_guard) = bucket_fence.lifecycle_guard {
         opts.add_bucket_lifecycle_lock_guard(bucket_lifecycle_guard);
     }

crates/ecstore/src/data_usage/mod.rs

@@ -176,6 +176,9 @@ static USAGE_CACHE_UPDATING: OnceLock<CacheUpdating> = OnceLock::new();
 static LIVE_BUCKET_USAGE_CACHE: OnceLock<LiveBucketUsageCache> = OnceLock::new();
 static USAGE_MEMORY_GENERATION: AtomicU64 = AtomicU64::new(0);
 
+/// Compatibility input for snapshot-protocol quota admission and capacity
+/// reporting. Durable quota checks and statistics do not consult this floor.
+///
 /// Best-available persisted usage for `bucket` when no authoritative source
 /// exists yet (issue #5716): after an upgrade from a pre-v2 release the only
 /// persisted usage data is the legacy `.usage.json`, which is demoted to
@@ -213,7 +216,7 @@ pub async fn lookup_degraded_bucket_usage_baseline(store: Arc<ECStore>, bucket:
     // Stale or empty cache: refresh through the TTL-bounded loader. A failed
     // refresh carries the previous baseline forward, so quota admission keeps
     // its last grounded values through a backend read outage.
-    let _ = load_data_usage_from_backend_cached(store).await;
+    let _ = project_data_usage_from_backend_cached(store, |_| ()).await;
     let cache = data_usage_snapshot_cache().read().await;
     cache
         .as_ref()
@@ -228,36 +231,40 @@ struct CachedDataUsageSnapshot {
     loaded_at: tokio::time::Instant,
     /// Pre-discard per-bucket sizes from the same load, retained even when the
     /// snapshot is incomplete and its bucket data is discarded. Consumed only
-    /// by [`lookup_degraded_bucket_usage_baseline`] for quota admission.
+    /// by [`lookup_degraded_bucket_usage_baseline`] for snapshot-protocol
+    /// admission and capacity reporting until their compatibility windows end.
     degraded_baseline: HashMap<String, u64>,
 }
 
 impl CachedDataUsageSnapshot {
-    fn result(&self) -> Result<DataUsageInfo, Error> {
+    fn result<T>(&self, project: impl FnOnce(&DataUsageInfo) -> T) -> Result<T, Error> {
         self.info
-            .clone()
+            .as_ref()
+            .map(project)
             .ok_or_else(|| Error::other("data usage snapshot load recently failed"))
     }
 }
 
-fn fresh_cached_data_usage_snapshot(
+fn fresh_cached_data_usage_snapshot<T>(
     cache: &Option<CachedDataUsageSnapshot>,
     now: tokio::time::Instant,
     ttl: Duration,
-) -> Option<Result<DataUsageInfo, Error>> {
+    project: impl FnOnce(&DataUsageInfo) -> T,
+) -> Option<Result<T, Error>> {
     cache
         .as_ref()
         .filter(|cached| now.duration_since(cached.loaded_at) < ttl)
-        .map(CachedDataUsageSnapshot::result)
+        .map(|cached| cached.result(project))
 }
 
-fn cache_data_usage_snapshot_result(
+fn cache_data_usage_snapshot_result<T>(
     cache: &mut Option<CachedDataUsageSnapshot>,
     result: Result<LoadedUsageBaseline, Error>,
     loaded_at: tokio::time::Instant,
     refresh_generation: u64,
     current_generation: u64,
-) -> Option<Result<DataUsageInfo, Error>> {
+    project: impl FnOnce(&DataUsageInfo) -> T,
+) -> Option<Result<T, Error>> {
     if current_generation != refresh_generation {
         return None;
     }
@@ -276,12 +283,13 @@ fn cache_data_usage_snapshot_result(
                     degraded_baseline.entry(bucket.clone()).or_insert(*size);
                 }
             }
+            let projected = project(&info);
             *cache = Some(CachedDataUsageSnapshot {
-                info: Some(info.clone()),
+                info: Some(info),
                 loaded_at,
                 degraded_baseline,
             });
-            Ok(info)
+            Ok(projected)
         }
         Err(e) => {
             // Keep the previous baseline through a failed refresh: quota
@@ -1482,12 +1490,21 @@ async fn normalize_loaded_data_usage(
 /// `save_data_usage_in_backend` invalidates the cache so a fresh scanner save
 /// is visible immediately in the saving process.
 pub async fn load_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
+    project_data_usage_from_backend_cached(store, DataUsageInfo::clone).await
+}
+
+// Project under the cache guard so scalar readers never copy the snapshot's
+// bucket maps. The owned API shares the same TTL, refresh and invalidation path.
+async fn project_data_usage_from_backend_cached<T>(
+    store: Arc<ECStore>,
+    project: impl Fn(&DataUsageInfo) -> T,
+) -> Result<T, Error> {
     let ttl = Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
 
     loop {
         {
             let cache = data_usage_snapshot_cache().read().await;
-            if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
+            if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl, &project) {
                 return result;
             }
         }
@@ -1499,7 +1516,7 @@ pub async fn load_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<
             .await;
         {
             let cache = data_usage_snapshot_cache().read().await;
-            if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
+            if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl, &project) {
                 return result;
             }
         }
@@ -1508,9 +1525,14 @@ pub async fn load_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<
         let result = load_data_usage_from_backend_with_baseline(store.clone()).await;
         let loaded_at = tokio::time::Instant::now();
         let mut cache = data_usage_snapshot_cache().write().await;
-        if let Some(result) =
-            cache_data_usage_snapshot_result(&mut cache, result, loaded_at, refresh_generation, data_usage_snapshot_generation())
-        {
+        if let Some(result) = cache_data_usage_snapshot_result(
+            &mut cache,
+            result,
+            loaded_at,
+            refresh_generation,
+            data_usage_snapshot_generation(),
+            &project,
+        ) {
             return result;
         }
         drop(cache);
@@ -1528,7 +1550,8 @@ pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> R
     loop {
         {
             let cache = admin_data_usage_snapshot_cache().read().await;
-            if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
+            if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl, DataUsageInfo::clone)
+            {
                 return result;
             }
         }
@@ -1539,7 +1562,8 @@ pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> R
             .await;
         {
             let cache = admin_data_usage_snapshot_cache().read().await;
-            if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
+            if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl, DataUsageInfo::clone)
+            {
                 return result;
             }
         }
@@ -1560,6 +1584,7 @@ pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> R
             loaded_at,
             refresh_generation,
             admin_data_usage_snapshot_generation(),
+            DataUsageInfo::clone,
         ) {
             return result;
         }
@@ -2273,8 +2298,10 @@ async fn get_persisted_bucket_usage(bucket: &str) -> Option<u64> {
 }
 
 async fn persisted_bucket_usage(store: Arc<ECStore>, bucket: &str) -> Option<u64> {
-    let info = load_data_usage_from_backend_cached(store).await.ok()?;
-    info.buckets_usage.get(bucket).map(|usage| usage.size)
+    project_data_usage_from_backend_cached(store, |info| info.buckets_usage.get(bucket).map(|usage| usage.size))
+        .await
+        .ok()
+        .flatten()
 }
 
 async fn reload_persisted_bucket_usage(store: Arc<ECStore>, bucket: &str) -> Option<u64> {
@@ -2371,6 +2398,7 @@ where
         tokio::time::Instant::now(),
         refresh_generation,
         data_usage_snapshot_generation(),
+        |_| (),
     )
     .is_some()
 }
@@ -2887,6 +2915,8 @@ pub async fn init_compression_total_memory_from_backend(store: Arc<ECStore>) {
 
 #[cfg(test)]
 mod tests {
+    mod snapshot_projection;
+
     use super::*;
     use crate::layout::endpoints::EndpointServerPools;
     use crate::runtime::instance::InstanceContext;
@@ -3330,6 +3360,7 @@ mod tests {
             pool_meta_save_gate: TokioMutex::default(),
             ctx,
             bucket_fence_registry: Arc::default(),
+            ordinary_classification: Default::default(),
         })
     }
 
@@ -4568,15 +4599,29 @@ mod tests {
             loaded_at,
             refresh_generation,
             data_usage_snapshot_generation(),
+            DataUsageInfo::clone,
         )
         .expect("an uninterrupted refresh should populate the cache");
         assert!(matches!(first, Err(Error::ErasureReadQuorum)));
 
-        let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
-            .expect("the failed load must remain cached within the TTL");
+        let cached = fresh_cached_data_usage_snapshot(
+            &cache,
+            loaded_at + Duration::from_secs(1),
+            Duration::from_secs(30),
+            DataUsageInfo::clone,
+        )
+        .expect("the failed load must remain cached within the TTL");
         assert!(cached.is_err());
 
-        assert!(fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(30), Duration::from_secs(30)).is_none());
+        assert!(
+            fresh_cached_data_usage_snapshot(
+                &cache,
+                loaded_at + Duration::from_secs(30),
+                Duration::from_secs(30),
+                DataUsageInfo::clone
+            )
+            .is_none()
+        );
     }
 
     #[test]
@@ -4597,14 +4642,20 @@ mod tests {
             loaded_at,
             refresh_generation,
             data_usage_snapshot_generation(),
+            DataUsageInfo::clone,
         )
         .expect("an uninterrupted refresh should populate the cache")
         .expect("successful load must be returned");
         assert_snapshot_bucket(&first, "bucket");
 
-        let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
-            .expect("successful load must remain cached within the TTL")
-            .expect("cached success must remain successful");
+        let cached = fresh_cached_data_usage_snapshot(
+            &cache,
+            loaded_at + Duration::from_secs(1),
+            Duration::from_secs(30),
+            DataUsageInfo::clone,
+        )
+        .expect("successful load must remain cached within the TTL")
+        .expect("cached success must remain successful");
         assert_snapshot_bucket(&cached, "bucket");
     }
 
@@ -4629,6 +4680,7 @@ mod tests {
             loaded_at,
             refresh_generation,
             data_usage_snapshot_generation(),
+            DataUsageInfo::clone,
         )
         .expect("an uninterrupted refresh should populate the cache")
         .expect("successful load must be returned");
@@ -4662,6 +4714,7 @@ mod tests {
             loaded_at,
             refresh_generation,
             data_usage_snapshot_generation(),
+            DataUsageInfo::clone,
         );
 
         assert!(stale_result.is_none());
@@ -6357,6 +6410,43 @@ mod tests {
         assert_eq!(get_bucket_usage_memory("bucket-a").await, None);
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn complete_scanner_cycle_with_carried_bucket_preserves_legacy_quota_floor() {
+        clear_usage_memory_cache_for_test().await;
+        let bucket = "partial-carried-quota";
+        let mut baseline = data_usage_info_for_test(bucket, 2, 100, SystemTime::UNIX_EPOCH + Duration::from_secs(100));
+        baseline.scanner_epoch = Some(7);
+        baseline.scanner_cycle = Some(10);
+        baseline.usage_snapshot_converged = Some(true);
+        replace_bucket_usage_memory_from_info(&baseline).await;
+        record_bucket_object_write_memory(bucket, None, 25).await;
+        record_bucket_object_delete_memory(bucket, 20, true).await;
+        assert_eq!(get_bucket_usage_memory(bucket).await, Some(125));
+        let mutation_time = memory_cache().read().await[bucket].usage_updated_at;
+
+        let mut carried = baseline;
+        carried.last_update = Some(mutation_time + Duration::from_secs(1));
+        carried.scanner_cycle = Some(11);
+        carried.usage_snapshot_converged = Some(false);
+        carried.usage_snapshot_pending_buckets = vec![bucket.to_string()];
+        assert!(carried.usage_snapshot_complete);
+        replace_bucket_usage_memory_from_info(&carried).await;
+        assert_eq!(
+            get_bucket_usage_memory(bucket).await,
+            Some(125),
+            "Complete with a carried certified value must retain positive deltas and the delete hold"
+        );
+
+        let mut reconciled = data_usage_info_for_test(bucket, 2, 105, mutation_time + Duration::from_secs(2));
+        reconciled.scanner_epoch = Some(7);
+        reconciled.scanner_cycle = Some(12);
+        reconciled.usage_snapshot_converged = Some(true);
+        replace_bucket_usage_memory_from_info(&reconciled).await;
+        assert_eq!(get_bucket_usage_memory(bucket).await, Some(105));
+        clear_usage_memory_cache_for_test().await;
+    }
+
     #[tokio::test]
     #[serial]
     async fn stale_quota_uses_complete_baseline_plus_positive_deltas() {

crates/ecstore/src/data_usage/tests/snapshot_projection.rs

@@ -0,0 +1,191 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::*;
+
+#[test]
+fn scalar_projection_borrows_snapshot_and_preserves_unknown_and_zero() {
+    let loaded_at = tokio::time::Instant::now();
+    let mut info = data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH);
+    info.buckets_usage.insert("empty".to_string(), BucketUsageInfo::default());
+    let cache = Some(CachedDataUsageSnapshot {
+        info: Some(info),
+        loaded_at,
+        degraded_baseline: HashMap::new(),
+    });
+    let stored = cache
+        .as_ref()
+        .expect("cached snapshot")
+        .info
+        .as_ref()
+        .expect("known snapshot");
+    let ttl = Duration::from_secs(30);
+
+    for (bucket, expected) in [("known", Some(42)), ("empty", Some(0)), ("missing", None)] {
+        let value = fresh_cached_data_usage_snapshot(&cache, loaded_at, ttl, |info| {
+            assert!(std::ptr::eq(info, stored), "projection must borrow the cached snapshot");
+            info.buckets_usage.get(bucket).map(|usage| usage.size)
+        })
+        .expect("fresh snapshot")
+        .expect("successful snapshot");
+        let owned = fresh_cached_data_usage_snapshot(&cache, loaded_at, ttl, DataUsageInfo::clone)
+            .expect("fresh snapshot")
+            .expect("successful snapshot");
+        assert_eq!(value, expected);
+        assert_eq!(value, owned.buckets_usage.get(bucket).map(|usage| usage.size));
+    }
+}
+
+#[test]
+fn projection_is_skipped_for_missing_failed_expired_or_invalidated_snapshots() {
+    let loaded_at = tokio::time::Instant::now();
+    let ttl = Duration::from_secs(30);
+    let never_project = |_: &DataUsageInfo| -> u64 { panic!("unusable snapshots must not be projected") };
+    assert!(fresh_cached_data_usage_snapshot(&None, loaded_at, ttl, never_project).is_none());
+
+    let mut cache = None;
+    let failure = cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), loaded_at, 1, 1, never_project)
+        .expect("matching generation");
+    assert!(matches!(failure, Err(Error::ErasureReadQuorum)));
+    assert!(
+        fresh_cached_data_usage_snapshot(&cache, loaded_at, ttl, never_project)
+            .expect("negative cache remains fresh")
+            .is_err()
+    );
+
+    cache = Some(CachedDataUsageSnapshot {
+        info: Some(data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH)),
+        loaded_at,
+        degraded_baseline: HashMap::new(),
+    });
+    assert!(fresh_cached_data_usage_snapshot(&cache, loaded_at + ttl, ttl, never_project).is_none());
+    cache = None;
+    let rejected = cache_data_usage_snapshot_result(
+        &mut cache,
+        Ok(LoadedUsageBaseline {
+            info: data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH),
+            degraded_baseline: HashMap::new(),
+            observed_unavailable: false,
+        }),
+        loaded_at,
+        1,
+        2,
+        never_project,
+    );
+    assert!(rejected.is_none());
+    assert!(cache.is_none(), "rejected generation must not republish stale data");
+}
+
+#[test]
+fn projected_publication_moves_bucket_maps_into_cache() {
+    let info = data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH);
+    let original_key = info.buckets_usage.get_key_value("known").expect("source bucket").0.as_ptr();
+    let mut cache = None;
+    let usage = cache_data_usage_snapshot_result(
+        &mut cache,
+        Ok(LoadedUsageBaseline {
+            info,
+            degraded_baseline: HashMap::new(),
+            observed_unavailable: false,
+        }),
+        tokio::time::Instant::now(),
+        1,
+        1,
+        |info| {
+            assert_eq!(
+                info.buckets_usage
+                    .get_key_value("known")
+                    .expect("projected bucket")
+                    .0
+                    .as_ptr(),
+                original_key
+            );
+            info.buckets_usage.get("known").map(|usage| usage.size)
+        },
+    )
+    .expect("matching generation")
+    .expect("successful publication");
+    assert_eq!(usage, Some(42));
+    let cached = cache
+        .as_ref()
+        .expect("published cache")
+        .info
+        .as_ref()
+        .expect("published snapshot");
+    assert_eq!(
+        cached.buckets_usage.get_key_value("known").expect("cached bucket").0.as_ptr(),
+        original_key
+    );
+}
+
+#[tokio::test]
+#[serial]
+async fn persisted_scalar_projection_shares_cache_refresh_and_invalidation() {
+    use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
+    use crate::config::com::save_config;
+
+    let (_dirs, store) = isolated_store_over_temp_disks().await;
+    let mut info = data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH);
+    info.buckets_usage.insert("empty".to_string(), BucketUsageInfo::default());
+    info.bucket_sizes.insert("empty".to_string(), 0);
+    info.buckets_count = 2;
+    save_config(
+        store.clone(),
+        DATA_USAGE_OBJ_NAME_PATH.as_str(),
+        serde_json::to_vec(&info).expect("encode snapshot"),
+    )
+    .await
+    .expect("persist snapshot");
+    invalidate_data_usage_snapshot_cache().await;
+
+    for (bucket, expected) in [("known", Some(42)), ("empty", Some(0)), ("missing", None)] {
+        assert_eq!(persisted_bucket_usage(store.clone(), bucket).await, expected);
+        let owned = load_data_usage_from_backend_cached(store.clone())
+            .await
+            .expect("owned cached snapshot");
+        assert_eq!(expected, owned.buckets_usage.get(bucket).map(|usage| usage.size));
+    }
+    info.buckets_usage.get_mut("known").expect("known bucket").size = 84;
+    info.bucket_sizes.insert("known".to_string(), 84);
+    save_config(
+        store.clone(),
+        DATA_USAGE_OBJ_NAME_PATH.as_str(),
+        serde_json::to_vec(&info).expect("encode new snapshot"),
+    )
+    .await
+    .expect("persist new snapshot without invalidating cache");
+    assert_eq!(
+        persisted_bucket_usage(store.clone(), "known").await,
+        Some(42),
+        "warm reads reuse the TTL cache"
+    );
+    invalidate_data_usage_snapshot_cache().await;
+    assert_eq!(
+        persisted_bucket_usage(store.clone(), "known").await,
+        Some(84),
+        "invalidation exposes the new snapshot"
+    );
+    invalidate_data_usage_snapshot_cache().await;
+    if let Some(cancel) = store.ctx.background_cancel_token() {
+        cancel.cancel();
+    }
+    for pool in &store.pools {
+        for set in &pool.disk_set {
+            let disks = set.disks.read().await.clone();
+            for disk in disks.iter().flatten() {
+                disk.close().await.expect("stop fixture disk monitoring");
+            }
+        }
+    }
+}

crates/ecstore/src/diagnostics/admin_server_info.rs

@@ -393,6 +393,10 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
     }
 }
 
+fn is_online_drive_state(state: &str) -> bool {
+    state == DriveState::Ok.to_string() || state == DriveState::Unformatted.to_string()
+}
+
 /// Classify every drive into online / offline / unknown buckets.
 ///
 /// `unknown` holds drives synthesized for a member whose properties RPC could
@@ -412,6 +416,7 @@ fn get_online_offline_disks_stats(disks_info: &[Disk]) -> (BackendDisks, Backend
         unknown_disks.entry(ep.clone()).or_insert(0);
     }
 
+    let mut online_root_disk_count = 0;
     for disk in disks_info {
         let ep = &disk.endpoint;
         let state = &disk.state;
@@ -419,30 +424,28 @@ fn get_online_offline_disks_stats(disks_info: &[Disk]) -> (BackendDisks, Backend
             *unknown_disks.get_mut(ep).expect("endpoint should be in disk map") += 1;
             continue;
         }
-        if *state != DriveState::Ok.to_string() && *state != DriveState::Unformatted.to_string() {
+        if !is_online_drive_state(state) {
             *offline_disks.get_mut(ep).expect("endpoint should be in disk map") += 1;
             continue;
         }
         *online_disks.get_mut(ep).expect("endpoint should be in disk map") += 1;
-    }
-
-    let mut root_disk_count = 0;
-    for di in disks_info {
-        if di.root_disk {
-            root_disk_count += 1;
+        if disk.root_disk {
+            online_root_disk_count += 1;
         }
     }
 
     // When every non-offline, non-unknown drive is a root mount, leave the
     // online tally as-is instead of demoting all of them (matches the prior
     // behavior; the unknown bucket is simply carried through untouched).
-    if disks_info.len() == (root_disk_count + offline_disks.values().sum::<usize>() + unknown_disks.values().sum::<usize>()) {
+    if disks_info.len()
+        == (online_root_disk_count + offline_disks.values().sum::<usize>() + unknown_disks.values().sum::<usize>())
+    {
         return (BackendDisks(online_disks), BackendDisks(offline_disks), BackendDisks(unknown_disks));
     }
 
     for disk in disks_info {
         let ep = &disk.endpoint;
-        if disk.root_disk {
+        if disk.root_disk && is_online_drive_state(&disk.state) {
             *offline_disks.get_mut(ep).expect("endpoint should be in disk map") += 1;
             *online_disks.get_mut(ep).expect("endpoint should be in disk map") -= 1;
         }
@@ -835,6 +838,35 @@ mod tests {
         );
     }
 
+    #[test]
+    fn disk_stats_only_demote_online_root_disks() {
+        let mut offline_root = disk_with_state("http://n3:9000/data", ITEM_OFFLINE);
+        offline_root.root_disk = true;
+        let mut unknown_root = disk_with_state("http://n4:9000/data", ITEM_UNKNOWN);
+        unknown_root.root_disk = true;
+        let mut online_root = disk_with_state("http://n5:9000/data", "ok");
+        online_root.root_disk = true;
+
+        let disks = vec![
+            disk_with_state("http://n1:9000/data", "ok"),
+            disk_with_state("http://n2:9000/data", "ok"),
+            offline_root,
+            unknown_root,
+            online_root,
+        ];
+
+        let (online, offline, unknown) = get_online_offline_disks_stats(&disks);
+
+        assert_eq!(online.sum(), 2, "only the two non-root online drives remain online");
+        assert_eq!(offline.sum(), 2, "the offline root drive and demoted online root drive are offline");
+        assert_eq!(unknown.sum(), 1, "the unknown root drive stays in unknown without underflowing online");
+        assert_eq!(
+            online.sum() + offline.sum() + unknown.sum(),
+            disks.len(),
+            "online + offline + unknown must equal the total drive count"
+        );
+    }
+
     #[test]
     fn topology_reconcile_synthesizes_missing_member_as_unknown() {
         let endpoints = topology_with_hosts(&["rustfs-1", "rustfs-2", "rustfs-3", "rustfs-4"]);

crates/ecstore/src/disk/cleanup_runtime.rs

@@ -16,6 +16,7 @@
 //! PUT admission precedes staging; receiver and GC disk work share a separate
 //! budget so coordinators waiting for peers cannot exhaust receiver capacity.
 
+use crate::disk::io_admission::{current_resource_unit, with_optional_resource_unit};
 use crate::disk::{LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_DISK};
 use std::collections::BTreeSet;
 use std::future::Future;
@@ -422,7 +423,7 @@ impl CleanupRuntime {
         // the result receiver's lifetime to discard work cancelled during
         // runtime scheduling, before its first poll can start filesystem I/O.
         let (result_tx, result_rx) = tokio::sync::oneshot::channel();
-        let task = runtime.spawn(async move {
+        let task = runtime.spawn(with_optional_resource_unit(current_resource_unit(), async move {
             if result_tx.is_closed() {
                 return;
             }
@@ -430,7 +431,7 @@ impl CleanupRuntime {
             // through I/O even if the waiter cancels and closes result_rx.
             let result = execute_disk(pending, Some(active), execution, started, future).await;
             let _ = result_tx.send(result);
-        });
+        }));
         match result_rx.await {
             Ok(result) => result,
             Err(err) => {
@@ -469,11 +470,11 @@ impl CleanupRuntime {
             gates: Some((self.disk_admission.clone(), self.disk_execution.clone())),
         };
         handle
-            .spawn(async move {
+            .spawn(with_optional_resource_unit(current_resource_unit(), async move {
                 let active = execution.acquire_owned().await.map_err(super::error::DiskError::other)?;
                 let _active = Admission::new(active, "rustfs_cleanup_gc_active");
                 scan(budget).await
-            })
+            }))
             .await
             .map_err(super::error::DiskError::other)?
     }
@@ -582,6 +583,7 @@ where
     F::Output: Send + 'static,
 {
     let started = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
+    let future = with_optional_resource_unit(current_resource_unit(), future);
     match reservation {
         Some(reservation) => reservation.handle.clone().spawn(async move {
             // This semaphore is never closed. Both queued work and overflow
@@ -620,6 +622,62 @@ mod tests {
     use std::time::Duration;
     use tokio::sync::oneshot;
 
+    #[tokio::test]
+    async fn cleanup_dispatch_retains_resource_units_without_inheriting_disk_policy() {
+        use crate::disk::io_admission::{current_disk_io_context, with_background_disk_io, with_background_resource_unit};
+        use rustfs_concurrency::adaptive::AdaptiveAdmission;
+        use std::num::NonZeroU16;
+        use tokio_util::sync::CancellationToken;
+
+        for mode in 0..4 {
+            let runtime = Arc::new(CleanupRuntime::new(config(1, 1, 1)).expect("cleanup runtime"));
+            let worker_runtime = runtime.clone();
+            let gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one unit"), 1);
+            let (started_tx, started_rx) = tokio::sync::oneshot::channel();
+            let (release_tx, release_rx) = tokio::sync::oneshot::channel();
+            let work = async move {
+                started_tx
+                    .send((current_resource_unit().is_some(), current_disk_io_context().is_none()))
+                    .expect("report worker context");
+                release_rx.await.expect("release owned worker");
+                Ok::<_, super::super::error::DiskError>(())
+            };
+            let task = tokio::spawn(with_background_disk_io(
+                CancellationToken::new(),
+                with_background_resource_unit(gate.acquire().await, async move {
+                    match mode {
+                        0 => worker_runtime
+                            .disk_operation(work, DiskAdmission::Request)
+                            .await
+                            .expect("disk operation"),
+                        1 => worker_runtime.gc_operation(move |_| work).await.expect("GC operation"),
+                        2 => {
+                            let reservation = worker_runtime.reserve().expect("cleanup reservation");
+                            run(Some(reservation), async { work.await.expect("cleanup operation") }).await;
+                        }
+                        _ => spawn(None, work)
+                            .await
+                            .expect("unisolated worker")
+                            .expect("unisolated operation"),
+                    }
+                }),
+            ));
+            let (inherited, original_policy) = deadline(started_rx).await.expect("owned work starts");
+            task.abort();
+            assert!(task.await.expect_err("waiter cancelled").is_cancelled());
+            let held = gate.snapshot().active;
+            release_tx.send(()).expect("owned work survives cancellation");
+            deadline(async {
+                while gate.snapshot().active != 0 {
+                    tokio::task::yield_now().await;
+                }
+            })
+            .await;
+            assert!(inherited, "mode {mode} lost resource ownership");
+            assert!(original_policy, "mode {mode} changed the detached worker's disk policy");
+            assert_eq!(held, 1, "mode {mode} released a live worker's unit");
+        }
+    }
     fn config(pending: usize, workers: usize, blocking_threads: usize) -> Config {
         Config {
             pending,

crates/ecstore/src/disk/fs.rs

@@ -159,10 +159,6 @@ pub fn lstat_std(path: impl AsRef<Path>) -> io::Result<Metadata> {
     std::fs::symlink_metadata(path)
 }
 
-pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
-    fs::create_dir_all(path.as_ref()).await
-}
-
 fn is_dir_error(e: &io::Error) -> bool {
     e.raw_os_error() == Some(libc::EISDIR)
         || e.kind() == io::ErrorKind::IsADirectory
@@ -210,10 +206,6 @@ pub fn remove_all_std(path: impl AsRef<Path>) -> io::Result<()> {
     }
 }
 
-pub async fn mkdir(path: impl AsRef<Path>) -> io::Result<()> {
-    fs::create_dir(path.as_ref()).await
-}
-
 pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
     fs::rename(from, to).await
 }
@@ -472,17 +464,6 @@ mod tests {
         assert!(metadata.file_type().is_symlink());
     }
 
-    #[tokio::test]
-    async fn test_make_dir_all() {
-        let temp_dir = TempDir::new().unwrap();
-        let nested_path = temp_dir.path().join("level1").join("level2").join("level3");
-
-        // Should create nested directories
-        assert!(make_dir_all(&nested_path).await.is_ok());
-        assert!(nested_path.exists());
-        assert!(nested_path.is_dir());
-    }
-
     #[tokio::test]
     async fn test_remove_file() {
         let temp_dir = TempDir::new().unwrap();
@@ -555,17 +536,6 @@ mod tests {
         assert!(!dir_path.exists());
     }
 
-    #[tokio::test]
-    async fn test_mkdir() {
-        let temp_dir = TempDir::new().unwrap();
-        let dir_path = temp_dir.path().join("test_mkdir");
-
-        // Create directory
-        assert!(mkdir(&dir_path).await.is_ok());
-        assert!(dir_path.exists());
-        assert!(dir_path.is_dir());
-    }
-
     #[tokio::test]
     async fn test_rename() {
         let temp_dir = TempDir::new().unwrap();

crates/ecstore/src/disk/io_admission.rs

@@ -24,6 +24,7 @@ use std::task::{Context, Poll};
 use std::time::Duration;
 
 use metrics::{counter, gauge, histogram};
+use rustfs_concurrency::adaptive::AdaptivePermit;
 use rustfs_rio::ChunkReader;
 use tokio::io::{AsyncRead, ReadBuf};
 use tokio::sync::Notify;
@@ -35,6 +36,7 @@ const BACKGROUND_MAX_WAIT: Duration = Duration::from_millis(100);
 
 tokio::task_local! {
     static DISK_IO_CONTEXT: DiskIoContext;
+    static RESOURCE_IO_UNIT: AdaptivePermit;
 }
 
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
@@ -49,6 +51,7 @@ pub(crate) struct DiskIoContext {
     pub(crate) cancellation: CancellationToken,
     pub(crate) admission: Option<Arc<DiskIoAdmission>>,
     pub(crate) permit: Option<DiskIoPermit>,
+    pub(crate) resource_unit: Option<AdaptivePermit>,
 }
 
 /// Mark disk I/O issued by `future` as background work for local-disk admission.
@@ -60,12 +63,43 @@ pub async fn with_background_disk_io<F: Future>(cancellation: CancellationToken,
             cancellation,
             admission: None,
             permit: None,
+            resource_unit: current_resource_unit(),
         },
         future,
     )
     .await
 }
 
+/// Retain an already admitted resource unit without changing disk classification,
+/// cancellation or admission. Acquire it before namespace, cache or commit locks.
+/// Existing background scopes and disk leases propagate it into native I/O.
+/// Use this for physical I/O ownership, not node-wide scheduling capacity that
+/// must return with a cancelled waiter so healthy disks can progress independently
+/// of a stalled native operation. Escaping buffers need their own memory leases.
+#[doc(hidden)]
+pub async fn with_background_resource_unit<F: Future>(unit: AdaptivePermit, future: F) -> F::Output {
+    RESOURCE_IO_UNIT.scope(unit, future).await
+}
+
+pub(crate) fn current_resource_unit() -> Option<AdaptivePermit> {
+    RESOURCE_IO_UNIT.try_with(Clone::clone).ok().or_else(|| {
+        DISK_IO_CONTEXT
+            .try_with(|context| context.resource_unit.clone())
+            .ok()
+            .flatten()
+    })
+}
+
+/// Transfer only resource ownership into an existing spawned task. Disk
+/// classification/permits are not inherited: changing those can introduce a
+/// wait behind a disk permit still held by the parent.
+pub(crate) async fn with_optional_resource_unit<F: Future>(unit: Option<AdaptivePermit>, future: F) -> F::Output {
+    match unit {
+        Some(unit) => with_background_resource_unit(unit, future).await,
+        None => future.await,
+    }
+}
+
 pub(crate) async fn with_disk_io_context<F: Future>(context: DiskIoContext, future: F) -> F::Output {
     DISK_IO_CONTEXT.scope(context, future).await
 }
@@ -79,7 +113,11 @@ pub(crate) async fn with_disk_io_permit<F: Future>(permit: Option<DiskIoPermit>,
         cancellation: CancellationToken::new(),
         admission: None,
         permit: None,
+        resource_unit: None,
     });
+    if context.resource_unit.is_none() {
+        context.resource_unit = permit.lease.resource_unit.clone();
+    }
     context.admission = Some(permit.admission());
     with_disk_io_context(
         DiskIoContext {
@@ -91,15 +129,32 @@ pub(crate) async fn with_disk_io_permit<F: Future>(permit: Option<DiskIoPermit>,
     .await
 }
 
-pub(crate) async fn with_optional_disk_io_context<F: Future>(context: Option<DiskIoContext>, future: F) -> F::Output {
-    match context {
-        Some(context) => with_disk_io_context(context, future).await,
-        None => future.await,
+pub(crate) fn with_optional_disk_io_context<F: Future>(
+    context: Option<DiskIoContext>,
+    future: F,
+) -> impl Future<Output = F::Output> {
+    // Capture at future construction, before JoinSet/spawn moves polling to a
+    // different task. A resource-only scope has no DiskIoContext to carry it.
+    let unit = current_resource_unit();
+    async move {
+        let future = with_optional_resource_unit(unit, future);
+        match context {
+            Some(context) => with_disk_io_context(context, future).await,
+            None => future.await,
+        }
     }
 }
 
 pub(crate) fn current_disk_io_context() -> Option<DiskIoContext> {
-    DISK_IO_CONTEXT.try_with(Clone::clone).ok()
+    DISK_IO_CONTEXT
+        .try_with(|context| {
+            let mut context = context.clone();
+            if let Ok(unit) = RESOURCE_IO_UNIT.try_with(Clone::clone) {
+                context.resource_unit = Some(unit);
+            }
+            context
+        })
+        .ok()
 }
 
 /// Acquire one low-level permit only when the current walker explicitly owns a gate.
@@ -175,6 +230,10 @@ impl DiskIoAdmission {
     }
 
     pub(crate) fn foreground(self: &Arc<Self>) -> DiskIoPermit {
+        self.foreground_with_resource(current_resource_unit())
+    }
+
+    fn foreground_with_resource(self: &Arc<Self>, resource_unit: Option<AdaptivePermit>) -> DiskIoPermit {
         self.foreground_active.fetch_add(1, Ordering::AcqRel);
         if self.background_waiters.load(Ordering::Acquire) > 0 {
             self.record_state();
@@ -183,11 +242,20 @@ impl DiskIoAdmission {
             lease: Arc::new(DiskIoPermitLease {
                 admission: self.clone(),
                 class: DiskIoClass::Foreground,
+                resource_unit,
             }),
         }
     }
 
     pub(crate) async fn background(self: &Arc<Self>, cancellation: &CancellationToken) -> std::io::Result<DiskIoPermit> {
+        self.background_with_resource(cancellation, current_resource_unit()).await
+    }
+
+    async fn background_with_resource(
+        self: &Arc<Self>,
+        cancellation: &CancellationToken,
+        resource_unit: Option<AdaptivePermit>,
+    ) -> std::io::Result<DiskIoPermit> {
         let started_at = Instant::now();
         let ticket = {
             let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
@@ -248,6 +316,7 @@ impl DiskIoAdmission {
                     lease: Arc::new(DiskIoPermitLease {
                         admission: self.clone(),
                         class: DiskIoClass::Background,
+                        resource_unit,
                     }),
                 });
             }
@@ -360,6 +429,7 @@ impl DiskIoPermit {
 struct DiskIoPermitLease {
     admission: Arc<DiskIoAdmission>,
     class: DiskIoClass,
+    resource_unit: Option<AdaptivePermit>,
 }
 
 pub(crate) fn spawn_blocking_with_disk_io_permit<T, F>(
@@ -370,8 +440,12 @@ where
     T: Send + 'static,
     F: FnOnce() -> T + Send + 'static,
 {
+    // A supplied disk permit can predate this resource scope or carry another
+    // owner's unit. Preserve the current unit independently until native I/O ends.
+    let resource_unit = current_resource_unit();
     tokio::task::spawn_blocking(move || {
         let _io_permit = io_permit;
+        let _resource_unit = resource_unit;
         operation()
     })
 }
@@ -436,7 +510,7 @@ impl<R> AdmissionReader<R> {
         }
 
         if self.context.class == DiskIoClass::Foreground {
-            self.permit = Some(self.admission.foreground());
+            self.permit = Some(self.admission.foreground_with_resource(self.context.resource_unit.clone()));
             return Poll::Ready(Ok(()));
         }
 
@@ -444,7 +518,10 @@ impl<R> AdmissionReader<R> {
         if wait.is_none() {
             let admission = self.admission.clone();
             let cancellation = self.context.cancellation.clone();
-            *wait = Some(Box::pin(async move { admission.background(&cancellation).await }));
+            let resource_unit = self.context.resource_unit.clone();
+            *wait = Some(Box::pin(
+                async move { admission.background_with_resource(&cancellation, resource_unit).await },
+            ));
         }
         let Some(wait_future) = wait.as_mut() else {
             return Poll::Ready(Err(io::Error::other("background disk I/O admission waiter is missing")));
@@ -514,8 +591,166 @@ impl ChunkReader for BoxedChunkReader {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use rustfs_concurrency::adaptive::AdaptiveAdmission;
+    use std::num::NonZeroU16;
     use tokio::io::{AsyncReadExt, AsyncWriteExt};
 
+    #[tokio::test]
+    async fn resource_unit_outlives_cancelled_scope_until_native_io_finishes() {
+        for permit_mode in 0..4 {
+            let gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one unit"), 1);
+            let previous_gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("previous owner"), 1);
+            let unit = gate.acquire().await;
+            let disk = DiskIoAdmission::new("resource-unit-native");
+            let existing_permit = match permit_mode {
+                2 => Some(disk.foreground()),
+                3 => Some(with_background_resource_unit(previous_gate.acquire().await, async { disk.foreground() }).await),
+                _ => None,
+            };
+            let (started_tx, started_rx) = tokio::sync::oneshot::channel();
+            let (release_tx, release_rx) = tokio::sync::oneshot::channel();
+            let task = tokio::spawn(with_background_resource_unit(unit, async move {
+                with_background_disk_io(CancellationToken::new(), async {
+                    let permit = if existing_permit.is_some() {
+                        existing_permit
+                    } else if permit_mode == 1 {
+                        Some(disk.background(&CancellationToken::new()).await.expect("disk permit"))
+                    } else {
+                        None
+                    };
+                    spawn_blocking_with_disk_io_permit(permit, move || {
+                        started_tx.send(()).expect("native I/O started");
+                        release_rx.blocking_recv().expect("native I/O release");
+                    })
+                    .await
+                    .expect("native I/O completed");
+                })
+                .await;
+            }));
+            started_rx.await.expect("native worker owns unit");
+            task.abort();
+            assert!(task.await.expect_err("scope was cancelled").is_cancelled());
+            let held = gate.snapshot().active;
+            let previous_held = previous_gate.snapshot().active;
+            let rejected = gate.try_acquire().is_none();
+            gate.set_limit(0);
+            release_tx.send(()).expect("release native I/O");
+            tokio::time::timeout(Duration::from_secs(5), async {
+                while gate.snapshot().active != 0 || previous_gate.snapshot().active != 0 {
+                    tokio::task::yield_now().await;
+                }
+            })
+            .await
+            .expect("native completion releases the unit");
+            gate.set_limit(1);
+            assert!(gate.try_acquire().is_some());
+            assert_eq!(held, 1, "mode {permit_mode}: native I/O still owns the current unit");
+            assert!(rejected, "mode {permit_mode}: a live native worker must retain admission");
+            assert_eq!(previous_held, u32::from(permit_mode == 3), "preserve any older disk lease's unit");
+        }
+    }
+
+    #[tokio::test]
+    async fn escaped_disk_permit_restores_resource_context_without_a_new_unit() {
+        let gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one unit"), 1);
+        let disk = DiskIoAdmission::new("resource-unit-escape");
+        let permit = with_background_resource_unit(gate.acquire().await, async {
+            disk.background(&CancellationToken::new()).await.expect("disk permit")
+        })
+        .await;
+        assert_eq!(gate.snapshot().active, 1);
+        with_background_disk_io(CancellationToken::new(), async {
+            with_disk_io_permit(Some(permit), async {
+                let unit = current_resource_unit().expect("escaped disk lease restores unit into an existing context");
+                assert_eq!(gate.snapshot().active, 1, "cloning retains one unit rather than charging twice");
+                drop(unit);
+            })
+            .await;
+        })
+        .await;
+        assert_eq!(gate.snapshot().active, 0);
+    }
+
+    #[tokio::test]
+    async fn escaped_stream_retains_resource_unit_outside_task_local_scope() {
+        let gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one unit"), 1);
+        let disk = DiskIoAdmission::new("resource-unit-stream");
+        let (mut writer, inner) = tokio::io::duplex(8);
+        let mut reader = with_background_resource_unit(gate.acquire().await, async {
+            with_background_disk_io(CancellationToken::new(), async {
+                AdmissionReader::new(inner, disk, current_disk_io_context().expect("resource context"))
+            })
+            .await
+        })
+        .await;
+        assert_eq!(gate.snapshot().active, 1);
+        writer.write_all(b"rust").await.expect("write complete stream");
+        drop(writer);
+        let mut bytes = Vec::new();
+        reader.read_to_end(&mut bytes).await.expect("read outside original scope");
+        assert_eq!(bytes, b"rust");
+        assert_eq!(gate.snapshot().active, 1, "reader still owns its context");
+        drop(reader);
+        assert_eq!(gate.snapshot().active, 0);
+    }
+
+    #[tokio::test]
+    async fn resource_scope_preserves_existing_disk_policy_and_cancellation() {
+        let gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one unit"), 1);
+        with_background_resource_unit(gate.acquire().await, async {
+            assert!(current_disk_io_context().is_none(), "resource ownership does not classify disk I/O");
+            assert!(current_resource_unit().is_some());
+        })
+        .await;
+        let cancellation = CancellationToken::new();
+        let existing = DiskIoContext {
+            class: DiskIoClass::Foreground,
+            cancellation: cancellation.clone(),
+            admission: None,
+            permit: None,
+            resource_unit: None,
+        };
+        with_disk_io_context(existing, async {
+            with_background_resource_unit(gate.acquire().await, async {
+                cancellation.cancel();
+                let context = current_disk_io_context().expect("existing disk context");
+                assert_eq!(context.class, DiskIoClass::Foreground);
+                assert!(
+                    context.resource_unit.is_some(),
+                    "captured disk context must retain the current resource unit"
+                );
+                assert!(context.cancellation.is_cancelled(), "retain the original cancellation token");
+                assert!(current_resource_unit().is_some());
+            })
+            .await;
+        })
+        .await;
+        assert_eq!(gate.snapshot().active, 0);
+    }
+
+    #[tokio::test]
+    async fn resource_only_scope_survives_existing_optional_context_spawn_boundary() {
+        let gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one unit"), 1);
+        let (release_tx, release_rx) = tokio::sync::oneshot::channel();
+        let mut worker = None;
+        with_background_resource_unit(gate.acquire().await, async {
+            worker = Some(tokio::spawn(with_optional_disk_io_context(current_disk_io_context(), async {
+                release_rx.await.expect("release child after caller scope ends");
+                assert!(current_disk_io_context().is_none());
+                assert!(current_resource_unit().is_some());
+            })));
+        })
+        .await;
+        let held = gate.snapshot().active;
+        release_tx.send(()).expect("child remains alive");
+        worker
+            .expect("scoped child was spawned")
+            .await
+            .expect("child preserves resource-only scope");
+        assert_eq!(held, 1);
+        assert_eq!(gate.snapshot().active, 0);
+    }
+
     struct BlockingWorker(tokio::task::JoinHandle<()>);
 
     async fn wait_for_background_waiters(admission: &DiskIoAdmission, expected: usize) {
@@ -636,6 +871,7 @@ mod tests {
                 cancellation: CancellationToken::new(),
                 admission: None,
                 permit: None,
+                resource_unit: None,
             },
         );
         let read = tokio::spawn(async move {

crates/ecstore/src/disk/local/artifact_lease.rs

@@ -0,0 +1,740 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::disk::error::{DiskError, Result};
+use std::{fs::File, sync::Arc};
+
+/// A target disk's admitted artifact publication. Physical mutation workers
+/// retain this Arc until their final syscall and rollback have finished.
+pub struct ArtifactLease {
+    _active: File,
+    _key_directory: File,
+    _seal_directory: File,
+    _root: File,
+}
+
+pub type ArtifactPutLease = ArtifactLease;
+pub type ArtifactRepairLease = ArtifactLease;
+
+/// Read-only census owns both already-closed physical mutation domains.
+pub(super) struct ArtifactInventoryLease {
+    _put: Arc<ArtifactLease>,
+    _repair: Arc<ArtifactLease>,
+}
+
+pub(super) async fn drain_closed(root: File, key: [u8; 32], deadline: tokio::time::Instant) -> Result<ArtifactInventoryLease> {
+    #[cfg(unix)]
+    {
+        unix::drain_closed(root, key, deadline).await
+    }
+    #[cfg(not(unix))]
+    {
+        let _ = (root, key, deadline);
+        Err(DiskError::UnsupportedDisk)
+    }
+}
+
+#[derive(Clone, Copy)]
+pub(crate) enum Domain {
+    Put,
+    Repair,
+}
+
+impl Domain {
+    fn directory(self) -> &'static str {
+        match self {
+            Self::Put => ".rustfs-artifact-put-seals",
+            Self::Repair => ".rustfs-artifact-repair-seals",
+        }
+    }
+
+    fn closed(self) -> &'static [u8] {
+        match self {
+            Self::Put => b"rustfs-artifact-put-sealed-v1\n",
+            Self::Repair => b"rustfs-artifact-repair-sealed-v1\n",
+        }
+    }
+}
+
+pub(super) async fn pinned_root(disk: &super::LocalDisk) -> Result<File> {
+    #[cfg(target_os = "linux")]
+    let root = disk.mount_lease.clone();
+    #[cfg(all(unix, not(target_os = "linux")))]
+    let root = disk.artifact_identity_root.clone();
+    #[cfg(not(unix))]
+    let _ = disk;
+    tokio::task::spawn_blocking(move || {
+        #[cfg(unix)]
+        {
+            root.try_clone().map_err(DiskError::from)
+        }
+        #[cfg(not(unix))]
+        {
+            Err(DiskError::other("artifact disk leases require Unix"))
+        }
+    })
+    .await
+    .map_err(DiskError::from)?
+}
+
+pub(super) async fn acquire(root: File, key: [u8; 32]) -> Result<Arc<ArtifactPutLease>> {
+    acquire_in(root, key, Domain::Put).await
+}
+
+pub(super) async fn acquire_in(root: File, key: [u8; 32], domain: Domain) -> Result<Arc<ArtifactLease>> {
+    #[cfg(unix)]
+    {
+        unix::acquire(root, key, domain).await
+    }
+    #[cfg(not(unix))]
+    {
+        let _ = (root, key, domain);
+        Err(DiskError::other("artifact disk leases require Unix"))
+    }
+}
+
+pub(super) async fn seal(root: File, key: [u8; 32]) -> Result<()> {
+    seal_in(root, key, Domain::Put).await
+}
+
+pub(super) async fn seal_in(root: File, key: [u8; 32], domain: Domain) -> Result<()> {
+    #[cfg(unix)]
+    {
+        unix::seal(root, key, domain).await
+    }
+    #[cfg(not(unix))]
+    {
+        let _ = (root, key, domain);
+        Err(DiskError::other("artifact disk leases require Unix"))
+    }
+}
+
+#[cfg(unix)]
+mod unix {
+    use super::*;
+    use rustix::fs::{FlockOperation, Mode, OFlags, flock, mkdirat, openat};
+    use rustix::io::Errno;
+    use std::{io, io::Read, io::Write, os::unix::fs::MetadataExt, time::Duration};
+
+    const POLL_INTERVAL: Duration = Duration::from_millis(10);
+    const LOCK_TIMEOUT: Duration = Duration::from_secs(30);
+
+    struct Opened {
+        root: File,
+        seals: File,
+        key: File,
+        admission: File,
+        active: File,
+        domain: Domain,
+    }
+
+    fn invalid() -> io::Error {
+        io::Error::new(io::ErrorKind::InvalidData, "invalid artifact seal filesystem state")
+    }
+
+    fn directory(parent: &File, name: &str, device: u64) -> io::Result<File> {
+        match mkdirat(parent, name, Mode::RWXU) {
+            Ok(()) | Err(Errno::EXIST) => {}
+            Err(error) => return Err(error.into()),
+        }
+        let file = File::from(openat(
+            parent,
+            name,
+            OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
+            Mode::empty(),
+        )?);
+        let metadata = file.metadata()?;
+        if !metadata.is_dir() || metadata.dev() != device || metadata.mode() & 0o077 != 0 {
+            return Err(invalid());
+        }
+        Ok(file)
+    }
+
+    fn regular(file: &File, device: u64) -> io::Result<std::fs::Metadata> {
+        let metadata = file.metadata()?;
+        if !metadata.is_file() || metadata.dev() != device || metadata.nlink() != 1 || metadata.mode() & 0o077 != 0 {
+            return Err(invalid());
+        }
+        Ok(metadata)
+    }
+
+    fn lock_file(parent: &File, name: &str, device: u64) -> io::Result<File> {
+        let file = File::from(openat(
+            parent,
+            name,
+            OFlags::CREATE | OFlags::RDWR | OFlags::NOFOLLOW | OFlags::CLOEXEC | OFlags::NONBLOCK,
+            Mode::RUSR | Mode::WUSR,
+        )?);
+        if regular(&file, device)?.len() != 0 {
+            return Err(invalid());
+        }
+        Ok(file)
+    }
+
+    impl Opened {
+        fn open(root: File, key: [u8; 32], domain: Domain) -> io::Result<Self> {
+            let metadata = root.metadata()?;
+            if !metadata.is_dir() {
+                return Err(invalid());
+            }
+            let seals = directory(&root, domain.directory(), metadata.dev())?;
+            let name = hex_simd::encode_to_string(key, hex_simd::AsciiCase::Lower);
+            let key = directory(&seals, &name, metadata.dev())?;
+            // Neither lock inode is ever removed or replaced, including after
+            // closing. Independent processes must always lock the same inode.
+            let admission = lock_file(&key, "admission.lock", metadata.dev())?;
+            let active = lock_file(&key, "active.lock", metadata.dev())?;
+            Ok(Self {
+                root,
+                seals,
+                key,
+                admission,
+                active,
+                domain,
+            })
+        }
+
+        fn closed(&self) -> io::Result<Option<File>> {
+            self.closed_with_flags(OFlags::RDWR)
+        }
+
+        fn closed_with_flags(&self, access: OFlags) -> io::Result<Option<File>> {
+            let mut file = match openat(
+                &self.key,
+                "closed",
+                access | OFlags::NOFOLLOW | OFlags::CLOEXEC | OFlags::NONBLOCK,
+                Mode::empty(),
+            ) {
+                Ok(file) => File::from(file),
+                Err(Errno::NOENT) => return Ok(None),
+                Err(error) => return Err(error.into()),
+            };
+            let length = u64::try_from(self.domain.closed().len()).map_err(|_| invalid())?;
+            if regular(&file, self.root.metadata()?.dev())?.len() != length {
+                return Err(invalid());
+            }
+            let mut bytes = vec![0; self.domain.closed().len()];
+            file.read_exact(&mut bytes)?;
+            if bytes != self.domain.closed() {
+                return Err(invalid());
+            }
+            Ok(Some(file))
+        }
+
+        fn admit(self) -> io::Result<Arc<ArtifactLease>> {
+            if self.closed()?.is_some() {
+                return Err(io::Error::new(
+                    io::ErrorKind::PermissionDenied,
+                    "artifact attempt mutation domain is closed",
+                ));
+            }
+            // Admission serializes the closed check with taking this shared
+            // physical owner. A sealer cannot pass between those two steps.
+            flock(&self.active, FlockOperation::NonBlockingLockShared)?;
+            flock(&self.admission, FlockOperation::Unlock)?;
+            Ok(Arc::new(ArtifactLease {
+                _active: self.active,
+                _key_directory: self.key,
+                _seal_directory: self.seals,
+                _root: self.root,
+            }))
+        }
+
+        fn close(self) -> io::Result<Self> {
+            let marker = match self.closed()? {
+                Some(marker) => marker,
+                None => {
+                    let mut marker = File::from(openat(
+                        &self.key,
+                        "closed",
+                        OFlags::CREATE | OFlags::EXCL | OFlags::RDWR | OFlags::NOFOLLOW | OFlags::CLOEXEC,
+                        Mode::RUSR | Mode::WUSR,
+                    )?);
+                    regular(&marker, self.root.metadata()?.dev())?;
+                    marker.write_all(self.domain.closed())?;
+                    marker
+                }
+            };
+            // No admission is released until the permanent denial and every
+            // ancestor entry are durable. A failed/cancelled close never removes it.
+            marker.sync_all()?;
+            self.admission.sync_all()?;
+            self.active.sync_all()?;
+            self.key.sync_all()?;
+            self.seals.sync_all()?;
+            self.root.sync_all()?;
+            flock(&self.admission, FlockOperation::Unlock)?;
+            Ok(self)
+        }
+    }
+
+    async fn lock(file: &File, operation: FlockOperation) -> io::Result<()> {
+        tokio::time::timeout(LOCK_TIMEOUT, async {
+            loop {
+                match flock(file, operation) {
+                    Ok(()) => return Ok(()),
+                    Err(Errno::WOULDBLOCK | Errno::INTR) => tokio::time::sleep(POLL_INTERVAL).await,
+                    Err(error) => return Err(error.into()),
+                }
+            }
+        })
+        .await
+        .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "artifact disk lease lock deadline"))?
+    }
+
+    async fn open(root: File, key: [u8; 32], domain: Domain) -> Result<Opened> {
+        tokio::task::spawn_blocking(move || Opened::open(root, key, domain))
+            .await
+            .map_err(DiskError::from)?
+            .map_err(DiskError::from)
+    }
+
+    fn read_error(error: io::Error) -> DiskError {
+        match error.kind() {
+            io::ErrorKind::NotFound => DiskError::other("artifact inventory requires closed PUT and repair domains"),
+            io::ErrorKind::InvalidData | io::ErrorKind::NotADirectory => DiskError::FileCorrupt,
+            io::ErrorKind::TimedOut => DiskError::Timeout,
+            _ if error.raw_os_error() == Some(Errno::LOOP.raw_os_error()) => DiskError::FileCorrupt,
+            _ => DiskError::from(error),
+        }
+    }
+
+    fn existing_directory(parent: &File, name: &str, device: u64) -> io::Result<File> {
+        let file = File::from(openat(
+            parent,
+            name,
+            OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
+            Mode::empty(),
+        )?);
+        let metadata = file.metadata()?;
+        if metadata.dev() != device || metadata.mode() & 0o077 != 0 {
+            return Err(invalid());
+        }
+        Ok(file)
+    }
+
+    fn existing(root: File, key: [u8; 32], domain: Domain) -> io::Result<Opened> {
+        let device = root.metadata()?.dev();
+        let seals = existing_directory(&root, domain.directory(), device)?;
+        let name = hex_simd::encode_to_string(key, hex_simd::AsciiCase::Lower);
+        let key = existing_directory(&seals, &name, device)?;
+        let lock = |name| -> io::Result<File> {
+            let file = File::from(openat(
+                &key,
+                name,
+                OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC | OFlags::NONBLOCK,
+                Mode::empty(),
+            )?);
+            if regular(&file, device)?.len() != 0 {
+                return Err(invalid());
+            }
+            Ok(file)
+        };
+        let admission = lock("admission.lock")?;
+        let active = lock("active.lock")?;
+        let opened = Opened {
+            root,
+            seals,
+            key,
+            admission,
+            active,
+            domain,
+        };
+        if opened.closed_with_flags(OFlags::RDONLY)?.is_none() {
+            return Err(io::Error::from(io::ErrorKind::NotFound));
+        }
+        Ok(opened)
+    }
+
+    fn retained(opened: Opened) -> Arc<ArtifactLease> {
+        Arc::new(ArtifactLease {
+            _active: opened.active,
+            _key_directory: opened.key,
+            _seal_directory: opened.seals,
+            _root: opened.root,
+        })
+    }
+
+    pub(super) async fn drain_closed(
+        root: File,
+        key: [u8; 32],
+        deadline: tokio::time::Instant,
+    ) -> Result<ArtifactInventoryLease> {
+        let (put, repair) = tokio::task::spawn_blocking(move || {
+            let put = existing(root.try_clone()?, key, Domain::Put)?;
+            let repair = existing(root, key, Domain::Repair)?;
+            Ok::<_, io::Error>((put, repair))
+        })
+        .await
+        .map_err(DiskError::from)?
+        .map_err(read_error)?;
+        // Lock order: closed PUT active exclusive -> closed repair active
+        // exclusive. No admission lock is acquired and no seal is created.
+        tokio::time::timeout_at(deadline, async {
+            lock(&put.active, FlockOperation::NonBlockingLockExclusive).await?;
+            lock(&repair.active, FlockOperation::NonBlockingLockExclusive).await
+        })
+        .await
+        .map_err(|_| DiskError::Timeout)?
+        .map_err(read_error)?;
+        Ok(ArtifactInventoryLease {
+            _put: retained(put),
+            _repair: retained(repair),
+        })
+    }
+
+    pub(super) async fn acquire(root: File, key: [u8; 32], domain: Domain) -> Result<Arc<ArtifactLease>> {
+        let opened = open(root, key, domain).await?;
+        // Lock order: admission exclusive -> active shared. Contention sleeps
+        // asynchronously; no blocking executor thread waits for another owner.
+        lock(&opened.admission, FlockOperation::NonBlockingLockExclusive).await?;
+        tokio::task::spawn_blocking(move || opened.admit())
+            .await
+            .map_err(DiskError::from)?
+            .map_err(DiskError::from)
+    }
+
+    pub(super) async fn seal(root: File, key: [u8; 32], domain: Domain) -> Result<()> {
+        let opened = open(root, key, domain).await?;
+        lock(&opened.admission, FlockOperation::NonBlockingLockExclusive).await?;
+        let opened = tokio::task::spawn_blocking(move || opened.close())
+            .await
+            .map_err(DiskError::from)??;
+        // Closing is already durable. Cancellation or timeout here only loses
+        // the confirmation; admission stays permanently closed across reopen.
+        lock(&opened.active, FlockOperation::NonBlockingLockExclusive).await?;
+        Ok(())
+    }
+}
+
+#[cfg(all(test, unix))]
+mod tests {
+    use super::*;
+    use std::{path::Path, time::Duration};
+
+    const KEY: [u8; 32] = [7; 32];
+
+    fn root(path: &Path) -> File {
+        use rustix::fs::{Mode, OFlags, open};
+        File::from(
+            open(
+                path,
+                OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
+                Mode::empty(),
+            )
+            .expect("open pinned test root"),
+        )
+    }
+
+    fn directory(path: &Path) -> std::path::PathBuf {
+        path.join(Domain::Put.directory())
+            .join(hex_simd::encode_to_string(KEY, hex_simd::AsciiCase::Lower))
+    }
+
+    async fn closed(path: &Path) {
+        tokio::time::timeout(Duration::from_secs(5), async {
+            while !directory(path).join("closed").exists() {
+                tokio::time::sleep(Duration::from_millis(5)).await;
+            }
+        })
+        .await
+        .expect("sealer reaches its monotonic close point");
+    }
+
+    #[tokio::test]
+    async fn artifact_repair_closing_domain_is_independent_and_survives_reopen() {
+        let temp = tempfile::tempdir().expect("disk root");
+        seal(root(temp.path()), KEY).await.expect("PUT closes first");
+        let repair = acquire_in(root(temp.path()), KEY, Domain::Repair)
+            .await
+            .expect("PUT closure permits existing-version repair");
+        assert!(acquire(root(temp.path()), KEY).await.is_err());
+        drop(repair);
+        seal_in(root(temp.path()), KEY, Domain::Repair)
+            .await
+            .expect("repair closes independently");
+        assert!(acquire_in(root(temp.path()), KEY, Domain::Repair).await.is_err());
+        seal_in(root(temp.path()), KEY, Domain::Repair)
+            .await
+            .expect("reopened repair close is durable");
+
+        let other = [8; 32];
+        seal_in(root(temp.path()), other, Domain::Repair)
+            .await
+            .expect("close only repair admission");
+        drop(
+            acquire(root(temp.path()), other)
+                .await
+                .expect("repair closure permits PUT domain"),
+        );
+    }
+
+    #[tokio::test]
+    async fn artifact_repair_cancelled_waiter_keeps_physical_owner_and_durable_close() {
+        let temp = tempfile::tempdir().expect("disk root");
+        let writer = acquire_in(root(temp.path()), KEY, Domain::Repair)
+            .await
+            .expect("repair admitted");
+        let (entered, started) = tokio::sync::oneshot::channel();
+        let (release, waiting) = std::sync::mpsc::channel();
+        let owner = tokio::spawn(async move {
+            tokio::task::spawn_blocking(move || {
+                let _owner = writer;
+                let _ = entered.send(());
+                let _ = waiting.recv();
+            })
+            .await
+            .expect("repair worker");
+        });
+        started.await.expect("physical repair entered");
+        owner.abort();
+        assert!(owner.await.expect_err("cancel repair waiter").is_cancelled());
+        let sealer = tokio::spawn(seal_in(root(temp.path()), KEY, Domain::Repair));
+        let marker = temp
+            .path()
+            .join(Domain::Repair.directory())
+            .join(hex_simd::encode_to_string(KEY, hex_simd::AsciiCase::Lower))
+            .join("closed");
+        tokio::time::timeout(Duration::from_secs(5), async {
+            while !marker.exists() {
+                tokio::time::sleep(Duration::from_millis(5)).await;
+            }
+        })
+        .await
+        .expect("repair admission durably closes");
+        assert!(!sealer.is_finished(), "ARTIFACT_REPAIR_CANCELLED_PHYSICAL_TAIL");
+        sealer.abort();
+        assert!(sealer.await.expect_err("cancel repair close waiter").is_cancelled());
+        assert!(acquire_in(root(temp.path()), KEY, Domain::Repair).await.is_err());
+        let pending = seal_in(root(temp.path()), KEY, Domain::Repair);
+        tokio::pin!(pending);
+        assert!(
+            tokio::time::timeout(Duration::from_millis(50), &mut pending).await.is_err(),
+            "ARTIFACT_REPAIR_REOPEN_MUST_DRAIN_PHYSICAL_TAIL"
+        );
+        release.send(()).expect("release physical repair");
+        tokio::time::timeout(Duration::from_secs(5), &mut pending)
+            .await
+            .expect("physical drain bounded")
+            .expect("reopened close confirms repair drain");
+        assert!(acquire_in(root(temp.path()), KEY, Domain::Repair).await.is_err());
+    }
+
+    #[tokio::test]
+    async fn artifact_put_seal_waits_for_independent_openers_and_last_clone() {
+        let temp = tempfile::tempdir().expect("disk root");
+        let first = acquire(root(temp.path()), KEY).await.expect("first admitted writer");
+        let tail = first.clone();
+        let second = acquire(root(temp.path()), KEY).await.expect("independent shared writer");
+        let pending = seal(root(temp.path()), KEY);
+        tokio::pin!(pending);
+        assert!(
+            tokio::time::timeout(Duration::from_millis(50), &mut pending).await.is_err(),
+            "ARTIFACT_PUT_ACTIVE_WRITERS_DRAIN"
+        );
+        drop(first);
+        drop(second);
+        assert!(
+            tokio::time::timeout(Duration::from_millis(50), &mut pending).await.is_err(),
+            "ARTIFACT_PUT_PHYSICAL_CLONE_RETAINED"
+        );
+        assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CLOSE_REJECTS_ADMISSION");
+        drop(tail);
+        tokio::time::timeout(Duration::from_secs(5), &mut pending)
+            .await
+            .expect("physical tail drain bounded")
+            .expect("seal complete");
+        seal(root(temp.path()), KEY)
+            .await
+            .expect("independent reopen preserves closed state");
+        assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CLOSED_AFTER_REOPEN");
+    }
+
+    #[tokio::test]
+    async fn artifact_put_close_cancellation_leaves_durable_denial() {
+        let temp = tempfile::tempdir().expect("disk root");
+        let writer = acquire(root(temp.path()), KEY).await.expect("admitted writer");
+        let sealer = tokio::spawn(seal(root(temp.path()), KEY));
+        closed(temp.path()).await;
+        sealer.abort();
+        assert!(sealer.await.expect_err("cancel close waiter").is_cancelled());
+        assert!(
+            acquire(root(temp.path()), KEY).await.is_err(),
+            "ARTIFACT_PUT_CANCELLED_CLOSE_STAYS_CLOSED"
+        );
+        drop(writer);
+        seal(root(temp.path()), KEY).await.expect("reopen finishes physical drain");
+        assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CANCELLED_CLOSE_REOPEN");
+    }
+
+    #[tokio::test]
+    async fn artifact_put_physical_worker_retains_lease_after_waiter_cancellation() {
+        let temp = tempfile::tempdir().expect("disk root");
+        let writer = acquire(root(temp.path()), KEY).await.expect("physical writer admitted");
+        let (entered, started) = tokio::sync::oneshot::channel();
+        let (release, waiting) = std::sync::mpsc::channel();
+        let owner = tokio::spawn(async move {
+            tokio::task::spawn_blocking(move || {
+                let _physical_owner = writer;
+                let _ = entered.send(());
+                let _ = waiting.recv();
+            })
+            .await
+            .expect("physical worker");
+        });
+        started.await.expect("physical syscall entered");
+        owner.abort();
+        assert!(owner.await.expect_err("cancel async writer").is_cancelled());
+        let pending = seal(root(temp.path()), KEY);
+        tokio::pin!(pending);
+        assert!(
+            tokio::time::timeout(Duration::from_millis(50), &mut pending).await.is_err(),
+            "ARTIFACT_PUT_CANCELLED_PHYSICAL_TAIL"
+        );
+        release.send(()).expect("release physical worker");
+        tokio::time::timeout(Duration::from_secs(5), &mut pending)
+            .await
+            .expect("physical worker drains")
+            .expect("seal confirms real completion");
+    }
+
+    #[tokio::test]
+    async fn artifact_put_seal_waits_for_another_process() {
+        const CHILD_ROOT: &str = "RUSTFS_TEST_ARTIFACT_PUT_SEAL_ROOT";
+        if let Some(path) = std::env::var_os(CHILD_ROOT) {
+            seal(root(Path::new(&path)), KEY)
+                .await
+                .expect("child closes and drains parent writer");
+            assert!(acquire(root(Path::new(&path)), KEY).await.is_err(), "ARTIFACT_PUT_CHILD_CLOSED");
+            return;
+        }
+        struct Child(std::process::Child);
+        impl Drop for Child {
+            fn drop(&mut self) {
+                let _ = self.0.kill();
+                let _ = self.0.wait();
+            }
+        }
+        let temp = tempfile::tempdir().expect("disk root");
+        let writer = acquire(root(temp.path()), KEY).await.expect("parent owns physical write");
+        let mut child = Child(
+            std::process::Command::new(std::env::current_exe().expect("test executable"))
+                .args([
+                    "--exact",
+                    "disk::local::artifact_lease::tests::artifact_put_seal_waits_for_another_process",
+                ])
+                .env(CHILD_ROOT, temp.path())
+                .spawn()
+                .expect("start independent sealer"),
+        );
+        closed(temp.path()).await;
+        assert!(child.0.try_wait().expect("probe child").is_none(), "ARTIFACT_PUT_CROSS_PROCESS_DRAIN");
+        assert!(
+            acquire(root(temp.path()), KEY).await.is_err(),
+            "ARTIFACT_PUT_CROSS_PROCESS_ADMISSION_CLOSED"
+        );
+        drop(writer);
+        let status = tokio::time::timeout(Duration::from_secs(5), async {
+            loop {
+                if let Some(status) = child.0.try_wait().expect("child wait") {
+                    break status;
+                }
+                tokio::time::sleep(Duration::from_millis(10)).await;
+            }
+        })
+        .await
+        .expect("child physically drains");
+        assert!(status.success(), "ARTIFACT_PUT_CROSS_PROCESS_SEAL");
+        assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_PROCESS_REOPEN_CLOSED");
+    }
+
+    #[tokio::test]
+    async fn artifact_put_lease_rejects_symlinks_at_every_seal_component() {
+        use std::os::unix::fs::{PermissionsExt, symlink};
+        for component in ["directory", "key", "admission.lock", "active.lock", "closed"] {
+            let temp = tempfile::tempdir().expect("disk root");
+            let outside = tempfile::tempdir().expect("symlink destination");
+            let sentinel = outside.path().join("untouched");
+            std::fs::write(&sentinel, b"unchanged").expect("sentinel");
+            let target = match component {
+                "directory" => temp.path().join(Domain::Put.directory()),
+                "key" => {
+                    std::fs::create_dir(temp.path().join(Domain::Put.directory())).expect("seal directory");
+                    std::fs::set_permissions(temp.path().join(Domain::Put.directory()), std::fs::Permissions::from_mode(0o700))
+                        .expect("private seal directory");
+                    directory(temp.path())
+                }
+                file => {
+                    drop(acquire(root(temp.path()), KEY).await.expect("initialize seal files"));
+                    let target = directory(temp.path()).join(file);
+                    if file != "closed" {
+                        std::fs::remove_file(&target).expect("replace fixture lock");
+                    }
+                    target
+                }
+            };
+            symlink(
+                if matches!(component, "directory" | "key") {
+                    outside.path()
+                } else {
+                    &sentinel
+                },
+                target,
+            )
+            .expect("symlink fixture");
+            assert!(
+                acquire(root(temp.path()), KEY).await.is_err(),
+                "ARTIFACT_PUT_SYMLINK_REJECTED: {component}"
+            );
+            assert!(
+                seal(root(temp.path()), KEY).await.is_err(),
+                "ARTIFACT_PUT_SYMLINK_SEAL_REJECTED: {component}"
+            );
+            assert_eq!(std::fs::read(&sentinel).expect("sentinel intact"), b"unchanged");
+        }
+    }
+
+    #[tokio::test]
+    async fn artifact_put_lease_rejects_corrupt_closed_marker() {
+        use std::os::unix::fs::PermissionsExt;
+        let temp = tempfile::tempdir().expect("disk root");
+        drop(acquire(root(temp.path()), KEY).await.expect("initialize seal files"));
+        let marker = directory(temp.path()).join("closed");
+        std::fs::write(&marker, b"partial").expect("corrupt marker fixture");
+        std::fs::set_permissions(&marker, std::fs::Permissions::from_mode(0o600)).expect("marker permissions");
+        assert!(acquire(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CORRUPT_MARKER_FAILS_CLOSED");
+        assert!(seal(root(temp.path()), KEY).await.is_err(), "ARTIFACT_PUT_CORRUPT_MARKER_NOT_CONFIRMED");
+        assert_eq!(std::fs::read(marker).expect("corruption retained"), b"partial");
+    }
+
+    #[tokio::test]
+    async fn artifact_put_lease_stays_on_pinned_root_after_path_replacement() {
+        let temp = tempfile::tempdir().expect("disk root parent");
+        let current = temp.path().join("current");
+        let original = temp.path().join("original");
+        std::fs::create_dir(&current).expect("initial disk root");
+        let pin = root(&current);
+        drop(
+            acquire(pin.try_clone().expect("root pin clone"), KEY)
+                .await
+                .expect("initialize pinned seal"),
+        );
+        std::fs::rename(&current, &original).expect("replace configured pathname");
+        std::fs::create_dir(&current).expect("replacement disk root");
+        seal(pin, KEY).await.expect("close original disk through its descriptor");
+        assert!(directory(&original).join("closed").exists(), "ARTIFACT_PUT_ROOT_PINNED");
+        assert!(!current.join(Domain::Put.directory()).exists(), "ARTIFACT_PUT_REPLACEMENT_UNTOUCHED");
+        assert!(acquire(root(&original), KEY).await.is_err());
+    }
+}

crates/ecstore/src/disk/local/commit.rs

@@ -21,10 +21,11 @@ use super::run_destination_commit_directory_preparation;
 use super::should_fail_local_inline_rollback_hardlink;
 use super::{
     EVENT_DISK_LOCAL_ACCESS_FAILED, EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, EVENT_DISK_LOCAL_RENAME_REJECTED, LOG_COMPONENT_ECSTORE,
-    LOG_SUBSYSTEM_DISK_LOCAL, LocalDisk, SyncMode, effective_durability, inline_metadata_rollback_dir, observe_old_current,
-    remove_dir_all_if_exists, remove_dst_base_before_commit, remove_file_if_exists, rename_data_versions_signature,
-    run_inline_preparation_before_backup, should_fail_after_metadata_commit, should_fail_before_old_metadata_backup,
-    should_fail_commit_rename, should_remove_staged_meta_before_commit, skip_access_checks, tmp_staging_mutation_path,
+    LOG_SUBSYSTEM_DISK_LOCAL, LocalDisk, SyncMode, effective_durability, has_inline_commit_test_failpoints,
+    inline_metadata_rollback_dir, observe_old_current, remove_dir_all_if_exists, remove_dst_base_before_commit,
+    remove_file_if_exists, rename_data_versions_signature, run_inline_preparation_before_backup,
+    should_fail_after_metadata_commit, should_fail_before_old_metadata_backup, should_fail_commit_rename,
+    should_remove_staged_meta_before_commit, skip_access_checks, tmp_staging_mutation_path,
 };
 #[cfg(test)]
 use super::{run_inline_before_file_sync_admission, run_owned_file_write_before_open, run_rename_data_after_first_publication};
@@ -105,16 +106,7 @@ impl StagedRollbackBackup {
 
 impl RenameCommitContext<'_> {
     fn rollback_metadata_std(&self, rollback_data_dir: Option<Uuid>) -> std::io::Result<()> {
-        if let Some(old_data_dir) = rollback_data_dir {
-            let Some(dst_parent) = self.destination_metadata.parent() else {
-                return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing object metadata parent"));
-            };
-            let backup_path = dst_parent.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP);
-            std::fs::rename(backup_path, self.destination_metadata)?;
-        } else {
-            remove_file_if_exists(self.destination_metadata)?;
-        }
-        Ok(())
+        rollback_metadata_std(self.destination_metadata, rollback_data_dir)
     }
 
     fn rollback_non_inline_std(&self, rollback_data_dir: Option<Uuid>) -> std::io::Result<()> {
@@ -130,16 +122,7 @@ impl RenameCommitContext<'_> {
         rollback_data_dir: Option<Uuid>,
         local_backup: Option<&StagedRollbackBackup>,
     ) -> std::io::Result<()> {
-        #[cfg(all(test, not(windows)))]
-        os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Rollback, self.destination_metadata);
-        if let Some(backup) = local_backup {
-            // The preceding commit used this staged directory on the same
-            // filesystem. Never unlink xl.meta before its atomic replacement.
-            std::fs::rename(backup.path(), self.destination_metadata)?;
-        } else {
-            self.rollback_metadata_std(rollback_data_dir)?;
-        }
-        Ok(())
+        rollback_inline_metadata_std(self.destination_metadata, rollback_data_dir, local_backup)
     }
 
     async fn restore_published_data_source(&self) -> Result<()> {
@@ -207,6 +190,106 @@ pub(super) fn create_local_inline_rollback_backup(
     Ok(backup_path)
 }
 
+fn rollback_metadata_std(destination_metadata: &Path, rollback_data_dir: Option<Uuid>) -> std::io::Result<()> {
+    if let Some(old_data_dir) = rollback_data_dir {
+        let Some(dst_parent) = destination_metadata.parent() else {
+            return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing object metadata parent"));
+        };
+        let backup_path = dst_parent.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP);
+        std::fs::rename(backup_path, destination_metadata)?;
+    } else {
+        remove_file_if_exists(destination_metadata)?;
+    }
+    Ok(())
+}
+
+fn rollback_inline_metadata_std(
+    destination_metadata: &Path,
+    rollback_data_dir: Option<Uuid>,
+    local_backup: Option<&StagedRollbackBackup>,
+) -> std::io::Result<()> {
+    #[cfg(all(test, not(windows)))]
+    os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Rollback, destination_metadata);
+    if let Some(backup) = local_backup {
+        // The preceding commit used this staged directory on the same
+        // filesystem. Never unlink xl.meta before its atomic replacement.
+        std::fs::rename(backup.path(), destination_metadata)?;
+        Ok(())
+    } else {
+        rollback_metadata_std(destination_metadata, rollback_data_dir)
+    }
+}
+
+fn fsync_inline_commit_std(
+    dst: &Path,
+    dst_volume_dir: &Path,
+    destination_was_absent: bool,
+    local_backup: Option<&StagedRollbackBackup>,
+    commit_guard: &os::RenameCommitGuard,
+) -> std::io::Result<()> {
+    if let Some(dst_parent) = dst.parent() {
+        let fsync_started = rustfs_io_metrics::put_stage_timer();
+        if let Err(err) = os::fsync_commit_directory_limited_std(dst_parent, commit_guard) {
+            rustfs_io_metrics::record_put_object_stage_duration_from(
+                rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC,
+                fsync_started,
+            );
+            rollback_inline_metadata_std(dst, None, local_backup)?;
+            return Err(err);
+        }
+        rustfs_io_metrics::record_put_object_stage_duration_from(
+            rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC,
+            fsync_started,
+        );
+    }
+
+    if destination_was_absent {
+        let mut ancestor = dst.parent().and_then(Path::parent);
+        while let Some(ancestor_dir) = ancestor {
+            if !ancestor_dir.starts_with(dst_volume_dir) {
+                break;
+            }
+            let fsync_started = rustfs_io_metrics::put_stage_timer();
+            if let Err(err) = os::fsync_dir_limited_std(ancestor_dir) {
+                rustfs_io_metrics::record_put_object_stage_duration_from(
+                    rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC,
+                    fsync_started,
+                );
+                rollback_inline_metadata_std(dst, None, local_backup)?;
+                return Err(err);
+            }
+            rustfs_io_metrics::record_put_object_stage_duration_from(
+                rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC,
+                fsync_started,
+            );
+            if ancestor_dir == dst_volume_dir {
+                break;
+            }
+            ancestor = ancestor_dir.parent();
+        }
+    }
+    Ok(())
+}
+
+type PreparedInlineMetadata = (
+    Option<Uuid>,
+    Option<Uuid>,
+    Option<Vec<u8>>,
+    Option<crate::disk::OldCurrentSize>,
+    bool,
+    Option<FileInfo>,
+    Option<StagedRollbackBackup>,
+    bool,
+    os::PreparedRenameSource,
+);
+
+enum InlinePreparationOutcome {
+    Committed(RenameDataResp),
+    CommitErr(DiskError),
+    Prepared(PreparedInlineMetadata),
+}
+
+#[cfg(test)]
 pub(super) async fn lock_rename_commit_directories(
     source_parent: &Path,
     destination_parent: &Path,
@@ -244,33 +327,74 @@ pub(super) async fn lock_rename_commit_directories(
     result.map_err(to_file_error).map_err(DiskError::from)
 }
 
-async fn prepare_rename_destination_metadata(
-    file_path: &Path,
-    rename_commit_guard: &os::RenameCommitGuard,
+async fn prepare_rename_commit_and_destination_metadata(
+    create_source_parent: bool,
+    source_parent: &Path,
+    destination_parent: &Path,
+    destination_metadata: &Path,
+    base_dir: &Path,
+    publication_root: &os::PublicationRoot,
     mutation_lease: Arc<os::NamespaceMutationLease>,
-) -> Result<Option<Bytes>> {
+) -> Result<(os::RenameCommitGuard, Option<Bytes>)> {
     #[cfg(any(windows, target_os = "linux"))]
     let result = {
-        let file_path = file_path.to_path_buf();
-        let rename_commit_guard = rename_commit_guard.clone();
+        let source_parent = source_parent.to_path_buf();
+        let destination_parent = destination_parent.to_path_buf();
+        let destination_metadata = destination_metadata.to_path_buf();
+        let base_dir = base_dir.to_path_buf();
+        let publication_root = publication_root.clone();
         os::run_blocking_namespace_operation(mutation_lease, move || {
-            os::read_destination_file_with_commit_guard(&file_path, &rename_commit_guard)
+            if create_source_parent {
+                #[cfg(test)]
+                super::volume_mutation_test_hooks::run(super::volume_mutation_test_hooks::Stage::Create, &source_parent);
+                std::fs::create_dir_all(&source_parent)?;
+            }
+            let guard = match os::prepare_rename_commit_guard(&source_parent, &destination_parent, &base_dir, &publication_root) {
+                Ok(guard) => guard,
+                Err(err) => {
+                    return Err(match std::fs::symlink_metadata(&base_dir) {
+                        Err(base_err) if base_err.kind() == ErrorKind::NotFound => base_err,
+                        _ => err,
+                    });
+                }
+            };
+            #[cfg(all(test, windows))]
+            run_destination_commit_directory_preparation(&destination_parent);
+            let data = os::read_destination_file_with_commit_guard(&destination_metadata, &guard)?;
+            Ok((guard, data.map(Bytes::from)))
         })
         .await
     };
     #[cfg(not(any(windows, target_os = "linux")))]
-    let _ = (rename_commit_guard, mutation_lease);
-    #[cfg(not(any(windows, target_os = "linux")))]
-    let result = match fs::read(file_path).await {
-        Ok(data) => Ok(Some(data)),
-        Err(err) if err.kind() == ErrorKind::NotFound => Ok(None),
-        Err(err) => Err(err),
+    let result = {
+        if create_source_parent {
+            let source_parent = source_parent.to_path_buf();
+            os::run_blocking_namespace_operation(mutation_lease, move || {
+                #[cfg(test)]
+                super::volume_mutation_test_hooks::run(super::volume_mutation_test_hooks::Stage::Create, &source_parent);
+                std::fs::create_dir_all(source_parent)
+            })
+            .await
+            .map_err(to_file_error)?;
+        }
+        let guard =
+            os::prepare_rename_commit_guard(source_parent, destination_parent, base_dir, publication_root).map_err(|err| {
+                match std::fs::symlink_metadata(base_dir) {
+                    Err(base_err) if base_err.kind() == ErrorKind::NotFound => base_err,
+                    _ => err,
+                }
+            });
+        match guard {
+            Ok(guard) => match fs::read(destination_metadata).await {
+                Ok(data) => Ok((guard, Some(Bytes::from(data)))),
+                Err(err) if err.kind() == ErrorKind::NotFound => Ok((guard, None)),
+                Err(err) => Err(err),
+            },
+            Err(err) => Err(err),
+        }
     };
 
-    result
-        .map(|data| data.map(Bytes::from))
-        .map_err(to_file_error)
-        .map_err(DiskError::from)
+    result.map_err(to_file_error).map_err(DiskError::from)
 }
 
 /// Proof produced only when the local rename returns at an existing access
@@ -282,24 +406,25 @@ pub(in crate::disk) struct LocalRenamePreflightRejection(());
 pub(super) struct RenameDataState {
     namespace_owner: Option<Arc<dyn Send + Sync>>,
     preflight_rejection: Option<LocalRenamePreflightRejection>,
+    prepared: Option<Arc<os::NamespaceMutationLease>>,
+}
+
+pub(in crate::disk) struct PreparedRenameData {
+    file_info: FileInfo,
+    mutation_lease: Arc<os::NamespaceMutationLease>,
 }
 
 impl LocalDisk {
-    #[tracing::instrument(name = "rename_data", target = "rustfs_ecstore::disk::local", level = "trace", skip_all)]
-    pub(super) async fn rename_data_inner(
+    async fn prepare_rename_data_admission(
         &self,
-        src_volume: &str,
-        src_path: &str,
         fi: FileInfo,
-        dst_volume: &str,
-        dst_path: &str,
-        state: &mut RenameDataState,
-    ) -> Result<RenameDataResp> {
-        crate::hp_guard!("LocalDisk::rename_data");
-        #[cfg(feature = "e2e-test-hooks")]
-        if fi.is_healing() {
-            wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
-        }
+        source: (&str, &str),
+        destination: (&str, &str),
+        source_volume: Option<&str>,
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<PreparedRenameData> {
+        let (src_volume, src_path) = source;
+        let (dst_volume, dst_path) = destination;
         let mut fi = fi;
         // A non-force DeleteBucket must not remove a directory while a local
         // object commit is publishing into it. The peer's empty scan remains
@@ -334,16 +459,32 @@ impl LocalDisk {
         } else {
             None
         };
-        let metadata_lease =
-            os::acquire_metadata_mutation_lease(&self.get_object_path(dst_volume, dst_path)?, state.namespace_owner.take()).await;
-        let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner(
-            &self.root,
-            dst_volume,
-            &destination_object_path,
-            staging_prefix.as_deref(),
-            Some(metadata_lease),
-        )
-        .await;
+        let root_destination_object_path = self.root.join(
+            destination_object_path
+                .strip_prefix(self.io_root())
+                .map_err(|_| DiskError::InvalidPath)?,
+        );
+        let metadata_lease = os::acquire_metadata_mutation_lease(&root_destination_object_path, namespace_owner).await;
+        let mutation_lease = if let Some(source_volume) = source_volume {
+            os::acquire_rename_data_mutation_lease_for_volumes(
+                &self.root,
+                source_volume,
+                dst_volume,
+                &destination_object_path,
+                staging_prefix.as_deref(),
+                Some(metadata_lease),
+            )
+            .await
+        } else {
+            os::acquire_rename_data_mutation_lease_with_owner(
+                &self.root,
+                dst_volume,
+                &destination_object_path,
+                staging_prefix.as_deref(),
+                Some(metadata_lease),
+            )
+            .await
+        };
         if let Some(claim) = quota_fence_claim {
             mutation_lease.attach_external_guard(claim);
         }
@@ -351,6 +492,99 @@ impl LocalDisk {
             fi.erasure.index = 0;
         }
         fi.validate_for_metadata_read()?;
+        Ok(PreparedRenameData {
+            file_info: fi,
+            mutation_lease,
+        })
+    }
+
+    pub(in crate::disk) async fn begin_legacy_rename_data(
+        &self,
+        source: (&str, &str),
+        fi: &FileInfo,
+        destination: (&str, &str),
+        permit: crate::disk::io_admission::DiskIoPermit,
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<PreparedRenameData> {
+        // Capture the same permit, mount FD and listener-instance owner before
+        // quota -> metadata -> namespace -> sorted volume READ admission.
+        #[cfg(target_os = "linux")]
+        let owner: Arc<dyn Send + Sync> = Arc::new((permit, self.mount_lease.clone(), namespace_owner));
+        #[cfg(not(target_os = "linux"))]
+        let owner: Arc<dyn Send + Sync> = Arc::new((permit, namespace_owner));
+        let prepared = self
+            .prepare_rename_data_admission(fi.clone(), source, destination, Some(source.0), Some(owner))
+            .await?;
+        for volume in [Some(source.0), (source.0 != destination.0).then_some(destination.0)]
+            .into_iter()
+            .flatten()
+        {
+            if !skip_access_checks(volume) {
+                let path = self.io_get_bucket_path(volume)?;
+                let metadata = fs::symlink_metadata(path)
+                    .await
+                    .map_err(crate::disk::error_conv::to_volume_error)?;
+                if !metadata.is_dir() {
+                    return Err(DiskError::VolumeNotFound);
+                }
+            }
+        }
+        Ok(prepared)
+    }
+
+    pub(in crate::disk) async fn rename_data_prepared(
+        &self,
+        source: (&str, &str),
+        destination: (&str, &str),
+        prepared: PreparedRenameData,
+    ) -> Result<RenameDataResp> {
+        let PreparedRenameData {
+            file_info,
+            mutation_lease,
+        } = prepared;
+        let mut state = RenameDataState {
+            prepared: Some(mutation_lease),
+            ..Default::default()
+        };
+        self.rename_data_inner(source.0, source.1, file_info, destination.0, destination.1, &mut state)
+            .await
+    }
+
+    #[tracing::instrument(name = "rename_data", target = "rustfs_ecstore::disk::local", level = "trace", skip_all)]
+    pub(super) async fn rename_data_inner(
+        &self,
+        src_volume: &str,
+        src_path: &str,
+        fi: FileInfo,
+        dst_volume: &str,
+        dst_path: &str,
+        state: &mut RenameDataState,
+    ) -> Result<RenameDataResp> {
+        crate::hp_guard!("LocalDisk::rename_data");
+        #[cfg(feature = "e2e-test-hooks")]
+        if fi.is_healing() {
+            wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
+        }
+        let PreparedRenameData {
+            file_info: fi,
+            mutation_lease,
+        } = if let Some(mutation_lease) = state.prepared.take() {
+            PreparedRenameData {
+                file_info: fi,
+                mutation_lease,
+            }
+        } else {
+            self.prepare_rename_data_admission(
+                fi,
+                (src_volume, src_path),
+                (dst_volume, dst_path),
+                None,
+                state.namespace_owner.take(),
+            )
+            .await?
+        };
+        let no_inline = fi.data.is_none() && fi.size > 0;
+        let fi_healing = fi.is_healing();
         // Snapshot the destination part paths before `fi` is consumed below. These
         // are the descriptors a reader may hold for the version this call is about
         // to replace (backlog#1145); readers build the identical string in
@@ -404,22 +638,23 @@ impl LocalDisk {
             return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
         }
 
-        // xl.meta path
-        let src_file_path = self.io_get_object_path(src_volume, format!("{}/{}", src_path, STORAGE_FORMAT_FILE).as_str())?;
+        // Keep recovery authority separate from active upload metadata so a
+        // committed upload cannot reopen when set-level cleanup fails.
+        let preserve_upload_metadata = src_volume == super::super::RUSTFS_META_MULTIPART_BUCKET;
+        let source_metadata_name = if preserve_upload_metadata {
+            super::super::MULTIPART_COMMIT_METADATA_FILE
+        } else {
+            STORAGE_FORMAT_FILE
+        };
+        let src_file_path = self.io_get_object_path(src_volume, format!("{src_path}/{source_metadata_name}").as_str())?;
         let dst_file_path = self.io_get_object_open_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?;
 
-        // data_dir path
-        let has_data_dir_path = {
-            let has_data_dir = {
-                if !fi.is_remote() {
-                    fi.data_dir
-                        .map(|dir| rustfs_utils::path::retain_slash(dir.to_string().as_str()))
-                } else {
-                    None
-                }
-            };
-
-            if let Some(data_dir) = has_data_dir {
+        // data_dir path (only used by the non-inline commit branch)
+        let has_data_dir_path = if no_inline && !fi.is_remote() {
+            if let Some(data_dir) = fi
+                .data_dir
+                .map(|dir| rustfs_utils::path::retain_slash(dir.to_string().as_str()))
+            {
                 let src_data_path = self.io_get_object_path(
                     src_volume,
                     rustfs_utils::path::retain_slash(format!("{}/{}", src_path, data_dir).as_str()).as_str(),
@@ -433,6 +668,8 @@ impl LocalDisk {
             } else {
                 None
             }
+        } else {
+            None
         };
 
         check_path_length(src_file_path.to_string_lossy().to_string().as_str())?;
@@ -452,14 +689,13 @@ impl LocalDisk {
         let dst_file_parent = dst_file_path
             .parent()
             .ok_or_else(|| DiskError::other("missing object metadata parent"))?;
-        if !no_inline {
-            fs::create_dir_all(src_file_parent).await.map_err(to_file_error)?;
-        }
         // Acquire the common trees before reading destination metadata. On
         // Windows this pins the object directory identity across publication.
-        let rename_commit_guard = lock_rename_commit_directories(
+        let (rename_commit_guard, has_dst_buf) = prepare_rename_commit_and_destination_metadata(
+            !no_inline,
             src_file_parent,
             dst_file_parent,
+            &dst_file_path,
             &dst_volume_dir,
             &self.publication_root,
             mutation_lease.clone(),
@@ -474,12 +710,35 @@ impl LocalDisk {
             source_volume_dir: &src_volume_dir,
             data_paths: has_data_dir_path.as_ref(),
         };
-        let has_dst_buf = prepare_rename_destination_metadata(
-            commit.destination_metadata,
-            &commit.directory_guard,
-            commit.mutation_lease.clone(),
-        )
-        .await?;
+
+        if preserve_upload_metadata {
+            let source_object_path = self.get_object_path(src_volume, src_path)?;
+            let authority_owner: Arc<dyn Send + Sync> = if source_object_path == self.get_object_path(dst_volume, dst_path)? {
+                commit.mutation_lease.clone()
+            } else {
+                os::try_acquire_metadata_mutation_lease_with_owner(&source_object_path, commit.mutation_lease.clone())?
+            };
+            let authority = self.io_get_object_path(src_volume, &format!("{src_path}/{STORAGE_FORMAT_FILE}"))?;
+            let recovery = self.io_get_object_path(
+                src_volume,
+                &format!("{src_path}/{}/{STORAGE_FORMAT_FILE}", super::super::MULTIPART_RECOVERY_DIR),
+            )?;
+            check_path_length(recovery.to_string_lossy().as_ref())?;
+            // Both volume READs are already held. Neither source key may wait
+            // behind a competing owner that is queued behind a volume WRITE.
+            let recovery_lease = os::try_acquire_namespace_mutation_lease_with_owner(&recovery, authority_owner.clone())?;
+            os::rename_all_with_lease(&authority, &recovery, &src_volume_dir, commit.publication_root, recovery_lease).await?;
+            // This is system metadata authority, so use its existing strict
+            // durability policy even when the object payload is relaxed.
+            if effective_durability(src_volume).syncs_commit_metadata() {
+                os::fsync_dir_with_owner(recovery.parent().ok_or(DiskError::InvalidPath)?, Some(authority_owner.clone()))
+                    .await
+                    .map_err(to_file_error)?;
+                os::fsync_dir_with_owner(src_file_parent, Some(authority_owner.clone()))
+                    .await
+                    .map_err(to_file_error)?;
+            }
+        }
 
         if no_inline {
             // Non-inline: read xl.meta, parse, write, rename data dir, rename xl.meta
@@ -689,7 +948,7 @@ impl LocalDisk {
             if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref()) {
                 let backup_parent = dst_file_parent.join(old_data_dir.to_string());
                 #[cfg(not(windows))]
-                if let Err(err) = os::make_dir_all(&backup_parent, &skip_parent).await {
+                if let Err(err) = os::make_dir_all_with_owner(&backup_parent, &skip_parent, commit.mutation_lease.clone()).await {
                     commit.restore_published_data_source().await?;
                     return Err(err);
                 }
@@ -868,14 +1127,8 @@ impl LocalDisk {
             #[cfg(windows)]
             drop(commit.directory_guard);
 
-            if let Some(src_file_path_parent) = src_file_path.parent() {
-                if src_volume != super::super::RUSTFS_META_MULTIPART_BUCKET {
-                    let _ = std::fs::remove_dir(src_file_path_parent);
-                } else {
-                    let _ = self
-                        .delete_file(&dst_volume_dir, &src_file_path_parent.to_path_buf(), true, false)
-                        .await;
-                }
+            if !preserve_upload_metadata && let Some(parent) = src_file_path.parent() {
+                let _ = std::fs::remove_dir(parent);
             }
 
             // Heal reuses a version's `data_dir` and lands the rebuilt shard on
@@ -908,14 +1161,12 @@ impl LocalDisk {
             // allowing a retry to reuse the deterministic staging path too early.
             let src = src_file_path.clone();
             let dst = dst_file_path.clone();
-            let cleanup_path = if src_volume == super::super::RUSTFS_META_MULTIPART_BUCKET {
-                src_file_path.parent().map(|p| p.to_path_buf())
-            } else {
-                None
-            };
             let dst_path_for_failpoint = dst_path.to_string();
             let dst_volume_for_source = dst_volume.to_string();
             let dst_path_for_source = dst_path.to_string();
+            let dst_volume_dir_for_commit = dst_volume_dir.clone();
+            let publication_root_for_commit = self.publication_root.clone();
+            let cleanup_src_parent = !preserve_upload_metadata;
             #[cfg(windows)]
             let source_parent = src_file_parent.to_path_buf();
             let rename_commit_guard_for_preparation = commit.directory_guard.clone();
@@ -932,7 +1183,7 @@ impl LocalDisk {
             } else {
                 None
             };
-            let prepare_inline_metadata = move || {
+            let prepare_inline_metadata = move |is_cancelled: &dyn Fn() -> bool| {
                 let mut prepared_metadata_source =
                     os::create_prepared_rename_source_with_commit_guard(&src, &dst, &rename_commit_guard_for_preparation)?;
                 #[cfg(windows)]
@@ -1012,7 +1263,71 @@ impl LocalDisk {
                     staged_rollback_path = Some(StagedRollbackBackup(backup_path));
                 }
 
-                Ok::<_, std::io::Error>((
+                let destination_was_absent = has_dst_buf.is_none();
+                if !is_cancelled()
+                    && rollback_data_dir.is_none()
+                    && !os::dst_dir_fsync_group_commit_enabled()
+                    && !has_inline_commit_test_failpoints(&dst_path_for_failpoint)
+                {
+                    let rename_res = if should_fail_commit_rename(&dst_path_for_failpoint) {
+                        drop(prepared_metadata_source);
+                        Err(DiskError::other("test fail during metadata commit rename"))
+                    } else {
+                        os::rename_prepared_source_on_thread(
+                            prepared_metadata_source,
+                            &src,
+                            &dst,
+                            &dst_volume_dir_for_commit,
+                            &publication_root_for_commit,
+                            &rename_commit_guard_for_preparation,
+                        )
+                    };
+                    if let Err(err) = rename_res {
+                        if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                            let _ = remove_file_if_exists(backup_path);
+                        }
+                        return Ok(InlinePreparationOutcome::CommitErr(err));
+                    }
+
+                    if should_fail_after_metadata_commit(&dst_path_for_failpoint) {
+                        let err = rollback_inline_metadata_std(&dst, None, staged_rollback_path.as_ref())
+                            .err()
+                            .unwrap_or_else(|| std::io::Error::other("test fail after metadata commit"));
+                        return Ok(InlinePreparationOutcome::CommitErr(DiskError::from(err)));
+                    }
+
+                    if sync
+                        && let Err(err) = fsync_inline_commit_std(
+                            &dst,
+                            &dst_volume_dir_for_commit,
+                            destination_was_absent,
+                            staged_rollback_path.as_ref(),
+                            &rename_commit_guard_for_preparation,
+                        )
+                    {
+                        return Ok(InlinePreparationOutcome::CommitErr(DiskError::from(err)));
+                    }
+
+                    if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                        let _ = remove_file_if_exists(backup_path);
+                    }
+                    #[cfg(not(windows))]
+                    if cleanup_src_parent && let Some(parent) = src.parent() {
+                        let _ = std::fs::remove_dir(parent);
+                    }
+
+                    return Ok::<_, std::io::Error>(InlinePreparationOutcome::Committed(RenameDataResp {
+                        old_data_dir,
+                        rollback_data_dir,
+                        cleanup_data_dir: old_data_dir,
+                        sign: version_signature,
+                        old_current_size,
+                        old_current_source_checked,
+                        old_current_source,
+                    }));
+                }
+
+                Ok::<_, std::io::Error>(InlinePreparationOutcome::Prepared((
                     rollback_data_dir,
                     old_data_dir,
                     version_signature,
@@ -1020,19 +1335,22 @@ impl LocalDisk {
                     old_current_source_checked,
                     old_current_source,
                     staged_rollback_path,
-                    has_dst_buf.is_none(),
+                    destination_was_absent,
                     prepared_metadata_source,
-                ))
+                )))
             };
             let inline_preparation = if let Some(admission) = file_sync_admission.as_ref() {
-                os::run_blocking_namespace_file_sync_operation(commit.mutation_lease.clone(), admission, prepare_inline_metadata)
-                    .await
+                os::run_cancellable_blocking_namespace_file_sync_operation(
+                    commit.mutation_lease.clone(),
+                    admission,
+                    prepare_inline_metadata,
+                )
+                .await
             } else {
-                os::run_blocking_namespace_operation(commit.mutation_lease.clone(), prepare_inline_metadata).await
+                os::run_cancellable_blocking_namespace_operation(commit.mutation_lease.clone(), prepare_inline_metadata).await
             }
             .map_err(to_file_error)
             .map_err(DiskError::from);
-
             let (
                 rollback_data_dir,
                 cleanup_data_dir,
@@ -1044,13 +1362,27 @@ impl LocalDisk {
                 destination_was_absent,
                 prepared_metadata_source,
             ) = match inline_preparation {
-                Ok(prepared) => prepared,
-                Err(err) => {
+                Ok(InlinePreparationOutcome::Committed(resp)) => {
+                    drop(file_sync_admission.take());
+                    #[cfg(windows)]
+                    drop(commit.directory_guard);
+                    #[cfg(windows)]
+                    if cleanup_src_parent && let Some(parent) = src_file_path.parent() {
+                        let _ = std::fs::remove_dir(parent);
+                    }
+                    for part_path in &invalidate_part_paths {
+                        self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
+                    }
+                    return Ok(resp);
+                }
+                Ok(InlinePreparationOutcome::CommitErr(err)) | Err(err) => {
+                    drop(file_sync_admission.take());
                     for part_path in &invalidate_part_paths {
                         self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
                     }
                     return Err(err);
                 }
+                Ok(InlinePreparationOutcome::Prepared(prepared)) => prepared,
             };
 
             commit.directory_guard = remove_dst_base_before_commit(
@@ -1083,18 +1415,25 @@ impl LocalDisk {
                 let backup_path = dst_parent
                     .join(rollback_data_dir.to_string())
                     .join(STORAGE_FORMAT_FILE_BACKUP);
-                // rename_all acquires the backup path's namespace lease. Do not
-                // hold a disk admission while acquiring another namespace lock.
+                // Keep backup mutations serialized, but never wait for a second
+                // namespace while holding the volume READ: its owner can be
+                // queued behind a volume DELETE that is waiting for this commit.
                 drop(file_sync_admission.take());
-                if let Err(err) = os::rename_all_with_owner(
-                    staged_backup,
-                    &backup_path,
-                    &dst_volume_dir,
-                    commit.publication_root,
-                    Some(commit.mutation_lease.clone()),
-                )
-                .await
-                {
+                let backup_result =
+                    match os::try_acquire_namespace_mutation_lease_with_owner(&backup_path, commit.mutation_lease.clone()) {
+                        Ok(lease) => {
+                            os::rename_all_with_lease(
+                                staged_backup,
+                                &backup_path,
+                                &dst_volume_dir,
+                                commit.publication_root,
+                                lease,
+                            )
+                            .await
+                        }
+                        Err(err) => Err(err),
+                    };
+                if let Err(err) = backup_result {
                     let _ = remove_file_if_exists(staged_backup);
                     return Err(err);
                 }
@@ -1254,9 +1593,7 @@ impl LocalDisk {
             }
 
             // Cleanup
-            if let Some(ref cleanup) = cleanup_path {
-                let _ = self.delete_file(&dst_volume_dir, cleanup, true, false).await;
-            } else if let Some(parent) = src_file_path.parent() {
+            if cleanup_src_parent && let Some(parent) = src_file_path.parent() {
                 let _ = std::fs::remove_dir(parent);
             }
 

crates/ecstore/src/disk/local/read_path_tests.rs

@@ -0,0 +1,202 @@
+// Copyright 2026 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+use super::*;
+use std::cell::RefCell;
+
+struct Observation {
+    rebuilds: usize,
+    normalizations: usize,
+    before_recheck: Option<Box<dyn FnOnce()>>,
+}
+
+thread_local! {
+    static OBSERVATION: RefCell<Option<Observation>> = const { RefCell::new(None) };
+}
+
+struct Observe;
+
+impl Observe {
+    fn new(before_recheck: Option<Box<dyn FnOnce()>>) -> Self {
+        OBSERVATION.with(|state| {
+            assert!(state.borrow().is_none(), "read-path observations must not nest");
+            *state.borrow_mut() = Some(Observation {
+                rebuilds: 0,
+                normalizations: 0,
+                before_recheck,
+            });
+        });
+        Self
+    }
+
+    fn normalizations(&self) -> usize {
+        OBSERVATION.with(|state| state.borrow().as_ref().expect("active observation").normalizations)
+    }
+
+    fn rebuilds(&self) -> usize {
+        OBSERVATION.with(|state| state.borrow().as_ref().expect("active observation").rebuilds)
+    }
+}
+
+impl Drop for Observe {
+    fn drop(&mut self) {
+        OBSERVATION.with(|state| *state.borrow_mut() = None);
+    }
+}
+
+pub(super) fn record_bucket_rebuild() {
+    OBSERVATION.with(|state| {
+        if let Some(observation) = state.borrow_mut().as_mut() {
+            observation.rebuilds += 1;
+        }
+    });
+}
+
+pub(super) fn record_normalization() {
+    OBSERVATION.with(|state| {
+        if let Some(observation) = state.borrow_mut().as_mut() {
+            observation.normalizations += 1;
+        }
+    });
+}
+
+pub(super) fn before_bucket_recheck() {
+    let hook = OBSERVATION.with(|state| state.borrow_mut().as_mut().and_then(|value| value.before_recheck.take()));
+    if let Some(hook) = hook {
+        hook();
+    }
+}
+
+async fn local_disk(root: &Path) -> LocalDisk {
+    let endpoint = Endpoint::try_from(root.to_string_lossy().as_ref()).expect("local endpoint");
+    let disk = LocalDisk::new(&endpoint, false).await.expect("local disk");
+    disk.make_volume("bucket").await.expect("bucket");
+    disk
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn checked_read_paths_reuse_bucket_construction() {
+    let root = tempfile::tempdir().expect("disk root");
+    let disk = local_disk(root.path()).await;
+    std::fs::create_dir_all(root.path().join("bucket/prefix/object")).expect("object directories");
+    for key in ["prefix/object/xl.meta", "", "prefix/../object/xl.meta"] {
+        let expected = build_local_disk_object_path(disk.io_root(), "bucket", key);
+        let observation = Observe::new(None);
+        let actual = disk.io_get_checked_read_paths("bucket", key).expect("checked read paths");
+        assert_eq!(actual, expected, "returned paths must retain their existing spelling");
+        assert_eq!(observation.rebuilds(), 0, "the validated tuple already owns its bucket path");
+        // path_join_buf already resolves dot components in object keys.
+        let expected_normalizations = if cfg!(windows) { 3 } else { 0 };
+        assert_eq!(observation.normalizations(), expected_normalizations);
+        #[cfg(not(target_os = "linux"))]
+        {
+            assert_eq!(
+                disk.io_get_bucket_and_object_open_path("bucket", key)
+                    .expect("read-version paths"),
+                expected
+            );
+            assert_eq!(observation.rebuilds(), 0, "read_version shares the same checked path core");
+            assert_eq!(observation.normalizations(), 2 * expected_normalizations);
+        }
+    }
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn checked_read_paths_preserve_containment_errors() {
+    let root = tempfile::tempdir().expect("disk root");
+    let disk = local_disk(root.path()).await;
+    for (bucket, key) in [("../outside", "object"), ("bucket", "../outside"), ("bucket", "/outside")] {
+        let old = disk
+            .io_get_object_path(bucket, key)
+            .and_then(|path| disk.io_get_bucket_path(bucket).map(|volume| (volume, path)));
+        let new = disk.io_get_checked_read_paths(bucket, key);
+        assert_eq!(new, old, "path and error behavior for {bucket}/{key}");
+    }
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn checked_read_paths_metadata_bytes_and_absence_errors() {
+    let root = tempfile::tempdir().expect("disk root");
+    let disk = local_disk(root.path()).await;
+    let object = root.path().join("bucket/prefix/object");
+    std::fs::create_dir_all(&object).expect("object");
+    let metadata = b"opaque metadata bytes passed through the read API";
+    std::fs::write(object.join(STORAGE_FORMAT_FILE), metadata).expect("metadata");
+    {
+        let observation = Observe::new(None);
+        let actual = disk
+            .read_metadata("bucket", "prefix/object/xl.meta")
+            .await
+            .expect("read metadata");
+        assert_eq!(actual.as_ref(), metadata);
+        assert_eq!(observation.rebuilds(), 0, "scanner metadata reads use the shared path core");
+        assert_eq!(observation.normalizations(), if cfg!(windows) { 3 } else { 0 });
+    }
+    assert!(matches!(
+        disk.read_metadata("bucket", "missing/xl.meta").await,
+        Err(DiskError::FileNotFound)
+    ));
+    assert!(matches!(
+        disk.read_metadata("missing", "object/xl.meta").await,
+        Err(DiskError::VolumeNotFound)
+    ));
+}
+
+#[cfg(unix)]
+#[tokio::test(flavor = "current_thread")]
+async fn checked_read_paths_recheck_bucket_after_object_validation() {
+    let root = tempfile::tempdir().expect("disk root");
+    let outside = tempfile::tempdir().expect("outside root");
+    let disk = local_disk(root.path()).await;
+    std::fs::create_dir_all(root.path().join("bucket/object")).expect("object");
+    std::fs::write(root.path().join("bucket/object/xl.meta"), b"inside").expect("inside metadata");
+    let bucket = root.path().join("bucket");
+    let saved = root.path().join("saved");
+    let target = outside.path().to_path_buf();
+    let _observation = Observe::new(Some(Box::new(move || {
+        std::fs::rename(&bucket, saved).expect("replace bucket after full-path validation");
+        std::os::unix::fs::symlink(target, bucket).expect("bucket replacement symlink");
+    })));
+    assert!(
+        matches!(disk.io_get_checked_read_paths("bucket", "object/xl.meta"), Err(DiskError::InvalidPath)),
+        "the second bucket check must remain effective before queuing native IO"
+    );
+}
+
+#[test]
+fn read_path_normalization_matches_component_normalizer() {
+    for value in [
+        "",
+        ".",
+        "..",
+        "./a",
+        "a/./b",
+        "a//b/",
+        "a/../b",
+        "../../a",
+        "/",
+        "//a///b/",
+        "/a/../b",
+        "/../../a",
+        r"C:\a\b",
+        r"C:\a\..\b",
+        r"\\server\share\a",
+        r"\\?\C:\a\.\b",
+    ] {
+        let path = Path::new(value);
+        let expected = normalize_path_components(path);
+        let observation = Observe::new(None);
+        let actual = normalize_path_cow(path);
+        assert_eq!(
+            actual.components().collect::<Vec<_>>(),
+            expected.components().collect::<Vec<_>>(),
+            "{value}"
+        );
+        let needs_buffer = cfg!(windows)
+            || path
+                .components()
+                .any(|component| matches!(component, std::path::Component::ParentDir | std::path::Component::CurDir));
+        assert_eq!(matches!(actual, std::borrow::Cow::Owned(_)), needs_buffer, "{value}");
+        assert_eq!(observation.normalizations(), usize::from(needs_buffer), "{value}");
+    }
+}

crates/ecstore/src/disk/mod.rs

@@ -27,12 +27,16 @@ mod io_admission;
 #[cfg(test)]
 pub(crate) mod lifecycle_test_io;
 pub mod local;
+pub use local::artifact_inventory::{ArtifactInventoryFrame, ArtifactInventoryVersion};
 pub mod os;
 
 #[cfg(test)]
 pub(crate) use io_admission::DiskIoClass;
-pub use io_admission::with_background_disk_io;
-pub(crate) use io_admission::{acquire_context_disk_io_permit, current_disk_io_context, with_optional_disk_io_context};
+pub(crate) use io_admission::{
+    acquire_context_disk_io_permit, current_disk_io_context, current_resource_unit, with_optional_disk_io_context,
+    with_optional_resource_unit,
+};
+pub use io_admission::{with_background_disk_io, with_background_resource_unit};
 
 pub const RUSTFS_META_BUCKET: &str = ".rustfs.sys";
 pub const MIGRATING_META_BUCKET: &str = ".minio.sys";
@@ -46,6 +50,9 @@ pub const FORMAT_CONFIG_FILE: &str = "format.json";
 pub const HEALING_MARKER_PATH: &str = "healing.bin";
 pub const STORAGE_FORMAT_FILE: &str = "xl.meta";
 pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp";
+/// Original upload authority after it stops admitting active part mutations.
+pub(crate) const MULTIPART_RECOVERY_DIR: &str = ".abort-authority";
+pub(crate) const MULTIPART_COMMIT_METADATA_FILE: &str = "xl.meta.complete";
 pub const PART_TRANSACTION_NEW_META: &str = "new.meta";
 pub const PART_TRANSACTION_OLD_META: &str = "old.meta";
 pub const PART_TRANSACTION_ROLLBACK: &str = "rollback";
@@ -67,6 +74,10 @@ use crate::cluster::rpc::RemoteDisk;
 use crate::cluster::rpc::build_internode_data_transport_from_env;
 use crate::disk::disk_store::DiskStoreRenameDataExt;
 use crate::disk::disk_store::LocalDiskWrapper;
+pub use crate::disk::disk_store::{
+    LegacyFileStream, LegacyFileWrite, LegacyMetadataWrite, LegacyPartMutation, LegacyPartOperation, LegacyPathDelete,
+    LegacyPathDeleteOperation, LegacyRenameData, LegacyRenameFile,
+};
 use crate::disk::health_state::RuntimeDriveHealthState;
 use crate::disk::local::ScanGuard;
 use bytes::Bytes;
@@ -79,12 +90,7 @@ use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
 use rustfs_madmin::info_commands::DiskMetrics;
 use rustfs_rio::ChunkReaderBox;
 use serde::{Deserialize, Serialize};
-use std::{
-    fmt::Debug,
-    path::PathBuf,
-    sync::{Arc, atomic::AtomicBool},
-    time::Duration,
-};
+use std::{fmt::Debug, path::PathBuf, sync::Arc, time::Duration};
 use time::OffsetDateTime;
 use tokio::io::{AsyncRead, AsyncWrite};
 use uuid::Uuid;
@@ -92,6 +98,7 @@ use uuid::Uuid;
 /// Independent admission and physical ownership for one disk rename.
 #[derive(Default)]
 pub(crate) struct RenameDataGuards {
+    pub(crate) artifact_put: Option<crate::bucket::artifact::ArtifactPutIdentity>,
     pub(crate) scanner_publication_lease_token: Option<Uuid>,
     pub(crate) external_guard: Option<Arc<dyn Send + Sync>>,
     pub(crate) namespace_owner: Option<Arc<dyn Send + Sync>>,
@@ -233,6 +240,117 @@ pub enum Disk {
 }
 
 impl Disk {
+    /// Capture local file-write admission before authoritative classification.
+    pub async fn begin_legacy_file_write(&self, volume: &str) -> Result<LegacyFileWrite> {
+        match self {
+            Self::Local(disk) => disk.begin_legacy_file_write(volume).await,
+            Self::Remote(_) => Err(DiskError::DiskNotFound),
+        }
+    }
+
+    pub(crate) async fn begin_legacy_file_write_with_namespace_owner(
+        &self,
+        volume: &str,
+        owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<LegacyFileWrite> {
+        match self {
+            Self::Local(disk) => disk.begin_legacy_file_write_with_namespace_owner(volume, owner).await,
+            Self::Remote(_) => Err(DiskError::DiskNotFound),
+        }
+    }
+
+    /// Capture metadata publication admission before authoritative classification.
+    pub async fn begin_legacy_metadata_write(&self, volume: &str, path: &str) -> Result<LegacyMetadataWrite> {
+        match self {
+            Self::Local(disk) => disk.begin_legacy_metadata_write(volume, path).await,
+            Self::Remote(_) => Err(DiskError::DiskNotFound),
+        }
+    }
+
+    pub(crate) async fn begin_legacy_metadata_write_with_namespace_owner(
+        &self,
+        volume: &str,
+        path: &str,
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<LegacyMetadataWrite> {
+        match self {
+            Self::Local(disk) => {
+                disk.begin_legacy_metadata_write_with_namespace_owner(volume, path, namespace_owner)
+                    .await
+            }
+            Self::Remote(_) => Err(DiskError::DiskNotFound),
+        }
+    }
+
+    pub(crate) fn record_legacy_admission_failure(
+        &self,
+        operation: &'static str,
+        error: &DiskError,
+        started: std::time::Instant,
+    ) {
+        if let Self::Local(disk) = self {
+            disk.record_legacy_admission_failure(operation, error, started);
+        }
+    }
+
+    pub(crate) async fn begin_legacy_path_delete(
+        &self,
+        volume: &str,
+        operation: LegacyPathDeleteOperation,
+        owner: Option<Arc<dyn Send + Sync>>,
+        deadline: tokio::time::Instant,
+    ) -> Result<LegacyPathDelete> {
+        match self {
+            Self::Local(disk) => disk.begin_legacy_path_delete(volume, operation, owner, deadline).await,
+            Self::Remote(_) => Err(DiskError::DiskNotFound),
+        }
+    }
+
+    pub(crate) async fn begin_legacy_part_mutation(
+        &self,
+        source: Option<(&str, &str)>,
+        destination: (&str, &str),
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
+        operation: LegacyPartOperation,
+    ) -> Result<LegacyPartMutation> {
+        match self {
+            Self::Local(disk) => {
+                disk.begin_legacy_part_mutation(source, destination, namespace_owner, operation)
+                    .await
+            }
+            Self::Remote(_) => Err(DiskError::DiskNotFound),
+        }
+    }
+
+    pub(crate) async fn begin_legacy_rename_file(
+        &self,
+        source: (&str, &str),
+        destination: (&str, &str),
+        durable: bool,
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<LegacyRenameFile> {
+        match self {
+            Self::Local(disk) => {
+                disk.begin_legacy_rename_file(source, destination, durable, namespace_owner)
+                    .await
+            }
+            Self::Remote(_) => Err(DiskError::DiskNotFound),
+        }
+    }
+
+    pub(crate) async fn begin_legacy_rename_data(
+        &self,
+        source: (&str, &str),
+        fi: &FileInfo,
+        destination: (&str, &str),
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<LegacyRenameData> {
+        match self {
+            Self::Local(disk) => disk.begin_legacy_rename_data(source, fi, destination, namespace_owner).await,
+            Self::Remote(_) => Err(DiskError::DiskNotFound),
+        }
+    }
+
     /// A read-only probe; the physical delete repeats the same leaf checks.
     pub async fn is_plain_orphan_data_dir(&self, volume: &str, path: &str) -> Result<bool> {
         match self {
@@ -1056,13 +1174,27 @@ impl Disk {
         dst_volume: &str,
         dst_path: &str,
     ) -> Result<RenameDataResp> {
-        let Some(scope) = crate::store::bucket_heal_scope_for_object(dst_volume, dst_path) else {
-            return self
-                .rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
-                .await;
+        let bucket_scope = crate::store::bucket_heal_scope_for_object(dst_volume, dst_path);
+        let repair_scope = crate::bucket::artifact::artifact_repair_scope(dst_volume, dst_path);
+        if let Some(scope) = &bucket_scope {
+            scope.check()?;
+        }
+        if let Some(scope) = &repair_scope {
+            scope.check().map_err(DiskError::other)?;
+        }
+        let repair_lease = if let (Self::Local(disk), Some(scope)) = (self, &repair_scope) {
+            if let Some(identity) = scope.receiver_identity() {
+                scope.check_version(fi.version_id).await.map_err(DiskError::other)?;
+                Some(disk.acquire_artifact_repair_lease(identity.receiver_key).await?)
+            } else {
+                None
+            }
+        } else {
+            None
         };
-        scope.check()?;
-        self.rename_data_borrowed_with_fence_and_guard(src_volume, src_path, fi, dst_volume, dst_path, None, Some(scope))
+        let owner: Option<Arc<dyn Send + Sync>> = (bucket_scope.is_some() || repair_scope.is_some())
+            .then(|| Arc::new((bucket_scope, repair_scope, repair_lease)) as Arc<dyn Send + Sync>);
+        self.rename_data_borrowed_with_fence_and_guard(src_volume, src_path, fi, dst_volume, dst_path, None, owner)
             .await
     }
 
@@ -1073,10 +1205,20 @@ impl Disk {
         fi: &FileInfo,
         dst_volume: &str,
         dst_path: &str,
-        guards: RenameDataGuards,
+        mut guards: RenameDataGuards,
     ) -> RenameDataObservation {
         match self {
             Disk::Local(local_disk) => {
+                if let Some(identity) = guards.artifact_put.take() {
+                    let lease = match local_disk.acquire_artifact_put_lease(identity.receiver_key).await {
+                        Ok(lease) => lease,
+                        Err(error) => return RenameDataObservation::unknown(Err(error)),
+                    };
+                    // Keep the existing timeout policy; physical workers own the
+                    // receiver lease even after the caller's disk deadline.
+                    guards.namespace_owner =
+                        Some(Arc::new((guards.external_guard.clone(), guards.namespace_owner.take(), lease)));
+                }
                 local_disk
                     .rename_data_observed_with_guards(src_volume, src_path, fi, dst_volume, dst_path, guards)
                     .await
@@ -1084,39 +1226,17 @@ impl Disk {
             Disk::Remote(remote_disk) => RenameDataObservation::unknown(
                 remote_disk
                     .rename_data_borrowed_with_fence(
-                        src_volume,
-                        src_path,
+                        (src_volume, src_path),
                         fi,
-                        dst_volume,
-                        dst_path,
+                        (dst_volume, dst_path),
                         guards.scanner_publication_lease_token,
+                        guards.artifact_put.map(|identity| identity.incarnation),
                     )
                     .await,
             ),
         }
     }
 
-    pub(crate) async fn rename_data_borrowed_with_fence(
-        &self,
-        src_volume: &str,
-        src_path: &str,
-        fi: &FileInfo,
-        dst_volume: &str,
-        dst_path: &str,
-        scanner_publication_lease_token: Option<Uuid>,
-    ) -> Result<RenameDataResp> {
-        self.rename_data_borrowed_with_fence_and_guard(
-            src_volume,
-            src_path,
-            fi,
-            dst_volume,
-            dst_path,
-            scanner_publication_lease_token,
-            None,
-        )
-        .await
-    }
-
     #[allow(clippy::too_many_arguments)]
     pub async fn rename_data_borrowed_with_fence_and_guard(
         &self,
@@ -1137,12 +1257,11 @@ impl Disk {
             Disk::Remote(remote_disk) => {
                 remote_disk
                     .rename_data_borrowed_with_fence(
-                        src_volume,
-                        src_path,
+                        (src_volume, src_path),
                         fi,
-                        dst_volume,
-                        dst_path,
+                        (dst_volume, dst_path),
                         scanner_publication_lease_token,
+                        None,
                     )
                     .await
             }
@@ -1907,12 +2026,6 @@ pub struct WalkDirOptions {
     // Override the remote stream stall timeout for long background walks.
     #[serde(default)]
     pub stall_timeout_ms: Option<u64>,
-
-    /// In-process completion state for bounded local walks. This is skipped
-    /// from RPC serialization; remote peers retain the legacy natural-EOF
-    /// behavior until they support an explicit capability.
-    #[serde(skip)]
-    pub producer_limit_reached: Option<Arc<AtomicBool>>,
 }
 
 impl WalkDirOptions {
@@ -2757,7 +2870,6 @@ mod tests {
             skip_total_timeout: false,
             timeout_ms: Some(10_000),
             stall_timeout_ms: Some(20_000),
-            producer_limit_reached: None,
         };
 
         assert_eq!(opts.bucket, "test-bucket");

crates/ecstore/src/ecstore_validation_blackbox.rs

@@ -153,10 +153,13 @@ async fn refresh_loss_lock_client_keeps_rejection_observable_for_late_waiters()
     );
 }
 
-/// Returns the backing [`tempfile::TempDir`]s alongside the set so callers keep
-/// them alive for the test's duration and the directories are removed on drop.
-pub(crate) async fn make_local_set_disks(drive_count: usize, parity_count: usize) -> (Vec<tempfile::TempDir>, Arc<SetDisks>) {
-    make_local_set_disks_with_ctx(drive_count, parity_count, crate::runtime::instance::bootstrap_ctx()).await
+/// Keep the real metadata owner and backing directories alive together.
+pub(crate) async fn make_local_set_disks(
+    drive_count: usize,
+    parity_count: usize,
+) -> (crate::set_disk::HermeticSetFixture, Arc<SetDisks>) {
+    let (fixture, _, set) = crate::set_disk::hermetic_set_disks_for_pool_with_default_parity(drive_count, 0, parity_count).await;
+    (fixture, set)
 }
 
 /// Like [`make_local_set_disks`], but binds the set to an explicit instance
@@ -756,6 +759,9 @@ async fn blackbox_range_read_restores_exact_slice_with_one_offline_disk() {
 #[tokio::test]
 async fn blackbox_delete_marker_hides_object_body_without_erasing_prior_version_metadata() {
     let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
+    let store = crate::bucket::metadata_sys::object_store_in(set_disks.instance_ctx())
+        .await
+        .expect("fixture metadata authority should belong to this set");
     let bucket = "bb-delete-marker-read-negative";
     let object = "object.bin";
     let opts = ObjectOptions {
@@ -765,13 +771,26 @@ async fn blackbox_delete_marker_hides_object_body_without_erasing_prior_version_
         ..Default::default()
     };
 
-    set_disks
+    store
         .make_bucket(bucket, &MakeBucketOptions::default())
         .await
         .expect("bucket should be created");
     let mut reader = PutObjReader::from_vec(b"body hidden by delete marker".to_vec());
     set_disks
-        .put_object(bucket, object, &mut reader, &opts)
+        .put_object(
+            bucket,
+            object,
+            &mut reader,
+            &ObjectOptions {
+                object_lock_config_snapshot: Some(
+                    store
+                        .object_lock_config_snapshot(bucket)
+                        .await
+                        .expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
+                ),
+                ..opts.clone()
+            },
+        )
         .await
         .expect("object should be written before delete marker");
 

crates/ecstore/src/erasure/codec/bridge.rs

@@ -15,6 +15,7 @@
 use crate::erasure::codec::workspace::RustfsCodecDecodeWorkspace;
 use crate::erasure::coding::Erasure;
 use rustfs_erasure_codec::galois_8::ReedSolomon;
+use smallvec::SmallVec;
 use std::io;
 use std::sync::{Arc, OnceLock};
 
@@ -213,7 +214,7 @@ impl RustfsCodecDecodeEngine {
         // All slots must be present here: the verification path reconstructs
         // missing parity alongside data (see `reconstruct_into`), so a `None`
         // slot is an internal invariant violation, not a degraded-read state.
-        let mut shard_refs = Vec::with_capacity(self.data_shards + self.parity_shards);
+        let mut shard_refs: SmallVec<[&[u8]; 16]> = SmallVec::with_capacity(self.data_shards + self.parity_shards);
         for (index, shard) in shards.iter().enumerate() {
             let shard = shard.as_ref().ok_or_else(|| {
                 io::Error::new(
@@ -423,6 +424,30 @@ mod tests {
         engine.reconstruct_into(shards, &mut workspace).map(|_| ())
     }
 
+    #[test]
+    fn reconstruct_verifies_sources_across_descriptor_boundary() {
+        let data = (0..1025)
+            .map(|index| u8::try_from((index * 29 + 17) % 251).expect("bounded byte"))
+            .collect::<Vec<_>>();
+        for (data_shards, parity_shards) in [(8, 8), (9, 8)] {
+            let erasure = Erasure::new(data_shards, parity_shards, 2048);
+            let expected = encoded_shards(&erasure, &data);
+            let engine = RustfsCodecDecodeEngine::new(&erasure).expect("decode engine");
+            let mut shards = expected.clone();
+            shards[0] = None;
+            shards[data_shards] = None;
+            reconstruct_with(&engine, &mut shards).expect("restore missing data and parity");
+            assert_eq!(shards, expected, "every restored byte must match at width {}", shards.len());
+
+            let mut corrupt = expected.clone();
+            corrupt[0] = None;
+            corrupt[data_shards] = None;
+            corrupt[data_shards + 1].as_mut().expect("surplus parity present")[0] ^= 1;
+            let error = reconstruct_with(&engine, &mut corrupt).expect_err("corrupt surplus source must fail closed");
+            assert_eq!(error.kind(), io::ErrorKind::InvalidData);
+        }
+    }
+
     #[test]
     fn data_shards_complete_ignores_parity_slots() {
         let shards = vec![Some(vec![1]), Some(vec![2]), None];

crates/ecstore/src/erasure/coding/erasure.rs

@@ -189,12 +189,11 @@ impl LegacyReedSolomonEncoder {
                 .is_some_and(|bytes| bytes <= LEGACY_REED_SOLOMON_CACHE_MAX_LOGICAL_SHARD_BYTES_PER_WORKSPACE)
     }
 
-    fn encode(&self, shards: SmallVec<[&mut [u8]; 16]>) -> io::Result<()> {
-        let mut shards_vec: Vec<&mut [u8]> = shards.into_vec();
-        if shards_vec.is_empty() {
+    fn encode(&self, mut shards: SmallVec<[&mut [u8]; 16]>) -> io::Result<()> {
+        if shards.is_empty() {
             return Ok(());
         }
-        let shard_len = shards_vec[0].len();
+        let shard_len = shards[0].len();
         let cached_encoder = self
             .encoder_cache
             .write()
@@ -214,7 +213,7 @@ impl LegacyReedSolomonEncoder {
                     .map_err(|e| io::Error::other(format!("Failed to create SIMD encoder: {e:?}")))?,
             }
         };
-        for (i, shard) in shards_vec.iter().enumerate().take(self.data_shards) {
+        for (i, shard) in shards.iter().enumerate().take(self.data_shards) {
             encoder
                 .add_original_shard(shard)
                 .map_err(|e| io::Error::other(format!("Failed to add shard {i}: {e:?}")))?;
@@ -223,8 +222,8 @@ impl LegacyReedSolomonEncoder {
             .encode()
             .map_err(|e| io::Error::other(format!("SIMD encoding failed: {e:?}")))?;
         for (i, recovery_shard) in result.recovery_iter().enumerate() {
-            if i + self.data_shards < shards_vec.len() {
-                shards_vec[i + self.data_shards].copy_from_slice(recovery_shard);
+            if i + self.data_shards < shards.len() {
+                shards[i + self.data_shards].copy_from_slice(recovery_shard);
             }
         }
         drop(result);
@@ -393,14 +392,13 @@ impl ReedSolomonEncoder {
     }
 
     /// Encode data shards with parity.
-    pub fn encode(&self, shards: SmallVec<[&mut [u8]; 16]>) -> io::Result<()> {
-        let mut shards_vec: Vec<&mut [u8]> = shards.into_vec();
-        if shards_vec.is_empty() {
+    pub fn encode(&self, mut shards: SmallVec<[&mut [u8]; 16]>) -> io::Result<()> {
+        if shards.is_empty() {
             return Ok(());
         }
 
         if let Some(ref rs) = self.encoder {
-            rs.encode(&mut shards_vec)
+            rs.encode(shards.as_mut_slice())
                 .map_err(|e| io::Error::other(format!("Reed-Solomon encode failed: {e:?}")))
         } else {
             Ok(())
@@ -1010,7 +1008,7 @@ impl Erasure {
             return Ok(true);
         }
 
-        let mut shard_refs = Vec::with_capacity(expected_shards);
+        let mut shard_refs: SmallVec<[&[u8]; 16]> = SmallVec::with_capacity(expected_shards);
         let mut shard_len = None;
         for (index, shard) in shards.iter().enumerate() {
             let shard = shard
@@ -1208,6 +1206,53 @@ mod tests {
     use std::task::{Context, Poll};
     use tokio::io::ReadBuf;
 
+    #[test]
+    fn encode_preserves_codec_bytes_across_descriptor_boundary() {
+        let data = (0..1025)
+            .map(|index| u8::try_from((index * 29 + 17) % 251).expect("bounded byte"))
+            .collect::<Vec<_>>();
+        for uses_legacy in [false, true] {
+            for (data_shards, parity_shards) in [(4, 2), (8, 8), (9, 8)] {
+                let erasure = Erasure::new_with_options(data_shards, parity_shards, 2048, uses_legacy);
+                let shards = erasure.encode_data(&data).expect("encode boundary layout");
+                let joined = shards
+                    .iter()
+                    .take(data_shards)
+                    .flat_map(|shard| shard.iter().copied())
+                    .collect::<Vec<_>>();
+                assert_eq!(&joined[..data.len()], &data);
+                assert!(joined[data.len()..].iter().all(|byte| *byte == 0));
+                if uses_legacy {
+                    let mut reference = reed_solomon_simd::ReedSolomonEncoder::new(data_shards, parity_shards, shards[0].len())
+                        .expect("independent legacy encoder");
+                    for shard in shards.iter().take(data_shards) {
+                        reference.add_original_shard(shard).expect("add original shard");
+                    }
+                    let result = reference.encode().expect("independent legacy parity");
+                    for (index, parity) in result.recovery_iter().enumerate() {
+                        assert_eq!(shards[data_shards + index].as_ref(), parity, "legacy parity {index}");
+                    }
+                } else {
+                    let reference = ReedSolomon::new(data_shards, parity_shards).expect("independent modern codec");
+                    let refs = shards.iter().map(Bytes::as_ref).collect::<Vec<_>>();
+                    assert!(reference.verify(&refs).expect("verify independently generated parity"));
+                }
+                let mut optional = optional_shards(&shards);
+                assert!(
+                    erasure
+                        .verify_data_and_parity(&optional)
+                        .expect("verify complete boundary layout")
+                );
+                optional[data_shards].as_mut().expect("parity present")[0] ^= 1;
+                assert!(
+                    !erasure
+                        .verify_data_and_parity(&optional)
+                        .expect("detect corrupt boundary parity")
+                );
+            }
+        }
+    }
+
     fn erasure_with_invalid_dimensions(data_shards: usize, parity_shards: usize, block_size: usize) -> Erasure {
         Erasure {
             data_shards,

crates/ecstore/src/lib.rs

@@ -55,8 +55,8 @@ mod set_disk;
 mod storage_api_contracts;
 mod store;
 
-pub use disk::with_background_disk_io;
-pub use store::PoolMetaWriteGateStatus;
+pub use disk::{with_background_disk_io, with_background_resource_unit};
+pub use store::{BucketIncarnationFenceGuard, PoolMetaWriteGateStatus};
 
 // pub mod checksum;
 mod event;

crates/ecstore/src/object_api/readers.rs

@@ -4473,3 +4473,7 @@ mod tests {
             .expect_err("a lying frame-layout marker must fail closed, not serve plaintext");
     }
 }
+
+#[cfg(test)]
+#[path = "readers/compression_index_compatibility_tests.rs"]
+mod compression_index_compatibility_tests;

crates/ecstore/src/object_api/readers/compression_index_compatibility_tests.rs

@@ -0,0 +1,185 @@
+use super::*;
+use rustfs_utils::http::metadata_compat::{SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, insert_str};
+use sha2::{Digest, Sha256};
+use std::io::Cursor;
+use tokio::io::AsyncReadExt;
+
+const STREAM: &[u8] = include_bytes!(concat!(
+    env!("CARGO_MANIFEST_DIR"),
+    "/tests/fixtures/compression-index/legacy-lz4-stream.bin"
+));
+const HISTORICAL_INDEX: &[u8] = include_bytes!(concat!(
+    env!("CARGO_MANIFEST_DIR"),
+    "/tests/fixtures/compression-index/pre-zigzag-index.bin"
+));
+#[cfg(not(feature = "rio-v2"))]
+const CURRENT_INDEX: &[u8] =
+    include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/compression-index/zigzag-index.bin"));
+const PLAINTEXT_LEN: usize = 3 * 1024 * 1024 + 123;
+const PLAINTEXT_SHA256: &str = "68f9edf6f9600395d6d2b5517d97570fecd2a0c6dbe6130617ef415a3b48d89c";
+
+fn fixture_info(index: &[u8]) -> ObjectInfo {
+    let mut metadata = HashMap::new();
+    insert_str(&mut metadata, SUFFIX_COMPRESSION, "lz4".to_owned());
+    insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, PLAINTEXT_LEN.to_string());
+    ObjectInfo {
+        size: i64::try_from(STREAM.len()).expect("fixed fixture size fits i64"),
+        parts: Arc::new(vec![ObjectPartInfo {
+            number: 1,
+            size: STREAM.len(),
+            actual_size: i64::try_from(PLAINTEXT_LEN).expect("fixed plaintext length fits i64"),
+            index: Some(Bytes::copy_from_slice(index)),
+            ..Default::default()
+        }]),
+        user_defined: Arc::new(metadata),
+        ..Default::default()
+    }
+}
+
+fn fixture_plaintext() -> Vec<u8> {
+    (0..PLAINTEXT_LEN)
+        .map(|offset| {
+            let chunk = offset / 1024;
+            let lane = offset % 1024;
+            if lane < 8 {
+                u64::try_from(chunk).expect("fixed chunk number fits u64").to_le_bytes()[lane]
+            } else {
+                b"abcdefghijklmnopqrstuvwxyz"[(offset + chunk) % 26]
+            }
+        })
+        .collect()
+}
+
+#[tokio::test]
+async fn historical_index_preserves_the_complete_compressed_body() {
+    let info = fixture_info(HISTORICAL_INDEX);
+    let (mut reader, storage_offset, _) = GetObjectReader::new(
+        Box::new(Cursor::new(STREAM.to_vec())),
+        None,
+        &info,
+        &ObjectOptions::default(),
+        &HeaderMap::new(),
+    )
+    .await
+    .expect("construct reader for fixed historical index");
+    let mut actual = Vec::new();
+    reader.read_to_end(&mut actual).await.expect("read complete historical body");
+    assert_eq!(storage_offset, 0);
+    assert_eq!(actual.len(), PLAINTEXT_LEN);
+    assert_eq!(
+        hex_simd::encode_to_string(Sha256::digest(&actual), hex_simd::AsciiCase::Lower),
+        PLAINTEXT_SHA256
+    );
+}
+
+#[tokio::test]
+async fn historical_index_preserves_ranges_across_compression_blocks() {
+    let info = fixture_info(HISTORICAL_INDEX);
+    let expected = fixture_plaintext();
+    for start in [7, 1024 * 1024 - 17, 2 * 1024 * 1024 + 31, PLAINTEXT_LEN - 97] {
+        let range = HTTPRangeSpec {
+            is_suffix_length: false,
+            start: i64::try_from(start).expect("fixed range start fits i64"),
+            end: i64::try_from(start + 63).expect("fixed range end fits i64"),
+        };
+        let plan = ReadPlan::build(Some(range.clone()), &info, &ObjectOptions::default(), &HeaderMap::new())
+            .await
+            .expect("plan historical compressed range");
+        let physical = plan.storage_offset;
+        let input = STREAM.get(physical..).expect("planned offset stays within the fixed stream");
+        let (mut reader, _, _) = GetObjectReader::new(
+            Box::new(Cursor::new(input.to_vec())),
+            Some(range),
+            &info,
+            &ObjectOptions::default(),
+            &HeaderMap::new(),
+        )
+        .await
+        .expect("construct historical range reader");
+        let mut actual = Vec::new();
+        reader.read_to_end(&mut actual).await.expect("read historical range");
+        assert_eq!(actual, expected[start..start + 64], "range starting at {start}");
+    }
+}
+
+#[cfg(not(feature = "rio-v2"))]
+#[tokio::test]
+async fn current_index_selects_a_fixed_compressed_block_for_a_range() {
+    let info = fixture_info(CURRENT_INDEX);
+    let start = 2 * 1024 * 1024 + 31;
+    let range = HTTPRangeSpec {
+        is_suffix_length: false,
+        start: i64::try_from(start).expect("fixed range start fits i64"),
+        end: i64::try_from(start + 63).expect("fixed range end fits i64"),
+    };
+    let plan = ReadPlan::build(Some(range.clone()), &info, &ObjectOptions::default(), &HeaderMap::new())
+        .await
+        .expect("plan range with fixed current index");
+    assert_eq!(plan.storage_offset, 34_663, "use the recorded second block boundary");
+    let physical = plan.storage_offset;
+    let (mut reader, _, _) = GetObjectReader::new(
+        Box::new(Cursor::new(STREAM[physical..].to_vec())),
+        Some(range),
+        &info,
+        &ObjectOptions::default(),
+        &HeaderMap::new(),
+    )
+    .await
+    .expect("construct indexed range reader");
+    let mut actual = Vec::new();
+    reader.read_to_end(&mut actual).await.expect("read indexed range");
+    assert_eq!(actual, fixture_plaintext()[start..start + 64]);
+}
+
+#[tokio::test]
+async fn truncated_index_falls_back_without_truncating_the_range() {
+    let info = fixture_info(&HISTORICAL_INDEX[..HISTORICAL_INDEX.len() - 3]);
+    let start = 2 * 1024 * 1024 + 31;
+    let range = HTTPRangeSpec {
+        is_suffix_length: false,
+        start: i64::try_from(start).expect("fixed range start fits i64"),
+        end: i64::try_from(start + 96).expect("fixed range end fits i64"),
+    };
+    let plan = ReadPlan::build(Some(range.clone()), &info, &ObjectOptions::default(), &HeaderMap::new())
+        .await
+        .expect("invalid index still permits a sequential range plan");
+    assert_eq!(plan.storage_offset, 0, "truncated index must not select a storage offset");
+    let (mut reader, _, _) = GetObjectReader::new(
+        Box::new(Cursor::new(STREAM.to_vec())),
+        Some(range),
+        &info,
+        &ObjectOptions::default(),
+        &HeaderMap::new(),
+    )
+    .await
+    .expect("construct sequential fallback reader");
+    let mut actual = Vec::new();
+    reader
+        .read_to_end(&mut actual)
+        .await
+        .expect("read range without the truncated index");
+    assert_eq!(actual, fixture_plaintext()[start..start + 97]);
+}
+
+#[tokio::test]
+async fn historical_index_fallback_does_not_hide_a_body_checksum_failure() {
+    let info = fixture_info(HISTORICAL_INDEX);
+    let mut corrupted = STREAM.to_vec();
+    corrupted[4] ^= 1; // Corrupt the first block's stored CRC, leaving its payload intact.
+    let (mut reader, _, _) = GetObjectReader::new(
+        Box::new(Cursor::new(corrupted)),
+        None,
+        &info,
+        &ObjectOptions::default(),
+        &HeaderMap::new(),
+    )
+    .await
+    .expect("construct reader before body checksum verification");
+    let mut actual = Vec::new();
+    let error = reader
+        .read_to_end(&mut actual)
+        .await
+        .expect_err("bad body CRC cannot become successful EOF");
+    assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
+    assert!(actual.len() < PLAINTEXT_LEN);
+}

crates/ecstore/src/object_api/types.rs

@@ -152,7 +152,7 @@ pub struct ObjectLockConfigSnapshot {
     config_revision: Option<OffsetDateTime>,
     state: crate::bucket::metadata_sys::ObjectLockConfigState,
     lifecycle_fence: NamespaceLockFence,
-    _lifecycle_guard: Option<rustfs_lock::NamespaceLockGuard>,
+    _lifecycle_guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
     metadata_transaction_guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
 }
 
@@ -210,11 +210,36 @@ impl ObjectLockConfigSnapshot {
             config_revision: Some(config_revision),
             state,
             lifecycle_fence,
-            _lifecycle_guard: Some(lifecycle_guard),
+            _lifecycle_guard: Some(Arc::new(lifecycle_guard)),
             metadata_transaction_guard: Some(Arc::new(metadata_transaction_guard)),
         }
     }
 
+    pub(crate) fn for_store_bucket_under_lifecycle_owner(
+        store_id: Uuid,
+        bucket: &str,
+        bucket_incarnation_id: Uuid,
+        config_revision: OffsetDateTime,
+        state: crate::bucket::metadata_sys::ObjectLockConfigState,
+        lifecycle_guard: Arc<rustfs_lock::NamespaceLockGuard>,
+        metadata_transaction_guard: Arc<rustfs_lock::NamespaceLockGuard>,
+    ) -> Self {
+        let mut lifecycle_fence = NamespaceLockFence::new();
+        if let Some(signal) = lifecycle_guard.lock_lost_signal() {
+            lifecycle_fence.add_signal(signal);
+        }
+        Self {
+            store_id: Some(store_id),
+            bucket: Some(bucket.to_owned()),
+            bucket_incarnation_id: Some(bucket_incarnation_id),
+            config_revision: Some(config_revision),
+            state,
+            lifecycle_fence,
+            _lifecycle_guard: Some(lifecycle_guard),
+            metadata_transaction_guard: Some(metadata_transaction_guard),
+        }
+    }
+
     pub(crate) fn for_store_bucket_under_lifecycle_fence(
         store_id: Uuid,
         bucket: &str,
@@ -254,6 +279,15 @@ impl ObjectLockConfigSnapshot {
         &self.state
     }
 
+    pub(crate) fn is_unbound(&self) -> bool {
+        self.store_id.is_none()
+            && self.bucket.is_none()
+            && self.bucket_incarnation_id.is_none()
+            && self.config_revision.is_none()
+            && self._lifecycle_guard.is_none()
+            && self.metadata_transaction_guard.is_none()
+    }
+
     pub(crate) fn is_valid_for_destructive_put(&self, store_id: Uuid, bucket: &str, bucket_incarnation_id: Uuid) -> bool {
         self.store_id == Some(store_id)
             && self.bucket.as_deref() == Some(bucket)
@@ -266,6 +300,11 @@ impl ObjectLockConfigSnapshot {
                 .is_some_and(|guard| !guard.is_lock_lost())
     }
 
+    pub(crate) fn owns_bucket_lifecycle_fence(&self, store_id: Uuid, bucket: &str, bucket_incarnation_id: Uuid) -> bool {
+        self.is_valid_for_destructive_put(store_id, bucket, bucket_incarnation_id)
+            && self._lifecycle_guard.as_ref().is_some_and(|guard| !guard.is_lock_lost())
+    }
+
     /// Share the held transaction lock without queuing another reader behind
     /// a metadata writer that is itself waiting for this snapshot to drop.
     pub(crate) fn metadata_transaction_guard_for(
@@ -283,6 +322,22 @@ impl ObjectLockConfigSnapshot {
         self.metadata_transaction_guard.clone()
     }
 
+    /// Returns the snapshot's validated bucket incarnation ID only while both the
+    /// lifecycle read fence and the metadata-transaction guard remain active and
+    /// unreleased for `(store_id, bucket)`.
+    pub(crate) fn validated_bucket_incarnation_for(
+        &self,
+        store_id: Uuid,
+        bucket: &str,
+        expected_incarnation_id: Option<Uuid>,
+    ) -> Option<Uuid> {
+        let guard = self.metadata_transaction_guard_for(store_id, bucket, expected_incarnation_id)?;
+        if guard.is_released() {
+            return None;
+        }
+        self.bucket_incarnation_id
+    }
+
     pub(crate) fn add_lock_fences(&self, opts: &mut ObjectOptions) {
         opts.bucket_lifecycle_lock_fence
             .get_or_insert_with(NamespaceLockFence::new)
@@ -946,6 +1001,12 @@ pub struct ObjectOptions {
     /// A genuine target write guard; loss fences alone do not authorize a borrowed write.
     #[doc(hidden)]
     pub write_lock_context: Option<WriteLockContext>,
+    /// Ordinary ECStore PUT target bound to its bucket's immutable artifact namespace.
+    #[doc(hidden)]
+    pub artifact_put_target: Option<Arc<crate::bucket::artifact::ArtifactPutTarget>>,
+    /// Owned, non-serializable authority for one fully sealed managed version.
+    #[doc(hidden)]
+    pub artifact_delete_target: Option<Arc<crate::bucket::artifact::ArtifactDeleteTarget>>,
     /// In-process delete ownership for late filesystem completion. Never
     /// serialized or copied into object metadata.
     #[doc(hidden)]
@@ -1025,6 +1086,9 @@ pub struct ObjectOptions {
     /// Proves an upper layer holds the bucket lifecycle sentinel. A separate
     /// fence avoids recursively acquiring the read lock behind a queued writer.
     pub bucket_lifecycle_lock_fence: Option<NamespaceLockFence>,
+    /// Actual generation owner retained by storage commits and their physical tails.
+    #[doc(hidden)]
+    pub bucket_incarnation_fence: Option<Arc<crate::store::BucketIncarnationFenceGuard>>,
     /// Retains the actual lifecycle guard for an owned multipart commit. A
     /// lock-loss fence alone does not keep a cancelled caller's lock alive. The
     /// owner also retains the continuous incarnation-validation coverage.

crates/ecstore/src/object_api/write_commit_context.rs

@@ -71,7 +71,7 @@ impl WriteCommitGuard {
 
     /// Namespace locks key the disk identity of the name (see `new_ns_lock`),
     /// so a guard proves every spelling that resolves to the same file.
-    fn matches(&self, bucket: &str, object: &str) -> bool {
+    pub(crate) fn matches(&self, bucket: &str, object: &str) -> bool {
         self.bucket.as_ref() == bucket && self.object.as_ref() == object_name_disk_identity(object).as_ref()
     }
 }

crates/ecstore/src/runtime/instance.rs

@@ -110,6 +110,82 @@ impl Drop for NamespaceCommitGuard {
     }
 }
 
+/// A durable decommission pause as seen by this instance's worker watchdog.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(crate) struct DecommissionPauseRecord {
+    pub(crate) reason: String,
+    /// The pool's settled progress when it paused; see
+    /// [`DecommissionCountLedger::progress`].
+    pub(crate) progress: u64,
+    /// Consecutive pauses on `reason` without progress in between.
+    pub(crate) repeats: u32,
+    /// The watchdog does not relaunch the pool before this instant.
+    pub(crate) relaunch_after: Instant,
+}
+
+/// What this instance counted for one decommission generation of a pool.
+///
+/// Progress counters persist only at checkpoints and a resumed pass relists
+/// the bucket, so a retained entry (one whose source stays) is seen again on
+/// every pass. The ledger keeps its counted versions so a relist in the same
+/// process does not count them twice.
+#[derive(Debug, Default)]
+pub(crate) struct DecommissionCountLedger {
+    generation: Option<time::OffsetDateTime>,
+    /// (bucket, object, version id, delete marker) of retained versions.
+    retained: HashSet<(String, String, Option<uuid::Uuid>, bool)>,
+    retained_key_bytes: usize,
+    /// Entries whose source was cleaned up. Only moving an entry off the pool
+    /// advances it; a relisted retained entry cannot.
+    cleaned_entries: u64,
+}
+
+impl DecommissionCountLedger {
+    const MAX_RETAINED_VERSIONS: usize = 1024;
+    const MAX_RETAINED_KEY_BYTES: usize = 1024 * 1024;
+
+    /// Whether this version is counted now: a retained version once, and a
+    /// cleaned-up version unless it was already counted while retained.
+    /// When the retained ledger is full, omit that progress count until source
+    /// cleanup. This bounds diagnostic memory without evicting deduplication
+    /// evidence or blocking data movement.
+    pub(crate) fn admit(&mut self, bucket: &str, object: &str, version: (Option<uuid::Uuid>, bool), cleaned: bool) -> bool {
+        let key_bytes = bucket.len().saturating_add(object.len());
+        if cleaned {
+            // Most successful entries never needed retained-version tracking.
+            if self.retained.is_empty() || key_bytes > Self::MAX_RETAINED_KEY_BYTES {
+                return true;
+            }
+            let key = (bucket.to_owned(), object.to_owned(), version.0, version.1);
+            if self.retained.remove(&key) {
+                self.retained_key_bytes -= key_bytes;
+                return false;
+            }
+            return true;
+        }
+        if self.retained.len() >= Self::MAX_RETAINED_VERSIONS
+            || key_bytes > Self::MAX_RETAINED_KEY_BYTES.saturating_sub(self.retained_key_bytes)
+        {
+            return false;
+        }
+        let key = (bucket.to_owned(), object.to_owned(), version.0, version.1);
+        if self.retained.insert(key) {
+            self.retained_key_bytes += key_bytes;
+            true
+        } else {
+            false
+        }
+    }
+
+    pub(crate) fn record_cleaned_entry(&mut self) {
+        self.cleaned_entries = self.cleaned_entries.saturating_add(1);
+    }
+
+    pub(crate) fn progress(&self) -> u64 {
+        self.cleaned_entries
+    }
+}
+
 /// Runtime state owned by a single `ECStore` instance.
 ///
 /// This is intentionally minimal in the first migration slice; subsequent
@@ -221,6 +297,12 @@ pub struct InstanceContext {
     /// start runs detached from its caller, so this keeps abandoned requests
     /// and their retries from stacking concurrent starts and preflight heals.
     decommission_start_in_flight: AtomicBool,
+    /// The last durable pause of each local decommission, keyed by pool
+    /// index. The worker watchdog backs off relaunching a pool that keeps
+    /// pausing on the same cause without progress.
+    decommission_pauses: std::sync::Mutex<HashMap<usize, DecommissionPauseRecord>>,
+    /// Per-pool counting ledger of the current decommission generation.
+    decommission_count_ledgers: std::sync::Mutex<HashMap<usize, DecommissionCountLedger>>,
     /// Remote scanner publication leases own a read guard until explicit
     /// release or bounded expiry.  Keeping the guard in storage-owned state
     /// makes a remote movement transition wait on the same fence as a local
@@ -295,6 +377,8 @@ impl InstanceContext {
             background_cancel_token: OnceLock::new(),
             data_movement_operation_gate: Arc::new(RwLock::new(())),
             decommission_start_in_flight: AtomicBool::new(false),
+            decommission_pauses: std::sync::Mutex::new(HashMap::new()),
+            decommission_count_ledgers: std::sync::Mutex::new(HashMap::new()),
             scanner_publication_leases: Arc::new(Mutex::new(HashMap::new())),
             data_movement_operation_epoch: AtomicU64::new(0),
             data_movement_operation_epoch_exhausted: AtomicBool::new(false),
@@ -380,6 +464,52 @@ impl InstanceContext {
         self.decommission_start_in_flight.load(Ordering::Acquire)
     }
 
+    /// Record a durable pause of `pool_index`; `update` derives the new record
+    /// from the previous one.
+    pub(crate) fn record_decommission_pause(
+        &self,
+        pool_index: usize,
+        update: impl FnOnce(Option<&DecommissionPauseRecord>) -> DecommissionPauseRecord,
+    ) -> DecommissionPauseRecord {
+        let mut pauses = self
+            .decommission_pauses
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        let record = update(pauses.get(&pool_index));
+        pauses.insert(pool_index, record.clone());
+        record
+    }
+
+    /// Run `update` on `pool_index`'s counting ledger for `generation`,
+    /// starting a fresh ledger when the generation changed.
+    pub(crate) fn with_decommission_count_ledger<T>(
+        &self,
+        pool_index: usize,
+        generation: time::OffsetDateTime,
+        update: impl FnOnce(&mut DecommissionCountLedger) -> T,
+    ) -> T {
+        let mut ledgers = self
+            .decommission_count_ledgers
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        let ledger = ledgers.entry(pool_index).or_default();
+        if ledger.generation != Some(generation) {
+            *ledger = DecommissionCountLedger {
+                generation: Some(generation),
+                ..Default::default()
+            };
+        }
+        update(ledger)
+    }
+
+    pub(crate) fn decommission_pause(&self, pool_index: usize) -> Option<DecommissionPauseRecord> {
+        self.decommission_pauses
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
+            .get(&pool_index)
+            .cloned()
+    }
+
     pub(crate) async fn install_scanner_publication_lease(
         &self,
         token: Uuid,
@@ -684,6 +814,14 @@ impl InstanceContext {
         self.tier_config_mgr.get_or_init(TierConfigMgr::new).clone()
     }
 
+    #[cfg(test)]
+    pub(crate) fn set_test_tier_config_mgr(&self, manager: Arc<RwLock<TierConfigMgr>>) {
+        assert!(
+            self.tier_config_mgr.set(manager).is_ok(),
+            "test tier manager must be installed before use"
+        );
+    }
+
     /// This instance's event notifier, materializing it on first access.
     /// Shared for the lifetime of the instance.
     pub fn event_notifier(&self) -> Arc<RwLock<EventNotifier>> {
@@ -919,6 +1057,38 @@ pub fn bootstrap_ctx() -> Arc<InstanceContext> {
 mod tests {
     use super::*;
 
+    #[test]
+    fn decommission_count_ledger_bounds_retained_entries() {
+        let mut ledger = super::DecommissionCountLedger::default();
+        for index in 0..1024 {
+            assert!(ledger.admit("bucket", &format!("key-{index}"), (None, false), false));
+        }
+        assert!(!ledger.admit("bucket", "overflow", (None, false), false));
+        assert_eq!(ledger.retained.len(), 1024);
+        // An uncounted retained version can still be counted after cleanup.
+        assert!(ledger.admit("bucket", "overflow", (None, false), true));
+        assert!(!ledger.admit("bucket", "key-0", (None, false), true));
+        assert!(ledger.admit("bucket", "next", (None, false), false));
+        assert!(!ledger.admit("bucket", "next", (None, false), false));
+        assert_eq!(ledger.retained.len(), 1024);
+    }
+
+    #[test]
+    fn decommission_count_ledger_bounds_retained_key_bytes() {
+        let mut ledger = super::DecommissionCountLedger::default();
+        let bucket = "b".repeat(63);
+        let object_prefix = "x".repeat(1020);
+        let mut admitted = 0;
+        for index in 0..1024 {
+            admitted += usize::from(ledger.admit(&bucket, &format!("{object_prefix}{index:04}"), (None, false), false));
+        }
+        assert!(admitted > 0 && admitted < 1024);
+        assert!(ledger.retained.iter().map(|(b, o, _, _)| b.len() + o.len()).sum::<usize>() <= 1024 * 1024);
+        // Removal releases the byte reservation as well as the entry slot.
+        assert!(!ledger.admit(&bucket, &format!("{object_prefix}0000"), (None, false), true));
+        assert!(ledger.admit(&bucket, &format!("{object_prefix}1024"), (None, false), false));
+    }
+
     #[test]
     fn namespace_commit_guards_are_instance_local_and_count_until_last_owner() {
         let first = Arc::new(InstanceContext::new());

crates/ecstore/src/services/rebalance/entry.rs

@@ -250,7 +250,7 @@ impl ECStore {
         }
 
         let bucket_incarnation_fence = match bucket_configs.bucket_incarnation_id {
-            Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?),
+            Some(expected) => Some(Arc::new(self.acquire_bucket_incarnation_fence(&bucket, expected).await?)),
             None => None,
         };
 
@@ -533,9 +533,8 @@ impl ECStore {
                         &cleanup_transferred_tier_versions,
                         data_movement::SourceCleanupBucketFence {
                             expected_incarnation_id: bucket_configs.bucket_incarnation_id,
-                            lifecycle_guard: bucket_incarnation_fence
-                                .as_ref()
-                                .and_then(|guard| guard.namespace_lock_guard()),
+                            lifecycle_owner: bucket_incarnation_fence.as_ref(),
+                            lifecycle_guard: None,
                             namespace_lock_lost_signal: lock_lost_signal.clone(),
                             ..Default::default()
                         },
@@ -1729,6 +1728,8 @@ mod tests {
         /// Both versions are on the target with their identities, the null
         /// version's content and SHA-256 are intact, and the source is gone.
         async fn assert_moved(&self) {
+            use crate::disk::DiskAPI as _;
+
             let target = self.store.pools[1]
                 .get_disks_by_key(self.object)
                 .load_file_info_versions_exact(self.bucket, self.object)
@@ -1769,6 +1770,16 @@ mod tests {
                 self.source_versions().await.is_none(),
                 "the source entry must be removed only after both versions moved"
             );
+            let source_disks = self.source_set.disk_inventory().await;
+            assert_eq!(source_disks.iter().flatten().count(), 2, "both source disks must be online");
+            for disk in source_disks.into_iter().flatten() {
+                assert!(
+                    !tokio::fs::try_exists(disk.path().join(self.bucket).join(self.object))
+                        .await
+                        .expect("inspect the physical rebalance source path"),
+                    "rebalance must remove source metadata and shards from every disk"
+                );
+            }
         }
     }
 
@@ -1782,6 +1793,141 @@ mod tests {
         fixture.assert_moved().await;
     }
 
+    /// An `ExpiredObjectAllVersions` expiry is a prefix delete that takes the
+    /// bucket lifecycle write lock, while the entry holds that lock for read
+    /// through its bucket incarnation fence. Applied inline, it waited out the
+    /// lock timeout, stalled every lifecycle read in the bucket behind it, and
+    /// failed the entry. The version moves instead and lifecycle expires it on
+    /// the target (rustfs/backlog#2873).
+    #[test]
+    #[serial_test::serial]
+    fn real_rebalance_moves_a_version_expired_by_a_delete_all_rule() {
+        run_large_stack_async_test("rebalance-delete-all-expiry", run_delete_all_expiry_case);
+    }
+
+    async fn run_delete_all_expiry_case() {
+        const REBALANCE_ID: &str = "rebalance-delete-all-expiry";
+        let (_temp_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None).await;
+        let bucket = "rebalance-delete-all";
+        let object = "expired.bin";
+        // An active run blocks the data-usage publication bucket creation needs.
+        Box::pin(store.make_bucket(
+            bucket,
+            &MakeBucketOptions {
+                versioning_enabled: true,
+                ..Default::default()
+            },
+        ))
+        .await
+        .expect("create bucket");
+        let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
+        let rebalance_meta = active_rebalance_meta(REBALANCE_ID);
+        rebalance_meta
+            .save(Arc::clone(&store.pools[0]))
+            .await
+            .expect("persist the active rebalance run");
+        *store.rebalance_meta.write().await = Some(rebalance_meta);
+        let source_set = store.pools[0].get_disks_by_key(object);
+        let payload = b"expired by an all-versions rule".to_vec();
+        let written = Box::pin(source_set.put_object(
+            bucket,
+            object,
+            &mut PutObjReader::from_vec(payload.clone()),
+            &ObjectOptions {
+                versioned: true,
+                mod_time: Some(OffsetDateTime::now_utc() - time::Duration::days(10)),
+                expected_bucket_incarnation_id: Some(incarnation),
+                ..Default::default()
+            },
+        ))
+        .await
+        .expect("seed the expired source version");
+        let lifecycle = BucketLifecycleConfiguration {
+            expiry_updated_at: None,
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: Some(LifecycleExpiration {
+                    days: Some(1),
+                    expired_object_all_versions: Some(true),
+                    ..Default::default()
+                }),
+                abort_incomplete_multipart_upload: None,
+                del_marker_expiration: None,
+                filter: None,
+                id: Some("delete-all".to_string()),
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: None,
+            }],
+        };
+        let entry = metacache_entry_from_source(&source_set, bucket, object).await;
+
+        let lock_timeout = crate::set_disk::get_lock_acquire_timeout();
+        let started = std::time::Instant::now();
+        let outcome = tokio::time::timeout(
+            lock_timeout,
+            Box::pin(Arc::clone(&store).rebalance_entry(
+                RebalanceEntryTarget {
+                    bucket: bucket.to_string(),
+                    pool_index: 0,
+                },
+                entry,
+                Arc::clone(&source_set),
+                Arc::new(RebalanceBucketConfigs {
+                    bucket_incarnation_id: Some(incarnation),
+                    lifecycle_config: Some(lifecycle),
+                    ..Default::default()
+                }),
+                Arc::from(REBALANCE_ID),
+                CancellationToken::new(),
+            )),
+        )
+        .await
+        .expect("the entry must not wait out the bucket lifecycle lock timeout")
+        .expect("rebalance entry");
+        let elapsed = started.elapsed();
+        assert_eq!(outcome, RebalanceEntryOutcome::Completed);
+        assert!(
+            elapsed < lock_timeout / 2,
+            "the entry must finish well under the {lock_timeout:?} lock timeout, took {elapsed:?}"
+        );
+
+        assert!(
+            source_set
+                .load_file_info_versions_exact(bucket, object)
+                .await
+                .expect("read source versions")
+                .is_none(),
+            "the source entry must be removed once the version moved"
+        );
+        let target = store.pools[1]
+            .get_disks_by_key(object)
+            .load_file_info_versions_exact(bucket, object)
+            .await
+            .expect("read target versions")
+            .expect("the version must reach the target pool");
+        assert_eq!(target.versions.len(), 1, "{:?}", target.versions);
+        assert_eq!(target.versions[0].version_id, written.version_id);
+        let mut reader = store.pools[1]
+            .get_object_reader(
+                bucket,
+                object,
+                None,
+                HeaderMap::new(),
+                &ObjectOptions {
+                    versioned: true,
+                    version_id: written.version_id.map(|version| version.to_string()),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("the moved version should be readable");
+        let mut body = Vec::new();
+        reader.stream.read_to_end(&mut body).await.expect("drain the moved version");
+        assert_eq!(body, payload);
+    }
+
     /// Overwriting after versioning was enabled leaves a newer version above
     /// the null version. The old `None` read selected that newer version, the
     /// target accepted it as an equivalent copy, and cleanup then deleted the

crates/ecstore/src/services/rebalance/mod.rs

@@ -88,6 +88,7 @@ pub async fn test_store_with_persisted_rebalance_meta(
         pool_meta_save_gate: tokio::sync::Mutex::default(),
         ctx,
         bucket_fence_registry: std::sync::Arc::default(),
+        ordinary_classification: Default::default(),
     });
     (temp_dirs, store)
 }
@@ -147,6 +148,19 @@ pub(crate) async fn test_three_pool_stores_with_isolated_node_contexts(
     test_pool_stores_with_contexts(rebalance_meta, true, 3, 2).await
 }
 
+/// Four drives per set, so a set can hold metadata on fewer than its read
+/// quorum of drives while each drive stays online.
+#[cfg(all(test, feature = "test-util"))]
+pub(crate) async fn test_three_pool_stores_with_four_disk_sets_with_isolated_node_contexts(
+    rebalance_meta: Option<RebalanceMeta>,
+) -> (
+    Vec<tempfile::TempDir>,
+    std::sync::Arc<crate::store::ECStore>,
+    std::sync::Arc<crate::store::ECStore>,
+) {
+    test_pool_stores_with_contexts(rebalance_meta, true, 3, 4).await
+}
+
 #[cfg(all(test, feature = "test-util"))]
 pub(crate) async fn test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(
     rebalance_meta: Option<RebalanceMeta>,
@@ -253,6 +267,7 @@ async fn test_pool_stores_with_contexts(
             pool_meta_save_gate: tokio::sync::Mutex::new(pool_meta_write_state.independent_clone_for_test()),
             ctx: store_ctx,
             bucket_fence_registry: std::sync::Arc::default(),
+            ordinary_classification: Default::default(),
         })
     };
     let store = make_store(ctx);

crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs

@@ -3415,6 +3415,7 @@ fn test_store_with_rebalance_meta(meta: RebalanceMeta) -> Arc<crate::store::ECSt
         pool_meta_save_gate: tokio::sync::Mutex::default(),
         ctx: crate::runtime::instance::bootstrap_ctx(),
         bucket_fence_registry: std::sync::Arc::default(),
+        ordinary_classification: Default::default(),
     })
 }
 

crates/ecstore/src/services/rebalance/worker.rs

@@ -641,7 +641,13 @@ impl SetDisks {
                         state = "agreed_entry",
                         "Rebalance listing agreed entry"
                     );
-                    Box::pin(cb1(entry))
+                    {
+                        let future = cb1(entry);
+                        Box::pin(async move {
+                            future.await;
+                            Ok(())
+                        })
+                    }
                 })),
                 partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
                     let resolver = resolver.clone();

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -3053,7 +3053,7 @@ impl RenameRollbackReceipt {
 
 struct RenameRollbackOwnership {
     receipt: Option<RenameRollbackReceipt>,
-    namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
+    namespace_commit_guard: Option<Arc<dyn Send + Sync>>,
 }
 
 async fn inspect_incomplete_rename_rollback(
@@ -3169,7 +3169,7 @@ async fn rollback_failed_rename(
                         old_data_dir: rollback_dir,
                         ..Default::default()
                     },
-                    disk_namespace_commit_guard.map(|owner| owner as Arc<dyn Send + Sync>),
+                    disk_namespace_commit_guard,
                 )
                 .await
             });
@@ -3281,10 +3281,11 @@ async fn record_rename_rollback_outcomes(
 /// quorum and optional scanner lease map together avoids widening either
 /// fanout helper's argument list while preserving the fence semantics.
 pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
+    artifact_put: Option<crate::bucket::artifact::ArtifactPutIdentity>,
     write_quorum: usize,
     scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
     scanner_publication_commit_scope: Option<crate::object_api::ScannerPublicationCommitScope>,
-    namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
+    namespace_commit_guard: Option<Arc<dyn Send + Sync>>,
     rollback_receipt: Option<RenameRollbackReceipt>,
 }
 
@@ -3294,6 +3295,7 @@ impl<'a> RenameDataFenceOptions<'a> {
         scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
     ) -> Self {
         Self {
+            artifact_put: None,
             write_quorum,
             scanner_publication_lease_tokens,
             scanner_publication_commit_scope: None,
@@ -3302,6 +3304,14 @@ impl<'a> RenameDataFenceOptions<'a> {
         }
     }
 
+    pub(in crate::set_disk) fn with_artifact_put(
+        mut self,
+        identity: Option<crate::bucket::artifact::ArtifactPutIdentity>,
+    ) -> Self {
+        self.artifact_put = identity;
+        self
+    }
+
     pub(in crate::set_disk) fn with_rollback_receipt(mut self, receipt: RenameRollbackReceipt) -> Self {
         self.rollback_receipt = Some(receipt);
         self
@@ -3317,7 +3327,7 @@ impl<'a> RenameDataFenceOptions<'a> {
 
     pub(in crate::set_disk) fn with_namespace_commit_guard(
         mut self,
-        namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
+        namespace_commit_guard: Option<Arc<dyn Send + Sync>>,
     ) -> Self {
         self.namespace_commit_guard = namespace_commit_guard;
         self
@@ -3770,6 +3780,7 @@ impl SetDisks {
         fence_options: RenameDataFenceOptions<'_>,
     ) -> disk::error::Result<RenameDataCommit> {
         let RenameDataFenceOptions {
+            artifact_put,
             write_quorum,
             scanner_publication_lease_tokens,
             scanner_publication_commit_scope: _scanner_publication_commit_scope,
@@ -3864,11 +3875,10 @@ impl SetDisks {
                                     &file_info,
                                     &dst_bucket,
                                     &dst_object,
-                                    disk::RenameDataGuards {
+                                    crate::disk::RenameDataGuards {
+                                        artifact_put,
                                         scanner_publication_lease_token,
-                                        namespace_owner: namespace_commit_guard
-                                            .clone()
-                                            .map(|owner| owner as Arc<dyn Send + Sync>),
+                                        namespace_owner: namespace_commit_guard.clone(),
                                         ..Default::default()
                                     },
                                 )
@@ -4177,6 +4187,7 @@ impl SetDisks {
             .await;
         }
         let RenameDataFenceOptions {
+            artifact_put,
             write_quorum,
             scanner_publication_lease_tokens,
             scanner_publication_commit_scope,
@@ -4297,11 +4308,10 @@ impl SetDisks {
                                     file_info,
                                     &dst_bucket,
                                     &dst_object,
-                                    disk::RenameDataGuards {
+                                    crate::disk::RenameDataGuards {
+                                        artifact_put,
                                         scanner_publication_lease_token,
-                                        namespace_owner: namespace_commit_guard
-                                            .clone()
-                                            .map(|owner| owner as Arc<dyn Send + Sync>),
+                                        namespace_owner: namespace_commit_guard.clone(),
                                         ..Default::default()
                                     },
                                 )
@@ -12815,6 +12825,118 @@ mod tests {
         assert!(missing.below_quorum, "missing disk slots should remain visible in the quorum lens");
     }
 
+    #[cfg(not(windows))]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    async fn legacy_path_delete_flow_prefix_delete_keeps_scanner_owner_until_drain() {
+        use crate::disk::os::{drain_namespace_for_test, prepared_publication_test_hooks as hooks};
+        use crate::object_api::{
+            ScannerPublicationCommitScope, ScannerPublicationCommitScopeGuard, ScannerPublicationCommitState,
+        };
+        use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated;
+        use crate::storage_api_contracts::bucket::BucketOperations;
+
+        let (_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
+        let bucket = "path-delete-scanner";
+        crate::bucket::metadata_sys::object_store_in(&set.ctx)
+            .await
+            .expect("fixture object store")
+            .make_bucket(bucket, &Default::default())
+            .await
+            .expect("authoritative bucket creation");
+        let mut paths = Vec::new();
+        for disk in &disks {
+            disk.write_all(bucket, "prefix/part.1", Bytes::from_static(b"old shard"))
+                .await
+                .expect("seed prefix on every disk");
+            paths.push(
+                disk.get_object_path_for_io_if_local(bucket, "prefix")
+                    .expect("local disk")
+                    .expect("prefix path"),
+            );
+        }
+        let movement = Arc::new(tokio::sync::RwLock::new(()));
+        let scope = ScannerPublicationCommitScope::new_storage_owned(
+            7,
+            tokio::time::Instant::now() + Duration::from_secs(30),
+            Vec::new(),
+            movement.clone().read_owned().await,
+        );
+        scope.try_begin().expect("begin scanner publication");
+        let caller_guard = ScannerPublicationCommitScopeGuard::new(scope.clone());
+        let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
+        let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
+        let _hook = hooks::install_at(hooks::Stage::Rename, &paths[0], move || {
+            let _ = entered_tx.send(());
+            let _ = release_rx.recv();
+        });
+        let owner_set = set.clone();
+        let owner_scope = scope.clone();
+        let caller = tokio::spawn(async move {
+            owner_set
+                .delete_prefix_with_scanner_publication_lease(bucket, "prefix", None, Some(owner_scope))
+                .await
+        });
+        let entered = tokio::time::timeout(Duration::from_secs(5), entered_rx).await;
+        caller.abort();
+        let _ = caller.await;
+        drop(caller_guard);
+        let state_while_blocked = scope.state();
+        let movement_while_blocked = movement.clone().try_write_owned().is_ok();
+        let disk::Disk::Local(local) = disks[0].as_ref() else {
+            unreachable!("fixture contains local disks");
+        };
+        let local = local.get_disk();
+        let volume_read_held = crate::disk::os::disk_volume_mutation_lock(&local.root, bucket)
+            .try_write_owned()
+            .is_err();
+        let mut deletion = Box::pin(local.delete_volume(bucket, true));
+        let early_deletion = futures::poll!(deletion.as_mut());
+        let deletion_was_blocked = early_deletion.is_pending();
+        // Always release and drain physical work before reporting an ownership failure.
+        let _ = release_tx.send(());
+        let mut drained = Vec::new();
+        for path in &paths {
+            drained.push(tokio::time::timeout(Duration::from_secs(5), drain_namespace_for_test(path)).await);
+        }
+        let deletion_result = match early_deletion {
+            std::task::Poll::Ready(result) => result,
+            std::task::Poll::Pending => tokio::time::timeout(Duration::from_secs(5), deletion)
+                .await
+                .expect("volume deletion drains"),
+        };
+        let terminal = tokio::time::timeout(Duration::from_secs(5), scope.wait_for_completion()).await;
+        assert!(
+            entered.is_ok_and(|result| result.is_ok()),
+            "real prefix deletion must reach its filesystem worker"
+        );
+        assert!(drained.into_iter().all(|result| result.is_ok()), "every physical namespace must drain");
+        deletion_result.expect("volume deletion after physical drain");
+        assert!(
+            volume_read_held && deletion_was_blocked,
+            "volume deletion must wait for the physical prefix worker"
+        );
+        assert_eq!(state_while_blocked, ScannerPublicationCommitState::InFlight);
+        assert!(!movement_while_blocked, "scanner movement remains fenced after caller cancellation");
+        assert_eq!(terminal.expect("scanner owner completion"), ScannerPublicationCommitState::Committed);
+        assert!(scope.release_movement_permit().await, "terminal scanner owner releases movement");
+        assert!(movement.try_write_owned().is_ok());
+        for path in paths.iter().skip(1) {
+            assert!(!path.exists(), "successful prefix fanout removes every surviving volume's prefix");
+        }
+        disks[0].make_volume(bucket).await.expect("recreate deleted volume");
+        disks[0]
+            .write_all(bucket, "prefix/sentinel", Bytes::from_static(b"new generation"))
+            .await
+            .expect("new generation sentinel");
+        assert_eq!(
+            disks[0]
+                .read_all(bucket, "prefix/sentinel")
+                .await
+                .expect("sentinel survives old deletion"),
+            b"new generation".as_slice()
+        );
+    }
+
     #[tokio::test]
     async fn delete_prefix_succeeds_when_present_disks_reach_quorum() {
         let bucket = "delete-prefix-bucket";

crates/ecstore/src/set_disk/core/metadata_read.rs

@@ -67,6 +67,7 @@ pub(in crate::set_disk) struct MetadataReadResult {
     pub(in crate::set_disk) slots: Vec<MetadataDiskObservation>,
     pub(in crate::set_disk) diagnostics: MetadataFanoutDiagnostics,
     pub(in crate::set_disk) condition_stopped: bool,
+    pub(in crate::set_disk) verified_inline_body: Option<bytes::Bytes>,
 }
 
 impl MetadataReadResult {
@@ -634,7 +635,7 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
     parts_metadata: &[FileInfo],
     disks: &[Option<DiskStore>],
 ) -> Option<&'static str> {
-    inline_body_miss_reason_from_observations(
+    verify_inline_body_from_observations(
         bucket,
         object,
         candidate,
@@ -645,17 +646,18 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
         disks,
     )
     .await
+    .err()
 }
 
-async fn inline_body_miss_reason_from_observations<'a>(
+async fn verify_inline_body_from_observations<'a>(
     bucket: &str,
     object: &str,
     candidate: &FileInfo,
     observations: impl IntoIterator<Item = (usize, Option<&'a FileInfo>)>,
     disks: &[Option<DiskStore>],
-) -> Option<&'static str> {
+) -> Result<bytes::Bytes, &'static str> {
     if let Some(reason) = data_read_early_stop_inline_candidate_miss_reason(candidate) {
-        return Some(reason);
+        return Err(reason);
     }
 
     let Ok(erasure) = coding::Erasure::try_new_with_options(
@@ -664,21 +666,21 @@ async fn inline_body_miss_reason_from_observations<'a>(
         candidate.erasure.block_size,
         candidate.uses_legacy_checksum,
     ) else {
-        return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
+        return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
     };
     let data_files =
         match collect_inline_data_shard_fileinfos_from_observations(observations, candidate, erasure.data_shards, |index| {
             disks.get(index).is_some_and(Option::is_some)
         }) {
             Ok(data_files) => data_files,
-            Err(reason) => return Some(reason),
+            Err(reason) => return Err(reason),
         };
 
     let Some(part) = candidate.parts.first() else {
-        return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
+        return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
     };
     let Ok(object_size) = usize::try_from(candidate.size) else {
-        return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
+        return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
     };
     let checksum_info = candidate.erasure.get_checksum_info(part.number);
     let checksum_algo = if candidate.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S {
@@ -698,14 +700,14 @@ async fn inline_body_miss_reason_from_observations<'a>(
     let Ok(mut readers) =
         build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await
     else {
-        return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
+        return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
     };
 
     #[cfg(test)]
     crate::set_disk::disk_call_counters::record(object, crate::set_disk::disk_call_counters::KIND_BODY_DECODE, 0);
     match try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size).await {
-        Some(body) if body.len() == object_size => None,
-        _ => Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY),
+        Some(body) if body.len() == object_size => Ok(body),
+        _ => Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY),
     }
 }
 
@@ -1080,6 +1082,8 @@ impl SetDisks {
                     let response_start = observe.then(Instant::now);
                     let result = if let Some(disk) = disk {
                         Self::record_read_version_call(&object, disk_index);
+                        #[cfg(test)]
+                        Self::read_version_fanout_barrier(&object, disk_index).await;
                         if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) {
                             Self::record_metadata_slowtail_fault(&object, disk_index);
                             tokio::time::sleep(delay).await;
@@ -1148,6 +1152,7 @@ impl SetDisks {
             slots,
             diagnostics,
             condition_stopped: false,
+            verified_inline_body: None,
         })
     }
 
@@ -1368,6 +1373,7 @@ impl SetDisks {
                                 ),
                             )
                     });
+                let mut verified_inline_body = None;
                 let should_return_early = if condition_stopped {
                     true
                 } else if force_full_wait {
@@ -1377,7 +1383,7 @@ impl SetDisks {
                         Some(_candidate) if non_inline_candidate_eligible == Some(true) => {
                             accumulator.candidate_has_read_reserve()
                         }
-                        Some(candidate) => match inline_body_miss_reason_from_observations(
+                        Some(candidate) => match verify_inline_body_from_observations(
                             bucket.as_ref(),
                             object.as_ref(),
                             candidate,
@@ -1386,8 +1392,11 @@ impl SetDisks {
                         )
                         .await
                         {
-                            None => true,
-                            Some(reason) => {
+                            Ok(body) => {
+                                verified_inline_body = Some(body);
+                                true
+                            }
+                            Err(reason) => {
                                 final_miss_reason_override = Some(reason);
                                 if bounded_fanout
                                     && reason == GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD
@@ -1445,6 +1454,7 @@ impl SetDisks {
                         slots,
                         diagnostics,
                         condition_stopped,
+                        verified_inline_body,
                     });
                 }
             }
@@ -1527,6 +1537,7 @@ impl SetDisks {
             slots,
             diagnostics,
             condition_stopped: false,
+            verified_inline_body: None,
         })
     }
 

crates/ecstore/src/set_disk/mod.rs

@@ -906,6 +906,8 @@ pub(crate) use ops::object::body_cache_plaintext_len;
 #[cfg(all(test, feature = "test-util"))]
 pub(crate) use ops::object::cleanup_rejected_transition_upload_durably;
 #[cfg(test)]
+pub(crate) use ops::object::hermetic_set_disks_support::{HermeticSetFixture, hermetic_set_disks_for_pool_with_default_parity};
+#[cfg(test)]
 pub(crate) use ops::object::{DeleteCleanupBarrier, DeleteObjectCommitBarrier};
 #[cfg(any(test, feature = "test-util"))]
 pub use ops::object::{PutObjectCommitBarrier, PutObjectCommitPause};
@@ -939,6 +941,7 @@ struct OwnedGetObjectFileInfo {
     fi: FileInfo,
     parts_metadata: Vec<FileInfo>,
     online_disks: Vec<Option<DiskStore>>,
+    verified_inline_body: Option<Bytes>,
 }
 
 enum LateMetadataFanout {
@@ -955,6 +958,7 @@ impl GetObjectFileInfo {
                 fi,
                 parts_metadata,
                 online_disks,
+                verified_inline_body: None,
             }),
             shared: None,
             condition_stopped: false,
@@ -1017,6 +1021,27 @@ impl GetObjectFileInfo {
         self.late_metadata_fanout.is_some()
     }
 
+    fn set_verified_inline_body(&mut self, body: Option<Bytes>) {
+        if let Some(owned) = self.owned.as_mut() {
+            owned.verified_inline_body = body;
+        }
+    }
+
+    fn take_verified_inline_body(&mut self) -> Option<Bytes> {
+        self.owned.as_mut().and_then(|owned| owned.verified_inline_body.take())
+    }
+
+    fn into_object_info(self, bucket: &str, object: &str, versioned: bool) -> ObjectInfo {
+        match (self.owned, self.shared) {
+            (Some(snapshot), None) => ObjectInfo::from_owned_file_info(snapshot.fi, bucket, object, versioned),
+            (None, Some(entry)) => match Arc::try_unwrap(entry) {
+                Ok(entry) => ObjectInfo::from_owned_file_info(entry.fi, bucket, object, versioned),
+                Err(entry) => ObjectInfo::from_file_info(&entry.fi, bucket, object, versioned),
+            },
+            _ => unreachable!("GET metadata snapshot representation must be exclusive"),
+        }
+    }
+
     fn into_owned(self) -> (FileInfo, Vec<FileInfo>, Vec<Option<DiskStore>>) {
         let (fi, parts_metadata, online_disks, _) = self.into_owned_with_late_metadata_fanout();
         (fi, parts_metadata, online_disks)
@@ -1122,6 +1147,310 @@ mod prepared_get_object_metadata_tests {
     use http::HeaderMap;
     use tokio::io::AsyncReadExt;
 
+    const OBSERVATION_HOOK_BODY_LEN: usize = 1024 * 1024 + 1;
+
+    struct ObservationBodyHook {
+        object: &'static str,
+        calls: Arc<AtomicU64>,
+        body: bytes::Bytes,
+    }
+
+    #[async_trait::async_trait]
+    impl crate::object_api::GetObjectBodyCacheHook for ObservationBodyHook {
+        async fn lookup(&self, bucket: &str, object: &str, info: &ObjectInfo) -> Option<bytes::Bytes> {
+            if bucket == crate::disk::RUSTFS_META_BUCKET && object == self.object {
+                assert_eq!(info.size, i64::try_from(self.body.len()).expect("hook fixture size"));
+                self.calls.fetch_add(1, Ordering::SeqCst);
+                Some(self.body.clone())
+            } else {
+                None
+            }
+        }
+    }
+
+    struct ObservationHookGuard;
+
+    impl Drop for ObservationHookGuard {
+        fn drop(&mut self) {
+            crate::object_api::unregister_get_object_body_cache_hook();
+        }
+    }
+
+    async fn ordinary_observation_get(set: &SetDisks, object: &str) -> Vec<u8> {
+        let mut reader = set
+            .get_object_reader(crate::disk::RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
+            .await
+            .expect("ordinary GET positive control");
+        let mut bytes = Vec::new();
+        reader.stream.read_to_end(&mut bytes).await.expect("ordinary GET body");
+        bytes
+    }
+
+    fn observation_register_hook(object: &'static str, body: Vec<u8>) -> (ObservationHookGuard, Arc<AtomicU64>) {
+        let calls = Arc::new(AtomicU64::new(0));
+        crate::object_api::register_get_object_body_cache_hook(Arc::new(ObservationBodyHook {
+            object,
+            calls: calls.clone(),
+            body: bytes::Bytes::from(body),
+        }));
+        (ObservationHookGuard, calls)
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(body_cache_hook)]
+    async fn locked_observation_uses_real_outer_guard() {
+        let object = "config/locked-observation-real.bin";
+        let payload = vec![b'p'; OBSERVATION_HOOK_BODY_LEN];
+        let (_dirs, set) = make_local_set_disks(4, 2).await;
+        let mut reader = PutObjReader::from_vec(payload.clone());
+        let written = set
+            .put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
+            .await
+            .expect("persist locked observation fixture");
+        let (_hook, calls) = observation_register_hook(object, vec![b'x'; payload.len()]);
+        assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; payload.len()]);
+        assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
+        let namespace = set
+            .new_ns_lock(crate::disk::RUSTFS_META_BUCKET, object)
+            .await
+            .expect("namespace");
+        let guard = crate::object_api::WriteCommitGuard::acquire(&namespace, std::time::Duration::from_secs(5))
+            .await
+            .expect("matching real owner");
+        let mut contender = Box::pin(crate::object_api::WriteCommitGuard::acquire(
+            &namespace,
+            std::time::Duration::from_secs(30),
+        ));
+        assert!(futures::poll!(contender.as_mut()).is_pending(), "I06L_OWNER_RETAINED");
+        let result = tokio::time::timeout(
+            std::time::Duration::from_secs(10),
+            crate::config::com::read_config_observation_no_lock(set.clone(), object, payload.len(), &guard),
+        )
+        .await
+        .expect("locked observation hang watchdog");
+        assert!(result.is_ok(), "I06L_NO_NESTED_LOCK: {result:?}");
+        let (bytes, info) = result.expect("locked observation");
+        assert!(bytes == payload, "I06L_HOOK_BYPASS");
+        assert!(calls.load(Ordering::SeqCst) == 0, "I06L_HOOK_BYPASS");
+        assert_eq!(info.size, written.size);
+        assert_eq!(info.etag, written.etag);
+        assert_eq!(info.data_dir, written.data_dir);
+        assert_eq!(info.name, object);
+        assert!(futures::poll!(contender.as_mut()).is_pending(), "I06L_OWNER_RETAINED");
+        drop(guard);
+        let next = tokio::time::timeout(std::time::Duration::from_secs(10), contender)
+            .await
+            .expect("owner release watchdog")
+            .expect("competing owner acquired");
+        drop(next);
+        assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; payload.len()]);
+        assert_eq!(calls.load(Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(body_cache_hook)]
+    async fn locked_observation_preserves_empty_missing_and_bound() {
+        let (_dirs, set) = make_local_set_disks(4, 2).await;
+        for (object, body) in [
+            ("config/locked-empty.bin", Vec::new()),
+            ("config/locked-bound.bin", b"raw".to_vec()),
+        ] {
+            let mut reader = PutObjReader::from_vec(body.clone());
+            let written = set
+                .put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
+                .await
+                .expect("persist boundary fixture");
+            let namespace = set
+                .new_ns_lock(crate::disk::RUSTFS_META_BUCKET, object)
+                .await
+                .expect("namespace");
+            let guard = crate::object_api::WriteCommitGuard::acquire(&namespace, std::time::Duration::from_secs(5))
+                .await
+                .expect("matching boundary owner");
+            let result = crate::config::com::read_config_observation_no_lock(set.clone(), object, body.len(), &guard).await;
+            assert!(result.is_ok(), "I06L_EMPTY_DISTINCT: {result:?}");
+            let (bytes, info) = result.expect("boundary observation");
+            assert_eq!(bytes, body);
+            assert_eq!(info.size, written.size);
+            if !body.is_empty() {
+                let result =
+                    crate::config::com::read_config_observation_no_lock(set.clone(), object, body.len() - 1, &guard).await;
+                assert!(matches!(result, Err(Error::CorruptedFormat)), "I06L_BOUND_ENFORCED");
+            }
+        }
+        let object = "config/locked-absent.bin";
+        let namespace = set
+            .new_ns_lock(crate::disk::RUSTFS_META_BUCKET, object)
+            .await
+            .expect("namespace");
+        let guard = crate::object_api::WriteCommitGuard::acquire(&namespace, std::time::Duration::from_secs(5))
+            .await
+            .expect("matching missing owner");
+        let result = crate::config::com::read_config_observation_no_lock(set, object, 0, &guard).await;
+        assert!(matches!(result, Err(Error::ConfigNotFound)), "I06L_EMPTY_DISTINCT");
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(body_cache_hook)]
+    async fn observation_real_backend_boundaries() {
+        let object = "config/observation-real.bin";
+        let payload = b"persisted config";
+        let (_dirs, set) = make_local_set_disks(4, 2).await;
+        let mut reader = PutObjReader::from_vec(payload.to_vec());
+        let written = set
+            .put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
+            .await
+            .expect("persist real config fixture");
+        let result = crate::config::com::read_config_observation(set.clone(), object, payload.len()).await;
+        assert!(result.is_ok(), "I05_OBSERVATION_SUCCESS: {result:?}");
+        let (bytes, info) = result.expect("real observation success");
+        assert_eq!(bytes, payload);
+        assert_eq!(info.size, written.size);
+        assert_eq!(info.etag, written.etag);
+        assert_eq!(info.data_dir, written.data_dir);
+        assert_eq!(info.name, object);
+        let over = crate::config::com::read_config_observation(set.clone(), object, payload.len() - 1).await;
+        assert!(over.is_err(), "I05_BOUND_ENFORCED");
+        assert_eq!(over.expect_err("over limit must fail"), Error::CorruptedFormat, "I05_BOUND_ENFORCED");
+        let absent = crate::config::com::read_config_observation(set.clone(), "config/observation-absent.bin", 32).await;
+        assert_eq!(absent.expect_err("absent config"), Error::ConfigNotFound);
+        let empty_object = "config/observation-empty.bin";
+        let mut empty = PutObjReader::from_vec(Vec::new());
+        set.put_object(crate::disk::RUSTFS_META_BUCKET, empty_object, &mut empty, &ObjectOptions::default())
+            .await
+            .expect("persist empty config");
+        let empty = crate::config::com::read_config_observation(set, empty_object, 0).await;
+        assert!(empty.is_ok(), "existing empty config must succeed: {empty:?}");
+        let (bytes, info) = empty.expect("empty observation");
+        assert!(bytes.is_empty());
+        assert_eq!(info.size, 0);
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(body_cache_hook)]
+    async fn observation_bypasses_registered_body_hook() {
+        let object = "config/observation-hook.bin";
+        let (_dirs, set) = make_local_set_disks(4, 2).await;
+        let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
+        set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
+            .await
+            .expect("persist real config fixture");
+        let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
+        assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
+        assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
+        let result = crate::config::com::read_config_observation(set, object, OBSERVATION_HOOK_BODY_LEN).await;
+        assert!(result.is_ok(), "I05_HOOK_BYPASS: {result:?}");
+        assert_eq!(
+            result.expect("observed persisted body").0,
+            vec![b'p'; OBSERVATION_HOOK_BODY_LEN],
+            "I05_HOOK_BYPASS"
+        );
+        assert_eq!(calls.load(Ordering::SeqCst), 0, "I05_HOOK_BYPASS");
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(body_cache_hook)]
+    async fn observation_hook_scope_restores_after_error() {
+        let object = "config/observation-hook-error.bin";
+        let (_dirs, set) = make_local_set_disks(4, 2).await;
+        let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
+        set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
+            .await
+            .expect("persist real config fixture");
+        let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
+        let result = crate::config::com::read_config_observation(set.clone(), object, OBSERVATION_HOOK_BODY_LEN - 1).await;
+        assert_eq!(result.expect_err("bounded observation error"), Error::CorruptedFormat);
+        assert_eq!(calls.load(Ordering::SeqCst), 0);
+        assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
+        assert_eq!(calls.load(Ordering::SeqCst), 1);
+    }
+
+    struct PendingObservationStore {
+        set: Arc<SetDisks>,
+        ready: std::sync::Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
+    }
+
+    impl std::fmt::Debug for PendingObservationStore {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            f.debug_struct("PendingObservationStore").finish()
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl crate::storage_api_contracts::object::ObjectIO for PendingObservationStore {
+        type Error = Error;
+        type RangeSpec = HTTPRangeSpec;
+        type HeaderMap = HeaderMap;
+        type ObjectOptions = ObjectOptions;
+        type ObjectInfo = ObjectInfo;
+        type GetObjectReader = GetObjectReader;
+        type PutObjectReader = PutObjReader;
+
+        async fn get_object_reader(
+            &self,
+            bucket: &str,
+            object: &str,
+            range: Option<HTTPRangeSpec>,
+            headers: HeaderMap,
+            opts: &ObjectOptions,
+        ) -> Result<GetObjectReader> {
+            let mut reader = self.set.get_object_reader(bucket, object, range, headers, opts).await?;
+            let mut body = Vec::new();
+            reader.stream.read_to_end(&mut body).await.expect("pending producer real GET");
+            assert_eq!(
+                body,
+                vec![b'p'; OBSERVATION_HOOK_BODY_LEN],
+                "I05_HOOK_BYPASS: producer must enter its own scope"
+            );
+            self.ready
+                .lock()
+                .expect("pending signal")
+                .take()
+                .expect("one GET")
+                .send(())
+                .expect("producer entry signal");
+            std::future::pending::<Result<GetObjectReader>>().await
+        }
+
+        async fn put_object(
+            &self,
+            _bucket: &str,
+            _object: &str,
+            _data: &mut PutObjReader,
+            _opts: &ObjectOptions,
+        ) -> Result<ObjectInfo> {
+            panic!("observation must not PUT");
+        }
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(body_cache_hook)]
+    async fn observation_hook_scope_restores_after_cancel() {
+        let object = "config/observation-hook-cancel.bin";
+        let (_dirs, set) = make_local_set_disks(4, 2).await;
+        let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
+        set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
+            .await
+            .expect("persist real config fixture");
+        let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
+        assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
+        assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
+        let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
+        let adapter = Arc::new(PendingObservationStore {
+            set: set.clone(),
+            ready: std::sync::Mutex::new(Some(ready_tx)),
+        });
+        let mut scope = Box::pin(crate::config::com::read_config_observation(adapter, object, OBSERVATION_HOOK_BODY_LEN));
+        tokio::select! {
+            result = ready_rx => { result.expect("producer reached pending GET"); }
+            result = &mut scope => { panic!("producer must remain pending until cancelled: {result:?}"); }
+        }
+        assert_eq!(calls.load(Ordering::SeqCst), 0);
+        drop(scope);
+        assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
+        assert_eq!(calls.load(Ordering::SeqCst), 1);
+    }
+
     const READ_VERSION_BARRIER_GUARD: std::time::Duration = std::time::Duration::from_secs(10);
 
     fn object_with_initial_data_shards(bucket: &str, prefix: &str) -> String {
@@ -2206,7 +2535,7 @@ mod prepared_get_object_metadata_tests {
         let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled();
         rustfs_io_metrics::set_get_stage_metrics_enabled(true);
 
-        let (restored, object_size, calls_total) = metrics::with_local_recorder(&recorder, || {
+        let (restored, object_size, calls_total, body_decodes) = metrics::with_local_recorder(&recorder, || {
             runtime.block_on(async {
                 let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
                 let opts = ObjectOptions {
@@ -2260,7 +2589,12 @@ mod prepared_get_object_metadata_tests {
                             .await
                             .expect("inline GET body should stream");
 
-                        (restored, object_size, calls.total(disk_call_counters::KIND_READ_VERSION))
+                        (
+                            restored,
+                            object_size,
+                            calls.total(disk_call_counters::KIND_READ_VERSION),
+                            calls.total(disk_call_counters::KIND_BODY_DECODE),
+                        )
                     },
                 )
                 .await
@@ -2274,6 +2608,10 @@ mod prepared_get_object_metadata_tests {
             calls_total, 4,
             "default production inline GET should schedule the initial bounded quorum plus one hedge"
         );
+        assert_eq!(
+            body_decodes, 1,
+            "cold inline early-stop GET must reuse the quorum-verified inline body instead of decoding shards twice"
+        );
         assert_eq!(
             recorder.histogram_values(
                 "rustfs_io_get_object_metadata_fanout_scheduled",
@@ -2300,6 +2638,124 @@ mod prepared_get_object_metadata_tests {
         );
     }
 
+    #[test]
+    #[serial_test::serial(body_cache_hook)]
+    fn get_object_info_uses_early_stop_for_locked_or_cache_safe_reads_and_full_wait_for_unlocked() {
+        use crate::storage_api_contracts::object::ObjectOperations;
+
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("current-thread runtime should build");
+        let bucket = "get-object-info-early-stop-bucket";
+        let object = object_with_initial_data_shards(bucket, "get-object-info-early-stop-object");
+        let payload = b"head object early-stop payload".repeat(128);
+
+        runtime.block_on(async {
+            let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
+            let write_opts = ObjectOptions {
+                no_lock: true,
+                ..Default::default()
+            };
+
+            set_disks
+                .make_bucket(bucket, &MakeBucketOptions::default())
+                .await
+                .expect("bucket should be created");
+            let mut put_reader = PutObjReader::from_vec(payload.clone());
+            set_disks
+                .put_object(bucket, &object, &mut put_reader, &write_opts)
+                .await
+                .expect("inline object should be written");
+
+            temp_env::async_with_vars(
+                [
+                    ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", None::<&str>),
+                    ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", None::<&str>),
+                    ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>),
+                ],
+                async {
+                    let slow_parity_disk = bounded_initial_parity_disk_index(bucket, &object);
+                    let barrier =
+                        rename_fanout_barrier::arm(&object, slow_parity_disk, rename_fanout_barrier::PHASE_READ_VERSION);
+                    let calls = disk_call_counters::observe(&object);
+                    let set_disks_for_head = Arc::clone(&set_disks);
+                    let object_for_head = object.clone();
+                    let head_opts = ObjectOptions {
+                        no_lock: true,
+                        metadata_cache_safe: true,
+                        ..Default::default()
+                    };
+                    let mut head_task =
+                        tokio::spawn(
+                            async move { set_disks_for_head.get_object_info(bucket, &object_for_head, &head_opts).await },
+                        );
+
+                    tokio::time::timeout(READ_VERSION_BARRIER_GUARD, barrier.wait_until_paused())
+                        .await
+                        .expect("cache-safe HEAD should pause a slow parity metadata read");
+                    let info = tokio::time::timeout(READ_VERSION_BARRIER_GUARD, &mut head_task)
+                        .await
+                        .expect("cache-safe HEAD should return before the paused parity metadata response")
+                        .expect("HEAD task should not panic")
+                        .expect("HEAD should succeed");
+                    assert_eq!(info.size, payload.len() as i64);
+                    assert_eq!(
+                        calls.total(disk_call_counters::KIND_READ_VERSION),
+                        4,
+                        "cache-safe HEAD should schedule bounded quorum plus one hedge"
+                    );
+                    assert_eq!(
+                        calls.total(disk_call_counters::KIND_BODY_DECODE),
+                        1,
+                        "early-stop HEAD verifies inline quorum once"
+                    );
+
+                    // Second cache-safe HEAD should hit the QuorumSelected metadata cache entry (0 new disk reads).
+                    let cached_info = set_disks
+                        .get_object_info(bucket, &object, &ObjectOptions::default())
+                        .await
+                        .expect("cached HEAD with default locked options should succeed");
+                    assert_eq!(cached_info.size, payload.len() as i64);
+                    assert_eq!(
+                        calls.total(disk_call_counters::KIND_READ_VERSION),
+                        4,
+                        "repeat locked HEAD must hit the QuorumSelected metadata cache entry"
+                    );
+
+                    // Release the slow parity barrier so full-wait can complete all 4 disks.
+                    barrier.release();
+
+                    // Unlocked internal preflight (no_lock=true, metadata_cache_safe=false) must bypass cache and full-wait.
+                    let unlocked_info = set_disks
+                        .get_object_info(
+                            bucket,
+                            &object,
+                            &ObjectOptions {
+                                no_lock: true,
+                                metadata_cache_safe: false,
+                                ..Default::default()
+                            },
+                        )
+                        .await
+                        .expect("unlocked preflight get_object_info should succeed");
+                    assert_eq!(unlocked_info.size, payload.len() as i64);
+                    assert_eq!(
+                        calls.total(disk_call_counters::KIND_READ_VERSION),
+                        8,
+                        "unlocked preflight get_object_info must query all 4 disks"
+                    );
+                    assert_eq!(
+                        calls.total(disk_call_counters::KIND_BODY_DECODE),
+                        1,
+                        "unlocked preflight full-wait must not run early-stop inline body decode"
+                    );
+                },
+            )
+            .await;
+        });
+    }
+
     #[test]
     #[serial_test::serial(body_cache_hook)]
     fn prepared_metadata_uses_full_fanout_even_when_data_read_early_stop_is_enabled() {
@@ -12603,7 +13059,7 @@ mod tests {
         );
     }
 
-    #[tokio::test(start_paused = true)]
+    #[tokio::test]
     async fn test_get_disks_info_probes_drives_concurrently() {
         use crate::disk::disk_store::DISK_INFO_PROBE_DELAY_FOR_TEST;
 
@@ -12618,6 +13074,8 @@ mod tests {
             disks.push(Some(disk));
         }
 
+        // Complete real filesystem setup before measuring virtual probe delays.
+        tokio::time::pause();
         let probe_delay = std::time::Duration::from_secs(2);
         let started = tokio::time::Instant::now();
         let info = DISK_INFO_PROBE_DELAY_FOR_TEST
@@ -16464,7 +16922,14 @@ mod tests {
 
     #[tokio::test]
     async fn set_level_versioned_delete_marker_hides_object_without_corrupting_version_metadata() {
-        let set_disks = make_local_bucket_test_set_disks_with_drive_count(4).await;
+        let (_fixture, _, set_disks) = hermetic_set_disks_for_pool_with_default_parity(4, 0, 2).await;
+        set_disks.set_test_storage_class_config(
+            storageclass::lookup_config_for_pools_without_env(&rustfs_config::server_config::KVS::new(), &[4])
+                .expect("test storage class should resolve for the local drive count"),
+        );
+        let store = crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
+            .await
+            .expect("fixture metadata authority should belong to this set");
         let bucket = "bucket-versioned-delete";
         let object = "object.txt";
         let opts = ObjectOptions {
@@ -16473,13 +16938,26 @@ mod tests {
             ..Default::default()
         };
 
-        set_disks
+        store
             .make_bucket(bucket, &MakeBucketOptions::default())
             .await
             .expect("bucket should be created");
         let mut reader = PutObjReader::from_vec(b"versioned object body".to_vec());
         let written = set_disks
-            .put_object(bucket, object, &mut reader, &opts)
+            .put_object(
+                bucket,
+                object,
+                &mut reader,
+                &ObjectOptions {
+                    object_lock_config_snapshot: Some(
+                        store
+                            .object_lock_config_snapshot(bucket)
+                            .await
+                            .expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
+                    ),
+                    ..opts.clone()
+                },
+            )
             .await
             .expect("versioned object should be written");
         assert!(written.version_id.is_some());
@@ -16796,7 +17274,14 @@ mod tests {
     /// `BucketNotEmpty` for a bucket the client has already emptied.
     #[tokio::test]
     async fn set_level_explicit_null_version_delete_purges_the_null_delete_marker() {
-        let set_disks = make_local_bucket_test_set_disks().await;
+        let (_fixture, _, set_disks) = hermetic_set_disks_for_pool_with_default_parity(2, 0, 1).await;
+        set_disks.set_test_storage_class_config(
+            storageclass::lookup_config_for_pools_without_env(&rustfs_config::server_config::KVS::new(), &[2])
+                .expect("test storage class should resolve for the local drive count"),
+        );
+        let store = crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
+            .await
+            .expect("fixture metadata authority should belong to this set");
         let bucket = "bucket-null-marker-purge";
         let object = "object.txt";
         // The delete path reads versioned/suspended from the bucket-config
@@ -16818,13 +17303,26 @@ mod tests {
             ..Default::default()
         };
 
-        set_disks
+        store
             .make_bucket(bucket, &MakeBucketOptions::default())
             .await
             .expect("bucket should be created");
         let mut reader = PutObjReader::from_vec(b"suspended version body".to_vec());
         set_disks
-            .put_object(bucket, object, &mut reader, &opts)
+            .put_object(
+                bucket,
+                object,
+                &mut reader,
+                &ObjectOptions {
+                    object_lock_config_snapshot: Some(
+                        store
+                            .object_lock_config_snapshot(bucket)
+                            .await
+                            .expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
+                    ),
+                    ..opts.clone()
+                },
+            )
             .await
             .expect("suspended-version object should be written");
 
@@ -16849,7 +17347,20 @@ mod tests {
 
         let mut replacement_reader = PutObjReader::from_vec(b"replacement suspended version body".to_vec());
         set_disks
-            .put_object(bucket, object, &mut replacement_reader, &opts)
+            .put_object(
+                bucket,
+                object,
+                &mut replacement_reader,
+                &ObjectOptions {
+                    object_lock_config_snapshot: Some(
+                        store
+                            .object_lock_config_snapshot(bucket)
+                            .await
+                            .expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
+                    ),
+                    ..opts.clone()
+                },
+            )
             .await
             .expect("a suspended-version PUT should replace the first null marker");
         let replacement_marker = set_disks
@@ -16909,7 +17420,14 @@ mod tests {
 
     #[tokio::test]
     async fn set_level_version_suspended_delete_creates_null_delete_marker() {
-        let set_disks = make_local_bucket_test_set_disks().await;
+        let (_fixture, _, set_disks) = hermetic_set_disks_for_pool_with_default_parity(2, 0, 1).await;
+        set_disks.set_test_storage_class_config(
+            storageclass::lookup_config_for_pools_without_env(&rustfs_config::server_config::KVS::new(), &[2])
+                .expect("test storage class should resolve for the local drive count"),
+        );
+        let store = crate::bucket::metadata_sys::object_store_in(&set_disks.ctx)
+            .await
+            .expect("fixture metadata authority should belong to this set");
         let bucket = "bucket-version-suspended-delete";
         let object = "object.txt";
         let opts = ObjectOptions {
@@ -16919,13 +17437,26 @@ mod tests {
             ..Default::default()
         };
 
-        set_disks
+        store
             .make_bucket(bucket, &MakeBucketOptions::default())
             .await
             .expect("bucket should be created");
         let mut reader = PutObjReader::from_vec(b"suspended version body".to_vec());
         set_disks
-            .put_object(bucket, object, &mut reader, &opts)
+            .put_object(
+                bucket,
+                object,
+                &mut reader,
+                &ObjectOptions {
+                    object_lock_config_snapshot: Some(
+                        store
+                            .object_lock_config_snapshot(bucket)
+                            .await
+                            .expect("PUT fixture should own the bucket lifecycle and Object Lock snapshot"),
+                    ),
+                    ..opts.clone()
+                },
+            )
             .await
             .expect("suspended-version object should be written");
 

crates/ecstore/src/set_disk/ops/delete_recovery.rs

@@ -113,6 +113,7 @@ pub(crate) struct DeleteRecoveryContext<'a> {
     pub repair_marker_source: bool,
     pub namespace_owner: Option<Arc<dyn Send + Sync>>,
     pub recovery_ticket: Option<crate::store::delete_recovery_runtime::DeleteRecoveryTicket>,
+    pub published_hint: Option<(Uuid, &'a [Result<DeleteRecoveryInspection>])>,
     pub can_commit: &'a (dyn Fn() -> bool + Sync),
 }
 
@@ -125,6 +126,7 @@ impl<'a> DeleteRecoveryContext<'a> {
             repair_marker_source: false,
             namespace_owner: None,
             recovery_ticket: None,
+            published_hint: None,
             can_commit,
         }
     }
@@ -421,7 +423,9 @@ impl SetDisks {
         let cleanup = loop {
             #[cfg(test)]
             test_hooks::record_cleanup_attempt(bucket, object);
-            match self.recover_delete_key(bucket, object, context.clone()).await {
+            let mut attempt_context = context.clone();
+            attempt_context.published_hint = (attempt == 1).then_some((transaction, published.as_slice()));
+            match self.recover_delete_key(bucket, object, attempt_context).await {
                 // A stale settlement fingerprint only means the snapshot raced
                 // another writer; each attempt re-inspects every participant.
                 Err(DiskError::OutdatedXLMeta) if attempt < PUBLISHED_CLEANUP_ATTEMPTS && can_commit() => attempt += 1,
@@ -551,6 +555,7 @@ impl SetDisks {
             explicit_delete,
             repair_marker_source,
             namespace_owner,
+            published_hint,
             can_commit,
             recovery_ticket: _,
         } = context;
@@ -589,7 +594,18 @@ impl SetDisks {
         let mut all_states = BTreeMap::new();
         for transaction in transactions {
             let mut complete = Vec::with_capacity(disks.len());
-            for state in inspect(&disks, bucket, object, transaction).await {
+            let inspections = if let Some((published_tx, published_states)) = published_hint
+                && !repair_marker_source
+                && !dry_run
+                && published_tx == transaction
+                && published_states.len() == disks.len()
+                && published_states.iter().all(Result::is_ok)
+            {
+                published_states.to_vec()
+            } else {
+                inspect(&disks, bucket, object, transaction).await
+            };
+            for state in inspections {
                 match state {
                     Ok(state) => {
                         if let Some(journal) = &state.journal {
@@ -832,6 +848,9 @@ impl SetDisks {
                     bucket,
                     copies,
                     |copy| {
+                        if receipt(copy.state) && copy.commit_data_safe {
+                            return None;
+                        }
                         Some(
                             if copy.state == DeleteRecoveryState::Absent
                                 || (certified_same_transaction
@@ -857,7 +876,10 @@ impl SetDisks {
                         &disks,
                         bucket,
                         copies,
-                        |_| Some(DeleteRecoveryAction::FinishCommit),
+                        |copy| {
+                            (copy.state != DeleteRecoveryState::CommitFinished || !copy.commit_data_safe)
+                                .then_some(DeleteRecoveryAction::FinishCommit)
+                        },
                         can_commit,
                         namespace_owner.as_ref(),
                         &mut summary.removed_versions,
@@ -922,13 +944,25 @@ impl SetDisks {
             // GC is a key-wide barrier: every predecessor must have settled
             // before the last surviving commit witness can disappear.
             let mut settled = Vec::with_capacity(all_states.len());
-            for transaction in all_states.keys() {
-                let fresh = inspect(&disks, bucket, object, *transaction).await;
-                if fresh.iter().any(|state| !matches!(state, Ok(state) if matches!(state.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished))) {
+            if all_states.len() == 1 {
+                let copies = all_states.into_values().next().ok_or(DiskError::FileCorrupt)?;
+                if copies
+                    .iter()
+                    .any(|state| !matches!(state.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished))
+                {
                     summary.pending += 1;
                     return Ok(summary);
                 }
-                settled.push(fresh.into_iter().collect::<Result<Vec<_>>>()?);
+                settled.push(copies);
+            } else {
+                for transaction in all_states.keys() {
+                    let fresh = inspect(&disks, bucket, object, *transaction).await;
+                    if fresh.iter().any(|state| !matches!(state, Ok(state) if matches!(state.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished))) {
+                        summary.pending += 1;
+                        return Ok(summary);
+                    }
+                    settled.push(fresh.into_iter().collect::<Result<Vec<_>>>()?);
+                }
             }
             for (index, mut copies) in settled.into_iter().enumerate() {
                 if index > 0 {
@@ -1783,6 +1817,56 @@ mod tests {
             .expect("returning old replica must not leave bucket residue");
     }
 
+    #[tokio::test]
+    async fn delete_recovery_set_replays_partial_gc_with_empty_transaction_directory() {
+        let bucket = "recover-empty-gc";
+        let (dirs, store, disks) = fixture(bucket).await;
+        put(&store, bucket, &[24; 4096], false).await;
+        let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
+        for (index, disk) in disks.iter().enumerate() {
+            publish(disk, bucket, transaction).await;
+            let state = disk.inspect_delete_transaction(bucket, "object", transaction).await.unwrap();
+            let finished = disk
+                .settle_delete_transaction(bucket, "object", transaction, state.fingerprint, DeleteRecoveryAction::FinishCommit)
+                .await
+                .unwrap();
+            if index > 0 {
+                disk.settle_delete_transaction(
+                    bucket,
+                    "object",
+                    transaction,
+                    finished.fingerprint,
+                    DeleteRecoveryAction::FinalizeCommit,
+                )
+                .await
+                .unwrap();
+            }
+        }
+        // A crash can persist proof removal before the final empty rmdir.
+        let empty = dirs[3].path().join(bucket).join("object").join(transaction.to_string());
+        tokio::fs::create_dir_all(&empty).await.unwrap();
+        for _ in 0..2 {
+            store
+                .recover_delete_transactions(bucket, &HealOpts::default())
+                .await
+                .expect("the remaining receipt must settle every participant without a fresh delete");
+            for disk in &disks {
+                assert!(discover(disk, bucket, Some("object")).await.unwrap().is_empty());
+                assert!(
+                    disk.inspect_delete_transaction(bucket, "object", transaction)
+                        .await
+                        .unwrap()
+                        .current_digest
+                        .is_none()
+                );
+            }
+        }
+        store
+            .delete_bucket(bucket, &DeleteBucketOptions::default())
+            .await
+            .expect("ordinary bucket removal must succeed after replay");
+    }
+
     #[tokio::test]
     async fn delete_recovery_tracks_only_observed_removed_version() {
         let bucket = "recover-actual-removal";

crates/ecstore/src/set_disk/ops/heal/shard_integrity_rollout_tests.rs

@@ -15,7 +15,9 @@
 use crate::disk::{DiskAPI as _, ReadOptions};
 use crate::object_api::{ObjectOptions, PutObjReader, ShardIntegrityWriteMode};
 use crate::set_disk::CompletePart;
-use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity_isolated;
+use crate::set_disk::ops::object::hermetic_set_disks_support::{
+    hermetic_set_disks_for_pool_with_default_parity_isolated, lifecycle_owned_put_options,
+};
 use crate::storage_api_contracts::multipart::MultipartOperations as _;
 use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
 use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
@@ -39,18 +41,19 @@ async fn remote_metadata_heal_does_not_certify_unread_tier_payload() {
     for disk in &disks {
         disk.make_volume(bucket).await.expect("fixture bucket");
     }
-    set.put_object(
+    let opts = lifecycle_owned_put_options(
+        &set,
         bucket,
-        object,
-        &mut PutObjReader::from_vec(vec![0x31; 1024 * 1024 + 17]),
-        &ObjectOptions {
+        ObjectOptions {
             no_lock: true,
             shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected),
             ..Default::default()
         },
     )
-    .await
-    .expect("protected source");
+    .await;
+    set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![0x31; 1024 * 1024 + 17]), &opts)
+        .await
+        .expect("protected source");
     for (disk, dir) in disks.iter().zip(&dirs) {
         let mut info = disk
             .read_version("", bucket, object, "", &ReadOptions::default())
@@ -94,10 +97,15 @@ async fn rollout_requires_both_flags_for_new_puts() {
     for disk in &disks {
         disk.make_volume(bucket).await.expect("fixture bucket");
     }
-    let opts = ObjectOptions {
-        no_lock: true,
-        ..Default::default()
-    };
+    let opts = lifecycle_owned_put_options(
+        &set,
+        bucket,
+        ObjectOptions {
+            no_lock: true,
+            ..Default::default()
+        },
+    )
+    .await;
 
     for (requested, confirmed, protected) in [
         (false, false, false),
@@ -159,12 +167,17 @@ async fn legacy_shard_repair_and_explicit_version_metadata_recovery_remain_avail
     for disk in &disks {
         disk.make_volume(bucket).await.expect("fixture bucket");
     }
-    let opts = ObjectOptions {
-        no_lock: true,
-        versioned: true,
-        shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
-        ..Default::default()
-    };
+    let opts = lifecycle_owned_put_options(
+        &set,
+        bucket,
+        ObjectOptions {
+            no_lock: true,
+            versioned: true,
+            shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
+            ..Default::default()
+        },
+    )
+    .await;
     let expected = vec![0x67; 1024 * 1024 + 37];
     for case in ["missing-part", "bad-bitrot", "missing-version-metadata"] {
         let object = case;
@@ -272,10 +285,15 @@ async fn multipart_mode_survives_rollout_switch_changes() {
     for disk in &disks {
         disk.make_volume(bucket).await.expect("fixture bucket");
     }
-    let opts = ObjectOptions {
-        no_lock: true,
-        ..Default::default()
-    };
+    let opts = lifecycle_owned_put_options(
+        &set,
+        bucket,
+        ObjectOptions {
+            no_lock: true,
+            ..Default::default()
+        },
+    )
+    .await;
     for protected in [false, true] {
         let object = format!("upload-{protected}");
         let upload =
@@ -354,11 +372,16 @@ async fn physical_copy_inherits_source_mode_instead_of_current_switch() {
         };
         let object = format!("source-{protected}");
         let expected = vec![0x39; 1024 * 1024 + 11];
-        let seed_opts = ObjectOptions {
-            no_lock: true,
-            shard_integrity_write_mode: Some(mode),
-            ..Default::default()
-        };
+        let seed_opts = lifecycle_owned_put_options(
+            &set,
+            bucket,
+            ObjectOptions {
+                no_lock: true,
+                shard_integrity_write_mode: Some(mode),
+                ..Default::default()
+            },
+        )
+        .await;
         let mut source = set
             .put_object(bucket, &object, &mut PutObjReader::from_vec(expected.clone()), &seed_opts)
             .await
@@ -366,8 +389,8 @@ async fn physical_copy_inherits_source_mode_instead_of_current_switch() {
         source.metadata_only = false;
         source.put_object_reader = Some(PutObjReader::from_vec(expected.clone()));
         let opts = ObjectOptions {
-            no_lock: true,
-            ..Default::default()
+            shard_integrity_write_mode: None,
+            ..seed_opts.clone()
         };
         let copied = temp_env::async_with_vars(
             rollout_vars(!protected, !protected),

crates/ecstore/src/set_disk/ops/heal_lock_wait_tests.rs

@@ -0,0 +1,194 @@
+// Copyright 2026 RustFS Team
+// SPDX-License-Identifier: Apache-2.0
+
+use super::*;
+use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated;
+use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
+use crate::storage_api_contracts::heal::HealOperations as _;
+use metrics_util::debugging::{DebugValue, DebuggingRecorder};
+use std::collections::BTreeMap;
+use std::future::Future;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use std::time::{Duration, Instant};
+
+const METRIC: &str = "rustfs_heal_object_lock_wait_seconds";
+
+async fn captured<F: Future>(recorder: &DebuggingRecorder, future: F) -> F::Output {
+    let mut future = std::pin::pin!(future);
+    std::future::poll_fn(|cx| metrics::with_local_recorder(recorder, || future.as_mut().poll(cx))).await
+}
+
+fn poll_once<F: Future>(recorder: &DebuggingRecorder, future: Pin<&mut F>) -> Poll<F::Output> {
+    let mut cx = Context::from_waker(futures::task::noop_waker_ref());
+    metrics::with_local_recorder(recorder, || future.poll(&mut cx))
+}
+
+fn samples(recorder: &DebuggingRecorder) -> Vec<(BTreeMap<String, String>, f64)> {
+    recorder
+        .snapshotter()
+        .snapshot()
+        .into_vec()
+        .into_iter()
+        .filter(|(key, _, _, _)| key.key().name() == METRIC)
+        .flat_map(|(key, _, _, value)| {
+            let labels: BTreeMap<_, _> = key
+                .key()
+                .labels()
+                .map(|label| (label.key().to_owned(), label.value().to_owned()))
+                .collect();
+            let DebugValue::Histogram(values) = value else { panic!("wait must be a histogram") };
+            values.into_iter().map(move |value| (labels.clone(), value.0))
+        })
+        .collect()
+}
+
+fn one_sample(recorder: &DebuggingRecorder, op: &str, mode: &str, result: &str) -> f64 {
+    let rows = samples(recorder);
+    assert_eq!(rows.len(), 1, "one observation per actual acquisition attempt");
+    assert_eq!(
+        rows[0].0,
+        BTreeMap::from([
+            ("op".to_owned(), op.to_owned()),
+            ("mode".to_owned(), mode.to_owned()),
+            ("result".to_owned(), result.to_owned()),
+        ]),
+        "no object, bucket, version, or owner labels"
+    );
+    assert!(rows[0].1.is_finite() && rows[0].1 >= 0.0);
+    rows[0].1
+}
+
+#[tokio::test]
+async fn heal_object_lock_metrics_cover_paths_and_skip_borrowed_locks() {
+    let (_dirs, _disks, set) = hermetic_set_disks_isolated(4).await;
+    let bucket = "heal-lock-metric-paths";
+    set.make_bucket(bucket, &MakeBucketOptions::default()).await.expect("bucket");
+    let recorder = DebuggingRecorder::new();
+    for (op, kind) in [
+        ("read_repair_data", HealObjectLockKind::Read),
+        ("read_repair_commit", HealObjectLockKind::Write),
+        ("heal_object", HealObjectLockKind::Write),
+    ] {
+        let guard = captured(&recorder, set.acquire_heal_object_lock(bucket, "missing", "", kind, op))
+            .await
+            .expect("namespace lock");
+        one_sample(&recorder, op, kind.as_str(), "acquired");
+        metrics::with_local_recorder(&recorder, || drop(guard));
+        assert!(samples(&recorder).is_empty(), "hold time must not become another wait sample");
+    }
+    let opts = HealOpts {
+        dry_run: true,
+        ..Default::default()
+    };
+    captured(&recorder, set.heal_object(bucket, "missing", "", &opts))
+        .await
+        .expect("missing object report");
+    one_sample(&recorder, "heal_object", "write", "acquired");
+    captured(&recorder, set.check_abandoned_parts(bucket, "missing", &opts))
+        .await
+        .expect("abandoned parts check");
+    one_sample(&recorder, "check_abandoned_parts", "write", "acquired");
+    captured(&recorder, set.heal_object_dir(bucket, "missing-dir/", true, false))
+        .await
+        .expect("directory report");
+    one_sample(&recorder, "heal_object_dir", "write", "acquired");
+
+    let no_lock = HealOpts { no_lock: true, ..opts };
+    captured(&recorder, set.heal_object(bucket, "missing", "", &no_lock))
+        .await
+        .expect("borrowed-lock report");
+    captured(&recorder, set.check_abandoned_parts(bucket, "missing", &no_lock))
+        .await
+        .expect("borrowed-lock check");
+    assert!(samples(&recorder).is_empty(), "no_lock must not invent an acquisition");
+}
+
+#[tokio::test]
+async fn heal_object_lock_metrics_preserve_contention_and_cancellation() {
+    let (_dirs, _disks, set) = hermetic_set_disks_isolated(4).await;
+    let namespace = set.new_ns_lock("metric-contention", "object").await.expect("namespace");
+    let recorder = DebuggingRecorder::new();
+    for cancel in [false, true] {
+        let blocker = namespace.get_write_lock(Duration::from_secs(30)).await.expect("blocker");
+        let mut waiting = Box::pin(set.acquire_heal_object_lock(
+            "metric-contention",
+            "object",
+            "",
+            HealObjectLockKind::Read,
+            "read_repair_data",
+        ));
+        assert!(poll_once(&recorder, waiting.as_mut()).is_pending(), "real holder must block heal");
+        let pending_at = Instant::now();
+        assert!(samples(&recorder).is_empty(), "unfinished attempt has no completed sample");
+        if cancel {
+            let blocked_for = pending_at.elapsed();
+            metrics::with_local_recorder(&recorder, || drop(waiting));
+            assert!(one_sample(&recorder, "read_repair_data", "read", "cancelled") >= blocked_for.as_secs_f64());
+            let mut successor = Box::pin(namespace.get_read_lock(Duration::from_secs(30)));
+            assert!(
+                poll_once(&recorder, successor.as_mut()).is_pending(),
+                "cancelling heal must not release another holder"
+            );
+            drop(blocker);
+            drop(successor.await.expect("cancelled waiter must not leak ownership"));
+        } else {
+            let blocked_for = pending_at.elapsed();
+            drop(blocker);
+            let guard = captured(&recorder, waiting).await.expect("acquire after blocker releases");
+            assert!(one_sample(&recorder, "read_repair_data", "read", "acquired") >= blocked_for.as_secs_f64());
+            let mut successor = Box::pin(namespace.get_write_lock(Duration::from_secs(30)));
+            assert!(
+                poll_once(&recorder, successor.as_mut()).is_pending(),
+                "observation must preserve the returned guard"
+            );
+            metrics::with_local_recorder(&recorder, || drop(guard));
+            drop(successor.await.expect("release at the original guard drop"));
+        }
+        assert!(samples(&recorder).is_empty(), "one outcome per attempt");
+    }
+}
+
+#[tokio::test]
+#[serial_test::serial]
+async fn heal_object_lock_metrics_retain_failed_acquisition() {
+    temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
+        let (_dirs, _disks, set) = hermetic_set_disks_isolated(4).await;
+        let namespace = set.new_ns_lock("metric-timeout", "object").await.expect("namespace");
+        let blocker = namespace.get_write_lock(Duration::from_secs(30)).await.expect("blocker");
+        let recorder = DebuggingRecorder::new();
+        let result = captured(
+            &recorder,
+            set.acquire_heal_object_lock("metric-timeout", "object", "", HealObjectLockKind::Write, "heal_object"),
+        )
+        .await;
+        assert!(result.is_err(), "measurement must not turn a timeout into success");
+        one_sample(&recorder, "heal_object", "write", "failed");
+        drop(blocker);
+    })
+    .await;
+}
+
+#[tokio::test]
+async fn heal_object_lock_metrics_separate_disabled_and_unpolled_acquisitions() {
+    use rustfs_lock::{GlobalLockManager, NamespaceLock, NamespaceLockWrapper, ObjectKey};
+    let manager = Arc::new(GlobalLockManager::Disabled(rustfs_lock::fast_lock::DisabledLockManager::new()));
+    let namespace = NamespaceLockWrapper::new(
+        NamespaceLock::with_local_manager("disabled-test".to_owned(), manager),
+        ObjectKey::new("bucket", "object"),
+        "owner".to_owned(),
+    );
+    let recorder = DebuggingRecorder::new();
+    let unpolled =
+        measure_heal_object_lock("heal_object", HealObjectLockKind::Write, namespace.get_write_lock(Duration::from_secs(1)));
+    metrics::with_local_recorder(&recorder, || drop(unpolled));
+    assert!(samples(&recorder).is_empty(), "a never-polled future is not an attempted acquisition");
+    let guard = captured(
+        &recorder,
+        measure_heal_object_lock("heal_object", HealObjectLockKind::Write, namespace.get_write_lock(Duration::from_secs(1))),
+    )
+    .await
+    .expect("disabled guard");
+    assert!(matches!(&guard, rustfs_lock::NamespaceLockGuard::Fast(guard) if guard.is_disabled()));
+    one_sample(&recorder, "heal_object", "write", "disabled");
+}